dwc-archive 0.1.3 → 1.1.9

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b35d0c900fbb815f4955e59306c6cc5a134f4c5094adfc98576ac72249e82566
+  data.tar.gz: 8f8f58062273f5d5e6afb1084aed358d7585e8efac107e7fdf723cfa93e0d0bc
+SHA512:
+  metadata.gz: 39cd3818632795b6d49ea9c7affbc565c8d8a7c9aa48b1acf2e00b6c8743b80ed58f5d6a0b7e26ea5375f1e3ba5e2dc91b59fd9820cda0918d254ae4739d930e
+  data.tar.gz: 82f055dd0ed152a516eade9df4efe12cc0aaded52c0c1054f25340d07f0a2cd9f11fd2ca6873e3cf9adab7b3404f892c8c07c5953ada133c2a53d071fc8fe858

data/.github/workflows/build.yml

@@ -0,0 +1,35 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: build
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['2.6', '2.7', '3.0']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+    # change this to (see https://github.com/ruby/setup-ruby#versioning):
+    # uses: ruby/setup-ruby@v1
+      uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+    - name: Run tests
+      run: bundle exec rake

data/.gitignore

@@ -17,5 +17,15 @@ tmtags
 coverage
 rdoc
 pkg
+install
+Gemfile.lock
 
 ## PROJECT::SPECIFIC
+tags
+bin
+.bundle
+bundle_bin
+Gemfile.lock
+.byebug_history
+
+

data/.rspec

@@ -0,0 +1,3 @@
+--format p
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,23 @@
+AllCops:
+  TargetRubyVersion: 2.6
+  Exclude:
+    - features/**/*
+    - .bundle/**/*
+    - bundle_bin/**/*
+    - lib/dwc_archive/ingester.rb
+    - lib/dwc_archive/generator_meta_xml.rb
+    - lib/dwc_archive/generator_eml_xml.rb
+    - lib/dwc_archive/classification_normalizer.rb
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Layout/DotPosition:
+  EnforcedStyle: trailing
+
+Metrics/BlockLength:
+  Exclude:
+    - 'Rakefile'
+    - '*.gemspec'
+    - '**/*.rake'
+    - 'spec/**/*spec.rb'

data/.ruby-version

@@ -0,0 +1 @@
+3.3.5

data/CHANGELOG

@@ -0,0 +1,21 @@
+1.1.0  Update name parser to go-based biodiversity
+
+1.0.1  Cleanup and gems update
+
+0.9.11 Removed VERSION duplicate
+
+0.9.7  Refactoring and tests improvements
+
+0.9.6  Added support for GNUB DwCA files
+
+0.9.4  Gem dependencies updated, added travis support
+
+0.9.0  Migrated code to ruby 1.9.3
+
+0.8.3  Updated outdated exception rasing
+
+0.8.2  Removed species info from linnean classification path
+
+0.8.1  Linnean classification path is now only for species and infraspecies with canonical forms. It ends with a canonical form of the taxon
+
+0.8.0  Added linnean classification path to normalized data from DwCA. It consists of data associated with clades like 'kingdom', 'order' etc.

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2010 Dmitry Mozzherin
+Copyright (c) 2010-2020 Dmitry Mozzherin
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -0,0 +1,203 @@
+Darwin Core Archive
+===================
+
+[![Gem Version][1]][2]
+[![Continuous Integration Status][3]][4]
+[![Coverage Status][5]][6]
+[![CodePolice][7]][8]
+[![Dependency Status][9]][10]
+
+Darwin Core Archive format is a current standard for information exchange
+between Global Names Architecture modules. This gem allows to work with
+Darwin Core Archive data compressed to either zip or tar.gz files.
+More information about Darwing Core Archive can be found on a [GBIF page:][11]
+
+Installation
+------------
+
+    sudo gem install dwc-archive
+
+### System Requirements
+
+You need [Redis Server][12] and unzip library installed
+
+
+Usage
+-----
+
+```ruby
+require 'rubygems'
+require 'dwc_archive'
+
+dwc = DarwinCore.new('/path_to_file/archive_file.tar.gz')
+dwc.archive.files      # the archive file list
+dwc.metadata.data      # summary of metadata from eml.xml if it exists
+dwc.metadata.authors   # authors of the archive
+dwc.core.data          # summary of DarwinCore main file
+dwc.core.file_path     # path to the DarwinCore main file
+dwc.extensions         # array of DarwinCore Star extensions
+dwc.extensions[0].data # summary for an extension
+
+# read content of the core data file into memory or used with a block
+# it returns array of arrays of data
+# rows that had a wrong encoding will be collected into errors array
+data, errors = dwc.core.read
+
+# read content using a block, getting back results in sets of 100 rows each
+results = []
+tail_data, tail_errors = dwc.core.read(100) do |data, errors|
+  results << [data, errors]
+end
+results << [tail_data, tail_errors]
+
+# read content of an extension data file into memory
+data, errors = dwc.core.extensions[0].read
+
+# read content of an extension data using block
+results = []
+tail_data, tail_errors = dwc.core.extensions[0](100) do |data, errors|
+  results << [data, errors]
+end
+results << [tail_data, tail_errors]
+
+# normalize names in classification collecting together synonyms,
+# canonical names, vernacular names and associating paths to taxons
+# in a classification distributed as DwCA file
+
+result = dwc.normalize_classification
+
+# for a finer control over normalization:
+
+cn = DarwinCore::ClassificationNormalizer.new(dwc)
+cn.normalize
+# if you don't want to generate path consisting of canonical forms
+# of ancestors to a taxon
+cn.normalize(:with_canonical_names => false)
+
+# if you don't want to ingest information from extensions
+cn.normalize(:with_extensions => false)
+
+# to get a flat hash of nodes with attached vernacular names and synonyms
+normalized_data = cn.normalized_data
+
+# to get a representation of tree organization as a hash
+classification_tree = cn.tree
+
+# to get list of all name strings used as scientific or vernacular names
+all_name_strings = cn.name_strings
+
+# to get list of errors generated during the normalization
+errors = cn.error_names
+
+DarwinCore.clean_all   # remove all expanded archives
+```
+
+Creating a DarwinCore Archive file
+----------------------------------
+
+```ruby
+gen = DarwinCore::Generator.new('/tmp/dwc_birches.tar.gz')
+
+core = [
+  ["http://rs.tdwg.org/dwc/terms/taxonID",
+   "http://rs.tdwg.org/dwc/terms/parentNameUsageID",
+   "http://rs.tdwg.org/dwc/terms/scientificName",
+   "http://rs.tdwg.org/dwc/terms/taxonRank"],
+  [1, 0, "Plantae", "kingdom"],
+  [2, 1, "Betula", "genus"],
+  [3, 2, "Betula verucosa", "species"]
+]
+
+vernacular_names = [
+  ["http://rs.tdwg.org/dwc/terms/TaxonID",
+  "http://rs.tdwg.org/dwc/terms/vernacularName"],
+  [1, "Plants"],
+  [1, "Растения"],
+  [2, "Birch"],
+  [2, "Береза"],
+  [3, "Wheeping Birch"],
+  [3, "Береза плакучая"]
+]
+
+eml = {
+  :id => '1234',
+  :license => 'http://creativecommons.org/licenses/by-sa/3.0/',
+  :title => 'Test Classification',
+  :authors => [
+    { :first_name => 'John',
+      :last_name => 'Doe',
+      :email => 'jdoe@example.com',
+      :organization => 'Example',
+      :position => 'Assistant Professor',
+      :url => 'http://example.org' },
+      { :first_name => 'Jane',
+        :last_name => 'Doe',
+        :email => 'jane@example.com' }
+],
+  :metadata_providers => [
+    { :first_name => 'Jim',
+      :last_name => 'Doe',
+      :email => 'jimdoe@example.com',
+      :url => 'http://aggregator.example.org' }],
+  :abstract => 'test classification',
+  :citation =>
+    'Test classification: Doe John, Doe Jane, Taxnonmy, 10, 1, 2010',
+  :url => 'http://example.com'
+}
+
+gen.add_core(core, 'core.txt')
+gen.add_extension(vernacular_names,
+                  'vernacular_names.txt',
+                  true, 'http://rs.gbif.org/terms/1.0/VernacularName')
+gen.add_meta_xml
+gen.add_eml_xml(eml)
+gen.pack
+```
+
+Logging
+-------
+
+Gem has ability to show logs of it's events:
+
+    require 'dwc-archive'
+    DarwinCore.logger = Logger.new($stdout)
+
+
+Note on Patches/Pull Requests
+-----------------------------
+
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump
+  version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+
+Copyright
+---------
+
+Author -- [@dimus][13]
+
+Contributors -- [@mjy][14], [@LocoDelAssembly][16]
+
+Copyright (c) 2010-2020 [@dimus][15]. See LICENSE for details.
+
+[1]: https://badge.fury.io/rb/dwc-archive.png
+[2]: http://badge.fury.io/rb/dwc-archive
+[3]: https://secure.travis-ci.org/GlobalNamesArchitecture/dwc-archive.png
+[4]: http://travis-ci.org/GlobalNamesArchitecture/dwc-archive
+[5]: https://coveralls.io/repos/GlobalNamesArchitecture/dwc-archive/badge.png
+[6]: https://coveralls.io/r/GlobalNamesArchitecture/dwc-archive
+[7]: https://codeclimate.com/github/GlobalNamesArchitecture/dwc-archive.png
+[8]: https://codeclimate.com/github/GlobalNamesArchitecture/dwc-archive
+[9]: https://gemnasium.com/GlobalNamesArchitecture/dwc-archive.png
+[10]: https://gemnasium.com/GlobalNamesArchitecture/dwc-archive
+[11]: http://bit.ly/2IxcBA
+[12]: http://redis.io/topics/quickstart
+[13]: https://github.com/dimus
+[14]: https://github.com/mjy
+[15]: http://mbl.edu
+[16]: https://github.com/LocoDelAssembly

data/Rakefile

@@ -1,57 +1,21 @@
-require 'rubygems'
-require 'rake'
+# frozen_string_literal: true
 
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name = "dwc-archive"
-    gem.summary = %Q{Handler of Darwin Core Archive files}
-    gem.description = %Q{Darwin Core Archive Files are current standard exchange format for GLobal Names Architecture modules. This project creates ways to work with such files.}
-    gem.email = "dmozzherin at gmail dot com"
-    gem.homepage = "http://github.com/dimus/dwc-archive"
-    gem.authors = ["Dmitry Mozzherin"]
-    gem.add_development_dependency "rspec", ">= 1.2.9"
-    gem.add_development_dependency "cucumber", ">= 0"
-    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
-  end
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
-end
-
-require 'spec/rake/spectask'
-Spec::Rake::SpecTask.new(:spec) do |spec|
-  spec.libs << 'lib' << 'spec'
-  spec.spec_files = FileList['spec/**/*_spec.rb']
-end
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+require "cucumber/rake/task"
 
-Spec::Rake::SpecTask.new(:rcov) do |spec|
-  spec.libs << 'lib' << 'spec'
-  spec.pattern = 'spec/**/*_spec.rb'
-  spec.rcov = true
+RSpec::Core::RakeTask.new(:rspec) do |rspec|
+  rspec.pattern = "spec/**/*_spec.rb"
 end
 
-task :spec => :check_dependencies
+Cucumber::Rake::Task.new(:features)
 
-begin
-  require 'cucumber/rake/task'
-  Cucumber::Rake::Task.new(:features)
+# task rspec: :check_dependencies
+task features: :check_dependencies
 
-  task :features => :check_dependencies
-rescue LoadError
-  task :features do
-    abort "Cucumber is not available. In order to run features, you must: sudo gem install cucumber"
-  end
+desc "open an irb session preloaded with this library"
+task :console do
+  sh "irb -I lib -I extra -r dwc_archive.rb"
 end
 
-task :default => :spec
-
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "dwc-archive #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end
+task default: :rspec

data/dwc-archive.gemspec

@@ -1,74 +1,36 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
-# -*- encoding: utf-8 -*-
+# frozen_string_literal: true
 
-Gem::Specification.new do |s|
-  s.name = %q{dwc-archive}
-  s.version = "0.1.3"
+require File.expand_path("lib/dwc_archive/version", __dir__)
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Dmitry Mozzherin"]
-  s.date = %q{2010-03-18}
-  s.description = %q{Darwin Core Archive Files are current standard exchange format for GLobal Names Architecture modules. This project creates ways to work with such files.}
-  s.email = %q{dmozzherin at gmail dot com}
-  s.extra_rdoc_files = [
-    "LICENSE",
-     "README.rdoc"
-  ]
-  s.files = [
-    ".document",
-     ".gitignore",
-     "LICENSE",
-     "README.rdoc",
-     "Rakefile",
-     "VERSION",
-     "dwc-archive.gemspec",
-     "features/dwc-archive.feature",
-     "features/step_definitions/dwc-archive_steps.rb",
-     "features/support/env.rb",
-     "lib/dwc-archive.rb",
-     "lib/dwc-archive/.expander.rb.swo",
-     "lib/dwc-archive/archive.rb",
-     "lib/dwc-archive/core.rb",
-     "lib/dwc-archive/expander.rb",
-     "lib/dwc-archive/extension.rb",
-     "lib/dwc-archive/metadata.rb",
-     "lib/ruby_extensions.rb",
-     "spec/dwc-archive_spec.rb",
-     "spec/files/data.tar.gz",
-     "spec/files/data.zip",
-     "spec/files/eml.xml",
-     "spec/files/meta.xml",
-     "spec/lib/ruby_extenstions_spec.rb",
-     "spec/spec.opts",
-     "spec/spec_helper.rb"
-  ]
-  s.homepage = %q{http://github.com/dimus/dwc-archive}
-  s.rdoc_options = ["--charset=UTF-8"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.6}
-  s.summary = %q{Handler of Darwin Core Archive files}
-  s.test_files = [
-    "spec/dwc-archive_spec.rb",
-     "spec/lib/ruby_extenstions_spec.rb",
-     "spec/spec_helper.rb"
-  ]
+Gem::Specification.new do |gem|
+  gem.name          = "dwc-archive"
+  gem.version       = DarwinCore::VERSION
+  gem.authors       = ["Dmitry Mozzherin"]
+  gem.email         = ["dmozzherin at gmail dot com"]
+  gem.description   = "Darwin Core Archive is the current standard exchange " \
+                      "format for GLobal Names Architecture modules. " \
+                      "This gem makes it easy to incorporate files in " \
+                      "Darwin Core Archive format into a ruby project."
+  gem.summary       = "Handler of Darwin Core Archive files"
+  gem.homepage      = "http://github.com/GlobalNamesArchitecture/dwc-archive"
+  gem.license       = "MIT"
 
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
+  gem.required_ruby_version = ">= 2.6.0"
+  gem.files         = `git ls-files`.split("\n").map(&:strip)
+  gem.executables   = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
 
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
-      s.add_development_dependency(%q<cucumber>, [">= 0"])
-    else
-      s.add_dependency(%q<rspec>, [">= 1.2.9"])
-      s.add_dependency(%q<cucumber>, [">= 0"])
-    end
-  else
-    s.add_dependency(%q<rspec>, [">= 1.2.9"])
-    s.add_dependency(%q<cucumber>, [">= 0"])
-  end
-end
+  # gem.add_runtime_dependency "minitar", "~> 0.6"
+  gem.add_runtime_dependency "biodiversity", "~> 6.0"
+  gem.add_runtime_dependency "nokogiri", "~> 1.16"
 
+  gem.add_development_dependency "bundler", "~> 2.5"
+  gem.add_development_dependency "byebug", "~> 11.1"
+  gem.add_development_dependency "cucumber", "~> 9"
+  gem.add_development_dependency "git", "~> 2.3"
+  gem.add_development_dependency "rake", "~> 13.2"
+  gem.add_development_dependency "rspec", "~> 3.13"
+  gem.add_development_dependency "rubocop", "~> 1.66"
+gem.add_development_dependency "ruby-lsp", "~> 0.17"
+end

data/features/dwca-creator.feature

@@ -0,0 +1,40 @@
+Feature: Creating and writing a Darwin Core Archive
+  In order to communicate with DwCA compatible programs
+  A User should be able to
+  Save data from ruby objects into Darwin Core Archive file
+
+  Scenario: Creating Core File
+    Given an array of urls for Darwin Core or other terms
+    And arrays of data in the order correpsonding to order of terms
+    When User creates generator
+    And User sends this data to core generator
+    Then these data should be saved as "darwin_core.txt" file
+
+  Scenario: Creating Extensions
+    Given 2 sets of data with terms as urls in the header
+    When User creates generator
+    And User adds extensions with file names "vernacular.txt" and "synonyms.txt"
+    Then data are saved as "vernacular.txt" and "synonyms.txt"
+
+  Scenario: Creating metadata.xml and eml.xml
+    Given an array of urls for Darwin Core or other terms
+    And arrays of data in the order correpsonding to order of terms
+    And 2 sets of data with terms as urls in the header
+    When User creates generator
+    And User sends this data to core generator
+    And User adds extensions with file names "vernacular.txt" and "synonyms.txt"
+    And User generates meta.xml and eml.xml
+    Then there should be "meta.xml" file with core and extensions informations
+    And there should be "eml.xml" file with authoriship information
+
+  Scenario: Making DarwinCore Archive file
+    Given an array of urls for Darwin Core or other terms
+    And arrays of data in the order correpsonding to order of terms
+    And 2 sets of data with terms as urls in the header
+    When User creates generator
+    And User sends this data to core generator
+    And User adds extensions with file names "vernacular.txt" and "synonyms.txt"
+    And User generates meta.xml and eml.xml
+    And generates archive
+    Then there should be a valid new archive file
+

data/features/dwca-reader.feature

@@ -0,0 +1,88 @@
+Feature: Reading of a Darwing Core Archive
+  In order to start working with Darwin Core Archive file
+  A user should be able initiate dwc object from a file
+  So I want to implement handling of dwc object creation
+
+  Scenario: Creating Darwin Core Archive object
+    Given path to a dwc file "data.tar.gz"
+    When I create a new DarwinCore::Archive instance
+    Then I should find that the archive is valid
+    And I should see what files the archive has
+
+    When I delete expanded files
+    Then they should disappear
+
+  Scenario: Instantiating DarwinCore with a file without "eml.xml"
+    Given path to a dwc file "minimal.tar.gz"
+    When I create a new DarwinCore instance
+    Then "DarwinCore_instance.metadata.data" should send instance of "NilClass" back
+
+  Scenario: Instantiating DarwinCore with tar.gz file
+    Given path to a dwc file "data.tar.gz"
+    When I create a new DarwinCore instance
+    Then instance should have a valid archive
+    And instance should have a core
+    And I should see checksum
+    When I check core data
+    Then I should find core.properties
+    And core.file_path
+    And core.id
+    And core.fields
+    And core.size
+    Then DarwinCore instance should have an extensions array
+    And every extension in array should be an instance of DarwinCore::Extension
+    And extension should have properties, data, file_path, coreid, fields
+    Then DarwinCore instance should have dwc.metadata object
+    And I should find id, title, creators, metadata provider
+
+  Scenario: Instantiating DawinCore with zip file
+    Given path to a dwc file "data.zip"
+    When I create a new DarwinCore instance
+    Then instance should have a valid archive
+
+  Scenario: Cleaning temporary directory from expanded archives
+    Given acces to DarwinCore gem
+    When I use DarwinCore.clean_all method
+    Then all temporary directories created by DarwinCore are deleted
+
+  Scenario: Importing data into memory
+    Given path to a dwc file "data.tar.gz"
+    When I create a new DarwinCore instance
+    Then I can read its content into memory
+    Then I can read extensions content into memory
+
+  Scenario: Importing data with quotes
+    Given path to a dwc file "data_with_quotes.tar.gz"
+    When I create a new DarwinCore instance
+    Then I can read its content into memory
+
+  Scenario: Importing data using block
+    Given path to a dwc file "data.tar.gz"
+    When I create a new DarwinCore instance
+    Then I can read its core content using block
+    Then I can read extensions content using block
+
+  Scenario: Normalizing classification
+    Given path to a dwc file "data.tar.gz"
+    When I create a new DarwinCore instance
+    Then I am able to use DarwinCore#normalize_classification method
+    And get normalized classification in expected format
+    And there are paths, synonyms and vernacular names in normalized classification
+    And there are local_id and global_id methods in taxons and synonyms
+    And names used in classification can be accessed by "name_strings" method
+    And vernacular names used in classification can be accessed by "vernacular_name_strings" method
+    And nodes_ids organized in trees can be accessed by "tree" method
+
+  Scenario: Normalizing classification without canonical names
+    Given path to a dwc file "data.tar.gz"
+    When I create a new DarwinCore instance
+    Then I am able to use DarwinCore#normalize_classification method
+    Then I am able to use normalize method without canonical names path
+    And get normalized classification in expected format
+    And there are id paths, no canonical names paths in normalized classification
+  
+  Scenario: Normalizing classification skipping extensions data
+    Given path to a dwc file "synonyms_in_extension.tar.gz"
+    When I create a new DarwinCore instance
+    Then I am able to use normalize method without ingesting extensions
+    And extension information is not ingested