dwc-archive 0.2.3 → 0.3.0

data/.gitignore

@@ -20,3 +20,4 @@ pkg
 *.gemspec
 
 ## PROJECT::SPECIFIC
+tags

data/VERSION

@@ -1 +1 @@
-0.2.3
+0.3.0

data/features/dwca-creator.feature

@@ -0,0 +1,40 @@
+Feature: Creating and writing a Darwin Core Archive
+  In order to communicate with DwCA compatible programs
+  A User should be able to
+  Save data from ruby objects into Darwin Core Archive file
+
+  Scenario: Creating Core File
+    Given an array of urls for Darwin Core or other terms
+    And arrays of data in the order correpsonding to order of terms
+    When User creates generator
+    And User sends this data to core generator
+    Then these data should be saved as "darwin_core.txt" file
+
+  Scenario: Creating Extensions
+    Given 2 sets of data with terms as urls in the header
+    When User creates generator
+    And User adds extensions with file names "vernacular.txt" and "synonyms.txt"
+    Then data are saved as "vernacular.txt" and "synonyms.txt"
+
+  Scenario: Creating metadata.xml and eml.xml
+    Given an array of urls for Darwin Core or other terms
+    And arrays of data in the order correpsonding to order of terms
+    And 2 sets of data with terms as urls in the header
+    When User creates generator
+    And User sends this data to core generator
+    And User adds extensions with file names "vernacular.txt" and "synonyms.txt"
+    And User generates meta.xml and eml.xml
+    Then there should be "meta.xml" file with core and extensions informations
+    And there should be "eml.xml" file with authoriship information
+
+  Scenario: Making DarwinCore Archive file
+    Given an array of urls for Darwin Core or other terms
+    And arrays of data in the order correpsonding to order of terms
+    And 2 sets of data with terms as urls in the header
+    When User creates generator
+    And User sends this data to core generator
+    And User adds extensions with file names "vernacular.txt" and "synonyms.txt"
+    And User generates meta.xml and eml.xml
+    And generates archive
+    Then there should be a valid new archive file
+

data/features/{dwc-archive.feature → dwca-reader.feature}

@@ -1,4 +1,4 @@
-Feature: Creation of a Darwing Core Archive
+Feature: Reading of a Darwing Core Archive
   In order to start working with Darwin Core Archive file
   A user should be able initiate dwc object from a file
   So I want to implement handling of dwc object creation

data/features/step_definitions/dwc-creator_steps.rb

@@ -0,0 +1,105 @@
+#encoding: utf-8
+require 'ruby-debug'
+
+
+Given /^an array of urls for Darwin Core or other terms$/ do
+  @rows = ["http://rs.tdwg.org/dwc/terms/taxonID", "http://rs.tdwg.org/dwc/terms/parentNameUsageID", "http://rs.tdwg.org/dwc/terms/scientificName", "http://rs.tdwg.org/dwc/terms/taxonRank"]
+end
+
+Given /^arrays of data in the order correpsonding to order of terms$/ do
+  @data = [
+     [1, 0, "Plantae", "kingdom"],
+     [2, 1, "Betula", "genus"],
+     [3, 2, "Betula verucosa", "species"]
+   ]
+end
+
+When /^User sends this data to core generator$/ do
+  @data = @data.unshift @rows
+  @gen.add_core(@data, 'darwin_core.txt')
+end
+
+Then /^these data should be saved as "([^\"]*)" file$/ do |file_name|
+  file = File.join(@gen.path, file_name)
+  @gen.files.include?(file_name).should be_true
+  csv = CSV.open(file).count.should == 4
+end
+
+Given /^2 sets of data with terms as urls in the header$/ do
+  @vernaculars = [
+      ["http://rs.tdwg.org/dwc/terms/TaxonID", "http://rs.tdwg.org/dwc/terms/vernacularName"],
+      [1, "Plants"],
+      [1, "Растения"],
+      [2, "Birch"],
+      [2, "Береза"],
+      [3, "Wheeping Birch"],
+      [3, "Береза плакучая"]
+    ]
+  @synonyms = [
+      ["http://rs.tdwg.org/dwc/terms/TaxonID", "http://rs.tdwg.org/dwc/terms/scientificName", "http://rs.tdwg.org/dwc/terms/taxonomicStatus"], 
+      [1, "Betila Linnaeus, 1753", 'misspelling']
+    ]
+end
+
+When /^User creates generator$/ do
+  @gen = DarwinCore::Generator.new('/tmp/dwc.tar.gz')
+end
+
+When /^User adds extensions with file names "([^\"]*)" and "([^\"]*)"$/ do |file_name_1, file_name_2|
+  @gen.add_extension(@vernaculars, file_name_1)
+  @gen.add_extension(@synonyms, file_name_2)
+end
+
+Then /^data are saved as "([^\"]*)" and "([^\"]*)"$/ do |file_name_1, file_name_2|
+  [file_name_1, file_name_2].each do |file_name|
+    file = File.join(@gen.path, file_name)
+    @gen.files.include?(file_name).should be_true
+    csv = CSV.open(file).count.should > 1
+  end
+end
+
+When /^User generates meta\.xml and eml.xml$/ do
+  @gen.add_meta_xml
+  @gen.add_eml_xml({
+    :id => '1234',
+    :title => 'Test Classification',
+    :authors => [
+      { :first_name => 'John',
+        :last_name => 'Doe',
+        :email => 'jdoe@example.com' },
+      { :first_name => 'Jane',
+        :last_name => 'Doe', 
+        :email => 'jane@example.com' }
+      ],
+    :abstract => 'test classification',
+    :citation => 'Test classification: Doe John, Doe Jane, Taxnonmy, 10, 1, 2010',
+    :url => 'http://example.com'
+  })
+end
+
+Then /^there should be "([^\"]*)" file with core and extensions informations$/ do |file_name|
+  meta = File.join(@gen.path, file_name)
+  @gen.files.include?(file_name).should be_true
+  dom = Nokogiri::XML(open(File.join(@gen.path, file_name)))
+  dom.xpath('//xmlns:core//xmlns:location').text.should == 'darwin_core.txt'
+  dom.xpath('//xmlns:extension[1]//xmlns:location').text.should == 'vernacular.txt'
+end
+
+Then /^there should be "([^\"]*)" file with authoriship information$/ do |file_name|
+  eml = File.join(@gen.path, file_name)
+  @gen.files.include?(file_name).should be_true
+end
+
+Given /^a path to a new file \- "([^\"]*)"$/ do |file_name|
+  @dwca_file = file_name
+end
+
+When /^generates archive$/ do
+  @gen.pack
+end
+
+Then /^there should be a valid new archive file$/ do
+  dwc = DarwinCore.new('/tmp/dwc.tar.gz')
+  dwc.archive.valid?.should be_true
+end
+

data/lib/dwc-archive.rb

@@ -1,13 +1,16 @@
 # encoding: UTF-8
 $:.unshift(File.dirname(__FILE__)) unless
    $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))   
+R19 = RUBY_VERSION.split('.')[0..1].join('').to_i > 18
 require 'ruby_extensions'
 require 'fileutils'
-begin
+require 'ostruct'
+
+if R19
+  require 'csv'
+else
   require 'fastercsv'
   CSV = FasterCSV
-rescue LoadError
-  require 'csv'
 end
 require 'dwc-archive/ingester'
 require 'dwc-archive/errors'
@@ -16,6 +19,9 @@ require 'dwc-archive/archive'
 require 'dwc-archive/core'
 require 'dwc-archive/extension'
 require 'dwc-archive/metadata'
+require 'dwc-archive/generator'
+require 'dwc-archive/generator_meta_xml'
+require 'dwc-archive/generator_eml_xml'
 
 class DarwinCore
   attr_reader :archive, :core, :metadata, :extensions

data/lib/dwc-archive/errors.rb

@@ -5,4 +5,5 @@ class DarwinCore
   class InvalidArchiveError < Error; end
   class CoreFileError < Error; end
   class ExtensionFileError < Error; end
+  class GeneratorError < Error; end
 end

data/lib/dwc-archive/generator.rb

@@ -0,0 +1,73 @@
+class DarwinCore
+  class Generator
+    attr_reader :eml_xml_data
+
+    #TODO refactor -- for now copying expander methods
+    def initialize(dwc_path, tmp_dir = DEFAULT_TMP_DIR)
+      @dwc_path = dwc_path
+      @path = File.join(tmp_dir, 'dwc_' + rand(10000000000).to_s)
+      FileUtils.mkdir(@path)
+      @meta_xml_data = {:extensions => []}
+      @eml_xml_data = {:id => nil, :title => nil, :authors => [], :abstract => nil, :citation => nil, :url => nil}
+      @write = R19 ? 'w:utf-8' : 'w'
+    end
+    
+    #TODO refactor!
+    def clean
+      FileUtils.rm_rf(@path) if FileTest.exists?(@path)
+    end
+
+    def add_core(data, file_name, keep_headers = true)
+      c = CSV.open(File.join(@path,file_name), @write)
+      header = data.shift
+      fields = header.map do |f|
+        f.strip!
+        raise GeneratorError("No header in core data, or header fields are not urls") unless f.match(/^http:\/\//)
+        f.split("/")[-1]
+      end
+      data.unshift(fields) if keep_headers
+      @meta_xml_data[:core] = {:fields => header, :ignoreHeaderLines => keep_headers, :location => file_name}
+      data.each {|d| c << d}
+      c.close
+    end
+
+    def add_extension(data, file_name, keep_headers = true)
+      c = CSV.open(File.join(@path,file_name), @write)
+      header = data.shift
+      fields = header.map do |f|
+        f.strip!
+        raise GeneratorError("No header in core data, or header fields are not urls") unless f.match(/^http:\/\//)
+        f.split("/")[-1]
+      end
+      data.unshift(fields) if keep_headers
+      @meta_xml_data[:extensions] << { :fields => header, :ignoreHeaderLines => keep_headers, :location => file_name }
+      data.each { |d| c << d }
+      c.close
+    end
+
+    def add_meta_xml
+      meta = DarwinCore::Generator::MetaXml.new(@meta_xml_data, @path)
+      meta.create
+    end
+
+    def add_eml_xml(data)
+      @eml_xml_data = data
+      eml = DarwinCore::Generator::EmlXml.new(@eml_xml_data, @path)
+      eml.create
+    end
+
+    def path
+      @path
+    end
+    
+    def files
+      return nil unless @path && FileTest.exists?(@path)
+      Dir.entries(@path).select {|e| e !~ /[\.]{1,2}$/}.sort
+    end
+
+    def pack
+      a = "cd #{@path}; tar -zcf #{@dwc_path} *"
+      system(a)
+    end
+  end
+end

data/lib/dwc-archive/generator_eml_xml.rb

@@ -0,0 +1,50 @@
+class DarwinCore
+  class Generator
+    class EmlXml
+      def initialize(data, path)
+        @data = data
+        @path = path
+        @write = R19 ? 'w:utf-8' : 'w'
+      end
+      def create
+        builder = Nokogiri::XML::Builder.new do |xml|
+          xml.eml( :packageId => "eml.1.1",
+                 :system => "knb",
+                 'xmlns:eml' => "eml://ecoinformatics.org/eml-2.1.0", 
+                 'xmlns:xsi' => "http://www.w3.org/2001/XMLSchema-instance",
+                 'xsi:schemaLocation' => "eml://ecoinformatics.org/eml-2.1.0 eml.xsd" ) do
+            xml.dataset(:id => @data[:id]) do
+              xml.title(@data[:title])
+              contacts = []
+              @data[:authors].each_with_index do |a, i|
+                creator_id = i + 1
+                contacts << creator_id
+                xml.creator(:id => creator_id, :scope => 'document') do
+                  xml.individualName do
+                    xml.givenName(a[:first_name])
+                    xml.surName(a[:last_name])
+                  end
+                  xml.electronicMailAddress(a[:email])
+                end
+              end
+              xml.abstract(@data[:abstract])
+              contacts.each do |contact|
+                xml.contact { xml.references(contact) }
+              end
+            end
+            xml.additionalMetadata do
+              xml.metadata do
+                xml.citation(@data[:citation])
+              end
+            end
+            xml.parent.namespace = xml.parent.namespace_definitions.first
+          end
+        end
+        data = builder.to_xml
+        f = open(File.join(@path, 'eml.xml'), @write)
+        f.write(data)
+        f.close
+      end
+    end
+  end
+end

data/lib/dwc-archive/generator_meta_xml.rb

@@ -0,0 +1,51 @@
+class DarwinCore
+  class Generator
+    class MetaXml
+      def initialize(data, path)
+        @data = data
+        @path = path
+        @write = R19 ? 'w:utf-8' : 'w'
+      end
+
+      def create
+        builder = Nokogiri::XML::Builder.new do |xml|
+          opts = { :encoding => "UTF-8", :fieldsTerminatedBy => ",", :fieldsEnclosedBy => '"', :linesTerminatedBy => "\n", :rowType => "http://rs.tdwg.org/dwc/tems/Taxon" }
+          xml.starArchive(:xmlns => "http://rs.tdwg.org/dwc/terms/xsd/archive/",
+            "xmlns:xsi" =>"http://www.w3.org/2001/XMLSchema-instance",
+            "xsi:schemaLocation" => "http://rs.tdwg.org/dwc/terms/xsd/archive/ http://darwincore.googlecode.com/svn/trunk/text/tdwg_dwc_text.xsd",
+            :fileRoot => ".") do
+            xml.core(opts.merge(:ignoreHeaderLines => @data[:core][:ignoreHeaderLines])) do
+              xml.files { xml.location(@data[:core][:location]) }
+              taxon_id, fields = find_taxon_id(@data[:core][:fields])
+              xml.id_(:term => taxon_id[0], :index => taxon_id[1])
+              fields.each { |f| xml.field(:term => f[0], :index => f[1]) }
+            end
+            @data[:extensions].each do |e|
+              xml.extension(opts.merge(:ignoreHeaderLines => e[:ignoreHeaderLines])) do
+                xml.files { xml.location(e[:location]) }
+                taxon_id, fields = find_taxon_id(e[:fields])
+                xml.coreid(:term => taxon_id[0], :index => taxon_id[1])
+                fields.each { |f| xml.field(:term => f[0], :index => f[1]) }
+              end
+            end
+          end
+        end
+        meta_xml_data = builder.to_xml
+        meta_file = open(File.join(@path, 'meta.xml'), @write)
+        meta_file.write(meta_xml_data)
+        meta_file.close
+      end
+
+      private
+      def find_taxon_id(data)
+        fields = []
+        data.each_with_index { |f, i| fields << [f.strip, i] }
+        taxon_id, fields = fields.partition { |f| f[0].match(/\/taxonid$/i) }
+        raise GeneratorError if taxon_id.size != 1
+        [taxon_id[0], fields]
+      end
+
+    end
+  end
+end
+

data/lib/dwc-archive/ingester.rb

@@ -24,12 +24,12 @@ class DarwinCore
     private
     def process_csv_row(result, errors, row)
       str = row.join('')
-      if defined? FasterCSV
-        require File.join(File.dirname(__FILE__), 'utf_regex_ruby18')
-        UTF8RGX === str ? result << row : errors << row
-      else
+      if R19
         str = str.force_encoding('utf-8')
         str.encoding.name == "UTF-8" && str.valid_encoding? ? result << row : errors << row
+      else
+        require File.join(File.dirname(__FILE__), 'utf_regex_ruby18')
+        UTF8RGX === str ? result << row : errors << row
       end
     end
 

data/lib/dwc-archive/metadata.rb

@@ -1,6 +1,6 @@
 class DarwinCore
   class Metadata
-    def initialize(archive)
+    def initialize(archive = nil)
       @archive = archive
       @metadata = @archive.eml
     end

data/lib/ruby_extensions.rb

@@ -9,7 +9,7 @@ class Hash
         result = Nokogiri::XML(xml_io)
         return { result.root.name.to_sym => xml_node_to_hash(result.root)} 
       rescue Exception => e
-        # raise your custom exception here
+        raise e
       end
     end 
 
@@ -53,8 +53,8 @@ class Hash
 
     def prepare(data)
       return data if data.class != String
-      data = true if data.strip == "true"
-      data = false if data.strip == "false"
+      return true if data.strip == "true"
+      return false if data.strip == "false"
       data.to_i.to_s == data ? data.to_i : data
     end
   end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: dwc-archive
 version: !ruby/object:Gem::Version 
-  hash: 17
+  hash: 19
   prerelease: false
   segments: 
   - 0
-  - 2
   - 3
-  version: 0.2.3
+  - 0
+  version: 0.3.0
 platform: ruby
 authors: 
 - Dmitry Mozzherin
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-27 00:00:00 -04:00
+date: 2010-07-12 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -64,8 +64,10 @@ files:
 - README.rdoc
 - Rakefile
 - VERSION
-- features/dwc-archive.feature
-- features/step_definitions/dwc-archive_steps.rb
+- features/dwca-creator.feature
+- features/dwca-reader.feature
+- features/step_definitions/dwc-creator_steps.rb
+- features/step_definitions/dwc-reader_steps.rb
 - features/support/env.rb
 - lib/dwc-archive.rb
 - lib/dwc-archive/.expander.rb.swo
@@ -74,6 +76,9 @@ files:
 - lib/dwc-archive/errors.rb
 - lib/dwc-archive/expander.rb
 - lib/dwc-archive/extension.rb
+- lib/dwc-archive/generator.rb
+- lib/dwc-archive/generator_eml_xml.rb
+- lib/dwc-archive/generator_meta_xml.rb
 - lib/dwc-archive/ingester.rb
 - lib/dwc-archive/metadata.rb
 - lib/dwc-archive/utf_regex_ruby18.rb