dwc-archive 0.1.6 → 0.2.0

data/README.rdoc

@@ -22,6 +22,28 @@ Update to latest rubygems (v >= 1.3.6) which adds gemcutter sources by default.
     dwc.extensions         # array of DarwinCore Star extensions
     dwc.extensions[0].data # summary for an extension
 
+    # read content of the core data file into memory or used with a block
+    # it returns array of arrays of data
+    # rows that had a wrong encoding will be collected into errors array
+    data, errors = dwc.core.read 
+
+    # read content using a block with getting back results in sets 100 rows each
+    results = []
+    tail_data, tail_errors = dwc.core.read(100) do |data, errors|
+      results << [data, errors]
+    end
+    results << [tail_data, tail_errors]
+
+    # read content of an extension data file into memory
+    data, errors = dwc.core.extensions[0].read
+
+    # read content of an extension data using block
+    results = []
+    tail_data, tail_errors = dwc.core.extensions[0](100) do |data, errors|
+      results << [data, errors]
+    end
+    results << [tail_data, tail_errors]
+
     DarwinCore.clean_all   # remove all expanded archives
 
 == Note on Patches/Pull Requests

data/VERSION

@@ -1 +1 @@
-0.1.6
+0.2.0

data/features/dwc-archive.feature

@@ -42,3 +42,15 @@ Feature: Creation of a Darwing Core Archive
     Given acces to DarwinCore gem
     When I use DarwinCore.clean_all method
     Then all temporary directories created by DarwinCore are deleted
+
+  Scenario: Importing data into memory
+    Given path to a dwc file "data.tar.gz"
+    When I create a new DarwinCore instance
+    Then I can read its content into memory
+    Then I can read extensions content into memory
+  
+  Scenario: Importing data using block
+    Given path to a dwc file "data.tar.gz"
+    When I create a new DarwinCore instance
+    Then I can read its core content using block
+    Then I can read extensions content using block

data/features/step_definitions/dwc-archive_steps.rb

@@ -124,3 +124,40 @@ Then /^"([^\"]*)" should send instance of "([^\"]*)" back$/ do |arg1, arg2|
   res.class.to_s.should == arg2
 end
 
+Then /^I can read its content into memory$/ do
+  core_data, core_errors = @dwc.core.read
+  core_data.class.should == Array
+  core_data.size.should == 584
+  core_errors.size.should == 1
+end
+
+Then /^I can read extensions content into memory$/ do
+  ext = @dwc.extensions
+  ext.class.should == Array
+  ext_data, ext_errors = ext[0].read
+  ext_data.class.should == Array
+  ext_data.size.should == 1
+  ext_errors.size.should == 0
+end
+
+Then /^I can read its core content using block$/ do
+  res = []
+  @dwc.core.ignore_headers.should be_true
+  tail_data, tail_errors = @dwc.core.read(200) do |r, err|
+    res << [r.size, err.size]
+  end
+  res << [tail_data.size, tail_errors.size]
+  res.should == [[200,0],[200,0],[184,1]]
+end
+
+Then /^I can read extensions content using block$/ do
+  res = []
+  ext = @dwc.extensions[0]
+  ext.ignore_headers.should be_true
+  tail_data, tail_errors = ext.read(200) do |r, err|
+    res << [r.size, err.size]
+  end
+  res << [tail_data.size, tail_errors.size]
+  res.should == [[1,0]]
+end
+

data/lib/dwc-archive.rb

@@ -3,6 +3,13 @@ $:.unshift(File.dirname(__FILE__)) unless
    $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))   
 require 'ruby_extensions'
 require 'fileutils'
+begin
+  require 'fastercsv'
+  CSV = FasterCSV
+rescue LoadError
+  require 'csv'
+end
+require 'dwc-archive/ingester'
 require 'dwc-archive/errors'
 require 'dwc-archive/expander'
 require 'dwc-archive/archive'
@@ -15,6 +22,17 @@ class DarwinCore
   alias :eml :metadata
   
   DEFAULT_TMP_DIR = "/tmp"
+  UTF8RGX = /\A(
+        [\x09\x0A\x0D\x20-\x7E]            # ASCII
+      | [\xC2-\xDF][\x80-\xBF]             # non-overlong 2-byte
+      |  \xE0[\xA0-\xBF][\x80-\xBF]        # excluding overlongs
+      | [\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}  # straight 3-byte
+      |  \xED[\x80-\x9F][\x80-\xBF]        # excluding surrogates
+      |  \xF0[\x90-\xBF][\x80-\xBF]{2}     # planes 1-3
+      | [\xF1-\xF3][\x80-\xBF]{3}          # planes 4-15
+      |  \xF4[\x80-\x8F][\x80-\xBF]{2}     # plane 16
+    )*\z/x unless defined? UTF8RGX
+
   
   def initialize(dwc_path, tmp_dir = DEFAULT_TMP_DIR)
     @archive = DarwinCore::Archive.new(dwc_path, tmp_dir) 
@@ -38,7 +56,7 @@ class DarwinCore
     root_key = @archive.meta.keys[0]
     ext = @archive.meta[root_key][:extension]
     return [] unless ext
-    ext = [ext] unless ext.class == Array
+    ext = [ext] if ext.class != Array
     ext.map { |e| DarwinCore::Extension.new(@archive, e) }
   end
 end

data/lib/dwc-archive/core.rb

@@ -1,34 +1,23 @@
 class DarwinCore
   class Core
+    include DarwinCore::Ingester
+    attr_reader :id 
     def initialize(archive)
       @archive = archive
       @path = @archive.files_path
       root_key = @archive.meta.keys[0]
-      @core = @archive.meta[root_key][:core]
-      raise "Cannot found core in meta.xml, is meta.xml valid?" unless @core
+      @data = @archive.meta[root_key][:core]
+      raise CoreFileError("Cannot found core in meta.xml, is meta.xml valid?") unless @data
+      @properties = @data[:attributes]
+      @encoding = @properties[:encoding] || 'UTF-8'
+      raise CoreFileError("No support for encodings other than utf-8 or utf-16 at the moment") unless ["utf-8", "utf8", "utf-16", "utf16"].include? @encoding.downcase
+      @id = @data[:id][:attributes]
+      @field_separator = get_field_separator
+      @quote_character = @properties[:fieldsEnclosedBy] || ""
+      @line_separator = @properties[:linesTerminatedBy] || "\n"
+      @ignore_headers = @properties[:ignoreHeaderLines] ? [1, true].include?(@properties[:ignoreHeaderLines]) : false
+      @file_path = get_file_path
+      @fields = get_fields
     end
-
-    def data
-      @core
-    end
-
-    def properties
-      @core[:attributes]
-    end
-
-    def file_path
-      file = @core[:files][:location] || @core[:location]
-      File.join(@path, file)
-    end
-
-    def id
-      @core[:id][:attributes]
-    end
-
-    def fields
-      @core[:field] = [@core[:field]] unless @core[:field].class == Array
-      @core[:field].map {|f| f[:attributes]}
-    end
-    
   end
 end

data/lib/dwc-archive/errors.rb

@@ -3,4 +3,5 @@ class DarwinCore
   class FileNotFoundError < Error; end
   class UnpackingError < Error; end
   class InvalidArchiveError < Error; end
+  class CoreFileError < Error; end
 end

data/lib/dwc-archive/extension.rb

@@ -1,31 +1,22 @@
 class DarwinCore
   class Extension
-    def initialize(archive, extension)
+    include DarwinCore::Ingester
+    attr_reader :coreid
+
+    def initialize(archive, data)
       @archive = archive
       @path = @archive.files_path
-      @extension = extension
-    end
-
-    def data
-      @extension
+      @data = data
+      @properties = @data[:attributes]
+      @coreid = @data[:coreid][:attributes]
+      @encoding = @properties[:encoding] || 'UTF-8'
+      @quote_character = @properties[:fieldsEnclosedBy] || ""
+      @line_separator = @properties[:linesTerminatedBy] || "\n"
+      @ignore_headers = @properties[:ignoreHeaderLines] ? [1, true].include?(@properties[:ignoreHeaderLines]) : false
+      @field_separator = get_field_separator
+      @file_path = get_file_path
+      @fields = get_fields
     end
 
-    def properties
-      @extension[:attributes]
-    end
-    
-    def file_path
-      file = @extension[:files][:location] 
-      File.join(@path, file)
-    end
-
-    def coreid
-      @extension[:coreid][:attributes]
-    end
-
-    def fields
-      @extension[:field] = [@extension[:field]] unless @extension[:field].class == Array
-      @extension[:field].map {|f| f[:attributes]}
-    end
   end
 end

data/lib/dwc-archive/ingester.rb

@@ -0,0 +1,46 @@
+class DarwinCore
+  module Ingester
+    attr_reader :data, :properties, :encoding, :fields_separator
+    attr_reader :file_path, :fields, :line_separator, :quote_character, :ignore_headers
+    def read(batch_size = 10000)
+      res = []
+      errors = []
+      index_fix = 1
+      args = {:col_sep => @field_separator}
+      args.merge!({:quote_char => @quote_character}) if @quote_character != ''
+      CSV.open(@file_path, args).each_with_index do |r, i|
+        index_fix = 0; next if @ignore_headers && i == 0
+        str = r.join('')
+        if defined? FasterCSV
+          UTF8RGX === str ? res << r : errors << r
+        else
+          str = str.force_encoding('utf-8')
+          str.encoding.name == "UTF-8" && str.valid_encoding? ? res << r : errors << r
+        end
+        if block_given? && (i + index_fix) % batch_size == 0
+          yield [res, errors]
+          res = []
+          errors = []
+        end
+      end
+      [res, errors]
+    end
+    
+    private
+    def get_file_path
+      file = @data[:location] || @data[:attributes][:location] || @data[:files][:location]
+      File.join(@path, file)
+    end
+
+    def get_fields
+      @data[:field] = [data[:field]] if data[:field].class != Array
+      @data[:field].map {|f| f[:attributes]}
+    end
+
+    def get_field_separator
+      res = @properties[:fieldsTerminatedBy] || ','
+      res = "\t" if res == "\\t"
+      res
+    end
+  end
+end

data/lib/ruby_extensions.rb

@@ -52,7 +52,10 @@ class Hash
     end          
 
     def prepare(data)
-      (data.class == String && data.to_i.to_s == data) ? data.to_i : data
+      return data if data.class != String
+      data = true if data.strip == "true"
+      data = false if data.strip == "false"
+      data.to_i.to_s == data ? data.to_i : data
     end
   end
   

data/spec/lib/dwc-archive_spec.rb

@@ -6,9 +6,8 @@ describe DarwinCore do
   end
 
   describe ".new" do 
-
     it "should create DarwinCore instance out of archive file" do
-      ['data.zip', 'data.tar.gz', 'minimal.tar.gz'].each do |file|
+      ['data.zip', 'data.tar.gz', 'minimal.tar.gz', 'junk_dir_inside.zip'].each do |file|
         file = File.join(@file_dir, file)
         dwc = DarwinCore.new(file)
         dwc.archive.valid?.should be_true
@@ -35,6 +34,5 @@ describe DarwinCore do
       dwc = DarwinCore.new(file)
       dwc.archive.valid?.should be_true
     end
-
   end
 end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 1
-  - 6
-  version: 0.1.6
+  - 2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Dmitry Mozzherin
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-06 00:00:00 -04:00
+date: 2010-05-19 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -69,6 +69,7 @@ files:
 - lib/dwc-archive/errors.rb
 - lib/dwc-archive/expander.rb
 - lib/dwc-archive/extension.rb
+- lib/dwc-archive/ingester.rb
 - lib/dwc-archive/metadata.rb
 - lib/ruby_extensions.rb
 - spec/files/broken.tar.gz
@@ -77,6 +78,7 @@ files:
 - spec/files/eml.xml
 - spec/files/file with characters(3).gz
 - spec/files/invalid.tar.gz
+- spec/files/junk_dir_inside.zip
 - spec/files/meta.xml
 - spec/files/minimal.tar.gz
 - spec/files/uncompressed