data_tools 0.6.0 → 0.6.4

data/.gitignore

@@ -3,3 +3,4 @@
 pkg
 todo.txt
 .DS_Store
+tmp

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    data_tools (0.6.0)
+    data_tools (0.6.2)
       awesome_print
       facets
 

data/lib/data_tools.rb

@@ -8,7 +8,7 @@ module DataTools
     $".grep(/data_tools/).each {|f| load(f)}
   end
 
-  def DataTools.scour(s, opts)
+  def DataTools.scour(s, opts = {})
     case s
     when nil
       nil
@@ -18,7 +18,7 @@ module DataTools
         # looks numeric
         s2 = s2.to_i.to_s
       end
-      (s2.empty? || opts[:junkwords].include?(s2)) ? nil : s2
+      (s2.empty? || (opts[:junkwords]||[]).include?(s2)) ? nil : s2
     when Numeric
       s.to_s
     else

data/lib/data_tools/array.rb

@@ -37,6 +37,4 @@ module DataTools::Array
     File.unlink(filename) if File.exists?(filename)
     File.open(filename, "w") {|f| f << Marshal.dump(self)}
   end
-
-
 end

data/lib/data_tools/array_of_hashes.rb

@@ -123,18 +123,18 @@ module DataTools::ArrayOfHashes
     File.open(filename, "w") {|f| f << Marshal.dump(self)}
   end
 
-  # attempt to dump out contents of this array-of-hashes as CSV to named file
-  # fields is list of attribute names to write out
-  # options headers is public names for the fields
-  def csvme(filename, fields, headers = fields)
-    CSV.open(filename, "wb") do |csv|
-      csv << headers unless headers.nil?
-      pluck(fields).each do |ary|
-        csv << ary
-      end
-    end
-    true
-  end
+  # # attempt to dump out contents of this array-of-hashes as CSV to named file
+  # # fields is list of attribute names to write out
+  # # options headers is public names for the fields
+  # def csvme(filename, fields, headers = fields)
+  #   CSV.open(filename, "wb") do |csv|
+  #     csv << headers unless headers.nil?
+  #     pluck(fields).each do |ary|
+  #       csv << ary
+  #     end
+  #   end
+  #   true
+  # end
 
   def tsvme(filename, fields, headers = fields)
     File.open(target) do |output|

data/lib/data_tools/enumerator.rb

@@ -4,7 +4,26 @@ module DataTools::Enumerator
     each do |hash|
       outputstream.puts hash.pluck(fields).to_csv
     end
+    outputstream.flush # otherwise missing rows might not get pushed out
     outputstream
+  rescue Errno::EPIPE
+    # output was closed, that's fine
+  end
+
+  def lazy_select(&block)
+    Enumerator.new do |yielder|
+      self.each do |val|
+        yielder.yield(val) if block.call(val)
+      end
+    end
+  end
+
+  def lazy_map(&block)
+    Enumerator.new do |yielder|
+      self.each do |value|
+        yielder.yield(block.call(value))
+      end
+    end
   end
 end
 

data/lib/data_tools/hash.rb

@@ -180,44 +180,23 @@ module DataTools::Hash
     end
   end
 
-  # # HASH OF ARRAYS
-  # def coalesce!(args)
-  #   rules = args[:per]
-  #   rules.each do |from, to|
-  #     if self[to].nil?
-  #       raise "cannot merge #{from} into #{to}, destination does not exist"
-  #     end
-  #     if self[from].nil?
-  #       $stderr.puts "cannot merge #{from} into #{to}, source does not exist, ignoring"
-  #       next
-  #     end
-  #     self[to] += self[from]
-  #     self.delete(from)
-  #   end
-  #   self
-  # end
-
   def cleanse(options = {})
     each_with_object({}) do |(k,v), out|
       out[k] = DataTools.scour(v, options)
       if dateformat = options[:datefields][k]
         begin
-          out[k] = DateTime.strptime(v, dateformat).to_time
-        rescue
-          warn "invalid #{k} (expected #{dateformat}): #{rec}"
+          out[k] = v && DateTime.strptime(v, dateformat).to_date
+        rescue ArgumentError
+          warn "expected '#{dateformat}' in #{k} = '#{v}' at [#{options[:line]}]: #{self}"
+          out[k] = nil
         end
       end
     end
   end
 
-  def subset(keys)
-    map do |h|
-      h.select {|k,v| keys.include? k}
-    end
-  end
-
-  def pluck(*keys)
-    keys.flatten.map {|k| self[k]}
+  def pluck(keys)
+    keys.map {|k| self[k]}
+    # keys.flatten.map {|k| self[k]}
   end
 end
 

data/lib/data_tools/io.rb

@@ -1,27 +1,32 @@
 require "csv"
 
 module DataTools::IO
+  attr_reader :headers, :import_options
+
   def unmarshal
     Marshal.load(self)
   end
 
-  def headers
-    @import_headers ||= @import_options[:headers] || behead
-  end
-
   def split(line)
-    case import_options[:format]
-    when :tsv
+    fields = case import_options[:format]
+    when :tsv # tab-delimited
       line.split("\t")
-    when :qcq
+    when :wsv # whitespace-delimited
+      line.split
+    when :qcq # quote-comma-quote (*not* the same as CSV)
       line.split('","')
-    else # default is CSV
+    else # default is :csv
       line.parse_csv
     end
+
+    fields.map {|f| DataTools.scour(f)}
   end
 
   def parseline(line)
-    split(line.chomp)
+    @linenumber += 1
+    # remove leading and trailing line endings (CR or LF)
+    # but NOT whitespace, because e.g. there could be leading or trailing blank fields delimited by tabs
+    split(line.gsub(/^[\n\r]*|[\n\r]*$/, ''))
   end
 
   def import_options
@@ -35,15 +40,20 @@ module DataTools::IO
     import_options.merge!(options)
   end
 
+  def line_to_record(line)
+    Hash[headers.zip(parseline(line)).select {|k,v| !v.nil?}]
+  end
+
   def import(opts = {}) # expects a block
     configure_import(opts)
-    headers = opts[:headers] || parseline(readline)
-    # warn "HEADERS ARE #{headers}"
+    @linenumber = 0
+    @headers = opts[:headers] || parseline(readline(opts[:rowsep] || $/))
     Enumerator.new do |yielder|
-      self.each do |line|
-        rec = Hash[headers.zip(parseline(line))]
+      self.each(opts[:rowsep] || $/) do |line|
+        rec = line_to_record(line)
+        next if rec.empty? # silently ignore blank records
         rec.extend DataTools::Hash
-        yielder.yield rec.cleanse(import_options)
+        yielder.yield rec.cleanse(import_options.merge(:line => @linenumber))
       end
       # need to emit anything to trigger a file-completed action? (such as pushing a batch to storage)
     end
@@ -53,3 +63,5 @@ end
 class IO
   include DataTools::IO
 end
+
+ARGF.extend DataTools::IO

data/lib/data_tools/version.rb

@@ -1,3 +1,3 @@
 module DataTools
-  VERSION = "0.6.0"
+  VERSION = "0.6.4"
 end

data/spec/import_spec.rb

@@ -3,8 +3,8 @@ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 require "json"
 
 describe "File Import" do
-  it "imports" do
-    f = File.open(File.dirname(__FILE__) + "/../hrhead.csv")
+  it "imports CSV" do
+    f = File.open(File.dirname(__FILE__) + "/../tmp/hrhead.csv")
     # sio = f.import.csvme(StringIO.new, ['Person Phone GUID', 'Person Address GUID'])
     # puts sio.string
 
@@ -12,4 +12,11 @@ describe "File Import" do
       puts slice.extend(DataTools::ArrayOfHashes).pluck('Person Phone GUID', 'Person Address GUID').to_json
     end
   end
+
+  it "import WSV" do
+    f = File.open(File.dirname(__FILE__) + "/../tmp/visits.txt")
+    recs = f.import(:format => :wsv, :datefields => {'admit_arrive_date' => '%Y-%m-%d'}).to_a
+    recs.count.should == 99
+    puts recs.sample
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_tools
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-23 00:00:00.000000000 Z
+date: 2013-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
@@ -118,3 +118,4 @@ signing_key:
 specification_version: 3
 summary: Miscellaneous data-munging utilities.
 test_files: []
+has_rdoc: