csv-autoparser 0.1.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b806b5d4df47de7532a01384fa357b998e53653a
-  data.tar.gz: 1ad970d72389cb2a061127d69c584ed8be1e0244
+  metadata.gz: 202d8f86d22651aeecb94ab9b5a839c1df8d0117
+  data.tar.gz: 06df0597fc5d4bd6df4f9f7da5ded7bdda5542c2
 SHA512:
-  metadata.gz: 9263abf623d4ffb74ad65ba9243f9fa36e9924ba7787f7c9619bbbf866a3843764ee2f0a1476af6331c16172eed75f948340562c8750ca37b47260b803594b5a
-  data.tar.gz: 07d207dc0bd482d688d721a714bb4caa8527925f521099f47dfe6bb0cf485645811c6d598a572e24eb86f222868909e0246fcf2040d42d9e5f6f63708f3afedc
+  metadata.gz: 409f3c4d1505698a2e248309d061a0c77c98c965081361a058756a215eb5eb3601b5385604065a231482dde5b55af48e4cf6b58d9a4200926c42c1ced7521a90
+  data.tar.gz: 9fc72adf4b37cd1b86fbe62e0f47e73731aa7c70963192bc4e6cf4f5afef6feeb71c9673a98c4f0a20a2880d376459f24abaf5a7f6239a9a99a36c310b0924f1

data/README.md

@@ -1,6 +1,12 @@
 # CSV::AutoParser
 
-CSV::AutoParser automatically parses a CSV file given a user specified header row.
+CSV::AutoParser automatically parses a CSV file given a user specified header row and
+adds method style accessors to the CSV::Row data. 
+
+All functionality of the standard Ruby CSV class is accessible since CSV::AutoParser
+inherits from CSV. CSV::AutoParser objects behave just like CSV objects when not
+provided a block (though method style accessors will still be avaliable on CSV::Row
+objects).
 
 ## Installation
 
@@ -8,18 +14,36 @@ CSV::AutoParser automatically parses a CSV file given a user specified header ro
 
 ## Usage
 
-    require 'csv/autoparser'
-
-    # ID header row by CSV line number.
-    csv = CSV::AutoParser.new("my_file.csv") {|csv_line_number, header_row| csv_line_number == 1 }
-    csv.rows.each {|row| puts row.full_name }
-
-    # -OR- ID header row by column header names.
-    csv = CSV::AutoParser.new(input_file) do |line_num, header_row| 
-      ["name", "Job title"].all? {|cell| header_row.include?(cell) } 
-    end
-    puts csv.rows.first.name # => "Jon Smith"
-    csv.rows.first.job_title # => "blacksmith"
+```ruby
+require 'csv/autoparser'
+
+data = <<CSV
+"this is",not,"the header"
+"the real header","is easy","to find"
+name,"Job title",age
+"Jon Smith",blacksmith,55
+"Jimmy Johnson",farmer,34
+"Kimmy Kimmson","pig wrangler",29
+CSV
+
+# ID header row by CSV line number.
+csv = CSV::AutoParser.new(data) {|line_number, header_row| line_number == 3 }
+csv.each {|row| puts "#{row.name} is a #{row.age} year old #{row.job_title}." }
+# Jon Smith is a 55 year old blacksmith.
+# Jimmy Johnson is a 34 year old farmer.
+# Kimmy Kimmson is a 29 year old pig wrangler.
+
+# -OR- ID header row by column header names.
+csv = CSV::AutoParser.new(data) do |line_num, header_row| 
+  ["name", "Job title"].all? {|field| header_row.include?(field) } 
+end
+csv.is_a?(CSV)        # => true
+table = csv.read      # => CSV::Table
+table.first.name      # => "Jon Smith"
+table[-1].job_title   # => "pig wrangler"
+```
+
+More usage examples can be seen in examples/.
 
 ## Contributing
 

data/examples/basic.rb

@@ -0,0 +1,72 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+
+require 'csv/autoparser'
+
+data = <<CSV
+"this is",not,"the header"
+"the real header","is easy","to find"
+name,"Job title",age
+"Jon Smith",blacksmith,55
+"Jimmy Johnson",farmer,34
+"Kimmy Kimmson","pig wrangler",29
+CSV
+
+########################################
+# Indentifying the Header Row
+########################################
+
+# ID header row by CSV line number.
+csv = CSV::AutoParser.new(data) {|line_number, header_row| line_number == 3 }
+csv.each {|row| puts "#{row.name} is a #{row.age} year old #{row.job_title}." }
+# Jon Smith is a 55 year old blacksmith.
+# Jimmy Johnson is a 34 year old farmer.
+# Kimmy Kimmson is a 29 year old pig wrangler.
+
+# -OR- ID header row by column header names.
+csv = CSV::AutoParser.new(data) do |line_num, header_row| 
+["name", "Job title"].all? {|field| header_row.include?(field) } 
+end
+csv.is_a?(CSV)        # => true
+table = csv.read      # => CSV::Table
+table.first.name      # => "Jon Smith"
+table[-1].job_title   # => "pig wrangler"
+
+
+################################################
+# Want to get at the data above the header row?
+################################################
+
+csv = CSV::AutoParser.new(File.expand_path("../../test/fixtures/persons.csv", __FILE__)) {|lineno, hr| lineno == 3 }
+csv.pre_header_rows.each do |row|
+  puts "#{row.file}(#{row.line}): #{row.inspect}"
+end
+
+
+########################################
+# Optional Header Columns
+########################################
+
+data2 = <<CSV
+name,"city  name","Job title"
+"Jon Smith",Sacramento,blacksmith
+"Jimmy Johnson","San Diego",farmer
+"Kimmy Kimmson",Austin,"pig wrangler"
+CSV
+
+# When CSV columns are optional, specify it through options so that a NoMethodError is not raised.
+def demo_optional_header_columns data
+  csv = CSV::AutoParser.new(data, optional_headers: ["City Name", :age]) {|lineno, hr| hr.include? "name" }
+  csv.each do |row| 
+    # row.birth_date # => raises NoMethodError
+    if row.city_name
+      puts "#{row.name} is a #{row.age || "?"} year old #{row.job_title} living in #{row.city_name}." 
+    else
+      puts "#{row.name} is a #{row.age || "?"} year old #{row.job_title}." 
+    end
+  end
+end
+
+demo_optional_header_columns(data)
+demo_optional_header_columns(data2)
+

data/lib/csv/autoparser.rb

@@ -2,46 +2,98 @@ require "csv"
 require "csv/autoparser/version"
 
 class CSV
-  class AutoParser
 
-    class Row < Array
-      attr_reader :csv_file, :csv_line
+  class Row
+    alias_method :orig_initialize, :initialize
+    # Defines method style accessors based on header row names.
+    def initialize(*args)
+      orig_initialize(*args)
+      if field_row?
+        headers.each do |h|
+          define_singleton_method(CSV::AutoParser.convert_header_to_method_name(h)) { fetch(h) }
+        end
+      end
+    end
+  end
+
+  class AutoParser < CSV
+
+    # This is the method called by AutoParser to turn header names into legal method names.
+    # Redefine as necessary.
+    def self.convert_header_to_method_name header
+      header.to_s.downcase.strip.gsub(/\s+/, '_').gsub(/-+/, '_').gsub(/[^\w]/, '').to_sym
+    end
+
+    # The rows found before the header row are paired with file and line information. These
+    # objects are available through CSV::AutoParser#pre_header_rows.
+    class PreHeaderRow < Array
+      attr_reader :file, :line
       def self.create original_row, file, line
-        row = Row.new(original_row)
-        row.instance_eval { @csv_file = file; @csv_line = line }
+        row = PreHeaderRow.new(original_row)
+        row.instance_eval { @file = file; @line = line }
         return row
       end
     end
 
     class HeaderRowNotFound < RuntimeError; end
 
-    attr_reader :pre_header_rows, :rows
+    attr_reader :pre_header_rows, :header_line_number
 
-    def initialize file, &is_header
-      map = {}
-      csv_line_number = 0
-      @rows = []
+    # +data+ can be path of CSV file in addition to a CSV String or an IO object like CSV.new.
+    # All CSV.new options are supported via +opts+. If an +&is_header+ block is provided, it 
+    # takes precedence over the CSV.new +:headers+ option. A +:optional_headers+ option has
+    # been added for specifying headers that may not be present in the CSV, but you do not want
+    # a NoMethodError to raise when accessing a field using the header method style accessor.
+    def initialize data, opts={}, &is_header
+      @header_line_number = nil
       @pre_header_rows = []
-      CSV.foreach(file) do |row| 
-        csv_line_number += 1
-        if map.empty?
+      @optional_headers = [opts.delete(:optional_headers)].flatten.compact
+      if data.is_a?(String) and File.exists?(data)
+        file = data
+        data = File.open(data) 
+      end
+      if block_given?
+        data_io = if data.is_a?(IO)
+                    data
+                  elsif data.is_a?(String)
+                    StringIO.new(data)
+                  else
+                    raise ArgumentError, "data must be a path to a CSV file, a CSV formatted String, or an IO object."
+                  end
+        header_pos = data_io.pos
+        csv_line_number = 0
+        header_finder = CSV.new(data_io, opts.merge(:headers => false)).each do |row| 
+          csv_line_number += 1
           if is_header.call(csv_line_number, row)
-            row.each_index {|index| map[row[index]] = index } 
+            @header_line_number = csv_line_number
+            break
           else
-            @pre_header_rows << Row::create(row, file, csv_line_number)
-          end
-        else
-          @rows << Row::create(row, file, csv_line_number)
-          map.each_pair do |column_name, column_offset|
-            @rows.last.define_singleton_method(column_to_method_name(column_name)) { self[column_offset] }
+            @pre_header_rows << CSV::AutoParser::PreHeaderRow.create(row, file, csv_line_number)
           end
+          header_pos = data_io.pos
         end
+        raise HeaderRowNotFound, "Could not find header row#{file ? " in #{file}" : "" }." if @header_line_number.nil?
+        data_io.seek header_pos
+        data_io = StringIO.new(data_io.read)
+        super(data_io, opts.merge(:headers => true))
+      else
+        @header_line_number = 1 if opts[:headers] == :first_row or opts[:headers] == true
+        super(data, opts)
       end
-      raise HeaderRowNotFound, "Could not find header row in #{file}." if map.empty?
     end
 
-    def column_to_method_name name
-      name.downcase.strip.gsub(/\s+/, '_').gsub(/-+/, '_').gsub(/[^\w]/, '')
+    alias_method :orig_shift, :shift
+
+    # Overriden to add methods for optional headers which were not present in the CSV.
+    def shift
+      row = orig_shift
+      [@optional_headers].flatten.compact.each do |h|
+        method_name = self.class.convert_header_to_method_name(h)
+        unless row.respond_to? method_name
+          row.define_singleton_method(method_name) {nil}
+        end
+      end
+      return row
     end
 
   end

data/lib/csv/autoparser/version.rb

@@ -1,5 +1,5 @@
 class CSV
-  class AutoParser
-    VERSION = "0.1.0"
+  class AutoParser < CSV
+    VERSION = "1.0.0"
   end
 end

data/test/test_csv/autoparser.rb

@@ -4,29 +4,34 @@ require 'minitest_helper'
 describe CSV::AutoParser do
 
   it "it can parse a csv automatically via csv line number id" do
-    csv = CSV::AutoParser.new(fixture_file_path('persons.csv')) {|csv_line_number, header_row| csv_line_number == 1 }
-    csv.rows.length.must_equal 5
-    csv.rows.first.full_name.must_equal "bob"
+    table = CSV::AutoParser.new(fixture_file_path('persons.csv')) {|csv_line_number, header_row| csv_line_number == 1 }.read
+    table.class.must_equal CSV::Table
+    table.length.must_equal 5
+    table.first["full-name"].must_equal "bob"
+    table.first.full_name.must_equal "bob"
   end
 
   it "it can parse a csv automatically via header row id" do
-    csv = CSV::AutoParser.new(fixture_file_path('persons.csv')) do |line_num, header_row| 
+    table = CSV::AutoParser.new(fixture_file_path('persons.csv'), header_converters: :symbol) do |line_num, header_row| 
       ["name", "Job title"].all? {|cell| header_row.include?(cell) } 
-    end
-    csv.rows.length.must_equal 3
-    csv.rows.first.name.must_equal "Jon Smith"
-    csv.rows.first.job_title.must_equal "blacksmith"
-    csv.rows.last.age.to_i.must_equal 29
+    end.read
+    table.length.must_equal 3
+    table.first.name.must_equal "Jon Smith"
+    table.first["Job title"].must_be_nil
+    table.first[:job_title].must_equal "blacksmith"
+    table.first.job_title.must_equal "blacksmith"
+    table[-1].age.to_i.must_equal 29
   end
 
   it "it will give you the rows found before the header row" do
-    csv = CSV::AutoParser.new(fixture_file_path('persons.csv')) do |line_num, header_row| 
+    parser = CSV::AutoParser.new(fixture_file_path('persons.csv')) do |line_num, header_row| 
       ["name", "Job title"].all? {|cell| header_row.include?(cell) } 
     end
-    csv.pre_header_rows.first.last.must_equal "years of age"
-    csv.pre_header_rows.last.first.must_equal "bob"
-    File.basename(csv.pre_header_rows.last.csv_file).must_equal "persons.csv"
-    csv.pre_header_rows.last.csv_line.must_equal 2
+    parser.header_line_number.must_equal 3
+    parser.pre_header_rows.first.last.must_equal "years of age"
+    parser.pre_header_rows.last.first.must_equal "bob"
+    File.basename(parser.pre_header_rows.last.file).must_equal "persons.csv"
+    parser.pre_header_rows.last.line.must_equal 2
   end
 
   it "will raise an exception if it can't find the header row" do
@@ -35,24 +40,79 @@ describe CSV::AutoParser do
   end
 
   it "will not confuse column information with another csv which is parsed simultaneously" do
-    csv = CSV::AutoParser.new(fixture_file_path('persons.csv')) do |line_num, header_row| 
+    table = CSV::AutoParser.new(fixture_file_path('persons.csv')) do |line_num, header_row| 
       ["name", "Job title"].all? {|cell| header_row.include?(cell) } 
-    end
-    csv2 = CSV::AutoParser.new(fixture_file_path('persons2.csv')) do |line_num, header_row| 
+    end.read
+    table2 = CSV::AutoParser.new(fixture_file_path('persons2.csv')) do |line_num, header_row| 
+      ["name", "Job title"].all? {|cell| header_row.include?(cell) } 
+    end.read
+    table.length.must_equal 3
+    table.first.name.must_equal "Jon Smith"
+    table.first.job_title.must_equal "blacksmith"
+    table[-1].age.to_i.must_equal 29
+    table[1].job_title.must_equal "farmer"
+
+    table2.length.must_equal 2
+    table2.first.name.must_equal "Kermy Frog"
+    table2.first.job_title.must_equal "frog"
+    table2[-1].age.to_i.must_equal 19
+  end
+
+  it "will define methods which return nil for optional columns not present in CSV" do
+    table1 = CSV::AutoParser.new(fixture_file_path('persons.csv')) do |line_num, header_row| 
+      ["name", "Job title"].all? {|cell| header_row.include?(cell) } 
+    end.read
+    lambda { table1.first.my_optional_header }.must_raise(NoMethodError)
+    table2 = CSV::AutoParser.new(fixture_file_path('persons.csv'), optional_headers: [:my_optional_header, "Zip Code", :name]) do |line_num, header_row| 
       ["name", "Job title"].all? {|cell| header_row.include?(cell) } 
+    end.read
+    table2.first.my_optional_header.must_be_nil
+    lambda { table1.first.my_optional_header }.must_raise(NoMethodError)
+    table2.first.name.must_equal "Jon Smith"
+    lambda { table2.first.my_mandatory_header }.must_raise(NoMethodError)
+    table2.first.zip_code.must_be_nil
+    table2[-1].my_optional_header.must_be_nil
+  end
+
+  it "will pass along CSV.new options" do
+    parser = CSV::AutoParser.new(fixture_file_path('persons.csv')) {|l, hr| l == 1 }
+    parser.field_size_limit.must_be_nil
+    parser = CSV::AutoParser.new(fixture_file_path('persons.csv'), field_size_limit: 100) {|l, hr| l == 1 }
+    parser.field_size_limit.must_equal 100
+  end
+
+  it "should work with a CSV string or an IO object too" do
+    input_objects = [fixture_file_path('persons.csv'), File.open(fixture_file_path('persons.csv')), File.open(fixture_file_path('persons.csv'))]
+    input_objects.each do |obj|
+      table = CSV::AutoParser.new(obj) do |line_num, header_row| 
+        ["name", "Job title"].all? {|cell| header_row.include?(cell) } 
+      end.read
+      table.length.must_equal 3
+      table.first.name.must_equal "Jon Smith"
+      table.first.job_title.must_equal "blacksmith"
+      table[-1].age.to_i.must_equal 29
+    end
+  end
+
+  it "should work just like CSV.new when not passed a block except that it can now take a file path as data too" do 
+    input_objects = [fixture_file_path('persons.csv'), File.open(fixture_file_path('persons.csv')), File.open(fixture_file_path('persons.csv'))]
+    input_objects.each do |obj|
+      parser = CSV::AutoParser.new(obj, header_converters: :symbol, headers: :first_row)
+      parser.header_line_number.must_equal 1
+      parser.pre_header_rows.must_be_empty
+      table = parser.read
+      table.length.must_equal 5
+      table.first[:fullname].must_equal "bob"
+      # method names are based off of converted header names!
+      table.first.fullname.must_equal "bob"
     end
-    csv.rows.length.must_equal 3
-    csv.rows.first.name.must_equal "Jon Smith"
-    csv.rows.first.job_title.must_equal "blacksmith"
-    csv.rows.last.age.to_i.must_equal 29
-    csv.rows[1].job_title.must_equal "farmer"
-    csv.rows[1].csv_line.must_equal 5
-
-    csv2.rows.length.must_equal 2
-    csv2.rows.first.name.must_equal "Kermy Frog"
-    csv2.rows.first.job_title.must_equal "frog"
-    csv2.rows.last.age.to_i.must_equal 19
-    File.basename(csv2.rows.first.csv_file).must_equal "persons2.csv"
+    parser = CSV::AutoParser.new(fixture_file_path('persons.csv'), header_converters: :symbol, headers: "first_col,second_col,third_col,fourth_col")
+    parser.header_line_number.must_equal nil
+    parser.pre_header_rows.must_be_empty
+    table = parser.read
+    table.length.must_equal 6
+    table[0].first_col.must_equal "full-name"
+    table[1].first_col.must_equal "bob"
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csv-autoparser
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Ben Delsol
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-27 00:00:00.000000000 Z
+date: 2014-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -49,6 +49,7 @@ files:
 - README.md
 - Rakefile
 - csv-autoparser.gemspec
+- examples/basic.rb
 - lib/csv/autoparser.rb
 - lib/csv/autoparser/version.rb
 - test/fixtures/persons.csv