csvreader 0.7.0 → 1.0.0

data/lib/csvreader/converter.rb

@@ -0,0 +1,155 @@
+# encoding: utf-8
+
+class CsvReader
+
+
+
+class Converter
+
+
+# A Regexp used to find and convert some common Date formats.
+  DATE_MATCHER     = / \A(?: (\w+,?\s+)?\w+\s+\d{1,2},?\s+\d{2,4}
+                                  |
+                            \d{4}-\d{2}-\d{2} )\z
+                    /x
+
+  # A Regexp used to find and convert some common DateTime formats.
+  DATE_TIME_MATCHER = / \A(?: (\w+,?\s+)?\w+\s+\d{1,2}\s+\d{1,2}:\d{1,2}:\d{1,2},?\s+\d{2,4}
+                      |
+            \d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}
+                      |
+            # ISO-8601
+            \d{4}-\d{2}-\d{2}
+              (?:T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?(?:[+-]\d{2}(?::\d{2})|Z)?)?)?
+        )\z
+         /x
+
+
+  CONVERTERS  = {
+       ##
+       ##  todo/fix: use regex INTEGER_MATCH / FLOAT_MATCH
+       ##    to avoid rescue (with exception and stacktrace) for every try!!!
+           integer: ->(value) {  Integer( value ) rescue value },
+           float:   ->(value) {  Float( value ) rescue value },
+           numeric:   [:integer, :float],
+           date:     ->(value) {
+             begin
+               value.match?( DATE_MATCHER ) ? Date.parse( value ) : value
+             rescue  #  date parse errors
+               value
+             end
+           },
+           date_time: ->(value) {
+             begin
+               value.match?( DATE_TIME_MATCHER ) ? DateTime.parse( value ) : value
+             rescue  # encoding conversion or date parse errors
+               value
+             end
+           },
+
+           ## new - add null and boolean (any others): why? why not?
+           null: -> (value) {
+              ## turn empty strings into nil
+              ##  rename to blank_to_nil or empty_to_nil or add both?
+              ##  todo: add NIL, nil too? or #NA, N/A etc. - why? why not?
+              if value.empty? || ['NULL', 'null', 'N/A', 'n/a', '#NA', '#na' ].include?( value )
+                 nil
+              else
+                value
+              end
+           },
+           boolean: -> (value) {
+             ## check yaml for possible true/value values - any missing?
+             ##  add more (or less) - why? why not?
+             if ['TRUE', 'true', 't', 'ON', 'on', 'YES', 'yes'].include?( value )
+               true
+             elsif
+               ['FALSE', 'false', 'f', 'OFF', 'off', 'NO', 'no'].include?( value )
+               false
+             else
+               value
+             end
+           },
+           bool: [:boolean],  ## bool convenience alias for boolean
+
+           all:  [:null, :boolean, :date_time, :numeric],
+         }
+
+
+   HEADER_CONVERTERS = {
+    downcase: ->(value) { value.downcase },
+    symbol:   ->(value) { value.downcase.gsub( /[^\s\w]+/, "" ).strip.
+                                          gsub( /\s+/, "_" ).to_sym
+                        }
+   }
+
+
+def self.create_header_converters( converters )
+  new( converters, HEADER_CONVERTERS )
+end
+
+def self.create_converters( converters )
+  new( converters, CONVERTERS )
+end
+
+
+
+def initialize( converters, registry=CONVERTERS )
+     converters = case converters
+                  when nil then []
+                  when Array then converters
+                  else [converters]
+                  end
+
+     @converters = []
+
+     converters.each do |converter|
+        if converter.is_a? Proc  # custom code block
+           add_converter( registry, &converter)
+        else   # by name
+           add_converter( converter, registry )
+        end
+     end
+   end
+
+   def to_a() @converters; end    ## todo: rename to/use converters attribute name - why? why not?
+   def empty?() @converters.empty?; end
+
+  def convert( value, index_or_header=nil )
+    return value if value.nil?
+
+    @converters.each do |converter|
+        value = if converter.arity == 1  # straight converter
+              converter.call( value )
+            else
+              ## note: for CsvReader pass in the zero-based field/column index (integer)
+              ##       for CsvHashReader pass in the header/field/column name (string)
+              converter.call( value, index_or_header )
+            end
+        break unless value.is_a?( String )  # note: short-circuit pipeline for speed
+    end
+    value  # final state of value, converted or original
+  end
+
+
+private
+
+  def add_converter( name=nil, registry, &converter )
+    if name.nil?  # custom converter
+      @converters << converter
+    else          # named converter
+      combo = registry[name]
+      case combo
+      when Array  # combo converter
+        combo.each do |converter_name|
+          add_converter( converter_name, registry )
+        end
+      else   # individual named converter
+        @converters << combo
+      end
+    end
+  end # method add_converter
+
+end  # class Converter
+
+end # class CsvReader

data/lib/csvreader/parser.rb

@@ -2,54 +2,6 @@
 
 class CsvReader
 
-class Parser
-
-## use/allow different "backends" e.g. ParserStd, ParserStrict, ParserTab, etc.
-##   parser must support parse method (with and without block)
-##    e.g.  records = parse( data )
-##             -or-
-##          parse( data ) do |record|
-##          end
-
-
-DEFAULT = ParserStd.new
-
-RFC4180 = ParserStrict.new
-STRICT  = ParserStrict.new  ## note: make strict its own instance (so you can change config without "breaking" rfc4180)
-EXCEL   = ParserStrict.new   ## note: make excel its own instance (so you can change configs without "breaking" rfc4180/strict)
-
-MYSQL   = ParserStrict.new( sep: "\t",
-                            quote: false,
-                            escape: true,
-                            null: "\\N" )
-
-POSTGRES = POSTGRESQL = ParserStrict.new( doublequote: false,
-                                          escape: true,
-                                          null: "" )
-
-POSTGRES_TEXT = POSTGRESQL_TEXT = ParserStrict.new( sep: "\t",
-                                                    quote: false,
-                                                    escape: true,
-                                                    null: "\\N" )
-
-TAB     = ParserTab.new
-
-
-def self.default()         DEFAULT;         end ## alternative alias for DEFAULT
-def self.strict()          STRICT;          end ## alternative alias for STRICT
-def self.rfc4180()         RFC4180;         end ## alternative alias for RFC4180
-def self.excel()           EXCEL;           end ## alternative alias for EXCEL
-def self.mysql()           MYSQL;           end
-def self.postgresql()      POSTGRESQL;      end
-def self.postgres()        postgresql;      end
-def self.postgresql_text() POSTGRESQL_TEXT; end
-def self.postgres_text()   postgresql_text; end
-def self.tab()             TAB;             end
-
-end # class Parser
-
-
-
 ####################################
 # define errors / exceptions
 #   for all parsers for (re)use

data/lib/csvreader/reader.rb

@@ -2,31 +2,19 @@
 
 class CsvReader
 
-  DEFAULT = CsvBuilder.new( Parser::DEFAULT )
-  STRICT  = CsvBuilder.new( Parser::STRICT )
-  RFC4180 = CsvBuilder.new( Parser::RFC4180 )
-  EXCEL   = CsvBuilder.new( Parser::EXCEL )
-  TAB     = CsvBuilder.new( Parser::TAB )
-
-  def self.default()  DEFAULT; end    ## alternative alias for DEFAULT
-  def self.strict()   STRICT; end     ## alternative alias for RFC4180
-  def self.rfc4180()  RFC4180; end    ## alternative alias for RFC4180
-  def self.excel()    EXCEL; end      ## alternative alias for EXCEL
-  def self.tab()      TAB; end        ## alternative alias for TAB
-
-
-
-
-
 #######
 ##  csv reader
 
-    def self.open( path, mode='r:bom|utf-8',
+    def self.open( path, mode=nil,
                    sep: nil,
                    converters: nil,
                    parser: nil, &block )   ## rename path to filename or name - why? why not?
-        f = File.open( path, mode )
-        csv = new(f, sep: sep, converters: converters, parser: parser )
+
+        ## note: default mode (if nil/not passed in) to 'r:bom|utf-8'
+        f = File.open( path, mode ? mode : 'r:bom|utf-8' )
+        csv = new(f, sep: sep,
+                     converters: converters,
+                     parser: parser )
 
         # handle blocks like Ruby's open(), not like the (old old) CSV library
         if block_given?
@@ -127,13 +115,15 @@ class CsvReader
           # create the IO object we will read from
           @io = data.is_a?(String) ? StringIO.new(data) : data
 
-          @sep        = sep
-          @converters = converters
+          @sep = sep
+
+          @converters  = Converter.create_converters( converters )
 
           @parser = parser.nil? ? Parser::DEFAULT : parser
     end
 
 
+
     ### IO and StringIO Delegation ###
     extend Forwardable
     def_delegators :@io,
@@ -152,13 +142,24 @@ class CsvReader
 
      def each( &block )
        if block_given?
-         kwargs = {
-           ##  converters: converters  ## todo: add converters
-         }
+         kwargs = {}
          ## note: only add separator if present/defined (not nil)
          kwargs[:sep] = @sep    if @sep && @parser.respond_to?( :'sep=' )
 
-         @parser.parse( @io, kwargs, &block )
+         ## check array / pipeline of converters is empty (size=0 e.g. is [])
+         if @converters.empty?
+           @parser.parse( @io, kwargs, &block )
+         else
+           ## add "post"-processing with converters pipeline
+           ##   that is, convert all strings to integer, float, date, ... if wanted
+           @parser.parse( @io, kwargs ) do |raw_record|
+             record = []
+             raw_record.each_with_index do | value, i |
+               record << @converters.convert( value, i )
+             end
+             block.call( record )
+           end
+         end
        else
          to_enum
        end

data/lib/csvreader/reader_hash.rb

@@ -2,87 +2,188 @@
 
 class CsvHashReader
 
-  def initialize( parser )
-    @parser = parser
-  end
 
-  DEFAULT = new( CsvReader::Parser::DEFAULT )
-  STRICT  = new( CsvReader::Parser::STRICT )
-  RFC4180 = new( CsvReader::Parser::RFC4180 )
+def self.open( path, mode=nil,
+               headers: nil,
+               sep: nil,
+               converters: nil,
+               header_converters: nil,
+               parser: nil, &block )   ## rename path to filename or name - why? why not?
+
+    ## note: default mode (if nil/not passed in) to 'r:bom|utf-8'
+    f = File.open( path, mode ? mode : 'r:bom|utf-8' )
+    csv = new(f, headers: headers,
+                 sep: sep,
+                 converters: converters,
+                 header_converters: header_converters,
+                 parser: parser )
+
+    # handle blocks like Ruby's open(), not like the (old old) CSV library
+    if block_given?
+      begin
+        block.call( csv )
+      ensure
+        csv.close
+      end
+    else
+      csv
+    end
+end # method self.open
+
+
+def self.read( path, headers: nil,
+                     sep: nil,
+                     converters: nil,
+                     header_converters: nil,
+                     parser: nil )
+    open( path,
+          headers: headers,
+          sep: sep,
+          converters: converters,
+          header_converters: header_converters,
+          parser: parser ) { |csv| csv.read }
+end
 
-  def self.default()  DEFAULT; end    ## alternative alias for DEFAULT
-  def self.strict()   STRICT;  end    ## alternative alias for STRICT
-  def self.rfc4180()  RFC4180; end    ## alternative alias for RFC4180
 
 
+def self.foreach( path, headers: nil,
+                        sep: nil,
+                        converters: nil,
+                        header_converters: nil,
+                        parser: nil, &block )
+  csv = open( path,
+              headers: headers,
+              sep: sep,
+              converters: converters,
+              header_converters: header_converters,
+              parser: parser )
 
-  def self.parse( data, sep: nil, headers: nil, &block )
-    DEFAULT.parse( data, sep: sep, headers: headers, &block )
+  if block_given?
+    begin
+      csv.each( &block )
+    ensure
+      csv.close
+    end
+  else
+    csv.to_enum    ## note: caller (responsible) must close file!!!
+    ## remove version without block given - why? why not?
+    ## use Csv.open().to_enum  or Csv.open().each
+    ##   or Csv.new( File.new() ).to_enum or Csv.new( File.new() ).each ???
   end
+end # method self.foreach
 
-  def self.read( path, sep: nil, headers: nil )
-    DEFAULT.read( path, sep: sep, headers: headers )
-  end
 
-  def self.foreach( path, sep: nil, headers: nil, &block )
-    DEFAULT.foreach( path,sep: sep, headers: headers, &block )
+def self.parse( data, headers: nil,
+                      sep: nil,
+                      converters: nil,
+                      header_converters: nil,
+                      parser: nil, &block )
+  csv = new( data,
+             headers: headers,
+             sep: sep,
+             converters: converters,
+             header_converters: header_converters,
+             parser: parser )
+
+  if block_given?
+    csv.each( &block )  ## note: caller (responsible) must close file!!! - add autoclose - why? why not?
+  else  # slurp contents, if no block is given
+    csv.read            ## note: caller (responsible) must close file!!! - add autoclose - why? why not?
   end
+end # method self.parse
 
 
 
-#############################
-## all "high-level" reader methods
-##
 
-def parse( data, sep: nil, headers: nil, &block )
-  if block_given?
-    parse_lines( data, sep: sep, headers: headers, &block )
-  else
-    records = []
-    parse_lines( data, sep: sep, headers: headers ) do |record|
-      records << record
-    end
-    records
-  end
-end
 
+def initialize( data, headers: nil, sep: nil,
+                      converters: nil,
+                      header_converters: nil,
+                      parser: nil )
+      raise ArgumentError.new( "Cannot parse nil as CSV" )  if data.nil?
+      ## todo: use (why? why not) - raise ArgumentError, "Cannot parse nil as CSV"     if data.nil?
 
-def read( path, sep: nil, headers: nil )
-  txt = File.open( path, 'r:bom|utf-8' ).read
-  parse( txt, sep: sep, headers: headers )
-end
+      # create the IO object we will read from
+      @io = data.is_a?(String) ? StringIO.new(data) : data
 
+      ## pass in headers as array e.g. ['A', 'B', 'C']
+      ##  double check: run header_converters on passed in headers?
+      ##    for now - do NOT auto-convert passed in headers - keep them as-is (1:1)
+      @names = headers ? headers : nil
 
-def foreach( path, sep: nil, headers: nil, &block )
-  File.open( path, 'r:bom|utf-8' ) do |file|
-    parse_lines( file, sep: sep, headers: headers, &block )
-  end
-end
+      @sep = sep
 
+      @converters        = CsvReader::Converter.create_converters( converters )
+      @header_converters = CsvReader::Converter.create_header_converters( header_converters )
 
-private
+      @parser = parser.nil? ? CsvReader::Parser::DEFAULT : parser
+end
 
-####################
-## parse_lines helper method to keep in one (central) place only (for easy editing/changing)
-##   - builds key/value pairs
 
-def parse_lines( data, sep: nil, headers: nil, &block)
-  ## pass in headers as array e.g. ['A', 'B', 'C']
-  names = headers ? headers : nil
 
-  kwargs = {
-    ##  converters: converters  ## todo: add converters
-  }
-  kwargs[:sep] = sep   if sep && @parser.respond_to?( :'sep=' )   ## note: only add separator if present/defined (not nil)
+### IO and StringIO Delegation ###
+extend Forwardable
+def_delegators :@io,
+               :close, :closed?, :eof, :eof?
+
+ ## add more - why? why not?
+ ## def_delegators :@io, :binmode, :binmode?, :close, :close_read, :close_write,
+ ##                                :closed?, :eof, :eof?, :external_encoding, :fcntl,
+ ##                                :fileno, :flock, :flush, :fsync, :internal_encoding,
+ ##                                :ioctl, :isatty, :path, :pid, :pos, :pos=, :reopen,
+ ##                                :seek, :stat, :string, :sync, :sync=, :tell, :to_i,
+ ##                                :to_io, :truncate, :tty?
+
+
+ include Enumerable
+
+
+ def each( &block )
+
+   ## todo/fix:
+   ##   add case for headers/names.size != values.size
+   ##   - add rest option? for if less headers than values (see python csv.DictReader - why? why not?)
+   ##
+   ##   handle case with duplicate and empty header names etc.
+
+
+   if block_given?
+     kwargs = {}
+     ## note: only add separator if present/defined (not nil)
+     kwargs[:sep] = @sep    if @sep && @parser.respond_to?( :'sep=' )
+
+     @parser.parse( @io, kwargs ) do |raw_values|     # sep: sep
+        if @names.nil?    ## check for (first) headers row
+          if @header_converters.empty?
+            @names = raw_values   ## store header row / a.k.a. field/column names
+          else
+            values = []
+            raw_values.each_with_index do |value,i|
+              values << @header_converters.convert( value, i )
+            end
+            @names = values
+          end
+        else    ## "regular" record
+          raw_record = @names.zip( raw_values ).to_h    ## todo/fix: check for more values than names/headers!!!
+          if @converters.empty?
+            block.call( raw_record )
+          else
+            ## add "post"-processing with converters pipeline
+            ##   that is, convert all strings to integer, float, date, ... if wanted
+            record = {}
+            raw_record.each do | key, value |
+              record[ key ] = @converters.convert( value, key )
+            end
+            block.call( record )
+          end
+        end
+     end
+   else
+     to_enum
+   end
+ end # method each
+
+ def read() to_a; end # method read
 
-  @parser.parse( data, kwargs ) do |values|     # sep: sep
-    if names.nil?
-      names = values   ## store header row / a.k.a. field/column names
-    else
-      record = names.zip( values ).to_h    ## todo/fix: check for more values than names/headers!!!
-      block.call( record )
-    end
-  end
-end
 
 end # class CsvHashReader