fastercsv 0.1.9 → 0.2.0

data/CHANGELOG

@@ -2,6 +2,25 @@
 
 Below is a complete listing of changes for each revision of FasterCSV.
 
+== 0.2.0
+
+* Added VERSION constant.
+* Significantly improved test speed.
+* Worked around Date::parse bug so tests will pass on Windows.
+* Documented test procedure.
+* Made FasterCSV#lineno CSV aware.
+* Added line numbers to MalformedCSVError messages.
+* <tt>:headers</tt> can now be set to an Array of headers to use.
+* <tt>:headers</tt> can now be set to an external CSV String of headers to use.
+* Added an <tt>:unconverted_fields</tt> options, so those can be returned
+  when needed.
+* Provided support for the serialization of custom Ruby objects using CSV.
+* Added CSV drop-in interface.
+* Added header information to FieldInfo Struct for conversions by header.
+* Added an alias to support <tt>require "fastercsv"</tt>.
+* Added FCSV alias for FasterCSV.
+* Added FasterCSV::instance and FasterCSV()/FCSV() shortcuts for easy output.
+
 == 0.1.9
 
 * Fixing the require "English" bug.

data/INSTALL

@@ -21,3 +21,15 @@ Download the latest version of FasterCSV from the
 the root project directory and enter:
 
 	$ sudo ruby setup.rb
+
+== Running the Tests
+
+If you would like to run FasterCSV's test suite on your system before installing
+and you have Rake installed, just issue the following command from the root of
+the project directory:
+
+  $ rake
+
+If you do not have rake, use the following command instead:
+
+  $ ruby -I lib:test test/ts_all.rb

data/Rakefile

@@ -37,14 +37,17 @@ end
 
 desc "Time FasterCSV and CSV"
 task :benchmark do
+  TESTS = 6
   path = "test/test_data.csv"
-	sh %Q{time ruby -r csv -e 'CSV.foreach("#{path}") { |row| }'}
-	sh %Q{time ruby -r lib/faster_csv -e 'FasterCSV.foreach("#{path}") { |row| }'}
+	sh %Q{time ruby -r csv -e } +
+	   %Q{'#{TESTS}.times { CSV.foreach("#{path}") { |row| } }'}
+	sh %Q{time ruby -r lib/faster_csv -e } +
+	   %Q{'#{TESTS}.times { FasterCSV.foreach("#{path}") { |row| } }'}
 end
 
 spec = Gem::Specification.new do |spec|
 	spec.name = "fastercsv"
-	spec.version = "0.1.9"
+	spec.version = "0.2.0"
 	spec.platform = Gem::Platform::RUBY
 	spec.summary = "FasterCSV is CSV, but faster, smaller, and cleaner."
 

data/TODO

@@ -28,3 +28,4 @@ order.
     "Experiment ID:	3",,,,,,,,,,,,0.92
     ....
 * Add calculated fields.
+* Examples, examples, examples...

data/examples/shortcut_interface.rb

@@ -0,0 +1,32 @@
+#!/usr/local/bin/ruby -w
+
+# shortcut_interface.rb
+#
+#  Created by James Edward Gray II on 2006-04-01.
+#  Copyright 2006 Gray Productions. All rights reserved.
+# 
+# Feature implementation and example code by Ara.T.Howard.
+
+require "faster_csv"
+
+#
+# So now it's this easy to write to STDOUT.
+#
+FCSV { |f| f << %w( a b c) << %w( d e f ) }
+
+#
+# Writing to a String.
+#
+FCSV(csv = '') do |f|
+  f << %w( q r s )
+  f << %w( x y z )
+end
+puts csv
+
+#
+# Writing to STDERR.
+#
+FCSV(STDERR) do |f|
+  f << %w( 0 1 2 )
+  f << %w( A B C )
+end

data/lib/faster_csv.rb

@@ -67,7 +67,16 @@ require "stringio"
 #   csv_string = ["CSV", "data"].to_csv   # to CSV
 #   csv_array  = "CSV,String".parse_csv   # from CSV
 # 
+# == Shortcut Interface
+# 
+#   FCSV            { |csv_out| csv_out << %w{my data here} }  # to STDOUT
+#   FCSV(csv = "")  { |csv_str| csv_str << %w{my data here} }  # to a String
+#   FCSV(STDERR)    { |csv_err| csv_err << %w{my data here} }  # to STDERR
+# 
 class FasterCSV
+  # The version of the installed library.
+  VERSION = "0.2.0".freeze
+  
   # 
   # A FasterCSV::Row is part Array and part Hash.  It retains an order for the
   # fields and allows duplicates just as an Array would, but also allows you to
@@ -330,8 +339,9 @@ class FasterCSV
   # 
   # <b><tt>index</tt></b>::  The zero-based index of the field in its row.
   # <b><tt>line</tt></b>::   The line of the data source this row is from.
+  # <b><tt>header</tt></b>:: The header for the column, when available.
   # 
-  FieldInfo = Struct.new(:index, :line)
+  FieldInfo = Struct.new(:index, :line, :header)
   
   # 
   # This Hash holds the built-in converters of FasterCSV that can be accessed by
@@ -390,16 +400,159 @@ class FasterCSV
   # <b><tt>:col_sep</tt></b>::            <tt>","</tt>
   # <b><tt>:row_sep</tt></b>::            <tt>:auto</tt>
   # <b><tt>:converters</tt></b>::         +nil+
+  # <b><tt>:unconverted_fields</tt></b>:: +nil+
   # <b><tt>:headers</tt></b>::            +false+
   # <b><tt>:return_headers</tt></b>::     +false+
   # <b><tt>:header_converters</tt></b>::  +nil+
   # 
-  DEFAULT_OPTIONS = { :col_sep           => ",",
-                      :row_sep           => :auto,
-                      :converters        => nil,
-                      :headers           => false,
-                      :return_headers    => false,
-                      :header_converters => nil }.freeze
+  DEFAULT_OPTIONS = { :col_sep            => ",",
+                      :row_sep            => :auto,
+                      :converters         => nil,
+                      :unconverted_fields => nil,
+                      :headers            => false,
+                      :return_headers     => false,
+                      :header_converters  => nil }.freeze
+  
+  # 
+  # This method will build a drop-in replacement for many of the standard CSV
+  # methods.  It allows you to write code like:
+  # 
+  #   begin
+  #     require "faster_csv"
+  #     FasterCSV.build_csv_interface
+  #   rescue LoadError
+  #     require "csv"
+  #   end
+  #   # ... use CSV here ...
+  # 
+  # This is not a complete interface with completely identical behavior.
+  # However, it is intended to be close enough that you won't notice the
+  # difference in most cases.  CSV methods supported are:
+  # 
+  # * foreach()
+  # * generate_line()
+  # * open()
+  # * parse()
+  # * parse_line()
+  # * readlines()
+  # 
+  # Be warned that this interface is slower than vanilla FasterCSV due to the
+  # extra layer of method calls.  Depending on usage, this can slow it down to 
+  # near CSV speeds.
+  # 
+  def self.build_csv_interface
+    Object.const_set(:CSV, Class.new).class_eval do
+      def self.foreach( path, rs = :auto, &block )  # :nodoc:
+        FasterCSV.foreach(path, :row_sep => rs, &block)
+      end
+      
+      def self.generate_line( row, fs = ",", rs = "" )  # :nodoc:
+        FasterCSV.generate_line(row, :col_sep => fs, :row_sep => rs)
+      end
+      
+      def self.open( path, mode, fs = ",", rs = :auto, &block )  # :nodoc:
+        if block and mode.include? "r"
+          FasterCSV.open(path, mode, :col_sep => fs, :row_sep => rs) do |csv|
+            csv.each(&block)
+          end
+        else
+          FasterCSV.open(path, mode, :col_sep => fs, :row_sep => rs, &block)
+        end
+      end
+      
+      def self.parse( str_or_readable, fs = ",", rs = :auto, &block )  # :nodoc:
+        FasterCSV.parse(str_or_readable, :col_sep => fs, :row_sep => rs, &block)
+      end
+      
+      def self.parse_line( src, fs = ",", rs = :auto )  # :nodoc:
+        FasterCSV.parse_line(src, :col_sep => fs, :row_sep => rs)
+      end
+      
+      def self.readlines( path, rs = :auto )  # :nodoc:
+        FasterCSV.readlines(path, :row_sep => rs)
+      end
+    end
+  end
+  
+  # 
+  # This method allows you to serialize an Array of Ruby objects to a String or
+  # File of CSV data.  This is not as powerful as Marshal or YAML, but perhaps
+  # useful for spreadsheet and database interaction.
+  # 
+  # Out of the box, this method is intended to work with simple data objects or
+  # Structs.  It will serialize a list of instance variables and/or
+  # Struct.members().
+  # 
+  # If you need need more complicated serialization, you can control the process
+  # by adding methods to the class to be serialized.
+  # 
+  # A class method csv_meta() is responsible for returning the first row of the
+  # document (as an Array).  This row is considered to be a Hash of the form
+  # key_1,value_1,key_2,value_2,...  FasterCSV::load() expects to find a class
+  # key with a value of the stringified class name and FasterCSV::dump() will
+  # create this, if you do not define this method.  This method is only called
+  # on the first object of the Array.
+  # 
+  # The next method you can provide is an instance method called csv_headers().
+  # This method is expected to return the second line of the document (again as
+  # an Array), which is to be used to give each column a header.  By default,
+  # FasterCSV::load() will set an instance variable if the field header starts
+  # with an @ character or call send() passing the header as the method name and
+  # the field value as an argument.  This method is only called on the first
+  # object of the Array.
+  # 
+  # Finally, you can provide an instance method called csv_dump(), which will
+  # be passed the headers.  This should return an Array of fields that can be
+  # serialized for this object.  This method is called once for every object in
+  # the Array.
+  # 
+  # The +io+ parameter can be used to serialize to a File, and +options+ can be
+  # anything FasterCSV::new() accepts.
+  # 
+  def self.dump( ary_of_objs, io = "", options = Hash.new )
+    obj_template = ary_of_objs.first
+    
+    csv = FasterCSV.new(io, options)
+    
+    # write meta information
+    begin
+      csv << obj_template.class.csv_meta
+    rescue NoMethodError
+      csv << [:class, obj_template.class]
+    end
+
+    # write headers
+    begin
+      headers = obj_template.csv_headers
+    rescue NoMethodError
+      headers = obj_template.instance_variables.sort
+      if obj_template.class.ancestors.find { |cls| cls.to_s =~ /\AStruct\b/ }
+        headers += obj_template.members.map { |mem| "#{mem}=" }.sort
+      end
+    end
+    csv << headers
+    
+    # serialize each object
+    ary_of_objs.each do |obj|
+      begin
+        csv << obj.csv_dump(headers)
+      rescue NoMethodError
+        csv << headers.map do |var|
+          if var[0] == ?@
+            obj.instance_variable_get(var)
+          else
+            obj[var[0..-2]]
+          end
+        end
+      end
+    end
+    
+    if io.is_a? String
+      csv.string
+    else
+      csv.close
+    end
+  end
   
   # 
   # :call-seq:
@@ -508,6 +661,77 @@ class FasterCSV
     (new("", options) << row).string
   end
   
+  # 
+  # This method will return a FasterCSV instance, just like FasterCSV::new(), 
+  # but the instance will be cached and returned for all future calls to this 
+  # method for the same +data+ object (tested by Object#object_id()) with the
+  # same +options+
+  # 
+  # If a block is given, the instance is passed to the block and the return
+  # value becomes the return value of the block.
+  # 
+  def self.instance( data = STDOUT, options = Hash.new )
+    # create a _signature_ for this method call, data object and options
+    sig = [data.object_id] +
+          options.values_at(*DEFAULT_OPTIONS.keys.sort_by { |sym| sym.to_s })
+    
+    # fetch or create the instance for this signature
+    @@instances ||= Hash.new
+    instance    =   (@@instances[sig] ||= new(data, options))
+
+    if block_given?
+      yield instance  # run block, if given, returning result
+    else
+      instance        # or return the instance
+    end
+  end
+  
+  # 
+  # This method is the reading counterpart to FasterCSV::dump().  See that
+  # method for a detailed description of the process.
+  # 
+  # You can customize loading by adding a class method called csv_load() which 
+  # will be passed a Hash of meta information, an Array of headers, and an Array
+  # of fields for the object the method is expected to return.
+  # 
+  # Remember that all fields will be Strings after this load.  If you need
+  # something else, use +options+ to setup converters or provide a custom
+  # csv_load() implementation.
+  # 
+  def self.load( io_or_str, options = Hash.new )
+    csv = FasterCSV.new(io_or_str, options)
+    
+    # load meta information
+    meta = Hash[*csv.shift]
+    cls  = meta["class"].split("::").inject(Object) do |c, const|
+      c.const_get(const)
+    end
+    
+    # load headers
+    headers = csv.shift
+    
+    # unserialize each object stored in the file
+    results = csv.inject(Array.new) do |all, row|
+      begin
+        obj = cls.csv_load(meta, headers, row)
+      rescue NoMethodError
+        obj = cls.allocate
+        headers.zip(row) do |name, value|
+          if name[0] == ?@
+            obj.instance_variable_set(name, value)
+          else
+            obj.send(name, value)
+          end
+        end
+      end
+      all << obj
+    end
+    
+    csv.close unless io_or_str.is_a? String
+    
+    results
+  end
+  
   # 
   # :call-seq:
   #   open( filename, mode="r", options = Hash.new ) { |faster_csv| ... }
@@ -541,7 +765,6 @@ class FasterCSV
   # * fsync()
   # * ioctl()
   # * isatty()
-  # * lineno()
   # * pid()
   # * pos()
   # * reopen()
@@ -663,10 +886,24 @@ class FasterCSV
   #                                       Hash and/or lambdas that handle custom
   #                                       conversion.  A single converter
   #                                       doesn't have to be in an Array.
+  # <b><tt>:unconverted_fields</tt></b>:: If set to +true+, an
+  #                                       unconverted_fields() method will be
+  #                                       added to all returned rows (Array or
+  #                                       FasterCSV::Row) that will return the
+  #                                       fields as they were before convertion.
+  #                                       Note that <tt>:headers</tt> supplied
+  #                                       by Array or String were not fields of
+  #                                       the document and thus will have an
+  #                                       empty Array attached.
   # <b><tt>:headers</tt></b>::            If set to <tt>:first_row</tt> or 
   #                                       +true+, the initial row of the CSV
   #                                       file will be treated as a row of
-  #                                       headers.  This setting causes
+  #                                       headers.  If set to an Array, the
+  #                                       contents will be used as the headers.
+  #                                       If set to a String, the String is run
+  #                                       through a call of
+  #                                       FasterCSV::parse_line() to produce an
+  #                                       Array of headers.  This setting causes
   #                                       FasterCSV.shift() to return rows as
   #                                       FasterCSV::Row objects instead of
   #                                       Arrays.
@@ -701,16 +938,24 @@ class FasterCSV
     unless options.empty?
       raise ArgumentError, "Unknown options:  #{options.keys.join(', ')}."
     end
+    
+    # track our own lineno since IO gets confused about line-ends is CSV fields
+    @lineno = 0
   end
   
+  # 
+  # The line number of the last row read from this file.  Fields with nested 
+  # line-end characters will not affect this count.
+  # 
+  attr_reader :lineno
+  
   ### IO and StringIO Delegation ###
   
   extend Forwardable
   def_delegators :@io, :binmode, :close, :close_read, :close_write, :closed?,
                        :eof, :eof?, :fcntl, :fileno, :flush, :fsync, :ioctl,
-                       :isatty, :lineno, :pid, :pos, :reopen, :rewind, :seek,
-                       :stat, :string, :sync, :sync=, :tell, :to_i, :to_io,
-                       :tty?
+                       :isatty, :pid, :pos, :reopen, :rewind, :seek, :stat,
+                       :string, :sync, :sync=, :tell, :to_i, :to_io, :tty?
 
   ### End Delegation ###
   
@@ -820,6 +1065,21 @@ class FasterCSV
   # The data source must be open for reading.
   # 
   def shift
+    #########################################################################
+    ### This method is purposefully kept a bit long as simple conditional ###
+    ### checks are faster than numerous (expensive) method calls.         ###
+    #########################################################################
+    
+    # handle headers not based on document content
+    if header_row? and @return_headers and
+       [Array, String].include? @use_headers.class
+       if @unconverted_fields
+         return add_unconverted_fields(parse_headers, Array.new)
+       else
+         return parse_headers
+       end
+    end
+    
     # begin with a blank line, so we can always add to it
     line = ""
 
@@ -838,7 +1098,14 @@ class FasterCSV
       # I believe a blank line should be an <tt>Array.new</tt>, not 
       # CSV's <tt>[nil]</tt>
       # 
-      return Array.new if parse.empty?
+      if parse.empty?
+        @lineno += 1
+        if @unconverted_fields
+          return add_unconverted_fields(Array.new, Array.new)
+        else
+          return Array.new
+        end
+      end
 
       # 
       # shave leading empty fields if needed, because the main parser chokes 
@@ -863,7 +1130,8 @@ class FasterCSV
               $2
             else
               # or throw an Exception
-              raise MalformedCSVError, 'Unquoted fields do not allow \r or \n.'
+              raise MalformedCSVError, "Unquoted fields do not allow " +
+                                       "\\r or \\n (line #{lineno + 1})."
             end
           end
         else                  # we found a quoted field...
@@ -874,15 +1142,28 @@ class FasterCSV
 
       # if parse is empty?(), we found all the fields on the line...
       if parse.empty?
-        # convert fields if needed...
-        csv = convert_fields(csv) unless header_row? or @converters.empty?
+        @lineno += 1
+
+        # save fields unconverted fields, if needed...
+        unconverted = csv.dup if @unconverted_fields
+
+        # convert fields, if needed...
+        csv = convert_fields(csv) unless @use_headers or @converters.empty?
         # parse out header rows and handle FasterCSV::Row conversions...
         csv = parse_headers(csv)  if     @use_headers
+
+        # inject unconverted fields and accessor, if requested...
+        if @unconverted_fields and not csv.respond_to? :unconverted_fields
+          add_unconverted_fields(csv, unconverted)
+        end
+
         # return the results
         break csv
       end
       # if we're not empty?() but at eof?(), a quoted field wasn't closed...
-      raise MalformedCSVError, "Unclosed quoted field." if @io.eof?
+      if @io.eof?
+        raise MalformedCSVError, "Unclosed quoted field on line #{lineno + 1}."
+      end
       # otherwise, we need to loop and pull some more data to complete the row
     end
   end
@@ -966,7 +1247,14 @@ class FasterCSV
   # are set.  When +field_name+ is <tt>:header_converters</tt> header converters
   # are added instead.
   # 
+  # The <tt>:unconverted_fields</tt> option is also actived for 
+  # <tt>:converters</tt> calls, if requested.
+  # 
   def init_converters( options, field_name = :converters )
+    if field_name == :converters
+      @unconverted_fields = options.delete(:unconverted_fields)
+    end
+
     instance_variable_set("@#{field_name}", Array.new)
     
     # find the correct method to add the coverters
@@ -996,6 +1284,7 @@ class FasterCSV
     @use_headers    = options.delete(:headers)
     @return_headers = options.delete(:return_headers)
 
+    # headers must be delayed until shift(), in case they need a row of content
     @headers = nil
     
     init_converters(options, :header_converters)
@@ -1028,24 +1317,22 @@ class FasterCSV
   
   # 
   # Processes +fields+ with <tt>@converters</tt>, or <tt>@header_converters</tt>
-  # if this is a header_row?(), returning the converted field set.  Any
+  # if +headers+ is passed as +true+, returning the converted field set.  Any
   # converter that changes the field into something other than a String halts
   # the pipeline of conversion for that field.  This is primarily an efficiency
   # shortcut.
   # 
-  def convert_fields( fields )
-    converters = if header_row?  # see if we are converting headers or fields
-      @header_converters
-    else
-      @converters
-    end
+  def convert_fields( fields, headers = false )
+    # see if we are converting headers or fields
+    converters = headers ? @header_converters : @converters
     
     fields.enum_for(:each_with_index).map do |field, index|  # map_with_index
       converters.each do |converter|
         field = if converter.arity == 1  # straight field converter
           converter[field]
         else                             # FieldInfo converter
-          converter[field, FieldInfo.new(index, @io.lineno)]
+          header = @use_headers && !headers ? @headers[index] : nil
+          converter[field, FieldInfo.new(index, lineno, header)]
         end
         break unless field.is_a? String  # short-curcuit pipeline for speed
       end
@@ -1060,18 +1347,56 @@ class FasterCSV
   # converters) or by reading past them to return a field row. Headers are also
   # saved in <tt>@headers</tt> for use in future rows.
   # 
-  def parse_headers( row )
-    if @headers.nil?  # header row
-      @headers = convert_fields(row)  # save
-      if @return_headers  # return the headers
-        FasterCSV::Row.new(@headers, row, true)
-      else                # skip to next field row
-        shift
+  # When +nil+, +row+ is assumed to be a header row not based on an actual row
+  # of the stream.
+  # 
+  def parse_headers( row = nil )
+    if @headers.nil?                # header row
+      @headers = case @use_headers  # save headers
+      when Array  then @use_headers                         # Array of headers
+      when String then self.class.parse_line(@use_headers)  # CSV header String
+      else             row                                  # first row headers
+      end
+      
+      # prepare converted and unconverted copies
+      row      = @headers                       if row.nil?
+      @headers = convert_fields(@headers, true)
+      
+      if @return_headers                                     # return headers
+        return FasterCSV::Row.new(@headers, row, true)
+      elsif not [Array, String].include? @use_headers.class  # skip to field row
+        return shift
       end
-    else              # field row
-      FasterCSV::Row.new(@headers, row)
     end
+
+    FasterCSV::Row.new(@headers, convert_fields(row))  # field row
   end
+  
+  # 
+  # Thiw methods injects an instance variable <tt>unconverted_fields</tt> into
+  # +row+ and an accessor method for it called unconverted_fields().  The
+  # variable is set to the contents of +fields+.
+  # 
+  def add_unconverted_fields( row, fields )
+    class << row
+      attr_reader :unconverted_fields
+    end
+    row.instance_eval { @unconverted_fields = fields }
+    row
+  end
+end
+
+# Another name for FasterCSV.
+FCSV = FasterCSV
+
+# Another name for FasterCSV::instance().
+def FasterCSV( *args, &block )
+  FasterCSV.instance(*args, &block)
+end
+
+# Another name for FCSV::instance().
+def FCSV( *args, &block )
+  FCSV.instance(*args, &block)
 end
 
 class Array