csvreader 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 47c630dbfe75b03e4f2d03710cca0f4b3c66ea84
-  data.tar.gz: 7b604c9b9144190b1b8b4a4dcd848ea5b7f88aa4
+  metadata.gz: 8d41f765a3d0091d6f4fe392f014e386ef35a166
+  data.tar.gz: 268048267f364829801e12f7c41331153ce8c138
 SHA512:
-  metadata.gz: 2dad1ae27b4273b8e5a22cf6eae60f141eca381229ed03bfbe6f403e99aae07a16aa4fc84e1f22a953cd9db5da1b9fa2fb6266666fdc6a756872e8bd4ec8dfb9
-  data.tar.gz: 38b0002ea3bdfff0b7ce994064d4fb4993e75a3b1225128a50ad9e18180c12cdae9d2a40b6f694e70ebccf4d65005fd5a2145450c20a565de8d3f5af7b398a58
+  metadata.gz: d891d31e0447639dfc1d70895c81eba369028b0a32c0daf7aba605d6a91f8e1fe1304eda3d2a2efcff018a1ceb3e8f9f964301269b150771550e5e372cb45c48
+  data.tar.gz: b8d51ae12ce5a8dbba0772ee6222338c9e6a7091585b2bf6480c88fd27f4738c5e4472f329568c92ff6c31397a8c297b157fc0b1c1e916e9497213efb2ef7245

data/Manifest.txt

@@ -9,6 +9,7 @@ lib/csvreader/buffer.rb
 lib/csvreader/builder.rb
 lib/csvreader/converter.rb
 lib/csvreader/parser.rb
+lib/csvreader/parser_json.rb
 lib/csvreader/parser_std.rb
 lib/csvreader/parser_strict.rb
 lib/csvreader/parser_tab.rb
@@ -34,3 +35,4 @@ test/test_reader.rb
 test/test_reader_converters.rb
 test/test_reader_hash.rb
 test/test_reader_hash_converters.rb
+test/test_samples.rb

data/lib/csvreader/base.rb

@@ -34,6 +34,10 @@ class Parser
   ##          end
 
   DEFAULT = ParserStd.new
+  NUMERIC = ParserStd.new( numeric: true,
+                           nan: ['#NAN', 'NAN', 'NaN', 'nan' ],
+                           null: "" )
+
 
   RFC4180 = ParserStrict.new
   STRICT  = ParserStrict.new  ## note: make strict its own instance (so you can change config without "breaking" rfc4180)
@@ -53,15 +57,14 @@ class Parser
                                                       escape: true,
                                                       null: "\\N" )
 
-  NUMERIC = ParserStrict.new( numeric: true,
-                              nan: ['#NAN', 'NAN', 'NaN', 'nan' ],
-                              null: "" )
-
 
   TAB     = ParserTab.new
 
 
   def self.default()         DEFAULT;         end ## alternative alias for DEFAULT
+  def self.numeric()         NUMERIC;         end
+  def self.num()             numeric;         end
+  def self.n()               numeric;         end
   def self.strict()          STRICT;          end ## alternative alias for STRICT
   def self.rfc4180()         RFC4180;         end ## alternative alias for RFC4180
   def self.excel()           EXCEL;           end ## alternative alias for EXCEL
@@ -70,7 +73,6 @@ class Parser
   def self.postgres()        postgresql;      end
   def self.postgresql_text() POSTGRESQL_TEXT; end
   def self.postgres_text()   postgresql_text; end
-  def self.numeric()         NUMERIC;         end
   def self.tab()             TAB;             end
 end # class Parser
 end # class CsvReader
@@ -80,6 +82,7 @@ end # class CsvReader
 class CsvReader
   ### pre-define CsvReader (built-in) formats/dialect
   DEFAULT = Builder.new( Parser::DEFAULT )
+  NUMERIC = Builder.new( Parser::NUMERIC )
 
   STRICT  = Builder.new( Parser::STRICT )
   RFC4180 = Builder.new( Parser::RFC4180 )
@@ -89,12 +92,14 @@ class CsvReader
   POSTGRES = POSTGRESQL           = Builder.new( Parser::POSTGRESQL )
   POSTGRES_TEXT = POSTGRESQL_TEXT = Builder.new( Parser::POSTGRESQL_TEXT )
 
-  NUMERIC = Builder.new( Parser::NUMERIC )
 
   TAB = Builder.new( Parser::TAB )
 
 
   def self.default()         DEFAULT;         end ## alternative alias for DEFAULT
+  def self.numeric()         NUMERIC;         end
+  def self.num()             numeric;         end
+  def self.n()               numeric;         end
   def self.strict()          STRICT;          end ## alternative alias for STRICT
   def self.rfc4180()         RFC4180;         end ## alternative alias for RFC4180
   def self.excel()           EXCEL;           end ## alternative alias for EXCEL
@@ -103,14 +108,15 @@ class CsvReader
   def self.postgres()        postgresql;      end
   def self.postgresql_text() POSTGRESQL_TEXT; end
   def self.postgres_text()   postgresql_text; end
-  def self.numeric()         NUMERIC;         end
   def self.tab()             TAB;             end
 end # class CsvReader
 
 
+
 class CsvHashReader
   ### pre-define CsvReader (built-in) formats/dialect
   DEFAULT = Builder.new( Parser::DEFAULT )
+  NUMERIC = Builder.new( Parser::NUMERIC )
 
   STRICT  = Builder.new( Parser::STRICT )
   RFC4180 = Builder.new( Parser::RFC4180 )
@@ -120,12 +126,14 @@ class CsvHashReader
   POSTGRES = POSTGRESQL           = Builder.new( Parser::POSTGRESQL )
   POSTGRES_TEXT = POSTGRESQL_TEXT = Builder.new( Parser::POSTGRESQL_TEXT )
 
-  NUMERIC = Builder.new( Parser::NUMERIC )
 
   TAB = Builder.new( Parser::TAB )
 
 
   def self.default()         DEFAULT;         end ## alternative alias for DEFAULT
+  def self.numeric()         NUMERIC;         end
+  def self.num()             numeric;         end
+  def self.n()               numeric;         end
   def self.strict()          STRICT;          end ## alternative alias for STRICT
   def self.rfc4180()         RFC4180;         end ## alternative alias for RFC4180
   def self.excel()           EXCEL;           end ## alternative alias for EXCEL
@@ -134,7 +142,6 @@ class CsvHashReader
   def self.postgres()        postgresql;      end
   def self.postgresql_text() POSTGRESQL_TEXT; end
   def self.postgres_text()   postgresql_text; end
-  def self.numeric()         NUMERIC;         end
   def self.tab()             TAB;             end
 end # class CsvHashReader
 

data/lib/csvreader/parser_json.rb

@@ -0,0 +1,2 @@
+##
+## todo: add parser for new CSV <3 JSON format

data/lib/csvreader/parser_std.rb

@@ -44,21 +44,29 @@ attr_reader :config   ## todo/fix: change config to proper dialect class/struct
 ##    null values - include NA - why? why not?
 ##        make null values case sensitive or add an option for case sensitive
 ##        or better allow a proc as option for checking too!!!
-def initialize( null: ['\N', 'NA']  ## note: set to nil for no null vales / not availabe (na)
+def initialize( null:     ['\N', 'NA'],  ## note: set to nil for no null vales / not availabe (na)
+                numeric:  false,   ## (auto-)convert all non-quoted values to float
+                nan:      nil      ## note: only if numeric - set mappings for Float::NAN (not a number) values
               )
   @config = {}   ## todo/fix: change config to proper dialect class/struct - why? why not?
 
   ## note: null values must get handled by parser
   ##   only get checked for unquoted strings (and NOT for quoted strings)
   ##   "higher-level" code only knows about strings and has no longer any info if string was quoted or unquoted
-  @config[:null] = null   ## null values
+  @config[:null]    = null   ## null values
+  @config[:numeric] = numeric
+  @config[:nan]     = nan   # not a number (NaN) e.g. Float::NAN
 end
 
+
+
 #########################################
 ## config convenience helpers
 ##   e.g. use like  Csv.defaultl.null = '\N'   etc.   instead of
 ##                  Csv.default.config[:null] = '\N'
 def null=( value )     @config[:null]=value; end
+def numeric=( value )     @config[:numeric]=value; end
+def nan=( value )         @config[:nan]=value; end
 
 
 
@@ -143,14 +151,25 @@ end
 
 
 def parse_field( input )
+  value = ""
+
+  numeric = config[:numeric]
+
   logger.debug "parse field"  if logger.debug?
 
-  value = ""
   skip_spaces( input )   ## strip leading spaces
 
+
   if (c=input.peek; c=="," || c==LF || c==CR || input.eof?) ## empty field
-    value = nil  if is_null?( value )   ## note: allows null = '' that is turn unquoted empty strings into null/nil
-     ## return value; do nothing
+    ## note: allows null = '' that is turn unquoted empty strings into null/nil
+    ##   or if using numeric into NotANumber (NaN)
+    if is_null?( value )
+      value = nil
+    elsif numeric && is_nan?( value )  ## todo: check - how to handle numeric? return nil, NaN, or "" ???
+      value = Float::NAN
+    else
+      # do nothing - keep value as is :-) e.g. "".
+    end
   elsif input.peek == DOUBLE_QUOTE
     logger.debug "start double_quote field - peek >#{input.peek}< (#{input.peek.ord})"  if logger.debug?
     value << parse_doublequote( input )
@@ -174,7 +193,23 @@ def parse_field( input )
     ##  note: only strip **trailing** spaces (space and tab only)
     ##    do NOT strip newlines etc. might have been added via escape! e.g. \\\n
     value = value.sub( /[ \t]+$/, '' )
-    value = nil  if is_null?( value )   ## note: null check only for UNQUOTED (not quoted/escaped) values
+
+    if is_null?( value )   ## note: null check only for UNQUOTED (not quoted/escaped) values
+      value = nil
+    elsif numeric
+      if is_nan?( value )
+        value = Float::NAN
+      else
+        ## numeric - (auto-convert) non-quoted values (if NOT nil) to floats
+        if numeric.is_a?( Proc )
+          value = numeric.call( value )   ## allow custom converter proc (e.g. how to handle NaN and conversion errors?)
+        else
+          value = convert_to_float( value ) # default (fails silently) keep string value if cannot convert - change - why? why not?
+        end
+      end
+    else
+      # do nothing - keep value as is :-).
+    end
 
     logger.debug "end reg field - peek >#{input.peek}< (#{input.peek.ord})"  if logger.debug?
   end
@@ -273,6 +308,25 @@ end # method parse_lines
 
 
 
+def convert_to_float( value ) Float( value ) rescue value; end
+
+def is_nan?( value )
+   nan = @config[:nan]
+   if nan.nil?
+     false  ## nothing set; return always false (not NaN)
+   elsif nan.is_a?( Proc )
+     nan.call( value )
+   elsif nan.is_a?( Array )
+     nan.include?( value )
+   elsif nan.is_a?( String )
+     value == nan
+   else  ## unknown config style / setting
+     ##  todo: issue a warning or error - why? why not?
+     false  ## nothing set; return always false (not nan)
+   end
+end
+
+
 def is_null?( value )
    null = @config[:null]
    if null.nil?

data/lib/csvreader/parser_strict.rb

@@ -36,9 +36,7 @@ def initialize( sep:         ',',
                 doublequote: true,
                 escape:      false,   ## true/false
                 null:        nil,     ## note: set to nil for no null vales / not availabe (na)
-                comment:     false,   ## note: comment char e.g. # or false/nil
-                numeric:     false,   ## (auto-)convert all non-quoted values to float
-                nan:         nil      ## note: only if numeric - set mappings for Float::NAN (not a number) values
+                comment:     false   ## note: comment char e.g. # or false/nil
                )
   @config = {}   ## todo/fix: change config to proper dialect class/struct - why? why not?
   @config[:sep]          = sep
@@ -47,8 +45,6 @@ def initialize( sep:         ',',
   @config[:escape]  = escape
   @config[:null]     = null
   @config[:comment] = comment
-  @config[:numeric] = numeric
-  @config[:nan]     = nan   # not a number (NaN) e.g. Float::NAN
 end
 
 
@@ -62,8 +58,6 @@ def doublequote=( value ) @config[:doublequote]=value; end
 def escape=( value )      @config[:escape]=value; end
 def null=( value )        @config[:null]=value; end
 def comment=( value )     @config[:comment]=value; end
-def numeric=( value )     @config[:numeric]=value; end
-def nan=( value )         @config[:nan]=value; end
 
 
 
@@ -158,20 +152,14 @@ def parse_field( input, sep: )
 
   quote   = config[:quote]
   escape  = config[:escape]
-  numeric = config[:numeric]
 
   logger.debug "parse field - sep: >#{sep}< (#{sep.ord})"  if logger.debug?
 
   if (c=input.peek; c==sep || c==LF || c==CR || input.eof?) ## empty unquoted field
     ## note: allows null = '' that is turn unquoted empty strings into null/nil
     ##   or if using numeric into NotANumber (NaN)
-    if is_null?( value )
-      value = nil
-    elsif numeric & is_nan?( value )
-      value = Float::NAN
-    else
-      # do nothing - keep value as is :-) e.g. "".
-    end
+    value = nil   if is_null?( value )
+    ##  do nothing - keep value as is :-) e.g. "".
   elsif quote && input.peek == quote
     logger.debug "start quote field - peek >#{input.peek}< (#{input.peek.ord})"  if logger.debug?
     value << parse_quote( input, sep: sep )
@@ -190,22 +178,8 @@ def parse_field( input, sep: )
     end
 
 
-    if is_null?( value )   ## note: null check only for UNQUOTED (not quoted/escaped) values
-      value = nil
-    elsif numeric
-      if is_nan?( value )
-        value = Float::NAN
-      else
-        ## numeric - (auto-convert) non-quoted values (if NOT nil) to floats
-        if numeric.is_a?( Proc )
-          value = numeric.call( value )   ## allow custom converter proc (e.g. how to handle NaN and conversion errors?)
-        else
-          value = convert_to_float( value ) # default (fails silently) keep string value if cannot convert - change - why? why not?
-        end
-      end
-    else
-      # do nothing - keep value as is :-).
-    end
+    value = nil  if is_null?( value )   ## note: null check only for UNQUOTED (not quoted/escaped) values
+    # do nothing - keep value as is :-).
 
     logger.debug "end reg field - peek >#{input.peek}< (#{input.peek.ord})"  if logger.debug?
   end
@@ -296,24 +270,6 @@ def parse_lines( input, sep:, &block )
 end # method parse_lines
 
 
-def convert_to_float( value ) Float( value ) rescue value; end
-
-def is_nan?( value )
-   nan = @config[:nan]
-   if nan.nil?
-     false  ## nothing set; return always false (not NaN)
-   elsif nan.is_a?( Proc )
-     nan.call( value )
-   elsif nan.is_a?( Array )
-     nan.include?( value )
-   elsif nan.is_a?( String )
-     value == nan
-   else  ## unknown config style / setting
-     ##  todo: issue a warning or error - why? why not?
-     false  ## nothing set; return always false (not nan)
-   end
-end
-
 def is_null?( value )
    null = @config[:null]
    if null.nil?

data/lib/csvreader/version.rb

@@ -5,7 +5,7 @@ class CsvReader   ## note: uses a class for now - change to module - why? why no
 
   MAJOR = 1    ## todo: namespace inside version or something - why? why not??
   MINOR = 0
-  PATCH = 1
+  PATCH = 2
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
 

data/test/test_parser_numeric.rb

@@ -26,6 +26,8 @@ def test_parse
                  [4.0,5.0,6.0]], parser.parse( "1,2,3\n4,5,6" )
    assert_equal [[1.0,2.0,3.0],
                  ["4","5","6"]], parser.parse( %Q{ 1,2 , 3\n"4","5","6"} )
+   assert_equal [[1.0,2.0,3.0],
+                ["4","5","6"]], parser.parse( %Q{ 1,2 , 3\n "4", "5" ,"6" } )
    assert_equal [["a","b","c"]], parser.parse( %Q{"a","b","c"} )
 end
 

data/test/test_samples.rb

@@ -0,0 +1,64 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_samples.rb
+
+
+require 'helper'
+
+class TestSamples < MiniTest::Test
+
+
+def test_cities11
+  records = CsvReader.read( "#{CsvReader.test_data_dir}/cities11.csv" )
+  pp records
+
+  assert_equal [["Los Angeles",   "34°03'N",      "118°15'W"],
+                ["New York City", %Q{40°42'46"N}, %Q{74°00'21"W}],
+                ["Paris",         %Q{48°51'24"N}, %Q{2°21'03"E}]], records
+end
+
+
+def test_cars11
+  records = CsvReader.read( "#{CsvReader.test_data_dir}/cars11.csv" )
+  pp records
+
+  assert_equal [["Year", "Make",  "Model",  "Description", "Price"],
+                ["1997", "Ford",  "E350",   "ac, abs, moon", "3000.00"],
+                ["1999", "Chevy", %Q{Venture "Extended Edition"}, "", "4900.00"],
+                ["1999", "Chevy", %Q{Venture "Extended Edition, Very Large"}, "", "5000.00"],
+                ["1996", "Jeep",  "Grand Cherokee", "MUST SELL!\nair, moon roof, loaded", "4799.00"]], records
+end
+
+
+def test_customers11
+  records = CsvReader.read( "#{CsvReader.test_data_dir}/customers11.csv" )
+  pp records
+
+  assert_equal [["Name",   "Times arrived", "Total $ spent", "Food feedback"],
+                ["Dan",      "34", "2548", "Lovin it!"],
+                ["Maria",    "55", "5054", "Good, delicious food"],
+                ["Carlos",   "22", "4352", %Q{I am "pleased", but could be better}],
+                ["Stephany", "34", "6542", "I want bigger steaks!!!!!"],
+                ["James",    "1",    "43", "Not bad"],
+                ["Robin",    "1",    "56", "Fish is tasty"],
+                ["Anna",     "1",    "79", "Good, better, the best!"]], records
+end
+
+def test_shakespeare11
+  records = CsvReader.read( "#{CsvReader.test_data_dir}/shakespeare.csv" )
+  pp records
+
+  assert_equal [["Quote", "Play", "Cite"],
+                ["Sweet are the uses of adversity", "As You Like It", "Act 2, scene 1, 12"],
+                ["All the world's a stage", "As You Like It", "Act 2, scene 7, 139"],
+                ["We few, we happy few", "Henry V", ""],
+                [%Q{"Seems," madam! Nay it is; I know not "seems."}, "Hamlet", "(1.ii.76)"],
+                ["To be, or not to be", "Hamlet", "Act 3, scene 1, 55"],
+                ["What's in a name? That which we call a rose by any other name would smell as sweet.", "Romeo and Juliet", "(II, ii, 1-2)"],
+                ["O Romeo, Romeo, wherefore art thou Romeo?", "Romeo and Juliet", "Act 2, scene 2, 33"],
+                ["Tomorrow, and tomorrow, and tomorrow", "Macbeth", "Act 5, scene 5, 19"]], records
+end
+
+end # class TestSamples

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csvreader
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-11 00:00:00.000000000 Z
+date: 2018-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rdoc
@@ -60,6 +60,7 @@ files:
 - lib/csvreader/builder.rb
 - lib/csvreader/converter.rb
 - lib/csvreader/parser.rb
+- lib/csvreader/parser_json.rb
 - lib/csvreader/parser_std.rb
 - lib/csvreader/parser_strict.rb
 - lib/csvreader/parser_tab.rb
@@ -85,6 +86,7 @@ files:
 - test/test_reader_converters.rb
 - test/test_reader_hash.rb
 - test/test_reader_hash_converters.rb
+- test/test_samples.rb
 homepage: https://github.com/csv11/csvreader
 licenses:
 - Public Domain