csvrecord 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ea4f015ade2a8b105eb89aec99e4a9bd7f5be01c
-  data.tar.gz: 2738c2e3655a29e3a626eb7aee228d9b7275a3bd
+  metadata.gz: f60b9d15d91c6d20b625bb343d895ad750f43a81
+  data.tar.gz: 5510e7921a509fe40906de7f533f185bf244e8bd
 SHA512:
-  metadata.gz: '0290ce3db55dfa7e3e24f00db9aa4c918be94b04c408415844015d09fa8f0eabee4f8989ac7f9dfb7fc917dbd428727da25ee1d84200057b5bbe6c9add3e18ce'
-  data.tar.gz: 6c47502227af693814a99f7d34bd9fede8afadc556d1797c1cf365437e7491312bd7d1037c23eb0923775884bb0ed5e55ac5a62642d940810db7fa9bdb3e4a5d
+  metadata.gz: 9853722319a53c73469238440eb2c249cd8c3829711acbd35efc60996316f8d8e763d5f61decdc4e3186baebf13536c46c72217fd92f08e6d675b7a6c088568b
+  data.tar.gz: bd2922bf8c44a458abe70b5dceee9b405904f878eacdaf34a1454a2313c43517329d31841c2bc4faca5c6d3492220d9c5c3091115171046c087a2889a4415339

data/Manifest.txt

@@ -6,7 +6,12 @@ Rakefile
 lib/csvrecord.rb
 lib/csvrecord/base.rb
 lib/csvrecord/builder.rb
+lib/csvrecord/reader.rb
 lib/csvrecord/version.rb
 test/data/beer.csv
+test/data/beer11.csv
 test/helper.rb
-test/test_beer.rb
+test/test_reader.rb
+test/test_record.rb
+test/test_record_auto.rb
+test/test_version.rb

data/README.md

@@ -74,11 +74,18 @@ pretty prints (pp):
 Or loop over the records. Example:
 
 ``` ruby
-Beer.read( data ).each do |rec|
+Beer.read( 'beer.csv' ).each do |rec|
+  puts "#{rec.name} (#{rec.abv}%) by #{rec.brewery}, #{rec.city}"
+end
+
+# -or-
+
+Beer.foreach( 'beer.csv' ) do |rec|
   puts "#{rec.name} (#{rec.abv}%) by #{rec.brewery}, #{rec.city}"
 end
 ```
 
+
 printing:
 
 ```
@@ -97,11 +104,11 @@ Or create new records from scratch. Example:
 beer = Beer.new( 'Andechser Klosterbrauerei',
                  'Andechs',
                  'Doppelbock Dunkel',
-                 '7.0%' )
+                 '7%' )
 
 # -or-
 
-values = ['Andechser Klosterbrauerei', 'Andechs', 'Doppelbock Dunkel', '7.0%']
+values = ['Andechser Klosterbrauerei', 'Andechs', 'Doppelbock Dunkel', '7%']
 beer = Beer.new( values )
 
 # -or-
@@ -109,14 +116,14 @@ beer = Beer.new( values )
 beer = Beer.new( brewery: 'Andechser Klosterbrauerei',
                  city:    'Andechs',
                  name:    'Doppelbock Dunkel',
-                 abv:     '7.0%' )
+                 abv:     '7%' )
 
 # -or-
 
 hash = { brewery: 'Andechser Klosterbrauerei',
          city:    'Andechs',
          name:    'Doppelbock Dunkel',
-         abv:     '7.0%' }
+         abv:     '7%' }
 beer = Beer.new( hash )
 
 
@@ -126,24 +133,24 @@ beer = Beer.new
 beer.update( brewery: 'Andechser Klosterbrauerei',
              city:    'Andechs',
              name:    'Doppelbock Dunkel' )
-beer.update( abv: 12.7 )
+beer.update( abv: 7.0 )
 
 # -or-
 
 beer = Beer.new
-beer.parse( ['Andechser Klosterbrauerei', 'Andechs', 'Doppelbock Dunkel', '7.0%'] )
+beer.parse( ['Andechser Klosterbrauerei', 'Andechs', 'Doppelbock Dunkel', '7%'] )
 
 # -or-
 
 beer = Beer.new
-beer.parse( 'Andechser Klosterbrauerei,Andechs,Doppelbock Dunkel,7.0%' )
+beer.parse( 'Andechser Klosterbrauerei,Andechs,Doppelbock Dunkel,7%' )
 
 # -or-
 
 beer = Beer.new
 beer.brewery = 'Andechser Klosterbrauerei'
 beer.name    = 'Doppelbock Dunkel'
-beer.abv     = 12.7
+beer.abv     = 7.0
 ```
 
 

data/lib/csvrecord.rb

@@ -10,6 +10,7 @@ require 'pp'
 require 'csvrecord/version' # let version always go first
 require 'csvrecord/base'
 require 'csvrecord/builder'
+require 'csvrecord/reader'
 
 
 

data/lib/csvrecord/base.rb

@@ -100,6 +100,40 @@ module CsvRecord
   end
 
 
+   ###########################################
+   ## "magic" lazy auto-build schema from headers versions
+
+   def self.build_class( headers )   ## check: find a better name - why? why not?
+     ## (auto-)build record class from an array of headers
+     ##   add fields (all types will be string for now)
+     clazz = Class.new(Base)
+     headers.each do |header|
+       ## downcase and remove all non-ascii chars etc.
+       ##  todo/fix: remove all non-ascii chars!!!
+       ##  todo: check if header starts with a number too!!
+       name = header.downcase.gsub( ' ', '_' )
+       name = name.to_sym   ## symbol-ify
+       clazz.field( name )
+     end
+     clazz
+   end
+
+   def self.read( path, sep: Csv.config.sep )
+     headers = CsvReader.header( path, sep: sep )
+
+     clazz = build_class( headers )
+     clazz.read( path, sep: sep )
+   end
+
+   def self.foreach( path, sep: Csv.config.sep, &block )
+     headers = CsvReader.header( path, sep: sep )
+
+     clazz = build_class( headers )
+     clazz.foreach( path, sep: sep, &block )
+   end
+
+
+
 
 class Base
 
@@ -234,11 +268,21 @@ def to_csv   ## use/rename/alias to to_row too - why? why not?
 end
 
 
+def self.foreach( path, sep: Csv.config.sep, headers: true )
+  CsvReader.foreach( path, sep: sep, headers: headers ) do |row|
+    rec = new
+    values = CsvReader.unwrap( row )
+    rec.parse( values )
+
+    yield( rec )    ## check: use block.class( rec ) - why? why not?
+  end
+end
+
 
-def self.parse( txt_or_rows )  ## note: returns an (lazy) enumarator
+def self.parse( txt_or_rows, sep: Csv.config.sep, headers: true )  ## note: returns an (lazy) enumarator
   if txt_or_rows.is_a? String
     txt = txt_or_rows
-    rows = CSV.parse( txt, headers: true )
+    rows = CsvReader.parse( txt, sep: sep, headers: headers )
   else
     ### todo/fix: use only self.create( array-like ) for array-like data  - why? why not?
     rows = txt_or_rows    ## assume array-like records that responds to :each
@@ -248,25 +292,19 @@ def self.parse( txt_or_rows )  ## note: returns an (lazy) enumarator
 
   Enumerator.new do |yielder|
     rows.each do |row|
-     ## check - check for to_h - why? why not?  supported/built-into by CSV::Row??
-     ## if row.respond_to?( :to_h )
-     ## else
-       ## pp row.fields
-       ## pp row.to_hash
-       ## fix/todo!!!!!!!!!!!!!
-       ##  check for CSV::Row etc. - use row.to_hash ?
-       rec = new
-       rec.parse( row.fields )
-     ## end
-     yielder.yield( rec )
+      rec = new
+      values = CsvReader.unwrap( row )
+      rec.parse( values )
+
+      yielder.yield( rec )
     end
   end
 end
 
 
-def self.read( path )  ## not returns an enumarator
+def self.read( path, sep: Csv.config.sep, headers: true )  ## not returns an enumarator
   txt  = File.open( path, 'r:utf-8' ).read
-  parse( txt )
+  parse( txt, sep: sep, headers: headers )
 end
 
 

data/lib/csvrecord/reader.rb

@@ -0,0 +1,188 @@
+# encoding: utf-8
+
+
+module Csv    ## check: rename to CsvSettings / CsvPref / CsvGlobals or similar - why? why not???
+
+  ## STD_CSV_ENGINE = CSV   ## to avoid name confusion use longer name - why? why not? find a better name?
+  ## use __CSV__ or similar? or just ::CSV ??
+
+  class Configuration
+
+    puts "CSV::VERSION:"
+    puts CSV::VERSION
+
+    puts "builtin CSV::Converters:"
+    pp CSV::Converters
+
+    puts "CSV::DEFAULT_OPTIONS:"
+    pp CSV::DEFAULT_OPTIONS
+
+    ##  register our own converters
+    ## check if strip gets called for nil values too?
+    CSV::Converters[:strip] = ->(field) { field.strip }
+
+
+    attr_accessor :sep   ## col_sep (column separator)
+
+    def initialize
+      @sep = ','
+      ## note: do NOT add headers as global - should ALWAYS be explicit
+      ##   headers (true/false) - changes resultset and requires different processing!!!
+
+      self  ## return self for chaining
+    end
+
+    def blank?( line )
+      ## note:  blank line does NOT include "blank" with spaces only!!
+      ##          use BLANK_REGEX in skip_lines to clean-up/skip/remove/ignore
+      ##  see skip_blanks in default_options
+      line.empty?
+    end
+
+    ## lines starting with # (note: only leading spaces allowed)
+    COMMENTS_REGEX = /^\s*#/
+    BLANK_REGEX    = /^\s*$/   ## skip all whitespace lines - note: use "" or , for a blank record!!!
+    SKIP_REGEX = Regexp.union( COMMENTS_REGEX, BLANK_REGEX )
+
+    def skip?( line )
+      ## check if comment line - skip comments
+      ##  see skip_lines in default_options
+      line =~ SKIP_REGEX
+    end
+
+    ## built-in (default) options
+    ##  todo: find a better name?
+    def default_options
+      ## note:
+      ##   do NOT include sep character and
+      ##   do NOT include headers true/false here
+      ##
+      ##  make default sep its own "global" default config
+      ##   e.g. Csv.config.sep =
+
+      ## common options
+      ##   skip comments starting with #
+      ##   skip blank lines
+      ##   strip leading and trailing spaces
+      ##    NOTE/WARN:  leading and trailing spaces NOT allowed/working with double quoted values!!!!
+      defaults = {
+        skip_blanks: true,    ## note: skips lines with no whitespaces only!! (e.g. line with space is NOT blank!!)
+        skip_lines:  SKIP_REGEX,
+        :converters => :strip
+      }
+      defaults
+    end
+  end # class Configuration
+
+
+  ## lets you use
+  ##   Csv.configure do |config|
+  ##      config.sep = ','   ## or "/t"
+  ##   end
+
+  def self.configure
+    yield( config )
+  end
+
+  def self.config
+    @config ||= Configuration.new
+  end
+end   # module Csvv
+
+
+
+####
+## use our own wrapper
+
+class CsvReader
+
+  ####################
+  # helper methods
+  def self.unwrap( row_or_array )   ## unwrap row - find a better name? why? why not?
+    ## return row values as array of strings
+    if row_or_array.is_a?( CSV::Row )
+      row = row_or_array
+      row.fields   ## gets array of string of field values
+    else  ## assume "classic" array of strings
+      array = row_or_array
+    end
+  end
+
+
+
+  def self.foreach( path, sep: Csv.config.sep, headers: true )
+    csv_options = Csv.config.default_options.merge(
+                     headers: headers,
+                     col_sep: sep,
+                     external_encoding: 'utf-8'  ## note:  always (auto-)add utf-8 external encoding for now!!!
+    )
+
+    CSV.foreach( path, csv_options ) do |row|
+      yield( row )    ## check/todo: use block.call( row ) ## why? why not?
+    end
+  end
+
+
+  def self.read( path, sep: Csv.config.sep, headers: true )
+    ## note: use our own file.open
+    ##   always use utf-8 for now
+    ##    check/todo: add skip option bom too - why? why not?
+    txt = File.open( path, 'r:utf-8' )
+    parse( txt, sep: sep, headers: headers )
+  end
+
+  def self.parse( txt, sep: Csv.config.sep, headers: true )
+    csv_options = Csv.config.default_options.merge(
+                     headers: headers,
+                     col_sep: sep
+    )
+    ## pp csv_options
+    CSV.parse( txt, csv_options )
+  end
+
+  def self.parse_line( txt, sep: Csv.config.sep )
+    ## note: do NOT include headers option (otherwise single row gets skipped as first header row :-)
+    csv_options = Csv.config.default_options.merge(
+                    headers: false,  ## note: always turn off headers!!!!!!
+                    col_sep: sep
+    )
+    ## pp csv_options
+    CSV.parse_line( txt, csv_options )
+  end
+
+
+
+  def self.header( path, sep: Csv.config.sep )   ## use header or headers - or use both (with alias)?
+      # read first lines (only)
+      #  and parse with csv to get header from csv library itself
+      #
+      #  check - if there's an easier or built-in way for the csv library
+
+      ## readlines until
+      ##  - NOT a comments line or
+      ##  - NOT a blank line
+
+      lines = ''
+      File.open( path, 'r:utf-8' ) do |f|
+
+        ## todo/fix: how to handle empty files or files without headers?!
+
+        ## todo/check if readline includes \n\r too??
+        ## yes! - line include \n e.g.
+        ##   "Brewery,City,Name,Abv\n" or
+        ##   "#######\n#  try with some comments\n#   and blank lines even before header\n\nBrewery,City,Name,Abv\n"
+        loop do
+          line = f.readline
+          lines << line
+          break unless  Csv.config.skip?( line ) || Csv.config.blank?( line )
+        end
+      end
+
+      ## puts "lines:"
+      ## pp lines
+
+      ## note: do NOT use headers: true to get "plain" data array (no hash records)
+      ##   hash record does NOT work for single line/row
+      parse_line( lines, sep: sep )
+    end  # method self.header
+end # class CsvReader

data/lib/csvrecord/version.rb

@@ -4,7 +4,7 @@
 module CsvRecord
 
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
-  MINOR = 2
+  MINOR = 3
   PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 

data/test/data/beer11.csv

@@ -0,0 +1,51 @@
+#######
+#  try with some comments
+#   and blank lines even before header
+
+Brewery,City,Name,Abv
+Andechser Klosterbrauerei,Andechs,Doppelbock Dunkel,7%
+Augustiner Bräu München,München,Edelstoff,5.6%
+Bayerische Staatsbrauerei Weihenstephan,Freising,Hefe Weissbier,5.4%
+
+Brauerei Spezial,      Bamberg,  Rauchbier Märzen,   5.1%
+
+Hacker-Pschorr Bräu,   München,  Münchner Dunkel,    5.0%
+
+## some more comments here
+
+Staatliches Hofbräuhaus München,München,Hofbräu Oktoberfestbier,6.3%
+
+## check for nil
+"", ,,"",
+
+## check for blank line with spaces
+##    yes, will get added as a record!! e.g. ["", nil, nil, nil]
+##    use regex to skip blank lines with spaces!!!!
+    
+
+## test double quotes and double quotes escaped
+##   note: double quotes do NOT work with leading AND/OR trailing spaces
+##    leads to:
+##    CSV::MalformedCSVError - Missing or stray quote in line xxx
+##
+##   note: for now double quote does not accept leading AND/OR trailing spaces!!!!
+##
+##  todo/fix: check liberal_quote option starting in csv ruby 2.4 ???
+##
+## examples:
+##  "value with comma, comma","some ""hello""","some ""hello""",
+##    works - but does NOT work (note the leading and trailing spaces for double quotes):
+##  "value with comma, comma" ,"some ""hello""",  "some ""hello""",
+##
+## check for "multi-line":
+## "hello
+##  and another line
+##  and another",two,three,
+
+
+"value with comma, comma","some ""hello""","some ""hello""",
+
+## check for "multi-line"
+"hello
+  and another line
+  and another",two,three,

data/test/test_reader.rb

@@ -0,0 +1,119 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_reader.rb
+
+
+require 'helper'
+
+class TestReader < MiniTest::Test
+
+def test_read
+  puts "== read: beer.csv:"
+  table = CsvReader.read( "#{CsvRecord.test_data_dir}/beer.csv" )   ## returns CSV::Table
+
+  pp table.class.name
+  pp table
+  pp table.to_a   ## note: includes header (first row with column names)
+
+  table.each do |row|   ## note: will skip (NOT include) header row!!
+    pp row
+  end
+  puts "  #{table.size} rows"  ## note: again will skip (NOT include) header row in count!!!
+  assert_equal 6, table.size
+end
+
+def test_read_header_false
+  puts "== read (headers: false): beer.csv:"
+  data = CsvReader.read( "#{CsvRecord.test_data_dir}/beer.csv", headers: false )
+
+  pp data.class.name
+  pp data
+
+  data.each do |row|
+    pp row
+  end
+  puts "  #{data.size} rows"
+  assert_equal 7, data.size   ## note: include header row in count
+end
+
+
+def test_read11
+  puts "== read: beer11.csv:"
+  table = CsvReader.read( "#{CsvRecord.test_data_dir}/beer11.csv" )
+  pp table
+  pp table.to_a   ## note: includes header (first row with column names)
+
+  assert true
+end
+
+
+def test_parse_line
+  puts "== parse_line:"
+  row = CsvReader.parse_line( <<TXT )
+Augustiner Bräu München,                 München,  Edelstoff,      5.6%
+Bayerische Staatsbrauerei Weihenstephan, Freising, Hefe Weissbier, 5.4%
+TXT
+
+  pp row
+  assert_equal ['Augustiner Bräu München', 'München', 'Edelstoff', '5.6%'], row
+end
+
+def test_parse_line11
+  puts "== parse_line:"
+  row = CsvReader.parse_line( <<TXT )
+#######
+#  try with some comments
+#   and blank lines even before header
+
+Augustiner Bräu München,                 München,   Edelstoff,      5.6%
+Bayerische Staatsbrauerei Weihenstephan, Freising,  Hefe Weissbier, 5.4%
+TXT
+
+  pp row
+  assert_equal ['Augustiner Bräu München', 'München', 'Edelstoff', '5.6%'], row
+end
+
+def test_header
+  puts "== header: beer.csv:"
+  header = CsvReader.header( "#{CsvRecord.test_data_dir}/beer.csv" )
+  pp header
+  assert_equal ['Brewery','City','Name','Abv'], header
+end
+
+def test_header11
+  puts "== header: beer11.csv:"
+  header = CsvReader.header( "#{CsvRecord.test_data_dir}/beer11.csv" )
+  pp header
+  assert_equal ['Brewery','City','Name','Abv'], header
+end
+
+
+def test_foreach
+  puts "== foreach: beer.csv:"
+  CsvReader.foreach( "#{CsvRecord.test_data_dir}/beer.csv" ) do |row|
+    pp row
+    pp row.fields
+  end
+  assert true
+end
+
+def test_foreach11
+  puts "== foreach: beer11.csv:"
+  CsvReader.foreach( "#{CsvRecord.test_data_dir}/beer11.csv" ) do |row|
+    pp row
+    pp row.fields
+  end
+  assert true
+end
+
+def test_foreach_header_false
+  puts "== foreach (headers: false): beer11.csv:"
+  CsvReader.foreach( "#{CsvRecord.test_data_dir}/beer11.csv", headers: false ) do |row|
+    pp row      ## note: is Array (no .fields available!!!!!)
+  end
+  assert true
+end
+
+end # class TestReader

data/test/{test_beer.rb → test_record.rb}

@@ -2,21 +2,12 @@
 
 ###
 #  to run use
-#     ruby -I ./lib -I ./test test/test_beer.rb
+#     ruby -I ./lib -I ./test test/test_record.rb
 
 
 require 'helper'
 
-class TestBeer < MiniTest::Test
-
-  def test_version
-    pp CsvRecord::VERSION
-    pp CsvRecord.banner
-    pp CsvRecord.root
-
-    assert true  ## assume ok if we get here
-  end
-
+class TestRecord < MiniTest::Test
 
   def test_class_style1
      clazz1 = CsvRecord.define do
@@ -104,6 +95,35 @@ class TestBeer < MiniTest::Test
     assert true  ## assume ok if we get here
   end
 
+
+  def test_foreach
+    puts "== foreach:"
+    Beer.foreach( "#{CsvRecord.test_data_dir}/beer.csv" ) do |rec|
+      puts "#{rec.name} (#{rec.abv}%) by #{rec.brewery}, #{rec.city}"
+    end
+
+    assert true  ## assume ok if we get here
+  end
+
+
+  def test_parse_array_or_arrays
+    data=[
+      ['Brewery','City','Name','Abv'],
+      ['Andechser Klosterbrauerei','Andechs','Doppelbock Dunkel','7%'],
+      ['Augustiner Bräu München','München','Edelstoff','5.6%'],
+      ['Bayerische Staatsbrauerei Weihenstephan','Freising','Hefe Weissbier','5.4%'],
+      ['Brauerei Spezial','Bamberg','Rauchbier Märzen','5.1%'],
+      ['Hacker-Pschorr Bräu','München','Münchner Dunkel','5.0%'],
+      ['Staatliches Hofbräuhaus München','München','Hofbräu Oktoberfestbier','6.3%']]
+
+    beers = Beer.parse( data ).to_a
+    puts "#{beers.size} beers:"
+    pp beers
+
+    assert true  ## assume ok if we get here
+  end
+
+
   def test_classic
     puts "== read( data ).to_a:"
     beers = BeerClassic.read( "#{CsvRecord.test_data_dir}/beer.csv" ).to_a
@@ -227,4 +247,4 @@ class TestBeer < MiniTest::Test
     assert_equal values[3].to_f, beer['Abv']
   end
 
-end # class TestBeer
+end # class TestRecord

data/test/test_record_auto.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_record_auto.rb
+
+
+require 'helper'
+
+class TestRecordAuto < MiniTest::Test
+
+
+def test_read
+  beers = CsvRecord.read( "#{CsvRecord.test_data_dir}/beer.csv" ).to_a
+  pp beers
+
+  assert_equal 6, beers.size
+  assert_equal 'Andechser Klosterbrauerei', beers[0].brewery
+  assert_equal 'Andechs',                   beers[0].city
+  assert_equal 'Doppelbock Dunkel',         beers[0].name
+  assert_equal '7%',                        beers[0].abv
+end
+
+
+def test_foreach
+  CsvRecord.foreach( "#{CsvRecord.test_data_dir}/beer.csv" ) do |rec|
+    pp rec
+    puts "#{rec.name} (#{rec.abv}%) by #{rec.brewery}, #{rec.city}"
+  end
+
+  assert true
+end
+
+end # class TestRecordAuto

data/test/test_version.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_version.rb
+
+
+require 'helper'
+
+class TestVersion < MiniTest::Test
+
+  def test_version
+    pp CsvRecord::VERSION
+    pp CsvRecord.banner
+    pp CsvRecord.root
+
+    assert true  ## assume ok if we get here
+  end
+
+end # class TestVersion

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csvrecord
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-13 00:00:00.000000000 Z
+date: 2018-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rdoc
@@ -57,10 +57,15 @@ files:
 - lib/csvrecord.rb
 - lib/csvrecord/base.rb
 - lib/csvrecord/builder.rb
+- lib/csvrecord/reader.rb
 - lib/csvrecord/version.rb
 - test/data/beer.csv
+- test/data/beer11.csv
 - test/helper.rb
-- test/test_beer.rb
+- test/test_reader.rb
+- test/test_record.rb
+- test/test_record_auto.rb
+- test/test_version.rb
 homepage: https://github.com/csv11/csvrecord
 licenses:
 - Public Domain