csvutils 0.2.2 → 0.3.0

data/lib/csvutils/test.rb

@@ -1,22 +1,19 @@
-# encoding: utf-8
-
-
-class CsvUtils
-
-  ## test or dry run to check if rows can get read/scanned
-  def self.test( path, sep: ',' )
-    i = 0
-    csv_options = { headers: true,
-                    col_sep: sep,
-                    external_encoding: 'utf-8'  ## note:  always (auto-)add utf-8 external encoding!!!
-                   }
-
-    CSV.foreach( path, csv_options ) do |row|
-      i += 1
-      print '.' if i % 100 == 0
-    end
-
-    puts " #{i} rows"
-  end
-
-end  # class CsvUtils
+# encoding: utf-8
+
+
+class CsvUtils
+
+  ## test or dry run to check if rows can get read/scanned
+  def self.test( path, sep: ',' )
+    i = 0
+    csv_options = { sep: sep }
+
+    CsvHash.foreach( path, csv_options ) do |rec|
+      i += 1
+      print '.' if i % 100 == 0
+    end
+
+    puts " #{i} rows"
+  end
+
+end  # class CsvUtils

data/lib/csvutils/utils.rb

@@ -1,13 +1,29 @@
-# encoding: utf-8
-
-
-class CsvUtils
-
-  def self.pp_header( headers )  ## check: rename to print_headers or prettyprint_header - why? why not?
-    puts "#{headers.size} columns:"
-    headers.each_with_index do |header,i|
-      puts "  #{i+1}: #{header}"
-    end
-  end
-
-end  # class CsvUtils
+# encoding: utf-8
+
+
+class CsvUtils
+
+  def self.pp_header( headers )  ## check: rename to print_headers or prettyprint_header - why? why not?
+    puts "#{headers.size} columns:"
+    headers.each_with_index do |header,i|
+      puts "  #{i+1}: #{header}"
+    end
+  end
+
+
+  ###################
+  ## (simple) helper for "csv-encoding" values / row
+  ##
+  ##  todo: check for newline in value too? why? why not?
+  def self.csv_row( *values, sep: ',' )
+    values.map do |value|
+       if value && (value.index( sep ) || value.index('"'))
+         ## double quotes and enclose in double qoutes
+         value = %Q{"#{value.gsub('"', '""')}"}
+       else
+         value
+       end
+    end
+  end
+
+end  # class CsvUtils

data/lib/csvutils/version.rb

@@ -1,24 +1,24 @@
-# encoding: utf-8
-
-
-## note: for now CsvUtils is a class!!! NOT a module - change - why? why not?
-class CsvUtils
-
-  MAJOR = 0    ## todo: namespace inside version or something - why? why not??
-  MINOR = 2
-  PATCH = 2
-  VERSION = [MAJOR,MINOR,PATCH].join('.')
-
-  def self.version
-    VERSION
-  end
-
-  def self.banner
-    "csvutils/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
-  end
-
-  def self.root
-    File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )
-  end
-
-end # class CsvUtils
+# encoding: utf-8
+
+
+## note: for now CsvUtils is a class!!! NOT a module - change - why? why not?
+class CsvUtils
+
+  MAJOR = 0    ## todo: namespace inside version or something - why? why not??
+  MINOR = 3
+  PATCH = 0
+  VERSION = [MAJOR,MINOR,PATCH].join('.')
+
+  def self.version
+    VERSION
+  end
+
+  def self.banner
+    "csvutils/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
+  end
+
+  def self.root
+    File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )
+  end
+
+end # class CsvUtils

data/test/helper.rb

@@ -1,16 +1,16 @@
-## $:.unshift(File.dirname(__FILE__))
-
-## minitest setup
-
-require 'minitest/autorun'
-
-
-## our own code
-require 'csvutils'
-
-## add test_data_dir helper
-class CsvUtils
-  def self.test_data_dir
-    "#{root}/test/data"
-  end
-end
+## $:.unshift(File.dirname(__FILE__))
+
+## minitest setup
+
+require 'minitest/autorun'
+
+
+## our own code
+require 'csvutils'
+
+## add test_data_dir helper
+class CsvUtils
+  def self.test_data_dir
+    "#{root}/datasets"
+  end
+end

data/test/test_cut.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_cut.rb
+
+
+require 'helper'
+
+class TestCut < MiniTest::Test
+
+  def test_eng
+    path = "#{CsvUtils.test_data_dir}/eng-england/2017-18/E0.csv"
+    columns = [ 'HomeTeam', 'FTHG', 'FTAG', 'AwayTeam', 'Date' ]
+    CsvUtils.cut( path, *columns, output: './tmp/cut_test_eng.csv' )
+  end
+
+  def test_at
+    path = "#{CsvUtils.test_data_dir}/at-austria/AUT.csv"
+    columns = [ 'Home', 'HG', 'AG', 'Away', 'Date', 'Time' ]
+    CsvUtils.cut( path, *columns, output: './tmp/cut_test_at.csv' )
+  end
+
+  def test_de
+    path = "#{CsvUtils.test_data_dir}/de-deutschland/bundesliga.csv"
+    columns = ['Saison', 'Spieltag',
+               'Heim', 'Ergebnis', 'Gast', 'Datum', 'Uhrzeit' ]
+    CsvUtils.cut( path, *columns, sep: ';', output: './tmp/cut_test_de.csv' )
+  end
+
+end # class TestHead

data/test/test_head.rb

@@ -0,0 +1,30 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_head.rb
+
+
+require 'helper'
+
+class TestHead < MiniTest::Test
+
+  def test_eng
+    path = "#{CsvUtils.test_data_dir}/eng-england/2017-18/E0.csv"
+
+    CsvUtils.head( path )
+  end
+
+  def test_at
+    path = "#{CsvUtils.test_data_dir}/at-austria/AUT.csv"
+
+    CsvUtils.head( path )
+  end
+
+  def test_de
+    path = "#{CsvUtils.test_data_dir}/de-deutschland/bundesliga.csv"
+
+    CsvUtils.head( path, sep: ';' )
+  end
+
+end # class TestHead

data/test/{test_headers.rb → test_header.rb}

@@ -1,50 +1,50 @@
-# encoding: utf-8
-
-###
-#  to run use
-#     ruby -I ./lib -I ./test test/test_headers.rb
-
-
-require 'helper'
-
-class TestHeaders < MiniTest::Test
-
-
-##
-# Div,Date,HomeTeam,AwayTeam,FTHG,FTAG,FTR,HTHG,HTAG,HTR,
-#   Referee,HS,AS,HST,AST,HF,AF,HC,AC,HY,AY,HR,AR,
-#    B365H,B365D,B365A,BWH,BWD,BWA,IWH,IWD,IWA,LBH,LBD,LBA,PSH,PSD,PSA,
-#    WHH,WHD,WHA,VCH,VCD,VCA,
-#    Bb1X2,BbMxH,BbAvH,BbMxD,BbAvD,BbMxA,BbAvA,BbOU,BbMx>2.5,BbAv>2.5,BbMx<2.5,BbAv<2.5,
-#    BbAH,BbAHh,BbMxAHH,BbAvAHH,BbMxAHA,BbAvAHA,PSCH,PSCD,PSCA
-  def test_eng
-    path = "#{CsvUtils.test_data_dir}/eng-england/2017-18/E0.csv"
-
-    headers = CsvUtils.header( path )
-    pp headers
-
-    assert_equal ['Date','HomeTeam','AwayTeam','FTHG','FTAG','HTHG','HTAG'], headers
-  end
-
-###
-# Country,League,Season,Date,Time,Home,Away,HG,AG,
-#  Res,PH,PD,PA,MaxH,MaxD,MaxA,AvgH,AvgD,AvgA
-  def test_at
-    path = "#{CsvUtils.test_data_dir}/at-austria/AUT.csv"
-
-    headers =  CsvUtils.header( path )
-    pp headers
-
-    assert_equal ['Season','Date','Time','Home','Away','HG','AG'], headers
-  end
-
-  def test_de
-    path = "#{CsvUtils.test_data_dir}/de-deutschland/bundesliga.csv"
-
-    headers =  CsvUtils.header( path, sep: ';' )
-    pp headers
-
-    assert_equal ['Spielzeit','Saison','Spieltag','Datum','Uhrzeit','Heim','Gast','Ergebnis','Halbzeit'], headers
-  end
-
-end # class TestHeaders
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_header.rb
+
+
+require 'helper'
+
+class TestHeader < MiniTest::Test
+
+
+##
+# Div,Date,HomeTeam,AwayTeam,FTHG,FTAG,FTR,HTHG,HTAG,HTR,
+#   Referee,HS,AS,HST,AST,HF,AF,HC,AC,HY,AY,HR,AR,
+#    B365H,B365D,B365A,BWH,BWD,BWA,IWH,IWD,IWA,LBH,LBD,LBA,PSH,PSD,PSA,
+#    WHH,WHD,WHA,VCH,VCD,VCA,
+#    Bb1X2,BbMxH,BbAvH,BbMxD,BbAvD,BbMxA,BbAvA,BbOU,BbMx>2.5,BbAv>2.5,BbMx<2.5,BbAv<2.5,
+#    BbAH,BbAHh,BbMxAHH,BbAvAHH,BbMxAHA,BbAvAHA,PSCH,PSCD,PSCA
+  def test_eng
+    path = "#{CsvUtils.test_data_dir}/eng-england/2017-18/E0.csv"
+
+    headers = CsvUtils.header( path )
+    pp headers
+
+    assert_equal ['Date','HomeTeam','AwayTeam','FTHG','FTAG','HTHG','HTAG'], headers
+  end
+
+###
+# Country,League,Season,Date,Time,Home,Away,HG,AG,
+#  Res,PH,PD,PA,MaxH,MaxD,MaxA,AvgH,AvgD,AvgA
+  def test_at
+    path = "#{CsvUtils.test_data_dir}/at-austria/AUT.csv"
+
+    headers =  CsvUtils.header( path )
+    pp headers
+
+    assert_equal ['Season','Date','Time','Home','Away','HG','AG'], headers
+  end
+
+  def test_de
+    path = "#{CsvUtils.test_data_dir}/de-deutschland/bundesliga.csv"
+
+    headers =  CsvUtils.header( path, sep: ';' )
+    pp headers
+
+    assert_equal ['Spielzeit','Saison','Spieltag','Datum','Uhrzeit','Heim','Gast','Ergebnis','Halbzeit'], headers
+  end
+
+end # class TestHeader

data/test/test_misc.rb

@@ -1,44 +1,44 @@
-# encoding: utf-8
-
-###
-#  to run use
-#     ruby -I ./lib -I ./test test/test_misc.rb
-
-
-require 'helper'
-
-class TestMiscellaneous < MiniTest::Test
-
-  def test_eng
-    path = "#{CsvUtils.test_data_dir}/eng-england/2017-18/E0.csv"
-
-    CsvUtils.test( path )
-
-    CsvUtils.stat( path )
-    CsvUtils.stat( path, 'HomeTeam', 'AwayTeam' )
-
-    assert true
-  end
-
-  def test_test_de
-    path = "#{CsvUtils.test_data_dir}/de-deutschland/bundesliga.csv"
-
-    CsvUtils.test( path, sep: ';' )
-
-    CsvUtils.stat( path, sep: ';' )
-    CsvUtils.stat( path, 'Spielzeit', 'Saison', 'Heim', 'Gast', sep: ';' )
-
-    assert true
-  end
-
-  def test_test_at
-    path = "#{CsvUtils.test_data_dir}/at-austria/AUT.csv"
-
-    CsvUtils.test( path )
-
-    CsvUtils.stat( path )
-    CsvUtils.stat( path, 'Season', 'Home', 'Away' )
-    assert true
-  end
-
-end # class TestMiscellaneous
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_misc.rb
+
+
+require 'helper'
+
+class TestMiscellaneous < MiniTest::Test
+
+  def test_eng
+    path = "#{CsvUtils.test_data_dir}/eng-england/2017-18/E0.csv"
+
+    CsvUtils.test( path )
+
+    CsvUtils.stat( path )
+    CsvUtils.stat( path, 'HomeTeam', 'AwayTeam' )
+
+    assert true
+  end
+
+  def test_test_de
+    path = "#{CsvUtils.test_data_dir}/de-deutschland/bundesliga.csv"
+
+    CsvUtils.test( path, sep: ';' )
+
+    CsvUtils.stat( path, sep: ';' )
+    CsvUtils.stat( path, 'Spielzeit', 'Saison', 'Heim', 'Gast', sep: ';' )
+
+    assert true
+  end
+
+  def test_test_at
+    path = "#{CsvUtils.test_data_dir}/at-austria/AUT.csv"
+
+    CsvUtils.test( path )
+
+    CsvUtils.stat( path )
+    CsvUtils.stat( path, 'Season', 'Home', 'Away' )
+    assert true
+  end
+
+end # class TestMiscellaneous

data/test/test_split.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_split.rb
+
+
+require 'helper'
+
+class TestSplit < MiniTest::Test
+
+  def test_eng
+    path = "#{CsvUtils.test_data_dir}/eng-england/2017-18/E0.csv"
+    columns = [ 'HomeTeam' ]
+    CsvUtils.split( path, *columns ) do |values, chunk|
+      pp values
+      pp chunk
+    end
+  end
+
+
+  def test_de
+    path = "#{CsvUtils.test_data_dir}/de-deutschland/bundesliga.csv"
+    columns = ['Saison', 'Spieltag' ]
+    CsvUtils.split( path, *columns, sep: ';' ) do |values, chunk|
+      pp values
+      pp chunk
+    end
+  end
+
+end # class TestSplit