csvjson 0.0.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 56559ebdff3af27b8342110f2086c854a858ba99
-  data.tar.gz: 82cb4535856f56c62be126da74655bddf54dabd1
+  metadata.gz: d79afe20fafe225a50aa9a270e6bee240b47566a
+  data.tar.gz: e62c8fc7c94431e8b4756535f93221b743dfac08
 SHA512:
-  metadata.gz: 40516df39243045d71dd4a150bfed56c40c2039e7381bfae558ab125cf6207f7b4b3094ab9041d0d79ac58931d6d225b2aac9fae1d76094ffe7dd3d7076ad432
-  data.tar.gz: 5a7cd3c63e571772634e6485b15e40aef4011fd30336a88e540111104a8a55b1d57b14b5167185eb5a432bc505451b522d4dfd23520924ae4025fb66d2245b93
+  metadata.gz: d642bd3599dd04e15abd6d4accc5f2c63106af5158ac4876d79f0583c689e33013b05adb762e668db757028fcaed435e248b5b25703445b4379d5e058b5f4fce
+  data.tar.gz: 671a42c6e0f4245142d691ad9c0b9b596eb6754c2d543a1c95b6d52055b274c14321e1992d311778907c7fd6a443440911560a6dd6b134d73e2bd666a859c36b

data/Manifest.txt

@@ -3,4 +3,10 @@ Manifest.txt
 README.md
 Rakefile
 lib/csvjson.rb
+lib/csvjson/parser.rb
 lib/csvjson/version.rb
+test/data/hello.json.csv
+test/data/hello11.json.csv
+test/helper.rb
+test/test_parser.rb
+test/test_parser_misc.rb

data/README.md

@@ -1,3 +1,155 @@
-# csvjson
+# CSV <3 JSON Parser / Reader
 
 csvjson library / gem - read tabular data in the CSV <3 JSON format, that is, comma-separated values CSV (line-by-line) records with javascript object notation (JSON) encoding rules
+
+* home  :: [github.com/csv11/csvjson](https://github.com/csv11/csvjson)
+* bugs  :: [github.com/csv11/csvjson/issues](https://github.com/csv11/csvjson/issues)
+* gem   :: [rubygems.org/gems/csvjson](https://rubygems.org/gems/csvjson)
+* rdoc  :: [rubydoc.info/gems/csvjson](http://rubydoc.info/gems/csvjson)
+* forum :: [wwwmake](http://groups.google.com/group/wwwmake)
+
+
+## What's CSV <3 JSON?
+
+CSV <3 JSON is a Comma-Separated Values (CSV)
+variant / format / dialect
+where the line-by-line records follow the
+JavaScript Object Notation (JSON) encoding rules.
+It's a modern (simple) tabular data format that
+includes arrays, numbers, booleans, nulls, nested structures, comments and more.
+Example:
+
+
+```
+# "Vanilla" CSV <3 JSON
+
+1,"John","12 Totem Rd. Aspen",true
+2,"Bob",null,false
+3,"Sue","Bigsby, 345 Carnival, WA 23009",false
+```
+
+or
+
+```
+# CSV <3 JSON with array values
+
+1,"directions",["north","south","east","west"]
+2,"colors",["red","green","blue"]
+3,"drinks",["soda","water","tea","coffe"]
+4,"spells",[]
+```
+
+For more see the [official CSV <3 JSON Format documentation »](https://github.com/csv11/csv-json)
+
+
+
+## Usage
+
+``` ruby
+txt <<=TXT
+# "Vanilla" CSV <3 JSON
+
+1,"John","12 Totem Rd. Aspen",true
+2,"Bob",null,false
+3,"Sue","Bigsby, 345 Carnival, WA 23009",false
+TXT
+
+records = CsvJson.parse( txt )     ## or CSV_JSON.parse or CSVJSON.parse
+pp records
+# =>  [[1,"John","12 Totem Rd. Aspen",true],
+#      [2,"Bob",nil,false],
+#      [3,"Sue","Bigsby, 345 Carnival, WA 23009",false]]
+
+# -or-
+
+records = CsvJson.read( "values.json.csv" )   ## or CSV_JSON.read or CSVJSON.read
+pp records
+# =>  [[1,"John","12 Totem Rd. Aspen",true],
+#      [2,"Bob",nil,false],
+#      [3,"Sue","Bigsby, 345 Carnival, WA 23009",false]]
+
+# -or-
+
+CsvJson.foreach( "values.json.csv" ) do |rec|   ## or CSV_JSON.foreach or CSVJSON.foreach
+  pp rec
+end
+# => [1,"John","12 Totem Rd. Aspen",true]
+# => [2,"Bob",nil,false]
+# => [3,"Sue","Bigsby, 345 Carnival, WA 23009",false]
+```
+
+
+
+### What about Enumerable?
+
+Yes, the reader / parser includes `Enumerable` and runs on `each`.
+Use `new` or `open` without a block
+to get the enumerator (iterator).
+Example:
+
+
+``` ruby
+csv = CsvJson.new( "1,2,3" )   ## or CSV_JSON.new or CSVJSON.new
+it  = csv.to_enum
+pp it.next  
+# => [1,2,3]
+
+# -or-
+
+csv = CsvJson.open( "values.json.csv" )  ## or CSV_JSON.open or CSVJSON.open
+it  = csv.to_enum
+pp it.next
+# => [1,"John","12 Totem Rd. Aspen",true]
+pp it.next
+# => [2,"Bob",nil,false]
+```
+
+
+
+### What about headers?
+
+Yes, you can. Use the `CsvHash`
+from the csvreader library / gem
+if the first line is a header (or if missing pass in the headers
+as an array) and you want your records as hashes instead of arrays of strings.
+Example:
+
+``` ruby
+txt <<=TXT
+"id","name","address","regular"
+1,"John","12 Totem Rd. Aspen",true
+2,"Bob",null,false
+3,"Sue","Bigsby, 345 Carnival, WA 23009",false
+TXT
+
+records = CsvHash.json.parse( txt )
+pp records
+
+# => [{"id":      1,
+#      "name":    "John",
+#      "address": "12 Totem Rd. Aspen",
+#      "regular": true},
+#     {"id":      2,
+#      "name":    "Bob",
+#      "address": null,
+#      "regular": false},
+#    ... ]
+```
+
+For more see the [official CsvHash documentation in the csvreader library / gem »](https://github.com/csv11/csvreader)
+
+
+
+
+
+
+## License
+
+The `csvjson` scripts are dedicated to the public domain.
+Use it as you please with no restrictions whatsoever.
+
+
+## Questions? Comments?
+
+Send them along to the [wwwmake forum](http://groups.google.com/group/wwwmake).
+Thanks!

data/lib/csvjson.rb

@@ -1,11 +1,22 @@
 # encoding: utf-8
 
 require 'pp'
+require 'json'
+require 'logger'
 
 
 ## our own code
+##   todo/check: use require_relative - why? why not?
 require 'csvjson/version'    # note: let version always go first
+require 'csvjson/parser'
 
 
-pp CsvJson.banner
-pp CsvJson.root
+## add some "alternative" shortcut aliases
+CSV_JSON = CsvJson
+CSVJSON  = CsvJson
+CSVJ     = CsvJson
+CsvJ     = CsvJson
+
+
+
+puts CsvJson.banner

data/lib/csvjson/parser.rb

@@ -0,0 +1,131 @@
+# encoding: utf-8
+
+
+class CsvJson
+
+###################################
+## add simple logger with debug flag/switch
+#
+#  use Parser.debug = true   # to turn on
+#
+#  todo/fix: use logutils instead of std logger - why? why not?
+
+def self.build_logger()
+  l = Logger.new( STDOUT )
+  l.level = :info    ## set to :info on start; note: is 0 (debug) by default
+  l
+end
+def self.logger() @@logger ||= build_logger; end
+def logger()  self.class.logger; end
+
+
+
+
+def self.open( path, mode=nil, &block )   ## rename path to filename or name - why? why not?
+
+    ## note: default mode (if nil/not passed in) to 'r:bom|utf-8'
+    f = File.open( path, mode ? mode : 'r:bom|utf-8' )
+    csv = new( f )
+
+    # handle blocks like Ruby's open()
+    if block_given?
+      begin
+        block.call( csv )
+      ensure
+        csv.close
+      end
+    else
+      csv
+    end
+end # method self.open
+
+
+def self.read( path )
+    open( path ) { |csv| csv.read }
+end
+
+
+def self.foreach( path, &block )
+  csv = open( path )
+
+  if block_given?
+    begin
+      csv.each( &block )
+    ensure
+      csv.close
+    end
+  else
+    csv.to_enum    ## note: caller (responsible) must close file!!!
+    ## remove version without block given - why? why not?
+    ## use Csv.open().to_enum  or Csv.open().each
+    ##   or Csv.new( File.new() ).to_enum or Csv.new( File.new() ).each ???
+  end
+end # method self.foreach
+
+
+def self.parse( data, &block )
+  csv = new( data )
+
+  if block_given?
+    csv.each( &block )  ## note: caller (responsible) must close file!!! - add autoclose - why? why not?
+  else  # slurp contents, if no block is given
+    csv.read            ## note: caller (responsible) must close file!!! - add autoclose - why? why not?
+  end
+end # method self.parse
+
+
+
+def initialize( data )
+  if data.is_a?( String )
+    @input = data   # note: just needs each for each_line
+  else  ## assume io
+    @input = data
+  end
+end
+
+
+
+include Enumerable
+
+def each( &block )
+  if block_given?
+    @input.each_line do |line|
+
+      logger.debug  "line:"             if logger.debug?
+      logger.debug line.pretty_inspect  if logger.debug?
+
+
+      ##  note: chomp('') if is an empty string,
+      ##    it will remove all trailing newlines from the string.
+      ##    use line.sub(/[\n\r]*$/, '') or similar instead - why? why not?
+      line = line.chomp( '' )
+      line = line.strip         ## strip leading and trailing whitespaces (space/tab) too
+      logger.debug line.pretty_inspect    if logger.debug?
+
+      if line.empty?             ## skip blank lines
+        logger.debug "skip blank line"    if logger.debug?
+        next
+      end
+
+      if line.start_with?( "#" )  ## skip comment lines
+        logger.debug "skip comment line"   if logger.debug?
+        next
+      end
+
+      ## note: auto-wrap in array e.g. with []
+      json = JSON.parse( "[#{line}]" )
+      logger.debug json.pretty_inspect   if logger.debug?
+      block.call( json )
+    end
+  else
+     to_enum
+  end
+end # method each
+
+def read() to_a; end # method read
+
+def close
+  @input.close   if @input.respond_to?(:close)   ## note: string needs no close
+end
+
+end # class CsvJson

data/lib/csvjson/version.rb

@@ -4,9 +4,9 @@
 
 class CsvJson
 
-  MAJOR = 0
+  MAJOR = 1
   MINOR = 0
-  PATCH = 1
+  PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/test/data/hello.json.csv

@@ -0,0 +1,3 @@
+1,"John","12 Totem Rd. Aspen",true
+2,"Bob",null,false
+3,"Sue","Bigsby, 345 Carnival, WA 23009",false

data/test/data/hello11.json.csv

@@ -0,0 +1,5 @@
+# hello world
+
+1, "John", "12 Totem Rd. Aspen",             true
+2, "Bob",  null,                             false
+3, "Sue",  "Bigsby, 345 Carnival, WA 23009", false

data/test/helper.rb

@@ -0,0 +1,20 @@
+## $:.unshift(File.dirname(__FILE__))
+
+## minitest setup
+
+require 'minitest/autorun'
+
+
+## our own code
+require 'csvjson'
+
+
+## add test_data_dir helper
+class CsvJson
+  def self.test_data_dir
+    "#{root}/test/data"
+  end
+end
+
+
+CsvJson.logger.level = :debug   ## turn on "global" logging

data/test/test_parser.rb

@@ -0,0 +1,104 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_parser.rb
+
+
+require 'helper'
+
+class TestParser < MiniTest::Test
+
+
+def parser
+  CsvJson
+end
+
+def records   ## "standard" records for testing
+  [[1, "John", "12 Totem Rd. Aspen",             true],
+   [2, "Bob",  nil,                              false],
+   [3, "Sue",  "Bigsby, 345 Carnival, WA 23009", false]]
+end
+
+
+
+def test_parse
+  assert_equal records, parser.parse( <<TXT )
+1,"John","12 Totem Rd. Aspen",true
+2,"Bob",null,false
+3,"Sue","Bigsby, 345 Carnival, WA 23009",false
+TXT
+
+  assert_equal records, parser.parse( <<TXT )
+# hello world
+
+1,"John","12 Totem Rd. Aspen",true
+2,"Bob",null,false
+3,"Sue","Bigsby, 345 Carnival, WA 23009",false
+TXT
+
+  assert_equal records, parser.parse( <<TXT )
+  # hello world (pretty printed)
+
+  1, "John", "12 Totem Rd. Aspen",             true
+  2, "Bob",  null,                             false
+  3, "Sue",  "Bigsby, 345 Carnival, WA 23009", false
+
+  # try more comments and empty lines
+
+TXT
+
+
+  txt =<<TXT
+  # hello world
+
+  1,"John","12 Totem Rd. Aspen",true
+  2,"Bob",null,false
+  3,"Sue","Bigsby, 345 Carnival, WA 23009",false
+TXT
+
+  recs = []
+  parser.parse( txt ) { |rec| recs << rec }
+  assert_equal records, recs
+end
+
+
+def test_read
+  assert_equal records, parser.read( "#{CsvJson.test_data_dir}/hello.json.csv" )
+  assert_equal records, parser.read( "#{CsvJson.test_data_dir}/hello11.json.csv" )
+end
+
+
+def test_open
+  assert_equal records, parser.open( "#{CsvJson.test_data_dir}/hello.json.csv", "r:bom|utf-8" ).read
+  assert_equal records, parser.open( "#{CsvJson.test_data_dir}/hello11.json.csv", "r:bom|utf-8" ).read
+end
+
+
+def test_foreach
+  recs = []
+  parser.foreach( "#{CsvJson.test_data_dir}/hello.json.csv" ) { |rec| recs << rec }
+  assert_equal records, recs
+
+  recs = []
+  parser.foreach( "#{CsvJson.test_data_dir}/hello11.json.csv" ) { |rec| recs << rec }
+  assert_equal records, recs
+end
+
+
+def test_enum
+  csv = CsvJson.new( <<TXT )
+  # hello world
+
+  1,"John","12 Totem Rd. Aspen",true
+  2,"Bob",null,false
+  3,"Sue","Bigsby, 345 Carnival, WA 23009",false
+TXT
+
+  it = csv.to_enum
+  assert_equal [1, "John", "12 Totem Rd. Aspen",             true], it.next
+  assert_equal [2, "Bob",  nil,                              false], it.next
+  assert_equal [3, "Sue",  "Bigsby, 345 Carnival, WA 23009", false], it.next
+end
+
+end # class TestParser

data/test/test_parser_misc.rb

@@ -0,0 +1,80 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_parser_misc.rb
+
+
+require 'helper'
+
+class TestParserMisc < MiniTest::Test
+
+
+def parser
+  CsvJson
+end
+
+
+def test_quotes_and_commas
+  assert_equal [
+    [1, "John", "12 Totem Rd., Aspen", true],
+    [2, "Bob",  nil,                   false],
+    [3, "Sue", "\"Bigsby\", 345 Carnival, WA 23009", false]
+    ], parser.parse( <<TXT )
+  1,"John","12 Totem Rd., Aspen",true
+  2,"Bob",null,false
+  3,"Sue","\\"Bigsby\\", 345 Carnival, WA 23009",false
+TXT
+end
+
+
+def test_arrays
+  assert_equal [
+    [1, "directions", ["north","south","east","west"]],
+    [2, "colors", ["red","green","blue"]],
+    [3, "drinks", ["soda","water","tea","coffe"]],
+    [4, "spells", []],
+  ], parser.parse( <<TXT )
+  # CSV <3 JSON with array values
+
+  1,"directions",["north","south","east","west"]
+  2,"colors",["red","green","blue"]
+  3,"drinks",["soda","water","tea","coffe"]
+  4,"spells",[]
+TXT
+end
+
+def test_misc
+  ## note:
+  ##   in the csv <3 json source text backslash needs to get doubled / escaped twice e.g.
+  ##   \\"  for quotes
+  ##   \\n  for newlines and so on
+
+  assert_equal [
+    ["index", "value1", "value2"],
+    ["number", 1, 2],
+    ["boolean", false, true],
+    ["null", nil, "non null"],
+    ["array of numbers", [1], [1,2]],
+    ["simple object", {"a" => 1}, {"a" => 1, "b" => 2}],
+    ["array with mixed objects", [1, nil,"ball"], [2,{"a" => 10, "b" => 20},"cube"]],
+    ["string with quotes", "a\"b", "alert(\"Hi!\")"],
+    ["string with bell&newlines","bell is \u0007","multi\nline\ntext"]
+  ], parser.parse( <<TXT )
+  # CSV with all kinds of values
+
+  "index","value1","value2"
+  "number",1,2
+  "boolean",false,true
+  "null",null,"non null"
+  "array of numbers",[1],[1,2]
+  "simple object",{"a": 1},{"a":1, "b":2}
+  "array with mixed objects",[1,null,"ball"],[2,{"a": 10, "b": 20},"cube"]
+  "string with quotes","a\\"b","alert(\\"Hi!\\")"
+  "string with bell&newlines","bell is \\u0007","multi\\nline\\ntext"
+TXT
+
+end
+
+
+end # class TestParserMisc

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: csvjson
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Gerald Bauer
@@ -54,7 +54,13 @@ files:
 - README.md
 - Rakefile
 - lib/csvjson.rb
+- lib/csvjson/parser.rb
 - lib/csvjson/version.rb
+- test/data/hello.json.csv
+- test/data/hello11.json.csv
+- test/helper.rb
+- test/test_parser.rb
+- test/test_parser_misc.rb
 homepage: https://github.com/csv11/csvjson
 licenses:
 - Public Domain