csvreader 1.2.1 → 1.2.5

data/lib/csvreader/base.rb

@@ -1,13 +1,21 @@
-# encoding: utf-8
 
 
 require 'pp'
-require 'logger'
+require 'logger'   ## todo/fix:  check why logger is required - use logutils!!!???
 require 'forwardable'
 require 'stringio'
+require 'fileutils'
+
+require 'time'
 require 'date'    ## use for Date.parse and DateTime.parse
 require 'yaml'    ## used for (optional) meta data blocks
+require 'json'
+
 
+## our own parser libs
+require 'tabreader'
+require 'csvjson'
+require 'csvyaml'
 
 
 ###
@@ -19,6 +27,7 @@ require 'csvreader/parser_strict'   # flexible (strict - no leading/trailing spa
 require 'csvreader/parser_tab'
 require 'csvreader/parser_fixed'
 require 'csvreader/parser_json'
+require 'csvreader/parser_yaml'
 require 'csvreader/parser_table'
 require 'csvreader/parser'
 require 'csvreader/converter'
@@ -68,6 +77,8 @@ class Parser
   TABLE   = ParserTable.new    ## space-separated e.g /[ \t]+/
   FIXED   = ParserFixed.new
 
+  JSON    = ParserJson.new
+  YAML    = ParserYaml.new
 
   def self.default()         DEFAULT;         end ## alternative alias for DEFAULT
   def self.numeric()         NUMERIC;         end
@@ -85,11 +96,17 @@ class Parser
   def self.postgres()        postgresql;      end
   def self.postgresql_text() POSTGRESQL_TEXT; end
   def self.postgres_text()   postgresql_text; end
+
   def self.tab()             TAB;             end
   def self.table()           TABLE;           end
   def self.fixed()           FIXED;           end
   def self.fix()             fixed;           end
   def self.f()               fixed;           end
+
+  def self.json()            JSON;            end
+  def self.j()               json;            end
+  def self.yaml()            YAML;            end
+  def self.y()               yaml;            end
 end # class Parser
 end # class CsvReader
 
@@ -114,6 +131,8 @@ class CsvReader
   TABLE = Builder.new( Parser::TABLE )
   FIXED = Builder.new( Parser::FIXED )
 
+  JSON  = Builder.new( Parser::JSON )
+  YAML  = Builder.new( Parser::YAML )
 
   def self.default()         DEFAULT;         end ## alternative alias for DEFAULT
   def self.numeric()         NUMERIC;         end
@@ -131,11 +150,17 @@ class CsvReader
   def self.postgres()        postgresql;      end
   def self.postgresql_text() POSTGRESQL_TEXT; end
   def self.postgres_text()   postgresql_text; end
+
   def self.tab()             TAB;             end
   def self.table()           TABLE;           end
   def self.fixed()           FIXED;           end
   def self.fix()             fixed;           end
   def self.f()               fixed;           end
+
+  def self.json()            JSON;            end
+  def self.j()               json;            end
+  def self.yaml()            YAML;            end
+  def self.y()               yaml;            end
 end # class CsvReader
 
 
@@ -158,6 +183,9 @@ class CsvHashReader
   TABLE = Builder.new( Parser::TABLE )
   FIXED = Builder.new( Parser::FIXED )
 
+  JSON  = Builder.new( Parser::JSON )
+  YAML  = Builder.new( Parser::YAML )
+
 
   def self.default()         DEFAULT;         end ## alternative alias for DEFAULT
   def self.numeric()         NUMERIC;         end
@@ -171,11 +199,17 @@ class CsvHashReader
   def self.postgres()        postgresql;      end
   def self.postgresql_text() POSTGRESQL_TEXT; end
   def self.postgres_text()   postgresql_text; end
+
   def self.tab()             TAB;             end
   def self.table()           TABLE;           end
   def self.fixed()           FIXED;           end
   def self.fix()             fixed;           end
   def self.f()               fixed;           end
+
+  def self.json()            JSON;            end
+  def self.j()               json;            end
+  def self.yaml()            YAML;            end
+  def self.y()               yaml;            end
 end # class CsvHashReader
 
 

data/lib/csvreader/buffer.rb

@@ -1,4 +1,3 @@
-# encoding: utf-8
 
 class CsvReader
 class Buffer   ## todo: find a better name:

data/lib/csvreader/builder.rb

@@ -1,4 +1,3 @@
-# encoding: utf-8
 
 class CsvReader
 class Builder

data/lib/csvreader/converter.rb

@@ -1,4 +1,3 @@
-# encoding: utf-8
 
 class CsvReader
 

data/lib/csvreader/parser.rb

@@ -1,33 +1,32 @@
-# encoding: utf-8
-
-class CsvReader
-
-class Parser
-  ## "forward" reference,
-  ##     see base.rb for more
-end
-
-
-####################################
-# define errors / exceptions
-#   for all parsers for (re)use
-
-class Error < StandardError
-end
-
-####
-# todo/check:
-#  use "common" error class - why? why not?
-
-class ParseError < Error
-  attr_reader :message
-
-  def initialize( message )
-    @message = message
-  end
-
-  def to_s
-    "*** csv parse error: #{@message}"
-  end
-end # class ParseError
-end # class CsvReader
+
+class CsvReader
+
+class Parser
+  ## "forward" reference,
+  ##     see base.rb for more
+end
+
+
+####################################
+# define errors / exceptions
+#   for all parsers for (re)use
+
+class Error < StandardError
+end
+
+####
+# todo/check:
+#  use "common" error class - why? why not?
+
+class ParseError < Error
+  attr_reader :message
+
+  def initialize( message )
+    @message = message
+  end
+
+  def to_s
+    "*** csv parse error: #{@message}"
+  end
+end # class ParseError
+end # class CsvReader

data/lib/csvreader/parser_fixed.rb

@@ -1,106 +1,105 @@
-# encoding: utf-8
-
-class CsvReader
-
-class ParserFixed
-
-###################################
-## add simple logger with debug flag/switch
-#
-#  use Parser.debug = true   # to turn on
-#
-#  todo/fix: use logutils instead of std logger - why? why not?
-
-def self.build_logger()
-  l = Logger.new( STDOUT )
-  l.level = :info    ## set to :info on start; note: is 0 (debug) by default
-  l
-end
-def self.logger() @@logger ||= build_logger; end
-def logger()  self.class.logger; end
-
-
-def parse( data, width:, &block )
-
-  ## note: input: required each_line (string or io/file for example)
-
-  input = data   ## assume it's a string or io/file handle
-
-  if block_given?
-    parse_lines( input, width: width, &block )
-  else
-    records = []
-
-    parse_lines( input, width: width ) do |record|
-      records << record
-    end
-
-    records
-  end
-end ## method parse
-
-
-
-private
-
-def parse_lines( input, width:, &block )
-
-  ## note: each line only works with \n (windows) or \r\n (unix)
-  ##   will NOT work with \r (old mac, any others?) only!!!!
-  input.each_line do |line|
-
-    ##  note: chomp('') if is an empty string,
-    ##    it will remove all trailing newlines from the string.
-    ##    use line.sub(/[\n\r]*$/, '') or similar instead - why? why not?
-    line = line.chomp( '' )
-    logger.debug "line:"                if logger.debug?
-    logger.debug line.pretty_inspect    if logger.debug?
-
-
-    ## skip empty lines and comments
-    if line =~ /^[ \t]*$/   ## skip blank lines (with whitespace only)
-       logger.debug "skip blank line"    if logger.debug?
-       next
-    end
-
-    if line =~ /^[ \t]*#/   # start_with?( "#" ) -- skip comment lines (note: allow leading whitespaces)
-       logger.debug "skip comment line"   if logger.debug?
-       next
-    end
-
-
-    if width.is_a?( String )
-      ## assume it's String#unpack format e.g.
-      ##   "209231-231992395    MoreData".unpack('aa5A1A9a4Z*')
-      ##     returns an array as follows :
-      ##   ["2", "09231", "-", "231992395", "    ", "MoreData"]
-      ##  see String#unpack
-
-      values = line.unpack( width )
-    else  ## assume array with integers
-      values = []
-      offset = 0  # start position / offset
-      width.each_with_index do |w,i|
-         logger.debug "[#{i}] start: #{offset}, width: #{w}"   if logger.debug?
-
-         if w < 0   ## convention - if width negative, skip column
-            # note: minus (-) and minus (-) equal plus (+)
-            ##   e.g. 2 - -2 = 4
-           offset -= w
-         else
-           value = line[offset, w]
-           value = value.strip   if value    ## note: if not nil strip; only use rstrip (for trailing only) - why? why not?
-           values << value
-           offset += w
-         end
-      end
-    end
-
-    ## note: requires block - enforce? how? why? why not?
-    block.call( values )
-  end
-end # method parse_lines
-
-
-end # class ParserFixed
-end # class CsvReader
+
+class CsvReader
+
+class ParserFixed
+
+###################################
+## add simple logger with debug flag/switch
+#
+#  use Parser.debug = true   # to turn on
+#
+#  todo/fix: use logutils instead of std logger - why? why not?
+
+def self.build_logger()
+  l = Logger.new( STDOUT )
+  l.level = :info    ## set to :info on start; note: is 0 (debug) by default
+  l
+end
+def self.logger() @@logger ||= build_logger; end
+def logger()  self.class.logger; end
+
+
+def parse( data, width:, &block )
+
+  ## note: input: required each_line (string or io/file for example)
+
+  input = data   ## assume it's a string or io/file handle
+
+  if block_given?
+    parse_lines( input, width: width, &block )
+  else
+    records = []
+
+    parse_lines( input, width: width ) do |record|
+      records << record
+    end
+
+    records
+  end
+end ## method parse
+
+
+
+private
+
+def parse_lines( input, width:, &block )
+
+  ## note: each line only works with \n (windows) or \r\n (unix)
+  ##   will NOT work with \r (old mac, any others?) only!!!!
+  input.each_line do |line|
+
+    ##  note: chomp('') if is an empty string,
+    ##    it will remove all trailing newlines from the string.
+    ##    use line.sub(/[\n\r]*$/, '') or similar instead - why? why not?
+    line = line.chomp( '' )
+    logger.debug "line:"                if logger.debug?
+    logger.debug line.pretty_inspect    if logger.debug?
+
+
+    ## skip empty lines and comments
+    if line =~ /^[ \t]*$/   ## skip blank lines (with whitespace only)
+       logger.debug "skip blank line"    if logger.debug?
+       next
+    end
+
+    if line =~ /^[ \t]*#/   # start_with?( "#" ) -- skip comment lines (note: allow leading whitespaces)
+       logger.debug "skip comment line"   if logger.debug?
+       next
+    end
+
+
+    if width.is_a?( String )
+      ## assume it's String#unpack format e.g.
+      ##   "209231-231992395    MoreData".unpack('aa5A1A9a4Z*')
+      ##     returns an array as follows :
+      ##   ["2", "09231", "-", "231992395", "    ", "MoreData"]
+      ##  see String#unpack
+
+      values = line.unpack( width )
+    else  ## assume array with integers
+      values = []
+      offset = 0  # start position / offset
+      width.each_with_index do |w,i|
+         logger.debug "[#{i}] start: #{offset}, width: #{w}"   if logger.debug?
+
+         if w < 0   ## convention - if width negative, skip column
+            # note: minus (-) and minus (-) equal plus (+)
+            ##   e.g. 2 - -2 = 4
+           offset -= w
+         else
+           value = line[offset, w]
+           value = value.strip   if value    ## note: if not nil strip; only use rstrip (for trailing only) - why? why not?
+           values << value
+           offset += w
+         end
+      end
+    end
+
+    ## note: requires block - enforce? how? why? why not?
+    block.call( values )
+  end
+end # method parse_lines
+
+
+end # class ParserFixed
+end # class CsvReader

data/lib/csvreader/parser_json.rb

@@ -1,5 +1,23 @@
-# encoding: utf-8
-
-
-#######
-## todo: add parser for new CSV <3 JSON format
+
+
+class CsvReader
+
+class ParserJson
+
+def parse( data, **kwargs, &block )
+  ## note: kwargs NOT used for now (but required for "protocol/interface" by other parsers)
+
+  ## note: input: required each_line (string or io/file for example)
+  ## assume data is a string or io/file handle
+  csv = CsvJson.new( data )
+
+  if block_given?
+    csv.each( &block )
+  else
+    csv.to_a
+  end
+end ## method parse
+
+
+end # class ParserJson
+end # class CsvReader