censive 0.7 → 0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8a757fa8bbc5ddf364889e4b7feca2001f3784e8d0b2ff70a1b0349691a34aae
-  data.tar.gz: 68dced562eb0dc9b7ad300447091ceb74c04a55201e88cc9fffbe1ba3bbc534d
+  metadata.gz: 6419efcdc9274ea8bcf7b8527001e33f8bdfea348dfd911686cab36984d507da
+  data.tar.gz: 3b59aead54517fd64d7ece3eaa6f459e301e1e48f1ae34772a7128c61fb739f2
 SHA512:
-  metadata.gz: c48d7e2bd3d1a7baa5fb2fae7b0553de665737849e9a50721f704a1a1f67c758c545dfe53d21f32ce386b20ea21f04c67ee8d765bf20653774b9475ebb60711f
-  data.tar.gz: 411d59006ebcb6a07161186b56f73a8dcc73beeaecbe14e786ad237935c62fd6ef0631483c8f297399098b0dea2387863f7be8c878568e0558804e5bd20b55ee
+  metadata.gz: 7910c09e76a81ed27870ea52fb6c8aea0316ed213c53a026d98adc64f93349477e6acab0a93b88c6f184ce1d317634ecdca9290d50bff9b117b98bedd3ac7b86
+  data.tar.gz: 358ab985947d486b5f486b1f7e9c1f591e3b8e906b9eab59a4ed151e5f5d9652c211f2d2a4ee36f0543227e2ae5e33ba57f1e4c178f6f7e72e05c14d7b46895f

data/censive.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = "censive"
-  s.version     = "0.7"
+  s.version     = "0.8"
   s.author      = "Steve Shreeve"
   s.email       = "steve.shreeve@gmail.com"
   s.summary     = "A quick and lightweight CSV handling library for Ruby"

data/lib/censive.rb

@@ -5,6 +5,8 @@
 #
 # Author: Steve Shreeve (steve.shreeve@gmail.com)
 #   Date: Jan 30, 2023
+#
+# Thanks: Crystal's CSV library, see https://crystal-lang.org/api/1.7.2/CSV.html
 # ==============================================================================
 # The goals are:
 #
@@ -15,10 +17,7 @@
 #
 # 1. Option to support IO streaming
 # 2. Option to strip whitespace
-# 3. Option to change output line endings
-# 4. Option to force quotes in output
-# 5. Option to allow reading excel CSV (="Text" for cells)
-# 6. Confirm file encodings such as UTF-8, UTF-16, etc.
+# 3. Confirm file encodings such as UTF-8, UTF-16, etc.
 #
 # NOTE: Only getch and scan_until advance strscan's position
 # ==============================================================================
@@ -39,6 +38,7 @@ class Censive < StringScanner
 
     drop:  false   , # enable to drop trailing separators
     eol:   "\n"    , # desired line endings for exports
+    excel: false   , # allow ,="0123" style columns
     mode:  :compact, # export mode: compact or full
     out:   nil     , # output IO/file
     relax: false   , # relax parsing of quotes
@@ -48,56 +48,61 @@ class Censive < StringScanner
     super(str || '')
     reset
 
-    @sep   = sep  .freeze
-    @quote = quote.freeze
+    @sep    = sep  .freeze
+    @quote  = quote.freeze
+
+    @drop   = drop
+    @eol    = eol.freeze
+    @mode   = mode
+    @out    = out
+    @relax  = relax
 
-    @drop  = drop
-    @eol   = eol.freeze
-    @mode  = mode
-    @out   = out
-    @relax = relax
+    @es     = ""   .freeze
+    @cr     = "\r" .freeze
+    @lf     = "\n" .freeze
+    @eq     = "="  .freeze
+    @esc    = (@quote * 2).freeze
 
-    @es    = ""   .freeze
-    @cr    = "\r" .freeze
-    @lf    = "\n" .freeze
-    @esc   = (@quote * 2).freeze
+    @tokens = [@sep,@quote,@cr,@lf,@es,nil]
+    @tokens << @eq if excel # See http://bit.ly/3Y7jIvc
   end
 
   def reset(str=nil)
     self.string = str if str
     super()
-    @char  = string[pos]
-    @flag  = nil
+    @char = peek(1)
+    @flag = nil
 
-    @rows  = nil
-    @cols  = @cells = 0
+    @rows = nil
+    @cols = @cells = 0
   end
 
   # ==[ Lexer ]==
 
   def next_char
     getch
-    @char = string[pos]
+    @char = peek(1)
   end
 
   def next_token
     case @flag
-    when @es then @flag = nil; [@cr,@lf,nil].include?(@char) and return @es
+    when @es then @flag = nil; [@cr,@lf,@es,nil].include?(@char) and return @es
     when @cr then @flag = nil; next_char == @lf and next_char
     when @lf then @flag = nil; next_char
     end if @flag
 
-    if [@sep,@quote,@cr,@lf,nil].include?(@char)
+    if @tokens.include?(@char)
       case @char
-      when @quote # consume_quoted_cell
+      when @quote, @eq # consume quoted cell
+        @char == @eq and next_char # excel mode: allows ,="012",
         match = ""
         while true
           getch # consume the quote (optimized by not calling next_char)
           match << (scan_until(/(?=#{@quote})/o) or bomb "unclosed quote")
           case next_char
-          when @sep        then @flag = @es; next_char; break
-          when @quote      then match << @quote
-          when @cr,@lf,nil then break
+          when @sep            then @flag = @es; next_char; break
+          when @quote          then match << @quote
+          when @cr,@lf,@es,nil then break
           else
             if @relax
               match << @quote + @char
@@ -107,14 +112,14 @@ class Censive < StringScanner
           end
         end
         match
-      when @sep then @flag = @es; next_char; @es
-      when @cr  then @flag = @cr; nil
-      when @lf  then @flag = @lf; nil
-      when nil  then nil
+      when @sep    then @flag = @es; next_char; @es
+      when @cr     then @flag = @cr; nil
+      when @lf     then @flag = @lf; nil
+      when @es,nil then              nil
       end
-    else # consume_unquoted_cell
+    else # consume unquoted cell
       match = scan_until(/(?=#{@sep}|#{@cr}|#{@lf}|\z)/o) or bomb "unexpected character"
-      @char = string[pos]
+      @char = peek(1)
       @char == @sep and @flag = @es and next_char
       match
     end
@@ -158,6 +163,9 @@ class Censive < StringScanner
   def <<(row)
     @out or return super
 
+    # drop trailing seps, if specified
+    row.pop while row.last.empty? if @drop
+
     # most compact export format
     s,q = @sep, @quote
     out = case @mode
@@ -178,9 +186,6 @@ class Censive < StringScanner
       row.map {|col| "#{q}#{col.gsub(q, @esc)}#{q}" }
     end.join(s)
 
-    # drop trailing seps, if specified
-    out.gsub!(/#{s}+\z/,'') if @drop
-
     # write output, using desired line endings
     @out << out + @eol
   end
@@ -203,81 +208,25 @@ __END__
 
 # ==[ Playground... ]==
 
-STDOUT.sync = true
-
-require 'fileutils'
-
-ARGV << "101.csv"
-
-rand = `LC_ALL=C tr -dc a-zA-Z0-9 < /dev/random | head -c12`
-
-rows = []
-cols = []
-coun = 0
-full = 0
-
-ARGV.each do |path|
-  File.file?(path) or next
-
-  print "Processing #{path.inspect}"
-
-  rows.clear
-  cols.clear
-  seen = 0
-  coun += 1
-
-  dest = "#{path}-#{rand}"
-
-  begin
-    Censive.writer(dest) do |file|
-      Censive.new(File.read(path), relax: true).each do |cols|
-        cols.each {|cell| cell && cell.size >= 3 && cell.sub!(/\A="/, '') && cell.sub!(/"\z/, '') }
-        file << cols
-        seen += 1
-        print "." if (seen % 1e5) == 0
-      end
-    end
-    FileUtils.mv(dest, path)
-    full += (seen - 1)
-    puts " (#{seen - 1} rows of data)"
-  rescue
-    puts " - unable to process (#{$!})"
-    FileUtils.rm_f(dest)
-  end
-end
-
-puts "Processed #{coun} files with a total of #{full} rows of data" if coun > 1
-
-__END__
-,"CHUI, LOK HANG "BENNY",   =>   ,"""CHUI, LOK HANG ""BENNY""",
+# STDOUT.sync = true
+#
+# data = File.read('1.csv')
+#
+# Censive.writer('out.csv') do |out|
+#   Censive.new(data, relax: true, excel: true).each do |row|
+#     out << row
+#   end
+# end
+#
+# __END__
 
-,"..............."B
+ARGV << "z.csv" if ARGV.empty?
 
-__END__
+path = ARGV.first
+data = File.read(path)
 
+csv = Censive.new(data)
 
-data = File.read('1.csv')
+data.size > 1e6 ? csv.parse : csv.parse.each {|cols| p cols }
 
-Censive.writer('out.csv', sep: ',', quote: "'") do |out|
-  Censive.new(data).each do |row|
-    out << row
-  end
-end
-
-# ARGV << "z.csv" if ARGV.empty?
-#
-# case 1
-# when 1
-#   path = ARGV.first
-#   data = File.read(path)
-# when 2
-#   data = DATA.gets("\n\n").rstrip
-# end
-#
-# STDOUT.sync = true
-#
-# csv = Censive.new(data)
-#
-# data.size > 1e6 ? csv.parse : csv.parse.each {|cols| p cols }
-#
-# csv.stats
+csv.stats

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: censive
 version: !ruby/object:Gem::Version
-  version: '0.7'
+  version: '0.8'
 platform: ruby
 authors:
 - Steve Shreeve