censive 0.8 → 0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6419efcdc9274ea8bcf7b8527001e33f8bdfea348dfd911686cab36984d507da
-  data.tar.gz: 3b59aead54517fd64d7ece3eaa6f459e301e1e48f1ae34772a7128c61fb739f2
+  metadata.gz: f4b06c1c42b5f813f8901c4e7240cdd43df1ccc22cf87327dc3ed7d850720eb4
+  data.tar.gz: 97ab27b79eead81517fa28a4c51923fa02ec2fa95922f6f61dc509c7a4890b2e
 SHA512:
-  metadata.gz: 7910c09e76a81ed27870ea52fb6c8aea0316ed213c53a026d98adc64f93349477e6acab0a93b88c6f184ce1d317634ecdca9290d50bff9b117b98bedd3ac7b86
-  data.tar.gz: 358ab985947d486b5f486b1f7e9c1f591e3b8e906b9eab59a4ed151e5f5d9652c211f2d2a4ee36f0543227e2ae5e33ba57f1e4c178f6f7e72e05c14d7b46895f
+  metadata.gz: a2f297ac516f5e01510a9ceb90cdb2cc1e782ff97a4f67515d73f6d56d8512cd4d9cbb5d04425bcdb8a7a5cdb63aeb2835e7bed2a76dcc149dae0bd63c4cc17b
+  data.tar.gz: 85762c69bc669db5a48f0e3b58e4319afdc9f1765cc18cd2b6c9501aaaccf3e41dddb58d3654bc4c76242632eaf8988f348bb652c6c40105e0373b8afdf463d3

data/README.md

@@ -2,7 +2,58 @@
 
 A quick and lightweight CSV handling library for Ruby
 
-## Writing CSV
+## Example
+
+```ruby
+#!/usr/bin/env ruby
+
+STDOUT.sync = true
+
+require 'censive'
+require 'fileutils'
+
+abort "usage: #{File.basename($0)} <files>" if ARGV.empty?
+
+rand = `LC_ALL=C tr -dc a-zA-Z0-9 < /dev/random | head -c12`
+
+rows = []
+cols = []
+coun = 0
+full = 0
+
+ARGV.each do |path|
+  File.file?(path) or next
+
+  print "Processing #{path.inspect}"
+
+  rows.clear
+  cols.clear
+  seen = 0
+  coun += 1
+
+  dest = "#{path}-#{rand}"
+
+  begin
+    Censive.writer(dest) do |file|
+      Censive.reader(path, excel: true, relax: true).each do |cols|
+        file << cols
+        seen += 1
+        print "." if (seen % 1e5) == 0 # give a status update every so often
+      end
+    end
+    FileUtils.mv(dest, path)
+    full += (seen - 1)
+    puts " (#{seen - 1} rows of data)"
+  rescue
+    puts " - unable to process (#{$!})"
+    FileUtils.rm_f(dest)
+  end
+end
+
+puts "Processed #{coun} files with a total of #{full} rows of data" if coun > 1
+```
+
+## Convert a CSV file to a TSV file
 
 ```ruby
 require 'censive'
@@ -12,7 +63,7 @@ data = File.read('data.csv')
 
 # write out a tab-separated tsv file
 Censive.writer('out.tsv', sep: "\t", mode: :full) do |out|
-  Censive.new(data).each do |row|
+  Censive.new(data, excel: true, relax: true).each do |row|
     out << row
   end
 end

data/censive.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = "censive"
-  s.version     = "0.8"
+  s.version     = "0.10"
   s.author      = "Steve Shreeve"
   s.email       = "steve.shreeve@gmail.com"
   s.summary     = "A quick and lightweight CSV handling library for Ruby"

data/lib/censive.rb

@@ -6,7 +6,8 @@
 # Author: Steve Shreeve (steve.shreeve@gmail.com)
 #   Date: Jan 30, 2023
 #
-# Thanks: Crystal's CSV library, see https://crystal-lang.org/api/1.7.2/CSV.html
+# Thanks to https://crystal-lang.org/api/1.7.2/CSV.html (Crystal's CSV library)
+# and, also https://github.com/ruby/strscan/blob/master/ext/strscan/strscan.c
 # ==============================================================================
 # The goals are:
 #
@@ -17,7 +18,8 @@
 #
 # 1. Option to support IO streaming
 # 2. Option to strip whitespace
-# 3. Confirm file encodings such as UTF-8, UTF-16, etc.
+# 3. Option to support headers in readers and writers
+# 4. Confirm file encodings such as UTF-8, UTF-16, etc.
 #
 # NOTE: Only getch and scan_until advance strscan's position
 # ==============================================================================
@@ -26,36 +28,36 @@ require 'strscan'
 
 class Censive < StringScanner
 
-  def self.writer(path, **opts)
-    File.open(path, 'w') do |file|
-      yield new(out: file, **opts)
+  def self.writer(obj=$stdout, **opts, &code)
+    case obj
+    when String then File.open(path, 'w') {|file| yield new(out: obj, **opts, &code) }
+    when IO     then new(out: obj, **opts, &code)
+    else abort "#{File.basename($0)}: invalid #{obj.class} object in writer"
     end
   end
 
   def initialize(str=nil,
-    sep:   ','     , # column separator character
-    quote: '"'     , # quote character
-
-    drop:  false   , # enable to drop trailing separators
-    eol:   "\n"    , # desired line endings for exports
-    excel: false   , # allow ,="0123" style columns
+    drop:  false   , # drop trailing empty fields?
+    eol:   "\n"    , # line endings for exports
+    excel: false   , # literals(="01") formulas(=A1 + B2); http://bit.ly/3Y7jIvc
     mode:  :compact, # export mode: compact or full
-    out:   nil     , # output IO/file
-    relax: false   , # relax parsing of quotes
-
+    out:   nil     , # output stream, needs to respond to <<
+    quote: '"'     , # quote character
+    relax: false   , # relax quote parsing so ,"Fo"o, => ,"Fo""o",
+    sep:   ','     , # column separator character
     **opts           # grab bag
   )
     super(str || '')
     reset
 
-    @sep    = sep  .freeze
-    @quote  = quote.freeze
-
     @drop   = drop
-    @eol    = eol.freeze
+    @eol    = eol  .freeze #!# TODO: are the '.freeze' statements helpful?
+    @excel  = excel
     @mode   = mode
     @out    = out
+    @quote  = quote.freeze
     @relax  = relax
+    @sep    = sep  .freeze
 
     @es     = ""   .freeze
     @cr     = "\r" .freeze
@@ -64,7 +66,6 @@ class Censive < StringScanner
     @esc    = (@quote * 2).freeze
 
     @tokens = [@sep,@quote,@cr,@lf,@es,nil]
-    @tokens << @eq if excel # See http://bit.ly/3Y7jIvc
   end
 
   def reset(str=nil)
@@ -85,30 +86,33 @@ class Censive < StringScanner
   end
 
   def next_token
+
+    # process and clear @flag
     case @flag
     when @es then @flag = nil; [@cr,@lf,@es,nil].include?(@char) and return @es
     when @cr then @flag = nil; next_char == @lf and next_char
     when @lf then @flag = nil; next_char
+    else          @flag = nil
     end if @flag
 
+    # Excel literals ="0123" and formulas =A1 + B2 (see http://bit.ly/3Y7jIvc)
+    if @excel && @char == @eq
+      @flag = @eq
+      next_char
+    end
+
     if @tokens.include?(@char)
       case @char
-      when @quote, @eq # consume quoted cell
-        @char == @eq and next_char # excel mode: allows ,="012",
+      when @quote # consume quoted cell
         match = ""
         while true
-          getch # consume the quote (optimized by not calling next_char)
+          getch # move past the quote that got us here
           match << (scan_until(/(?=#{@quote})/o) or bomb "unclosed quote")
           case next_char
           when @sep            then @flag = @es; next_char; break
           when @quote          then match << @quote
           when @cr,@lf,@es,nil then break
-          else
-            if @relax
-              match << @quote + @char
-            else
-              bomb "invalid character after quote"
-            end
+          else @relax ? match << (@quote + @char) : bomb("invalid character after quote")
           end
         end
         match
@@ -119,6 +123,7 @@ class Censive < StringScanner
       end
     else # consume unquoted cell
       match = scan_until(/(?=#{@sep}|#{@cr}|#{@lf}|\z)/o) or bomb "unexpected character"
+      match = @eq + match and @flag = nil if @flag == @eq
       @char = peek(1)
       @char == @sep and @flag = @es and next_char
       match
@@ -132,12 +137,12 @@ class Censive < StringScanner
   # ==[ Parser ]==
 
   def parse
-    @rows ||= []
+    @rows = []
     while row = next_row
       @rows << row
-      size = row.size
-      @cols = size if size > @cols
-      @cells += size
+      count = row.size
+      @cols = count if count > @cols
+      @cells += count
     end
     @rows
   end
@@ -151,28 +156,34 @@ class Censive < StringScanner
 
   # ==[ Helpers ]==
 
-  # grok returns: 2 for seps and quotes, 1 for seps only, and 0 for neither
+  # grok returns: 2 (must be quoted and escaped), 1 (must be quoted), 0 (neither)
   def grok(str)
-    if pos = str.index(/(#{@quote})|#{@sep}/o)
+    if pos = str.index(/(#{@quote})|#{@sep}|#{@cr}|#{@lf}/o)
       $1 ? 2 : str.index(/#{@quote}/o, pos) ? 2 : 1
     else
       0
     end
   end
 
+  # output a row
   def <<(row)
     @out or return super
 
-    # drop trailing seps, if specified
+    # drop trailing empty columns
     row.pop while row.last.empty? if @drop
 
-    # most compact export format
+    #!# FIXME: Excel output needs to protect 0-leading numbers
+
     s,q = @sep, @quote
     out = case @mode
     when :compact
       case grok(row.join)
-      when 0 then row
-      when 1 then row.map {|col| col.include?(s) ? "#{q}#{col}#{q}" : col }
+      when 0
+        row
+      when 1
+        row.map do |col|
+          col.match?(/#{@sep}|#{@cr}|#{@lf}/o) ? "#{q}#{col}#{q}" : col
+        end
       else
         row.map do |col|
           case grok(col)
@@ -186,7 +197,7 @@ class Censive < StringScanner
       row.map {|col| "#{q}#{col.gsub(q, @esc)}#{q}" }
     end.join(s)
 
-    # write output, using desired line endings
+    # add line ending
     @out << out + @eol
   end
 
@@ -195,6 +206,11 @@ class Censive < StringScanner
     @rows.each {|row| yield row }
   end
 
+  def export(...)
+    out = self.class.writer(...)
+    each {|row| out << row }
+  end
+
   def stats
     wide = string.size.to_s.size
     puts "%#{wide}d rows"    % @rows.size
@@ -204,9 +220,35 @@ class Censive < StringScanner
   end
 end
 
+# ==[ Command line ]==
+
+if __FILE__ == $0
+  raw = DATA.gets("\n\n").chomp
+  csv = Censive.new(raw, excel: true)
+  csv.export # (sep: "\t", excel: true)
+end
+
 __END__
+Name,Age,Shoe
+Alice,27,5
+Bob,33,10 1/2
+Charlie or "Chuck",=B2 + B3,9
+"Doug E Fresh",="007",10
+Subtotal,=sum(B2:B5),="01234"
+
+
+
+
+path = '../test/a-uses-tabs-and-single-quotes-and-no-trailing-newline.tsv'
+data = File.read(path)
+
+out = Censive.writer
 
-# ==[ Playground... ]==
+Censive.new(data, sep: "\t", quote: "'").each do |row|
+  p row
+end
+
+Censive.reader(path, sep: "\t", quote: "'").each {|r| p r}
 
 # STDOUT.sync = true
 #
@@ -219,14 +261,14 @@ __END__
 # end
 #
 # __END__
-
-ARGV << "z.csv" if ARGV.empty?
-
-path = ARGV.first
-data = File.read(path)
-
-csv = Censive.new(data)
-
-data.size > 1e6 ? csv.parse : csv.parse.each {|cols| p cols }
-
-csv.stats
+#
+# ARGV << "z.csv" if ARGV.empty?
+#
+# path = ARGV.first
+# data = File.read(path)
+#
+# csv = Censive.new(data)
+#
+# data.size > 1e6 ? csv.parse : csv.parse.each {|cols| p cols }
+#
+# csv.stats

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: censive
 version: !ruby/object:Gem::Version
-  version: '0.8'
+  version: '0.10'
 platform: ruby
 authors:
 - Steve Shreeve
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-31 00:00:00.000000000 Z
+date: 2023-02-02 00:00:00.000000000 Z
 dependencies: []
 description: A quick and lightweight CSV handling library for Ruby
 email: steve.shreeve@gmail.com
@@ -20,7 +20,7 @@ files:
 - README.md
 - censive.gemspec
 - lib/censive.rb
-- test/a-uses-tabs-and-single-quotes-and-no-trailing-newline.csv
+- test/a-uses-tabs-and-single-quotes-and-no-trailing-newline.tsv
 homepage: https://github.com/shreeve/censive
 licenses:
 - MIT