censive 0.9 → 0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cbca33c415269ae1fafea5297f2b409879a46c37c884a0a7017bca322bcff2a6
-  data.tar.gz: ac021ddf3d7503aebc5791b0912c6409a0888627060b532e65f6eb72b94965a3
+  metadata.gz: 91f3884b92c79f8cca37066a420c872b612c37ed736f61fa7d11c0f0fd861d8e
+  data.tar.gz: 71789d006309c1d87e681a7078e718342cb36e1f8a18690c59e51595c49d2e59
 SHA512:
-  metadata.gz: 8095c0c7704e3a6ee66930b36f0131b38d52a68cdd066d9677e8ceb58c4ecd7ce7eed496c78b1841cabe845b8c82624ca808b33a7cf7ec4c8fd504b287b3ffb5
-  data.tar.gz: 2e363b63b37977784a38c06e091f3201a1cd7a13138e8101e0e41ca49c47b3c4b433e7e6f2843a6816ddcbf9c1c8293da0d858f6be38bd0d3d82ed5dbd904bfe
+  metadata.gz: d5fe33889abe08bc6aa57bb4dec2d65ef3e8935da5aa209a0bcb6060696de07cf44189ad9be6f43ffcd2897672668bd3aff7e8b713f5070199627e40edd3704f
+  data.tar.gz: 2c79d30b4682da07800c19eb93067c5c72d471f9abc7b824f7525b3d36b8fb1108d0e733706852f750baac57360af2d5b77527a1daab871e2313d558928ce240

data/README.md

@@ -2,7 +2,58 @@
 
 A quick and lightweight CSV handling library for Ruby
 
-## Writing CSV
+## Example
+
+```ruby
+#!/usr/bin/env ruby
+
+STDOUT.sync = true
+
+require 'censive'
+require 'fileutils'
+
+abort "usage: #{File.basename($0)} <files>" if ARGV.empty?
+
+rand = `LC_ALL=C tr -dc a-zA-Z0-9 < /dev/random | head -c12`
+
+rows = []
+cols = []
+coun = 0
+full = 0
+
+ARGV.each do |path|
+  File.file?(path) or next
+
+  print "Processing #{path.inspect}"
+
+  rows.clear
+  cols.clear
+  seen = 0
+  coun += 1
+
+  dest = "#{path}-#{rand}"
+
+  begin
+    Censive.writer(dest) do |file|
+      Censive.reader(path, excel: true, relax: true).each do |cols|
+        file << cols
+        seen += 1
+        print "." if (seen % 1e5) == 0 # give a status update every so often
+      end
+    end
+    FileUtils.mv(dest, path)
+    full += (seen - 1)
+    puts " (#{seen - 1} rows of data)"
+  rescue
+    puts " - unable to process (#{$!})"
+    FileUtils.rm_f(dest)
+  end
+end
+
+puts "Processed #{coun} files with a total of #{full} rows of data" if coun > 1
+```
+
+## Convert a CSV file to a TSV file
 
 ```ruby
 require 'censive'
@@ -17,3 +68,12 @@ Censive.writer('out.tsv', sep: "\t", mode: :full) do |out|
   end
 end
 ```
+
+Or, you can be more succinct with:
+
+```ruby
+require 'censive'
+
+csv = Censive.new(File.read('data.csv'))
+csv.export(sep: "\t")
+```

data/censive.gemspec

@@ -2,13 +2,14 @@
 
 Gem::Specification.new do |s|
   s.name        = "censive"
-  s.version     = "0.9"
+  s.version     = "0.11"
   s.author      = "Steve Shreeve"
   s.email       = "steve.shreeve@gmail.com"
-  s.summary     = "A quick and lightweight CSV handling library for Ruby"
+  s.summary     =
   s.description = "A quick and lightweight CSV handling library for Ruby"
   s.homepage    = "https://github.com/shreeve/censive"
   s.license     = "MIT"
   s.files       = `git ls-files`.split("\n") - %w[.gitignore]
   s.executables = `(cd bin 2>&1) > /dev/null && git ls-files .`.split("\n")
+  s.add_dependency "strscan", ">= 3.0.6"
 end

data/lib/censive.rb

@@ -1,137 +1,116 @@
 #!/usr/bin/env ruby
 
-# ==============================================================================
+# ============================================================================
 # censive - A quick and lightweight CSV handling library for Ruby
 #
 # Author: Steve Shreeve (steve.shreeve@gmail.com)
-#   Date: Jan 30, 2023
-#
-# Thanks to https://crystal-lang.org/api/1.7.2/CSV.html (Crystal's CSV library)
-# and, also https://github.com/ruby/strscan/blob/master/ext/strscan/strscan.c
-# ==============================================================================
-# The goals are:
+#   Date: Feb 3, 2023
 #
+# https://crystal-lang.org/api/1.7.2/CSV.html (Crystal's CSV library)
+# https://github.com/ruby/strscan/blob/master/ext/strscan/strscan.c
+# https://github.com/ruby/strscan/issues/53 for details
+# https://github.com/ruby/strscan/pull/54 for code
+# ============================================================================
+# GOALS:
 # 1. Faster than Ruby's default CSV library
-# 2. Lightweight code base with streamlined method calls
-#
-# To consider:
-#
-# 1. Option to support IO streaming
-# 2. Option to strip whitespace
-# 3. Option to support headers in readers and writers
-# 4. Confirm file encodings such as UTF-8, UTF-16, etc.
+# 2. Lightweight code base with streamlined logic
+# 3. Support for most non-compliant CSV variations (eg - @relax, @excel)
 #
-# NOTE: Only getch and scan_until advance strscan's position
-# ==============================================================================
+# TODO:
+# 1. Support IO streaming
+# 2. Add option to strip whitespace
+# 3. Support CSV headers in first row
+# ============================================================================
 
+require 'bundler/setup'
 require 'strscan'
 
 class Censive < StringScanner
 
-  def self.writer(path, **opts)
-    File.open(path, 'w') do |file|
-      yield new(out: file, **opts)
+  def self.writer(obj=nil, **opts, &code)
+    case obj
+    when String then File.open(path, 'w') {|file| yield new(out: obj, **opts, &code) }
+    when IO,nil then new(out: obj, **opts, &code)
+    else abort "#{File.basename($0)}: invalid #{obj.class} object in writer"
     end
   end
 
   def initialize(str=nil,
-    sep:   ','     , # column separator character
-    quote: '"'     , # quote character
-
-    drop:  false   , # enable to drop trailing separators
-    eol:   "\n"    , # desired line endings for exports
-    excel: false   , # literals (="01"), formulas (=A1 + B2), see http://bit.ly/3Y7jIvc
+    drop:  false   , # drop trailing empty fields?
+    eol:   "\n"    , # line endings for exports
+    excel: false   , # literals ="01" formulas =A1 + B2 http://bit.ly/3Y7jIvc
     mode:  :compact, # export mode: compact or full
-    out:   nil     , # output IO/file
-    relax: false   , # relax parsing of quotes
-
+    out:   nil     , # output stream, needs to respond to <<
+    quote: '"'     , # quote character
+    relax: false   , # relax quote parsing so ,"Fo"o, => ,"Fo""o",
+    sep:   ','     , # column separator character
     **opts           # grab bag
   )
     super(str || '')
     reset
 
-    @sep    = sep  .freeze
-    @quote  = quote.freeze
-
     @drop   = drop
-    @eol    = eol.freeze
+    @eol    = eol
     @excel  = excel
     @mode   = mode
-    @out    = out
+    @out    = out || $stdout
+    @quote  = quote
     @relax  = relax
+    @sep    = sep
 
-    @es     = ""   .freeze
-    @cr     = "\r" .freeze
-    @lf     = "\n" .freeze
-    @eq     = "="  .freeze
-    @esc    = (@quote * 2).freeze
-
-    @tokens = [@sep,@quote,@cr,@lf,@es,nil]
+    @cr     = "\r"
+    @lf     = "\n"
+    @es     = ""
+    @eq     = "="
+    @esc    = (@quote * 2)
   end
 
   def reset(str=nil)
     self.string = str if str
     super()
-    @char = peek(1)
-    @flag = nil
-
+    @char = curr_char
     @rows = nil
     @cols = @cells = 0
   end
 
   # ==[ Lexer ]==
 
-  def next_char
-    getch
-    @char = peek(1)
-  end
-
-  def next_token
+  # pure ruby versions for debugging
+  # def curr_char;             @char = string[pos]; end
+  # def next_char; scan(/./m); @char = string[pos]; end
 
-    # process and clear @flag
-    case @flag
-    when @es then @flag = nil; [@cr,@lf,@es,nil].include?(@char) and return @es
-    when @cr then @flag = nil; next_char == @lf and next_char
-    when @lf then @flag = nil; next_char
-    else          @flag = nil
-    end if @flag
+  def curr_char; @char = currchar; end
+  def next_char; @char = nextchar; end
 
-    # See http://bit.ly/3Y7jIvc
+  def next_token
     if @excel && @char == @eq
-      @flag = @eq
+      excel = true
       next_char
     end
 
-    if @tokens.include?(@char)
+    if @char == @quote # consume quoted cell
+      match = ""
+      while true
+        next_char
+        match << (scan_until(/(?=#{@quote})/o) or bomb "unclosed quote")
+        match << @quote and next if next_char == @quote
+        break if [@sep,@cr,@lf,@es,nil].include?(@char)
+        @relax or bomb "invalid character after quote"
+        match << @quote + scan_until(/(?=#{@quote})/o) + @quote
+      end
+      next_char if @char == @sep
+      match
+    elsif [@sep,@cr,@lf,@es,nil].include?(@char)
       case @char
-      when @quote # consume quoted cell
-        match = ""
-        while true
-          getch # consume the quote that got us here
-          match << (scan_until(/(?=#{@quote})/o) or bomb "unclosed quote")
-          case next_char
-          when @sep            then @flag = @es; next_char; break
-          when @quote          then match << @quote
-          when @cr,@lf,@es,nil then break
-          else
-            if @relax
-              match << @quote + @char
-            else
-              bomb "invalid character after quote"
-            end
-          end
-        end
-        match
-      when @sep    then @flag = @es; next_char; @es
-      when @cr     then @flag = @cr; nil
-      when @lf     then @flag = @lf; nil
-      when @es,nil then              nil
+      when @sep then next_char; @es
+      when @cr  then next_char == @lf and next_char; nil
+      when @lf  then next_char; nil
+      else nil
       end
     else # consume unquoted cell
       match = scan_until(/(?=#{@sep}|#{@cr}|#{@lf}|\z)/o) or bomb "unexpected character"
-      match = @eq + match if @flag == @eq # preserve @eq for excel formulas
-      @char = peek(1)
-      @char == @sep and @flag = @es and next_char
+      match.prepend(@eq) if excel
+      next_char if curr_char == @sep
       match
     end
   end
@@ -143,12 +122,12 @@ class Censive < StringScanner
   # ==[ Parser ]==
 
   def parse
-    @rows ||= []
+    @rows = []
     while row = next_row
       @rows << row
-      size = row.size
-      @cols = size if size > @cols
-      @cells += size
+      count = row.size
+      @cols = count if count > @cols
+      @cells += count
     end
     @rows
   end
@@ -162,26 +141,25 @@ class Censive < StringScanner
 
   # ==[ Helpers ]==
 
-  # grok returns: 2 (must be quoted and escaped), 1 (must be quoted), 0 (neither)
+  # returns 2 (must be quoted and escaped), 1 (must be quoted), 0 (neither)
   def grok(str)
-    if pos = str.index(/(#{@quote})|#{@sep}|#{@cr}|#{@lf}/o)
-      $1 ? 2 : str.index(/#{@quote}/o, pos) ? 2 : 1
+    if idx = str.index(/(#{@quote})|#{@sep}|#{@cr}|#{@lf}/o) #!# FIXME: regex injection is possible
+      $1 ? 2 : str.index(/#{@quote}/o, idx) ? 2 : 1
     else
       0
     end
   end
 
+  # output a row
   def <<(row)
-    @out or return super
 
-    # drop trailing seps, if specified
+    # drop trailing empty columns
     row.pop while row.last.empty? if @drop
 
-    # most compact export format
     s,q = @sep, @quote
     out = case @mode
     when :compact
-      case grok(row.join)
+      case @excel ? 2 : grok(row.join)
       when 0
         row
       when 1
@@ -190,6 +168,7 @@ class Censive < StringScanner
         end
       else
         row.map do |col|
+          @excel && col =~ /\A0\d*\z/ ? "=#{q}#{col}#{q}" :
           case grok(col)
           when 0 then col
           when 1 then "#{q}#{col}#{q}"
@@ -201,7 +180,6 @@ class Censive < StringScanner
       row.map {|col| "#{q}#{col.gsub(q, @esc)}#{q}" }
     end.join(s)
 
-    # write output, using desired line endings
     @out << out + @eol
   end
 
@@ -210,6 +188,11 @@ class Censive < StringScanner
     @rows.each {|row| yield row }
   end
 
+  def export(**opts)
+    out = opts.empty? ? self : self.class.writer(**opts)
+    each {|row| out << row }
+  end
+
   def stats
     wide = string.size.to_s.size
     puts "%#{wide}d rows"    % @rows.size
@@ -219,29 +202,25 @@ class Censive < StringScanner
   end
 end
 
-__END__
-
-# ==[ Playground... ]==
-
-# STDOUT.sync = true
-#
-# data = File.read('1.csv')
-#
-# Censive.writer('out.csv') do |out|
-#   Censive.new(data, relax: true, excel: true).each do |row|
-#     out << row
-#   end
-# end
-#
-# __END__
-
-ARGV << "z.csv" if ARGV.empty?
-
-path = ARGV.first
-data = File.read(path)
-
-csv = Censive.new(data)
-
-data.size > 1e6 ? csv.parse : csv.parse.each {|cols| p cols }
+if __FILE__ == $0
+  raw = DATA.gets("\n\n").chomp
+# raw = File.read(ARGV.first || "lc-2023.csv")
+  csv = Censive.new(raw, excel: true, relax: true)
+  csv.export # (sep: ",", excel: true)
+end
 
-csv.stats
+__END__
+Name,Age,Shoe
+Alice,27,5
+Bob,33,10 1/2
+Charlie or "Chuck",=B2 + B3,9
+"Doug E Fresh",="007",10
+Subtotal,=sum(B2:B5),="01234"
+
+# first line works in "relax" mode, bottom line is compliant
+123,"CHO, JOELLE "JOJO"",456
+123,"CHO, JOELLE ""JOJO""",456
+
+# Excel mode checking
+=,=x,x=,="x",="","","=",123,0123,="123",="0123"
+,=x,x=,x,,,,,,=,,123,="0123",123,,="0123" # <= a little off

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: censive
 version: !ruby/object:Gem::Version
-  version: '0.9'
+  version: '0.11'
 platform: ruby
 authors:
 - Steve Shreeve
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-31 00:00:00.000000000 Z
-dependencies: []
+date: 2023-02-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: strscan
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.6
 description: A quick and lightweight CSV handling library for Ruby
 email: steve.shreeve@gmail.com
 executables: []
@@ -20,7 +34,7 @@ files:
 - README.md
 - censive.gemspec
 - lib/censive.rb
-- test/a-uses-tabs-and-single-quotes-and-no-trailing-newline.csv
+- test/a-uses-tabs-and-single-quotes-and-no-trailing-newline.tsv
 homepage: https://github.com/shreeve/censive
 licenses:
 - MIT