censive 0.10 → 0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4b06c1c42b5f813f8901c4e7240cdd43df1ccc22cf87327dc3ed7d850720eb4
-  data.tar.gz: 97ab27b79eead81517fa28a4c51923fa02ec2fa95922f6f61dc509c7a4890b2e
+  metadata.gz: 3343af27996e08f7c83ae7327e60fc8c225095c457be3141a471f105d596e165
+  data.tar.gz: 4ccdfe64a1314628ca9070c5a50678c29933ba87c11b160ec25494a21772090b
 SHA512:
-  metadata.gz: a2f297ac516f5e01510a9ceb90cdb2cc1e782ff97a4f67515d73f6d56d8512cd4d9cbb5d04425bcdb8a7a5cdb63aeb2835e7bed2a76dcc149dae0bd63c4cc17b
-  data.tar.gz: 85762c69bc669db5a48f0e3b58e4319afdc9f1765cc18cd2b6c9501aaaccf3e41dddb58d3654bc4c76242632eaf8988f348bb652c6c40105e0373b8afdf463d3
+  metadata.gz: 6b5e4d4e3f3f0ec4898fe91ef0f4033c068c68f79fe1acf0f41e45a9d96380751fed6a852920e2a9aa1ae2e477750d972a2838ba5fc7212ea940af534560d183
+  data.tar.gz: f58137689118ff3546d9d973ffdd5f9a7f686aa61e4d813e2adf7557b6d0e6303b8f9d9d21bc084ba82f73c50119c522049a066757ab7cead0d5d69dc6037d63

data/README.md

@@ -68,3 +68,12 @@ Censive.writer('out.tsv', sep: "\t", mode: :full) do |out|
   end
 end
 ```
+
+Or, you can be more succinct with:
+
+```ruby
+require 'censive'
+
+csv = Censive.new(File.read('data.csv'))
+csv.export(sep: "\t")
+```

data/censive.gemspec

@@ -2,13 +2,14 @@
 
 Gem::Specification.new do |s|
   s.name        = "censive"
-  s.version     = "0.10"
+  s.version     = "0.12"
   s.author      = "Steve Shreeve"
   s.email       = "steve.shreeve@gmail.com"
-  s.summary     = "A quick and lightweight CSV handling library for Ruby"
+  s.summary     =
   s.description = "A quick and lightweight CSV handling library for Ruby"
   s.homepage    = "https://github.com/shreeve/censive"
   s.license     = "MIT"
   s.files       = `git ls-files`.split("\n") - %w[.gitignore]
   s.executables = `(cd bin 2>&1) > /dev/null && git ls-files .`.split("\n")
+  s.add_dependency "strscan", ">= 3.0.6"
 end

data/lib/censive.rb

@@ -1,37 +1,36 @@
 #!/usr/bin/env ruby
 
-# ==============================================================================
+# ============================================================================
 # censive - A quick and lightweight CSV handling library for Ruby
 #
 # Author: Steve Shreeve (steve.shreeve@gmail.com)
-#   Date: Jan 30, 2023
-#
-# Thanks to https://crystal-lang.org/api/1.7.2/CSV.html (Crystal's CSV library)
-# and, also https://github.com/ruby/strscan/blob/master/ext/strscan/strscan.c
-# ==============================================================================
-# The goals are:
+#   Date: Feb 3, 2023
 #
+# https://crystal-lang.org/api/1.7.2/CSV.html (Crystal's CSV library)
+# https://github.com/ruby/strscan/blob/master/ext/strscan/strscan.c
+# https://github.com/ruby/strscan/issues/53 for details
+# https://github.com/ruby/strscan/pull/54 for code
+# ============================================================================
+# GOALS:
 # 1. Faster than Ruby's default CSV library
-# 2. Lightweight code base with streamlined method calls
-#
-# To consider:
-#
-# 1. Option to support IO streaming
-# 2. Option to strip whitespace
-# 3. Option to support headers in readers and writers
-# 4. Confirm file encodings such as UTF-8, UTF-16, etc.
+# 2. Lightweight code base with streamlined logic
+# 3. Support for most non-compliant CSV variations (eg - @relax, @excel)
 #
-# NOTE: Only getch and scan_until advance strscan's position
-# ==============================================================================
+# TODO:
+# 1. Support IO streaming
+# 2. Add option to strip whitespace
+# 3. Support CSV headers in first row
+# ============================================================================
 
+require 'bundler/setup'
 require 'strscan'
 
 class Censive < StringScanner
 
-  def self.writer(obj=$stdout, **opts, &code)
+  def self.writer(obj=nil, **opts, &code)
     case obj
     when String then File.open(path, 'w') {|file| yield new(out: obj, **opts, &code) }
-    when IO     then new(out: obj, **opts, &code)
+    when IO,nil then new(out: obj, **opts, &code)
     else abort "#{File.basename($0)}: invalid #{obj.class} object in writer"
     end
   end
@@ -39,7 +38,7 @@ class Censive < StringScanner
   def initialize(str=nil,
     drop:  false   , # drop trailing empty fields?
     eol:   "\n"    , # line endings for exports
-    excel: false   , # literals(="01") formulas(=A1 + B2); http://bit.ly/3Y7jIvc
+    excel: false   , # literals ="01" formulas =A1 + B2 http://bit.ly/3Y7jIvc
     mode:  :compact, # export mode: compact or full
     out:   nil     , # output stream, needs to respond to <<
     quote: '"'     , # quote character
@@ -51,81 +50,67 @@ class Censive < StringScanner
     reset
 
     @drop   = drop
-    @eol    = eol  .freeze #!# TODO: are the '.freeze' statements helpful?
+    @eol    = eol
     @excel  = excel
     @mode   = mode
-    @out    = out
-    @quote  = quote.freeze
+    @out    = out || $stdout
+    @quote  = quote
     @relax  = relax
-    @sep    = sep  .freeze
+    @sep    = sep
 
-    @es     = ""   .freeze
-    @cr     = "\r" .freeze
-    @lf     = "\n" .freeze
-    @eq     = "="  .freeze
-    @esc    = (@quote * 2).freeze
-
-    @tokens = [@sep,@quote,@cr,@lf,@es,nil]
+    @cr     = "\r"
+    @lf     = "\n"
+    @es     = ""
+    @eq     = "="
+    @esc    = (@quote * 2)
   end
 
   def reset(str=nil)
     self.string = str if str
     super()
-    @char = peek(1)
-    @flag = nil
-
+    @char = curr_char
     @rows = nil
     @cols = @cells = 0
   end
 
   # ==[ Lexer ]==
 
-  def next_char
-    getch
-    @char = peek(1)
-  end
-
-  def next_token
+  # pure ruby versions for debugging
+  # def curr_char;             @char = string[pos]; end
+  # def next_char; scan(/./m); @char = string[pos]; end
 
-    # process and clear @flag
-    case @flag
-    when @es then @flag = nil; [@cr,@lf,@es,nil].include?(@char) and return @es
-    when @cr then @flag = nil; next_char == @lf and next_char
-    when @lf then @flag = nil; next_char
-    else          @flag = nil
-    end if @flag
+  def curr_char; @char = currchar; end
+  def next_char; @char = nextchar; end
 
-    # Excel literals ="0123" and formulas =A1 + B2 (see http://bit.ly/3Y7jIvc)
+  def next_token
     if @excel && @char == @eq
-      @flag = @eq
+      excel = true
       next_char
     end
 
-    if @tokens.include?(@char)
+    if @char == @quote # consume quoted cell
+      match = ""
+      while true
+        next_char
+        match << (scan_until(/(?=#{@quote})/o) or bomb "unclosed quote")
+        match << @quote and next if next_char == @quote
+        break if [@sep,@cr,@lf,@es,nil].include?(@char)
+        @relax or bomb "invalid character after quote"
+        match << @quote + scan_until(/(?=#{@quote})/o) + @quote
+      end
+      next_char if @char == @sep
+      match
+    elsif [@sep,@cr,@lf,@es,nil].include?(@char)
       case @char
-      when @quote # consume quoted cell
-        match = ""
-        while true
-          getch # move past the quote that got us here
-          match << (scan_until(/(?=#{@quote})/o) or bomb "unclosed quote")
-          case next_char
-          when @sep            then @flag = @es; next_char; break
-          when @quote          then match << @quote
-          when @cr,@lf,@es,nil then break
-          else @relax ? match << (@quote + @char) : bomb("invalid character after quote")
-          end
-        end
-        match
-      when @sep    then @flag = @es; next_char; @es
-      when @cr     then @flag = @cr; nil
-      when @lf     then @flag = @lf; nil
-      when @es,nil then              nil
+      when @sep then next_char; @es
+      when @cr  then next_char == @lf and next_char; nil
+      when @lf  then next_char; nil
+      else nil
       end
     else # consume unquoted cell
       match = scan_until(/(?=#{@sep}|#{@cr}|#{@lf}|\z)/o) or bomb "unexpected character"
-      match = @eq + match and @flag = nil if @flag == @eq
-      @char = peek(1)
-      @char == @sep and @flag = @es and next_char
+      match.prepend(@eq) if excel
+      next_char if curr_char == @sep
       match
     end
   end
@@ -156,10 +141,10 @@ class Censive < StringScanner
 
   # ==[ Helpers ]==
 
-  # grok returns: 2 (must be quoted and escaped), 1 (must be quoted), 0 (neither)
+  # returns 2 (must be quoted and escaped), 1 (must be quoted), 0 (neither)
   def grok(str)
-    if pos = str.index(/(#{@quote})|#{@sep}|#{@cr}|#{@lf}/o)
-      $1 ? 2 : str.index(/#{@quote}/o, pos) ? 2 : 1
+    if idx = str.index(/(#{@quote})|#{@sep}|#{@cr}|#{@lf}/o) #!# FIXME: regex injection?
+      $1 ? 2 : str.index(/#{@quote}/o, idx) ? 2 : 1
     else
       0
     end
@@ -167,17 +152,14 @@ class Censive < StringScanner
 
   # output a row
   def <<(row)
-    @out or return super
 
     # drop trailing empty columns
     row.pop while row.last.empty? if @drop
 
-    #!# FIXME: Excel output needs to protect 0-leading numbers
-
     s,q = @sep, @quote
     out = case @mode
     when :compact
-      case grok(row.join)
+      case @excel ? 2 : grok(row.join)
       when 0
         row
       when 1
@@ -186,6 +168,7 @@ class Censive < StringScanner
         end
       else
         row.map do |col|
+          @excel && col =~ /\A0\d*\z/ ? "=#{q}#{col}#{q}" :
           case grok(col)
           when 0 then col
           when 1 then "#{q}#{col}#{q}"
@@ -194,10 +177,15 @@ class Censive < StringScanner
         end
       end
     when :full
-      row.map {|col| "#{q}#{col.gsub(q, @esc)}#{q}" }
+      if @excel
+        row.map do |col|
+          col =~ /\A0\d*\z/ ? "=#{q}#{col}#{q}" : "#{q}#{col.gsub(q, @esc)}#{q}"
+        end
+      else
+        row.map {|col| "#{q}#{col.gsub(q, @esc)}#{q}" }
+      end
     end.join(s)
 
-    # add line ending
     @out << out + @eol
   end
 
@@ -206,8 +194,8 @@ class Censive < StringScanner
     @rows.each {|row| yield row }
   end
 
-  def export(...)
-    out = self.class.writer(...)
+  def export(**opts)
+    out = opts.empty? ? self : self.class.writer(**opts)
     each {|row| out << row }
   end
 
@@ -220,12 +208,11 @@ class Censive < StringScanner
   end
 end
 
-# ==[ Command line ]==
-
 if __FILE__ == $0
   raw = DATA.gets("\n\n").chomp
-  csv = Censive.new(raw, excel: true)
-  csv.export # (sep: "\t", excel: true)
+# raw = File.read(ARGV.first || "lc-2023.csv")
+  csv = Censive.new(raw, excel: true, relax: true)
+  csv.export # (sep: ",", excel: true)
 end
 
 __END__
@@ -236,39 +223,10 @@ Charlie or "Chuck",=B2 + B3,9
 "Doug E Fresh",="007",10
 Subtotal,=sum(B2:B5),="01234"
 
+# first line works in "relax" mode, bottom line is compliant
+123,"CHO, JOELLE "JOJO"",456
+123,"CHO, JOELLE ""JOJO""",456
 
-
-
-path = '../test/a-uses-tabs-and-single-quotes-and-no-trailing-newline.tsv'
-data = File.read(path)
-
-out = Censive.writer
-
-Censive.new(data, sep: "\t", quote: "'").each do |row|
-  p row
-end
-
-Censive.reader(path, sep: "\t", quote: "'").each {|r| p r}
-
-# STDOUT.sync = true
-#
-# data = File.read('1.csv')
-#
-# Censive.writer('out.csv') do |out|
-#   Censive.new(data, relax: true, excel: true).each do |row|
-#     out << row
-#   end
-# end
-#
-# __END__
-#
-# ARGV << "z.csv" if ARGV.empty?
-#
-# path = ARGV.first
-# data = File.read(path)
-#
-# csv = Censive.new(data)
-#
-# data.size > 1e6 ? csv.parse : csv.parse.each {|cols| p cols }
-#
-# csv.stats
+# Excel mode checking
+=,=x,x=,="x",="","","=",123,0123,="123",="0123"
+,=x,x=,x,,,,,,=,,123,="0123",123,,="0123" # <= a little off

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: censive
 version: !ruby/object:Gem::Version
-  version: '0.10'
+  version: '0.12'
 platform: ruby
 authors:
 - Steve Shreeve
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-02 00:00:00.000000000 Z
-dependencies: []
+date: 2023-02-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: strscan
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.6
 description: A quick and lightweight CSV handling library for Ruby
 email: steve.shreeve@gmail.com
 executables: []
@@ -40,7 +54,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.5
+rubygems_version: 3.4.6
 signing_key:
 specification_version: 4
 summary: A quick and lightweight CSV handling library for Ruby