xsv 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9493c51a049db036f7fcfacbdf911d69255008cb15fd4853eafd12a6d9eeaaf1
-  data.tar.gz: '008dfbfa456e9a07e7e301290388a23bb18e3e61f79ce480e5cff5f8b050253f'
+  metadata.gz: 34a568089851462504ab294931b59d454ed2df2788282e8aa15cc166e0c45271
+  data.tar.gz: 48e148855403abc349d62093d8351d68497681b43726c5894dc45b87c964a9e7
 SHA512:
-  metadata.gz: 154e166f904d5c964d44ad7886d8b5dbc350073d8c01b22f6490c1bf65ec376e06474b249f781314deb3541ee14bbb103e3acd096938bd1b7d8bb118b94b1ab1
-  data.tar.gz: ac24357f40b3aba550668a4d2ba4be75625751868f180a07b981dc931fcf28c2a714eac72443d123dd0a22ac28e1e6c67e56ca3423eb89a716b4e78b7ce8de59
+  metadata.gz: c634494fbff9d65dc9f16af6cdc457fb8caddd56da46bdaf256562a4b4d96ffe271ece897cec1ec153befa30eb617c3d39da70c5b5250039838848464108d14c
+  data.tar.gz: f3bde0d89ddd1d8b5badf35b7da02025cddcc41b3f6127f48ec56bfcef31d22e16ff4378f3c3a79502c42089c41dea35bce8b0245920ba23c4ab7e303c6f9340

data/Gemfile.lock

@@ -1,17 +1,15 @@
 PATH
   remote: .
   specs:
-    xsv (0.2.2)
-      nokogiri (~> 1.10)
+    xsv (0.2.3)
+      ox (~> 2.13)
       rubyzip (~> 2.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    mini_portile2 (2.4.0)
     minitest (5.14.0)
-    nokogiri (1.10.8)
-      mini_portile2 (~> 2.4.0)
+    ox (2.13.2)
     rake (10.5.0)
     rubyzip (2.2.0)
 

data/README.md

@@ -1,12 +1,12 @@
 # Xsv .xlsx reader
 
-Xsv is a very basic parser for Excel files in the .xlsx format that strives to
-provide feature parity with common CSV readers and nothing more. This means
-it only parses values to basic Ruby types and does not deal with most formatting
-or more advanced functionality. The goal is to allow for fast parsing of large
-worksheets with minimal RAM and CPU consumption.
+Xsv is a very basic parser for Office Open XML spreadsheet files (.xlsx files)
+that aims to provide feature parity with common CSV readers with high
+performance. This means it only parses values to basic Ruby types and does not
+deal with most formatting or more advanced functionality. The goal is to allow
+for fast parsing of large worksheets with minimal RAM and CPU consumption.
 
-Xsv stands for 'Excel Separated Values' because Excel just gets in the way.
+Xsv stands for 'Excel Separated Values', because Excel just gets in the way.
 
 ## Installation
 
@@ -99,7 +99,9 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/xsv.
+Bug reports and pull requests are welcome on GitHub at https://github.com/martijn/xsv.
+Please provide an .xlsx file with a minimum breaking example that is acceptable
+for inclusion in the source code repository.
 
 ## License
 

data/lib/xsv.rb

@@ -1,7 +1,12 @@
 require "date"
+require "ox"
 
 require "xsv/helpers"
+require "xsv/shared_strings_parser"
 require "xsv/sheet"
+require "xsv/sheet_bounds_handler"
+require "xsv/sheet_rows_handler"
+require "xsv/styles_handler"
 require "xsv/version"
 require "xsv/workbook"
 

data/lib/xsv/shared_strings_parser.rb

@@ -0,0 +1,37 @@
+module Xsv
+  class SharedStringsParser < Ox::Sax
+    def self.parse(io)
+      strings = []
+      handler = new { |s| strings << s }
+      Ox.sax_parse(handler, io)
+      return strings
+    end
+
+    def initialize(&block)
+      @block = block
+      @state = nil
+    end
+
+    def start_element(name)
+      case name
+      when :si
+        @current_string = ""
+      when :t
+        @state = name
+      end
+    end
+
+    def text(value)
+      @current_string += value if @state == :t
+    end
+
+    def end_element(name)
+      case name
+      when :si
+        @block.call(@current_string)
+      when :t
+        @state = nil
+      end
+    end
+  end
+end

data/lib/xsv/sheet.rb

@@ -3,26 +3,19 @@ module Xsv
     include Enumerable
     include Xsv::Helpers
 
-    attr_reader :xml, :mode
+    attr_reader :mode
 
     # Set a number of rows to skip at the top of the sheet (header row offset)
     attr_accessor :row_skip
 
-    def initialize(workbook, xml)
+    def initialize(workbook, io)
       @workbook = workbook
-      @xml = xml
+      @io = io
       @headers = []
       @mode = :array
       @row_skip = 0
 
-      @has_cells = !xml.at_css("sheetData c").nil?
-
-      if @has_cells
-        @column_count, @last_row = get_sheet_dimensions
-      else
-        @column_count = 0
-        @last_row = 0
-      end
+      @last_row, @column_count = SheetBoundsHandler.get_bounds(@io, @workbook)
     end
 
     def inspect
@@ -31,30 +24,14 @@ module Xsv
 
     # Iterate over rows
     def each_row
-      row_index = 0 - @row_skip
-
-      @xml.css("sheetData row").each do |row_xml|
-        if row_index < 0
-          row_index += 1
-          next
-        end
-
-        row_index += 1
-
-        next if row_index == 1 && @mode == :hash
-
-        # pad empty rows
-        while row_index < row_xml["r"].to_i - @row_skip do
-          yield(empty_row)
-          row_index += 1
-        end
+      @io.rewind
 
-        yield(parse_row(row_xml))
-
-        # Do not return empty trailing rows
-        break if row_index == @last_row - @row_skip
+      handler = SheetRowsHandler.new(@mode, empty_row, @workbook, @row_skip, @last_row) do |row|
+        yield(row)
       end
 
+      Ox.sax_parse(handler, @io)
+
       true
     end
 
@@ -62,13 +39,11 @@ module Xsv
 
     # Get row by number, starting at 0
     def [](number)
-      row_xml = xml.at_css("sheetData row[r=#{number + @row_skip + 1}]")
-
-      if row_xml
-        parse_row(row_xml)
-      else
-        empty_row
+      each_with_index do |row, i|
+        return row if i == number
       end
+
+      return empty_row
     end
 
     # Load headers in the top row of the worksheet. After parsing of headers
@@ -91,7 +66,12 @@ module Xsv
     private
 
     def parse_headers
-      parse_row(@xml.css("sheetData row")[@row_skip], :array)
+      if @mode == :array
+        first
+      elsif @mode == :hash
+        @mode == :array
+        headers.tap { @mode = :hash }
+      end
     end
 
     def empty_row
@@ -102,71 +82,5 @@ module Xsv
         @headers.zip([]).to_h
       end
     end
-
-    def parse_row(xml, mode = nil)
-      mode ||= @mode
-      row = empty_row
-
-      xml.css("c").first(@column_count).each do |c_xml|
-        value = case c_xml["t"]
-          when "s"
-            @workbook.shared_strings[c_xml.css("v").inner_text.to_i]
-          when "str"
-            c_xml.css("v").inner_text.to_s
-          when "e" # N/A
-            nil
-          when nil
-            v = c_xml.at_css("v")
-
-            if v.nil?
-              nil
-            elsif c_xml["s"]
-              style = @workbook.xfs[c_xml["s"].to_i]
-              numFmt = @workbook.numFmts[style[:numFmtId].to_i]
-
-              parse_number_format(v.inner_text, numFmt)
-            else
-              parse_number(v.inner_text)
-            end
-          else
-            raise Xsv::Error, "Encountered unknown column type #{c_xml["t"]}"
-          end
-
-        # Determine column position and pad row with nil values
-        col_index = column_index(c_xml["r"])
-
-        case mode
-        when :array
-          row[col_index] = value
-        when :hash
-          row[@headers[col_index]] = value
-        end
-      end
-
-      row
-    end
-
-    # Read or estimate outer bounds of sheet
-    def get_sheet_dimensions
-      dimension = xml.at_css("dimension")
-
-      if dimension
-        _firstCell, lastCell = dimension["ref"].split(":")
-      end
-
-      if lastCell
-        # Assume the dimension reflects the content
-        column_count = column_index(lastCell) + 1
-      else
-        # Find the last cell in every row that has a value
-        rightmost_cells = @xml.xpath("//xmlns:row/xmlns:c[*[local-name() = 'v']][last()]").map { |c| column_index(c["r"]) }
-        column_count = rightmost_cells.max + 1
-      end
-
-      # Find the last row that contains actual values
-      last_row = @xml.at_xpath("//xmlns:row[*[xmlns:v]][last()]")["r"].to_i
-
-      return [column_count, last_row]
-    end
   end
 end

data/lib/xsv/sheet_bounds_handler.rb

@@ -0,0 +1,76 @@
+module Xsv
+  # SheetBoundsHandler scans a sheet looking for the outer bounds of the content within
+  class SheetBoundsHandler < Ox::Sax
+    include Xsv::Helpers
+
+    def self.get_bounds(sheet, workbook)
+      rows = 0
+      cols = 0
+
+      handler = new(workbook.trim_empty_rows) do |row, col|
+        rows = row
+        cols = col == 0 ? 0 : col + 1
+
+        return rows, cols
+      end
+
+      sheet.rewind
+      Ox.sax_parse(handler, sheet)
+
+      return rows, cols
+    end
+
+    # Ox::Sax implementation
+
+    def initialize(trim_empty_rows, &block)
+      @block = block
+      @state = nil
+      @cell = nil
+      @row = nil
+      @maxRow = 0
+      @maxColumn = 0
+      @trim_empty_rows = trim_empty_rows
+    end
+
+    def start_element(name)
+      case name
+      when :c
+        @state = name
+        @cell = nil
+      when :v
+        col = column_index(@cell)
+        @maxColumn = col if col > @maxColumn
+        @maxRow = @row if @row > @maxRow
+      when :row
+        @state = name
+        @row = nil
+      when :dimension
+        @state = name
+      end
+    end
+
+    def end_element(name)
+      if name == :sheetData
+        @block.call(@maxRow, @maxColumn)
+      end
+    end
+
+    def attr(name, value)
+      if @state == :c && name == :r
+        @cell = value
+      elsif @state == :row && name == :r
+        @row = value.to_i
+      elsif @state == :dimension && name == :ref
+        _firstCell, lastCell = value.split(":")
+
+        if lastCell
+          @maxColumn = column_index(lastCell)
+          unless @trim_empty_rows
+            @maxRow = lastCell[/\d+$/].to_i
+            @block.call(@maxRow, @maxColumn)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/xsv/sheet_rows_handler.rb

@@ -0,0 +1,121 @@
+module Xsv
+  class SheetRowsHandler < Ox::Sax
+    include Xsv::Helpers
+
+    def format_cell
+      case @current_cell[:t]
+      when "s"
+        @workbook.shared_strings[@current_value.to_i]
+      when "str"
+        @current_value
+      when "e" # N/A
+        nil
+      when nil
+        if @current_value == ""
+          nil
+        elsif @current_cell[:s]
+          style = @workbook.xfs[@current_cell[:s].to_i]
+          numFmt = @workbook.numFmts[style[:numFmtId].to_i]
+
+          parse_number_format(@current_value, numFmt)
+        else
+          parse_number(@current_value)
+        end
+      else
+        raise Xsv::Error, "Encountered unknown column type #{@current_cell[:t]}"
+      end
+    end
+
+    # Ox::Sax implementation below
+
+    def initialize(mode, empty_row, workbook, row_skip, last_row, &block)
+      @block = block
+
+      # :sheetData
+      # :row
+      # :c
+      # :v
+      @state = nil
+
+      @mode = mode
+      @empty_row = empty_row
+      @workbook = workbook
+      @row_skip = row_skip
+      @row_index = 0 - @row_skip
+      @current_row = {}
+      @current_row_attrs = {}
+      @current_cell = {}
+      @current_value = nil
+      @last_row = last_row
+
+      if @mode == :hash
+        @headers = @empty_row.keys
+      end
+    end
+
+    def start_element(name)
+      case name
+      when :c
+        @state = name
+        @current_cell = {}
+        @current_value = ""
+      when :v
+        @state = name
+      when :row
+        @state = name
+        @current_row = @empty_row.dup
+        @current_row_attrs = {}
+      else
+        @state = nil
+      end
+    end
+
+    def text(value)
+      if @state == :v
+        @current_value += value
+      end
+    end
+
+    def attr(name, value)
+      case @state
+      when :c
+        @current_cell[name] = value
+      when :row
+        @current_row_attrs[name] = value
+      end
+    end
+
+    def end_element(name)
+      case name
+      when :c
+        col_index = column_index(@current_cell[:r])
+
+        case @mode
+        when :array
+          @current_row[col_index] = format_cell
+        when :hash
+          @current_row[@headers[col_index]] = format_cell
+        end
+      when :row
+        if @row_index < 0
+          @row_index += 1
+          return
+        end
+
+        @row_index += 1
+
+        # Skip first row if we're in hash mode
+        return if @row_index == 1 && @mode == :hash
+
+        # Pad empty rows
+        while @row_index < @current_row_attrs[:r].to_i - @row_skip
+          @block.call(@empty_row)
+          @row_index += 1
+        end
+
+        # Do not return empty trailing rows
+        @block.call(@current_row) unless @row_index > @last_row - @row_skip
+      end
+    end
+  end
+end

data/lib/xsv/styles_handler.rb

@@ -0,0 +1,58 @@
+module Xsv
+  # StylesHandler interprets the relevant parts of styles.xml
+  class StylesHandler < Ox::Sax
+    def self.get_styles(io, numFmts)
+      @xfs = nil
+      @numFmts = nil
+      handler = new(numFmts) do |xfs, numFmts|
+         @xfs = xfs
+         @numFmts = numFmts
+      end
+
+      Ox.sax_parse(handler, io)
+      return @xfs, @numFmts
+    end
+
+    # Ox::Sax implementation
+
+    def initialize(numFmts, &block)
+      @block = block
+      @state = nil
+      @xfs = []
+      @numFmts = numFmts
+
+      @xf = {}
+      @numFmt = {}
+    end
+
+    def start_element(name)
+      case name
+      when :cellXfs, :numFmts
+        @state = name
+      when :xf
+        @xf = {}
+      when :numFmt
+        @numFmt = {}
+      end
+    end
+
+    def attr(name, value)
+      case @state
+      when :cellXfs
+        @xf[name] = value
+      when :numFmts
+        @numFmt[name] = value
+      end
+    end
+
+    def end_element(name)
+      if @state == :cellXfs && name == :xf
+        @xfs << @xf
+      elsif @state == :numFmts && name == :numFmt
+        @numFmts[@numFmt[:numFmtId].to_i] = @numFmt[:formatCode]
+      elsif name == :styleSheet
+        @block.call(@xfs, @numFmts)
+      end
+    end
+  end
+end

data/lib/xsv/version.rb

@@ -1,3 +1,3 @@
 module Xsv
-  VERSION = "0.2.3"
+  VERSION = "0.3.0"
 end

data/lib/xsv/workbook.rb

@@ -4,22 +4,28 @@ require 'zip'
 module Xsv
   class Workbook
 
-    attr_reader :sheets, :shared_strings, :xfs, :numFmts
+    attr_reader :sheets, :shared_strings, :xfs, :numFmts, :trim_empty_rows
 
     # Open the workbook of the given filename, string or buffer
-    def self.open(data)
+    def self.open(data, **kws)
       if data.is_a?(IO)
-        @workbook = self.new(Zip::File.open_buffer(data))
+        @workbook = self.new(Zip::File.open_buffer(data), kws)
       elsif data.start_with?("PK\x03\x04")
-        @workbook = self.new(Zip::File.open_buffer(data))
+        @workbook = self.new(Zip::File.open_buffer(data), kws)
       else
-        @workbook = self.new(Zip::File.open(data))
+        @workbook = self.new(Zip::File.open(data), kws)
       end
     end
 
     # Open a workbook from an instance of Zip::File
-    def initialize(zip)
+    #
+    # Options:
+    #
+    #    trim_empty_rows (false) Scan sheet for end of content and don't return trailing rows
+    #
+    def initialize(zip, trim_empty_rows: false)
       @zip = zip
+      @trim_empty_rows = trim_empty_rows
 
       @sheets = []
       @xfs = []
@@ -38,35 +44,22 @@ module Xsv
 
     def fetch_shared_strings
       stream = @zip.glob("xl/sharedStrings.xml").first.get_input_stream
-      xml = Nokogiri::XML(stream)
-      expected_count = xml.at_css("sst")["uniqueCount"].to_i
-      @shared_strings = xml.css("sst si").map { |si| si.css("t").map(&:inner_text).join }
-
-      if @shared_strings.count != expected_count
-        raise Xsv::AssertionFailed, "Mismatch in shared strings count! #{expected_count} <> #{@shared_strings.count}"
-      end
+      @shared_strings = SharedStringsParser.parse(stream)
 
       stream.close
     end
 
     def fetch_styles
       stream = @zip.glob("xl/styles.xml").first.get_input_stream
-      xml = Nokogiri::XML(stream)
 
-      xml.css("cellXfs xf").each do |xf|
-        @xfs << xf.attributes.map { |k, v| [k.to_sym, v.value] }.to_h
-      end
-
-      xml.css("numFmts numFmt").each do |numFmt|
-        @numFmts[numFmt["numFmtId"].to_i] = numFmt["formatCode"]
-      end
+      @xfs, @numFmts = StylesHandler.get_styles(stream, @numFmts)
     end
 
     def fetch_sheets
       @zip.glob("xl/worksheets/sheet*.xml").sort do |a, b|
         a.name[/\d+/].to_i <=> b.name[/\d+/].to_i
       end.each do |entry|
-        @sheets << Xsv::Sheet.new(self, Nokogiri::XML(entry.get_input_stream))
+        @sheets << Xsv::Sheet.new(self, entry.get_input_stream)
       end
     end
   end

data/xsv.gemspec

@@ -34,7 +34,7 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '~> 2.6'
 
   spec.add_dependency "rubyzip", "~> 2.2"
-  spec.add_dependency "nokogiri", "~> 1.10"
+  spec.add_dependency "ox", "~> 2.13"
 
   spec.add_development_dependency "bundler", "~> 1.17"
   spec.add_development_dependency "rake", "~> 10.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: xsv
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - Martijn Storck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-02-22 00:00:00.000000000 Z
+date: 2020-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip
@@ -25,19 +25,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.2'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: ox
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '2.13'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '2.13'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -98,7 +98,11 @@ files:
 - bin/setup
 - lib/xsv.rb
 - lib/xsv/helpers.rb
+- lib/xsv/shared_strings_parser.rb
 - lib/xsv/sheet.rb
+- lib/xsv/sheet_bounds_handler.rb
+- lib/xsv/sheet_rows_handler.rb
+- lib/xsv/styles_handler.rb
 - lib/xsv/version.rb
 - lib/xsv/workbook.rb
 - test.sh