xsv 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 58e5d405e39f42d0e5287d47dd65c65b39a0ab5a2fc7fde3fd85c7211469e6e1
-  data.tar.gz: 7100a73ce192536f81a34ffbb1b431a793edf9cb71c1612a547a64a686a8330f
+  metadata.gz: 1ff63ab6263b0486a400d5d36eb53d0af7265532b67536c7024b414b2bcf3ad2
+  data.tar.gz: a885af30f1a874818bb80f984115d7bc47e178007bf9848407e279af3effcd3b
 SHA512:
-  metadata.gz: 3b8fcbab2e2aa1f02dc0b51051a9b60dd2518b18b72007c2f3e77fa99248e864069d54b0bb43d783f8bb6ef79b6c2504c8cd05c244a2e9c85cddb882de224556
-  data.tar.gz: 3ec5120d8b6e365996985c75f4c291e3f4805e9876fd93ae2dfe071c5bd69ad751677cf71121ac23f6e9bed75ab80296b12346d79696cb6105974345d289bb7e
+  metadata.gz: 37b9b49b66fcec0c393a56382b162d0fb3c77e2342798292d3a4c73c49939ce3d0827c783a7b9b5c139b52553e3a622f6e47f510e4839963609c571e7ab53067
+  data.tar.gz: ecd5e49c82d802e2a654c7d6321cdf81b5ea81482a83821ca6ddd19ae5036a6e2a516569356ded600b2181b0c4161c5c7e4e36a43570b4bef82f7c9d58038392

data/.github/workflows/ruby.yml

@@ -19,10 +19,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby-version: ['2.6', '2.7', '3.0', '3.1', 'jruby', 'truffleruby']
+        ruby-version: ['2.7', '3.0', '3.1', '3.2', '3.3', 'jruby', 'truffleruby']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:

data/.standard.yml

@@ -1 +1 @@
-ruby_version: 2.6.9
+ruby_version: 2.7.8

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Xsv Changelog
 
+## 1.3.0 2023-12-16
+
+- Ruby 2.6 is no longer supported. Xsv is compatible with Ruby 2.7 through 3.3, latest JRuby, and latest TruffleRuby
+- Easier access worksheets using `Xsv::Workbook#[]` and `Enumerable` methods on `Xsv::Workbook`. The old `sheets` and `sheet_by_name` method have been retained for backward compatibility.
+- Update of development dependencies including minitest and standardrb
+- Various performance improvements, especially with YJIT on Ruby 3.3
+
+## 1.2.1 2023-05-09
+
+- Handle columns without `r` attribute (issue #48)
+- Access sheets by UTF-8 name (thanks @sebikeller)
+
 ## 1.2.0 2023-01-01
 
 **This release contains the following minor breaking changes**

data/README.md

@@ -1,23 +1,20 @@
-# Xsv .xlsx reader
-
-
+# Xsv .xlsx reader for Ruby
 
 [![Test badge](https://img.shields.io/github/actions/workflow/status/martijn/xsv/ruby.yml?branch=main)](https://github.com/martijn/xsv/actions/workflows/ruby.yml)
-[![Codecov badge](https://img.shields.io/codecov/c/github/martijn/xsv/main)](https://app.codecov.io/gh/martijn/xsv)
 [![Yard Docs badge](http://img.shields.io/badge/yard-docs-blue.svg)](https://rubydoc.info/github/martijn/xsv)
 [![Gem Version badge](https://badge.fury.io/rb/xsv.svg)](https://badge.fury.io/rb/xsv)
 
-Xsv is a fast, lightweight, pure Ruby parser for ISO/IEC 29500 Office Open XML spreadsheet files
-(commonly known as Excel or .xlsx files). It strives to be minimal in the
-sense that it provides nothing a CSV reader wouldn't, meaning it only
-deals with minimal formatting and cannot create or modify documents.
+Xsv is a high performance, lightweight, pure Ruby parser for ISO/IEC 29500 Office Open XML spreadsheets
+(commonly known as Excel or .xlsx files). It strives to be minimal in the sense that it provides nothing a
+CSV reader wouldn't. This means it only deals with the minimal required formatting and cannot create or modify
+documents.
+Xsv can handle very large Excel files with minimal resources thanks to a custom streaming XML parser that
+is optimized for the Excel file format.
 
 Xsv is designed for worksheets with a single table of data, optionally
 with a header row. It only casts values to basic Ruby types (integer, float,
 date and time) and does not deal with most formatting or more advanced
-functionality. It strives for fast processing of large worksheets with
-minimal RAM and CPU consumption and has been in production use since the earliest
-versions.
+functionality. Xsv has been production-ready since the initial release.
 
 Xsv stands for 'Excel Separated Values', because Excel just gets in the way.
 
@@ -37,20 +34,24 @@ Or install it yourself as:
 
     $ gem install xsv
 
-Xsv targets ruby >= 2.6 and has a just single dependency, `rubyzip`. It has been
+Xsv targets ruby >= 2.7 and has a just single dependency, `rubyzip`. It has been
 tested successfully with MRI, JRuby, and TruffleRuby. It has no native extensions
 and is designed to be thread-safe.
 
 ## Usage
 
 ### Array and hash mode
+
 Xsv has two modes of operation. By default, it returns an array for
 each row in the sheet:
 
 ```ruby
-x = Xsv.open("sheet.xlsx") # => #<Xsv::Workbook sheets=1>
+workbook = Xsv.open("sheet.xlsx") # => #<Xsv::Workbook sheets=1>
 
-sheet = x.sheets[0]
+# Access worksheet by index, 0 is the first sheet
+sheet = workbook[0]
+# or, access worksheet by name
+sheet = workbook["Sheet1"]
 
 # Iterate over rows
 sheet.each do |row|
@@ -68,29 +69,35 @@ option on open:
 ```ruby
 # Parse headers for all sheets on open
 
-x = Xsv.open("sheet.xlsx", parse_headers: true)
+workbook = Xsv.open("sheet.xlsx", parse_headers: true)
 
-x.sheets[0][1]   # => {"header1" => "value1", "header2" => "value2"}
+# Get the first row from the first sheet
+workbook.first.first # => {"header1" => "value1", "header2" => "value2"}
 
 # Manually parse headers for a single sheet
 
-x = Xsv.open("sheet.xlsx")
+workbook = Xsv.open("sheet.xlsx")
 
-sheet = x.sheets[0]
+sheet = workbook.first
 
-sheet[0]  # => ["header1", "header2"]
+sheet.first # => ["header1", "header2"]
 
 sheet.parse_headers!
 
-sheet[0] # => {"header1" => "value1", "header2" => "value2"}
+sheet.first # => {"header1" => "value1", "header2" => "value2"}
 ```
 
-Because of the way Ruby hashes work will raise `Xsv::DuplicateHeaders` if it detects
-duplicate values in the header row when calling `#parse_headers!` or when opening
-a workbook with `parse_headers: true`.
+Xsv will raise `Xsv::DuplicateHeaders` if it detects duplicate values in the header row when calling
+`#parse_headers!` or when opening a workbook with `parse_headers: true` to ensure hash keys are unique.
 
 `Xsv::Sheet` implements `Enumerable` so along with `#each`
-you can call methods like `#first`, `#filter`/`#select`, and `#map` on it.
+you can call methods like `#first`, `#filter`/`#select`, and `#map` on it. Likewise these methods can
+be used on `Xsv::Workbook` to iterate over sheets, for example:
+
+```ruby
+# Get the name of all the sheets in a workbook
+sheet_names = @workbook.map(&:name)
+```
 
 ### Opening a string or buffer instead of filename
 
@@ -116,24 +123,6 @@ end
 Prior to Xsv 1.1.0, `Xsv::Workbook.open` was used instead of `Xsv.open`. The parameters are identical and
 the former is maintained for backwards compatibility.
 
-### Accessing sheets by name
-
-The sheets can be accessed by index or by name:
-
-```ruby
-x = Xsv.open("sheet.xlsx")
-
-sheet = x.sheets[0] # gets sheet by index
-
-sheet = x.sheets_by_name('Name').first # gets sheet by name
-```
-
-To get all the sheets names:
-
-```ruby
-sheet_names = x.sheets.map(&:name)
-```
-
 ### Assumptions
 
 Since Xsv treats worksheets like csv files it makes certain assumptions about your
@@ -147,26 +136,35 @@ If your data or headers do not start on the first row of the sheet you can
 tell Xsv to skip a number of rows:
 
 ```ruby
-workbook.sheets[0].row_skip = 1
+sheet = workbook[0]
+sheet.row_skip = 1
 ```
 
 All operations will honour this offset, making the skipped rows unreachable.
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can
+also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the
+version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version,
+push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Performance and Benchmarks
 
-Xsv is faster and more memory efficient than other gems because of two things: it only _reads values_ from Excel files and it's based on a SAX-based parser instead of a DOM-based parser. If you want to read some background on this, check out my blog post on
+Xsv is faster and more memory efficient than other gems because of two things: it only _reads values_ from Excel files
+and it's based on a SAX-based parser instead of a DOM-based parser. If you want to read some background on this, check
+out my blog post on
 [Efficient XML parsing in Ruby](https://storck.io/posts/efficient-xml-parsing-in-ruby/).
 
-Jamie Schembri did a shootout of Xsv against various other Excel reading gems comparing parsing speed, memory usage, and allocations.
+Jamie Schembri did a shootout of Xsv against various other Excel reading gems comparing parsing speed, memory usage, and
+allocations.
 Check our his blog post: [Faster Excel parsing in Ruby](https://blog.schembri.me/post/faster-excel-parsing-in-ruby/).
 
-Pre-1.0, Xsv used a native extension for XML parsing, which was faster than the native Ruby one (on MRI). But even with the native Ruby version generally Xsv still outperforms other Ruby parsing gems.
+Pre-1.0, Xsv used a native extension for XML parsing, which was faster than the native Ruby one (on MRI). But even
+the current native Ruby parser generally outperforms the competition. For maximum performance, it is recommended to
+enable YJIT.
 
 ## Contributing
 
@@ -176,4 +174,6 @@ for inclusion in the source code repository.
 
 ## License
 
+Copyright © Martijn Storck and Xsv contributors
+
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/benchmark.rb

@@ -14,7 +14,6 @@ def bench_perf(sheet)
   result = Benchmark::Perf.cpu(repeat: 5) do
     sheet.each do |row|
       row.each do |cell|
-        cell
       end
     end
   end
@@ -27,13 +26,14 @@ def bench_mem(sheet)
     bm.report do
       sheet.each do |row|
         row.each do |cell|
-          cell
         end
       end
     end
   end
 end
 
+puts RUBY_DESCRIPTION
+
 file = File.read("test/files/10k-sheet.xlsx")
 
 workbook = Xsv.open(file)

data/lib/xsv/helpers.rb

@@ -45,11 +45,14 @@ module Xsv
     EPOCH = Date.new(1899, 12, 30).freeze
 
     # Return the index number for the given Excel column name (i.e. "A1" => 0)
+    # @param col [String] Column name in A1 notation
     def column_index(col)
-      col.each_codepoint.reduce(0) do |sum, n|
-        break sum - 1 if n < A_CODEPOINT # reached a number
+      chars = col.bytes
+      sum = 0
+      while (char = chars.delete_at(0))
+        break sum - 1 if char < A_CODEPOINT # reached the number
 
-        sum * 26 + (n - A_CODEPOINT + 1)
+        sum = sum * 26 + (char - A_CODEPOINT + 1)
       end
     end
 

data/lib/xsv/sax_parser.rb

@@ -62,15 +62,27 @@ module Xsv
               args = nil
             end
 
-            stripped_tag_name = strip_namespace(tag_name)
+            is_close_tag = tag_name.delete_prefix!("/")
 
-            if tag_name.start_with?("/")
-              end_element(strip_namespace(tag_name[1..])) if responds_to_end_element
+            # Strip XML namespace from tag
+            if (offset = tag_name.index(":"))
+              tag_name.slice!(0, offset + 1)
+            end
+
+            if is_close_tag
+              end_element(tag_name) if responds_to_end_element
             elsif args.nil?
-              start_element(stripped_tag_name, nil)
+              start_element(tag_name, nil)
             else
-              start_element(stripped_tag_name, args.scan(ATTR_REGEX).each_with_object({}) { |(_, k, v), h| h[k.to_sym] = v })
-              end_element(stripped_tag_name) if responds_to_end_element && args.end_with?("/")
+              attribute_buffer = {}
+              attributes = args.scan(ATTR_REGEX)
+              while (attr = attributes.delete_at(0))
+                attribute_buffer[attr[1].to_sym] = attr[2]
+              end
+
+              start_element(tag_name, attribute_buffer)
+
+              end_element(tag_name) if responds_to_end_element && args.end_with?("/")
             end
 
             state = :look_start
@@ -82,16 +94,5 @@ module Xsv
         end
       end
     end
-
-    private
-
-    # I am not proud of this, but there's simply no need to deal with xmlns for this application ¯\_(ツ)_/¯
-    def strip_namespace(tag)
-      if (offset = tag.index(":"))
-        tag[offset + 1..]
-      else
-        tag
-      end
-    end
   end
 end

data/lib/xsv/sheet.rb

@@ -30,13 +30,11 @@ module Xsv
     #
     # @param workbook [Workbook] The Workbook with shared data such as shared strings and styles
     # @param io [IO] A handle to an open worksheet XML file
-    # @param size [Number] size of the XML file
-    def initialize(workbook, io, size, ids)
+    def initialize(workbook, io, ids)
       @workbook = workbook
       @id = ids[:sheetId].to_i
       @io = io
       @name = ids[:name]
-      @size = size
       @headers = []
       @mode = :array
       @row_skip = 0
@@ -58,11 +56,7 @@ module Xsv
     # Iterate over rows, returning either hashes or arrays based on the current mode.
     def each_row(&block)
       @io.rewind
-
-      handler = SheetRowsHandler.new(@mode, empty_row, @workbook, @row_skip, @last_row, &block)
-
-      handler.parse(@io)
-
+      SheetRowsHandler.new(@mode, empty_row, @workbook, @row_skip, @last_row, &block).parse(@io)
       true
     end
 

data/lib/xsv/sheet_rows_handler.rb

@@ -17,6 +17,7 @@ module Xsv
       @store_characters = false
 
       @row_index = 0
+      @col_index = 0
       @current_row = {}
       @current_row_number = 0
       @current_cell = {}
@@ -33,7 +34,7 @@ module Xsv
       when "v", "is", "t"
         @store_characters = true
       when "row"
-        @current_row = @empty_row.dup
+        @current_row = @mode == :array ? [] : @empty_row.dup
         @current_row_number = attrs[:r].to_i
       end
     end
@@ -47,19 +48,22 @@ module Xsv
       when "v", "is", "t"
         @store_characters = false
       when "c"
-        col_index = column_index(@current_cell[:r])
+        col_index = @current_cell[:r] ? column_index(@current_cell[:r]) : @col_index
 
         if @mode == :array
           @current_row[col_index] = format_cell
         else
           @current_row[@headers[col_index]] = format_cell
         end
+
+        @col_index += 1
       when "row"
         return if @current_row_number <= @row_skip
 
         adjusted_row_number = @current_row_number - @row_skip
 
         @row_index += 1
+        @col_index = 0
 
         # Skip first row if we're in hash mode
         return if adjusted_row_number == 1 && @mode == :hash
@@ -72,7 +76,14 @@ module Xsv
         end
 
         # Do not return empty trailing rows
-        @block.call(@current_row) unless @row_index > @last_row
+        return if @row_index > @last_row
+
+        # Add trailing empty columns
+        if @mode == :array && @current_row.length < @empty_row.length
+          @block.call(@current_row + @empty_row[@current_row.length..])
+        else
+          @block.call(@current_row)
+        end
       end
     end
 

data/lib/xsv/sheets_ids_handler.rb

@@ -9,7 +9,9 @@ module Xsv
 
       new { |sheet_ids| sheets_ids << sheet_ids }.parse(io)
 
-      sheets_ids
+      sheets_ids.each do |sheet|
+        sheet[:name].force_encoding(Encoding::UTF_8)
+      end
     end
 
     def initialize(&block)

data/lib/xsv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Xsv
-  VERSION = "1.2.0"
+  VERSION = "1.3.0"
 end

data/lib/xsv/workbook.rb

@@ -6,6 +6,8 @@ module Xsv
   # An OOXML Spreadsheet document is called a Workbook. A Workbook consists of
   # multiple Sheets that are available in the array that's accessible through {#sheets}
   class Workbook
+    include Enumerable
+
     # Access the Sheet objects contained in the workbook
     # @return [Array<Sheet>]
     attr_reader :sheets
@@ -69,6 +71,24 @@ module Xsv
       @num_fmts[@xfs[style][:numFmtId]]
     end
 
+    def each(&block)
+      sheets.each(&block)
+    end
+
+    # Get a sheet by index or name
+    # @param [String, Integer] index_or_name The name of the sheet or index in the workbook
+    # @return [<Xsv::Sheet>, nil] returns the sheet instance or nil if it was not found
+    def [](index_or_name)
+      case index_or_name
+      when Integer
+        sheets[index_or_name]
+      when String
+        sheets_by_name(index_or_name).first
+      else
+        raise ArgumentError, "Sheets can be accessed by Integer of String only"
+      end
+    end
+
     private
 
     def fetch_shared_strings
@@ -98,7 +118,7 @@ module Xsv
             r[:Type].end_with?("worksheet")
         end
         sheet_ids = @sheet_ids.detect { |i| i[:id] == rel[:Id] }
-        Xsv::Sheet.new(self, entry.get_input_stream, entry.size, sheet_ids).tap do |sheet|
+        Xsv::Sheet.new(self, entry.get_input_stream, sheet_ids).tap do |sheet|
           sheet.parse_headers! if mode == :hash
         end
       end

data/xsv.gemspec

@@ -36,13 +36,12 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.required_ruby_version = ">= 2.6"
+  spec.required_ruby_version = ">= 2.7"
 
   spec.add_dependency "rubyzip", ">= 1.3", "< 3"
 
   spec.add_development_dependency "bundler", "< 3"
-  spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "minitest", "~> 5.14.2"
-  spec.add_development_dependency "standard", "~> 1.6.0"
-  spec.add_development_dependency "codecov", ">= 0.6.0"
+  spec.add_development_dependency "rake", "~> 13.1.0"
+  spec.add_development_dependency "minitest", "~> 5.20.0"
+  spec.add_development_dependency "standard", "~> 1.32.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: xsv
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Martijn Storck
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-01-01 00:00:00.000000000 Z
+date: 2023-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip
@@ -50,56 +50,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: 13.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: 13.1.0
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.14.2
+        version: 5.20.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.14.2
+        version: 5.20.0
 - !ruby/object:Gem::Dependency
   name: standard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.32.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.0
-- !ruby/object:Gem::Dependency
-  name: codecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.6.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 1.32.1
 description: |2
                               Xsv is a fast, lightweight parser for Office Open XML spreadsheet files
                               (commonly known as Excel or .xlsx files). It strives to be minimal in the
@@ -150,14 +136,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.5.1
 signing_key:
 specification_version: 4
 summary: A fast and lightweight xlsx parser that provides nothing a CSV parser wouldn't