calcsheet-native 0.1.0

data/lib/calcsheet/native/xlsx_reader.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+module Calcsheet
+  module Native
+    # The native XLSX backend: calcsheet's Ruby reader (the zip archive and its limits, the workbook, styles and
+    # shared strings) with the XML of each sheet read in Rust instead of by Ox. The Rust side reads a plain subset of
+    # XML and the values calcsheet accepts; on anything else (a comment, CDATA, a reference in an attribute, a value
+    # calcsheet refuses, an error of the zip stream...) the sheet is read again by calcsheet's own parser, which
+    # gives its exact rows or error. The rows already yielded are not yielded again, and the bytes the first attempt
+    # inflated are not charged twice against +max_uncompressed_size+.
+    #
+    # A read from a later row (the rows after the header, +start_at:+) parses the rows before it too, where calcsheet
+    # passes over them with RowSkipper: what the subset reads, RowSkipper passes over without an error, and anything
+    # else is again left to calcsheet.
+    #
+    # It depends on the private methods of Calcsheet::Readers::XlsxReader and the internals of ZipContainer of the
+    # calcsheet minor version it supports.
+    class XlsxReader < Calcsheet::Readers::XlsxReader
+      # calcsheet's SharedStrings over a StringTable read in Rust, which the sheet readers share: one copy of the
+      # strings.
+      class SharedStrings < Readers::Xlsx::SharedStrings
+        # @return [StringTable]
+        attr_reader :native
+
+        # @param table [StringTable]
+        def initialize(table) # rubocop:disable Lint/MissingSuper -- the buffer and offsets of the superclass are in Rust
+          @native = table
+          @size = table.size
+          @bytesize = table.bytesize
+        end
+
+        # @see Readers::Xlsx::SharedStrings#unchecked
+        def unchecked(index)
+          @native.unchecked(index)
+        end
+      end
+
+      # Thrown when calcsheet's parser must read the sheet.
+      FALLBACK = :calcsheet_native_fallback
+      # Rows per call into Rust, and bytes read from the zip stream at a time.
+      ROWS = 512
+      CHUNK = 64 * 1024
+      # The chunks of shared strings between two minor GCs (see #read_shared_strings).
+      GC_CHUNKS = 32
+      # The kinds of XlsxSheet's cells; the text of a SHARED cell is its shared string.
+      NUMBER = 0
+      SHARED = 1
+      INLINE = 2
+      BOOLEAN = 3
+      ERROR = 4
+      DATE = 5
+      # The inflated total of ZipContainer, restored before calcsheet's parser reads the sheet again.
+      TOTAL = :@inflated_total
+
+      @fallbacks = 0
+
+      class << self
+        # @return [Integer] the sheet reads handed to calcsheet's parser in this process, for specs and diagnostics
+        attr_accessor :fallbacks
+      end
+
+      private
+
+      def each_row(index, cast, types, start_row, &block)
+        path = @meta.sheet_path(index)
+        return super unless @zip.entry?(path)
+
+        charged = @zip.instance_variable_get(TOTAL)
+        strings = @shared_strings
+        last = 0
+        finished = catch(FALLBACK) do
+          read_natively(path, cast, types, start_row) do |row|
+            last = row.number
+            yield row
+          end
+          true
+        end
+        return if finished
+
+        self.class.fallbacks += 1
+        @zip.instance_variable_set(TOTAL, charged)
+        @shared_strings = strings
+        super(index, cast, types, start_row) { |row| yield row if row.number > last }
+      end
+
+      # Yields the rows read by XlsxSheet; throws FALLBACK when calcsheet's parser must read the sheet. The errors of
+      # the caller's block are its own.
+      def read_natively(path, cast, types, start_row)
+        stream = guarded { @zip.stream(path) }
+        sheet = XlsxSheet.new(@max_cell_bytes)
+        builder = RowBuilder.new(cast, types, number_errors: :as_read)
+        @style_formats = {}
+        # Read into one String: a new one per chunk would leave the whole part as garbage before the GC runs.
+        buffer = String.new(capacity: CHUNK)
+        loop do
+          batch = guarded { sheet.rows(ROWS) }
+          throw FALLBACK if batch == false
+          if batch == :strings
+            guarded do
+              table = shared_strings
+              if table.is_a?(SharedStrings)
+                sheet.table(table.native)
+              else
+                sheet.strings(table.instance_variable_get(:@buffer), table.instance_variable_get(:@offsets))
+              end
+            end
+            next
+          end
+          if batch.nil?
+            chunk = guarded { stream.read(CHUNK, buffer) }
+            chunk ? sheet.feed(chunk) : sheet.finish
+            next
+          end
+
+          numbers, ends, cells, done = batch
+          from = 0
+          numbers.each_with_index do |number, position|
+            to = ends[position]
+            guarded { add_cells(builder, cells, from, to) }
+            from = to
+            row = builder.finish(number)
+            yield row if row && number >= start_row
+          end
+          break if done
+        end
+      ensure
+        stream&.close
+      end
+
+      # Runs the block, throwing FALLBACK for any error it raises: calcsheet's parser raises it again, where it
+      # would.
+      def guarded
+        yield
+      rescue StandardError
+        throw FALLBACK
+      end
+
+      # Adds the cells +from...to+ of a batch to the builder, as SheetParser#finish_cell does.
+      def add_cells(builder, cells, from, to)
+        while from < to
+          column = cells[from]
+          style = cells[from + 2]
+          text = cells[from + 3]
+          text = read_text(text) if cells[from + 4]
+          case cells[from + 1]
+          when NUMBER then builder.add(column, :number, text, format(style))
+          when SHARED then builder.add(column, :text, text)
+          when INLINE then builder.add(column, :text, Readers::Xlsx::Escapes.decode!(text))
+          when BOOLEAN then builder.add(column, :boolean, text)
+          when ERROR then builder.add(column, :error, text)
+          when DATE then builder.add(column, :iso_date, text, format(style))
+          else builder.add(column, :text, text)
+          end
+          from += 6
+        end
+      end
+
+      # The line ends and references of a text, as SheetParser#text reads them.
+      def read_text(text)
+        text = Readers::References.line_ends(text) if text.include?("\r")
+        text.include?("&") ? Readers::References.decode(text) : text
+      end
+
+      # The +[kind, pad]+ of a number or date cell's style (SheetParser#style_kind): 0 without one.
+      def format(style)
+        return @meta.style_kind(0) if style.nil?
+
+        @style_formats[style] ||= begin
+          throw FALLBACK unless style.zero? || style.between?(1, @meta.style_count - 1)
+          @meta.style_kind(style)
+        end
+      end
+
+      # The shared strings read in Rust, as calcsheet's SharedStrings; calcsheet loads them itself when the Rust side
+      # does not read them, with the inflated total set back first.
+      def load_shared_strings
+        return super unless @zip.entry?(SHARED_STRINGS)
+
+        charged = @zip.instance_variable_get(TOTAL)
+        table = catch(FALLBACK) { read_shared_strings }
+        return table if table
+
+        @zip.instance_variable_set(TOTAL, charged)
+        super
+      end
+
+      def read_shared_strings
+        stream = guarded { @zip.stream(SHARED_STRINGS, ratio_floor: false) }
+        reader = XlsxStrings.new(@max_cell_bytes)
+        buffer = String.new(capacity: CHUNK)
+        chunks = 0
+        loop do
+          result = guarded { reader.read }
+          throw FALLBACK if result == false
+          return SharedStrings.new(result) if result
+
+          chunk = guarded { stream.read(CHUNK, buffer) }
+          chunk ? reader.feed(chunk) : reader.finish
+          # The inflater leaves its pieces as garbage, and this loop allocates too little for the GC to run: a
+          # minor GC every GC_CHUNKS keeps the peak memory at the Ruby backend's.
+          GC.start(full_mark: false) if (chunks += 1) % GC_CHUNKS == 0
+        end
+      ensure
+        stream&.close
+      end
+    end
+  end
+end

data/lib/calcsheet/native.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "calcsheet"
+require_relative "native/version"
+
+begin
+  # A precompiled platform gem holds one extension per Ruby minor version; a gem built from source holds one.
+  require_relative "native/#{RUBY_VERSION[/\A\d+\.\d+/]}/calcsheet_native"
+rescue LoadError
+  require_relative "native/calcsheet_native"
+end
+require_relative "native/csv_reader"
+require_relative "native/ods_reader"
+require_relative "native/xlsx_reader"
+
+module Calcsheet
+  # The native reader backends of calcsheet, which Calcsheet.open finds through +require "calcsheet/native"+ (see
+  # Calcsheet::NativeBackends in calcsheet).
+  module Native
+    # The version of the interface with calcsheet: Calcsheet::NativeBackends::PROTOCOLS lists those it accepts.
+    PROTOCOL = 1
+
+    # The formats read natively.
+    BACKENDS = { csv: CsvReader, xlsx: XlsxReader, ods: OdsReader }.freeze
+
+    # @return [Hash{Symbol => Class}] the backend class of each format read natively
+    def self.backends
+      BACKENDS
+    end
+  end
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: calcsheet-native
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Demetra Opinioni.net Srl
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: calcsheet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: rb_sys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.130
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.130
+description: |
+  calcsheet-native reads CSV, XLSX and ODS for calcsheet with a Rust core that runs without the GVL, and gives
+  the same rows as calcsheet's Ruby backends.
+email:
+- developers@opinioni.net
+executables: []
+extensions:
+- ext/calcsheet_native/extconf.rb
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Cargo.lock
+- Cargo.toml
+- LICENSE.txt
+- README.md
+- ext/calcsheet_native/Cargo.toml
+- ext/calcsheet_native/extconf.rb
+- ext/calcsheet_native/src/csv.rs
+- ext/calcsheet_native/src/gvl.rs
+- ext/calcsheet_native/src/lib.rs
+- ext/calcsheet_native/src/ods.rs
+- ext/calcsheet_native/src/xlsx.rs
+- ext/calcsheet_native/src/xml.rs
+- lib/calcsheet/native.rb
+- lib/calcsheet/native/csv_reader.rb
+- lib/calcsheet/native/ods_reader.rb
+- lib/calcsheet/native/version.rb
+- lib/calcsheet/native/xlsx_reader.rb
+homepage: https://gitlab.opinioni.net/demetra-developers/rubygems/calcsheet-native
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  source_code_uri: https://gitlab.opinioni.net/demetra-developers/rubygems/calcsheet-native/-/tree/main
+  bug_tracker_uri: https://gitlab.opinioni.net/demetra-developers/rubygems/calcsheet-native/-/issues
+  changelog_uri: https://gitlab.opinioni.net/demetra-developers/rubygems/calcsheet-native/-/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.16
+specification_version: 4
+summary: Native reader backends for calcsheet.
+test_files: []