calcsheet 0.1.0 → 0.2.0

data/lib/calcsheet/native_backends.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require_relative "errors"
+require_relative "version"
+
+module Calcsheet
+  # The reader backends of the optional calcsheet-native gem, which Calcsheet.open uses in place of the Ruby ones
+  # under +backend: :auto+ (when calcsheet-native is installed and compatible) and +backend: :native+ (or raises).
+  #
+  # calcsheet-native is found once per process, at the first open that asks for it: Calcsheet::Native when it is
+  # already defined, else +require "calcsheet/native"+ (a LoadError for that library itself means not installed; one
+  # for a file it requires, such as its extension, means it is unusable). Its +Calcsheet::Native::PROTOCOL+ must be an Integer of PROTOCOLS and +Calcsheet::Native.backends+ a
+  # Hash mapping formats among FORMATS to classes implementing the backend contract of Reading; otherwise, or when its
+  # code raises, it is unusable: the first +:auto+ open warns once and the Ruby backends read. A native class replaces
+  # only calcsheet's own reader: when an application registered another one in Calcsheet.backends for that format,
+  # the application's one reads.
+  #
+  # calcsheet-native depends on this interface (the protocol, +Calcsheet::Native.backends+ and the backend contract of
+  # Reading): change it only in a minor release and together with the protocol version.
+  #
+  # @api private
+  module NativeBackends
+    # The protocols of calcsheet-native this calcsheet supports.
+    PROTOCOLS = [ 1 ].freeze
+    # The library of calcsheet-native.
+    LIBRARY = "calcsheet/native"
+    # The formats calcsheet-native may read.
+    FORMATS = %i[csv xlsx ods].freeze
+    # What an open from the code of calcsheet-native finds while that code is being required: nothing yet.
+    LOADING = [ {}.freeze, "calcsheet-native is being loaded (an open from its own code)", nil ].freeze
+
+    @mutex = Mutex.new
+    @library = LIBRARY
+    @loaded = nil
+    @warning = nil
+
+    class << self
+      # @param format [Symbol] a key of Calcsheet.backends
+      # @param mode [Symbol] +:auto+ or +:native+
+      # @return [Class, nil] the native backend of +format+; nil when none applies under +:auto+
+      # @raise [BackendUnavailableError] under +:native+ when none applies
+      def backend_for(format, mode)
+        backends, reason, cause = loaded
+        reason ||= refusal(backends, format)
+        return backends.fetch(format) if reason.nil?
+        raise BackendUnavailableError, "backend: :native: #{reason}", cause: cause if mode == :native
+
+        warn_once
+        nil
+      end
+
+      # For specs: forgets what was loaded, so that the next open looks for calcsheet-native again, requiring
+      # +library+ if Calcsheet::Native is not defined.
+      # @param library [String]
+      # @return [void]
+      def reset!(library: LIBRARY)
+        @mutex.synchronize do
+          @library = library
+          @loaded = nil
+          @warning = nil
+        end
+      end
+
+      private
+
+      # @return [Array(Hash{Symbol => Class}, String, Exception)] the native backends by format and, when
+      #   calcsheet-native cannot be used, the reason (its backends are then empty) and the error that made it
+      #   unusable, if any
+      def loaded
+        return @loaded if @loaded
+        # The thread that is requiring calcsheet-native opens a book from its code: the lock cannot be taken again,
+        # and nothing is known yet (nor memoized).
+        return LOADING if @mutex.owned?
+
+        @mutex.synchronize { @loaded ||= load_native }
+      end
+
+      # Any error of calcsheet-native's code (its extension's initialization, a NameError, a failing backends) makes
+      # it unusable, once: it is not required again at the next open.
+      def load_native
+        native = native_module
+        return [ {}.freeze, "calcsheet-native is not installed (add it to the Gemfile)", nil ].freeze if native.nil?
+
+        # A Calcsheet::Native without an Integer PROTOCOL, or without backends, is not one this calcsheet can use.
+        protocol = native.const_get(:PROTOCOL, false) if native.const_defined?(:PROTOCOL, false)
+        unless protocol.is_a?(Integer) && PROTOCOLS.include?(protocol)
+          return unusable("calcsheet-native protocol #{protocol.inspect} is not supported by calcsheet #{VERSION} " \
+                          "(supported: #{PROTOCOLS.join(", ")})")
+        end
+
+        backends = native.backends if native.respond_to?(:backends)
+        unless backends.is_a?(Hash)
+          return unusable("calcsheet-native protocol #{protocol} does not answer Calcsheet::Native.backends with a Hash")
+        end
+
+        [ backends.slice(*FORMATS).freeze, nil, nil ].freeze
+      rescue StandardError, ScriptError => error # a LoadError for the library itself is not an error (#native_module)
+        unusable("calcsheet-native cannot be loaded (#{error.class}: #{error.message})", error)
+      end
+
+      # @return [Array(Hash, String, Exception)] no backends, +reason+ (which the first +:auto+ open warns of) and
+      #   +cause+
+      def unusable(reason, cause = nil)
+        @warning = "calcsheet: #{reason}; using the Ruby backends"
+        [ {}.freeze, reason, cause ].freeze
+      end
+
+      # @return [Module, nil] Calcsheet::Native; nil when the library itself cannot be found (calcsheet-native is not
+      #   installed) or defines no Calcsheet::Native
+      # @raise [LoadError] when the library is found but cannot be loaded (its extension or a dependency is missing)
+      def native_module
+        require @library unless Calcsheet.const_defined?(:Native, false)
+        Calcsheet::Native if Calcsheet.const_defined?(:Native, false)
+      rescue LoadError => error
+        raise unless error.path == @library # a file the library requires is missing: it is installed but unusable
+
+        nil
+      end
+
+      # @return [String, nil] why the loaded calcsheet-native does not read +format+; nil when it does
+      def refusal(backends, format)
+        return "calcsheet-native does not read #{format}" unless backends.key?(format)
+        return if Calcsheet.backends[format].equal?(RUBY_BACKENDS[format])
+
+        "Calcsheet.backends[#{format.inspect}] is not the reader calcsheet-native replaces"
+      end
+
+      # Warns of an unusable calcsheet-native, at the first open that falls back to the Ruby backends.
+      def warn_once
+        return if @warning.nil?
+
+        @mutex.synchronize do
+          warn(@warning) if @warning # another thread may have warned meanwhile
+          @warning = nil
+        end
+      end
+    end
+  end
+end

data/lib/calcsheet/option_validation.rb

@@ -17,7 +17,9 @@ module Calcsheet
     # The zip-bomb limits: always on, so that a blank configuration value cannot switch them off.
     ZIP_LIMITS = %i[max_uncompressed_size max_zip_entries].freeze
     # The options that are true or false.
-    BOOLEANS = %i[strip_formula_prefix excel_it sanitize_sheet_names].freeze
+    BOOLEANS = %i[strip_formula_prefix excel_it sanitize_sheet_names precheck_sheets].freeze
+    # The values of +backend:+.
+    BACKENDS = %i[auto ruby native].freeze
     # The decimal separators of text read as numbers and of CSV output.
     DECIMAL_SEPARATORS = [ ".", "," ].freeze
     # The thousands separators: none, ".", ",", a space, an apostrophe, the no-break space U+00A0 and the narrow
@@ -44,6 +46,7 @@ module Calcsheet
       positive(:batch_size, options[:batch_size])
       non_negative(:sample_rows, options[:sample_rows], nil_allowed: false)
       one_of(:decimals, options[:decimals], %i[bigdecimal float])
+      one_of(:backend, options[:backend], BACKENDS)
       TimeZone.wrap(options[:time_zone])
       separators(options[:decimal_separator], options[:thousands_separator])
       %i[date_formats datetime_formats].each { |key| formats(key, options[key]) }

data/lib/calcsheet/raw_row.rb

@@ -5,7 +5,7 @@ module Calcsheet
   #
   # @attr_reader number [Integer] the row number the user sees
   # @attr_reader line [Integer, nil] the physical line where it starts (CSV); nil otherwise
-  # @attr_reader values [Array] in file order, already cast, with trailing blank cells trimmed
+  # @attr_reader values [Array] in file order, already cast, trimmed at the end (see Reading)
   # @attr_reader errors [Array<Array(Integer, String, Symbol)>, nil] nil, or +[column_index, raw_value, reason]+ for
   #   each cell whose cast failed (its value is nil), in column order
   RawRow = Data.define(:number, :line, :values, :errors) do

data/lib/calcsheet/readers/csv_reader.rb

@@ -2,14 +2,14 @@
 
 require "csv"
 require "stringio"
-require_relative "../cast"
 require_relative "../errors"
 require_relative "../formula_protection"
-require_relative "../raw_row"
 require_relative "../reading"
+require_relative "../row_builder"
 require_relative "../sheet_info"
 require_relative "csv_dialect"
 require_relative "csv_input"
+require_relative "csv_records"
 require_relative "row_batches"
 
 module Calcsheet
@@ -20,7 +20,8 @@ module Calcsheet
   # @api private
   module Readers
     # The CSV reader backend (see Reading for the contract): one sheet, named after the file without its extension
-    # ("csv" for an IO without a path), parsed by the stdlib CSV in the detected dialect.
+    # ("csv" for an IO without a path), parsed in the detected dialect by CsvRecords, or by the stdlib CSV when the
+    # sample showed no line ending.
     #
     # Rows are numbered by CSV record, counting the header and blank lines, and +line+ is the physical line where
     # the record starts. A seekable source is read again from its start whenever a request needs a record
@@ -78,6 +79,13 @@ module Calcsheet
       end
 
       # Drops the parser and the sample; the book closes the source.
+      # The row that would follow the last record, once a read reached the end of the file: the blank records at the
+      # end are rows of the file (see Reading).
+      # @return [Array(Integer, Integer)] its number and line
+      def end_row(_sheet_index)
+        [ @records + 1, @line ]
+      end
+
       def close
         @csv = nil
         @rows = nil
@@ -95,6 +103,7 @@ module Calcsheet
         # forced dialect or a parser that failed starts again from the first record.
         restart(forced) if @csv.nil? || start <= @records || forced != @forced
         strip = cast.options.strip_formula_prefix
+        builder = RowBuilder.new(cast, types)
         while (fields = shift)
           number = @records += 1
           line = @line
@@ -108,8 +117,7 @@ module Calcsheet
           last -= 1 while last >= 0 && blank_field?(fields[last], number, last)
           next if last.negative?
 
-          fields = fields.first(last + 1) if last + 1 < fields.size
-          yield raw_row(number, line, fields, cast, types, strip)
+          yield raw_row(builder, number, line, fields, last, strip)
         end
       end
 
@@ -125,13 +133,24 @@ module Calcsheet
           @forced = forced
         end
         @started = true
-        @csv = CSV.new(input, col_sep: @dialect.col_sep, quote_char: @dialect.quote_char, row_sep: @dialect.row_sep)
+        @csv = records(input)
         @stray = nil
         @rows = nil
         @records = 0
         @line = 1
       end
 
+      # The parser of the records: CsvRecords for a line ending the sample showed, the stdlib parser, which finds the
+      # line ending itself, otherwise.
+      def records(input)
+        dialect = @dialect
+        if dialect.row_sep.is_a?(String)
+          CsvRecords.new(input, col_sep: dialect.col_sep, quote_char: dialect.quote_char, row_sep: dialect.row_sep)
+        else
+          CSV.new(input, col_sep: dialect.col_sep, quote_char: dialect.quote_char, row_sep: dialect.row_sep)
+        end
+      end
+
       # The text the parser reads: the source after the byte order mark, or the sample (consumed here) and the rest of
       # the stream.
       def input
@@ -204,25 +223,29 @@ module Calcsheet
         raise ArgumentError, "encoding: cannot read #{value.inspect} (#{error.message})"
       end
 
-      def raw_row(number, line, fields, cast, types, strip)
-        errors = nil
-        # Array.new with a block rather than each_with_index.map, which allocates an Enumerator per row.
-        values = Array.new(fields.size) do |index|
+      # The row of a record whose last non-blank field is at +last+: the builder casts the fields, so that the row
+      # equals those of the other readers. A field is checked and unprotected first, when a limit or
+      # +strip_formula_prefix+ asks for it.
+      def raw_row(builder, number, line, fields, last, strip)
+        prepare(fields, last, number, strip) if strip || @max_cell_bytes
+        builder.text_row(fields, last, number, line: line)
+      end
+
+      # Checks the size of the fields up to +last+ and removes the formula protection from them, in place.
+      def prepare(fields, last, number, strip)
+        index = 0
+        while index <= last
           field = fields[index]
-          next if field.nil? || field.empty?
-
-          check_size(field, number, index) if @max_cell_bytes && field.bytesize > @max_cell_bytes
-          field = FormulaProtection.unprotect(field) if strip
-          cast.text(field, types&.[](index))
-        rescue Cast::Failure => failure
-          (errors ||= []) << [ index, field, failure.reason ]
-          nil
+          unless field.nil?
+            check_size(field, number, index) if @max_cell_bytes && field.bytesize > @max_cell_bytes
+            fields[index] = FormulaProtection.unprotect(field) if strip
+          end
+          index += 1
         end
-        RawRow.new(number: number, line: line, values: values, errors: errors)
       end
 
       # Every field is measured before a regexp sees it: here those from the end to the last non-blank one, the
-      # others in raw_row.
+      # others in #prepare.
       def blank_field?(field, number, index)
         return true if field.nil?
 

data/lib/calcsheet/readers/csv_records.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Calcsheet
+  module Readers
+    # The records of a CSV text, as the stdlib CSV parser gives them with +liberal_parsing: false+, for a known line
+    # ending: the same fields (nil for an empty unquoted one, "" for an empty quoted one), the same records ([] for an
+    # empty line) and a CSV::MalformedCSVError with the same message where it refuses a record. The stdlib parser is
+    # general (converters, headers, skipped lines...) and costs about half of a CSV read; a record without the quote
+    # character is split in one call here, and one with it is scanned field by field.
+    #
+    # It stands in for the CSV object of CsvReader: #each gives an object whose #next is the next record, #line the
+    # raw text of the record #next returned, #row_sep the line ending.
+    #
+    # @api private
+    class CsvRecords
+      # The text read from the input at a time, so that a long record is gathered in linear time.
+      CHUNK_BYTES = 64 * 1024
+
+      # @return [String] the line ending
+      attr_reader :row_sep
+      # @return [String, nil] the raw text of the last record, line ending included
+      attr_reader :line
+
+      # @param input [#gets] UTF-8 text (CsvInput)
+      # @param col_sep [String] the separator
+      # @param quote_char [String] the quote character
+      # @param row_sep [String] "\n", "\r\n" or "\r"
+      def initialize(input, col_sep:, quote_char:, row_sep:)
+        @input = input
+        @col_sep = col_sep
+        @quote = quote_char
+        @row_sep = row_sep
+        # The line break that is not the line ending, which an unquoted field keeps (none in a CRLF text).
+        @other_break = { "\n" => "\r", "\r" => "\n" }[row_sep]
+        @records = 0
+        @line = nil
+        @eof = false
+      end
+
+      # @return [CsvRecords] self: #next gives the records
+      def each
+        self
+      end
+
+      # @return [Array<String, nil>] the fields of the next record
+      # @raise [StopIteration] at the end
+      # @raise [CSV::MalformedCSVError] for a record the stdlib parser refuses
+      # @raise [EncodingError] from the input
+      def next
+        text = read_line or raise StopIteration
+        fields = plain(text) || quoted(text)
+        @records += 1
+        fields
+      end
+
+      private
+
+      # The next line, ending with the line ending unless the text ends first; nil at the end.
+      def read_line
+        return if @eof
+
+        line = @input.gets(@row_sep, CHUNK_BYTES)
+        return @eof = nil if line.nil?
+
+        until line.end_with?(@row_sep)
+          more = @input.gets(@row_sep, CHUNK_BYTES)
+          if more.nil?
+            @eof = true
+            break
+          end
+          line << more
+        end
+        line
+      end
+
+      # The fields of a line without the quote character or a line break inside it (but the other line break, which
+      # an unquoted field keeps): a split. nil for another line.
+      def plain(text)
+        return if text.include?(@quote)
+
+        body = text.end_with?(@row_sep) ? text.byteslice(0, text.bytesize - @row_sep.bytesize) : text
+        return if @other_break.nil? ? body.include?("\r") || body.include?("\n") : body.include?(@row_sep)
+
+        @line = text
+        fields = body.split(@col_sep, -1)
+        fields.map! { _1.empty? ? nil : _1 }
+      end
+
+      # The fields of a record that holds the quote character or a stray line break, scanned field by field, reading
+      # further lines while a quoted field is open.
+      def quoted(text)
+        loop do
+          result = parse(text)
+          case result
+          when Array
+            @line = text
+            return result
+          when String then raise CSV::MalformedCSVError.new(result, @records + 1)
+          end
+
+          more = read_line
+          return finish_unclosed(text) if more.nil?
+
+          text << more
+        end
+      end
+
+      # A quoted field still open at the end of the text.
+      def finish_unclosed(text)
+        @eof = true
+        result = parse(text, at_end: true)
+        raise CSV::MalformedCSVError.new(result, @records + 1) if result.is_a?(String)
+
+        @line = text
+        result
+      end
+
+      # The record that +text+ holds: its fields, the stdlib's message for a malformed one, or :more when a quoted
+      # field is still open and the text may go on. The text ends with the line ending, or is the end of the input.
+      def parse(text, at_end: @eof)
+        size = text.bytesize
+        ending = text.end_with?(@row_sep) ? size - @row_sep.bytesize : size
+        fields = []
+        position = 0
+        return fields if ending.zero? && size.positive?
+
+        loop do
+          if at?(text, position, @quote)
+            close = position + @quote.bytesize
+            value = nil
+            loop do
+              found = text.byteindex(@quote, close)
+              return at_end ? "Unclosed quoted field" : :more if found.nil?
+
+              piece = text.byteslice(close, found - close)
+              after = found + @quote.bytesize
+              if at?(text, after, @quote)
+                # A doubled quote: the field is joined from its pieces.
+                (value ||= +"") << piece << @quote
+                close = after + @quote.bytesize
+                next
+              end
+              # The closing quote: what follows it must end the field or the record.
+              fields << (value ? value << piece : piece)
+              return fields if after == size || after == ending
+              return "Any value after quoted field isn't allowed" unless at?(text, after, @col_sep)
+
+              position = after + @col_sep.bytesize
+              break
+            end
+            next
+          end
+
+          stop = text.byteindex(@col_sep, position)
+          stop = ending if stop.nil? || stop > ending
+          field = text.byteslice(position, stop - position)
+          if (message = refusal(field))
+            return message
+          end
+
+          fields << (field.empty? ? nil : field)
+          return fields if stop == ending
+
+          position = stop + @col_sep.bytesize
+        end
+      end
+
+      # Whether +token+ is at +position+ of +text+; a one-byte token (the usual separator and quote) is compared as a
+      # byte, without a slice.
+      def at?(text, position, token)
+        return text.getbyte(position) == token.getbyte(0) if token.bytesize == 1
+
+        text.byteslice(position, token.bytesize) == token
+      end
+
+      # The stdlib's message for an unquoted field it refuses, by the first character it stops at: a quote, or in a
+      # CRLF text a line break of another ending (named a wrong line ending when it starts the field).
+      def refusal(field)
+        quote = field.index(@quote)
+        line_break = field.index(/[\r\n]/) if @other_break.nil?
+        return if quote.nil? && line_break.nil?
+        return "Illegal quoting" if line_break.nil? || (quote && quote < line_break)
+
+        shown = field[line_break] == "\n" ? "\\n" : "\\r"
+        return %(New line must be <"\\r\\n"> not <"#{shown}">) if line_break.zero?
+
+        %(Unquoted fields do not allow new line <"#{shown}">)
+      end
+    end
+  end
+end