calcsheet-native 0.1.0

data/ext/calcsheet_native/src/xml.rs

@@ -0,0 +1,370 @@
+//! The plain subset of XML the sheet readers handle: elements, attributes and text, a leading XML declaration, well
+//! formed. Anything else (a comment, CDATA, a DOCTYPE, another processing instruction, a reference, "<" or a line
+//! break in an attribute value, an odd name, a tag longer than calcsheet's scanners accept) is `Unsupported`, and
+//! the Ruby side then reads the sheet with calcsheet's own parser.
+
+use memchr::memchr;
+
+/// TagScan::MAX_TAG_BYTES: calcsheet's scanners refuse a longer tag before the row a read starts from.
+const MAX_TAG_BYTES: usize = 65_536;
+/// Deeper XML is left to the Ruby parser.
+const MAX_DEPTH: usize = 64;
+
+/// Why the subset reader gives up: the Ruby parser decides.
+pub struct Unsupported;
+
+pub type Result<T> = std::result::Result<T, Unsupported>;
+
+pub enum Item {
+    /// A start tag: the span of its raw name in the buffer, and true for an empty element (`<a/>`), which ends right
+    /// after. Its attributes are in `Lexer::attrs` until the next item.
+    Start(usize, usize, bool),
+    /// An end tag: the span of its raw name.
+    End(usize, usize),
+    Text(usize, usize),
+}
+
+pub struct Lexer {
+    pub buf: Vec<u8>,
+    pub pos: usize,
+    pub eof: bool,
+    seen_root: bool,
+    started: bool,
+    // The raw names of the open elements, end to end.
+    stack: Vec<u8>,
+    stack_ends: Vec<usize>,
+    /// The attributes of the start tag just read: name and value spans in `buf`.
+    pub attrs: Vec<(usize, usize, usize, usize)>,
+    // The end tag just read: where it starts and its name, for #unread_end.
+    last_end: Option<(usize, usize, usize)>,
+}
+
+impl Lexer {
+    pub fn new() -> Self {
+        Lexer {
+            buf: Vec::new(),
+            pos: 0,
+            eof: false,
+            seen_root: false,
+            started: false,
+            stack: Vec::new(),
+            stack_ends: Vec::new(),
+            attrs: Vec::new(),
+            last_end: None,
+        }
+    }
+
+    /// Appends text, dropping what was read. Spans of earlier items are invalid afterwards.
+    pub fn feed(&mut self, chunk: &[u8]) {
+        if self.pos > 0 {
+            self.buf.drain(..self.pos);
+            self.pos = 0;
+        }
+        self.buf.extend_from_slice(chunk);
+    }
+
+    /// Whether the text ends after a complete document, as an Ox parse that meets no end of its own does.
+    pub fn complete(&self) -> bool {
+        self.stack_ends.is_empty()
+            && self.seen_root
+            && self.buf[self.pos..].iter().all(u8::is_ascii_whitespace)
+    }
+
+    /// The next item, `None` when the text fed so far ends inside it.
+    pub fn item(&mut self) -> Result<Option<Item>> {
+        loop {
+            let n = self.buf.len();
+            let start = self.pos;
+            if start == n {
+                return Ok(None);
+            }
+            if self.buf[start] != b'<' {
+                let end = memchr(b'<', &self.buf[start..]).map_or(n, |k| start + k);
+                if end == n && !self.eof {
+                    return Ok(None);
+                }
+                if self.stack_ends.is_empty()
+                    && !self.buf[start..end].iter().all(u8::is_ascii_whitespace)
+                {
+                    return Err(Unsupported);
+                }
+                self.pos = end;
+                self.started = true;
+                return Ok(Some(Item::Text(start, end)));
+            }
+            let Some(close) = memchr(b'>', &self.buf[start..]).map(|k| start + k) else {
+                return if self.eof || n - start >= MAX_TAG_BYTES {
+                    Err(Unsupported)
+                } else {
+                    Ok(None)
+                };
+            };
+            if close - start >= MAX_TAG_BYTES {
+                return Err(Unsupported);
+            }
+            match self.buf.get(start + 1).copied() {
+                Some(b'?') => {
+                    // Only the XML declaration, first in the document.
+                    let tag = &self.buf[start + 1..close];
+                    if self.started || !tag.starts_with(b"?xml ") || !tag.ends_with(b"?") {
+                        return Err(Unsupported);
+                    }
+                    self.started = true;
+                    self.pos = close + 1;
+                }
+                Some(b'!') => return Err(Unsupported),
+                Some(b'/') => {
+                    let name_start = start + 2;
+                    let name_end = name_start + trim_end(&self.buf[name_start..close]).len();
+                    let open_start = match self.stack_ends.len() {
+                        0 => return Err(Unsupported),
+                        1 => 0,
+                        depth => self.stack_ends[depth - 2],
+                    };
+                    if self.stack[open_start..] != self.buf[name_start..name_end] {
+                        return Err(Unsupported);
+                    }
+                    self.stack.truncate(open_start);
+                    self.stack_ends.pop();
+                    self.started = true;
+                    self.pos = close + 1;
+                    self.last_end = Some((start, name_start, name_end));
+                    return Ok(Some(Item::End(name_start, name_end)));
+                }
+                _ => {
+                    let empty = self.buf[close - 1] == b'/' && close - 1 > start;
+                    let body_end = if empty { close - 1 } else { close };
+                    let name_start = start + 1;
+                    let name_end = (name_start..body_end)
+                        .find(|&i| is_space(self.buf[i]))
+                        .unwrap_or(body_end);
+                    name(&self.buf[name_start..name_end])?;
+                    self.attrs.clear();
+                    attributes(&self.buf, name_end, body_end, &mut self.attrs)?;
+                    if empty {
+                        if self.stack_ends.is_empty() {
+                            return Err(Unsupported);
+                        }
+                    } else {
+                        if self.stack_ends.len() >= MAX_DEPTH {
+                            return Err(Unsupported);
+                        }
+                        self.stack
+                            .extend_from_slice(&self.buf[name_start..name_end]);
+                        self.stack_ends.push(self.stack.len());
+                    }
+                    self.seen_root = true;
+                    self.started = true;
+                    self.pos = close + 1;
+                    return Ok(Some(Item::Start(name_start, name_end, empty)));
+                }
+            }
+        }
+    }
+
+    /// Reads the end tag just read again at the next #item: its element is open again.
+    pub fn unread_end(&mut self) {
+        if let Some((start, name_start, name_end)) = self.last_end.take() {
+            self.stack
+                .extend_from_slice(&self.buf[name_start..name_end]);
+            self.stack_ends.push(self.stack.len());
+            self.pos = start;
+        }
+    }
+
+    /// The attributes of the start tag just read, refused when one is repeated (Ox would read them its own way).
+    pub fn check_unique(&self) -> Result<()> {
+        for (index, &(start, end, _, _)) in self.attrs.iter().enumerate() {
+            let name = &self.buf[start..end];
+            if self.attrs[..index]
+                .iter()
+                .any(|&(s, e, _, _)| &self.buf[s..e] == name)
+            {
+                return Err(Unsupported);
+            }
+        }
+        Ok(())
+    }
+}
+
+pub fn is_space(c: u8) -> bool {
+    matches!(c, b' ' | b'\t' | b'\n' | b'\r')
+}
+
+fn trim_end(bytes: &[u8]) -> &[u8] {
+    let end = bytes
+        .iter()
+        .rposition(|&c| !is_space(c))
+        .map_or(0, |k| k + 1);
+    &bytes[..end]
+}
+
+/// The bytes of a name, and those an attribute value of the subset may not hold, as lookup tables: one load per
+/// byte instead of a chain of comparisons.
+const NAME_BYTES: [bool; 256] =
+    byte_table(b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-.:");
+const NOT_IN_VALUES: [bool; 256] = byte_table(b"<&\r\n\t");
+
+const fn byte_table(bytes: &[u8]) -> [bool; 256] {
+    let mut table = [false; 256];
+    let mut index = 0;
+    while index < bytes.len() {
+        table[bytes[index] as usize] = true;
+        index += 1;
+    }
+    table
+}
+
+/// A name of ASCII letters, digits, "_", "-", "." and at most one ":" between a prefix and a local name.
+pub fn name(raw: &[u8]) -> Result<()> {
+    let Some(&first) = raw.first() else {
+        return Err(Unsupported);
+    };
+    if !(first.is_ascii_alphabetic() || first == b'_') {
+        return Err(Unsupported);
+    }
+    let mut colon = None;
+    for (index, &c) in raw.iter().enumerate() {
+        if !NAME_BYTES[c as usize] {
+            return Err(Unsupported);
+        }
+        if c == b':' {
+            if colon.is_some() {
+                return Err(Unsupported);
+            }
+            colon = Some(index);
+        }
+    }
+    match colon {
+        Some(index) if index + 1 == raw.len() => Err(Unsupported),
+        _ => Ok(()),
+    }
+}
+
+/// The name without its namespace prefix, as Ox's `strip_namespace` gives it.
+pub fn local(raw: &[u8]) -> &[u8] {
+    memchr(b':', raw).map_or(raw, |colon| &raw[colon + 1..])
+}
+
+/// The attributes of a start tag in `buf[from..to]`: `name="value"` or `name='value'`, separated by white space,
+/// the values without "<", "&", line breaks or tabs, which Ox would read in ways of its own. Spans are appended to
+/// `attrs`.
+fn attributes(
+    buf: &[u8],
+    mut at: usize,
+    to: usize,
+    attrs: &mut Vec<(usize, usize, usize, usize)>,
+) -> Result<()> {
+    loop {
+        let skipped = (at..to).find(|&i| !is_space(buf[i])).unwrap_or(to);
+        if skipped == to {
+            return Ok(());
+        }
+        if skipped == at {
+            return Err(Unsupported);
+        }
+        at = skipped;
+        let eq = memchr(b'=', &buf[at..to])
+            .map(|k| at + k)
+            .ok_or(Unsupported)?;
+        name(&buf[at..eq])?;
+        let quote = *buf.get(eq + 1).filter(|_| eq + 1 < to).ok_or(Unsupported)?;
+        if quote != b'"' && quote != b'\'' {
+            return Err(Unsupported);
+        }
+        let value_start = eq + 2;
+        let value_end = memchr(quote, &buf[value_start..to])
+            .map(|k| value_start + k)
+            .ok_or(Unsupported)?;
+        let value = &buf[value_start..value_end];
+        if value.iter().any(|&c| NOT_IN_VALUES[c as usize]) {
+            return Err(Unsupported);
+        }
+        attrs.push((at, eq, value_start, value_end));
+        at = value_end + 1;
+    }
+}
+
+/// Valid UTF-8, or unsupported.
+pub fn utf8(bytes: &[u8]) -> Result<&[u8]> {
+    std::str::from_utf8(bytes)
+        .map(|_| bytes)
+        .map_err(|_| Unsupported)
+}
+
+/// References.decode: every "&" starts a reference to a predefined entity or to a code point other than NUL, a
+/// surrogate or one beyond U+10FFFF; anything else is left to calcsheet, which refuses it.
+pub fn decode(text: &[u8]) -> Result<Vec<u8>> {
+    let mut decoded = Vec::with_capacity(text.len());
+    let mut at = 0;
+    while let Some(offset) = memchr(b'&', &text[at..]) {
+        let amp = at + offset;
+        decoded.extend_from_slice(&text[at..amp]);
+        let rest = &text[amp + 1..];
+        let semicolon = memchr(b';', rest).ok_or(Unsupported)?;
+        let body = &rest[..semicolon];
+        match body {
+            b"lt" => decoded.push(b'<'),
+            b"gt" => decoded.push(b'>'),
+            b"amp" => decoded.push(b'&'),
+            b"quot" => decoded.push(b'"'),
+            b"apos" => decoded.push(b'\''),
+            _ => {
+                let code = reference(body).ok_or(Unsupported)?;
+                let character = char::from_u32(code)
+                    .filter(|_| code != 0)
+                    .ok_or(Unsupported)?;
+                let mut bytes = [0; 4];
+                decoded.extend_from_slice(character.encode_utf8(&mut bytes).as_bytes());
+            }
+        }
+        at = amp + 1 + semicolon + 1;
+    }
+    decoded.extend_from_slice(&text[at..]);
+    Ok(decoded)
+}
+
+/// The code point of `#0*[0-9]{1,7}` or `#x0*[0-9a-fA-F]{1,6}`.
+fn reference(body: &[u8]) -> Option<u32> {
+    let (digits, radix, most) = match body {
+        [b'#', b'x', rest @ ..] => (rest, 16, 6),
+        [b'#', rest @ ..] => (rest, 10, 7),
+        _ => return None,
+    };
+    let significant = &digits[digits
+        .iter()
+        .position(|&c| c != b'0')
+        .unwrap_or(digits.len().saturating_sub(1))..];
+    if digits.is_empty() || significant.is_empty() || significant.len() > most {
+        return None;
+    }
+    let text = std::str::from_utf8(significant).ok()?;
+    if !text.bytes().all(|c| {
+        if radix == 16 {
+            c.is_ascii_hexdigit()
+        } else {
+            c.is_ascii_digit()
+        }
+    }) {
+        return None;
+    }
+    u32::from_str_radix(text, radix).ok()
+}
+
+/// References.line_ends: CR LF and a CR alone read as LF.
+pub fn line_ends(text: &[u8]) -> Vec<u8> {
+    let mut lines = Vec::with_capacity(text.len());
+    let mut index = 0;
+    while index < text.len() {
+        if text[index] == b'\r' {
+            lines.push(b'\n');
+            if text.get(index + 1) == Some(&b'\n') {
+                index += 1;
+            }
+        } else {
+            lines.push(text[index]);
+        }
+        index += 1;
+    }
+    lines
+}

data/lib/calcsheet/native/csv_reader.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Calcsheet
+  module Native
+    # The native CSV backend: calcsheet's Ruby reader (dialect detection, encodings, numbering, checks and the
+    # conversion of the fields) with the records tokenized in Rust instead of by the stdlib CSV parser. A dialect the
+    # tokenizer does not handle (a separator or quote that is not one ASCII character, or a line ending the sample
+    # did not show) is parsed by the stdlib, as the Ruby backend does.
+    #
+    # It depends on the private methods of Calcsheet::Readers::CsvReader of the calcsheet minor version it supports.
+    class CsvReader < Calcsheet::Readers::CsvReader
+      # The line endings the tokenizer handles.
+      ROW_SEPARATORS = [ "\n", "\r", "\r\n" ].freeze
+
+      # The stdlib CSV parser's place for the reader: CsvReader#shift pulls records with +each+ and +next+, and
+      # reads +row_sep+ when a record is malformed.
+      class Records
+        # Records parsed per call into Rust.
+        BATCH = 1024
+        EMPTY = [].freeze
+
+        # @return [String] the line ending
+        attr_reader :row_sep
+        # @return [Integer] the physical lines the last record ends
+        attr_reader :breaks
+        # @return [Boolean] whether the last record has the stray line break outside quoted fields
+        attr_reader :stray
+
+        # @param input [Calcsheet::Readers::CsvInput]
+        # @param dialect [Calcsheet::Readers::CsvDialect::Dialect]
+        def initialize(input, dialect)
+          @input = input
+          @row_sep = dialect.row_sep
+          @tokenizer = CsvTokenizer.new(dialect.col_sep, dialect.quote_char, dialect.row_sep)
+          @fields = @all_breaks = @strays = EMPTY
+          @index = 0
+          @breaks = 0
+          @stray = false
+        end
+
+        # @return [Records] self: #next gives the records
+        def each
+          self
+        end
+
+        # @return [Array<String, nil>] the fields of the next record
+        # @raise [StopIteration] at the end
+        # @raise [CSV::MalformedCSVError] for a malformed record
+        # @raise [EncodingError] from the input, at the record that holds the invalid bytes
+        def next
+          if @index == @fields.size
+            @fields, @all_breaks, @strays = batch
+            @index = 0
+            raise StopIteration if @fields.empty?
+          end
+          index = @index
+          @index += 1
+          @breaks = @all_breaks[index]
+          @stray = @strays[index]
+          @fields[index]
+        end
+
+        private
+
+        # The next records, reading the input only when the text read ends inside a record.
+        def batch
+          loop do
+            records = @tokenizer.records(BATCH)
+            return records if records
+            raise CSV::MalformedCSVError.new(*@tokenizer.error) if records == false
+
+            chunk = @input.gets(nil, Calcsheet::Readers::CsvInput::CHUNK_SIZE)
+            chunk ? @tokenizer.feed(chunk) : @tokenizer.finish
+          end
+        end
+      end
+
+      private
+
+      # Parses with the tokenizer when it handles the dialect.
+      def restart(forced)
+        super
+        @csv = Records.new(@input, @dialect) if tokenizes?(@dialect)
+      end
+
+      # The text the parser reads, kept for the tokenizer.
+      def input
+        @input = super
+      end
+
+      def tokenizes?(dialect)
+        ROW_SEPARATORS.include?(dialect.row_sep) && [ dialect.col_sep, dialect.quote_char ].all? { _1.bytesize == 1 }
+      end
+
+      # CsvReader#each_row, with the line breaks and the stray line break of each record from the tokenizer instead
+      # of its raw text.
+      def each_row(sheet_index, cast, types, start, &block)
+        forced = forced_dialect(cast.options)
+        restart(forced) if @csv.nil? || start <= @records || forced != @forced
+        return super unless @csv.is_a?(Records)
+
+        strip = cast.options.strip_formula_prefix
+        builder = RowBuilder.new(cast, types)
+        while (fields = shift)
+          number = @records += 1
+          line = @line
+          @line += @csv.breaks
+          mixed_line_endings(number, line, @csv.row_sep) if @csv.stray
+          next if number < start
+
+          last = fields.size - 1
+          last -= 1 while last >= 0 && blank_field?(fields[last], number, last)
+          next if last.negative?
+
+          yield raw_row(builder, number, line, fields, last, strip)
+        end
+      end
+    end
+  end
+end

data/lib/calcsheet/native/ods_reader.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module Calcsheet
+  module Native
+    # The native ODS backend: calcsheet's Ruby reader (the zip archive and its limits, the Scanner that finds the
+    # tables, the rows of +start_at:+ and the cell styles) with the XML of each table read in Rust instead of by Ox.
+    # The Rust side reads a plain subset of XML and the values calcsheet accepts; on anything else (a comment, CDATA,
+    # a reference in an attribute, a value calcsheet refuses, an error of the zip stream...) the table is read again
+    # by calcsheet's own parser, which gives its exact rows or error. The rows already yielded are not yielded again,
+    # and the inflated total of the container and what the Scanner learned are set back to their state before the
+    # first attempt, so that the second one charges and caches exactly as calcsheet does.
+    #
+    # It depends on the private methods of Calcsheet::Readers::OdsReader and the internals of ZipContainer and
+    # Ods::Scanner of the calcsheet minor version it supports.
+    class OdsReader < Calcsheet::Readers::OdsReader
+      # Thrown when calcsheet's parser must read the table.
+      FALLBACK = :calcsheet_native_fallback
+      # Rows per call into Rust, and bytes read from the stream at a time.
+      ROWS = 512
+      CHUNK = 64 * 1024
+      # The RowBuilder kinds of OdsTable's cells, and the reasons of a missing value.
+      KINDS = %i[text number iso_date duration boolean error missing].freeze
+      MISSING = %i[invalid_decimal invalid_date invalid_boolean].freeze
+      # The inflated total of ZipContainer.
+      TOTAL = :@inflated_total
+
+      @fallbacks = 0
+
+      class << self
+        # @return [Integer] the table reads handed to calcsheet's parser in this process, for specs and diagnostics
+        attr_accessor :fallbacks
+      end
+
+      private
+
+      def each_row(index, cast, types, start_row, &block)
+        charged = @zip.instance_variable_get(TOTAL)
+        learned = @scanner.instance_variables.to_h { [ _1, @scanner.instance_variable_get(_1) ] }
+        last = 0
+        finished = catch(FALLBACK) do
+          read_natively(index, cast, types, start_row) do |row|
+            last = row.number
+            yield row
+          end
+          true
+        end
+        return if finished
+
+        self.class.fallbacks += 1
+        @zip.instance_variable_set(TOTAL, charged)
+        (@scanner.instance_variables - learned.keys).each { @scanner.remove_instance_variable(_1) }
+        learned.each { |name, value| @scanner.instance_variable_set(name, value) }
+        super(index, cast, types, start_row) { |row| yield row if row.number > last }
+      end
+
+      # Yields the rows read by OdsTable, as ContentParser yields them; throws FALLBACK when calcsheet's parser must
+      # read the table. The errors of the caller's block are its own.
+      def read_natively(index, cast, types, start_row)
+        guarded { sheet_at(index) }
+        io = guarded { @scanner.io_at_sheet(index, row: start_row) } or return
+        table = guarded do
+          prefixes = @scanner.prefixes
+          @formats = @scanner.cell_formats
+          OdsTable.new(prefixes.values_at(:table, :office, :text, :calcext), @formats.keys, @max_cell_bytes,
+                       io.first_row, start_row)
+        end
+        @format_list = @formats.values
+        @column_formats = []
+        builder = RowBuilder.new(cast, types, number_errors: :raw)
+        # Read into one String: a new one per chunk would leave the whole table as garbage before the GC runs.
+        buffer = String.new(capacity: CHUNK)
+        loop do
+          batch = guarded { table.rows(ROWS) }
+          throw FALLBACK if batch == false
+          if batch.nil?
+            chunk = guarded { io.read(CHUNK, buffer) }
+            chunk ? table.feed(chunk) : table.finish
+            next
+          end
+
+          firsts, lasts, ends, cells, @column_formats, done = batch
+          from = 0
+          firsts.each_with_index do |number, position|
+            to = ends[position]
+            guarded { add_cells(builder, cells, from, to) }
+            from = to
+            last = lasts[position]
+            row = builder.finish(last)
+            next if row.nil? || last < start_row
+
+            [ number, start_row ].max.upto(last - 1) { |each| yield RawRow.new(number: each, values: row.values.dup, errors: row.errors&.dup) }
+            yield row
+          end
+          break if done
+        end
+      ensure
+        io&.close
+      end
+
+      # Runs the block, throwing FALLBACK for any error it raises: calcsheet's parser raises it again, where it
+      # would.
+      def guarded
+        yield
+      rescue StandardError
+        throw FALLBACK
+      end
+
+      # Adds the cells +from...to+ of a batch to the builder, as ContentParser#finish_cell does.
+      def add_cells(builder, cells, from, to)
+        while from < to
+          column = cells[from]
+          repeat = cells[from + 1]
+          kind = KINDS[cells[from + 2]]
+          missing = MISSING[cells[from + 3]] if kind == :missing
+          raw = cells[from + 4]
+          style = cells[from + 5]
+          if repeat == 1
+            builder.add(column, kind, raw, format_of(kind, style, column), missing: missing)
+          elsif kind == :number || kind == :iso_date
+            builder.add_run(column, column + repeat - 1, kind, raw, missing: missing) { format_of(kind, style, _1) }
+          else
+            builder.add_run(column, column + repeat - 1, kind, raw, missing: missing)
+          end
+          from += 6
+        end
+      end
+
+      # ContentParser#format_of: a number takes only the padding of a zero-padding style, a date its style's format.
+      def format_of(kind, style, column)
+        return unless kind == :number || kind == :iso_date
+
+        format = format_at(style, column)
+        kind == :iso_date || (format && format[0] == :general) ? format : nil
+      end
+
+      # ContentParser#format_at: the format of the cell's or row's style (-1: a style without one), else of the
+      # default cell style of its column (-2).
+      def format_at(style, column)
+        return if @formats.empty?
+        return @format_list[style] if style >= 0
+        return if style == -1
+
+        position = (0...@column_formats.size / 3).bsearch { |entry| @column_formats[entry * 3 + 1] >= column }
+        @format_list[@column_formats[position * 3 + 2]] if position && @column_formats[position * 3] <= column
+      end
+    end
+  end
+end

data/lib/calcsheet/native/version.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Calcsheet
+  module Native
+    # The version of the gem; the Rust core (Calcsheet::Native.core_version) has the same.
+    VERSION = "0.1.0"
+  end
+end