calcsheet-native 0.1.0

data/ext/calcsheet_native/src/lib.rs

@@ -0,0 +1,470 @@
+//! The Rust core of calcsheet-native, loaded by `lib/calcsheet/native.rb`.
+
+mod csv;
+mod gvl;
+mod ods;
+mod xlsx;
+mod xml;
+
+use std::cell::RefCell;
+use std::sync::Arc;
+
+use magnus::{Error, IntoValue, RArray, RString, Ruby, Value, function, method, prelude::*};
+
+use crate::csv::{Field, RowSep, Stop, Tokenizer};
+use crate::ods::TableReader;
+use crate::xlsx::{SheetReader, StringsReader, StringsStop};
+
+/// The version of the Rust core, which must equal the version of the gem.
+fn core_version() -> &'static str {
+    env!("CARGO_PKG_VERSION")
+}
+
+/// The CSV tokenizer of `Calcsheet::Native::CsvReader`: it is fed UTF-8 text and hands out records in batches,
+/// parsed without the GVL.
+#[magnus::wrap(class = "Calcsheet::Native::CsvTokenizer", free_immediately, size)]
+struct CsvTokenizer(RefCell<Tokenizer>);
+
+impl CsvTokenizer {
+    /// `col_sep` and `quote_char` are one ASCII character, `row_sep` "\n", "\r" or "\r\n".
+    fn new(
+        ruby: &Ruby,
+        col_sep: String,
+        quote_char: String,
+        row_sep: String,
+    ) -> Result<Self, Error> {
+        let byte = |name: &str, value: &str| match value.as_bytes() {
+            [b] if b.is_ascii() => Ok(*b),
+            _ => Err(Error::new(
+                ruby.exception_arg_error(),
+                format!("{name}: one ASCII character expected, got {value:?}"),
+            )),
+        };
+        let row_sep = match row_sep.as_str() {
+            "\n" => RowSep::Lf,
+            "\r" => RowSep::Cr,
+            "\r\n" => RowSep::CrLf,
+            other => {
+                return Err(Error::new(
+                    ruby.exception_arg_error(),
+                    format!("row_sep: \"\\n\", \"\\r\" or \"\\r\\n\" expected, got {other:?}"),
+                ));
+            }
+        };
+        Ok(CsvTokenizer(RefCell::new(Tokenizer::new(
+            byte("col_sep", &col_sep)?,
+            byte("quote_char", &quote_char)?,
+            row_sep,
+        ))))
+    }
+
+    fn tokenizer(&self, ruby: &Ruby) -> Result<std::cell::RefMut<'_, Tokenizer>, Error> {
+        self.0.try_borrow_mut().map_err(|_| {
+            Error::new(
+                ruby.exception_runtime_error(),
+                "a CSV tokenizer is used by two threads at once",
+            )
+        })
+    }
+
+    /// Appends a chunk of UTF-8 text.
+    fn feed(ruby: &Ruby, rb_self: &Self, chunk: RString) -> Result<(), Error> {
+        let mut tokenizer = rb_self.tokenizer(ruby)?;
+        // SAFETY: the bytes are copied before any Ruby code runs.
+        tokenizer.feed(unsafe { chunk.as_slice() });
+        Ok(())
+    }
+
+    /// No more text comes.
+    fn finish(ruby: &Ruby, rb_self: &Self) -> Result<(), Error> {
+        rb_self.tokenizer(ruby)?.finish();
+        Ok(())
+    }
+
+    /// Up to `max` records: `[fields, breaks, strays]`, an Array of the fields (Arrays of Strings and nil), one of
+    /// Integers and one of booleans; an empty batch at the end of the text. nil when the text ends inside a record
+    /// (feed more, or finish), false when the next record is malformed (see #error).
+    fn records(ruby: &Ruby, rb_self: &Self, max: usize) -> Result<Value, Error> {
+        let mut tokenizer = rb_self.tokenizer(ruby)?;
+        let (records, stop) = gvl::without_gvl(|| tokenizer.batch(max)).map_err(|panic| {
+            Error::new(
+                ruby.exception_runtime_error(),
+                format!("calcsheet-native: {panic}"),
+            )
+        })?;
+        if records.is_empty() {
+            match stop {
+                Stop::NeedMore => return Ok(ruby.qnil().as_value()),
+                Stop::Error(_) => return Ok(ruby.qfalse().as_value()),
+                Stop::End | Stop::Full => {}
+            }
+        }
+
+        let text = tokenizer.buffer();
+        let utf8 = ruby.utf8_encindex();
+        let all = ruby.ary_new_capa(records.len());
+        let breaks = ruby.ary_new_capa(records.len());
+        let strays = ruby.ary_new_capa(records.len());
+        for record in records {
+            let fields = ruby.ary_new_capa(record.fields.len());
+            for field in record.fields {
+                match field {
+                    Field::Nil => fields.push(ruby.qnil())?,
+                    Field::Span(start, end) => {
+                        fields.push(ruby.enc_str_new(&text[start..end], utf8))?
+                    }
+                    Field::Owned(bytes) => fields.push(ruby.enc_str_new(bytes, utf8))?,
+                }
+            }
+            all.push(fields)?;
+            breaks.push(record.breaks)?;
+            strays.push(record.stray)?;
+        }
+        Ok(ruby
+            .ary_new_from_values(&[all.as_value(), breaks.as_value(), strays.as_value()])
+            .as_value())
+    }
+
+    /// `[message, line]` of the malformed record that #records stopped at: the arguments of CSV::MalformedCSVError.
+    fn error(ruby: &Ruby, rb_self: &Self) -> Result<Option<(String, usize)>, Error> {
+        let tokenizer = rb_self.tokenizer(ruby)?;
+        Ok(tokenizer
+            .pending_error()
+            .map(|message| (message, tokenizer.line())))
+    }
+}
+
+/// The cells of an XLSX sheet part for `Calcsheet::Native::XlsxReader`: it is fed the inflated XML and hands out
+/// rows in batches, parsed without the GVL, or tells that calcsheet's own parser must read the sheet.
+#[magnus::wrap(class = "Calcsheet::Native::XlsxSheet", free_immediately, size)]
+struct XlsxSheet(RefCell<SheetReader>);
+
+impl XlsxSheet {
+    fn new(max_cell_bytes: Option<usize>) -> Self {
+        XlsxSheet(RefCell::new(SheetReader::new(max_cell_bytes)))
+    }
+
+    fn reader(&self, ruby: &Ruby) -> Result<std::cell::RefMut<'_, SheetReader>, Error> {
+        self.0.try_borrow_mut().map_err(|_| {
+            Error::new(
+                ruby.exception_runtime_error(),
+                "an XLSX sheet reader is used by two threads at once",
+            )
+        })
+    }
+
+    fn feed(ruby: &Ruby, rb_self: &Self, chunk: RString) -> Result<(), Error> {
+        // SAFETY: the bytes are copied before any Ruby code runs.
+        rb_self.reader(ruby)?.feed(unsafe { chunk.as_slice() });
+        Ok(())
+    }
+
+    fn finish(ruby: &Ruby, rb_self: &Self) -> Result<(), Error> {
+        rb_self.reader(ruby)?.finish();
+        Ok(())
+    }
+
+    /// The shared strings table read in Rust.
+    fn table(ruby: &Ruby, rb_self: &Self, table: &StringTable) -> Result<(), Error> {
+        rb_self.reader(ruby)?.set_table(Arc::clone(&table.0));
+        Ok(())
+    }
+
+    /// The shared strings table that calcsheet loaded: the buffer and packed offsets of its SharedStrings.
+    fn strings(
+        ruby: &Ruby,
+        rb_self: &Self,
+        buffer: RString,
+        offsets: RString,
+    ) -> Result<(), Error> {
+        // SAFETY: the bytes are copied before any Ruby code runs.
+        rb_self
+            .reader(ruby)?
+            .set_strings(unsafe { buffer.as_slice() }, unsafe { offsets.as_slice() });
+        Ok(())
+    }
+
+    /// Up to `max` rows: `[numbers, ends, cells, done]`, the row numbers, the end of each row's cells in `cells`, the
+    /// cells as six values each (column, kind, style, text, raw, type text) and whether the sheet ended. nil when the
+    /// text fed so far ends inside a row and no row is complete (feed more, or finish); false when calcsheet's
+    /// parser must read the sheet; `:strings` when a cell needs the shared strings and no row is complete (see
+    /// #strings). A shared string is frozen, as calcsheet's SharedStrings hands it out.
+    fn rows(ruby: &Ruby, rb_self: &Self, max: usize) -> Result<Value, Error> {
+        let mut reader = rb_self.reader(ruby)?;
+        let (rows, stop) = gvl::without_gvl(|| reader.rows(max)).map_err(|panic| {
+            Error::new(
+                ruby.exception_runtime_error(),
+                format!("calcsheet-native: {panic}"),
+            )
+        })?;
+        let done = match stop {
+            xlsx::Stop::Unsupported => return Ok(ruby.qfalse().as_value()),
+            xlsx::Stop::NeedMore if rows.is_empty() => return Ok(ruby.qnil().as_value()),
+            xlsx::Stop::NeedStrings if rows.is_empty() => {
+                return Ok(ruby.to_symbol("strings").as_value());
+            }
+            xlsx::Stop::End => true,
+            xlsx::Stop::NeedMore | xlsx::Stop::NeedStrings | xlsx::Stop::Full => false,
+        };
+        let utf8 = ruby.utf8_encindex();
+        let numbers = ruby.ary_new_capa(rows.len());
+        let ends = ruby.ary_new_capa(rows.len());
+        let cells = ruby.ary_new();
+        let mut count = 0;
+        for row in rows {
+            numbers.push(row.number)?;
+            for cell in row.cells {
+                cells.push(cell.column)?;
+                cells.push(cell.kind as u8)?;
+                cells.push(cell.style)?;
+                let text = ruby.enc_str_new(reader.text(&cell), utf8);
+                if cell.kind == xlsx::Kind::Shared {
+                    text.freeze();
+                }
+                cells.push(text)?;
+                cells.push(cell.raw)?;
+                cells.push(cell.type_text.map(|text| ruby.enc_str_new(text, utf8)))?;
+                count += 6;
+            }
+            ends.push(count)?;
+        }
+        Ok(ruby
+            .ary_new_from_values(&[
+                numbers.as_value(),
+                ends.as_value(),
+                cells.as_value(),
+                done.into_value_with(ruby),
+            ])
+            .as_value())
+    }
+}
+
+/// The cells of one ODS table for `Calcsheet::Native::OdsReader`: it is fed the table's XML from calcsheet's Scanner
+/// and hands out rows in batches, parsed without the GVL, or tells that calcsheet's parser must read the table.
+#[magnus::wrap(class = "Calcsheet::Native::OdsTable", free_immediately, size)]
+struct OdsTable(RefCell<TableReader>);
+
+impl OdsTable {
+    /// `prefixes`: those of the table, office, text and calcext namespaces; `formats`: the names of the cell styles
+    /// with a format, in the order of the formats the Ruby side holds.
+    fn new(
+        prefixes: (String, String, String, String),
+        formats: RArray,
+        max_cell_bytes: Option<usize>,
+        first_row: u64,
+        start_row: u64,
+    ) -> Result<Self, Error> {
+        let formats = formats
+            .to_vec::<String>()?
+            .into_iter()
+            .map(String::into_bytes)
+            .collect();
+        let (table, office, text, calcext) = &prefixes;
+        Ok(OdsTable(RefCell::new(TableReader::new(
+            [table, office, text, calcext],
+            formats,
+            max_cell_bytes,
+            first_row,
+            start_row,
+        ))))
+    }
+
+    fn reader(&self, ruby: &Ruby) -> Result<std::cell::RefMut<'_, TableReader>, Error> {
+        self.0.try_borrow_mut().map_err(|_| {
+            Error::new(
+                ruby.exception_runtime_error(),
+                "an ODS table reader is used by two threads at once",
+            )
+        })
+    }
+
+    fn feed(ruby: &Ruby, rb_self: &Self, chunk: RString) -> Result<(), Error> {
+        // SAFETY: the bytes are copied before any Ruby code runs.
+        rb_self.reader(ruby)?.feed(unsafe { chunk.as_slice() });
+        Ok(())
+    }
+
+    fn finish(ruby: &Ruby, rb_self: &Self) -> Result<(), Error> {
+        rb_self.reader(ruby)?.finish();
+        Ok(())
+    }
+
+    /// Up to `max` rows with cells: `[firsts, lasts, ends, cells, column_formats, done]`, the first and last number
+    /// of each row (a repeated row), the end of each row's cells in `cells`, the cells as six values each (column,
+    /// repeat, kind, missing, value, format), the column formats so far as `[first, last, format]` triples flattened,
+    /// and whether the table ended. nil when the text fed so far ends inside a row and no row is complete; false when
+    /// calcsheet's parser must read the table.
+    fn rows(ruby: &Ruby, rb_self: &Self, max: usize) -> Result<Value, Error> {
+        let mut reader = rb_self.reader(ruby)?;
+        let (rows, stop) = gvl::without_gvl(|| reader.rows(max)).map_err(|panic| {
+            Error::new(
+                ruby.exception_runtime_error(),
+                format!("calcsheet-native: {panic}"),
+            )
+        })?;
+        let done = match stop {
+            ods::Stop::Unsupported => return Ok(ruby.qfalse().as_value()),
+            ods::Stop::NeedMore if rows.is_empty() => return Ok(ruby.qnil().as_value()),
+            ods::Stop::End => true,
+            ods::Stop::NeedMore | ods::Stop::Full => false,
+        };
+        let utf8 = ruby.utf8_encindex();
+        let firsts = ruby.ary_new_capa(rows.len());
+        let lasts = ruby.ary_new_capa(rows.len());
+        let ends = ruby.ary_new_capa(rows.len());
+        let cells = ruby.ary_new();
+        let mut count = 0;
+        for row in rows {
+            firsts.push(row.number)?;
+            lasts.push(row.last)?;
+            for cell in row.cells {
+                cells.push(cell.column)?;
+                cells.push(cell.repeat)?;
+                cells.push(cell.kind as u8)?;
+                cells.push(cell.missing)?;
+                cells.push(ruby.enc_str_new(reader.value(&cell), utf8))?;
+                cells.push(cell.format)?;
+                count += 6;
+            }
+            ends.push(count)?;
+        }
+        let formats = ruby.ary_new_capa(reader.column_formats.len() * 3);
+        for &(first, last, format) in &reader.column_formats {
+            formats.push(first)?;
+            formats.push(last)?;
+            formats.push(format)?;
+        }
+        Ok(ruby
+            .ary_new_from_values(&[
+                firsts.as_value(),
+                lasts.as_value(),
+                ends.as_value(),
+                cells.as_value(),
+                formats.as_value(),
+                done.into_value_with(ruby),
+            ])
+            .as_value())
+    }
+}
+
+/// The shared strings table read in Rust: `Calcsheet::Native::SharedStrings` hands its strings out as calcsheet's
+/// SharedStrings does, and the sheet readers share it.
+#[magnus::wrap(class = "Calcsheet::Native::StringTable", free_immediately, size)]
+struct StringTable(Arc<xlsx::Strings>);
+
+impl StringTable {
+    fn size(&self) -> usize {
+        self.0.size()
+    }
+
+    fn bytesize(&self) -> usize {
+        self.0.buffer.len()
+    }
+
+    /// SharedStrings#unchecked: a new frozen String tagged UTF-8, not validated.
+    fn unchecked(ruby: &Ruby, rb_self: &Self, index: usize) -> Result<RString, Error> {
+        if index >= rb_self.0.size() {
+            return Err(Error::new(
+                ruby.exception_index_error(),
+                format!("no shared string {index}"),
+            ));
+        }
+        let text = ruby.enc_str_new(rb_self.0.get(index), ruby.utf8_encindex());
+        text.freeze();
+        Ok(text)
+    }
+}
+
+/// The shared strings of an XLSX file for `Calcsheet::Native::XlsxReader`: fed the XML of `xl/sharedStrings.xml`, it
+/// gives a StringTable, or tells that calcsheet must load the strings.
+#[magnus::wrap(class = "Calcsheet::Native::XlsxStrings", free_immediately, size)]
+struct XlsxStrings(RefCell<StringsReader>);
+
+impl XlsxStrings {
+    fn new(max_cell_bytes: Option<usize>) -> Self {
+        XlsxStrings(RefCell::new(StringsReader::new(max_cell_bytes)))
+    }
+
+    fn reader(&self, ruby: &Ruby) -> Result<std::cell::RefMut<'_, StringsReader>, Error> {
+        self.0.try_borrow_mut().map_err(|_| {
+            Error::new(
+                ruby.exception_runtime_error(),
+                "a shared strings reader is used by two threads at once",
+            )
+        })
+    }
+
+    fn feed(ruby: &Ruby, rb_self: &Self, chunk: RString) -> Result<(), Error> {
+        // SAFETY: the bytes are copied before any Ruby code runs.
+        rb_self.reader(ruby)?.feed(unsafe { chunk.as_slice() });
+        Ok(())
+    }
+
+    fn finish(ruby: &Ruby, rb_self: &Self) -> Result<(), Error> {
+        rb_self.reader(ruby)?.finish();
+        Ok(())
+    }
+
+    /// The StringTable once the XML is read; nil when it needs more, false when calcsheet must load the table.
+    fn read(ruby: &Ruby, rb_self: &Self) -> Result<Value, Error> {
+        let mut reader = rb_self.reader(ruby)?;
+        let stop = gvl::without_gvl(|| reader.read()).map_err(|panic| {
+            Error::new(
+                ruby.exception_runtime_error(),
+                format!("calcsheet-native: {panic}"),
+            )
+        })?;
+        match stop {
+            StringsStop::NeedMore => Ok(ruby.qnil().as_value()),
+            StringsStop::Unsupported => Ok(ruby.qfalse().as_value()),
+            StringsStop::Done => {
+                let mut strings = xlsx::Strings {
+                    buffer: std::mem::take(&mut reader.buffer),
+                    offsets: std::mem::take(&mut reader.offsets),
+                };
+                strings.buffer.shrink_to_fit();
+                strings.offsets.shrink_to_fit();
+                Ok(StringTable(Arc::new(strings)).into_value_with(ruby))
+            }
+        }
+    }
+}
+
+#[magnus::init]
+fn init(ruby: &Ruby) -> Result<(), Error> {
+    let native = ruby.define_module("Calcsheet")?.define_module("Native")?;
+    native.define_singleton_method("core_version", function!(core_version, 0))?;
+
+    let tokenizer = native.define_class("CsvTokenizer", ruby.class_object())?;
+    tokenizer.define_singleton_method("new", function!(CsvTokenizer::new, 3))?;
+    tokenizer.define_method("feed", method!(CsvTokenizer::feed, 1))?;
+    tokenizer.define_method("finish", method!(CsvTokenizer::finish, 0))?;
+    tokenizer.define_method("records", method!(CsvTokenizer::records, 1))?;
+    tokenizer.define_method("error", method!(CsvTokenizer::error, 0))?;
+
+    let sheet = native.define_class("XlsxSheet", ruby.class_object())?;
+    sheet.define_singleton_method("new", function!(XlsxSheet::new, 1))?;
+    sheet.define_method("feed", method!(XlsxSheet::feed, 1))?;
+    sheet.define_method("finish", method!(XlsxSheet::finish, 0))?;
+    sheet.define_method("rows", method!(XlsxSheet::rows, 1))?;
+    sheet.define_method("strings", method!(XlsxSheet::strings, 2))?;
+    sheet.define_method("table", method!(XlsxSheet::table, 1))?;
+
+    let string_table = native.define_class("StringTable", ruby.class_object())?;
+    string_table.define_method("size", method!(StringTable::size, 0))?;
+    string_table.define_method("bytesize", method!(StringTable::bytesize, 0))?;
+    string_table.define_method("unchecked", method!(StringTable::unchecked, 1))?;
+
+    let strings = native.define_class("XlsxStrings", ruby.class_object())?;
+    strings.define_singleton_method("new", function!(XlsxStrings::new, 1))?;
+    strings.define_method("feed", method!(XlsxStrings::feed, 1))?;
+    strings.define_method("finish", method!(XlsxStrings::finish, 0))?;
+    strings.define_method("read", method!(XlsxStrings::read, 0))?;
+
+    let table = native.define_class("OdsTable", ruby.class_object())?;
+    table.define_singleton_method("new", function!(OdsTable::new, 5))?;
+    table.define_method("feed", method!(OdsTable::feed, 1))?;
+    table.define_method("finish", method!(OdsTable::finish, 0))?;
+    table.define_method("rows", method!(OdsTable::rows, 1))?;
+    Ok(())
+}