calcsheet-native 0.1.0

data/ext/calcsheet_native/src/csv.rs

@@ -0,0 +1,309 @@
+//! A CSV tokenizer that accepts and refuses what the parser of Ruby's stdlib CSV does with `liberal_parsing: false`,
+//! for a one-byte separator and quote and a known row separator. It works on UTF-8 bytes, fed in chunks, and never
+//! touches Ruby: the bindings in `lib.rs` turn its records into Ruby objects.
+
+/// The line ending of the records.
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub enum RowSep {
+    Lf,
+    Cr,
+    CrLf,
+}
+
+/// A field of a record: nil for an empty unquoted field, else its text as a span of the buffer or, for a quoted
+/// field with doubled quotes, the unescaped bytes.
+pub enum Field {
+    Nil,
+    Span(usize, usize),
+    Owned(Vec<u8>),
+}
+
+/// A record and what calcsheet reads from its raw text: the physical lines it ends ("\r\n", "\n" and a lone "\r"
+/// each end one, inside quoted fields too) and whether the line break that does not belong to the row separator
+/// appears outside quoted fields.
+pub struct Record {
+    pub fields: Vec<Field>,
+    pub breaks: usize,
+    pub stray: bool,
+}
+
+/// What a batch ends with.
+pub enum Stop {
+    /// The text ends inside a record: feed more, or finish.
+    NeedMore,
+    /// No record is left.
+    End,
+    /// The next record is malformed: the message of the stdlib's CSV::MalformedCSVError.
+    Error(String),
+    /// The batch has its records.
+    Full,
+}
+
+enum Step {
+    Record(Record, usize),
+    Stop(Stop),
+}
+
+/// Whether the row separator starts at a position.
+enum SepAt {
+    Yes(usize),
+    No,
+    /// A CR at the end of the text, which may start a CRLF.
+    Partial,
+}
+
+pub struct Tokenizer {
+    col_sep: u8,
+    quote: u8,
+    row_sep: RowSep,
+    buf: Vec<u8>,
+    pos: usize,
+    eof: bool,
+    records: usize,
+}
+
+impl Tokenizer {
+    pub fn new(col_sep: u8, quote: u8, row_sep: RowSep) -> Self {
+        Tokenizer {
+            col_sep,
+            quote,
+            row_sep,
+            buf: Vec::new(),
+            pos: 0,
+            eof: false,
+            records: 0,
+        }
+    }
+
+    /// Appends text, dropping the records already handed out.
+    pub fn feed(&mut self, chunk: &[u8]) {
+        if self.pos > 0 {
+            self.buf.drain(..self.pos);
+            self.pos = 0;
+        }
+        self.buf.extend_from_slice(chunk);
+    }
+
+    /// No more text comes.
+    pub fn finish(&mut self) {
+        self.eof = true;
+    }
+
+    /// The text of the buffer that the spans of the last batch point into.
+    pub fn buffer(&self) -> &[u8] {
+        &self.buf
+    }
+
+    /// The stdlib's line number for an error: the record that fails, counted from 1.
+    pub fn line(&self) -> usize {
+        self.records + 1
+    }
+
+    /// The message of the malformed record at the position, if the next record is one.
+    pub fn pending_error(&self) -> Option<String> {
+        match self.record(self.pos) {
+            Step::Stop(Stop::Error(message)) => Some(message),
+            _ => None,
+        }
+    }
+
+    /// Parses up to `max` records from the text fed so far.
+    pub fn batch(&mut self, max: usize) -> (Vec<Record>, Stop) {
+        let mut records = Vec::new();
+        while records.len() < max {
+            match self.record(self.pos) {
+                Step::Record(record, end) => {
+                    records.push(record);
+                    self.pos = end;
+                    self.records += 1;
+                }
+                Step::Stop(stop) => return (records, stop),
+            }
+        }
+        (records, Stop::Full)
+    }
+
+    fn sep_at(&self, at: usize) -> SepAt {
+        let b = &self.buf;
+        match (self.row_sep, b[at]) {
+            (RowSep::Lf, b'\n') | (RowSep::Cr, b'\r') => SepAt::Yes(1),
+            (RowSep::CrLf, b'\r') => match b.get(at + 1) {
+                Some(b'\n') => SepAt::Yes(2),
+                None if !self.eof => SepAt::Partial,
+                _ => SepAt::No,
+            },
+            _ => SepAt::No,
+        }
+    }
+
+    fn need_more(&self) -> Step {
+        Step::Stop(if self.eof { Stop::End } else { Stop::NeedMore })
+    }
+
+    fn done(&self, fields: Vec<Field>, start: usize, end: usize) -> Step {
+        let (breaks, stray) = self.raw_facts(start, end);
+        Step::Record(
+            Record {
+                fields,
+                breaks,
+                stray,
+            },
+            end,
+        )
+    }
+
+    fn malformed(message: String) -> Step {
+        Step::Stop(Stop::Error(message))
+    }
+
+    /// The record that starts at `start`, the position after it, or why there is none.
+    fn record(&self, start: usize) -> Step {
+        let b = &self.buf;
+        let n = b.len();
+        if start == n {
+            return self.need_more();
+        }
+        // An empty line is a record without fields.
+        match self.sep_at(start) {
+            SepAt::Yes(len) => return self.done(Vec::new(), start, start + len),
+            SepAt::Partial => return Step::Stop(Stop::NeedMore),
+            SepAt::No => {}
+        }
+
+        let mut fields = Vec::new();
+        let mut i = start;
+        loop {
+            if i == n {
+                // After a separator at the end of the text: an empty last field.
+                if !self.eof {
+                    return Step::Stop(Stop::NeedMore);
+                }
+                fields.push(Field::Nil);
+                return self.done(fields, start, n);
+            }
+            if b[i] == self.quote {
+                let mut owned: Option<Vec<u8>> = None;
+                let mut segment = i + 1;
+                let mut j = i + 1;
+                let close = loop {
+                    let Some(q) = b[j..].iter().position(|&c| c == self.quote).map(|k| j + k)
+                    else {
+                        if !self.eof {
+                            return Step::Stop(Stop::NeedMore);
+                        }
+                        return Self::malformed("Unclosed quoted field".into());
+                    };
+                    if q + 1 == n && !self.eof {
+                        return Step::Stop(Stop::NeedMore);
+                    }
+                    if q + 1 < n && b[q + 1] == self.quote {
+                        owned
+                            .get_or_insert_with(Vec::new)
+                            .extend_from_slice(&b[segment..=q]);
+                        j = q + 2;
+                        segment = j;
+                        continue;
+                    }
+                    break q;
+                };
+                let field = match owned {
+                    Some(mut bytes) => {
+                        bytes.extend_from_slice(&b[segment..close]);
+                        Field::Owned(bytes)
+                    }
+                    None => Field::Span(i + 1, close),
+                };
+                fields.push(field);
+                let after = close + 1;
+                if after == n {
+                    return self.done(fields, start, n);
+                }
+                if b[after] == self.col_sep {
+                    i = after + 1;
+                    continue;
+                }
+                return match self.sep_at(after) {
+                    SepAt::Yes(len) => self.done(fields, start, after + len),
+                    SepAt::Partial => Step::Stop(Stop::NeedMore),
+                    SepAt::No => {
+                        Self::malformed("Any value after quoted field isn't allowed".into())
+                    }
+                };
+            }
+
+            let mut j = i;
+            loop {
+                if j == n {
+                    if !self.eof {
+                        return Step::Stop(Stop::NeedMore);
+                    }
+                    fields.push(unquoted(i, n));
+                    return self.done(fields, start, n);
+                }
+                let c = b[j];
+                if c == self.col_sep {
+                    fields.push(unquoted(i, j));
+                    i = j + 1;
+                    break;
+                }
+                if c == self.quote {
+                    return Self::malformed("Illegal quoting".into());
+                }
+                if c == b'\n' || c == b'\r' {
+                    match self.sep_at(j) {
+                        SepAt::Yes(len) => {
+                            fields.push(unquoted(i, j));
+                            return self.done(fields, start, j + len);
+                        }
+                        SepAt::Partial => return Step::Stop(Stop::NeedMore),
+                        // The stdlib keeps the line break of another ending in an unquoted field, except in a CRLF
+                        // file, where it names it as a wrong line ending when it starts the field.
+                        SepAt::No if self.row_sep == RowSep::CrLf => {
+                            let shown = if c == b'\n' { "\\n" } else { "\\r" };
+                            return Self::malformed(if j == i {
+                                format!("New line must be <\"\\r\\n\"> not <\"{shown}\">")
+                            } else {
+                                format!("Unquoted fields do not allow new line <\"{shown}\">")
+                            });
+                        }
+                        SepAt::No => {}
+                    }
+                }
+                j += 1;
+            }
+        }
+    }
+
+    /// The physical lines the raw record ends and whether it has the stray line break outside quoted fields: the
+    /// parts between quote characters alternate outside and inside.
+    fn raw_facts(&self, start: usize, end: usize) -> (usize, bool) {
+        let raw = &self.buf[start..end];
+        let stray = match self.row_sep {
+            RowSep::Lf => Some(b'\r'),
+            RowSep::Cr => Some(b'\n'),
+            RowSep::CrLf => None,
+        };
+        let mut breaks = 0;
+        let mut inside = false;
+        let mut found = false;
+        for (index, &c) in raw.iter().enumerate() {
+            if c == b'\n' || (c == b'\r' && raw.get(index + 1) != Some(&b'\n')) {
+                breaks += 1;
+            }
+            if c == self.quote {
+                inside = !inside;
+            } else if !inside && Some(c) == stray {
+                found = true;
+            }
+        }
+        (breaks, found)
+    }
+}
+
+fn unquoted(start: usize, end: usize) -> Field {
+    if start == end {
+        Field::Nil
+    } else {
+        Field::Span(start, end)
+    }
+}

data/ext/calcsheet_native/src/gvl.rs

@@ -0,0 +1,47 @@
+//! Running Rust code without Ruby's global VM lock.
+
+use std::ffi::c_void;
+use std::panic::{AssertUnwindSafe, catch_unwind};
+
+/// Runs `f` with the GVL released, so that other Ruby threads run meanwhile. `f` must not touch Ruby objects. A
+/// panic in `f` is caught (it must not unwind through Ruby's C frames) and returned as an error.
+pub fn without_gvl<F: FnOnce() -> R, R>(f: F) -> Result<R, String> {
+    struct Call<F, R> {
+        f: Option<F>,
+        result: Option<std::thread::Result<R>>,
+    }
+
+    unsafe extern "C" fn trampoline<F: FnOnce() -> R, R>(data: *mut c_void) -> *mut c_void {
+        // SAFETY: `data` is the `Call` below, alive and borrowed by nothing else until this returns.
+        let call = unsafe { &mut *(data as *mut Call<F, R>) };
+        let f = call.f.take();
+        call.result = Some(catch_unwind(AssertUnwindSafe(|| {
+            (f.expect("called once"))()
+        })));
+        std::ptr::null_mut()
+    }
+
+    let mut call = Call {
+        f: Some(f),
+        result: None,
+    };
+    // SAFETY: the trampoline only uses `call`, which outlives this blocking call; no unblocking function is given,
+    // so an interrupt waits for `f` to return.
+    unsafe {
+        rb_sys::rb_thread_call_without_gvl(
+            Some(trampoline::<F, R>),
+            &mut call as *mut Call<F, R> as *mut c_void,
+            None,
+            std::ptr::null_mut(),
+        );
+    }
+    match call.result {
+        Some(Ok(value)) => Ok(value),
+        Some(Err(panic)) => Err(panic
+            .downcast_ref::<&str>()
+            .map(|s| s.to_string())
+            .or_else(|| panic.downcast_ref::<String>().cloned())
+            .unwrap_or_else(|| "a panic without a message".into())),
+        None => Err("the call did not run".into()),
+    }
+}