roo-calamine 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +8 -0
- data/Cargo.lock +292 -0
- data/Cargo.toml +7 -0
- data/LICENSE +21 -0
- data/README.md +330 -0
- data/ext/roo_calamine/Cargo.toml +18 -0
- data/ext/roo_calamine/extconf.rb +14 -0
- data/ext/roo_calamine/src/convert.rs +365 -0
- data/ext/roo_calamine/src/lib.rs +254 -0
- data/ext/roo_calamine/src/reader.rs +1182 -0
- data/lib/roo/calamine/context.rb +57 -0
- data/lib/roo/calamine/sheet_doc.rb +66 -0
- data/lib/roo/calamine/version.rb +7 -0
- data/lib/roo/calamine.rb +47 -0
- metadata +90 -0
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "mkmf"
|
|
4
|
+
|
|
5
|
+
# JRuby cannot load a native extension. Write a no-op Makefile so the gem
|
|
6
|
+
# installs, and lib/roo/calamine.rb stays inert.
|
|
7
|
+
if RUBY_ENGINE != "ruby"
|
|
8
|
+
File.write("Makefile", "all install clean:\n\t@:\n")
|
|
9
|
+
return
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
require "rb_sys/mkmf"
|
|
13
|
+
|
|
14
|
+
create_rust_makefile("roo/calamine/roo_calamine")
|
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
//! Builds the `Roo::Excelx::Cell` objects that
|
|
2
|
+
//! `Roo::Excelx::SheetDoc#cell_from_xml` builds, by calling roo's own
|
|
3
|
+
//! constructors with the same arguments.
|
|
4
|
+
|
|
5
|
+
use std::cell::{Cell as StdCell, RefCell};
|
|
6
|
+
|
|
7
|
+
use magnus::gc::Marker;
|
|
8
|
+
use magnus::value::{Lazy, Opaque, ReprValue};
|
|
9
|
+
use magnus::{
|
|
10
|
+
prelude::*, DataTypeFunctions, Error, IntoValue, RArray, RClass, RHash, RString, Ruby,
|
|
11
|
+
StaticSymbol, Symbol, TypedData, Value,
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
use crate::reader::{parse_index, Body, Kind, Mode, RawCell, RawRow, Reference, Style};
|
|
15
|
+
|
|
16
|
+
fn constant<T: magnus::TryConvert>(ruby: &Ruby, path: &str) -> T {
|
|
17
|
+
ruby.eval(path).expect("roo is loaded before roo/calamine")
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
static COORDINATE: Lazy<RClass> = Lazy::new(|r| constant(r, "Roo::Excelx::Coordinate"));
|
|
21
|
+
static CELL_STRING: Lazy<RClass> = Lazy::new(|r| constant(r, "Roo::Excelx::Cell::String"));
|
|
22
|
+
static CELL_BOOLEAN: Lazy<RClass> = Lazy::new(|r| constant(r, "Roo::Excelx::Cell::Boolean"));
|
|
23
|
+
static CELL_NUMBER: Lazy<RClass> = Lazy::new(|r| constant(r, "Roo::Excelx::Cell::Number"));
|
|
24
|
+
static CELL_DATE: Lazy<RClass> = Lazy::new(|r| constant(r, "Roo::Excelx::Cell::Date"));
|
|
25
|
+
static CELL_DATETIME: Lazy<RClass> = Lazy::new(|r| constant(r, "Roo::Excelx::Cell::DateTime"));
|
|
26
|
+
static CELL_TIME: Lazy<RClass> = Lazy::new(|r| constant(r, "Roo::Excelx::Cell::Time"));
|
|
27
|
+
static CELL_EMPTY: Lazy<RClass> = Lazy::new(|r| constant(r, "Roo::Excelx::Cell::Empty"));
|
|
28
|
+
static NUMERIC_OR_FORMULA: Lazy<StaticSymbol> = Lazy::new(|r| r.sym_new("numeric_or_formula"));
|
|
29
|
+
|
|
30
|
+
/// `string[/<([biu]|sup|sub)>/]`, the test in `SharedStrings#use_html?`.
|
|
31
|
+
fn has_html_tag(string: &[u8]) -> bool {
|
|
32
|
+
string.windows(3).enumerate().any(|(i, w)| {
|
|
33
|
+
w[0] == b'<'
|
|
34
|
+
&& (matches!(w, [_, b'b' | b'i' | b'u', b'>'])
|
|
35
|
+
|| string[i..].starts_with(b"<sup>")
|
|
36
|
+
|| string[i..].starts_with(b"<sub>"))
|
|
37
|
+
})
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
#[derive(Clone, Copy, PartialEq, Eq)]
|
|
41
|
+
enum ValueType {
|
|
42
|
+
Date,
|
|
43
|
+
DateTime,
|
|
44
|
+
Time,
|
|
45
|
+
/// `:float`, `:percentage`, and any other format type roo builds a
|
|
46
|
+
/// `Cell::Number` for.
|
|
47
|
+
Number,
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/// Turns raw rows into roo cells for one worksheet. The Ruby objects it keeps
|
|
51
|
+
/// are pinned by [`DataTypeFunctions::mark`]; caches that may be filled while
|
|
52
|
+
/// Ruby runs live in `Cell`/`RefCell` fields that `mark` does not borrow.
|
|
53
|
+
#[derive(TypedData)]
|
|
54
|
+
#[magnus(
|
|
55
|
+
class = "Roo::Calamine::Native::Converter",
|
|
56
|
+
mark,
|
|
57
|
+
free_immediately,
|
|
58
|
+
size
|
|
59
|
+
)]
|
|
60
|
+
pub struct Converter {
|
|
61
|
+
mode: Mode,
|
|
62
|
+
empty_cell: bool,
|
|
63
|
+
context: Opaque<Value>,
|
|
64
|
+
hyperlinks: Option<Opaque<RHash>>,
|
|
65
|
+
base_date: Opaque<Value>,
|
|
66
|
+
base_timestamp: Opaque<Value>,
|
|
67
|
+
/// Format string per style index, filled on first use.
|
|
68
|
+
formats: Opaque<RArray>,
|
|
69
|
+
types: RefCell<Vec<Option<ValueType>>>,
|
|
70
|
+
/// roo's own shared-string arrays: plain text, and the HTML version
|
|
71
|
+
/// (`nil` when the HTML wrapper is disabled). Fetched on first use.
|
|
72
|
+
shared_plain: StdCell<Option<Opaque<RArray>>>,
|
|
73
|
+
shared_html: StdCell<Option<Opaque<Value>>>,
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
impl DataTypeFunctions for Converter {
|
|
77
|
+
fn mark(&self, marker: &Marker) {
|
|
78
|
+
marker.mark(self.context);
|
|
79
|
+
if let Some(hyperlinks) = self.hyperlinks {
|
|
80
|
+
marker.mark(hyperlinks);
|
|
81
|
+
}
|
|
82
|
+
marker.mark(self.base_date);
|
|
83
|
+
marker.mark(self.base_timestamp);
|
|
84
|
+
marker.mark(self.formats);
|
|
85
|
+
if let Some(plain) = self.shared_plain.get() {
|
|
86
|
+
marker.mark(plain);
|
|
87
|
+
}
|
|
88
|
+
if let Some(html) = self.shared_html.get() {
|
|
89
|
+
marker.mark(html);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
impl Converter {
|
|
95
|
+
/// `context` is a `Roo::Calamine::Context`.
|
|
96
|
+
pub fn new(ruby: &Ruby, context: Value, stream: bool) -> Result<Self, Error> {
|
|
97
|
+
let hyperlinks: RHash = context.funcall("hyperlinks", ())?;
|
|
98
|
+
Ok(Self {
|
|
99
|
+
mode: if stream { Mode::Stream } else { Mode::Document },
|
|
100
|
+
empty_cell: context.funcall::<_, _, Value>("empty_cell", ())?.to_bool(),
|
|
101
|
+
context: context.into(),
|
|
102
|
+
hyperlinks: (!hyperlinks.is_empty()).then(|| hyperlinks.into()),
|
|
103
|
+
base_date: context.funcall::<_, _, Value>("base_date", ())?.into(),
|
|
104
|
+
base_timestamp: context.funcall::<_, _, Value>("base_timestamp", ())?.into(),
|
|
105
|
+
formats: ruby.ary_new().into(),
|
|
106
|
+
types: RefCell::new(Vec::new()),
|
|
107
|
+
shared_plain: StdCell::new(None),
|
|
108
|
+
shared_html: StdCell::new(None),
|
|
109
|
+
})
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/// `SheetDoc#each_cell` for a streamed row: every cell, empty ones too.
|
|
113
|
+
pub fn row_cells(ruby: &Ruby, rb_self: &Self, row: &crate::Row) -> Result<RArray, Error> {
|
|
114
|
+
let cells = ruby.ary_new_capa(row.raw().cells.len());
|
|
115
|
+
for (index, raw) in row.raw().cells.iter().enumerate() {
|
|
116
|
+
if let Some((_, cell)) = rb_self.cell(ruby, row.raw(), raw, 0, index + 1)? {
|
|
117
|
+
cells.push(cell)?;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
Ok(cells)
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/// `SheetDoc#extract_cells` for one row, adding to `cells`.
|
|
124
|
+
pub fn add_row(
|
|
125
|
+
&self,
|
|
126
|
+
ruby: &Ruby,
|
|
127
|
+
cells: RHash,
|
|
128
|
+
row: &RawRow,
|
|
129
|
+
row_number: usize,
|
|
130
|
+
) -> Result<(), Error> {
|
|
131
|
+
for (index, raw) in row.cells.iter().enumerate() {
|
|
132
|
+
if let Some((coordinate, cell)) = self.cell(ruby, row, raw, row_number, index + 1)? {
|
|
133
|
+
cells.aset(coordinate, cell)?;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
Ok(())
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
fn context(&self, ruby: &Ruby) -> Value {
|
|
140
|
+
ruby.get_inner(self.context)
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
fn coordinate(
|
|
144
|
+
&self,
|
|
145
|
+
ruby: &Ruby,
|
|
146
|
+
raw: &RawCell,
|
|
147
|
+
row: usize,
|
|
148
|
+
column: usize,
|
|
149
|
+
) -> Result<Value, Error> {
|
|
150
|
+
let coordinate = ruby.get_inner(&COORDINATE);
|
|
151
|
+
match (&raw.reference, self.mode) {
|
|
152
|
+
(Reference::At { row, column }, _) => coordinate.new_instance((*row, *column)),
|
|
153
|
+
(Reference::Invalid(reference), _) => self
|
|
154
|
+
.context(ruby)
|
|
155
|
+
.funcall("coordinate", (ruby.str_new(reference),)),
|
|
156
|
+
(Reference::Missing, Mode::Document) => coordinate.new_instance((row, column)),
|
|
157
|
+
(Reference::Missing, Mode::Stream) => {
|
|
158
|
+
self.context(ruby).funcall("coordinate", (ruby.qnil(),))
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
fn style(&self, ruby: &Ruby, raw: &RawCell) -> Result<Value, Error> {
|
|
164
|
+
match &raw.style {
|
|
165
|
+
Style::Index(index) => Ok(index.into_value_with(ruby)),
|
|
166
|
+
Style::Other(text) => self
|
|
167
|
+
.context(ruby)
|
|
168
|
+
.funcall("style_index", (ruby.str_new(text),)),
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/// `[format.to_s, Excelx::Format.to_type(format)]` for a style index.
|
|
173
|
+
fn style_info(&self, ruby: &Ruby, style: Value) -> Result<(Value, ValueType), Error> {
|
|
174
|
+
let formats = ruby.get_inner(self.formats);
|
|
175
|
+
let cacheable = usize::try_convert(style).ok().filter(|i| *i < 1 << 16);
|
|
176
|
+
if let Some(index) = cacheable {
|
|
177
|
+
if let Some(Some(value_type)) = self.types.borrow().get(index) {
|
|
178
|
+
return Ok((formats.entry(index as isize)?, *value_type));
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
let info: RArray = self.context(ruby).funcall("style", (style,))?;
|
|
182
|
+
let format: Value = info.entry(0)?;
|
|
183
|
+
let type_symbol: Symbol = info.entry(1)?;
|
|
184
|
+
let value_type = match type_symbol.name()?.as_ref() {
|
|
185
|
+
"date" => ValueType::Date,
|
|
186
|
+
"datetime" => ValueType::DateTime,
|
|
187
|
+
"time" => ValueType::Time,
|
|
188
|
+
_ => ValueType::Number,
|
|
189
|
+
};
|
|
190
|
+
if let Some(index) = cacheable {
|
|
191
|
+
formats.store(index as isize, format)?;
|
|
192
|
+
let mut types = self.types.borrow_mut();
|
|
193
|
+
if types.len() <= index {
|
|
194
|
+
types.resize(index + 1, None);
|
|
195
|
+
}
|
|
196
|
+
types[index] = Some(value_type);
|
|
197
|
+
}
|
|
198
|
+
Ok((format, value_type))
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/// `shared_strings.use_html?(i) ? shared_strings.to_html[i] : shared_strings[i]`,
|
|
202
|
+
/// decided again for every cell, as roo does, since a caller may change a
|
|
203
|
+
/// shared string in place.
|
|
204
|
+
fn shared_string(&self, ruby: &Ruby, text: &str) -> Result<Value, Error> {
|
|
205
|
+
if let Some(index) = parse_index(text) {
|
|
206
|
+
if self.shared_plain.get().is_none() {
|
|
207
|
+
let tables: RArray = self.context(ruby).funcall("shared_strings", ())?;
|
|
208
|
+
let plain: RArray = tables.entry(0)?;
|
|
209
|
+
let html: Value = tables.entry(1)?;
|
|
210
|
+
self.shared_plain.set(Some(plain.into()));
|
|
211
|
+
self.shared_html.set(Some(html.into()));
|
|
212
|
+
}
|
|
213
|
+
let plain = ruby.get_inner(self.shared_plain.get().expect("fetched above"));
|
|
214
|
+
let html = ruby.get_inner(self.shared_html.get().expect("fetched above"));
|
|
215
|
+
let chosen = match RArray::from_value(html) {
|
|
216
|
+
None => Some(plain),
|
|
217
|
+
Some(html) if index < html.len() => {
|
|
218
|
+
match RString::from_value(html.entry(index as isize)?) {
|
|
219
|
+
Some(string) if has_html_tag(unsafe { string.as_slice() }) => {
|
|
220
|
+
return Ok(string.as_value());
|
|
221
|
+
}
|
|
222
|
+
Some(_) => Some(plain),
|
|
223
|
+
None => None,
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
Some(_) => None,
|
|
227
|
+
};
|
|
228
|
+
if let Some(plain) = chosen.filter(|plain| index < plain.len()) {
|
|
229
|
+
return plain.entry(index as isize);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
self.context(ruby)
|
|
233
|
+
.funcall("shared_string", (ruby.str_new(text),))
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/// Builds the cell `SheetDoc#cell_from_xml` builds for `raw`. `None` means
|
|
237
|
+
/// roo stores no cell (an empty cell without the `empty_cell` option).
|
|
238
|
+
fn cell(
|
|
239
|
+
&self,
|
|
240
|
+
ruby: &Ruby,
|
|
241
|
+
row: &RawRow,
|
|
242
|
+
raw: &RawCell,
|
|
243
|
+
row_number: usize,
|
|
244
|
+
column: usize,
|
|
245
|
+
) -> Result<Option<(Value, Value)>, Error> {
|
|
246
|
+
let coordinate = self.coordinate(ruby, raw, row_number, column)?;
|
|
247
|
+
let style = self.style(ruby, raw)?;
|
|
248
|
+
let link = match self.hyperlinks {
|
|
249
|
+
Some(hyperlinks) => ruby
|
|
250
|
+
.get_inner(hyperlinks)
|
|
251
|
+
.get(coordinate)
|
|
252
|
+
.unwrap_or_else(|| ruby.qnil().as_value()),
|
|
253
|
+
None => ruby.qnil().as_value(),
|
|
254
|
+
};
|
|
255
|
+
let formula = raw.formula.as_ref().map(|f| ruby.str_new(row.text(f)));
|
|
256
|
+
let cell: Value =
|
|
257
|
+
match &raw.body {
|
|
258
|
+
Body::Empty => {
|
|
259
|
+
if !self.empty_cell {
|
|
260
|
+
return Ok(None);
|
|
261
|
+
}
|
|
262
|
+
ruby.get_inner(&CELL_EMPTY).new_instance((coordinate,))?
|
|
263
|
+
}
|
|
264
|
+
Body::Inline(text) => ruby.get_inner(&CELL_STRING).new_instance((
|
|
265
|
+
ruby.str_new(row.text(text)),
|
|
266
|
+
formula,
|
|
267
|
+
style,
|
|
268
|
+
link,
|
|
269
|
+
coordinate,
|
|
270
|
+
))?,
|
|
271
|
+
// roo looks up the style format for every `v` cell, which raises when
|
|
272
|
+
// the workbook has no styles part, so this does too.
|
|
273
|
+
Body::Value(text) => {
|
|
274
|
+
let text = row.text(text);
|
|
275
|
+
match (raw.kind, self.style_info(ruby, style)?) {
|
|
276
|
+
(Kind::Shared, _) => {
|
|
277
|
+
let value = self.shared_string(ruby, text)?;
|
|
278
|
+
ruby.get_inner(&CELL_STRING)
|
|
279
|
+
.new_instance((value, formula, style, link, coordinate))?
|
|
280
|
+
}
|
|
281
|
+
(Kind::Boolean, _) => ruby.get_inner(&CELL_BOOLEAN).new_instance((
|
|
282
|
+
ruby.str_new(text),
|
|
283
|
+
formula,
|
|
284
|
+
style,
|
|
285
|
+
link,
|
|
286
|
+
coordinate,
|
|
287
|
+
))?,
|
|
288
|
+
(Kind::Str, _) => ruby.get_inner(&CELL_STRING).new_instance((
|
|
289
|
+
ruby.str_new(text),
|
|
290
|
+
formula,
|
|
291
|
+
style,
|
|
292
|
+
link,
|
|
293
|
+
coordinate,
|
|
294
|
+
))?,
|
|
295
|
+
(kind, (format, format_type)) => {
|
|
296
|
+
let value_type = if kind == Kind::InlineStr {
|
|
297
|
+
ValueType::Number
|
|
298
|
+
} else {
|
|
299
|
+
format_type
|
|
300
|
+
};
|
|
301
|
+
let numeric_or_formula = ruby.get_inner(&NUMERIC_OR_FORMULA).as_value();
|
|
302
|
+
let excelx_type =
|
|
303
|
+
ruby.ary_new_from_values(&[numeric_or_formula, format]);
|
|
304
|
+
let content = ruby.str_new(text);
|
|
305
|
+
match value_type {
|
|
306
|
+
ValueType::Number => ruby.get_inner(&CELL_NUMBER).new_instance(
|
|
307
|
+
(content, formula, excelx_type, style, link, coordinate),
|
|
308
|
+
)?,
|
|
309
|
+
ValueType::Date => ruby.get_inner(&CELL_DATE).new_instance((
|
|
310
|
+
content,
|
|
311
|
+
formula,
|
|
312
|
+
excelx_type,
|
|
313
|
+
style,
|
|
314
|
+
link,
|
|
315
|
+
ruby.get_inner(self.base_date),
|
|
316
|
+
coordinate,
|
|
317
|
+
))?,
|
|
318
|
+
ValueType::DateTime | ValueType::Time => {
|
|
319
|
+
let number: f64 = content.funcall("to_f", ())?;
|
|
320
|
+
let (class, base) = if number < 1.0 {
|
|
321
|
+
(&CELL_TIME, self.base_timestamp)
|
|
322
|
+
} else if (number - number.floor()).abs() > 0.000001 {
|
|
323
|
+
(&CELL_DATETIME, self.base_timestamp)
|
|
324
|
+
} else {
|
|
325
|
+
(&CELL_DATE, self.base_date)
|
|
326
|
+
};
|
|
327
|
+
ruby.get_inner(class).new_instance((
|
|
328
|
+
ruby.float_from_f64(number),
|
|
329
|
+
formula,
|
|
330
|
+
excelx_type,
|
|
331
|
+
style,
|
|
332
|
+
link,
|
|
333
|
+
ruby.get_inner(base),
|
|
334
|
+
coordinate,
|
|
335
|
+
))?
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
};
|
|
342
|
+
Ok(Some((coordinate, cell)))
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
#[cfg(test)]
|
|
347
|
+
mod tests {
|
|
348
|
+
use super::has_html_tag;
|
|
349
|
+
|
|
350
|
+
#[test]
|
|
351
|
+
fn finds_the_tags_roo_looks_for() {
|
|
352
|
+
for html in [
|
|
353
|
+
"<html><b>x</b></html>",
|
|
354
|
+
"a<i>",
|
|
355
|
+
"<u>",
|
|
356
|
+
"x<sup>2</sup>",
|
|
357
|
+
"<sub>",
|
|
358
|
+
] {
|
|
359
|
+
assert!(has_html_tag(html.as_bytes()), "{html}");
|
|
360
|
+
}
|
|
361
|
+
for plain in ["<html>x</html>", "<br>", "<bb>", "b>", "<s>", "<su>", "<"] {
|
|
362
|
+
assert!(!has_html_tag(plain.as_bytes()), "{plain}");
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
}
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
//! Native fast path for `roo-calamine`.
|
|
2
|
+
//!
|
|
3
|
+
//! [`reader`] turns worksheet XML into raw cell records without touching Ruby,
|
|
4
|
+
//! on a thread of its own. [`convert`] turns those records into roo's cells.
|
|
5
|
+
|
|
6
|
+
pub mod convert;
|
|
7
|
+
pub mod reader;
|
|
8
|
+
|
|
9
|
+
use std::cell::{Cell as StdCell, RefCell};
|
|
10
|
+
use std::ffi::c_void;
|
|
11
|
+
use std::panic::{self, AssertUnwindSafe};
|
|
12
|
+
use std::path::Path;
|
|
13
|
+
use std::sync::atomic::{AtomicBool, Ordering};
|
|
14
|
+
use std::sync::mpsc::{self, Receiver, RecvTimeoutError, TryRecvError};
|
|
15
|
+
use std::time::Duration;
|
|
16
|
+
|
|
17
|
+
use magnus::value::Lazy;
|
|
18
|
+
use magnus::{function, method, prelude::*, Error, ExceptionClass, RArray, RHash, Ruby};
|
|
19
|
+
|
|
20
|
+
use convert::Converter;
|
|
21
|
+
use reader::{Mode, RawRow, SheetReader, Unsupported};
|
|
22
|
+
|
|
23
|
+
static UNSUPPORTED: Lazy<ExceptionClass> = Lazy::new(|r| {
|
|
24
|
+
r.eval("Roo::Calamine::UnsupportedError")
|
|
25
|
+
.expect("roo/calamine defines its error before the extension loads")
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
fn unsupported(ruby: &Ruby, error: Unsupported) -> Error {
|
|
29
|
+
Error::new(ruby.get_inner(&UNSUPPORTED), error.0)
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/// Runs `work` with the GVL released. `work` must not touch any Ruby object,
|
|
33
|
+
/// and should return soon after the flag it is given turns true: Ruby sets
|
|
34
|
+
/// it when it has something to deliver to this thread (`Thread#raise`,
|
|
35
|
+
/// `Timeout`, Ctrl-C). Such an exception comes back as `Err` instead of
|
|
36
|
+
/// unwinding through the caller's Rust frames.
|
|
37
|
+
fn without_gvl<F: FnOnce(&AtomicBool) -> R, R>(work: F) -> Result<R, Error> {
|
|
38
|
+
struct Call<'a, F, R> {
|
|
39
|
+
work: Option<F>,
|
|
40
|
+
interrupted: &'a AtomicBool,
|
|
41
|
+
result: Option<std::thread::Result<R>>,
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
unsafe extern "C" fn trampoline<F: FnOnce(&AtomicBool) -> R, R>(
|
|
45
|
+
data: *mut c_void,
|
|
46
|
+
) -> *mut c_void {
|
|
47
|
+
let call = unsafe { &mut *(data as *mut Call<'_, F, R>) };
|
|
48
|
+
let work = call.work.take().expect("called once");
|
|
49
|
+
let interrupted = call.interrupted;
|
|
50
|
+
call.result = Some(panic::catch_unwind(AssertUnwindSafe(|| work(interrupted))));
|
|
51
|
+
std::ptr::null_mut()
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
unsafe extern "C" fn unblock(data: *mut c_void) {
|
|
55
|
+
unsafe { &*(data as *const AtomicBool) }.store(true, Ordering::Release);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
let interrupted = AtomicBool::new(false);
|
|
59
|
+
let mut call = Call {
|
|
60
|
+
work: Some(work),
|
|
61
|
+
interrupted: &interrupted,
|
|
62
|
+
result: None,
|
|
63
|
+
};
|
|
64
|
+
let data = &mut call as *mut Call<'_, F, R> as *mut c_void;
|
|
65
|
+
let flag = &interrupted as *const AtomicBool as *mut c_void;
|
|
66
|
+
magnus::rb_sys::protect(|| unsafe {
|
|
67
|
+
rb_sys::rb_thread_call_without_gvl(Some(trampoline::<F, R>), data, Some(unblock), flag);
|
|
68
|
+
rb_sys::Qnil as rb_sys::VALUE
|
|
69
|
+
})?;
|
|
70
|
+
match call.result.expect("trampoline ran") {
|
|
71
|
+
Ok(result) => Ok(result),
|
|
72
|
+
Err(payload) => panic::resume_unwind(payload),
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
enum Wait {
|
|
77
|
+
Batch(Batch),
|
|
78
|
+
/// Ruby interrupted the wait without raising; wait again.
|
|
79
|
+
Interrupted,
|
|
80
|
+
/// The reading thread ended without sending a last batch.
|
|
81
|
+
Stopped,
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/// Waits for the next batch with the GVL released.
|
|
85
|
+
fn wait_for(batches: &Receiver<Batch>) -> Result<Wait, Error> {
|
|
86
|
+
without_gvl(|interrupted| loop {
|
|
87
|
+
match batches.recv_timeout(Duration::from_millis(20)) {
|
|
88
|
+
Ok(batch) => return Wait::Batch(batch),
|
|
89
|
+
Err(RecvTimeoutError::Disconnected) => return Wait::Stopped,
|
|
90
|
+
Err(RecvTimeoutError::Timeout) if interrupted.load(Ordering::Acquire) => {
|
|
91
|
+
return Wait::Interrupted
|
|
92
|
+
}
|
|
93
|
+
Err(RecvTimeoutError::Timeout) => {}
|
|
94
|
+
}
|
|
95
|
+
})
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/// One streamed `<row>`, converted to cells only when roo asks for them.
|
|
99
|
+
#[magnus::wrap(class = "Roo::Calamine::Native::Row", free_immediately, size)]
|
|
100
|
+
pub struct Row(RawRow);
|
|
101
|
+
|
|
102
|
+
impl Row {
|
|
103
|
+
pub fn raw(&self) -> &RawRow {
|
|
104
|
+
&self.0
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
type Batch = Result<Vec<RawRow>, Unsupported>;
|
|
109
|
+
|
|
110
|
+
/// Reads a worksheet on its own thread, one batch ahead of Ruby: while Ruby
|
|
111
|
+
/// turns one batch into cells, the thread parses the next, without the GVL.
|
|
112
|
+
/// Ruby releases the GVL only when it has to wait for a batch, so a busy Ruby
|
|
113
|
+
/// thread elsewhere does not slow every batch down.
|
|
114
|
+
#[magnus::wrap(class = "Roo::Calamine::Native::Reader", free_immediately, size)]
|
|
115
|
+
struct Reader {
|
|
116
|
+
/// `None` once closed.
|
|
117
|
+
batches: RefCell<Option<Receiver<Batch>>>,
|
|
118
|
+
/// Set while a Ruby thread waits on `batches` without the GVL; the
|
|
119
|
+
/// receiver must not be used from two threads at once.
|
|
120
|
+
in_use: StdCell<bool>,
|
|
121
|
+
done: StdCell<bool>,
|
|
122
|
+
rows_read: StdCell<usize>,
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
impl Reader {
|
|
126
|
+
/// `batch` is the number of cells parsed per batch.
|
|
127
|
+
fn new(ruby: &Ruby, path: String, stream: bool, batch: usize) -> Result<Self, Error> {
|
|
128
|
+
let mode = if stream { Mode::Stream } else { Mode::Document };
|
|
129
|
+
let mut sheet =
|
|
130
|
+
SheetReader::open(Path::new(&path), mode).map_err(|e| unsupported(ruby, e))?;
|
|
131
|
+
let (sender, batches) = mpsc::sync_channel::<Batch>(1);
|
|
132
|
+
std::thread::Builder::new()
|
|
133
|
+
.name("roo-calamine".into())
|
|
134
|
+
.spawn(move || loop {
|
|
135
|
+
let rows = sheet.next_rows(batch);
|
|
136
|
+
let last = !matches!(&rows, Ok(rows) if !rows.is_empty());
|
|
137
|
+
// A send fails once the Reader is garbage collected.
|
|
138
|
+
if sender.send(rows).is_err() || last {
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
})
|
|
142
|
+
.map_err(|e| Error::new(ruby.exception_runtime_error(), e.to_string()))?;
|
|
143
|
+
Ok(Self {
|
|
144
|
+
batches: RefCell::new(Some(batches)),
|
|
145
|
+
in_use: StdCell::new(false),
|
|
146
|
+
done: StdCell::new(false),
|
|
147
|
+
rows_read: StdCell::new(0),
|
|
148
|
+
})
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
fn read_batch(&self, ruby: &Ruby) -> Result<Vec<RawRow>, Error> {
|
|
152
|
+
if self.done.get() {
|
|
153
|
+
return Ok(Vec::new());
|
|
154
|
+
}
|
|
155
|
+
if self.in_use.replace(true) {
|
|
156
|
+
return Err(Error::new(
|
|
157
|
+
ruby.exception_runtime_error(),
|
|
158
|
+
"reader is already in use",
|
|
159
|
+
));
|
|
160
|
+
}
|
|
161
|
+
let received = match self.batches.borrow().as_ref() {
|
|
162
|
+
None => Ok(Wait::Batch(Ok(Vec::new()))),
|
|
163
|
+
Some(batches) => match batches.try_recv() {
|
|
164
|
+
Ok(batch) => Ok(Wait::Batch(batch)),
|
|
165
|
+
Err(TryRecvError::Disconnected) => Ok(Wait::Stopped),
|
|
166
|
+
Err(TryRecvError::Empty) => loop {
|
|
167
|
+
match wait_for(batches) {
|
|
168
|
+
Ok(Wait::Interrupted) => continue,
|
|
169
|
+
other => break other,
|
|
170
|
+
}
|
|
171
|
+
},
|
|
172
|
+
},
|
|
173
|
+
};
|
|
174
|
+
self.in_use.set(false);
|
|
175
|
+
let rows = match received? {
|
|
176
|
+
Wait::Batch(batch) => batch.map_err(|e| unsupported(ruby, e))?,
|
|
177
|
+
Wait::Stopped | Wait::Interrupted => {
|
|
178
|
+
return Err(unsupported(
|
|
179
|
+
ruby,
|
|
180
|
+
Unsupported("the reader thread stopped".into()),
|
|
181
|
+
));
|
|
182
|
+
}
|
|
183
|
+
};
|
|
184
|
+
if rows.is_empty() {
|
|
185
|
+
self.done.set(true);
|
|
186
|
+
}
|
|
187
|
+
Ok(rows)
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/// Stops the reading thread, which closes the file and frees the rows it
|
|
191
|
+
/// read ahead, without waiting for garbage collection.
|
|
192
|
+
fn close(ruby: &Ruby, rb_self: &Self) -> Result<(), Error> {
|
|
193
|
+
if rb_self.in_use.get() {
|
|
194
|
+
return Err(Error::new(
|
|
195
|
+
ruby.exception_runtime_error(),
|
|
196
|
+
"reader is already in use",
|
|
197
|
+
));
|
|
198
|
+
}
|
|
199
|
+
rb_self.batches.borrow_mut().take();
|
|
200
|
+
rb_self.done.set(true);
|
|
201
|
+
Ok(())
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/// Document mode: adds the next batch of rows to `cells`. Returns `false`
|
|
205
|
+
/// once the sheet is exhausted.
|
|
206
|
+
fn fill_cells(
|
|
207
|
+
ruby: &Ruby,
|
|
208
|
+
rb_self: &Self,
|
|
209
|
+
cells: RHash,
|
|
210
|
+
converter: &Converter,
|
|
211
|
+
) -> Result<bool, Error> {
|
|
212
|
+
let rows = rb_self.read_batch(ruby)?;
|
|
213
|
+
for row in &rows {
|
|
214
|
+
let row_number = rb_self.rows_read.get() + 1;
|
|
215
|
+
rb_self.rows_read.set(row_number);
|
|
216
|
+
converter.add_row(ruby, cells, row, row_number)?;
|
|
217
|
+
}
|
|
218
|
+
Ok(!rows.is_empty())
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/// Stream mode: the next batch of rows, or `nil` at the end of the sheet.
|
|
222
|
+
fn next_rows(ruby: &Ruby, rb_self: &Self) -> Result<Option<RArray>, Error> {
|
|
223
|
+
let rows = rb_self.read_batch(ruby)?;
|
|
224
|
+
if rows.is_empty() {
|
|
225
|
+
return Ok(None);
|
|
226
|
+
}
|
|
227
|
+
let out = ruby.ary_new_capa(rows.len());
|
|
228
|
+
for row in rows {
|
|
229
|
+
out.push(Row(row))?;
|
|
230
|
+
}
|
|
231
|
+
Ok(Some(out))
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
#[magnus::init]
|
|
236
|
+
fn init(ruby: &Ruby) -> Result<(), Error> {
|
|
237
|
+
let native = ruby
|
|
238
|
+
.define_module("Roo")?
|
|
239
|
+
.define_module("Calamine")?
|
|
240
|
+
.define_module("Native")?;
|
|
241
|
+
|
|
242
|
+
let reader = native.define_class("Reader", ruby.class_object())?;
|
|
243
|
+
reader.define_singleton_method("new", function!(Reader::new, 3))?;
|
|
244
|
+
reader.define_method("fill_cells", method!(Reader::fill_cells, 2))?;
|
|
245
|
+
reader.define_method("next_rows", method!(Reader::next_rows, 0))?;
|
|
246
|
+
reader.define_method("close", method!(Reader::close, 0))?;
|
|
247
|
+
|
|
248
|
+
native.define_class("Row", ruby.class_object())?;
|
|
249
|
+
|
|
250
|
+
let converter = native.define_class("Converter", ruby.class_object())?;
|
|
251
|
+
converter.define_singleton_method("new", function!(Converter::new, 2))?;
|
|
252
|
+
converter.define_method("row_cells", method!(Converter::row_cells, 1))?;
|
|
253
|
+
Ok(())
|
|
254
|
+
}
|