imprimatur 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/Cargo.lock +4 -3
- data/Cargo.toml +1 -1
- data/README.md +27 -5
- data/ext/imprimatur_native/cli/Cargo.toml +2 -2
- data/ext/imprimatur_native/cli/src/cli/grep.rs +1 -1
- data/ext/imprimatur_native/cli/src/cli/markdown.rs +2 -2
- data/ext/imprimatur_native/cli/src/cli.rs +5 -5
- data/ext/imprimatur_native/cli/src/main.rs +1 -1
- data/ext/imprimatur_native/core/Cargo.toml +7 -2
- data/ext/imprimatur_native/core/README.md +6 -1
- data/ext/imprimatur_native/core/src/archive.rs +292 -0
- data/ext/imprimatur_native/core/src/error.rs +1 -1
- data/ext/imprimatur_native/core/src/lib.rs +2 -0
- data/ext/imprimatur_native/core/src/read/content.rs +16 -1
- data/ext/imprimatur_native/core/src/read/json.rs +3 -1
- data/ext/imprimatur_native/core/src/read/markdown.rs +1 -1
- data/ext/imprimatur_native/core/src/read/{xlsx/numbers.rs → numbers.rs} +4 -3
- data/ext/imprimatur_native/core/src/read/ods.rs +357 -0
- data/ext/imprimatur_native/core/src/read/ole.rs +244 -0
- data/ext/imprimatur_native/core/src/read/package.rs +0 -5
- data/ext/imprimatur_native/core/src/read/xls/strings.rs +156 -0
- data/ext/imprimatur_native/core/src/read/xls.rs +342 -0
- data/ext/imprimatur_native/core/src/read/xlsx.rs +1 -2
- data/ext/imprimatur_native/core/src/read.rs +64 -20
- data/ext/imprimatur_native/core/src/test_support.rs +87 -0
- data/ext/imprimatur_native/ffi/Cargo.toml +2 -2
- data/ext/imprimatur_native/ffi/src/archive.rs +121 -0
- data/ext/imprimatur_native/ffi/src/blocking.rs +63 -0
- data/ext/imprimatur_native/ffi/src/lib.rs +14 -0
- data/ext/imprimatur_native/ffi/src/read.rs +1 -3
- data/lib/imprimatur/error.rb +4 -3
- data/lib/imprimatur/reader.rb +1 -1
- data/lib/imprimatur/version.rb +1 -1
- data/lib/imprimatur/zip.rb +127 -0
- data/lib/imprimatur.rb +3 -2
- metadata +13 -4
|
@@ -161,3 +161,90 @@ pub fn tiff_of(data: &[u8]) -> (std::collections::BTreeMap<u16, u32>, &[u8]) {
|
|
|
161
161
|
let (offset, length) = (tags[&273] as usize, tags[&279] as usize);
|
|
162
162
|
(tags, &data[offset..offset + length])
|
|
163
163
|
}
|
|
164
|
+
|
|
165
|
+
/// A compound file (MS-CFB, version 3) holding one stream named `name` in
|
|
166
|
+
/// its root storage; a stream under 4096 bytes goes to the mini stream, as
|
|
167
|
+
/// writers put it.
|
|
168
|
+
pub fn ole_of(name: &str, stream: &[u8]) -> Vec<u8> {
|
|
169
|
+
const SECTOR: usize = 512;
|
|
170
|
+
const END: u32 = 0xFFFF_FFFE;
|
|
171
|
+
let small = stream.len() < 4096;
|
|
172
|
+
let pad = |data: &[u8], size: usize| {
|
|
173
|
+
let mut data = data.to_vec();
|
|
174
|
+
data.resize(data.len().div_ceil(size).max(1) * size, 0);
|
|
175
|
+
data
|
|
176
|
+
};
|
|
177
|
+
// Sectors: FAT, directory, mini FAT (when small), then the data: the
|
|
178
|
+
// stream itself, or the mini stream holding it.
|
|
179
|
+
let data = if small { pad(stream, 64) } else { stream.to_vec() };
|
|
180
|
+
let data_sectors = data.len().div_ceil(SECTOR).max(1);
|
|
181
|
+
let first_data = if small { 3 } else { 2 };
|
|
182
|
+
let mut fat = vec![0xFFFF_FFFD_u32, END];
|
|
183
|
+
if small {
|
|
184
|
+
fat.push(END);
|
|
185
|
+
}
|
|
186
|
+
fat.extend((0..data_sectors).map(|index| {
|
|
187
|
+
if index + 1 == data_sectors { END } else { u32::try_from(first_data + index + 1).unwrap() }
|
|
188
|
+
}));
|
|
189
|
+
fat.resize(SECTOR / 4, 0xFFFF_FFFF);
|
|
190
|
+
let mut header = vec![0_u8; SECTOR];
|
|
191
|
+
header[..8].copy_from_slice(&[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]);
|
|
192
|
+
let put = |bytes: &mut Vec<u8>, at: usize, value: u32| {
|
|
193
|
+
bytes[at..at + 4].copy_from_slice(&value.to_le_bytes());
|
|
194
|
+
};
|
|
195
|
+
header[0x18..0x1A].copy_from_slice(&0x3E_u16.to_le_bytes());
|
|
196
|
+
header[0x1A..0x1C].copy_from_slice(&3_u16.to_le_bytes());
|
|
197
|
+
header[0x1C..0x1E].copy_from_slice(&0xFFFE_u16.to_le_bytes());
|
|
198
|
+
header[0x1E..0x20].copy_from_slice(&9_u16.to_le_bytes());
|
|
199
|
+
header[0x20..0x22].copy_from_slice(&6_u16.to_le_bytes());
|
|
200
|
+
put(&mut header, 0x2C, 1);
|
|
201
|
+
put(&mut header, 0x30, 1);
|
|
202
|
+
put(&mut header, 0x38, 4096);
|
|
203
|
+
put(&mut header, 0x3C, if small { 2 } else { END });
|
|
204
|
+
put(&mut header, 0x40, u32::from(small));
|
|
205
|
+
put(&mut header, 0x44, END);
|
|
206
|
+
put(&mut header, 0x4C, 0);
|
|
207
|
+
for index in 1..109 {
|
|
208
|
+
put(&mut header, 0x4C + index * 4, 0xFFFF_FFFF);
|
|
209
|
+
}
|
|
210
|
+
let entry = |name: &str, kind: u8, child: u32, start: u32, size: usize| {
|
|
211
|
+
let mut bytes = vec![0_u8; 128];
|
|
212
|
+
let units: Vec<u16> = name.encode_utf16().collect();
|
|
213
|
+
for (index, unit) in units.iter().enumerate() {
|
|
214
|
+
bytes[index * 2..index * 2 + 2].copy_from_slice(&unit.to_le_bytes());
|
|
215
|
+
}
|
|
216
|
+
bytes[0x40..0x42]
|
|
217
|
+
.copy_from_slice(&u16::try_from(units.len() * 2 + 2).unwrap().to_le_bytes());
|
|
218
|
+
bytes[0x42] = kind;
|
|
219
|
+
for at in [0x44, 0x48, 0x4C] {
|
|
220
|
+
put(&mut bytes, at, 0xFFFF_FFFF);
|
|
221
|
+
}
|
|
222
|
+
put(&mut bytes, 0x4C, child);
|
|
223
|
+
put(&mut bytes, 0x74, start);
|
|
224
|
+
put(&mut bytes, 0x78, u32::try_from(size).unwrap());
|
|
225
|
+
bytes
|
|
226
|
+
};
|
|
227
|
+
let first = u32::try_from(first_data).unwrap();
|
|
228
|
+
let mut directory = if small {
|
|
229
|
+
let mut directory = entry("Root Entry", 5, 1, first, data.len());
|
|
230
|
+
directory.extend(entry(name, 2, 0xFFFF_FFFF, 0, stream.len()));
|
|
231
|
+
directory
|
|
232
|
+
} else {
|
|
233
|
+
let mut directory = entry("Root Entry", 5, 1, END, 0);
|
|
234
|
+
directory.extend(entry(name, 2, 0xFFFF_FFFF, first, stream.len()));
|
|
235
|
+
directory
|
|
236
|
+
};
|
|
237
|
+
directory.resize(SECTOR, 0);
|
|
238
|
+
let mut file = header;
|
|
239
|
+
file.extend(fat.iter().flat_map(|word| word.to_le_bytes()));
|
|
240
|
+
file.extend(directory);
|
|
241
|
+
if small {
|
|
242
|
+
let mut mini_fat: Vec<u32> =
|
|
243
|
+
(1..data.len() / 64).map(|next| u32::try_from(next).unwrap()).collect();
|
|
244
|
+
mini_fat.push(END);
|
|
245
|
+
mini_fat.resize(SECTOR / 4, 0xFFFF_FFFF);
|
|
246
|
+
file.extend(mini_fat.iter().flat_map(|word| word.to_le_bytes()));
|
|
247
|
+
}
|
|
248
|
+
file.extend(pad(&data, SECTOR));
|
|
249
|
+
file
|
|
250
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "imprimatur_native"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.2.0"
|
|
4
4
|
description = "Ruby bindings for the imprimatur crate"
|
|
5
5
|
publish = false
|
|
6
6
|
edition.workspace = true
|
|
@@ -15,7 +15,7 @@ crate-type = ["cdylib"]
|
|
|
15
15
|
workspace = true
|
|
16
16
|
|
|
17
17
|
[dependencies]
|
|
18
|
-
imprimatur = { workspace = true, features = ["json"] }
|
|
18
|
+
imprimatur = { workspace = true, features = ["archive", "json"] }
|
|
19
19
|
serde_json.workspace = true
|
|
20
20
|
magnus.workspace = true
|
|
21
21
|
rb-sys.workspace = true
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
//! `Imprimatur::Native::ZipWriter` and `ZipReader`: zip archives of any
|
|
2
|
+
//! files, compressed and read without the GVL.
|
|
3
|
+
|
|
4
|
+
use std::cell::RefCell;
|
|
5
|
+
use std::fs::File;
|
|
6
|
+
|
|
7
|
+
use imprimatur::archive::{Archive, Writer};
|
|
8
|
+
use magnus::{Error, RString, Ruby};
|
|
9
|
+
|
|
10
|
+
use crate::blocking::without_gvl;
|
|
11
|
+
use crate::{OrRaise, borrow};
|
|
12
|
+
|
|
13
|
+
#[magnus::wrap(class = "Imprimatur::Native::ZipWriter", free_immediately, size)]
|
|
14
|
+
pub struct ZipWriter(RefCell<Option<Writer<File>>>);
|
|
15
|
+
|
|
16
|
+
fn finished(ruby: &Ruby) -> Error {
|
|
17
|
+
Error::new(ruby.exception_runtime_error(), "the archive is finished")
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
#[expect(clippy::needless_pass_by_value, reason = "magnus passes Ruby arguments as owned values")]
|
|
21
|
+
impl ZipWriter {
|
|
22
|
+
/// A new archive at `path`.
|
|
23
|
+
pub fn new(ruby: &Ruby, path: String) -> Result<Self, Error> {
|
|
24
|
+
Ok(Self(RefCell::new(Some(Writer::create(path).or_raise(ruby)?))))
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/// Adds the file at `source` as `name`, streamed.
|
|
28
|
+
pub fn add_file(
|
|
29
|
+
ruby: &Ruby,
|
|
30
|
+
rb_self: &Self,
|
|
31
|
+
name: String,
|
|
32
|
+
source: String,
|
|
33
|
+
comment: Option<String>,
|
|
34
|
+
) -> Result<(), Error> {
|
|
35
|
+
let mut slot = borrow(ruby, &rb_self.0, "zip writer")?;
|
|
36
|
+
let writer = slot.as_mut().ok_or_else(|| finished(ruby))?;
|
|
37
|
+
without_gvl(ruby, || writer.add_file(&name, &source, comment.as_deref()))?.or_raise(ruby)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/// Adds `data` as `name`.
|
|
41
|
+
pub fn add(
|
|
42
|
+
ruby: &Ruby,
|
|
43
|
+
rb_self: &Self,
|
|
44
|
+
name: String,
|
|
45
|
+
data: RString,
|
|
46
|
+
comment: Option<String>,
|
|
47
|
+
) -> Result<(), Error> {
|
|
48
|
+
// SAFETY: the bytes are copied out at once, while the GVL is held.
|
|
49
|
+
#[allow(unsafe_code)]
|
|
50
|
+
let data = unsafe { data.as_slice() }.to_vec();
|
|
51
|
+
let mut slot = borrow(ruby, &rb_self.0, "zip writer")?;
|
|
52
|
+
let writer = slot.as_mut().ok_or_else(|| finished(ruby))?;
|
|
53
|
+
without_gvl(ruby, || writer.add(&name, &data, comment.as_deref()))?.or_raise(ruby)
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/// The archive's own comment.
|
|
57
|
+
pub fn set_comment(ruby: &Ruby, rb_self: &Self, comment: String) -> Result<(), Error> {
|
|
58
|
+
let mut slot = borrow(ruby, &rb_self.0, "zip writer")?;
|
|
59
|
+
slot.as_mut().ok_or_else(|| finished(ruby))?.set_comment(&comment).or_raise(ruby)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/// Writes the archive's directory; the writer takes no more entries.
|
|
63
|
+
pub fn finish(ruby: &Ruby, rb_self: &Self) -> Result<(), Error> {
|
|
64
|
+
let writer =
|
|
65
|
+
borrow(ruby, &rb_self.0, "zip writer")?.take().ok_or_else(|| finished(ruby))?;
|
|
66
|
+
without_gvl(ruby, || writer.finish())?.or_raise(ruby).map(drop)
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/// An entry for Ruby: name, size, compressed size, CRC-32, comment.
|
|
71
|
+
type EntryRow = (String, u64, u64, u32, String);
|
|
72
|
+
|
|
73
|
+
#[magnus::wrap(class = "Imprimatur::Native::ZipReader", free_immediately, size)]
|
|
74
|
+
pub struct ZipReader(RefCell<Archive<File>>);
|
|
75
|
+
|
|
76
|
+
#[expect(clippy::needless_pass_by_value, reason = "magnus passes Ruby arguments as owned values")]
|
|
77
|
+
impl ZipReader {
|
|
78
|
+
/// The archive at `path`.
|
|
79
|
+
pub fn new(ruby: &Ruby, path: String) -> Result<Self, Error> {
|
|
80
|
+
Ok(Self(RefCell::new(Archive::open(path).or_raise(ruby)?)))
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
pub fn names(ruby: &Ruby, rb_self: &Self) -> Result<Vec<String>, Error> {
|
|
84
|
+
Ok(borrow(ruby, &rb_self.0, "zip archive")?.names())
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/// Each entry's name, size, compressed size, CRC-32 and comment.
|
|
88
|
+
pub fn entries(ruby: &Ruby, rb_self: &Self) -> Result<Vec<EntryRow>, Error> {
|
|
89
|
+
let entries = borrow(ruby, &rb_self.0, "zip archive")?.entries().or_raise(ruby)?;
|
|
90
|
+
Ok(entries
|
|
91
|
+
.into_iter()
|
|
92
|
+
.map(|entry| {
|
|
93
|
+
(entry.name, entry.size, entry.compressed_size, entry.crc32, entry.comment)
|
|
94
|
+
})
|
|
95
|
+
.collect())
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/// The entry's bytes, `nil` when there is none.
|
|
99
|
+
pub fn read(
|
|
100
|
+
ruby: &Ruby,
|
|
101
|
+
rb_self: &Self,
|
|
102
|
+
name: String,
|
|
103
|
+
limit: u64,
|
|
104
|
+
) -> Result<Option<RString>, Error> {
|
|
105
|
+
let mut archive = borrow(ruby, &rb_self.0, "zip archive")?;
|
|
106
|
+
let data = without_gvl(ruby, || archive.read(&name, limit))?.or_raise(ruby)?;
|
|
107
|
+
Ok(data.map(|data| ruby.str_from_slice(&data)))
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/// Writes the entry to `path`; whether there was one.
|
|
111
|
+
pub fn extract(
|
|
112
|
+
ruby: &Ruby,
|
|
113
|
+
rb_self: &Self,
|
|
114
|
+
name: String,
|
|
115
|
+
path: String,
|
|
116
|
+
limit: u64,
|
|
117
|
+
) -> Result<bool, Error> {
|
|
118
|
+
let mut archive = borrow(ruby, &rb_self.0, "zip archive")?;
|
|
119
|
+
without_gvl(ruby, || archive.extract(&name, &path, limit))?.or_raise(ruby)
|
|
120
|
+
}
|
|
121
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
//! Work run without the GVL that cannot stop midway (compressing a file):
|
|
2
|
+
//! Ruby's other threads keep running, and an interrupt waits for the end.
|
|
3
|
+
|
|
4
|
+
use std::ffi::c_void;
|
|
5
|
+
use std::panic::{AssertUnwindSafe, catch_unwind};
|
|
6
|
+
|
|
7
|
+
use magnus::{Error, Ruby};
|
|
8
|
+
|
|
9
|
+
/// Runs interrupted before they start are retried this many times.
|
|
10
|
+
pub const MAX_ATTEMPTS: usize = 64;
|
|
11
|
+
|
|
12
|
+
/// The job, and what it returned (or that it panicked).
|
|
13
|
+
struct Slot<F, T> {
|
|
14
|
+
job: Option<F>,
|
|
15
|
+
result: Option<std::thread::Result<T>>,
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/// Runs `job` on this thread with the GVL released.
|
|
19
|
+
pub fn without_gvl<F: FnOnce() -> T, T>(ruby: &Ruby, job: F) -> Result<T, Error> {
|
|
20
|
+
let mut slot = Slot { job: Some(job), result: None };
|
|
21
|
+
let pointer: *mut Slot<F, T> = &raw mut slot;
|
|
22
|
+
for _ in 0..MAX_ATTEMPTS {
|
|
23
|
+
// SAFETY: `slot` outlives the call, which returns once `run` has
|
|
24
|
+
// finished or never started; `run` touches no Ruby object. Without
|
|
25
|
+
// an unblocking function the job runs to its end; the `2` variant
|
|
26
|
+
// raises no pending interrupt across these Rust frames.
|
|
27
|
+
#[allow(unsafe_code)]
|
|
28
|
+
unsafe {
|
|
29
|
+
rb_sys::rb_thread_call_without_gvl2(
|
|
30
|
+
Some(run::<F, T>),
|
|
31
|
+
pointer.cast(),
|
|
32
|
+
None,
|
|
33
|
+
std::ptr::null_mut(),
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
match slot.result.take() {
|
|
37
|
+
Some(Ok(value)) => return Ok(value),
|
|
38
|
+
Some(Err(_)) => {
|
|
39
|
+
return Err(Error::new(
|
|
40
|
+
ruby.exception_runtime_error(),
|
|
41
|
+
"imprimatur failed on this archive",
|
|
42
|
+
));
|
|
43
|
+
}
|
|
44
|
+
// An interrupt was pending, so the job never started: let Ruby
|
|
45
|
+
// handle it (it may raise), then start.
|
|
46
|
+
None => ruby.thread_check_ints()?,
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
Err(Error::new(ruby.exception_runtime_error(), "interrupted before it could start"))
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/// The job itself, without the GVL; a panic becomes an error, not an
|
|
53
|
+
/// abort at this C boundary.
|
|
54
|
+
#[allow(unsafe_code)]
|
|
55
|
+
unsafe extern "C" fn run<F: FnOnce() -> T, T>(data: *mut c_void) -> *mut c_void {
|
|
56
|
+
// SAFETY: `data` is the `Slot` without_gvl passed, alive and not
|
|
57
|
+
// otherwise borrowed until this returns.
|
|
58
|
+
let slot = unsafe { &mut *data.cast::<Slot<F, T>>() };
|
|
59
|
+
if let Some(job) = slot.job.take() {
|
|
60
|
+
slot.result = Some(catch_unwind(AssertUnwindSafe(job)));
|
|
61
|
+
}
|
|
62
|
+
std::ptr::null_mut()
|
|
63
|
+
}
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
//! [`imprimatur::xlsx::Format`]. The Ruby DSL in `lib/imprimatur/xlsx.rb`
|
|
4
4
|
//! drives them; sheets cross as indices.
|
|
5
5
|
|
|
6
|
+
mod archive;
|
|
7
|
+
mod blocking;
|
|
6
8
|
mod convert;
|
|
7
9
|
mod document;
|
|
8
10
|
mod read;
|
|
@@ -284,6 +286,18 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
|
|
|
284
286
|
native.define_singleton_method("markdown", function!(read::markdown_native, 4))?;
|
|
285
287
|
native.define_singleton_method("content", function!(read::content_native, 4))?;
|
|
286
288
|
native.define_singleton_method("images", function!(read::images_native, 4))?;
|
|
289
|
+
let zip_writer = native.define_class("ZipWriter", ruby.class_object())?;
|
|
290
|
+
zip_writer.define_singleton_method("new", function!(archive::ZipWriter::new, 1))?;
|
|
291
|
+
zip_writer.define_method("add_file", method!(archive::ZipWriter::add_file, 3))?;
|
|
292
|
+
zip_writer.define_method("add", method!(archive::ZipWriter::add, 3))?;
|
|
293
|
+
zip_writer.define_method("comment=", method!(archive::ZipWriter::set_comment, 1))?;
|
|
294
|
+
zip_writer.define_method("finish", method!(archive::ZipWriter::finish, 0))?;
|
|
295
|
+
let zip_reader = native.define_class("ZipReader", ruby.class_object())?;
|
|
296
|
+
zip_reader.define_singleton_method("new", function!(archive::ZipReader::new, 1))?;
|
|
297
|
+
zip_reader.define_method("names", method!(archive::ZipReader::names, 0))?;
|
|
298
|
+
zip_reader.define_method("entries", method!(archive::ZipReader::entries, 0))?;
|
|
299
|
+
zip_reader.define_method("read", method!(archive::ZipReader::read, 2))?;
|
|
300
|
+
zip_reader.define_method("extract", method!(archive::ZipReader::extract, 3))?;
|
|
287
301
|
document::define(ruby, native)?;
|
|
288
302
|
let workbook = native.define_class("Workbook", ruby.class_object())?;
|
|
289
303
|
workbook.define_singleton_method("new", function!(NativeWorkbook::new, 0))?;
|
|
@@ -13,11 +13,9 @@ use imprimatur::read::{Limits, content, images, json, markdown};
|
|
|
13
13
|
use magnus::{Error, IntoValue, RArray, RString, Ruby};
|
|
14
14
|
use serde_json::Value as Json;
|
|
15
15
|
|
|
16
|
+
use crate::blocking::MAX_ATTEMPTS;
|
|
16
17
|
use crate::{argument_error, domain_error};
|
|
17
18
|
|
|
18
|
-
/// Reads interrupted without anything to raise are retried this many times.
|
|
19
|
-
const MAX_ATTEMPTS: usize = 64;
|
|
20
|
-
|
|
21
19
|
/// A read of a document's bytes within limits, run without the GVL.
|
|
22
20
|
type Job<T> = fn(&[u8], &Limits) -> imprimatur::Result<T>;
|
|
23
21
|
|
data/lib/imprimatur/error.rb
CHANGED
|
@@ -9,11 +9,12 @@ module Imprimatur
|
|
|
9
9
|
# A PDF without a text layer: its pages are images (scans), which need OCR.
|
|
10
10
|
class NoTextLayer < Error; end
|
|
11
11
|
|
|
12
|
-
# A PDF encrypted with a user password
|
|
12
|
+
# A PDF encrypted with a user password (only an empty one is tried), or
|
|
13
|
+
# an encrypted legacy +.xls+.
|
|
13
14
|
class PasswordRequired < Error; end
|
|
14
15
|
|
|
15
|
-
# A format {Imprimatur.markdown} does not read: legacy +.
|
|
16
|
-
#
|
|
16
|
+
# A format {Imprimatur.markdown} does not read: legacy +.doc+, binary
|
|
17
|
+
# +.xlsb+, presentations, anything not a PDF, XLSX, XLS, ODS or DOCX.
|
|
17
18
|
class UnsupportedDocument < Error; end
|
|
18
19
|
|
|
19
20
|
# The read hit one of its limits (size, decompressed size, nesting, time),
|
data/lib/imprimatur/reader.rb
CHANGED
|
@@ -14,7 +14,7 @@ module Imprimatur
|
|
|
14
14
|
module Reader
|
|
15
15
|
# File signatures a document's bytes start with; a String that starts
|
|
16
16
|
# with neither is taken for a path.
|
|
17
|
-
SIGNATURES = ["%PDF-".b, "PK\x03\x04".b, "PK\x05\x06".b].freeze
|
|
17
|
+
SIGNATURES = ["%PDF-".b, "PK\x03\x04".b, "PK\x05\x06".b, "\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1".b].freeze
|
|
18
18
|
private_constant :SIGNATURES
|
|
19
19
|
|
|
20
20
|
# @see Imprimatur.markdown
|
data/lib/imprimatur/version.rb
CHANGED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'fileutils'
|
|
4
|
+
|
|
5
|
+
module Imprimatur
|
|
6
|
+
# Zip archives of any files: entries written with their modification times
|
|
7
|
+
# and comments (ZIP64 when large), read one at a time within a size limit.
|
|
8
|
+
# Compressing and reading run without the GVL.
|
|
9
|
+
#
|
|
10
|
+
# Imprimatur::Zip.write('export.zip') do |zip|
|
|
11
|
+
# zip.add_file('contracts/0contract.pdf', '/tmp/contract.pdf', comment: 'Contract')
|
|
12
|
+
# zip.add('manifest.json', JSON.generate(manifest))
|
|
13
|
+
# end
|
|
14
|
+
#
|
|
15
|
+
# Imprimatur::Zip.open('geolite.zip') do |zip|
|
|
16
|
+
# zip.read(zip.glob('**/*-Locations-en.csv').first)
|
|
17
|
+
# end
|
|
18
|
+
module Zip
|
|
19
|
+
# Bytes an entry may hold once decompressed by default: 1 GiB.
|
|
20
|
+
MAX_ENTRY_BYTES = 1024**3
|
|
21
|
+
|
|
22
|
+
# An entry added to an archive being written.
|
|
23
|
+
Added = Data.define(:name, :comment)
|
|
24
|
+
|
|
25
|
+
# An entry of an archive being read, as its directory states it.
|
|
26
|
+
Entry = Data.define(:name, :size, :compressed_size, :crc32, :comment)
|
|
27
|
+
|
|
28
|
+
# Writes an archive at +path+, the block adding its entries. On an
|
|
29
|
+
# error, the partial file is removed.
|
|
30
|
+
#
|
|
31
|
+
# @yieldparam zip [Writer]
|
|
32
|
+
# @return the path
|
|
33
|
+
def self.write(path)
|
|
34
|
+
writer = Writer.new(path)
|
|
35
|
+
yield writer
|
|
36
|
+
writer.finish
|
|
37
|
+
path
|
|
38
|
+
rescue Exception # any failure, an interrupt included: the partial file goes, then the error
|
|
39
|
+
FileUtils.rm_f(path.to_s) if writer
|
|
40
|
+
raise
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# The archive at +path+.
|
|
44
|
+
#
|
|
45
|
+
# @yieldparam zip [Reader] when a block is given
|
|
46
|
+
# @return [Reader, Object] the reader, or what the block returns
|
|
47
|
+
# @raise [Errno::ENOENT] when there is no such file
|
|
48
|
+
# @raise [Error] when it is no zip
|
|
49
|
+
def self.open(path)
|
|
50
|
+
reader = Reader.new(path)
|
|
51
|
+
block_given? ? yield(reader) : reader
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Adds entries to an archive being written; see {Zip.write}.
|
|
55
|
+
class Writer
|
|
56
|
+
# @return [Array<Added>] the entries added so far, in order
|
|
57
|
+
attr_reader :entries
|
|
58
|
+
|
|
59
|
+
def initialize(path)
|
|
60
|
+
@native = Native::ZipWriter.new(path.to_s)
|
|
61
|
+
@entries = []
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Adds the file at +source+ as +name+, streamed from disk, with its
|
|
65
|
+
# modification time.
|
|
66
|
+
def add_file(name, source, comment: nil)
|
|
67
|
+
@native.add_file(name.to_s, source.to_s, comment&.to_s)
|
|
68
|
+
@entries << Added.new(name: name.to_s, comment: comment&.to_s)
|
|
69
|
+
self
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Adds +data+ as +name+, dated now.
|
|
73
|
+
def add(name, data, comment: nil)
|
|
74
|
+
@native.add(name.to_s, data.to_s.b, comment&.to_s)
|
|
75
|
+
@entries << Added.new(name: name.to_s, comment: comment&.to_s)
|
|
76
|
+
self
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# The archive's own comment, shown by most zip tools.
|
|
80
|
+
def comment=(comment)
|
|
81
|
+
@native.comment = comment.to_s
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# @api private
|
|
85
|
+
def finish = @native.finish
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# An archive's entries, by name; see {Zip.open}.
|
|
89
|
+
class Reader
|
|
90
|
+
# @return [Array<String>] the entries' names, in the archive's order
|
|
91
|
+
attr_reader :names
|
|
92
|
+
|
|
93
|
+
def initialize(path)
|
|
94
|
+
raise Errno::ENOENT, path.to_s unless File.file?(path.to_s)
|
|
95
|
+
|
|
96
|
+
@native = Native::ZipReader.new(path.to_s)
|
|
97
|
+
@names = @native.names.freeze
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Each entry's name, sizes, CRC-32 and comment, nothing decompressed.
|
|
101
|
+
#
|
|
102
|
+
# @return [Array<Entry>]
|
|
103
|
+
def entries
|
|
104
|
+
@native.entries.map { |row| Entry.new(*row) }
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# The names matching a shell pattern, +**/+ spanning folders.
|
|
108
|
+
def glob(pattern) = names.select { File.fnmatch?(pattern, it, File::FNM_PATHNAME | File::FNM_EXTGLOB) }
|
|
109
|
+
|
|
110
|
+
# The entry's bytes, binary.
|
|
111
|
+
#
|
|
112
|
+
# @raise [KeyError] for an entry the archive does not have
|
|
113
|
+
# @raise [LimitExceeded] when it holds more than +max_bytes+
|
|
114
|
+
def read(name, max_bytes: MAX_ENTRY_BYTES)
|
|
115
|
+
@native.read(name.to_s, Integer(max_bytes)) or raise KeyError, "no entry #{name} in the archive"
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Writes the entry to +path+, streamed; returns +path+.
|
|
119
|
+
#
|
|
120
|
+
# @raise (see #read)
|
|
121
|
+
def extract(name, path, max_bytes: MAX_ENTRY_BYTES)
|
|
122
|
+
@native.extract(name.to_s, path.to_s, Integer(max_bytes)) or raise KeyError, "no entry #{name} in the archive"
|
|
123
|
+
path
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
data/lib/imprimatur.rb
CHANGED
|
@@ -9,6 +9,7 @@ require_relative 'imprimatur/validation'
|
|
|
9
9
|
require_relative 'imprimatur/xlsx'
|
|
10
10
|
require_relative 'imprimatur/document'
|
|
11
11
|
require_relative 'imprimatur/reader'
|
|
12
|
+
require_relative 'imprimatur/zip'
|
|
12
13
|
|
|
13
14
|
# Document authoring with a native core: XLSX workbooks, and PDF and DOCX from
|
|
14
15
|
# one document model.
|
|
@@ -46,7 +47,7 @@ module Imprimatur
|
|
|
46
47
|
# As {.pdf}, rendered as DOCX.
|
|
47
48
|
def docx(path = nil, **options, &) = Document.build(:docx, path, **options, &)
|
|
48
49
|
|
|
49
|
-
# A PDF, XLSX or DOCX as Markdown: a PDF's text layer (headings by font
|
|
50
|
+
# A PDF, workbook (XLSX, XLS, ODS) or DOCX as Markdown: a PDF's text layer (headings by font
|
|
50
51
|
# size, running headers and footers left out, pages marked with
|
|
51
52
|
# +<!-- page N -->+), each sheet of a workbook as a table, a Word
|
|
52
53
|
# document's headings, lists, emphasis, links and tables. The format is
|
|
@@ -67,7 +68,7 @@ module Imprimatur
|
|
|
67
68
|
# @return [String] Markdown, UTF-8
|
|
68
69
|
# @raise [NoTextLayer] a PDF of scanned pages: run OCR instead
|
|
69
70
|
# @raise [PasswordRequired] a PDF that needs a password to open
|
|
70
|
-
# @raise [UnsupportedDocument] not a PDF, XLSX or DOCX
|
|
71
|
+
# @raise [UnsupportedDocument] not a PDF, XLSX, XLS, ODS or DOCX
|
|
71
72
|
# @raise [LimitExceeded] the document is too large, too deep or too slow
|
|
72
73
|
# @raise [Error] a damaged document
|
|
73
74
|
def markdown(document, ...) = Reader.markdown(document, ...)
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: imprimatur
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Abdelkader Boudih
|
|
@@ -25,7 +25,8 @@ dependencies:
|
|
|
25
25
|
version: 0.9.130
|
|
26
26
|
description: 'A Ruby DSL over a native core that writes Excel workbooks, and PDF and
|
|
27
27
|
Word documents from one model, with its own XML, zip and PDF writers: no third-party
|
|
28
|
-
format stack.
|
|
28
|
+
format stack. It reads PDF, XLSX, XLS, ODS and DOCX back: as Markdown, as typed
|
|
29
|
+
structure, and their images. Precompiled for Linux, macOS and FreeBSD; CRuby only.'
|
|
29
30
|
email:
|
|
30
31
|
- terminale@gmail.com
|
|
31
32
|
executables: []
|
|
@@ -50,6 +51,7 @@ files:
|
|
|
50
51
|
- ext/imprimatur_native/core/Cargo.toml
|
|
51
52
|
- ext/imprimatur_native/core/LICENSE
|
|
52
53
|
- ext/imprimatur_native/core/README.md
|
|
54
|
+
- ext/imprimatur_native/core/src/archive.rs
|
|
53
55
|
- ext/imprimatur_native/core/src/doc.rs
|
|
54
56
|
- ext/imprimatur_native/core/src/doc/paragraph.rs
|
|
55
57
|
- ext/imprimatur_native/core/src/doc/table.rs
|
|
@@ -73,6 +75,9 @@ files:
|
|
|
73
75
|
- ext/imprimatur_native/core/src/read/inflate.rs
|
|
74
76
|
- ext/imprimatur_native/core/src/read/json.rs
|
|
75
77
|
- ext/imprimatur_native/core/src/read/markdown.rs
|
|
78
|
+
- ext/imprimatur_native/core/src/read/numbers.rs
|
|
79
|
+
- ext/imprimatur_native/core/src/read/ods.rs
|
|
80
|
+
- ext/imprimatur_native/core/src/read/ole.rs
|
|
76
81
|
- ext/imprimatur_native/core/src/read/package.rs
|
|
77
82
|
- ext/imprimatur_native/core/src/read/pdf.rs
|
|
78
83
|
- ext/imprimatur_native/core/src/read/pdf/cmap.rs
|
|
@@ -92,8 +97,9 @@ files:
|
|
|
92
97
|
- ext/imprimatur_native/core/src/read/pdf/object/name.rs
|
|
93
98
|
- ext/imprimatur_native/core/src/read/pdf/xref.rs
|
|
94
99
|
- ext/imprimatur_native/core/src/read/span.rs
|
|
100
|
+
- ext/imprimatur_native/core/src/read/xls.rs
|
|
101
|
+
- ext/imprimatur_native/core/src/read/xls/strings.rs
|
|
95
102
|
- ext/imprimatur_native/core/src/read/xlsx.rs
|
|
96
|
-
- ext/imprimatur_native/core/src/read/xlsx/numbers.rs
|
|
97
103
|
- ext/imprimatur_native/core/src/style.rs
|
|
98
104
|
- ext/imprimatur_native/core/src/test_support.rs
|
|
99
105
|
- ext/imprimatur_native/core/src/xlsx.rs
|
|
@@ -113,6 +119,8 @@ files:
|
|
|
113
119
|
- ext/imprimatur_native/core/src/zip/reader.rs
|
|
114
120
|
- ext/imprimatur_native/extconf.rb
|
|
115
121
|
- ext/imprimatur_native/ffi/Cargo.toml
|
|
122
|
+
- ext/imprimatur_native/ffi/src/archive.rs
|
|
123
|
+
- ext/imprimatur_native/ffi/src/blocking.rs
|
|
116
124
|
- ext/imprimatur_native/ffi/src/convert.rs
|
|
117
125
|
- ext/imprimatur_native/ffi/src/document.rs
|
|
118
126
|
- ext/imprimatur_native/ffi/src/lib.rs
|
|
@@ -129,6 +137,7 @@ files:
|
|
|
129
137
|
- lib/imprimatur/validation.rb
|
|
130
138
|
- lib/imprimatur/version.rb
|
|
131
139
|
- lib/imprimatur/xlsx.rb
|
|
140
|
+
- lib/imprimatur/zip.rb
|
|
132
141
|
homepage: https://github.com/seuros/imprimatur
|
|
133
142
|
licenses:
|
|
134
143
|
- BSD-3-Clause
|
|
@@ -153,5 +162,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
153
162
|
requirements: []
|
|
154
163
|
rubygems_version: 4.0.20
|
|
155
164
|
specification_version: 4
|
|
156
|
-
summary: XLSX, PDF and DOCX authoring
|
|
165
|
+
summary: XLSX, PDF and DOCX authoring, and office documents read back. Let it be printed.
|
|
157
166
|
test_files: []
|