imprimatur 0.7.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9812ca055242755ab6117d2c5eba9bb1991e02a21572682f9f36c2a0481e2139
-  data.tar.gz: 8e1d32d440ec79ea732c62dd0a830c0b13f76655df7f9b85d60d539e0d429936
+  metadata.gz: 20a7d3843214018bf15d889668ad1d33ba8dfee183b86e041d56b5cc2db88832
+  data.tar.gz: 6699fc743a0a04fffbaf67d68551ca680c416cdc56f1fd71984ec2f7cba8e5b1
 SHA512:
-  metadata.gz: 3572b1a17819ae0fa85346788343f19ef9410d19f48b7257edb95dc00544ae1234991a5d0b50284ed3d10e21f34ca991f965aa7956fecb6a03244805870cea13
-  data.tar.gz: 9399ac644f99a699a12a2a8bdcefe73977bddcec75571e8053d984d829ddc33817fd40bb6c9c79fd15ce89f56fbe472035c276e3177430b9a35ab20023b66f16
+  metadata.gz: ef0a7f9c8d545aa513881626e798702095c66a42a5665b663a56b4c6205e5a0c09596366d9dae05c6ac6539f46ba242744feb2ab99080e906b418c0af2550625
+  data.tar.gz: 38ccdff0d6b4d2bfcb516660de96f1274ac0d1cef01b9d97f2790b0223b041ec2fd5b747be21a053a368c0bf58e33e2f8b9c608ed1bd5d5ea2e1da7bfd218615

data/CHANGELOG.md

@@ -11,6 +11,26 @@
 - Allow a bounded exit grace when macOS cannot inspect an exiting worker;
   persistent monitor failures still terminate and reap a live worker.
 
+## [0.8.1](https://github.com/seuros/imprimatur/compare/v0.8.0...v0.8.1) (2026-10-29)
+
+
+### Bug Fixes
+
+* **pdf:** recover timeout neighbours within the overall deadline ([#16](https://github.com/seuros/imprimatur/issues/16)) ([e86fbbf](https://github.com/seuros/imprimatur/commit/e86fbbfef85f2c645090c43359c0744962e951a1))
+* **pdf:** step timeout recovery so a Ruby wakeup cannot restart it ([75c3894](https://github.com/seuros/imprimatur/commit/75c3894c8a791c5ef4af562dc99de4e745770ad3))
+
+## [0.8.0](https://github.com/seuros/imprimatur/compare/v0.7.0...v0.8.0) (2026-10-27)
+
+
+### Features
+
+* **reader:** accept drop_repeated in the Ruby API ([4e537d6](https://github.com/seuros/imprimatur/commit/4e537d641bafd8d3cdb8b1c20d16f4cb766ad073))
+
+
+### Bug Fixes
+
+* **pdf:** read printed page numbers from headers too, not just footers ([c036434](https://github.com/seuros/imprimatur/commit/c0364341ee8fc36be2c97d25fd06da0aa76a4ad6))
+
 ## [0.7.0](https://github.com/seuros/imprimatur/compare/v0.6.1...v0.7.0) (2026-10-26)
 
 

data/Cargo.lock

@@ -513,7 +513,7 @@ dependencies = [
 
 [[package]]
 name = "imprimatur"
-version = "0.7.0"
+version = "0.8.1"
 dependencies = [
  "aes",
  "crc32fast",
@@ -536,7 +536,7 @@ dependencies = [
 
 [[package]]
 name = "imprimatur-cli"
-version = "0.7.0"
+version = "0.8.1"
 dependencies = [
  "imprimatur",
  "regex",
@@ -548,7 +548,7 @@ dependencies = [
 
 [[package]]
 name = "imprimatur_native"
-version = "0.7.0"
+version = "0.8.1"
 dependencies = [
  "imprimatur",
  "magnus",

data/Cargo.toml

@@ -15,7 +15,7 @@ crc32fast = "1.5.2"
 datamatrix = "0.3.3"
 font-test-data = "0.10.0"
 hayro = { version = "0.8.0", default-features = false, features = ["embed-fonts", "embed-cmaps"] }
-imprimatur = { path = "ext/imprimatur_native/core", version = "0.7.0" }
+imprimatur = { path = "ext/imprimatur_native/core", version = "0.8.1" }
 libproc = "0.14.11"
 magnus = { version = "0.9.2", default-features = false }
 md-5 = "0.11.0"

data/README.md

@@ -32,7 +32,7 @@ end
 gem "imprimatur"
 ```
 
-Precompiled gems ship for Linux (x86_64 and aarch64, glibc 2.34+ and musl),
+Precompiled gems ship for Linux (x86_64 and aarch64, glibc 2.34+),
 macOS 13+ (arm64, x86_64) and FreeBSD 15.1 (amd64): nothing to compile.
 Anywhere else, the source gem compiles the extension on install, with the
 toolchain from [rustup.rs](https://rustup.rs) (1.99+).
@@ -384,6 +384,28 @@ after metadata was read preserves completed pages and explicitly marks the
 unreached pages; an expired overall deadline marks them `skipped`. Cancellation
 still raises rather than returning a partial successful call.
 
+For timeout-neighbour recovery, opt in to a shorter worker-attempt budget:
+
+```ruby
+Imprimatur.pdf_pages(bytes, timeout: 300, batch_timeout: 60)
+```
+
+Here `timeout:` is the **overall** deadline, including recovery; `batch_timeout:`
+caps the initial worker and each singleton retry. Completed pages survive a
+batch timeout. Unread later neighbours are tried once before the first missing
+page (which may have stalled the worker). Recovery is best effort, never an
+extension of the overall deadline. Output-limit skips are not retried, and all
+attempts share `max_output_bytes:`. Input, depth, process-memory and per-page
+limits still apply; `max_inflated:` remains per worker/recovery attempt, not a
+cumulative decompression allowance.
+
+With `batch_timeout:` enabled, an explicit finite `timeout:` replaces the
+ordinary 60-second process ceiling for this call; `timeout: nil` retains that
+60-second ceiling. Both budgets must be positive; zero overall time still fails
+immediately. Omitting `batch_timeout:` preserves the existing behaviour. Rust
+callers can use `pages_isolated_recovering` with an explicit attempt duration
+and overall `Limits`/`ProcessLimits`.
+
 This is resource containment, **not a filesystem/network security sandbox**.
 The raw crate `info`, `pages` and `render_page` APIs remain in-process;
 `render_page` cannot interrupt Hayro.

data/ext/imprimatur_native/cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "imprimatur-cli"
-version = "0.7.0"
+version = "0.8.1"
 description = "Read PDF, XLSX, XLS, ODS and DOCX documents as Markdown or JSON, search them, extract their images."
 keywords = ["pdf", "xlsx", "docx", "markdown", "cli"]
 categories = ["command-line-utilities", "text-processing"]

data/ext/imprimatur_native/cli/src/cli/markdown.rs

@@ -16,7 +16,8 @@ use crate::app::{self, App, Out, Status};
 /// PDF: the text layer, placed glyph by glyph, with headings by font size,
 /// running headers and footers left out, and pages after the first marked
 /// `<!-- page N -->` (or `<!-- page N (label L) -->` with its printed page
-/// label, from `/PageLabels` or a footer that looks like one). Encrypted
+/// label, from `/PageLabels` or a header or footer line that looks like
+/// one). Encrypted
 /// PDFs open when their user password is empty. XLSX, XLS and ODS: each
 /// sheet as a table, values as shown, then its comments. DOCX: headings,
 /// lists, emphasis, links and tables.

data/ext/imprimatur_native/core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "imprimatur"
-version = "0.7.0"
+version = "0.8.1"
 description = "XLSX, PDF and DOCX authoring, and PDF, XLSX, XLS, ODS and DOCX reading, with its own XML, zip and PDF code."
 keywords = ["xlsx", "excel", "pdf", "docx", "spreadsheet"]
 categories = ["encoding", "text-processing"]

data/ext/imprimatur_native/core/src/read/budget.rs

@@ -45,6 +45,12 @@ impl Default for Limits {
 }
 
 impl Limits {
+    /// The configured overall read timeout, if enabled.
+    #[must_use]
+    pub const fn timeout_duration(&self) -> Option<Duration> {
+        self.timeout
+    }
+
     /// The largest document accepted, in bytes; 512 MiB by default.
     #[must_use]
     pub const fn max_input(mut self, bytes: usize) -> Self {

data/ext/imprimatur_native/core/src/read/markdown.rs

@@ -44,7 +44,10 @@ pub enum Mode {
 }
 
 fn pdf(out: &mut String, pages: &[Page]) {
-    for page in pages.iter().filter(|page| !page.blocks.is_empty()) {
+    // Every page past the first gets its marker, blocks or none: an empty
+    // page (left bare by `--drop-repeated`, or genuinely blank) still needs
+    // one, so the markers stay a reliable page count and index.
+    for page in pages {
         if page.number > 1 {
             end_block(out);
             // Writing to a String cannot fail.

data/ext/imprimatur_native/core/src/read/pdf/isolation/protocol.rs

@@ -60,7 +60,7 @@ impl Request {
         if process.memory_bytes == 0 || process.memory_bytes >= i64::MAX as u64 {
             return Err(Error::InvalidPdfOption("memory limit must be positive and finite"));
         }
-        let wall = limits.timeout.unwrap_or(process.wall_time).min(process.wall_time);
+        let wall = process.wall_time_for(limits);
         let cpu_seconds = wall
             .as_secs()
             .checked_add(u64::from(wall.subsec_nanos() != 0))
@@ -165,6 +165,7 @@ pub(super) fn failure(error: &Error) -> (u8, String) {
     let kind = match error {
         Error::PasswordRequired => 1,
         Error::UnsupportedDocument(_) => 2,
+        Error::LimitExceeded("time") => 6,
         Error::LimitExceeded(_) | Error::Cancelled => 3,
         Error::InvalidPdfOption(_) => 4,
         _ => 5,
@@ -184,6 +185,7 @@ pub(super) fn error(kind: u8, message: String) -> Error {
         2 => Error::UnsupportedDocument(message),
         3 => Error::LimitExceeded("PDF worker limit"),
         4 => Error::InvalidPdfOption("the PDF worker rejected the options"),
+        6 => Error::LimitExceeded("time"),
         _ => Error::malformed("PDF worker", message),
     }
 }

data/ext/imprimatur_native/core/src/read/pdf/isolation/recovery.rs

@@ -0,0 +1,158 @@
+//! Optional timeout-neighbour recovery under one overall supervisor deadline.
+
+use std::iter;
+use std::path::Path;
+use std::time::{Duration, Instant};
+
+use super::{ProcessLimits, invoke, protocol, stream};
+use crate::read::Limits;
+use crate::read::pdf::{Pages, ReadOptions, Status, bounded};
+use crate::{Error, Result};
+
+#[cfg(test)]
+mod tests;
+
+/// Reads with a shorter worker-attempt timeout, recovering unread neighbours.
+///
+/// `limits.timeout` and `process.wall_time` bound the entire operation, not
+/// each attempt. Completed records survive; timeout-unread pages are tried
+/// once individually, later neighbours before the first missing page.
+/// All workers retain the input, decompression, depth and process-memory caps.
+/// Decompression is bounded per worker, as in ordinary page-local recovery;
+/// retained text shares `options.max_output_bytes` across all attempts.
+///
+/// # Errors
+/// As [`super::pages_isolated`], or a zero attempt timeout.
+pub fn pages_isolated_recovering(
+    worker: &Path,
+    data: &[u8],
+    options: &ReadOptions,
+    limits: &Limits,
+    process: &ProcessLimits,
+    attempt_timeout: Duration,
+) -> Result<Pages> {
+    let mut recovery = Recovery::new(options, attempt_timeout)?;
+    loop {
+        if let Some(pages) = recovery.step(worker, data, limits, process)? {
+            return Ok(pages);
+        }
+    }
+}
+
+/// [`pages_isolated_recovering`], one worker attempt per [`Recovery::step`].
+///
+/// A step that fails keeps the progress before it, and the deadline set by
+/// the first step: a caller woken spuriously can step again.
+pub struct Recovery {
+    options: ReadOptions,
+    attempt_timeout: Duration,
+    deadline: Option<Instant>,
+    result: Option<Pages>,
+    pending: Vec<usize>,
+    remaining: usize,
+}
+
+impl Recovery {
+    /// # Errors
+    /// A zero attempt timeout.
+    pub fn new(options: &ReadOptions, attempt_timeout: Duration) -> Result<Self> {
+        if attempt_timeout.is_zero() {
+            return Err(Error::InvalidPdfOption("batch timeout must be positive"));
+        }
+        Ok(Self {
+            options: options.clone(),
+            attempt_timeout,
+            deadline: None,
+            result: None,
+            pending: Vec::new(),
+            remaining: options.max_output_bytes,
+        })
+    }
+
+    /// Runs the next worker attempt; the pages once recovery is over.
+    ///
+    /// # Errors
+    /// As [`pages_isolated_recovering`].
+    pub fn step(
+        &mut self,
+        worker: &Path,
+        data: &[u8],
+        limits: &Limits,
+        process: &ProcessLimits,
+    ) -> Result<Option<Pages>> {
+        let deadline = self.deadline(limits, process)?;
+        let attempt_timeout = self.attempt_timeout;
+        let attempt = |selection: &ReadOptions| {
+            let wall_time = deadline.saturating_duration_since(Instant::now()).min(attempt_timeout);
+            if wall_time.is_zero() {
+                return Err(Error::LimitExceeded("time"));
+            }
+            let process = ProcessLimits { wall_time, ..process.clone() };
+            let operation = protocol::Operation::Pages(selection.clone());
+            stream::decode_recoverable(
+                invoke(worker, data, operation, limits, &process)?,
+                selection,
+            )
+        };
+        let Some(result) = &mut self.result else {
+            let (pages, first_missing) = attempt(&self.options)?;
+            let Some(first_missing) = first_missing else { return Ok(Some(pages)) };
+            let count = pages.pages.len();
+            self.pending = (first_missing + 1..count)
+                .chain(iter::once(first_missing))
+                .filter(|&index| index < count)
+                .rev()
+                .collect();
+            self.remaining -= pages.pages.iter().map(|page| page.text.len()).sum::<usize>();
+            self.result = Some(pages);
+            return Ok(self.settle());
+        };
+        let Some(&index) = self.pending.last() else { return Ok(self.result.take()) };
+        let number = result.pages[index].number;
+        let selection = ReadOptions {
+            first_page: number,
+            last_page: number,
+            max_pages: 1,
+            max_output_bytes: self.remaining,
+            ..self.options.clone()
+        };
+        match attempt(&selection) {
+            Ok((mut single, _)) => {
+                if single.info != result.info || single.pages.len() != 1 {
+                    return Err(Error::malformed("PDF worker", "inconsistent recovery metadata"));
+                }
+                let page = single.pages.remove(0);
+                self.remaining -= page.text.len();
+                result.pages[index] = page;
+            }
+            Err(error @ Error::LimitExceeded(reason)) if reason != "PDF worker output" => {
+                // A retry can expire while spooling, before metadata exists.
+                // Keep the prefix even then; no attempt renews the outer clock.
+                if Instant::now() >= deadline {
+                    return Ok(self.result.take());
+                }
+                let status = if stream::expired(&error) { Status::Skipped } else { Status::Failed };
+                result.pages[index] = bounded::unread(number, status, error.to_string());
+            }
+            // Valid worker failures with metadata already decode to page
+            // records. Infrastructure/protocol errors are not page damage.
+            Err(error) => return Err(error),
+        }
+        self.pending.pop();
+        Ok(self.settle())
+    }
+
+    fn deadline(&mut self, limits: &Limits, process: &ProcessLimits) -> Result<Instant> {
+        if let Some(deadline) = self.deadline {
+            return Ok(deadline);
+        }
+        let deadline = Instant::now()
+            .checked_add(process.wall_time_for(limits))
+            .ok_or(Error::InvalidPdfOption("worker wall time is too large"))?;
+        Ok(*self.deadline.insert(deadline))
+    }
+
+    const fn settle(&mut self) -> Option<Pages> {
+        if self.pending.is_empty() || self.remaining == 0 { self.result.take() } else { None }
+    }
+}

data/ext/imprimatur_native/core/src/read/pdf/isolation/stream.rs

@@ -25,6 +25,15 @@ pub(super) fn write(out: &mut impl Write, frame: &Frame) -> Result<()> {
 }
 
 pub(super) fn decode(output: Output, options: &ReadOptions) -> Result<Pages> {
+    decode_recoverable(output, options).map(|(pages, _)| pages)
+}
+
+// The index comes from the supervisor/protocol failure, not human-readable
+// page reasons: output-limit skips must never become recovery candidates.
+pub(super) fn decode_recoverable(
+    output: Output,
+    options: &ReadOptions,
+) -> Result<(Pages, Option<usize>)> {
     let mut failure = output.failure;
     if output.data.len() < protocol::HEADER.len() {
         return Err(failure.unwrap_or_else(|| malformed("missing response")));
@@ -67,14 +76,12 @@ pub(super) fn decode(output: Output, options: &ReadOptions) -> Result<Pages> {
         info.ok_or_else(|| failure.take().unwrap_or_else(|| malformed("missing metadata")))?;
     let count = selected(info, options);
     if pages.len() == count && ended {
-        return Ok(Pages { info, pages });
+        return Ok((Pages { info, pages }, None));
     }
     let error = failure.ok_or_else(|| malformed("incomplete response"))?;
     let reason = error.to_string();
-    let timed_out = matches!(
-        error,
-        Error::LimitExceeded("time" | "render worker wall time" | "PDF worker limit")
-    );
+    let recoverable = expired(&error);
+    let timed_out = recoverable || matches!(error, Error::LimitExceeded("PDF worker limit"));
     let first_missing = pages.len();
     for index in first_missing..count {
         let number =
@@ -83,7 +90,11 @@ pub(super) fn decode(output: Output, options: &ReadOptions) -> Result<Pages> {
             if timed_out || index > first_missing { Status::Skipped } else { Status::Failed };
         pages.push(bounded::unread(number, status, reason.clone()));
     }
-    Ok(Pages { info, pages })
+    Ok((Pages { info, pages }, recoverable.then_some(first_missing)))
+}
+
+pub(super) fn expired(error: &Error) -> bool {
+    matches!(error, Error::LimitExceeded("time" | "render worker wall time"))
 }
 
 fn selected(info: Info, options: &ReadOptions) -> usize {

data/ext/imprimatur_native/core/src/read/pdf/isolation.rs

@@ -2,9 +2,12 @@
 //! occurs in the supervisor.
 
 mod protocol;
+mod recovery;
 mod stream;
 mod system;
 #[cfg(test)]
+mod test_support;
+#[cfg(test)]
 mod tests;
 mod worker;
 
@@ -18,6 +21,7 @@ use std::time::{Duration, Instant};
 use super::{Info, Pages, ReadOptions, RenderOptions};
 use crate::read::{Budget, Image, Limits};
 use crate::{Error, Result};
+pub use recovery::{Recovery, pages_isolated_recovering};
 pub use worker::worker_main;
 
 /// Inspects a PDF in the trusted, resource-limited worker.
@@ -71,6 +75,12 @@ impl Default for ProcessLimits {
     }
 }
 
+impl ProcessLimits {
+    fn wall_time_for(&self, limits: &Limits) -> Duration {
+        limits.timeout.unwrap_or(self.wall_time).min(self.wall_time)
+    }
+}
+
 /// Renders through the explicitly selected, trusted worker executable.
 ///
 /// The child is killed and reaped on timeout, cancellation, resource failure

data/ext/imprimatur_native/core/src/read/pdf/layout/blocks.rs

@@ -0,0 +1,259 @@
+use std::collections::HashMap;
+
+use super::{Line, PageLines};
+use crate::read::content::{Block, Bounds, Page, Placed, Text};
+
+/// The document's pages, from their lines: blocks of the lines that are
+/// not running headers or footers, and, with `drop_repeated_anywhere`, not
+/// text repeated at the same spot on most pages anywhere on the page.
+pub fn pages(
+    pages: &[PageLines],
+    labels: Option<&[Option<String>]>,
+    drop_repeated_anywhere: bool,
+) -> Vec<Page> {
+    let boilerplate = repeated_keys(pages, |line| line.margin);
+    let margin_kept =
+        |line: &&Line| line.margin == 0 || boilerplate.binary_search(&line.margin).is_err();
+    let repeated =
+        if drop_repeated_anywhere { repeated_keys(pages, |line| line.repeat) } else { Vec::new() };
+    let kept = |line: &&Line| {
+        margin_kept(line) && (repeated.is_empty() || repeated.binary_search(&line.repeat).is_err())
+    };
+    let body = body_size(pages, &margin_kept);
+    let levels = heading_sizes(pages, &margin_kept, body);
+    let mut out = Vec::with_capacity(pages.len());
+    let mut paragraph: Vec<&Line> = Vec::new();
+    // Each page's blocks, collected here, then moved out at their size.
+    let mut blocks = Vec::new();
+    for (index, (number, page)) in (1..).zip(pages).enumerate() {
+        paragraph.clear();
+        for line in page.lines.iter().filter(kept) {
+            if let Some(last) = paragraph.last()
+                && !continues(last, line)
+            {
+                blocks.push(block(page, &paragraph, body, &levels));
+                paragraph.clear();
+            }
+            paragraph.push(line);
+        }
+        if !paragraph.is_empty() {
+            blocks.push(block(page, &paragraph, body, &levels));
+        }
+        #[expect(
+            clippy::drain_collect,
+            clippy::iter_with_drain,
+            reason = "`blocks` keeps its buffer for the next page"
+        )]
+        let blocks = blocks.drain(..).collect();
+        let label = labels
+            .map_or_else(|| printed_label(page), |labels| labels.get(index).cloned().flatten());
+        out.push(Page { number, blocks, images: page.images, label });
+    }
+    out
+}
+
+/// The page's own printed label, from a single header or footer line that
+/// looks like one: a bare page number or roman numeral ("`-i-`", "`7`"), or
+/// the same after a short label word ("`Page 70`" -> "`70`"). Unlike the
+/// boilerplate this document's other margin lines are checked against, a
+/// running "`Page 70`", "`Page 71`", ... header is not noise here: each
+/// instance *is* that page's label, not a repeated line to drop.
+fn printed_label(page: &PageLines) -> Option<String> {
+    let mut candidates =
+        page.lines.iter().filter(|line: &&Line| line.margin != 0).filter_map(|line| {
+            let text = page.text(line).trim();
+            page_number_label(text)
+        });
+    let label = candidates.next()?;
+    candidates.next().is_none().then_some(label)
+}
+
+/// `text` as a page label, if it looks like one, stripped of a leading
+/// label word such as "Page" when there is one.
+fn page_number_label(text: &str) -> Option<String> {
+    if looks_like_page_number(text) {
+        return Some(text.to_owned());
+    }
+    let word_end = text.find(|ch: char| !ch.is_ascii_alphabetic())?;
+    if word_end == 0 {
+        return None;
+    }
+    let rest = text[word_end..].trim_start_matches(|ch: char| !ch.is_alphanumeric());
+    looks_like_page_number(rest).then(|| rest.to_owned())
+}
+
+fn looks_like_page_number(text: &str) -> bool {
+    let core = text.trim_matches(|ch: char| !ch.is_alphanumeric());
+    !core.is_empty()
+        && core.len() <= 8
+        && (core.chars().all(|ch| ch.is_ascii_digit())
+            || core.chars().all(|ch| {
+                matches!(ch.to_ascii_lowercase(), 'i' | 'v' | 'x' | 'l' | 'c' | 'd' | 'm')
+            }))
+}
+
+/// Whether `next` carries on the paragraph `last` is in: same direction,
+/// similar size, and no more than a line and a half below it.
+fn continues(last: &Line, next: &Line) -> bool {
+    let size = last.size.max(next.size).max(1.0);
+    last.turns == next.turns
+        && (last.size - next.size).abs() <= 0.15 * size
+        && last.y - next.y > 0.0
+        && last.y - next.y <= 1.55 * size
+        && last.bold == next.bold
+}
+
+/// A paragraph of lines as a block: a heading when its few short lines
+/// are set in a heading size, otherwise a paragraph with its lines on lines
+/// of their own, words hyphenated across them joined again.
+fn block(page: &PageLines, lines: &[&Line], body: f64, levels: &[f64]) -> Placed {
+    let first = lines[0];
+    let length: usize = lines.iter().map(|line| line.span.len()).sum();
+    let short = lines.len() <= 3 && length <= 200;
+    let text = paragraph_text(page, lines);
+    let level =
+        if short { levels.iter().position(|&size| first.size >= size - 0.25) } else { None };
+    let sized = level
+        .filter(|_| first.size >= body * 1.15)
+        .map(|level| u8::try_from(level.min(2) + 1).unwrap_or(3));
+    let numbered = (short && first.bold && first.size >= body - 0.25)
+        .then(|| numbered_heading_level(&text))
+        .flatten();
+    let block = match sized.or(numbered) {
+        Some(level) => Block::Heading { level, text: Text::plain(text) },
+        None => Block::Paragraph { text: Text::plain(text) },
+    };
+    Placed { block, bounds: bounds(page, lines) }
+}
+
+fn paragraph_text(page: &PageLines, lines: &[&Line]) -> String {
+    let length: usize = lines.iter().map(|line| line.span.len()).sum();
+    let mut text = String::with_capacity(length + lines.len());
+    for (index, line) in lines.iter().enumerate() {
+        let line = page.text(line);
+        if index > 0 {
+            let joined = text.ends_with('-') && {
+                let mut before = text.chars().rev().skip(1);
+                before.next().is_some_and(char::is_lowercase)
+                    && line.chars().next().is_some_and(char::is_lowercase)
+            };
+            if joined {
+                text.pop();
+            } else {
+                text.push('\n');
+            }
+        }
+        text.push_str(line);
+    }
+    text
+}
+
+fn numbered_heading_level(text: &str) -> Option<u8> {
+    let (depth, rest) = numbered_prefix(text)?;
+    if depth < 2 || !(rest.is_empty() || rest.starts_with(' ')) {
+        return None;
+    }
+    let title = rest.trim_start();
+    if title.contains("....") || title.starts_with(|ch: char| !ch.is_alphanumeric()) {
+        return None;
+    }
+    Some(u8::try_from(depth).unwrap_or(6).min(6))
+}
+
+fn numbered_prefix(text: &str) -> Option<(usize, &str)> {
+    let bytes = text.as_bytes();
+    let mut groups = 0_usize;
+    let mut pos = 0_usize;
+    loop {
+        let start = pos;
+        while bytes.get(pos).is_some_and(u8::is_ascii_digit) {
+            pos += 1;
+        }
+        if pos == start {
+            break;
+        }
+        groups += 1;
+        if groups >= 4
+            || bytes.get(pos) != Some(&b'.')
+            || !bytes.get(pos + 1).is_some_and(u8::is_ascii_digit)
+        {
+            break;
+        }
+        pos += 1;
+    }
+    (groups > 0).then(|| (groups, &text[pos..]))
+}
+
+/// Where upright lines lie on the page, from its top-left corner: from the
+/// top of the first line's capitals to below the last line's descenders.
+fn bounds(page: &PageLines, lines: &[&Line]) -> Option<Bounds> {
+    if lines.iter().any(|line| line.turns != 0) {
+        return None;
+    }
+    let (first, last) = (lines.first()?, lines.last()?);
+    let left = lines.iter().map(|line| line.left).fold(f64::MAX, f64::min);
+    let right = lines.iter().map(|line| line.right).fold(f64::MIN, f64::max);
+    let top = page.height - first.size.mul_add(0.8, first.y);
+    let bottom = page.height - last.size.mul_add(-0.25, last.y);
+    Some(Bounds { left, top, right, bottom })
+}
+
+/// The keys `key` returns for lines that repeat on enough pages to be
+/// boilerplate rather than coincidence, sorted.
+fn repeated_keys(pages: &[PageLines], key: impl Fn(&Line) -> u64) -> Vec<u64> {
+    if pages.len() < 3 {
+        return Vec::new();
+    }
+    let mut counts: HashMap<u64, usize> = HashMap::new();
+    let mut seen: Vec<u64> = Vec::new();
+    for page in pages {
+        seen.clear();
+        seen.extend(page.lines.iter().map(&key).filter(|&key| key != 0));
+        seen.sort_unstable();
+        seen.dedup();
+        for &key in &seen {
+            *counts.entry(key).or_default() += 1;
+        }
+    }
+    let threshold = (pages.len() * 2 / 5).max(3);
+    let mut repeated: Vec<u64> =
+        counts.into_iter().filter(|&(_, count)| count >= threshold).map(|(key, _)| key).collect();
+    repeated.sort_unstable();
+    repeated
+}
+
+/// Every page's lines that `kept` keeps.
+fn kept_lines<'p>(
+    pages: &'p [PageLines],
+    kept: &'p impl Fn(&&Line) -> bool,
+) -> impl Iterator<Item = &'p Line> {
+    pages.iter().flat_map(|page| &page.lines).filter(kept)
+}
+
+/// The most common font size, by characters.
+fn body_size(pages: &[PageLines], kept: &impl Fn(&&Line) -> bool) -> f64 {
+    let mut counts: HashMap<i64, usize> = HashMap::new();
+    for line in kept_lines(pages, kept) {
+        #[expect(clippy::cast_possible_truncation, reason = "font sizes are small")]
+        let key = (line.size * 2.0).round() as i64;
+        *counts.entry(key).or_default() += line.span.len();
+    }
+    #[expect(clippy::cast_precision_loss, reason = "a small integer")]
+    counts
+        .into_iter()
+        .max_by_key(|&(size, count)| (count, -size))
+        .map_or(0.0, |(size, _)| size as f64 / 2.0)
+}
+
+/// The sizes of headings, largest first: distinct sizes clearly above the
+/// body's, three at most.
+fn heading_sizes(pages: &[PageLines], kept: &impl Fn(&&Line) -> bool, body: f64) -> Vec<f64> {
+    let mut sizes: Vec<f64> = kept_lines(pages, kept)
+        .filter(|line| line.size >= body * 1.15 && line.span.len() <= 200)
+        .map(|line| (line.size * 2.0).round() / 2.0)
+        .collect();
+    sizes.sort_by(|a, b| b.total_cmp(a));
+    sizes.dedup();
+    sizes.truncate(3);
+    sizes
+}

data/ext/imprimatur_native/core/src/read/pdf/layout.rs

@@ -3,15 +3,15 @@
 //! the running headers and footers left out.
 
 mod bidi;
+mod blocks;
 #[cfg(test)]
 mod tests;
 
-use std::collections::HashMap;
 use std::ops::Range;
 
 use self::bidi::{Scratch, is_mark};
+pub use self::blocks::pages;
 use super::content::{Glyph, PageText};
-use crate::read::content::{Block, Bounds, Page, Placed, Text};
 use crate::read::span;
 use crate::script::Counts;
 
@@ -250,244 +250,3 @@ fn repeat_key(text: &str, left: f64, y: f64, turns: u8) -> u64 {
     }
     hash.max(1)
 }
-
-/// The document's pages, from their lines: blocks of the lines that are
-/// not running headers or footers, and, with `drop_repeated_anywhere`, not
-/// text repeated at the same spot on most pages anywhere on the page.
-pub fn pages(
-    pages: &[PageLines],
-    labels: Option<&[Option<String>]>,
-    drop_repeated_anywhere: bool,
-) -> Vec<Page> {
-    let boilerplate = repeated_keys(pages, |line| line.margin);
-    let margin_kept =
-        |line: &&Line| line.margin == 0 || boilerplate.binary_search(&line.margin).is_err();
-    let repeated =
-        if drop_repeated_anywhere { repeated_keys(pages, |line| line.repeat) } else { Vec::new() };
-    let kept = |line: &&Line| {
-        margin_kept(line) && (repeated.is_empty() || repeated.binary_search(&line.repeat).is_err())
-    };
-    let body = body_size(pages, &margin_kept);
-    let levels = heading_sizes(pages, &margin_kept, body);
-    let mut out = Vec::with_capacity(pages.len());
-    let mut paragraph: Vec<&Line> = Vec::new();
-    // Each page's blocks, collected here, then moved out at their size.
-    let mut blocks = Vec::new();
-    for (index, (number, page)) in (1..).zip(pages).enumerate() {
-        paragraph.clear();
-        for line in page.lines.iter().filter(kept) {
-            if let Some(last) = paragraph.last()
-                && !continues(last, line)
-            {
-                blocks.push(block(page, &paragraph, body, &levels));
-                paragraph.clear();
-            }
-            paragraph.push(line);
-        }
-        if !paragraph.is_empty() {
-            blocks.push(block(page, &paragraph, body, &levels));
-        }
-        #[expect(
-            clippy::drain_collect,
-            clippy::iter_with_drain,
-            reason = "`blocks` keeps its buffer for the next page"
-        )]
-        let blocks = blocks.drain(..).collect();
-        let label = labels.map_or_else(
-            || footer_label(page, &boilerplate),
-            |labels| labels.get(index).cloned().flatten(),
-        );
-        out.push(Page { number, blocks, images: page.images, label });
-    }
-    out
-}
-
-fn footer_label(page: &PageLines, boilerplate: &[u64]) -> Option<String> {
-    let mut candidates = page.lines.iter().filter(|line: &&Line| {
-        line.margin != 0
-            && line.y < page.height * EDGE
-            && boilerplate.binary_search(&line.margin).is_err()
-    });
-    let line = candidates.next()?;
-    if candidates.next().is_some() {
-        return None;
-    }
-    let text = page.text(line).trim();
-    looks_like_page_number(text).then(|| text.to_owned())
-}
-
-fn looks_like_page_number(text: &str) -> bool {
-    let core = text.trim_matches(|ch: char| !ch.is_alphanumeric());
-    !core.is_empty()
-        && core.len() <= 8
-        && (core.chars().all(|ch| ch.is_ascii_digit())
-            || core.chars().all(|ch| {
-                matches!(ch.to_ascii_lowercase(), 'i' | 'v' | 'x' | 'l' | 'c' | 'd' | 'm')
-            }))
-}
-
-/// Whether `next` carries on the paragraph `last` is in: same direction,
-/// similar size, and no more than a line and a half below it.
-fn continues(last: &Line, next: &Line) -> bool {
-    let size = last.size.max(next.size).max(1.0);
-    last.turns == next.turns
-        && (last.size - next.size).abs() <= 0.15 * size
-        && last.y - next.y > 0.0
-        && last.y - next.y <= 1.55 * size
-        && last.bold == next.bold
-}
-
-/// A paragraph of lines as a block: a heading when its few short lines
-/// are set in a heading size, otherwise a paragraph with its lines on lines
-/// of their own, words hyphenated across them joined again.
-fn block(page: &PageLines, lines: &[&Line], body: f64, levels: &[f64]) -> Placed {
-    let first = lines[0];
-    let length: usize = lines.iter().map(|line| line.span.len()).sum();
-    let short = lines.len() <= 3 && length <= 200;
-    let text = paragraph_text(page, lines);
-    let level =
-        if short { levels.iter().position(|&size| first.size >= size - 0.25) } else { None };
-    let sized = level
-        .filter(|_| first.size >= body * 1.15)
-        .map(|level| u8::try_from(level.min(2) + 1).unwrap_or(3));
-    let numbered = (short && first.bold && first.size >= body - 0.25)
-        .then(|| numbered_heading_level(&text))
-        .flatten();
-    let block = match sized.or(numbered) {
-        Some(level) => Block::Heading { level, text: Text::plain(text) },
-        None => Block::Paragraph { text: Text::plain(text) },
-    };
-    Placed { block, bounds: bounds(page, lines) }
-}
-
-fn paragraph_text(page: &PageLines, lines: &[&Line]) -> String {
-    let length: usize = lines.iter().map(|line| line.span.len()).sum();
-    let mut text = String::with_capacity(length + lines.len());
-    for (index, line) in lines.iter().enumerate() {
-        let line = page.text(line);
-        if index > 0 {
-            let joined = text.ends_with('-') && {
-                let mut before = text.chars().rev().skip(1);
-                before.next().is_some_and(char::is_lowercase)
-                    && line.chars().next().is_some_and(char::is_lowercase)
-            };
-            if joined {
-                text.pop();
-            } else {
-                text.push('\n');
-            }
-        }
-        text.push_str(line);
-    }
-    text
-}
-
-fn numbered_heading_level(text: &str) -> Option<u8> {
-    let (depth, rest) = numbered_prefix(text)?;
-    if depth < 2 || !(rest.is_empty() || rest.starts_with(' ')) {
-        return None;
-    }
-    let title = rest.trim_start();
-    if title.contains("....") || title.starts_with(|ch: char| !ch.is_alphanumeric()) {
-        return None;
-    }
-    Some(u8::try_from(depth).unwrap_or(6).min(6))
-}
-
-fn numbered_prefix(text: &str) -> Option<(usize, &str)> {
-    let bytes = text.as_bytes();
-    let mut groups = 0_usize;
-    let mut pos = 0_usize;
-    loop {
-        let start = pos;
-        while bytes.get(pos).is_some_and(u8::is_ascii_digit) {
-            pos += 1;
-        }
-        if pos == start {
-            break;
-        }
-        groups += 1;
-        if groups >= 4
-            || bytes.get(pos) != Some(&b'.')
-            || !bytes.get(pos + 1).is_some_and(u8::is_ascii_digit)
-        {
-            break;
-        }
-        pos += 1;
-    }
-    (groups > 0).then(|| (groups, &text[pos..]))
-}
-
-/// Where upright lines lie on the page, from its top-left corner: from the
-/// top of the first line's capitals to below the last line's descenders.
-fn bounds(page: &PageLines, lines: &[&Line]) -> Option<Bounds> {
-    if lines.iter().any(|line| line.turns != 0) {
-        return None;
-    }
-    let (first, last) = (lines.first()?, lines.last()?);
-    let left = lines.iter().map(|line| line.left).fold(f64::MAX, f64::min);
-    let right = lines.iter().map(|line| line.right).fold(f64::MIN, f64::max);
-    let top = page.height - first.size.mul_add(0.8, first.y);
-    let bottom = page.height - last.size.mul_add(-0.25, last.y);
-    Some(Bounds { left, top, right, bottom })
-}
-
-/// The keys `key` returns for lines that repeat on enough pages to be
-/// boilerplate rather than coincidence, sorted.
-fn repeated_keys(pages: &[PageLines], key: impl Fn(&Line) -> u64) -> Vec<u64> {
-    if pages.len() < 3 {
-        return Vec::new();
-    }
-    let mut counts: HashMap<u64, usize> = HashMap::new();
-    let mut seen: Vec<u64> = Vec::new();
-    for page in pages {
-        seen.clear();
-        seen.extend(page.lines.iter().map(&key).filter(|&key| key != 0));
-        seen.sort_unstable();
-        seen.dedup();
-        for &key in &seen {
-            *counts.entry(key).or_default() += 1;
-        }
-    }
-    let threshold = (pages.len() * 2 / 5).max(3);
-    let mut repeated: Vec<u64> =
-        counts.into_iter().filter(|&(_, count)| count >= threshold).map(|(key, _)| key).collect();
-    repeated.sort_unstable();
-    repeated
-}
-
-/// Every page's lines that `kept` keeps.
-fn kept_lines<'p>(
-    pages: &'p [PageLines],
-    kept: &'p impl Fn(&&Line) -> bool,
-) -> impl Iterator<Item = &'p Line> {
-    pages.iter().flat_map(|page| &page.lines).filter(kept)
-}
-
-/// The most common font size, by characters.
-fn body_size(pages: &[PageLines], kept: &impl Fn(&&Line) -> bool) -> f64 {
-    let mut counts: HashMap<i64, usize> = HashMap::new();
-    for line in kept_lines(pages, kept) {
-        #[expect(clippy::cast_possible_truncation, reason = "font sizes are small")]
-        let key = (line.size * 2.0).round() as i64;
-        *counts.entry(key).or_default() += line.span.len();
-    }
-    #[expect(clippy::cast_precision_loss, reason = "a small integer")]
-    counts
-        .into_iter()
-        .max_by_key(|&(size, count)| (count, -size))
-        .map_or(0.0, |(size, _)| size as f64 / 2.0)
-}
-
-/// The sizes of headings, largest first: distinct sizes clearly above the
-/// body's, three at most.
-fn heading_sizes(pages: &[PageLines], kept: &impl Fn(&&Line) -> bool, body: f64) -> Vec<f64> {
-    let mut sizes: Vec<f64> = kept_lines(pages, kept)
-        .filter(|line| line.size >= body * 1.15 && line.span.len() <= 200)
-        .map(|line| (line.size * 2.0).round() / 2.0)
-        .collect();
-    sizes.sort_by(|a, b| b.total_cmp(a));
-    sizes.dedup();
-    sizes.truncate(3);
-    sizes
-}

data/ext/imprimatur_native/ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "imprimatur_native"
-version = "0.7.0"
+version = "0.8.1"
 description = "Ruby bindings for the imprimatur crate"
 publish = false
 edition.workspace = true

data/ext/imprimatur_native/ffi/src/lib.rs

@@ -323,13 +323,13 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     barcode::define(ruby, imprimatur)?;
 
     let native = imprimatur.define_module("Native")?;
-    native.define_singleton_method("markdown", function!(read::markdown_native, 4))?;
-    native.define_singleton_method("content", function!(read::content_native, 5))?;
-    native.define_singleton_method("images", function!(read::images_native, 4))?;
+    native.define_singleton_method("markdown", function!(read::markdown_native, 5))?;
+    native.define_singleton_method("content", function!(read::content_native, 6))?;
+    native.define_singleton_method("images", function!(read::images_native, 5))?;
     native.define_singleton_method("pdf_info", function!(pdf::info, 4))?;
     native.define_singleton_method("pdf_pages", function!(pdf::pages, 5))?;
     native.define_singleton_method("render_pdf_page", function!(pdf::render, 5))?;
-    native.define_singleton_method("sheets", function!(sheets::sheets_native, 5))?;
+    native.define_singleton_method("sheets", function!(sheets::sheets_native, 6))?;
     let zip_writer = native.define_class("ZipWriter", ruby.class_object())?;
     zip_writer.define_singleton_method("new", function!(archive::ZipWriter::new, 1))?;
     zip_writer.define_method("add_file", method!(archive::ZipWriter::add_file, 3))?;

data/ext/imprimatur_native/ffi/src/pdf.rs

@@ -4,10 +4,12 @@ use imprimatur::read::{Limits, pdf};
 use magnus::{Error, RString, Ruby};
 use serde_json::json;
 use std::path::PathBuf;
+use std::time::Duration;
 
 use crate::read::without_gvl;
+use crate::{argument_error, domain_error};
 
-type Keywords = (Option<usize>, Option<u64>, Option<f64>);
+type Keywords = (Option<usize>, Option<u64>, Option<f64>, bool);
 
 pub fn info(
     ruby: &Ruby,
@@ -29,39 +31,61 @@ pub fn pages(
     ruby: &Ruby,
     document: RString,
     keywords: Keywords,
-    options: (u32, u32, usize, usize, usize),
+    options: (u32, u32, usize, usize, usize, Option<f64>),
     worker: String,
     memory_bytes: u64,
 ) -> Result<RString, Error> {
-    let (first_page, last_page, max_pages, max_page_bytes, max_output_bytes) = options;
+    let (first_page, last_page, max_pages, max_page_bytes, max_output_bytes, batch_timeout) =
+        options;
     let options =
         pdf::ReadOptions { first_page, last_page, max_pages, max_page_bytes, max_output_bytes };
     let worker = PathBuf::from(worker);
     let process = process_limits(memory_bytes);
-    let value = without_gvl(ruby, document, keywords, |data, limits| {
-        let result = pdf::isolation::pages_isolated(&worker, data, &options, limits, &process)?;
-        let pages: Vec<_> = result
-            .pages
-            .into_iter()
-            .map(|page| {
-                json!({
-                    "page": page.number,
-                    "status": page.status.as_str(),
-                    "text": page.text,
-                    "images": page.images,
-                    "truncated": page.truncated,
-                    "reason": page.reason,
-                })
+    let Some(seconds) = batch_timeout else {
+        let result = without_gvl(ruby, document, keywords, |data, limits| {
+            pdf::isolation::pages_isolated(&worker, data, &options, limits, &process)
+        })?;
+        return Ok(ruby.str_new(&pages_json(result)));
+    };
+    let attempt = Duration::try_from_secs_f64(seconds)
+        .map_err(|_| argument_error(ruby, "batch timeout must be finite and positive"))?;
+    let mut recovery = pdf::isolation::Recovery::new(&options, attempt)
+        .map_err(|error| domain_error(ruby, &error))?;
+    loop {
+        let step = without_gvl(ruby, document, keywords, |data, limits| {
+            let process = pdf::isolation::ProcessLimits {
+                wall_time: limits.timeout_duration().unwrap_or(process.wall_time),
+                ..process.clone()
+            };
+            recovery.step(&worker, data, limits, &process)
+        })?;
+        if let Some(result) = step {
+            return Ok(ruby.str_new(&pages_json(result)));
+        }
+    }
+}
+
+fn pages_json(result: pdf::Pages) -> String {
+    let pages: Vec<_> = result
+        .pages
+        .into_iter()
+        .map(|page| {
+            json!({
+                "page": page.number,
+                "status": page.status.as_str(),
+                "text": page.text,
+                "images": page.images,
+                "truncated": page.truncated,
+                "reason": page.reason,
             })
-            .collect();
-        Ok(json!({
-            "page_count": result.info.page_count,
-            "encrypted": result.info.encrypted,
-            "pages": pages,
         })
-        .to_string())
-    })?;
-    Ok(ruby.str_new(&value))
+        .collect();
+    json!({
+        "page_count": result.info.page_count,
+        "encrypted": result.info.encrypted,
+        "pages": pages,
+    })
+    .to_string()
 }
 
 pub fn render(

data/ext/imprimatur_native/ffi/src/read.rs

@@ -25,15 +25,19 @@ struct Read<T, F> {
 }
 
 /// Called by `Imprimatur.markdown`, which owns the keyword interface.
-/// `timeout` is in seconds; `nil` limits keep their defaults.
+/// `timeout` is in seconds; `nil` limits keep their defaults. `drop_repeated`
+/// drops PDF text repeated at the same spot on most pages, anywhere on the
+/// page (watermarks); off by default.
 pub fn markdown_native(
     ruby: &Ruby,
     document: RString,
     max_input: Option<usize>,
     max_inflated: Option<u64>,
     timeout: Option<f64>,
+    drop_repeated: bool,
 ) -> Result<RString, Error> {
-    let text = without_gvl(ruby, document, (max_input, max_inflated, timeout), markdown)?;
+    let text =
+        without_gvl(ruby, document, (max_input, max_inflated, timeout, drop_repeated), markdown)?;
     Ok(ruby.str_new(&text))
 }
 
@@ -45,9 +49,11 @@ pub fn content_native(
     max_input: Option<usize>,
     max_inflated: Option<u64>,
     timeout: Option<f64>,
+    drop_repeated: bool,
     styles: bool,
 ) -> Result<RString, Error> {
-    let read = without_gvl(ruby, document, (max_input, max_inflated, timeout), content)?;
+    let read =
+        without_gvl(ruby, document, (max_input, max_inflated, timeout, drop_repeated), content)?;
     let options = json::Options { styles };
     Ok(ruby.str_new(&Json::Object(json::content_with(&read, options)).to_string()))
 }
@@ -60,9 +66,13 @@ pub fn images_native(
     max_input: Option<usize>,
     max_inflated: Option<u64>,
     timeout: Option<f64>,
+    drop_repeated: bool,
 ) -> Result<RArray, Error> {
-    let found =
-        without_gvl(ruby, document, (max_input, max_inflated, timeout), |bytes, limits| {
+    let found = without_gvl(
+        ruby,
+        document,
+        (max_input, max_inflated, timeout, drop_repeated),
+        |bytes, limits| {
             let mut found = Vec::new();
             images(bytes, limits, |image| {
                 let description = Json::Object(json::image(&image)).to_string();
@@ -70,7 +80,8 @@ pub fn images_native(
                 Ok(())
             })?;
             Ok(found)
-        })?;
+        },
+    )?;
     let pairs = ruby.ary_new_capa(found.len());
     for (data, description) in found {
         let pair = [
@@ -86,9 +97,14 @@ pub fn images_native(
 fn limits(
     ruby: &Ruby,
     cancel: &Arc<AtomicBool>,
-    (max_input, max_inflated, timeout): (Option<usize>, Option<u64>, Option<f64>),
+    (max_input, max_inflated, timeout, drop_repeated): (
+        Option<usize>,
+        Option<u64>,
+        Option<f64>,
+        bool,
+    ),
 ) -> Result<Limits, Error> {
-    let mut limits = Limits::default().cancel_on(Arc::clone(cancel));
+    let mut limits = Limits::default().cancel_on(Arc::clone(cancel)).drop_repeated(drop_repeated);
     if let Some(bytes) = max_input {
         limits = limits.max_input(bytes);
     }
@@ -107,11 +123,11 @@ fn limits(
 pub fn without_gvl<T, F>(
     ruby: &Ruby,
     document: RString,
-    keywords: (Option<usize>, Option<u64>, Option<f64>),
+    keywords: (Option<usize>, Option<u64>, Option<f64>, bool),
     job: F,
 ) -> Result<T, Error>
 where
-    F: Fn(&[u8], &Limits) -> imprimatur::Result<T>,
+    F: FnMut(&[u8], &Limits) -> imprimatur::Result<T>,
 {
     let document = crate::bytes(document);
     let cancel = Arc::new(AtomicBool::new(false));
@@ -158,7 +174,7 @@ where
 #[allow(unsafe_code)]
 unsafe extern "C" fn run<T, F>(data: *mut c_void) -> *mut c_void
 where
-    F: Fn(&[u8], &Limits) -> imprimatur::Result<T>,
+    F: FnMut(&[u8], &Limits) -> imprimatur::Result<T>,
 {
     // SAFETY: `data` is the `Read<T>` without_gvl passed, alive and not
     // otherwise borrowed until this returns.

data/ext/imprimatur_native/ffi/src/sheets.rs

@@ -23,9 +23,11 @@ pub fn sheets_native(
     max_input: Option<usize>,
     max_inflated: Option<u64>,
     timeout: Option<f64>,
+    drop_repeated: bool,
     styles: bool,
 ) -> Result<RArray, Error> {
-    let read = without_gvl(ruby, document, (max_input, max_inflated, timeout), content)?;
+    let read =
+        without_gvl(ruby, document, (max_input, max_inflated, timeout, drop_repeated), content)?;
     let Some(sheets) = read.sheets() else {
         let kind = if matches!(read, Content::Pdf(_)) { "a PDF" } else { "a Word document" };
         let error = imprimatur::Error::UnsupportedDocument(format!("{kind} is no spreadsheet"));

data/lib/imprimatur/pdf_reader.rb

@@ -11,8 +11,9 @@ module Imprimatur
 
     def self.pages(document, first_page: 1, last_page: 0xFFFFFFFF, max_pages: 500,
                    max_page_bytes: 256 * 1024, max_output_bytes: 8 * 1024 * 1024,
-                   memory_bytes: 1024 * 1024 * 1024, max_bytes: nil, **limits)
-      options = [first_page, last_page, max_pages, max_page_bytes, max_output_bytes]
+                   memory_bytes: 1024 * 1024 * 1024, max_bytes: nil, batch_timeout: nil, **limits)
+      options = [first_page, last_page, max_pages, max_page_bytes, max_output_bytes,
+                 batch_timeout && Float(batch_timeout)]
       JSON.parse(Native.pdf_pages(Reader.read(document, max_bytes), Reader.limits(max_bytes:, **limits),
                                   options, RENDER_WORKER, memory_bytes),
                  symbolize_names: true)

data/lib/imprimatur/reader.rb

@@ -41,8 +41,8 @@ module Imprimatur
     end
 
     # @api private
-    def self.limits(timeout: 60, max_bytes: nil, max_inflated: nil)
-      [max_bytes, max_inflated, timeout && Float(timeout)]
+    def self.limits(timeout: 60, max_bytes: nil, max_inflated: nil, drop_repeated: false)
+      [max_bytes, max_inflated, timeout && Float(timeout), drop_repeated ? true : false]
     end
 
     # A path or an IO is read no further than +max_bytes+ (one byte past it

data/lib/imprimatur/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Imprimatur
-  VERSION = '0.7.0'
+  VERSION = '0.8.1'
 end

data/lib/imprimatur.rb

@@ -72,6 +72,8 @@ module Imprimatur
     # @param timeout [Numeric, nil] seconds the read may take (60), +nil+ for none
     # @param max_bytes [Integer, nil] the largest document accepted (512 MiB)
     # @param max_inflated [Integer, nil] decompressed bytes allowed in all (1 GiB)
+    # @param drop_repeated [Boolean] PDF only: drop text repeated at the same
+    #   spot on most pages, anywhere on the page (watermarks); off by default
     # @return [String] Markdown, UTF-8
     # @raise [NoTextLayer] a PDF of scanned pages: run OCR instead
     # @raise [PasswordRequired] a PDF that needs a password to open
@@ -113,6 +115,8 @@ module Imprimatur
     # image count (nil when unknown), truncation flag and failure reason.
     # Takes +first_page:+, +last_page:+, +max_pages:+ (500),
     # +max_page_bytes:+ (256 KiB), +max_output_bytes:+ (8 MiB), and read limits.
+    # Optional +batch_timeout:+ caps each worker attempt; +timeout:+ then bounds
+    # the whole call, including one-pass recovery of timeout-unread neighbours.
     # @return [Hash] +page_count:+, +encrypted:+ and +pages:+
     def pdf_pages(document, ...) = PdfReader.pages(document, ...)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: imprimatur
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Abdelkader Boudih
@@ -115,11 +115,13 @@ files:
 - ext/imprimatur_native/core/src/read/pdf/images.rs
 - ext/imprimatur_native/core/src/read/pdf/isolation.rs
 - ext/imprimatur_native/core/src/read/pdf/isolation/protocol.rs
+- ext/imprimatur_native/core/src/read/pdf/isolation/recovery.rs
 - ext/imprimatur_native/core/src/read/pdf/isolation/stream.rs
 - ext/imprimatur_native/core/src/read/pdf/isolation/system.rs
 - ext/imprimatur_native/core/src/read/pdf/isolation/worker.rs
 - ext/imprimatur_native/core/src/read/pdf/layout.rs
 - ext/imprimatur_native/core/src/read/pdf/layout/bidi.rs
+- ext/imprimatur_native/core/src/read/pdf/layout/blocks.rs
 - ext/imprimatur_native/core/src/read/pdf/lexer.rs
 - ext/imprimatur_native/core/src/read/pdf/object.rs
 - ext/imprimatur_native/core/src/read/pdf/object/name.rs