imprimatur 0.1.0

data/ext/imprimatur_native/cli/src/cli/grep.rs

@@ -0,0 +1,359 @@
+use std::collections::VecDeque;
+use std::fmt::Write as _;
+use std::path::{Path, PathBuf};
+use std::process::ExitCode;
+
+use imprimatur::read::{Block, Content};
+use regex::{Regex, RegexBuilder};
+use serde_json::json;
+use usage_rs::{Args, RunWith};
+
+use super::limits::LimitFlags;
+use crate::app::{self, App, Out};
+
+/// Search documents for a pattern
+///
+/// Searches what `imprimatur markdown` reads, line by line: a PDF's
+/// paragraphs and headings by page, a workbook's cells and comments by
+/// sheet and cell, a Word document's blocks by section. Each match is
+/// printed with where it is: `p3` (page 3), `Sales!B2` (a cell, or
+/// `Sales!B2 (comment)` its comment), `§Scope` (the heading it falls under)
+/// or `¶12` (the 12th block, before any heading).
+///
+/// The exit status follows grep: 0 when a line matched, 1 when none did,
+/// 2 for a wrong command line or a document that could not be searched (a
+/// scanned PDF, a damaged file); with `--json`, its reason is reported.
+#[derive(Args, Debug)]
+#[expect(clippy::struct_excessive_bools, reason = "grep's switches, each a flag of its own")]
+#[usage(
+    effect = "read",
+    args_override_self = false,
+    example("imprimatur grep -i 'non.?conformity' audits/*.pdf", help = "Regular expression"),
+    example("imprimatur grep -F 'ISO 17065' -C 1 cert.pdf", help = "A fixed string, with context"),
+    example("imprimatur grep -l GOTS *.docx *.xlsx", help = "Only the documents that match"),
+    example("imprimatur --json grep 'Total' sales.xlsx", help = "Matches as JSON, with offsets")
+)]
+pub struct Grep {
+    /// A regular expression (Rust syntax), or with `-F` a fixed string
+    #[usage(arg, value_name = "PATTERN")]
+    pub pattern: String,
+
+    /// Documents to search; `-` reads standard input
+    #[usage(arg, required, value_name = "FILE", value_hint = usage_rs::ValueHint::FilePath)]
+    pub files: Vec<PathBuf>,
+
+    /// Match regardless of case
+    #[usage(short = 'i', long)]
+    pub ignore_case: bool,
+
+    /// Take the pattern as a fixed string
+    #[usage(short = 'F', long)]
+    pub fixed_strings: bool,
+
+    /// Match whole words only
+    #[usage(short = 'w', long)]
+    pub word_regexp: bool,
+
+    /// Lines of context around each match
+    #[usage(short = 'C', long, default = "0", value_name = "LINES")]
+    pub context: usize,
+
+    /// Stop each document after this many matching lines
+    #[usage(short = 'm', long, value_name = "NUMBER")]
+    pub max_count: Option<usize>,
+
+    /// Print only the documents with a match
+    #[usage(short = 'l', long, conflicts("--count"))]
+    pub files_with_matches: bool,
+
+    /// Print how many lines match in each document
+    #[usage(short = 'c', long)]
+    pub count: bool,
+
+    /// Print nothing; the exit status tells whether anything matched
+    #[usage(short = 'q', long)]
+    pub quiet: bool,
+
+    #[usage(flatten)]
+    pub limits: LimitFlags,
+}
+
+/// Where a line is.
+#[derive(Debug, Clone)]
+enum Location<'c> {
+    Page(u32),
+    Cell { sheet: &'c str, cell: String },
+    Comment { sheet: &'c str, cell: String },
+    Section(&'c str),
+    Block(usize),
+}
+
+impl Location<'_> {
+    fn label(&self, out: &mut String) {
+        // Writing to a String cannot fail.
+        let _ = match self {
+            Self::Page(page) => write!(out, "p{page}"),
+            Self::Cell { sheet, cell } => write!(out, "{sheet}!{cell}"),
+            Self::Comment { sheet, cell } => write!(out, "{sheet}!{cell} (comment)"),
+            Self::Section(heading) => write!(out, "§{heading}"),
+            Self::Block(index) => write!(out, "¶{index}"),
+        };
+    }
+
+    fn json(&self) -> serde_json::Value {
+        match self {
+            Self::Page(page) => json!({ "page": page }),
+            Self::Cell { sheet, cell } => json!({ "sheet": sheet, "cell": cell }),
+            Self::Comment { sheet, cell } => {
+                json!({ "sheet": sheet, "cell": cell, "comment": true })
+            }
+            Self::Section(heading) => json!({ "section": heading }),
+            Self::Block(index) => json!({ "block": index }),
+        }
+    }
+}
+
+/// Every line of a document's content, with where it is.
+fn lines<'c>(content: &'c Content, mut each: impl FnMut(Location<'c>, &str) -> bool) {
+    match content {
+        Content::Pdf(pages) => {
+            for page in pages {
+                for placed in &page.blocks {
+                    for line in placed.block.plain_text().lines() {
+                        if !each(Location::Page(page.number), line) {
+                            return;
+                        }
+                    }
+                }
+            }
+        }
+        Content::Xlsx(sheets) => {
+            for sheet in sheets {
+                for cell in sheet.cells.cells() {
+                    let reference = cell.reference();
+                    for line in sheet.cells.text(cell).lines() {
+                        let location =
+                            Location::Cell { sheet: &sheet.name, cell: reference.clone() };
+                        if !each(location, line) {
+                            return;
+                        }
+                    }
+                }
+                for note in &sheet.comments {
+                    for line in note.text.lines() {
+                        let location =
+                            Location::Comment { sheet: &sheet.name, cell: note.reference() };
+                        if !each(location, line) {
+                            return;
+                        }
+                    }
+                }
+            }
+        }
+        Content::Docx(blocks) => {
+            let mut section = None;
+            for (index, block) in (1..).zip(blocks) {
+                if let Block::Heading { text, .. } = block {
+                    section = Some(text.text.as_str());
+                }
+                let location = section.map_or(Location::Block(index), Location::Section);
+                for line in block.plain_text().lines() {
+                    if !each(location.clone(), line) {
+                        return;
+                    }
+                }
+            }
+        }
+        _ => {}
+    }
+}
+
+/// grep's statuses.
+const MATCHED: u8 = 0;
+const NO_MATCH: u8 = 1;
+const TROUBLE: u8 = 2;
+
+impl RunWith<&App> for Grep {
+    type Output = ExitCode;
+
+    fn run_with(self, app: &App) -> ExitCode {
+        let limits = match self.limits.checked() {
+            Ok(limits) => limits,
+            Err(status) => return status,
+        };
+        let pattern = match self.regex() {
+            Ok(pattern) => pattern,
+            Err(message) => {
+                eprintln!("imprimatur: {message}");
+                return ExitCode::from(TROUBLE);
+            }
+        };
+        let (mut matched, mut trouble) = (false, false);
+        let mut out = Out::new();
+        let named = self.files.len() > 1;
+        for file in &self.files {
+            let document = match app::content(file, &limits) {
+                Ok(document) => document,
+                Err(error) => {
+                    trouble = true;
+                    if !self.quiet && !app.report_failure(&mut out, file, &error).1 {
+                        break;
+                    }
+                    continue;
+                }
+            };
+            let search = Search { grep: &self, app: *app, file, named, pattern: &pattern };
+            let found = search.run(&document, &mut out);
+            matched |= found.count > 0;
+            if !found.open || (self.quiet && matched) {
+                break;
+            }
+        }
+        ExitCode::from(match (matched, trouble) {
+            (true, _) if self.quiet => MATCHED,
+            (_, true) => TROUBLE,
+            (true, false) => MATCHED,
+            (false, false) => NO_MATCH,
+        })
+    }
+}
+
+impl Grep {
+    fn regex(&self) -> Result<Regex, String> {
+        let escaped;
+        let pattern = if self.fixed_strings {
+            escaped = regex::escape(&self.pattern);
+            &escaped
+        } else {
+            &self.pattern
+        };
+        let pattern =
+            if self.word_regexp { format!(r"\b(?:{pattern})\b") } else { pattern.clone() };
+        RegexBuilder::new(&pattern)
+            .case_insensitive(self.ignore_case)
+            .build()
+            .map_err(|error| format!("invalid pattern: {error}"))
+    }
+}
+
+struct Search<'g> {
+    grep: &'g Grep,
+    app: App,
+    file: &'g Path,
+    named: bool,
+    pattern: &'g Regex,
+}
+
+/// What searching one document found.
+struct Found {
+    count: usize,
+    open: bool,
+}
+
+impl Search<'_> {
+    fn run(&self, content: &Content, out: &mut Out) -> Found {
+        let grep = self.grep;
+        let listing = grep.quiet || grep.files_with_matches || grep.count;
+        let limit = grep.max_count.unwrap_or(usize::MAX);
+        let mut found = Found { count: 0, open: true };
+        // Lines before the next match, and how many after the last to show.
+        let mut before: VecDeque<(Location<'_>, String)> = VecDeque::with_capacity(grep.context);
+        let (mut after, mut last_shown) = (0_usize, None::<usize>);
+        let mut line_number = 0_usize;
+        let mut line = String::new();
+        lines(content, |location, text| {
+            line_number += 1;
+            if found.count >= limit && after == 0 {
+                return false;
+            }
+            let hit = found.count < limit && self.pattern.is_match(text);
+            if hit {
+                found.count += 1;
+                if listing {
+                    return !(grep.quiet || grep.files_with_matches);
+                }
+                if grep.context > 0
+                    && let Some(shown) = last_shown
+                    && line_number - before.len() > shown + 1
+                    && !self.app.json
+                {
+                    found.open &= out.write("--\n");
+                }
+                let first = line_number - before.len();
+                for (offset, (location, text)) in before.iter().enumerate() {
+                    found.open &= self.print(out, &mut line, location, text, None, first + offset);
+                }
+                before.clear();
+                found.open &=
+                    self.print(out, &mut line, &location, text, Some(self.pattern), line_number);
+                last_shown = Some(line_number);
+                after = grep.context;
+            } else if after > 0 && !listing {
+                after -= 1;
+                found.open &= self.print(out, &mut line, &location, text, None, line_number);
+                last_shown = Some(line_number);
+            } else if grep.context > 0 && !listing {
+                if before.len() == grep.context {
+                    before.pop_front();
+                }
+                before.push_back((location, text.to_owned()));
+            }
+            found.open
+        });
+        if grep.files_with_matches && found.count > 0 {
+            found.open &= if self.app.json {
+                out.json(&json!({ "file": self.file.to_string_lossy(), "matched": true }))
+            } else {
+                out.write(&format!("{}\n", self.file.display()))
+            };
+        } else if grep.count && !grep.quiet {
+            found.open &= if self.app.json {
+                out.json(&json!({ "file": self.file.to_string_lossy(), "count": found.count }))
+            } else if self.named {
+                out.write(&format!("{}:{}\n", self.file.display(), found.count))
+            } else {
+                out.write(&format!("{}\n", found.count))
+            };
+        }
+        found
+    }
+
+    /// One line of output: a match (`pattern` given) or context.
+    fn print(
+        &self,
+        out: &mut Out,
+        buffer: &mut String,
+        location: &Location<'_>,
+        text: &str,
+        pattern: Option<&Regex>,
+        line: usize,
+    ) -> bool {
+        if self.app.json {
+            let mut object = json!({
+                "file": self.file.to_string_lossy(),
+                "type": if pattern.is_some() { "match" } else { "context" },
+                "location": location.json(),
+                "line": line,
+                "text": text,
+            });
+            if let Some(pattern) = pattern {
+                let spans: Vec<_> = pattern
+                    .find_iter(text)
+                    .map(|found| json!({ "start": found.start(), "end": found.end() }))
+                    .collect();
+                object["matches"] = spans.into();
+            }
+            return out.json(&object);
+        }
+        buffer.clear();
+        if self.named {
+            buffer.push_str(&self.file.to_string_lossy());
+            buffer.push(':');
+        }
+        location.label(buffer);
+        buffer.push_str(if pattern.is_some() { ": " } else { "- " });
+        buffer.push_str(text.trim());
+        buffer.push('\n');
+        out.write(buffer)
+    }
+}

data/ext/imprimatur_native/cli/src/cli/images.rs

@@ -0,0 +1,137 @@
+use std::collections::HashSet;
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::process::ExitCode;
+
+use imprimatur::read::{Image, ImageLocation, images, json};
+use usage_rs::{Args, RunWith};
+
+use super::limits::LimitFlags;
+use crate::app::{self, App, Out, Status};
+
+/// Extract documents' images into a directory
+///
+/// A PDF's images, once each, from the first page that draws them: JPEG
+/// and JPEG 2000 as they are stored, fax-compressed scans as TIFF, raw
+/// pixels as PNG (CMYK converted to RGB, soft masks as alpha). A scanned
+/// PDF gives its page images, ready for OCR. The media of a Word or Excel
+/// document are copied, with the sheet that shows each one.
+///
+/// Files are named after their document: `NAME-p3-1.png` is the first image
+/// of page 3; `NAME-image1.png` is a medium of the same name. The paths
+/// written are printed, one per line; with `--json`, each image is a JSON
+/// object with its format, size and position on the page.
+#[derive(Args, Debug)]
+#[usage(
+    effect = "write",
+    args_override_self = false,
+    after_long_help = super::EXIT_STATUS,
+    example("imprimatur images -o figures/ report.pdf", help = "Every image of a PDF"),
+    example("imprimatur images -o pages/ --min-size 600 scan.pdf", help = "Page scans, for OCR"),
+    example("imprimatur --json images -o media/ deck.docx", help = "With sizes and positions")
+)]
+pub struct Images {
+    /// Documents to read; `-` reads standard input
+    #[usage(arg, required, value_name = "FILE", value_hint = usage_rs::ValueHint::FilePath)]
+    pub files: Vec<PathBuf>,
+
+    /// Directory to write the images into (created if missing)
+    #[usage(short = 'o', long, value_name = "DIR", value_hint = usage_rs::ValueHint::DirPath)]
+    pub output: PathBuf,
+
+    /// Skip images narrower or shorter than this, in pixels
+    #[usage(long, default = "0", value_name = "PIXELS")]
+    pub min_size: u32,
+
+    #[usage(flatten)]
+    pub limits: LimitFlags,
+}
+
+impl RunWith<&App> for Images {
+    type Output = ExitCode;
+
+    fn run_with(self, app: &App) -> ExitCode {
+        let limits = match self.limits.checked() {
+            Ok(limits) => limits,
+            Err(status) => return status,
+        };
+        if let Err(status) = app::create_dir(&self.output) {
+            return status;
+        }
+        let mut status = Status::default();
+        let mut written = HashSet::new();
+        let mut out = Out::new();
+        for file in &self.files {
+            let mut open = true;
+            // The page of the last image, and how many it had so far.
+            let mut last = (0, 0);
+            let result = app::read(file).and_then(|document| {
+                images(&document, &limits, |image| {
+                    if image
+                        .width
+                        .zip(image.height)
+                        .is_some_and(|(width, height)| width.min(height) < self.min_size)
+                    {
+                        return Ok(());
+                    }
+                    let target = self.target(file, &image, &mut last, &mut written);
+                    fs::write(&target, &image.data)?;
+                    open &= if app.json {
+                        out.json(&describe(file, &target, &image))
+                    } else {
+                        out.write(&format!("{}\n", target.display()))
+                    };
+                    // A closed pipe ends the run.
+                    if open { Ok(()) } else { Err(imprimatur::Error::Cancelled) }
+                })
+            });
+            if !open {
+                break;
+            }
+            if let Err(error) = result {
+                let (failure, still_open) = app.report_failure(&mut out, file, &error);
+                status.record(failure);
+                if !still_open {
+                    break;
+                }
+            }
+        }
+        status.exit_code()
+    }
+}
+
+impl Images {
+    /// `DIR/NAME-p3-1.png` for a page's images, `DIR/NAME-image1.png` for a
+    /// package's; `-2`, `-3`... when a name is taken.
+    fn target(
+        &self,
+        file: &Path,
+        image: &Image,
+        last: &mut (u32, usize),
+        written: &mut HashSet<PathBuf>,
+    ) -> PathBuf {
+        let stem = app::stem(file);
+        let base = match &image.location {
+            ImageLocation::Page { number, .. } => {
+                // Images come page by page.
+                *last = if last.0 == *number { (*number, last.1 + 1) } else { (*number, 1) };
+                format!("{stem}-p{number}-{}", last.1)
+            }
+            ImageLocation::Part { name, .. } => {
+                let part = Path::new(name)
+                    .file_stem()
+                    .map_or_else(|| "image".into(), |part| part.to_string_lossy());
+                format!("{stem}-{part}")
+            }
+            _ => format!("{stem}-image"),
+        };
+        app::unique_path(&self.output, &base, image.format.extension(), written)
+    }
+}
+
+fn describe(file: &Path, target: &Path, image: &Image) -> serde_json::Value {
+    let mut object = json::image(image);
+    object.insert("file".into(), file.to_string_lossy().into());
+    object.insert("image".into(), target.to_string_lossy().into());
+    serde_json::Value::Object(object)
+}

data/ext/imprimatur_native/cli/src/cli/limits.rs

@@ -0,0 +1,49 @@
+use std::process::ExitCode;
+use std::time::Duration;
+
+use imprimatur::read::Limits;
+use usage_rs::Args;
+
+/// The bounds a read runs under, shared by every command that reads.
+#[derive(Args, Debug)]
+pub struct LimitFlags {
+    /// Seconds one document may take
+    #[usage(long, default = "60", value_name = "SECONDS", help_heading = "Limits")]
+    pub timeout: f64,
+
+    /// Largest document accepted, in bytes (512 MiB by default)
+    #[usage(long, value_name = "BYTES", help_heading = "Limits")]
+    pub max_bytes: Option<usize>,
+
+    /// Decompressed data allowed per document, in bytes (1 GiB by default)
+    #[usage(long, value_name = "BYTES", help_heading = "Limits")]
+    pub max_inflated: Option<u64>,
+}
+
+impl LimitFlags {
+    /// The limits, or the exit status of a wrong command line, reported.
+    pub fn checked(&self) -> Result<Limits, ExitCode> {
+        self.limits().map_err(|message| {
+            eprintln!("imprimatur: {message}");
+            ExitCode::from(2)
+        })
+    }
+
+    /// The limits, or what is wrong with the flags.
+    pub fn limits(&self) -> Result<Limits, String> {
+        let timeout = Duration::try_from_secs_f64(self.timeout)
+            .ok()
+            .filter(|timeout| !timeout.is_zero())
+            .ok_or_else(|| {
+                format!("--timeout must be a positive number of seconds, not {}", self.timeout)
+            })?;
+        let mut limits = Limits::default().timeout(Some(timeout));
+        if let Some(bytes) = self.max_bytes {
+            limits = limits.max_input(bytes);
+        }
+        if let Some(bytes) = self.max_inflated {
+            limits = limits.max_inflated(bytes);
+        }
+        Ok(limits)
+    }
+}

data/ext/imprimatur_native/cli/src/cli/markdown.rs

@@ -0,0 +1,128 @@
+use std::collections::HashSet;
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::process::ExitCode;
+use std::time::Instant;
+
+use imprimatur::read::json;
+use serde_json::Value as Json;
+use usage_rs::{Args, RunWith};
+
+use super::limits::LimitFlags;
+use crate::app::{self, App, Out, Status};
+
+/// Print documents as Markdown
+///
+/// PDF: the text layer, placed glyph by glyph, with headings by font size,
+/// running headers and footers left out, and pages after the first marked
+/// `<!-- page N -->`. Encrypted PDFs open when their user password is
+/// empty. XLSX: each sheet as a table, values as Excel shows them, then its
+/// comments. DOCX: headings, lists, emphasis, links and tables.
+///
+/// With several documents, each is introduced by `<!-- file: PATH -->`. A
+/// document that fails is reported on standard error and the others are
+/// still read; the exit status is the first failure's.
+///
+/// With `--json`, each document is one JSON object instead: its structure,
+/// not Markdown. A PDF's pages of blocks (heading, paragraph) with their
+/// position on the page; a workbook's sheets of rows of typed cells
+/// (number, date, bool, text, error, formula) and comments; a Word
+/// document's blocks (heading, paragraph, list item, table) with the byte
+/// ranges of their bold, italic and linked text.
+#[derive(Args, Debug)]
+#[usage(
+    effect = "read",
+    // Per command: the root's setting covers only the root's own flags.
+    args_override_self = false,
+    after_long_help = super::EXIT_STATUS,
+    example("imprimatur markdown audit.pdf", help = "Print a PDF's text"),
+    example("imprimatur md -o notes/ *.docx", help = "Write notes/NAME.md for each"),
+    example("curl -s $URL | imprimatur md -", help = "Read standard input"),
+    example("imprimatur --json md sales.xlsx", help = "Typed cells, as JSON")
+)]
+pub struct Markdown {
+    /// Documents to read; `-` reads standard input
+    #[usage(arg, required, value_name = "FILE", value_hint = usage_rs::ValueHint::FilePath)]
+    pub files: Vec<PathBuf>,
+
+    /// Write each document to DIR/NAME.md instead of standard output
+    #[usage(short = 'o', long, value_name = "DIR", value_hint = usage_rs::ValueHint::DirPath)]
+    pub output: Option<PathBuf>,
+
+    #[usage(flatten)]
+    pub limits: LimitFlags,
+}
+
+impl RunWith<&App> for Markdown {
+    type Output = ExitCode;
+
+    fn run_with(self, app: &App) -> ExitCode {
+        let limits = match self.limits.checked() {
+            Ok(limits) => limits,
+            Err(status) => return status,
+        };
+        if let Some(directory) = &self.output
+            && let Err(status) = app::create_dir(directory)
+        {
+            return status;
+        }
+        let mut status = Status::default();
+        let mut written = HashSet::new();
+        let mut out = Out::new();
+        let introduce = self.files.len() > 1 && self.output.is_none() && !app.json;
+        for file in &self.files {
+            let started = Instant::now();
+            let read = app::content(file, &limits);
+            let milliseconds = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX);
+            let shown = read.and_then(|content| {
+                let target = match &self.output {
+                    Some(directory) => {
+                        let target =
+                            app::unique_path(directory, &app::stem(file), "md", &mut written);
+                        fs::write(&target, content.to_markdown())?;
+                        Some(target)
+                    }
+                    None => None,
+                };
+                Ok((content, target))
+            });
+            let open = match shown {
+                Ok((content, target)) if app.json => {
+                    let mut object = json::content(&content);
+                    object.insert("file".into(), file.to_string_lossy().into());
+                    object.insert("ok".into(), true.into());
+                    object.insert("milliseconds".into(), milliseconds.into());
+                    if let Some(target) = target {
+                        object.insert("written".into(), target.to_string_lossy().into());
+                    }
+                    out.json(&Json::Object(object))
+                }
+                Ok((_, Some(_))) => true,
+                Ok((content, None)) => out.write(&printed(file, &content.to_markdown(), introduce)),
+                Err(error) => {
+                    let (failure, open) = app.report_failure(&mut out, file, &error);
+                    status.record(failure);
+                    open
+                }
+            };
+            if !open {
+                break;
+            }
+        }
+        status.exit_code()
+    }
+}
+
+fn printed(file: &Path, text: &str, introduce: bool) -> String {
+    let mut shown = String::with_capacity(text.len() + 64);
+    if introduce {
+        shown.push_str("<!-- file: ");
+        shown.push_str(&file.to_string_lossy());
+        shown.push_str(" -->\n\n");
+    }
+    shown.push_str(text);
+    if !text.ends_with('\n') {
+        shown.push('\n');
+    }
+    shown
+}