@arnilo/prism-tools 0.0.0-stage → 0.13.1

package/work/docling/ocr.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""One-shot local Docling OCR. Stdin is the document. Stdout is one JSON object.
+
+Host pins docling==2.130.0 and CPU torch (https://download.pytorch.org/whl/cpu),
+prefetches models into --artifacts (~1.4 GiB), and runs this with egress off.
+This process never opens a path or URL from the document. No npm dependency.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+import sys
+from io import BytesIO
+from pathlib import Path
+
+SUFFIXES = {
+    "pdf": "pdf",
+    "png": "png",
+    "jpg": "jpg",
+    "jpeg": "jpeg",
+    "tif": "tif",
+    "tiff": "tiff",
+    "bmp": "bmp",
+    "webp": "webp",
+}
+
+
+def main() -> None:
+    logging.disable(logging.INFO)
+    os.environ["HF_HUB_OFFLINE"] = "1"
+    os.environ["TRANSFORMERS_OFFLINE"] = "1"
+    os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"
+    os.environ["TQDM_DISABLE"] = "1"
+    real_out = sys.stdout
+
+    def emit(payload: dict[str, object], code: int) -> None:
+        real_out.write(json.dumps(payload, ensure_ascii=False) + "\n")
+        real_out.flush()
+        raise SystemExit(code)
+
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument("--format", required=True)
+    parser.add_argument("--max-pages", required=True, type=int)
+    parser.add_argument("--max-bytes", required=True, type=int)
+    parser.add_argument("--max-text-bytes", required=True, type=int)
+    parser.add_argument("--timeout-s", required=True, type=int)
+    parser.add_argument("--artifacts", required=True)
+    args = parser.parse_args()
+    suffix = SUFFIXES.get(args.format)
+    artifacts = args.artifacts
+    if (
+        suffix is None
+        or args.max_pages < 1
+        or args.max_bytes < 1
+        or args.max_text_bytes < 1
+        or args.timeout_s < 1
+        or not artifacts
+        or "\0" in artifacts
+        or "://" in artifacts
+    ):
+        emit({"ok": False, "error": "unsupported"}, 1)
+
+    data = sys.stdin.buffer.read(args.max_bytes + 1)
+    if len(data) > args.max_bytes:
+        emit({"ok": False, "error": "input_limit"}, 1)
+    if not data:
+        emit({"ok": False, "error": "io"}, 1)
+    if not Path(artifacts).is_dir():
+        emit({"ok": False, "error": "missing_models"}, 1)
+
+    sys.stdout = sys.stderr
+    try:
+        markdown, pages = convert(data, suffix, args.max_pages, args.max_bytes, args.timeout_s, artifacts)
+    except OcrFail as error:
+        sys.stdout = real_out
+        emit({"ok": False, "error": error.code}, 1)
+    except Exception:
+        sys.stdout = real_out
+        emit({"ok": False, "error": "io"}, 1)
+    sys.stdout = real_out
+    if pages < 1 or pages > args.max_pages:
+        emit({"ok": False, "error": "partial"}, 1)
+    if len(markdown.encode("utf-8")) > args.max_text_bytes:
+        emit({"ok": False, "error": "output_limit"}, 1)
+    emit({"ok": True, "markdown": markdown, "pages": pages}, 0)
+
+
+class OcrFail(Exception):
+    def __init__(self, code: str) -> None:
+        self.code = code
+
+
+def convert(
+    data: bytes,
+    suffix: str,
+    max_pages: int,
+    max_bytes: int,
+    timeout_s: int,
+    artifacts: str,
+) -> tuple[str, int]:
+    from docling.datamodel.base_models import ConversionStatus, DocumentStream, InputFormat
+    from docling.datamodel.pipeline_options import PdfPipelineOptions, RapidOcrOptions
+    from docling.document_converter import DocumentConverter, ImageFormatOption, PdfFormatOption
+
+    opts = PdfPipelineOptions(
+        artifacts_path=artifacts,
+        enable_remote_services=False,
+        allow_external_plugins=False,
+        do_ocr=True,
+        generate_page_images=False,
+        do_picture_description=False,
+        do_picture_classification=False,
+        ocr_options=RapidOcrOptions(backend="torch"),
+        document_timeout=float(timeout_s),
+    )
+    converter = DocumentConverter(
+        allowed_formats=[InputFormat.PDF, InputFormat.IMAGE],
+        format_options={
+            InputFormat.PDF: PdfFormatOption(pipeline_options=opts),
+            InputFormat.IMAGE: ImageFormatOption(pipeline_options=opts),
+        },
+    )
+    result = converter.convert(
+        DocumentStream(name=f"input.{suffix}", stream=BytesIO(data)),
+        max_num_pages=max_pages,
+        max_file_size=max_bytes,
+        raises_on_error=False,
+    )
+    if result.status != ConversionStatus.SUCCESS:
+        raise OcrFail(failure_code(result))
+    markdown = result.document.export_to_markdown()
+    if not isinstance(markdown, str):
+        raise OcrFail("io")
+    return markdown, int(result.document.num_pages())
+
+
+def failure_code(result: object) -> str:
+    errors = getattr(result, "errors", None) or []
+    for err in errors:
+        blob = f"{getattr(err, 'error_category', '')} {getattr(err, 'error_message', '')}".lower()
+        if "policy" in blob or "max_num_pages" in blob or "max_file_size" in blob:
+            return "over_page"
+    return "partial"
+
+
+if __name__ == "__main__":
+    main()

package/work/sandbox/Dockerfile

@@ -0,0 +1,36 @@
+# Work sandbox image. Host builds and pins: `name@sha256:<64-hex>`.
+# Prism never pulls this image. Runtime network is none (createDockerSandbox).
+FROM node:22-bookworm-slim AS node
+
+FROM python:3.12-slim-bookworm
+
+COPY --from=node /usr/local/bin/node /usr/local/bin/node
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      passwd \
+      libreoffice-writer \
+      libreoffice-calc \
+      libreoffice-impress \
+      poppler-utils \
+      zip \
+      unzip \
+      fonts-liberation \
+      ca-certificates \
+    && rm -rf /var/lib/apt/lists/* \
+    && python3 -m pip install --break-system-packages --no-cache-dir \
+      python-docx==1.1.2 \
+      openpyxl==3.1.5 \
+      python-pptx==1.0.2 \
+    && groupadd --gid 65532 nonroot \
+    && useradd --uid 65532 --gid 65532 --home-dir /home/nonroot --create-home --shell /usr/sbin/nologin nonroot \
+    && mkdir -p /opt/prism-work/skills /workspace \
+    && chown -R 65532:65532 /opt/prism-work /workspace /home/nonroot
+
+COPY sandbox/soffice.sh /usr/local/bin/soffice
+RUN chmod 755 /usr/local/bin/soffice
+COPY vendor/hermes-agent /opt/prism-work/skills
+RUN chown -R 65532:65532 /opt/prism-work/skills
+
+USER 65532:65532
+ENV HOME=/home/nonroot
+WORKDIR /workspace

package/work/sandbox/soffice.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -eu
+for arg in "$@"; do
+  case "$arg" in
+    --accept|--accept=*|*[Mm]acro*)
+      echo "soffice macro/socket flags refused" >&2
+      exit 2
+      ;;
+  esac
+done
+mkdir -p /tmp/lo-profile
+exec /usr/bin/soffice -env:UserInstallation=file:///tmp/lo-profile --headless --norestore --nologo --nolockcheck "$@"

package/work/src/command.ts

@@ -0,0 +1,29 @@
+import { statSync } from "node:fs";
+import { delimiter, isAbsolute, join } from "node:path";
+
+/**
+ * Resolve a host-owned connector CLI the same way a shell would: an absolute path must be an
+ * executable file; a bare name is searched on the absolute `PATH` entries only. No shell, no
+ * relative directory entries. Returns `undefined` when the command cannot be resolved.
+ */
+export function resolveCommand(name: string): string | undefined {
+  if (name.length === 0 || name.includes("\0") || name.length > 4096) return undefined;
+  const executable = (path: string): boolean => {
+    try {
+      const stat = statSync(path);
+      if (!stat.isFile()) return false;
+      return process.platform === "win32" || (stat.mode & 0o111) !== 0;
+    } catch {
+      return false;
+    }
+  };
+  if (isAbsolute(name)) return executable(name) ? name : undefined;
+  if (name.includes("/") || name.includes("\\")) return undefined;
+  const pathExt = process.platform === "win32" ? (process.env.PATHEXT ?? ".EXE;.CMD;.BAT").split(";") : [];
+  for (const dir of (process.env.PATH ?? "").split(delimiter)) {
+    if (!dir || !isAbsolute(dir)) continue;
+    const candidates = pathExt.length > 0 ? pathExt.map((ext) => join(dir, `${name}${ext.toLowerCase()}`)) : [join(dir, name)];
+    for (const candidate of candidates) if (executable(candidate)) return candidate;
+  }
+  return undefined;
+}

package/work/src/config.ts

@@ -0,0 +1,138 @@
+import type { JsonObject, JsonSchema } from "@arnilo/prism";
+
+/** Office tool group settings. `enabled: false` registers no `office_*` tool. */
+export interface ToolsWorkOfficeConfig {
+  /** Default `true`. */
+  readonly enabled?: boolean;
+  /** Document byte cap handed to the OOXML stack. */
+  readonly maxBytes?: number;
+}
+
+/** One connector's settings. `credential` is ref-only; the rest is host-declared adapter config. */
+export interface ToolsWorkConnectorConfig {
+  /** `env:NAME` or `keyring:NAME`; a literal fails config validation. */
+  readonly credential: string;
+  /** CLI binary name or absolute path. Default `m365` / `gws`. */
+  readonly binary?: string;
+  /** Host config directory for the CLI. Required together with `identity`. */
+  readonly configDir?: string;
+  /** `AgentIdentity` fields as JSON; `verified: true` is added by setup. */
+  readonly identity?: JsonObject;
+  /** Provider operations to allow; default is the adapter's default set. */
+  readonly allowedOps?: readonly string[];
+}
+
+/** Connector group settings. A provider with no resolvable credential registers nothing. */
+export interface ToolsWorkConnectorsConfig {
+  readonly microsoft365?: ToolsWorkConnectorConfig;
+  readonly google?: ToolsWorkConnectorConfig;
+}
+
+/** Reader service settings. The reader is provided as `work.reader`, not registered as a tool. */
+export interface ToolsWorkReaderConfig {
+  /** Formats to keep. Default all four. A format whose optional peer is missing is dropped with a note. */
+  readonly formats?: readonly ("pdf" | "docx" | "xlsx" | "pptx")[];
+  readonly maxBytes?: number;
+  readonly maxPages?: number;
+  readonly maxTextBytes?: number;
+  /** Mistral OCR parser; off by default (one external call per page). */
+  readonly ocr?: {
+    readonly provider: "mistral";
+    /** `env:NAME` or `keyring:NAME`; a literal fails config validation. */
+    readonly apiKey: string;
+    readonly baseUrl?: string;
+    readonly model?: string;
+  };
+}
+
+/** Validated `packageConfig` for `@arnilo/prism-tools/work`. */
+export interface ToolsWorkConfig {
+  /** Absolute workspace root the file-touching tools operate in. Required; checked in `setup`. */
+  readonly root?: string;
+  /** Office document tools. Default `{ enabled: true }`. */
+  readonly office?: ToolsWorkOfficeConfig;
+  /** `work_exec` inside the host-injected `work.sandbox` service. Default `{ enabled: true }`. */
+  readonly work?: { readonly enabled?: boolean };
+  /** Connector tools; each provider registers only with a resolvable credential. */
+  readonly connectors?: ToolsWorkConnectorsConfig;
+  /** The `work.reader` service. */
+  readonly reader?: ToolsWorkReaderConfig;
+  /** Non-token env entries for the sandbox. `M365_*` / `GOOGLE_*` names are refused. */
+  readonly env?: Readonly<Record<string, string>>;
+}
+
+const secretString: JsonSchema = {
+  type: "string",
+  minLength: 1,
+  maxLength: 4096,
+  pattern: "^(env|keyring):",
+  "x-prism-secret": true,
+};
+
+const connectorProperties: Readonly<Record<string, JsonSchema>> = {
+  credential: secretString,
+  binary: { type: "string", minLength: 1, maxLength: 4096 },
+  configDir: { type: "string", minLength: 1, maxLength: 4096 },
+  identity: { type: "object", additionalProperties: true },
+  allowedOps: { type: "array", maxItems: 64, items: { type: "string", minLength: 1, maxLength: 128 } },
+};
+
+const connectorObject: JsonSchema = {
+  type: "object",
+  required: ["credential"],
+  properties: connectorProperties,
+};
+
+/**
+ * The config schema. `root` is deliberately not declared `required`: a required field with no
+ * default would fail package conformance, so `setup` checks it and names the field instead.
+ * `connectors.*.credential` and `reader.ocr.apiKey` are ref-only secret fields; a literal fails validation.
+ */
+export const toolsWorkConfigSchema: JsonSchema = {
+  type: "object",
+  properties: {
+    root: { type: "string", minLength: 1, maxLength: 4096 },
+    office: {
+      type: "object",
+      properties: {
+        enabled: { type: "boolean", default: true },
+        maxBytes: { type: "number", minimum: 1 },
+      },
+    },
+    work: {
+      type: "object",
+      properties: { enabled: { type: "boolean", default: true } },
+    },
+    connectors: {
+      type: "object",
+      properties: {
+        microsoft365: connectorObject,
+        google: connectorObject,
+      },
+    },
+    reader: {
+      type: "object",
+      properties: {
+        formats: {
+          type: "array",
+          maxItems: 4,
+          items: { type: "string", enum: ["pdf", "docx", "xlsx", "pptx"] },
+        },
+        maxBytes: { type: "number", minimum: 1 },
+        maxPages: { type: "number", minimum: 1 },
+        maxTextBytes: { type: "number", minimum: 1 },
+        ocr: {
+          type: "object",
+          required: ["provider", "apiKey"],
+          properties: {
+            provider: { type: "string", enum: ["mistral"] },
+            apiKey: secretString,
+            baseUrl: { type: "string", minLength: 1, maxLength: 2048 },
+            model: { type: "string", minLength: 1, maxLength: 256 },
+          },
+        },
+      },
+    },
+    env: { type: "object", additionalProperties: { type: "string", maxLength: 4096 } },
+  },
+};