@hraness/wordcell 0.20.0-bootstrap.1

package/src/pdf/capture.ts

@@ -0,0 +1,298 @@
+import { createHash } from "node:crypto";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { countWords } from "../clip/extract.js";
+import { slugify } from "../clip/lib.js";
+import { sanitizeTerminalText } from "../clip/terminal.js";
+import { inspectPdf } from "./extract.js";
+import {
+  layoutBlocks,
+  parsePdfImageInterpretations,
+} from "./layout.js";
+import {
+  buildPdfMarkdown,
+  type ResolvedPdfImage,
+} from "./markdown.js";
+import type {
+  PdfCaptureDependencies,
+  PdfCaptureManifest,
+  PdfCaptureOptions,
+  PdfCaptureOutcome,
+  PdfImageCandidate,
+  PdfImageInterpretation,
+  PdfImageSemanticMetadata,
+  PdfManifestImage,
+} from "./model.js";
+import {
+  PDF_CAPTURE_MANIFEST_SCHEMA_VERSION,
+  PDF_CAPTURE_ANNOTATIONS_FILENAME,
+  PDF_CAPTURE_SOURCE_FILENAME,
+} from "./model.js";
+import { ocrPdfImage } from "./ocr.js";
+import {
+  pdfImageAssetPath,
+  pdfMarkdownFilename,
+  persistPdfCapture,
+} from "./persist.js";
+import { resolvePdfTools, runPdfToolCommand } from "./tools.js";
+
+function parsedInterpretations(
+  raw: readonly PdfImageInterpretation[] | undefined,
+  images: readonly PdfImageCandidate[],
+): {
+  readonly byId: ReadonlyMap<string, PdfImageInterpretation>;
+  readonly values: readonly PdfImageInterpretation[];
+} {
+  const parsed = raw === undefined ? [] : parsePdfImageInterpretations(raw);
+  const candidates = new Map(images.map((image) => [image.id, image]));
+  const output = new Map<string, PdfImageInterpretation>();
+  for (const interpretation of parsed) {
+    const image = candidates.get(interpretation.id);
+    if (image === undefined) {
+      throw new Error(`PDF image annotation does not match an extracted image: ${interpretation.id}`);
+    }
+    if (image.sha256 !== interpretation.sha256) {
+      throw new Error(`PDF image annotation hash does not match the extracted image: ${interpretation.id}`);
+    }
+    output.set(interpretation.id, interpretation);
+  }
+  return { byId: output, values: parsed };
+}
+
+function sortedImages(images: readonly PdfImageCandidate[]): readonly PdfImageCandidate[] {
+  return [...images].sort((left, right) =>
+    left.page - right.page
+    || left.top - right.top
+    || left.left - right.left
+    || left.id.localeCompare(right.id));
+}
+
+function normalizedMetadata(
+  value: PdfImageSemanticMetadata | undefined,
+): PdfImageSemanticMetadata | null {
+  return value ?? null;
+}
+
+function resolvedAnnotation(
+  image: PdfImageCandidate,
+  annotation: PdfImageInterpretation,
+): ResolvedPdfImage {
+  const markdown = annotation.kind === "visual" ? "" : annotation.markdown;
+  return {
+    image,
+    kind: annotation.kind,
+    method: annotation.method ?? "manual",
+    markdown,
+    alt: annotation.kind === "visual" ? (annotation.alt ?? null) : null,
+    confidence: null,
+    wordCount: annotation.kind === "visual" ? 0 : countWords(markdown),
+    metadata: normalizedMetadata(annotation.metadata),
+  };
+}
+
+function embeddedPlatforms(images: readonly ResolvedPdfImage[]): readonly string[] {
+  const byIdentity = new Map<string, string>();
+  for (const image of images) {
+    const platform = image.metadata?.platform?.trim();
+    if (platform === undefined || platform === "") continue;
+    const key = platform.toLocaleLowerCase("en-US");
+    if (!byIdentity.has(key)) byIdentity.set(key, platform);
+  }
+  return [...byIdentity.entries()]
+    .sort(([left], [right]) => left.localeCompare(right))
+    .map(([, value]) => value);
+}
+
+function manifestImage(image: ResolvedPdfImage): PdfManifestImage {
+  return {
+    id: image.image.id,
+    page: image.image.page,
+    top: image.image.top,
+    left: image.image.left,
+    width: image.image.width,
+    height: image.image.height,
+    asset: {
+      path: pdfImageAssetPath(image.image),
+      mimeType: image.image.mimeType,
+      bytes: image.image.bytes,
+      sha256: image.image.sha256,
+    },
+    kind: image.kind,
+    method: image.method,
+    confidence: image.confidence,
+    wordCount: image.wordCount,
+    metadata: image.metadata,
+  };
+}
+
+/** Inspect, OCR/annotate, render, and atomically persist one prepared PDF source. */
+export async function runPdfCapture(
+  options: PdfCaptureOptions,
+  dependencies: PdfCaptureDependencies = {},
+): Promise<PdfCaptureOutcome> {
+  const callerOwnedWorkspace = options.workspaceDirectory !== undefined;
+  const workspaceDirectory = options.workspaceDirectory
+    ?? mkdtempSync(join(tmpdir(), "hraness-wordcell-pdf-"));
+  try {
+    const inspection = await inspectPdf({
+      inputPath: options.inputPath,
+      workspaceDirectory,
+      ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),
+      ...(options.maxPdfBytes === undefined ? {} : { maxPdfBytes: options.maxPdfBytes }),
+      ...(options.maxPages === undefined ? {} : { maxPages: options.maxPages }),
+      ...(options.maxImages === undefined ? {} : { maxImages: options.maxImages }),
+      ...(options.maxAssetBytes === undefined ? {} : { maxAssetBytes: options.maxAssetBytes }),
+      ...(options.maxTotalAssetBytes === undefined ? {} : { maxTotalAssetBytes: options.maxTotalAssetBytes }),
+    }, dependencies);
+    const tools = resolvePdfTools(dependencies);
+    const runTool = dependencies.runTool ?? runPdfToolCommand;
+    const images = sortedImages(inspection.pages.flatMap((page) => page.images));
+    const annotations = parsedInterpretations(options.interpretations, images);
+    const resolvedImages: ResolvedPdfImage[] = [];
+    const warnings = [...inspection.warnings];
+    let unclassifiedImages = 0;
+    let ocrWarningCount = 0;
+
+    for (const image of images) {
+      const annotation = annotations.byId.get(image.id);
+      if (annotation !== undefined) {
+        resolvedImages.push(resolvedAnnotation(image, annotation));
+        continue;
+      }
+      const ocr = await ocrPdfImage(image, {
+        tesseractPath: tools.tesseract,
+        timeoutMs: options.timeoutMs ?? 120_000,
+        runTool,
+      });
+      if (tools.tesseract === null) unclassifiedImages += 1;
+      for (const warning of ocr.warnings) {
+        warnings.push(`Page ${image.page} image ${image.id}: ${warning}`);
+        ocrWarningCount += 1;
+      }
+      resolvedImages.push({
+        image,
+        kind: ocr.kind,
+        method: tools.tesseract === null ? "unclassified" : "tesseract",
+        markdown: ocr.markdown,
+        alt: null,
+        confidence: ocr.confidence,
+        wordCount: ocr.wordCount,
+        metadata: null,
+      });
+    }
+    const platforms = embeddedPlatforms(resolvedImages);
+    const status = warnings.length > 0 || unclassifiedImages > 0 || ocrWarningCount > 0
+      ? "partial"
+      : "complete";
+    const filenameTitle = inspection.originalFilename.replace(/\.pdf$/iu, "").trim();
+    const requestedSlug = options.slug ?? inspection.metadata.title ?? filenameTitle;
+    const slug = slugify(sanitizeTerminalText(requestedSlug));
+    if (slug === "") throw new Error("could not derive a safe PDF capture slug; pass an explicit slug");
+    const blocks = layoutBlocks(inspection.pages);
+    const byId = new Map(resolvedImages.map((image) => [image.image.id, image]));
+    const capturedAt = (dependencies.now ?? (() => new Date()))().toISOString();
+    const built = buildPdfMarkdown({
+      slug,
+      originalFilename: inspection.originalFilename,
+      sourceSha256: inspection.sourceSha256,
+      ...(options.remoteSource === undefined ? {} : { sourceUrl: options.remoteSource.finalUrl }),
+      capturedDate: capturedAt.slice(0, 10),
+      status,
+      metadata: inspection.metadata,
+      blocks,
+      images: byId,
+      embeddedPlatforms: platforms,
+    });
+    const textImageCount = resolvedImages.filter((image) => image.kind === "text").length;
+    const mixedImageCount = resolvedImages.filter((image) => image.kind === "mixed").length;
+    const visualImageCount = resolvedImages.filter((image) => image.kind === "visual").length;
+    const annotationCount = resolvedImages.filter((image) =>
+      image.method === "agent" || image.method === "manual").length;
+    const tesseractCount = resolvedImages.filter((image) => image.method === "tesseract").length;
+    const ocr: PdfCaptureManifest["extraction"]["ocr"] = annotationCount > 0 && tesseractCount > 0
+      ? "mixed"
+      : annotationCount > 0
+        ? "annotations"
+        : tesseractCount > 0
+          ? "tesseract"
+          : "unavailable";
+    const finalWarnings = [...new Set(warnings)];
+    const annotationsJson = annotations.values.length === 0
+      ? null
+      : `${JSON.stringify(annotations.values, null, 2)}\n`;
+    const annotationsManifest: PdfCaptureManifest["annotations"] = annotationsJson === null
+      ? null
+      : {
+          path: PDF_CAPTURE_ANNOTATIONS_FILENAME,
+          count: annotations.values.length,
+          bytes: Buffer.byteLength(annotationsJson),
+          sha256: createHash("sha256").update(annotationsJson).digest("hex"),
+        };
+    const manifest: PdfCaptureManifest = {
+      schemaVersion: PDF_CAPTURE_MANIFEST_SCHEMA_VERSION,
+      kind: "pdf",
+      capturedAt,
+      status,
+      source: {
+        originalFilename: sanitizeTerminalText(inspection.originalFilename).slice(0, 4_096),
+        path: PDF_CAPTURE_SOURCE_FILENAME,
+        mimeType: "application/pdf",
+        bytes: inspection.sourceBytes,
+        sha256: inspection.sourceSha256,
+        ...(options.remoteSource === undefined
+          ? {}
+          : {
+              requestedUrl: options.remoteSource.requestedUrl,
+              finalUrl: options.remoteSource.finalUrl,
+            }),
+      },
+      document: {
+        ...inspection.metadata,
+        processedPages: inspection.processedPages,
+      },
+      extraction: {
+        layout: "pdftohtml-xml",
+        popplerVersion: inspection.popplerVersion,
+        ocr,
+        headingCount: built.headingCount,
+        textBlockCount: built.textBlockCount,
+        imageCount: resolvedImages.length,
+        textImageCount,
+        mixedImageCount,
+        visualImageCount,
+      },
+      images: resolvedImages.map(manifestImage),
+      annotations: annotationsManifest,
+      embeddedPlatforms: platforms,
+      warnings: finalWarnings,
+    };
+    const outputDirectory = persistPdfCapture({
+      outputBase: options.outputBase,
+      slug,
+      force: options.force ?? false,
+      sourcePath: inspection.inputPath,
+      markdown: built.markdown,
+      manifest,
+      images,
+      ...(annotationsJson === null ? {} : { annotationsJson }),
+    });
+    return {
+      status,
+      slug,
+      outputDirectory,
+      markdownPath: join(outputDirectory, pdfMarkdownFilename(slug)),
+      sourcePath: join(outputDirectory, PDF_CAPTURE_SOURCE_FILENAME),
+      wordCount: countWords(built.markdown),
+      pageCount: inspection.metadata.pageCount,
+      processedPages: inspection.processedPages,
+      imageCount: images.length,
+      warnings: finalWarnings,
+      markdown: built.markdown,
+      manifest,
+    };
+  } finally {
+    if (!callerOwnedWorkspace) rmSync(workspaceDirectory, { recursive: true, force: true });
+  }
+}

package/src/pdf/cli.ts

@@ -0,0 +1,159 @@
+import { lstatSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
+
+import { redactSensitiveText } from "../clip/persist.js";
+import { sanitizeTerminalLine, sanitizeTerminalText } from "../clip/terminal.js";
+import { parsePdfArguments, pdfUsage } from "./args.js";
+import { runPdfCapture } from "./capture.js";
+import { parsePdfImageInterpretations } from "./layout.js";
+import { preparePdfSource, type PreparedPdfSource } from "./source.js";
+import type {
+  PdfCaptureDependencies,
+  PdfCaptureOptions,
+  PdfCaptureOutcome,
+  PdfImageInterpretation,
+} from "./model.js";
+
+export type PdfCliOutput = {
+  readonly stdout: (value: string) => void;
+  readonly stderr: (value: string) => void;
+};
+
+const defaultOutput: PdfCliOutput = {
+  stdout: (value) => process.stdout.write(value),
+  stderr: (value) => process.stderr.write(value),
+};
+
+export type PdfCliDependencies = {
+  readonly runPdfCapture?: typeof runPdfCapture;
+  readonly captureDependencies?: PdfCaptureDependencies;
+  readonly readInterpretations?: (path: string) => readonly PdfImageInterpretation[];
+  readonly preparePdfSource?: typeof preparePdfSource;
+};
+
+function safe(value: string): string {
+  return sanitizeTerminalLine(redactSensitiveText(value));
+}
+
+function terminalSafeJson(value: unknown): string {
+  return `${JSON.stringify(
+    value,
+    (_key, candidate: unknown) => typeof candidate === "string"
+      ? sanitizeTerminalText(redactSensitiveText(candidate))
+      : candidate,
+    2,
+  )}\n`;
+}
+
+function readInterpretations(path: string): readonly PdfImageInterpretation[] {
+  const absolute = resolve(path);
+  const stats = lstatSync(absolute);
+  if (!stats.isFile() || stats.isSymbolicLink()) {
+    throw new Error("PDF image annotations must be a regular JSON file");
+  }
+  if (stats.size > 16 * 1024 * 1024) {
+    throw new Error("PDF image annotations exceed the 16MB limit");
+  }
+  let value: unknown;
+  try {
+    value = JSON.parse(readFileSync(absolute, "utf8")) as unknown;
+  } catch {
+    throw new Error("PDF image annotations are not valid JSON");
+  }
+  return parsePdfImageInterpretations(value);
+}
+
+export function pdfCaptureSummary(outcome: PdfCaptureOutcome): Record<string, unknown> {
+  return {
+    ok: true,
+    status: outcome.status,
+    slug: outcome.slug,
+    outputDirectory: outcome.outputDirectory,
+    markdownPath: outcome.markdownPath,
+    sourcePath: outcome.sourcePath,
+    wordCount: outcome.wordCount,
+    pageCount: outcome.pageCount,
+    processedPages: outcome.processedPages,
+    imageCount: outcome.imageCount,
+    warnings: outcome.warnings,
+    manifest: outcome.manifest,
+  };
+}
+
+/** Delegated CLI entry point for the root `wordcell pdf` command. */
+export async function main(
+  rawArguments: readonly string[] = process.argv.slice(2),
+  environment: Readonly<Record<string, string | undefined>> = process.env,
+  output: PdfCliOutput = defaultOutput,
+  dependencies: PdfCliDependencies = {},
+): Promise<number> {
+  const parsed = parsePdfArguments(rawArguments, environment);
+  if (!parsed.ok) {
+    output.stderr(`error: ${safe(parsed.message)}\n\n${sanitizeTerminalText(pdfUsage)}`);
+    return 2;
+  }
+  const arguments_ = parsed.value;
+  if (arguments_.command === "help") {
+    output.stdout(sanitizeTerminalText(pdfUsage));
+    return 0;
+  }
+  if (!arguments_.quiet && !arguments_.json) {
+    output.stderr(`Saving PDF ${safe(arguments_.input)} ...\n`);
+  }
+  let preparedSource: PreparedPdfSource | null = null;
+  try {
+    const interpretations = arguments_.interpretationsPath === undefined
+      ? undefined
+      : (dependencies.readInterpretations ?? readInterpretations)(arguments_.interpretationsPath);
+    preparedSource = await (dependencies.preparePdfSource ?? preparePdfSource)(
+      arguments_.input,
+      {
+        ...(arguments_.timeoutMs === undefined ? {} : { timeoutMs: arguments_.timeoutMs }),
+        ...(arguments_.maxPdfBytes === undefined ? {} : { maxPdfBytes: arguments_.maxPdfBytes }),
+      },
+    );
+    const options: PdfCaptureOptions = {
+      inputPath: preparedSource.inputPath,
+      outputBase: arguments_.outputBase,
+      ...(preparedSource.remoteSource === undefined ? {} : { remoteSource: preparedSource.remoteSource }),
+      ...(arguments_.slug === undefined ? {} : { slug: arguments_.slug }),
+      ...(interpretations === undefined ? {} : { interpretations }),
+      force: arguments_.force,
+      ...(arguments_.timeoutMs === undefined ? {} : { timeoutMs: arguments_.timeoutMs }),
+      ...(arguments_.maxPdfBytes === undefined ? {} : { maxPdfBytes: arguments_.maxPdfBytes }),
+      ...(arguments_.maxPages === undefined ? {} : { maxPages: arguments_.maxPages }),
+      ...(arguments_.maxImages === undefined ? {} : { maxImages: arguments_.maxImages }),
+      ...(arguments_.maxAssetBytes === undefined ? {} : { maxAssetBytes: arguments_.maxAssetBytes }),
+      ...(arguments_.maxTotalAssetBytes === undefined
+        ? {}
+        : { maxTotalAssetBytes: arguments_.maxTotalAssetBytes }),
+    };
+    const outcome = await (dependencies.runPdfCapture ?? runPdfCapture)(
+      options,
+      dependencies.captureDependencies,
+    );
+    if (arguments_.json) output.stdout(terminalSafeJson(pdfCaptureSummary(outcome)));
+    else {
+      output.stdout(`Done: ${safe(outcome.markdownPath)}\n`);
+      const pages = outcome.processedPages === outcome.pageCount
+        ? `${outcome.pageCount} pages`
+        : `${outcome.processedPages} of ${outcome.pageCount} pages processed`;
+      output.stdout(
+        `Status: ${outcome.status}; ${pages}; ${outcome.wordCount} words; ${outcome.imageCount} images.\n`,
+      );
+    }
+    if (!arguments_.quiet && !arguments_.json) {
+      for (const warning of outcome.warnings) output.stderr(`warning: ${safe(warning)}\n`);
+    }
+    return 0;
+  } catch (error) {
+    const message = safe(error instanceof Error ? error.message : String(error));
+    if (arguments_.json) output.stdout(terminalSafeJson({ ok: false, error: message }));
+    else output.stderr(`error: ${message}\n`);
+    return 1;
+  } finally {
+    preparedSource?.dispose();
+  }
+}
+
+if (import.meta.main) process.exitCode = await main();