@semiont/content 0.5.23 → 0.5.25

package/README.md

@@ -87,25 +87,69 @@ When the project has `[git] sync = true` in `.semiont/config`, the store keeps t
 
 Every method accepts `{ noGit: true }` to skip staging for a single call. Without git sync, the store falls back to plain filesystem operations.
 
-## PDF Text-Layer Extraction
+## PDF Extraction
 
-For native (non-scanned) PDFs, `extractPdfTextLayer()` extracts positioned text using [pdfjs-dist](https://www.npmjs.com/package/pdfjs-dist). It returns `null` for scanned/image-only PDFs.
+`EXTRACTORS['pdf-text-layer']` turns a PDF into text plus the geometry that
+indexes it, routing by what the document actually holds:
+
+| Class | Document | Read by |
+|---|---|---|
+| A | native text layer | pdf.js, directly |
+| B | scanned — pixels only | OCR (Tesseract) |
+| C | hybrid — some pages scanned | both; pages still unread are reported |
+| D | tables | grid pages rewritten as markdown rows |
+| E | forms | AcroForm values folded in, anchored to their widgets |
+| F / G | encrypted, corrupt | declined by name, from the parser error |
 
 ```typescript
-import { extractPdfTextLayer, locate } from '@semiont/content';
+import { EXTRACTORS } from '@semiont/content';
+import { textExtractionOf, locate } from '@semiont/core';
 
-const layer = await extractPdfTextLayer(pdfBytes);
-if (layer) {
-  console.log(layer.text);          // Full extracted text
-  console.log(layer.pages.length);  // Page dimensions in PDF points
+const extracted = await EXTRACTORS[textExtractionOf('application/pdf')]!
+  .extract(pdfBytes, 'application/pdf');
 
-  // Find bounding rectangles for a span of the text (one per line)
-  const rects = locate(layer, 120, 178);
-  // => PdfCoordinate[] in PDF point space (origin: bottom-left)
+if (!('declined' in extracted)) {
+  extracted.text;          // reading-order text
+  extracted.items;         // positioned runs indexing it
+  extracted.method;        // 'pdf-text-layer' | 'ocr' | 'table' | 'form'
+  extracted.unreadPages;   // class C: pages no reader could recover
 }
 ```
 
-Coordinates are in PDF point space, originating from the bottom-left of the page. The Y-flip to canvas pixels happens downstream in the browser; the server has no canvas. The `PdfCoordinate` geometry type lives in `@semiont/core` alongside the viewrect FragmentSelector codec.
+A decline is named (`'no-text-layer' | 'encrypted' | 'corrupt' | 'too-large'`)
+rather than a bare null, so a caller can settle with the reason.
+`'no-text-layer'` means recognition ran and came up empty — not that it was
+never attempted.
+
+`extractPdfTextLayer()` is the lower-level reader underneath class A, returning
+`null` for a document with no text operators anywhere.
+
+Coordinates are PDF points, origin bottom-left; the Y-flip to canvas pixels
+happens in the browser. The vocabulary these produce — `AnchoredText`,
+`PdfTextItem` — and the `locate` / `textUnder` pair that reads it are exported
+from [`@semiont/core`](../core/README.md), so the browser can reason over
+geometry without importing this package's extraction stack.
+
+## Anchored-text store
+
+OCR costs ~2.9 s per scanned page and six consumers read the same document, so
+what the engine produced is kept rather than re-derived:
+
+```typescript
+import { createAnchoredTextStore } from '@semiont/content';
+
+const store = createAnchoredTextStore(dir, logger);
+await store.write(checksum, { text, items });
+const map = await store.read(checksum);   // null on any miss
+```
+
+Derived values only, keyed by content checksum and stamped with the versions of
+this package, the engine and its traineddata. A stamp mismatch, a corrupt file
+and an absent one are all the same answer: a miss. The store may make things
+faster, never make them fail.
+
+See **[ANCHORING.md](../../docs/system/ANCHORING.md)** for the pipeline this
+sits in.
 
 ## Utilities
 

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import { SemiontProject } from '@semiont/core/node';
-import { Logger, SupportedMediaType, PdfCoordinate } from '@semiont/core';
+import { Logger, SupportedMediaType, ExtractionOutcome, PdfTextItem, TextExtraction, IContentTransport, AnchoredText } from '@semiont/core';
 
 /**
  * WorkingTreeStore - Manages files in the project working tree
@@ -165,6 +165,299 @@ declare function calculateChecksum(content: string | Buffer): string;
  */
 declare function verifyChecksum(content: string | Buffer, checksum: string): boolean;
 
+/**
+ * Anchored-text cache — the persistent half of ANCHORED-TEXT-CACHE.md Lane 2.
+ *
+ * OCR costs ~2.9 s per scanned page, and six passes read the same document (five
+ * detection motivations plus the smelter's embed), each its own job in its own
+ * process. This stores what the engine produced so only the first pass pays.
+ *
+ * **Derived values only.** Everything here is reproducible from the source
+ * bytes, which is what makes a stamp miss safe. An authored coordinate map is
+ * embedded in the PDF Semiont generated, not stored alongside one — see
+ * `PDF-GENERATION.md`, which owns that decision and states the negative:
+ * never this store.
+ *
+ * The seam is `extract()` (PERSIST-ANCHORS D1/P2b): the record is the FINISHED
+ * extraction outcome — classification, geometry, provenance, or a named
+ * decline — so a hit skips the native parse and the engine both, and every
+ * geometry-yielding extraction stores an entry, native documents included.
+ * That is what makes the anchored-text endpoint answer for every resource
+ * whose extraction yields geometry, and what lets the reconcile planner treat
+ * "no entry under the current checksum" as work (P0's third drift class).
+ */
+
+/** The two halves of the wire record, split for storage. */
+type SuccessOutcome = Exclude<ExtractionOutcome, {
+    declined: string;
+}>;
+type DeclineOutcome = Extract<ExtractionOutcome, {
+    declined: string;
+}>;
+/**
+ * One line of recognized text: the geometry every word on it shares, plus the
+ * per-word parts that differ.
+ *
+ * Grouping is by *contiguous runs* of equal `(y, h)`, never by scanning for all
+ * items at a given y. That makes the codec lossless and order-preserving for
+ * any input — compression is the only thing that depends on words actually
+ * arriving in reading order, and correctness never is.
+ *
+ * Sharing `y`/`h` is measured-safe rather than assumed: within-line word-height
+ * spread is 0.0pt in both native and OCR'd output, because the engine already
+ * normalizes word boxes to the line. Per-word `x` and `width` are stored
+ * explicitly and NOT derived from neighbouring split positions — deriving width
+ * from the gap to the next word would widen every box to touch its neighbour,
+ * which would silently change the coverage arithmetic `textUnder` is calibrated
+ * on (RUN_COVERAGE_THRESHOLD, tuned against ink-tight boxes).
+ */
+interface CachedLine {
+    /** 1-indexed page. */
+    p: number;
+    /** PDF points, bottom-left origin — shared by every word on the line. */
+    y: number;
+    h: number;
+    /** `[x, width, start, end]` per word; offsets index `CachedAnchoredText.text`. */
+    words: [number, number, number, number][];
+}
+/**
+ * The stored record: one extraction OUTCOME for the whole resource
+ * (PERSIST-ANCHORS decision D1) — the anchored text with its provenance
+ * (`method`, `pdfClass`, `ocrConfidence`, `unreadPages`), or a named decline.
+ *
+ * Whole-resource on every side, deliberately. The producer's own shape is a
+ * per-page map, but that is an artifact of how `ocrPages` iterates, and letting
+ * it reach storage would have forced every consumer — the transport, the
+ * browser, a headless client — to reassemble pages it never asked to see.
+ *
+ * The `ocrConfidence` SUMMARY is stored (v2) — this repairs the regression
+ * OCR-CONFIDENCE-LOST.md records, where a hit answered with no confidence at
+ * all. Per-word confidences remain unstored: the summary is the record's
+ * quality provenance; the word list is operator log detail.
+ *
+ * v1 records (bare `{ text, lines }`, no provenance) read as misses under the
+ * v2 prefix; the reconcile planner's third drift class re-derives them.
+ */
+type CachedAnchoredText = ({
+    v: 2;
+    /** Engine + traineddata + our assembly code. A mismatch is a clean miss. */
+    stamp: string;
+    text: string;
+    lines: CachedLine[];
+} & Omit<SuccessOutcome, 'text' | 'items'>) | ({
+    v: 2;
+    stamp: string;
+} & DeclineOutcome);
+interface AnchoredTextStore {
+    /**
+     * The stored map for this key, or null for any miss. Never throws.
+     *
+     * The key is the **content checksum of the bytes the map derives from**
+     * (PERSIST-ANCHORS decision A): a representation is its bytes, so the
+     * checksum is its identity, and geometry derived from one revision of the
+     * bytes is unreachable by a reader holding a different revision — by
+     * construction, not by invalidation. Callers holding some other handle
+     * (a resource id) reach the artifact through an index, not by a second
+     * key scheme here.
+     */
+    read(key: string): Promise<ExtractionOutcome | null>;
+    /** Record an extraction outcome under the content checksum of its source
+     *  bytes. A store that cannot write is still a store. */
+    write(key: string, outcome: ExtractionOutcome): Promise<void>;
+    /**
+     * Every key `read()` would currently HIT — entries under a stale stamp or
+     * unreadable files are excluded, exactly as `read()` would exclude them.
+     * That equivalence is load-bearing: the reconcile planner treats a listed
+     * key as "artifact present" and plans re-derivation for the rest
+     * (PERSIST-ANCHORS P0, the third drift class), so a key listed here but
+     * missed by `read()` would be a permanent loss the diff can never see —
+     * the exact shape of the post-engine-upgrade hole this filter closes.
+     * One bulk call per reconcile, never a probe per resource. Never throws.
+     */
+    list(): Promise<string[]>;
+}
+/**
+ * A file-backed store under `dir` — one file per content key, sharded as
+ * `{ab}/{cd}/{key}.json` via the same `getShardPath` the event log uses
+ * (PERSIST-ANCHORS decision E). Same convention, separate tree: `.semiont/`
+ * is the KB's committed system of record; everything here is derived,
+ * reclaimable, and never a source of truth.
+ *
+ * `dir` is the caller's, out of `Project.anchoredTextDir`: this package has no idea
+ * which project it is serving. Every failure path is a miss rather than an
+ * error, matching the rule extraction already follows for unreadable pages —
+ * the cache may make things faster, never make them fail.
+ */
+declare function createAnchoredTextStore(dir: string, logger?: Logger): AnchoredTextStore;
+
+/**
+ * ContentExtractor — strategy-keyed text extraction for embedding.
+ *
+ * The registry is keyed by `TextExtraction` from `@semiont/core` — the
+ * media-type registry's dispatch vocabulary — never by a second media-type
+ * list (SMELTER-MEDIA-TYPES.md, Design §1): there is exactly one media-type
+ * table in the system, and this registry consumes it. The Smelter resolves
+ * `textExtractionOf(contentType)` and looks the extractor up by strategy; a
+ * `null` slot means decline (settle skipped, reason 'no-extractor').
+ *
+ * Extraction is ephemeral: `extract` runs at read time, its output feeds the
+ * chunker, and is discarded — no stored derived representation. Annotations
+ * anchor to native geometry (`items`), never to extracted-text offsets, so
+ * re-extraction can never break an anchor.
+ */
+
+interface ExtractedText {
+    /** Reading-order plain text, ready for the chunker. */
+    text: string;
+    /**
+     * Positioned text runs indexing `text`, for callers that anchor; absent for
+     * pure text, where character offsets are the anchor. Named `items` to match
+     * `AnchoredText`/`PdfTextLayer` — one concept, one name, and no collision
+     * with the OCR engine's own "blocks" (which are page regions, not runs).
+     */
+    items?: PdfTextItem[];
+    method: 'text-passthrough' | 'pdf-text-layer' | 'table' | 'form' | 'ocr';
+    pdfClass?: 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G';
+    /**
+     * How well the engine read the pixels, when any of this text came from OCR.
+     *
+     * Extraction quality, deliberately NOT anchor confidence: the two answer
+     * different questions. `AnchorConfidence` asks whether the renderer
+     * relocated a stored span in the current text, and for a PDF the answer is
+     * always "exactly" — the viewrect is absolute. This asks whether the glyphs
+     * under that box were read correctly, which no client can recompute.
+     * Reported for operators rather than stored on annotations, following the
+     * existing rule that anchor-audit detail belongs in logs.
+     */
+    ocrConfidence?: {
+        /** Mean per-word confidence, 0–100. */
+        mean: number;
+        /** Words the engine was unsure of — the number worth acting on. */
+        lowConfidenceWords: number;
+        totalWords: number;
+    };
+    /**
+     * 1-indexed pages this extraction could not read — present only when a
+     * document is partially covered (class C). Naming the gap is the point:
+     * without it a hybrid document embeds its native pages and says nothing
+     * about the rest, so coverage silently overstates what search can see.
+     * This is the work list OCR consumes.
+     */
+    unreadPages?: number[];
+}
+/**
+ * A named decline — an extractor that ran and decided it cannot yield text
+ * says why, so the settled signal can carry the class reason (a bare null
+ * could not name its class; SMELTER-MEDIA-TYPES Phase 0 log, note a).
+ */
+interface ExtractionDecline {
+    declined: 'no-text-layer' | 'encrypted' | 'corrupt' | 'too-large';
+}
+/**
+ * Where a strategy may reuse an earlier recognition, and under what key.
+ *
+ * The caller supplies the key, and derives it from the bytes it actually
+ * holds — `calculateChecksum` over the same Buffer it passes to `extract()` —
+ * never from a descriptor's claim. A catalog-derived key can race a byte
+ * change (bytes fetched at one moment, descriptor read at another) and file
+ * or read geometry under an identity that does not describe the bytes being
+ * extracted. The write path made recompute-over-claim the rule
+ * (PERSIST-ANCHORS P1b); readers mirror it (P1c). One SHA-256 over bytes
+ * already in memory is noise against the engine pass a hit avoids.
+ *
+ * Optional throughout: a caller that passes nothing extracts uncached and is
+ * unaffected. The seam is `extract()` itself (PERSIST-ANCHORS D1/P2b): a hit
+ * returns the FINISHED outcome — classification, geometry, provenance, or a
+ * named decline — so neither the native parse nor the engine runs. Every
+ * geometry-yielding extraction produces an entry, native documents included;
+ * the 'decode' strategy ignores the cache (no geometry, nothing expensive).
+ */
+interface ExtractionCache {
+    key: string;
+    store: AnchoredTextStore;
+}
+interface ContentExtractor {
+    /**
+     * Whether this strategy's extractions carry positioned runs (`items`) — the
+     * geometry an anchored-text artifact is made of. Declared, not probed:
+     * the reconcile planner must know "should an artifact exist?" without
+     * running the extractor (PERSIST-ANCHORS P0, the third drift class), and
+     * the declaration keeps the planner's gate and the live fetch's behavior
+     * twins by construction. Text strategies anchor by character offset and
+     * declare false.
+     */
+    yieldsGeometry: boolean;
+    /**
+     * Extract embeddable/annotatable text, or decline with the class reason
+     * (scanned-without-OCR, encrypted, corrupt). The caller skips embedding
+     * and settles skipped with that reason.
+     */
+    extract(content: Buffer, mediaType: string, cache?: ExtractionCache): Promise<ExtractedText | ExtractionDecline>;
+}
+/**
+ * Strategy → extractor. A `null` slot is a decline: the strategy names a
+ * capability nothing currently provides ('none' permanently).
+ */
+declare const EXTRACTORS: Record<TextExtraction, ContentExtractor | null>;
+
+/**
+ * PDF extractor — the 'pdf-text-layer' strategy (SMELTER-MEDIA-TYPES).
+ *
+ * Wraps the shared `extractPdfTextLayer` reader (detection's other consumer)
+ * and turns a PDF into text plus the geometry that indexes it, by class:
+ *
+ *   A native text layer   → read directly
+ *   B scanned             → read the page pixels by OCR
+ *   C hybrid              → both, with any page still unread reported
+ *   D tables              → grid pages rewritten as markdown rows
+ *   E forms               → AcroForm values folded in, anchored to widgets
+ *   F/G encrypted, corrupt → declined by name, from the parser error
+ *
+ * Everything runs inline. OCR was originally planned off the hot path, but
+ * the Smelter's lanes are per-resource and concurrent, so a slow page delays
+ * only its own resource — see SMELTER-MEDIA-TYPES Design §4 (revised).
+ */
+
+/**
+ * Largest PDF this will attempt, in bytes.
+ *
+ * A PDF is a compressed container, so input size bounds nothing on its own —
+ * but it is the one number available before the parser touches the file, and
+ * refusing here means a hostile or pathological document never gets to expand
+ * inside pdf.js. Chosen to sit above real corpora (a few hundred pages of
+ * scanned FOIA material runs tens of megabytes) while still being a ceiling.
+ *
+ * A starting point, not a measured optimum — revisit against a real corpus
+ * (SMELTER-MEDIA-TYPES, live-testing follow-up). The per-image budget in
+ * `pdf-page-images` guards the decoded side, which is where the unbounded
+ * growth actually lives.
+ */
+declare const MAX_PDF_BYTES: number;
+/** Whether a document is small enough to attempt. Exported because the
+ *  threshold is a judgement, and judgements deserve tests that do not have to
+ *  materialize two hundred megabytes to ask the question. */
+declare function withinByteBudget(bytes: number): boolean;
+
+/**
+ * `AnchoredTextStore` over `IContentTransport` — how an out-of-process
+ * extraction seam reaches the one real store (PERSIST-ANCHORS P2c).
+ *
+ * Every cache consumer runs outside the backend — the smelter worker and the
+ * detection workers — while the KnowledgeSystem owns the storage. This
+ * adapter maps the store contract onto the transport's three
+ * checksum-addressed calls, so `ExtractionCache { key, store }` works
+ * identically in-process (LocalContentTransport → the store directly) and
+ * over the wire (HttpContentTransport → the /anchored-text routes).
+ *
+ * It honors the store contract's failure rule — the cache may make things
+ * faster, never make them fail: a read or list failure is a miss, a write
+ * failure is swallowed (debug-logged). Callers that need a write to be LOUD
+ * — the re-anchor path, whose artifact IS the job — use the transport's
+ * `putAnchoredText` directly, not this adapter.
+ */
+
+declare function anchoredTextStoreOverTransport(content: IContentTransport, logger?: Logger): AnchoredTextStore;
+
 /**
  * PDF Text Layer Types
  *
@@ -173,37 +466,58 @@ declare function verifyChecksum(content: string | Buffer, checksum: string): boo
  * (Y increases upward). The Y-flip to canvas pixels happens downstream in the
  * browser; the server has no canvas.
  *
- * `PdfCoordinate` — the geometry `locate()` emits — lives in `@semiont/core`
- * alongside the viewrect FragmentSelector codec.
+ * The anchoring vocabulary these build on — `AnchoredText`, `PdfTextItem`,
+ * `PdfCoordinate`, and the `locate`/`textUnder` pair — lives in `@semiont/core`
+ * alongside the viewrect FragmentSelector codec, so the browser can reason over
+ * geometry without importing this package's extraction stack.
  */
+
 /**
- * A single text item (one text run, roughly a word) from the PDF text layer.
- * Character offsets refer to positions in `PdfTextLayer.text`.
+ * One filled AcroForm field: the value a form carries outside its drawn text
+ * layer. Geometry is the widget's rectangle, in the same PDF-point,
+ * bottom-left-origin space as `PdfTextItem` — so a consumer that folds field
+ * values into text can anchor them like any other run.
  */
-interface PdfTextItem {
-    start: number;
-    end: number;
+interface PdfFormField {
+    name: string;
+    value: string;
     page: number;
     x: number;
     y: number;
     width: number;
     height: number;
 }
-/** Page dimensions in PDF points */
+/** Page dimensions in PDF points, plus the page's span of `PdfTextLayer.text` */
 interface PdfPageInfo {
     pageNumber: number;
     widthPt: number;
     heightPt: number;
+    textStart: number;
+    textEnd: number;
+    /**
+     * Whether this page carries text-showing operators. False means the page
+     * is scanned: its characters exist only as pixels, so reading it needs
+     * OCR rather than `getTextContent`. Per-PAGE, deliberately: a document
+     * mixing native and scanned pages (class C) must route each page
+     * separately, and a single document-level flag cannot express that.
+     */
+    hasTextLayer: boolean;
 }
 /**
  * The full extracted text layer for a PDF.
  * `text` is the reading-order concatenation across all pages.
  * Each `item` is one text run carrying its character range into `text` plus PDF-point geometry.
  */
-interface PdfTextLayer {
+interface PdfTextLayer extends AnchoredText {
     pages: PdfPageInfo[];
-    text: string;
-    items: PdfTextItem[];
+    /**
+     * Filled AcroForm field values, empty for a document without a form.
+     * Deliberately NOT folded into `text`: the reader reports what the
+     * document holds, and each consumer projects what it needs — detection
+     * reads `text`/`items` and is unaffected by a form's presence, while the
+     * embedding extractor folds these in (SMELTER-MEDIA-TYPES class E).
+     */
+    fields: PdfFormField[];
 }
 
 /**
@@ -218,22 +532,5 @@ interface PdfTextLayer {
 
 declare function extractPdfTextLayer(bytes: Uint8Array | Buffer): Promise<PdfTextLayer | null>;
 
-/**
- * Locates bounding rectangles for a span of text in a PdfTextLayer
- * (single-line or multi-line).
- *
- * Finds all overlapping items [start, end), groups them by page and line, and
- * records one bounding rectangle per line as a PdfCoordinate.
- *
- * Returns both the per-line `rects` and the `overlap` items they were computed
- * from — so a caller that also needs the covered text (e.g. buildPdfAnnotation's
- * geometry↔text containment invariant) reuses this single `layer.items` scan
- * instead of re-filtering. Both arrays are empty if no item overlaps the span.
- */
-declare function locate(layer: PdfTextLayer, start: number, end: number): {
-    rects: PdfCoordinate[];
-    overlap: PdfTextItem[];
-};
-
-export { ChecksumMismatchError, WorkingTreeStore, calculateChecksum, deriveStorageUri, extractPdfTextLayer, locate, verifyChecksum };
-export type { PdfPageInfo, PdfTextItem, PdfTextLayer, StoredResource };
+export { ChecksumMismatchError, EXTRACTORS, MAX_PDF_BYTES, WorkingTreeStore, anchoredTextStoreOverTransport, calculateChecksum, createAnchoredTextStore, deriveStorageUri, extractPdfTextLayer, verifyChecksum, withinByteBudget };
+export type { AnchoredTextStore, CachedAnchoredText, CachedLine, ContentExtractor, ExtractedText, ExtractionCache, ExtractionDecline, PdfFormField, PdfPageInfo, PdfTextLayer, StoredResource };