@semiont/content 0.5.29 → 0.5.31

package/dist/index.d.ts

@@ -1,155 +1,102 @@
 import { Readable } from 'stream';
 import { SemiontProject, ArchivistAddressConfig } from '@semiont/core/node';
-import { Logger, ExtractionOutcome, PdfTextItem, TextExtraction, IContentTransport, AnchoredText } from '@semiont/core';
+import { Logger, StoredResource, ExtractionOutcome, PdfTextItem, IContentTransport, AnchoredText } from '@semiont/core';
 
 /**
- * WorkingTreeStore - Manages files in the project working tree
- *
- * Unlike the old content-addressed RepresentationStore, this store treats
- * the working tree (project root) as the source of truth for file content.
- * Resources are identified by their file:// URI, which is stable across
- * content changes and moves (tracked by events).
- *
- * Two write paths:
- * - store(content, storageUri): Write bytes to disk (API/GUI/AI path).
- *   Used when the file does not yet exist and the caller provides content.
- * - register(storageUri, expectedChecksum?): Adopt a file already on disk and
- *   return its metadata. The CLI path (the file arrived by other means) and
- *   the event-apply path (the Stower staging bytes an event names) both use
- *   it. Streams the file to hash it — never holds it. If expectedChecksum is
- *   provided, throws on mismatch.
- *
- * Storage layout:
- *   {projectRoot}/{path-from-uri}
- *
- * For example, storageUri "file://docs/overview.md" resolves to
- *   {projectRoot}/docs/overview.md
- */
-
-/**
- * Result of store() or register()
+ * Deferred, deduped `git add`. The index is for humans who commit by hand, so
+ * it must be current within seconds, not after every change.
+ *
+ * Serialized per repo — git's index is single-writer, and concurrent `git add`
+ * fails on `index.lock` rather than retrying. Created on first use, never at
+ * import: consumers of this package may never stage anything.
  */
-interface StoredResource {
-    storageUri: string;
-    checksum: string;
-    byteSize: number;
-    created: string;
+interface StagerOptions {
+    /** Quiet period after the last change before staging. */
+    flushMs?: number;
+    /** Ceiling on staleness: stage this long after the OLDEST pending path even
+     *  if changes keep arriving. Without it a continuous append stream resets
+     *  the debounce forever and the index never updates. */
+    maxWaitMs?: number;
+}
+interface Stager {
+    /** Queue a path. Returns immediately; deduped against what is pending. */
+    add(path: string): void;
+    /** Run an order-sensitive command (`mv`, `rm`): pending adds flush first,
+     *  then this runs alone — it must not overtake the adds it depends on. */
+    run(args: string[]): Promise<void>;
+    /** Stage everything pending now. */
+    flush(): Promise<void>;
+    /** Paths queued and not yet staged. */
+    pending(): number;
+    /** Drain and stop. A stopped process must leave nothing unstaged. */
+    dispose(): Promise<void>;
 }
+declare function createStager(cwd: string, options?: StagerOptions): Stager;
+
 /**
- * Manages files in the project working tree
+ * Files in the project working tree, addressed by file:// URI —
+ * "file://docs/overview.md" is {projectRoot}/docs/overview.md.
+ *
+ * `store` writes bytes the caller supplies; `register` adopts a file already
+ * on disk. Both stream to hash, neither holds a representation in memory.
  */
+
 declare class WorkingTreeStore {
     private projectRoot;
     private gitSync;
     private logger?;
-    constructor(project: SemiontProject, logger?: Logger);
+    private _stager?;
+    private readonly staging;
+    /** `staging` is policy — how stale the index may get is the caller's call. */
+    constructor(project: SemiontProject, logger?: Logger, staging?: StagerOptions);
+    /** Created on first use — importers of this package may never stage. */
+    private stager;
+    /** Stage everything pending now — for a caller that wants the index current. */
+    flushStaging(): Promise<void>;
+    /** Drain and stop. A stopped process must leave nothing unstaged. */
+    dispose(): Promise<void>;
     private shouldRunGit;
     /**
-     * Write content to disk at the location indicated by storageUri.
-     *
-     * API/GUI/AI path: caller provides bytes — as a Buffer it already holds, or
-     * as a stream (the Archivist's write endpoint hands the request body
-     * straight through, SINGLE-KB-MOUNT P2/D7: memory stays bounded by the
-     * chunk, never the representation).
+     * Write bytes to the path storageUri names, whole or streamed.
      *
-     * Atomic either way: bytes stream into a temp file beside the target and
-     * are renamed into place only once complete — and only once
-     * `expectedChecksum`, when given, agrees with what actually arrived. A
-     * mismatch or a torn stream leaves the target untouched (a version being
-     * overwritten survives) and no temp file behind, so the Stower's `register`
-     * can never find partial bytes an event names.
+     * Atomic: bytes land in a temp file and are renamed into place only once
+     * complete and once `expectedChecksum`, when given, agrees. A mismatch or a
+     * torn stream leaves the target untouched, so `register` can never find
+     * partial bytes an event names.
      *
-     * @param content - Raw bytes to write, whole or streamed
-     * @param storageUri - file:// URI (e.g. "file://docs/overview.md")
      * @throws ChecksumMismatchError when expectedChecksum disagrees with the body
-     * @returns Stored resource metadata
      */
     store(content: Buffer | Readable, storageUri: string, options?: {
         noGit?: boolean;
         expectedChecksum?: string;
     }): Promise<StoredResource>;
     /**
-     * Read an existing file and return its metadata.
+     * Adopt a file already on disk: stream it to hash it, then stage it.
      *
-     * The file is already on disk; this hashes it by streaming to confirm what
-     * it is, then stages it. If expectedChecksum is provided, throws
-     * ChecksumMismatchError on mismatch.
-     *
-     * @param storageUri - file:// URI (e.g. "file://docs/overview.md")
-     * @param expectedChecksum - Optional SHA-256 to verify against
-     * @returns Stored resource metadata
-     * @throws ChecksumMismatchError if expectedChecksum is provided and does not match
-     * @throws Error if file does not exist
+     * @throws ChecksumMismatchError if expectedChecksum is given and disagrees
      */
     register(storageUri: string, expectedChecksum?: string, options?: {
         noGit?: boolean;
     }): Promise<StoredResource>;
     /**
-     * Read file content by URI.
-     *
-     * @param storageUri - file:// URI
-     * @returns Raw bytes
-     */
-    /**
-     * The same bytes as `retrieve`, streamed — for the byte paths that must not
-     * hold a whole representation in memory (SINGLE-KB-MOUNT D7: the Archivist
-     * serves content for every reader now, so its memory cannot be bounded by
-     * the largest file anyone asks for).
-     *
-     * Lazy by construction: the stream is created here but nothing is read
-     * until the caller iterates, so a missing file surfaces as an `error` event
-     * on the stream rather than a rejected promise. Callers that need the
-     * distinction up front should resolve the descriptor first — which is what
-     * `resolveRepresentation` does.
+     * The same bytes as `retrieve`, streamed. Lazy: a missing file surfaces as
+     * an `error` event on the stream, not a rejected promise — callers needing
+     * that up front should resolve the descriptor first.
      */
     retrieveStream(storageUri: string): Readable;
     retrieve(storageUri: string): Promise<Buffer>;
-    /**
-     * Move a file from one URI to another.
-     *
-     * If .git/ exists in the project root and noGit is not set, runs `git mv`.
-     * Otherwise (no .git/ or noGit: true), runs fs.rename.
-     *
-     * @param fromUri - Current file:// URI
-     * @param toUri - New file:// URI
-     * @param options.noGit - Skip git mv even if .git/ is present
-     */
+    /** `git mv` when the project syncs git, `fs.rename` otherwise. */
     move(fromUri: string, toUri: string, options?: {
         noGit?: boolean;
     }): Promise<void>;
-    /**
-     * Remove a file from the working tree.
-     *
-     * If .git/ exists and noGit is not set:
-     *   - keepFile false (default): runs `git rm` (removes from index and disk)
-     *   - keepFile true: runs `git rm --cached` (removes from index only, file stays on disk)
-     * If no .git/ or noGit: true:
-     *   - keepFile false: runs fs.unlink
-     *   - keepFile true: no-op on filesystem
-     *
-     * @param storageUri - file:// URI
-     * @param options.noGit - Skip git rm even if .git/ is present
-     * @param options.keepFile - Remove from git index only; leave file on disk
-     */
+    /** @param options.keepFile - Drop from the index only; leave the file on disk. */
     remove(storageUri: string, options?: {
         noGit?: boolean;
         keepFile?: boolean;
     }): Promise<void>;
-    /**
-     * Convert a file:// URI to an absolute filesystem path.
-     *
-     * "file://docs/overview.md" → "{projectRoot}/docs/overview.md"
-     *
-     * @param storageUri - file:// URI
-     * @returns Absolute path
-     */
     resolveUri(storageUri: string): string;
 }
-/**
- * Thrown when a registered file's checksum does not match the expected value.
- * This indicates the file on disk differs from what was recorded (e.g. modified
- * after staging, or wrong file path provided).
- */
+/** The file on disk is not the file the checksum names. */
 declare class ChecksumMismatchError extends Error {
     readonly storageUri: string;
     readonly expected: string;
@@ -308,19 +255,23 @@ interface AnchoredTextStore {
 declare function createAnchoredTextStore(dir: string, logger?: Logger): AnchoredTextStore;
 
 /**
- * ContentExtractor — strategy-keyed text extraction for embedding.
- *
- * The registry is keyed by `TextExtraction` from `@semiont/core` — the
- * media-type registry's dispatch vocabulary — never by a second media-type
- * list (SMELTER-MEDIA-TYPES.md, Design §1): there is exactly one media-type
- * table in the system, and this registry consumes it. The Smelter resolves
- * `textExtractionOf(contentType)` and looks the extractor up by strategy; a
- * `null` slot means decline (settle skipped, reason 'no-extractor').
- *
- * Extraction is ephemeral: `extract` runs at read time, its output feeds the
- * chunker, and is discarded — no stored derived representation. Annotations
- * anchor to native geometry (`items`), never to extracted-text offsets, so
- * re-extraction can never break an anchor.
+ * TextExtractor — DERIVING text from bytes that carry none of their own.
+ *
+ * Scope note (READ-VS-EXTRACT P2/P3): this file used to hold a registry covering
+ * both ways a resource yields text — decoding (charset-aware `Buffer → string`)
+ * and deriving (parse a PDF, OCR it when there is no text layer). Those shared a
+ * name and almost nothing else: microseconds vs. minutes,
+ * total determinism vs. none across engine versions, no canonical artifact vs.
+ * exactly one, and anyone-with-bytes vs. the Smelter alone. The registry made
+ * them interchangeable at every call site.
+ *
+ * Decoding left: it is `decodeRepresentation` in `@semiont/core`, called
+ * directly. What remains here is the deriving half, reached through
+ * `derivingExtractorFor` and callable only with the store that persists its
+ * output.
+ *
+ * Anchoring is unaffected: annotations anchor to native geometry (`items`),
+ * never to extracted-text offsets, so re-derivation can never break an anchor.
  */
 
 interface ExtractedText {
@@ -375,7 +326,7 @@ interface ExtractionDecline {
     declined: 'no-text-layer' | 'encrypted' | 'corrupt' | 'too-large';
 }
 /**
- * Where a strategy may reuse an earlier recognition, and under what key.
+ * Where a derivation may reuse an earlier recognition, and under what key.
  *
  * The caller supplies the key, and derives it from the bytes it actually
  * holds — `calculateChecksum` over the same Buffer it passes to `extract()` —
@@ -386,40 +337,72 @@ interface ExtractionDecline {
  * (PERSIST-ANCHORS P1b); readers mirror it (P1c). One SHA-256 over bytes
  * already in memory is noise against the engine pass a hit avoids.
  *
- * Optional throughout: a caller that passes nothing extracts uncached and is
- * unaffected. The seam is `extract()` itself (PERSIST-ANCHORS D1/P2b): a hit
- * returns the FINISHED outcome — classification, geometry, provenance, or a
- * named decline — so neither the native parse nor the engine runs. Every
- * geometry-yielding extraction produces an entry, native documents included;
- * the 'decode' strategy ignores the cache (no geometry, nothing expensive).
+ * REQUIRED, and that is the ownership rule (READ-VS-EXTRACT P2). It carries an
+ * `AnchoredTextStore`, and only the Smelter holds one — so deriving is reachable
+ * exactly to the process that can persist what it derived. The restriction is a
+ * capability the caller must already hold, not a convention it must remember:
+ * a would-be second producer cannot construct the argument, so it cannot compile.
+ *
+ * The seam is `extract()` itself (PERSIST-ANCHORS D1/P2b): a hit returns the
+ * FINISHED outcome — classification, geometry, provenance, or a named decline —
+ * so neither the native parse nor the engine runs.
  */
 interface ExtractionCache {
     key: string;
     store: AnchoredTextStore;
 }
-interface ContentExtractor {
-    /**
-     * Whether this strategy's extractions carry positioned runs (`items`) — the
-     * geometry an anchored-text artifact is made of. Declared, not probed:
-     * the reconcile planner must know "should an artifact exist?" without
-     * running the extractor (PERSIST-ANCHORS P0, the third drift class), and
-     * the declaration keeps the planner's gate and the live fetch's behavior
-     * twins by construction. Text strategies anchor by character offset and
-     * declare false.
-     */
-    yieldsGeometry: boolean;
+/**
+ * Deriving text from bytes that carry none of their own.
+ *
+ * Named `ContentExtractor` until READ-VS-EXTRACT P3: the `Content` prefix named
+ * the INPUT, when what distinguishes this type is that it produces TEXT — by
+ * deriving, which since P3 is the only thing "extraction" means here. WHERE a
+ * media type's text comes from at all is core's `TextSource`, which spans both
+ * routes and is therefore not called extraction.
+ *
+ * Whether a text source yields positioned runs lives in `@semiont/core`'s
+ * `yieldsGeometryOf`, NOT here (P1). It is a property of the source, and that
+ * vocabulary is core's — declaring it per-implementation made it two facts that
+ * could disagree, and forced consumers asking about a media type to resolve an
+ * implementation to find out. `text-extractor.test.ts` gates core's answer
+ * against what these extractors actually produce.
+ */
+interface TextExtractor {
     /**
-     * Extract embeddable/annotatable text, or decline with the class reason
-     * (scanned-without-OCR, encrypted, corrupt). The caller skips embedding
-     * and settles skipped with that reason.
+     * Derive text WITH geometry from bytes that carry no text of their own, or
+     * decline with the class reason (scanned-without-OCR, encrypted, corrupt).
+     * The caller skips embedding and settles skipped with that reason.
+     *
+     * Expensive, non-deterministic across engine versions, and the sole producer
+     * of a canonical artifact — which is why `cache` is required rather than
+     * optional (see `ExtractionCache`).
      */
-    extract(content: Buffer, mediaType: string, cache?: ExtractionCache): Promise<ExtractedText | ExtractionDecline>;
+    extract(content: Buffer, mediaType: string, cache: ExtractionCache): Promise<ExtractedText | ExtractionDecline>;
 }
 /**
- * Strategy → extractor. A `null` slot is a decline: the strategy names a
- * capability nothing currently provides ('none' permanently).
+ * The deriving extractor for a media type, or `null` when its text needs no
+ * deriving.
+ *
+ * **This replaced a `Record<TextSource, TextExtractor | null>` keyed by
+ * strategy (READ-VS-EXTRACT P2), and the deletion is the point.** That map held
+ * one real extractor, a `null`, and — under 'decode' — a one-line wrapper around
+ * core's `decodeRepresentation`, which five sites in `@semiont/make-meaning`
+ * already called directly. Resolving "give me an extractor for this media type"
+ * therefore returned, half the time, a trivial function dressed as the same
+ * capability as OCR: identical at the call site, wildly different in cost,
+ * determinism, and who is allowed to run it. That symmetry is what let a
+ * detection worker OCR scanned PDFs for four months without anyone reading it as
+ * a category error (#739).
+ *
+ * Decoding is now a direct `decodeRepresentation()` call at the two sites that
+ * need it. There is no registry to resolve, so there is no way to reach OCR by
+ * asking a generic question — and a caller that gets a non-null answer here still
+ * cannot run it without an `AnchoredTextStore`.
+ *
+ * Keyed by P1's `yieldsGeometryOf`, so this and the Smelter's publish gate cannot
+ * disagree about which media types have a canonical artifact.
  */
-declare const EXTRACTORS: Record<TextExtraction, ContentExtractor | null>;
+declare function derivingExtractorFor(mediaType: string): TextExtractor | null;
 
 /**
  * PDF extractor — the 'pdf-text-layer' strategy (SMELTER-MEDIA-TYPES).
@@ -583,5 +566,5 @@ interface PdfTextLayer extends AnchoredText {
 
 declare function extractPdfTextLayer(bytes: Uint8Array | Buffer): Promise<PdfTextLayer | null>;
 
-export { ChecksumMismatchError, EXTRACTORS, MAX_PDF_BYTES, RepresentationMissing, WorkingTreeStore, archivistContentReads, calculateChecksum, createAnchoredTextStore, extractPdfTextLayer, verifyChecksum, withinByteBudget };
-export type { AnchoredTextStore, CachedAnchoredText, CachedLine, ContentExtractor, ContentReads, ExtractedText, ExtractionCache, ExtractionDecline, MissingReason, PdfFormField, PdfPageInfo, PdfTextLayer, StoredResource };
+export { ChecksumMismatchError, MAX_PDF_BYTES, RepresentationMissing, WorkingTreeStore, archivistContentReads, calculateChecksum, createAnchoredTextStore, createStager, derivingExtractorFor, extractPdfTextLayer, verifyChecksum, withinByteBudget };
+export type { AnchoredTextStore, CachedAnchoredText, CachedLine, ContentReads, ExtractedText, ExtractionCache, ExtractionDecline, MissingReason, PdfFormField, PdfPageInfo, PdfTextLayer, Stager, StagerOptions, TextExtractor };