@semiont/content 0.5.30 → 0.5.32

package/dist/index.js

@@ -1,19 +1,125 @@
 // src/working-tree-store.ts
 import { promises as fs, createReadStream, createWriteStream } from "fs";
-import { execFileSync } from "child_process";
 import { createHash, randomUUID } from "crypto";
 import { Readable } from "stream";
 import { pipeline } from "stream/promises";
 import path from "path";
-import { recordGitCommand } from "@semiont/observability";
-function git(args, cwd) {
-  const started = performance.now();
-  try {
-    execFileSync("git", args, { cwd });
-  } finally {
-    recordGitCommand(args[0] ?? "git", performance.now() - started);
-  }
+
+// src/git-staging.ts
+import { execFile } from "child_process";
+import { resolve } from "path";
+import { promisify } from "util";
+import { recordGitCommand, recordGitStagingFailure } from "@semiont/observability";
+var run = promisify(execFile);
+var LOCK_RETRY_DELAYS_MS = [50, 100, 200, 400, 800, 1600];
+var isIndexLockContention = (error) => typeof error === "object" && error !== null && error.code === 128 && String(error.stderr ?? "").includes("index.lock");
+var DEFAULT_FLUSH_MS = 250;
+var DEFAULT_MAX_WAIT_MS = 2e3;
+var stagers = /* @__PURE__ */ new Map();
+function createStager(cwd, options = {}) {
+  const key = resolve(cwd);
+  const existing = stagers.get(key);
+  if (existing) return existing;
+  const stager = buildStager(key, options);
+  stagers.set(key, stager);
+  return stager;
+}
+function buildStager(cwd, options = {}) {
+  const flushMs = options.flushMs ?? DEFAULT_FLUSH_MS;
+  const maxWaitMs = options.maxWaitMs ?? DEFAULT_MAX_WAIT_MS;
+  const queued = /* @__PURE__ */ new Set();
+  let timer;
+  let oldestAt;
+  let inFlight = Promise.resolve();
+  let disposed = false;
+  const git = async (args) => {
+    const started = performance.now();
+    try {
+      for (let attempt = 0; ; attempt++) {
+        try {
+          await run("git", args, { cwd });
+          return;
+        } catch (error) {
+          const last = attempt >= LOCK_RETRY_DELAYS_MS.length;
+          if (last || !isIndexLockContention(error)) throw error;
+          await new Promise((r) => setTimeout(r, LOCK_RETRY_DELAYS_MS[attempt]));
+        }
+      }
+    } finally {
+      recordGitCommand(args[0] ?? "git", performance.now() - started);
+    }
+  };
+  const serialize = (work) => {
+    inFlight = inFlight.then(work, work);
+    return inFlight;
+  };
+  const clearTimer = () => {
+    if (timer) {
+      clearTimeout(timer);
+      timer = void 0;
+    }
+  };
+  const drain = () => {
+    clearTimer();
+    if (queued.size === 0) return inFlight;
+    const batch = [...queued];
+    queued.clear();
+    oldestAt = void 0;
+    return serialize(
+      () => git(["add", ...batch]).catch((error) => {
+        const lock = isIndexLockContention(error);
+        if (lock) {
+          for (const path4 of batch) queued.add(path4);
+          if (oldestAt === void 0) oldestAt = Date.now();
+          arm();
+        }
+        recordGitStagingFailure(lock ? "index-lock" : "other");
+      })
+    );
+  };
+  const arm = () => {
+    clearTimer();
+    if (disposed || queued.size === 0) return;
+    const sinceOldest = oldestAt === void 0 ? 0 : Date.now() - oldestAt;
+    const wait = Math.max(0, Math.min(flushMs, maxWaitMs - sinceOldest));
+    timer = setTimeout(() => {
+      void drain();
+    }, wait);
+    timer.unref?.();
+  };
+  return {
+    add(path4) {
+      if (disposed) return;
+      if (queued.size === 0) oldestAt = Date.now();
+      queued.add(path4);
+      arm();
+    },
+    run(args) {
+      return drain().then(
+        () => serialize(
+          () => git(args).catch((error) => {
+            recordGitStagingFailure(isIndexLockContention(error) ? "index-lock" : "other");
+          })
+        )
+      );
+    },
+    flush() {
+      return drain();
+    },
+    pending() {
+      return queued.size;
+    },
+    async dispose() {
+      stagers.delete(cwd);
+      await drain();
+      disposed = true;
+      clearTimer();
+      await inFlight;
+    }
+  };
 }
+
+// src/working-tree-store.ts
 function hashingTap() {
   const hash = createHash("sha256");
   let byteSize = 0;
@@ -34,33 +140,40 @@ var WorkingTreeStore = class {
   projectRoot;
   gitSync;
   logger;
-  constructor(project, logger) {
+  _stager;
+  staging;
+  /** `staging` is policy — how stale the index may get is the caller's call. */
+  constructor(project, logger, staging = {}) {
     this.projectRoot = project.root;
     this.gitSync = project.gitSync;
     this.logger = logger;
+    this.staging = staging;
+  }
+  /** Created on first use — importers of this package may never stage. */
+  stager() {
+    if (!this._stager) this._stager = createStager(this.projectRoot, this.staging);
+    return this._stager;
+  }
+  /** Stage everything pending now — for a caller that wants the index current. */
+  flushStaging() {
+    return this._stager ? this._stager.flush() : Promise.resolve();
+  }
+  /** Drain and stop. A stopped process must leave nothing unstaged. */
+  async dispose() {
+    if (this._stager) await this._stager.dispose();
   }
   shouldRunGit(noGit) {
     return this.gitSync && !noGit;
   }
   /**
-   * Write content to disk at the location indicated by storageUri.
+   * Write bytes to the path storageUri names, whole or streamed.
    *
-   * API/GUI/AI path: caller provides bytes — as a Buffer it already holds, or
-   * as a stream (the Archivist's write endpoint hands the request body
-   * straight through, SINGLE-KB-MOUNT P2/D7: memory stays bounded by the
-   * chunk, never the representation).
+   * Atomic: bytes land in a temp file and are renamed into place only once
+   * complete and once `expectedChecksum`, when given, agrees. A mismatch or a
+   * torn stream leaves the target untouched, so `register` can never find
+   * partial bytes an event names.
    *
-   * Atomic either way: bytes stream into a temp file beside the target and
-   * are renamed into place only once complete — and only once
-   * `expectedChecksum`, when given, agrees with what actually arrived. A
-   * mismatch or a torn stream leaves the target untouched (a version being
-   * overwritten survives) and no temp file behind, so the Stower's `register`
-   * can never find partial bytes an event names.
-   *
-   * @param content - Raw bytes to write, whole or streamed
-   * @param storageUri - file:// URI (e.g. "file://docs/overview.md")
    * @throws ChecksumMismatchError when expectedChecksum disagrees with the body
-   * @returns Stored resource metadata
    */
   async store(content, storageUri, options) {
     const filePath = this.resolveUri(storageUri);
@@ -87,7 +200,7 @@ var WorkingTreeStore = class {
       }
       await fs.rename(tempPath, filePath);
       if (this.shouldRunGit(options?.noGit)) {
-        git(["add", filePath], this.projectRoot);
+        this.stager().add(filePath);
       }
       this.logger?.info("Resource stored", { storageUri, checksum, byteSize });
       return {
@@ -102,17 +215,9 @@ var WorkingTreeStore = class {
     }
   }
   /**
-   * Read an existing file and return its metadata.
-   *
-   * The file is already on disk; this hashes it by streaming to confirm what
-   * it is, then stages it. If expectedChecksum is provided, throws
-   * ChecksumMismatchError on mismatch.
+   * Adopt a file already on disk: stream it to hash it, then stage it.
    *
-   * @param storageUri - file:// URI (e.g. "file://docs/overview.md")
-   * @param expectedChecksum - Optional SHA-256 to verify against
-   * @returns Stored resource metadata
-   * @throws ChecksumMismatchError if expectedChecksum is provided and does not match
-   * @throws Error if file does not exist
+   * @throws ChecksumMismatchError if expectedChecksum is given and disagrees
    */
   async register(storageUri, expectedChecksum, options) {
     const filePath = this.resolveUri(storageUri);
@@ -126,7 +231,7 @@ var WorkingTreeStore = class {
       throw new ChecksumMismatchError(storageUri, expectedChecksum, checksum);
     }
     if (this.shouldRunGit(options?.noGit)) {
-      git(["add", filePath], this.projectRoot);
+      this.stager().add(filePath);
     }
     const byteSize = tap.byteSize;
     this.logger?.info("Resource registered", { storageUri, checksum, byteSize });
@@ -138,22 +243,9 @@ var WorkingTreeStore = class {
     };
   }
   /**
-   * Read file content by URI.
-   *
-   * @param storageUri - file:// URI
-   * @returns Raw bytes
-   */
-  /**
-   * The same bytes as `retrieve`, streamed — for the byte paths that must not
-   * hold a whole representation in memory (SINGLE-KB-MOUNT D7: the Archivist
-   * serves content for every reader now, so its memory cannot be bounded by
-   * the largest file anyone asks for).
-   *
-   * Lazy by construction: the stream is created here but nothing is read
-   * until the caller iterates, so a missing file surfaces as an `error` event
-   * on the stream rather than a rejected promise. Callers that need the
-   * distinction up front should resolve the descriptor first — which is what
-   * `resolveRepresentation` does.
+   * The same bytes as `retrieve`, streamed. Lazy: a missing file surfaces as
+   * an `error` event on the stream, not a rejected promise — callers needing
+   * that up front should resolve the descriptor first.
    */
   retrieveStream(storageUri) {
     return createReadStream(this.resolveUri(storageUri));
@@ -169,42 +261,20 @@ var WorkingTreeStore = class {
       throw error;
     }
   }
-  /**
-   * Move a file from one URI to another.
-   *
-   * If .git/ exists in the project root and noGit is not set, runs `git mv`.
-   * Otherwise (no .git/ or noGit: true), runs fs.rename.
-   *
-   * @param fromUri - Current file:// URI
-   * @param toUri - New file:// URI
-   * @param options.noGit - Skip git mv even if .git/ is present
-   */
+  /** `git mv` when the project syncs git, `fs.rename` otherwise. */
   async move(fromUri, toUri, options) {
     const fromPath = this.resolveUri(fromUri);
     const toPath = this.resolveUri(toUri);
     this.logger?.debug("Moving resource", { fromUri, toUri });
     await fs.mkdir(path.dirname(toPath), { recursive: true });
     if (this.shouldRunGit(options?.noGit)) {
-      git(["mv", fromPath, toPath], this.projectRoot);
+      await this.stager().run(["mv", fromPath, toPath]);
     } else {
       await fs.rename(fromPath, toPath);
     }
     this.logger?.info("Resource moved", { fromUri, toUri });
   }
-  /**
-   * Remove a file from the working tree.
-   *
-   * If .git/ exists and noGit is not set:
-   *   - keepFile false (default): runs `git rm` (removes from index and disk)
-   *   - keepFile true: runs `git rm --cached` (removes from index only, file stays on disk)
-   * If no .git/ or noGit: true:
-   *   - keepFile false: runs fs.unlink
-   *   - keepFile true: no-op on filesystem
-   *
-   * @param storageUri - file:// URI
-   * @param options.noGit - Skip git rm even if .git/ is present
-   * @param options.keepFile - Remove from git index only; leave file on disk
-   */
+  /** @param options.keepFile - Drop from the index only; leave the file on disk. */
   async remove(storageUri, options) {
     const filePath = this.resolveUri(storageUri);
     const keepFile = options?.keepFile ?? false;
@@ -212,7 +282,7 @@ var WorkingTreeStore = class {
     const useGit = this.shouldRunGit(options?.noGit);
     if (useGit) {
       const gitArgs = keepFile ? ["rm", "--cached", filePath] : ["rm", filePath];
-      git(gitArgs, this.projectRoot);
+      await this.stager().run(gitArgs);
       this.logger?.info("Resource removed", { storageUri, keepFile, git: true });
       return;
     }
@@ -231,14 +301,6 @@ var WorkingTreeStore = class {
       throw error;
     }
   }
-  /**
-   * Convert a file:// URI to an absolute filesystem path.
-   *
-   * "file://docs/overview.md" → "{projectRoot}/docs/overview.md"
-   *
-   * @param storageUri - file:// URI
-   * @returns Absolute path
-   */
   resolveUri(storageUri) {
     if (!storageUri.startsWith("file://")) {
       throw new Error(`Invalid storage URI (must start with file://): ${storageUri}`);
@@ -274,8 +336,8 @@ function verifyChecksum(content, checksum) {
   return calculateChecksum(content) === checksum;
 }
 
-// src/content-extractor.ts
-import { decodeRepresentation } from "@semiont/core";
+// src/text-extractor.ts
+import { yieldsGeometryOf } from "@semiont/core";
 
 // src/pdf-extractor.ts
 import { isObject as isObject4 } from "@semiont/core";
@@ -574,11 +636,11 @@ function toRgb(image) {
 var IMAGE_RESOLVE_TIMEOUT_MS = 3e4;
 function resolveImage(page, ref) {
   const scope = ref.startsWith("g_") ? page.commonObjs : page.objs;
-  return new Promise((resolve) => {
-    const timer = setTimeout(() => resolve(null), IMAGE_RESOLVE_TIMEOUT_MS);
+  return new Promise((resolve2) => {
+    const timer = setTimeout(() => resolve2(null), IMAGE_RESOLVE_TIMEOUT_MS);
     const settle = (value) => {
       clearTimeout(timer);
-      resolve(value);
+      resolve2(value);
     };
     try {
       scope.get(ref, settle);
@@ -824,18 +886,17 @@ function shapeTables(layer) {
 }
 var pdfExtractor = {
   // Every non-declined PDF extraction carries positioned runs — native text
-  // layers and OCR both anchor by page geometry.
-  yieldsGeometry: true,
+  // layers and OCR both anchor by page geometry. That fact is declared in core
+  // ('pdf-text-layer' → true) rather than here; this comment records the
+  // behavior the census gate holds core's answer to.
   async extract(content, _mediaType, cache) {
-    const hit = await cache?.store.read(cache.key);
+    const hit = await cache.store.read(cache.key);
     if (hit) return hit;
     const outcome = await extractPdf(content);
-    if (cache) {
-      try {
-        if (outcome.kind === "declined") await cache.store.write(cache.key, outcome);
-        else if (outcome.items) await cache.store.write(cache.key, { ...outcome, items: outcome.items });
-      } catch {
-      }
+    try {
+      if (outcome.kind === "declined") await cache.store.write(cache.key, outcome);
+      else if (outcome.items) await cache.store.write(cache.key, { ...outcome, items: outcome.items });
+    } catch {
     }
     return outcome;
   }
@@ -890,18 +951,10 @@ async function extractPdf(content) {
   };
 }
 
-// src/content-extractor.ts
-var passthroughExtractor = {
-  yieldsGeometry: false,
-  async extract(content, mediaType) {
-    return { kind: "extracted", text: decodeRepresentation(content, mediaType), method: "text-passthrough" };
-  }
-};
-var EXTRACTORS = {
-  "decode": passthroughExtractor,
-  "pdf-text-layer": pdfExtractor,
-  "none": null
-};
+// src/text-extractor.ts
+function derivingExtractorFor(mediaType) {
+  return yieldsGeometryOf(mediaType) ? pdfExtractor : null;
+}
 
 // src/anchored-text-store.ts
 import fs2 from "fs";
@@ -1103,13 +1156,14 @@ function archivistContentReads(config) {
 }
 export {
   ChecksumMismatchError,
-  EXTRACTORS,
   MAX_PDF_BYTES,
   RepresentationMissing,
   WorkingTreeStore,
   archivistContentReads,
   calculateChecksum,
   createAnchoredTextStore,
+  createStager,
+  derivingExtractorFor,
   extractPdfTextLayer,
   verifyChecksum,
   withinByteBudget