@kontourai/lookout 0.6.0 → 0.7.0

package/README.md

@@ -180,25 +180,54 @@ resource-scoped by Forage's validators, so only the hash path needs this
 guard.)
 
 A fetch that repeats the prior capture exactly is not appended to snapshot
-history: same URL, status, body bytes, redirects, render state, and `etag` /
-`last-modified` validators, with only the fetch time differing. Its
+history: same URL, status, body bytes, text decoding, redirects, render state,
+and `etag` / `last-modified` validators, with only the fetch time differing. Its
 `unchanged-hash` result names the stored capture as both `priorSnapshotRef` and
 `currentSnapshotRef`, and `checkedAt` records the check. Other response headers
 of the repeat, such as `Date`, are not kept. A stable source therefore adds no
-records, however often it is checked.
+records, however often it is checked. The text decoding is the encoding the
+declared `Content-Type` charset resolves to, so the same bytes under another
+charset (other text for an extractor) are a new capture, while `utf8` and
+`utf-8` are the same decoding.
+
+Changed captures still grow history. Pass `retention` to bound it with Forage's
+`prune`: after each stored capture, the runner keeps the newest `keepLast`
+snapshots, the latest, the prior capture its result names, and every reference
+`cited(sourceId)` returns. Cite what your observations, review rounds, and
+exported receipts still reference. If `cited` throws or returns anything but
+snapshot references, nothing is pruned and the result carries a warning.
+`cited` is read just before each prune, outside the store's lock, so record a
+citation before handing its reference out.
+
+```ts
+const runner = createCheckRunner({
+  store,
+  retention: { keepLast: 20, cited: (sourceId) => loadCitedSnapshotRefs(sourceId) },
+});
+```
+
+**Upgrading to 0.7 (Forage 1.0).** New text captures hash their exact bytes.
+A text page whose bytes are not plain UTF-8 (a non-UTF-8 charset, a byte-order
+mark, or invalid UTF-8) therefore reports `changed` once, on its first check
+after the upgrade, and is extracted again. Drift comes from proposal diffs, so
+a page whose extracted content is identical emits nothing. Plain UTF-8 pages
+keep their stored capture.
 
 `error` results preserve provenance: `origin: "forage"` carries Forage's
 discriminated `FetchError` verbatim (its `kind`, and `status` when present);
 `origin: "lookout"` carries a typed `kind` — `prior-read`, `persistence`,
-`dependency-contract`, or `unexpected`.
+`history-full`, `dependency-contract`, or `unexpected`. `history-full` means the
+source holds Forage's `maxHistoryFiles` records; with `retention` the runner
+prunes once and retries before reporting it.
 
 Snapshot references (`snapshotRef` / `priorSnapshotRef` / `currentSnapshotRef`)
 are portable logical refs from Forage's `buildSnapshotSourceRef` — never
 filesystem paths. Snapshots are stored via Forage's filesystem store, rooted by
 default at `<cwd>/.kontourai/lookout/snapshots` (override with the CLI
 `--snapshot-root` flag or by injecting a store in library use). Lookout adds no
-custom filenames or retention; `createLookoutSnapshotStore(root, {
-maxHistoryFiles })` passes Forage's per-source record ceiling through. `resolveLookoutSnapshot()` replays one exact
+custom filenames or storage format; `createLookoutSnapshotStore(root, {
+maxHistoryFiles })` passes Forage's per-source record ceiling through and
+returns a store with Forage's `prune`. `resolveLookoutSnapshot()` replays one exact
 reference through an injected store or Lookout snapshot root, authenticates its
 body and replay metadata, and never fetches. References emitted before Forage's
 replay-envelope digest remain resolvable with `integrity: "body-and-identity"`;
@@ -366,6 +395,8 @@ import { createObserveExtractDiff } from "@kontourai/lookout";
 
 const composition = createObserveExtractDiff({
   acquisition: { check: runner.check },
+  // The store acquisition persists to; used to compare captures by reference.
+  snapshots: store,
   extraction: {
     async extract({ source, snapshotRef }) {
       // Resolve `snapshotRef`, prepare content, and invoke a caller-selected
@@ -390,8 +421,10 @@ const result = await composition.observe(source);
 
 `unchanged-304` and `unchanged-hash` are recorded as `unchanged` and never call
 the extraction capability, so they make zero preparation and provider calls,
-when the capture has the same URL and body hash as the recorder's
-`lastExtractedSnapshotRef`. That is the last snapshot an observation fully
+when the capture has the same URL, body hash, and text decoding as the
+recorder's `lastExtractedSnapshotRef`. Both references are resolved through
+`snapshots`; one that does not resolve (for example, pruned) counts as a
+different capture and is extracted. That is the last snapshot an observation fully
 handled: the current snapshot of a `completed`, `partial`, or `unchanged`
 observation, as `extractedSnapshotRef(observation)` returns. Otherwise the
 capture was persisted but never extracted (for example, a provider failed on the

package/dist/src/capture-decoding.d.ts

@@ -0,0 +1,12 @@
+import type { Snapshot } from "@kontourai/forage/fetch";
+/**
+ * The encoding a text capture's body was decoded with, or `null` for a binary
+ * capture. Two captures with the same bytes but a different decoding give the
+ * extractor different text, so they are different captures.
+ *
+ * Text captures that carry `bytes` were decoded with their declared charset;
+ * the label is resolved the way Forage decodes it, so `utf8` and `utf-8` are
+ * the same decoding. Text captures without `bytes` (the earlier Forage record
+ * format, and rendered pages) were decoded or serialized as UTF-8.
+ */
+export declare function captureDecoding(snapshot: Snapshot): string | null;

package/dist/src/capture-decoding.js

@@ -0,0 +1,18 @@
+import { decodeTextBody } from "@kontourai/forage/fetch";
+/**
+ * The encoding a text capture's body was decoded with, or `null` for a binary
+ * capture. Two captures with the same bytes but a different decoding give the
+ * extractor different text, so they are different captures.
+ *
+ * Text captures that carry `bytes` were decoded with their declared charset;
+ * the label is resolved the way Forage decodes it, so `utf8` and `utf-8` are
+ * the same decoding. Text captures without `bytes` (the earlier Forage record
+ * format, and rendered pages) were decoded or serialized as UTF-8.
+ */
+export function captureDecoding(snapshot) {
+    if (typeof snapshot.body !== "string")
+        return null;
+    if (snapshot.bytes === undefined)
+        return "utf-8";
+    return decodeTextBody(new Uint8Array(0), snapshot.declaredCharset ?? null).encoding;
+}

package/dist/src/check-result.d.ts

@@ -20,7 +20,7 @@ export interface ChangedResult extends CheckResultCommon {
     currentSnapshotRef: string;
     changeBasis: "initial" | "hash";
 }
-export type LookoutErrorKind = "prior-read" | "persistence" | "dependency-contract" | "unexpected";
+export type LookoutErrorKind = "prior-read" | "persistence" | "history-full" | "dependency-contract" | "unexpected";
 export interface ErrorResult extends CheckResultCommon {
     kind: "error";
     origin: "forage" | "lookout";

package/dist/src/check-runner.d.ts

@@ -3,8 +3,28 @@ import type { CheckResult } from "./check-result.js";
 import type { ProviderResolver } from "./provider-resolution.js";
 import type { LookoutSource } from "./registry.js";
 export type FetchSource = (config: SourceConfig, options?: FetchSourceOptions) => Promise<FetchResult>;
+/**
+ * Bounds each source's snapshot history with the store's `prune` capability.
+ * After every stored capture, all but the newest `keepLast` snapshots are
+ * removed, except the latest, the captures the result names, and every
+ * snapshot `cited` returns. `cited` is read just before the prune, outside the
+ * store's lock: a citation recorded after that read is not protected by that
+ * prune, so record a citation before handing its reference out.
+ */
+export interface SnapshotRetention {
+    /** Newest snapshots kept per source (a non-negative integer; the latest is always kept). */
+    readonly keepLast: number;
+    /**
+     * Snapshot references the caller still cites for this source: recorded
+     * observations, review rounds, exported receipts. Never pruned. When this
+     * throws or returns anything but canonical references, nothing is pruned.
+     */
+    readonly cited: (sourceId: string) => Promise<readonly string[]> | readonly string[];
+}
 export interface CreateCheckRunnerOptions {
     store: SnapshotStore;
+    /** Prune history after each stored capture. Without it, history is kept in full. */
+    retention?: SnapshotRetention;
     fetchSource?: FetchSource;
     fetchOptions?: Omit<FetchSourceOptions, "store">;
     clock?: () => string;

package/dist/src/check-runner.js

@@ -1,5 +1,6 @@
 import { isDeepStrictEqual } from "node:util";
-import { buildSnapshotSourceRef, fetchSource as forageFetchSource, } from "@kontourai/forage/fetch";
+import { buildSnapshotSourceRef, fetchSource as forageFetchSource, isSnapshotHistoryFullError, parseSnapshotSourceRef, } from "@kontourai/forage/fetch";
+import { captureDecoding } from "./capture-decoding.js";
 /**
  * The egress policy for lookout's registered-source fetches. Registered source
  * URLs are operator- / aggregator-supplied and not fully trusted, so a source
@@ -15,6 +16,40 @@ export function createCheckRunner(options) {
     const fetchImpl = options.fetchSource ?? forageFetchSource;
     const clock = options.clock ?? (() => new Date().toISOString());
     const fetchOptions = { ...options.fetchOptions };
+    const retention = options.retention;
+    if (retention !== undefined && (!Number.isSafeInteger(retention.keepLast) || retention.keepLast < 0 || typeof retention.cited !== "function")) {
+        throw new TypeError("retention requires a non-negative integer keepLast and a cited function");
+    }
+    /** Apply retention for one source. Returns a warning instead of throwing: the capture is already stored. */
+    async function prune(sourceId, keepAlso) {
+        if (retention === undefined)
+            return undefined;
+        if (typeof options.store.prune !== "function")
+            return "snapshot retention skipped: the store cannot prune";
+        const keep = keepAlso.map((snapshot) => lookupOf(buildSnapshotSourceRef(snapshot)));
+        try {
+            const cited = await retention.cited(sourceId);
+            if (!Array.isArray(cited))
+                return "snapshot retention skipped: cited references are not a list";
+            for (const reference of cited) {
+                const lookup = lookupOf(reference);
+                if (lookup === undefined)
+                    return "snapshot retention skipped: a cited reference is not a snapshot reference";
+                if (lookup.sourceId === sourceId)
+                    keep.push(lookup);
+            }
+        }
+        catch (error) {
+            return `snapshot retention skipped: cited references could not be read: ${error instanceof Error ? error.message : String(error)}`;
+        }
+        try {
+            await options.store.prune(sourceId, { keepLast: retention.keepLast, keep });
+            return undefined;
+        }
+        catch (error) {
+            return `snapshot retention failed: ${error instanceof Error ? error.message : String(error)}`;
+        }
+    }
     async function check(source) {
         const common = () => ({
             sourceId: source.id,
@@ -73,12 +108,36 @@ export function createCheckRunner(options) {
                 const priorSnapshotRef = buildSnapshotSourceRef(prior);
                 return { ...base, kind: "unchanged-hash", priorSnapshotRef, currentSnapshotRef: priorSnapshotRef };
             }
+            // The capture just stored is kept even when it is not the newest (clock
+            // skew, or another check storing a later capture first): its result names it.
+            const keepAlso = prior === undefined ? [snapshot] : [snapshot, prior];
             try {
                 await options.store.put(snapshot);
             }
             catch (error) {
-                return lookoutError(base, "persistence", error);
+                if (!isSnapshotHistoryFullError(error))
+                    return lookoutError(base, "persistence", error);
+                // A full history refuses every later capture. Retention frees space
+                // (for example when it is enabled on a store that already reached the
+                // ceiling), so prune once and retry; otherwise say what to do.
+                const warning = await prune(source.id, keepAlso);
+                if (retention === undefined || warning !== undefined) {
+                    if (warning !== undefined)
+                        base.warnings.push(warning);
+                    return historyFull(base, error.maxHistoryFiles);
+                }
+                try {
+                    await options.store.put(snapshot);
+                }
+                catch (retryError) {
+                    return isSnapshotHistoryFullError(retryError)
+                        ? historyFull(base, retryError.maxHistoryFiles)
+                        : lookoutError(base, "persistence", retryError);
+                }
             }
+            const retentionWarning = await prune(source.id, keepAlso);
+            if (retentionWarning !== undefined)
+                base.warnings.push(retentionWarning);
             const currentSnapshotRef = buildSnapshotSourceRef(snapshot);
             if (!prior) {
                 return {
@@ -123,10 +182,24 @@ function isRepeatCapture(prior, current) {
         prior.url === current.url &&
         prior.status === current.status &&
         prior.bodyHash === current.bodyHash &&
+        // Same bytes under another charset decode to other text: a new capture.
+        captureDecoding(prior) === captureDecoding(current) &&
         prior.rendered === current.rendered &&
         isDeepStrictEqual(prior.redirects, current.redirects) &&
         REVALIDATION_HEADERS.every((name) => prior.headers?.[name] === current.headers?.[name]);
 }
+function lookupOf(reference) {
+    if (typeof reference !== "string")
+        return undefined;
+    const parsed = parseSnapshotSourceRef(reference);
+    if (parsed === undefined)
+        return undefined;
+    const { sourceId, url, bodyHash, fetchedAt, snapshotDigest } = parsed;
+    return { sourceId, url, bodyHash, fetchedAt, ...(snapshotDigest === undefined ? {} : { snapshotDigest }) };
+}
+function historyFull(common, maxHistoryFiles) {
+    return lookoutError(common, "history-full", `snapshot history for this source holds its maximum of ${maxHistoryFiles} records; configure snapshot retention or prune the store`);
+}
 function bodyEncoding(snapshot) {
     const descriptor = Object.getOwnPropertyDescriptor(snapshot, "body");
     if (descriptor === undefined || !("value" in descriptor))

package/dist/src/index.d.ts

@@ -1,7 +1,7 @@
 export { runCli } from "./cli.js";
 export type { RunCliOptions } from "./cli.js";
 export { createCheckRunner } from "./check-runner.js";
-export type { CheckRunner, CreateCheckRunnerOptions, FetchSource } from "./check-runner.js";
+export type { CheckRunner, CreateCheckRunnerOptions, FetchSource, SnapshotRetention } from "./check-runner.js";
 export type { ChangedResult, CheckResult, CheckResultCommon, ErrorResult, LookoutErrorKind, Unchanged304Result, UnchangedHashResult, } from "./check-result.js";
 export { defaultProviderResolver } from "./provider-resolution.js";
 export type { ProviderResolver } from "./provider-resolution.js";

package/dist/src/observe-extract-diff.d.ts

@@ -1,4 +1,5 @@
 import type { ExtractionPartial, ExtractionProviderFailure, ExtractionResult, PreparedArtifact } from "@kontourai/traverse";
+import type { SnapshotStore } from "@kontourai/forage/fetch";
 import type { CheckResult } from "./check-result.js";
 import type { ProposalSetObservation } from "./proposal-diff.js";
 import type { LookoutSource } from "./registry.js";
@@ -69,8 +70,8 @@ export interface ObserveExtractRecorder {
      * for which `extractedSnapshotRef(observation)` is not null, or `null` when
      * there is none.
      *
-     * An unchanged check is only skipped when its capture has the same URL and
-     * body hash as this snapshot. Otherwise a change that acquisition already
+     * An unchanged check is only skipped when its capture has the same URL, body
+     * hash, and text decoding as this snapshot. Otherwise a change that acquisition already
      * persisted, but that was never extracted (for example because a provider
      * failed), would be reported as unchanged forever.
      */
@@ -86,6 +87,14 @@ export interface ObserveExtractDiffOptions {
     readonly acquisition: ObserveExtractAcquisition;
     readonly extraction: ObserveExtractExtraction;
     readonly recorder: ObserveExtractRecorder;
+    /**
+     * The snapshot store acquisition persists to. An unchanged check is compared
+     * with the last extracted capture by resolving both references here, since a
+     * reference does not name the charset its text was decoded with. A reference
+     * that does not resolve counts as a different capture, so it is extracted.
+     * With snapshot retention, cite the last extracted reference to keep it.
+     */
+    readonly snapshots: SnapshotStore;
 }
 export type ObserveExtractErrorKind = "acquisition-threw" | "recording-failed" | "dependency-contract";
 export interface ObserveExtractError {

package/dist/src/observe-extract-diff.js

@@ -1,5 +1,6 @@
 import { validatePreparedArtifact } from "@kontourai/traverse";
-import { parseSnapshotSourceRef } from "@kontourai/forage/fetch";
+import { resolveSnapshotSourceRef } from "@kontourai/forage/fetch";
+import { captureDecoding } from "./capture-decoding.js";
 /**
  * The snapshot an observation's extraction fully handled: the current snapshot
  * of a `completed`, `partial`, or `unchanged` observation, else `null`.
@@ -10,6 +11,11 @@ export function extractedSnapshotRef(observation) {
     return handled && observation.sourceSnapshot !== null ? observation.sourceSnapshot.currentSnapshotRef : null;
 }
 export function createObserveExtractDiff(options) {
+    // Without a store every unchanged capture would silently count as new and be
+    // extracted again, so a missing one is refused up front.
+    if (typeof options?.snapshots?.findExact !== "function") {
+        throw new TypeError("createObserveExtractDiff requires snapshots: the snapshot store acquisition persists to, with exact lookup");
+    }
     return {
         async observe(source) {
             try {
@@ -44,7 +50,7 @@ export function createObserveExtractDiff(options) {
                     if (extracted !== null && (typeof extracted !== "string" || extracted === "")) {
                         return { ok: false, error: error("dependency-contract", "Observation recorder returned an invalid last extracted snapshot reference") };
                     }
-                    if (extracted !== null && sameCapture(extracted, sourceSnapshot.currentSnapshotRef)) {
+                    if (extracted !== null && await sameCapture(options.snapshots, extracted, sourceSnapshot.currentSnapshotRef)) {
                         return record(options.recorder, baseObservation(source, check, "unchanged", sourceSnapshot, null, null, null));
                     }
                     sourceSnapshot = { priorSnapshotRef: extracted, currentSnapshotRef: sourceSnapshot.currentSnapshotRef };
@@ -122,14 +128,19 @@ async function record(recorder, observation) {
 function observationSource(source) {
     return { id: source.id, url: source.url, kind: source.kind };
 }
-/** Two references name the same capture content: same source, resource URL, and body hash. */
-function sameCapture(left, right) {
+/**
+ * Two references name the same capture content: same source, resource URL,
+ * body hash, and text decoding. Same bytes under another declared charset are
+ * other text, so they are a different capture.
+ */
+async function sameCapture(store, left, right) {
     if (left === right)
         return true;
-    const a = parseSnapshotSourceRef(left);
-    const b = parseSnapshotSourceRef(right);
-    return a !== undefined && b !== undefined &&
-        a.sourceId === b.sourceId && a.url === b.url && a.bodyHash === b.bodyHash;
+    const [a, b] = await Promise.all([resolveSnapshotSourceRef(store, left), resolveSnapshotSourceRef(store, right)]);
+    if (!a.ok || !b.ok)
+        return false;
+    return a.snapshot.sourceId === b.snapshot.sourceId && a.snapshot.url === b.snapshot.url &&
+        a.snapshot.bodyHash === b.snapshot.bodyHash && captureDecoding(a.snapshot) === captureDecoding(b.snapshot);
 }
 function snapshotFor(check) {
     if (check.kind === "unchanged-304")

package/dist/src/snapshot-store.d.ts

@@ -1,4 +1,4 @@
-import type { SnapshotStore } from "@kontourai/forage";
+import type { ExactSnapshotStore, PrunableSnapshotStore, SnapshotStore } from "@kontourai/forage";
 import { type SnapshotSourceRefResolution } from "@kontourai/forage/fetch";
 export interface LookoutSnapshotStoreOptions {
     /**
@@ -8,7 +8,7 @@ export interface LookoutSnapshotStoreOptions {
      */
     readonly maxHistoryFiles?: number;
 }
-export declare function createLookoutSnapshotStore(root?: string, options?: LookoutSnapshotStoreOptions): SnapshotStore;
+export declare function createLookoutSnapshotStore(root?: string, options?: LookoutSnapshotStoreOptions): ExactSnapshotStore & PrunableSnapshotStore;
 export type ResolveLookoutSnapshotOptions = {
     store: SnapshotStore;
     root?: never;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kontourai/lookout",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "A small source registry and drift-check runner built on Forage snapshots.",
   "license": "Apache-2.0",
   "type": "module",
@@ -46,12 +46,12 @@
     "workflow:validate-artifacts": "flow-agents-validate-artifacts"
   },
   "dependencies": {
-    "@kontourai/datum": "0.7.0",
-    "@kontourai/forage": "0.6.1",
+    "@kontourai/datum": "0.8.0",
+    "@kontourai/forage": "1.0.0",
     "@kontourai/traverse": "0.25.1"
   },
   "devDependencies": {
-    "@kontourai/flow-agents": "5.9.0",
+    "@kontourai/flow-agents": "6.4.0",
     "@kontourai/survey": "3.0.0",
     "@types/node": "^26.1.2",
     "ajv": "8.20.0",