@gmickel/gno 1.19.0 → 1.21.0

package/src/core/capsule-reverification-scheduler.ts

@@ -0,0 +1,218 @@
+/** Bounded, coalescing resident scheduler for evidence-triggered reverification. */
+
+import type {
+  SavedCapsuleRegistrationRecord,
+  StorePort,
+  StoreResult,
+} from "../store/types";
+import type {
+  SavedCapsuleReverificationDeps,
+  SavedCapsuleReverificationOutcome,
+} from "./capsule-reverification";
+
+import { SavedCapsuleRegistryError } from "./capsule-registry";
+import { reverifySavedCapsule } from "./capsule-reverification";
+import {
+  decodeDocumentChangeCursor,
+  encodeDocumentChangeCursor,
+} from "./change-journal";
+
+const MAX_REGISTRATIONS_PER_DRAIN = 10_000;
+
+type SchedulerStore = StorePort &
+  Pick<
+    StorePort,
+    | "getSavedCapsuleReverificationState"
+    | "listDocumentChanges"
+    | "listSavedCapsuleIdsAffectedByChanges"
+    | "listSavedCapsuleRegistrations"
+    | "setSavedCapsuleReverificationSequence"
+  >;
+
+export interface SavedCapsuleReverificationDrain {
+  fromSequence: number;
+  throughSequence: number;
+  cursorExpired: boolean;
+  affected: number;
+  completed: number;
+  failed: number;
+}
+
+export interface SavedCapsuleReverificationSchedulerOptions {
+  deps: Omit<SavedCapsuleReverificationDeps, "store"> & {
+    store: SchedulerStore & SavedCapsuleReverificationDeps["store"];
+  };
+  startBackgroundWork: (
+    operation: (signal: AbortSignal) => Promise<void>
+  ) => boolean;
+  onDrain?: (result: SavedCapsuleReverificationDrain) => void;
+}
+
+const unwrapStore = <T>(result: StoreResult<T>, operation: string): T => {
+  if (result.ok) return result.value;
+  throw new SavedCapsuleRegistryError(
+    "store_failed",
+    `${operation}: ${result.error.message}`,
+    result.error.cause
+  );
+};
+
+export class SavedCapsuleReverificationScheduler {
+  readonly #options: SavedCapsuleReverificationSchedulerOptions;
+  #pending = false;
+  #running = false;
+  #disposed = false;
+
+  constructor(options: SavedCapsuleReverificationSchedulerOptions) {
+    this.#options = options;
+  }
+
+  notifySyncSettled(): void {
+    if (this.#disposed) return;
+    this.#pending = true;
+    if (this.#running) return;
+    this.#running = true;
+    const started = this.#options.startBackgroundWork(async (signal) => {
+      await this.#run(signal);
+    });
+    if (!started) {
+      this.#running = false;
+    }
+  }
+
+  async triggerNow(signal: AbortSignal = new AbortController().signal) {
+    if (this.#disposed) return [];
+    const results: SavedCapsuleReverificationDrain[] = [];
+    do {
+      this.#pending = false;
+      results.push(await this.#drain(signal));
+    } while (this.#pending && !signal.aborted && !this.#disposed);
+    return results;
+  }
+
+  async dispose(): Promise<void> {
+    this.#disposed = true;
+    await Promise.resolve();
+  }
+
+  async #run(signal: AbortSignal): Promise<void> {
+    const operation = (async () => {
+      try {
+        while (this.#pending && !signal.aborted && !this.#disposed) {
+          this.#pending = false;
+          const result = await this.#drain(signal);
+          this.#options.onDrain?.(result);
+        }
+      } finally {
+        this.#running = false;
+        if (this.#pending && !this.#disposed) {
+          this.notifySyncSettled();
+        }
+      }
+    })();
+    await operation;
+  }
+
+  async #drain(signal: AbortSignal): Promise<SavedCapsuleReverificationDrain> {
+    const store = this.#options.deps.store;
+    const schedulerState = unwrapStore(
+      await store.getSavedCapsuleReverificationState(),
+      "Failed to read saved Capsule scheduler state"
+    );
+    const fromSequence = schedulerState.lastProcessedSequence;
+    const journal = unwrapStore(
+      await store.listDocumentChanges({
+        cursor: encodeDocumentChangeCursor(fromSequence),
+        limit: 1,
+      }),
+      "Failed to read document change journal"
+    );
+    const throughSequence = decodeDocumentChangeCursor(journal.latestCursor);
+    if (throughSequence <= fromSequence) {
+      return {
+        fromSequence,
+        throughSequence,
+        cursorExpired: journal.cursorExpired,
+        affected: 0,
+        completed: 0,
+        failed: 0,
+      };
+    }
+
+    let registrations: SavedCapsuleRegistrationRecord[];
+    if (journal.cursorExpired) {
+      registrations = unwrapStore(
+        await store.listSavedCapsuleRegistrations(),
+        "Failed to list saved Context Capsules"
+      );
+      if (registrations.length > MAX_REGISTRATIONS_PER_DRAIN) {
+        throw new SavedCapsuleRegistryError(
+          "store_failed",
+          "Saved Capsule scheduler registration bound exceeded"
+        );
+      }
+    } else {
+      const affected = unwrapStore(
+        await store.listSavedCapsuleIdsAffectedByChanges(
+          fromSequence,
+          throughSequence,
+          MAX_REGISTRATIONS_PER_DRAIN
+        ),
+        "Failed to resolve changed saved Capsule evidence"
+      );
+      if (affected.truncated) {
+        throw new SavedCapsuleRegistryError(
+          "store_failed",
+          "Saved Capsule scheduler registration bound exceeded"
+        );
+      }
+      const all = unwrapStore(
+        await store.listSavedCapsuleRegistrations(),
+        "Failed to list saved Context Capsules"
+      );
+      const ids = new Set(affected.registrationIds);
+      registrations = all.filter((registration) =>
+        ids.has(registration.registrationId)
+      );
+    }
+
+    const outcomes: SavedCapsuleReverificationOutcome[] = [];
+    for (const registration of registrations) {
+      if (signal.aborted) break;
+      if (registration.lastAttemptedSequence >= throughSequence) continue;
+      outcomes.push(
+        await reverifySavedCapsule(
+          registration.registrationId,
+          {
+            kind: "journal",
+            fromSequence,
+            throughSequence,
+          },
+          this.#options.deps
+        )
+      );
+    }
+    if (!signal.aborted) {
+      const advanced = unwrapStore(
+        await store.setSavedCapsuleReverificationSequence(
+          throughSequence,
+          schedulerState.registrationEpoch
+        ),
+        "Failed to advance saved Capsule scheduler state"
+      );
+      if (!advanced) this.#pending = true;
+    }
+    return {
+      fromSequence,
+      throughSequence,
+      cursorExpired: journal.cursorExpired,
+      affected: outcomes.length,
+      completed: outcomes.filter(
+        (outcome) => outcome.verification.operationStatus === "completed"
+      ).length,
+      failed: outcomes.filter(
+        (outcome) => outcome.verification.operationStatus === "failed"
+      ).length,
+    };
+  }
+}

package/src/core/capsule-reverification.ts

@@ -0,0 +1,289 @@
+/** Non-generative reverification for metadata-only saved Capsule records. */
+
+import type { ContextCapsuleRuntimeDeps } from "../app/context-runtime";
+import type {
+  SavedCapsuleRegistrationRecord,
+  SavedCapsuleRegistrationSnapshot,
+  SavedCapsuleTriggerKind,
+  SavedCapsuleVerificationRecord,
+  StorePort,
+  StoreResult,
+} from "../store/types";
+import type { ContextCapsuleVerification } from "./context-capsule";
+
+import { DEFAULT_INDEX_NAME } from "../app/constants";
+import {
+  canonicalVerifiedContextCapsuleJson,
+  verifyContextCapsuleRuntime,
+} from "../app/context-runtime";
+import { canonicalizeIndexName } from "../app/index-name";
+import {
+  getSavedCapsule,
+  loadSavedCapsuleFile,
+  SavedCapsuleRegistryError,
+} from "./capsule-registry";
+import { decodeDocumentChangeCursor } from "./change-journal";
+import { sha256Text } from "./context-capsule-validation";
+
+type ReverificationStore = StorePort &
+  ContextCapsuleRuntimeDeps["store"] &
+  Pick<
+    StorePort,
+    | "getSavedCapsuleRegistration"
+    | "getSavedCapsuleRegistrationSnapshot"
+    | "listDocumentChanges"
+    | "upsertSavedCapsuleVerification"
+  >;
+
+const MAX_REGISTRATION_CONFLICT_ATTEMPTS = 2;
+
+export interface SavedCapsuleReverificationNotification {
+  type: "capsule-reverified";
+  registrationId: string;
+  capsuleId: string;
+  operationStatus: "completed" | "failed";
+  affectedQuestionState: "unaffected" | "affected" | "unknown";
+  changedAt: string;
+}
+
+export interface SavedCapsuleReverificationOutcome {
+  registration: SavedCapsuleRegistrationRecord;
+  verification: SavedCapsuleVerificationRecord;
+  receipt: ContextCapsuleVerification | null;
+}
+
+export interface SavedCapsuleReverificationDeps extends Omit<
+  ContextCapsuleRuntimeDeps,
+  "store" | "indexName"
+> {
+  store: ReverificationStore;
+  indexName: string;
+  now?: () => number;
+  notify?: (event: SavedCapsuleReverificationNotification) => void;
+}
+
+export interface SavedCapsuleReverificationTrigger {
+  kind: SavedCapsuleTriggerKind;
+  fromSequence: number;
+  throughSequence: number;
+}
+
+const unwrapStore = <T>(result: StoreResult<T>, operation: string): T => {
+  if (result.ok) return result.value;
+  throw new SavedCapsuleRegistryError(
+    "store_failed",
+    `${operation}: ${result.error.message}`,
+    result.error.cause
+  );
+};
+
+const currentSequence = async (store: ReverificationStore): Promise<number> => {
+  const page = unwrapStore(
+    await store.listDocumentChanges({ limit: 1 }),
+    "Failed to read document change journal"
+  );
+  return decodeDocumentChangeCursor(page.latestCursor);
+};
+
+const getVerificationSnapshot = async (
+  store: ReverificationStore,
+  registrationId: string
+): Promise<SavedCapsuleRegistrationSnapshot> => {
+  const snapshot = unwrapStore(
+    await store.getSavedCapsuleRegistrationSnapshot(registrationId),
+    "Failed to read saved Context Capsule verification snapshot"
+  );
+  if (!snapshot) {
+    throw new SavedCapsuleRegistryError(
+      "registration_not_found",
+      "Saved Context Capsule registration not found"
+    );
+  }
+  return snapshot;
+};
+
+const affected = (
+  receipt: ContextCapsuleVerification
+): {
+  state: "unaffected" | "affected";
+  reasons: string[];
+} => {
+  const reasons: string[] = [];
+  if (receipt.contentStatus === "stale") reasons.push("content_stale");
+  if (receipt.contentStatus === "missing") reasons.push("content_missing");
+  if (receipt.rankingStatus === "reranked") reasons.push("ranking_changed");
+  if (receipt.fingerprintStatus === "drifted") {
+    reasons.push("fingerprint_changed");
+  }
+  return {
+    state: reasons.length === 0 ? "unaffected" : "affected",
+    reasons,
+  };
+};
+
+const errorIdentity = (error: unknown): { code: string; message: string } => {
+  if (
+    error instanceof Error &&
+    "code" in error &&
+    typeof error.code === "string"
+  ) {
+    return { code: error.code, message: error.message.slice(0, 4096) };
+  }
+  return {
+    code: "verification_failed",
+    message: (error instanceof Error
+      ? error.message
+      : "Saved Capsule verification failed"
+    ).slice(0, 4096),
+  };
+};
+
+const persist = async (
+  deps: SavedCapsuleReverificationDeps,
+  registration: SavedCapsuleRegistrationRecord,
+  registrationGeneration: number,
+  verification: SavedCapsuleVerificationRecord
+): Promise<boolean> => {
+  const persisted = unwrapStore(
+    await deps.store.upsertSavedCapsuleVerification(verification, {
+      registrationGeneration,
+    }),
+    "Failed to persist saved Context Capsule verification"
+  );
+  if (!persisted) return false;
+  if (registration.notificationPreference === "local") {
+    deps.notify?.({
+      type: "capsule-reverified",
+      registrationId: registration.registrationId,
+      capsuleId: registration.capsuleId,
+      operationStatus: verification.operationStatus,
+      affectedQuestionState: verification.affectedQuestionState,
+      changedAt: new Date(verification.verifiedAtMs).toISOString(),
+    });
+  }
+  return true;
+};
+
+const reverifySavedCapsuleAttempt = async (
+  registrationId: string,
+  trigger: SavedCapsuleReverificationTrigger,
+  deps: SavedCapsuleReverificationDeps
+): Promise<SavedCapsuleReverificationOutcome | null> => {
+  const snapshot = await getVerificationSnapshot(deps.store, registrationId);
+  const registration = snapshot.registration;
+  const runtimeIndex = canonicalizeIndexName(
+    deps.indexName || DEFAULT_INDEX_NAME
+  );
+  const verifiedAtMs = (deps.now ?? Date.now)();
+  let receipt: ContextCapsuleVerification | null = null;
+  let verification: SavedCapsuleVerificationRecord;
+  try {
+    if (runtimeIndex !== registration.indexName) {
+      throw Object.assign(
+        new Error(
+          `Saved Context Capsule index ${registration.indexName} does not match runtime index ${runtimeIndex}`
+        ),
+        { code: "invalid_filter" }
+      );
+    }
+    const loaded = await loadSavedCapsuleFile(
+      registration.filePath,
+      registration.fileHash
+    );
+    if (loaded.capsule.capsuleId !== registration.capsuleId) {
+      throw Object.assign(
+        new Error("Saved Context Capsule file changed after registration"),
+        { code: "capsule_file_changed" }
+      );
+    }
+    receipt = await verifyContextCapsuleRuntime(loaded.capsule, {
+      ...deps,
+      store: deps.store,
+      indexName: runtimeIndex,
+    });
+    const receiptJson = canonicalVerifiedContextCapsuleJson(receipt);
+    const projection = affected(receipt);
+    verification = {
+      registrationId,
+      triggerKind: trigger.kind,
+      fromSequence: trigger.fromSequence,
+      throughSequence: trigger.throughSequence,
+      operationStatus: "completed",
+      affectedQuestionState: projection.state,
+      affectedReasons: projection.reasons,
+      receiptJson,
+      receiptHash: sha256Text(receiptJson),
+      errorCode: null,
+      errorMessage: null,
+      verifiedAtMs,
+    };
+  } catch (error) {
+    const failure = errorIdentity(error);
+    verification = {
+      registrationId,
+      triggerKind: trigger.kind,
+      fromSequence: trigger.fromSequence,
+      throughSequence: trigger.throughSequence,
+      operationStatus: "failed",
+      affectedQuestionState: "unknown",
+      affectedReasons: [],
+      receiptJson: null,
+      receiptHash: null,
+      errorCode: failure.code,
+      errorMessage: failure.message,
+      verifiedAtMs,
+    };
+  }
+  if (
+    !(await persist(
+      deps,
+      registration,
+      snapshot.registrationGeneration,
+      verification
+    ))
+  ) {
+    return null;
+  }
+  const refreshed = await getSavedCapsule(deps.store, registrationId);
+  return { registration: refreshed, verification, receipt };
+};
+
+export const reverifySavedCapsule = async (
+  registrationId: string,
+  trigger: SavedCapsuleReverificationTrigger,
+  deps: SavedCapsuleReverificationDeps
+): Promise<SavedCapsuleReverificationOutcome> => {
+  for (
+    let attempt = 0;
+    attempt < MAX_REGISTRATION_CONFLICT_ATTEMPTS;
+    attempt += 1
+  ) {
+    const outcome = await reverifySavedCapsuleAttempt(
+      registrationId,
+      trigger,
+      deps
+    );
+    if (outcome) return outcome;
+  }
+  throw new SavedCapsuleRegistryError(
+    "store_failed",
+    "Saved Context Capsule registration changed repeatedly during verification"
+  );
+};
+
+export const reverifySavedCapsuleManually = async (
+  registrationId: string,
+  deps: SavedCapsuleReverificationDeps
+): Promise<SavedCapsuleReverificationOutcome> => {
+  const sequence = await currentSequence(deps.store);
+  const registration = await getSavedCapsule(deps.store, registrationId);
+  return reverifySavedCapsule(
+    registrationId,
+    {
+      kind: "manual",
+      fromSequence: registration.lastAttemptedSequence,
+      throughSequence: sequence,
+    },
+    deps
+  );
+};

package/src/core/change-diff.ts

@@ -0,0 +1,182 @@
+/**
+ * Bounded, content-free structural snapshots for document change journaling.
+ *
+ * The snapshot values are normalized summaries only. Source bodies are used
+ * transiently during sync and are never included in a journal delta.
+ */
+
+import type {
+  DocumentChangeStructureDelta,
+  DocumentChangeSet,
+} from "../store/types";
+
+import { parseFrontmatter } from "../ingestion/frontmatter";
+import { buildLineOffsets } from "../ingestion/position";
+import { getExcludedRanges } from "../ingestion/strip";
+import { normalizeMarkdownPath, normalizeWikiName, parseLinks } from "./links";
+import { extractSections } from "./sections";
+
+export type RelationMap = Record<string, string[]>;
+
+export interface DocumentStructureSnapshot {
+  headings: string[];
+  links: string[];
+  typedEdges: string[];
+  dates: Record<string, string>;
+}
+
+export interface DocumentStructureDeltaResult {
+  delta: DocumentChangeStructureDelta;
+  history: "available" | "unavailable";
+}
+
+const RELATION_EDGE_TYPE_PATTERN = /^[a-z][a-z0-9_]*$/;
+
+export function isRelationMap(value: unknown): value is RelationMap {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return false;
+  }
+  return Object.values(value).every(
+    (targets) =>
+      Array.isArray(targets) &&
+      targets.every((target) => typeof target === "string")
+  );
+}
+
+export function normalizeRelationTarget(raw: string): string {
+  const trimmed = raw.trim();
+  if (trimmed.startsWith("[[") && trimmed.endsWith("]]")) {
+    return trimmed.slice(2, -2).split("|")[0]?.trim() ?? "";
+  }
+  return trimmed;
+}
+
+export function normalizeRelationEdgeType(raw: string): string {
+  return raw
+    .trim()
+    .toLowerCase()
+    .replace(/[-\s]+/g, "_");
+}
+
+const withOccurrences = (values: readonly string[]): string[] => {
+  const counts = new Map<string, number>();
+  return values
+    .map((value) => {
+      const count = (counts.get(value) ?? 0) + 1;
+      counts.set(value, count);
+      return count === 1 ? value : `${value} [${count}]`;
+    })
+    .sort();
+};
+
+const linkSummary = (
+  link: ReturnType<typeof parseLinks>[number],
+  relPath: string
+): string | null => {
+  const anchor = link.targetAnchor
+    ? `#${normalizeWikiName(link.targetAnchor)}`
+    : "";
+  if (link.kind === "wiki") {
+    const collection = link.targetCollection
+      ? `${normalizeWikiName(link.targetCollection)}:`
+      : "";
+    return `wiki:${collection}${normalizeWikiName(link.targetRef)}${anchor}`;
+  }
+  const target = normalizeMarkdownPath(link.targetRef, relPath);
+  return target ? `markdown:${target}${anchor}` : null;
+};
+
+const extractTypedEdges = (markdown: string): string[] => {
+  const relations = parseFrontmatter(markdown).metadata.relations;
+  if (!isRelationMap(relations)) return [];
+
+  const values: string[] = [];
+  for (const [rawEdgeType, targets] of Object.entries(relations)) {
+    const edgeType = normalizeRelationEdgeType(rawEdgeType);
+    if (!RELATION_EDGE_TYPE_PATTERN.test(edgeType)) continue;
+    for (const rawTarget of targets) {
+      const target = normalizeRelationTarget(rawTarget);
+      if (target) {
+        values.push(`${edgeType}:${normalizeWikiName(target)}`);
+      }
+    }
+  }
+  return [...new Set(values)].sort();
+};
+
+export const extractDocumentStructure = (
+  markdown: string,
+  relPath: string,
+  dateFields: Readonly<Record<string, string>> | null | undefined
+): DocumentStructureSnapshot => {
+  const excludedRanges = getExcludedRanges(markdown);
+  const links = parseLinks(markdown, buildLineOffsets(markdown), excludedRanges)
+    .map((link) => linkSummary(link, relPath))
+    .filter((value): value is string => value !== null);
+
+  return {
+    headings: withOccurrences(
+      extractSections(markdown).map(
+        ({ level, title }) => `${"#".repeat(level)} ${title.normalize("NFC")}`
+      )
+    ),
+    links: withOccurrences(links),
+    typedEdges: extractTypedEdges(markdown),
+    dates: { ...dateFields },
+  };
+};
+
+const diffValues = (
+  previous: readonly string[],
+  next: readonly string[]
+): DocumentChangeSet => {
+  const previousValues = new Set(previous);
+  const nextValues = new Set(next);
+  return {
+    added: next.filter((value) => !previousValues.has(value)),
+    removed: previous.filter((value) => !nextValues.has(value)),
+  };
+};
+
+export const diffDocumentStructure = (
+  previous: DocumentStructureSnapshot | null | undefined,
+  next: DocumentStructureSnapshot
+): DocumentStructureDeltaResult => {
+  if (previous === undefined) {
+    return {
+      history: "unavailable",
+      delta: {
+        headings: { added: [], removed: [] },
+        links: { added: [], removed: [] },
+        typedEdges: { added: [], removed: [] },
+        dates: { added: [], removed: [], changed: [] },
+        truncated: true,
+      },
+    };
+  }
+
+  const prior = previous ?? {
+    headings: [],
+    links: [],
+    typedEdges: [],
+    dates: {},
+  };
+  const priorDateKeys = Object.keys(prior.dates).sort();
+  const nextDateKeys = Object.keys(next.dates).sort();
+  const dates = diffValues(priorDateKeys, nextDateKeys);
+  const changed = priorDateKeys.filter(
+    (key) =>
+      Object.hasOwn(next.dates, key) && prior.dates[key] !== next.dates[key]
+  );
+
+  return {
+    history: "available",
+    delta: {
+      headings: diffValues(prior.headings, next.headings),
+      links: diffValues(prior.links, next.links),
+      typedEdges: diffValues(prior.typedEdges, next.typedEdges),
+      dates: { ...dates, changed },
+      truncated: false,
+    },
+  };
+};