@gmickel/gno 1.20.0 → 1.22.0

package/src/core/capsule-reverification.ts

@@ -0,0 +1,289 @@
+/** Non-generative reverification for metadata-only saved Capsule records. */
+
+import type { ContextCapsuleRuntimeDeps } from "../app/context-runtime";
+import type {
+  SavedCapsuleRegistrationRecord,
+  SavedCapsuleRegistrationSnapshot,
+  SavedCapsuleTriggerKind,
+  SavedCapsuleVerificationRecord,
+  StorePort,
+  StoreResult,
+} from "../store/types";
+import type { ContextCapsuleVerification } from "./context-capsule";
+
+import { DEFAULT_INDEX_NAME } from "../app/constants";
+import {
+  canonicalVerifiedContextCapsuleJson,
+  verifyContextCapsuleRuntime,
+} from "../app/context-runtime";
+import { canonicalizeIndexName } from "../app/index-name";
+import {
+  getSavedCapsule,
+  loadSavedCapsuleFile,
+  SavedCapsuleRegistryError,
+} from "./capsule-registry";
+import { decodeDocumentChangeCursor } from "./change-journal";
+import { sha256Text } from "./context-capsule-validation";
+
+type ReverificationStore = StorePort &
+  ContextCapsuleRuntimeDeps["store"] &
+  Pick<
+    StorePort,
+    | "getSavedCapsuleRegistration"
+    | "getSavedCapsuleRegistrationSnapshot"
+    | "listDocumentChanges"
+    | "upsertSavedCapsuleVerification"
+  >;
+
+const MAX_REGISTRATION_CONFLICT_ATTEMPTS = 2;
+
+export interface SavedCapsuleReverificationNotification {
+  type: "capsule-reverified";
+  registrationId: string;
+  capsuleId: string;
+  operationStatus: "completed" | "failed";
+  affectedQuestionState: "unaffected" | "affected" | "unknown";
+  changedAt: string;
+}
+
+export interface SavedCapsuleReverificationOutcome {
+  registration: SavedCapsuleRegistrationRecord;
+  verification: SavedCapsuleVerificationRecord;
+  receipt: ContextCapsuleVerification | null;
+}
+
+export interface SavedCapsuleReverificationDeps extends Omit<
+  ContextCapsuleRuntimeDeps,
+  "store" | "indexName"
+> {
+  store: ReverificationStore;
+  indexName: string;
+  now?: () => number;
+  notify?: (event: SavedCapsuleReverificationNotification) => void;
+}
+
+export interface SavedCapsuleReverificationTrigger {
+  kind: SavedCapsuleTriggerKind;
+  fromSequence: number;
+  throughSequence: number;
+}
+
+const unwrapStore = <T>(result: StoreResult<T>, operation: string): T => {
+  if (result.ok) return result.value;
+  throw new SavedCapsuleRegistryError(
+    "store_failed",
+    `${operation}: ${result.error.message}`,
+    result.error.cause
+  );
+};
+
+const currentSequence = async (store: ReverificationStore): Promise<number> => {
+  const page = unwrapStore(
+    await store.listDocumentChanges({ limit: 1 }),
+    "Failed to read document change journal"
+  );
+  return decodeDocumentChangeCursor(page.latestCursor);
+};
+
+const getVerificationSnapshot = async (
+  store: ReverificationStore,
+  registrationId: string
+): Promise<SavedCapsuleRegistrationSnapshot> => {
+  const snapshot = unwrapStore(
+    await store.getSavedCapsuleRegistrationSnapshot(registrationId),
+    "Failed to read saved Context Capsule verification snapshot"
+  );
+  if (!snapshot) {
+    throw new SavedCapsuleRegistryError(
+      "registration_not_found",
+      "Saved Context Capsule registration not found"
+    );
+  }
+  return snapshot;
+};
+
+const affected = (
+  receipt: ContextCapsuleVerification
+): {
+  state: "unaffected" | "affected";
+  reasons: string[];
+} => {
+  const reasons: string[] = [];
+  if (receipt.contentStatus === "stale") reasons.push("content_stale");
+  if (receipt.contentStatus === "missing") reasons.push("content_missing");
+  if (receipt.rankingStatus === "reranked") reasons.push("ranking_changed");
+  if (receipt.fingerprintStatus === "drifted") {
+    reasons.push("fingerprint_changed");
+  }
+  return {
+    state: reasons.length === 0 ? "unaffected" : "affected",
+    reasons,
+  };
+};
+
+const errorIdentity = (error: unknown): { code: string; message: string } => {
+  if (
+    error instanceof Error &&
+    "code" in error &&
+    typeof error.code === "string"
+  ) {
+    return { code: error.code, message: error.message.slice(0, 4096) };
+  }
+  return {
+    code: "verification_failed",
+    message: (error instanceof Error
+      ? error.message
+      : "Saved Capsule verification failed"
+    ).slice(0, 4096),
+  };
+};
+
+const persist = async (
+  deps: SavedCapsuleReverificationDeps,
+  registration: SavedCapsuleRegistrationRecord,
+  registrationGeneration: number,
+  verification: SavedCapsuleVerificationRecord
+): Promise<boolean> => {
+  const persisted = unwrapStore(
+    await deps.store.upsertSavedCapsuleVerification(verification, {
+      registrationGeneration,
+    }),
+    "Failed to persist saved Context Capsule verification"
+  );
+  if (!persisted) return false;
+  if (registration.notificationPreference === "local") {
+    deps.notify?.({
+      type: "capsule-reverified",
+      registrationId: registration.registrationId,
+      capsuleId: registration.capsuleId,
+      operationStatus: verification.operationStatus,
+      affectedQuestionState: verification.affectedQuestionState,
+      changedAt: new Date(verification.verifiedAtMs).toISOString(),
+    });
+  }
+  return true;
+};
+
+const reverifySavedCapsuleAttempt = async (
+  registrationId: string,
+  trigger: SavedCapsuleReverificationTrigger,
+  deps: SavedCapsuleReverificationDeps
+): Promise<SavedCapsuleReverificationOutcome | null> => {
+  const snapshot = await getVerificationSnapshot(deps.store, registrationId);
+  const registration = snapshot.registration;
+  const runtimeIndex = canonicalizeIndexName(
+    deps.indexName || DEFAULT_INDEX_NAME
+  );
+  const verifiedAtMs = (deps.now ?? Date.now)();
+  let receipt: ContextCapsuleVerification | null = null;
+  let verification: SavedCapsuleVerificationRecord;
+  try {
+    if (runtimeIndex !== registration.indexName) {
+      throw Object.assign(
+        new Error(
+          `Saved Context Capsule index ${registration.indexName} does not match runtime index ${runtimeIndex}`
+        ),
+        { code: "invalid_filter" }
+      );
+    }
+    const loaded = await loadSavedCapsuleFile(
+      registration.filePath,
+      registration.fileHash
+    );
+    if (loaded.capsule.capsuleId !== registration.capsuleId) {
+      throw Object.assign(
+        new Error("Saved Context Capsule file changed after registration"),
+        { code: "capsule_file_changed" }
+      );
+    }
+    receipt = await verifyContextCapsuleRuntime(loaded.capsule, {
+      ...deps,
+      store: deps.store,
+      indexName: runtimeIndex,
+    });
+    const receiptJson = canonicalVerifiedContextCapsuleJson(receipt);
+    const projection = affected(receipt);
+    verification = {
+      registrationId,
+      triggerKind: trigger.kind,
+      fromSequence: trigger.fromSequence,
+      throughSequence: trigger.throughSequence,
+      operationStatus: "completed",
+      affectedQuestionState: projection.state,
+      affectedReasons: projection.reasons,
+      receiptJson,
+      receiptHash: sha256Text(receiptJson),
+      errorCode: null,
+      errorMessage: null,
+      verifiedAtMs,
+    };
+  } catch (error) {
+    const failure = errorIdentity(error);
+    verification = {
+      registrationId,
+      triggerKind: trigger.kind,
+      fromSequence: trigger.fromSequence,
+      throughSequence: trigger.throughSequence,
+      operationStatus: "failed",
+      affectedQuestionState: "unknown",
+      affectedReasons: [],
+      receiptJson: null,
+      receiptHash: null,
+      errorCode: failure.code,
+      errorMessage: failure.message,
+      verifiedAtMs,
+    };
+  }
+  if (
+    !(await persist(
+      deps,
+      registration,
+      snapshot.registrationGeneration,
+      verification
+    ))
+  ) {
+    return null;
+  }
+  const refreshed = await getSavedCapsule(deps.store, registrationId);
+  return { registration: refreshed, verification, receipt };
+};
+
+export const reverifySavedCapsule = async (
+  registrationId: string,
+  trigger: SavedCapsuleReverificationTrigger,
+  deps: SavedCapsuleReverificationDeps
+): Promise<SavedCapsuleReverificationOutcome> => {
+  for (
+    let attempt = 0;
+    attempt < MAX_REGISTRATION_CONFLICT_ATTEMPTS;
+    attempt += 1
+  ) {
+    const outcome = await reverifySavedCapsuleAttempt(
+      registrationId,
+      trigger,
+      deps
+    );
+    if (outcome) return outcome;
+  }
+  throw new SavedCapsuleRegistryError(
+    "store_failed",
+    "Saved Context Capsule registration changed repeatedly during verification"
+  );
+};
+
+export const reverifySavedCapsuleManually = async (
+  registrationId: string,
+  deps: SavedCapsuleReverificationDeps
+): Promise<SavedCapsuleReverificationOutcome> => {
+  const sequence = await currentSequence(deps.store);
+  const registration = await getSavedCapsule(deps.store, registrationId);
+  return reverifySavedCapsule(
+    registrationId,
+    {
+      kind: "manual",
+      fromSequence: registration.lastAttemptedSequence,
+      throughSequence: sequence,
+    },
+    deps
+  );
+};

package/src/core/change-diff.ts

@@ -0,0 +1,182 @@
+/**
+ * Bounded, content-free structural snapshots for document change journaling.
+ *
+ * The snapshot values are normalized summaries only. Source bodies are used
+ * transiently during sync and are never included in a journal delta.
+ */
+
+import type {
+  DocumentChangeStructureDelta,
+  DocumentChangeSet,
+} from "../store/types";
+
+import { parseFrontmatter } from "../ingestion/frontmatter";
+import { buildLineOffsets } from "../ingestion/position";
+import { getExcludedRanges } from "../ingestion/strip";
+import { normalizeMarkdownPath, normalizeWikiName, parseLinks } from "./links";
+import { extractSections } from "./sections";
+
+export type RelationMap = Record<string, string[]>;
+
+export interface DocumentStructureSnapshot {
+  headings: string[];
+  links: string[];
+  typedEdges: string[];
+  dates: Record<string, string>;
+}
+
+export interface DocumentStructureDeltaResult {
+  delta: DocumentChangeStructureDelta;
+  history: "available" | "unavailable";
+}
+
+const RELATION_EDGE_TYPE_PATTERN = /^[a-z][a-z0-9_]*$/;
+
+export function isRelationMap(value: unknown): value is RelationMap {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return false;
+  }
+  return Object.values(value).every(
+    (targets) =>
+      Array.isArray(targets) &&
+      targets.every((target) => typeof target === "string")
+  );
+}
+
+export function normalizeRelationTarget(raw: string): string {
+  const trimmed = raw.trim();
+  if (trimmed.startsWith("[[") && trimmed.endsWith("]]")) {
+    return trimmed.slice(2, -2).split("|")[0]?.trim() ?? "";
+  }
+  return trimmed;
+}
+
+export function normalizeRelationEdgeType(raw: string): string {
+  return raw
+    .trim()
+    .toLowerCase()
+    .replace(/[-\s]+/g, "_");
+}
+
+const withOccurrences = (values: readonly string[]): string[] => {
+  const counts = new Map<string, number>();
+  return values
+    .map((value) => {
+      const count = (counts.get(value) ?? 0) + 1;
+      counts.set(value, count);
+      return count === 1 ? value : `${value} [${count}]`;
+    })
+    .sort();
+};
+
+const linkSummary = (
+  link: ReturnType<typeof parseLinks>[number],
+  relPath: string
+): string | null => {
+  const anchor = link.targetAnchor
+    ? `#${normalizeWikiName(link.targetAnchor)}`
+    : "";
+  if (link.kind === "wiki") {
+    const collection = link.targetCollection
+      ? `${normalizeWikiName(link.targetCollection)}:`
+      : "";
+    return `wiki:${collection}${normalizeWikiName(link.targetRef)}${anchor}`;
+  }
+  const target = normalizeMarkdownPath(link.targetRef, relPath);
+  return target ? `markdown:${target}${anchor}` : null;
+};
+
+const extractTypedEdges = (markdown: string): string[] => {
+  const relations = parseFrontmatter(markdown).metadata.relations;
+  if (!isRelationMap(relations)) return [];
+
+  const values: string[] = [];
+  for (const [rawEdgeType, targets] of Object.entries(relations)) {
+    const edgeType = normalizeRelationEdgeType(rawEdgeType);
+    if (!RELATION_EDGE_TYPE_PATTERN.test(edgeType)) continue;
+    for (const rawTarget of targets) {
+      const target = normalizeRelationTarget(rawTarget);
+      if (target) {
+        values.push(`${edgeType}:${normalizeWikiName(target)}`);
+      }
+    }
+  }
+  return [...new Set(values)].sort();
+};
+
+export const extractDocumentStructure = (
+  markdown: string,
+  relPath: string,
+  dateFields: Readonly<Record<string, string>> | null | undefined
+): DocumentStructureSnapshot => {
+  const excludedRanges = getExcludedRanges(markdown);
+  const links = parseLinks(markdown, buildLineOffsets(markdown), excludedRanges)
+    .map((link) => linkSummary(link, relPath))
+    .filter((value): value is string => value !== null);
+
+  return {
+    headings: withOccurrences(
+      extractSections(markdown).map(
+        ({ level, title }) => `${"#".repeat(level)} ${title.normalize("NFC")}`
+      )
+    ),
+    links: withOccurrences(links),
+    typedEdges: extractTypedEdges(markdown),
+    dates: { ...dateFields },
+  };
+};
+
+const diffValues = (
+  previous: readonly string[],
+  next: readonly string[]
+): DocumentChangeSet => {
+  const previousValues = new Set(previous);
+  const nextValues = new Set(next);
+  return {
+    added: next.filter((value) => !previousValues.has(value)),
+    removed: previous.filter((value) => !nextValues.has(value)),
+  };
+};
+
+export const diffDocumentStructure = (
+  previous: DocumentStructureSnapshot | null | undefined,
+  next: DocumentStructureSnapshot
+): DocumentStructureDeltaResult => {
+  if (previous === undefined) {
+    return {
+      history: "unavailable",
+      delta: {
+        headings: { added: [], removed: [] },
+        links: { added: [], removed: [] },
+        typedEdges: { added: [], removed: [] },
+        dates: { added: [], removed: [], changed: [] },
+        truncated: true,
+      },
+    };
+  }
+
+  const prior = previous ?? {
+    headings: [],
+    links: [],
+    typedEdges: [],
+    dates: {},
+  };
+  const priorDateKeys = Object.keys(prior.dates).sort();
+  const nextDateKeys = Object.keys(next.dates).sort();
+  const dates = diffValues(priorDateKeys, nextDateKeys);
+  const changed = priorDateKeys.filter(
+    (key) =>
+      Object.hasOwn(next.dates, key) && prior.dates[key] !== next.dates[key]
+  );
+
+  return {
+    history: "available",
+    delta: {
+      headings: diffValues(prior.headings, next.headings),
+      links: diffValues(prior.links, next.links),
+      typedEdges: diffValues(prior.typedEdges, next.typedEdges),
+      dates: { ...dates, changed },
+      truncated: false,
+    },
+  };
+};

package/src/core/change-journal.ts

@@ -0,0 +1,228 @@
+/**
+ * Shared, storage-agnostic contracts for the document change journal.
+ */
+
+import type {
+  DocumentChangeDateDelta,
+  DocumentChangeSet,
+  DocumentChangeStructureDelta,
+  DocumentChangeRetentionPolicy,
+} from "../store/types";
+
+const CURSOR_PREFIX = "gno-change-v1.";
+const MAX_DELTA_ITEMS_PER_SIDE = 16;
+const MAX_DELTA_VALUE_JSON_BYTES = 256;
+export const MAX_DOCUMENT_CHANGE_DELTA_JSON_BYTES = 16 * 1024;
+const UTF8_ENCODER = new TextEncoder();
+
+export const DEFAULT_DOCUMENT_CHANGE_RETENTION: DocumentChangeRetentionPolicy =
+  {
+    maxAgeDays: 30,
+    maxEntries: 10_000,
+    maxBytes: 16 * 1024 * 1024,
+  };
+
+export const EMPTY_DOCUMENT_CHANGE_SET: DocumentChangeSet = {
+  added: [],
+  removed: [],
+};
+
+export const EMPTY_DOCUMENT_CHANGE_DATE_DELTA: DocumentChangeDateDelta = {
+  added: [],
+  removed: [],
+  changed: [],
+};
+
+export const EMPTY_DOCUMENT_CHANGE_STRUCTURE_DELTA: DocumentChangeStructureDelta =
+  {
+    headings: EMPTY_DOCUMENT_CHANGE_SET,
+    links: EMPTY_DOCUMENT_CHANGE_SET,
+    typedEdges: EMPTY_DOCUMENT_CHANGE_SET,
+    dates: EMPTY_DOCUMENT_CHANGE_DATE_DELTA,
+    truncated: false,
+  };
+
+const normalizeValues = (
+  values: readonly string[] | undefined
+): { values: string[]; truncated: boolean } => {
+  const unique = [
+    ...new Set(
+      (values ?? [])
+        .map((value) => value.trim())
+        .filter((value) => value.length > 0)
+    ),
+  ].sort();
+  const selected: string[] = [];
+  const selectedValues = new Set<string>();
+  let truncated = false;
+  for (const value of unique) {
+    let normalized = "";
+    let jsonByteLength = 0;
+    for (const character of value) {
+      const escaped = JSON.stringify(character).slice(1, -1);
+      const characterBytes = UTF8_ENCODER.encode(escaped).byteLength;
+      if (jsonByteLength + characterBytes > MAX_DELTA_VALUE_JSON_BYTES) {
+        truncated = true;
+        break;
+      }
+      normalized += character;
+      jsonByteLength += characterBytes;
+    }
+    if (normalized !== value) {
+      truncated = true;
+    }
+    if (selectedValues.has(normalized)) {
+      truncated = true;
+      continue;
+    }
+    if (selected.length === MAX_DELTA_ITEMS_PER_SIDE) {
+      truncated = true;
+      continue;
+    }
+    selected.push(normalized);
+    selectedValues.add(normalized);
+  }
+  return {
+    values: selected,
+    truncated,
+  };
+};
+
+export interface SerializedDocumentChangeStructureDelta {
+  delta: DocumentChangeStructureDelta;
+  headingDeltaJson: string;
+  linkDeltaJson: string;
+  typedEdgeDeltaJson: string;
+  dateDeltaJson: string;
+}
+
+const normalizeSet = (
+  value: Partial<DocumentChangeSet> | undefined
+): { value: DocumentChangeSet; truncated: boolean } => {
+  const added = normalizeValues(value?.added);
+  const removed = normalizeValues(value?.removed);
+  return {
+    value: { added: added.values, removed: removed.values },
+    truncated: added.truncated || removed.truncated,
+  };
+};
+
+const normalizeDates = (
+  value: Partial<DocumentChangeDateDelta> | undefined
+): { value: DocumentChangeDateDelta; truncated: boolean } => {
+  const added = normalizeValues(value?.added);
+  const removed = normalizeValues(value?.removed);
+  const changed = normalizeValues(value?.changed);
+  return {
+    value: {
+      added: added.values,
+      removed: removed.values,
+      changed: changed.values,
+    },
+    truncated: added.truncated || removed.truncated || changed.truncated,
+  };
+};
+
+export const normalizeDocumentChangeStructureDelta = (
+  value?: Partial<DocumentChangeStructureDelta>
+): DocumentChangeStructureDelta => {
+  const headings = normalizeSet(value?.headings);
+  const links = normalizeSet(value?.links);
+  const typedEdges = normalizeSet(value?.typedEdges);
+  const dates = normalizeDates(value?.dates);
+  return {
+    headings: headings.value,
+    links: links.value,
+    typedEdges: typedEdges.value,
+    dates: dates.value,
+    truncated:
+      (value?.truncated ?? false) ||
+      headings.truncated ||
+      links.truncated ||
+      typedEdges.truncated ||
+      dates.truncated,
+  };
+};
+
+/**
+ * Canonical storage projection for migration 015's UTF-8 byte constraints.
+ * Callers must use this instead of independently stringifying normalized deltas.
+ */
+export const serializeDocumentChangeStructureDelta = (
+  value?: Partial<DocumentChangeStructureDelta>
+): SerializedDocumentChangeStructureDelta => {
+  const delta = normalizeDocumentChangeStructureDelta(value);
+  const serialized = {
+    headingDeltaJson: JSON.stringify(delta.headings),
+    linkDeltaJson: JSON.stringify(delta.links),
+    typedEdgeDeltaJson: JSON.stringify(delta.typedEdges),
+    dateDeltaJson: JSON.stringify(delta.dates),
+  };
+  for (const json of Object.values(serialized)) {
+    if (
+      UTF8_ENCODER.encode(json).byteLength >
+      MAX_DOCUMENT_CHANGE_DELTA_JSON_BYTES
+    ) {
+      throw new RangeError(
+        "Normalized document change structure exceeds its UTF-8 storage limit"
+      );
+    }
+  }
+  return { delta, ...serialized };
+};
+
+export const encodeDocumentChangeCursor = (sequence: number): string => {
+  if (!Number.isSafeInteger(sequence) || sequence < 0) {
+    throw new RangeError(
+      "Document change cursor sequence must be non-negative"
+    );
+  }
+  return `${CURSOR_PREFIX}${btoa(JSON.stringify({ sequence }))}`;
+};
+
+export const decodeDocumentChangeCursor = (cursor: string): number => {
+  if (!cursor.startsWith(CURSOR_PREFIX)) {
+    throw new TypeError("Invalid document change cursor");
+  }
+  try {
+    const parsed: unknown = JSON.parse(
+      atob(cursor.slice(CURSOR_PREFIX.length))
+    );
+    if (
+      !parsed ||
+      typeof parsed !== "object" ||
+      !("sequence" in parsed) ||
+      !Number.isSafeInteger(parsed.sequence) ||
+      (parsed.sequence as number) < 0
+    ) {
+      throw new TypeError("Invalid document change cursor");
+    }
+    return parsed.sequence as number;
+  } catch (cause) {
+    if (cause instanceof TypeError) {
+      throw cause;
+    }
+    throw new TypeError("Invalid document change cursor", { cause });
+  }
+};
+
+export const validateDocumentChangeRetentionPolicy = (
+  policy: DocumentChangeRetentionPolicy,
+  nowMs: number
+): void => {
+  if (
+    !Number.isSafeInteger(nowMs) ||
+    nowMs < 0 ||
+    !Number.isSafeInteger(policy.maxAgeDays) ||
+    policy.maxAgeDays < 1 ||
+    policy.maxAgeDays > 3650 ||
+    !Number.isSafeInteger(policy.maxEntries) ||
+    policy.maxEntries < 1 ||
+    policy.maxEntries > 1_000_000 ||
+    !Number.isSafeInteger(policy.maxBytes) ||
+    policy.maxBytes < 1 ||
+    policy.maxBytes > 1024 * 1024 * 1024
+  ) {
+    throw new RangeError("Invalid document change retention policy");
+  }
+};