@gmickel/gno 1.12.3 → 1.13.0

package/src/core/activation-probe-plan.ts

@@ -0,0 +1,321 @@
+/**
+ * Ephemeral activation probe planning shared by local and connector proofs.
+ *
+ * Raw probe terms never leave this in-memory plan and must not be serialized,
+ * logged, or copied into activation receipts.
+ */
+
+import type { FtsTokenizer } from "../config/types";
+import type {
+  ActivationIndexDocument,
+  ActivationIndexIdentity,
+  StorePort,
+  StoreResult,
+} from "../store/types";
+
+import { err, ok } from "../store/types";
+import {
+  extractActivationProbeTerms,
+  fingerprintActivationIndex,
+} from "./activation-probe";
+
+const MAX_PROBE_DOCUMENTS = 16;
+const MAX_PROBE_DOCUMENT_SCANS = 64;
+const MAX_PROBE_ATTEMPTS = 64;
+const MAX_PROBE_CONTENT_CHARS = 32_768;
+const PROBE_RESULT_LIMIT = 8;
+
+interface ProbeDocumentReference {
+  document: ActivationIndexDocument;
+  mirrorHash: string;
+}
+
+interface ProbeDocumentCandidates extends ProbeDocumentReference {
+  terms: Array<{ term: string; occurrence: number }>;
+}
+
+export interface EphemeralActivationProbeCandidate {
+  /** Sensitive corpus-derived term. Never serialize or log. */
+  term: string;
+  documents: ProbeDocumentReference[];
+}
+
+export interface EphemeralActivationProbePlan {
+  collection: string;
+  fingerprint: string;
+  identity: ActivationIndexIdentity;
+  activeDocuments: ActivationIndexDocument[];
+  candidates: EphemeralActivationProbeCandidate[];
+}
+
+export interface EphemeralActivationProbeMatch {
+  /** Sensitive corpus-derived term. Never serialize or log. */
+  term: string;
+  probeHash: string;
+  resultUri: string;
+  resultSourceHash: string;
+}
+
+export interface ActivationProbePlanRevalidation {
+  stable: boolean;
+  currentPlan: EphemeralActivationProbePlan;
+}
+
+export type ActivationProbeMatchResult =
+  | { kind: "matched"; value: EphemeralActivationProbeMatch }
+  | { kind: "no_probe_term" }
+  | { kind: "retrieval_mismatch"; probeHash?: string };
+
+function sha256(value: string): string {
+  const hasher = new Bun.CryptoHasher("sha256");
+  hasher.update(value);
+  return hasher.digest("hex");
+}
+
+function compareText(left: string, right: string): number {
+  if (left === right) {
+    return 0;
+  }
+  return left < right ? -1 : 1;
+}
+
+async function collectProbeCandidates(
+  store: StorePort,
+  documents: ActivationIndexDocument[],
+  ftsTokenizer: FtsTokenizer
+): Promise<StoreResult<EphemeralActivationProbeCandidate[]>> {
+  const probeDocuments: ProbeDocumentCandidates[] = [];
+  let probeDocumentReads = 0;
+  for (const document of documents) {
+    if (!document.mirrorHash) {
+      continue;
+    }
+    if (probeDocumentReads >= MAX_PROBE_DOCUMENT_SCANS) {
+      break;
+    }
+    probeDocumentReads += 1;
+    const content = await store.getContentPrefix(
+      document.mirrorHash,
+      MAX_PROBE_CONTENT_CHARS
+    );
+    if (!content.ok) {
+      return err(
+        content.error.code,
+        content.error.message,
+        content.error.cause
+      );
+    }
+    if (content.value === null) {
+      continue;
+    }
+    const terms = extractActivationProbeTerms(content.value, ftsTokenizer);
+    if (terms.length === 0) {
+      continue;
+    }
+    probeDocuments.push({
+      document,
+      mirrorHash: document.mirrorHash,
+      terms: terms.map((term, occurrence) => ({ term, occurrence })),
+    });
+    if (probeDocuments.length >= MAX_PROBE_DOCUMENTS) {
+      break;
+    }
+  }
+
+  const documentsByTerm = new Map<string, ProbeDocumentReference[]>();
+  for (const probeDocument of probeDocuments) {
+    for (const { term } of probeDocument.terms) {
+      const references = documentsByTerm.get(term) ?? [];
+      references.push({
+        document: probeDocument.document,
+        mirrorHash: probeDocument.mirrorHash,
+      });
+      documentsByTerm.set(term, references);
+    }
+  }
+
+  for (const probeDocument of probeDocuments) {
+    probeDocument.terms.sort((left, right) => {
+      const frequencyDifference =
+        (documentsByTerm.get(left.term)?.length ?? 0) -
+        (documentsByTerm.get(right.term)?.length ?? 0);
+      if (frequencyDifference !== 0) {
+        return frequencyDifference;
+      }
+      if (left.occurrence !== right.occurrence) {
+        return left.occurrence - right.occurrence;
+      }
+      return compareText(left.term, right.term);
+    });
+  }
+
+  const candidates: EphemeralActivationProbeCandidate[] = [];
+  const seenTerms = new Set<string>();
+  const rounds = Math.max(
+    0,
+    ...probeDocuments.map(({ terms }) => terms.length)
+  );
+  for (let round = 0; round < rounds; round += 1) {
+    for (const probeDocument of probeDocuments) {
+      const term = probeDocument.terms[round]?.term;
+      if (!term || seenTerms.has(term)) {
+        continue;
+      }
+      seenTerms.add(term);
+      candidates.push({
+        term,
+        documents: documentsByTerm.get(term) ?? [],
+      });
+      if (candidates.length >= MAX_PROBE_ATTEMPTS) {
+        return ok(candidates);
+      }
+    }
+  }
+  return ok(candidates);
+}
+
+export async function createEphemeralActivationProbePlan(
+  store: StorePort,
+  collection: string,
+  options: { collectCandidates?: boolean } = {}
+): Promise<StoreResult<EphemeralActivationProbePlan>> {
+  const snapshotResult = await store.getActivationIndexSnapshot(collection);
+  if (!snapshotResult.ok) {
+    return err(
+      snapshotResult.error.code,
+      snapshotResult.error.message,
+      snapshotResult.error.cause
+    );
+  }
+  const { documents, identity } = snapshotResult.value;
+  const activeDocuments = documents
+    .filter((document) => document.active)
+    .sort((left, right) => {
+      const uriDifference = compareText(left.uri, right.uri);
+      return uriDifference === 0 ? left.id - right.id : uriDifference;
+    });
+  const fingerprint = fingerprintActivationIndex({
+    collection,
+    indexName: identity.indexName,
+    schemaVersion: identity.schemaVersion,
+    ftsTokenizer: identity.ftsTokenizer,
+    ftsStateHash: identity.ftsStateHash,
+    documents: activeDocuments,
+  });
+  const candidates =
+    options.collectCandidates === false
+      ? ok([])
+      : await collectProbeCandidates(
+          store,
+          activeDocuments,
+          identity.ftsTokenizer
+        );
+  if (!candidates.ok) {
+    return candidates;
+  }
+  return ok({
+    collection,
+    fingerprint,
+    identity,
+    activeDocuments,
+    candidates: candidates.value,
+  });
+}
+
+export async function populateEphemeralActivationProbePlan(
+  store: StorePort,
+  plan: EphemeralActivationProbePlan
+): Promise<StoreResult<EphemeralActivationProbePlan>> {
+  const candidates = await collectProbeCandidates(
+    store,
+    plan.activeDocuments,
+    plan.identity.ftsTokenizer
+  );
+  if (!candidates.ok) {
+    return candidates;
+  }
+  return ok({ ...plan, candidates: candidates.value });
+}
+
+/**
+ * Re-read the metadata-only activation snapshot and compare it with a plan
+ * created before an asynchronous proof. Callers use this optimistic guard
+ * immediately before accepting or persisting proof derived from that plan.
+ */
+export async function revalidateEphemeralActivationProbePlan(
+  store: StorePort,
+  plan: EphemeralActivationProbePlan
+): Promise<StoreResult<ActivationProbePlanRevalidation>> {
+  const current = await createEphemeralActivationProbePlan(
+    store,
+    plan.collection,
+    { collectCandidates: false }
+  );
+  if (!current.ok) {
+    return current;
+  }
+  return ok({
+    stable: current.value.fingerprint === plan.fingerprint,
+    currentPlan: current.value,
+  });
+}
+
+export async function findEphemeralActivationProbeMatch(
+  store: StorePort,
+  plan: EphemeralActivationProbePlan
+): Promise<StoreResult<ActivationProbeMatchResult>> {
+  if (plan.candidates.length === 0) {
+    return ok({ kind: "no_probe_term" });
+  }
+
+  const activeDocumentsByUri = new Map(
+    plan.activeDocuments.map((document) => [document.uri, document])
+  );
+  let finalProbeHash: string | undefined;
+  for (const candidate of plan.candidates) {
+    const digestKey = candidate.documents
+      .map(({ mirrorHash }) => mirrorHash)
+      .sort(compareText)
+      .join("\0");
+    finalProbeHash = sha256(`${digestKey}\0${candidate.term}`);
+    const search = await store.searchFts(candidate.term, {
+      collection: plan.collection,
+      limit: PROBE_RESULT_LIMIT,
+      snippet: false,
+    });
+    if (!search.ok) {
+      return err(search.error.code, search.error.message, search.error.cause);
+    }
+    const matchedResult = search.value.find((result) => {
+      const activeDocument = result.uri
+        ? activeDocumentsByUri.get(result.uri)
+        : undefined;
+      return (
+        activeDocument !== undefined &&
+        activeDocument.mirrorHash !== null &&
+        result.sourceHash === activeDocument.sourceHash &&
+        result.mirrorHash === activeDocument.mirrorHash
+      );
+    });
+    if (!matchedResult) {
+      continue;
+    }
+    const matchedDocument = matchedResult.uri
+      ? activeDocumentsByUri.get(matchedResult.uri)
+      : undefined;
+    if (!matchedDocument) {
+      continue;
+    }
+    return ok({
+      kind: "matched",
+      value: {
+        term: candidate.term,
+        probeHash: finalProbeHash,
+        resultUri: matchedResult.uri ?? matchedDocument.uri,
+        resultSourceHash:
+          matchedResult.sourceHash ?? matchedDocument.sourceHash,
+      },
+    });
+  }
+  return ok({ kind: "retrieval_mismatch", probeHash: finalProbeHash });
+}

package/src/core/activation-probe.ts

@@ -0,0 +1,138 @@
+import type { FtsTokenizer } from "../config/types";
+import type { ActivationIndexDocument } from "../store/types";
+
+const RECEIPT_SCHEMA_VERSION = "1.0";
+const INDEX_IMPLEMENTATION_ID = "documents_fts-bm25-v1";
+const MAX_CONTENT_CHARS_PER_DOCUMENT = 32_768;
+const MAX_CANDIDATES_PER_DOCUMENT = 32;
+const MAX_TERM_CODEPOINTS = 64;
+const CJK_PREFIX_CODEPOINTS = 4;
+const TOKEN_PATTERN = /[\p{L}\p{N}_]+/gu;
+const LETTER_PATTERN = /\p{L}/u;
+const CJK_PATTERN =
+  /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
+
+const STOPWORDS = new Set([
+  "a",
+  "an",
+  "and",
+  "are",
+  "as",
+  "at",
+  "be",
+  "by",
+  "for",
+  "from",
+  "in",
+  "is",
+  "it",
+  "of",
+  "on",
+  "or",
+  "that",
+  "the",
+  "this",
+  "to",
+  "with",
+  "der",
+  "die",
+  "das",
+  "und",
+  "ein",
+  "eine",
+  "le",
+  "la",
+  "les",
+  "et",
+  "un",
+  "une",
+  "il",
+  "e",
+  "per",
+  "con",
+]);
+
+function sha256(value: string): string {
+  const hasher = new Bun.CryptoHasher("sha256");
+  hasher.update(value);
+  return hasher.digest("hex");
+}
+
+function compareText(left: string, right: string): number {
+  if (left === right) {
+    return 0;
+  }
+  return left < right ? -1 : 1;
+}
+
+function normalizeTerm(raw: string, ftsTokenizer: FtsTokenizer): string | null {
+  const normalized = raw.normalize("NFC").toLocaleLowerCase("und");
+  const codepoints = Array.from(normalized);
+  const minimumCodepoints = ftsTokenizer === "trigram" ? 3 : 2;
+  if (
+    codepoints.length < minimumCodepoints ||
+    !LETTER_PATTERN.test(normalized) ||
+    STOPWORDS.has(normalized)
+  ) {
+    return null;
+  }
+  if (codepoints.length <= MAX_TERM_CODEPOINTS) {
+    return normalized;
+  }
+  if (CJK_PATTERN.test(normalized)) {
+    return codepoints.slice(0, CJK_PREFIX_CODEPOINTS).join("");
+  }
+  return null;
+}
+
+/** Extract bounded Unicode FTS-compatible candidates in occurrence order. */
+export function extractActivationProbeTerms(
+  text: string,
+  ftsTokenizer: FtsTokenizer = "unicode61"
+): string[] {
+  const terms: string[] = [];
+  const seen = new Set<string>();
+  const input = text.slice(0, MAX_CONTENT_CHARS_PER_DOCUMENT);
+  for (const match of input.matchAll(TOKEN_PATTERN)) {
+    const term = normalizeTerm(match[0], ftsTokenizer);
+    if (!term || seen.has(term)) {
+      continue;
+    }
+    seen.add(term);
+    terms.push(term);
+    if (terms.length >= MAX_CANDIDATES_PER_DOCUMENT) {
+      break;
+    }
+  }
+  return terms;
+}
+
+export function fingerprintActivationIndex(input: {
+  collection: string;
+  indexName: string;
+  schemaVersion: number;
+  ftsTokenizer: string;
+  ftsStateHash: string;
+  documents: ActivationIndexDocument[];
+}): string {
+  const documents = input.documents
+    .filter((document) => document.active)
+    .map((document) => ({
+      uri: document.uri,
+      sourceHash: document.sourceHash,
+      mirrorHash: document.mirrorHash,
+    }))
+    .sort((left, right) => compareText(left.uri, right.uri));
+  return sha256(
+    JSON.stringify({
+      receiptSchemaVersion: RECEIPT_SCHEMA_VERSION,
+      indexImplementation: INDEX_IMPLEMENTATION_ID,
+      indexName: input.indexName,
+      schemaVersion: input.schemaVersion,
+      ftsTokenizer: input.ftsTokenizer,
+      ftsStateHash: input.ftsStateHash,
+      collection: input.collection,
+      documents,
+    })
+  );
+}

package/src/core/activation-receipt-store.ts

@@ -0,0 +1,39 @@
+import type {
+  ActivationVerificationReceipt,
+  StorePort,
+  StoreResult,
+} from "../store/types";
+
+import { err, ok } from "../store/types";
+
+/**
+ * Persist a receipt only after the collection has been synchronized into the
+ * store. Config can name a collection before its first sync creates the parent
+ * row required by the receipt table's foreign key.
+ */
+export async function persistActivationReceiptForKnownCollection(
+  store: StorePort,
+  receipt: ActivationVerificationReceipt
+): Promise<StoreResult<ActivationVerificationReceipt>> {
+  const collections = await store.getCollections();
+  if (!collections.ok) {
+    return err(
+      collections.error.code,
+      collections.error.message,
+      collections.error.cause
+    );
+  }
+  if (!collections.value.some(({ name }) => name === receipt.collection)) {
+    return ok(receipt);
+  }
+
+  const persisted = await store.upsertActivationReceipt(receipt);
+  if (!persisted.ok) {
+    return err(
+      persisted.error.code,
+      persisted.error.message,
+      persisted.error.cause
+    );
+  }
+  return ok(receipt);
+}