@gmickel/gno 1.19.0 → 1.21.0

package/src/core/change-journal.ts

@@ -0,0 +1,228 @@
+/**
+ * Shared, storage-agnostic contracts for the document change journal.
+ */
+
+import type {
+  DocumentChangeDateDelta,
+  DocumentChangeSet,
+  DocumentChangeStructureDelta,
+  DocumentChangeRetentionPolicy,
+} from "../store/types";
+
+const CURSOR_PREFIX = "gno-change-v1.";
+const MAX_DELTA_ITEMS_PER_SIDE = 16;
+const MAX_DELTA_VALUE_JSON_BYTES = 256;
+export const MAX_DOCUMENT_CHANGE_DELTA_JSON_BYTES = 16 * 1024;
+const UTF8_ENCODER = new TextEncoder();
+
+export const DEFAULT_DOCUMENT_CHANGE_RETENTION: DocumentChangeRetentionPolicy =
+  {
+    maxAgeDays: 30,
+    maxEntries: 10_000,
+    maxBytes: 16 * 1024 * 1024,
+  };
+
+export const EMPTY_DOCUMENT_CHANGE_SET: DocumentChangeSet = {
+  added: [],
+  removed: [],
+};
+
+export const EMPTY_DOCUMENT_CHANGE_DATE_DELTA: DocumentChangeDateDelta = {
+  added: [],
+  removed: [],
+  changed: [],
+};
+
+export const EMPTY_DOCUMENT_CHANGE_STRUCTURE_DELTA: DocumentChangeStructureDelta =
+  {
+    headings: EMPTY_DOCUMENT_CHANGE_SET,
+    links: EMPTY_DOCUMENT_CHANGE_SET,
+    typedEdges: EMPTY_DOCUMENT_CHANGE_SET,
+    dates: EMPTY_DOCUMENT_CHANGE_DATE_DELTA,
+    truncated: false,
+  };
+
+const normalizeValues = (
+  values: readonly string[] | undefined
+): { values: string[]; truncated: boolean } => {
+  const unique = [
+    ...new Set(
+      (values ?? [])
+        .map((value) => value.trim())
+        .filter((value) => value.length > 0)
+    ),
+  ].sort();
+  const selected: string[] = [];
+  const selectedValues = new Set<string>();
+  let truncated = false;
+  for (const value of unique) {
+    let normalized = "";
+    let jsonByteLength = 0;
+    for (const character of value) {
+      const escaped = JSON.stringify(character).slice(1, -1);
+      const characterBytes = UTF8_ENCODER.encode(escaped).byteLength;
+      if (jsonByteLength + characterBytes > MAX_DELTA_VALUE_JSON_BYTES) {
+        truncated = true;
+        break;
+      }
+      normalized += character;
+      jsonByteLength += characterBytes;
+    }
+    if (normalized !== value) {
+      truncated = true;
+    }
+    if (selectedValues.has(normalized)) {
+      truncated = true;
+      continue;
+    }
+    if (selected.length === MAX_DELTA_ITEMS_PER_SIDE) {
+      truncated = true;
+      continue;
+    }
+    selected.push(normalized);
+    selectedValues.add(normalized);
+  }
+  return {
+    values: selected,
+    truncated,
+  };
+};
+
+export interface SerializedDocumentChangeStructureDelta {
+  delta: DocumentChangeStructureDelta;
+  headingDeltaJson: string;
+  linkDeltaJson: string;
+  typedEdgeDeltaJson: string;
+  dateDeltaJson: string;
+}
+
+const normalizeSet = (
+  value: Partial<DocumentChangeSet> | undefined
+): { value: DocumentChangeSet; truncated: boolean } => {
+  const added = normalizeValues(value?.added);
+  const removed = normalizeValues(value?.removed);
+  return {
+    value: { added: added.values, removed: removed.values },
+    truncated: added.truncated || removed.truncated,
+  };
+};
+
+const normalizeDates = (
+  value: Partial<DocumentChangeDateDelta> | undefined
+): { value: DocumentChangeDateDelta; truncated: boolean } => {
+  const added = normalizeValues(value?.added);
+  const removed = normalizeValues(value?.removed);
+  const changed = normalizeValues(value?.changed);
+  return {
+    value: {
+      added: added.values,
+      removed: removed.values,
+      changed: changed.values,
+    },
+    truncated: added.truncated || removed.truncated || changed.truncated,
+  };
+};
+
+export const normalizeDocumentChangeStructureDelta = (
+  value?: Partial<DocumentChangeStructureDelta>
+): DocumentChangeStructureDelta => {
+  const headings = normalizeSet(value?.headings);
+  const links = normalizeSet(value?.links);
+  const typedEdges = normalizeSet(value?.typedEdges);
+  const dates = normalizeDates(value?.dates);
+  return {
+    headings: headings.value,
+    links: links.value,
+    typedEdges: typedEdges.value,
+    dates: dates.value,
+    truncated:
+      (value?.truncated ?? false) ||
+      headings.truncated ||
+      links.truncated ||
+      typedEdges.truncated ||
+      dates.truncated,
+  };
+};
+
+/**
+ * Canonical storage projection for migration 015's UTF-8 byte constraints.
+ * Callers must use this instead of independently stringifying normalized deltas.
+ */
+export const serializeDocumentChangeStructureDelta = (
+  value?: Partial<DocumentChangeStructureDelta>
+): SerializedDocumentChangeStructureDelta => {
+  const delta = normalizeDocumentChangeStructureDelta(value);
+  const serialized = {
+    headingDeltaJson: JSON.stringify(delta.headings),
+    linkDeltaJson: JSON.stringify(delta.links),
+    typedEdgeDeltaJson: JSON.stringify(delta.typedEdges),
+    dateDeltaJson: JSON.stringify(delta.dates),
+  };
+  for (const json of Object.values(serialized)) {
+    if (
+      UTF8_ENCODER.encode(json).byteLength >
+      MAX_DOCUMENT_CHANGE_DELTA_JSON_BYTES
+    ) {
+      throw new RangeError(
+        "Normalized document change structure exceeds its UTF-8 storage limit"
+      );
+    }
+  }
+  return { delta, ...serialized };
+};
+
+export const encodeDocumentChangeCursor = (sequence: number): string => {
+  if (!Number.isSafeInteger(sequence) || sequence < 0) {
+    throw new RangeError(
+      "Document change cursor sequence must be non-negative"
+    );
+  }
+  return `${CURSOR_PREFIX}${btoa(JSON.stringify({ sequence }))}`;
+};
+
+export const decodeDocumentChangeCursor = (cursor: string): number => {
+  if (!cursor.startsWith(CURSOR_PREFIX)) {
+    throw new TypeError("Invalid document change cursor");
+  }
+  try {
+    const parsed: unknown = JSON.parse(
+      atob(cursor.slice(CURSOR_PREFIX.length))
+    );
+    if (
+      !parsed ||
+      typeof parsed !== "object" ||
+      !("sequence" in parsed) ||
+      !Number.isSafeInteger(parsed.sequence) ||
+      (parsed.sequence as number) < 0
+    ) {
+      throw new TypeError("Invalid document change cursor");
+    }
+    return parsed.sequence as number;
+  } catch (cause) {
+    if (cause instanceof TypeError) {
+      throw cause;
+    }
+    throw new TypeError("Invalid document change cursor", { cause });
+  }
+};
+
+export const validateDocumentChangeRetentionPolicy = (
+  policy: DocumentChangeRetentionPolicy,
+  nowMs: number
+): void => {
+  if (
+    !Number.isSafeInteger(nowMs) ||
+    nowMs < 0 ||
+    !Number.isSafeInteger(policy.maxAgeDays) ||
+    policy.maxAgeDays < 1 ||
+    policy.maxAgeDays > 3650 ||
+    !Number.isSafeInteger(policy.maxEntries) ||
+    policy.maxEntries < 1 ||
+    policy.maxEntries > 1_000_000 ||
+    !Number.isSafeInteger(policy.maxBytes) ||
+    policy.maxBytes < 1 ||
+    policy.maxBytes > 1024 * 1024 * 1024
+  ) {
+    throw new RangeError("Invalid document change retention policy");
+  }
+};

package/src/core/context-budget.ts

@@ -6,6 +6,8 @@
  * canonical Capsule payload, including coverage, omissions, and guidance.
  */
 
+import type { FusionSource } from "../pipeline/types";
+
 export const CONTEXT_OMISSION_REASONS = [
   "duplicate",
   "overlap",
@@ -43,6 +45,10 @@ export interface MaterializedContextCandidate<
   text: string;
   facets: string[];
   retrievalRank: number;
+  /** Absent only for legacy results that predate planner provenance metadata. */
+  retrievalSources?: FusionSource[];
+  /** Absent only for legacy results that predate planner provenance metadata. */
+  graphExpanded?: boolean;
   value: T;
 }
 

package/src/core/context-capsule-retrieval-schema.ts

@@ -59,10 +59,14 @@ export const contextCapsuleRetrievalSchema = z
       .object({
         author: z.string().min(1).max(256).nullable(),
         lang: z.string().min(1).max(64).nullable(),
+        intent: z.string().min(1).max(16_384).nullable().optional(),
+        exclude: z.array(nonEmptyText.max(256)).max(128).optional(),
+        minScore: z.number().min(0).max(1).nullable().optional(),
         queryModes: z.array(queryModeSchema).max(128),
         limit: z.number().int().positive(),
         candidateLimit: z.number().int().positive(),
         graphRequested: z.boolean(),
+        rerankRequested: z.boolean().optional(),
       })
       .strict(),
     capabilityStates: z

package/src/core/context-capsule-schema.ts

@@ -29,6 +29,14 @@ const positiveIntegerSchema = z.number().int().positive();
 const nonNegativeIntegerSchema = z.number().int().nonnegative();
 const COLLECTION_PATTERN = /^[a-z0-9][a-z0-9_-]{0,63}$/;
 const collectionSchema = nonEmptyTextSchema.max(64).regex(COLLECTION_PATTERN);
+const retrievalSourceSchema = z.enum([
+  "bm25",
+  "vector",
+  "bm25_variant",
+  "vector_variant",
+  "hyde",
+  "graph",
+]);
 const compareCodeUnits = (left: string, right: string): number =>
   left < right ? -1 : left > right ? 1 : 0;
 
@@ -244,6 +252,15 @@ export const contextCapsuleEvidenceSchema = z
     contextIds: z.array(sha256Schema).max(128),
     retrievalRank: positiveIntegerSchema,
     selectionRank: positiveIntegerSchema,
+    retrievalSources: z
+      .array(retrievalSourceSchema)
+      .min(1)
+      .max(6)
+      .refine((sources) => new Set(sources).size === sources.length, {
+        message: "retrievalSources must be unique",
+      })
+      .optional(),
+    graphExpanded: z.boolean().optional(),
     facets: z.array(nonEmptyTextSchema.max(512)).max(128),
     trust: z.literal("untrusted"),
     egress: z.enum([

package/src/core/context-capsule-validation.ts

@@ -352,11 +352,12 @@ export const validateContextCapsulePayload = (
     });
   }
   const semanticRequested = value.retrieval.depthPolicy !== "fast";
+  const rerankRequested =
+    value.retrieval.request.rerankRequested ?? semanticRequested;
   if (
     value.retrieval.capabilityStates.semanticSearch.requested !==
       semanticRequested ||
-    value.retrieval.capabilityStates.reranking.requested !==
-      semanticRequested ||
+    value.retrieval.capabilityStates.reranking.requested !== rerankRequested ||
     value.retrieval.capabilityStates.graphExpansion.requested !==
       value.retrieval.request.graphRequested
   ) {

package/src/core/context-capsule.ts

@@ -84,6 +84,17 @@ const normalizePayload = (
         value.retrieval.request.lang === null
           ? null
           : normalizeText(value.retrieval.request.lang),
+      ...(value.retrieval.request.intent === undefined
+        ? {}
+        : {
+            intent:
+              value.retrieval.request.intent === null
+                ? null
+                : normalizeText(value.retrieval.request.intent),
+          }),
+      ...(value.retrieval.request.exclude === undefined
+        ? {}
+        : { exclude: normalizeSet(value.retrieval.request.exclude) }),
       queryModes: value.retrieval.request.queryModes.map((mode) => ({
         ...mode,
         text: normalizeText(mode.text),
@@ -110,6 +121,13 @@ const normalizePayload = (
     documentDate: normalizeDocumentDate(item.documentDate),
     observedAt: normalizeDate(item.observedAt),
     contextIds: normalizeSet(item.contextIds),
+    ...(item.retrievalSources === undefined
+      ? {}
+      : {
+          retrievalSources: [...new Set(item.retrievalSources)].sort(
+            compareCodeUnits
+          ),
+        }),
     facets: normalizeSet(item.facets),
   })),
   guidance: {

package/src/core/context-compiler.ts

@@ -64,8 +64,8 @@ export interface ContextRetrievalRequest extends HybridSearchOptions {
 export interface ContextRetrievalCandidate {
   result: SearchResult;
   retrievalRank: number;
-  retrievalSources: FusionSource[];
-  graphExpanded: boolean;
+  retrievalSources?: FusionSource[];
+  graphExpanded?: boolean;
   contextIds: string[];
   observedAt: string | null;
 }
@@ -97,9 +97,13 @@ export interface ContextCompilerInput {
   categories?: string[];
   author?: string;
   lang?: string;
+  intent?: string;
+  exclude?: string[];
+  minScore?: number;
   since?: string;
   until?: string;
   graph?: boolean;
+  noRerank?: boolean;
   limit?: number;
   candidateLimit?: number;
   /** Frozen once by the caller; never defaulted from wall-clock time. */
@@ -164,16 +168,9 @@ const compareCodeUnits = (left: string, right: string): number => {
 };
 
 const plannerMeta = (
-  result: SearchResult,
-  fallbackRank: number
-): SearchResultPlannerMetadata =>
-  result[SEARCH_RESULT_PLANNER_METADATA] ?? {
-    retrievalRank: fallbackRank,
-    mirrorHash: result.conversion?.mirrorHash ?? "",
-    seq: 0,
-    sources: [],
-    graphExpanded: false,
-  };
+  result: SearchResult
+): SearchResultPlannerMetadata | undefined =>
+  result[SEARCH_RESULT_PLANNER_METADATA];
 
 const compareSearchResults = (
   left: SearchResult,
@@ -219,7 +216,9 @@ const referenceFromResult = (
 const normalizeMaterialized = <T>(
   draft: ContextMaterializedDraft<T>,
   facets: string[],
-  retrievalRank: number
+  retrievalRank: number,
+  retrievalSources: FusionSource[] | undefined,
+  graphExpanded: boolean | undefined
 ): MaterializedContextCandidate<T> => {
   const text = draft.text;
   if (
@@ -250,6 +249,10 @@ const normalizeMaterialized = <T>(
     text,
     facets,
     retrievalRank,
+    ...(retrievalSources === undefined
+      ? {}
+      : { retrievalSources: [...retrievalSources].sort(compareCodeUnits) }),
+    ...(graphExpanded === undefined ? {} : { graphExpanded }),
     value: draft.value,
   };
 };
@@ -352,10 +355,13 @@ export const planContextEvidence = async <T, P>(
         categories: input.categories,
         author: input.author,
         lang: input.lang,
+        intent: input.intent,
+        exclude: input.exclude,
+        minScore: input.minScore,
         since: temporalRange.since,
         until: temporalRange.until,
         graph: hasRerankBudget ? input.graph : false,
-        noRerank: hasRerankBudget ? undefined : true,
+        noRerank: input.noRerank || !hasRerankBudget ? true : undefined,
         limit: resultLimit === undefined ? undefined : Math.max(1, resultLimit),
         candidateLimit:
           rerankLimit === undefined ? undefined : Math.max(1, rerankLimit),
@@ -377,7 +383,7 @@ export const planContextEvidence = async <T, P>(
   const results = decoratedResults
     .map((result, index) => ({
       result,
-      retrievalRank: plannerMeta(result, index + 1).retrievalRank,
+      retrievalRank: plannerMeta(result)?.retrievalRank ?? index + 1,
     }))
     .sort(
       (left, right) =>
@@ -407,8 +413,8 @@ export const planContextEvidence = async <T, P>(
   const referencesByCandidate: ContextCandidateReference[] = [];
 
   for (const [index, result] of results.entries()) {
-    const meta = plannerMeta(result, index + 1);
-    for (const source of meta.sources) retrievalSources.add(source);
+    const meta = plannerMeta(result);
+    for (const source of meta?.sources ?? []) retrievalSources.add(source);
     const retrievalReference = referenceFromResult(result);
     if (!isContextUriInScope(result.uri, indexName, collections, uriPrefix)) {
       for (const facet of facetPlan) {
@@ -431,9 +437,13 @@ export const planContextEvidence = async <T, P>(
     }
     plannedCandidates.push({
       result,
-      retrievalRank: meta.retrievalRank,
-      retrievalSources: [...meta.sources].sort(compareCodeUnits),
-      graphExpanded: meta.graphExpanded,
+      retrievalRank: meta?.retrievalRank ?? index + 1,
+      ...(meta === undefined
+        ? {}
+        : {
+            retrievalSources: [...meta.sources].sort(compareCodeUnits),
+            graphExpanded: meta.graphExpanded,
+          }),
       contextIds:
         guidance.idsByResultIdentity.get(
           contextGuidanceResultIdentity(result)
@@ -486,7 +496,9 @@ export const planContextEvidence = async <T, P>(
       normalizeMaterialized(
         outcome.candidate,
         matchedFacets,
-        plannedCandidate.retrievalRank
+        plannedCandidate.retrievalRank,
+        plannedCandidate.retrievalSources,
+        plannedCandidate.graphExpanded
       )
     );
   }

package/src/core/context-evidence.ts

@@ -490,6 +490,12 @@ export const toContextCapsuleEvidence = (
     text: candidate.text,
     retrievalRank: candidate.retrievalRank,
     selectionRank,
+    ...(candidate.retrievalSources === undefined
+      ? {}
+      : { retrievalSources: [...candidate.retrievalSources] }),
+    ...(candidate.graphExpanded === undefined
+      ? {}
+      : { graphExpanded: candidate.graphExpanded }),
     facets: [...candidate.facets],
   };
 };