@gmickel/gno 2.4.0 → 2.6.0

package/src/app/compiled-context-files.ts

@@ -0,0 +1,361 @@
+// Bun has no lstat, exclusive-create, hard-link or atomic-rename primitives.
+import { lstat, open, link, rename, unlink } from "node:fs/promises";
+import { dirname, parse, resolve, join } from "node:path";
+import { z } from "zod";
+
+import type {
+  CompiledContextCheck,
+  CompiledContextPreview,
+} from "../core/compiled-context";
+import type { ContextCapsuleV1 } from "../core/context-capsule";
+import type { ContextCapsuleBuildInput } from "./context-runtime";
+
+import { withWriteLock } from "../core/file-lock";
+import {
+  checkCompiledContext,
+  compiledContextRefreshRequest,
+  previewCompiledContext,
+  type CompiledContextRuntimeDeps,
+} from "./compiled-context";
+
+const MAX_BYTES = 4 * 1024 * 1024;
+const snapshotSchema = z
+  .object({
+    capsulePath: z.string().min(1),
+    outputDigest: z.string().regex(/^[a-f0-9]{64}$/),
+    settings: z
+      .object({
+        budgetTokens: z.number().int().positive(),
+        budgetBytes: z.number().int().positive().optional(),
+      })
+      .strict(),
+  })
+  .strict();
+const sidecarSchema = snapshotSchema
+  .extend({
+    artifactKind: z.literal("gno_compiled_context_sidecar"),
+    schemaVersion: z.literal("1.0"),
+    rendererVersion: z.literal("1"),
+    previous: snapshotSchema.optional(),
+  })
+  .strict();
+type Snapshot = z.infer<typeof snapshotSchema>;
+type Sidecar = z.infer<typeof sidecarSchema>;
+export interface CompiledContextFileResult {
+  status: "written" | "unchanged";
+  outputPath: string;
+  capsulePath: string;
+  sidecarPath: string;
+  preview: CompiledContextPreview;
+}
+const digest = (text: string): string =>
+  new Bun.CryptoHasher("sha256").update(text).digest("hex");
+const isMissing = (error: unknown): boolean =>
+  !!error &&
+  typeof error === "object" &&
+  "code" in error &&
+  error.code === "ENOENT";
+
+/** Refuse links in every existing path component; never create parent directories. */
+async function safePath(path: string, allowMissing = false): Promise<string> {
+  const absolute = resolve(path);
+  const root = parse(absolute).root;
+  const parts = absolute.slice(root.length).split(/[\\/]/).filter(Boolean);
+  let current = root;
+  for (const [index, part] of parts.entries()) {
+    current = join(current, part);
+    try {
+      const stat = await lstat(current);
+      if (
+        stat.isSymbolicLink() ||
+        (index < parts.length - 1 ? !stat.isDirectory() : !stat.isFile())
+      ) {
+        throw new Error(`Unsafe compiled-context path: ${current}`);
+      }
+    } catch (error) {
+      if (allowMissing && index === parts.length - 1 && isMissing(error))
+        return absolute;
+      throw error;
+    }
+  }
+  return absolute;
+}
+async function readBounded(path: string): Promise<string> {
+  await safePath(path);
+  const file = Bun.file(path);
+  if (file.size > MAX_BYTES)
+    throw new Error("Compiled-context input exceeds 4 MiB");
+  return file.text();
+}
+async function absent(path: string): Promise<void> {
+  await safePath(path, true);
+  try {
+    await lstat(path);
+  } catch (error) {
+    if (isMissing(error)) return;
+    throw error;
+  }
+  throw new Error(`Destination already exists: ${path}`);
+}
+function outputTarget(path: string): string {
+  if (!path.endsWith(".gno-context.md"))
+    throw new Error("Output must end in .gno-context.md");
+  return resolve(path);
+}
+async function stage(path: string, content: string): Promise<string> {
+  await safePath(path, true);
+  const temp = join(dirname(path), `.${crypto.randomUUID()}.gno-context.tmp`);
+  const handle = await open(temp, "wx", 0o600);
+  try {
+    await Bun.write(Bun.file(handle.fd), content);
+    await handle.sync();
+  } catch (error) {
+    await unlink(temp);
+    throw error;
+  } finally {
+    await handle.close();
+  }
+  return temp;
+}
+async function removeTemp(path: string): Promise<void> {
+  try {
+    await unlink(path);
+  } catch (error) {
+    if (!isMissing(error)) throw error;
+  }
+}
+function sidecar(snapshot: Snapshot, previous?: Snapshot): Sidecar {
+  return {
+    artifactKind: "gno_compiled_context_sidecar",
+    schemaVersion: "1.0",
+    rendererVersion: "1",
+    ...snapshot,
+    ...(previous ? { previous } : {}),
+  };
+}
+async function owned(
+  outputPath: string
+): Promise<{ markdown: string; snapshot: Snapshot; sidecarText: string }> {
+  const markdown = await readBounded(outputPath);
+  const sidecarText = await readBounded(`${outputPath}.json`);
+  const metadata = sidecarSchema.parse(JSON.parse(sidecarText));
+  const outputDigest = digest(markdown);
+  const snapshot =
+    metadata.outputDigest === outputDigest
+      ? snapshotSchema.parse({
+          capsulePath: metadata.capsulePath,
+          outputDigest: metadata.outputDigest,
+          settings: metadata.settings,
+        })
+      : metadata.previous;
+  if (!snapshot || snapshot.outputDigest !== outputDigest)
+    throw new Error("Conflict: compiled context was manually changed");
+  return { markdown, snapshot, sidecarText };
+}
+async function unchanged(
+  outputPath: string,
+  original: Awaited<ReturnType<typeof owned>>
+): Promise<void> {
+  if (
+    (await readBounded(outputPath)) !== original.markdown ||
+    (await readBounded(`${outputPath}.json`)) !== original.sidecarText
+  ) {
+    throw new Error("Conflict: compiled context changed during publication");
+  }
+}
+function result(
+  outputPath: string,
+  capsulePath: string,
+  preview: CompiledContextPreview,
+  status: "written" | "unchanged"
+): CompiledContextFileResult {
+  return {
+    status,
+    outputPath,
+    capsulePath,
+    sidecarPath: `${outputPath}.json`,
+    preview,
+  };
+}
+
+export async function compileContextFile(
+  input: {
+    capsulePath: string;
+    outputPath: string;
+    budgetTokens: number;
+    budgetBytes?: number;
+  },
+  deps: CompiledContextRuntimeDeps
+): Promise<CompiledContextFileResult> {
+  const outputPath = outputTarget(input.outputPath);
+  const capsulePath = await safePath(input.capsulePath);
+  await absent(outputPath);
+  await absent(`${outputPath}.json`);
+  await safePath(`${outputPath}.lock`, true);
+  return withWriteLock(`${outputPath}.lock`, async () => {
+    await absent(outputPath);
+    await absent(`${outputPath}.json`);
+    const capsuleText = await readBounded(capsulePath);
+    const capsule = JSON.parse(capsuleText);
+    const settings = {
+      budgetTokens: input.budgetTokens,
+      ...(input.budgetBytes === undefined
+        ? {}
+        : { budgetBytes: input.budgetBytes }),
+    };
+    const preview = await previewCompiledContext(
+      { capsule, ...settings },
+      deps
+    );
+    const metadata = sidecar({
+      capsulePath,
+      outputDigest: preview.digest,
+      settings,
+    });
+    const stagedOutput = await stage(outputPath, preview.markdown);
+    let stagedSidecar: string | undefined;
+    let publishedSidecar = false;
+    try {
+      stagedSidecar = await stage(
+        `${outputPath}.json`,
+        JSON.stringify(metadata)
+      );
+      const checked = await checkCompiledContext(
+        { capsule, markdown: preview.markdown },
+        deps
+      );
+      if (
+        checked.status !== "current" ||
+        (await readBounded(capsulePath)) !== capsuleText
+      )
+        throw new Error("Source or policy changed before publication");
+      await absent(outputPath);
+      await absent(`${outputPath}.json`);
+      await link(stagedSidecar, `${outputPath}.json`);
+      publishedSidecar = true;
+      await link(stagedOutput, outputPath);
+      return result(outputPath, capsulePath, preview, "written");
+    } catch (error) {
+      if (publishedSidecar) await removeTemp(`${outputPath}.json`);
+      throw error;
+    } finally {
+      await removeTemp(stagedOutput);
+      if (stagedSidecar) await removeTemp(stagedSidecar);
+    }
+  });
+}
+
+export async function checkContextFile(
+  input: { outputPath: string; capsulePath?: string },
+  deps: CompiledContextRuntimeDeps
+): Promise<CompiledContextCheck> {
+  try {
+    const outputPath = outputTarget(input.outputPath);
+    const saved = await owned(outputPath);
+    const capsule = JSON.parse(
+      await readBounded(input.capsulePath ?? saved.snapshot.capsulePath)
+    );
+    return await checkCompiledContext(
+      { capsule, markdown: saved.markdown },
+      deps
+    );
+  } catch (error) {
+    const message =
+      error instanceof Error ? error.message : "Unavailable compiled context";
+    return {
+      schemaVersion: "1.0",
+      status: message.startsWith("Conflict:") ? "conflict" : "unverifiable",
+      reasons: [message],
+      digest: null,
+      capsuleId: null,
+    };
+  }
+}
+
+export async function refreshContextFile(
+  input: { outputPath: string; capsuleOutputPath: string },
+  deps: CompiledContextRuntimeDeps,
+  build: (request: ContextCapsuleBuildInput) => Promise<ContextCapsuleV1>
+): Promise<CompiledContextFileResult> {
+  const outputPath = outputTarget(input.outputPath);
+  if (!input.capsuleOutputPath.endsWith(".gno-context.capsule.json"))
+    throw new Error("Refreshed Capsule must end in .gno-context.capsule.json");
+  const capsulePath = await safePath(input.capsuleOutputPath, true);
+  await safePath(`${outputPath}.lock`, true);
+  return withWriteLock(`${outputPath}.lock`, async () => {
+    const saved = await owned(outputPath);
+    const priorCapsuleText = await readBounded(saved.snapshot.capsulePath);
+    const priorCapsule = JSON.parse(priorCapsuleText);
+    const current = await checkCompiledContext(
+      { capsule: priorCapsule, markdown: saved.markdown },
+      deps
+    );
+    if (current.status === "current") {
+      const preview = await previewCompiledContext(
+        { capsule: priorCapsule, ...saved.snapshot.settings },
+        deps
+      );
+      await unchanged(outputPath, saved);
+      if ((await readBounded(saved.snapshot.capsulePath)) !== priorCapsuleText)
+        throw new Error("Capsule changed during refresh");
+      return result(
+        outputPath,
+        saved.snapshot.capsulePath,
+        preview,
+        "unchanged"
+      );
+    }
+    if (current.status !== "stale")
+      throw new Error(
+        `Cannot refresh ${current.status} context: ${current.reasons.join("; ")}`
+      );
+    await absent(capsulePath);
+    // Stage first so an unwritable Capsule destination fails before retrieval/publication.
+    const stagedCapsule = await stage(capsulePath, "");
+    let stagedOutput: string | undefined;
+    let stagedSidecar: string | undefined;
+    try {
+      const capsule = await build(
+        compiledContextRefreshRequest(priorCapsule, deps)
+      );
+      const preview = await previewCompiledContext(
+        { capsule, ...saved.snapshot.settings },
+        deps
+      );
+      await Bun.write(stagedCapsule, JSON.stringify(capsule));
+      stagedOutput = await stage(outputPath, preview.markdown);
+      stagedSidecar = await stage(
+        `${outputPath}.json`,
+        JSON.stringify(
+          sidecar(
+            {
+              capsulePath,
+              outputDigest: preview.digest,
+              settings: saved.snapshot.settings,
+            },
+            saved.snapshot
+          )
+        )
+      );
+      const checked = await checkCompiledContext(
+        { capsule, markdown: preview.markdown },
+        deps
+      );
+      if (checked.status !== "current")
+        throw new Error("Source or policy changed before publication");
+      await unchanged(outputPath, saved);
+      if ((await readBounded(saved.snapshot.capsulePath)) !== priorCapsuleText)
+        throw new Error("Capsule changed during refresh");
+      await absent(capsulePath);
+      await link(stagedCapsule, capsulePath);
+      // Retain one prior reference: either output remains verifiable after interruption.
+      await rename(stagedSidecar, `${outputPath}.json`);
+      await rename(stagedOutput, outputPath);
+      return result(outputPath, capsulePath, preview, "written");
+    } finally {
+      await removeTemp(stagedCapsule);
+      if (stagedOutput) await removeTemp(stagedOutput);
+      if (stagedSidecar) await removeTemp(stagedSidecar);
+    }
+  });
+}

package/src/app/compiled-context.ts

@@ -0,0 +1,240 @@
+/** Verification and caller authority shared by all compiled-context surfaces. */
+import type { ContextCapsuleV1 } from "../core/context-capsule";
+import type {
+  EgressCallerContext,
+  EgressDestinationZone,
+} from "../core/egress-policy";
+import type {
+  ContextCapsuleBuildInput,
+  ContextCapsuleRuntimeDeps,
+} from "./context-runtime";
+
+import { currentEgressSources } from "../core/collection-egress-policy-service";
+import {
+  compiledContextPreviewInputSchema,
+  compiledContextCheckInputSchema,
+  COMPILED_CONTEXT_MAX_BYTES,
+  readCompiledContextMetadata,
+  compiledContextBodyMatches,
+  renderCompiledContext,
+  type CompiledContextCheck,
+  type CompiledContextPreview,
+} from "../core/compiled-context";
+import { sha256Text } from "../core/context-capsule-validation";
+import {
+  parseCanonicalContextCapsuleForVerification,
+  rawCanonicalContextJson,
+} from "../core/context-verifier-input";
+import { evaluateEgressPolicy } from "../core/egress-policy";
+import { resolveEgressLineage } from "../core/egress-provenance";
+import {
+  verifyContextCapsuleRuntime,
+  canonicalVerifiedContextCapsuleJson,
+} from "./context-runtime";
+export interface CompiledContextRuntimeDeps extends ContextCapsuleRuntimeDeps {
+  destinationZone?: EgressDestinationZone;
+  caller?: EgressCallerContext;
+}
+class CompiledContextStateError extends Error {
+  constructor(
+    readonly state: "stale" | "unverifiable",
+    message: string
+  ) {
+    super(message);
+    this.name = "CompiledContextStateError";
+  }
+}
+function parse(
+  input: unknown,
+  deps: CompiledContextRuntimeDeps
+): ContextCapsuleV1 {
+  if (
+    new TextEncoder().encode(rawCanonicalContextJson(input)).byteLength >
+    COMPILED_CONTEXT_MAX_BYTES
+  )
+    throw new Error("Capsule exceeds 4 MiB");
+  const capsule = parseCanonicalContextCapsuleForVerification(input, deps);
+  if (capsule.schemaVersion === "1.0")
+    throw new Error(
+      "Rebuild legacy Capsule with current provenance before compiling"
+    );
+  return capsule;
+}
+function authorize(
+  capsule: ContextCapsuleV1,
+  deps: CompiledContextRuntimeDeps
+): string {
+  if (capsule.schemaVersion === "1.0")
+    throw new Error("Capsule lineage unavailable");
+  const names = [
+    ...new Set([
+      ...capsule.scope.collections,
+      ...capsule.evidence.map((item) => item.collection),
+      ...capsule.egressLineage.sources.map((item) => item.collection),
+    ]),
+  ];
+  if (
+    names.some(
+      (name) =>
+        !deps.config.collections.some((collection) => collection.name === name)
+    )
+  )
+    throw new CompiledContextStateError(
+      "unverifiable",
+      "Collection scope is no longer configured; rebuild within eligible scope"
+    );
+  const sources = currentEgressSources(deps.config, names);
+  const decision = evaluateEgressPolicy({
+    collections: sources,
+    action: "export",
+    destination: { zone: deps.destinationZone ?? "local_process" },
+    caller: deps.caller ?? { authenticated: true, operationAuthorized: true },
+    contentClass: "capsule",
+  });
+  if (!decision.allowed)
+    throw new CompiledContextStateError(
+      "unverifiable",
+      "Current collection policy or caller authority denies this context"
+    );
+  const lineage = resolveEgressLineage(sources, names);
+  if (lineage.digest !== capsule.egressLineage.digest)
+    throw new CompiledContextStateError(
+      "stale",
+      "Collection policy changed; rebuild and verify the Capsule"
+    );
+  return lineage.digest;
+}
+async function verified(
+  capsule: ContextCapsuleV1,
+  deps: CompiledContextRuntimeDeps
+): Promise<{ verificationDigest: string; lineageDigest: string }> {
+  const lineageDigest = authorize(capsule, deps);
+  const receipt = await verifyContextCapsuleRuntime(capsule, deps);
+  if (
+    receipt.contentStatus !== "unchanged" ||
+    receipt.fingerprintStatus !== "unchanged"
+  )
+    throw new CompiledContextStateError(
+      "stale",
+      `Capsule requires refresh: ${[receipt.contentCode, ...receipt.fingerprintReasons].join(", ")}`
+    );
+  authorize(capsule, deps);
+  return {
+    verificationDigest: sha256Text(
+      canonicalVerifiedContextCapsuleJson(receipt)
+    ),
+    lineageDigest,
+  };
+}
+export async function previewCompiledContext(
+  input: unknown,
+  deps: CompiledContextRuntimeDeps
+): Promise<CompiledContextPreview> {
+  const settings = compiledContextPreviewInputSchema.parse(input);
+  const capsule = parse(settings.capsule, deps);
+  const identity = await verified(capsule, deps);
+  const preview = renderCompiledContext(
+    capsule,
+    settings,
+    identity.verificationDigest,
+    identity.lineageDigest,
+    deps.countTokens
+  );
+  const finalIdentity = await verified(capsule, deps);
+  if (finalIdentity.verificationDigest !== identity.verificationDigest)
+    throw new CompiledContextStateError(
+      "stale",
+      "Index changed during compilation; retry after indexing completes"
+    );
+  return preview;
+}
+export async function checkCompiledContext(
+  input: unknown,
+  deps: CompiledContextRuntimeDeps
+): Promise<CompiledContextCheck> {
+  const result = (
+    status: CompiledContextCheck["status"],
+    reasons: string[],
+    digest: string | null = null,
+    capsuleId: string | null = null
+  ): CompiledContextCheck => ({
+    schemaVersion: "1.0",
+    status,
+    reasons,
+    digest,
+    capsuleId,
+  });
+  try {
+    const { capsule: raw, markdown } =
+      compiledContextCheckInputSchema.parse(input);
+    const capsule = parse(raw, deps);
+    // Authorize before reporting identifiers from supplied private evidence.
+    authorize(capsule, deps);
+    const meta = readCompiledContextMetadata(markdown);
+    if (!compiledContextBodyMatches(markdown))
+      return result("conflict", [
+        "Output bytes were edited; preserve edits before recompiling",
+      ]);
+    const expected = await previewCompiledContext(
+      {
+        capsule,
+        budgetTokens: meta.budgetTokens,
+        budgetBytes: meta.budgetBytes,
+      },
+      deps
+    );
+    if (expected.markdown !== markdown)
+      return result("conflict", [
+        "Artifact metadata, renderer settings, or Capsule identity differs",
+      ]);
+    return result("current", [], expected.digest, capsule.capsuleId);
+  } catch (error) {
+    if (error instanceof CompiledContextStateError)
+      return result(error.state, [error.message]);
+    // Do not echo parser paths/values or supplied private source material.
+    return result("unverifiable", [
+      "Invalid or unavailable Capsule, artifact metadata, index, or recorded tokenizer; rebuild or supply the original matching inputs",
+    ]);
+  }
+}
+export function compiledContextRefreshRequest(
+  input: unknown,
+  authority: Pick<
+    CompiledContextRuntimeDeps,
+    "countTokens" | "tokenizerFingerprint"
+  > = {}
+): ContextCapsuleBuildInput {
+  // Refresh cannot trust stale fingerprints, but must preserve the saved request exactly.
+  const capsule = parseCanonicalContextCapsuleForVerification(input, authority);
+  if (capsule.schemaVersion === "1.0")
+    throw new Error("Rebuild legacy Capsule before refresh");
+  const request = capsule.retrieval.request;
+  return {
+    goal: capsule.goal,
+    query: capsule.query,
+    indexName: capsule.scope.indexName,
+    collections: capsule.scope.collections,
+    uriPrefix: capsule.scope.uriPrefix,
+    tagsAll: capsule.scope.tagsAll,
+    tagsAny: capsule.scope.tagsAny,
+    categories: capsule.scope.categories,
+    since: capsule.scope.since ?? undefined,
+    until: capsule.scope.until ?? undefined,
+    filter: "filter" in capsule.scope ? capsule.scope.filter : undefined,
+    author: request.author ?? undefined,
+    lang: request.lang ?? undefined,
+    intent: request.intent ?? undefined,
+    exclude: request.exclude,
+    minScore: request.minScore ?? undefined,
+    queryModes: request.queryModes,
+    limit: request.limit,
+    candidateLimit: request.candidateLimit,
+    graph: request.graphRequested,
+    noRerank: request.rerankRequested === false,
+    depthPolicy: capsule.retrieval.depthPolicy,
+    budgetTokens: capsule.budget.requestedTokens,
+    budgetBytes: capsule.budget.requestedBytes,
+    safetyMarginTokens: capsule.budget.safetyMarginTokens,
+    safetyMarginBytes: capsule.budget.safetyMarginBytes,
+  };
+}

package/src/app/context-surface.ts

@@ -5,7 +5,10 @@ import { z } from "zod";
 import type { ContextCapsuleBuildInput } from "./context-runtime-types";
 
 import { ContextCapsuleContractError } from "../core/context-capsule";
-import { metadataPredicateSchema } from "../core/typed-metadata";
+import {
+  metadataPredicateSchema,
+  METADATA_FILTER_DESCRIPTION,
+} from "../core/typed-metadata";
 
 const queryModeSchema = z
   .object({
@@ -29,7 +32,9 @@ export const contextBuildSurfaceSchema = z
     tagsAll: stringList.optional(),
     tagsAny: stringList.optional(),
     categories: stringList.optional(),
-    filter: metadataPredicateSchema.optional(),
+    filter: metadataPredicateSchema
+      .optional()
+      .describe(METADATA_FILTER_DESCRIPTION),
     author: z.string().optional(),
     lang: z.string().optional(),
     intent: z.string().optional(),

package/src/cli/commands/ask.ts

@@ -15,6 +15,7 @@ import type {
 } from "../../llm/types";
 import type { AskOptions, AskResult, Citation } from "../../pipeline/types";
 
+import { decorateUriForIndex } from "../../app/constants";
 import { buildVerifiedAsk } from "../../app/verified-ask";
 import { resolveCliProjectAffinity } from "../../core/project-affinity-surface";
 import {
@@ -96,7 +97,7 @@ export async function ask(
   const globals = getGlobals();
 
   const initResult = await initStore({
-    configPath: options.configPath,
+    configPath: options.configPath ?? globals.config,
     indexName: globals.index,
     collection: options.collection,
     syncConfig: false,
@@ -434,9 +435,20 @@ export async function ask(
         return { success: false, error: finalized.error.message };
       }
     }
+    // URIs carry `?index=` like search and Capsule results on a named index.
     return {
       success: true,
-      data: askResult,
+      data: {
+        ...askResult,
+        results: askResult.results.map((result) => ({
+          ...result,
+          uri: decorateUriForIndex(result.uri, globals.index),
+        })),
+        citations: askResult.citations?.map((citation) => ({
+          ...citation,
+          uri: decorateUriForIndex(citation.uri, globals.index),
+        })),
+      },
       metadata: traceSession?.metadata(),
     };
   } catch (cause) {

package/src/cli/commands/audit.ts

@@ -21,6 +21,7 @@ import {
 import { runWorkspaceAudit } from "../../core/audit-workspace";
 import { normalizeTag, validateTag } from "../../core/tags";
 import { normalizeCollectionName } from "../../core/validation";
+import { windowsPrivatePath } from "../../core/windows-private-path";
 import { SqliteAdapter } from "../../store/sqlite/adapter";
 
 export interface AuditCommandOptions {
@@ -218,12 +219,15 @@ export const writeAuditReport = async (
   );
   const temporaryPath = join(temporaryDirectory, "report");
   try {
+    if (process.platform === "win32")
+      await windowsPrivatePath(temporaryDirectory, true);
     await Bun.write(temporaryPath, `${formatAuditReport(report, options)}\n`, {
       createPath: false,
       mode: 0o600,
     });
     await chmod(temporaryPath, 0o600);
     await rename(temporaryPath, path);
+    if (process.platform === "win32") await windowsPrivatePath(path);
   } finally {
     await unlink(temporaryPath).catch(() => undefined);
     await rmdir(temporaryDirectory).catch(() => undefined);