@gmickel/gno 1.12.3 → 1.13.0

package/src/serve/status.ts

@@ -13,9 +13,18 @@ import type {
 } from "./status-model";
 
 import { getModelsCachePath } from "../app/constants";
+import {
+  type ActivationStatus,
+  buildActivationStatus,
+} from "../core/activation-status";
 import { ModelCache } from "../llm/cache";
 import { envIsSet, resolveDownloadPolicy } from "../llm/policy";
 import { getActivePreset, resolveModelUri } from "../llm/registry";
+import {
+  buildActivationCheck,
+  buildConnectorActivationCheck,
+} from "./activation-health";
+import { getConnectorVerificationTargets } from "./connectors";
 import { downloadState, type ServerContext } from "./context";
 
 const GIGABYTE = 1024 * 1024 * 1024;
@@ -60,6 +69,8 @@ export interface StatusBuildDeps {
   inspectDisk?: (path: string) => Promise<DiskSnapshot | null>;
   isModelCached?: (uri: string) => Promise<boolean>;
   listSuggestedCollections?: () => Promise<SuggestedCollection[]>;
+  buildActivation?: typeof buildActivationStatus;
+  listConnectorTargets?: typeof getConnectorVerificationTargets;
 }
 
 function formatBytes(bytes: number): string {
@@ -490,6 +501,7 @@ async function buildBootstrapState(
 
   const cachedCount = modelEntries.filter((entry) => entry.cached).length;
   const totalCount = modelEntries.length;
+  const totalSizeBytes = await cache.totalSize();
 
   const policySummary = policy.offline
     ? "Offline mode. Cached models only."
@@ -517,8 +529,8 @@ async function buildBootstrapState(
     },
     cache: {
       path: cache.dir,
-      totalSizeBytes: await cache.totalSize(),
-      totalSizeLabel: formatBytes(await cache.totalSize()),
+      totalSizeBytes,
+      totalSizeLabel: formatBytes(totalSizeBytes),
     },
     models: {
       activePresetId: preset.id,
@@ -540,12 +552,23 @@ function buildOnboarding(
   status: IndexStatus,
   modelCheck: HealthCheck,
   suggestions: SuggestedCollection[],
-  presetName: string
+  presetName: string,
+  activation: ActivationStatus
 ): AppStatusResponse["onboarding"] {
-  const foldersReady = status.collections.length > 0;
+  const foldersReady = activation.collections.length > 0;
   const modelsReady = modelCheck.status === "ok";
-  const indexedReady =
-    status.activeDocuments > 0 && status.embeddingBacklog === 0;
+  const indexedReady = activation.healthy;
+  const semanticStates = [
+    ...new Set(
+      activation.collections.map(
+        ({ semanticAvailability }) => semanticAvailability.code
+      )
+    ),
+  ];
+  const failedActivation = activation.collections.find(({ ready }) => !ready);
+  const activationDetail = failedActivation?.remediation
+    ? `${failedActivation.collection}: ${failedActivation.remediation.stage}/${failedActivation.remediation.code}. Run: ${failedActivation.remediation.command}`
+    : "Run the first sync to populate a searchable lexical index.";
 
   const steps = [
     {
@@ -555,6 +578,7 @@ function buildOnboarding(
       detail: foldersReady
         ? `${summarizeCount(status.collections.length, "folder")} connected.`
         : "Choose the folders you want GNO to watch and index.",
+      action: "add-collection",
     },
     {
       id: "preset",
@@ -567,21 +591,20 @@ function buildOnboarding(
           {
             id: "models",
             title: "Prepare local models",
-            status: foldersReady ? "current" : "upcoming",
+            status: "upcoming",
             detail: modelCheck.detail,
+            action: "download-models",
           },
         ] satisfies AppStatusResponse["onboarding"]["steps"])
       : []),
     {
       id: "indexing",
-      title: "Finish first index",
+      title: "Prove lexical retrieval",
       status: indexedReady ? "complete" : foldersReady ? "current" : "upcoming",
-      detail:
-        status.activeDocuments > 0 && status.embeddingBacklog > 0
-          ? `${summarizeCount(status.embeddingBacklog, "chunk")} still waiting on embeddings.`
-          : status.activeDocuments > 0
-            ? `${summarizeCount(status.activeDocuments, "document")} indexed so far.`
-            : "Run the first sync to scan your folders and build the search index.",
+      detail: indexedReady
+        ? `${summarizeCount(activation.collections.length, "folder")} passed a corpus-derived lexical proof.`
+        : activationDetail,
+      action: "sync",
     },
   ] satisfies AppStatusResponse["onboarding"]["steps"];
 
@@ -597,28 +620,14 @@ function buildOnboarding(
     };
   }
 
-  if (!modelsReady) {
-    return {
-      ready: false,
-      stage: "models",
-      headline: "Your folders are connected. Finish model setup next",
-      detail: modelCheck.detail,
-      suggestedCollections: suggestions,
-      steps,
-    };
-  }
-
   if (!indexedReady) {
     return {
       ready: false,
       stage: "indexing",
-      headline: "GNO is almost ready. Finish the first indexing run",
-      detail:
-        status.activeDocuments > 0 && status.embeddingBacklog > 0
-          ? `Finish embeddings for ${summarizeCount(status.embeddingBacklog, "chunk")} to unlock semantic search and local answers.`
-          : status.activeDocuments > 0
-            ? "The first sync started. Let embeddings finish so semantic search and answers can fully light up."
-            : "Run the first sync to populate the index from the folders you connected.",
+      headline: activation.usable
+        ? "Search works in some folders. Repair the remaining retrieval proof"
+        : "Finish the first lexical retrieval proof",
+      detail: activationDetail,
       suggestedCollections: suggestions,
       steps,
     };
@@ -628,7 +637,9 @@ function buildOnboarding(
     ready: true,
     stage: "ready",
     headline: "Workspace ready",
-    detail: "Your folders, local models, and first index are all in place.",
+    detail: modelsReady
+      ? `Every folder passed lexical retrieval. Local model files are cached; semantic retrieval remains separate (${semanticStates.join(", ")}).`
+      : `Every folder passed lexical retrieval. Semantic retrieval remains an optional next step (${semanticStates.join(", ")}).`,
     suggestedCollections: suggestions,
     steps,
   };
@@ -647,17 +658,43 @@ export async function buildAppStatus(
 
   const status = result.value;
   const preset = getActivePreset(ctx.config);
-  const [modelCheck, diskCheck, suggestions, bootstrap] = await Promise.all([
+  const cache = new ModelCache(getModelsCachePath());
+  const isModelCached =
+    deps.isModelCached ?? ((uri: string) => cache.isCached(uri));
+  const [
+    modelCheck,
+    diskCheck,
+    suggestions,
+    bootstrap,
+    embedCached,
+    connectorTargets,
+  ] = await Promise.all([
     buildModelCheck(ctx, deps),
     buildDiskCheck(status, deps),
     deps.listSuggestedCollections?.() ?? listSuggestedCollections(),
     buildBootstrapState(ctx),
+    isModelCached(preset.embed),
+    (deps.listConnectorTargets ?? getConnectorVerificationTargets)(),
   ]);
+  const activation = await (deps.buildActivation ?? buildActivationStatus)(
+    ctx.store,
+    ctx.config.collections.map(({ name }) => name),
+    {
+      semantic: {
+        modelsCached: embedCached,
+        embeddingBacklog: status.embeddingBacklog,
+        vectorAvailable: ctx.capabilities.vector,
+      },
+      connectorTargets,
+    }
+  );
 
   const backgroundCheck = buildBackgroundCheck(ctx, status);
   const checks = [
     buildCollectionCheck(status),
     buildIndexingCheck(status),
+    buildActivationCheck(activation),
+    buildConnectorActivationCheck(activation),
     modelCheck,
     buildVectorCheck(ctx),
     diskCheck,
@@ -698,7 +735,14 @@ export async function buildAppStatus(
       name: preset.name,
     },
     capabilities: ctx.capabilities,
-    onboarding: buildOnboarding(status, modelCheck, suggestions, preset.name),
+    activation,
+    onboarding: buildOnboarding(
+      status,
+      modelCheck,
+      suggestions,
+      preset.name,
+      activation
+    ),
     health: {
       state: healthState,
       summary: healthSummary,

package/src/store/activation-receipts.ts

@@ -0,0 +1,390 @@
+import type {
+  ActivationStageReceipt,
+  ActivationVerificationReceipt,
+} from "./types";
+
+const ACTIVATION_RECEIPT_MAX_BYTES = 16_384;
+const HASH_PATTERN = /^[a-f0-9]{64}$/;
+const RESULT_URI_PATTERN = /^gno:\/\/[^/]+\/.+/;
+const CONNECTOR_TARGET_PATTERN =
+  /^(?:mcp|skill):[a-z0-9][a-z0-9._-]{0,63}:(?:user|project):[a-f0-9]{64}$/;
+const DATE_TIME_PATTERN =
+  /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|[+-](\d{2}):(\d{2}))$/;
+const INDEX_FAILURE_CODES = new Set(["no_documents", "index_out_of_sync"]);
+const LEXICAL_FAILURE_CODES = new Set([
+  "no_probe_term",
+  "index_query_failed",
+  "retrieval_mismatch",
+]);
+const CONNECTOR_SKIPPED_CODES = new Set([
+  "connector_not_configured",
+  "connector_probe_unavailable",
+  "target_runtime_unverifiable",
+]);
+const CONNECTOR_FAILURE_CODES = new Set([
+  "connector_probe_unavailable",
+  "connector_unsupported_config",
+  "connector_start_failed",
+  "connector_timeout",
+  "connector_missing_tools",
+  "connector_status_failed",
+  "connector_search_failed",
+  "connector_result_mismatch",
+]);
+
+function projectStage(stage: ActivationStageReceipt): ActivationStageReceipt {
+  return {
+    status: stage.status,
+    startedAt: stage.startedAt,
+    completedAt: stage.completedAt,
+    latencyMs: stage.latencyMs,
+    ...(stage.code ? { code: stage.code } : {}),
+  };
+}
+
+function projectReceipt(
+  receipt: ActivationVerificationReceipt
+): ActivationVerificationReceipt {
+  return {
+    schemaVersion: "1.0",
+    collection: receipt.collection,
+    fingerprint: receipt.fingerprint,
+    ready: receipt.ready,
+    generatedAt: receipt.generatedAt,
+    stages: {
+      index: projectStage(receipt.stages.index),
+      lexical: projectStage(receipt.stages.lexical),
+      semantic: projectStage(receipt.stages.semantic),
+      connector: projectStage(receipt.stages.connector),
+    },
+    evidence: {
+      ...(receipt.evidence.probeHash
+        ? { probeHash: receipt.evidence.probeHash }
+        : {}),
+      ...(receipt.evidence.resultUri
+        ? { resultUri: receipt.evidence.resultUri }
+        : {}),
+      ...(receipt.evidence.resultSourceHash
+        ? { resultSourceHash: receipt.evidence.resultSourceHash }
+        : {}),
+      ...(receipt.evidence.connectorTarget
+        ? { connectorTarget: receipt.evidence.connectorTarget }
+        : {}),
+    },
+  };
+}
+
+function hasOnlyKeys(value: Record<string, unknown>, allowed: Set<string>) {
+  return Object.keys(value).every((key) => allowed.has(key));
+}
+
+function isDateTime(value: unknown): value is string {
+  if (typeof value !== "string") {
+    return false;
+  }
+  const match = DATE_TIME_PATTERN.exec(value);
+  if (!match) {
+    return false;
+  }
+  const [, yearText, monthText, dayText, hourText, minuteText, secondText] =
+    match;
+  const year = Number(yearText);
+  const month = Number(monthText);
+  const day = Number(dayText);
+  const hour = Number(hourText);
+  const minute = Number(minuteText);
+  const second = Number(secondText);
+  const offsetHour = Number(match[7] ?? 0);
+  const offsetMinute = Number(match[8] ?? 0);
+  const leapYear = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
+  const daysInMonth = [
+    31,
+    leapYear ? 29 : 28,
+    31,
+    30,
+    31,
+    30,
+    31,
+    31,
+    30,
+    31,
+    30,
+    31,
+  ];
+  return (
+    month >= 1 &&
+    month <= 12 &&
+    day >= 1 &&
+    day <= (daysInMonth[month - 1] ?? 0) &&
+    hour <= 23 &&
+    minute <= 59 &&
+    second <= 59 &&
+    offsetHour <= 23 &&
+    offsetMinute <= 59
+  );
+}
+
+function isNullableDateTime(value: unknown): value is string | null {
+  return value === null || isDateTime(value);
+}
+
+function isStageShape(value: unknown): value is ActivationStageReceipt {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+  const stage = value as Record<string, unknown>;
+  return (
+    hasOnlyKeys(
+      stage,
+      new Set(["status", "startedAt", "completedAt", "latencyMs", "code"])
+    ) &&
+    typeof stage.status === "string" &&
+    isNullableDateTime(stage.startedAt) &&
+    isNullableDateTime(stage.completedAt) &&
+    (stage.latencyMs === null ||
+      (typeof stage.latencyMs === "number" &&
+        Number.isInteger(stage.latencyMs) &&
+        stage.latencyMs >= 0)) &&
+    (stage.code === undefined || typeof stage.code === "string")
+  );
+}
+
+function isTimedStage(stage: ActivationStageReceipt): boolean {
+  return (
+    isDateTime(stage.startedAt) &&
+    isDateTime(stage.completedAt) &&
+    typeof stage.latencyMs === "number"
+  );
+}
+
+function isIndexStage(stage: ActivationStageReceipt): boolean {
+  if (!isTimedStage(stage)) {
+    return false;
+  }
+  return (
+    (stage.status === "passed" && stage.code === undefined) ||
+    (stage.status === "failed" &&
+      stage.code !== undefined &&
+      INDEX_FAILURE_CODES.has(stage.code))
+  );
+}
+
+function isLexicalStage(stage: ActivationStageReceipt): boolean {
+  if (stage.status === "passed") {
+    return isTimedStage(stage) && stage.code === undefined;
+  }
+  if (stage.status === "failed") {
+    return (
+      isTimedStage(stage) &&
+      stage.code !== undefined &&
+      LEXICAL_FAILURE_CODES.has(stage.code)
+    );
+  }
+  return (
+    stage.status === "skipped" &&
+    stage.startedAt === null &&
+    isDateTime(stage.completedAt) &&
+    stage.latencyMs === null &&
+    stage.code !== undefined &&
+    INDEX_FAILURE_CODES.has(stage.code)
+  );
+}
+
+function isSemanticStage(stage: ActivationStageReceipt): boolean {
+  return (
+    stage.status === "pending" &&
+    stage.startedAt === null &&
+    stage.completedAt === null &&
+    stage.latencyMs === null &&
+    stage.code === "semantic_not_checked"
+  );
+}
+
+function isConnectorStage(stage: ActivationStageReceipt): boolean {
+  if (stage.status === "passed") {
+    return isTimedStage(stage) && stage.code === undefined;
+  }
+  if (stage.status === "failed") {
+    return (
+      isTimedStage(stage) &&
+      stage.code !== undefined &&
+      CONNECTOR_FAILURE_CODES.has(stage.code)
+    );
+  }
+  if (stage.status !== "skipped") {
+    return false;
+  }
+  if (stage.code === "connector_not_requested") {
+    return (
+      stage.startedAt === null &&
+      stage.completedAt === null &&
+      stage.latencyMs === null
+    );
+  }
+  return (
+    isTimedStage(stage) &&
+    stage.code !== undefined &&
+    CONNECTOR_SKIPPED_CODES.has(stage.code)
+  );
+}
+
+function isReceipt(value: unknown): value is ActivationVerificationReceipt {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+  const receipt = value as Record<string, unknown>;
+  const stages = receipt.stages as Record<string, unknown> | undefined;
+  const evidence = receipt.evidence as Record<string, unknown> | undefined;
+  if (
+    !hasOnlyKeys(
+      receipt,
+      new Set([
+        "schemaVersion",
+        "collection",
+        "fingerprint",
+        "ready",
+        "generatedAt",
+        "stages",
+        "evidence",
+      ])
+    ) ||
+    receipt.schemaVersion !== "1.0" ||
+    typeof receipt.collection !== "string" ||
+    receipt.collection.length < 1 ||
+    receipt.collection.length > 128 ||
+    typeof receipt.fingerprint !== "string" ||
+    !HASH_PATTERN.test(receipt.fingerprint) ||
+    typeof receipt.ready !== "boolean" ||
+    !isDateTime(receipt.generatedAt) ||
+    !stages ||
+    !hasOnlyKeys(
+      stages,
+      new Set(["index", "lexical", "semantic", "connector"])
+    ) ||
+    !isStageShape(stages.index) ||
+    !isStageShape(stages.lexical) ||
+    !isStageShape(stages.semantic) ||
+    !isStageShape(stages.connector) ||
+    !evidence ||
+    !hasOnlyKeys(
+      evidence,
+      new Set(["probeHash", "resultUri", "resultSourceHash", "connectorTarget"])
+    )
+  ) {
+    return false;
+  }
+
+  const indexStage = stages.index as ActivationStageReceipt;
+  const lexicalStage = stages.lexical as ActivationStageReceipt;
+  const semanticStage = stages.semantic as ActivationStageReceipt;
+  const connectorStage = stages.connector as ActivationStageReceipt;
+  if (
+    !isIndexStage(indexStage) ||
+    !isLexicalStage(lexicalStage) ||
+    !isSemanticStage(semanticStage) ||
+    !isConnectorStage(connectorStage)
+  ) {
+    return false;
+  }
+  const expectedReady =
+    indexStage.status === "passed" && lexicalStage.status === "passed";
+  if (receipt.ready !== expectedReady) {
+    return false;
+  }
+  const connectorCode = connectorStage.code;
+  const connectorNeedsLexicalProof =
+    connectorStage.status === "passed" ||
+    (connectorStage.status === "failed" &&
+      connectorCode !== "connector_unsupported_config");
+  const unavailableBeforeConnectorProbe =
+    connectorStage.status === "skipped" &&
+    connectorCode === "connector_probe_unavailable";
+  if (
+    (connectorNeedsLexicalProof && !expectedReady) ||
+    (unavailableBeforeConnectorProbe && expectedReady)
+  ) {
+    return false;
+  }
+
+  const validEvidence =
+    (evidence.probeHash === undefined ||
+      (typeof evidence.probeHash === "string" &&
+        HASH_PATTERN.test(evidence.probeHash))) &&
+    (evidence.resultUri === undefined ||
+      (typeof evidence.resultUri === "string" &&
+        evidence.resultUri.length <= 2048 &&
+        RESULT_URI_PATTERN.test(evidence.resultUri))) &&
+    (evidence.resultSourceHash === undefined ||
+      (typeof evidence.resultSourceHash === "string" &&
+        HASH_PATTERN.test(evidence.resultSourceHash))) &&
+    (evidence.connectorTarget === undefined ||
+      (typeof evidence.connectorTarget === "string" &&
+        CONNECTOR_TARGET_PATTERN.test(evidence.connectorTarget)));
+  if (!validEvidence) {
+    return false;
+  }
+
+  const connectorWasRequested =
+    connectorStage.code !== "connector_not_requested";
+  if (
+    connectorWasRequested !==
+    (typeof evidence.connectorTarget === "string")
+  ) {
+    return false;
+  }
+
+  const hasProbeHash = typeof evidence.probeHash === "string";
+  const hasResultUri = typeof evidence.resultUri === "string";
+  const hasResultSourceHash = typeof evidence.resultSourceHash === "string";
+  if (expectedReady) {
+    return hasProbeHash && hasResultUri && hasResultSourceHash;
+  }
+  if (lexicalStage.code === "retrieval_mismatch") {
+    return hasProbeHash && !hasResultUri && !hasResultSourceHash;
+  }
+  return !hasProbeHash && !hasResultUri && !hasResultSourceHash;
+}
+
+export function parseActivationReceipt(
+  raw: string
+): ActivationVerificationReceipt | null {
+  try {
+    const parsed: unknown = JSON.parse(raw);
+    return isReceipt(parsed) ? projectReceipt(parsed) : null;
+  } catch {
+    return null;
+  }
+}
+
+export function serializeActivationReceipt(
+  receipt: ActivationVerificationReceipt
+):
+  | {
+      ok: true;
+      json: string;
+      projected: ActivationVerificationReceipt;
+      connectorTarget: string;
+    }
+  | { ok: false; error: string } {
+  let projected: ActivationVerificationReceipt;
+  try {
+    projected = projectReceipt(receipt);
+  } catch {
+    return { ok: false, error: "Activation receipt is schema-invalid" };
+  }
+  if (!isReceipt(projected)) {
+    return { ok: false, error: "Activation receipt is schema-invalid" };
+  }
+  const json = JSON.stringify(projected);
+  if (
+    new TextEncoder().encode(json).byteLength > ACTIVATION_RECEIPT_MAX_BYTES
+  ) {
+    return { ok: false, error: "Activation receipt exceeds 16 KiB" };
+  }
+  return {
+    ok: true,
+    json,
+    projected,
+    connectorTarget: projected.evidence.connectorTarget ?? "",
+  };
+}

package/src/store/index.ts

@@ -19,6 +19,14 @@ export {
 export { SqliteAdapter } from "./sqlite";
 // Types and interfaces
 export type {
+  ActivationIndexDocument,
+  ActivationIndexIdentity,
+  ActivationIndexSnapshot,
+  ActivationStageName,
+  ActivationStageReceipt,
+  ActivationStageStatus,
+  ActivationVerificationCode,
+  ActivationVerificationReceipt,
   ChunkInput,
   ChunkRow,
   CleanupStats,

package/src/store/migrations/012-activation-receipts.ts

@@ -0,0 +1,38 @@
+/**
+ * Migration: privacy-bounded retrieval activation receipts.
+ *
+ * @module src/store/migrations/012-activation-receipts
+ */
+
+import type { Database } from "bun:sqlite";
+
+import type { Migration } from "./runner";
+
+export const migration: Migration = {
+  version: 12,
+  name: "activation_receipts",
+
+  up(db: Database): void {
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS activation_receipts (
+        collection TEXT NOT NULL,
+        connector_target TEXT NOT NULL DEFAULT '',
+        schema_version TEXT NOT NULL,
+        fingerprint TEXT NOT NULL,
+        receipt_json TEXT NOT NULL CHECK (length(receipt_json) <= 16384),
+        updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+        PRIMARY KEY (collection, connector_target),
+        FOREIGN KEY (collection) REFERENCES collections(name) ON DELETE CASCADE
+      )
+    `);
+    db.exec(`
+      CREATE INDEX IF NOT EXISTS idx_activation_receipts_fingerprint
+      ON activation_receipts(fingerprint)
+    `);
+  },
+
+  down(db: Database): void {
+    db.exec("DROP INDEX IF EXISTS idx_activation_receipts_fingerprint");
+    db.exec("DROP TABLE IF EXISTS activation_receipts");
+  },
+};

package/src/store/migrations/013-fts-sync-marker.ts

@@ -0,0 +1,39 @@
+/**
+ * Migration: metadata-only FTS synchronization marker.
+ *
+ * The marker records which mirror hash was transactionally written through the
+ * supported FTS writers. This one-time migration compares legacy FTS bodies
+ * before trusting them; passive activation checks remain metadata-only.
+ */
+
+import type { Database } from "bun:sqlite";
+
+import type { Migration } from "./runner";
+
+export const migration: Migration = {
+  version: 13,
+  name: "fts_sync_marker",
+
+  up(db: Database): void {
+    db.exec("ALTER TABLE documents ADD COLUMN fts_mirror_hash TEXT");
+    db.exec(`
+      UPDATE documents
+      SET fts_mirror_hash = mirror_hash
+      WHERE active = 1
+        AND mirror_hash IS NOT NULL
+        AND EXISTS (
+          SELECT 1
+          FROM documents_fts f
+          JOIN content c ON c.mirror_hash = documents.mirror_hash
+          WHERE f.rowid = documents.id
+            AND f.filepath = documents.rel_path
+            AND f.title = COALESCE(documents.title, '')
+            AND f.body = c.markdown
+        )
+    `);
+  },
+
+  down(db: Database): void {
+    db.exec("ALTER TABLE documents DROP COLUMN fts_mirror_hash");
+  },
+};