@gmickel/gno 1.17.0 → 1.19.0

package/src/serve/routes/api.ts

@@ -16,6 +16,7 @@ import type {
   Config,
   ModelPreset,
 } from "../../config/types";
+import type { JobManager } from "../../core/job-manager";
 import type {
   AskResult,
   Citation,
@@ -32,6 +33,7 @@ import type {
 import type { DocumentEventBus } from "../doc-events";
 import type { EmbedScheduler } from "../embed-scheduler";
 import type { StartJobError } from "../jobs";
+import type { ResidentStatus } from "../status-model";
 import type { CollectionWatchService } from "../watch-service";
 
 import { modelsPull } from "../../cli/commands/models/pull";
@@ -76,6 +78,11 @@ import {
   planMoveRefactor,
   planRenameRefactor,
 } from "../../core/file-refactors";
+import {
+  hasContentMutation,
+  recordContentMutation,
+  recordIndexMutation,
+} from "../../core/mutation-generations";
 import {
   resolveNoteCreatePlan,
   sanitizeNoteFilename,
@@ -87,6 +94,14 @@ import {
   resolveNotePreset,
   type NotePresetId,
 } from "../../core/note-presets";
+import {
+  retrievalTraceFilters,
+  startRetrievalTraceRequest,
+} from "../../core/retrieval-trace-request";
+import {
+  evidenceFromExactDocument,
+  RetrievalTraceSession,
+} from "../../core/retrieval-trace-session";
 import { extractSections } from "../../core/sections";
 import { normalizeStructuredQueryInput } from "../../core/structured-query";
 import {
@@ -111,7 +126,7 @@ import {
 } from "../../llm/registry";
 import {
   generateGroundedAnswer,
-  processAnswerResult,
+  processAnswerResultWithTrace,
 } from "../../pipeline/answer";
 import { diagnoseQueryTarget } from "../../pipeline/diagnose";
 import { searchHybrid } from "../../pipeline/hybrid";
@@ -138,6 +153,10 @@ import {
 } from "../context";
 import { analyzeImportPath } from "../import-preview";
 import { getActiveJob, getJobStatus, startJob } from "../jobs";
+import {
+  requestRetrievalTraceId,
+  withRetrievalTraceHeader,
+} from "../retrieval-trace";
 import { buildAppStatus, type StatusBuildDeps } from "../status";
 
 /** Mutable context holder for hot-reloading presets */
@@ -147,6 +166,26 @@ export interface ContextHolder {
   scheduler: EmbedScheduler | null;
   eventBus: DocumentEventBus | null;
   watchService: CollectionWatchService | null;
+  jobManager?: JobManager;
+  markContentMutation?: () => void;
+  markIndexMutation?: () => void;
+  startBackgroundWork?: (
+    operation: (signal: AbortSignal) => Promise<void>
+  ) => boolean;
+}
+
+async function syncResidentCollection(
+  ctxHolder: ContextHolder,
+  collection: Collection,
+  store: SqliteAdapter,
+  options: Parameters<typeof defaultSyncService.syncCollection>[2],
+  syncCollection: typeof defaultSyncService.syncCollection = defaultSyncService.syncCollection.bind(
+    defaultSyncService
+  )
+): ReturnType<typeof defaultSyncService.syncCollection> {
+  const result = await syncCollection(collection, store, options);
+  recordContentMutation(result, ctxHolder.markContentMutation);
+  return result;
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -433,6 +472,65 @@ function errorResponse(
   );
 }
 
+interface RestTraceStart {
+  session: RetrievalTraceSession | null;
+  error: Response | null;
+}
+
+async function startRestTrace(
+  ctx: ServerContext,
+  input: {
+    query: string;
+    goal?: string;
+    filters?: Record<string, unknown>;
+    pipeline: string;
+    modelUris?: string[];
+  }
+): Promise<RestTraceStart> {
+  const started = await startRetrievalTraceRequest({
+    store: ctx.store,
+    config: ctx.config,
+    query: input.query,
+    goal: input.goal,
+    filters: input.filters,
+    pipeline: input.pipeline,
+    indexName: ctx.indexName,
+    modelUris: input.modelUris,
+  });
+  return started.ok
+    ? { session: started.value, error: null }
+    : {
+        session: null,
+        error: errorResponse(
+          "RUNTIME",
+          `Retrieval trace start failed: ${started.error.message}`,
+          500
+        ),
+      };
+}
+
+async function finishRestTrace(
+  request: Request,
+  session: RetrievalTraceSession | null,
+  status: "completed" | "partial" | "failed",
+  response: Response
+): Promise<Response> {
+  if (!session) return response;
+  const terminalStatus = request.signal.aborted ? "cancelled" : status;
+  const finished = await session.finish(terminalStatus);
+  if (!finished.ok) {
+    return withRetrievalTraceHeader(
+      errorResponse(
+        "RUNTIME",
+        `Retrieval trace finalization failed: ${finished.error.message}`,
+        500
+      ),
+      session
+    );
+  }
+  return withRetrievalTraceHeader(response, session);
+}
+
 function jobConflictResponse(jobResult: StartJobError): Response {
   return errorResponse("CONFLICT", jobResult.error, 409, {
     activeJobId: jobResult.activeJobId,
@@ -815,6 +913,12 @@ export async function handleStatus(
   }
 }
 
+export function handleResidentStatus(
+  getStatus: () => ResidentStatus
+): Response {
+  return jsonResponse(getStatus());
+}
+
 /**
  * GET /api/collections
  * Returns list of collections.
@@ -990,33 +1094,38 @@ export async function handleCreateCollection(
   if (!collection) {
     return errorResponse("RUNTIME", "Collection not found after add", 500);
   }
-  const jobResult = startJob("add", async (): Promise<SyncResult> => {
-    const result = await defaultSyncService.syncCollection(
-      collection,
-      store,
-      withContentTypeRules(
-        {
-          gitPull: body.gitPull,
-          runUpdateCmd: true,
-        },
-        syncResult.config
-      )
-    );
-    if (result.filesAdded > 0 || result.filesUpdated > 0) {
-      if (ctxHolder.scheduler) {
-        await ctxHolder.scheduler.triggerNow();
+  const jobResult = await startJob(
+    "add",
+    async (): Promise<SyncResult> => {
+      const result = await syncResidentCollection(
+        ctxHolder,
+        collection,
+        store,
+        withContentTypeRules(
+          {
+            gitPull: body.gitPull,
+            runUpdateCmd: true,
+          },
+          syncResult.config
+        )
+      );
+      if (result.filesAdded > 0 || result.filesUpdated > 0) {
+        if (ctxHolder.scheduler) {
+          await ctxHolder.scheduler.triggerNow();
+        }
       }
-    }
-    return {
-      collections: [result],
-      totalDurationMs: result.durationMs,
-      totalFilesProcessed: result.filesProcessed,
-      totalFilesAdded: result.filesAdded,
-      totalFilesUpdated: result.filesUpdated,
-      totalFilesErrored: result.filesErrored,
-      totalFilesSkipped: result.filesSkipped,
-    };
-  });
+      return {
+        collections: [result],
+        totalDurationMs: result.durationMs,
+        totalFilesProcessed: result.filesProcessed,
+        totalFilesAdded: result.filesAdded,
+        totalFilesUpdated: result.filesUpdated,
+        totalFilesErrored: result.filesErrored,
+        totalFilesSkipped: result.filesSkipped,
+      };
+    },
+    ctxHolder.jobManager
+  );
 
   if (!jobResult.ok) {
     return jobConflictResponse(jobResult);
@@ -1100,7 +1209,8 @@ export async function handleDeleteCollection(
     const status = statusMap[syncResult.code] ?? 500;
     return errorResponse(syncResult.code, syncResult.error, status);
   }
-
+  ctxHolder.markContentMutation?.();
+  ctxHolder.markIndexMutation?.();
   return jsonResponse({
     success: true,
     collection: name,
@@ -1231,6 +1341,7 @@ export async function handleClearCollectionEmbeddings(
     const status = result.error.code === "INVALID_INPUT" ? 400 : 500;
     return errorResponse(result.error.code, result.error.message, status);
   }
+  recordIndexMutation(result.value.deletedVectors, ctxHolder.markIndexMutation);
 
   return jsonResponse({
     success: true,
@@ -1292,25 +1403,30 @@ export async function handleSync(
   }
 
   // Start background sync job
-  const jobResult = startJob("sync", async (): Promise<SyncResult> => {
-    const result = await defaultSyncService.syncAll(
-      collections,
-      store,
-      withContentTypeRules(
-        {
-          gitPull: body.gitPull,
-          runUpdateCmd: true,
-        },
-        ctxHolder.config
-      )
-    );
-    if (result.totalFilesAdded > 0 || result.totalFilesUpdated > 0) {
-      if (ctxHolder.scheduler) {
-        await ctxHolder.scheduler.triggerNow();
+  const jobResult = await startJob(
+    "sync",
+    async (): Promise<SyncResult> => {
+      const result = await defaultSyncService.syncAll(
+        collections,
+        store,
+        withContentTypeRules(
+          {
+            gitPull: body.gitPull,
+            runUpdateCmd: true,
+          },
+          ctxHolder.config
+        )
+      );
+      recordContentMutation(result, ctxHolder.markContentMutation);
+      if (hasContentMutation(result)) {
+        if (ctxHolder.scheduler) {
+          await ctxHolder.scheduler.triggerNow();
+        }
       }
-    }
-    return result;
-  });
+      return result;
+    },
+    ctxHolder.jobManager
+  );
 
   if (!jobResult.ok) {
     return jobConflictResponse(jobResult);
@@ -1594,7 +1710,8 @@ export async function handleDocsAutocomplete(
 export async function handleDoc(
   store: SqliteAdapter,
   config: Config,
-  url: URL
+  url: URL,
+  request?: Request
 ): Promise<Response> {
   const uri = url.searchParams.get("uri");
   if (!uri) {
@@ -1634,7 +1751,7 @@ export async function handleDoc(
   });
   const source = await buildSourceMeta(config.collections, doc);
 
-  return jsonResponse({
+  const responseData = {
     docid: doc.docid,
     uri: doc.uri,
     title: doc.title,
@@ -1645,7 +1762,52 @@ export async function handleDoc(
     tags,
     source,
     capabilities,
+  };
+  const continuationTraceId = request
+    ? requestRetrievalTraceId(request)
+    : undefined;
+  if (!continuationTraceId) return jsonResponse(responseData);
+
+  const resumed = await RetrievalTraceSession.resume({
+    store,
+    config: config.retrievalTraces,
+    traceId: continuationTraceId,
   });
+  if (!resumed.ok) {
+    return errorResponse("RUNTIME", resumed.error.message, 500);
+  }
+  const traceSession = resumed.value;
+  if (!traceSession) return jsonResponse(responseData);
+
+  if (content !== null) {
+    const endLine = content.split("\n").length;
+    const evidence = evidenceFromExactDocument({
+      docid: doc.docid,
+      uri: doc.uri,
+      sourceHash: doc.sourceHash,
+      mirrorHash: doc.mirrorHash ?? undefined,
+      content,
+      startLine: 1,
+      endLine,
+    });
+    if (evidence) {
+      const got = await traceSession.recordEvidence("get", [evidence]);
+      if (!got.ok) {
+        return withRetrievalTraceHeader(
+          errorResponse("RUNTIME", got.error.message, 500),
+          traceSession
+        );
+      }
+      const opened = await traceSession.recordEvidence("open", [evidence]);
+      if (!opened.ok) {
+        return withRetrievalTraceHeader(
+          errorResponse("RUNTIME", opened.error.message, 500),
+          traceSession
+        );
+      }
+    }
+  }
+  return withRetrievalTraceHeader(jsonResponse(responseData), traceSession);
 }
 
 /**
@@ -1797,6 +1959,7 @@ export async function handleTags(
  * Deactivate a document (soft delete - does not remove file from disk).
  */
 export async function handleDeactivateDoc(
+  ctxHolder: ContextHolder,
   store: SqliteAdapter,
   docId: string,
   req?: Request
@@ -1821,6 +1984,10 @@ export async function handleDeactivateDoc(
   if (!result.ok) {
     return errorResponse("RUNTIME", result.error.message, 500);
   }
+  if (result.value > 0) {
+    ctxHolder.markContentMutation?.();
+    ctxHolder.markIndexMutation?.();
+  }
 
   return jsonResponse({
     success: true,
@@ -1929,10 +2096,12 @@ export async function handleRenameDoc(
     const nextUri = `gno://${collection.name}/${nextRelPath}`;
     let warning: string | undefined;
     try {
-      await syncCollection(
+      await syncResidentCollection(
+        ctxHolder,
         collection,
         store,
-        withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
+        withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config),
+        syncCollection
       );
     } catch {
       warning =
@@ -2151,10 +2320,12 @@ export async function handleTrashDoc(
     }
     let warning: string | undefined;
     try {
-      await syncCollection(
+      await syncResidentCollection(
+        ctxHolder,
         collection,
         store,
-        withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
+        withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config),
+        syncCollection
       );
     } catch {
       warning =
@@ -2274,7 +2445,8 @@ export async function handleMoveDoc(
     await renameFilePath(fullPath, nextFullPath);
     let warning: string | undefined;
     try {
-      await defaultSyncService.syncCollection(
+      await syncResidentCollection(
+        ctxHolder,
         collection,
         store,
         withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
@@ -2397,7 +2569,8 @@ export async function handleDuplicateDoc(
     await copyFilePath(fullPath, nextFullPath);
     let warning: string | undefined;
     try {
-      await defaultSyncService.syncCollection(
+      await syncResidentCollection(
+        ctxHolder,
         collection,
         store,
         withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
@@ -2546,7 +2719,10 @@ export async function handleUpdateDoc(
   ctxHolder: ContextHolder,
   store: SqliteAdapter,
   docId: string,
-  req: Request
+  req: Request,
+  deps?: {
+    syncCollection?: typeof defaultSyncService.syncCollection;
+  }
 ): Promise<Response> {
   let body: UpdateDocRequestBody;
   try {
@@ -2726,32 +2902,38 @@ export async function handleUpdateDoc(
     // Note: embedding handled separately by embed-scheduler (not inline)
     let jobId: string | null = null;
     if (contentToWrite !== undefined) {
-      const jobResult = startJob("sync", async (): Promise<SyncResult> => {
-        const result = await defaultSyncService.syncCollection(
-          collection,
-          store,
-          withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
-        );
-        // Notify scheduler after sync completes
-        ctxHolder.scheduler?.notifySyncComplete([doc.docid]);
-        ctxHolder.eventBus?.emit({
-          type: "document-changed",
-          uri: doc.uri,
-          collection: doc.collection,
-          relPath: doc.relPath,
-          origin: "save",
-          changedAt: new Date().toISOString(),
-        });
-        return {
-          collections: [result],
-          totalDurationMs: result.durationMs,
-          totalFilesProcessed: result.filesProcessed,
-          totalFilesAdded: result.filesAdded,
-          totalFilesUpdated: result.filesUpdated,
-          totalFilesErrored: result.filesErrored,
-          totalFilesSkipped: result.filesSkipped,
-        };
-      });
+      const jobResult = await startJob(
+        "sync",
+        async (): Promise<SyncResult> => {
+          const result = await syncResidentCollection(
+            ctxHolder,
+            collection,
+            store,
+            withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config),
+            deps?.syncCollection
+          );
+          // Notify scheduler after sync completes
+          ctxHolder.scheduler?.notifySyncComplete([doc.docid]);
+          ctxHolder.eventBus?.emit({
+            type: "document-changed",
+            uri: doc.uri,
+            collection: doc.collection,
+            relPath: doc.relPath,
+            origin: "save",
+            changedAt: new Date().toISOString(),
+          });
+          return {
+            collections: [result],
+            totalDurationMs: result.durationMs,
+            totalFilesProcessed: result.filesProcessed,
+            totalFilesAdded: result.filesAdded,
+            totalFilesUpdated: result.filesUpdated,
+            totalFilesErrored: result.filesErrored,
+            totalFilesSkipped: result.filesSkipped,
+          };
+        },
+        ctxHolder.jobManager
+      );
       jobId = jobResult.ok ? jobResult.jobId : null;
     }
 
@@ -2906,7 +3088,10 @@ export async function handleCreateEditableCopy(
 export async function handleCreateCapture(
   ctxHolder: ContextHolder,
   store: SqliteAdapter,
-  req: Request
+  req: Request,
+  deps?: {
+    syncCollection?: typeof defaultSyncService.syncCollection;
+  }
 ): Promise<Response> {
   let body: CreateCaptureRequestBody;
   try {
@@ -2994,31 +3179,37 @@ export async function handleCreateCapture(
     await writeCapturePlanFile(plan, fullPath);
 
     const gnoUri = `gno://${collection.name}/${plan.relPath}`;
-    const jobResult = startJob("sync", async (): Promise<SyncResult> => {
-      const result = await defaultSyncService.syncCollection(
-        collection,
-        store,
-        withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
-      );
-      ctxHolder.scheduler?.notifySyncComplete([plan.relPath]);
-      ctxHolder.eventBus?.emit({
-        type: "document-changed",
-        uri: gnoUri,
-        collection: collection.name,
-        relPath: plan.relPath,
-        origin: "create",
-        changedAt: new Date().toISOString(),
-      });
-      return {
-        collections: [result],
-        totalDurationMs: result.durationMs,
-        totalFilesProcessed: result.filesProcessed,
-        totalFilesAdded: result.filesAdded,
-        totalFilesUpdated: result.filesUpdated,
-        totalFilesErrored: result.filesErrored,
-        totalFilesSkipped: result.filesSkipped,
-      };
-    });
+    const jobResult = await startJob(
+      "sync",
+      async (): Promise<SyncResult> => {
+        const result = await syncResidentCollection(
+          ctxHolder,
+          collection,
+          store,
+          withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config),
+          deps?.syncCollection
+        );
+        ctxHolder.scheduler?.notifySyncComplete([plan.relPath]);
+        ctxHolder.eventBus?.emit({
+          type: "document-changed",
+          uri: gnoUri,
+          collection: collection.name,
+          relPath: plan.relPath,
+          origin: "create",
+          changedAt: new Date().toISOString(),
+        });
+        return {
+          collections: [result],
+          totalDurationMs: result.durationMs,
+          totalFilesProcessed: result.filesProcessed,
+          totalFilesAdded: result.filesAdded,
+          totalFilesUpdated: result.filesUpdated,
+          totalFilesErrored: result.filesErrored,
+          totalFilesSkipped: result.filesSkipped,
+        };
+      },
+      ctxHolder.jobManager
+    );
 
     return jsonResponse(
       buildCaptureReceipt({
@@ -3058,7 +3249,10 @@ export async function handleCreateCapture(
 export async function handleCreateDoc(
   ctxHolder: ContextHolder,
   store: SqliteAdapter,
-  req: Request
+  req: Request,
+  deps?: {
+    syncCollection?: typeof defaultSyncService.syncCollection;
+  }
 ): Promise<Response> {
   let body: CreateDocRequestBody;
   try {
@@ -3230,34 +3424,40 @@ export async function handleCreateDoc(
 
     // Run sync via job system (non-blocking)
     // Note: embedding handled separately by embed-scheduler (not inline)
-    const jobResult = startJob("sync", async (): Promise<SyncResult> => {
-      const result = await defaultSyncService.syncCollection(
-        collection,
-        store,
-        withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
-      );
-      // Notify scheduler after sync completes (use gnoUri as docid placeholder)
-      // The sync will create a proper docid, but we don't have it here yet
-      // Using normalizedRelPath as identifier since docid is generated during sync
-      ctxHolder.scheduler?.notifySyncComplete([normalizedRelPath]);
-      ctxHolder.eventBus?.emit({
-        type: "document-changed",
-        uri: gnoUri,
-        collection: collection.name,
-        relPath: normalizedRelPath,
-        origin: "create",
-        changedAt: new Date().toISOString(),
-      });
-      return {
-        collections: [result],
-        totalDurationMs: result.durationMs,
-        totalFilesProcessed: result.filesProcessed,
-        totalFilesAdded: result.filesAdded,
-        totalFilesUpdated: result.filesUpdated,
-        totalFilesErrored: result.filesErrored,
-        totalFilesSkipped: result.filesSkipped,
-      };
-    });
+    const jobResult = await startJob(
+      "sync",
+      async (): Promise<SyncResult> => {
+        const result = await syncResidentCollection(
+          ctxHolder,
+          collection,
+          store,
+          withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config),
+          deps?.syncCollection
+        );
+        // Notify scheduler after sync completes (use gnoUri as docid placeholder)
+        // The sync will create a proper docid, but we don't have it here yet
+        // Using normalizedRelPath as identifier since docid is generated during sync
+        ctxHolder.scheduler?.notifySyncComplete([normalizedRelPath]);
+        ctxHolder.eventBus?.emit({
+          type: "document-changed",
+          uri: gnoUri,
+          collection: collection.name,
+          relPath: normalizedRelPath,
+          origin: "create",
+          changedAt: new Date().toISOString(),
+        });
+        return {
+          collections: [result],
+          totalDurationMs: result.durationMs,
+          totalFilesProcessed: result.filesProcessed,
+          totalFilesAdded: result.filesAdded,
+          totalFilesUpdated: result.filesUpdated,
+          totalFilesErrored: result.filesErrored,
+          totalFilesSkipped: result.filesSkipped,
+        };
+      },
+      ctxHolder.jobManager
+    );
 
     return jsonResponse(
       {
@@ -3289,9 +3489,13 @@ export async function handleCreateDoc(
  * Returns search results.
  */
 export async function handleSearch(
-  store: SqliteAdapter,
+  contextOrStore: ServerContext | SqliteAdapter,
   req: Request
 ): Promise<Response> {
+  const context =
+    "store" in contextOrStore ? contextOrStore : (null as ServerContext | null);
+  const store =
+    "store" in contextOrStore ? contextOrStore.store : contextOrStore;
   let body: SearchRequestBody;
   try {
     body = (await req.json()) as SearchRequestBody;
@@ -3403,13 +3607,30 @@ export async function handleSearch(
     author,
   };
 
-  const result = await searchBm25(store, rawQuery, options);
+  const trace = context
+    ? await startRestTrace(context, {
+        query: rawQuery,
+        filters: retrievalTraceFilters(options),
+        pipeline: "bm25",
+      })
+    : { session: null, error: null };
+  if (trace.error) return trace.error;
+
+  const result = await searchBm25(store, rawQuery, {
+    ...options,
+    traceSession: trace.session ?? undefined,
+  });
 
   if (!result.ok) {
-    return errorResponse("RUNTIME", result.error.message, 500);
+    return finishRestTrace(
+      req,
+      trace.session,
+      "failed",
+      errorResponse("RUNTIME", result.error.message, 500)
+    );
   }
 
-  return jsonResponse(result.value);
+  return withRetrievalTraceHeader(jsonResponse(result.value), trace.session);
 }
 
 /**
@@ -3543,6 +3764,41 @@ export async function handleQuery(
     : undefined;
   const author = body.author?.trim() || undefined;
 
+  const queryOptions = {
+    limit: Math.min(body.limit ?? 20, 50),
+    minScore: body.minScore,
+    collection: body.collection,
+    lang: body.lang,
+    intent: body.intent?.trim() || undefined,
+    candidateLimit:
+      body.candidateLimit !== undefined
+        ? Math.min(body.candidateLimit, 100)
+        : undefined,
+    exclude,
+    queryModes: normalizedQueryModes,
+    noExpand: body.noExpand,
+    noRerank: body.noRerank,
+    graph: body.graph === true,
+    noGraph: body.noGraph,
+    tagsAll,
+    tagsAny,
+    since: body.since,
+    until: body.until,
+    categories,
+    author,
+  };
+  const trace = await startRestTrace(ctx, {
+    query: normalizedQuery,
+    filters: retrievalTraceFilters(queryOptions),
+    pipeline: "hybrid",
+    modelUris: [
+      ctx.embedPort?.modelUri,
+      ctx.expandPort?.modelUri,
+      ctx.rerankPort?.modelUri,
+    ].filter((value): value is string => Boolean(value)),
+  });
+  if (trace.error) return trace.error;
+
   const result = await searchHybrid(
     {
       store: ctx.store,
@@ -3554,35 +3810,21 @@ export async function handleQuery(
     },
     normalizedQuery,
     {
-      limit: Math.min(body.limit ?? 20, 50),
-      minScore: body.minScore,
-      collection: body.collection,
-      lang: body.lang,
-      intent: body.intent?.trim() || undefined,
-      candidateLimit:
-        body.candidateLimit !== undefined
-          ? Math.min(body.candidateLimit, 100)
-          : undefined,
-      exclude,
-      queryModes: normalizedQueryModes,
-      noExpand: body.noExpand,
-      noRerank: body.noRerank,
-      graph: body.graph === true,
-      noGraph: body.noGraph,
-      tagsAll,
-      tagsAny,
-      since: body.since,
-      until: body.until,
-      categories,
-      author,
+      ...queryOptions,
+      traceSession: trace.session ?? undefined,
     }
   );
 
   if (!result.ok) {
-    return errorResponse("RUNTIME", result.error.message, 500);
+    return finishRestTrace(
+      req,
+      trace.session,
+      "failed",
+      errorResponse("RUNTIME", result.error.message, 500)
+    );
   }
 
-  return jsonResponse(result.value);
+  return withRetrievalTraceHeader(jsonResponse(result.value), trace.session);
 }
 
 /**
@@ -3795,15 +4037,6 @@ export async function handleAsk(
     return errorResponse("VALIDATION", "Query cannot be empty");
   }
 
-  // Check if answer generation is available
-  if (!ctx.capabilities.answer) {
-    return errorResponse(
-      "UNAVAILABLE",
-      "Answer generation not available. No answer model loaded.",
-      503
-    );
-  }
-
   // Parse tag filters
   let tagsAll: string[] | undefined;
   let tagsAny: string[] | undefined;
@@ -3890,6 +4123,64 @@ export async function handleAsk(
   const author = body.author?.trim() || undefined;
 
   const limit = Math.min(body.limit ?? 5, 20);
+  const askOptions = {
+    limit,
+    collection: body.collection,
+    lang: body.lang,
+    intent: body.intent?.trim() || undefined,
+    noExpand: body.noExpand,
+    noRerank: body.noRerank,
+    candidateLimit:
+      body.candidateLimit !== undefined
+        ? Math.min(body.candidateLimit, 100)
+        : undefined,
+    exclude,
+    queryModes: normalizedQueryModes,
+    tagsAll,
+    tagsAny,
+    since: body.since,
+    until: body.until,
+    categories,
+    author,
+  };
+  const trace = await startRestTrace(ctx, {
+    query: normalizedQuery,
+    filters: retrievalTraceFilters(askOptions),
+    pipeline: "ask",
+    modelUris: [
+      ctx.embedPort?.modelUri,
+      ctx.expandPort?.modelUri,
+      ctx.answerPort?.modelUri,
+      ctx.rerankPort?.modelUri,
+    ].filter((value): value is string => Boolean(value)),
+  });
+  if (trace.error) return trace.error;
+
+  if (!ctx.capabilities.answer) {
+    const unavailable = await trace.session?.recordCapability(
+      "answer_generation",
+      "unavailable",
+      "model_unavailable"
+    );
+    if (unavailable && !unavailable.ok) {
+      return finishRestTrace(
+        req,
+        trace.session,
+        "failed",
+        errorResponse("RUNTIME", unavailable.error.message, 500)
+      );
+    }
+    return finishRestTrace(
+      req,
+      trace.session,
+      "failed",
+      errorResponse(
+        "UNAVAILABLE",
+        "Answer generation not available. No answer model loaded.",
+        503
+      )
+    );
+  }
 
   // Run hybrid search first
   const searchResult = await searchHybrid(
@@ -3903,29 +4194,18 @@ export async function handleAsk(
     },
     normalizedQuery,
     {
-      limit,
-      collection: body.collection,
-      lang: body.lang,
-      intent: body.intent?.trim() || undefined,
-      noExpand: body.noExpand,
-      noRerank: body.noRerank,
-      candidateLimit:
-        body.candidateLimit !== undefined
-          ? Math.min(body.candidateLimit, 100)
-          : undefined,
-      exclude,
-      queryModes: normalizedQueryModes,
-      tagsAll,
-      tagsAny,
-      since: body.since,
-      until: body.until,
-      categories,
-      author,
+      ...askOptions,
+      traceSession: trace.session ?? undefined,
     }
   );
 
   if (!searchResult.ok) {
-    return errorResponse("RUNTIME", searchResult.error.message, 500);
+    return finishRestTrace(
+      req,
+      trace.session,
+      "failed",
+      errorResponse("RUNTIME", searchResult.error.message, 500)
+    );
   }
 
   const results = searchResult.value.results;
@@ -3937,20 +4217,93 @@ export async function handleAsk(
   let answerGenerated = false;
 
   if (ctx.answerPort) {
-    const maxTokens = body.maxAnswerTokens ?? 512;
-    const rawResult = await generateGroundedAnswer(
-      { genPort: ctx.answerPort, store: ctx.store },
-      normalizedQuery,
-      results,
-      maxTokens
+    const attempted = await trace.session?.recordCapability(
+      "answer_generation",
+      "attempted"
     );
+    if (attempted && !attempted.ok) {
+      return finishRestTrace(
+        req,
+        trace.session,
+        "failed",
+        errorResponse("RUNTIME", attempted.error.message, 500)
+      );
+    }
+    const maxTokens = body.maxAnswerTokens ?? 512;
+    let rawResult: Awaited<ReturnType<typeof generateGroundedAnswer>>;
+    try {
+      rawResult = await generateGroundedAnswer(
+        { genPort: ctx.answerPort, store: ctx.store },
+        normalizedQuery,
+        results,
+        maxTokens
+      );
+    } catch (error) {
+      await trace.session?.recordCapability(
+        "answer_generation",
+        "failed",
+        "generation_failed"
+      );
+      return finishRestTrace(
+        req,
+        trace.session,
+        "failed",
+        errorResponse(
+          "RUNTIME",
+          error instanceof Error ? error.message : String(error),
+          500
+        )
+      );
+    }
 
     if (rawResult) {
-      const processed = processAnswerResult(rawResult);
-      answer = processed.answer;
-      citations = processed.citations;
-      answerContext = processed.answerContext;
-      answerGenerated = true;
+      try {
+        const processed = await processAnswerResultWithTrace(
+          rawResult,
+          trace.session ?? undefined
+        );
+        answer = processed.answer;
+        citations = processed.citations;
+        answerContext = processed.answerContext;
+        answerGenerated = true;
+        const used = await trace.session?.recordCapability(
+          "answer_generation",
+          "used"
+        );
+        if (used && !used.ok) {
+          throw new Error(`Trace recording failed: ${used.error.message}`);
+        }
+      } catch (error) {
+        await trace.session?.recordCapability(
+          "answer_generation",
+          "failed",
+          "generation_failed"
+        );
+        return finishRestTrace(
+          req,
+          trace.session,
+          "failed",
+          errorResponse(
+            "RUNTIME",
+            error instanceof Error ? error.message : String(error),
+            500
+          )
+        );
+      }
+    } else {
+      const failed = await trace.session?.recordCapability(
+        "answer_generation",
+        "failed",
+        "generation_failed"
+      );
+      if (failed && !failed.ok) {
+        return finishRestTrace(
+          req,
+          trace.session,
+          "failed",
+          errorResponse("RUNTIME", failed.error.message, 500)
+        );
+      }
     }
   }
 
@@ -3975,7 +4328,13 @@ export async function handleAsk(
     },
   };
 
-  return jsonResponse(askResult);
+  const complete = answerGenerated && (citations?.length ?? 0) > 0;
+  return finishRestTrace(
+    req,
+    trace.session,
+    complete ? "completed" : "partial",
+    jsonResponse(askResult)
+  );
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -4141,7 +4500,13 @@ export function handleModelStatus(): Response {
  * Start downloading models for current preset.
  * Returns immediately; poll /api/models/status for progress.
  */
-export function handleModelPull(ctxHolder: ContextHolder): Response {
+export function handleModelPull(
+  ctxHolder: ContextHolder,
+  deps?: {
+    modelsPullFn?: typeof modelsPull;
+    reloadServerContextFn?: typeof reloadServerContext;
+  }
+): Response {
   // Don't start if already downloading
   if (downloadState.active) {
     return errorResponse("CONFLICT", "Download already in progress", 409);
@@ -4152,17 +4517,20 @@ export function handleModelPull(ctxHolder: ContextHolder): Response {
   downloadState.active = true;
   downloadState.startedAt = Date.now();
 
-  // Run download in background (don't await)
-  // Pass current config so it uses the active preset from UI
-  modelsPull({
-    config: ctxHolder.config,
-    all: true,
-    onProgress: (type, progress) => {
-      downloadState.currentType = type;
-      downloadState.progress = progress;
-    },
-  })
-    .then(async (result) => {
+  const operation = async (signal: AbortSignal): Promise<void> => {
+    try {
+      const result = await (deps?.modelsPullFn ?? modelsPull)({
+        config: ctxHolder.config,
+        all: true,
+        signal,
+        onProgress: (type, progress) => {
+          if (signal.aborted) return;
+          downloadState.currentType = type;
+          downloadState.progress = progress;
+        },
+      });
+      if (signal.aborted) return;
+
       // Track results
       for (const r of result.results) {
         if (r.ok) {
@@ -4180,35 +4548,39 @@ export function handleModelPull(ctxHolder: ContextHolder): Response {
       // Reload context to pick up new models
       console.log("Models downloaded, reloading context...");
       try {
-        ctxHolder.current = await reloadServerContext(
-          ctxHolder.current,
-          ctxHolder.config
-        );
+        ctxHolder.current = await (
+          deps?.reloadServerContextFn ?? reloadServerContext
+        )(ctxHolder.current, ctxHolder.config);
+        if (signal.aborted) return;
         console.log("Context reloaded");
         if (ctxHolder.scheduler) {
-          void ctxHolder.scheduler.triggerNow().catch((error) => {
-            console.error(
-              "Failed to trigger embedding after model download:",
-              error
-            );
-          });
+          await ctxHolder.scheduler.triggerNow();
         }
       } catch (e) {
         console.error("Failed to reload context:", e);
       }
-
-      downloadState.active = false;
-      downloadState.currentType = null;
-      downloadState.progress = null;
-    })
-    .catch((e) => {
+    } catch (e) {
       console.error("Model download failed:", e);
-      downloadState.active = false;
       downloadState.failed.push({
         type: downloadState.currentType ?? "embed",
         error: e instanceof Error ? e.message : String(e),
       });
-    });
+    } finally {
+      downloadState.active = false;
+      downloadState.currentType = null;
+      downloadState.progress = null;
+    }
+  };
+
+  const started = ctxHolder.startBackgroundWork?.(operation) ?? false;
+  if (!started) {
+    resetDownloadState();
+    return errorResponse(
+      "UNAVAILABLE",
+      "Resident runtime is shutting down",
+      503
+    );
+  }
 
   return jsonResponse({
     started: true,
@@ -4224,8 +4596,8 @@ export function handleModelPull(ctxHolder: ContextHolder): Response {
  * GET /api/jobs/:id
  * Poll job status for async operations.
  */
-export function handleJob(jobId: string): Response {
-  const status = getJobStatus(jobId);
+export function handleJob(jobId: string, jobManager?: JobManager): Response {
+  const status = getJobStatus(jobId, jobManager);
   if (!status) {
     return errorResponse("NOT_FOUND", "Job not found or expired", 404);
   }
@@ -4236,9 +4608,9 @@ export function handleJob(jobId: string): Response {
  * GET /api/jobs/active
  * Returns the current active job, or null when idle.
  */
-export function handleActiveJob(): Response {
+export function handleActiveJob(jobManager?: JobManager): Response {
   return jsonResponse({
-    activeJob: getActiveJob(),
+    activeJob: getActiveJob(jobManager),
   });
 }
 
@@ -4575,7 +4947,7 @@ export async function routeApi(
   }
 
   if (path === "/api/doc") {
-    return handleDoc(store, config, url);
+    return handleDoc(store, config, url, req);
   }
 
   if (path === "/api/doc-asset") {
@@ -4583,7 +4955,25 @@ export async function routeApi(
   }
 
   if (path === "/api/search" && req.method === "POST") {
-    return handleSearch(store, req);
+    return handleSearch(
+      {
+        store,
+        config,
+        indexName: "default",
+        vectorIndex: null,
+        embedPort: null,
+        expandPort: null,
+        answerPort: null,
+        rerankPort: null,
+        capabilities: {
+          bm25: true,
+          vector: false,
+          hybrid: false,
+          answer: false,
+        },
+      },
+      req
+    );
   }
 
   // Unknown API route