@remnic/core 9.3.769 → 9.3.771

package/src/access-operations-batch.ts

@@ -14,10 +14,12 @@ import { z } from "zod";
 import type { DreamsPhase, RecallDisclosure, RecallPlanMode } from "./types.js";
 import type { RemnicChatGptMemoryInspectorInput } from "./mcp-memory-inspector-app.js";
 import { defineOperation } from "./access-boundary.js";
+import { EngramAccessForbiddenError } from "./access-errors.js";
 import { EngramAccessInputError, type EngramAccessService } from "./access-service.js";
 import { projectTagProjectId } from "./coding/coding-namespace.js";
 import { expandTildePath } from "./utils/path.js";
 import { resolvePrincipal } from "./namespaces/principal.js";
+import { getRecallTimingStatus, isRecallTimingsOperator } from "./recall-timings.js";
 import { validateBriefingFormat } from "./briefing.js";
 import {
   buildChatGptMemoryInspectorActionRequest,
@@ -254,6 +256,23 @@ defineOperation({ name: "offline_sync_apply_file_content", description: "Apply f
 defineOperation({ name: "offline_sync_apply", description: "Apply sync.", schema: strictSchema({ changeset: z.unknown().optional(), namespace: S.str }), handler: async (input, ctx) => ({ result: await ctx.service.offlineSyncApply({ changeset: input.changeset, namespace: optStr(input.namespace), principal: ctx.authenticatedPrincipal }) }) });
 defineOperation({ name: "lcm_status", description: "LCM status.", schema: strictSchema({}), handler: async (_i, ctx) => ({ result: await ctx.service.lcmStatus() }) });
 defineOperation({ name: "memory_list", description: "List memories.", schema: strictSchema({}), handler: async (_i, ctx) => ({ result: await ctx.service.memoryBrowse() }) });
+defineOperation({
+  name: "recall_timings",
+  description: "List recent recall timings.",
+  schema: strictSchema({}),
+  handler: async (_input, ctx) => {
+    if (
+      !isRecallTimingsOperator(
+        ctx.service.configRef,
+        ctx.authenticatedPrincipal,
+        ctx.operatorPrincipal,
+      )
+    ) {
+      throw new EngramAccessForbiddenError("recall timings require the configured operator principal");
+    }
+    return { result: getRecallTimingStatus(ctx.service.configRef) };
+  },
+});
 defineOperation({ name: "entity_list", description: "List entities.", schema: strictSchema({}), handler: async (_i, ctx) => ({ result: await ctx.service.entityList() }) });
 defineOperation({ name: "maintenance_status", description: "Maintenance status.", schema: strictSchema({}), handler: async (_i, ctx) => ({ result: await ctx.service.maintenance() }) });
 defineOperation({ name: "quality_status", description: "Quality status.", schema: strictSchema({}), handler: async (_i, ctx) => ({ result: await ctx.service.quality() }) });

package/src/access-surface-catalog.ts

@@ -200,6 +200,7 @@ export const HTTP_ROUTES: readonly HttpRouteEntry[] = [
   { method: "POST", pathname: "/engram/v1/coding/delta", operation: "coding_delta" },
   { method: "POST", pathname: "/engram/v1/suggestions", operation: "suggestion_submit" },
   { method: "GET", pathname: "/engram/v1/memories", operation: "memory_list" },
+  { method: "GET", pathname: "/engram/v1/recall/timings", operation: "recall_timings" },
   { method: "GET", pathname: "/engram/v1/memories/:id", operation: "memory_get" },
   { method: "GET", pathname: "/engram/v1/memories/:id/timeline", operation: "memory_timeline" },
   { method: "GET", pathname: "/engram/v1/entities", operation: "entity_list" },

package/src/orchestration/recall-internal.ts

@@ -51,6 +51,7 @@ import { createRecallSectionMetricRecorder } from "../recall-qos.js";
 import { buildRecallQueryPolicy } from "../recall-query-policy.js";
 import { type GraphRecallExpandedEntry, type LastRecallBudgetSummary, type LastRecallSnapshot, LastRecallStore, RecallHandleHistoryStore } from "../recall-state.js";
 import { type RecallFilterTrace, type RecallXrayResult, type RecallXrayScoreDecomposition, type RecallXraySnapshot, buildXraySnapshot } from "../recall-xray.js";
+import { recordRecallTiming } from "../recall-timings.js";
 import { findUnresolvedEntityRefs } from "../reconstruct.js";
 import { RerankCache, rerankLocalOrNoop } from "../rerank.js";
 import { buildResponseGuidanceRecallSection, shouldRecallResponseGuidance } from "../response-guidance-recall.js";
@@ -752,6 +753,9 @@ export class RecallInternalCoordinator {
     // Catalog read touch (issue #1499) is recorded LATER — after the Phase 1
     // abort gate — so it fires only when retrieval actually runs, not for
     // aborted / short-circuited recalls.
+    const codingContext = sessionKey
+      ? this.deps.getCodingContextForSession(sessionKey)
+      : null;
     const scopePlan = resolveScopePlan({
       config: this.deps.config,
       sessionKey,
@@ -761,9 +765,7 @@ export class RecallInternalCoordinator {
           && options.principalOverride.length > 0
           ? options.principalOverride
           : undefined,
-      codingContext: sessionKey
-        ? this.deps.getCodingContextForSession(sessionKey)
-        : null,
+      codingContext,
       namespacesEnabled,
     });
     const {
@@ -867,6 +869,14 @@ export class RecallInternalCoordinator {
       const earlySessionKey = sessionKey ?? "default";
       this.deps._recallWorkspaceOverrides.delete(earlySessionKey);
       timings.total = `${Date.now() - recallStart}ms`;
+      recordRecallTiming(this.deps.config, {
+        ...timings,
+        timestamp: new Date().toISOString(),
+        namespace: selfNamespace,
+        total: timings.total,
+        recallPlan: timings.recallPlan,
+        queryPolicy: timings.queryPolicy,
+      });
       // X-ray capture for the `no_recall` early-return path
       // (issue #570 PR 1).  `no_recall` skips retrieval entirely, so
       // the snapshot carries zero results and an empty-budget accounting
@@ -5146,6 +5156,14 @@ export class RecallInternalCoordinator {
       .map(([k, v]) => `${k}=${v}`)
       .join(", ");
     log.info(`recall timings: ${timingParts}`);
+    recordRecallTiming(this.deps.config, {
+      ...timings,
+      timestamp: new Date().toISOString(),
+      namespace: selfNamespace,
+      total: timings.total,
+      recallPlan: timings.recallPlan,
+      queryPolicy: timings.queryPolicy,
+    });
 
     const assembledRecall = this.deps.assembleRecallSections(
       sectionBuckets,

package/src/recall-timings-access.test.ts

@@ -0,0 +1,301 @@
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { getOperation } from "./access-boundary.js";
+import { EngramAccessHttpServer } from "./access-http.js";
+import { EngramAccessService } from "./access-service.js";
+import { parseConfig } from "./config.js";
+import { Orchestrator } from "./orchestrator.js";
+import {
+  getRecallTimings,
+  getRecallTimingStatus,
+  recordRecallTiming,
+  resolveRecallTimingsOperatorPrincipal,
+  type RecallTimingRecord,
+} from "./recall-timings.js";
+
+function makeConfig(memoryDir: string, overrides: Record<string, unknown> = {}) {
+  return parseConfig({
+    openaiApiKey: "sk-test",
+    memoryDir,
+    workspaceDir: path.join(memoryDir, "workspace"),
+    qmdEnabled: false,
+    embeddingFallbackEnabled: false,
+    recallPlannerEnabled: false,
+    sharedContextEnabled: false,
+    initGateTimeoutMs: 1000,
+    ...overrides,
+  });
+}
+
+test("authenticated recall timings route returns two recalls without user-correlatable inputs", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-recall-timings-"));
+  const config = makeConfig(memoryDir);
+  const orchestrator = new Orchestrator(config);
+  const service = new EngramAccessService(orchestrator);
+  const server = new EngramAccessHttpServer({
+    service,
+    host: "127.0.0.1",
+    port: 0,
+    authToken: "secret-token",
+    principal: "operator",
+    trustPrincipalHeader: true,
+    maxBodyBytes: 1024,
+  });
+  const started = await server.start();
+  const url = `http://${started.host}:${started.port}/engram/v1/recall/timings`;
+  try {
+    await orchestrator.recall("private first query", "private:first:session");
+    await orchestrator.recall("private second query", "private:second:session", { mode: "no_recall" });
+
+    const denied = await fetch(url);
+    assert.equal(denied.status, 401);
+
+    const nonOperator = await fetch(url, {
+      headers: {
+        Authorization: "Bearer secret-token",
+        "X-Engram-Principal": "reader",
+      },
+    });
+    assert.equal(nonOperator.status, 403);
+    assert.equal(nonOperator.headers.get("cache-control"), "no-store");
+
+    const response = await fetch(url, {
+      headers: {
+        Authorization: "Bearer secret-token",
+        "X-Engram-Principal": "operator",
+      },
+    });
+    assert.equal(response.status, 200);
+    assert.equal(response.headers.get("cache-control"), "no-store");
+    const payload = await response.json() as {
+      generatedAt: string;
+      processStartedAt: string;
+      capacity: number;
+      count: number;
+      order: string;
+      records: RecallTimingRecord[];
+    };
+    assert.ok(Number.isFinite(Date.parse(payload.generatedAt)));
+    assert.ok(Number.isFinite(Date.parse(payload.processStartedAt)));
+    assert.ok(Date.parse(payload.processStartedAt) <= Date.parse(payload.generatedAt));
+    assert.equal(payload.capacity, 50);
+    assert.equal(payload.order, "newest-first");
+    assert.equal(payload.count, 2);
+    assert.equal(payload.records.length, 2);
+    const operation = getOperation("recall_timings");
+    assert.ok(operation);
+    await assert.rejects(
+      operation.run({}, {
+        service,
+        authenticatedPrincipal: "reader",
+        operatorPrincipal: "operator",
+      }),
+      /configured operator principal/,
+    );
+    const operationOutput = await operation.run({}, {
+      service,
+      authenticatedPrincipal: "operator",
+      operatorPrincipal: "operator",
+    }) as { result: { count: number; records: RecallTimingRecord[] } };
+    // generatedAt legitimately differs between the two calls; the data must not.
+    assert.equal(operationOutput.result.count, payload.count);
+    assert.deepEqual(operationOutput.result.records, payload.records);
+    for (const record of payload.records) {
+      assert.equal(record.namespace, "default");
+      assert.ok(Number.isFinite(Date.parse(record.timestamp)));
+      assert.equal(typeof record.timingsMs.total, "number");
+      assert.ok(record.timingsMs.total >= 0);
+      assert.equal(typeof record.recallPlan, "string");
+      assert.equal(typeof record.queryPolicy, "string");
+      assert.equal("query" in record, false);
+      assert.equal("prompt" in record, false);
+      assert.equal("sessionKey" in record, false);
+      assert.equal("retrievalQuery" in record, false);
+      assert.equal("query" in record.timingsMs, false);
+      assert.equal("prompt" in record.timingsMs, false);
+    }
+  } finally {
+    await server.stop();
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("recall timing history keeps only the newest 50 records", () => {
+  const config = makeConfig(path.join(os.tmpdir(), "remnic-recall-timing-ring"));
+  for (let index = 0; index < 51; index += 1) {
+    recordRecallTiming(config, {
+      timestamp: new Date(index * 1000).toISOString(),
+      namespace: "default",
+      total: index,
+      recallPlan: "full",
+      queryPolicy: "general/full",
+    });
+  }
+
+  const records = getRecallTimings(config);
+  assert.equal(records.length, 50);
+  assert.equal(records[0]?.timingsMs.total, 50);
+  assert.equal(records[49]?.timingsMs.total, 1);
+});
+
+test("operator principal resolution agrees for standalone and embedded access", () => {
+  const standaloneConfig = makeConfig(path.join(os.tmpdir(), "remnic-recall-timing-standalone"));
+  assert.equal(
+    resolveRecallTimingsOperatorPrincipal(standaloneConfig, "standalone-operator"),
+    "standalone-operator",
+  );
+
+  const embeddedConfig = makeConfig(path.join(os.tmpdir(), "remnic-recall-timing-embedded"), {
+    agentAccessHttp: {
+      enabled: true,
+      host: "127.0.0.1",
+      port: 0,
+      principal: "embedded-operator",
+      maxBodyBytes: 1024,
+    },
+  });
+  assert.equal(
+    resolveRecallTimingsOperatorPrincipal(embeddedConfig),
+    "embedded-operator",
+  );
+  assert.equal(
+    resolveRecallTimingsOperatorPrincipal(
+      embeddedConfig,
+      "fallback-transport-principal",
+    ),
+    "embedded-operator",
+  );
+});
+
+test("recall timing records expose only the fixed telemetry schema", () => {
+  const config = makeConfig(path.join(os.tmpdir(), "remnic-recall-timing-schema"));
+  recordRecallTiming(config, {
+    timestamp: new Date(0).toISOString(),
+    namespace: "default",
+    total: "7ms",
+    qmd: "3ms",
+    qmdPost: "2ms-cache",
+    recallPlan: "full",
+    queryPolicy: "general/full",
+    queryAware: "1ms;helped=private prompt detail",
+    prompt: "private prompt",
+    sessionKey: "user-session",
+    unknownTiming: "9ms",
+  });
+
+  const record = getRecallTimings(config)[0];
+  assert.deepEqual(record, {
+    timestamp: new Date(0).toISOString(),
+    namespace: "default",
+    recallPlan: "full",
+    queryPolicy: "general/full",
+    timingsMs: {
+      total: 7,
+      qmd: 3,
+      qmdPost: 2,
+    },
+  });
+});
+
+test("phases that did not run are omitted while measured zeros survive", () => {
+  const config = makeConfig(path.join(os.tmpdir(), "remnic-recall-timing-phases"));
+  recordRecallTiming(config, {
+    timestamp: new Date(0).toISOString(),
+    namespace: "default",
+    total: "5ms",
+    qmd: "0ms",
+    recallPlan: "full",
+    queryPolicy: "general/full",
+  });
+
+  const record = getRecallTimings(config)[0];
+  assert.ok(record);
+  assert.equal(record.timingsMs.qmd, 0);
+  assert.equal("qmdPost" in record.timingsMs, false);
+  assert.equal("ki" in record.timingsMs, false);
+});
+
+test("status envelope reports a stable process start and live generation time", () => {
+  const config = makeConfig(path.join(os.tmpdir(), "remnic-recall-timing-envelope"));
+  const first = getRecallTimingStatus(config);
+  const second = getRecallTimingStatus(config);
+  assert.equal(first.processStartedAt, second.processStartedAt);
+  assert.ok(Number.isFinite(Date.parse(first.processStartedAt)));
+  assert.ok(Date.parse(first.processStartedAt) <= Date.now());
+  assert.equal(first.capacity, 50);
+  assert.equal(first.order, "newest-first");
+  assert.equal(first.count, 0);
+  assert.deepEqual(first.records, []);
+});
+
+test("configured operator principal outranks a divergent server principal override", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-recall-timings-div-"));
+  const config = makeConfig(memoryDir, {
+    agentAccessHttp: {
+      enabled: true,
+      host: "127.0.0.1",
+      port: 0,
+      principal: "config-operator",
+      maxBodyBytes: 1024,
+    },
+  });
+  const orchestrator = new Orchestrator(config);
+  const service = new EngramAccessService(orchestrator);
+  const server = new EngramAccessHttpServer({
+    service,
+    host: "127.0.0.1",
+    port: 0,
+    authToken: "secret-token",
+    principal: "cli-override",
+    trustPrincipalHeader: true,
+    maxBodyBytes: 1024,
+  });
+  const started = await server.start();
+  const url = `http://${started.host}:${started.port}/engram/v1/recall/timings`;
+  try {
+    // The server's own default identity is NOT the operator when the config
+    // explicitly names a different one. Settled precedence: config first.
+    const overrideOnly = await fetch(url, {
+      headers: { Authorization: "Bearer secret-token" },
+    });
+    assert.equal(overrideOnly.status, 403);
+
+    const configOperator = await fetch(url, {
+      headers: {
+        Authorization: "Bearer secret-token",
+        "X-Engram-Principal": "config-operator",
+      },
+    });
+    assert.equal(configOperator.status, 200);
+  } finally {
+    await server.stop();
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("batch context without transport operator fails closed when config names no operator", async () => {
+  const config = makeConfig(path.join(os.tmpdir(), "remnic-recall-timing-batch-closed"));
+  const orchestrator = new Orchestrator(config);
+  const service = new EngramAccessService(orchestrator);
+  const operation = getOperation("recall_timings");
+  assert.ok(operation);
+  // No config principal and no transport operator: nobody can pass the gate,
+  // not even a caller whose authenticated principal would match elsewhere.
+  await assert.rejects(
+    operation.run({}, { service, authenticatedPrincipal: "operator" }),
+    /configured operator principal/,
+  );
+  // The same caller passes once the transport supplies the operator context,
+  // as the HTTP route does by dispatching with operatorPrincipal set to the
+  // server's own principal.
+  const output = await operation.run({}, {
+    service,
+    authenticatedPrincipal: "operator",
+    operatorPrincipal: "operator",
+  }) as { result: { count: number } };
+  assert.equal(output.result.count, 0);
+});

package/src/recall-timings.ts

@@ -0,0 +1,141 @@
+import type { PluginConfig } from "./types.js";
+
+export interface RecallTimingRecord {
+  readonly timestamp: string;
+  readonly namespace: string;
+  readonly recallPlan: string;
+  readonly queryPolicy: string;
+  readonly timingsMs: Readonly<Record<string, number>>;
+}
+
+export interface RecallTimingStatus {
+  readonly generatedAt: string;
+  readonly processStartedAt: string;
+  readonly capacity: number;
+  readonly count: number;
+  readonly order: "newest-first";
+  readonly records: RecallTimingRecord[];
+}
+
+const RECALL_TIMING_HISTORY_LIMIT = 50;
+const TIMING_FIELD_ALLOWLIST = [
+  "total",
+  "sharedCtx",
+  "profile",
+  "peerProfile",
+  "identityContinuity",
+  "entityRetrieval",
+  "ki",
+  "artifacts",
+  "objectiveState",
+  "causalTrajectories",
+  "cmcCausalChains",
+  "calibrationRules",
+  "trustZones",
+  "harmonicRetrieval",
+  "verifiedRecall",
+  "verifiedRules",
+  "workProducts",
+  "qmd",
+  "transcript",
+  "summaries",
+  "nativeKnowledge",
+  "convRecall",
+  "compounding",
+  "graphShadow",
+  "qmdPost",
+] as const;
+// The ring buffer is process-local: a daemon restart or haproxy failover to
+// the other backend starts an empty history. processStartedAt lets consumers
+// detect that discontinuity. Derived from the process time origin, not module
+// initialization, so lazy imports cannot skew it.
+const PROCESS_STARTED_AT = new Date(Date.now() - process.uptime() * 1000).toISOString();
+const histories = new WeakMap<PluginConfig, RecallTimingRecord[]>();
+
+function numericMilliseconds(value: unknown): number | undefined {
+  if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
+    return value;
+  }
+  if (typeof value !== "string") return undefined;
+  const match = /^(\d+)ms/.exec(value);
+  return match ? Number(match[1]) : undefined;
+}
+
+function sanitizeRecallTiming(
+  input: Record<string, unknown>,
+): RecallTimingRecord {
+  // Serialization boundary: ONLY allowlisted numeric phases reach timingsMs,
+  // and the four dimension strings are copied field-by-field. Never spread
+  // the internal record into the public shape.
+  const timingsMs: Record<string, number> = {
+    total: numericMilliseconds(input.total) ?? 0,
+  };
+  for (const field of TIMING_FIELD_ALLOWLIST) {
+    if (field === "total") continue;
+    const value = numericMilliseconds(input[field]);
+    // Phases that did not run are omitted; 0 means "ran, measured zero".
+    if (value !== undefined) timingsMs[field] = value;
+  }
+  return {
+    timestamp: typeof input.timestamp === "string" ? input.timestamp : "",
+    namespace: typeof input.namespace === "string" ? input.namespace : "",
+    recallPlan: typeof input.recallPlan === "string" ? input.recallPlan : "",
+    queryPolicy: typeof input.queryPolicy === "string" ? input.queryPolicy : "",
+    timingsMs,
+  };
+}
+
+export function recordRecallTiming(
+  config: PluginConfig,
+  input: Record<string, unknown>,
+): void {
+  const history = histories.get(config) ?? [];
+  history.push(sanitizeRecallTiming(input));
+  if (history.length > RECALL_TIMING_HISTORY_LIMIT) history.shift();
+  histories.set(config, history);
+}
+
+export function resolveRecallTimingsOperatorPrincipal(
+  config: PluginConfig,
+  transportPrincipal?: string,
+): string | undefined {
+  return config.agentAccessHttp.principal?.trim()
+    || transportPrincipal?.trim()
+    || undefined;
+}
+
+export function isRecallTimingsOperator(
+  config: PluginConfig,
+  authenticatedPrincipal?: string,
+  transportPrincipal?: string,
+): boolean {
+  const operatorPrincipal = resolveRecallTimingsOperatorPrincipal(
+    config,
+    transportPrincipal,
+  );
+  return Boolean(
+    operatorPrincipal
+    && authenticatedPrincipal
+    && authenticatedPrincipal === operatorPrincipal,
+  );
+}
+
+export function getRecallTimings(config: PluginConfig): RecallTimingRecord[] {
+  const history = histories.get(config) ?? [];
+  return history.slice().reverse().map((record) => ({
+    ...record,
+    timingsMs: { ...record.timingsMs },
+  }));
+}
+
+export function getRecallTimingStatus(config: PluginConfig): RecallTimingStatus {
+  const records = getRecallTimings(config);
+  return {
+    generatedAt: new Date().toISOString(),
+    processStartedAt: PROCESS_STARTED_AT,
+    capacity: RECALL_TIMING_HISTORY_LIMIT,
+    count: records.length,
+    order: "newest-first",
+    records,
+  };
+}