@remnic/core 9.3.712 → 9.3.714

package/src/access-mcp.ts

@@ -34,6 +34,7 @@ import {
   buildChatGptMemoryInspectorResult,
   type RemnicChatGptMemoryInspectorInput,
 } from "./mcp-memory-inspector-app.js";
+import { processChatMessage } from "./chat/chat-factory.js";
 
 type JsonRpcId = string | number | null;
 
@@ -50,6 +51,12 @@ type McpRequestOptions = {
   sessionKeyOverride?: string;
   sessionId?: string;
   correlationId?: string;
+  /**
+   * Write-quota enforcer forwarded into observe's idempotency lock
+   * (issue #1649). Only the MCP-over-HTTP transport supplies this (it owns
+   * the write-rate-limit window); the standalone MCP server has no quota.
+   */
+  enforceWriteQuota?: () => void | Promise<void>;
 };
 
 type McpTool = {
@@ -365,6 +372,13 @@ export class EngramMcpServer {
    */
   private readonly correctionVisible: boolean;
 
+  /**
+   * Whether the `memory_chat` tool should appear in `tools/list` (issue
+   * #1583). Gated on `chat.enabled`. When false the tools array is
+   * byte-identical to pre-feature (rule 39).
+   */
+  private readonly chatVisible: boolean;
+
   constructor(
     private readonly service: EngramAccessService,
     options: {
@@ -377,6 +391,7 @@ export class EngramMcpServer {
       codegraphVisible?: boolean;
       sessionDeltaVisible?: boolean;
       correctionVisible?: boolean;
+      chatVisible?: boolean;
     } = {},
   ) {
     this.citationsEnabled = options.citationsEnabled === true;
@@ -388,6 +403,7 @@ export class EngramMcpServer {
     this.sessionDeltaVisible = options.sessionDeltaVisible === true;
     // correction defaults to visible (enabled by default — plan is read-only).
     this.correctionVisible = options.correctionVisible !== false;
+    this.chatVisible = options.chatVisible === true;
     this.authenticatedPrincipal =
       options.principal?.trim() ||
       readEnvVar("OPENCLAW_ENGRAM_ACCESS_PRINCIPAL")?.trim() ||
@@ -1003,6 +1019,11 @@ export class EngramMcpServer {
           properties: {
             memoryId: { type: "string" },
             namespace: { type: "string" },
+            // Issue #1582 — when memoryId is a `[m:xxxx]` handle, sessionKey
+            // scopes resolution to this session's recent recall history.
+            // Advertising it here lets the transport inject its session key
+            // (toolAcceptsArgument) so MCP callers can cite handles (codex review).
+            sessionKey: { type: "string" },
           },
           required: ["memoryId"],
           additionalProperties: false,
@@ -1156,6 +1177,11 @@ export class EngramMcpServer {
               },
               description: "Conversation messages to observe",
             },
+            idempotencyKey: {
+              type: "string",
+              description:
+                "Optional idempotency key (issue #1649). Deduplicates a retried observe POST server-side so the batch is ingested once even when the HTTP response is lost. Reusing the key with a different payload is rejected.",
+            },
             namespace: { type: "string" },
             skipExtraction: { type: "boolean" },
             cwd: { type: "string", description: "Working directory for auto git-context resolution." },
@@ -2214,6 +2240,26 @@ export class EngramMcpServer {
       );
       this.tools = [...this.tools, ...planTool, ...applyTool];
     }
+    if (this.chatVisible) {
+      const chatTools = withToolAliases(
+        {
+          name: "engram.memory_chat",
+          description:
+            "Conversational memory inspection and correction (issue #1583). Send a message to chat about what Remnic remembers, inspect memories, or request corrections. Returns {reply, chatSessionId, pendingPlan?}.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              message: { type: "string", description: "The user's message — a question about memories, a correction request, or a confirmation (yes/apply) to proceed with a pending plan." },
+              chatSessionId: { type: "string", description: "Optional existing chat session id to resume. Omit to start a new session." },
+            },
+            required: ["message"],
+            additionalProperties: false,
+          },
+        },
+        this.emitLegacyTools,
+      );
+      this.tools = [...this.tools, ...chatTools];
+    }
   }
 
   /** Get clientInfo for a specific MCP session. Returns undefined for non-MCP requests. */
@@ -2374,7 +2420,13 @@ export class EngramMcpServer {
           argumentsObject = { ...argumentsObject, sessionKey: options.sessionKeyOverride };
         }
         const effectivePrincipal = options?.principalOverride ?? this.authenticatedPrincipal;
-        const result = await this.callTool(name, argumentsObject, effectivePrincipal, options?.sessionId);
+        // Forward the MCP session scope (namespace/sessionKey overrides) so
+        // tools like memory_chat bind the caller active scope (Thread 17).
+        const mcpScope = {
+          ...(options?.namespaceOverride ? { namespace: options.namespaceOverride } : {}),
+          ...(options?.sessionKeyOverride ? { sessionKey: options.sessionKeyOverride } : {}),
+        };
+        const result = await this.callTool(name, argumentsObject, effectivePrincipal, options?.sessionId, mcpScope, options?.enforceWriteQuota);
         return {
           jsonrpc: "2.0",
           id,
@@ -2562,7 +2614,7 @@ export class EngramMcpServer {
       }));
   }
 
-  private async callTool(name: string, args: Record<string, unknown>, effectivePrincipal?: string, mcpSessionId?: string): Promise<unknown> {
+  private async callTool(name: string, args: Record<string, unknown>, effectivePrincipal?: string, mcpSessionId?: string, scope?: { namespace?: string; sessionKey?: string }, enforceWriteQuota?: () => void | Promise<void>): Promise<unknown> {
     // Migrated operations dispatch through the access boundary (issue #1525):
     // one registry entry owns schema validation, normalization (rules
     // 17/28/36/48/51), and error mapping for every surface. The switch
@@ -3104,21 +3156,27 @@ export class EngramMcpServer {
         );
       case "engram.observe": {
         const body = parseMcpRequest("observe", args);
-        return this.service.observe({
-          sessionKey: body.sessionKey,
-          messages: body.messages.map((message) => ({
-            role: message.role,
-            content: message.content,
-            parts: message.parts ?? undefined,
-            rawContent: message.rawContent ?? undefined,
-            sourceFormat: message.sourceFormat ?? undefined,
-          })),
-          namespace: body.namespace,
-          authenticatedPrincipal: effectivePrincipal,
-          skipExtraction: body.skipExtraction === true,
-          cwd: body.cwd,
-          projectTag: body.projectTag,
-        });
+        return this.service.observe(
+          {
+            sessionKey: body.sessionKey,
+            messages: body.messages.map((message) => ({
+              role: message.role,
+              content: message.content,
+              parts: message.parts ?? undefined,
+              rawContent: message.rawContent ?? undefined,
+              sourceFormat: message.sourceFormat ?? undefined,
+            })),
+            skipExtraction: body.skipExtraction === true,
+            idempotencyKey: body.idempotencyKey,
+            namespace: body.namespace,
+            authenticatedPrincipal: effectivePrincipal,
+            cwd: body.cwd,
+            projectTag: body.projectTag,
+          },
+          // Forward the transport's write-quota enforcer into observe's
+          // idempotency lock so a replay is never 429'd (issue #1649).
+          enforceWriteQuota ? { enforceWriteQuota } : undefined,
+        );
       }
       case "engram.lcm_search":
         return this.service.lcmSearch({
@@ -3725,6 +3783,30 @@ export class EngramMcpServer {
           authenticatedPrincipal: effectivePrincipal,
         });
       }
+      case "engram.memory_chat": {
+        const message = typeof args.message === "string" ? args.message : "";
+        if (!message) {
+          throw new EngramAccessInputError("message is required");
+        }
+        const chatSessionId = typeof args.chatSessionId === "string" ? args.chatSessionId : undefined;
+        const chatResult = await processChatMessage({
+          service: this.service,
+          config: this.service.configRef?.chat,
+          memoryDir: this.service.memoryDir,
+          message,
+          ...(chatSessionId ? { chatSessionId } : {}),
+          ...(effectivePrincipal ? { principal: effectivePrincipal } : {}),
+          ...(scope?.namespace ? { namespace: scope.namespace } : {}),
+          ...(scope?.sessionKey ? { sessionKey: scope.sessionKey } : {}),
+        });
+        // Strip internal error details — MCP clients get the tagged reply only,
+        // matching the HTTP handler (cursor security review, Thread 17).
+        if (chatResult && typeof chatResult === "object" && "error" in chatResult) {
+          const { error: _stripped, ...wireResult } = chatResult as unknown as Record<string, unknown>;
+          return wireResult;
+        }
+        return chatResult;
+      }
       default:
         throw new Error(`unknown tool: ${name}`);
     }

package/src/access-operations-batch.ts

@@ -1466,3 +1466,17 @@ defineOperation({
   schema: looseSchema,
   handler: async (_input, _ctx) => ({ result: { stream: "sse" } }),
 });
+
+defineOperation({
+  name: "chat_message",
+  description: "Send a message to Remnic Chat (issue #1583). Endpoint registration only — the handler enforces the chat_disabled gate, message validation, and the confirmation protocol.",
+  schema: looseSchema,
+  handler: async (_input, _ctx) => ({ result: { ok: true } }),
+});
+
+defineOperation({
+  name: "chat_events",
+  description: "Stream chat session transcript events (SSE, issue #1583). Endpoint registration only.",
+  schema: looseSchema,
+  handler: async (_input, _ctx) => ({ result: { stream: "sse" } }),
+});

package/src/access-operations.ts

@@ -58,11 +58,15 @@ import {
  */
 const memoryGetSchema = z.object({
   memoryId: z.string().trim().min(1, "memoryId is required").max(512),
+  // Issue #1582 — when memoryId is a `[m:xxxx]` handle, sessionKey scopes the
+  // resolution to this session's recent recall history.
+  sessionKey: z.string().trim().min(1).max(512).nullable().optional(),
   namespace: z.string().trim().max(256).nullable().optional(),
 });
 
 export interface MemoryGetInput {
   readonly memoryId: string;
+  readonly sessionKey?: string | null;
   readonly namespace?: string | null;
 }
 
@@ -79,6 +83,7 @@ export const memoryGetOperation = defineOperation<MemoryGetInput, MemoryGetOutpu
       input.memoryId,
       input.namespace ?? undefined,
       ctx.authenticatedPrincipal,
+      input.sessionKey ?? undefined,
     );
     return { result };
   },

package/src/access-schema.ts

@@ -216,6 +216,14 @@ export const observeRequestSchema = z.object({
   messages: z.array(messageSchema).min(1, "messages must be a non-empty array"),
   namespace: namespaceSchema,
   skipExtraction: z.boolean().optional(),
+  /**
+   * Optional idempotency key for server-side dedup of retried observe POSTs
+   * (issue #1649). A retry that reaches the daemon after the first attempt
+   * already processed is replayed from cache instead of re-ingested. When the
+   * key is reused with a DIFFERENT payload the request is rejected as a
+   * conflict (same contract as memory_store/suggestion_submit).
+   */
+  idempotencyKey: idempotencyKeySchema,
   /** Working directory for auto git-context resolution (issue #569). */
   cwd: z.string().trim().min(1, "cwd must be non-empty when provided").max(2048).optional(),
   /**

package/src/access-service-observe-idempotency.test.ts

@@ -0,0 +1,308 @@
+/**
+ * #1649: a retried observe POST (same `idempotencyKey`) must be deduplicated
+ * server-side — the batch is ingested exactly once even when the HTTP response
+ * is lost and the client replays the request. Without the key the daemon
+ * re-runs every side effect under `skipDedupeCheck: true` and the turn is
+ * queued for extraction twice.
+ *
+ * Verified here against the `EngramAccessService.observe` path with a stub
+ * orchestrator that records every namespace-bearing side effect (LCM enqueue +
+ * extraction replay), the same probe shape used by the #1495 observe-scope
+ * tests. The idempotency store is filesystem-backed, so `memoryDir` is a real
+ * temp directory (not the synthetic path the scope tests use, which never touch
+ * the store because they omit the key).
+ */
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+
+import { EngramAccessService } from "./access-service.js";
+import { EngramAccessInputError } from "./access-service.js";
+import { Orchestrator } from "./orchestrator.js";
+import type { EngramAccessObserveRequest } from "./access-service.js";
+import type { CodingContext, PluginConfig } from "./types.js";
+
+interface ObserveProbe {
+  orch: Orchestrator;
+  contexts: Map<string, CodingContext>;
+  lcmCalls: Array<{ sessionKey: string }>;
+  extractionCalls: Array<{
+    sessionKeys: string[];
+    writeNamespaceOverride?: string;
+    principalOverride?: string;
+  }>;
+}
+
+function makeObserveProbe(memoryDir: string): ObserveProbe {
+  const contexts = new Map<string, CodingContext>();
+  const lcmCalls: ObserveProbe["lcmCalls"] = [];
+  const extractionCalls: ObserveProbe["extractionCalls"] = [];
+
+  const config = {
+    namespacesEnabled: true,
+    defaultNamespace: "default",
+    sharedNamespace: "shared",
+    namespacePolicies: [
+      { name: "pi-geek", readPrincipals: ["pi-geek"], writePrincipals: ["pi-geek"] },
+    ],
+    codingMode: { projectScope: true },
+    memoryDir,
+    // Disable objective-state snapshots so the probe doesn't need a writable
+    // storage backend — the idempotency contract under test is LCM + extraction.
+    objectiveStateMemoryEnabled: false,
+    objectiveStateSnapshotWritesEnabled: false,
+    principalFromSessionKeyMode: "prefix",
+    principalFromSessionKeyRules: [{ match: "pi-geek:", principal: "pi-geek" }],
+    recallCrossNamespaceBudgetEnabled: false,
+    recallCrossNamespaceBudgetWindowMs: 60_000,
+    recallCrossNamespaceBudgetSoftLimit: 10,
+    recallCrossNamespaceBudgetHardLimit: 30,
+  } as unknown as PluginConfig;
+
+  const orch = {
+    config,
+    getCodingContextForSession: (sk: string | undefined) =>
+      (sk ? contexts.get(sk) : null) ?? null,
+    setCodingContextForSession: (sk: string, ctx: CodingContext | null) => {
+      if (ctx === null) contexts.delete(sk);
+      else contexts.set(sk, ctx);
+    },
+    applyCodingNamespaceOverlay: (sk: string | undefined, base: string) =>
+      Orchestrator.prototype.applyCodingNamespaceOverlay.call(orch, sk, base),
+    getStorage: async (ns: string) => ({ dir: join(memoryDir, "storage", ns) }),
+    lcmEngine: {
+      enabled: true,
+      enqueueObserveMessages: (sessionKey: string) => {
+        lcmCalls.push({ sessionKey });
+      },
+    },
+    ingestReplayBatch: async (
+      turns: Array<{ sessionKey: string }>,
+      options: { writeNamespaceOverride?: string; principalOverride?: string } = {},
+    ) => {
+      extractionCalls.push({
+        sessionKeys: turns.map((t) => t.sessionKey),
+        writeNamespaceOverride: options.writeNamespaceOverride,
+        principalOverride: options.principalOverride,
+      });
+    },
+  } as unknown as Orchestrator;
+
+  return { orch, contexts, lcmCalls, extractionCalls };
+}
+
+function observeRequest(
+  overrides: Partial<EngramAccessObserveRequest>,
+): EngramAccessObserveRequest {
+  return {
+    sessionKey: "pi-geek:abc123",
+    messages: [
+      { role: "user", content: "what database are we using?" },
+      { role: "assistant", content: "we use postgres for the primary store" },
+    ],
+    ...overrides,
+  } as EngramAccessObserveRequest;
+}
+
+test("#1649 a retried observe with the same idempotencyKey is deduplicated to a single ingest", async () => {
+  const memoryDir = mkdtempSync(join(tmpdir(), "remnic-observe-idem-"));
+  try {
+    const probe = makeObserveProbe(memoryDir);
+    const service = new EngramAccessService(probe.orch);
+    const key = "observe-batch-pi-geek-abc123-#1";
+
+    const first = await service.observe(observeRequest({ idempotencyKey: key }));
+    // First attempt is a real ingest.
+    assert.equal(first.idempotencyReplay, undefined, "first attempt is not a replay");
+    assert.equal(probe.extractionCalls.length, 1, "first attempt ingests once");
+    assert.equal(probe.lcmCalls.length, 1, "first attempt archives LCM once");
+
+    // Retry: same key, same payload — the daemon already processed the body, so
+    // this is the response-lost-after-process case the issue describes.
+    const second = await service.observe(observeRequest({ idempotencyKey: key }));
+    assert.equal(second.idempotencyReplay, true, "retry is served from the cache");
+    assert.equal(second.accepted, first.accepted, "retry reports the same accepted count");
+    assert.equal(
+      second.effectiveNamespace,
+      first.effectiveNamespace,
+      "retry reports the same effective namespace",
+    );
+
+    // The defining contract: NO second ingest, NO second LCM archive.
+    assert.equal(probe.extractionCalls.length, 1, "retry must not queue extraction a second time");
+    assert.equal(probe.lcmCalls.length, 1, "retry must not archive LCM a second time");
+  } finally {
+    rmSync(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("#1649 observe without an idempotencyKey is never deduplicated (backward compatible)", async () => {
+  const memoryDir = mkdtempSync(join(tmpdir(), "remnic-observe-nokey-"));
+  try {
+    const probe = makeObserveProbe(memoryDir);
+    const service = new EngramAccessService(probe.orch);
+
+    await service.observe(observeRequest({}));
+    await service.observe(observeRequest({}));
+
+    assert.equal(probe.extractionCalls.length, 2, "no key ⇒ both calls ingest");
+    assert.equal(probe.lcmCalls.length, 2, "no key ⇒ both calls archive LCM");
+  } finally {
+    rmSync(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("#1649 reusing an idempotencyKey with a divergent payload is rejected as a conflict", async () => {
+  const memoryDir = mkdtempSync(join(tmpdir(), "remnic-observe-conflict-"));
+  try {
+    const probe = makeObserveProbe(memoryDir);
+    const service = new EngramAccessService(probe.orch);
+    const key = "observe-batch-pi-geek-abc123-#2";
+
+    await service.observe(observeRequest({ idempotencyKey: key }));
+
+    // Same key, DIFFERENT messages — a stale key must never silently mask a
+    // unrelated batch. Same contract as memory_store/suggestion_submit.
+    await assert.rejects(
+      service.observe(
+        observeRequest({
+          idempotencyKey: key,
+          messages: [
+            { role: "user", content: "completely different question" },
+            { role: "assistant", content: "completely different answer" },
+          ],
+        }),
+      ),
+      (err: unknown) =>
+        err instanceof EngramAccessInputError &&
+        /idempotencyKey reuse conflict/.test(err.message),
+      "divergent payload under a reused key must throw a conflict",
+    );
+  } finally {
+    rmSync(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("#1649 review fix: enforceWriteQuota runs only on a cache miss, never on a replay", async () => {
+  const memoryDir = mkdtempSync(join(tmpdir(), "remnic-observe-quota-hook-"));
+  try {
+    const probe = makeObserveProbe(memoryDir);
+    const service = new EngramAccessService(probe.orch);
+    const key = "observe-batch-pi-geek-abc123-#3";
+    let quotaChecks = 0;
+    const enforceWriteQuota = () => {
+      quotaChecks += 1;
+    };
+
+    await service.observe(observeRequest({ idempotencyKey: key }), { enforceWriteQuota });
+    assert.equal(quotaChecks, 1, "quota is enforced exactly once on the real ingest");
+
+    const replay = await service.observe(observeRequest({ idempotencyKey: key }), { enforceWriteQuota });
+    assert.equal(replay.idempotencyReplay, true, "second call is a replay");
+    assert.equal(quotaChecks, 1, "quota is NOT re-checked on a replay (response-lost retry must not 429)");
+    assert.equal(probe.extractionCalls.length, 1, "replay did not re-ingest");
+  } finally {
+    rmSync(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("#1649 review fix: the same idempotencyKey under a different principal is a conflict, not a silent replay", async () => {
+  const memoryDir = mkdtempSync(join(tmpdir(), "remnic-observe-principal-conflict-"));
+  try {
+    const probe = makeObserveProbe(memoryDir);
+    const service = new EngramAccessService(probe.orch);
+    const key = "observe-batch-shared-key-#4";
+
+    // Principal alice ingests under the key.
+    await service.observe(
+      observeRequest({ idempotencyKey: key, authenticatedPrincipal: "alice" }),
+    );
+
+    // A different principal reusing the same key + same body must NOT replay
+    // alice's cached response — the fingerprint folds in authenticatedPrincipal,
+    // so this is a conflict (defends against cross-identity replay when the
+    // principal is supplied out-of-band via HTTP/MCP auth).
+    await assert.rejects(
+      service.observe(
+        observeRequest({ idempotencyKey: key, authenticatedPrincipal: "bob" }),
+      ),
+      (err: unknown) =>
+        err instanceof EngramAccessInputError &&
+        /idempotencyKey reuse conflict/.test(err.message),
+      "cross-principal key reuse must throw a conflict",
+    );
+  } finally {
+    rmSync(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("#1649 review fix: rebinding the session's ambient coding context makes the same key a conflict", async () => {
+  const memoryDir = mkdtempSync(join(tmpdir(), "remnic-observe-ambient-scope-"));
+  try {
+    const probe = makeObserveProbe(memoryDir);
+    const service = new EngramAccessService(probe.orch);
+    const key = "observe-batch-ambient-scope-#5";
+    const req = observeRequest({ idempotencyKey: key });
+
+    // No explicit namespace/cwd/projectTag and no ambient context yet — the
+    // scope resolves to the default store. This caches the response.
+    await service.observe(req);
+    assert.equal(probe.extractionCalls.length, 1, "first observe ingests once");
+
+    // The session is now rebound to a coding project by an external caller
+    // (e.g. a memory_store with cwd, or a direct setCodingContextForSession).
+    // The resolved writeNamespace changed, so reusing the same key + same body
+    // must NOT silently replay — it is a conflict.
+    probe.contexts.set(req.sessionKey, {
+      projectId: "tag:remnic",
+      branch: null,
+      rootPath: "/projects/remnic",
+      defaultBranch: null,
+    });
+
+    await assert.rejects(
+      service.observe(req),
+      (err: unknown) =>
+        err instanceof EngramAccessInputError &&
+        /idempotencyKey reuse conflict/.test(err.message),
+      "key reuse after ambient scope rebind must throw a conflict, not silently replay",
+    );
+  } finally {
+    rmSync(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("#1649 review fix: projectTag observe replays correctly despite scope resolution side effects", async () => {
+  // The fingerprint folds in the RESOLVED writeNamespace (not the raw ambient
+  // context). resolveMemoryScopePlan seeds the session context from projectTag
+  // on the first call; the replay reads the SAME seeded context and resolves the
+  // SAME writeNamespace. This proves the fingerprint is stable across replays
+  // even though the scope resolution mutates session state on the first call.
+  const memoryDir = mkdtempSync(join(tmpdir(), "remnic-observe-tag-replay-"));
+  try {
+    const probe = makeObserveProbe(memoryDir);
+    const service = new EngramAccessService(probe.orch);
+    const key = "observe-batch-tag-replay-#6";
+    const req = observeRequest({ idempotencyKey: key, projectTag: "remnic" });
+
+    const first = await service.observe(req);
+    assert.equal(first.idempotencyReplay, undefined, "first attempt is a real ingest");
+
+    // The first observe's resolveMemoryScopePlan seeded a coding context for
+    // this session. Verify it was actually set (the side effect we depend on).
+    assert.ok(
+      probe.contexts.get(req.sessionKey),
+      "first observe seeded a coding context via resolveMemoryScopePlan",
+    );
+
+    // Retry with the same key + same body + same projectTag — the pre-resolved
+    // writeNamespace matches because the seeded context produces the same scope.
+    const replay = await service.observe(req);
+    assert.equal(replay.idempotencyReplay, true, "projectTag observe replays despite scope-resolution side effects");
+    assert.equal(probe.extractionCalls.length, 1, "replay did not re-ingest");
+  } finally {
+    rmSync(memoryDir, { recursive: true, force: true });
+  }
+});