@remnic/core 9.6.0 → 9.6.2

package/src/connector-provenance.test.ts

@@ -0,0 +1,219 @@
+/**
+ * Phase 1 connector-provenance tests.
+ *
+ * Verifies that a server-resolved `sourceConnector` identity threads through
+ * the memory_store and suggestion_submit write paths — from the MCP request
+ * boundary, through the operation handler (where it overrides any client-
+ * supplied value), down to YAML frontmatter on disk.
+ *
+ * All fixtures are synthetic — no real user data.
+ */
+
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { EngramMcpServer } from "./access-mcp.js";
+import { memoryStoreOperation } from "./access-operations.js";
+import type {
+  EngramAccessMemoryStoreRequest,
+  EngramAccessService,
+  EngramAccessWriteResponse,
+} from "./access-service.js";
+import { StorageManager } from "./storage.js";
+
+// ---------------------------------------------------------------------------
+// Mock service that captures memoryStore calls
+// ---------------------------------------------------------------------------
+
+function makeCaptureService(): {
+  captured: EngramAccessMemoryStoreRequest[];
+  service: EngramAccessService;
+} {
+  const captured: EngramAccessMemoryStoreRequest[] = [];
+  const service = {
+    memoryStore: (
+      req: EngramAccessMemoryStoreRequest,
+    ): Promise<EngramAccessWriteResponse> => {
+      captured.push(req);
+      return Promise.resolve({
+        schemaVersion: 1,
+        operation: "memory_store",
+        namespace: "default",
+        dryRun: false,
+        accepted: true,
+        queued: false,
+        status: "stored",
+      });
+    },
+  } as unknown as EngramAccessService;
+  return { captured, service };
+}
+
+function memoryStoreRequest(
+  overrides: Record<string, unknown> = {},
+): { jsonrpc: "2.0"; id: number; method: "tools/call"; params: Record<string, unknown> } {
+  return {
+    jsonrpc: "2.0",
+    id: 1,
+    method: "tools/call",
+    params: {
+      name: "engram.memory_store",
+      arguments: {
+        content: "The user prefers dark mode for all coding tools",
+        schemaVersion: 1,
+        idempotencyKey: `key-${overrides.idempotencyKey ?? Math.random().toString(36).slice(2)}`,
+        ...overrides,
+      },
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Tests 1–3: MCP layer threads sourceConnector to the service
+// ---------------------------------------------------------------------------
+
+test("memory_store via MCP with sourceConnector 'chatgpt' → service receives it", async () => {
+  const { captured, service } = makeCaptureService();
+  const server = new EngramMcpServer(service, { principal: "test" });
+
+  await server.handleRequest(memoryStoreRequest(), {
+    sourceConnector: "chatgpt",
+  });
+
+  assert.equal(captured.length, 1, "memoryStore should have been called once");
+  assert.equal(
+    captured[0]!.sourceConnector,
+    "chatgpt",
+    "service should receive the server-resolved sourceConnector",
+  );
+});
+
+test("memory_store via MCP with sourceConnector 'codex-cli' → service receives it", async () => {
+  const { captured, service } = makeCaptureService();
+  const server = new EngramMcpServer(service, { principal: "test" });
+
+  await server.handleRequest(memoryStoreRequest(), {
+    sourceConnector: "codex-cli",
+  });
+
+  assert.equal(captured.length, 1);
+  assert.equal(captured[0]!.sourceConnector, "codex-cli");
+});
+
+test("memory_store via MCP with no sourceConnector (operator) → service receives undefined", async () => {
+  const { captured, service } = makeCaptureService();
+  const server = new EngramMcpServer(service, { principal: "test" });
+
+  await server.handleRequest(memoryStoreRequest(), {});
+
+  assert.equal(captured.length, 1);
+  assert.equal(
+    captured[0]!.sourceConnector,
+    undefined,
+    "operator calls should not carry a sourceConnector",
+  );
+});
+
+// ---------------------------------------------------------------------------
+// Test 4: client-supplied sourceConnector in args is IGNORED
+// ---------------------------------------------------------------------------
+
+test("client-supplied sourceConnector in args → IGNORED, server value wins", async () => {
+  const captured: EngramAccessMemoryStoreRequest[] = [];
+  const service = {
+    memoryStore: (req: EngramAccessMemoryStoreRequest) => {
+      captured.push(req);
+      return Promise.resolve({
+        schemaVersion: 1,
+        operation: "memory_store",
+        namespace: "default",
+        dryRun: false,
+        accepted: true,
+        queued: false,
+        status: "stored",
+      } satisfies EngramAccessWriteResponse);
+    },
+  } as unknown as EngramAccessService;
+
+  // Call the handler directly, simulating a malicious client that managed to
+  // inject sourceConnector: "evil" into the parsed input. The handler must
+  // override it with the server-resolved ctx.sourceConnector.
+  await memoryStoreOperation.spec.handler(
+    {
+      content: "The user prefers vim over emacs",
+      schemaVersion: 1,
+      idempotencyKey: "evil-key",
+      sourceConnector: "evil",
+    } as unknown as Parameters<typeof memoryStoreOperation.spec.handler>[0],
+    {
+      service,
+      sourceConnector: "chatgpt",
+    },
+  );
+
+  assert.equal(captured.length, 1);
+  assert.equal(
+    captured[0]!.sourceConnector,
+    "chatgpt",
+    "server-resolved sourceConnector must override client-supplied value",
+  );
+});
+
+// ---------------------------------------------------------------------------
+// Tests 5–6: StorageManager.writeMemory persists sourceConnector to frontmatter
+// ---------------------------------------------------------------------------
+
+test("StorageManager.writeMemory with sourceConnector 'omp' → frontmatter contains it", async () => {
+  StorageManager.clearAllStaticCaches();
+  const baseDir = await mkdtemp(path.join(os.tmpdir(), "prov-connector-"));
+  try {
+    const storage = new StorageManager(baseDir);
+    await storage.ensureDirectories();
+
+    const { id } = await storage.writeMemory(
+      "fact",
+      "The user works from the Pacific time zone",
+      { sourceConnector: "omp" },
+    );
+
+    const memory = await storage.getMemoryById(id);
+    assert.ok(memory, "getMemoryById must find the just-written memory");
+    assert.equal(
+      memory!.frontmatter.sourceConnector,
+      "omp",
+      "frontmatter should contain the sourceConnector",
+    );
+  } finally {
+    StorageManager.clearAllStaticCaches();
+    await rm(baseDir, { recursive: true, force: true });
+  }
+});
+
+test("StorageManager.writeMemory without sourceConnector → frontmatter omits it", async () => {
+  StorageManager.clearAllStaticCaches();
+  const baseDir = await mkdtemp(path.join(os.tmpdir(), "prov-connector-"));
+  try {
+    const storage = new StorageManager(baseDir);
+    await storage.ensureDirectories();
+
+    const { id } = await storage.writeMemory(
+      "fact",
+      "The user enjoys hiking on weekends",
+      {},
+    );
+
+    const memory = await storage.getMemoryById(id);
+    assert.ok(memory, "getMemoryById must find the just-written memory");
+    assert.equal(
+      memory!.frontmatter.sourceConnector,
+      undefined,
+      "frontmatter should omit sourceConnector when not provided",
+    );
+  } finally {
+    StorageManager.clearAllStaticCaches();
+    await rm(baseDir, { recursive: true, force: true });
+  }
+});

package/src/dedup/semantic.test.ts

@@ -245,6 +245,111 @@ test("semantic dedup: normalizes malformed numeric options before lookup", async
   assert.equal(clampedLowThreshold.action, "skip");
 });
 
+// ── Connector-aware semantic dedup (PR #1852 review finding on 7e0eb1a0) ──────
+//
+// A provenance-bearing candidate (sourceConnector present) may be suppressed
+// only by a near-duplicate from the SAME connector. A different or absent
+// neighbor connector must NOT suppress the write. Operator/unattributed
+// writes (no candidate connector) preserve the original behavior.
+
+test("semantic dedup connector-aware: skips same-connector paraphrase", async () => {
+  // Candidate from chatgpt; top neighbor is also chatgpt at 0.96 → skip.
+  const decision = await decideSemanticDedup(
+    "tabs are preferred by the user for indentation",
+    makeLookup([{ id: "existing-pref", score: 0.96, sourceConnector: "chatgpt" }]),
+    { ...DEFAULT_OPTS, sourceConnector: "chatgpt" },
+  );
+  assert.equal(decision.action, "skip");
+  if (decision.action === "skip") {
+    assert.equal(decision.reason, "near_duplicate");
+    assert.equal(decision.topId, "existing-pref");
+    assert.equal(decision.topScore, 0.96);
+  }
+});
+
+test("semantic dedup connector-aware: keeps different-connector paraphrase (B vs A)", async () => {
+  // Candidate from chatgpt; only neighbor is codex at 0.96 → must NOT skip.
+  const decision = await decideSemanticDedup(
+    "tabs are preferred by the user for indentation",
+    makeLookup([{ id: "codex-pref", score: 0.96, sourceConnector: "codex" }]),
+    { ...DEFAULT_OPTS, sourceConnector: "chatgpt" },
+  );
+  assert.equal(decision.action, "keep");
+  assert.equal(decision.reason, "connector_mismatch");
+  if (decision.action === "keep") {
+    assert.equal(decision.topId, "codex-pref");
+  }
+});
+
+test("semantic dedup connector-aware: keeps provenance candidate when neighbor connector absent", async () => {
+  // Candidate from chatgpt; neighbor has no connector at 0.97 → must NOT skip.
+  const decision = await decideSemanticDedup(
+    "tabs are preferred by the user for indentation",
+    makeLookup([{ id: "operator-pref", score: 0.97 }]),
+    { ...DEFAULT_OPTS, sourceConnector: "chatgpt" },
+  );
+  assert.equal(decision.action, "keep");
+  assert.equal(decision.reason, "connector_mismatch");
+});
+
+test("semantic dedup connector-aware: operator (no candidate connector) preserves skip regardless of neighbor connector", async () => {
+  // No candidate connector → existing behavior: skip on any near-duplicate,
+  // even one tagged with a different connector.
+  const decision = await decideSemanticDedup(
+    "tabs are preferred by the user for indentation",
+    makeLookup([{ id: "codex-pref", score: 0.96, sourceConnector: "codex" }]),
+    DEFAULT_OPTS,
+  );
+  assert.equal(decision.action, "skip");
+  if (decision.action === "skip") {
+    assert.equal(decision.topId, "codex-pref");
+  }
+});
+
+test("semantic dedup connector-aware: same-connector dedup survives when a different-connector hit scores higher", async () => {
+  // Candidate from chatgpt; codex neighbor scores 0.98 (higher) but a chatgpt
+  // neighbor also clears threshold at 0.94 → skip against the chatgpt hit,
+  // not the codex one. This preserves same-connector dedup.
+  const decision = await decideSemanticDedup(
+    "tabs are preferred by the user for indentation",
+    makeLookup([
+      { id: "codex-pref", score: 0.98, sourceConnector: "codex" },
+      { id: "chatgpt-pref", score: 0.94, sourceConnector: "chatgpt" },
+    ]),
+    { ...DEFAULT_OPTS, sourceConnector: "chatgpt" },
+  );
+  assert.equal(decision.action, "skip");
+  if (decision.action === "skip") {
+    assert.equal(decision.topId, "chatgpt-pref");
+    assert.equal(decision.topScore, 0.94);
+  }
+});
+
+test("semantic dedup connector-aware: keeps when candidate connector set but no same-connector neighbor clears threshold", async () => {
+  // Candidate from chatgpt; codex neighbor at 0.97 clears threshold, but the
+  // only chatgpt neighbor is at 0.80 (below) → keep (connector_mismatch).
+  const decision = await decideSemanticDedup(
+    "tabs are preferred by the user for indentation",
+    makeLookup([
+      { id: "codex-pref", score: 0.97, sourceConnector: "codex" },
+      { id: "chatgpt-unrelated", score: 0.8, sourceConnector: "chatgpt" },
+    ]),
+    { ...DEFAULT_OPTS, sourceConnector: "chatgpt" },
+  );
+  assert.equal(decision.action, "keep");
+  assert.equal(decision.reason, "connector_mismatch");
+});
+
+test("semantic dedup connector-aware: empty/whitespace candidate connector treated as operator", async () => {
+  // A whitespace-only candidate connector is absent → operator behavior → skip.
+  const wsDecision = await decideSemanticDedup(
+    "tabs are preferred by the user for indentation",
+    makeLookup([{ id: "codex-pref", score: 0.96, sourceConnector: "codex" }]),
+    { ...DEFAULT_OPTS, sourceConnector: "   " },
+  );
+  assert.equal(wsDecision.action, "skip");
+});
+
 test("semantic dedup: candidates=0 short-circuits without calling lookup", async () => {
   let called = 0;
   const decision = await decideSemanticDedup(

package/src/dedup/semantic.ts

@@ -23,6 +23,13 @@ export interface SemanticDedupHit {
   score: number;
   /** Optional source path, purely informational. */
   path?: string;
+  /**
+   * Source connector of the existing neighbor memory's provenance, used by
+   * the connector-aware skip gate (PR #1852 review finding on 7e0eb1a0).
+   * Absent when the neighbor is operator/unattributed or its frontmatter
+   * could not be read.
+   */
+  sourceConnector?: string;
 }
 
 /**
@@ -45,6 +52,15 @@ export interface SemanticDedupOptions {
   threshold: number;
   /** How many nearest neighbors to compare against. */
   candidates: number;
+  /**
+   * Source connector of the CANDIDATE fact being evaluated (its provenance).
+   * When present, a near-duplicate skip is permitted only against a neighbor
+   * carrying the SAME connector; a different or absent neighbor connector
+   * must NOT suppress the write (PR #1852 review finding on 7e0eb1a0).
+   * Absent for operator / unattributed writes, which preserve the original
+   * behavior (skip on any near-duplicate regardless of neighbor connector).
+   */
+  sourceConnector?: string;
 }
 
 export type SemanticDedupDecision =
@@ -54,7 +70,8 @@ export type SemanticDedupDecision =
         | "disabled"
         | "backend_unavailable"
         | "no_candidates"
-        | "no_near_duplicate";
+        | "no_near_duplicate"
+        | "connector_mismatch";
       topScore?: number;
       topId?: string;
     }
@@ -135,8 +152,21 @@ export async function decideSemanticDedup(
     return { action: "keep", reason: "no_candidates" };
   }
 
+  // Resolve the candidate's provenance connector. An empty/whitespace
+  // value is treated as absent (operator / unattributed).
+  const candidateConnector =
+    typeof options.sourceConnector === "string" &&
+    options.sourceConnector.trim().length > 0
+      ? options.sourceConnector.trim()
+      : undefined;
+
   // Defensive: callers ought to return sorted, but don't trust it.
   let top: SemanticDedupHit | undefined;
+  // Best same-connector neighbor — only tracked when the candidate is
+  // provenance-bearing. Tracked in the same pass so a connector-aware skip
+  // matches the best neighbor from the SAME connector even when a higher-
+  // scoring neighbor from a DIFFERENT connector would otherwise win.
+  let sameConnectorTop: SemanticDedupHit | undefined;
   for (const hit of hits) {
     if (
       !hit ||
@@ -150,12 +180,50 @@ export async function decideSemanticDedup(
     if (!top || hit.score > top.score) {
       top = hit;
     }
+    if (
+      candidateConnector !== undefined &&
+      typeof hit.sourceConnector === "string" &&
+      hit.sourceConnector.trim() === candidateConnector &&
+      (!sameConnectorTop || hit.score > sameConnectorTop.score)
+    ) {
+      sameConnectorTop = hit;
+    }
   }
   if (!top) {
     return { action: "keep", reason: "no_near_duplicate" };
   }
 
   if (top.score >= threshold) {
+    // Connector-aware skip gate (PR #1852 review finding on 7e0eb1a0):
+    // A provenance-bearing candidate may be suppressed ONLY by a near-
+    // duplicate from the SAME connector. A different or absent neighbor
+    // connector must NOT suppress the write — otherwise a connector B
+    // paraphrase would be dropped against connector A's memory before the
+    // write ever happens. When the candidate carries no connector (operator
+    // / unattributed), preserve the original behavior and skip on any match.
+    if (candidateConnector !== undefined) {
+      const match =
+        sameConnectorTop !== undefined && sameConnectorTop.score >= threshold
+          ? sameConnectorTop
+          : undefined;
+      if (match) {
+        return {
+          action: "skip",
+          reason: "near_duplicate",
+          topScore: match.score,
+          topId: match.id,
+          topPath: match.path,
+        };
+      }
+      // Best overall hit cleared threshold but came from a different or
+      // absent connector; no same-connector near-duplicate exists. Keep.
+      return {
+        action: "keep",
+        reason: "connector_mismatch",
+        topScore: top.score,
+        topId: top.id,
+      };
+    }
     return {
       action: "skip",
       reason: "near_duplicate",

package/src/explicit-capture.ts

@@ -13,6 +13,7 @@ export type ExplicitCaptureInput = {
   namespace?: string;
   tags?: string[];
   entityRef?: string;
+  sourceConnector?: string;
   ttl?: string;
   sourceReason?: string;
 };
@@ -24,6 +25,7 @@ export type ValidExplicitCapture = {
   namespace?: string;
   tags: string[];
   entityRef?: string;
+  sourceConnector?: string;
   expiresAt?: string;
   sourceReason?: string;
   /**
@@ -371,8 +373,9 @@ export function validateExplicitCaptureInput(
     namespace: asTrimmed(input.namespace),
     tags: Array.from(new Set((input.tags ?? []).map((tag) => tag.trim()).filter(Boolean))),
     entityRef: asTrimmed(input.entityRef),
-    expiresAt,
     sourceReason: asTrimmed(input.sourceReason),
+    sourceConnector: input.sourceConnector,
+    expiresAt,
   };
 }
 
@@ -410,7 +413,14 @@ async function findDuplicateExplicitCapture(
     const status = memory.frontmatter.status ?? "active";
     if (status !== "active") return false;
     if (memory.frontmatter.category !== candidate.category) return false;
-    return normalizeCaptureContent(memory.content) === normalizedCandidate;
+    if (normalizeCaptureContent(memory.content) !== normalizedCandidate) return false;
+    // Connector-aware dedup: same content from different connectors is NOT
+    // a duplicate. Operator (undefined) and connector-tagged memories are
+    // always distinct. Normalize empty strings to undefined.
+    const existingConnector = memory.frontmatter.sourceConnector?.trim() || undefined;
+    const newConnector = candidate.sourceConnector?.trim() || undefined;
+    if (existingConnector !== newConnector) return false;
+    return true;
   });
   return match?.frontmatter.id ?? null;
 }
@@ -438,6 +448,7 @@ export async function persistExplicitCapture(
     entityRef: candidate.entityRef,
     expiresAt: candidate.expiresAt,
     source: source === "inline" ? "explicit-inline" : "explicit",
+    ...(candidate.sourceConnector ? { sourceConnector: candidate.sourceConnector } : {}),
   });
   // #1522: catalog touch handled at the storage chokepoint — the StorageManager's
   // post-write hook records the namespace touch automatically.
@@ -492,6 +503,7 @@ async function findQueuedExplicitCaptureDuplicate(
   orchestrator: Orchestrator,
   namespace: string | undefined,
   content: string,
+  sourceConnector?: string,
 ): Promise<string | null> {
   const storage = await orchestrator.getStorage(namespace);
   const existing = await storage.readAllMemories();
@@ -500,7 +512,11 @@ async function findQueuedExplicitCaptureDuplicate(
     const status = memory.frontmatter.status ?? "active";
     if (status !== "pending_review") return false;
     if (!(memory.frontmatter.tags ?? []).includes("queued-review")) return false;
-    return normalizeCaptureContent(memory.content) === normalized;
+    if (normalizeCaptureContent(memory.content) !== normalized) return false;
+    const existingConnector = memory.frontmatter.sourceConnector?.trim() || undefined;
+    const newConnector = sourceConnector?.trim() || undefined;
+    if (existingConnector !== newConnector) return false;
+    return true;
   });
   return match?.frontmatter.id ?? null;
 }
@@ -520,7 +536,7 @@ export async function queueExplicitCaptureForReview(
     ? requestedNamespace
     : resolveExplicitCaptureReviewNamespace(orchestrator, requestedNamespace);
   const content = buildExplicitCaptureReviewContent(input, reason);
-  const duplicateOf = await findQueuedExplicitCaptureDuplicate(orchestrator, queueNamespace, content);
+  const duplicateOf = await findQueuedExplicitCaptureDuplicate(orchestrator, queueNamespace, content, input.sourceConnector);
   if (duplicateOf) {
     return { id: duplicateOf, duplicateOf };
   }
@@ -536,6 +552,7 @@ export async function queueExplicitCaptureForReview(
     tags: Array.from(new Set([...EXPLICIT_CAPTURE_REVIEW_TAGS, ...requestedTags])),
     entityRef: sanitizeReviewMetadata(input.entityRef),
     source: source === "inline" ? "explicit-inline-review" : "explicit-review",
+    ...(input.sourceConnector ? { sourceConnector: input.sourceConnector } : {}),
   });
   try {
     const created = await storage.getMemoryById(id);