@remnic/core 9.7.14 → 9.8.0

package/src/access-admin-ops-surface.ts

@@ -17,6 +17,7 @@ import { type CapsuleListEntry, defaultCapsulesDir } from "./capsule-cli.js";
 import { resolveCodingNamespaceOverlay } from "./coding/coding-namespace.js";
 import { type GraphSnapshotNodeMetadata, type GraphSnapshotRequest, type GraphSnapshotResponse, buildGraphSnapshot } from "./graph-snapshot.js";
 import { log } from "./logger.js";
+import { composeMemoryEnvelope } from "./write-envelope.js";
 import { buildQualityScore, groupActionsByStatus, listMemoryGovernanceRuns, readMemoryGovernanceRunArtifact, runMemoryGovernance } from "./maintenance/memory-governance.js";
 import { resolveScopeProfilePlan } from "./namespaces/scope-profiles.js";
 import type { Orchestrator } from "./orchestrator.js";
@@ -779,21 +780,30 @@ export class AccessAdminOpsSurface {
       },
       writePromotedMemory: async (namespace, memory) => {
         const resolved = await this.deps.orchestrator.getStorage(namespace);
-        const { id, tombstoneBlocked } = await resolved.writeMemory(
-          memory.category,
-          memory.content,
+        // Sealed-envelope write (issue #1989 PR4): an admin promotion
+        // REPLAYS a stored memory into another namespace — legacy rows may
+        // predate current field limits, so salvage (drops warn-logged).
+        const promotionEnvelope = composeMemoryEnvelope(
           {
+            content: memory.content,
+            category: memory.category,
             confidence: memory.confidence,
             tags: memory.tags,
             entityRef: memory.entityRef,
-            source: `admin-promotion:${memory.sourceNamespace}:${memory.reason.slice(0, 120)}`,
-            lineage: memory.lineage,
-            sourceMemoryId: memory.sourceMemoryId,
-            actor: memory.actor,
             validAt: memory.validAt,
             ...(memory.sourceConnector ? { sourceConnector: memory.sourceConnector } : {}),
           },
+          { source: `admin-promotion:${memory.sourceNamespace}:${memory.reason.slice(0, 120)}` },
+          { salvage: true },
         );
+        if (promotionEnvelope.salvageNotes.length > 0) {
+          log.warn(`admin-promotion write salvaged invalid fields: ${promotionEnvelope.salvageNotes.join("; ")}`);
+        }
+        const { id, tombstoneBlocked } = await resolved.writeSealedMemory(promotionEnvelope, {
+          lineage: memory.lineage,
+          sourceMemoryId: memory.sourceMemoryId,
+          actor: memory.actor,
+        });
         // #1645: a tombstone-blocked promotion lands pending_review (no active
         // copy in the target). Report it as a failed promotion so the admin
         // sees an honest result — the content is queued for review, not

package/src/access-boundary.test.ts

@@ -272,3 +272,31 @@ test("fleetWide flag is inert for normal (non-fleet-wide) ops under a scoped tok
   assert.deepEqual(result, { ok: true });
   assert.equal(ran, true, "a non-fleet-wide op runs normally for a scoped token");
 });
+
+test("allowedByOps: an op is granted by any listed op, not just its own name", async () => {
+  let ran = false;
+  __resetRegistryForTest();
+  const op = defineOperation({
+    name: "namespace_writable",
+    description: "test allowedByOps",
+    allowedByOps: ["namespace_writable", "observe", "memory_store"],
+    schema: z.object({}),
+    handler: async () => {
+      ran = true;
+      return { ok: true };
+    },
+  });
+  // A token scoped to observe (but NOT namespace_writable) reaches the handler.
+  const result = await tokenCapabilityStore.run({ version: 1, ops: ["observe"] }, () =>
+    op.run({}, ctx(makeMockService())),
+  );
+  assert.deepEqual(result, { ok: true });
+  assert.equal(ran, true, "an observe-scoped token is granted via allowedByOps");
+  // A token carrying none of the granting ops is rejected before the handler.
+  ran = false;
+  await assert.rejects(
+    () => tokenCapabilityStore.run({ version: 1, ops: ["recall"] }, () => op.run({}, ctx(makeMockService()))),
+    (err: unknown) => err instanceof EngramAccessForbiddenError,
+  );
+  assert.equal(ran, false, "a token without any granting op is rejected before the handler");
+});

package/src/access-boundary.ts

@@ -16,8 +16,14 @@
  */
 
 import { z } from "zod";
+import { EngramAccessForbiddenError } from "./access-errors.js";
 import { EngramAccessInputError, type EngramAccessService } from "./access-service.js";
-import { assertFleetWideOperationAllowed, assertOperationAllowed, tokenCapabilityStore } from "./access-token-capabilities.js";
+import {
+  assertFleetWideOperationAllowed,
+  assertOperationAllowed,
+  capabilityAllowsOp,
+  tokenCapabilityStore,
+} from "./access-token-capabilities.js";
 import { expandTildePath } from "./utils/path.js";
 
 // ---------------------------------------------------------------------------
@@ -63,6 +69,7 @@ export const OPERATION_NAMES = [
   "set_coding_context",
   "recall_tier_explain",
   "recall_xray",
+  "namespace_writable",
   "wearables_status",
   "wearables_sync",
   "transcript_day",
@@ -235,6 +242,17 @@ export interface OperationSpec<In, Out> {
    * (cron, internal callers, no capability record) are unaffected (issue #1850).
    */
   readonly fleetWide?: boolean;
+  /**
+   * Alternate op allow-list: when set, the token may invoke this operation if
+   * its ops allow-list permits ANY of these ops (instead of the default rule
+   * that requires the op's own {@link name}). Lets a read-only diagnostic be
+   * granted by the write ops it diagnoses — e.g. `namespace_writable` is
+   * runnable by any token permitted to `observe`/`memory_store`, so a token
+   * minted before the diagnostic op existed can still call it. Unrestricted /
+   * legacy tokens (no ops axis) are unaffected. Keeps the boundary and any HTTP
+   * transport for the same op enforcing ONE policy.
+   */
+  readonly allowedByOps?: readonly OperationName[];
 }
 
 export interface BoundOperation<In = unknown, Out = unknown> {
@@ -408,7 +426,18 @@ export function defineOperation<In, Out>(spec: OperationSpec<In, Out>): BoundOpe
       // unrestricted. Default-deny ONLY when an ops axis is present — legacy
       // tokens (absent record) and explicit-unrestricted new tokens are
       // unaffected.
-      assertOperationAllowed(tokenCapabilityStore.getStore(), spec.name);
+      // When the op declares alternate grant ops, permit any of them (so the
+      // batch/MCP path matches an HTTP transport that relaxes the same op);
+      // otherwise the token must carry the op's own name. Unrestricted/legacy
+      // tokens pass either way (capabilityAllowsOp is true when no ops axis).
+      if (spec.allowedByOps && spec.allowedByOps.length > 0) {
+        const store = tokenCapabilityStore.getStore();
+        if (!spec.allowedByOps.some((op) => capabilityAllowsOp(store, op))) {
+          throw new EngramAccessForbiddenError(`token is not permitted to call operation: ${spec.name}`);
+        }
+      } else {
+        assertOperationAllowed(tokenCapabilityStore.getStore(), spec.name);
+      }
       // Fleet-wide / global maintenance ops (issue #1850 round 10): these run
       // across all namespaces with no `namespace` arg, so the tools/call
       // effective-namespace gate never fires. A namespace-SCOPED token must not

package/src/access-citation.test.ts

@@ -0,0 +1,72 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  recordCitationUsage,
+  type CitationUsageDependencies,
+} from "./access-citation.js";
+
+test("recordCitationUsage matches each citation to its cited path before fallback", async () => {
+  let tracked: { ids: string[]; paths: string[] } | null = null;
+  const deps: CitationUsageDependencies = {
+    resolveNamespace: () => "main",
+    getStorage: async () => ({
+      findExistingMemoryPaths: async () =>
+        new Map([["same-id", ["facts/other/same-id.md", "facts/cited/same-id.md"]]]),
+    }),
+    trackMemoryAccess: (ids, paths) => {
+      tracked = { ids, paths };
+    },
+  };
+  const result = await recordCitationUsage(deps, {
+    namespace: "main",
+    entries: [
+      { path: "facts/cited/same-id.md", lineStart: 1, lineEnd: 1, note: "cited" },
+      { path: "facts/other/same-id.md", lineStart: 1, lineEnd: 1, note: "other" },
+    ],
+    rolloutIds: [],
+  });
+
+  assert.deepEqual(result, { submitted: 2, matched: 2 });
+  assert.deepEqual(tracked, {
+    ids: ["same-id", "same-id"],
+    paths: ["facts/cited/same-id.md", "facts/other/same-id.md"],
+  });
+});
+
+test("recordCitationUsage resolves each cited path before selecting storage", async () => {
+  const storageCalls: string[] = [];
+  let tracked: { ids: string[]; paths: string[]; namespaces: Array<string | undefined> } | null = null;
+  const deps: CitationUsageDependencies = {
+    resolveNamespace: () => "main",
+    resolveNamespaceForPath: async (path) =>
+      path.startsWith("namespaces/team/") ? "team" : "main",
+    getStorage: async (namespace) => {
+      storageCalls.push(namespace);
+      return {
+        findExistingMemoryPaths: async (ids) =>
+          new Map(ids.map((id) => [id, [`${namespace}/${id}.md`]])),
+      };
+    },
+    trackMemoryAccess: (ids, paths, namespaces) => {
+      tracked = { ids, paths, namespaces: namespaces ?? [] };
+    },
+  };
+
+  const result = await recordCitationUsage(deps, {
+    namespace: "main",
+    entries: [
+      { path: "namespaces/team/facts/shared.md", lineStart: 1, lineEnd: 1, note: "team" },
+      { path: "facts/main.md", lineStart: 1, lineEnd: 1, note: "main" },
+    ],
+    rolloutIds: [],
+  });
+
+  assert.deepEqual(result, { submitted: 2, matched: 2 });
+  assert.deepEqual(storageCalls, ["team", "main"]);
+  assert.deepEqual(tracked, {
+    ids: ["shared", "main"],
+    paths: ["team/shared.md", "main/main.md"],
+    namespaces: ["team", "main"],
+  });
+});

package/src/access-citation.ts

@@ -0,0 +1,139 @@
+import { log } from "./logger.js";
+
+export interface CitationUsageRequest {
+  sessionId?: string;
+  namespace?: string;
+  authenticatedPrincipal?: string;
+  entries: Array<{ path: string; lineStart: number; lineEnd: number; note: string }>;
+  rolloutIds: string[];
+}
+
+export interface CitationStorage {
+  findExistingMemoryPaths(
+    ids: string[],
+    preferredPaths?: Map<string, string[]>,
+  ): Promise<Map<string, string[]>>;
+}
+
+export interface CitationUsageDependencies {
+  resolveNamespace: (
+    namespace: string | undefined,
+    sessionId: string | undefined,
+    authenticatedPrincipal: string | undefined,
+  ) => string;
+  resolveNamespaceForPath?: (
+    path: string,
+    fallbackNamespace: string,
+    sessionId: string | undefined,
+    authenticatedPrincipal: string | undefined,
+  ) => Promise<string>;
+  getStorage: (namespace: string) => Promise<CitationStorage>;
+  trackMemoryAccess: (
+    memoryIds: string[],
+    memoryPaths: string[],
+    memoryNamespaces?: Array<string | undefined>,
+  ) => void;
+}
+
+export interface CitationUsageResult {
+  submitted: number;
+  matched: number;
+}
+
+export async function recordCitationUsage(
+  deps: CitationUsageDependencies,
+  request: CitationUsageRequest,
+): Promise<CitationUsageResult> {
+  if (request.entries.length === 0) return { submitted: 0, matched: 0 };
+
+  const fallbackNamespace = deps.resolveNamespace(
+    request.namespace,
+    request.sessionId,
+    request.authenticatedPrincipal,
+  );
+  const memoryEntries = (await Promise.all(
+    request.entries.map(async (entry) => {
+      const basename = entry.path.split(/[\\/]/).pop() ?? entry.path;
+      const id = basename.endsWith(".md") ? basename.slice(0, -3) : basename;
+      if (id.length === 0) return null;
+      const namespace = deps.resolveNamespaceForPath
+        ? await deps.resolveNamespaceForPath(
+            entry.path,
+            fallbackNamespace,
+            request.sessionId,
+            request.authenticatedPrincipal,
+          )
+        : fallbackNamespace;
+      // Strip the resolved namespace prefix so citedPath is storage-relative for
+      // both preferred-path matching and the exact-match lookup below. Namespace
+      // names may contain "/", which the storage helper cannot strip alone (#2020).
+      const citedPath =
+        namespace && entry.path.startsWith(`${namespace}/`)
+          ? entry.path.slice(namespace.length + 1)
+          : entry.path;
+      return { id, citedPath, namespace };
+    }),
+  )).filter(
+    (
+      entry,
+    ): entry is { id: string; citedPath: string; namespace: string } =>
+      entry !== null,
+  );
+
+  if (memoryEntries.length === 0) return { submitted: 0, matched: 0 };
+
+  const matchedEntries: Array<{
+    id: string;
+    path: string;
+    namespace: string;
+  }> = [];
+  const entriesByNamespace = new Map<string, typeof memoryEntries>();
+  for (const entry of memoryEntries) {
+    const entries = entriesByNamespace.get(entry.namespace) ?? [];
+    entries.push(entry);
+    entriesByNamespace.set(entry.namespace, entries);
+  }
+
+  for (const [namespace, namespaceEntries] of entriesByNamespace) {
+    const storage = await deps.getStorage(namespace);
+    const ids = [...new Set(namespaceEntries.map((entry) => entry.id))];
+    const preferredPaths = new Map<string, string[]>();
+    for (const entry of namespaceEntries) {
+      const paths = preferredPaths.get(entry.id) ?? [];
+      paths.push(entry.citedPath);
+      preferredPaths.set(entry.id, paths);
+    }
+    const pathsById = await storage.findExistingMemoryPaths(ids, preferredPaths);
+    const remainingPathsById = new Map(
+      Array.from(pathsById.entries()).map(([id, paths]) => [id, [...paths]]),
+    );
+    for (const entry of namespaceEntries) {
+      const paths = remainingPathsById.get(entry.id);
+      if (!paths || paths.length === 0) continue;
+      const preferredIndex = paths.indexOf(entry.citedPath);
+      const pathIndex = preferredIndex >= 0 ? preferredIndex : 0;
+      const [memoryPath] = paths.splice(pathIndex, 1);
+      if (memoryPath) {
+        matchedEntries.push({
+          id: entry.id,
+          path: memoryPath,
+          namespace,
+        });
+      }
+    }
+  }
+
+  if (matchedEntries.length > 0) {
+    try {
+      deps.trackMemoryAccess(
+        matchedEntries.map((entry) => entry.id),
+        matchedEntries.map((entry) => entry.path),
+        matchedEntries.map((entry) => entry.namespace),
+      );
+    } catch {
+      log.debug("citation usage tracking: failed to record access for cited memories");
+    }
+  }
+
+  return { submitted: memoryEntries.length, matched: matchedEntries.length };
+}

package/src/access-errors.ts

@@ -1,2 +1,23 @@
 /** Authenticated caller lacks the required authorization (HTTP 403). */
 export class EngramAccessForbiddenError extends Error {}
+
+/** Invalid caller input (HTTP 400). */
+export class EngramAccessInputError extends Error {}
+
+/**
+ * A write rejected by the namespace write-ACL (issue #1888). Subclasses
+ * EngramAccessInputError so every existing catch/HTTP-400 mapping still
+ * applies, but carries the attempted namespace + principal so the observe/
+ * write surfaces can dead-letter the payload before re-throwing (fail-closed
+ * placement is unchanged; only the destroyed-payload behavior is fixed).
+ */
+export class NamespaceNotWritableError extends EngramAccessInputError {
+  constructor(
+    readonly attemptedNamespace: string,
+    readonly principal: string | undefined,
+    message?: string
+  ) {
+    super(message ?? `namespace is not writable: ${attemptedNamespace}`);
+    this.name = "NamespaceNotWritableError";
+  }
+}

package/src/access-fingerprint-parity.test.ts

@@ -0,0 +1,211 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import os from "node:os";
+import path from "node:path";
+import { mkdtemp, rm } from "node:fs/promises";
+import {
+  AccessIdempotencyStore,
+  hashAccessIdempotencyPayload,
+} from "./access-idempotency.js";
+import {
+  buildAccessWriteRequestFingerprint,
+  buildObserveRequestFingerprint,
+} from "./write-envelope.js";
+
+/**
+ * Issue #1989 PR3 — fingerprint byte-parity safety net.
+ *
+ * The access surfaces' idempotency hashes are persisted indefinitely
+ * (AccessIdempotencyStore: last-512 LRU, no time TTL). The shared builders
+ * MUST reproduce the historical inline payload literals exactly, or every
+ * stored key silently stops replaying / starts conflicting after upgrade.
+ *
+ * The `legacy*` literals below are verbatim copies of the inline objects
+ * that lived at the call sites before PR3 (access-observe-write-surface.ts
+ * memoryStore/suggestionSubmit, access-service.ts peek twins and observe).
+ * Do NOT "sync" them with the builders — their whole point is to pin the
+ * historical shape.
+ */
+
+type WriteRequestish = {
+  content: string;
+  category?: string;
+  confidence?: number;
+  tags?: string[];
+  entityRef?: string;
+  ttl?: string;
+  sourceReason?: string;
+  sourceConnector?: string;
+};
+
+function legacyWriteFingerprint(
+  request: WriteRequestish,
+  namespace: string,
+  schemaVersion: number,
+): Record<string, unknown> {
+  // Verbatim pre-PR3 shape — field order as it appeared in the source.
+  return {
+    schemaVersion,
+    content: request.content,
+    category: request.category,
+    confidence: request.confidence,
+    namespace,
+    tags: request.tags,
+    entityRef: request.entityRef,
+    ttl: request.ttl,
+    sourceReason: request.sourceReason,
+    sourceConnector: request.sourceConnector,
+  };
+}
+
+const FULL_WRITE: WriteRequestish = {
+  content: "User prefers dark mode",
+  category: "preference",
+  confidence: 0.8,
+  tags: ["ui", "preference", "ui"],
+  entityRef: "person-test-user",
+  ttl: "2027-01-01T00:00:00.000Z",
+  sourceReason: "operator request",
+  sourceConnector: "cli",
+};
+
+const SPARSE_WRITE: WriteRequestish = {
+  content: "Sparse fact with every optional absent",
+};
+
+test("memory_store/suggestion_submit fingerprints hash identically to the pre-PR3 literals", () => {
+  for (const request of [FULL_WRITE, SPARSE_WRITE]) {
+    for (const operation of ["memory_store", "suggestion_submit"] as const) {
+      const legacy = hashAccessIdempotencyPayload({
+        operation,
+        request: legacyWriteFingerprint(request, "default", 1),
+      });
+      const migrated = hashAccessIdempotencyPayload({
+        operation,
+        request: buildAccessWriteRequestFingerprint({
+          schemaVersion: 1,
+          namespace: "default",
+          content: request.content,
+          category: request.category,
+          confidence: request.confidence,
+          tags: request.tags,
+          entityRef: request.entityRef,
+          ttl: request.ttl,
+          sourceReason: request.sourceReason,
+          sourceConnector: request.sourceConnector,
+        }),
+      });
+      assert.equal(migrated, legacy, `${operation} fingerprint diverged for ${request.content}`);
+    }
+  }
+});
+
+test("write fingerprints keep raw values: tag order and duplicates are identity", () => {
+  const base = {
+    schemaVersion: 1,
+    namespace: "default",
+    content: "same content",
+  };
+  const a = hashAccessIdempotencyPayload(
+    buildAccessWriteRequestFingerprint({ ...base, tags: ["b", "a"] }),
+  );
+  const b = hashAccessIdempotencyPayload(
+    buildAccessWriteRequestFingerprint({ ...base, tags: ["a", "b"] }),
+  );
+  // The legacy shape hashed the RAW tags array; preserving stored state
+  // means preserving that (documented per-surface quirk until a versioned
+  // payload migration; the envelope's v1 payload sorts tags instead).
+  assert.notEqual(a, b, "raw tag order must remain identity, matching stored pre-PR3 hashes");
+});
+
+test("observe fingerprint hashes identically to the pre-PR3 literal", () => {
+  const request = {
+    sessionKey: "agent:main:test",
+    messages: [
+      { role: "user", content: "hello" },
+      { role: "assistant", content: "hi there" },
+    ],
+    namespace: undefined as string | undefined,
+    skipExtraction: false,
+    authenticatedPrincipal: "operator",
+    cwd: "/home/user/project",
+    projectTag: "acme",
+    sourceConnector: undefined as string | undefined,
+  };
+  const codingContext = { projectId: "origin:abc", branch: "main" };
+  for (const effective of [codingContext, null]) {
+    const legacy = hashAccessIdempotencyPayload({
+      operation: "observe",
+      request: {
+        sessionKey: request.sessionKey,
+        messages: request.messages,
+        namespace: request.namespace,
+        skipExtraction: request.skipExtraction,
+        authenticatedPrincipal: request.authenticatedPrincipal,
+        cwd: request.cwd,
+        projectTag: request.projectTag,
+        effectiveCodingContext: effective ?? null,
+        sourceConnector: request.sourceConnector,
+      },
+    });
+    const migrated = hashAccessIdempotencyPayload({
+      operation: "observe",
+      request: buildObserveRequestFingerprint({
+        sessionKey: request.sessionKey,
+        messages: request.messages,
+        namespace: request.namespace,
+        skipExtraction: request.skipExtraction,
+        authenticatedPrincipal: request.authenticatedPrincipal,
+        cwd: request.cwd,
+        projectTag: request.projectTag,
+        effectiveCodingContext: effective ?? null,
+        sourceConnector: request.sourceConnector,
+      }),
+    });
+    assert.equal(migrated, legacy);
+  }
+});
+
+test("stored idempotency state from before PR3 still replays after PR3 (state-file round-trip)", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-idem-parity-"));
+  try {
+    const statePath = path.join(dir, "state", "access-idempotency.json");
+    const preUpgrade = new AccessIdempotencyStore(statePath);
+    // A pre-PR3 deployment recorded this response under the inline-literal hash.
+    const preHash = hashAccessIdempotencyPayload({
+      operation: "memory_store",
+      request: legacyWriteFingerprint(FULL_WRITE, "default", 1),
+    });
+    const storedResponse = { operation: "memory_store", memoryId: "fact-pre-upgrade", accepted: true };
+    await preUpgrade.put("idem-key-1", preHash, storedResponse);
+
+    // Post-upgrade process: fresh store instance over the same state file,
+    // hash computed through the PR3 builder.
+    const postUpgrade = new AccessIdempotencyStore(statePath);
+    const postHash = hashAccessIdempotencyPayload({
+      operation: "memory_store",
+      request: buildAccessWriteRequestFingerprint({
+        schemaVersion: 1,
+        namespace: "default",
+        ...FULL_WRITE,
+      }),
+    });
+    const replay = await postUpgrade.get("idem-key-1", postHash);
+    assert.equal(replay.conflict, false, "same request must not conflict across the upgrade");
+    assert.deepEqual(replay.response, storedResponse, "pre-upgrade response must replay");
+
+    // And a DIFFERENT request on the same key still conflicts (hash still discriminates).
+    const differentHash = hashAccessIdempotencyPayload({
+      operation: "memory_store",
+      request: buildAccessWriteRequestFingerprint({
+        schemaVersion: 1,
+        namespace: "default",
+        content: "entirely different content",
+      }),
+    });
+    const conflict = await postUpgrade.get("idem-key-1", differentHash);
+    assert.equal(conflict.conflict, true, "different request on the same key must conflict");
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});