@remnic/core 9.7.15 → 9.9.0

package/src/access-memory-search-fanout.test.ts

@@ -0,0 +1,220 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  mergeMemorySearchDefaultFallback,
+  resolveMemorySearchDefaultFallback,
+} from "./access-memory-search-fanout.js";
+import type { ResolvedScopeProfilePlan } from "./namespaces/scope-profiles.js";
+import type { PluginConfig } from "./types.js";
+
+function pluginConfig(overrides: Partial<PluginConfig> = {}): PluginConfig {
+  return {
+    namespacesEnabled: true,
+    defaultNamespace: "default",
+    sharedNamespace: "shared",
+    namespacePolicies: [],
+    memoryDir: "/synthetic/mem",
+    ...overrides,
+  } as unknown as PluginConfig;
+}
+
+function profilePlan(overrides: Partial<ResolvedScopeProfilePlan> = {}): ResolvedScopeProfilePlan {
+  const base = overrides.baseNamespace ?? "operator-x";
+  const readOrder = overrides.profile?.readOrder ?? ["userProject", "userGlobal", "serverShared"];
+  // Build a layer per readOrder entry so the resolved-userGlobal-readable gate
+  // mirrors what resolveScopeProfilePlan produces (the unit under test reads
+  // profilePlan.layers, not the raw readOrder).
+  const layerKind: Record<string, "user-project" | "user-global" | "server-shared"> = {
+    userProject: "user-project",
+    userGlobal: "user-global",
+    serverShared: "server-shared",
+  };
+  const layerNamespace: Record<string, string> = {
+    userProject: `${base}-project`,
+    userGlobal: base,
+    serverShared: "shared",
+  };
+  const layers = readOrder.map((id) => ({
+    id,
+    kind: layerKind[id],
+    namespace: layerNamespace[id],
+    readable: true,
+    writable: id === "userProject",
+    promotable: id === "userGlobal",
+    reason: "test fixture",
+  }));
+  return {
+    profileId: "standard",
+    profile: {
+      readOrder,
+      writeDefault: "userProject",
+      promotionTargets: ["userGlobal", "serverShared"],
+      autoPromote: {
+        enabled: false,
+        targets: ["userGlobal"],
+        categories: ["fact"],
+        minConfidenceTier: "inferred",
+      },
+    },
+    baseNamespace: base,
+    writeLayer: "userProject",
+    writeNamespace: "",
+    readNamespaces: [base, "shared"],
+    promotionTargets: [],
+    warnings: [],
+    ...overrides,
+    // Recompute layers from the (possibly overridden) readOrder/baseNamespace
+    // unless the caller supplied their own.
+    layers: overrides.layers ?? layers,
+  } as unknown as ResolvedScopeProfilePlan;
+}
+
+// Wrapper that defaults defaultAtFlatRoot=true so each gate condition is
+// exercised independently (a `null` result means a NON-flat-root gate blocked,
+// not the flat-root gate). The dedicated flat-root-false test covers that gate.
+// `probed` records whether the provider was invoked, so the lazy-probe tests
+// can assert ineligible profiles never touch the default store.
+async function applyFallback(
+  plan: ResolvedScopeProfilePlan,
+  config: PluginConfig,
+  principal: string | undefined,
+  defaultAtFlatRoot = true,
+  probed: { value: boolean } = { value: false },
+): Promise<{ fallback: string | null; probed: boolean }> {
+  const result = await resolveMemorySearchDefaultFallback({
+    profilePlan: plan,
+    config,
+    principal,
+    defaultAtFlatRootProvider: () => {
+      probed.value = true;
+      return Promise.resolve(defaultAtFlatRoot);
+    },
+  });
+  return { fallback: result, probed: probed.value };
+}
+
+test("resolveMemorySearchDefaultFallback returns the default namespace on a flat-root legacy deployment (#2018)", async () => {
+  const { fallback, probed } = await applyFallback(profilePlan(), pluginConfig(), "operator-x");
+  assert.equal(fallback, "default");
+  assert.equal(probed, true, "the flat-root provider must be probed when cheap gates pass");
+});
+
+test("resolveMemorySearchDefaultFallback returns null when the default namespace is not at the flat root (#2056 r4)", async () => {
+  const { fallback } = await applyFallback(profilePlan(), pluginConfig(), "operator-x", false);
+  assert.equal(fallback, null);
+});
+
+test("resolveMemorySearchDefaultFallback returns null for a project-only lockdown and never probes storage (#1501, #2056 r6)", async () => {
+  const plan = profilePlan({
+    profile: {
+      readOrder: ["userProject"],
+      writeDefault: "userProject",
+      promotionTargets: [],
+      autoPromote: {
+        enabled: false,
+        targets: [],
+        categories: ["fact"],
+        minConfidenceTier: "explicit",
+      },
+    },
+  });
+  const { fallback, probed } = await applyFallback(plan, pluginConfig(), "operator-x");
+  assert.equal(fallback, null);
+  assert.equal(probed, false, "ineligible profiles must not probe the default store");
+});
+
+test("resolveMemorySearchDefaultFallback returns null when the profile self is already the default", async () => {
+  const { fallback } = await applyFallback(profilePlan({ baseNamespace: "default" }), pluginConfig(), "default");
+  assert.equal(fallback, null);
+});
+
+test("resolveMemorySearchDefaultFallback returns null when only serverShared (no userGlobal) is in readOrder", async () => {
+  // serverShared maps to sharedNamespace, NOT the default: a profile that
+  // reads only userProject + serverShared deliberately reads the shared
+  // namespace, not the default, so the fallback must not fire.
+  const plan = profilePlan({
+    profile: {
+      readOrder: ["userProject", "serverShared"],
+      writeDefault: "userProject",
+      promotionTargets: ["serverShared"],
+      autoPromote: {
+        enabled: false,
+        targets: ["serverShared"],
+        categories: ["fact"],
+        minConfidenceTier: "inferred",
+      },
+    },
+  });
+  const { fallback } = await applyFallback(plan, pluginConfig(), "operator-x");
+  assert.equal(fallback, null);
+});
+
+test("resolveMemorySearchDefaultFallback returns null when readOrder omits userGlobal even if a readable userGlobal layer is materialized (#2056 r3)", async () => {
+  // resolveScopeProfilePlan always materializes a userGlobal layer regardless
+  // of readOrder. A profile that intentionally omits userGlobal from readOrder
+  // must not get the fallback just because the materialized layer is readable.
+  const plan = profilePlan({
+    profile: {
+      readOrder: ["userProject"],
+      writeDefault: "userProject",
+      promotionTargets: [],
+      autoPromote: {
+        enabled: false,
+        targets: [],
+        categories: ["fact"],
+        minConfidenceTier: "explicit",
+      },
+    },
+    layers: [
+      { id: "userProject", kind: "user-project", namespace: "operator-x-project", readable: false, writable: false, promotable: false, reason: "no coding context" },
+      { id: "userGlobal", kind: "user-global", namespace: "operator-x", readable: true, writable: true, promotable: true, reason: "materialized but not in readOrder" },
+    ],
+  });
+  const { fallback } = await applyFallback(plan, pluginConfig(), "operator-x");
+  assert.equal(fallback, null);
+});
+
+test("resolveMemorySearchDefaultFallback ACL-gates the default namespace", async () => {
+  const config = pluginConfig({
+    defaultNamespace: "root",
+    namespacePolicies: [
+      { name: "root", readPrincipals: ["owner"], writePrincipals: ["owner"] },
+    ],
+  } as Partial<PluginConfig>);
+  const { fallback } = await applyFallback(profilePlan(), config, "operator-x");
+  assert.equal(fallback, null);
+});
+
+test("resolveMemorySearchDefaultFallback returns null when the resolved userGlobal layer is unreadable (#2056 r2)", async () => {
+  // userGlobal is listed in readOrder but resolved unreadable (e.g. a policy
+  // withholding the principal): a deliberate omission must not trigger the
+  // fallback.
+  const plan = profilePlan({
+    layers: [
+      { id: "userGlobal", kind: "user-global", namespace: "operator-x", readable: false, writable: false, promotable: false, reason: "policy withholds principal" },
+    ],
+  });
+  const { fallback } = await applyFallback(plan, pluginConfig(), "operator-x");
+  assert.equal(fallback, null);
+});
+
+test("resolveMemorySearchDefaultFallback returns null when the principal owns a dedicated namespace (#2056 r2 / #1501 privateOnly)", async () => {
+  // Multi-tenant scope-profile deployment: the principal has its own policy,
+  // so its self namespace is NOT the configured default. Reaching into the
+  // default would read another namespace — the fallback must not fire even
+  // though userGlobal is readable and baseNamespace differs from default.
+  const config = pluginConfig({
+    namespacePolicies: [
+      { name: "operator-x", readPrincipals: ["operator-x"], writePrincipals: ["operator-x"] },
+    ],
+  } as Partial<PluginConfig>);
+  const { fallback } = await applyFallback(profilePlan(), config, "operator-x");
+  assert.equal(fallback, null);
+});
+
+test("mergeMemorySearchDefaultFallback dedupes and passes through", () => {
+  assert.deepEqual(mergeMemorySearchDefaultFallback(["a", "b"], "default"), ["a", "b", "default"]);
+  assert.deepEqual(mergeMemorySearchDefaultFallback(["default", "b"], "default"), ["default", "b"]);
+  assert.deepEqual(mergeMemorySearchDefaultFallback(["a", "b"], null), ["a", "b"]);
+});

package/src/access-memory-search-fanout.ts

@@ -0,0 +1,168 @@
+import { canReadNamespace, defaultNamespaceForPrincipal } from "./namespaces/principal.js";
+import type { ResolvedScopeProfilePlan } from "./namespaces/scope-profiles.js";
+import { log } from "./logger.js";
+import type { SearchDegradation, SearchExecutionOptions } from "./search/port.js";
+import type { PluginConfig } from "./types.js";
+
+/**
+ * Issue #2018: `memory_search` has no sessionKey, so — unlike recall — it
+ * cannot resolve a coding overlay. With scope profiles active and no overlay,
+ * `expandScopeProfileReadNamespaces` derives a per-principal self namespace.
+ * On a legacy flat-root deployment the corpus still lives under the configured
+ * DEFAULT namespace's base collection, which that derived set excludes, so
+ * every query silently returned 0.
+ *
+ * Returns the configured default namespace to merge into the profile read set
+ * when the base collection must stay reachable, or `null` otherwise. ALL of:
+ *
+ *   - the default namespace is THIS principal's own self store — i.e. the
+ *     principal has no dedicated namespace policy, so
+ *     `defaultNamespaceForPrincipal` resolves to the configured default. On a
+ *     multi-tenant scope-profile deployment every principal has its own
+ *     policy; reaching into the default would read ANOTHER namespace, so the
+ *     fallback must not fire (#2056 r2 / #1501 privateOnly);
+ *   - the profile resolved `userGlobal` as a READABLE layer (not just listed
+ *     it in `readOrder`). `serverShared` maps to `sharedNamespace`, not the
+ *     default, and an unreadable resolved layer is a deliberate omission —
+ *     neither may trigger this fallback (#2056 r2);
+ *   - the profile's resolved self namespace is NOT already the default
+ *     (otherwise it is already in the set);
+ *   - the principal is authorized to read the default namespace.
+ *
+ * The flat-root signal is resolved lazily via {@link defaultAtFlatRootProvider}
+ * — the provider is only invoked once the cheap gates (readOrder consent,
+ * resolved-layer readability, principal-self, self-resolved-away, ACL) all
+ * pass, so a project-only / serverShared-only / unreadable-userGlobal
+ * profile never touches the default store (#2056 r6).
+ */
+export async function resolveMemorySearchDefaultFallback(options: {
+  profilePlan: ResolvedScopeProfilePlan;
+  config: PluginConfig;
+  principal: string | undefined;
+  /**
+   * True only when the configured default namespace's storage root equals
+   * `config.memoryDir` (the legacy flat-root layout). Invoked ONLY after the
+   * cheap gates pass, so ineligible profiles never probe the default store.
+   */
+  defaultAtFlatRootProvider: () => Promise<boolean>;
+}): Promise<string | null> {
+  const { profilePlan, config, principal } = options;
+  // Defense-in-depth: principalSelfIsDefault is only ever true alongside a
+  // flat-root default on legacy deployments (a hosted scope-profile principal
+  // with its own policy fails this even when default is at the flat root). It
+  // is NOT redundant — it keeps a policy-having principal isolated from the
+  // flat default corpus.
+  const principalSelfIsDefault =
+    defaultNamespaceForPrincipal(principal, config) === config.defaultNamespace;
+  // readOrder must explicitly intend userGlobal (resolveScopeProfilePlan
+  // always materializes a userGlobal layer even when readOrder omits it, so
+  // the layer's presence alone is not consent), AND that layer must have
+  // resolved readable (a listed-but-unreadable layer is a deliberate
+  // omission). serverShared maps to sharedNamespace, not the default.
+  const profileIntendsUserGlobal =
+    profilePlan.profile.readOrder.includes("userGlobal");
+  const userGlobalLayerReadable = profilePlan.layers.some(
+    (layer) => layer.id === "userGlobal" && layer.readable && Boolean(layer.namespace),
+  );
+  const selfResolvedAwayFromDefault =
+    profilePlan.baseNamespace !== config.defaultNamespace;
+  if (
+    !(
+      profileIntendsUserGlobal &&
+      userGlobalLayerReadable &&
+      principalSelfIsDefault &&
+      selfResolvedAwayFromDefault &&
+      canReadNamespace(principal, config.defaultNamespace, config)
+    )
+  ) {
+    // Cheap gates failed — never probe the default store.
+    return null;
+  }
+  const defaultAtFlatRoot = await options.defaultAtFlatRootProvider();
+  return defaultAtFlatRoot ? config.defaultNamespace : null;
+}
+
+/**
+ * Issue #2018 r4: the explicit legacy-flat-root signal — the configured
+ * default namespace's storage root equals `config.memoryDir`. Resolved from
+ * storage (not inferred from config/policy shape) so a hosted scope-profile
+ * deployment never reaches into the default corpus.
+ */
+export async function defaultNamespaceAtFlatRoot(
+  getStorage: (namespace: string) => Promise<{ dir: string }>,
+  config: { defaultNamespace: string; memoryDir: string },
+): Promise<boolean> {
+  const storage = await getStorage(config.defaultNamespace);
+  return storage.dir === config.memoryDir;
+}
+
+/**
+ * Merge the {@link resolveMemorySearchDefaultFallback} result into a profile
+ * read-namespace set without duplicates. Returns the original set unchanged
+ * when no fallback applies.
+ */
+export function mergeMemorySearchDefaultFallback(
+  profileNamespaces: string[],
+  fallback: string | null,
+): string[] {
+  if (!fallback || profileNamespaces.includes(fallback)) return profileNamespaces;
+  return [...profileNamespaces, fallback];
+}
+
+
+/**
+ * Run a memory_search namespace fanout with the observability required by
+ * issue #2018: a silent empty fanout is surfaced as a warning (never silently
+ * empty), and per-namespace backend degradations — which memory_search
+ * previously swallowed — are collected and warned so a missing collection
+ * looks different from a genuine no-matches.
+ *
+ * `search` is injected so this stays free of the orchestrator type.
+ */
+export async function runMemorySearchFanout<TResult>(options: {
+  query: string;
+  namespaces: string[];
+  maxResults?: number;
+  principal?: string;
+  requestedNamespace?: string;
+  collection?: string;
+  search(params: {
+    query: string;
+    namespaces: string[];
+    maxResults?: number;
+    mode: "search";
+    execution: SearchExecutionOptions;
+  }): Promise<TResult[]>;
+}): Promise<TResult[]> {
+  const { query, namespaces, maxResults, principal, requestedNamespace, collection } = options;
+  if (namespaces.length === 0) {
+    log.warn("memory_search resolved zero readable namespaces", {
+      principal: principal ?? null,
+      requestedNamespace: requestedNamespace ?? null,
+      collection: collection ?? null,
+    });
+    return [];
+  }
+  log.debug("memory_search fanout", {
+    principal: principal ?? null,
+    namespaces,
+    collection: collection ?? null,
+  });
+  const degradations: SearchDegradation[] = [];
+  const results = await options.search({
+    query,
+    namespaces,
+    maxResults,
+    mode: "search",
+    execution: {
+      onDegradation: (degradation) => degradations.push(degradation),
+    },
+  });
+  if (degradations.length > 0) {
+    log.warn("memory_search backend degradation", {
+      principal: principal ?? null,
+      degradations,
+    });
+  }
+  return results;
+}

package/src/access-observe-quarantine.test.ts

@@ -0,0 +1,172 @@
+/**
+ * Issue #1888: a write rejected by the namespace write-ACL must be
+ * dead-lettered (payload preserved for replay) AND still throw loudly — the
+ * fail-closed placement is unchanged; only the destroyed-payload behavior is
+ * fixed.
+ */
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { memoryStoreRequestSchema, observeRequestSchema } from "./access-schema.js";
+import { EngramAccessService, NamespaceNotWritableError } from "./access-service.js";
+import { Orchestrator } from "./orchestrator.js";
+import type { CodingContext, PluginConfig } from "./types.js";
+import { WriteQuarantineStore } from "./write-quarantine.js";
+
+function makeService(memoryDir: string): EngramAccessService {
+  const orch = Object.create(Orchestrator.prototype) as Orchestrator;
+  const internals = orch as unknown as {
+    config: PluginConfig;
+    _codingContextBySession: Map<string, CodingContext>;
+  };
+  internals.config = {
+    namespacesEnabled: true,
+    defaultNamespace: "default",
+    sharedNamespace: "shared",
+    namespacePolicies: [],
+    codingMode: { projectScope: true },
+    memoryDir,
+  } as unknown as PluginConfig;
+  internals._codingContextBySession = new Map();
+  return new EngramAccessService(orch);
+}
+
+async function withService(fn: (service: EngramAccessService, dir: string) => Promise<void>): Promise<void> {
+  const dir = await mkdtemp(path.join(tmpdir(), "remnic-1888-"));
+  try {
+    await fn(makeService(dir), dir);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+}
+
+test("memory_store rejected by the namespace ACL is quarantined and still throws (#1888)", async () => {
+  await withService(async (service, dir) => {
+    await assert.rejects(
+      service.memoryStore({
+        content: "secret data",
+        namespace: "victim-secret",
+        authenticatedPrincipal: "alice",
+        sessionKey: "s-1",
+      } as unknown as Parameters<EngramAccessService["memoryStore"]>[0]),
+      NamespaceNotWritableError
+    );
+
+    const records = await new WriteQuarantineStore(dir).list();
+    assert.equal(records.length, 1);
+    assert.equal(records[0]?.operation, "memory_store");
+    assert.equal(records[0]?.principal, "alice");
+    assert.equal(records[0]?.attemptedNamespace, "victim-secret");
+    assert.equal((records[0]?.payload as { content?: string } | undefined)?.content, "secret data");
+  });
+});
+
+test("suggestion_submit rejected by the namespace ACL is quarantined and still throws (#1888)", async () => {
+  await withService(async (service, dir) => {
+    await assert.rejects(
+      service.suggestionSubmit({
+        content: "a suggestion",
+        namespace: "victim-secret",
+        authenticatedPrincipal: "alice",
+        sessionKey: "s-2",
+      } as unknown as Parameters<EngramAccessService["suggestionSubmit"]>[0]),
+      NamespaceNotWritableError
+    );
+
+    const records = await new WriteQuarantineStore(dir).list();
+    assert.equal(records.length, 1);
+    assert.equal(records[0]?.operation, "suggestion_submit");
+  });
+});
+
+test("a non-ACL input error on a writable namespace is NOT quarantined (#1888)", async () => {
+  await withService(async (service, dir) => {
+    // `default` is writable, so namespace resolution succeeds; the request then
+    // fails a later validation (unsupported schemaVersion). That rejection is
+    // not a namespace-ACL denial, so nothing is quarantined.
+    await assert.rejects(
+      service.memoryStore({
+        content: "fine",
+        namespace: "default",
+        schemaVersion: 999,
+        authenticatedPrincipal: "alice",
+        sessionKey: "s-3",
+      } as unknown as Parameters<EngramAccessService["memoryStore"]>[0]),
+      (err: unknown) => err instanceof Error && !(err instanceof NamespaceNotWritableError)
+    );
+
+    assert.equal(await new WriteQuarantineStore(dir).count(), 0);
+  });
+});
+
+test("a dry-run rejected write is NOT quarantined (#1888)", async () => {
+  await withService(async (service, dir) => {
+    await assert.rejects(
+      service.memoryStore({
+        content: "dry run",
+        namespace: "victim-secret",
+        authenticatedPrincipal: "alice",
+        sessionKey: "s-4",
+        dryRun: true,
+      } as unknown as Parameters<EngramAccessService["memoryStore"]>[0]),
+      NamespaceNotWritableError
+    );
+
+    // A dry run is a no-persist validation, so nothing is parked.
+    assert.equal(await new WriteQuarantineStore(dir).count(), 0);
+  });
+});
+
+test("suppressQuarantine skips dead-lettering an ACL-rejected write; without it it is parked (#1888 replay)", async () => {
+  await withService(async (service, dir) => {
+    const store = new WriteQuarantineStore(dir);
+    // Replay re-submits with suppressQuarantine set: the ACL rejection still
+    // throws, but the attempt is NOT re-parked, so replay cannot duplicate it.
+    await assert.rejects(
+      service.memoryStore({
+        content: "replayed",
+        namespace: "victim-secret",
+        authenticatedPrincipal: "alice",
+        sessionKey: "s-5",
+        suppressQuarantine: true,
+      } as unknown as Parameters<EngramAccessService["memoryStore"]>[0]),
+      NamespaceNotWritableError
+    );
+    assert.equal(await store.count(), 0);
+
+    // The identical write WITHOUT the flag is dead-lettered as before.
+    await assert.rejects(
+      service.memoryStore({
+        content: "replayed",
+        namespace: "victim-secret",
+        authenticatedPrincipal: "alice",
+        sessionKey: "s-5",
+      } as unknown as Parameters<EngramAccessService["memoryStore"]>[0]),
+      NamespaceNotWritableError
+    );
+    assert.equal(await store.count(), 1);
+  });
+});
+
+test("public write schemas strip suppressQuarantine — external callers cannot skip quarantine", () => {
+  // The write surface honors suppressQuarantine (in-process replay sets it), but
+  // it is NOT a field on the public HTTP/MCP request schemas. zod strips unknown
+  // keys, so a body carrying it is scrubbed before the service — and the access
+  // boundary re-validates the cleaned envelope — keeping ACL-rejected external
+  // writes parked rather than silently discarded.
+  const stored = memoryStoreRequestSchema.parse({
+    sessionKey: "s",
+    content: "hello",
+    suppressQuarantine: true,
+  } as Record<string, unknown>);
+  assert.equal("suppressQuarantine" in stored, false);
+  const observed = observeRequestSchema.parse({
+    sessionKey: "s",
+    messages: [{ role: "user", content: "hi" }],
+    suppressQuarantine: true,
+  } as Record<string, unknown>);
+  assert.equal("suppressQuarantine" in observed, false);
+});