@remnic/core 1.1.31 → 9.3.515

package/src/namespaces/principal.test.ts

@@ -12,7 +12,7 @@
 import assert from "node:assert/strict";
 import test from "node:test";
 
-import { canReadNamespace } from "./principal.js";
+import { canReadNamespace, resolvePrincipal } from "./principal.js";
 import type { PluginConfig } from "../types.js";
 
 // ──────────────────────────────────────────────────────────────────────────
@@ -26,6 +26,7 @@ import type { PluginConfig } from "../types.js";
 function makeConfig(
   namespacesEnabled: boolean,
   policies: Array<{ name: string; readPrincipals: string[]; writePrincipals: string[] }> = [],
+  overrides: Partial<PluginConfig> = {},
 ): PluginConfig {
   return {
     namespacesEnabled,
@@ -46,6 +47,7 @@ function makeConfig(
     consolidation: { enabled: false },
     extraction: { enabled: false },
     lcm: { enabled: false },
+    ...overrides,
   } as unknown as PluginConfig;
 }
 
@@ -193,3 +195,37 @@ test("canReadNamespace: namespacesEnabled=true + no policy for namespace → all
   assert.equal(canReadNamespace("alice", "shared", config), true, "shared namespace is implicitly readable");
   assert.equal(canReadNamespace("alice", "unknown-ns", config), false, "unknown namespace without policy is denied");
 });
+
+test("resolvePrincipal: safe regex rules can resolve a principal", () => {
+  const config = makeConfig(true, [], {
+    principalFromSessionKeyMode: "regex",
+    principalFromSessionKeyRules: [
+      { match: "^codex-session$", principal: "codex" },
+    ],
+  });
+
+  assert.equal(resolvePrincipal("codex-session", config), "codex");
+});
+
+test("resolvePrincipal: unsafe regex rules are skipped without evaluating crafted session keys", () => {
+  const config = makeConfig(true, [], {
+    principalFromSessionKeyMode: "regex",
+    principalFromSessionKeyRules: [
+      { match: "(a+)+$", principal: "blocked" },
+    ],
+  });
+  const craftedSessionKey = `${"a".repeat(10_000)}!`;
+
+  assert.equal(resolvePrincipal(craftedSessionKey, config), "default");
+});
+
+test("resolvePrincipal: regex rules are skipped for overlong session keys", () => {
+  const config = makeConfig(true, [], {
+    principalFromSessionKeyMode: "regex",
+    principalFromSessionKeyRules: [
+      { match: "^a+$", principal: "blocked" },
+    ],
+  });
+
+  assert.equal(resolvePrincipal("a".repeat(1_000), config), "default");
+});

package/src/namespaces/principal.ts

@@ -1,4 +1,16 @@
 import type { PluginConfig } from "../types.js";
+import { isLikelyUnsafeRegex } from "../routing/engine.js";
+
+const MAX_REGEX_SESSION_KEY_LENGTH = 512;
+
+function compileSafePrincipalRegex(pattern: string): RegExp | null {
+  if (isLikelyUnsafeRegex(pattern)) return null;
+  try {
+    return new RegExp(pattern);
+  } catch {
+    return null;
+  }
+}
 
 export function resolvePrincipal(sessionKey: string | undefined, config: PluginConfig): string {
   if (!config.namespacesEnabled) return "default";
@@ -17,12 +29,12 @@ export function resolvePrincipal(sessionKey: string | undefined, config: PluginC
       if (sk === r.match) return r.principal;
     }
   } else if (mode === "regex") {
-    for (const r of rules) {
-      try {
-        const re = new RegExp(r.match);
-        if (re.test(sk)) return r.principal;
-      } catch {
-        // ignore invalid regex
+    if (sk.length <= MAX_REGEX_SESSION_KEY_LENGTH) {
+      for (const r of rules) {
+        const re = compileSafePrincipalRegex(r.match);
+        if (re?.test(sk)) {
+          return r.principal;
+        }
       }
     }
   }
@@ -83,4 +95,3 @@ export function recallNamespacesForPrincipal(principal: string, config: PluginCo
 
   return out;
 }
-

package/src/namespaces/search.test.ts

@@ -6,8 +6,12 @@ import type { PluginConfig, QmdSearchResult } from "../types.js";
 
 class FakeBackend implements SearchBackend {
   updates = 0;
+  calls: Array<{ method: string; collection: string | undefined }> = [];
 
-  constructor(private readonly globalUpdate: boolean) {}
+  constructor(
+    private readonly globalUpdate: boolean,
+    private readonly results: QmdSearchResult[] = [],
+  ) {}
 
   async probe(): Promise<boolean> {
     return true;
@@ -21,23 +25,27 @@ class FakeBackend implements SearchBackend {
     return "fake";
   }
 
-  async search(): Promise<QmdSearchResult[]> {
-    return [];
+  async search(_query?: string, collection?: string): Promise<QmdSearchResult[]> {
+    this.calls.push({ method: "search", collection });
+    return this.results;
   }
 
   async searchGlobal(): Promise<QmdSearchResult[]> {
     return [];
   }
 
-  async bm25Search(): Promise<QmdSearchResult[]> {
+  async bm25Search(_query?: string, collection?: string): Promise<QmdSearchResult[]> {
+    this.calls.push({ method: "bm25", collection });
     return [];
   }
 
-  async vectorSearch(): Promise<QmdSearchResult[]> {
+  async vectorSearch(_query?: string, collection?: string): Promise<QmdSearchResult[]> {
+    this.calls.push({ method: "vector", collection });
     return [];
   }
 
-  async hybridSearch(): Promise<QmdSearchResult[]> {
+  async hybridSearch(_query?: string, collection?: string): Promise<QmdSearchResult[]> {
+    this.calls.push({ method: "hybrid", collection });
     return [];
   }
 
@@ -103,3 +111,65 @@ test("updateNamespaces still updates every namespace for scoped backends", async
   assert.equal(updated, 3);
   assert.equal(created.reduce((sum, backend) => sum + backend.updates, 0), 3);
 });
+
+test("searchAcrossNamespaces preserves same path results from distinct namespaces", async () => {
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+    (scopedConfig) => {
+      const namespace = scopedConfig.memoryDir.endsWith("/shared") ? "shared" : "main";
+      return new FakeBackend(false, [
+        {
+          path: "facts/a.md",
+          docid: "a",
+          score: namespace === "main" ? 0.9 : 0.8,
+          snippet: namespace,
+        },
+      ]);
+    },
+  );
+
+  const results = await router.searchAcrossNamespaces({
+    query: "a",
+    namespaces: ["main", "shared"],
+    maxResults: 10,
+  });
+
+  assert.deepEqual(
+    results.map((result) => result.snippet),
+    ["main", "shared"],
+  );
+});
+
+test("searchAcrossNamespaces passes scoped collection to backend search methods", async () => {
+  const created: FakeBackend[] = [];
+  const router = new NamespaceSearchRouter(
+    config(),
+    { storageFor: async (namespace: string) => ({ dir: `/tmp/remnic/${namespace}` }) },
+    () => {
+      const backend = new FakeBackend(false);
+      created.push(backend);
+      return backend;
+    },
+  );
+
+  for (const mode of ["search", "hybrid", "bm25", "vector"] as const) {
+    router.clearCache();
+    created.length = 0;
+    await router.searchAcrossNamespaces({
+      query: "a",
+      namespaces: ["main", "shared"],
+      maxResults: 10,
+      mode,
+    });
+
+    assert.deepEqual(
+      created.flatMap((backend) => backend.calls.map((call) => call.collection)),
+      [
+        "openclaw-engram--ns-6d61696e",
+        "openclaw-engram--ns-736861726564",
+      ],
+      mode,
+    );
+  }
+});

package/src/namespaces/search.ts

@@ -1,6 +1,7 @@
 import type { PluginConfig, QmdSearchResult } from "../types.js";
 import type { SearchBackend, SearchExecutionOptions, SearchQueryOptions } from "../search/port.js";
 import { createSearchBackend } from "../search/factory.js";
+import { namespaceIdentityToken, normalizeNamespaceIdentity } from "./identity.js";
 
 export function namespaceCollectionName(
   baseCollection: string,
@@ -10,8 +11,8 @@ export function namespaceCollectionName(
     useLegacyDefaultCollection?: boolean;
   },
 ): string {
-  const trimmed = namespace.trim();
-  const defaultNamespace = options?.defaultNamespace?.trim() || "default";
+  const trimmed = normalizeNamespaceIdentity(namespace);
+  const defaultNamespace = normalizeNamespaceIdentity(options?.defaultNamespace ?? "") || "default";
   if (
     options?.useLegacyDefaultCollection === true &&
     trimmed === defaultNamespace
@@ -19,15 +20,7 @@ export function namespaceCollectionName(
     return baseCollection;
   }
 
-  const normalized = trimmed
-    .toLowerCase()
-    .replace(/[^a-z0-9._-]+/g, "-");
-  let start = 0;
-  let end = normalized.length;
-  while (start < end && normalized[start] === "-") start += 1;
-  while (end > start && normalized[end - 1] === "-") end -= 1;
-  const token = normalized.slice(start, end) || defaultNamespace;
-  return `${baseCollection}--ns--${token}`;
+  return `${baseCollection}--${namespaceIdentityToken(trimmed || defaultNamespace)}`;
 }
 
 type StorageRouterLike = {
@@ -75,23 +68,31 @@ export class NamespaceSearchRouter {
     const resultsByNamespace = await Promise.all(
       namespaces.map(async (namespace) => {
         const record = await this.backendRecordFor(namespace);
-        if (!record.available || record.collectionState === "missing") return [] as QmdSearchResult[];
+        if (!record.available || record.collectionState === "missing") {
+          return { namespace, results: [] as QmdSearchResult[] };
+        }
+        let results: QmdSearchResult[];
         switch (method) {
           case "hybrid":
-            return await record.backend.hybridSearch(query, undefined, maxResults, options.execution);
+            results = await record.backend.hybridSearch(query, record.collection, maxResults, options.execution);
+            break;
           case "bm25":
-            return await record.backend.bm25Search(query, undefined, maxResults, options.execution);
+            results = await record.backend.bm25Search(query, record.collection, maxResults, options.execution);
+            break;
           case "vector":
-            return await record.backend.vectorSearch(query, undefined, maxResults, options.execution);
+            results = await record.backend.vectorSearch(query, record.collection, maxResults, options.execution);
+            break;
           default:
-            return await record.backend.search(
+            results = await record.backend.search(
               query,
-              undefined,
+              record.collection,
               maxResults,
               options.searchOptions,
               options.execution,
             );
+            break;
         }
+        return { namespace, results };
       }),
     );
 
@@ -205,14 +206,14 @@ export class NamespaceSearchRouter {
 }
 
 function mergeNamespaceSearchResults(
-  lists: QmdSearchResult[][],
+  lists: Array<{ namespace: string; results: QmdSearchResult[] }>,
   maxResults: number,
 ): QmdSearchResult[] {
   const merged = new Map<string, QmdSearchResult>();
 
-  for (const list of lists) {
-    for (const result of list) {
-      const key = result.path || result.docid;
+  for (const { namespace, results } of lists) {
+    for (const result of results) {
+      const key = `${namespace}\0${result.path || result.docid}`;
       const existing = merged.get(key);
       if (!existing) {
         merged.set(key, result);

package/src/namespaces/storage.ts

@@ -1,8 +1,10 @@
 import path from "node:path";
-import { access } from "node:fs/promises";
+import { access, lstat, readdir } from "node:fs/promises";
 import { isSafeRouteNamespace } from "../routing/engine.js";
 import { StorageManager } from "../storage.js";
 import type { PluginConfig } from "../types.js";
+import { ALL_CATEGORY_DIRS } from "../utils/category-dir.js";
+import { namespaceIdentityToken, normalizeNamespaceIdentity } from "./identity.js";
 
 async function exists(p: string): Promise<boolean> {
   try {
@@ -13,11 +15,67 @@ async function exists(p: string): Promise<boolean> {
   }
 }
 
+async function hasStoredEntries(p: string): Promise<boolean> {
+  try {
+    const entry = await lstat(p);
+    if (entry.isSymbolicLink()) return true;
+    if (!entry.isDirectory()) return true;
+    const children = await readdir(p, { withFileTypes: true });
+    for (const child of children) {
+      const childPath = path.join(p, child.name);
+      if (child.isSymbolicLink() || child.isFile()) return true;
+      if (child.isDirectory() && (await hasStoredEntries(childPath))) return true;
+    }
+    return false;
+  } catch {
+    return false;
+  }
+}
+
+const LEGACY_NAMESPACE_CONTENT_CHILDREN = [
+  ...ALL_CATEGORY_DIRS,
+  "entities",
+  "artifacts",
+  "identity",
+  "config",
+  "summaries",
+  "profile.md",
+] as const;
+
+const LEGACY_NAMESPACE_RUNTIME_CHILDREN = ["state"] as const;
+
+async function hasAnyLegacyData(
+  rootDir: string,
+  options: { includeRuntimeState?: boolean } = {},
+): Promise<boolean> {
+  const children = options.includeRuntimeState === true
+    ? [...LEGACY_NAMESPACE_CONTENT_CHILDREN, ...LEGACY_NAMESPACE_RUNTIME_CHILDREN]
+    : LEGACY_NAMESPACE_CONTENT_CHILDREN;
+  for (const child of children) {
+    if (await hasStoredEntries(path.join(rootDir, child))) return true;
+  }
+  return false;
+}
+
+async function hasAnyNamespaceStorageMarker(
+  rootDir: string,
+  options: { includeRuntimeState?: boolean } = {},
+): Promise<boolean> {
+  const children = options.includeRuntimeState === true
+    ? [...LEGACY_NAMESPACE_CONTENT_CHILDREN, ...LEGACY_NAMESPACE_RUNTIME_CHILDREN]
+    : LEGACY_NAMESPACE_CONTENT_CHILDREN;
+  for (const child of children) {
+    if (await exists(path.join(rootDir, child))) return true;
+  }
+  return false;
+}
+
 /**
  * Storage routing for namespaces.
  *
  * Compatibility note:
- * - When namespaces are enabled, non-default namespaces live under `memoryDir/namespaces/<ns>`.
+ * - When namespaces are enabled, existing raw namespace roots are preserved.
+ *   New namespace roots use tokenized names under `memoryDir/namespaces/<token>`.
  * - The default namespace continues to use the legacy `memoryDir` root unless the caller
  *   has created `memoryDir/namespaces/<defaultNamespace>` (in which case we use that).
  *
@@ -31,38 +89,62 @@ export class NamespaceStorageRouter {
   constructor(private readonly config: PluginConfig) {}
 
   private async defaultNamespaceRoot(): Promise<string> {
-    if (this.defaultNsRootResolved) return this.defaultNsRootResolved;
     if (!this.config.namespacesEnabled) {
       this.defaultNsRootResolved = this.config.memoryDir;
       return this.defaultNsRootResolved;
     }
 
-    const nsDir = path.join(this.config.memoryDir, "namespaces", this.config.defaultNamespace);
-    this.defaultNsRootResolved = (await exists(nsDir)) ? nsDir : this.config.memoryDir;
+    const legacyNsDir = path.join(this.config.memoryDir, "namespaces", this.config.defaultNamespace);
+    const tokenizedNsDir = path.join(
+      this.config.memoryDir,
+      "namespaces",
+      namespaceIdentityToken(this.config.defaultNamespace),
+    );
+    const tokenizedHasData =
+      (await exists(tokenizedNsDir)) && (await hasAnyNamespaceStorageMarker(tokenizedNsDir, { includeRuntimeState: true }));
+    const nsDir = tokenizedHasData
+      ? tokenizedNsDir
+      : (await exists(legacyNsDir)) ? legacyNsDir : tokenizedNsDir;
+    this.defaultNsRootResolved =
+      (await exists(nsDir)) && !(await hasAnyLegacyData(this.config.memoryDir))
+        ? nsDir
+        : this.config.memoryDir;
     return this.defaultNsRootResolved;
   }
 
-  private namespaceRootSync(namespace: string): string {
+  private async namespaceRoot(namespace: string): Promise<string> {
     // NOTE: only used after defaultNamespaceRoot() resolution.
     if (!this.config.namespacesEnabled) return this.config.memoryDir;
     if (namespace === this.config.defaultNamespace) {
       return this.defaultNsRootResolved ?? this.config.memoryDir;
     }
-    return path.join(this.config.memoryDir, "namespaces", namespace);
+    const legacyRoot = path.join(this.config.memoryDir, "namespaces", namespace);
+    const tokenizedRoot = path.join(this.config.memoryDir, "namespaces", namespaceIdentityToken(namespace));
+    if ((await exists(tokenizedRoot)) && (await hasAnyNamespaceStorageMarker(tokenizedRoot, { includeRuntimeState: true }))) {
+      return tokenizedRoot;
+    }
+    return (await exists(legacyRoot)) ? legacyRoot : tokenizedRoot;
   }
 
   async storageFor(namespace: string): Promise<StorageManager> {
-    const ns = namespace || this.config.defaultNamespace;
+    const ns = normalizeNamespaceIdentity(namespace || this.config.defaultNamespace);
     if (ns !== this.config.defaultNamespace && !isSafeRouteNamespace(ns)) {
       throw new Error(`unsafe namespace: ${ns}`);
     }
-    if (this.cache.has(ns)) return this.cache.get(ns)!;
 
+    let root: string;
     if (ns === this.config.defaultNamespace) {
-      await this.defaultNamespaceRoot();
+      root = await this.defaultNamespaceRoot();
+      const cached = this.cache.get(ns);
+      if (cached && cached.dir === root) {
+        return cached;
+      }
+    } else {
+      const cached = this.cache.get(ns);
+      root = await this.namespaceRoot(ns);
+      if (cached && cached.dir === root) return cached;
     }
 
-    const root = this.namespaceRootSync(ns);
     const sm = new StorageManager(root, this.config.entitySchemas);
     // Propagate the inline-attribution template so that router-created storages
     // (used by extraction and shared-promotion paths) strip citations consistently,

package/src/native-knowledge.ts

@@ -472,10 +472,26 @@ async function resolveCandidatePaths(options: {
   identityVariantMode: "recall" | "disk";
 }): Promise<string[]> {
   const out = new Set<string>();
+  const workspaceRoot = path.resolve(options.workspaceDir);
+  const resolveWorkspaceFile = (relativePath: string): string | null => {
+    if (path.isAbsolute(relativePath)) return null;
+    const resolved = path.resolve(workspaceRoot, relativePath);
+    const relative = path.relative(workspaceRoot, resolved);
+    if (relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
+      return null;
+    }
+    return resolved;
+  };
+  const safeIdentityNamespace = (namespace: string): string | null => {
+    if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(namespace)) return null;
+    if (namespace.includes("..")) return null;
+    return namespace;
+  };
   for (const rel of options.includeFiles) {
     const trimmed = rel.trim();
     if (!trimmed) continue;
-    const candidatePath = path.join(options.workspaceDir, trimmed);
+    const candidatePath = resolveWorkspaceFile(trimmed);
+    if (candidatePath === null) continue;
     out.add(candidatePath);
     if (path.basename(trimmed).toLowerCase() !== "identity.md") continue;
 
@@ -484,7 +500,10 @@ async function resolveCandidatePaths(options: {
       if (!Array.isArray(options.recallNamespaces)) continue;
       for (const namespace of options.recallNamespaces) {
         if (!namespace || namespace === options.defaultNamespace) continue;
-        out.add(path.join(options.workspaceDir, relativeDir, `IDENTITY.${namespace}.md`));
+        const safeNamespace = safeIdentityNamespace(namespace);
+        if (safeNamespace === null) continue;
+        const variantPath = resolveWorkspaceFile(path.join(relativeDir, `IDENTITY.${safeNamespace}.md`));
+        if (variantPath !== null) out.add(variantPath);
       }
       continue;
     }
@@ -498,7 +517,8 @@ async function resolveCandidatePaths(options: {
     }
     for (const entry of entries) {
       if (!/^identity\.[^.\/]+\.md$/i.test(entry)) continue;
-      out.add(path.join(options.workspaceDir, relativeDir, entry));
+      const variantPath = resolveWorkspaceFile(path.join(relativeDir, entry));
+      if (variantPath !== null) out.add(variantPath);
     }
   }
   return Array.from(out);

package/src/negative.ts

@@ -10,9 +10,51 @@ export interface NegativeExampleEntry {
 
 type NegativeState = Record<string, NegativeExampleEntry>;
 
+const POISONOUS_NEGATIVE_MEMORY_IDS = new Set(["__proto__", "prototype", "constructor"]);
+
+function createNegativeState(): NegativeState {
+  return Object.create(null) as NegativeState;
+}
+
+function isSafeNegativeMemoryId(memoryId: unknown): memoryId is string {
+  return (
+    typeof memoryId === "string" &&
+    memoryId.length > 0 &&
+    !POISONOUS_NEGATIVE_MEMORY_IDS.has(memoryId)
+  );
+}
+
+function sanitizeNegativeState(value: unknown): NegativeState {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return createNegativeState();
+
+  const state = createNegativeState();
+  for (const [memoryId, rawEntry] of Object.entries(value)) {
+    if (!isSafeNegativeMemoryId(memoryId)) continue;
+    if (!rawEntry || typeof rawEntry !== "object" || Array.isArray(rawEntry)) continue;
+    const entry = rawEntry as Partial<NegativeExampleEntry>;
+    if (
+      typeof entry.notUseful !== "number" ||
+      !Number.isFinite(entry.notUseful) ||
+      entry.notUseful < 0
+    ) {
+      continue;
+    }
+    if (typeof entry.lastUpdatedAt !== "string") continue;
+    const notes = Array.isArray(entry.notes)
+      ? entry.notes.filter((note): note is string => typeof note === "string")
+      : undefined;
+    state[memoryId] = {
+      notUseful: Math.floor(entry.notUseful),
+      lastUpdatedAt: entry.lastUpdatedAt,
+      notes: notes && notes.length > 0 ? notes : undefined,
+    };
+  }
+  return state;
+}
+
 export class NegativeExampleStore {
   private readonly statePath: string;
-  private state: NegativeState = {};
+  private state: NegativeState = createNegativeState();
 
   constructor(memoryDir: string) {
     this.statePath = path.join(memoryDir, "state", "negative_examples.json");
@@ -21,10 +63,9 @@ export class NegativeExampleStore {
   async load(): Promise<void> {
     try {
       const raw = await readFile(this.statePath, "utf-8");
-      const parsed = JSON.parse(raw) as NegativeState;
-      if (parsed && typeof parsed === "object") this.state = parsed;
+      this.state = sanitizeNegativeState(JSON.parse(raw));
     } catch {
-      this.state = {};
+      this.state = createNegativeState();
     }
   }
 
@@ -36,6 +77,7 @@ export class NegativeExampleStore {
     const now = new Date().toISOString();
 
     for (const memoryId of memoryIds) {
+      if (!isSafeNegativeMemoryId(memoryId)) continue;
       const existing = this.state[memoryId] ?? { notUseful: 0, lastUpdatedAt: now };
       const next: NegativeExampleEntry = {
         notUseful: existing.notUseful + 1,
@@ -60,8 +102,12 @@ export class NegativeExampleStore {
    * Intended as a soft bias, not a hard filter.
    */
   penalty(memoryId: string, opts: { perHit: number; cap: number }): number {
+    if (!isSafeNegativeMemoryId(memoryId)) return 0;
     const entry = this.state[memoryId];
     if (!entry) return 0;
+    if (!Number.isFinite(entry.notUseful) || entry.notUseful < 0) return 0;
+    if (!Number.isFinite(opts.perHit) || opts.perHit < 0) return 0;
+    if (!Number.isFinite(opts.cap) || opts.cap < 0) return 0;
 
     // Cap effect to avoid runaway ranking distortion.
     const hits = Math.min(10, entry.notUseful);
@@ -69,4 +115,3 @@ export class NegativeExampleStore {
     return Math.min(opts.cap, raw);
   }
 }
-