@remnic/core 9.45.3 → 9.45.5

package/src/external-wiki-config.ts

@@ -0,0 +1,100 @@
+import path from "node:path";
+import { assertExternalWikiRootOutsideMemoryDir } from "./external-wiki-guard.js";
+import { expandTildePath } from "./utils/path.js";
+
+export interface ExternalWikiRoot {
+  id: string;
+  rootDir: string;
+  enabled: boolean;
+  label?: string;
+  pagesDir: string;
+  indexFile: string;
+  indexInQmd: boolean;
+  includeInDefaultRecall: false;
+}
+
+const EXTERNAL_WIKI_ID_PATTERN = /^[a-z0-9][a-z0-9_-]{0,63}$/;
+
+function requireObject(value: unknown, keyName: string): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error(`${keyName} must be an object`);
+  }
+  return value as Record<string, unknown>;
+}
+
+function requireString(value: unknown, keyName: string): string {
+  if (typeof value !== "string" || value.trim().length === 0) {
+    throw new Error(`${keyName} must be a non-empty string`);
+  }
+  return value.trim();
+}
+
+function parseBoolean(value: unknown, fallback: boolean, keyName: string): boolean {
+  if (value === undefined) return fallback;
+  if (typeof value !== "boolean") throw new Error(`${keyName} must be a boolean`);
+  return value;
+}
+
+function parseRootDir(value: unknown, keyName: string): string {
+  const raw = requireString(value, keyName);
+  if (!path.isAbsolute(raw) && !raw.startsWith("~/")) {
+    throw new Error(`${keyName} must be an absolute path or start with ~/`);
+  }
+  return path.resolve(expandTildePath(raw));
+}
+
+function parseRootRelativePath(value: unknown, fallback: string, keyName: string): string {
+  const candidate = value === undefined ? fallback : requireString(value, keyName);
+  if (candidate.includes("\\")) {
+    throw new Error(`${keyName} must use POSIX separators`);
+  }
+  const normalized = path.normalize(candidate);
+  if (
+    path.isAbsolute(candidate) ||
+    normalized === ".." ||
+    normalized.startsWith(`..${path.sep}`) ||
+    normalized === "."
+  ) {
+    throw new Error(`${keyName} must be a relative path within rootDir`);
+  }
+  return normalized;
+}
+
+export function parseExternalWikiRoots(value: unknown, memoryDir?: string): ExternalWikiRoot[] {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) throw new Error("externalWikis must be an array");
+
+  const seenIds = new Set<string>();
+  return value.map((entry, index) => {
+    const keyName = `externalWikis[${index}]`;
+    const raw = requireObject(entry, keyName);
+    const id = requireString(raw.id, `${keyName}.id`);
+    if (!EXTERNAL_WIKI_ID_PATTERN.test(id)) {
+      throw new Error(`${keyName}.id must match ${EXTERNAL_WIKI_ID_PATTERN}`);
+    }
+    if (seenIds.has(id)) throw new Error(`externalWikis contains duplicate id "${id}"`);
+    seenIds.add(id);
+
+    const includeInDefaultRecall = parseBoolean(raw.includeInDefaultRecall, false, `${keyName}.includeInDefaultRecall`);
+    if (includeInDefaultRecall) {
+      throw new Error(`${keyName}.includeInDefaultRecall=true is not supported yet`);
+    }
+
+    const rootDir = parseRootDir(raw.rootDir, `${keyName}.rootDir`);
+    if (memoryDir !== undefined) {
+      assertExternalWikiRootOutsideMemoryDir(memoryDir, rootDir);
+    }
+
+    const label = raw.label === undefined ? undefined : requireString(raw.label, `${keyName}.label`);
+    return {
+      id,
+      rootDir,
+      enabled: parseBoolean(raw.enabled, true, `${keyName}.enabled`),
+      ...(label === undefined ? {} : { label }),
+      pagesDir: parseRootRelativePath(raw.pagesDir, "wiki", `${keyName}.pagesDir`),
+      indexFile: parseRootRelativePath(raw.indexFile, "INDEX.md", `${keyName}.indexFile`),
+      indexInQmd: parseBoolean(raw.indexInQmd, false, `${keyName}.indexInQmd`),
+      includeInDefaultRecall: false,
+    };
+  });
+}

package/src/external-wiki-guard.ts

@@ -0,0 +1,83 @@
+import { realpath } from "node:fs/promises";
+import path from "node:path";
+import { coerceBool } from "./connectors/coerce.js";
+
+export const EXTERNAL_WIKI_COLLECTION_PREFIX = "external-wiki-";
+
+export function isExternalWikiCollectionName(collection: string): boolean {
+  return collection.startsWith(EXTERNAL_WIKI_COLLECTION_PREFIX);
+}
+
+export function parseExternalWikiRecallGuard(config: Record<string, unknown>): {
+  wikiMergeIntoRecall: false;
+  qmdCollection: string;
+  qmdColdCollection: string;
+} {
+  const rawMerge = config.wikiMergeIntoRecall;
+  const merge = rawMerge == null ? false : coerceBool(rawMerge);
+  if (merge === undefined) {
+    throw new Error(
+      `wikiMergeIntoRecall must be a boolean-like value (true/false/1/0/yes/no/on/off); got ${JSON.stringify(rawMerge)}`,
+    );
+  }
+  if (merge) {
+    throw new Error(
+      "wikiMergeIntoRecall=true is not supported; external wiki content is available only through on-demand search",
+    );
+  }
+
+  const qmdCollection =
+    typeof config.qmdCollection === "string"
+      ? config.qmdCollection
+      : "openclaw-engram";
+  const qmdColdCollection =
+    typeof config.qmdColdCollection === "string" && config.qmdColdCollection.length > 0
+      ? config.qmdColdCollection
+      : "openclaw-engram-cold";
+  if (isExternalWikiCollectionName(qmdCollection)) {
+    throw new Error(
+      "qmdCollection must be a memory collection; external wiki collections are on-demand only",
+    );
+  }
+  if (isExternalWikiCollectionName(qmdColdCollection)) {
+    throw new Error(
+      "qmdColdCollection must be a memory collection; external wiki collections are on-demand only",
+    );
+  }
+  return { wikiMergeIntoRecall: false, qmdCollection, qmdColdCollection };
+}
+
+export function assertExternalWikiRootOutsideMemoryDir(memoryDir: string, rootDir: string): void {
+  const resolvedMemoryDir = path.resolve(memoryDir);
+  const relativeRoot = path.relative(resolvedMemoryDir, path.resolve(rootDir));
+  const isInsideMemoryDir =
+    relativeRoot === "" ||
+    (!path.isAbsolute(relativeRoot) && relativeRoot !== ".." && !relativeRoot.startsWith(`..${path.sep}`));
+  if (isInsideMemoryDir) {
+    throw new Error(`external wiki rootDir must be outside memoryDir: ${rootDir}`);
+  }
+}
+
+async function canonicalPath(candidate: string): Promise<string> {
+  let existing = path.resolve(candidate);
+  const suffix: string[] = [];
+  while (true) {
+    try {
+      return path.join(await realpath(existing), ...suffix);
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+      const parent = path.dirname(existing);
+      if (parent === existing) throw error;
+      suffix.unshift(path.basename(existing));
+      existing = parent;
+    }
+  }
+}
+
+export async function assertExternalWikiRootOutsideMemoryDirCanonical(
+  memoryDir: string,
+  rootDir: string
+): Promise<void> {
+  const [canonicalMemoryDir, canonicalRootDir] = await Promise.all([canonicalPath(memoryDir), canonicalPath(rootDir)]);
+  assertExternalWikiRootOutsideMemoryDir(canonicalMemoryDir, canonicalRootDir);
+}

package/src/external-wiki-mcp-tools.ts

@@ -0,0 +1,23 @@
+import type { McpTool } from "./access-mcp.js";
+
+export const EXTERNAL_WIKI_MCP_TOOLS: McpTool[] = [
+  {
+    name: "engram.external_wiki_search",
+    description:
+      "Search configured external compiled-wiki roots on demand. Returns ranked snippets with root-relative citations and never changes default recall.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        query: { type: "string", description: "Required topical query." },
+        limit: { type: "number", description: "Maximum hits (default 6, maximum 20)." },
+        wikiId: { type: "string", description: "Optional configured wiki id to search exclusively." },
+        maxCharsPerHit: {
+          type: "number",
+          description: "Maximum characters in each snippet (default 1000, maximum 8000).",
+        },
+      },
+      required: ["query"],
+      additionalProperties: false,
+    },
+  },
+];

package/src/external-wiki-search.test.ts

@@ -0,0 +1,367 @@
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { runExternalWikiCliCommand } from "./external-wiki-cli.js";
+import {
+  type ExternalWikiCandidateProvider,
+  type ExternalWikiRoot,
+  searchExternalWikis,
+} from "./external-wiki-search.js";
+
+async function createWiki(id: string, files: Readonly<Record<string, string>>): Promise<ExternalWikiRoot> {
+  const rootDir = await mkdtemp(path.join(os.tmpdir(), `remnic-external-wiki-${id}-`));
+  for (const [relativePath, content] of Object.entries(files)) {
+    const absolutePath = path.join(rootDir, relativePath);
+    await mkdir(path.dirname(absolutePath), { recursive: true });
+    await writeFile(absolutePath, content, "utf8");
+  }
+  return {
+    id,
+    rootDir,
+    enabled: true,
+    pagesDir: "wiki",
+    indexFile: "INDEX.md",
+    indexInQmd: false,
+    includeInDefaultRecall: false,
+  };
+}
+
+async function removeWikis(roots: readonly ExternalWikiRoot[]): Promise<void> {
+  await Promise.all(roots.map((root) => rm(root.rootDir, { recursive: true, force: true })));
+}
+
+test("external wiki search ranks topical pages and returns bounded citations", async () => {
+  const root = await createWiki("reading", {
+    "INDEX.md": [
+      "# Catalog",
+      "- [[wiki/retrieval-architecture|Retrieval Architecture]] - Hybrid retrieval design and ranking.",
+      "- [[wiki/cooking|Cooking Notes]] - Pantry and meal planning.",
+    ].join("\n"),
+    "wiki/retrieval-architecture.md": [
+      "# Retrieval Architecture",
+      "",
+      "## Candidate generation",
+      "A hybrid index combines lexical overlap with semantic candidates.",
+      "The planner fans out over the best cited pages.",
+    ].join("\n"),
+    "wiki/cooking.md": "# Cooking Notes\n\nKeep a stocked pantry.\n",
+  });
+
+  try {
+    const result = await searchExternalWikis([root], {
+      query: "hybrid retrieval planner",
+      limit: 2,
+      maxCharsPerHit: 180,
+    });
+
+    assert.equal(result.query, "hybrid retrieval planner");
+    assert.equal(result.count, 1);
+    assert.equal(result.hits[0]?.wikiId, "reading");
+    assert.equal(result.hits[0]?.title, "Retrieval Architecture");
+    assert.equal(result.hits[0]?.path, "wiki/retrieval-architecture.md");
+    assert.match(result.hits[0]?.snippet ?? "", /hybrid index/i);
+    assert.ok((result.hits[0]?.snippet.length ?? 0) <= 180);
+    assert.deepEqual(result.hits[0]?.citations, [
+      {
+        path: "wiki/retrieval-architecture.md",
+        lineStart: 3,
+        lineEnd: 5,
+        note: "Retrieval Architecture",
+      },
+    ]);
+    assert.equal(result.hits[0]?.indexBlurb, "Hybrid retrieval design and ranking.");
+    assert.equal(result.hits[0]?.rank, 1);
+    assert.ok((result.hits[0]?.score ?? 0) > 0);
+    assert.deepEqual(result.degradedWikiIds, []);
+  } finally {
+    await removeWikis([root]);
+  }
+});
+
+test("external wiki search spans enabled roots and wikiId scopes one root", async () => {
+  const reading = await createWiki("reading", {
+    "INDEX.md": "- [[wiki/agents|Agent Design]] - Delegation and planner fan-out.\n",
+    "wiki/agents.md": "# Agent Design\n\nA planner can delegate independent research.\n",
+  });
+  const operations = await createWiki("operations", {
+    "INDEX.md": "- [[wiki/agents|Operations Agents]] - Delegation during incidents.\n",
+    "wiki/agents.md": "# Operations Agents\n\nDelegate independent incident checks.\n",
+  });
+
+  try {
+    const all = await searchExternalWikis([reading, operations], {
+      query: "delegate independent agents",
+      limit: 4,
+    });
+    assert.deepEqual(new Set(all.hits.map((hit) => hit.wikiId)), new Set(["reading", "operations"]));
+
+    const scoped = await searchExternalWikis([reading, operations], {
+      query: "delegate independent agents",
+      wikiId: "operations",
+    });
+    assert.ok(scoped.hits.length > 0);
+    assert.ok(scoped.hits.every((hit) => hit.wikiId === "operations"));
+
+    await assert.rejects(
+      searchExternalWikis([reading, operations], { query: "agents", wikiId: "missing" }),
+      /unknown external wiki.*missing/i
+    );
+  } finally {
+    await removeWikis([reading, operations]);
+  }
+});
+
+test("external wiki search degrades to markdown page titles when INDEX.md is missing", async () => {
+  const root = await createWiki("degraded", {
+    "wiki/distributed-systems.md": "# Distributed Systems\n\nQuorum protects replicated writes.\n",
+    "wiki/irrelevant.md": "# Gardening\n\nWater the seedlings.\n",
+  });
+
+  try {
+    const result = await searchExternalWikis([root], { query: "distributed quorum" });
+    assert.equal(result.hits[0]?.title, "Distributed Systems");
+    assert.equal(result.hits[0]?.path, "wiki/distributed-systems.md");
+    assert.deepEqual(result.degradedWikiIds, ["degraded"]);
+  } finally {
+    await removeWikis([root]);
+  }
+});
+
+test("external wiki search isolates a missing configured root", async () => {
+  const missingRoot = path.join(os.tmpdir(), `remnic-missing-wiki-${Date.now()}`);
+  const result = await searchExternalWikis(
+    [
+      {
+        id: "missing",
+        rootDir: missingRoot,
+        enabled: true,
+        pagesDir: "wiki",
+        indexFile: "INDEX.md",
+        indexInQmd: false,
+        includeInDefaultRecall: false,
+      },
+    ],
+    { query: "anything" }
+  );
+
+  assert.deepEqual(result.hits, []);
+  assert.deepEqual(result.degradedWikiIds, ["missing"]);
+});
+
+test("external wiki search rejects empty queries and an empty enabled-root set", async () => {
+  await assert.rejects(searchExternalWikis([], { query: "topic" }), /no enabled external wiki roots/i);
+  await assert.rejects(
+    searchExternalWikis(
+      [
+        {
+          id: "disabled",
+          rootDir: os.tmpdir(),
+          enabled: false,
+          pagesDir: "wiki",
+          indexFile: "INDEX.md",
+          indexInQmd: false,
+          includeInDefaultRecall: false,
+        },
+      ],
+      { query: "topic" }
+    ),
+    /no enabled external wiki roots/i
+  );
+  await assert.rejects(
+    searchExternalWikis(
+      [
+        {
+          id: "enabled",
+          rootDir: os.tmpdir(),
+          enabled: true,
+          pagesDir: "wiki",
+          indexFile: "INDEX.md",
+          indexInQmd: false,
+          includeInDefaultRecall: false,
+        },
+      ],
+      { query: "   " }
+    ),
+    /query is required/i
+  );
+});
+
+test("external wiki search never follows catalog traversal outside pagesDir", async () => {
+  const parent = await mkdtemp(path.join(os.tmpdir(), "remnic-external-wiki-traversal-"));
+  const rootDir = path.join(parent, "root");
+  await mkdir(path.join(rootDir, "wiki"), { recursive: true });
+  await writeFile(path.join(rootDir, "INDEX.md"), "- [[../secret|Secret Plan]] - do not read\n", "utf8");
+  await writeFile(path.join(parent, "secret.md"), "# Secret Plan\n\nEXFILTRATION_SENTINEL\n", "utf8");
+  const root: ExternalWikiRoot = {
+    id: "safe",
+    rootDir,
+    enabled: true,
+    pagesDir: "wiki",
+    indexFile: "INDEX.md",
+    indexInQmd: false,
+    includeInDefaultRecall: false,
+  };
+
+  try {
+    const result = await searchExternalWikis([root], { query: "EXFILTRATION_SENTINEL" });
+    assert.equal(
+      result.hits.some((hit) => hit.snippet.includes("EXFILTRATION_SENTINEL")),
+      false
+    );
+    assert.equal(
+      result.hits.some((hit) => hit.path.includes("..")),
+      false
+    );
+  } finally {
+    await rm(parent, { recursive: true, force: true });
+  }
+});
+
+test("external wiki search enforces the requested snippet length cap", async () => {
+  const root = await createWiki("caps", {
+    "INDEX.md": "- [[wiki/long-page|Long Page]] - bounded snippets\n",
+    "wiki/long-page.md": `# Long Page\n\n## Match\nneedle ${"content ".repeat(100)}`,
+  });
+
+  try {
+    const result = await searchExternalWikis([root], {
+      query: "needle content",
+      maxCharsPerHit: 80,
+    });
+    assert.equal(result.hits.length, 1);
+    assert.ok((result.hits[0]?.snippet.length ?? 0) <= 80);
+  } finally {
+    await removeWikis([root]);
+  }
+});
+
+test("external wiki search fails open to filesystem candidates when a dedicated provider fails", async () => {
+  const root = await createWiki("fallback", {
+    "INDEX.md": "- [[wiki/fallback|Filesystem Fallback]] - resilient retrieval\n",
+    "wiki/fallback.md": "# Filesystem Fallback\n\nThe catalog remains available.\n",
+  });
+  const provider: ExternalWikiCandidateProvider = {
+    async search() {
+      throw new Error("backend unavailable");
+    },
+  };
+
+  try {
+    const result = await searchExternalWikis([root], { query: "filesystem fallback" }, { candidateProvider: provider });
+    assert.equal(result.hits[0]?.title, "Filesystem Fallback");
+  } finally {
+    await removeWikis([root]);
+  }
+});
+
+test("external wiki CLI preserves text contracts and supports cited JSON", async () => {
+  const root = await createWiki("cli", {
+    "INDEX.md": "- [[wiki/agents|Agent Planning]] - planner fan-out with citations\n",
+    "wiki/agents.md": "# Agent Planning\n\nPlanner fan-out uses cited retrieval evidence.\n",
+  });
+  try {
+    const jsonOutput: string[] = [];
+    const jsonErrors: string[] = [];
+    const jsonCode = await runExternalWikiCliCommand(
+      [root],
+      ["search", "planner", "fan-out", "--wiki-id", "cli", "--limit", "1", "--json"],
+      {
+        stdout: { write: (chunk) => jsonOutput.push(chunk) },
+        stderr: { write: (chunk) => jsonErrors.push(chunk) },
+      }
+    );
+    assert.equal(jsonCode, 0);
+    assert.equal(jsonErrors.join(""), "");
+    const parsed: unknown = JSON.parse(jsonOutput.join(""));
+    assert.ok(parsed && typeof parsed === "object" && "hits" in parsed);
+    assert.ok(Array.isArray(parsed.hits));
+    assert.equal(parsed.hits[0]?.path, "wiki/agents.md");
+
+    const textOutput: string[] = [];
+    const textCode = await runExternalWikiCliCommand([root], ["search", "retrieval", "evidence"], {
+      stdout: { write: (chunk) => textOutput.push(chunk) },
+      stderr: { write: () => undefined },
+    });
+    assert.equal(textCode, 0);
+    assert.match(textOutput.join(""), /\[cli\] wiki\/agents\.md \(score [\d.]+\)/);
+
+    const invalidErrors: string[] = [];
+    const invalidCode = await runExternalWikiCliCommand([root], ["search", "--limit", "0", "query"], {
+      stdout: { write: () => undefined },
+      stderr: { write: (chunk) => invalidErrors.push(chunk) },
+    });
+    assert.equal(invalidCode, 2);
+    assert.match(invalidErrors.join(""), /--limit must be an integer from 1 to 20/);
+
+    const noQueryErrors: string[] = [];
+    const noQueryCode = await runExternalWikiCliCommand([root], ["search"], {
+      stdout: { write: () => undefined },
+      stderr: { write: (chunk) => noQueryErrors.push(chunk) },
+    });
+    assert.equal(noQueryCode, 1);
+    assert.equal(noQueryErrors.join(""), "external-wiki: provide a search query\n");
+
+    const disabledErrors: string[] = [];
+    const disabledCode = await runExternalWikiCliCommand([{ ...root, enabled: false }], ["search", "planner"], {
+      stdout: { write: () => undefined },
+      stderr: { write: (chunk) => disabledErrors.push(chunk) },
+    });
+    assert.equal(disabledCode, 1);
+    assert.equal(disabledErrors.join(""), "external-wiki: no enabled wiki roots configured\n");
+
+    const emptyOutput: string[] = [];
+    const emptyCode = await runExternalWikiCliCommand([root], ["search", "unmatched-token"], {
+      stdout: { write: (chunk) => emptyOutput.push(chunk) },
+      stderr: { write: () => undefined },
+    });
+    assert.equal(emptyCode, 0);
+    assert.equal(emptyOutput.join(""), "No results.\n");
+
+    const searchErrors: string[] = [];
+    const searchErrorCode = await runExternalWikiCliCommand([root], ["search", "planner", "--wiki-id", "missing"], {
+      stdout: { write: () => undefined },
+      stderr: { write: (chunk) => searchErrors.push(chunk) },
+    });
+    assert.equal(searchErrorCode, 1);
+    assert.equal(searchErrors.join(""), "external-wiki: unknown external wiki: missing\n");
+
+    const legacyOutput: string[] = [];
+    const legacyCode = await runExternalWikiCliCommand([root], ["planner", "fan-out"], {
+      stdout: { write: (chunk) => legacyOutput.push(chunk) },
+      stderr: { write: () => undefined },
+    });
+    assert.equal(legacyCode, 0);
+    assert.match(legacyOutput.join(""), /^\[cli\] wiki\/agents\.md \(score [\d.]+\)/);
+
+    const degradedErrors: string[] = [];
+    const degradedCode = await runExternalWikiCliCommand(
+      [root, { ...root, id: "broken", rootDir: path.join(root.rootDir, "missing") }],
+      ["search", "planner"],
+      {
+        stdout: { write: () => undefined },
+        stderr: { write: (chunk) => degradedErrors.push(chunk) },
+      }
+    );
+    assert.equal(degradedCode, 0);
+    assert.equal(degradedErrors.join(""), "Warning: degraded roots: broken\n");
+
+    const allDegradedOutput: string[] = [];
+    const allDegradedErrors: string[] = [];
+    const allDegradedCode = await runExternalWikiCliCommand(
+      [{ ...root, id: "broken", rootDir: path.join(root.rootDir, "missing") }],
+      ["search", "planner"],
+      {
+        stdout: { write: (chunk) => allDegradedOutput.push(chunk) },
+        stderr: { write: (chunk) => allDegradedErrors.push(chunk) },
+      }
+    );
+    assert.equal(allDegradedCode, 0);
+    assert.equal(allDegradedOutput.join(""), "No results.\n");
+    assert.equal(allDegradedErrors.join(""), "Warning: degraded roots: broken\n");
+  } finally {
+    await removeWikis([root]);
+  }
+});