@remnic/core 9.45.3 → 9.45.5

package/src/external-wiki-search.ts

@@ -0,0 +1,362 @@
+import path from "node:path";
+import type { CitationEntry } from "./citations.js";
+import {
+  type ExternalWikiCatalogEntry,
+  type ExternalWikiRoot,
+  loadExternalWikiCatalog,
+  readExternalWikiPage,
+} from "./external-wiki.js";
+
+export type { ExternalWikiRoot } from "./external-wiki.js";
+
+const DEFAULT_LIMIT = 6;
+const MAX_LIMIT = 20;
+const DEFAULT_MAX_CHARS_PER_HIT = 1_000;
+const MAX_CHARS_PER_HIT = 8_000;
+const DEFAULT_MAX_CANDIDATE_FILES = 100;
+const MAX_CANDIDATE_FILES = 100;
+
+export interface ExternalWikiSearchInput {
+  readonly query: string;
+  readonly limit?: number;
+  readonly wikiId?: string;
+  readonly maxCharsPerHit?: number;
+}
+
+interface ExternalWikiSnippet {
+  readonly text: string;
+  readonly lineStart: number;
+  readonly lineEnd: number;
+}
+
+export interface ExternalWikiSearchHit {
+  readonly wikiId: string;
+  readonly title: string;
+  readonly path: string;
+  readonly snippet: string;
+  readonly score: number;
+  readonly rank: number;
+  readonly citations: readonly CitationEntry[];
+  readonly indexBlurb?: string;
+}
+
+export interface ExternalWikiSearchResult {
+  readonly query: string;
+  readonly hits: readonly ExternalWikiSearchHit[];
+  readonly count: number;
+  readonly degradedWikiIds: readonly string[];
+}
+
+export interface ExternalWikiCandidate {
+  readonly path: string;
+  readonly score?: number;
+}
+
+export interface ExternalWikiCandidateProvider {
+  search(root: ExternalWikiRoot, query: string, limit: number): Promise<readonly ExternalWikiCandidate[]>;
+}
+
+export interface ExternalWikiSearchOptions {
+  readonly candidateProvider?: ExternalWikiCandidateProvider;
+  readonly maxCandidateFiles?: number;
+}
+
+export class ExternalWikiSearchError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ExternalWikiSearchError";
+  }
+}
+
+interface RankedCatalogEntry {
+  readonly entry: ExternalWikiCatalogEntry;
+  readonly catalogScore: number;
+  readonly providerScore: number;
+  readonly providerRank?: number;
+}
+
+interface UnrankedHit extends Omit<ExternalWikiSearchHit, "rank"> {}
+
+export async function searchExternalWikis(
+  roots: readonly ExternalWikiRoot[],
+  input: ExternalWikiSearchInput,
+  options: ExternalWikiSearchOptions = {}
+): Promise<ExternalWikiSearchResult> {
+  const query = requireQuery(input.query);
+  const limit = boundedInteger(input.limit, DEFAULT_LIMIT, 1, MAX_LIMIT, "limit");
+  const maxCharsPerHit = boundedInteger(
+    input.maxCharsPerHit,
+    DEFAULT_MAX_CHARS_PER_HIT,
+    1,
+    MAX_CHARS_PER_HIT,
+    "maxCharsPerHit"
+  );
+  const maxCandidateFiles = boundedInteger(
+    options.maxCandidateFiles,
+    DEFAULT_MAX_CANDIDATE_FILES,
+    1,
+    MAX_CANDIDATE_FILES,
+    "maxCandidateFiles"
+  );
+  const selectedRoots = selectRoots(roots, input.wikiId);
+  const queryTokens = tokenize(query);
+  const degradedWikiIds = new Set<string>();
+  const perRootHits = await Promise.all(
+    selectedRoots.map(async (root) => {
+      try {
+        const catalog = await loadExternalWikiCatalog(root, { maxEntries: maxCandidateFiles });
+        if (!catalog.indexPresent) degradedWikiIds.add(root.id);
+        const candidates = await rankCandidates(
+          root,
+          catalog.entries,
+          query,
+          queryTokens,
+          maxCandidateFiles,
+          options.candidateProvider
+        );
+        return await readCandidateHits(root, candidates, query, queryTokens, maxCharsPerHit);
+      } catch {
+        degradedWikiIds.add(root.id);
+        return [];
+      }
+    })
+  );
+
+  const hits = perRootHits
+    .flat()
+    .sort(compareHits)
+    .slice(0, limit)
+    .map((hit, index): ExternalWikiSearchHit => ({ ...hit, rank: index + 1 }));
+
+  return {
+    query,
+    hits,
+    count: hits.length,
+    degradedWikiIds: [...degradedWikiIds].sort(),
+  };
+}
+
+function requireQuery(value: string): string {
+  if (typeof value !== "string" || value.trim().length === 0) {
+    throw new ExternalWikiSearchError("external wiki search query is required");
+  }
+  return value.trim();
+}
+
+function boundedInteger(
+  value: number | undefined,
+  fallback: number,
+  minimum: number,
+  maximum: number,
+  field: string
+): number {
+  const resolved = value ?? fallback;
+  if (!Number.isInteger(resolved) || resolved < minimum || resolved > maximum) {
+    throw new ExternalWikiSearchError(`${field} must be an integer from ${minimum} to ${maximum}`);
+  }
+  return resolved;
+}
+
+function selectRoots(roots: readonly ExternalWikiRoot[], wikiId: string | undefined): ExternalWikiRoot[] {
+  const enabled = roots.filter((root) => root.enabled);
+  if (enabled.length === 0) {
+    throw new ExternalWikiSearchError("no enabled external wiki roots are configured");
+  }
+  if (wikiId === undefined) return enabled;
+  const normalizedWikiId = wikiId.trim();
+  const selected = enabled.find((root) => root.id === normalizedWikiId);
+  if (!selected) {
+    throw new ExternalWikiSearchError(`unknown external wiki: ${normalizedWikiId || "(empty)"}`);
+  }
+  return [selected];
+}
+
+async function rankCandidates(
+  root: ExternalWikiRoot,
+  entries: readonly ExternalWikiCatalogEntry[],
+  query: string,
+  queryTokens: readonly string[],
+  maxCandidateFiles: number,
+  provider: ExternalWikiCandidateProvider | undefined
+): Promise<RankedCatalogEntry[]> {
+  let provided: readonly ExternalWikiCandidate[] = [];
+  if (provider) {
+    try {
+      provided = await provider.search(root, query, maxCandidateFiles);
+    } catch {
+      provided = [];
+    }
+  }
+
+  const providerByPath = new Map<string, { score: number; rank: number }>();
+  for (const [index, candidate] of provided.slice(0, maxCandidateFiles).entries()) {
+    if (!providerByPath.has(candidate.path)) {
+      providerByPath.set(candidate.path, { score: finiteScore(candidate.score), rank: index });
+    }
+  }
+
+  return entries
+    .map((entry): RankedCatalogEntry => {
+      const providerCandidate = providerByPath.get(entry.path);
+      return {
+        entry,
+        catalogScore: scoreCatalogEntry(entry, query, queryTokens),
+        providerScore: providerCandidate?.score ?? 0,
+        ...(providerCandidate ? { providerRank: providerCandidate.rank } : {}),
+      };
+    })
+    .sort((left, right) => {
+      const leftProvided = left.providerRank !== undefined;
+      const rightProvided = right.providerRank !== undefined;
+      if (leftProvided !== rightProvided) return leftProvided ? -1 : 1;
+      if (left.providerRank !== right.providerRank) {
+        return (left.providerRank ?? Number.MAX_SAFE_INTEGER) - (right.providerRank ?? Number.MAX_SAFE_INTEGER);
+      }
+      if (right.catalogScore !== left.catalogScore) return right.catalogScore - left.catalogScore;
+      return left.entry.path.localeCompare(right.entry.path);
+    })
+    .slice(0, maxCandidateFiles);
+}
+
+async function readCandidateHits(
+  root: ExternalWikiRoot,
+  candidates: readonly RankedCatalogEntry[],
+  query: string,
+  queryTokens: readonly string[],
+  maxCharsPerHit: number
+): Promise<UnrankedHit[]> {
+  const results = await Promise.all(
+    candidates.map(async (candidate): Promise<UnrankedHit | undefined> => {
+      try {
+        const page = await readExternalWikiPage(root, candidate.entry.path);
+        const pageScore = scorePage(page.title, page.content, query, queryTokens);
+        const score = candidate.catalogScore + candidate.providerScore + pageScore;
+        if (score <= 0) return undefined;
+        const snippet = extractSnippet(page.content, queryTokens, maxCharsPerHit);
+        const rootRelativePath = path.posix.join(root.pagesDir.replaceAll("\\", "/"), page.path);
+        const citation: CitationEntry = {
+          path: rootRelativePath,
+          lineStart: snippet.lineStart,
+          lineEnd: snippet.lineEnd,
+          note: page.title,
+        };
+        return {
+          wikiId: root.id,
+          title: page.title,
+          path: rootRelativePath,
+          snippet: snippet.text,
+          score: roundScore(score),
+          citations: [citation],
+          ...(candidate.entry.indexBlurb === undefined ? {} : { indexBlurb: candidate.entry.indexBlurb }),
+        };
+      } catch {
+        return undefined;
+      }
+    })
+  );
+  return results.filter((result): result is UnrankedHit => result !== undefined);
+}
+
+function scoreCatalogEntry(entry: ExternalWikiCatalogEntry, query: string, queryTokens: readonly string[]): number {
+  return (
+    scoreText(entry.title, query, queryTokens, 16, 20) +
+    scoreText(entry.path, query, queryTokens, 8, 8) +
+    scoreText(entry.indexBlurb ?? "", query, queryTokens, 4, 6)
+  );
+}
+
+function scorePage(title: string, content: string, query: string, queryTokens: readonly string[]): number {
+  const titleScore = scoreText(title, query, queryTokens, 12, 16);
+  const lines = content.split(/\r?\n/u);
+  let bestLineScore = 0;
+  const matchedTokens = new Set<string>();
+  for (const line of lines) {
+    const normalizedLine = line.toLocaleLowerCase();
+    let lineScore = 0;
+    for (const token of queryTokens) {
+      if (!normalizedLine.includes(token)) continue;
+      matchedTokens.add(token);
+      lineScore += line.startsWith("#") ? 6 : 3;
+    }
+    if (normalizedLine.includes(query.toLocaleLowerCase())) lineScore += 10;
+    bestLineScore = Math.max(bestLineScore, lineScore);
+  }
+  return titleScore + bestLineScore + matchedTokens.size * 2;
+}
+
+function scoreText(
+  value: string,
+  query: string,
+  queryTokens: readonly string[],
+  tokenWeight: number,
+  phraseWeight: number
+): number {
+  const normalized = value.toLocaleLowerCase();
+  let score = normalized.includes(query.toLocaleLowerCase()) ? phraseWeight : 0;
+  for (const token of queryTokens) {
+    if (normalized.includes(token)) score += tokenWeight;
+  }
+  return score;
+}
+
+function tokenize(value: string): string[] {
+  const tokens = value.toLocaleLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
+  return [...new Set(tokens)];
+}
+
+function extractSnippet(content: string, queryTokens: readonly string[], maxChars: number): ExternalWikiSnippet {
+  const lines = content.split(/\r?\n/u);
+  if (lines.length === 0) return { text: "", lineStart: 1, lineEnd: 1 };
+  let bestIndex = 0;
+  let bestScore = -1;
+  for (const [index, line] of lines.entries()) {
+    const normalized = line.toLocaleLowerCase();
+    const matchCount = queryTokens.reduce((sum, token) => sum + (normalized.includes(token) ? 1 : 0), 0);
+    const score = matchCount * 2 + (matchCount > 0 && !/^#{1,6}\s/u.test(line) ? 1 : 0);
+    if (score > bestScore) {
+      bestScore = score;
+      bestIndex = index;
+    }
+  }
+
+  let headingIndex = bestIndex;
+  while (headingIndex > 0 && !/^#{1,6}\s/u.test(lines[headingIndex] ?? "")) headingIndex -= 1;
+  const bodyStart = Math.max(0, bestIndex - 1);
+  const endExclusive = Math.min(lines.length, bestIndex + 3);
+  const start = headingIndex < bodyStart ? headingIndex : bodyStart;
+  const selected =
+    headingIndex < bodyStart
+      ? [lines[headingIndex] ?? "", ...lines.slice(bodyStart, endExclusive)]
+      : lines.slice(start, endExclusive);
+  const snippet = selected.join("\n").trim();
+  if (snippet.length <= maxChars) {
+    return { text: snippet, lineStart: start + 1, lineEnd: endExclusive };
+  }
+
+  const normalizedSnippet = snippet.toLocaleLowerCase();
+  const firstMatch =
+    queryTokens
+      .map((token) => normalizedSnippet.indexOf(token))
+      .filter((index) => index >= 0)
+      .sort((left, right) => left - right)[0] ?? 0;
+  const windowStart = Math.max(0, Math.min(firstMatch - Math.floor(maxChars / 3), snippet.length - maxChars));
+  return {
+    text: snippet.slice(windowStart, windowStart + maxChars).trim(),
+    lineStart: start + 1,
+    lineEnd: endExclusive,
+  };
+}
+
+function finiteScore(score: number | undefined): number {
+  return typeof score === "number" && Number.isFinite(score) ? Math.max(0, score) : 0;
+}
+
+function roundScore(score: number): number {
+  return Math.round(score * 1_000) / 1_000;
+}
+
+function compareHits(left: UnrankedHit, right: UnrankedHit): number {
+  if (right.score !== left.score) return right.score - left.score;
+  const wikiComparison = left.wikiId.localeCompare(right.wikiId);
+  return wikiComparison !== 0 ? wikiComparison : left.path.localeCompare(right.path);
+}

package/src/external-wiki.test.ts

@@ -0,0 +1,240 @@
+import assert from "node:assert/strict";
+import { mkdir, symlink, writeFile } from "node:fs/promises";
+import path from "node:path";
+import test from "node:test";
+import * as core from "./index.js";
+import type { ExternalWikiRoot } from "./external-wiki-config.js";
+import type {
+  loadExternalWikiCatalog,
+  parseExternalWikiCatalog,
+  readExternalWikiPage,
+  validateExternalWikiLayout,
+} from "./external-wiki.js";
+import { withTempDir } from "./testing/tmp-dir.js";
+
+const externalWiki = core as typeof core & {
+  loadExternalWikiCatalog: typeof loadExternalWikiCatalog;
+  parseExternalWikiCatalog: typeof parseExternalWikiCatalog;
+  readExternalWikiPage: typeof readExternalWikiPage;
+  validateExternalWikiLayout: typeof validateExternalWikiLayout;
+};
+
+function wikiConfig(rootDir: string, overrides: Partial<ExternalWikiRoot> = {}): ExternalWikiRoot {
+  return {
+    id: "reading",
+    rootDir,
+    enabled: true,
+    pagesDir: "wiki",
+    indexFile: "INDEX.md",
+    indexInQmd: false,
+    includeInDefaultRecall: false,
+    ...overrides,
+  };
+}
+
+async function withWiki(run: (rootDir: string) => Promise<void>): Promise<void> {
+  await withTempDir(run, "remnic-external-wiki-");
+}
+
+test("external wiki reader APIs are exported from core", () => {
+  assert.equal(typeof externalWiki.validateExternalWikiLayout, "function");
+  assert.equal(typeof externalWiki.parseExternalWikiCatalog, "function");
+  assert.equal(typeof externalWiki.loadExternalWikiCatalog, "function");
+  assert.equal(typeof externalWiki.readExternalWikiPage, "function");
+});
+
+test("validates the required pages directory and optional wiki layout", async () => {
+  await withWiki(async (rootDir) => {
+    await mkdir(path.join(rootDir, "wiki"));
+    await mkdir(path.join(rootDir, "raw"));
+    await mkdir(path.join(rootDir, "outputs"));
+    await writeFile(path.join(rootDir, "INDEX.md"), "# Catalog\n");
+
+    const layout = await externalWiki.validateExternalWikiLayout(wikiConfig(rootDir));
+
+    assert.equal(layout.rootDir, rootDir);
+    assert.equal(layout.pagesDir, path.join(rootDir, "wiki"));
+    assert.equal(layout.indexFile, path.join(rootDir, "INDEX.md"));
+    assert.equal(layout.indexPresent, true);
+    assert.equal(layout.rawDir, path.join(rootDir, "raw"));
+    assert.equal(layout.outputsDir, path.join(rootDir, "outputs"));
+  });
+});
+
+test("rejects missing pages directories and symlink escapes", async () => {
+  await withWiki(async (rootDir) => {
+    await assert.rejects(
+      () => externalWiki.validateExternalWikiLayout(wikiConfig(rootDir)),
+      /pages directory does not exist/
+    );
+
+    await withTempDir(async (outsideDir) => {
+      await symlink(outsideDir, path.join(rootDir, "wiki"));
+      await assert.rejects(
+        () => externalWiki.validateExternalWikiLayout(wikiConfig(rootDir)),
+        /pages directory escapes rootDir/
+      );
+    }, "remnic-wiki-outside-");
+  });
+});
+
+test("parses markdown and wiki-link catalog entries deterministically", () => {
+  const parsed = externalWiki.parseExternalWikiCatalog(
+    [
+      "# Catalog",
+      "- [Cache consistency](wiki/cache-consistency.md) - Invalidation and ownership.",
+      "- [[distributed-systems|Distributed systems]]: Coordination foundations.",
+      "- [External](https://example.com/external.md) - ignored",
+      "- [Escape](../outside.md) - ignored",
+      "- [Duplicate](wiki/cache-consistency.md) - ignored",
+    ].join("\n"),
+    wikiConfig("/srv/wiki")
+  );
+
+  assert.deepEqual(parsed, [
+    {
+      title: "Cache consistency",
+      path: "cache-consistency.md",
+      indexBlurb: "Invalidation and ownership.",
+      indexLine: 2,
+    },
+    {
+      title: "Distributed systems",
+      path: "distributed-systems.md",
+      indexBlurb: "Coordination foundations.",
+      indexLine: 3,
+    },
+  ]);
+});
+
+test("resolves markdown catalog links relative to a nested index file", () => {
+  const parsed = externalWiki.parseExternalWikiCatalog(
+    "- [Nested concept](../wiki/nested.md) - Nested catalog entry.\n",
+    wikiConfig("/srv/wiki", { indexFile: "catalog/INDEX.md" })
+  );
+
+  assert.deepEqual(parsed, [
+    {
+      title: "Nested concept",
+      path: "nested.md",
+      indexBlurb: "Nested catalog entry.",
+      indexLine: 1,
+    },
+  ]);
+});
+
+test("loads a bounded index and falls back to a sorted page listing", async () => {
+  await withWiki(async (rootDir) => {
+    await mkdir(path.join(rootDir, "wiki", "nested"), { recursive: true });
+    await writeFile(path.join(rootDir, "wiki", "zeta.md"), "# Zeta\n");
+    await writeFile(path.join(rootDir, "wiki", "nested", "alpha.md"), "# Alpha\n");
+    await writeFile(path.join(rootDir, "wiki", "ignored.txt"), "ignored\n");
+
+    const fallback = await externalWiki.loadExternalWikiCatalog(wikiConfig(rootDir));
+    assert.equal(fallback.wikiId, "reading");
+    assert.equal(fallback.indexPresent, false);
+    assert.deepEqual(fallback.entries, [
+      { title: "Alpha", path: "nested/alpha.md" },
+      { title: "Zeta", path: "zeta.md" },
+    ]);
+
+    await assert.rejects(
+      () => externalWiki.loadExternalWikiCatalog(wikiConfig(rootDir), { maxEntries: 1 }),
+      /more than 1 markdown files/
+    );
+    await assert.rejects(
+      () => externalWiki.loadExternalWikiCatalog(wikiConfig(rootDir, { enabled: false })),
+      /external wiki "reading" is disabled/
+    );
+    await assert.rejects(
+      () => externalWiki.loadExternalWikiCatalog(wikiConfig(rootDir), { maxEntries: 100_001 }),
+      /maxEntries must be at most 100000/
+    );
+    await assert.rejects(
+      () => externalWiki.loadExternalWikiCatalog(wikiConfig(rootDir), { maxDepth: 129 }),
+      /maxDepth must be at most 128/
+    );
+
+    await writeFile(path.join(rootDir, "INDEX.md"), "- [Zeta](wiki/zeta.md) - Last page.\n");
+    const indexed = await externalWiki.loadExternalWikiCatalog(wikiConfig(rootDir));
+    assert.equal(indexed.indexPresent, true);
+    assert.deepEqual(indexed.entries, [{ title: "Zeta", path: "zeta.md", indexBlurb: "Last page.", indexLine: 1 }]);
+
+    await assert.rejects(
+      () => externalWiki.loadExternalWikiCatalog(wikiConfig(rootDir), { maxIndexBytes: 8 }),
+      /INDEX\.md exceeds 8 bytes/
+    );
+  });
+});
+
+test("bounds all filesystem entries visited by catalog fallback", async () => {
+  await withWiki(async (rootDir) => {
+    await mkdir(path.join(rootDir, "wiki"));
+    await Promise.all(
+      Array.from({ length: 101 }, (_, index) =>
+        writeFile(path.join(rootDir, "wiki", `ignored-${index}.txt`), "ignored\n")
+      )
+    );
+
+    await assert.rejects(
+      () => externalWiki.loadExternalWikiCatalog(wikiConfig(rootDir), { maxEntries: 1 }),
+      /more than 100 filesystem entries/
+    );
+  });
+});
+
+test("loads concept pages on demand with containment and byte limits", async () => {
+  await withWiki(async (rootDir) => {
+    await mkdir(path.join(rootDir, "wiki"));
+    await writeFile(path.join(rootDir, "wiki", "concept.md"), "# Concept\n\nSource-backed synthesis.\n");
+    await writeFile(path.join(rootDir, "secret.md"), "not a concept page\n");
+
+    const page = await externalWiki.readExternalWikiPage(wikiConfig(rootDir), "concept.md", 1_024);
+    assert.deepEqual(page, {
+      wikiId: "reading",
+      path: "concept.md",
+      title: "Concept",
+      content: "# Concept\n\nSource-backed synthesis.\n",
+      bytes: 36,
+    });
+
+    await assert.rejects(
+      () => externalWiki.readExternalWikiPage(wikiConfig(rootDir), "../secret.md", 1_024),
+      /page path must stay within the pages directory/
+    );
+    await assert.rejects(
+      () => externalWiki.readExternalWikiPage(wikiConfig(rootDir), "concept.txt", 1_024),
+      /page path must end in \.md/
+    );
+    await assert.rejects(
+      () => externalWiki.readExternalWikiPage(wikiConfig(rootDir), "concept.md", 8),
+      /concept\.md exceeds 8 bytes/
+    );
+  });
+});
+
+test("rejects non-portable paths, symlink escapes, invalid UTF-8, and excessive limits", async () => {
+  await withWiki(async (rootDir) => {
+    await mkdir(path.join(rootDir, "wiki"));
+    await writeFile(path.join(rootDir, "wiki", "invalid.md"), Buffer.from([0xc3, 0x28]));
+    await writeFile(path.join(rootDir, "outside.md"), "# Outside\n");
+    await symlink(path.join(rootDir, "outside.md"), path.join(rootDir, "wiki", "escape.md"));
+
+    await assert.rejects(
+      () => externalWiki.readExternalWikiPage(wikiConfig(rootDir), "nested\\page.md"),
+      /page path must use POSIX separators/
+    );
+    await assert.rejects(
+      () => externalWiki.readExternalWikiPage(wikiConfig(rootDir), "escape.md"),
+      /page path must stay within the pages directory/
+    );
+    await assert.rejects(
+      () => externalWiki.readExternalWikiPage(wikiConfig(rootDir), "invalid.md"),
+      /invalid\.md is not valid UTF-8/
+    );
+    await assert.rejects(
+      () => externalWiki.readExternalWikiPage(wikiConfig(rootDir), "invalid.md", 16_777_217),
+      /maxBytes must be at most 16777216/
+    );
+  });
+});