@hicaru/pi-rlm 0.2.2 → 0.3.0

package/src/context/listing.ts

@@ -0,0 +1,70 @@
+/**
+ * Compact human-readable listing of currently loaded context files for the parent LLM.
+ * Shows paths and token estimates — NOT full file contents.
+ */
+
+import type { ContextFile } from "./types.ts";
+import { isContextFile } from "./namespace.ts";
+
+/** Maximum files shown in the compact LLM listing before truncation. */
+const MAX_LLM_LISTING_FILES = 200;
+
+/**
+ * Format a context payload (ContextFile[] or empty) for injection into the parent agent's
+ * message stream. Empty state points at autoSeedCwd / external add_context — never
+ * `add_context(".")`, which would re-pack the already-seeded cwd under a ctx/ prefix.
+ */
+export function formatContextListing(context: unknown): string {
+  const files = toFileList(context);
+  if (files.length === 0) {
+    return [
+      "RLM `context` is EMPTY — no files loaded yet.",
+      "The working directory seeds automatically on the first `repl()` call (autoSeedCwd).",
+      'Use `add_context("/path/to/dir")` / `add_context("docs.pdf")` / `add_context("https://…")` for external sources.',
+      "Documents (PDF, DOCX, XLSX, PPTX, CSV, …) are converted to Markdown on the way in.",
+      "",
+      "Use repl({code}) and delegate semantic reading to llm_query / llm_query_batched / llm_query_chunked.",
+    ].join("\n");
+  }
+
+  let totalTokens = 0;
+  let totalChars = 0;
+  for (let i = 0; i < files.length; i++) {
+    totalTokens += files[i].tokens;
+    totalChars += files[i].content.length;
+  }
+
+  const shown = files.slice(0, MAX_LLM_LISTING_FILES);
+  const truncated = files.length > MAX_LLM_LISTING_FILES
+    ? `... and ${files.length - MAX_LLM_LISTING_FILES} more files (truncated)`
+    : "";
+
+  const listingParts = new Array<string>(shown.length);
+  for (let i = 0; i < shown.length; i++) {
+    const f = shown[i];
+    listingParts[i] =
+      `${f.path} (${f.tokens.toLocaleString()} tok, ${f.content.length.toLocaleString()} chars)`;
+  }
+
+  return [
+    `Context: ${files.length.toLocaleString()} files, ${totalTokens.toLocaleString()} estimated tokens, ${totalChars.toLocaleString()} total characters.`,
+    "",
+    listingParts.join("\n"),
+    truncated,
+    "",
+    "File contents are loaded in the REPL `context` variable — file-reading tools are disabled.",
+    "Use repl({code}) and delegate semantic reading to llm_query / llm_query_batched / llm_query_chunked.",
+  ].join("\n");
+}
+
+function toFileList(context: unknown): readonly ContextFile[] {
+  if (!Array.isArray(context)) return Object.freeze([]);
+  const out = new Array<ContextFile>(context.length);
+  let n = 0;
+  for (let i = 0; i < context.length; i++) {
+    const entry: unknown = context[i];
+    if (isContextFile(entry)) out[n++] = entry;
+  }
+  out.length = n;
+  return out;
+}

package/src/context/md-cache.ts

@@ -0,0 +1,112 @@
+/**
+ * On-disk Markdown cache for anydoc conversions, keyed by (size, mtimeMs).
+ *
+ * Cache dir: $XDG_CACHE_HOME/pi-rlm/anydoc (else ~/.cache/…).
+ * Per source: <name>-<sha8(absPath)>.md + .json stamp {source, size, mtimeMs}.
+ *
+ * The stamp is ALWAYS the pre-conversion (size, mtimeMs). Capturing after toMarkdown races
+ * with mid-conversion edits and would stamp the new mtime against the old body — every future
+ * read would be a permanent stale hit. writeMdCache also refuses to write when a post-conversion
+ * stat disagrees with the captured stamp.
+ *
+ * Write ordering is load-bearing: Markdown body first, stamp second. A crash between them
+ * leaves stale-body-no-stamp, which reads as a miss. All cache failures are swallowed —
+ * the cache is an optimisation, never a dependency.
+ */
+
+import { createHash } from "node:crypto";
+import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
+import { basename, join } from "node:path";
+import { homedir } from "node:os";
+
+export interface FileStamp {
+  readonly size: number;
+  readonly mtimeMs: number;
+}
+
+interface CacheStamp {
+  readonly source: string;
+  readonly size: number;
+  readonly mtimeMs: number;
+}
+
+function cacheRoot(): string {
+  const xdg = process.env.XDG_CACHE_HOME;
+  if (typeof xdg === "string" && xdg.trim() !== "") return join(xdg, "pi-rlm", "anydoc");
+  return join(homedir(), ".cache", "pi-rlm", "anydoc");
+}
+
+function entryBase(absPath: string): string {
+  const name = basename(absPath).replace(/[^\w.-]+/g, "-").slice(0, 80) || "doc";
+  const sha8 = createHash("sha256").update(absPath).digest("hex").slice(0, 8);
+  return `${name}-${sha8}`;
+}
+
+function isStamp(value: unknown): value is CacheStamp {
+  if (value === null || typeof value !== "object") return false;
+  const r = value as Record<string, unknown>;
+  return typeof r.source === "string"
+    && typeof r.size === "number"
+    && typeof r.mtimeMs === "number";
+}
+
+/** Capture (size, mtimeMs) for a source file. Returns undefined on I/O failure. */
+export async function captureStamp(absPath: string): Promise<FileStamp | undefined> {
+  try {
+    const s = await stat(absPath);
+    return Object.freeze({ size: s.size, mtimeMs: s.mtimeMs });
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Read a cached Markdown body if the stamp still matches a fresh stat of `absPath`.
+ * Returns undefined on any miss or I/O failure.
+ */
+export async function readMdCache(absPath: string): Promise<string | undefined> {
+  try {
+    const base = join(cacheRoot(), entryBase(absPath));
+    const stampRaw = await readFile(`${base}.json`, "utf-8");
+    const stamp: unknown = JSON.parse(stampRaw);
+    if (!isStamp(stamp)) return undefined;
+    const s = await stat(absPath);
+    if (s.size !== stamp.size || s.mtimeMs !== stamp.mtimeMs) return undefined;
+    return await readFile(`${base}.md`, "utf-8");
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Write Markdown body then the PRE-CAPTURED stamp. Failures are swallowed.
+ *
+ * If a post-conversion stat disagrees with `stamp`, the write is skipped (file changed
+ * mid-conversion — stamping the new mtime against the old body would freeze stale content).
+ * Body-before-stamp is load-bearing (see module docstring).
+ */
+export async function writeMdCache(
+  absPath: string,
+  markdown: string,
+  stamp: FileStamp,
+): Promise<void> {
+  try {
+    // Refuse to cache if the source moved under us during conversion.
+    const s = await stat(absPath);
+    if (s.size !== stamp.size || s.mtimeMs !== stamp.mtimeMs) return;
+
+    const root = cacheRoot();
+    await mkdir(root, { recursive: true });
+    const base = join(root, entryBase(absPath));
+    // Body first, stamp second — crash between leaves a miss, never a wrong hit.
+    await writeFile(`${base}.md`, markdown, "utf-8");
+    const out: CacheStamp = Object.freeze({
+      source: absPath,
+      size: stamp.size,
+      mtimeMs: stamp.mtimeMs,
+    });
+    await writeFile(`${base}.json`, `${JSON.stringify(out)}\n`, "utf-8");
+  } catch {
+    // optimisation only
+  }
+}

package/src/context/merge.ts

@@ -0,0 +1,97 @@
+/**
+ * Merge / filter helpers for the live context list.
+ *
+ * Lifted from the old library-context module with `lib/` → `ctx/` and the empty-prefix
+ * short-circuit for cwd sources.
+ */
+
+import { estimateTokens } from "../text/tokens.ts";
+import {
+  contextEntryPath,
+  isContextFile,
+  namespaceContextFiles,
+  payloadPrefix,
+} from "./namespace.ts";
+import type { ContextFile } from "./types.ts";
+
+export interface FilteredContext {
+  readonly files: readonly ContextFile[];
+  /** Prefixes that selected zero files — the caller decides whether that is fatal. */
+  readonly unmatched: readonly string[];
+}
+
+/**
+ * Narrow a context payload to entries under any of `prefixes` (plain prefix match, no globs).
+ *
+ * Backs `rlm_query(prompt, paths=[…])`. Prefix-only is deliberate: the sandbox's own filters use
+ * Python `fnmatch`, which has no host-side equivalent here, and a subtree prefix is what callers
+ * actually want — the child can still `search()` inside the slice.
+ */
+export function filterContextByPaths(context: unknown, prefixes: readonly string[]): FilteredContext {
+  if (!Array.isArray(context) || prefixes.length === 0) {
+    return Object.freeze({ files: Object.freeze([]), unmatched: Object.freeze(Array.from(prefixes)) });
+  }
+  const hit = new Array<boolean>(prefixes.length).fill(false);
+  const out = new Array<ContextFile>(context.length); // pre-allocated, trimmed once below
+  let n = 0;
+  for (let i = 0; i < context.length; i++) {
+    const entry: unknown = context[i];
+    if (!isContextFile(entry)) continue;
+    for (let p = 0; p < prefixes.length; p++) {
+      if (!entry.path.startsWith(prefixes[p])) continue;
+      hit[p] = true;
+      out[n++] = entry;
+      break;
+    }
+  }
+  out.length = n;
+  const unmatched = new Array<string>(prefixes.length); // pre-allocated, no .push()
+  let u = 0;
+  for (let p = 0; p < prefixes.length; p++) {
+    if (!hit[p]) unmatched[u++] = prefixes[p];
+  }
+  unmatched.length = u;
+  return Object.freeze({ files: Object.freeze(out), unmatched: Object.freeze(unmatched) });
+}
+
+/**
+ * Append a source payload into an existing list context.
+ * Skips a payload whose `ctx/<id>/` prefix is already present, so a repeat load is a no-op.
+ *
+ * Cwd-seeded (un-prefixed) files never carry a `ctx/<id>/` prefix, so `payloadPrefix` returns
+ * undefined for them and this path-scan does not fire. Idempotency for the cwd seed is owned
+ * by the host `loaded` set (sentinel `""`) and the seeded-cwd absolute-path short-circuit in
+ * bridge/add-context.ts — not by this merge.
+ */
+export function mergeIntoContext(base: unknown, sourcePayload: unknown): unknown {
+  if (!Array.isArray(base)) return base;
+  if (Array.isArray(sourcePayload)) {
+    if (sourcePayload.length === 0) return base;
+    const prefix = payloadPrefix(sourcePayload);
+    // Only `ctx/<id>/` prefixes participate in path-scan dedup (payloadPrefix never returns "").
+    if (prefix !== undefined) {
+      for (let i = 0; i < base.length; i++) {
+        const path = contextEntryPath(base[i]);
+        if (path !== undefined && path.startsWith(prefix)) return base; // already present
+      }
+    }
+    const merged = new Array<unknown>(base.length + sourcePayload.length);
+    for (let i = 0; i < base.length; i++) merged[i] = base[i];
+    for (let i = 0; i < sourcePayload.length; i++) merged[base.length + i] = sourcePayload[i];
+    return merged;
+  }
+  if (typeof sourcePayload === "string") {
+    // Raw string payload: wrap once under the unknown prefix.
+    return mergeIntoContext(base, namespaceContextFiles(sourcePayload, "unknown"));
+  }
+  return base;
+}
+
+/** Build a frozen ContextFile from path + content (shared by every source-* producer). */
+export function makeContextFile(path: string, content: string): ContextFile {
+  return Object.freeze({
+    path,
+    content,
+    tokens: Math.max(1, estimateTokens(content.length)),
+  });
+}

package/src/context/namespace.ts

@@ -0,0 +1,180 @@
+/**
+ * Source-id / path-prefix derivation for add_context sources.
+ *
+ * Lifted from the old library-context module with `lib/` → `ctx/`. Fingerprinted source ids
+ * keep two sources that share a basename from colliding.
+ */
+
+import { createHash } from "node:crypto";
+import { basename, isAbsolute, resolve } from "node:path";
+import { estimateTokens } from "../text/tokens.ts";
+import { LEGACY_UNKNOWN_PREFIX, type ContextFile } from "./types.ts";
+
+export interface ContextNamespace {
+  readonly sourceId: string;
+  readonly pathPrefix: string;
+}
+
+/** https://host/… or git@host:… — option-injection safe (never starts with "-"). */
+export const GIT_URL = /^(https:\/\/|git@)[\w.-]+[:/]\S+$/;
+
+/** Short, stable discriminator so two sources never share a namespace. */
+function sourceFingerprint(canonical: string): string {
+  return createHash("sha256").update(canonical).digest("hex").slice(0, 8);
+}
+
+/**
+ * Sanitize a path/url basename into a stable, filesystem-safe source id.
+ * `resolvedPath` (absolute path) or the git URL is fingerprinted into the id so
+ * distinct sources sharing a basename get distinct namespaces
+ * (`ctx/utils-3f9a1c02/` vs `ctx/utils-a1b2c3d4/`).
+ */
+export function contextSourceId(source: string, resolvedPath?: string): string {
+  const trimmed = source.trim();
+  const isGit = GIT_URL.test(trimmed);
+  let raw: string;
+  if (isGit) {
+    const m = trimmed.match(/(?:\/|:)([\w.-]+?)(?:\.git)?\/?\s*$/);
+    raw = m?.[1] ?? "repo";
+  } else {
+    raw = basename(resolvedPath ?? trimmed);
+  }
+  const cleaned = raw
+    .replace(/\.git$/i, "")
+    .replace(/[^\w.-]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 80);
+  const canonical = isGit ? trimmed : (resolvedPath ?? trimmed);
+  return `${cleaned.length > 0 ? cleaned : "ctx"}-${sourceFingerprint(canonical)}`;
+}
+
+export function pathPrefixFor(sourceId: string): string {
+  return `ctx/${sourceId}/`;
+}
+
+/**
+ * Derive the namespace for a source string without packing (host-side idempotency).
+ * Returns both sourceId and pathPrefix so callers never un-parse the prefix.
+ */
+export function contextNamespace(source: string, cwd: string): ContextNamespace {
+  const trimmed = source.trim();
+  const sourceId = GIT_URL.test(trimmed)
+    ? contextSourceId(trimmed)
+    : contextSourceId(trimmed, isAbsolute(trimmed) ? trimmed : resolve(cwd, trimmed));
+  return Object.freeze({ sourceId, pathPrefix: pathPrefixFor(sourceId) });
+}
+
+/** Derive the path prefix for a source string without packing. */
+export function contextPathPrefix(source: string, cwd: string): string {
+  return contextNamespace(source, cwd).pathPrefix;
+}
+
+/**
+ * Apply a path prefix. Empty prefix is identity — the cwd source stays un-prefixed so
+ * search() hits remain real paths that edit/write can act on.
+ */
+export function applyPathPrefix(relPath: string, pathPrefix: string): string {
+  const cleaned = relPath.replace(/^\/+/, "");
+  if (pathPrefix === "") return cleaned;
+  if (cleaned.startsWith(pathPrefix)) return cleaned;
+  return `${pathPrefix}${cleaned}`;
+}
+
+/** Namespaced files plus summed content chars (one pass). */
+export function namespaceContextFilesWithChars(
+  payload: unknown,
+  sourceId: string,
+): { readonly files: readonly ContextFile[]; readonly chars: number } {
+  const prefix = pathPrefixFor(sourceId);
+  if (typeof payload === "string") {
+    return Object.freeze({
+      files: Object.freeze([
+        Object.freeze({
+          path: `${prefix}content`,
+          content: payload,
+          tokens: Math.max(1, estimateTokens(payload.length)),
+        }),
+      ]),
+      chars: payload.length,
+    });
+  }
+  if (!Array.isArray(payload)) return Object.freeze({ files: Object.freeze([]), chars: 0 });
+  const out = new Array<ContextFile>(payload.length);
+  let n = 0;
+  let chars = 0;
+  for (const item of payload) {
+    if (item === null || typeof item !== "object") continue;
+    const rec = item as Record<string, unknown>;
+    const content = typeof rec.content === "string" ? rec.content : String(rec.content ?? "");
+    let path = typeof rec.path === "string" ? rec.path : "unknown";
+    path = applyPathPrefix(path, prefix);
+    const tokens = typeof rec.tokens === "number" && Number.isFinite(rec.tokens)
+      ? Math.max(0, Math.floor(rec.tokens))
+      : Math.max(1, estimateTokens(content.length));
+    out[n++] = Object.freeze({ path, content, tokens });
+    chars += content.length;
+  }
+  out.length = n;
+  return Object.freeze({ files: Object.freeze(out), chars });
+}
+
+/** Namespace file entries under `ctx/<sourceId>/…` (single shared implementation). */
+export function namespaceContextFiles(
+  payload: unknown,
+  sourceId: string,
+): readonly ContextFile[] {
+  return namespaceContextFilesWithChars(payload, sourceId).files;
+}
+
+/** The one `ctx/<id>/` matcher. Never re-declare this regex; use the helpers below. */
+const CTX_PREFIX_RE = /^(ctx\/[^/]+\/)/;
+
+/** Narrow an unknown context entry to a ContextFile. Type guard, never a cast. */
+export function isContextFile(entry: unknown): entry is ContextFile {
+  if (entry === null || typeof entry !== "object") return false;
+  return "path" in entry && typeof entry.path === "string"
+    && "content" in entry && typeof entry.content === "string";
+}
+
+/** `path` of a context entry, or undefined when the entry is not file-shaped. */
+export function contextEntryPath(entry: unknown): string | undefined {
+  return isContextFile(entry) ? entry.path : undefined;
+}
+
+/** The `ctx/<id>/` prefix owning this path, or undefined. Skips the legacy catch-all. */
+function ctxPrefixOf(path: string): string | undefined {
+  const prefix = CTX_PREFIX_RE.exec(path)?.[1];
+  // `ctx/unknown/` is the legacy catch-all: never treat it as an identity.
+  return prefix === undefined || prefix === LEGACY_UNKNOWN_PREFIX ? undefined : prefix;
+}
+
+/** First `ctx/<id>/` prefix found in the payload, or undefined. Skips `ctx/unknown/`. */
+export function payloadPrefix(payload: readonly unknown[]): string | undefined {
+  for (let i = 0; i < payload.length; i++) {
+    const path = contextEntryPath(payload[i]);
+    if (path === undefined) continue;
+    const prefix = ctxPrefixOf(path);
+    if (prefix !== undefined) return prefix;
+  }
+  return undefined;
+}
+
+/**
+ * Every distinct `ctx/<id>/` prefix present in a context payload.
+ *
+ * The loaded-prefix set in bridge/add-context.ts is a CACHE of this — derived state, never
+ * independent state, so it may only be cleared by re-deriving it from the live payload.
+ * Empty-prefix (cwd) sources are intentionally invisible here — the host `loaded` set holds
+ * `""` as its sentinel for those.
+ */
+export function contextPrefixesIn(context: unknown): readonly string[] {
+  if (!Array.isArray(context)) return Object.freeze([]);
+  const seen = new Set<string>();
+  for (let i = 0; i < context.length; i++) {
+    const path = contextEntryPath(context[i]);
+    if (path === undefined) continue;
+    const prefix = ctxPrefixOf(path);
+    if (prefix !== undefined) seen.add(prefix);
+  }
+  return Object.freeze(Array.from(seen));
+}

package/src/context/resolve.ts

@@ -0,0 +1,122 @@
+/**
+ * One router: source string → SourceResult.
+ * git URL → source-git · dir → source-dir · file → document? source-doc : source-text.
+ */
+
+import { basename, isAbsolute, resolve } from "node:path";
+import { stat } from "node:fs/promises";
+import type { Result } from "../util/errors.ts";
+import { documentExtFromPath, getAnydoc } from "./anydoc.ts";
+import { GIT_URL, contextSourceId, pathPrefixFor } from "./namespace.ts";
+import { documentToContextFile } from "./source-doc.ts";
+import { sourceDir } from "./source-dir.ts";
+import { sourceGit } from "./source-git.ts";
+import { textToContextFile } from "./source-text.ts";
+import { checkPathSafety, isSensitivePath } from "./walk.ts";
+import {
+  MAX_CONTEXT_FILE_BYTES,
+  type ResolveOpts,
+  type SourceResult,
+} from "./types.ts";
+
+function emptySkipped(): SourceResult["skipped"] {
+  return Object.freeze([]);
+}
+
+function singleFileResult(
+  file: SourceResult["payload"][number],
+  sourceId: string,
+  pathPrefix: string,
+  documents: number,
+  converted: number,
+  skipped: SourceResult["skipped"] = emptySkipped(),
+): SourceResult {
+  return Object.freeze({
+    payload: Object.freeze([file]),
+    files: 1,
+    chars: file.content.length,
+    sourceId,
+    pathPrefix,
+    documents,
+    converted,
+    skipped,
+  });
+}
+
+/**
+ * Resolve a source string (local path or git URL) into a sandbox-ready SourceResult.
+ * `pathPrefix: ""` marks the primary/cwd source (un-prefixed paths).
+ */
+export async function resolveSource(
+  source: string,
+  opts: ResolveOpts,
+): Promise<Result<SourceResult, string>> {
+  const trimmed = source.trim();
+  if (trimmed === "") return { ok: false, error: "add_context: empty source" };
+  if (GIT_URL.test(trimmed)) return await sourceGit(trimmed, opts);
+  if (/^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed)) {
+    return { ok: false, error: `unsupported URL scheme (only https:// and git@ are allowed): ${trimmed}` };
+  }
+  const path = isAbsolute(trimmed) ? trimmed : resolve(opts.cwd, trimmed);
+  let s: Awaited<ReturnType<typeof stat>>;
+  try {
+    s = await stat(path);
+  } catch {
+    return { ok: false, error: `add_context: path not found: ${path}` };
+  }
+  const sourceId = contextSourceId(trimmed, path);
+  const pathPrefix = opts.pathPrefix !== undefined ? opts.pathPrefix : pathPrefixFor(sourceId);
+
+  if (s.isDirectory()) {
+    return { ok: true, value: await sourceDir(path, trimmed, { ...opts, pathPrefix }) };
+  }
+
+  // Single-file secret refuse — never pack a .env / key into context for a sub-LLM.
+  const rel = basename(path);
+  if (isSensitivePath(rel) || isSensitivePath(trimmed.replace(/^\.\//, ""))) {
+    return { ok: false, error: `add_context: refused sensitive path: ${path}` };
+  }
+
+  // Symlink safety for single-file: refuse escape of cwd / resolved sensitive targets.
+  const safety = await checkPathSafety(path, opts.cwd);
+  if (!safety.ok) {
+    return {
+      ok: false,
+      error: `add_context: refused ${safety.reason} path: ${path}`,
+    };
+  }
+
+  // Single file — preserve oversize wording (phase-context asserts on limit + llm_query_chunked).
+  if (s.size > MAX_CONTEXT_FILE_BYTES) {
+    return {
+      ok: false,
+      error: `add_context: ${path} is ${s.size.toLocaleString()} bytes `
+        + `(limit ${MAX_CONTEXT_FILE_BYTES.toLocaleString()}) — `
+        + "open() it in the REPL and delegate with llm_query_chunked instead",
+    };
+  }
+
+  const anydoc = await getAnydoc();
+  // formatFromPath is extension-based; one call on the path is enough.
+  const format = anydoc?.formatFromPath(path) ?? documentExtFromPath(path);
+  if (format !== null) {
+    const doc = await documentToContextFile(safety.realAbs, rel, pathPrefix, anydoc);
+    if (!doc.ok) {
+      return { ok: false, error: `add_context: could not convert ${path} (${doc.skipped.reason})` };
+    }
+    return {
+      ok: true,
+      value: singleFileResult(
+        doc.value, sourceId, pathPrefix,
+        1, // documents
+        doc.converted ? 1 : 0,
+      ),
+    };
+  }
+
+  const text = await textToContextFile(safety.realAbs, rel, pathPrefix, MAX_CONTEXT_FILE_BYTES);
+  if (!text.ok) {
+    return { ok: false, error: `add_context: could not read ${path} (${text.skipped.reason})` };
+  }
+  return { ok: true, value: singleFileResult(text.value, sourceId, pathPrefix, 0, 0) };
+}