@mrdrmccoy/xerj-opencode-plugin 0.0.0-stage → 0.4.0

package/src/index/brain.ts

@@ -0,0 +1,225 @@
+// Opt-in second brain. `autoindex --brain` writes documents under the brain
+// prefix and edges to `.xerj-memory-<name>-edges`; it cannot run in watch mode
+// (watch requires --no-graph), so brains are one-shot runs after the project
+// index and on explicit reindex.
+
+import { spawn } from "node:child_process";
+import { stat } from "node:fs/promises";
+import { join } from "node:path";
+import type { Config } from "../config.ts";
+import type { Logger } from "../log.ts";
+import { XerjClient, type ClientObserver } from "../node/client.ts";
+import type { BrainSpec } from "../paths.ts";
+
+export interface BrainRun {
+  readonly name: string;
+  readonly prefix: string;
+  /** Epoch ms of the last attempt; 0 means it has not run yet. */
+  readonly at: number;
+  readonly docs: number;
+  readonly edges: number;
+  readonly error: string | null;
+}
+
+export interface BrainRunSummary {
+  readonly docs: number;
+  readonly edges: number;
+}
+
+export interface BrainIndexerOptions {
+  readonly binaryPath: string;
+  readonly nodeUrl: string;
+  readonly dataRoot: string;
+  readonly config: Config;
+  readonly specs: ReadonlyArray<BrainSpec>;
+  readonly logger: Logger;
+  readonly observer?: ClientObserver;
+  readonly onChange?: () => void;
+  readonly onRun?: () => void;
+}
+
+const MIN_TIMEOUT_MS = 60_000;
+
+export class BrainIndexer {
+  private readonly runs = new Map<string, BrainRun>();
+  private readonly client: XerjClient;
+  private running = false;
+
+  constructor(private readonly options: BrainIndexerOptions) {
+    this.client = new XerjClient(options.nodeUrl, undefined, options.observer);
+  }
+
+  /** Configured brains, with zeros for the ones that have not run yet. */
+  snapshot(): ReadonlyArray<BrainRun> {
+    return this.options.specs.map(
+      (spec) =>
+        this.runs.get(spec.name) ?? {
+          name: spec.name,
+          prefix: spec.prefix,
+          at: 0,
+          docs: 0,
+          edges: 0,
+          error: null,
+        },
+    );
+  }
+
+  async runAll(): Promise<void> {
+    if (this.running) {
+      this.options.logger.debug("brain indexing is already running");
+      return;
+    }
+
+    this.running = true;
+    try {
+      for (const spec of this.options.specs) await this.run(spec);
+    } finally {
+      this.running = false;
+    }
+  }
+
+  private async run(spec: BrainSpec): Promise<void> {
+    const info = await stat(spec.directory).catch(() => null);
+    if (!info?.isDirectory()) {
+      this.record(spec, {
+        at: Date.now(),
+        docs: 0,
+        edges: 0,
+        error: `${spec.label} is not a directory`,
+      });
+      return;
+    }
+
+    const args = [
+      "autoindex",
+      spec.directory,
+      "--url",
+      this.options.nodeUrl,
+      "--prefix",
+      spec.prefix,
+      "--brain",
+      spec.name,
+      "--state-dir",
+      join(this.options.dataRoot, "state", spec.prefix),
+      "--json",
+      "--progress",
+      "none",
+    ];
+
+    if (!this.options.config.index.semantic) args.push("--no-semantic");
+    if (this.options.config.index.workers !== null) {
+      args.push("--workers", String(this.options.config.index.workers));
+    }
+
+    try {
+      const output = await run(this.options.binaryPath, args, this.timeoutMs());
+      const summary = await this.summarize(spec, parseBrainRun(output));
+      this.record(spec, { at: Date.now(), ...summary, error: null });
+      this.options.onRun?.();
+      this.options.logger.info(`brain ${spec.name} indexed`, summary);
+    } catch (error) {
+      this.record(spec, {
+        at: Date.now(),
+        docs: 0,
+        edges: 0,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      this.options.logger.warn(`brain ${spec.name} indexing failed`, String(error));
+    }
+  }
+
+  /** The run document reports edges written, not the graph's live total. */
+  private async summarize(
+    spec: BrainSpec,
+    fallback: BrainRunSummary,
+  ): Promise<BrainRunSummary> {
+    try {
+      const overview = await this.client.get(
+        `/_graph/${encodeURIComponent(spec.name)}/overview`,
+        "graph overview failed",
+      );
+      if (!isRecord(overview)) return fallback;
+
+      // `nodes.total` counts section records, not files; keep the run's doc
+      // total and take only the live edge count from the overview.
+      const edges = isRecord(overview.edges)
+        ? numberField(overview.edges, "total")
+        : null;
+
+      return {
+        docs: fallback.docs,
+        edges: edges ?? fallback.edges,
+      };
+    } catch (error) {
+      this.options.logger.debug("could not read the graph overview", String(error));
+      return fallback;
+    }
+  }
+
+  private record(spec: BrainSpec, run: Omit<BrainRun, "name" | "prefix">): void {
+    this.runs.set(spec.name, { name: spec.name, prefix: spec.prefix, ...run });
+    this.options.onChange?.();
+  }
+
+  private timeoutMs(): number {
+    return Math.max(MIN_TIMEOUT_MS, this.options.config.index.maxMinutes * 60_000);
+  }
+}
+
+/** Reads the run document `autoindex --json` prints as its last JSON line. */
+export function parseBrainRun(output: string): BrainRunSummary {
+  const line = [...output.trim().split("\n")]
+    .reverse()
+    .find((entry) => entry.trim().startsWith("{"));
+
+  if (!line) return { docs: 0, edges: 0 };
+
+  try {
+    const document = JSON.parse(line) as Record<string, unknown>;
+    const graph = isRecord(document.graph) ? document.graph : {};
+    return {
+      docs:
+        numberField(document, "records_total") ??
+        numberField(document, "records_submitted_this_run") ??
+        0,
+      edges: numberField(graph, "edges_written") ?? 0,
+    };
+  } catch {
+    return { docs: 0, edges: 0 };
+  }
+}
+
+function run(
+  command: string,
+  args: ReadonlyArray<string>,
+  timeoutMs: number,
+): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, [...args], { stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+
+    const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
+    child.stdout.on("data", (chunk: Buffer) => (stdout += chunk.toString()));
+    child.stderr.on("data", (chunk: Buffer) => (stderr += chunk.toString()));
+    child.on("error", (error) => {
+      clearTimeout(timer);
+      reject(error);
+    });
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      // Exit 3 completed with junk recorded; the run still counts.
+      if (code === 0 || code === 3) resolve(stdout);
+      else reject(new Error(`autoindex exited ${code}: ${stderr.trim().slice(0, 300)}`));
+    });
+  });
+}
+
+function numberField(value: Record<string, unknown>, key: string): number | null {
+  const found = value[key];
+  return typeof found === "number" && Number.isFinite(found) ? found : null;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/index/corpus.ts

@@ -0,0 +1,164 @@
+// Corpus layout and the autoindex command line. Pure pieces live here so the
+// tests can pin the exact flags without spawning anything.
+//
+// Verified against v1.0.0-rc.100:
+// - `--watch` requires `--no-graph` and emits one run document per pass on
+//   stdout when `--json` is set.
+// - exit 3 is success with junk recorded; exit 4 is a decision request with a
+//   JSON body on stdout.
+
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { Config } from "../config.ts";
+import { projectStateDir } from "../paths.ts";
+
+export interface CorpusLayout {
+  readonly projectRoot: string;
+  readonly prefix: string;
+  readonly stateDir: string;
+  readonly ignoreFile: string;
+}
+
+export function corpusLayout(
+  projectRoot: string,
+  prefix: string,
+  dataRoot: string,
+): CorpusLayout {
+  return {
+    projectRoot,
+    prefix,
+    // `--watch` refuses a state dir inside the watched tree: journal writes are
+    // filesystem events and would restart the pass forever. Keep the journal
+    // beside the node data instead.
+    stateDir: join(dataRoot, "state", prefix),
+    ignoreFile: join(projectRoot, ".xerjignore"),
+  };
+}
+
+/** Secret-shaped names XERJ's own ignore rules do not cover. */
+const IGNORE_DEFAULTS = [
+  "# Written once by the XERJ OpenCode plugin. Edit freely; it is not rewritten.",
+  ".env",
+  ".env.*",
+  "*.pem",
+  "*.key",
+  "**/secrets/**",
+  "",
+].join("\n");
+
+export async function ensureIgnoreFile(layout: CorpusLayout): Promise<boolean> {
+  await mkdir(projectStateDir(layout.projectRoot), { recursive: true });
+  try {
+    await writeFile(layout.ignoreFile, IGNORE_DEFAULTS, { flag: "wx" });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export interface AutoindexOptions {
+  readonly layout: CorpusLayout;
+  readonly nodeUrl: string;
+  readonly config: Config;
+  readonly watch: boolean;
+}
+
+export function autoindexArgs(options: AutoindexOptions): ReadonlyArray<string> {
+  const { layout, nodeUrl, config, watch } = options;
+  const args = [
+    "autoindex",
+    layout.projectRoot,
+    "--url",
+    nodeUrl,
+    "--prefix",
+    layout.prefix,
+    "--state-dir",
+    layout.stateDir,
+    "--no-graph",
+    "--json",
+    "--progress",
+    "none",
+  ];
+
+  if (watch) {
+    args.push("--watch", "--debounce", String(config.index.debounceMs));
+  }
+
+  if (config.index.workers !== null) {
+    args.push("--workers", String(config.index.workers));
+  }
+
+  if (!config.index.semantic) {
+    args.push("--no-semantic");
+  }
+
+  if (config.index.maxBytes > 0) {
+    args.push("--max-index-size", String(config.index.maxBytes));
+  }
+
+  args.push("--max-minutes", String(config.index.maxMinutes));
+  return args;
+}
+
+export interface CorpusRun {
+  readonly at: number;
+  readonly generation: number | null;
+  readonly files: number | null;
+  readonly records: number | null;
+  readonly junk: number | null;
+  readonly reason: string | null;
+}
+
+/** Parses one line of `autoindex --json` output. Returns null for anything else. */
+export function parseRunDocument(line: string): CorpusRun | null {
+  const trimmed = line.trim();
+  if (!trimmed.startsWith("{")) return null;
+
+  let value: unknown;
+  try {
+    value = JSON.parse(trimmed);
+  } catch {
+    return null;
+  }
+
+  if (typeof value !== "object" || value === null) return null;
+  const record = value as Record<string, unknown>;
+
+  // Watch mode emits a run document per pass; decision requests carry a
+  // different shape and are handled by the caller.
+  if (record.doc_kind !== "run") return null;
+
+  const started = typeof record.started === "string" ? Date.parse(record.started) : Number.NaN;
+
+  return {
+    at: Number.isFinite(started) ? started : Date.now(),
+    generation: numberOrNull(record.generation),
+    files: numberOrNull(record.files_indexed ?? record.files_total),
+    records: numberOrNull(record.records_total),
+    junk: numberOrNull(record.junk_records_total),
+    reason: stringOrNull(record.reason),
+  };
+}
+
+export type ExitMeaning = "ok" | "junk" | "decision" | "usage" | "error";
+
+export function exitMeaning(code: number | null): ExitMeaning {
+  if (code === 0) return "ok";
+  if (code === 3) return "junk";
+  if (code === 4) return "decision";
+  if (code === 2) return "usage";
+  return "error";
+}
+
+function numberOrNull(value: unknown): number | null {
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  if (typeof value === "string") {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) ? parsed : null;
+  }
+  return null;
+}
+
+function stringOrNull(value: unknown): string | null {
+  return typeof value === "string" ? value : null;
+}

package/src/index/hidden.ts

@@ -0,0 +1,318 @@
+// Hidden documents: autoindex skips hidden files and directories, so text under
+// `.agents`, `.github`, or any other dot-directory never reaches the project
+// corpus. This scanner walks configured folders and indexes text through
+// `_bulk` into `<prefix>-hidden-docs`.
+//
+// The index is separate from the autoindex corpus so deletions can be
+// reconciled by id. It is still matched by the project prefix, so search and
+// citations use the same paths relative to the project root.
+
+import { createHash } from "node:crypto";
+import { watch, type FSWatcher } from "node:fs";
+import { readdir, readFile, stat } from "node:fs/promises";
+import { basename, join, relative, resolve, sep } from "node:path";
+import type { Logger } from "../log.ts";
+import { NodeRequestError, type XerjClient } from "../node/client.ts";
+import type { DocumentRoot } from "../paths.ts";
+import { isSecretPath, isTextPath } from "../remote/spool.ts";
+
+export interface HiddenIndexOptions {
+  readonly client: XerjClient;
+  readonly projectRoot: string;
+  readonly prefix: string;
+  readonly roots: ReadonlyArray<DocumentRoot>;
+  readonly maxBytes: number;
+  readonly logger: Logger;
+  readonly onChange?: () => void;
+  readonly onRun?: () => void;
+}
+
+export interface HiddenSnapshot {
+  readonly docs: number;
+  readonly lastRunAt: number | null;
+  readonly error: string | null;
+}
+
+interface HiddenFile {
+  readonly id: string;
+  readonly path: string;
+  readonly absolute: string;
+  readonly body: string;
+  readonly mtime: number;
+}
+
+const SKIP_DIRS = new Set([
+  ".git",
+  ".xerj",
+  "node_modules",
+  "dist",
+  "target",
+  "vendor",
+  "coverage",
+  ".venv",
+  "venv",
+  "__pycache__",
+  ".cache",
+  ".next",
+  ".turbo",
+  ".idea",
+  "secrets",
+]);
+
+const MAX_FILES = 2_000;
+const MAX_INDEXED_DOCS = 5_000;
+const BATCH_SIZE = 50;
+const DEBOUNCE_MS = 1_500;
+const FALLBACK_POLL_MS = 30_000;
+
+export class HiddenIndex {
+  readonly index: string;
+
+  private readonly watchers: FSWatcher[] = [];
+  private interval: ReturnType<typeof setInterval> | null = null;
+  private timer: ReturnType<typeof setTimeout> | null = null;
+  private running = false;
+  private queued = false;
+  private mapped = false;
+  private state: HiddenSnapshot = { docs: 0, lastRunAt: null, error: null };
+
+  constructor(private readonly options: HiddenIndexOptions) {
+    this.index = `${options.prefix}-hidden-docs`;
+  }
+
+  snapshot(): HiddenSnapshot {
+    return this.state;
+  }
+
+  async start(): Promise<void> {
+    if (this.options.roots.length === 0) return;
+
+    await this.run();
+    this.watchRoots();
+  }
+
+  async stop(): Promise<void> {
+    for (const watcher of this.watchers) watcher.close();
+    this.watchers.length = 0;
+
+    if (this.interval) clearInterval(this.interval);
+    if (this.timer) clearTimeout(this.timer);
+    this.interval = null;
+    this.timer = null;
+  }
+
+  async reindex(): Promise<void> {
+    await this.run();
+  }
+
+  private watchRoots(): void {
+    for (const root of this.options.roots) {
+      try {
+        const watcher = watch(root.directory, { recursive: true }, () =>
+          this.schedule(),
+        );
+        watcher.on("error", (error) => {
+          this.options.logger.debug(
+            `hidden watch failed for ${root.label}; polling instead`,
+            String(error),
+          );
+          this.useFallback();
+        });
+        this.watchers.push(watcher);
+      } catch (error) {
+        this.options.logger.debug(
+          `hidden watch unavailable for ${root.label}; polling instead`,
+          String(error),
+        );
+        this.useFallback();
+      }
+    }
+  }
+
+  private useFallback(): void {
+    if (this.interval) return;
+    this.interval = setInterval(() => void this.run(), FALLBACK_POLL_MS);
+  }
+
+  private schedule(): void {
+    if (this.timer) return;
+    this.timer = setTimeout(() => {
+      this.timer = null;
+      void this.run();
+    }, DEBOUNCE_MS);
+  }
+
+  private async run(): Promise<void> {
+    if (this.running) {
+      this.queued = true;
+      return;
+    }
+
+    this.running = true;
+    try {
+      await this.ensureMapping();
+      const files = await this.scan();
+      await this.upload(files);
+      await this.removeStale(new Set(files.map((file) => file.id)));
+
+      this.state = { docs: files.length, lastRunAt: Date.now(), error: null };
+      this.options.onRun?.();
+      this.options.onChange?.();
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      this.state = { ...this.state, lastRunAt: Date.now(), error: message };
+      this.options.logger.warn("hidden document indexing failed", message);
+    } finally {
+      this.running = false;
+      if (this.queued) {
+        this.queued = false;
+        void this.run();
+      }
+    }
+  }
+
+  private async ensureMapping(): Promise<void> {
+    if (this.mapped) return;
+
+    try {
+      await this.options.client.put(
+        `/${this.index}`,
+        {
+          mappings: {
+            properties: {
+              body: { type: "semantic_text" },
+              title: { type: "keyword" },
+              path: { type: "keyword" },
+              abs_path: { type: "keyword" },
+              mtime: { type: "date" },
+              hidden: { type: "boolean" },
+            },
+          },
+        },
+        "could not create the hidden index",
+      );
+    } catch (error) {
+      // The index survives restarts; 400/409 mean it already exists.
+      if (
+        !(error instanceof NodeRequestError) ||
+        (error.status !== 400 && error.status !== 409)
+      ) {
+        throw error;
+      }
+    }
+
+    this.mapped = true;
+  }
+
+  private async scan(): Promise<HiddenFile[]> {
+    const files: HiddenFile[] = [];
+    const root = resolve(this.options.projectRoot);
+
+    for (const documentRoot of this.options.roots) {
+      await this.walk(documentRoot.directory, root, files);
+      if (files.length >= MAX_FILES) {
+        this.options.logger.warn(
+          `hidden indexing stopped at ${MAX_FILES} files; ` +
+            "raise the limit only if the tree is small",
+        );
+        break;
+      }
+    }
+
+    return files;
+  }
+
+  private async walk(
+    directory: string,
+    projectRoot: string,
+    files: HiddenFile[],
+  ): Promise<void> {
+    const entries = await readdir(directory, { withFileTypes: true }).catch(
+      () => [],
+    );
+
+    for (const entry of entries) {
+      if (files.length >= MAX_FILES) return;
+
+      const absolute = join(directory, entry.name);
+
+      if (entry.isDirectory()) {
+        if (SKIP_DIRS.has(entry.name)) continue;
+        await this.walk(absolute, projectRoot, files);
+        continue;
+      }
+
+      if (!entry.isFile() || !isTextPath(entry.name) || isSecretPath(entry.name)) {
+        continue;
+      }
+
+      const info = await stat(absolute).catch(() => null);
+      if (!info || info.size === 0 || info.size > this.options.maxBytes) {
+        continue;
+      }
+
+      const raw = await readFile(absolute, "utf8").catch(() => null);
+      if (raw === null || raw.includes("\u0000")) continue;
+
+      const path = relative(projectRoot, absolute).split(sep).join("/");
+      files.push({
+        id: createHash("sha256").update(path).digest("hex").slice(0, 24),
+        path,
+        absolute,
+        body: raw,
+        mtime: info.mtimeMs,
+      });
+    }
+  }
+
+  private async upload(files: ReadonlyArray<HiddenFile>): Promise<void> {
+    for (let index = 0; index < files.length; index += BATCH_SIZE) {
+      const batch = files.slice(index, index + BATCH_SIZE);
+      await this.options.client.bulk(
+        this.index,
+        batch.map((file) => ({
+          id: file.id,
+          source: {
+            body: file.body,
+            title: basename(file.path),
+            path: file.path,
+            abs_path: file.absolute,
+            mtime: new Date(file.mtime).toISOString(),
+            hidden: true,
+          },
+        })),
+      );
+    }
+  }
+
+  private async removeStale(current: ReadonlySet<string>): Promise<void> {
+    const existing = await this.existingIds();
+    const stale = existing.filter((id) => !current.has(id));
+    if (stale.length > 0) await this.options.client.bulkDelete(this.index, stale);
+  }
+
+  private async existingIds(): Promise<string[]> {
+    try {
+      const response = await this.options.client.search(this.index, {
+        size: MAX_INDEXED_DOCS,
+        _source: ["path"],
+        query: { match_all: {} },
+      });
+
+      if (!isRecord(response)) return [];
+      const hits = response.hits;
+      if (!isRecord(hits) || !Array.isArray(hits.hits)) return [];
+
+      return hits.hits.flatMap((hit): string[] =>
+        isRecord(hit) && typeof hit._id === "string" ? [hit._id] : [],
+      );
+    } catch (error) {
+      if (error instanceof NodeRequestError && error.status === 404) return [];
+      throw error;
+    }
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}