@a-t-h-i/bot-lobby 0.6.37 → 0.6.38

package/src/scripts/library.ts

@@ -0,0 +1,168 @@
+/**
+ * The scripts library: scripts every agent shares for jobs that come back
+ * (seeding data, regenerating fixtures, a set of checks, a migration), so a
+ * step done once is run the next time, not worked out again, which saves
+ * tokens and time. They live in `<data>/scripts/`, one file each, named for
+ * what they do in snake_case with the language's extension
+ * (`seed_public_holidays.py`, `run_changed_tests.sh`). The first lines of each
+ * file say what it is for and how to run it, and the library's index is read
+ * from them, so a script edited by hand stays listed. Only the oracle adds or
+ * replaces a script; workers propose one. A replaced script is kept under
+ * `archive/`.
+ */
+import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+/** The languages a script may be written in: how it is run and how its header lines are commented. */
+export const SCRIPT_LANGUAGES = {
+  py: { runner: "python3", comment: "#" },
+  sh: { runner: "bash", comment: "#" },
+  bash: { runner: "bash", comment: "#" },
+  js: { runner: "node", comment: "//" },
+  mjs: { runner: "node", comment: "//" },
+  cjs: { runner: "node", comment: "//" },
+  ts: { runner: "node", comment: "//" },
+  rb: { runner: "ruby", comment: "#" },
+  pl: { runner: "perl", comment: "#" },
+  ps1: { runner: "pwsh -File", comment: "#" },
+} as const;
+export type ScriptLanguage = keyof typeof SCRIPT_LANGUAGES;
+
+/** Longest name, extension included. */
+export const MAX_SCRIPT_NAME = 64;
+/** Largest script the library takes. */
+export const MAX_SCRIPT_BYTES = 64 * 1024;
+
+const NAME = new RegExp(`^[a-z][a-z0-9]*(?:_[a-z0-9]+)+\\.(${Object.keys(SCRIPT_LANGUAGES).join("|")})$`);
+/** How many lines from the top the header is looked for in. */
+const HEADER_LINES = 12;
+
+export interface ScriptEntry {
+  /** The file name: `seed_public_holidays.py`. */
+  name: string;
+  path: string;
+  language: ScriptLanguage;
+  /** What it is for, one line. */
+  purpose: string;
+  /** How to run it, with its arguments; the runner and path when the header gives none. */
+  usage: string;
+}
+
+export function scriptsDir(dataRoot: string): string {
+  return join(dataRoot, "scripts");
+}
+
+/** The words of a name (`seed_public_holidays.py` → `seed public holidays`), for matching it against a step. */
+export function nameWords(name: string): string {
+  return name.replace(/\.[a-z0-9]+$/, "").replace(/_/g, " ");
+}
+
+/**
+ * The library name for what was asked: lower-cased, spaces and hyphens as
+ * underscores (`Seed public-holidays.py` → `seed_public_holidays.py`). Throws
+ * with what is wrong when it still does not read `<what_it_does>.<ext>`.
+ */
+export function scriptName(asked: string): string {
+  const name = asked.trim().toLowerCase().replace(/[\s-]+/g, "_").replace(/_+/g, "_");
+  const languages = Object.keys(SCRIPT_LANGUAGES).map((ext) => `.${ext}`).join(", ");
+  if (!/\.[a-z0-9]+$/.test(name) || !(name.split(".").pop()! in SCRIPT_LANGUAGES)) {
+    throw new Error(`a script name ends in its language's extension (${languages}): "${asked}"`);
+  }
+  if (name.length > MAX_SCRIPT_NAME) throw new Error(`a script name is at most ${MAX_SCRIPT_NAME} characters: "${asked}"`);
+  if (!NAME.test(name)) {
+    throw new Error(`a script is named for what it does, in snake_case words (seed_public_holidays.py): "${asked}"`);
+  }
+  return name;
+}
+
+function language(name: string): ScriptLanguage {
+  return name.split(".").pop() as ScriptLanguage;
+}
+
+function headerValue(lines: readonly string[], key: string): string | undefined {
+  const line = lines.find((entry) => new RegExp(`^\\s*(?:#|//)\\s*${key}\\s*:`, "i").test(entry));
+  return line?.replace(new RegExp(`^\\s*(?:#|//)\\s*${key}\\s*:\\s*`, "i"), "").trim() || undefined;
+}
+
+/** The command that runs a script when its header gives none. */
+export function runCommand(entry: Pick<ScriptEntry, "path" | "language">): string {
+  return `${SCRIPT_LANGUAGES[entry.language].runner} ${entry.path}`;
+}
+
+function readEntry(dir: string, name: string): ScriptEntry | undefined {
+  const path = join(dir, name);
+  try {
+    if (!statSync(path).isFile()) return undefined;
+    const lines = readFileSync(path, "utf8").split("\n").slice(0, HEADER_LINES);
+    const entry = { name, path, language: language(name) };
+    return { ...entry, purpose: headerValue(lines, "purpose") ?? nameWords(name), usage: headerValue(lines, "usage") ?? runCommand(entry) };
+  } catch {
+    return undefined;
+  }
+}
+
+/** Every script in the library, by name; files not named like a script are left out. */
+export function listScripts(dataRoot: string): ScriptEntry[] {
+  const dir = scriptsDir(dataRoot);
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir)
+    .filter((name) => NAME.test(name))
+    .sort()
+    .map((name) => readEntry(dir, name))
+    .filter((entry): entry is ScriptEntry => entry !== undefined);
+}
+
+/** The body with any header lines of its own taken out, and a shebang it starts with kept apart. */
+function splitBody(body: string): { shebang?: string; rest: string } {
+  const lines = body.replace(/\r\n/g, "\n").split("\n");
+  const shebang = lines[0]?.startsWith("#!") ? lines.shift() : undefined;
+  const rest = lines.filter((line, index) => index >= HEADER_LINES || !/^\s*(?:#|\/\/)\s*(?:purpose|usage)\s*:/i.test(line));
+  while (rest.length > 0 && rest[0]!.trim() === "") rest.shift();
+  return { ...(shebang ? { shebang } : {}), rest: rest.join("\n") };
+}
+
+export interface SaveScript {
+  dataRoot: string;
+  name: string;
+  purpose: string;
+  /** How to run it, with its arguments; the runner and path when absent. */
+  usage?: string;
+  body: string;
+}
+
+export interface SavedScript {
+  entry: ScriptEntry;
+  /** Where the version it replaced was kept, when there was one. */
+  archived?: string;
+}
+
+/**
+ * Add a script to the library, or replace the one of that name (the old one
+ * is kept under `archive/`). The engine writes its header: purpose and usage
+ * as comments after any shebang, so the index never depends on the agent
+ * having written one.
+ */
+export function saveScript(input: SaveScript, now = new Date()): SavedScript {
+  const name = scriptName(input.name);
+  const purpose = input.purpose.replace(/\s+/g, " ").trim();
+  if (!purpose) throw new Error("a script needs its purpose: one line on what it does");
+  const { shebang, rest } = splitBody(input.body);
+  if (!rest.trim()) throw new Error("the script is empty");
+  const dir = scriptsDir(input.dataRoot);
+  const path = join(dir, name);
+  const lang = language(name);
+  const usage = input.usage?.replace(/\s+/g, " ").trim() || runCommand({ path, language: lang });
+  const { comment } = SCRIPT_LANGUAGES[lang];
+  const text = [...(shebang ? [shebang] : []), `${comment} purpose: ${purpose}`, `${comment} usage: ${usage}`, "", rest.replace(/\s+$/, ""), ""].join("\n");
+  if (Buffer.byteLength(text) > MAX_SCRIPT_BYTES) throw new Error(`a script is at most ${MAX_SCRIPT_BYTES / 1024} KB`);
+  mkdirSync(dir, { recursive: true });
+  let archived: string | undefined;
+  if (existsSync(path)) {
+    if (readFileSync(path, "utf8") === text) return { entry: readEntry(dir, name)! };
+    mkdirSync(join(dir, "archive"), { recursive: true });
+    archived = join(dir, "archive", `${name}.${now.toISOString().replace(/[:.]/g, "-")}`);
+    copyFileSync(path, archived);
+  }
+  writeFileSync(path, text, { mode: 0o755 });
+  return { entry: readEntry(dir, name)!, ...(archived ? { archived } : {}) };
+}

package/src/scripts/pick.ts

@@ -0,0 +1,79 @@
+/**
+ * Which of the library's scripts fit a step, and how an agent's prompt shows
+ * them. Jev judges each script against the step when it is on (see
+ * `classifier/scripts.ts`); without it, or when it fails, a script fits when
+ * its name or purpose shares words with the step. Either way the prompt names
+ * the library, so an agent can look for itself.
+ */
+import { score, tokenize } from "../knowledge/selector.ts";
+import { listScripts, nameWords, scriptsDir, type ScriptEntry } from "./library.ts";
+
+/** Most scripts one prompt suggests. */
+export const MAX_SUGGESTED = 6;
+/** Most scripts the oracle's prompt lists. */
+export const MAX_LISTED = 40;
+
+/** The scripts whose name or purpose shares the most words with the step, best first. */
+export function keywordScripts(step: string, entries: readonly ScriptEntry[], limit = MAX_SUGGESTED): ScriptEntry[] {
+  const tokens = tokenize(step);
+  return entries
+    .map((entry, index) => ({ entry, index, hits: score(`${nameWords(entry.name)} ${entry.purpose}`, tokens) }))
+    .filter((scored) => scored.hits > 0)
+    .sort((a, b) => b.hits - a.hits || a.index - b.index)
+    .slice(0, limit)
+    .map((scored) => scored.entry);
+}
+
+/** Picks the scripts that fit a step. */
+export interface ScriptPicker {
+  pick(step: string, entries: readonly ScriptEntry[], options?: { signal?: AbortSignal }): Promise<ScriptEntry[]>;
+}
+
+/** The picker that goes by shared words alone. */
+export const keywordPicker: ScriptPicker = { pick: async (step, entries) => keywordScripts(step, entries) };
+
+function line(entry: ScriptEntry): string {
+  return `- \`${entry.name}\` — ${entry.purpose}\n  run: \`${entry.usage}\``;
+}
+
+/**
+ * The Scripts layer of a worker's, scout's or QA agent's prompt: the scripts
+ * that fit this step and how to run each, then where the rest are. "" while
+ * the library is empty.
+ */
+export function scriptsBlock(dataRoot: string, entries: readonly ScriptEntry[], picked: readonly ScriptEntry[]): string {
+  if (entries.length === 0) return "";
+  const where = `The shared scripts library (${entries.length} script${entries.length === 1 ? "" : "s"}) is \`${scriptsDir(dataRoot)}\`; each file's first lines say what it does and how to run it.`;
+  if (picked.length === 0) return `None of the shared scripts fits this step. ${where}`;
+  return [
+    "These shared scripts do part of this step: run one instead of doing its work by hand, and check its output as you would your own.",
+    ...picked.map(line),
+    where,
+  ].join("\n");
+}
+
+/** The oracle's view of the library: every script by name and purpose, so it can point a brief at one. */
+export function libraryBlock(dataRoot: string, entries: readonly ScriptEntry[] = listScripts(dataRoot)): string {
+  const dir = scriptsDir(dataRoot);
+  if (entries.length === 0) return `The shared scripts library (\`${dir}\`) is empty.`;
+  const shown = entries.slice(0, MAX_LISTED).map((entry) => `- \`${entry.name}\` — ${entry.purpose}`);
+  const more = entries.length > MAX_LISTED ? [`…and ${entries.length - MAX_LISTED} more in \`${dir}\`.`] : [];
+  return [`Shared scripts (\`${dir}\`):`, ...shown, ...more].join("\n");
+}
+
+/**
+ * The Scripts layer for one agent's step: the library read, the picker's
+ * choice (shared words when it fails), and the block. "" with no scripts.
+ */
+export async function scriptsFor(dataRoot: string | undefined, step: string, picker: ScriptPicker = keywordPicker, signal?: AbortSignal): Promise<string> {
+  if (!dataRoot) return "";
+  const entries = listScripts(dataRoot);
+  if (entries.length === 0) return "";
+  let picked: ScriptEntry[];
+  try {
+    picked = await picker.pick(step, entries, signal ? { signal } : {});
+  } catch {
+    picked = keywordScripts(step, entries);
+  }
+  return scriptsBlock(dataRoot, entries, picked.slice(0, MAX_SUGGESTED));
+}

package/src/workflow/workflow.ts

@@ -1,5 +1,5 @@
-import { existsSync, mkdirSync, realpathSync } from "node:fs";
-import { isAbsolute, join, relative } from "node:path";
+import { existsSync, mkdirSync, readFileSync, realpathSync } from "node:fs";
+import { isAbsolute, join, relative, resolve, sep } from "node:path";
 import { profileFor, type BotLobbyConfig, type ProfileResolver } from "../schemas/configuration.ts";
 import type { AgentRun, Pushback, ResearchResult, ReviewResult, WorkerResult } from "../schemas/findings.ts";
 import {
@@ -24,6 +24,7 @@ import { ExcalidrawBook } from "../excalidraw/sessions.ts";
 import { knowledgeDir, type KnowledgeAgent } from "../knowledge/paths.ts";
 import { lobbyTopics } from "../lobby/topics.ts";
 import { writeScratchpad } from "../state/persistence.ts";
+import { saveScript } from "../scripts/library.ts";
 import { spawnPiProcess, type ProcessRunner, type RelayAsk } from "../execution/pi-runner.ts";
 import { previewDir } from "../ask/relay.ts";
 import type { AskQuestion, AskResult } from "../ask/types.ts";
@@ -90,6 +91,7 @@ export const ORCHESTRATE_ACTIONS = [
   "qa",
   "knowledge",
   "compact",
+  "script",
   "whiteboard",
   "complete",
   "block",
@@ -126,10 +128,14 @@ export interface OrchestrateParams {
   minutes?: number;
   /** knowledge: which persistent file the text belongs to. */
   kind?: KnowledgeKind;
-  /** compact: the knowledge file being rewritten. */
+  /** compact: the knowledge file being rewritten; script: a drafted script to take instead of `text`. */
   file?: string;
-  /** whiteboard: name for the new Excalidraw session. */
+  /** whiteboard: name for the new Excalidraw session; script: the script's file name (`seed_public_holidays.py`). */
   name?: string;
+  /** script: one line on what the script does. */
+  purpose?: string;
+  /** script: how to run it, with its arguments. */
+  usage?: string;
   approvalId?: string;
   decision?: "approved" | "rejected";
   note?: string;
@@ -635,6 +641,9 @@ function workerReport(outcome: WorkerOutcome, approvals: Approval[], pushback?:
     result.knowledgeProposals.length > 0
       ? `Knowledge proposals (accept with action=knowledge, or ignore): ${result.knowledgeProposals.map((proposal) => `${proposal.kind}: ${truncate(proposal.content, 160)}`).join("; ")}`
       : "",
+    result.scriptProposals?.length
+      ? `Script proposals (save with action=script name purpose usage file=<draft>, when the job will come back; or ignore): ${result.scriptProposals.map((proposal) => `${proposal.name}${proposal.purpose ? ` — ${truncate(proposal.purpose, 120)}` : ""}${proposal.draft ? ` (draft: ${proposal.draft})` : ""}`).join("; ")}`
+      : "",
     issues.length > 0 ? `Issues: ${issues.join("; ")}` : "",
     pushback && objection
       ? `Pushback recorded (${pushback.id}): ${truncate(objection.reason, 240)}. Resolve with action=resolve_approval before re-delegating ${result.domain}.`
@@ -1481,6 +1490,34 @@ function handleKnowledge(task: Task | undefined, params: OrchestrateParams, deps
   return `Recorded ${kind} for ${agent} in ${outcome.path}.`;
 }
 
+/**
+ * The oracle adds a script to the shared library, or replaces one, so every
+ * agent can run it the next time the job comes up. Its body is `text`, or a
+ * file a worker drafted (`file`, inside the project or the task's directory).
+ */
+function handleScript(task: Task | undefined, params: OrchestrateParams, deps: WorkflowDeps): string {
+  const name = params.name?.trim();
+  if (!name) throw new Error("script requires name: what it does in snake_case, with its extension (seed_public_holidays.py)");
+  const purpose = params.purpose?.trim();
+  if (!purpose) throw new Error("script requires purpose: one line on what it does");
+  const body = params.file?.trim() ? readDraft(params.file.trim(), task, deps) : params.text;
+  if (!body?.trim()) throw new Error("script requires text (the script) or file (a drafted script)");
+  const saved = saveScript({ dataRoot: dataRoot(deps.root, deps.configDir), name, purpose, body, ...(params.usage?.trim() ? { usage: params.usage.trim() } : {}) });
+  const { entry } = saved;
+  if (task) recordDecision(task, `${saved.archived ? "Replaced" : "Saved"} the shared script ${entry.name}: ${truncate(entry.purpose, 160)}`);
+  lobbyTopics.bump("knowledge");
+  return [`${saved.archived ? "Replaced" : "Saved"} ${entry.name} in the shared scripts library (${entry.path}); run: ${entry.usage}.`, saved.archived ? `The previous version is kept at ${saved.archived}.` : "", "Agents whose steps it fits are told to run it."].filter(Boolean).join(" ");
+}
+
+/** A drafted script, read only from inside the project, its worktree or the task's directory. */
+function readDraft(file: string, task: Task | undefined, deps: WorkflowDeps): string {
+  const roots = [deps.cwd, deps.root, ...(task ? [taskDirFor(deps.root, deps.configDir, task.id)] : [])].map((dir) => resolve(dir));
+  const path = isAbsolute(file) ? resolve(file) : resolve(deps.cwd, file);
+  if (!roots.some((root) => path === root || path.startsWith(`${root}${sep}`))) throw new Error(`the drafted script must be inside the project: ${file}`);
+  if (!existsSync(path)) throw new Error(`no drafted script at ${file}`);
+  return readFileSync(path, "utf8");
+}
+
 /**
  * §24: the Master rewrites the file (asking the user about ambiguity), the
  * engine archives the previous version and writes the compacted text.
@@ -1990,6 +2027,7 @@ const HANDLERS: Record<OrchestrateAction, (task: Task, params: OrchestrateParams
   resolve_approval: handleResolveApproval,
   qa: handleQa,
   knowledge: handleKnowledge,
+  script: handleScript,
   compact: handleCompact,
   whiteboard: handleWhiteboard,
   complete: handleComplete,
@@ -2008,9 +2046,9 @@ const HANDLERS: Record<OrchestrateAction, (task: Task, params: OrchestrateParams
  */
 export async function runWorkflowAction(params: OrchestrateParams, deps: WorkflowDeps): Promise<WorkflowResult> {
   // Project knowledge can be learned without starting or changing a task.
-  if ((params.action === "knowledge" || params.action === "compact") && !params.taskId) {
+  if ((params.action === "knowledge" || params.action === "compact" || params.action === "script") && !params.taskId) {
     try {
-      const message = params.action === "knowledge" ? handleKnowledge(undefined, params, deps) : handleCompact(undefined, params, deps);
+      const message = params.action === "knowledge" ? handleKnowledge(undefined, params, deps) : params.action === "script" ? handleScript(undefined, params, deps) : handleCompact(undefined, params, deps);
       return { ok: true, taskId: "", state: "created", message };
     } catch (error) {
       return { ok: false, taskId: "", state: "created", message: `Rejected: ${(error as Error).message}` };
@@ -2050,6 +2088,11 @@ export async function runWorkflowAction(params: OrchestrateParams, deps: Workflo
     ...(deps.askQuestions ? { askQuestions: (...args: Parameters<NonNullable<WorkflowDeps["askQuestions"]>>) => wait(() => deps.askQuestions!(...args)) } : {}),
     // Every agent, diff and desk of a task with its own worktree works there.
     cwd: taskCwd(task, deps.cwd),
+    // An agent can work for an hour: what the action has done so far (implementing, reviewing) is saved before it starts, so every window reads it.
+    runProcess: (args, options) => {
+      saveTask(deps.root, deps.configDir, task);
+      return (deps.runProcess ?? spawnPiProcess)(args, options);
+    },
     onUpdate: (run) => {
       if (run.status !== "running") {
         finished.set(run.runId, run);