@dirwin517/bot-farm 0.1.0

package/src/tasks.mjs

@@ -0,0 +1,191 @@
+// Tasks are the primitive. A pipeline is a recipe that emits them.
+//
+// That is the whole architectural bet here. If stages were their own thing,
+// the QA bot finding an unrelated defect would have nowhere to put it, and a
+// "pipeline" would only ever be a straight line. Because every stage is just
+// a task with dependencies, the board can hold pipeline work and ad-hoc work
+// side by side, work can fan out in parallel, and a late defect is simply a
+// new task that blocks an existing one.
+
+export const COLUMNS = ["backlog", "blocked", "waiting", "queued", "active", "review", "done", "cancelled"]
+
+let seq = 0
+const newId = () => "tsk_" + Date.now().toString(36) + (++seq).toString(36)
+
+export class Tasks {
+  constructor(db, { onChange = () => {} } = {}) {
+    this.db = db
+    this.onChange = onChange
+  }
+
+  /**
+   * Ownership has three shapes, and they are not the same thing:
+   *
+   *   session  a named session does this
+   *   role     whichever bot plays this part picks it up — so a dev task does
+   *            not die because one particular dev session was closed
+   *   human    you do this, or you answer this; it is never dispatched
+   */
+  create({
+    title,
+    brief = "",
+    projectId = null,
+    runId = null,
+    stage = null,
+    persona = null,
+    assignee = null,
+    role = null,
+    owner = null,
+    kind = "work",
+    options = [],
+    askedBy = null,
+    dependsOn = [],
+    createdBy = "operator",
+    acceptance = [],
+    order = Date.now(),
+  }) {
+    if (!title?.trim()) throw new TaskError("bad_request", "A task needs a title.")
+    if (owner === "human" || kind === "question") owner = "human"
+    else if (assignee) owner = "session"
+    else if (role) owner = "role"
+    else owner = owner ?? "unassigned"
+    const task = {
+      id: newId(),
+      title: title.trim(),
+      brief,
+      projectId,
+      runId,
+      stage,
+      persona,
+      assignee, // session id, when one session in particular owns it
+      role, // or a part, played by whoever is playing it
+      owner, // session | role | human | unassigned
+      kind, // work | question
+      options, // for a question: the answers offered
+      askedBy, // session that asked
+      answer: null,
+      dependsOn,
+      acceptance,
+      createdBy,
+      status: dependsOn.length ? "blocked" : owner === "human" ? "waiting" : assignee || role ? "queued" : "backlog",
+      queuedAt: dependsOn.length ? null : Date.now(),
+      handoff: null,
+      notes: [],
+      files: [],
+      created: Date.now(),
+      updated: Date.now(),
+      order,
+    }
+    this.db.put("tasks", task)
+    this.onChange()
+    return task
+  }
+
+  get(id) {
+    return this.db.get("tasks", id)
+  }
+
+  all({ projectId, runId, assignee, status, owner, role } = {}) {
+    return this.db
+      .all("tasks")
+      .filter((t) => (!projectId || t.projectId === projectId) && (!runId || t.runId === runId))
+      .filter((t) => (!assignee || t.assignee === assignee) && (!status || t.status === status))
+      .filter((t) => (!owner || t.owner === owner) && (!role || t.role === role))
+      .sort((a, b) => a.order - b.order)
+  }
+
+  /** Everything waiting on the operator: questions first, they block someone. */
+  forHuman() {
+    return this.all({ owner: "human" })
+      .filter((t) => !["done", "cancelled"].includes(t.status))
+      .sort((a, b) => (a.kind === "question" ? -1 : 1) - (b.kind === "question" ? -1 : 1) || a.created - b.created)
+  }
+
+  save(task) {
+    task.updated = Date.now()
+    this.db.put("tasks", task)
+    this.onChange()
+    return task
+  }
+
+  update(id, patch) {
+    const task = this.get(id)
+    if (!task) throw new TaskError("unknown_task", `No task ${id}.`)
+    if (patch.status && !COLUMNS.includes(patch.status)) {
+      throw new TaskError("bad_status", `status must be one of: ${COLUMNS.join(", ")}`)
+    }
+    Object.assign(task, patch)
+    return this.save(task)
+  }
+
+  note(id, text, by = "operator") {
+    const task = this.get(id)
+    if (!task) return null
+    task.notes.push({ at: Date.now(), by, text: String(text).slice(0, 2000) })
+    if (task.notes.length > 50) task.notes.shift()
+    return this.save(task)
+  }
+
+  /** A task is ready when every dependency has landed in `done`. */
+  ready(task) {
+    return task.dependsOn.every((id) => this.get(id)?.status === "done")
+  }
+
+  /**
+   * Move anything whose dependencies just cleared into `queued`. Queued is
+   * deliberately not `active`: the assignee is told about it when it next goes
+   * idle, so a running agent is never interrupted mid-thought.
+   */
+  unblock() {
+    const freed = []
+    for (const task of this.all()) {
+      if (task.status !== "blocked") continue
+      if (!this.ready(task)) continue
+      task.status = task.owner === "human" ? "waiting" : task.assignee || task.role ? "queued" : "backlog"
+      task.queuedAt = Date.now()
+      this.save(task)
+      freed.push(task)
+    }
+    return freed
+  }
+
+  /**
+   * The next thing this session should pick up: work addressed to it by name,
+   * or work addressed to the part it plays that nobody has taken yet.
+   */
+  nextFor(sessionId, { role = null, projectId = null, alive = null, now = Date.now() } = {}) {
+    const mine = this.all({ assignee: sessionId })
+    // Work addressed to a part stays in its workstream: another workstream's
+    // dev bot must not pick up this one's cards. A card sent back for rework
+    // prefers the bot that did it first: others take it only once that bot is
+    // gone or has had its chance (preferUntil).
+    const mayTake = (t) => !t.prefer || t.prefer === sessionId || now > (t.preferUntil ?? 0) || (alive && !alive(t.prefer))
+    const byRole = role
+      ? this.all({ role }).filter((t) => !t.assignee && t.owner === "role" && (!projectId || !t.projectId || t.projectId === projectId) && mayTake(t))
+      : []
+    return [...mine, ...byRole]
+      .filter((t) => t.status === "queued" && this.ready(t))
+      .sort((a, b) => a.order - b.order)[0]
+  }
+
+  /** Kanban projection: columns with their cards, newest activity first. */
+  board({ projectId, runId } = {}) {
+    const tasks = this.all({ projectId, runId })
+    return COLUMNS.map((status) => ({
+      status,
+      tasks: tasks.filter((t) => t.status === status),
+    }))
+  }
+
+  remove(id) {
+    this.db.delete("tasks", id)
+    this.onChange()
+  }
+}
+
+export class TaskError extends Error {
+  constructor(code, message) {
+    super(message)
+    this.code = code
+  }
+}

package/src/template.mjs

@@ -0,0 +1,86 @@
+// Mustache subset, dependency-free. Supports {{var}} (HTML-safe is pointless
+// here, so nothing is escaped), {{#section}}...{{/section}} for truthy values,
+// lists and objects, {{^inverted}}, dotted paths, and {{.}} inside a list of
+// scalars. Comments are {{! like this }}.
+//
+// This is how a pipeline step decides which earlier handoffs it receives: the
+// template names them, so dropping a stage's output from a later prompt is an
+// edit to one line of YAML rather than a code change.
+
+const TAG = /\{\{([#^/!&]?)\s*([^}]*?)\s*\}\}/g
+
+export function render(template, context = {}) {
+  const { out } = section(String(template ?? ""), 0, [context], true)
+  return out
+}
+
+function section(tpl, from, stack, emit) {
+  let out = ""
+  let i = from
+  TAG.lastIndex = from
+  let m
+  while ((m = TAG.exec(tpl))) {
+    if (emit) out += tpl.slice(i, m.index)
+    i = TAG.lastIndex
+    const [, sigil, name] = m
+
+    if (sigil === "!") continue
+    if (sigil === "/") return { out, next: i, closed: name }
+
+    if (sigil === "#" || sigil === "^") {
+      const value = lookup(stack, name)
+      const truthy = Array.isArray(value) ? value.length > 0 : !!value
+      const wanted = sigil === "#" ? truthy : !truthy
+
+      if (sigil === "#" && Array.isArray(value) && value.length) {
+        // Re-render the body once per item, then skip past it.
+        let last = null
+        for (const item of value) {
+          last = section(tpl, i, [...stack, item], true)
+          out += last.out
+        }
+        i = last.next
+      } else {
+        const body = section(tpl, i, sigil === "#" && isObject(value) ? [...stack, value] : stack, emit && wanted)
+        out += body.out
+        i = body.next
+      }
+      TAG.lastIndex = i
+      continue
+    }
+
+    if (emit) {
+      const v = lookup(stack, name)
+      out += v === null || v === undefined ? "" : Array.isArray(v) ? v.join(", ") : String(v)
+    }
+  }
+  if (emit) out += tpl.slice(i)
+  return { out, next: tpl.length }
+}
+
+function lookup(stack, name) {
+  if (name === ".") return stack[stack.length - 1]
+  const path = name.split(".")
+  for (let i = stack.length - 1; i >= 0; i--) {
+    let v = stack[i]
+    let ok = true
+    for (const key of path) {
+      if (v && typeof v === "object" && key in v) v = v[key]
+      else { ok = false; break }
+    }
+    if (ok) return v
+  }
+  return undefined
+}
+
+const isObject = (v) => v && typeof v === "object" && !Array.isArray(v)
+
+/** Names a template refers to — used to warn about typos in pipeline YAML. */
+export function variables(template) {
+  const names = new Set()
+  for (const m of String(template ?? "").matchAll(TAG)) {
+    if (m[1] === "!" || m[1] === "/") continue
+    names.add(m[2].split(".")[0])
+  }
+  return [...names]
+}

package/src/tools.mjs

@@ -0,0 +1,241 @@
+// A workspace's own tools: every .js / .mjs file in <workspace>/botfarm/mcps/
+// is loaded into BotFarm's MCP server and offered to the bots working in that
+// workspace (as botfarm_<name>), next to the built-in ones.
+//
+// A tool file exports:
+//   name         letters, digits, _ and - (defaults to the file name)
+//   description  what the model reads to decide when to call it
+//   inputSchema  JSON Schema for the arguments (default: no arguments)
+//   execute      async (args, ctx) => result   (also accepted: handler, default)
+//   timeoutMs    optional, default 120000
+//
+// ctx: { repoRoot, workspace, bot, signal, log(...), exec(cmd, args, opts) }
+//
+// Files are re-imported when they change (fs.watch, plus a rescan as a safety
+// net), so saving a file is enough; bots get the new list on their next turn.
+
+import { readdir, readFile, writeFile, stat, mkdir, unlink } from "node:fs/promises"
+import { watch } from "node:fs"
+import { join, basename, extname } from "node:path"
+import { pathToFileURL } from "node:url"
+import { execFile } from "node:child_process"
+
+const NAME_RE = /^[A-Za-z][A-Za-z0-9_-]{0,62}$/
+const FILE_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,80}\.m?js$/
+
+export class WorkspaceTools {
+  constructor({ supervisor, dirFor, reserved = [] }) {
+    this.sup = supervisor
+    this.dirFor = dirFor // (ws) => absolute tools folder
+    this.reserved = new Set(reserved) // built-in tool names a file may not take
+    this.sets = new Map() // workspace id -> { dir, tools: Map<name, tool>, files: Map<file, {mtime, name?, error?}>, watcher, version }
+    this.listeners = new Set()
+  }
+
+  onChange(fn) { this.listeners.add(fn) }
+
+  /** Load (or re-check) one workspace's tools. Returns true when the list changed. */
+  async load(ws) {
+    const dir = this.dirFor(ws)
+    let set = this.sets.get(ws.id)
+    if (!set || set.dir !== dir) {
+      set?.watcher?.close?.()
+      set = { dir, tools: new Map(), files: new Map(), watcher: null, version: 0, ws: ws.id }
+      this.sets.set(ws.id, set)
+    }
+    let names = []
+    try { names = (await readdir(dir)).filter((n) => FILE_RE.test(n)) } catch { names = [] }
+    let changed = false
+    const seen = new Set()
+    for (const file of names) {
+      seen.add(file)
+      const path = join(dir, file)
+      const mtime = await stat(path).then((s) => s.mtimeMs, () => 0)
+      const prev = set.files.get(file)
+      if (prev && prev.mtime === mtime) continue
+      changed = true
+      if (prev?.name) set.tools.delete(prev.name)
+      try {
+        const mod = await import(`${pathToFileURL(path).href}?v=${mtime}`)
+        const name = String(mod.name ?? mod.title ?? basename(file, extname(file)))
+        const exec = mod.execute ?? mod.handler ?? (typeof mod.default === "function" ? mod.default : null)
+        if (!NAME_RE.test(name)) throw new Error(`name "${name}" must be letters, digits, _ and - (starting with a letter)`)
+        if (this.reserved.has(name)) throw new Error(`"${name}" is a built-in BotFarm tool; pick another name`)
+        if (typeof exec !== "function") throw new Error("no execute(args, ctx) function exported")
+        const clash = [...set.tools.values()].find((t) => t.name === name && t.file !== file)
+        if (clash) throw new Error(`${clash.file} already defines "${name}"`)
+        const tool = {
+          name,
+          description: String(mod.description ?? "").trim() || `(no description in ${file})`,
+          inputSchema: mod.inputSchema && typeof mod.inputSchema === "object" ? mod.inputSchema : { type: "object", properties: {} },
+          timeoutMs: Number(mod.timeoutMs) > 0 ? Number(mod.timeoutMs) : 120_000,
+          execute: exec,
+          file,
+        }
+        set.tools.set(name, tool)
+        set.files.set(file, { mtime, name })
+      } catch (e) {
+        set.files.set(file, { mtime, error: e.message })
+      }
+    }
+    for (const [file, info] of [...set.files]) {
+      if (seen.has(file)) continue
+      set.files.delete(file)
+      if (info.name) set.tools.delete(info.name)
+      changed = true
+    }
+    if (!set.watcher) {
+      try {
+        let t = null
+        set.watcher = watch(dir, () => { clearTimeout(t); t = setTimeout(() => this.refresh(ws), 250) })
+        set.watcher.on("error", () => { set.watcher = null })
+      } catch { set.watcher = null } // no folder yet: the rescan picks it up
+    }
+    if (changed) set.version++
+    return changed
+  }
+
+  async refresh(ws) {
+    const changed = await this.load(ws).catch(() => false)
+    if (changed) for (const fn of this.listeners) fn(ws.id)
+    return changed
+  }
+
+  /** Rescan every workspace (the safety net for missed fs events). */
+  async rescan(workspaces) {
+    for (const ws of workspaces) await this.refresh(ws)
+  }
+
+  list(wsId) {
+    const set = this.sets.get(wsId)
+    if (!set) return { dir: null, tools: [], errors: [], version: 0 }
+    return {
+      dir: set.dir,
+      version: set.version,
+      tools: [...set.tools.values()].map(({ execute, ...t }) => t).sort((a, b) => a.name.localeCompare(b.name)),
+      errors: [...set.files].filter(([, f]) => f.error).map(([file, f]) => ({ file, error: f.error })),
+    }
+  }
+
+  get(wsId, name) {
+    return this.sets.get(wsId)?.tools.get(name) ?? null
+  }
+
+  /** MCP tool definitions for a workspace. */
+  mcpList(wsId) {
+    const set = this.sets.get(wsId)
+    if (!set) return []
+    return [...set.tools.values()].map((t) => ({ name: t.name, description: t.description, inputSchema: t.inputSchema }))
+  }
+
+  /**
+   * Run a tool. Never throws: the outcome is { ok, result?, error?, ms, logs }.
+   * `repoRoot` is the checkout the tool works in (the bot's worktree).
+   */
+  async run(wsId, name, args = {}, { repoRoot, workspace = null, bot = null, signal = null } = {}) {
+    const tool = this.get(wsId, name)
+    const started = Date.now()
+    const logs = []
+    if (!tool) return { ok: false, error: `no tool "${name}" in this workspace`, ms: 0, logs }
+    const ctl = new AbortController()
+    const onAbort = () => ctl.abort()
+    signal?.addEventListener?.("abort", onAbort)
+    let timer
+    const ctx = {
+      repoRoot,
+      workspace,
+      bot,
+      signal: ctl.signal,
+      log: (...a) => { if (logs.length < 200) logs.push(a.map((x) => (typeof x === "string" ? x : JSON.stringify(x))).join(" ").slice(0, 2000)) },
+      exec: (cmd, argv = [], opts = {}) => new Promise((resolve) => {
+        execFile(cmd, argv, { cwd: opts.cwd ?? repoRoot, timeout: opts.timeoutMs ?? tool.timeoutMs, maxBuffer: 16 * 1024 * 1024, signal: ctl.signal, env: { ...process.env, ...(opts.env ?? {}) } }, (err, stdout, stderr) => {
+          resolve({ code: err ? (typeof err.code === "number" ? err.code : 1) : 0, stdout: String(stdout ?? ""), stderr: String(stderr ?? "") || (err && !stderr ? err.message : "") })
+        })
+      }),
+    }
+    try {
+      const result = await Promise.race([
+        Promise.resolve().then(() => tool.execute(args ?? {}, ctx)),
+        new Promise((_, reject) => { timer = setTimeout(() => { ctl.abort(); reject(new Error(`timed out after ${Math.round(tool.timeoutMs / 1000)}s`)) }, tool.timeoutMs) }),
+      ])
+      return { ok: true, result: result === undefined ? null : result, ms: Date.now() - started, logs }
+    } catch (e) {
+      return { ok: false, error: e?.message ?? String(e), stack: String(e?.stack ?? "").split("\n").slice(1, 6).join("\n"), ms: Date.now() - started, logs }
+    } finally {
+      clearTimeout(timer)
+      signal?.removeEventListener?.("abort", onAbort)
+    }
+  }
+
+  // --- files, for the test bench ---------------------------------------------
+
+  checkFile(file) {
+    if (!FILE_RE.test(file ?? "")) throw new Error("a tool file is a name ending in .js or .mjs (letters, digits, _ - .)")
+  }
+
+  async source(ws, file) {
+    this.checkFile(file)
+    const path = join(this.dirFor(ws), file)
+    return { file, path, text: await readFile(path, "utf8").catch(() => "") }
+  }
+
+  async save(ws, file, text) {
+    this.checkFile(file)
+    const dir = this.dirFor(ws)
+    await mkdir(dir, { recursive: true })
+    await writeFile(join(dir, file), String(text ?? ""))
+    await this.refresh(ws)
+    return this.list(ws.id)
+  }
+
+  async create(ws, name) {
+    if (!NAME_RE.test(name ?? "")) throw new Error("a tool name is letters, digits, _ and - (starting with a letter)")
+    const file = `${name}.js`
+    const path = join(this.dirFor(ws), file)
+    if (await stat(path).then(() => true, () => false)) throw new Error(`${file} already exists`)
+    return { ...(await this.save(ws, file, template(name))), file }
+  }
+
+  async remove(ws, file) {
+    this.checkFile(file)
+    await unlink(join(this.dirFor(ws), file))
+    await this.refresh(ws)
+    return this.list(ws.id)
+  }
+
+  close() {
+    for (const s of this.sets.values()) s.watcher?.close?.()
+  }
+}
+
+/** Plain JSON for MCP: strings as text, everything else pretty-printed (and structured when it is an object). */
+export function toolResult(out) {
+  if (!out.ok) return { content: [{ type: "text", text: `Tool error: ${out.error}${out.logs.length ? `\n\nlog:\n${out.logs.join("\n")}` : ""}` }], isError: true }
+  const r = out.result
+  if (typeof r === "string") return { content: [{ type: "text", text: r }] }
+  const text = JSON.stringify(r, null, 2) ?? "null"
+  return r && typeof r === "object" && !Array.isArray(r)
+    ? { content: [{ type: "text", text }], structuredContent: r }
+    : { content: [{ type: "text", text }] }
+}
+
+const template = (name) => `// BotFarm tool: offered to every bot in this workspace as botfarm_${name}.
+// Saving this file reloads it. Try it from Tools in the BotFarm sidebar.
+
+export const name = "${name}"
+export const description = "Say what this does and when a bot should use it — this is what the model reads."
+export const inputSchema = {
+  type: "object",
+  properties: {
+    path: { type: "string", description: "A path relative to the worktree." },
+  },
+  required: ["path"],
+}
+
+// ctx.repoRoot: the calling bot's worktree. ctx.exec(cmd, args): run a program there.
+// ctx.log(...): lines shown in the test bench and returned with errors.
+export async function execute({ path }, ctx) {
+  const { stdout } = await ctx.exec("ls", ["-la", path])
+  return { path, listing: stdout.split("\\n").filter(Boolean) }
+}
+`