@jx0/agency 0.2.1 → 0.4.0

package/src/api/routes/agents.ts

@@ -1,50 +1,138 @@
+import * as fs from "fs";
+import * as path from "path";
 import { Hono } from "hono";
 import { db } from "../db/client.js";
 import { resolveAgent } from "../lib/resolve-agent.js";
 import {
   writeAgentToFleet,
   removeAgentFromFleet,
+  readFleet,
 } from "../lib/fleet-sync.js";
-import { startLocal, stopLocal } from "../lib/processes.js";
-import { startTunnel, stopTunnel } from "../lib/tunnels.js";
-import { readFleet } from "../lib/fleet-sync.js";
+import { listRoles, getRoleConfig } from "../lib/fs-store.js";
+import { getMetrics } from "../lib/metrics.js";
 
 const NAME_RE = /^[a-z][a-z0-9-]{1,30}$/;
-const VALID_LOCATIONS = new Set(["docker", "ec2", "local"]);
+const VALID_RUNTIMES = new Set(["system", "docker"]);
 
-// Map of allowed file names to role_config config_type values
+// Map of allowed file names to config_type values
 const FILE_TO_CONFIG_TYPE: Record<string, string> = {
   "SOUL.md": "soul",
   "USER.md": "identity",
   "AGENTS.md": "agents",
-  "MEMORY.md": "soul", // memory is part of soul for now
+  "MEMORY.md": "soul",
   "TOOLS.md": "tools",
 };
 
+/**
+ * Write a docker-compose.agents.yml with a service for the given agent.
+ * Merges into any existing services so multiple agents can coexist.
+ */
+export function generateAgentCompose(agentName: string): void {
+  const composePath = path.join(process.cwd(), "docker-compose.agents.yml");
+  const serviceName = `agent-${agentName}`;
+
+  // Parse existing compose file to preserve other agent services
+  let existing: { services: Record<string, Record<string, unknown>> } = { services: {} };
+  if (fs.existsSync(composePath)) {
+    try {
+      const raw = fs.readFileSync(composePath, "utf-8");
+      // Parse our simple YAML format: extract service blocks
+      let currentService = "";
+      let currentVolumes = false;
+      for (const line of raw.split("\n")) {
+        const svcMatch = line.match(/^  (agent-[\w-]+):$/);
+        if (svcMatch) {
+          currentService = svcMatch[1];
+          existing.services[currentService] = {};
+          currentVolumes = false;
+          continue;
+        }
+        if (currentService) {
+          const kvMatch = line.match(/^    (\w+): (.+)$/);
+          if (kvMatch) {
+            existing.services[currentService][kvMatch[1]] = kvMatch[2];
+            currentVolumes = false;
+            continue;
+          }
+          if (line.match(/^    volumes:$/)) {
+            currentVolumes = true;
+            existing.services[currentService].volumes = [];
+            continue;
+          }
+          if (currentVolumes) {
+            const volMatch = line.match(/^      - (.+)$/);
+            if (volMatch) {
+              (existing.services[currentService].volumes as string[]).push(volMatch[1]);
+            }
+          }
+        }
+      }
+    } catch {}
+  }
+
+  const service = {
+    image: "agency-agent",
+    container_name: serviceName,
+    restart: "unless-stopped",
+    extra_hosts: ["host.docker.internal:host-gateway"],
+    volumes: [`/root/.openclaw-${agentName}:/root/.openclaw-${agentName}`],
+  };
+
+  existing.services[serviceName] = service;
+
+  // Write as YAML manually (simple enough structure)
+  let yaml = "services:\n";
+  for (const [name, svc] of Object.entries(existing.services)) {
+    const s = svc as Record<string, unknown>;
+    yaml += `  ${name}:\n`;
+    yaml += `    image: ${s.image}\n`;
+    yaml += `    container_name: ${s.container_name}\n`;
+    yaml += `    restart: ${s.restart}\n`;
+    if (Array.isArray(s.extra_hosts) && s.extra_hosts.length > 0) {
+      yaml += `    extra_hosts:\n`;
+      for (const h of s.extra_hosts) {
+        yaml += `      - ${h}\n`;
+      }
+    }
+    if (Array.isArray(s.volumes) && s.volumes.length > 0) {
+      yaml += `    volumes:\n`;
+      for (const v of s.volumes) {
+        yaml += `      - ${v}\n`;
+      }
+    }
+  }
+
+  fs.writeFileSync(composePath, yaml);
+}
+
 export const agents = new Hono();
 
-// List available roles (from DB role_configs)
+// List available roles (from filesystem)
 agents.get("/roles", async (c) => {
-  const rows = await db
-    .selectFrom("role_configs")
-    .select("role")
-    .distinct()
-    .execute();
-  return c.json(rows.map((r) => r.role));
+  return c.json(listRoles());
 });
 
 agents.get("/", async (c) => {
   const rows = await db.selectFrom("agents").selectAll().execute();
-  return c.json(rows);
+
+  const enriched = rows.map((r) => ({
+    ...r,
+    metrics: getMetrics(r.name),
+  }));
+  return c.json(enriched);
 });
 
 agents.get("/:name", async (c) => {
   const agent = await resolveAgent(c.req.param("name"));
   if (!agent) return c.json({ error: "not found" }, 404);
-  return c.json(agent);
+
+  return c.json({
+    ...agent,
+    metrics: getMetrics(agent.name),
+  });
 });
 
-// Get agent config file from DB
+// Get agent config file from filesystem
 agents.get("/:name/files/:filename", async (c) => {
   const filename = c.req.param("filename");
   const configType = FILE_TO_CONFIG_TYPE[filename];
@@ -54,17 +142,11 @@ agents.get("/:name/files/:filename", async (c) => {
   const agent = await resolveAgent(c.req.param("name"));
   if (!agent) return c.json({ error: "not found" }, 404);
 
-  const config = await db
-    .selectFrom("role_configs")
-    .where("role", "=", agent.role)
-    .where("config_type", "=", configType)
-    .selectAll()
-    .executeTakeFirst();
-
-  if (!config) {
+  const content = getRoleConfig(agent.role, configType);
+  if (content === null) {
     return c.json({ error: "file not found" }, 404);
   }
-  return c.json({ filename, content: config.content });
+  return c.json({ filename, content });
 });
 
 // Get agent's role config by config_type directly
@@ -72,15 +154,10 @@ agents.get("/:name/config/:configType", async (c) => {
   const agent = await resolveAgent(c.req.param("name"));
   if (!agent) return c.json({ error: "not found" }, 404);
 
-  const config = await db
-    .selectFrom("role_configs")
-    .where("role", "=", agent.role)
-    .where("config_type", "=", c.req.param("configType"))
-    .selectAll()
-    .executeTakeFirst();
-
-  if (!config) return c.json({ error: "not found" }, 404);
-  return c.json(config);
+  const configType = c.req.param("configType");
+  const content = getRoleConfig(agent.role, configType);
+  if (content === null) return c.json({ error: "not found" }, 404);
+  return c.json({ role: agent.role, config_type: configType, content });
 });
 
 // Create agent
@@ -88,7 +165,8 @@ agents.post("/", async (c) => {
   const body = await c.req.json<{
     name: string;
     role: string;
-    location: string;
+    runtime: string;      // "system" | "docker"
+    machine: string;      // required machine name
     slack_bot_token?: string;
     slack_app_token?: string;
   }>();
@@ -96,8 +174,16 @@ agents.post("/", async (c) => {
   if (!NAME_RE.test(body.name)) {
     return c.json({ error: "invalid name: must match /^[a-z][a-z0-9-]{1,30}$/" }, 400);
   }
-  if (!VALID_LOCATIONS.has(body.location)) {
-    return c.json({ error: "invalid location: must be docker, ec2, or local" }, 400);
+  if (!VALID_RUNTIMES.has(body.runtime)) {
+    return c.json({ error: "invalid runtime: must be system or docker" }, 400);
+  }
+  if (!body.machine) {
+    return c.json({ error: "machine is required" }, 400);
+  }
+  // Validate machine exists
+  const { readMachines } = await import("../routes/machines.js");
+  if (!readMachines().find((m: any) => m.name === body.machine)) {
+    return c.json({ error: `machine "${body.machine}" not found` }, 400);
   }
 
   const existing = await db
@@ -112,7 +198,8 @@ agents.post("/", async (c) => {
   // Write to fleet.json
   await writeAgentToFleet(body.name, {
     role: body.role,
-    location: body.location,
+    runtime: body.runtime,
+    machine: body.machine,
     ...(body.slack_bot_token ? { slackBotToken: body.slack_bot_token } : {}),
     ...(body.slack_app_token ? { slackAppToken: body.slack_app_token } : {}),
   });
@@ -124,7 +211,8 @@ agents.post("/", async (c) => {
       id: crypto.randomUUID(),
       name: body.name,
       role: body.role,
-      location: body.location,
+      runtime: body.runtime,
+      machine: body.machine,
       slack_bot_token: body.slack_bot_token ?? null,
       slack_app_token: body.slack_app_token ?? null,
       status: "idle",
@@ -146,7 +234,8 @@ agents.patch("/:name", async (c) => {
     status?: string;
     current_task?: string | null;
     role?: string;
-    location?: string;
+    runtime?: string;
+    machine?: string | null;
     slack_bot_token?: string | null;
     slack_app_token?: string | null;
   }>();
@@ -155,11 +244,14 @@ agents.patch("/:name", async (c) => {
   if (body.status !== undefined) q = q.set("status", body.status);
   if (body.current_task !== undefined) q = q.set("current_task", body.current_task);
   if (body.role !== undefined) q = q.set("role", body.role);
-  if (body.location !== undefined) {
-    if (!VALID_LOCATIONS.has(body.location)) {
-      return c.json({ error: "invalid location" }, 400);
+  if (body.runtime !== undefined) {
+    if (!VALID_RUNTIMES.has(body.runtime)) {
+      return c.json({ error: "invalid runtime" }, 400);
     }
-    q = q.set("location", body.location);
+    q = q.set("runtime", body.runtime);
+  }
+  if (body.machine !== undefined) {
+    q = q.set("machine", body.machine);
   }
   if (body.slack_bot_token !== undefined) q = q.set("slack_bot_token", body.slack_bot_token);
   if (body.slack_app_token !== undefined) q = q.set("slack_app_token", body.slack_app_token);
@@ -168,12 +260,17 @@ agents.patch("/:name", async (c) => {
   const updated = await q.returningAll().executeTakeFirstOrThrow();
 
   // Sync config fields to fleet.json
-  if (body.role !== undefined || body.location !== undefined ||
+  if (body.role !== undefined || body.runtime !== undefined ||
+      body.machine !== undefined ||
       body.slack_bot_token !== undefined ||
       body.slack_app_token !== undefined) {
+    const fleet = readFleet();
+    const existing = fleet.agents[agent.name] ?? { role: updated.role, runtime: updated.runtime ?? "system" };
     await writeAgentToFleet(agent.name, {
+      ...existing,
       role: updated.role,
-      location: updated.location ?? "local",
+      runtime: updated.runtime ?? "system",
+      machine: body.machine !== undefined ? (body.machine ?? undefined) : (existing.machine ?? undefined),
       ...(updated.slack_bot_token ? { slackBotToken: updated.slack_bot_token } : {}),
       ...(updated.slack_app_token ? { slackAppToken: updated.slack_app_token } : {}),
     });
@@ -187,17 +284,28 @@ agents.delete("/:name", async (c) => {
   const agent = await resolveAgent(c.req.param("name"));
   if (!agent) return c.json({ error: "not found" }, 404);
 
-  // Stop docker container if running
-  if (agent.location === "docker") {
+  // Stop agent if running
+  if (agent.status === "active") {
     try {
-      const proc = Bun.spawn(
-        ["docker", "compose", "-f", "docker-compose.agents.yml", "stop", `agent-${agent.name}`],
-        { cwd: process.cwd(), stdout: "inherit", stderr: "inherit" }
-      );
-      await proc.exited;
+      const { stop } = await import("../lib/deploy.js");
+      await stop({
+        name: agent.name,
+        role: agent.role,
+        runtime: agent.runtime ?? "system",
+        machine: agent.machine ?? "",
+      });
     } catch {}
   }
 
+  // Unlink related records before deleting
+  await db.updateTable("tasks").set({ created_by: null }).where("created_by", "=", agent.id).execute();
+  await db.deleteFrom("task_assignees").where("agent_id", "=", agent.id).execute();
+  await db.updateTable("messages").set({ from_agent: null }).where("from_agent", "=", agent.id).execute();
+  await db.updateTable("activities").set({ agent_id: null }).where("agent_id", "=", agent.id).execute();
+  await db.deleteFrom("notifications").where("target_agent", "=", agent.id).execute();
+  await db.updateTable("notifications").set({ source_agent: null }).where("source_agent", "=", agent.id).execute();
+  await db.updateTable("documents").set({ created_by: null }).where("created_by", "=", agent.id).execute();
+
   await db.deleteFrom("agents").where("id", "=", agent.id).execute();
   await removeAgentFromFleet(agent.name);
 
@@ -209,57 +317,25 @@ agents.post("/:name/deploy", async (c) => {
   const agent = await resolveAgent(c.req.param("name"));
   if (!agent) return c.json({ error: "not found" }, 404);
 
-  if (agent.location === "local") {
-    startLocal(agent.name, agent.role);
-    await db
-      .updateTable("agents")
-      .where("id", "=", agent.id)
-      .set({ status: "active", updated_at: new Date().toISOString() })
-      .execute();
-    return c.json({ status: "deployed", method: "local" });
-  }
-
-  if (agent.location === "ec2") {
-    const fleet = readFleet();
-    const fleetAgent = fleet.agents[agent.name];
-    const host = fleetAgent?.host;
-    if (!host) {
-      return c.json({ error: "No host configured for EC2 agent. Set 'host' in fleet.json or agent config." }, 400);
-    }
-
-    // Open reverse SSH tunnel so agent can reach our API at localhost:3100
-    try {
-      await startTunnel(agent.name, host);
-    } catch (err: any) {
-      return c.json({ error: `Failed to open SSH tunnel: ${err.message}` }, 500);
-    }
-
-    await db
-      .updateTable("agents")
-      .where("id", "=", agent.id)
-      .set({ status: "active", updated_at: new Date().toISOString() })
-      .execute();
-    return c.json({ status: "deployed", method: "ec2", tunnel: true });
-  }
-
-  if (agent.location === "docker") {
-    const proc = Bun.spawn(
-      ["docker", "compose", "-f", "docker-compose.agents.yml", "up", "-d", `agent-${agent.name}`],
-      { cwd: process.cwd(), stdout: "inherit", stderr: "inherit" }
-    );
-    const exitCode = await proc.exited;
-    if (exitCode !== 0) {
-      return c.json({ error: "docker compose up failed" }, 500);
-    }
+  try {
+    const { deploy } = await import("../lib/deploy.js");
+    const result = await deploy({
+      name: agent.name,
+      role: agent.role,
+      runtime: agent.runtime ?? "system",
+      machine: agent.machine ?? "",
+      slack_bot_token: agent.slack_bot_token,
+      slack_app_token: agent.slack_app_token,
+    });
     await db
       .updateTable("agents")
       .where("id", "=", agent.id)
       .set({ status: "active", updated_at: new Date().toISOString() })
       .execute();
-    return c.json({ status: "deployed", method: "docker" });
+    return c.json({ status: "deployed", method: result.method });
+  } catch (err: any) {
+    return c.json({ error: err.message }, 500);
   }
-
-  return c.json({ error: "unsupported location" }, 400);
 });
 
 // Stop agent
@@ -267,45 +343,21 @@ agents.post("/:name/stop", async (c) => {
   const agent = await resolveAgent(c.req.param("name"));
   if (!agent) return c.json({ error: "not found" }, 404);
 
-  if (agent.location === "local") {
-    const stopped = stopLocal(agent.name);
-    if (!stopped) {
-      return c.json({ error: "no local process found" }, 400);
-    }
-    await db
-      .updateTable("agents")
-      .where("id", "=", agent.id)
-      .set({ status: "idle", updated_at: new Date().toISOString() })
-      .execute();
-    return c.json({ status: "stopped" });
-  }
-
-  if (agent.location === "docker") {
-    const proc = Bun.spawn(
-      ["docker", "compose", "-f", "docker-compose.agents.yml", "stop", `agent-${agent.name}`],
-      { cwd: process.cwd(), stdout: "inherit", stderr: "inherit" }
-    );
-    const exitCode = await proc.exited;
-    if (exitCode !== 0) {
-      return c.json({ error: "docker compose stop failed" }, 500);
-    }
+  try {
+    const { stop } = await import("../lib/deploy.js");
+    await stop({
+      name: agent.name,
+      role: agent.role,
+      runtime: agent.runtime ?? "system",
+      machine: agent.machine ?? "",
+    });
     await db
       .updateTable("agents")
       .where("id", "=", agent.id)
       .set({ status: "idle", updated_at: new Date().toISOString() })
       .execute();
     return c.json({ status: "stopped" });
+  } catch (err: any) {
+    return c.json({ error: err.message }, 500);
   }
-
-  if (agent.location === "ec2") {
-    stopTunnel(agent.name);
-    await db
-      .updateTable("agents")
-      .where("id", "=", agent.id)
-      .set({ status: "idle", updated_at: new Date().toISOString() })
-      .execute();
-    return c.json({ status: "stopped", tunnel: "closed" });
-  }
-
-  return c.json({ error: "stop not supported for this agent location" }, 400);
 });

package/src/api/routes/documents.ts

@@ -22,10 +22,11 @@ documents.get("/:id", async (c) => {
 documents.post("/", async (c) => {
   const body = await c.req.json<{
     title: string; content: string; doc_type?: string;
-    task_id?: string; from: string;
+    task_id?: string; from?: string;
   }>();
-  const agent = await resolveAgent(body.from);
-  if (!agent) return c.json({ error: "unknown agent" }, 400);
+  // "human" or missing from = NULL (human user)
+  const agent = body.from && body.from !== "human" ? await resolveAgent(body.from) : null;
+  if (body.from && body.from !== "human" && !agent) return c.json({ error: "unknown agent" }, 400);
 
   const doc = await db.insertInto("documents").values({
     id: crypto.randomUUID(),
@@ -33,9 +34,27 @@ documents.post("/", async (c) => {
     content: body.content,
     doc_type: body.doc_type ?? "general",
     task_id: body.task_id ?? null,
-    created_by: agent.id,
+    created_by: agent?.id ?? null,
   }).returningAll().executeTakeFirstOrThrow();
 
-  await logActivity("document_created", agent.id, `Created doc: ${doc.title}`, doc.task_id);
+  if (agent) {
+    await logActivity("document_created", agent.id, `Created doc: ${doc.title}`, doc.task_id);
+  }
   return c.json(doc, 201);
 });
+
+documents.put("/:id", async (c) => {
+  const id = c.req.param("id");
+  const body = await c.req.json<{ title?: string; content?: string; doc_type?: string }>();
+
+  const existing = await db.selectFrom("documents").where("id", "=", id).selectAll().executeTakeFirst();
+  if (!existing) return c.json({ error: "not found" }, 404);
+
+  const updated = await db.updateTable("documents").set({
+    ...(body.title !== undefined && { title: body.title }),
+    ...(body.content !== undefined && { content: body.content }),
+    ...(body.doc_type !== undefined && { doc_type: body.doc_type }),
+  }).where("id", "=", id).returningAll().executeTakeFirstOrThrow();
+
+  return c.json(updated);
+});

package/src/api/routes/knowledge.ts

@@ -32,25 +32,27 @@ knowledge.get("/", async (c) => {
 
 knowledge.post("/", async (c) => {
   const body = await c.req.json<{
-    key: string; content: string; from: string;
+    key: string; content: string; from?: string;
     task_id?: string; tags?: string[];
   }>();
-  const agent = await resolveAgent(body.from);
-  if (!agent) return c.json({ error: "unknown agent" }, 400);
+  // "human" or missing from = NULL (human user)
+  const agent = body.from && body.from !== "human" ? await resolveAgent(body.from) : null;
+  if (body.from && body.from !== "human" && !agent) return c.json({ error: "unknown agent" }, 400);
 
   const tagsJson = JSON.stringify(body.tags ?? []);
+  const sourceId = agent?.id ?? null;
 
   const row = await db.insertInto("knowledge").values({
     id: crypto.randomUUID(),
     key: body.key,
     content: body.content,
-    source: agent.id,
+    source: sourceId ?? "human",
     task_id: body.task_id ?? null,
     tags: tagsJson,
   })
   .onConflict((oc) => oc.column("key").doUpdateSet({
     content: body.content,
-    source: agent.id,
+    source: sourceId ?? "human",
     tags: tagsJson,
   }))
   .returningAll()

package/src/api/routes/machines.ts

@@ -0,0 +1,107 @@
+import { Hono } from "hono";
+import * as fs from "fs";
+import * as path from "path";
+
+export interface Machine {
+  name: string;
+  host: string;
+  user: string;
+  port: number;
+  auth: "local" | "key" | "password";
+  ssh_key?: string;
+  password?: string;
+}
+
+function resolveMachinesPath(): string {
+  return process.env.MACHINES_PATH ?? path.resolve(process.cwd(), ".agency", "machines.json");
+}
+
+export function readMachines(): Machine[] {
+  const p = resolveMachinesPath();
+  if (!fs.existsSync(p)) return [];
+  return JSON.parse(fs.readFileSync(p, "utf-8"));
+}
+
+function writeMachines(machines: Machine[]): void {
+  const p = resolveMachinesPath();
+  fs.mkdirSync(path.dirname(p), { recursive: true });
+  fs.writeFileSync(p, JSON.stringify(machines, null, 2) + "\n");
+}
+
+function mask(m: Machine): Record<string, unknown> {
+  return {
+    name: m.name,
+    host: m.host,
+    user: m.user,
+    port: m.port,
+    auth: m.auth,
+    ssh_key: m.ssh_key ? "********" : undefined,
+    password: m.password ? "********" : undefined,
+  };
+}
+
+export const machines = new Hono();
+
+machines.get("/", (c) => {
+  return c.json(readMachines().map(mask));
+});
+
+machines.post("/", async (c) => {
+  const body = await c.req.json<Machine>();
+  if (!body.name || !body.host || !body.user) {
+    return c.json({ error: "name, host, and user are required" }, 400);
+  }
+
+  const list = readMachines();
+  if (list.find((m) => m.name === body.name)) {
+    return c.json({ error: "machine already exists" }, 409);
+  }
+
+  if (body.auth === "local" && list.some((m) => m.auth === "local")) {
+    return c.json({ error: "a local machine already exists" }, 409);
+  }
+
+  list.push({
+    name: body.name,
+    host: body.host,
+    user: body.user,
+    port: body.port || 22,
+    auth: body.auth || "key",
+    ssh_key: body.ssh_key,
+    password: body.password,
+  });
+  writeMachines(list);
+  return c.json(mask(list[list.length - 1]), 201);
+});
+
+machines.put("/:name", async (c) => {
+  const name = c.req.param("name");
+  const list = readMachines();
+  const idx = list.findIndex((m) => m.name === name);
+  if (idx === -1) return c.json({ error: "not found" }, 404);
+
+  const body = await c.req.json<Partial<Machine>>();
+  const existing = list[idx];
+  list[idx] = {
+    name: existing.name,
+    host: body.host ?? existing.host,
+    user: body.user ?? existing.user,
+    port: body.port ?? existing.port,
+    auth: body.auth ?? existing.auth,
+    ssh_key: body.ssh_key !== undefined ? body.ssh_key : existing.ssh_key,
+    password: body.password !== undefined ? body.password : existing.password,
+  };
+  writeMachines(list);
+  return c.json(mask(list[idx]));
+});
+
+machines.delete("/:name", (c) => {
+  const name = c.req.param("name");
+  const list = readMachines();
+  const filtered = list.filter((m) => m.name !== name);
+  if (filtered.length === list.length) {
+    return c.json({ error: "not found" }, 404);
+  }
+  writeMachines(filtered);
+  return c.json({ ok: true });
+});