@m8i-51/shoal 0.1.0

package/server/index.ts

@@ -0,0 +1,227 @@
+import "dotenv/config";
+import express from "express";
+import { fileURLToPath } from "url";
+import { dirname, join } from "path";
+import { existsSync, readFileSync, writeFileSync } from "fs";
+import { listRuns, getReportPath } from "./runs.js";
+import { activeSessions, spawnRun, cancelSession } from "./runner.js";
+
+function specFilePath(baseUrl: string): string {
+  try {
+    const host = new URL(baseUrl).host.replace(/[^a-zA-Z0-9]/g, "-");
+    return join(process.cwd(), "product-specs", `${host}.json`);
+  } catch {
+    return "";
+  }
+}
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const app = express();
+const PORT = parseInt(process.env.PORT ?? "4000", 10);
+
+app.use(express.json());
+
+// ----------------------------------------------------------------
+// API: product spec (goals)
+// ----------------------------------------------------------------
+app.get("/api/spec", (_req, res) => {
+  const baseUrl = process.env.BASE_URL ?? "http://localhost:3000";
+  const filePath = specFilePath(baseUrl);
+  if (!filePath || !existsSync(filePath)) {
+    res.status(404).json({ error: "spec not found" });
+    return;
+  }
+  try {
+    res.json(JSON.parse(readFileSync(filePath, "utf-8")));
+  } catch {
+    res.status(500).json({ error: "failed to read spec" });
+  }
+});
+
+app.patch("/api/spec/goals", (req, res) => {
+  const baseUrl = process.env.BASE_URL ?? "http://localhost:3000";
+  const filePath = specFilePath(baseUrl);
+  if (!filePath || !existsSync(filePath)) {
+    res.status(404).json({ error: "spec not found" });
+    return;
+  }
+  const { goals } = req.body as { goals?: unknown };
+  if (!Array.isArray(goals) || !goals.every((g) => typeof g === "string")) {
+    res.status(400).json({ error: "goals must be an array of strings" });
+    return;
+  }
+  try {
+    const spec = JSON.parse(readFileSync(filePath, "utf-8")) as Record<string, unknown>;
+    spec.appGoals = goals;
+    writeFileSync(filePath, JSON.stringify(spec, null, 2), "utf-8");
+    res.json({ ok: true });
+  } catch {
+    res.status(500).json({ error: "failed to update spec" });
+  }
+});
+
+// ----------------------------------------------------------------
+// API: list runs（アクティブなセッション情報で補完）
+// ----------------------------------------------------------------
+app.get("/api/runs", (_req, res) => {
+  const runs = listRuns();
+
+  // activeSessions が持つ isLive フラグで補完（インメモリ情報が優先）
+  const enriched = runs.map((r) => {
+    const session = activeSessions.get(r.runId);
+    if (session) return { ...r, isLive: !session.done };
+    return r;
+  });
+
+  res.json(enriched);
+});
+
+// ----------------------------------------------------------------
+// API: serve HTML report for a run
+// ----------------------------------------------------------------
+app.get("/api/runs/:runId/report", (req, res) => {
+  const reportPath = getReportPath(req.params.runId);
+  if (!reportPath) {
+    res.status(404).json({ error: "report not found" });
+    return;
+  }
+  res.sendFile(reportPath);
+});
+
+// ----------------------------------------------------------------
+// API: start a run
+// ----------------------------------------------------------------
+app.post("/api/runs/start", (req, res) => {
+  const { baseUrl, maxBrowsers, maxExplorers, llmBaseUrl, llmApiKey, llmModel } = req.body as {
+    baseUrl?: string;
+    maxBrowsers?: number;
+    maxExplorers?: number;
+    llmBaseUrl?: string;
+    llmApiKey?: string;
+    llmModel?: string;
+  };
+  const sessionId = spawnRun({ baseUrl, maxBrowsers, maxExplorers, llmBaseUrl, llmApiKey, llmModel });
+  res.json({ sessionId });
+});
+
+// ----------------------------------------------------------------
+// API: cancel a running run
+// ----------------------------------------------------------------
+app.post("/api/runs/:runId/cancel", (req, res) => {
+  const ok = cancelSession(req.params.runId);
+  res.json({ ok });
+});
+
+// ----------------------------------------------------------------
+// SSE ヘルパー
+// ----------------------------------------------------------------
+function sseStream(req: express.Request, res: express.Response, sessionId: string) {
+  const session = activeSessions.get(sessionId);
+  if (!session) {
+    res.status(404).json({ error: "session not found" });
+    return;
+  }
+
+  res.setHeader("Content-Type", "text/event-stream");
+  res.setHeader("Cache-Control", "no-cache");
+  res.setHeader("Connection", "keep-alive");
+  res.flushHeaders();
+
+  const send = (line: string) => {
+    if (res.destroyed) return;
+    try { res.write(`data: ${JSON.stringify(line)}\n\n`); } catch { /* ignore */ }
+  };
+
+  const sendDone = () => {
+    if (res.destroyed) return;
+    try {
+      res.write(`event: done\ndata: ${JSON.stringify({ exitCode: session.exitCode })}\n\n`);
+      res.end();
+    } catch { /* ignore */ }
+  };
+
+  for (const line of session.lines) send(line);
+
+  if (session.done) { sendDone(); return; }
+
+  session.listeners.push(send);
+  session.doneListeners.push(sendDone);
+
+  req.on("close", () => {
+    session.listeners = session.listeners.filter((l) => l !== send);
+    session.doneListeners = session.doneListeners.filter((l) => l !== sendDone);
+  });
+}
+
+// ----------------------------------------------------------------
+// API: SSE — /api/sessions/:sessionId/events（後方互換）
+// ----------------------------------------------------------------
+app.get("/api/sessions/:sessionId/events", (req, res) => {
+  sseStream(req, res, req.params.sessionId);
+});
+
+// ----------------------------------------------------------------
+// API: SSE — /api/runs/:runId/events（詳細ページ用）
+// ----------------------------------------------------------------
+app.get("/api/runs/:runId/events", (req, res) => {
+  sseStream(req, res, req.params.runId);
+});
+
+// ----------------------------------------------------------------
+// API: ログ行をまとめて返す（完了後・再起動後もファイルから参照可能）
+// ----------------------------------------------------------------
+app.get("/api/runs/:runId/log", (req, res) => {
+  const { runId } = req.params;
+
+  // 1. アクティブセッション（インメモリ）を優先
+  const session = activeSessions.get(runId);
+  if (session) {
+    res.json({ lines: session.lines, done: session.done, exitCode: session.exitCode });
+    return;
+  }
+
+  // 2. 保存済みログファイルにフォールバック
+  const logFilePath = join(process.cwd(), "logs", `log_${runId}.txt`);
+  if (existsSync(logFilePath)) {
+    const lines = readFileSync(logFilePath, "utf-8").split("\n").filter((l) => l !== "");
+    res.json({ lines, done: true, exitCode: null });
+    return;
+  }
+
+  res.status(404).json({ error: "no log found" });
+});
+
+// ----------------------------------------------------------------
+// Static: serve built React app
+// ----------------------------------------------------------------
+const distPath = join(__dirname, "..", "web", "dist");
+if (existsSync(distPath)) {
+  app.use(express.static(distPath));
+  app.get("/{*splat}", (_req, res) => {
+    res.sendFile(join(distPath, "index.html"));
+  });
+} else {
+  app.get("/{*splat}", (_req, res) => {
+    res.status(503).send("Frontend not built. Run: npm run build:web");
+  });
+}
+
+// Express エラーハンドラ（クラッシュ防止）
+app.use((err: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
+  console.error("[server] unhandled error:", err.message);
+  if (!res.headersSent) {
+    res.status(500).json({ error: "internal server error" });
+  }
+});
+
+// Node.js uncaught exception / rejection をログだけしてサーバーを落とさない
+process.on("uncaughtException", (err) => {
+  console.error("[server] uncaughtException:", err.message);
+});
+process.on("unhandledRejection", (reason) => {
+  console.error("[server] unhandledRejection:", reason);
+});
+
+app.listen(PORT, () => {
+  console.log(`\nshoal dashboard → http://localhost:${PORT}\n`);
+});

package/server/runner.ts

@@ -0,0 +1,125 @@
+import { spawn, type ChildProcess } from "child_process";
+import { fileURLToPath } from "url";
+import { dirname, join } from "path";
+import { existsSync, mkdirSync, writeFileSync, appendFileSync, unlinkSync } from "fs";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const packageRoot = join(__dirname, "..");
+
+export interface Session {
+  sessionId: string;
+  startedAt: string;
+  completedAt: string | null;
+  done: boolean;
+  exitCode: number | null;
+  lines: string[];
+  listeners: ((line: string) => void)[];
+  doneListeners: (() => void)[];
+  child: ChildProcess | null;
+}
+
+export const activeSessions = new Map<string, Session>();
+
+export function spawnRun(opts: {
+  baseUrl?: string;
+  maxBrowsers?: number;
+  maxExplorers?: number;
+  llmBaseUrl?: string;
+  llmApiKey?: string;
+  llmModel?: string;
+}): string {
+  const sessionId = `run_${Date.now()}`;
+
+  const session: Session = {
+    sessionId,
+    startedAt: new Date().toISOString(),
+    completedAt: null,
+    done: false,
+    exitCode: null,
+    lines: [],
+    listeners: [],
+    doneListeners: [],
+    child: null,
+  };
+  activeSessions.set(sessionId, session);
+
+  const logsDir = join(process.cwd(), "logs");
+  mkdirSync(logsDir, { recursive: true });
+
+  // ログをリアルタイムでファイルに書き出す（サーバー再起動後もポーリングで読める）
+  const logFilePath = join(logsDir, `log_${sessionId}.txt`);
+  console.log(`[runner] spawning ${sessionId}, log → ${logFilePath}`);
+
+  // running_*.json で実行中フラグをディスクに残す
+  const pendingPath = join(logsDir, `running_${sessionId}.json`);
+  writeFileSync(pendingPath, JSON.stringify({ runId: sessionId, startedAt: session.startedAt }));
+
+  const tsxBin = join(packageRoot, "node_modules", ".bin", "tsx");
+  const bin = existsSync(tsxBin) ? tsxBin : "tsx";
+  const script = join(packageRoot, "run.ts");
+
+  const env: NodeJS.ProcessEnv = {
+    ...process.env,
+    SHOAL_RUN_ID: sessionId,
+    ...(opts.baseUrl ? { BASE_URL: opts.baseUrl } : {}),
+    ...(opts.maxBrowsers != null ? { MAX_BROWSERS: String(opts.maxBrowsers) } : {}),
+    ...(opts.maxExplorers != null ? { MAX_EXPLORERS: String(opts.maxExplorers) } : {}),
+    ...(opts.llmBaseUrl ? { LLM_BASE_URL: opts.llmBaseUrl } : {}),
+    ...(opts.llmApiKey ? { LLM_API_KEY: opts.llmApiKey } : {}),
+    ...(opts.llmModel ? { LLM_MODEL: opts.llmModel } : {}),
+  };
+
+  const child = spawn(bin, [script], {
+    env,
+    cwd: process.cwd(),
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+  session.child = child;
+
+  const emit = (line: string) => {
+    session.lines.push(line);
+    try { appendFileSync(logFilePath, line + "\n"); } catch { /* ignore */ }
+    for (const listener of session.listeners) {
+      listener(line);
+    }
+  };
+
+  for (const stream of [child.stdout, child.stderr]) {
+    let buf = "";
+    stream.on("data", (chunk: Buffer) => {
+      buf += chunk.toString();
+      const parts = buf.split("\n");
+      buf = parts.pop() ?? "";
+      for (const line of parts) emit(line);
+    });
+    stream.on("end", () => {
+      if (buf) emit(buf);
+    });
+  }
+
+  child.on("exit", (code) => {
+    session.done = true;
+    session.completedAt = new Date().toISOString();
+    session.exitCode = code ?? 0;
+    try { unlinkSync(pendingPath); } catch { /* ignore */ }
+    for (const listener of session.doneListeners) listener();
+  });
+
+  return sessionId;
+}
+
+export function cancelSession(sessionId: string): boolean {
+  const session = activeSessions.get(sessionId);
+  if (!session || session.done || !session.child) return false;
+  try {
+    session.child.kill("SIGTERM");
+    setTimeout(() => {
+      if (!session.done) {
+        try { session.child?.kill("SIGKILL"); } catch { /* ignore */ }
+      }
+    }, 4000);
+    return true;
+  } catch {
+    return false;
+  }
+}

package/server/runs.ts

@@ -0,0 +1,103 @@
+import * as fs from "fs";
+import * as path from "path";
+import type { RunLog, Finding } from "../framework/types";
+
+export interface RunSummary {
+  runId: string;
+  startedAt: string;
+  completedAt: string | null;
+  status: "completed" | "running";
+  agentCount: number;
+  completedAgents: number;
+  errorAgents: number;
+  findingCount: number;
+  findingsByCategory: Record<string, number>;
+  hasReport: boolean;
+  isLive?: boolean;
+  estimatedCostUSD: number | null;
+}
+
+function countFindings(runId: string): { total: number; byCategory: Record<string, number> } {
+  const dir = path.join(process.cwd(), "findings", runId);
+  if (!fs.existsSync(dir)) return { total: 0, byCategory: {} };
+  const byCategory: Record<string, number> = {};
+  let total = 0;
+  for (const file of fs.readdirSync(dir)) {
+    if (!file.endsWith(".json")) continue;
+    try {
+      const f: Finding = JSON.parse(fs.readFileSync(path.join(dir, file), "utf-8"));
+      byCategory[f.category] = (byCategory[f.category] ?? 0) + 1;
+      total++;
+    } catch {
+      // skip malformed
+    }
+  }
+  return { total, byCategory };
+}
+
+export function listRuns(): RunSummary[] {
+  const logsDir = path.join(process.cwd(), "logs");
+  if (!fs.existsSync(logsDir)) return [];
+
+  const summaries: RunSummary[] = [];
+  const seenRunIds = new Set<string>();
+
+  for (const file of fs.readdirSync(logsDir)) {
+    // 実行中の pending ファイル（running_*.json）
+    if (file.startsWith("running_") && file.endsWith(".json")) {
+      try {
+        const raw = fs.readFileSync(path.join(logsDir, file), "utf-8");
+        const { runId, startedAt } = JSON.parse(raw);
+        if (!seenRunIds.has(runId)) {
+          seenRunIds.add(runId);
+          summaries.push({
+            runId,
+            startedAt,
+            completedAt: null,
+            status: "running",
+            agentCount: 0,
+            completedAgents: 0,
+            errorAgents: 0,
+            findingCount: 0,
+            findingsByCategory: {},
+            hasReport: false,
+            isLive: true,
+            estimatedCostUSD: null,
+          });
+        }
+      } catch { /* skip */ }
+      continue;
+    }
+
+    // 通常のランログ（YYYY-MM-DDTHH-MM-SS_run_*.json）
+    if (!file.endsWith(".json") || file.startsWith("report_")) continue;
+    try {
+      const raw = fs.readFileSync(path.join(logsDir, file), "utf-8");
+      const log: RunLog = JSON.parse(raw);
+      if (seenRunIds.has(log.runId)) continue;
+      seenRunIds.add(log.runId);
+      const { total, byCategory } = countFindings(log.runId);
+      const reportPath = path.join(logsDir, `report_${log.runId}.html`);
+      summaries.push({
+        runId: log.runId,
+        startedAt: log.startedAt,
+        completedAt: log.completedAt,
+        status: log.completedAt ? "completed" : "running",
+        agentCount: log.agents.length,
+        completedAgents: log.agents.filter((a) => a.status === "completed").length,
+        errorAgents: log.agents.filter((a) => a.status === "error").length,
+        findingCount: total,
+        findingsByCategory: byCategory,
+        hasReport: fs.existsSync(reportPath),
+        estimatedCostUSD: log.summary?.cost?.estimatedUSD ?? null,
+      });
+    } catch { /* skip */ }
+  }
+
+  return summaries.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
+}
+
+export function getReportPath(runId: string): string | null {
+  const p = path.join(process.cwd(), "logs", `report_${runId}.html`);
+  return fs.existsSync(p) ? p : null;
+}

package/targets/example.ts

@@ -0,0 +1,55 @@
+/**
+ * Example target — copy this file and adapt it to your app.
+ *
+ * A target connects shoal to your app's API.
+ * Define the tools agents can call, then implement the execute() handler.
+ */
+import type { TargetConfig } from "./types";
+
+const BASE_URL = process.env.BASE_URL ?? "http://localhost:3000";
+
+async function api(endpoint: string, method = "GET", body?: unknown): Promise<unknown> {
+  const res = await fetch(`${BASE_URL}${endpoint}`, {
+    method,
+    headers: { "Content-Type": "application/json" },
+    body: body ? JSON.stringify(body) : undefined,
+  });
+  const text = await res.text();
+  return text ? JSON.parse(text) : { ok: res.ok };
+}
+
+export const exampleConfig: TargetConfig = {
+  appTools: [
+    // Define the API tools agents can call.
+    // Each tool is described in plain English so agents know when and how to use it.
+    {
+      name: "get_items",
+      description: "Get a list of items from the app.",
+      input_schema: { type: "object", properties: {}, required: [] },
+    },
+    {
+      name: "create_item",
+      description: "Create a new item.",
+      input_schema: {
+        type: "object",
+        properties: {
+          title: { type: "string", description: "Item title" },
+          description: { type: "string", description: "Item description (optional)" },
+        },
+        required: ["title"],
+      },
+    },
+    // Add more tools as needed...
+  ],
+
+  async execute(toolName, input, agentId) {
+    switch (toolName) {
+      case "get_items":
+        return api("/api/items");
+      case "create_item":
+        return api("/api/items", "POST", { ...input, createdBy: agentId });
+      default:
+        return { error: "unknown tool" };
+    }
+  },
+};

package/targets/index.ts

@@ -0,0 +1,17 @@
+import type { TargetConfig } from "./types";
+import { exampleConfig } from "./example";
+import { noopTarget } from "./noop";
+
+const TARGETS: Record<string, TargetConfig> = {
+  "example": exampleConfig,
+  "none": noopTarget,
+};
+
+export function loadTarget(name: string): TargetConfig {
+  const target = TARGETS[name];
+  if (!target) {
+    console.warn(`[target] "${name}" not found, falling back to noop`);
+    return noopTarget;
+  }
+  return target;
+}

package/targets/noop.ts

@@ -0,0 +1,6 @@
+import type { TargetConfig } from "./types";
+
+export const noopTarget: TargetConfig = {
+  appTools: [],
+  execute: async () => ({ error: "no API tools configured for this target" }),
+};

package/targets/types.ts

@@ -0,0 +1,19 @@
+import type { Tool } from "../framework/llm-client";
+
+export interface Credentials {
+  email: string;
+  password: string;
+}
+
+export interface TargetConfig {
+  appTools: Tool[];
+  execute(toolName: string, input: Record<string, unknown>, agentId: string): Promise<unknown>;
+  /** Optional: absolute path to the project repository on the local filesystem.
+   *  If set, product-discovery will scan for README/docs/openapi files here.
+   *  If not set but GITHUB_REPO is set, it will fetch the README from GitHub instead. */
+  projectPath?: string;
+  /** Optional: seed credentials for the Account Manager agent.
+   *  If set, shoal will log in, discover roles, create test accounts per role,
+   *  and run all browser agents in authenticated sessions. */
+  credentials?: Credentials;
+}

package/triage-only.ts

@@ -0,0 +1,57 @@
+/**
+ * triage-only.ts
+ * 指定runIdのfindingsを読み込んでトリアージエージェントだけを実行する
+ *
+ * 使い方:
+ *   RUN_ID=run_xxx npx tsx scripts/triage-only.ts
+ *   (RUN_ID省略時は最新のrunを使用)
+ */
+
+import "dotenv/config";
+import * as fs from "fs";
+import * as path from "path";
+import { createLLMClient } from "./framework/llm-client";
+import { runTriageAgent } from "./framework/triage";
+import type { Finding } from "./framework/types";
+
+const GITHUB_TOKEN = process.env.GITHUB_TOKEN ?? "";
+const GITHUB_REPO = process.env.GITHUB_REPO ?? "";
+
+function loadFindings(runId: string): Finding[] {
+  const dir = path.join(process.cwd(), "findings", runId);
+  if (!fs.existsSync(dir)) {
+    console.error(`findings/${runId} が見つかりません`);
+    process.exit(1);
+  }
+  const files = fs.readdirSync(dir).filter((f) => f.endsWith(".json") && f !== "triage_result.json");
+  return files.map((f) => JSON.parse(fs.readFileSync(path.join(dir, f), "utf-8")) as Finding);
+}
+
+function getLatestRunId(): string {
+  const findingsDir = path.join(process.cwd(), "findings");
+  const runs = fs.readdirSync(findingsDir).filter((d) => d.startsWith("run_")).sort();
+  if (runs.length === 0) {
+    console.error("findingsディレクトリにrunが見つかりません");
+    process.exit(1);
+  }
+  return runs[runs.length - 1];
+}
+
+async function main() {
+  const runId = process.env.RUN_ID ?? getLatestRunId();
+  console.log(`[トリアージ単体実行] runId: ${runId}`);
+
+  const findings = loadFindings(runId);
+  console.log(`[トリアージ単体実行] findings読み込み: ${findings.length}件`);
+  findings.forEach((f) => console.log(`  - ${f.agentName}: ${f.title.slice(0, 50)}`));
+
+  const { client, defaultModel } = createLLMClient();
+  const result = await runTriageAgent(findings, client, defaultModel, { token: GITHUB_TOKEN, repo: GITHUB_REPO });
+
+  console.log("\n=== トリアージ結果 ===");
+  console.log(`  Issue作成: ${result.issuesCreated}件`);
+  console.log(`  スキップ: ${result.skipped.length}件`);
+  console.log(`  未処理: ${result.unprocessed.length}件`);
+}
+
+main().catch(console.error);