@recombine-ai/bosun 0.1.9 → 0.2.0-draft

package/build/server/chunks/_server.ts-iuWHhHZx.js

@@ -0,0 +1,629 @@
+import * as promptfoo from 'promptfoo';
+import { createContext } from '@recombine-ai/engine';
+import { glob } from 'glob';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { mkdir, writeFile, readFile } from 'node:fs/promises';
+import { t as tryReadConfig, g as getProjectRootPath } from './index3-CVYj52gl.js';
+import { readSuite, getSuiteFilePath } from './suite-4--Gr0jF.js';
+import deepmerge from 'deepmerge';
+
+const legacyPath = "src/bosun/workflows.ts";
+async function loadProjects() {
+  let projectFiles = [];
+  try {
+    projectFiles = await glob("projects/*/package.json", { absolute: true });
+  } catch {
+  }
+  if (!projectFiles.length) {
+    const singlePkg = path.resolve(process.cwd(), "package.json");
+    try {
+      const raw = await readFile(singlePkg, "utf8");
+      const pkg = JSON.parse(raw);
+      if (pkg?.name !== "@recombine-ai/bosun") {
+        projectFiles = [singlePkg];
+      }
+    } catch {
+    }
+  }
+  console.log("loadProjects candidates:", projectFiles);
+  const agentFactories = [];
+  for (const file of projectFiles) {
+    let pkg;
+    try {
+      const raw = await readFile(file, "utf8");
+      pkg = JSON.parse(raw);
+    } catch (e) {
+      console.error("Failed to read package.json:", file, e);
+      continue;
+    }
+    const agentsRoot = pkg?.bosun?.root ?? legacyPath;
+    const projectName = path.basename(path.dirname(file));
+    const agentsRootPath = path.resolve(path.dirname(file), agentsRoot);
+    const modUrl = pathToFileURL(agentsRootPath).href;
+    let mod;
+    try {
+      mod = await import(modUrl);
+    } catch (e) {
+      const candidates = [
+        // Replace src/.../workflows.ts -> build/.../workflows.js
+        agentsRootPath.replace(`${path.sep}src${path.sep}`, `${path.sep}build${path.sep}`).replace(/\.ts$/, ".js"),
+        // Explicit build/bosun/workflows.js
+        path.resolve(path.dirname(file), "build", "bosun", "workflows.js"),
+        // dist/bosun/workflows.js
+        path.resolve(path.dirname(file), "dist", "bosun", "workflows.js")
+      ];
+      let imported = false;
+      for (const cand of candidates) {
+        try {
+          const url = pathToFileURL(cand).href;
+          mod = await import(url);
+          imported = true;
+          break;
+        } catch {
+        }
+      }
+      if (!imported) {
+        try {
+          const { build } = await import('./main-DL75vM4l.js').then(function (n) { return n.m; });
+          const outDir = path.resolve(process.cwd(), "build", "academy-workflows");
+          await mkdir(outDir, { recursive: true });
+          const outFile = path.join(outDir, `${projectName}.cjs`);
+          await build({
+            entryPoints: [agentsRootPath],
+            outfile: outFile,
+            bundle: true,
+            platform: "node",
+            format: "cjs",
+            target: ["node18"],
+            sourcemap: false,
+            logLevel: "silent"
+          });
+          const bundledUrl = pathToFileURL(outFile).href;
+          mod = await import(bundledUrl);
+        } catch (e2) {
+          console.error("Failed to import/bundle agents module:", modUrl, e2);
+          continue;
+        }
+      }
+    }
+    const unwrap = (m) => {
+      let v = m?.default ?? m;
+      if (v && typeof v === "object" && "default" in v && Object.keys(v).length <= 2) {
+        v = v.default;
+      }
+      return v;
+    };
+    const factories = unwrap(mod);
+    const basePath = (mod?.basePath || mod?.default?.basePath) ?? path.dirname(agentsRootPath);
+    if (projectName && factories && typeof factories === "object") {
+      agentFactories.push({
+        basePath,
+        project: projectName,
+        factories
+      });
+    }
+  }
+  return agentFactories;
+}
+async function loadAgents(props) {
+  const {
+    agentId,
+    getMessages,
+    sendMessage,
+    stepTracer,
+    logger: providedLogger,
+    context,
+    timeoutSeconds
+  } = props;
+  const projects = await loadProjects();
+  if (!projects.length) throw new Error("No projects found");
+  let proj = projects[0];
+  let name = Object.keys(proj.factories)[0];
+  if (agentId) {
+    if (agentId.includes("/")) {
+      const [p, n] = agentId.split("/");
+      const found = projects.find((x) => x.project === p);
+      if (!found) throw new Error(`Project not found: ${p}`);
+      if (!found.factories[n]) throw new Error(`Agent not found: ${agentId}`);
+      proj = found;
+      name = n;
+    } else {
+      if (!proj.factories[agentId])
+        throw new Error(`Agent not found in default project: ${agentId}`);
+      name = agentId;
+    }
+  }
+  const factory = proj.factories[name];
+  if (!factory) throw new Error("Factory not found");
+  const logger = providedLogger || {
+    log: () => {
+    },
+    debug: () => {
+    },
+    error: () => {
+    },
+    warn: () => {
+    }
+  };
+  const scheduler = {
+    clearAllPendingActions: async () => {
+    },
+    registerAction: () => async () => {
+    },
+    triggerAction: async () => {
+    },
+    serialize: () => ({}),
+    setState: (_) => {
+    }
+  };
+  const ctx = createContext({});
+  if (context && typeof context === "object") {
+    ctx.swap(context);
+  }
+  const history = getMessages();
+  const getMessagesWithHistory = () => history;
+  const sendMessageWithHistory = async (message) => {
+    const m = typeof message === "string" ? { sender: "agent", text: message } : message;
+    try {
+      history.push(m);
+    } catch {
+      logger.log("Failed to push message to history:", m);
+    }
+    await sendMessage(m);
+  };
+  const agent = await factory({
+    stepTracer,
+    tracer: {
+      addStep: (step) => {
+      }
+    },
+    logger,
+    scheduler,
+    getMessages: getMessagesWithHistory,
+    sendMessage: sendMessageWithHistory,
+    sendAction: async () => {
+    },
+    ctx
+  });
+  process.on("unhandledRejection", (r) => {
+    console.error("UNHANDLED", r);
+  });
+  try {
+    await agent.respondToMessage();
+    await new Promise((resolve) => setTimeout(resolve, timeoutSeconds * 1e3));
+  } catch (err) {
+    const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err);
+    const stack = err instanceof Error ? err.stack : void 0;
+    try {
+      logger.error("Error in agent.respondToMessage():", msg);
+    } catch {
+    }
+    if (stack) {
+      try {
+        logger.error(stack);
+      } catch {
+      }
+    }
+  }
+}
+const mergeContexts = (baseCtx, testCtx) => deepmerge(baseCtx || {}, testCtx || {}, { arrayMerge: (_d, s, _o) => s });
+function parseConversationFromSheet(conversation) {
+  if (!conversation) return [];
+  return conversation.split("\n").filter((l) => l.trim()).map((line) => {
+    const [sender, ...content] = line.split(": ");
+    const lower = (sender || "").toLowerCase();
+    const role = lower === "agent" || lower === "assistant" ? "agent" : lower === "system" ? "system" : "user";
+    return { sender: role, text: content.join(": ") };
+  }).filter((m) => m.sender === "user" || m.sender === "agent" || m.sender === "system");
+}
+function toTestInputs(rows, defaultContext, scope = "pinned") {
+  return rows.filter((r) => scope === "all" ? true : r.pinned === "TRUE").map((row) => ({
+    id: row.id,
+    conversation: parseConversationFromSheet(row.conversation),
+    assertions: [
+      {
+        type: row.assertion_type,
+        value: row.assertion_value
+      },
+      {
+        type: "similar",
+        value: String(row.expected_output),
+        threshold: Number(row.similarity_threshold)
+      }
+    ],
+    context: (() => {
+      const testCtx = row.context ? JSON.parse(row.context) : {};
+      return defaultContext ? mergeContexts(defaultContext, testCtx) : testCtx;
+    })(),
+    timeoutSeconds: (() => {
+      const rawValue = row.timeout;
+      if (rawValue === void 0 || rawValue === null || String(rawValue).trim() === "") {
+        throw new Error("Timeout value is required");
+      }
+      const raw = Number(rawValue);
+      if (isNaN(raw)) {
+        throw new Error(`Invalid timeout value: ${rawValue}`);
+      }
+      if (raw > 1e3) {
+        throw new Error(`Timeout value exceeds maximum of 1000: ${raw}`);
+      }
+      if (raw < 0) {
+        throw new Error(`Timeout value must be non-negative: ${raw}`);
+      }
+      return raw;
+    })()
+  }));
+}
+class RecombineAgentProvider {
+  providerId;
+  config;
+  runLoadAgents;
+  constructor(options = {}) {
+    this.providerId = options.id || "recombine-agent";
+    this.config = options.config || {};
+    this.runLoadAgents = options.loadAgents || loadAgents;
+  }
+  id() {
+    return this.providerId;
+  }
+  async callApi(_prompt, context) {
+    const vars = context?.vars || {};
+    let captured = "";
+    const capturedLogs = [];
+    const capturedStepTraces = [];
+    let logStream = null;
+    let logsFilePath = "";
+    const agentCompositeId = `${vars.project}/${vars.agent}`;
+    const conv = Array.isArray(vars.conversation) ? vars.conversation.map((m) => ({ ...m })) : [];
+    const lastUserIndex = (() => {
+      for (let i = conv.length - 1; i >= 0; i--) {
+        if (conv[i]?.sender === "user") return i;
+      }
+      return -1;
+    })();
+    if (lastUserIndex < 0) {
+      return { output: "", error: "No user message found in conversation" };
+    }
+    conv[lastUserIndex];
+    {
+      const { createWriteStream } = await import('node:fs');
+      const runId = `${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+      const projectRoot = await getProjectRootPath(vars.project);
+      const logsDir = path.join(projectRoot, "data", "logs");
+      await mkdir(logsDir, { recursive: true });
+      logsFilePath = path.join(logsDir, `capturedLogs_${runId}.log`);
+      logStream = createWriteStream(logsFilePath, { flags: "a" });
+    }
+    const stepTracer = {
+      addStepTrace: (stepTrace) => {
+        try {
+          console.log(`trace: ${stepTrace.name}`);
+          capturedStepTraces.push(stepTrace);
+        } catch {
+        }
+      }
+    };
+    const push = (level, args) => {
+      try {
+        const raw = args.map((a) => {
+          try {
+            return typeof a === "string" ? a : JSON.stringify(a);
+          } catch {
+            return String(a);
+          }
+        }).join(" ");
+        const prefix = `[${level.toUpperCase()}] `;
+        let line = `${prefix}${raw}`;
+        if (line.length > 5e3) {
+          line = line.slice(0, 5e3) + "... [TRUNCATED]";
+        }
+        capturedLogs.push(line);
+        try {
+          logStream.write(line + "\n");
+        } catch {
+        }
+      } catch {
+      }
+    };
+    const logger = {
+      log: (...args) => push("log", args),
+      debug: (...args) => push("debug", args),
+      warn: (...args) => push("warn", args),
+      error: (...args) => push("error", args)
+    };
+    try {
+      await this.runLoadAgents({
+        agentId: agentCompositeId,
+        getMessages: () => conv,
+        sendMessage: async (message) => {
+          const m = typeof message === "string" ? { sender: "agent", text: message } : message;
+          if (m.sender === "agent") {
+            if (true) {
+              captured = typeof m.text === "string" ? m.text : JSON.stringify(m);
+              logger.log("Agent message captured:", captured);
+            }
+          }
+        },
+        stepTracer,
+        logger,
+        context: vars.context || {},
+        timeoutSeconds: Number(vars.timeoutSeconds)
+      });
+      const fs = await import('fs/promises');
+      const projectRoot = await getProjectRootPath(vars.project);
+      const logsDir = path.join(projectRoot, "data", "logs");
+      await mkdir(logsDir, { recursive: true });
+      const filePath = path.join(logsDir, `capturedStepTraces_${Date.now()}.json`);
+      await fs.writeFile(filePath, JSON.stringify(capturedStepTraces, null, 2));
+      const sanitizeTrace = (t) => {
+        try {
+          const clone = { ...t };
+          const isMain = clone?.name === "main" || clone?.name === "mainReply";
+          const truncate = (s, max = 5e3) => {
+            if (typeof s !== "string") return s;
+            return s.length > max ? s.slice(0, max) + "... [TRUNCATED]" : s;
+          };
+          if (isMain) {
+            if ("renderedPrompt" in clone) delete clone.renderedPrompt;
+            if ("receivedPrompt" in clone) delete clone.receivedPrompt;
+          } else {
+            if (typeof clone.renderedPrompt !== "undefined") {
+              clone.renderedPrompt = truncate(clone.renderedPrompt);
+            }
+            if (typeof clone.receivedPrompt !== "undefined") {
+              clone.receivedPrompt = truncate(clone.receivedPrompt);
+            }
+          }
+          return clone;
+        } catch {
+          return t;
+        }
+      };
+      const sanitizedStepTraces = Array.isArray(capturedStepTraces) ? capturedStepTraces.map(sanitizeTrace) : [];
+      if (captured && captured.trim()) {
+        return {
+          output: captured,
+          metadata: {
+            logs: capturedLogs,
+            stepTraces: sanitizedStepTraces,
+            rawStepTraces: capturedStepTraces
+          }
+        };
+      }
+      return {
+        output: "",
+        error: "No agent reply captured",
+        metadata: {
+          logs: capturedLogs,
+          stepTraces: sanitizedStepTraces,
+          rawStepTraces: capturedStepTraces
+        }
+      };
+    } catch (err) {
+      const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err);
+      capturedLogs.push(`[ERROR] ${msg}`);
+      try {
+        logStream.write(`[ERROR] ${msg}
+`);
+      } catch {
+      }
+      return {
+        output: "",
+        error: msg,
+        metadata: { logs: capturedLogs, stepTraces: capturedStepTraces }
+      };
+    } finally {
+      if (logStream) {
+        await new Promise((resolve) => {
+          try {
+            logStream.end(resolve);
+          } catch {
+            resolve();
+          }
+        });
+      }
+    }
+  }
+}
+function buildTableRows(evals, csvRows, spreadsheetId) {
+  const bucket = /* @__PURE__ */ new Map();
+  for (const ev of evals) {
+    for (const res of ev.results) {
+      const testId = String(res.metadata?.testId ?? "");
+      (bucket.get(testId) ?? bucket.set(testId, []).get(testId)).push(res);
+    }
+  }
+  const toStringSafe = (v) => {
+    if (typeof v === "string") return v;
+    try {
+      return JSON.stringify(v);
+    } catch {
+      return String(v);
+    }
+  };
+  return csvRows.map((row) => {
+    const rowId = row.id;
+    const evalResults = bucket.get(String(rowId)) ?? [];
+    const passCount = evalResults.filter((r) => r.success).length;
+    const totalRuns = evalResults.length;
+    const pct = totalRuns ? Math.round(passCount / totalRuns * 100) : 0;
+    const rateStr = `${passCount}/${totalRuns}`;
+    const results = evalResults.map((r) => {
+      const rawTraces = Array.isArray(r.metadata?.rawStepTraces) ? r.metadata?.rawStepTraces : Array.isArray(r.metadata?.stepTraces) ? r.metadata?.stepTraces : [];
+      const lastMain = (() => {
+        for (let i = rawTraces.length - 1; i >= 0; i--) {
+          const t = rawTraces[i];
+          if (t && t.name === "main") return t;
+        }
+        return void 0;
+      })();
+      const rendered_prompt = lastMain && typeof lastMain.renderedPrompt !== "undefined" ? toStringSafe(lastMain.renderedPrompt) : "NO RENDERED PROMPT";
+      const received_prompt = lastMain && typeof lastMain.receivedPrompt !== "undefined" ? toStringSafe(lastMain.receivedPrompt) : "NO RECEIVED PROMPT";
+      const stringified_conversation = lastMain && typeof lastMain.stringifiedConversation !== "undefined" ? toStringSafe(lastMain.stringifiedConversation) : "NO STRINGIFIED CONVERSATION";
+      const received_context = lastMain && typeof lastMain.receivedContext !== "undefined" ? toStringSafe(lastMain.receivedContext) : "NO RECEIVED CONTEXT";
+      const logs = Array.isArray(r.metadata?.logs) ? r.metadata?.logs : [];
+      const traces = Array.isArray(r.metadata?.stepTraces) ? r.metadata?.stepTraces : [];
+      return {
+        success: !!r.success,
+        result: String(r.response?.output ?? ""),
+        evaluation: String(r.gradingResult?.reason ?? ""),
+        rendered_prompt,
+        received_prompt,
+        stringified_conversation,
+        received_context,
+        traces,
+        errors: String(r.error ?? ""),
+        logs
+      };
+    });
+    return {
+      ...row,
+      prompts: `suite://${spreadsheetId}?testId=${rowId}`,
+      success_rate_percent: pct,
+      success_rate: rateStr,
+      results
+    };
+  });
+}
+async function runPromptTests(project, agent, opts, deps) {
+  const backendId = `${project}/${agent}`;
+  const suite = await readSuite(project, agent);
+  const rows = suite.tests;
+  const cfg = await tryReadConfig(project, agent);
+  if (!cfg) {
+    const suitePath = await getSuiteFilePath(project, agent);
+    const expectedPath = path.join(path.dirname(suitePath), `${agent}.config.json`);
+    throw new Error(`Config not found at ${expectedPath}`);
+  }
+  const repeats = Number(cfg?.repeats ?? 1);
+  const concurrency = Number(cfg?.concurrency ?? 1);
+  const baseContext = suite?.meta?.defaultContext || {};
+  const typedRows = rows;
+  const indexedRows = typedRows.map((row, index) => ({ ...row, index }));
+  const tests = toTestInputs(indexedRows, baseContext, opts?.scope);
+  const providerInstance = new RecombineAgentProvider({
+    id: "recombine-agent",
+    loadAgents: deps?.loadAgents
+  });
+  const providerFn = Object.assign(
+    (_promptStr, ctx) => providerInstance.callApi(_promptStr, ctx),
+    { label: "recombine-agent" }
+  );
+  const testSuites = tests.map((test) => ({
+    prompts: ["agent-run"],
+    // placeholder; provider ignores
+    tests: [
+      {
+        metadata: { testId: test.id },
+        assert: test.assertions,
+        model: "not used!",
+        vars: {
+          project,
+          agent,
+          context: test.context,
+          conversation: test.conversation,
+          timeoutSeconds: test.timeoutSeconds
+        }
+      }
+    ],
+    providers: [providerFn]
+  }));
+  const results = [];
+  for (const suite2 of testSuites) {
+    try {
+      const ev = await promptfoo.evaluate(suite2, {
+        repeat: repeats,
+        cache: false,
+        maxConcurrency: concurrency
+      });
+      results.push(ev);
+    } catch (err) {
+      const msg = err instanceof Error ? `${err.name}: ${err.message}` : String(err);
+      results.push({
+        results: [
+          {
+            success: false,
+            response: { output: "" },
+            error: msg,
+            metadata: { testId: suite2.tests?.[0]?.metadata?.testId }
+          }
+        ]
+      });
+    }
+  }
+  const rowsToWrite = buildTableRows(results, typedRows, backendId);
+  const projectRoot = await getProjectRootPath(project);
+  const outDir = path.join(projectRoot, "data", "test_runs");
+  await mkdir(outDir, { recursive: true });
+  const iso = (/* @__PURE__ */ new Date()).toISOString();
+  const isoFile = iso.replace(/:/g, "-");
+  const outPath = path.join(outDir, `${agent}.${isoFile}.report.json`);
+  const withProvenance = rowsToWrite.map((row, idx) => {
+    const test_id = row.id ?? String(idx);
+    const pinned_at_run = String(row.pinned || "").toUpperCase() === "TRUE" ? "TRUE" : "FALSE";
+    const run_scope = opts?.scope;
+    return {
+      ...row,
+      project,
+      agent,
+      test_id,
+      run_timestamp: iso,
+      run_config: JSON.stringify({ repeats, concurrency, defaultContext: baseContext }),
+      pinned_at_run,
+      run_scope
+    };
+  });
+  const body = {
+    project,
+    agent,
+    run_timestamp: iso,
+    run_scope: opts?.scope,
+    run_config: { repeats, concurrency, defaultContext: baseContext },
+    rows: withProvenance
+  };
+  await writeFile(outPath, JSON.stringify(body, null, 2), "utf8");
+}
+const GET = async ({ params, url }) => {
+  const { project, agent } = params;
+  const scope = url.searchParams.get("scope") || "pinned";
+  const stream = new ReadableStream({
+    start(controller) {
+      function send(obj, event) {
+        const payload = `data: ${JSON.stringify(obj)}
+` + (event ? `event: ${event}
+` : "") + "\n";
+        controller.enqueue(new TextEncoder().encode(payload));
+      }
+      (async () => {
+        try {
+          send({
+            level: "info",
+            text: `Starting run: project=${project}, agent=${agent}, scope=${scope}`
+          });
+          await runPromptTests(project, agent, { scope });
+          send({ level: "info", text: "Completed runPromptTests successfully" });
+          send({ code: 0 }, "done");
+          controller.close();
+        } catch (err) {
+          send({ level: "error", text: `runPromptTests failed: ${err?.message || String(err)}` });
+          send({ code: 1 }, "done");
+          try {
+            controller.close();
+          } catch {
+          }
+        }
+      })();
+    }
+  });
+  return new Response(stream, {
+    headers: {
+      "content-type": "text/event-stream",
+      "cache-control": "no-cache, no-transform",
+      connection: "keep-alive"
+    }
+  });
+};
+
+export { GET };
+//# sourceMappingURL=_server.ts-iuWHhHZx.js.map