@recombine-ai/bosun 0.16.2-cloud-1 → 0.16.2-cloud-2

package/dist/index.js

@@ -627,8 +627,6 @@ function toText(event) {
 }
 
 // src/backend/roleplay/router.ts
-import fs3 from "fs";
-import path8 from "path";
 import { z as z4 } from "zod";
 
 // src/backend/lib/trpc-context.ts
@@ -646,7 +644,7 @@ function writeProcedure(t2) {
 import "openai";
 
 // src/backend/tests/data.ts
-import { createHash, randomUUID as randomUUID2 } from "node:crypto";
+import { createHash as createHash2, randomUUID as randomUUID2 } from "node:crypto";
 import { readdir as readdir2, readFile as readFile3, stat, writeFile as writeFile2 } from "node:fs/promises";
 import path7 from "node:path";
 
@@ -927,6 +925,33 @@ function queueReportCompression(filePath) {
   return compression;
 }
 
+// src/backend/tests/test-edit-source.ts
+import { createHash } from "node:crypto";
+function canonicalize(value) {
+  if (Array.isArray(value)) return value.map(canonicalize);
+  if (!value || typeof value !== "object") return value;
+  return Object.fromEntries(
+    Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, entry]) => [key, canonicalize(entry)])
+  );
+}
+function testEditSourceFingerprint(test, overrides) {
+  const source = canonicalize({
+    step: test.step,
+    evaluationMode: test.evaluation_mode,
+    assertionType: test.assertion_type,
+    assertionValue: test.assertion_value,
+    caseSensitive: test.case_sensitive,
+    contextMode: test.context_mode,
+    conversation: overrides?.conversation ?? test.conversation,
+    expectedOutput: test.expected_output,
+    timeout: test.timeout,
+    similarityThreshold: test.similarity_threshold,
+    context: test.context,
+    mocks: test.mocks ?? null
+  });
+  return createHash("sha256").update(JSON.stringify(source)).digest("hex");
+}
+
 // src/backend/tests/data.ts
 var defaultTestRow = {
   order: 0,
@@ -983,7 +1008,7 @@ async function loadTests(ctx, agent) {
       "context",
       "mocks"
     ];
-    const workspaceId = createHash("sha256").update(process.cwd()).digest("hex").slice(0, 16);
+    const workspaceId = createHash2("sha256").update(process.cwd()).digest("hex").slice(0, 16);
     return { rows, headers, settingsKey: `bosun:test-settings:${workspaceId}:${agent}` };
   });
 }
@@ -999,6 +1024,22 @@ async function patchTest(ctx, agent, id, patch) {
     return updated;
   });
 }
+async function patchTestConversationIfCurrent(ctx, agent, id, expectedConversation, conversation, allowedSourceFingerprints) {
+  return withSuiteMutationLock(agent, async () => {
+    const suite = await readSuite(ctx, agent);
+    const current = suite.tests.find((row) => row.id === id);
+    if (!current) return { status: "not-found" };
+    if (current.conversation !== expectedConversation || allowedSourceFingerprints && !allowedSourceFingerprints.includes(testEditSourceFingerprint(current))) {
+      return { status: "stale", currentConversation: current.conversation };
+    }
+    const updated = preparePatch(current, { conversation });
+    await writeSuite(ctx, agent, {
+      ...suite,
+      tests: suite.tests.map((row) => row.id === id ? updated : row)
+    });
+    return { status: "changed", test: updated };
+  });
+}
 function preparePatch(current, patch) {
   const updated = {
     ...defaultTestRow,
@@ -1029,7 +1070,7 @@ async function patchTestsBatch(ctx, agent, updates) {
     return updatedRows;
   });
 }
-async function addTest(ctx, agent, seed) {
+async function addTest(ctx, agent, seed, options = {}) {
   return withSuiteMutationLock(agent, async () => {
     const suite = await loadSuiteOrEmpty(ctx, agent);
     const id = randomUUID2();
@@ -1037,17 +1078,37 @@ async function addTest(ctx, agent, seed) {
       (minimum, row) => Math.min(minimum, row.order),
       Number.POSITIVE_INFINITY
     );
+    let existingTests = suite.tests;
+    let order = Number.isFinite(firstOrder) ? firstOrder - 1024 : 0;
+    if (options.beforeTestId) {
+      const beforeTest = suite.tests.find((row) => row.id === options.beforeTestId);
+      if (!beforeTest) throw new Error(`Test '${options.beforeTestId}' was not found`);
+      const stepTests = suite.tests.filter((row) => row.step === beforeTest.step);
+      const beforeIndex = stepTests.findIndex((row) => row.id === beforeTest.id);
+      const previousTest = stepTests[beforeIndex - 1];
+      const candidate = previousTest ? previousTest.order + (beforeTest.order - previousTest.order) / 2 : beforeTest.order - 1024;
+      if (Number.isFinite(candidate) && candidate < beforeTest.order && (!previousTest || candidate > previousTest.order)) {
+        order = candidate;
+      } else {
+        const rebalancedOrders = new Map(stepTests.map((row, index) => [row.id, index * 1024]));
+        existingTests = suite.tests.map((row) => {
+          const rebalancedOrder = rebalancedOrders.get(row.id);
+          return typeof rebalancedOrder === "number" ? { ...row, order: rebalancedOrder } : row;
+        });
+        order = beforeIndex === 0 ? -1024 : beforeIndex * 1024 - 512;
+      }
+    }
     const newRow = {
       ...defaultTestRow,
       id,
-      order: Number.isFinite(firstOrder) ? firstOrder - 1024 : 0,
       ...seed,
+      order,
       context: makeContext(seed?.context)
     };
     if (typeof seed?.mocks !== "undefined") {
       newRow.mocks = makeContext(seed.mocks);
     }
-    await writeSuite(ctx, agent, { tests: [newRow, ...suite.tests] });
+    await writeSuite(ctx, agent, { tests: [newRow, ...existingTests] });
     return newRow;
   });
 }
@@ -1828,6 +1889,52 @@ function envIntOrDefault(key, defaultValue, opts) {
   return Math.max(opts.min, Math.min(opts.max, rounded));
 }
 
+// src/backend/roleplay/prompt-file.ts
+import fs3 from "node:fs";
+import path8 from "node:path";
+var promptMutationQueue = /* @__PURE__ */ new Map();
+async function resolveEditablePrompt(ctx, agentId, stepName) {
+  const { basePath, steps } = await getAgentSteps(ctx, agentId);
+  const targetStep = steps.find((step) => step?.name === stepName);
+  if (!targetStep || targetStep?.prompt?.type !== "file") {
+    throw new Error(`Step '${stepName}' does not have an editable file prompt`);
+  }
+  return {
+    promptPath: path8.resolve(basePath, targetStep.prompt.path),
+    readContent: () => targetStep.prompt.content()
+  };
+}
+async function readEditablePrompt(ctx, agentId, stepName) {
+  const { readContent } = await resolveEditablePrompt(ctx, agentId, stepName);
+  return readContent();
+}
+async function withPromptFileLock(ctx, agentId, stepName, operation) {
+  const { promptPath } = await resolveEditablePrompt(ctx, agentId, stepName);
+  const canonicalPath = await fs3.promises.realpath(promptPath);
+  const previous = promptMutationQueue.get(canonicalPath) ?? Promise.resolve();
+  let release;
+  const gate = new Promise((resolve) => {
+    release = resolve;
+  });
+  const queued = previous.catch(() => {
+  }).then(() => gate);
+  promptMutationQueue.set(canonicalPath, queued);
+  await previous.catch(() => {
+  });
+  try {
+    return await operation({
+      path: canonicalPath,
+      read: () => fs3.promises.readFile(canonicalPath, "utf-8"),
+      write: (prompt) => fs3.promises.writeFile(canonicalPath, prompt, "utf-8")
+    });
+  } finally {
+    release();
+    if (promptMutationQueue.get(canonicalPath) === queued) {
+      promptMutationQueue.delete(canonicalPath);
+    }
+  }
+}
+
 // src/backend/roleplay/variants.ts
 async function generateReplyVariants(ctx, props) {
   const { conversation, count } = props;
@@ -1874,12 +1981,24 @@ function createPromptEditProposal(proposal) {
   removeExpiredPromptEditProposals();
   const stored = { ...proposal, id: randomUUID3(), createdAt: Date.now() };
   proposals.set(stored.id, stored);
-  return { ...stored };
+  return publicProposal(stored);
 }
 function getPromptEditProposal(id) {
   removeExpiredPromptEditProposals();
   const proposal = proposals.get(id);
-  return proposal ? { ...proposal } : null;
+  return proposal ? publicProposal(proposal) : null;
+}
+function getPromptEditProposalForAcceptance(id) {
+  removeExpiredPromptEditProposals();
+  const proposal = proposals.get(id);
+  return proposal ? {
+    ...proposal,
+    testSourceGuard: proposal.testSourceGuard ? {
+      suiteKey: proposal.testSourceGuard.suiteKey,
+      testId: proposal.testSourceGuard.testId,
+      allowedFingerprints: [...proposal.testSourceGuard.allowedFingerprints]
+    } : void 0
+  } : null;
 }
 function deletePromptEditProposal(id) {
   return proposals.delete(id);
@@ -1889,8 +2008,12 @@ function removeExpiredPromptEditProposals(now = Date.now()) {
     if (proposal.createdAt + PROPOSAL_TTL_MS <= now) proposals.delete(id);
   }
 }
+function publicProposal(proposal) {
+  const { testSourceGuard: _testSourceGuard, ...result } = proposal;
+  return { ...result };
+}
 
-// src/backend/roleplay/suggest-prompt-edit.ts
+// src/backend/roleplay/suggest-file-edit.ts
 import "openai";
 
 // src/backend/lib/apply-v4a-diff.ts
@@ -2026,13 +2149,12 @@ function applyChunks(input, chunks) {
   return output.join("\n");
 }
 
-// src/backend/roleplay/suggest-prompt-edit.ts
+// src/backend/roleplay/suggest-file-edit.ts
 var MODEL = "gpt-5.4";
-var VIRTUAL_PROMPT_PATH = "prompt.md";
 var MAX_PATCH_ROUNDS = 8;
-async function suggestPromptEdit(ctx, input) {
+async function suggestFileEdit(ctx, input) {
   const client = ctx.openai;
-  let virtualPrompt = input.originalPrompt;
+  let virtualContent = input.originalContent;
   let activity = 0;
   let modelRound = 0;
   let reasoningSummary = "";
@@ -2048,22 +2170,23 @@ async function suggestPromptEdit(ctx, input) {
     });
   };
   let response = await createResponse(client, {
-    input: buildPrompt(input.request, input.originalPrompt),
+    input: input.request,
+    instructions: input.instructions,
     onEvent: handleEvent
   });
   for (let patchRound = 0; patchRound < MAX_PATCH_ROUNDS; patchRound += 1) {
     const patchCalls = response.output.filter(
       (item) => item?.type === "apply_patch_call"
     );
-    if (!patchCalls.length) return virtualPrompt;
+    if (!patchCalls.length) return virtualContent;
     const outputs = patchCalls.map((call) => {
       try {
-        virtualPrompt = applyPromptOperation(virtualPrompt, call.operation);
+        virtualContent = applyFileOperation(virtualContent, input.virtualPath, call.operation);
         return {
           type: "apply_patch_call_output",
           call_id: call.call_id,
           status: "completed",
-          output: `Updated ${VIRTUAL_PROMPT_PATH} in memory`
+          output: `Updated ${input.virtualPath} in memory`
         };
       } catch (error) {
         return {
@@ -2080,29 +2203,20 @@ async function suggestPromptEdit(ctx, input) {
     response = await createResponse(client, {
       previousResponseId: response.id,
       input: outputs,
+      instructions: input.instructions,
       onEvent: handleEvent
     });
   }
-  throw new Error("The model did not finish editing the prompt");
-}
-function buildPrompt(request, originalPrompt) {
-  return `The user is asking you to modify an LLM prompt given below. Try and make minimal changes. You are not allowed to change the shape of variables, their names or their set - this is set outside. Note that this is a Nunjucks template and it will be rendered according to context. You can change/append the template logic if necessary.
-
-This is their request:
-${request}
-
-This is the original prompt:
-${originalPrompt}
-`;
+  throw new Error(`The model did not finish editing ${input.virtualPath}`);
 }
 async function createResponse(client, input) {
   const stream = await client.responses.create({
     model: MODEL,
-    instructions: "The original prompt is the existing virtual file prompt.md. Use the Apply Patch tool to make the requested edit to that file. Do not create, delete, or modify any other file.",
+    instructions: input.instructions,
     input: input.input,
     ...input.previousResponseId ? { previous_response_id: input.previousResponseId } : {},
     tools: [{ type: "apply_patch" }],
-    reasoning: { effort: "xhigh", summary: "auto" },
+    reasoning: { effort: "high", summary: "auto" },
     text: { verbosity: "low" },
     stream: true
   });
@@ -2111,14 +2225,14 @@ async function createResponse(client, input) {
     input.onEvent(event);
     if (event?.type === "response.completed") completedResponse = event.response;
     if (event?.type === "response.failed") {
-      throw new Error(event.response?.error?.message || "OpenAI failed to suggest prompt edits");
+      throw new Error(event.response?.error?.message || "OpenAI failed to suggest edits");
     }
     if (event?.type === "response.incomplete") {
-      throw new Error("OpenAI returned an incomplete prompt edit");
+      throw new Error("OpenAI returned an incomplete edit");
     }
     if (event?.type === "error") throw new Error(event.message || "OpenAI streaming failed");
   }
-  if (!completedResponse) throw new Error("OpenAI stream ended before the prompt edit completed");
+  if (!completedResponse) throw new Error("OpenAI stream ended before the edit completed");
   return completedResponse;
 }
 function progressPhase(event, round) {
@@ -2127,18 +2241,140 @@ function progressPhase(event, round) {
   if (eventType.includes("apply_patch") || itemType === "apply_patch_call") return "drafting";
   return round > 0 ? "reviewing" : "thinking";
 }
-function applyPromptOperation(currentPrompt, operation) {
+function applyFileOperation(currentContent, virtualPath, operation) {
   const normalizedPath = String(operation.path || "").replace(/\\/g, "/").replace(/^\.\//, "").replace(/^\//, "");
-  if (normalizedPath !== VIRTUAL_PROMPT_PATH) {
-    throw new Error(`Only ${VIRTUAL_PROMPT_PATH} may be edited`);
+  if (normalizedPath !== virtualPath) {
+    throw new Error(`Only ${virtualPath} may be edited`);
   }
   if (operation.type !== "update_file") {
-    throw new Error(`${VIRTUAL_PROMPT_PATH} already exists and may only be updated`);
+    throw new Error(`${virtualPath} already exists and may only be updated`);
   }
   if (typeof operation.diff !== "string" || !operation.diff.trim()) {
     throw new Error("Apply Patch returned an empty diff");
   }
-  return applyV4ADiff(currentPrompt, operation.diff);
+  return applyV4ADiff(currentContent, operation.diff);
+}
+
+// src/backend/roleplay/suggest-prompt-edit.ts
+var VIRTUAL_PROMPT_PATH = "prompt.md";
+async function suggestPromptEdit(ctx, input) {
+  return suggestFileEdit(ctx, {
+    request: buildPrompt(input.request, input.originalPrompt),
+    originalContent: input.originalPrompt,
+    virtualPath: VIRTUAL_PROMPT_PATH,
+    instructions: "The original prompt is the existing virtual file prompt.md. Use the Apply Patch tool to make the requested edit to that file. Do not create, delete, or modify any other file.",
+    onProgress: input.onProgress
+  });
+}
+function buildPrompt(request, originalPrompt) {
+  return `The user is asking you to modify an LLM prompt given below. Try and make minimal changes. You are not allowed to change the shape of variables, their names or their set - this is set outside. Note that this is a Nunjucks template and it will be rendered according to context. You can change/append the template logic if necessary.
+
+This is their request:
+${request}
+
+This is the original prompt:
+${originalPrompt}
+`;
+}
+
+// src/lib/edit/test-ai-edit.ts
+function formatTestFixAssertion(label, value) {
+  const normalizedLabel = label.trim() === "LLM" ? "" : label.trim();
+  return [normalizedLabel, value.trim()].filter(Boolean).join(" ");
+}
+function buildTestFixRequest(input) {
+  const assertion = formatTestFixAssertion(input.assertionLabel, input.assertionValue);
+  return `You will be given the context, the conversation and the desired output and assertion about it. First, find why the prompt is not producing the correct result. It's usually one concrete instruction within the relevant section, or a contradiction. Then, change the prompt minimally so this will work, but keeping the prompt clean and comprehensive is more important. Try to provide as general solution as possible. Try to change just one part of the prompt, not multiple parts. Avoid adding exceptions and overrides. Avoid duplicating information that already exists in the prompt.
+
+Context:
+${input.context.trim()}
+
+Conversation:
+${input.conversation.trim()}
+
+Desired outcome:
+${input.expectedOutput.trim()}
+
+Assertion about outcome:
+${assertion}
+
+Current incorrect outcome:
+${input.currentIncorrectOutcome.trim()}`;
+}
+function buildPromptFixRequest(enteredRequest, input, originalConversation) {
+  const canonicalOriginal = buildTestFixRequest({ ...input, conversation: originalConversation });
+  const canonicalCorrected = buildTestFixRequest(input);
+  if (enteredRequest.trim() === canonicalOriginal.trim()) return canonicalCorrected;
+  return `${enteredRequest.trim()}
+
+The conversation history has now been checked and may have been corrected. Use the following test case as authoritative instead of any earlier version included above:
+
+${canonicalCorrected}`;
+}
+
+// src/backend/tests/suggest-test-history-edit.ts
+var VIRTUAL_HISTORY_PATH = "history.md";
+async function suggestTestHistoryEdit(ctx, input) {
+  return suggestFileEdit(ctx, {
+    request: buildTestHistoryPrompt(input),
+    originalContent: input.conversation,
+    virtualPath: VIRTUAL_HISTORY_PATH,
+    instructions: "The conversation history is the existing virtual file history.md. Decide whether it conflicts with the current LLM prompt or contains agent behaviour that the current prompt would not produce. If it does, use the Apply Patch tool to update history.md minimally. Otherwise, do not call Apply Patch. Preserve the test intent and never add or change the expected outcome. Do not create, delete, or modify any other file.",
+    onProgress: input.onProgress
+  });
+}
+function buildTestHistoryPrompt(input) {
+  return `The user wants you to update an LLM test's conversation history when needed. The history must reflect behaviour that is valid under the current prompt. Make the smallest useful change. The expected outcome is immutable and is provided only so you can preserve the test's intent.
+
+This is the user's request:
+${input.request}
+
+This is the current LLM prompt:
+${input.prompt}
+
+This is the test context:
+${input.context}
+
+This is the conversation history in history.md:
+${input.conversation}
+
+This is the immutable expected outcome:
+${input.expectedOutput}
+`;
+}
+
+// src/backend/roleplay/suggest-test-fix.ts
+var defaultDependencies = {
+  suggestHistory: suggestTestHistoryEdit,
+  suggestPrompt: suggestPromptEdit
+};
+async function suggestTestFix(ctx, input, dependencies = defaultDependencies) {
+  const suggestedConversation = await dependencies.suggestHistory(ctx, {
+    request: input.request,
+    prompt: input.originalPrompt,
+    context: input.context,
+    conversation: input.conversation,
+    expectedOutput: input.expectedOutput,
+    onProgress: input.onProgress
+  });
+  const promptRequest = buildPromptFixRequest(
+    input.request,
+    {
+      context: input.context,
+      conversation: suggestedConversation,
+      expectedOutput: input.expectedOutput,
+      assertionLabel: input.assertionLabel,
+      assertionValue: input.assertionValue,
+      currentIncorrectOutcome: input.currentIncorrectOutcome
+    },
+    input.conversation
+  );
+  const suggestedPrompt = await dependencies.suggestPrompt(ctx, {
+    request: promptRequest,
+    originalPrompt: input.originalPrompt,
+    onProgress: input.onProgress
+  });
+  return { suggestedConversation, suggestedPrompt };
 }
 
 // src/backend/roleplay/prompt-edit-progress.ts
@@ -2159,6 +2395,107 @@ function removeExpiredProgress(now = Date.now()) {
   }
 }
 
+// src/backend/tests/test-history-edit-proposals.ts
+import { randomUUID as randomUUID4 } from "node:crypto";
+var proposals2 = /* @__PURE__ */ new Map();
+var PROPOSAL_TTL_MS2 = 12 * 60 * 60 * 1e3;
+function createTestHistoryEditProposal(proposal) {
+  removeExpiredTestHistoryEditProposals();
+  const stored = { ...proposal, id: randomUUID4(), createdAt: Date.now() };
+  proposals2.set(stored.id, stored);
+  return publicProposal2(stored);
+}
+function getTestHistoryEditProposalForAcceptance(id) {
+  removeExpiredTestHistoryEditProposals();
+  const proposal = proposals2.get(id);
+  return proposal ? {
+    ...proposal,
+    sourceGuard: {
+      suiteKey: proposal.sourceGuard.suiteKey,
+      allowedTestFingerprints: [...proposal.sourceGuard.allowedTestFingerprints],
+      allowedPromptContents: [...proposal.sourceGuard.allowedPromptContents]
+    }
+  } : null;
+}
+function deleteTestHistoryEditProposal(id) {
+  return proposals2.delete(id);
+}
+function removeExpiredTestHistoryEditProposals(now = Date.now()) {
+  for (const [id, proposal] of proposals2) {
+    if (proposal.createdAt + PROPOSAL_TTL_MS2 <= now) proposals2.delete(id);
+  }
+}
+function publicProposal2(proposal) {
+  const { sourceGuard: _sourceGuard, ...result } = proposal;
+  return { ...result };
+}
+
+// src/lib/tests/assertion-types.ts
+var baseAssertionTypes = [
+  { value: "llm-rubric", label: "LLM" },
+  { value: "answer-relevance", label: "Answer Relevance (LLM)" },
+  { value: "context-faithfulness", label: "Context Faithfulness (LLM)" },
+  { value: "context-recall", label: "Context Recall (LLM)" },
+  { value: "context-relevance", label: "Context Relevance (LLM)" },
+  { value: "model-graded-closedqa", label: "Model-Graded Closed QA" },
+  { value: "model-graded-factuality", label: "Model-Graded Factuality" },
+  { value: "moderation", label: "Moderation" },
+  { value: "classifier", label: "Classifier" },
+  { value: "g-eval", label: "G-Eval" },
+  { value: "factuality", label: "Factuality" },
+  { value: "conversation-relevance", label: "Conversation Relevance" },
+  { value: "equals", label: "Equals" },
+  { value: "contains", label: "Contains" },
+  { value: "icontains", label: "Contains (case-insensitive)" },
+  { value: "contains-all", label: "Contains All" },
+  { value: "contains-any", label: "Contains Any" },
+  { value: "icontains-all", label: "Contains All (iCase)" },
+  { value: "icontains-any", label: "Contains Any (iCase)" },
+  { value: "starts-with", label: "Starts With" },
+  { value: "regex", label: "Regex" },
+  { value: "levenshtein", label: "Levenshtein Distance" },
+  { value: "is-refusal", label: "Is Refusal" },
+  { value: "is-html", label: "Is HTML" },
+  { value: "contains-html", label: "Contains HTML" },
+  { value: "is-json", label: "Is JSON" },
+  { value: "contains-json", label: "Contains JSON" },
+  { value: "is-xml", label: "Is XML" },
+  { value: "contains-xml", label: "Contains XML" },
+  { value: "is-sql", label: "Is SQL" },
+  { value: "contains-sql", label: "Contains SQL" },
+  { value: "similar", label: "Semantic Similarity" },
+  { value: "rouge-n", label: "ROUGE-N" },
+  { value: "bleu", label: "BLEU" },
+  { value: "gleu", label: "GLEU" },
+  { value: "meteor", label: "METEOR" },
+  { value: "perplexity", label: "Perplexity" },
+  { value: "perplexity-score", label: "Perplexity Score" },
+  { value: "latency", label: "Latency" },
+  { value: "cost", label: "Cost" },
+  { value: "finish-reason", label: "Finish Reason" },
+  { value: "trace-error-spans", label: "Trace Error Spans" },
+  { value: "trace-span-count", label: "Trace Span Count" },
+  { value: "trace-span-duration", label: "Trace Span Duration" },
+  { value: "is-valid-function-call", label: "Is Valid Function Call" },
+  { value: "is-valid-openai-function-call", label: "Is Valid OpenAI Function Call" },
+  { value: "is-valid-openai-tools-call", label: "Is Valid OpenAI Tools Call" },
+  { value: "guardrails", label: "Guardrails" },
+  { value: "javascript", label: "JavaScript (custom)" },
+  { value: "python", label: "Python (custom)" },
+  { value: "webhook", label: "Webhook (custom)" },
+  { value: "pi", label: "Pi Scorer" }
+];
+var assertionTypes = [
+  ...baseAssertionTypes,
+  ...baseAssertionTypes.map((option) => ({
+    value: `not-${option.value}`,
+    label: `Not ${option.label}`
+  }))
+];
+function assertionLabel(value) {
+  return assertionTypes.find((option) => option.value === value)?.label ?? value;
+}
+
 // src/backend/roleplay/router.ts
 var AgentIdSchema = z4.string().trim().min(1).max(512);
 var ConversationIdSchema = z4.string().trim().min(1).max(64);
@@ -2232,7 +2569,7 @@ function createRolePlayRouter(t2) {
         prompt: z4.string().max(1e6)
       })
     ).mutation(async ({ input, ctx }) => {
-      await persistPromptUpdate(ctx, input.agentId, input.step, input.prompt);
+      await withPromptFileLock(ctx, input.agentId, input.step, (file) => file.write(input.prompt));
       return { ok: true };
     }),
     suggestPromptEdit: t2.procedure.input(
@@ -2258,12 +2595,109 @@ function createRolePlayRouter(t2) {
       });
       return { status: "changed", proposal };
     }),
+    suggestTestFix: t2.procedure.input(
+      z4.object({
+        agentId: AgentIdSchema,
+        testId: z4.string(),
+        step: z4.string().trim().min(1).max(512),
+        request: z4.string().trim().min(1).max(1e5),
+        prompt: z4.string().max(1e6),
+        basePrompt: z4.string().max(1e6),
+        currentIncorrectOutcome: z4.string().max(1e6),
+        progressId: z4.string().uuid()
+      })
+    ).mutation(async ({ input, ctx }) => {
+      const suiteKey = input.agentId;
+      const test = (await readSuite(ctx, suiteKey)).tests.find((row) => row.id === input.testId);
+      if (!test) throw new Error("This test is no longer available.");
+      await withPromptFileLock(ctx, input.agentId, input.step, async (file) => {
+        const currentPrompt = await file.read();
+        if (currentPrompt !== input.basePrompt && currentPrompt !== input.prompt) {
+          throw new Error(
+            "The prompt changed before Bosun could start this Fix. Check the prompt, then try again."
+          );
+        }
+        if (currentPrompt !== input.prompt) await file.write(input.prompt);
+      });
+      const originalPrompt = input.prompt;
+      const context = JSON.stringify(test.context ?? {});
+      const assertionLabel2 = test.evaluation_mode === "json-deep-equal" ? "" : assertionLabel(test.assertion_type);
+      const assertionValue = test.evaluation_mode === "json-deep-equal" ? "" : test.assertion_value;
+      const suggestion = await suggestTestFix(ctx, {
+        request: input.request,
+        originalPrompt,
+        context,
+        conversation: test.conversation,
+        expectedOutput: test.expected_output,
+        assertionLabel: assertionLabel2,
+        assertionValue,
+        currentIncorrectOutcome: input.currentIncorrectOutcome,
+        onProgress: (progress) => updatePromptEditProgress(input.progressId, progress)
+      });
+      const originalTestFingerprint = testEditSourceFingerprint(test);
+      const historyTestFingerprint = testEditSourceFingerprint(test, {
+        conversation: suggestion.suggestedConversation
+      });
+      const history = suggestion.suggestedConversation === test.conversation ? { status: "unchanged" } : {
+        status: "changed",
+        proposal: createTestHistoryEditProposal({
+          agentId: input.agentId,
+          testId: input.testId,
+          step: input.step,
+          originalConversation: test.conversation,
+          suggestedConversation: suggestion.suggestedConversation,
+          expectedOutput: test.expected_output,
+          sourceGuard: {
+            suiteKey,
+            allowedTestFingerprints: [originalTestFingerprint],
+            allowedPromptContents: [originalPrompt, suggestion.suggestedPrompt]
+          }
+        })
+      };
+      const prompt = suggestion.suggestedPrompt === originalPrompt ? { status: "unchanged" } : {
+        status: "changed",
+        proposal: createPromptEditProposal({
+          agentId: input.agentId,
+          step: input.step,
+          originalPrompt,
+          suggestedPrompt: suggestion.suggestedPrompt,
+          testSourceGuard: {
+            suiteKey,
+            testId: input.testId,
+            allowedFingerprints: [originalTestFingerprint, historyTestFingerprint]
+          }
+        })
+      };
+      return { history, prompt };
+    }),
     promptEditProgress: t2.procedure.input(z4.object({ id: z4.string().uuid() })).query(({ input }) => getPromptEditProgress(input.id)),
     promptEditProposal: t2.procedure.input(z4.object({ id: z4.string().uuid() })).query(({ input }) => getPromptEditProposal(input.id)),
     acceptPromptEdit: write.input(z4.object({ id: z4.string().uuid() })).mutation(async ({ input, ctx }) => {
-      const proposal = getPromptEditProposal(input.id);
+      const proposal = getPromptEditProposalForAcceptance(input.id);
       if (!proposal) throw new Error("This prompt suggestion is no longer available");
-      await persistPromptUpdate(ctx, proposal.agentId, proposal.step, proposal.suggestedPrompt);
+      await withPromptFileLock(ctx, proposal.agentId, proposal.step, async (file) => {
+        if (await file.read() !== proposal.originalPrompt) {
+          throw new Error(
+            "The prompt changed while this suggestion was open. Reject this suggestion, then ask Bosun to create a new one."
+          );
+        }
+        const guard = proposal.testSourceGuard;
+        if (!guard) {
+          await file.write(proposal.suggestedPrompt);
+          return;
+        }
+        await withSuiteMutationLock(guard.suiteKey, async () => {
+          const test = (await readSuite(ctx, guard.suiteKey)).tests.find(
+            (row) => row.id === guard.testId
+          );
+          if (!test || !guard.allowedFingerprints.includes(testEditSourceFingerprint(test))) {
+            throw new Error(
+              "This test changed while the suggestion was open. Reject this suggestion, then ask Bosun to create a new one."
+            );
+          }
+          await file.write(proposal.suggestedPrompt);
+        });
+      });
       deletePromptEditProposal(proposal.id);
       return { ok: true, prompt: proposal.suggestedPrompt };
     }),
@@ -2309,12 +2743,7 @@ function createRolePlayRouter(t2) {
         promptOverrides: input.promptOverrides ?? {}
       });
     }),
-    commitVariant: write.input(
-      z4.object({
-        conversationId: ConversationIdSchema,
-        entries: z4.array(z4.any())
-      })
-    ).mutation(async ({ input, ctx }) => {
+    commitVariant: write.input(z4.object({ conversationId: ConversationIdSchema, entries: z4.array(z4.any()) })).mutation(async ({ input, ctx }) => {
       const conversation = await loadConversation(ctx, input.conversationId);
       const turn = lastTurn(conversation);
       if (!turn) throw new Error("This conversation has no reply to replace");
@@ -2333,21 +2762,6 @@ function createRolePlayRouter(t2) {
     })
   });
 }
-async function persistPromptUpdate(ctx, agentId, stepName, prompt) {
-  const { basePath, steps } = await getAgentSteps(ctx, agentId);
-  const targetStep = steps.find((step) => step?.name === stepName);
-  if (!targetStep || targetStep?.prompt?.type !== "file") return;
-  const promptPath = path8.resolve(basePath, targetStep.prompt.path);
-  await fs3.promises.writeFile(promptPath, prompt, "utf-8");
-}
-async function readEditablePrompt(ctx, agentId, stepName) {
-  const { steps } = await getAgentSteps(ctx, agentId);
-  const targetStep = steps.find((step) => step?.name === stepName);
-  if (!targetStep || targetStep?.prompt?.type !== "file") {
-    throw new Error(`Step '${stepName}' does not have an editable file prompt`);
-  }
-  return targetStep.prompt.content();
-}
 
 // src/backend/roleplay/conversations-router.ts
 import { z as z5 } from "zod";
@@ -2405,7 +2819,7 @@ import "@recombine-ai/telescope";
 
 // src/backend/prompt-cache.ts
 import { AsyncLocalStorage } from "node:async_hooks";
-import { randomUUID as randomUUID4 } from "node:crypto";
+import { randomUUID as randomUUID5 } from "node:crypto";
 var cacheBustContext = new AsyncLocalStorage();
 function withTestPromptCacheBust(run) {
   return cacheBustContext.run(true, run);
@@ -2423,7 +2837,7 @@ function wrapCreate(resource) {
       return Reflect.apply(originalCreate, this, [body, ...args]);
     }
     return Reflect.apply(originalCreate, this, [
-      { ...body, prompt_cache_key: `bosun-test-${randomUUID4()}` },
+      { ...body, prompt_cache_key: `bosun-test-${randomUUID5()}` },
       ...args
     ]);
   };
@@ -2774,6 +3188,12 @@ function gradeJsonExpectedFields(output, expectedOutput, caseSensitive = false)
 }
 
 // src/backend/tests/academy.ts
+function applyTestHistoryOverrides(rows, overrides) {
+  return rows.map((row) => {
+    const override = overrides[typeof row.id === "string" ? row.id : ""];
+    return override ? { ...row, conversation: override.conversation } : row;
+  });
+}
 var activeRuns = new ActiveTestRuns();
 function getPromptTestsProgress(suiteKey) {
   const progress = activeRuns.getProgress(suiteKey);
@@ -2787,13 +3207,14 @@ function getPromptTestsProgress(suiteKey) {
 function cancelPromptTests(suiteKey) {
   return activeRuns.cancel(suiteKey);
 }
-async function runPromptTestsManaged(ctx, suite, selection, requestedConfig, promptOverrides = {}) {
+async function runPromptTestsManaged(ctx, suite, selection, requestedConfig, promptOverrides = {}, testHistoryOverrides = {}) {
   const run = activeRuns.start(suite.suiteKey);
   if (!run) throw new Error("Tests already running for this agent");
   try {
     return await runPromptTests(ctx, suite, selection, requestedConfig, {
       signal: run.controller.signal,
       promptOverrides,
+      testHistoryOverrides,
       setProgress: (progress) => activeRuns.setProgress(suite.suiteKey, run, progress)
     });
   } finally {
@@ -2863,12 +3284,7 @@ function toTestInputs(rows, selection = { scope: "pinned" }) {
         type: "javascript",
         value: (output) => gradeJsonExpectedFields(output, row.expected_output, row.case_sensitive === true)
       }
-    ] : [
-      {
-        type: row.assertion_type,
-        value: row.assertion_value
-      }
-    ];
+    ] : [{ type: row.assertion_type, value: row.assertion_value }];
     return {
       id: row.id,
       scenario: row.scenario,
@@ -3118,7 +3534,7 @@ async function runPromptTests(ctx, suite, selection, requestedConfig, opts) {
   const onlyTestId = "testId" in selection ? selection.testId : void 0;
   const suiteKey = suite.suiteKey;
   const loadedSuite = await readSuite(ctx, suiteKey);
-  const rows = loadedSuite.tests;
+  const rows = applyTestHistoryOverrides(loadedSuite.tests, opts?.testHistoryOverrides ?? {});
   const repeats = Math.min(500, Math.max(1, Math.floor(Number(requestedConfig.repeats) || 1)));
   const concurrency = Math.min(
     75,
@@ -3140,9 +3556,7 @@ async function runPromptTests(ctx, suite, selection, requestedConfig, opts) {
     completedRunsByTestId: /* @__PURE__ */ new Map()
   };
   if (opts?.setProgress && !opts.setProgress(progress)) throw new Error("Operation cancelled");
-  const providerInstance = new RecombineAgentProvider(ctx, {
-    id: "recombine-agent"
-  });
+  const providerInstance = new RecombineAgentProvider(ctx, { id: "recombine-agent" });
   providerInstance.setAbortSignal(signal);
   providerInstance.setOnTestStarted((testId) => {
     if (progress.completedTestIds.includes(testId)) return;
@@ -3207,10 +3621,7 @@ async function runPromptTests(ctx, suite, selection, requestedConfig, opts) {
       await rename3(tmpPath, outPath);
     }
   }
-  const runConfig = {
-    repeats,
-    concurrency
-  };
+  const runConfig = { repeats, concurrency };
   const body = {
     agent: suiteKey,
     run_timestamp: iso,
@@ -3571,7 +3982,7 @@ var PIIFindingSchema = z7.object({
 }).strict();
 
 // src/backend/tests/pii-sanitized-test-drafts.ts
-import { randomUUID as randomUUID5 } from "node:crypto";
+import { randomUUID as randomUUID6 } from "node:crypto";
 
 // src/backend/tests/pii-sanitizer.ts
 import { z as z8 } from "zod";
@@ -3795,7 +4206,7 @@ async function createPIISanitizedTestDraft(ctx, input, dependencies = {}) {
   pruneExpiredDrafts(dependencies.now?.() ?? Date.now());
   const seed = TraceTestSeedSchema.parse(input.seed);
   const sanitized = await (dependencies.sanitize ?? ((s) => sanitizeTraceTestSeed(ctx, s)))(seed);
-  const draftId = randomUUID5();
+  const draftId = randomUUID6();
   const expiresAt = (dependencies.now?.() ?? Date.now()) + DRAFT_TTL_MS;
   drafts.set(draftId, {
     suiteKey: input.suiteKey,
@@ -3863,12 +4274,7 @@ function createTestsRouter(t2) {
     latestResult: t2.procedure.input(z9.object({ agent: z9.string() })).query(async ({ input, ctx }) => loadLatestResultsByTest(ctx, input.agent)),
     runProgress: t2.procedure.input(z9.object({ agent: z9.string() })).query(async ({ input }) => getPromptTestsProgress(input.agent)),
     historyList: t2.procedure.input(z9.object({ agent: z9.string() })).query(async ({ input, ctx }) => listRunHistory(ctx, input.agent)),
-    historyRun: t2.procedure.input(
-      z9.object({
-        agent: z9.string(),
-        timestamp: z9.string()
-      })
-    ).query(async ({ input, ctx }) => getRunHistory(ctx, input.agent, input.timestamp)),
+    historyRun: t2.procedure.input(z9.object({ agent: z9.string(), timestamp: z9.string() })).query(async ({ input, ctx }) => getRunHistory(ctx, input.agent, input.timestamp)),
     resultEntry: t2.procedure.input(
       z9.object({
         agent: z9.string(),
@@ -3903,28 +4309,59 @@ function createTestsRouter(t2) {
         timestamp: input.timestamp
       });
     }),
-    findPromptIssues: t2.procedure.input(
+    findPromptIssues: t2.procedure.input(z9.object({ agent: z9.string(), step: z9.string(), prompt: z9.string() })).mutation(async ({ input, ctx }) => {
+      return findPromptIssues(ctx, input);
+    }),
+    suggestHistoryEdit: t2.procedure.input(
       z9.object({
         agent: z9.string(),
-        step: z9.string(),
-        prompt: z9.string()
+        testId: z9.string(),
+        step: z9.string().trim().min(1).max(512),
+        request: z9.string().trim().min(1).max(1e5),
+        prompt: z9.string().max(1e6),
+        basePrompt: z9.string().max(1e6),
+        progressId: z9.string().uuid()
       })
     ).mutation(async ({ input, ctx }) => {
-      return findPromptIssues(ctx, input);
+      const suiteKey = input.agent;
+      const test = (await readSuite(ctx, suiteKey)).tests.find((row) => row.id === input.testId);
+      if (!test) throw new Error("This test is no longer available.");
+      const suggestedConversation = await suggestTestHistoryEdit(ctx, {
+        request: input.request,
+        prompt: input.prompt,
+        context: JSON.stringify(test.context ?? {}, null, 2),
+        conversation: test.conversation,
+        expectedOutput: test.expected_output,
+        onProgress: (progress) => updatePromptEditProgress(input.progressId, progress)
+      });
+      return suggestedConversation === test.conversation ? { status: "unchanged" } : {
+        status: "changed",
+        proposal: createTestHistoryEditProposal({
+          agentId: input.agent,
+          testId: input.testId,
+          step: input.step,
+          originalConversation: test.conversation,
+          suggestedConversation,
+          expectedOutput: test.expected_output,
+          sourceGuard: {
+            suiteKey,
+            allowedTestFingerprints: [testEditSourceFingerprint(test)],
+            allowedPromptContents: [input.basePrompt, input.prompt]
+          }
+        })
+      };
     }),
     suite: t2.procedure.input(z9.object({ agent: z9.string() })).query(async ({ input, ctx }) => readSuite(ctx, input.agent)),
     addTest: write.input(
       z9.object({
         agent: z9.string(),
+        beforeTestId: z9.string().optional(),
         seed: TestRowSchema.omit({ id: true, order: true }).partial()
       })
-    ).mutation(async ({ input, ctx }) => addTest(ctx, input.agent, input.seed)),
-    createPIISanitizedTestDraft: t2.procedure.input(
-      z9.object({
-        agent: z9.string(),
-        seed: TraceTestSeedSchema
-      })
-    ).mutation(async ({ input, ctx }) => {
+    ).mutation(
+      async ({ input, ctx }) => addTest(ctx, input.agent, input.seed, { beforeTestId: input.beforeTestId })
+    ),
+    createPIISanitizedTestDraft: t2.procedure.input(z9.object({ agent: z9.string(), seed: TraceTestSeedSchema })).mutation(async ({ input, ctx }) => {
       try {
         const draft = await createPIISanitizedTestDraft(ctx, {
           suiteKey: input.agent,
@@ -3960,6 +4397,41 @@ function createTestsRouter(t2) {
         patch: TestRowSchema.omit({ id: true, order: true }).partial()
       })
     ).mutation(async ({ input, ctx }) => patchTest(ctx, input.agent, input.id, input.patch)),
+    acceptHistoryEdit: write.input(z9.object({ id: z9.string().uuid() })).mutation(async ({ input, ctx }) => {
+      const proposal = getTestHistoryEditProposalForAcceptance(input.id);
+      if (!proposal) throw new Error("This test history suggestion is no longer available.");
+      const result = await withPromptFileLock(
+        ctx,
+        proposal.agentId,
+        proposal.step,
+        async (file) => {
+          if (!proposal.sourceGuard.allowedPromptContents.includes(await file.read())) {
+            throw new Error(
+              "The prompt changed while this suggestion was open. Reject this suggestion, then ask Bosun to create a new one."
+            );
+          }
+          return patchTestConversationIfCurrent(
+            ctx,
+            proposal.sourceGuard.suiteKey,
+            proposal.testId,
+            proposal.originalConversation,
+            proposal.suggestedConversation,
+            proposal.sourceGuard.allowedTestFingerprints
+          );
+        }
+      );
+      if (result.status === "not-found") {
+        throw new Error("This test is no longer available.");
+      }
+      if (result.status === "stale") {
+        throw new Error(
+          "This test changed while the suggestion was open. Reject this suggestion, then ask Bosun to create a new one."
+        );
+      }
+      deleteTestHistoryEditProposal(proposal.id);
+      return { ok: true, test: result.test };
+    }),
+    rejectHistoryEdit: t2.procedure.input(z9.object({ id: z9.string().uuid() })).mutation(({ input }) => ({ ok: deleteTestHistoryEditProposal(input.id) })),
     patchTestsBatch: write.input(
       z9.object({
         agent: z9.string(),
@@ -3980,6 +4452,7 @@ function createTestsRouter(t2) {
           step: z9.string(),
           scope: z9.enum(["pinned", "all"]).optional(),
           promptOverrides: z9.record(z9.string(), z9.string().max(1e6)).optional(),
+          testHistoryOverrides: z9.record(z9.string(), z9.object({ conversation: z9.string().max(1e6) }).strict()).optional(),
           repeats: z9.number().int().min(1).max(500),
           concurrency: z9.number().int().min(1).max(75)
         }).strict(),
@@ -3987,6 +4460,7 @@ function createTestsRouter(t2) {
           agent: z9.string(),
           testId: z9.string(),
           promptOverrides: z9.record(z9.string(), z9.string().max(1e6)).optional(),
+          testHistoryOverrides: z9.record(z9.string(), z9.object({ conversation: z9.string().max(1e6) }).strict()).optional(),
           repeats: z9.number().int().min(1).max(500),
           concurrency: z9.number().int().min(1).max(75)
         }).strict()
@@ -4000,18 +4474,17 @@ function createTestsRouter(t2) {
           selection,
           { testId: input.testId },
           runConfig,
-          input.promptOverrides
+          input.promptOverrides,
+          input.testHistoryOverrides
         );
       }
       return runPromptTestsManaged(
         ctx,
         selection,
-        {
-          scope: input.scope ?? "pinned",
-          step: input.step
-        },
+        { scope: input.scope ?? "pinned", step: input.step },
         runConfig,
-        input.promptOverrides
+        input.promptOverrides,
+        input.testHistoryOverrides
       );
     }),
     cancelRun: t2.procedure.input(z9.object({ agent: z9.string() })).mutation(async ({ input }) => cancelPromptTests(input.agent))
@@ -4047,6 +4520,21 @@ var PaginatedConversationsResponseSchema = z10.object({
   items: z10.array(ConversationSchema),
   lastConversationId: z10.string().optional()
 });
+var AnalyticsScalarSchema = z10.union([z10.boolean(), z10.number(), z10.string()]);
+var AnalyticsFieldSchema = z10.object({
+  key: z10.string(),
+  type: z10.enum(["boolean", "number", "string"]),
+  variantCount: z10.number(),
+  values: z10.array(
+    z10.object({
+      value: AnalyticsScalarSchema,
+      count: z10.number()
+    })
+  )
+});
+var AnalyticsFieldsResponseSchema = z10.object({
+  items: z10.array(AnalyticsFieldSchema)
+});
 var LlmUsageSchema = z10.object({
   promptTokens: z10.number().optional(),
   completionTokens: z10.number().optional(),
@@ -4130,6 +4618,11 @@ var PaginatedLatencyTracesResponseSchema = z10.object({
 var SourceIdSchema = z11.string().trim().max(128).optional();
 var ConversationIdSchema3 = z11.string().trim().min(1).max(256);
 var LimitSchema = z11.number().int().positive().max(1e3).optional();
+var AnalyticsFilterSchema = z11.object({
+  key: z11.string().trim().min(1).max(256),
+  type: z11.enum(["boolean", "number", "string"]),
+  values: z11.array(AnalyticsScalarSchema).min(1)
+});
 var ConversationsPageSchema = PaginatedConversationsResponseSchema.extend({
   items: z11.array(ConversationSchema).optional()
 });
@@ -4160,13 +4653,20 @@ function createTelescopeRouter(t2) {
         limit: LimitSchema,
         lastConversationId: z11.string().trim().min(1).optional(),
         phoneNumber: z11.string().trim().min(1).optional(),
-        agentId: z11.string().trim().min(1).optional()
+        agentId: z11.string().trim().min(1).optional(),
+        analyticsFilters: z11.array(AnalyticsFilterSchema).optional()
       })
     ).query(async ({ input, ctx }) => {
       const search = new URLSearchParams({ limit: String(input.limit ?? 10) });
       if (input.lastConversationId) search.set("lastConversationId", input.lastConversationId);
       if (input.phoneNumber) search.set("phoneNumber", input.phoneNumber);
       if (input.agentId) search.set("agentId", input.agentId);
+      if (input.analyticsFilters?.length) {
+        search.set(
+          "filters",
+          JSON.stringify(input.analyticsFilters.map((filter) => ({ ...filter, operator: "in" })))
+        );
+      }
       const page = await get(ctx, {
         sourceId: input.sourceId,
         path: "conversations",
@@ -4182,6 +4682,16 @@ function createTelescopeRouter(t2) {
         }))
       };
     }),
+    analyticsFields: t2.procedure.input(z11.object({ sourceId: SourceIdSchema, agentId: z11.string().trim().min(1).optional() })).query(({ input, ctx }) => {
+      const search = new URLSearchParams();
+      if (input.agentId) search.set("agentId", input.agentId);
+      return get(ctx, {
+        sourceId: input.sourceId,
+        path: "analytics/fields",
+        search,
+        schema: AnalyticsFieldsResponseSchema
+      });
+    }),
     conversation: t2.procedure.input(z11.object({ sourceId: SourceIdSchema, conversationId: ConversationIdSchema3 })).query(
       ({ input, ctx }) => get(ctx, {
         sourceId: input.sourceId,