@granular-software/sdk 0.4.41 → 0.4.42

package/dist/agent-evals.mjs

@@ -1,4 +1,4 @@
-import { writeFile, mkdir } from 'fs/promises';
+import { readFile, writeFile, mkdir } from 'fs/promises';
 import path from 'path';
 import OpenAI from 'openai';
 import * as Automerge from '@automerge/automerge';
@@ -16060,7 +16060,275 @@ var Granular = class _Granular {
   }
 };
 
+// src/agent-harness-templates/action-presentation/0.1.0/manifest.json
+var manifest_default = {
+  id: "action-presentation",
+  version: "0.1.0",
+  status: "candidate",
+  owner: "granular",
+  createdAt: "2026-05-25T00:00:00.000Z",
+  changelog: "Candidate harness template focused on action-request detection and mutation result presentation. It exists to harden cases where the model answers with a terse completion such as Done instead of using tools and reporting the target/action/result.",
+  promptBuilder: "buildGranularAgentSystemPrompt:action-presentation",
+  continuationBuilder: "buildContinuationInstruction:action-presentation",
+  modelOutputInstruction: "agent-evals:modelOutputInstruction",
+  codeReviewPolicy: "reviewGeneratedJobCode",
+  defaultModel: "gpt-5.4",
+  modelMatrix: ["gpt-5.4"],
+  temperature: 0,
+  compatibility: {
+    minSdkVersion: "0.4.40",
+    capabilities: [
+      "executeCode",
+      "readEntities",
+      "workflowHelpers",
+      "savedData",
+      "showRecords"
+    ]
+  },
+  evalGates: {
+    requiredSuites: [
+      "agent-harness-hardening",
+      "agent-harness-runtime-e2e",
+      "agent-production-readiness-e2e"
+    ],
+    criticalBuckets: [
+      "relationship-traversal",
+      "cross-turn-reference",
+      "ambiguous-target-choice",
+      "confirmation-gated-mutation",
+      "denied-action-refusal",
+      "mutation-presentation"
+    ],
+    maxRegressionPct: 0,
+    minPassK: {
+      "critical-mutation": 0.9,
+      "permission-boundary": 1
+    }
+  }
+};
+
+// src/agent-harness-templates/experimental-compact/0.1.0/manifest.json
+var manifest_default2 = {
+  id: "experimental-compact",
+  version: "0.1.0",
+  status: "candidate",
+  owner: "granular",
+  createdAt: "2026-05-24T00:00:00.000Z",
+  changelog: "Candidate harness template used for champion/challenger testing. It adds a compact decision discipline section on top of the stable renderer.",
+  promptBuilder: "buildGranularAgentSystemPrompt:experimental-compact",
+  continuationBuilder: "buildContinuationInstruction:experimental-compact",
+  modelOutputInstruction: "agent-evals:modelOutputInstruction",
+  codeReviewPolicy: "reviewGeneratedJobCode",
+  defaultModel: "gpt-5.4",
+  modelMatrix: ["gpt-5.4"],
+  temperature: 0,
+  compatibility: {
+    minSdkVersion: "0.4.40",
+    capabilities: [
+      "executeCode",
+      "readEntities",
+      "workflowHelpers",
+      "savedData",
+      "showRecords"
+    ]
+  },
+  evalGates: {
+    requiredSuites: [
+      "agent-harness-hardening",
+      "agent-harness-runtime-e2e",
+      "agent-production-readiness-e2e"
+    ],
+    criticalBuckets: [
+      "relationship-traversal",
+      "cross-turn-reference",
+      "ambiguous-target-choice",
+      "confirmation-gated-mutation",
+      "denied-action-refusal",
+      "mutation-presentation"
+    ],
+    maxRegressionPct: 0,
+    minPassK: {
+      "critical-mutation": 0.9,
+      "permission-boundary": 1
+    }
+  }
+};
+
+// src/agent-harness-templates/stable/1.0.0/manifest.json
+var manifest_default3 = {
+  id: "stable",
+  version: "1.0.0",
+  status: "stable",
+  owner: "granular",
+  createdAt: "2026-05-24T00:00:00.000Z",
+  changelog: "Baseline template wrapping the existing Granular agent harness prompt, continuation instruction, output contract, and generated-job review policy.",
+  promptBuilder: "buildGranularAgentSystemPrompt",
+  continuationBuilder: "buildContinuationInstruction",
+  modelOutputInstruction: "agent-evals:modelOutputInstruction",
+  codeReviewPolicy: "reviewGeneratedJobCode",
+  defaultModel: "gpt-5.4",
+  modelMatrix: ["gpt-5.4"],
+  temperature: 0,
+  compatibility: {
+    minSdkVersion: "0.4.40",
+    capabilities: [
+      "executeCode",
+      "readEntities",
+      "workflowHelpers",
+      "savedData",
+      "showRecords"
+    ]
+  },
+  evalGates: {
+    requiredSuites: [
+      "agent-harness-hardening",
+      "agent-harness-runtime-e2e",
+      "agent-production-readiness-e2e"
+    ],
+    criticalBuckets: [
+      "relationship-traversal",
+      "cross-turn-reference",
+      "ambiguous-target-choice",
+      "confirmation-gated-mutation",
+      "denied-action-refusal",
+      "mutation-presentation"
+    ],
+    maxRegressionPct: 0,
+    minPassK: {
+      "critical-mutation": 0.9,
+      "permission-boundary": 1
+    }
+  }
+};
+
 // src/agent-harness.ts
+var HARNESS_TEMPLATE_STATUSES = [
+  "draft",
+  "candidate",
+  "release-candidate",
+  "stable",
+  "deprecated"
+];
+function requireRecord(value, context) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error(`${context} must be an object.`);
+  }
+  return value;
+}
+function requiredString(record, key, context) {
+  const value = record[key];
+  if (typeof value !== "string" || !value.trim()) {
+    throw new Error(`${context}.${key} must be a non-empty string.`);
+  }
+  return value;
+}
+function requiredNumber(record, key, context) {
+  const value = record[key];
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    throw new Error(`${context}.${key} must be a finite number.`);
+  }
+  return value;
+}
+function requiredStringArray(record, key, context) {
+  const value = record[key];
+  if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || !item.trim())) {
+    throw new Error(`${context}.${key} must be an array of non-empty strings.`);
+  }
+  return [...value];
+}
+function requiredNumberRecord(record, key, context) {
+  const value = requireRecord(record[key], `${context}.${key}`);
+  const output = {};
+  for (const [entryKey, entryValue] of Object.entries(value)) {
+    if (typeof entryValue !== "number" || !Number.isFinite(entryValue) || entryValue < 0 || entryValue > 1) {
+      throw new Error(
+        `${context}.${key}.${entryKey} must be a number between 0 and 1.`
+      );
+    }
+    output[entryKey] = entryValue;
+  }
+  return output;
+}
+function validateHarnessTemplateManifest(value, context = "HarnessTemplateManifest") {
+  const record = requireRecord(value, context);
+  const status = requiredString(record, "status", context);
+  if (!HARNESS_TEMPLATE_STATUSES.includes(status)) {
+    throw new Error(
+      `${context}.status must be one of ${HARNESS_TEMPLATE_STATUSES.join(", ")}.`
+    );
+  }
+  const createdAt = requiredString(record, "createdAt", context);
+  if (Number.isNaN(Date.parse(createdAt))) {
+    throw new Error(`${context}.createdAt must be an ISO timestamp.`);
+  }
+  const compatibility = requireRecord(
+    record.compatibility,
+    `${context}.compatibility`
+  );
+  const evalGates = requireRecord(record.evalGates, `${context}.evalGates`);
+  const maxRegressionPct = requiredNumber(
+    evalGates,
+    "maxRegressionPct",
+    `${context}.evalGates`
+  );
+  if (maxRegressionPct < 0 || maxRegressionPct > 1) {
+    throw new Error(
+      `${context}.evalGates.maxRegressionPct must be between 0 and 1.`
+    );
+  }
+  return {
+    id: requiredString(record, "id", context),
+    version: requiredString(record, "version", context),
+    status,
+    owner: requiredString(record, "owner", context),
+    createdAt,
+    changelog: requiredString(record, "changelog", context),
+    promptBuilder: requiredString(record, "promptBuilder", context),
+    continuationBuilder: requiredString(record, "continuationBuilder", context),
+    modelOutputInstruction: requiredString(
+      record,
+      "modelOutputInstruction",
+      context
+    ),
+    codeReviewPolicy: requiredString(record, "codeReviewPolicy", context),
+    defaultModel: requiredString(record, "defaultModel", context),
+    modelMatrix: requiredStringArray(record, "modelMatrix", context),
+    temperature: requiredNumber(record, "temperature", context),
+    compatibility: {
+      minSdkVersion: requiredString(
+        compatibility,
+        "minSdkVersion",
+        `${context}.compatibility`
+      ),
+      capabilities: requiredStringArray(
+        compatibility,
+        "capabilities",
+        `${context}.compatibility`
+      )
+    },
+    evalGates: {
+      requiredSuites: requiredStringArray(
+        evalGates,
+        "requiredSuites",
+        `${context}.evalGates`
+      ),
+      criticalBuckets: requiredStringArray(
+        evalGates,
+        "criticalBuckets",
+        `${context}.evalGates`
+      ),
+      maxRegressionPct,
+      minPassK: requiredNumberRecord(
+        evalGates,
+        "minPassK",
+        `${context}.evalGates`
+      )
+    }
+  };
+}
+function defineHarnessTemplateManifest(value, context) {
+  return validateHarnessTemplateManifest(value, context);
+}
 function asRecord4(value) {
   if (!value || typeof value !== "object" || Array.isArray(value)) return null;
   return value;
@@ -16084,6 +16352,16 @@ function uniqueStrings(values, maxCount) {
   }
   return output;
 }
+function stableStringify(value) {
+  if (value === null || typeof value !== "object") {
+    return JSON.stringify(value);
+  }
+  if (Array.isArray(value)) {
+    return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;
+  }
+  const record = value;
+  return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`).join(",")}}`;
+}
 function renderConstBlock(name, value) {
   return `const ${name} = ${JSON.stringify(value, null, 2)} as const;`;
 }
@@ -18023,6 +18301,156 @@ ${knownFactsBlock}
 [Request]
 ${input.request?.trim() || "Use the latest user message in the conversation."}`;
 }
+var STABLE_AGENT_HARNESS_TEMPLATE_MANIFEST = defineHarnessTemplateManifest(
+  manifest_default3,
+  "stable@1.0.0 manifest"
+);
+function hashHarnessTemplateValue(value) {
+  return hashString(stableStringify(value)) || "00000000";
+}
+function renderStableHarnessPrompt(input) {
+  const prompt = buildGranularAgentSystemPrompt(input);
+  const templateHash = hashHarnessTemplateValue(
+    STABLE_AGENT_HARNESS_TEMPLATE_MANIFEST
+  );
+  return {
+    templateId: STABLE_AGENT_HARNESS_TEMPLATE_MANIFEST.id,
+    templateVersion: STABLE_AGENT_HARNESS_TEMPLATE_MANIFEST.version,
+    templateHash,
+    promptInstanceHash: hashHarnessTemplateValue({
+      templateHash,
+      input,
+      prompt
+    }),
+    prompt
+  };
+}
+function renderStableHarnessContinuation(resultPreview) {
+  const instruction = buildContinuationInstruction(resultPreview);
+  const templateHash = hashHarnessTemplateValue(
+    STABLE_AGENT_HARNESS_TEMPLATE_MANIFEST
+  );
+  return {
+    templateId: STABLE_AGENT_HARNESS_TEMPLATE_MANIFEST.id,
+    templateVersion: STABLE_AGENT_HARNESS_TEMPLATE_MANIFEST.version,
+    templateHash,
+    instruction
+  };
+}
+var STABLE_AGENT_HARNESS_TEMPLATE = {
+  manifest: STABLE_AGENT_HARNESS_TEMPLATE_MANIFEST,
+  renderPrompt: renderStableHarnessPrompt,
+  renderContinuation: renderStableHarnessContinuation
+};
+var EXPERIMENTAL_COMPACT_AGENT_HARNESS_TEMPLATE_MANIFEST = defineHarnessTemplateManifest(
+  manifest_default2,
+  "experimental-compact@0.1.0 manifest"
+);
+function renderExperimentalCompactHarnessPrompt(input) {
+  const prompt = `${buildGranularAgentSystemPrompt(input)}
+
+[Candidate Harness Delta: Compact Decision Discipline]
+- Prefer the smallest action that satisfies the current request.
+- When several records could match, ask one structured choice question before mutating.
+- Before any irreversible or outbound mutation, obtain explicit confirmation unless the policy surface already requires it.
+- Do not compensate for missing tools with raw network calls, synthetic records, or hidden side channels.
+- Keep the final user-facing reply focused on what was done, what was not done, and any remaining blocker.`;
+  const templateHash = hashHarnessTemplateValue(
+    EXPERIMENTAL_COMPACT_AGENT_HARNESS_TEMPLATE_MANIFEST
+  );
+  return {
+    templateId: EXPERIMENTAL_COMPACT_AGENT_HARNESS_TEMPLATE_MANIFEST.id,
+    templateVersion: EXPERIMENTAL_COMPACT_AGENT_HARNESS_TEMPLATE_MANIFEST.version,
+    templateHash,
+    promptInstanceHash: hashHarnessTemplateValue({
+      templateHash,
+      input,
+      prompt
+    }),
+    prompt
+  };
+}
+function renderExperimentalCompactHarnessContinuation(resultPreview) {
+  const instruction = `${buildContinuationInstruction(resultPreview)}
+
+Keep the continuation compact: either finish, ask the one blocking question, or run the next smallest safe action.`;
+  const templateHash = hashHarnessTemplateValue(
+    EXPERIMENTAL_COMPACT_AGENT_HARNESS_TEMPLATE_MANIFEST
+  );
+  return {
+    templateId: EXPERIMENTAL_COMPACT_AGENT_HARNESS_TEMPLATE_MANIFEST.id,
+    templateVersion: EXPERIMENTAL_COMPACT_AGENT_HARNESS_TEMPLATE_MANIFEST.version,
+    templateHash,
+    instruction
+  };
+}
+var EXPERIMENTAL_COMPACT_AGENT_HARNESS_TEMPLATE = {
+  manifest: EXPERIMENTAL_COMPACT_AGENT_HARNESS_TEMPLATE_MANIFEST,
+  renderPrompt: renderExperimentalCompactHarnessPrompt,
+  renderContinuation: renderExperimentalCompactHarnessContinuation
+};
+var ACTION_PRESENTATION_AGENT_HARNESS_TEMPLATE_MANIFEST = defineHarnessTemplateManifest(
+  manifest_default,
+  "action-presentation@0.1.0 manifest"
+);
+function renderActionPresentationHarnessPrompt(input) {
+  const prompt = `${buildGranularAgentSystemPrompt(input)}
+
+[Candidate Harness Delta: Action Request And Result Presentation]
+- Treat requests like "handle it", "take care of it", "send it", "update it", "fix it", "process it", or "do it" as action requests when the message names a business object, target, workflow, or mutation verb.
+- For action requests involving session data, generated files, records, effects, or workflows, do not use text-only completion. Generate and run code, ask the blocking human question, or clearly refuse if policy/tooling prevents the action.
+- Never answer only "Done", "OK", "Handled", or similar terse completion text for a mutation request. The final user-facing reply must name the action attempted, the grounded target or blocker, and the actual result.
+- After a mutation/effect call, base the reply on the returned action result and include a visible target label or identifier when one exists.
+- If no target can be grounded, say what was searched and what exact identifier or choice is needed; do not pretend the action completed.`;
+  const templateHash = hashHarnessTemplateValue(
+    ACTION_PRESENTATION_AGENT_HARNESS_TEMPLATE_MANIFEST
+  );
+  return {
+    templateId: ACTION_PRESENTATION_AGENT_HARNESS_TEMPLATE_MANIFEST.id,
+    templateVersion: ACTION_PRESENTATION_AGENT_HARNESS_TEMPLATE_MANIFEST.version,
+    templateHash,
+    promptInstanceHash: hashHarnessTemplateValue({
+      templateHash,
+      input,
+      prompt
+    }),
+    prompt
+  };
+}
+function renderActionPresentationHarnessContinuation(resultPreview) {
+  const instruction = `${buildContinuationInstruction(resultPreview)}
+
+Before finishing, check whether the latest user request asked for an action. If it did, do not finish with a bare completion token; either continue with the needed tool/code step, ask the blocking question, refuse with the policy reason, or report the grounded action result with the target label.`;
+  const templateHash = hashHarnessTemplateValue(
+    ACTION_PRESENTATION_AGENT_HARNESS_TEMPLATE_MANIFEST
+  );
+  return {
+    templateId: ACTION_PRESENTATION_AGENT_HARNESS_TEMPLATE_MANIFEST.id,
+    templateVersion: ACTION_PRESENTATION_AGENT_HARNESS_TEMPLATE_MANIFEST.version,
+    templateHash,
+    instruction
+  };
+}
+var ACTION_PRESENTATION_AGENT_HARNESS_TEMPLATE = {
+  manifest: ACTION_PRESENTATION_AGENT_HARNESS_TEMPLATE_MANIFEST,
+  renderPrompt: renderActionPresentationHarnessPrompt,
+  renderContinuation: renderActionPresentationHarnessContinuation
+};
+var AGENT_HARNESS_TEMPLATES = {
+  [STABLE_AGENT_HARNESS_TEMPLATE.manifest.id]: STABLE_AGENT_HARNESS_TEMPLATE,
+  [EXPERIMENTAL_COMPACT_AGENT_HARNESS_TEMPLATE.manifest.id]: EXPERIMENTAL_COMPACT_AGENT_HARNESS_TEMPLATE,
+  [ACTION_PRESENTATION_AGENT_HARNESS_TEMPLATE.manifest.id]: ACTION_PRESENTATION_AGENT_HARNESS_TEMPLATE
+};
+function resolveHarnessTemplate(templateId = "stable", options) {
+  const resolved = AGENT_HARNESS_TEMPLATES[templateId];
+  if (resolved) return resolved;
+  {
+    const known = Object.keys(AGENT_HARNESS_TEMPLATES).join(", ");
+    throw new Error(
+      `Unknown harness template "${templateId}". Known templates: ${known}`
+    );
+  }
+}
 
 // src/openai-usage.ts
 var OPENAI_PRICING_SOURCE_URL = "https://developers.openai.com/api/docs/models/gpt-5.4/";
@@ -18144,6 +18572,98 @@ function matchesPattern(text, matcher) {
   if (typeof matcher === "string") return text.includes(matcher);
   return matcher.test(text);
 }
+function matcherFromConfig(value) {
+  const match = value.match(/^\/([\s\S]*)\/([dgimsuvy]*)$/);
+  if (!match) return value;
+  return new RegExp(match[1] || "", match[2] || "");
+}
+function matchersFromConfig(values) {
+  if (!values?.length) return void 0;
+  return values.map((value) => matcherFromConfig(value));
+}
+function expectationsFromConfig(input) {
+  if (!input) return void 0;
+  return {
+    replyIncludes: matchersFromConfig(input.replyIncludes),
+    replyExcludes: matchersFromConfig(input.replyExcludes),
+    actionIncludes: matchersFromConfig(input.actionIncludes),
+    actionExcludes: matchersFromConfig(input.actionExcludes),
+    codeIncludes: matchersFromConfig(input.codeIncludes),
+    codeExcludes: matchersFromConfig(input.codeExcludes),
+    behaviorBuckets: input.behaviorBuckets,
+    actions: input.actions ? {
+      required: matchersFromConfig(input.actions.required),
+      forbidden: matchersFromConfig(input.actions.forbidden)
+    } : void 0,
+    prompts: input.prompts ? {
+      ...input.prompts,
+      requiredConfirmationBefore: matchersFromConfig(
+        input.prompts.requiredConfirmationBefore
+      )
+    } : void 0,
+    presentation: input.presentation ? {
+      mustMention: matchersFromConfig(input.presentation.mustMention),
+      mustNotMention: matchersFromConfig(input.presentation.mustNotMention),
+      mustDisplayOrSave: matchersFromConfig(
+        input.presentation.mustDisplayOrSave
+      )
+    } : void 0
+  };
+}
+function promptResponderFromConfig(rules) {
+  if (!rules?.length) return void 0;
+  return createHumanResponder(
+    rules.map((rule) => ({
+      type: rule.type,
+      when: Array.isArray(rule.when) ? rule.when.map((matcher) => matcherFromConfig(matcher)) : rule.when ? matcherFromConfig(rule.when) : void 0,
+      answer: rule.answer
+    }))
+  );
+}
+function inspectionFromConfig(inspection) {
+  if (!inspection) return void 0;
+  return {
+    code: inspection.code,
+    includes: matchersFromConfig(inspection.includes),
+    excludes: matchersFromConfig(inspection.excludes)
+  };
+}
+function inspectionsFromConfig(inspections) {
+  if (!inspections) return void 0;
+  if (Array.isArray(inspections)) {
+    return inspections.map((inspection) => inspectionFromConfig(inspection)).filter(
+      (inspection) => Boolean(inspection)
+    );
+  }
+  return inspectionFromConfig(inspections);
+}
+function scenariosFromAgentEvalFile(file) {
+  return file.scenarios.map((scenario) => ({
+    id: scenario.id,
+    description: scenario.description,
+    request: scenario.request,
+    behaviorBuckets: scenario.behaviorBuckets,
+    human: promptResponderFromConfig(scenario.human),
+    prepareRecords: scenario.prepareRecords,
+    expect: expectationsFromConfig(scenario.expect),
+    verify: inspectionFromConfig(scenario.verify),
+    inspect: inspectionsFromConfig(scenario.inspect),
+    steps: scenario.steps?.map((step) => ({
+      id: step.id,
+      request: step.request,
+      behaviorBuckets: step.behaviorBuckets || scenario.behaviorBuckets,
+      human: promptResponderFromConfig(step.human || scenario.human),
+      expect: expectationsFromConfig(step.expect || scenario.expect),
+      inspect: inspectionsFromConfig(step.inspect)
+    }))
+  }));
+}
+async function loadAgentEvalScenarioFile(filePath) {
+  const raw = JSON.parse(
+    await readFile(filePath, "utf8")
+  );
+  return scenariosFromAgentEvalFile(raw);
+}
 function assertMatches(label, text, includes = [], excludes = []) {
   for (const matcher of includes) {
     if (!matchesPattern(text, matcher)) {
@@ -18176,6 +18696,23 @@ function asArray3(value) {
   if (!value) return [];
   return Array.isArray(value) ? value : [value];
 }
+function progressEvent(input) {
+  return {
+    ...input,
+    id: input.id || [
+      input.scenarioId || "suite",
+      input.stepId || input.phase,
+      input.iteration ? `iteration-${input.iteration}` : "",
+      input.jobId || "",
+      input.title,
+      Date.now()
+    ].filter(Boolean).join(":"),
+    timestamp: (/* @__PURE__ */ new Date()).toISOString()
+  };
+}
+async function emitProgress(handler, event) {
+  await handler?.(progressEvent(event));
+}
 var GPT_54_TOKEN_PRICING_USD_PER_MILLION = {
   input: 2.5,
   cachedInput: 0.25,
@@ -18418,7 +18955,7 @@ function filterPromptsByBoundary(liveDoc, prompts, boundaryTimestamp) {
   });
 }
 function createScriptedPromptResponder(rules, fallback) {
-  return async ({ prompt, history }) => {
+  const responder = async ({ prompt, history }) => {
     const promptText = `${prompt.title || ""}
 ${prompt.message || ""}`;
     for (const rule of rules) {
@@ -18435,6 +18972,11 @@ ${prompt.message || ""}`;
       `No scripted prompt responder matched prompt ${prompt.id}: ${promptText}`
     );
   };
+  Object.defineProperty(responder, "__granularScriptedPromptRules", {
+    enumerable: false,
+    value: rules
+  });
+  return responder;
 }
 function extractJsonObject(text) {
   const start = text.indexOf("{");
@@ -18721,6 +19263,22 @@ ${checkpoint.latestJobResult}` : null
   ];
   return lines.filter(Boolean).join("\n\n");
 }
+function readableAgentMessage(message) {
+  const record = asRecord6(message);
+  if (!record) return JSON.stringify(message);
+  if (typeof record.reply === "string" && record.reply.trim()) {
+    return record.reply;
+  }
+  const show = asRecord6(record.show);
+  const variableNames = asArray3(show?.variableNames).map((value) => String(value)).filter(Boolean);
+  if (variableNames.length) {
+    return `Displayed ${variableNames.join(", ")}`;
+  }
+  if (typeof record.kind === "string") {
+    return `Agent ${record.kind} message`;
+  }
+  return JSON.stringify(message);
+}
 async function waitForJobOutcome(input) {
   const stdout = [];
   const stderr = [];
@@ -18728,6 +19286,11 @@ async function waitForJobOutcome(input) {
   let lastPromptCount = 0;
   let lastMessageCount = 0;
   let lastJobSummary = null;
+  let lastActionSummaryKey = "";
+  let lastActionSummaryLength = 0;
+  let lastAgentMessageKey = "";
+  let lastAgentMessageCount = 0;
+  let lastJobStatus = "";
   input.job.on("stdout", (line) => stdout.push(String(line)));
   input.job.on("stderr", (line) => stderr.push(String(line)));
   const startedAt = Date.now();
@@ -18743,7 +19306,71 @@ async function waitForJobOutcome(input) {
     const messages = asArray3(asRecord6(liveDoc.conversation)?.messages);
     lastMessageCount = messages.length;
     lastJobSummary = asRecord6(asRecord6(liveDoc.jobs)?.byId)?.[input.job.id] || null;
+    const jobRecord = asRecord6(lastJobSummary);
+    const jobStatus = typeof jobRecord?.status === "string" ? jobRecord.status : "";
+    if (jobStatus && jobStatus !== lastJobStatus) {
+      lastJobStatus = jobStatus;
+      input.onProgress?.({
+        id: `${input.job.id}:status:${jobStatus}:${Date.now()}`,
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        phase: "job",
+        status: "running",
+        jobId: input.job.id,
+        title: `Job ${jobStatus}`,
+        message: `Granular job ${input.job.id}`,
+        data: jobRecord,
+        ...input.progressContext
+      });
+    }
+    const actionSummary = getActionSummary(liveDoc, input.job.id);
+    const actionSummaryKey = JSON.stringify(actionSummary);
+    if (actionSummary.length && actionSummaryKey !== lastActionSummaryKey) {
+      const newActions = actionSummary.slice(lastActionSummaryLength);
+      lastActionSummaryLength = actionSummary.length;
+      lastActionSummaryKey = actionSummaryKey;
+      input.onProgress?.({
+        id: `${input.job.id}:actions:${Date.now()}`,
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        phase: "job",
+        status: "running",
+        jobId: input.job.id,
+        title: newActions.length === 1 ? "Action observed" : "Actions observed",
+        message: (newActions.length ? newActions : actionSummary).join("\n"),
+        data: { actionSummary, newActions },
+        ...input.progressContext
+      });
+    }
+    const agentMessages = getJobAgentMessages(liveDoc, input.job.id);
+    const agentMessageKey = JSON.stringify(agentMessages);
+    if (agentMessages.length && agentMessageKey !== lastAgentMessageKey) {
+      const newMessages = agentMessages.slice(lastAgentMessageCount);
+      lastAgentMessageCount = agentMessages.length;
+      lastAgentMessageKey = agentMessageKey;
+      const latestMessage = newMessages.at(-1) || agentMessages.at(-1);
+      input.onProgress?.({
+        id: `${input.job.id}:messages:${Date.now()}`,
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        phase: "job",
+        status: "running",
+        jobId: input.job.id,
+        title: "Agent message",
+        message: readableAgentMessage(latestMessage),
+        data: { agentMessages, newMessages },
+        ...input.progressContext
+      });
+    }
     if (prompts.length > 0) {
+      input.onProgress?.({
+        id: `${input.job.id}:prompt:${prompts[0]?.id || Date.now()}`,
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        phase: "prompt",
+        status: "running",
+        jobId: input.job.id,
+        title: "Waiting for prompt answer",
+        message: prompts[0]?.message || prompts[0]?.title,
+        data: { prompts },
+        ...input.progressContext
+      });
       return { kind: "prompt", prompts, liveDoc, stdout, stderr };
     }
     try {
@@ -18752,6 +19379,17 @@ async function waitForJobOutcome(input) {
         input.pollIntervalMs,
         `job ${input.job.id} tick`
       );
+      input.onProgress?.({
+        id: `${input.job.id}:completed:${Date.now()}`,
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        phase: "job",
+        status: "passed",
+        jobId: input.job.id,
+        title: "Job completed",
+        message: JSON.stringify(result)?.slice(0, 800),
+        data: { result, stdout, stderr },
+        ...input.progressContext
+      });
       return { kind: "completed", result, liveDoc, stdout, stderr };
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
@@ -18983,6 +19621,10 @@ function buildSessionLogReport(input) {
     `- Environment id: \`${conversation.environment.environmentId}\``,
     `- Sandbox id: \`${conversation.environment.sandboxId}\``,
     `- Status: ${result?.status || (error ? "failed" : "unknown")}`,
+    ...systemPrompts[0]?.iteration.templateId ? [
+      `- Harness template: \`${systemPrompts[0].iteration.templateId}@${systemPrompts[0].iteration.templateVersion || "unknown"}\``,
+      `- Template hash: \`${systemPrompts[0].iteration.templateHash || "unknown"}\``
+    ] : [],
     ...result?.error || error ? [`- Error: ${result?.error || error}`] : [],
     "",
     "## Conversation"
@@ -19090,6 +19732,12 @@ function buildSessionLogReport(input) {
       lines.push(
         `### Turn ${turn.turnNumber}, Generation ${iteration.iteration}`,
         "",
+        ...iteration.templateId ? [
+          `- Template: \`${iteration.templateId}@${iteration.templateVersion || "unknown"}\``,
+          `- Template hash: \`${iteration.templateHash || "unknown"}\``,
+          `- Prompt hash: \`${iteration.promptInstanceHash || "unknown"}\``,
+          ""
+        ] : [],
         fenced(iteration.systemPrompt, "text"),
         ""
       );
@@ -19131,9 +19779,167 @@ function isTransientEvalError(error) {
   const message = error instanceof Error ? error.message : String(error);
   return /socket connection was closed unexpectedly/i.test(message) || /timed out after/i.test(message) || /ECONNRESET/i.test(message) || /network/i.test(message) || /429/.test(message);
 }
+function assertionPassed(id, run) {
+  try {
+    run(id);
+    return { id, label: id, status: "passed" };
+  } catch (error) {
+    return {
+      id,
+      label: id,
+      status: "failed",
+      message: error instanceof Error ? error.message : String(error)
+    };
+  }
+}
+function hasMatchers(matchers) {
+  return Boolean(matchers?.length);
+}
+function evaluateExpectationAssertions(input) {
+  const expect = input.expect;
+  if (!expect) return [];
+  const prefix = `${input.scenarioId} step ${input.stepIndex + 1}`;
+  const assertions = [];
+  const actionText = input.completed.actionSummary.join("\n");
+  const codeText = input.completed.finalCode || "";
+  const displaySurface = [
+    input.completed.responseText,
+    actionText,
+    codeText,
+    JSON.stringify(input.completed.result)
+  ].join("\n");
+  const promptSurface = input.completed.promptInteractions.map(
+    (interaction) => [interaction.type, interaction.title, interaction.message].join(" ")
+  ).join("\n");
+  if (hasMatchers(expect.replyIncludes) || hasMatchers(expect.replyExcludes)) {
+    assertions.push(
+      assertionPassed(
+        "reply",
+        () => assertMatches(
+          `Reply for ${prefix}`,
+          input.completed.responseText,
+          expect.replyIncludes,
+          expect.replyExcludes
+        )
+      )
+    );
+  }
+  if (hasMatchers(expect.actionIncludes) || hasMatchers(expect.actionExcludes)) {
+    assertions.push(
+      assertionPassed(
+        "actions",
+        () => assertMatches(
+          `Action summary for ${prefix}`,
+          actionText,
+          expect.actionIncludes,
+          expect.actionExcludes
+        )
+      )
+    );
+  }
+  if (hasMatchers(expect.codeIncludes) || hasMatchers(expect.codeExcludes)) {
+    assertions.push(
+      assertionPassed(
+        "code",
+        () => assertMatches(
+          `Generated code for ${prefix}`,
+          codeText,
+          expect.codeIncludes,
+          expect.codeExcludes
+        )
+      )
+    );
+  }
+  if (hasMatchers(expect.actions?.required) || hasMatchers(expect.actions?.forbidden)) {
+    assertions.push(
+      assertionPassed(
+        "required-actions",
+        () => assertMatches(
+          `Required actions for ${prefix}`,
+          actionText,
+          expect.actions?.required,
+          expect.actions?.forbidden
+        )
+      )
+    );
+  }
+  if (hasMatchers(expect.presentation?.mustMention) || hasMatchers(expect.presentation?.mustNotMention)) {
+    assertions.push(
+      assertionPassed(
+        "presentation",
+        () => assertMatches(
+          `Presentation for ${prefix}`,
+          input.completed.responseText,
+          expect.presentation?.mustMention,
+          expect.presentation?.mustNotMention
+        )
+      )
+    );
+  }
+  if (hasMatchers(expect.presentation?.mustDisplayOrSave)) {
+    assertions.push(
+      assertionPassed(
+        "display",
+        () => assertMatches(
+          `Displayed or saved records for ${prefix}`,
+          displaySurface,
+          expect.presentation?.mustDisplayOrSave
+        )
+      )
+    );
+  }
+  if (expect.prompts?.requiredChoiceWhenAmbiguous) {
+    assertions.push(
+      assertionPassed("choice", () => {
+        if (!input.completed.promptInteractions.some(
+          (interaction) => interaction.type === "choice"
+        )) {
+          throw new Error(`Expected ${prefix} to use a choice prompt.`);
+        }
+      })
+    );
+  }
+  for (const forbiddenType of expect.prompts?.forbiddenPromptTypes || []) {
+    assertions.push(
+      assertionPassed(`forbidden-prompt:${forbiddenType}`, () => {
+        if (input.completed.promptInteractions.some(
+          (interaction) => interaction.type === forbiddenType
+        )) {
+          throw new Error(
+            `Prompt interactions for ${prefix} used forbidden prompt type ${forbiddenType}.`
+          );
+        }
+      })
+    );
+  }
+  if (expect.prompts?.requiredConfirmationBefore?.length) {
+    assertions.push(
+      assertionPassed(
+        "confirmation",
+        () => assertMatches(
+          `Confirmation prompts for ${prefix}`,
+          [actionText, promptSurface].join("\n"),
+          expect.prompts?.requiredConfirmationBefore
+        )
+      )
+    );
+  }
+  return assertions;
+}
 async function runAgentEvalSuite(options) {
   const results = [];
   for (const scenario of options.scenarios) {
+    await options.harness.emitProgress?.({
+      phase: "scenario",
+      status: "running",
+      scenarioId: scenario.id,
+      title: "Scenario started",
+      message: scenario.description || scenario.request || scenario.id,
+      data: {
+        scenarioId: scenario.id,
+        behaviorBuckets: scenario.behaviorBuckets
+      }
+    });
     let attempt = 0;
     let finalResult = null;
     while (attempt < 2 && !finalResult) {
@@ -19204,27 +20010,38 @@ async function runAgentEvalSuite(options) {
             },
             assertMatches
           };
-          if (step.expect) {
-            assertMatches(
-              `Reply for ${scenario.id} step ${index + 1}`,
-              completed.responseText,
-              step.expect.replyIncludes,
-              step.expect.replyExcludes
-            );
-            assertMatches(
-              `Action summary for ${scenario.id} step ${index + 1}`,
-              completed.actionSummary.join("\n"),
-              step.expect.actionIncludes,
-              step.expect.actionExcludes
-            );
-            assertMatches(
-              `Generated code for ${scenario.id} step ${index + 1}`,
-              completed.finalCode || "",
-              step.expect.codeIncludes,
-              step.expect.codeExcludes
-            );
+          const assertions = evaluateExpectationAssertions({
+            scenarioId: scenario.id,
+            stepIndex: index,
+            expect: step.expect,
+            completed
+          });
+          for (const assertion of assertions) {
+            await options.harness.emitProgress?.({
+              phase: "assertion",
+              status: assertion.status,
+              scenarioId: scenario.id,
+              stepId: step.id || `step-${index + 1}`,
+              title: assertion.label,
+              message: assertion.message,
+              data: assertion
+            });
+          }
+          const failedAssertion = assertions.find(
+            (assertion) => assertion.status === "failed"
+          );
+          if (failedAssertion) {
+            throw new Error(failedAssertion.message || failedAssertion.label);
           }
           for (const inspection of stepInspections) {
+            await options.harness.emitProgress?.({
+              phase: "inspection",
+              status: "running",
+              scenarioId: scenario.id,
+              stepId: step.id || `step-${index + 1}`,
+              title: "Running step inspection",
+              message: inspection.code.slice(0, 500)
+            });
             const inspectionResult = await context.inspect(inspection.code);
             const inspectionText = JSON.stringify(inspectionResult, null, 2);
             assertMatches(
@@ -19240,9 +20057,32 @@ async function runAgentEvalSuite(options) {
               });
             }
             inspectionResults.push(inspectionResult);
+            await options.harness.emitProgress?.({
+              phase: "inspection",
+              status: "passed",
+              scenarioId: scenario.id,
+              stepId: step.id || `step-${index + 1}`,
+              title: "Step inspection passed",
+              message: inspectionText.slice(0, 800),
+              data: inspectionResult
+            });
           }
           for (const check of stepChecks) {
+            await options.harness.emitProgress?.({
+              phase: "check",
+              status: "running",
+              scenarioId: scenario.id,
+              stepId: step.id || `step-${index + 1}`,
+              title: "Running custom check"
+            });
             await check(context);
+            await options.harness.emitProgress?.({
+              phase: "check",
+              status: "passed",
+              scenarioId: scenario.id,
+              stepId: step.id || `step-${index + 1}`,
+              title: "Custom check passed"
+            });
           }
           stepResults.push({
             id: step.id || `step-${index + 1}`,
@@ -19253,6 +20093,7 @@ async function runAgentEvalSuite(options) {
             actionSummary: completed.actionSummary,
             promptInteractions: completed.promptInteractions,
             inspectionResults,
+            assertions,
             turnDir: completed.turnDir
           });
         }
@@ -19271,6 +20112,7 @@ async function runAgentEvalSuite(options) {
           actionSummary: lastStep.actionSummary,
           promptInteractions: lastStep.promptInteractions,
           verification: lastStep.inspectionResults.length <= 1 ? lastStep.inspectionResults[0] ?? null : lastStep.inspectionResults,
+          assertions: stepResults.flatMap((step) => step.assertions || []),
           tokenUsage: aggregateConversationTokenUsage(conversation),
           steps: stepResults,
           turnDir: conversation.artifactDir
@@ -19292,6 +20134,14 @@ async function runAgentEvalSuite(options) {
           conversation,
           result
         });
+        await options.harness.emitProgress?.({
+          phase: "scenario",
+          status: "passed",
+          scenarioId: scenario.id,
+          title: "Scenario passed",
+          message: `${result.assertions?.filter((assertion) => assertion.status === "passed").length || 0}/${result.assertions?.length || 0} assertions`,
+          data: result
+        });
         finalResult = result;
       } catch (error) {
         const failureMessage = error instanceof Error ? error.message : String(error);
@@ -19332,6 +20182,14 @@ async function runAgentEvalSuite(options) {
           result: failed,
           error: failureMessage
         });
+        await options.harness.emitProgress?.({
+          phase: "scenario",
+          status: "failed",
+          scenarioId: scenario.id,
+          title: "Scenario failed",
+          message: failureMessage,
+          data: failed
+        });
         finalResult = failed;
       } finally {
         await options.harness.closeConversation(conversation);
@@ -19380,6 +20238,11 @@ function createAgentEvalHarness(options) {
   const chatTimeoutMs = options.chatTimeoutMs ?? 12e4;
   const jobTimeoutMs = options.jobTimeoutMs ?? 9e4;
   const pollIntervalMs = options.pollIntervalMs ?? 250;
+  const resolvedTemplate = resolveHarnessTemplate(
+    options.harnessTemplateId || process.env.GRANULAR_AGENT_HARNESS_TEMPLATE || "stable");
+  const promptRenderer = options.promptRenderer || resolvedTemplate.renderPrompt;
+  const continuationRenderer = options.continuationRenderer || resolvedTemplate.renderContinuation;
+  const onProgress = options.onProgress;
   async function openConversation(label) {
     await ensureDir(artifactDir);
     const clientId = `${slugify(label)}-${Date.now()}`;
@@ -19448,6 +20311,13 @@ function createAgentEvalHarness(options) {
     };
   }
   async function runInspection(conversation, inspection, completed, turnDir) {
+    await emitProgress(onProgress, {
+      phase: "inspection",
+      status: "running",
+      scenarioId: conversation.label,
+      title: "Running inspection",
+      message: inspection.code.slice(0, 500)
+    });
     let result = null;
     let lastError = null;
     for (let attempt = 0; attempt < 10; attempt += 1) {
@@ -19475,6 +20345,14 @@ function createAgentEvalHarness(options) {
       });
     }
     await writeJson(path.join(turnDir, "verification.json"), result);
+    await emitProgress(onProgress, {
+      phase: "inspection",
+      status: "passed",
+      scenarioId: conversation.label,
+      title: "Inspection passed",
+      message: JSON.stringify(result)?.slice(0, 800),
+      data: result
+    });
     return result;
   }
   async function resumePendingTurn(pending, responder) {
@@ -19484,6 +20362,16 @@ function createAgentEvalHarness(options) {
       prompt,
       history: pending.promptInteractions
     });
+    await emitProgress(onProgress, {
+      phase: "interaction",
+      status: "running",
+      scenarioId: pending.conversation.label,
+      stepId: path.basename(pending.turnDir),
+      jobId: pending.job.id,
+      title: "Prompt answered",
+      message: `${prompt.type}: ${prompt.message || prompt.title} -> ${JSON.stringify(answer)}`,
+      data: { prompt, answer }
+    });
     const session = pending.conversation.environment;
     await session.answerPrompt(prompt.id, answer);
     pending.promptInteractions.push({
@@ -19503,7 +20391,12 @@ function createAgentEvalHarness(options) {
       job: pending.job,
       boundaryTimestamp: pending.boundaryTimestamp,
       timeoutMs: jobTimeoutMs,
-      pollIntervalMs
+      pollIntervalMs,
+      onProgress: (event) => void onProgress?.(event),
+      progressContext: {
+        scenarioId: pending.conversation.label,
+        stepId: path.basename(pending.turnDir)
+      }
     });
     if (resumed.kind === "prompt") {
       return {
@@ -19584,12 +20477,44 @@ function createAgentEvalHarness(options) {
     };
     conversation.logTurns.push(turnLog);
     if (input.prepareRecords?.length) {
+      await emitProgress(onProgress, {
+        phase: "setup",
+        status: "running",
+        scenarioId: conversation.label,
+        stepId: turnId,
+        title: "Recording setup records",
+        message: `${input.prepareRecords.length} records`,
+        data: input.prepareRecords
+      });
       await conversation.environment.recordObjects(input.prepareRecords);
+      await emitProgress(onProgress, {
+        phase: "setup",
+        status: "passed",
+        scenarioId: conversation.label,
+        stepId: turnId,
+        title: "Setup records recorded",
+        message: `${input.prepareRecords.length} records`
+      });
     }
     if (input.prepareTools?.length) {
+      await emitProgress(onProgress, {
+        phase: "setup",
+        status: "running",
+        scenarioId: conversation.label,
+        stepId: turnId,
+        title: "Registering effect handlers",
+        message: `${input.prepareTools.length} handlers`
+      });
       await options.granular.ontology(conversation.environment.sandboxId).effects.registerMany(input.prepareTools);
     }
     if (input.prepare) {
+      await emitProgress(onProgress, {
+        phase: "setup",
+        status: "running",
+        scenarioId: conversation.label,
+        stepId: turnId,
+        title: "Running custom setup"
+      });
       await input.prepare({
         conversation,
         environment: conversation.environment,
@@ -19598,6 +20523,14 @@ function createAgentEvalHarness(options) {
     }
     const boundaryTimestamp = Date.now();
     conversation.history.push({ role: "user", content: input.request });
+    await emitProgress(onProgress, {
+      phase: "step",
+      status: "running",
+      scenarioId: conversation.label,
+      stepId: turnId,
+      title: "User request",
+      message: input.request
+    });
     await writeJson(path.join(turnDir, "request.json"), {
       request: input.request,
       boundaryTimestamp
@@ -19639,7 +20572,7 @@ function createAgentEvalHarness(options) {
         inputSchema: tool.inputSchema,
         outputSchema: tool.outputSchema
       }));
-      const systemPrompt = buildGranularAgentSystemPrompt({
+      const renderedPrompt = promptRenderer({
         domainDocumentation: await conversation.environment.getDomainDocumentation(),
         sessionContext: {
           sandboxId: conversation.environment.sandboxId,
@@ -19660,9 +20593,39 @@ function createAgentEvalHarness(options) {
         tools,
         checkpoint: latestCheckpoint
       });
-      const request = iteration === 0 ? input.request : buildContinuationInstruction(
+      const systemPrompt = renderedPrompt.prompt;
+      await emitProgress(onProgress, {
+        phase: "prompt",
+        status: "passed",
+        scenarioId: conversation.label,
+        stepId: turnId,
+        iteration: iteration + 1,
+        templateId: renderedPrompt.templateId,
+        templateVersion: renderedPrompt.templateVersion,
+        title: "Rendered harness prompt",
+        message: `${systemPrompt.split("\n").length} lines`,
+        data: {
+          templateId: renderedPrompt.templateId,
+          templateVersion: renderedPrompt.templateVersion,
+          templateHash: renderedPrompt.templateHash,
+          promptInstanceHash: renderedPrompt.promptInstanceHash,
+          prompt: systemPrompt
+        }
+      });
+      const request = iteration === 0 ? input.request : continuationRenderer(
         buildContinuationPreview(latestCheckpoint, noProgressCount)
-      );
+      ).instruction;
+      await emitProgress(onProgress, {
+        phase: "generation",
+        status: "running",
+        scenarioId: conversation.label,
+        stepId: turnId,
+        iteration: iteration + 1,
+        templateId: renderedPrompt.templateId,
+        templateVersion: renderedPrompt.templateVersion,
+        title: iteration === 0 ? "Generating agent response" : "Generating continuation",
+        message: request
+      });
       const generation = await withTimeout2(
         generateTurnWithRepair(options.generator, {
           systemPrompt,
@@ -19681,10 +20644,31 @@ function createAgentEvalHarness(options) {
         chatTimeoutMs,
         `chat generation for ${conversation.label} iteration ${iteration + 1}`
       );
+      await emitProgress(onProgress, {
+        phase: "generation",
+        status: "passed",
+        scenarioId: conversation.label,
+        stepId: turnId,
+        iteration: iteration + 1,
+        templateId: renderedPrompt.templateId,
+        templateVersion: renderedPrompt.templateVersion,
+        title: generation.code ? "Generated job code" : "Generated text reply",
+        message: generation.code || generation.reply || "",
+        data: {
+          reply: generation.reply,
+          code: generation.code,
+          attempts: generation.generationAttempts,
+          usage: tokenUsageForGenerationOutput(generation)
+        }
+      });
       const iterationLog = {
         iteration: iteration + 1,
         request,
         systemPrompt,
+        templateId: renderedPrompt.templateId,
+        templateVersion: renderedPrompt.templateVersion,
+        templateHash: renderedPrompt.templateHash,
+        promptInstanceHash: renderedPrompt.promptInstanceHash,
         generationReply: generation.reply,
         generatedCode: generation.code,
         rawGeneration: generation.raw,
@@ -19731,6 +20715,15 @@ function createAgentEvalHarness(options) {
           result: completed.result
         };
         await writeJson(path.join(turnDir, "result.json"), completed);
+        await emitProgress(onProgress, {
+          phase: "step",
+          status: "passed",
+          scenarioId: conversation.label,
+          stepId: turnId,
+          title: "Step completed with text reply",
+          message: responseText2,
+          data: completed
+        });
         return completed;
       }
       const session = conversation.environment;
@@ -19751,12 +20744,33 @@ function createAgentEvalHarness(options) {
           )
         }
       });
+      await emitProgress(onProgress, {
+        phase: "job",
+        status: "running",
+        scenarioId: conversation.label,
+        stepId: turnId,
+        iteration: iteration + 1,
+        jobId: job.id,
+        templateId: renderedPrompt.templateId,
+        templateVersion: renderedPrompt.templateVersion,
+        title: "Submitted Granular job",
+        message: job.id,
+        data: { code: generation.code }
+      });
       const outcome = await waitForJobOutcome({
         environment: conversation.environment,
         job,
         boundaryTimestamp,
         timeoutMs: jobTimeoutMs,
-        pollIntervalMs
+        pollIntervalMs,
+        onProgress: (event) => void onProgress?.(event),
+        progressContext: {
+          scenarioId: conversation.label,
+          stepId: turnId,
+          iteration: iteration + 1,
+          templateId: renderedPrompt.templateId,
+          templateVersion: renderedPrompt.templateVersion
+        }
       });
       if (outcome.kind === "prompt") {
         if (!autoAnswerPrompts) {
@@ -19866,6 +20880,23 @@ function createAgentEvalHarness(options) {
         controllerReason: continuation.reason,
         noProgressCount: continuation.nextNoProgressCount
       };
+      await emitProgress(onProgress, {
+        phase: "continuation",
+        status: continuation.shouldContinue ? "running" : "passed",
+        scenarioId: conversation.label,
+        stepId: turnId,
+        iteration: iteration + 1,
+        jobId: job.id,
+        templateId: renderedPrompt.templateId,
+        templateVersion: renderedPrompt.templateVersion,
+        title: continuation.shouldContinue ? "Harness requested another loop" : "Harness accepted completion",
+        message: `${continuation.reason}; ${continuation.outcome}`,
+        data: {
+          continuation,
+          checkpoint: latestCheckpoint,
+          verifierSnapshot
+        }
+      });
       previousSnapshot = verifierSnapshot;
       noProgressCount = continuation.nextNoProgressCount;
       conversation.history.push({
@@ -19919,6 +20950,17 @@ function createAgentEvalHarness(options) {
           result: outcome.result
         };
         await writeJson(path.join(turnDir, "result.json"), completed);
+        await emitProgress(onProgress, {
+          phase: "step",
+          status: "passed",
+          scenarioId: conversation.label,
+          stepId: turnId,
+          iteration: iteration + 1,
+          jobId: job.id,
+          title: "Step completed",
+          message: responseText,
+          data: completed
+        });
         return completed;
       }
       iteration += 1;
@@ -19930,6 +20972,7 @@ function createAgentEvalHarness(options) {
   return {
     artifactDir,
     granular: options.granular,
+    emitProgress: (event) => emitProgress(onProgress, event),
     openConversation,
     closeConversation,
     runTurn,
@@ -19995,7 +21038,11 @@ function createAgentTester(options) {
     controllerBudgets: options.controllerBudgets,
     chatTimeoutMs: options.chatTimeoutMs,
     jobTimeoutMs: options.jobTimeoutMs,
-    pollIntervalMs: options.pollIntervalMs
+    pollIntervalMs: options.pollIntervalMs,
+    harnessTemplateId: options.harnessTemplateId,
+    promptRenderer: options.promptRenderer,
+    continuationRenderer: options.continuationRenderer,
+    onProgress: options.onProgress
   });
   return {
     ...harness,
@@ -20021,6 +21068,6 @@ var createHumanResponder = createScriptedPromptResponder;
 var createOpenAIGenerator = createOpenAIChatTurnGenerator;
 var createTestArtifactsDirectory = createTimestampedArtifactDirectory;
 
-export { createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, generateTurnWithRepair, runAgentEvalSuite, runAgentTests };
+export { createAgentEvalHarness, createAgentTester, createHumanResponder, createOpenAIChatTurnGenerator, createOpenAIGenerator, createScriptedPromptResponder, createTestArtifactsDirectory, createTimestampedArtifactDirectory, generateTurnWithRepair, loadAgentEvalScenarioFile, runAgentEvalSuite, runAgentTests, scenariosFromAgentEvalFile };
 //# sourceMappingURL=agent-evals.mjs.map
 //# sourceMappingURL=agent-evals.mjs.map