@alexeiled/pi-fusion 0.3.0 → 0.5.0

package/src/report.ts

@@ -1,9 +1,16 @@
 import type { FailedPanelSummary, PanelOutput } from "./run-builder.js";
-import type { FusionRun } from "./types.js";
+import { summarizeProviderFailures } from "./run-observations.js";
+import type { FusionRun, ProviderFailure, RunObservation } from "./types.js";
 
 type ReportRun = Pick<
   FusionRun,
-  "id" | "prompt" | "profileName" | "chainRunId" | "panelRunId" | "judgeRunId"
+  | "id"
+  | "prompt"
+  | "profileName"
+  | "chainRunId"
+  | "panelRunId"
+  | "judgeRunId"
+  | "panelStopReason"
 > &
   Partial<Pick<FusionRun, "phase" | "createdAt" | "updatedAt">>;
 
@@ -27,6 +34,7 @@ export interface RenderJudgeReportInput {
   panelOutputs?: readonly PanelOutput[];
   failures?: readonly FailedPanelSummary[];
   judgeModel?: string;
+  judgeObservation?: RunObservation;
 }
 
 export interface RenderFailureReportInput {
@@ -56,6 +64,7 @@ type ReportSectionTitle =
   | "Recommendation"
   | "Risks"
   | "Next Step"
+  | "Run Details"
   | "Run Metadata";
 
 interface ReportSection {
@@ -127,7 +136,7 @@ export function renderSinglePanelReport(
   input: RenderSinglePanelReportInput,
 ): string {
   const panelName = formatPanelName(input.output);
-  return renderReport([
+  const sections: ReportSection[] = [
     {
       title: "Summary",
       content:
@@ -177,7 +186,13 @@ export function renderSinglePanelReport(
         "Use this single-panel result directly, or rerun /fusion if you need judge synthesis.",
     },
     { title: "Run Metadata", content: formatRunMetadata(input.run) },
-  ]);
+  ];
+  const runDetails = formatRunDetails({
+    panelOutputs: [input.output],
+    failures: input.failures,
+  });
+  if (runDetails) sections.splice(-1, 0, runDetails);
+  return renderReport(sections);
 }
 
 export function renderJudgeReport(input: RenderJudgeReportInput): string {
@@ -190,7 +205,7 @@ export function renderJudgeReport(input: RenderJudgeReportInput): string {
       ? unsectionedOutput
       : "Judge completed without a recommendation.";
 
-  return renderReport([
+  const reportSections: ReportSection[] = [
     {
       title: "Summary",
       content: sections.get("Summary") ?? judgeSummary(panelOutputs, failures),
@@ -235,7 +250,17 @@ export function renderJudgeReport(input: RenderJudgeReportInput): string {
         "Review the recommendation and decide whether to act on it.",
     },
     { title: "Run Metadata", content: formatRunMetadata(input.run) },
-  ]);
+  ];
+  const runDetails = formatRunDetails({
+    panelOutputs,
+    failures,
+    ...(input.judgeModel ? { judgeModel: input.judgeModel } : {}),
+    ...(input.judgeObservation
+      ? { judgeObservation: input.judgeObservation }
+      : {}),
+  });
+  if (runDetails) reportSections.splice(-1, 0, runDetails);
+  return renderReport(reportSections);
 }
 
 export function renderFailureReport(input: RenderFailureReportInput): string {
@@ -357,6 +382,118 @@ function formatSectionContent(content: string | readonly string[]): string {
   return text.trim() || "None.";
 }
 
+interface RunDetailsInput {
+  panelOutputs: readonly PanelOutput[];
+  failures: readonly FailedPanelSummary[];
+  judgeModel?: string;
+  judgeObservation?: RunObservation;
+}
+
+function formatRunDetails(input: RunDetailsInput): ReportSection | undefined {
+  const entries = [
+    ...input.panelOutputs.map((item) => ({
+      label: formatPanelName(item),
+      status: "completed",
+      configuredModel: item.configuredModel ?? item.model,
+      observation: item.observation,
+    })),
+    ...input.failures.map((item) => ({
+      label: formatPanelName(item),
+      status: item.reason === "stopped-after-agreement" ? "stopped" : "failed",
+      configuredModel: item.configuredModel ?? item.model,
+      observation: item.observation,
+    })),
+  ];
+  if (input.judgeObservation) {
+    entries.push({
+      label: "Judge",
+      status: "completed",
+      configuredModel: input.judgeModel,
+      observation: input.judgeObservation,
+    });
+  }
+  if (!entries.some((entry) => entry.observation)) return undefined;
+
+  const observations = entries.map((entry) => entry.observation ?? {});
+  const providerFailures = summarizeProviderFailures(
+    observations.flatMap((observation) => observation.providerFailures ?? []),
+  );
+  const lines = entries.map(
+    (entry) =>
+      `- ${entry.label} (${entry.status}): ${formatObservation(entry.observation ?? {}, entry.configuredModel)}`,
+  );
+  lines.push(
+    `- Aggregate model time: ${formatTotal(observations, (observation) => observation.durationMs, formatDuration)}`,
+    `- Total input tokens: ${formatTotal(observations, (observation) => observation.usage?.inputTokens, formatTokens)}`,
+    `- Total output tokens: ${formatTotal(observations, (observation) => observation.usage?.outputTokens, formatTokens)}`,
+    `- Total estimated cost: ${formatTotal(observations, (observation) => observation.usage?.costUsd, formatCost)}`,
+  );
+  if (providerFailures.length > 0) {
+    lines.push("- Model issues:");
+    lines.push(
+      ...providerFailures.map(
+        (failure) => `  - ${formatProviderFailure(failure)}`,
+      ),
+    );
+  }
+  return { title: "Run Details", content: lines };
+}
+
+function formatObservation(
+  observation: RunObservation,
+  configuredModel?: string,
+): string {
+  return [
+    observation.model ??
+      (configuredModel ? `${configuredModel} (configured)` : "model unknown"),
+    observation.durationMs !== undefined
+      ? formatDuration(observation.durationMs)
+      : "time unknown",
+    formatUsage(observation),
+  ].join(" · ");
+}
+
+function formatUsage(observation: RunObservation): string {
+  const input = observation.usage?.inputTokens;
+  const output = observation.usage?.outputTokens;
+  const cost = observation.usage?.costUsd;
+  return [
+    input !== undefined ? `in ${formatTokens(input)}` : "input unknown",
+    output !== undefined ? `out ${formatTokens(output)}` : "output unknown",
+    cost !== undefined ? formatCost(cost) : "cost unknown",
+  ].join(", ");
+}
+
+function formatProviderFailure(failure: ProviderFailure): string {
+  const target = failure.model
+    ? `${failure.provider}/${failure.model.split("/").slice(1).join("/")}`
+    : failure.provider;
+  return `${target}: ${failure.message}${failure.count && failure.count > 1 ? ` (x${failure.count})` : ""}`;
+}
+
+function formatTotal(
+  observations: readonly RunObservation[],
+  read: (observation: RunObservation) => number | undefined,
+  format: (value: number) => string,
+): string {
+  const values = observations.map(read);
+  if (values.some((value) => value === undefined)) return "unknown";
+  const total = values.reduce<number>((sum, value) => sum + (value ?? 0), 0);
+  return format(total);
+}
+
+function formatDuration(value: number): string {
+  return value >= 100 ? `${(value / 1000).toFixed(1)}s` : `${value}ms`;
+}
+
+function formatTokens(value: number): string {
+  return value.toLocaleString("en-US");
+}
+
+function formatCost(value: number): string {
+  return `$${value.toFixed(4)}`;
+}
+
 function formatAgentStatus(options: AgentStatusOptions): string[] {
   const hasPanelStatus =
     options.panelOutputs !== undefined || options.failures !== undefined;
@@ -382,7 +519,9 @@ function formatAgentStatus(options: AgentStatusOptions): string[] {
   }
 
   lines.push(`- Judge: ${options.judgeStatus}`);
-  if (options.judgeModel) lines.push(`  Model: ${options.judgeModel}`);
+  if (options.judgeModel) {
+    lines.push(`  Configured model: ${options.judgeModel}`);
+  }
   if (options.extra) lines.push(...options.extra);
   return lines;
 }
@@ -390,13 +529,28 @@ function formatAgentStatus(options: AgentStatusOptions): string[] {
 function formatPanelDetails(
   item: Pick<
     PanelOutput,
-    "agent" | "role" | "model" | "artifactPath" | "sessionPath"
+    | "agent"
+    | "role"
+    | "model"
+    | "configuredModel"
+    | "observation"
+    | "artifactPath"
+    | "sessionPath"
   >,
 ): string[] {
   return [
     `  Agent: ${item.agent}`,
     ...(item.role ? [`  Role: ${item.role}`] : []),
-    ...(item.model ? [`  Model: ${item.model}`] : []),
+    ...(item.observation?.model
+      ? [`  Model: ${item.observation.model}`]
+      : (item.configuredModel ?? item.model)
+        ? [`  Configured model: ${item.configuredModel ?? item.model}`]
+        : []),
+    ...(item.configuredModel &&
+    item.observation?.model &&
+    item.configuredModel !== item.observation.model
+      ? [`  Configured model: ${item.configuredModel}`]
+      : []),
     ...(item.artifactPath ? [`  Artifact: ${item.artifactPath}`] : []),
     ...(item.sessionPath ? [`  Session: ${item.sessionPath}`] : []),
   ];
@@ -410,7 +564,14 @@ function formatRunMetadata(run: ReportRun): string[] {
     `- Prompt: ${firstLine(run.prompt)}`,
     ...(run.chainRunId ? [`- Chain run: ${run.chainRunId}`] : []),
     ...(run.panelRunId ? [`- Panel run: ${run.panelRunId}`] : []),
-    ...(run.judgeRunId ? [`- Fallback judge run: ${run.judgeRunId}`] : []),
+    ...(run.panelStopReason === "agreement"
+      ? ["- Panel stopped after strong agreement"]
+      : []),
+    ...(run.judgeRunId
+      ? [
+          `- ${run.chainRunId ? "Fallback judge run" : "Judge run"}: ${run.judgeRunId}`,
+        ]
+      : []),
     ...(typeof run.createdAt === "number"
       ? [`- Created: ${formatTimestamp(run.createdAt)}`]
       : []),

package/src/result-extract.ts

@@ -3,6 +3,10 @@ import {
   type FailedPanelSummary,
   type PanelOutput,
 } from "./run-builder.js";
+import {
+  extractPanelDecision,
+  extractRunObservation,
+} from "./run-observations.js";
 import type { PanelMemberConfig } from "./types.js";
 
 export type ResultExtractErrorCode =
@@ -17,16 +21,19 @@ export interface ResultExtractError {
 export interface ExtractPanelResultsOptions {
   panel?: readonly PanelMemberConfig[];
   limit?: number;
+  completedOnly?: boolean;
+  stoppedPanelIndices?: readonly number[];
 }
 
+export type ExtractPanelResultsSuccess = {
+  ok: true;
+  outputs: PanelOutput[];
+  failures: FailedPanelSummary[];
+  runId?: string;
+};
+
 export type ExtractPanelResultsResult =
-  | {
-      ok: true;
-      outputs: PanelOutput[];
-      failures: FailedPanelSummary[];
-      runId?: string;
-    }
-  | { ok: false; error: ResultExtractError };
+  ExtractPanelResultsSuccess | { ok: false; error: ResultExtractError };
 
 interface ResultsContainer {
   payload: Record<string, unknown>;
@@ -36,6 +43,17 @@ interface ResultsContainer {
 
 type ChildStatus = "success" | "failed";
 
+function isCompletedResult(value: unknown): boolean {
+  if (!isRecord(value)) return false;
+  const status = firstString(value.status, value.state);
+  return !(
+    status === "running" ||
+    status === "active" ||
+    status === "pending" ||
+    status === "queued"
+  );
+}
+
 export function extractPanelResults(
   payload: unknown,
   options: ExtractPanelResultsOptions = {},
@@ -50,6 +68,7 @@ export function extractPanelResults(
       ? container.results
       : container.results.slice(0, options.limit);
   for (const [index, rawResult] of results.entries()) {
+    if (options.completedOnly && !isCompletedResult(rawResult)) continue;
     const child = normalizeChildResult(rawResult, index, options);
     if (!child.ok) return child;
     if (child.status === "success") outputs.push(child.output);
@@ -78,11 +97,32 @@ function findResultsContainer(
     );
   }
 
+  if (Array.isArray(payload.results) && payload.results.length > 0) {
+    return { ok: true, payload, results: payload.results, path: "$.results" };
+  }
+
+  if (
+    isRecord(payload.details) &&
+    Array.isArray(payload.details.results) &&
+    payload.details.results.length > 0
+  ) {
+    return {
+      ok: true,
+      payload: { ...payload, ...payload.details },
+      results: payload.details.results,
+      path: "$.details.results",
+    };
+  }
+
+  if (Array.isArray(payload.steps)) {
+    return { ok: true, payload, results: payload.steps, path: "$.steps" };
+  }
+
   if (Array.isArray(payload.results)) {
     return { ok: true, payload, results: payload.results, path: "$.results" };
   }
 
-  if ("results" in payload) {
+  if ("results" in payload && !Array.isArray(payload.results)) {
     return error(
       "unknown-result-shape",
       "Subagents result payload results field must be an array.",
@@ -91,6 +131,25 @@ function findResultsContainer(
   }
 
   if (isRecord(payload.details)) {
+    if (
+      Array.isArray(payload.details.results) &&
+      payload.details.results.length > 0
+    ) {
+      return {
+        ok: true,
+        payload: { ...payload, ...payload.details },
+        results: payload.details.results,
+        path: "$.details.results",
+      };
+    }
+    if (Array.isArray(payload.details.steps)) {
+      return {
+        ok: true,
+        payload: { ...payload, ...payload.details },
+        results: payload.details.steps,
+        path: "$.details.steps",
+      };
+    }
     if (Array.isArray(payload.details.results)) {
       return {
         ok: true,
@@ -99,7 +158,10 @@ function findResultsContainer(
         path: "$.details.results",
       };
     }
-    if ("results" in payload.details) {
+    if (
+      "results" in payload.details &&
+      !Array.isArray(payload.details.results)
+    ) {
       return error(
         "unknown-result-shape",
         "Subagents result details.results field must be an array.",
@@ -149,13 +211,19 @@ function normalizeChildResult(
   const status = classifyChildStatus(rawResult);
 
   if (status === "success") {
+    const rawOutput = firstNonBlankString(
+      rawResult.output,
+      rawResult.finalOutput,
+      rawResult.summary,
+      rawResult.text,
+      recentOutputText(rawResult.recentOutput),
+    );
+    const decision =
+      extractPanelDecision(rawResult.structuredOutput) ??
+      extractPanelDecision(rawOutput);
     const output =
-      firstNonBlankString(
-        rawResult.output,
-        rawResult.finalOutput,
-        rawResult.summary,
-        rawResult.text,
-      ) ?? artifactOutput(artifactPath);
+      firstNonBlankString(decision?.answerMarkdown, rawOutput) ??
+      artifactOutput(artifactPath);
     if (!output) {
       return error(
         "missing-result-field",
@@ -171,12 +239,18 @@ function normalizeChildResult(
         member,
         agent,
         output,
+        decision,
+        observation: extractRunObservation(rawResult),
         artifactPath,
         sessionPath,
       }),
     };
   }
 
+  const stoppedAfterAgreement =
+    options.stoppedPanelIndices?.includes(index) === true;
+  const observation = extractRunObservation(rawResult);
+  if (stoppedAfterAgreement) delete observation.providerFailures;
   return {
     ok: true,
     status,
@@ -184,7 +258,11 @@ function normalizeChildResult(
       index,
       member,
       agent,
-      summary: failureSummary(rawResult, artifactPath),
+      summary: stoppedAfterAgreement
+        ? "Stopped after strong panel agreement."
+        : failureSummary(rawResult, artifactPath),
+      reason: failureReason(rawResult, stoppedAfterAgreement),
+      observation,
       artifactPath,
       sessionPath,
     }),
@@ -238,13 +316,14 @@ function buildPanelOutput(input: {
   member: PanelMemberConfig | undefined;
   agent: string;
   output: string;
+  decision: PanelOutput["decision"];
+  observation: PanelOutput["observation"];
   artifactPath: string | undefined;
   sessionPath: string | undefined;
 }): PanelOutput {
-  const model = input.member
-    ? appendThinkingSuffix(input.member.model, input.member.thinking)
-    : undefined;
-  return {
+  const configuredModel = configuredMemberModel(input.member);
+  const model = input.observation?.model ?? configuredModel;
+  const output: PanelOutput = {
     index: input.index,
     agent: input.agent,
     output: input.output,
@@ -252,9 +331,15 @@ function buildPanelOutput(input: {
     ...(input.member?.label ? { label: input.member.label } : {}),
     ...(input.member?.role ? { role: input.member.role } : {}),
     ...(model ? { model } : {}),
+    ...(configuredModel ? { configuredModel } : {}),
+    ...(input.decision ? { decision: input.decision } : {}),
     ...(input.artifactPath ? { artifactPath: input.artifactPath } : {}),
     ...(input.sessionPath ? { sessionPath: input.sessionPath } : {}),
   };
+  if (hasObservation(input.observation)) {
+    output.observation = input.observation;
+  }
+  return output;
 }
 
 function buildFailedPanelSummary(input: {
@@ -262,13 +347,14 @@ function buildFailedPanelSummary(input: {
   member: PanelMemberConfig | undefined;
   agent: string;
   summary: string;
+  reason: FailedPanelSummary["reason"];
+  observation: FailedPanelSummary["observation"];
   artifactPath: string | undefined;
   sessionPath: string | undefined;
 }): FailedPanelSummary {
-  const model = input.member
-    ? appendThinkingSuffix(input.member.model, input.member.thinking)
-    : undefined;
-  return {
+  const configuredModel = configuredMemberModel(input.member);
+  const model = input.observation?.model ?? configuredModel;
+  const failure: FailedPanelSummary = {
     index: input.index,
     agent: input.agent,
     summary: input.summary,
@@ -276,9 +362,46 @@ function buildFailedPanelSummary(input: {
     ...(input.member?.label ? { label: input.member.label } : {}),
     ...(input.member?.role ? { role: input.member.role } : {}),
     ...(model ? { model } : {}),
+    ...(configuredModel ? { configuredModel } : {}),
+    ...(input.reason ? { reason: input.reason } : {}),
     ...(input.artifactPath ? { artifactPath: input.artifactPath } : {}),
     ...(input.sessionPath ? { sessionPath: input.sessionPath } : {}),
   };
+  if (hasObservation(input.observation)) {
+    failure.observation = input.observation;
+  }
+  return failure;
+}
+
+function failureReason(
+  result: Record<string, unknown>,
+  stoppedAfterAgreement = false,
+): FailedPanelSummary["reason"] {
+  if (stoppedAfterAgreement) return "stopped-after-agreement";
+  if (result.timedOut === true) return "timeout";
+  if (result.interrupted === true) return "interrupted";
+  return undefined;
+}
+
+function hasObservation(
+  observation: PanelOutput["observation"] | undefined,
+): observation is NonNullable<PanelOutput["observation"]> {
+  return Boolean(
+    observation &&
+    (observation.model ||
+      observation.durationMs !== undefined ||
+      observation.usage ||
+      observation.attempts ||
+      observation.providerFailures),
+  );
+}
+
+function configuredMemberModel(
+  member: PanelMemberConfig | undefined,
+): string | undefined {
+  return member
+    ? appendThinkingSuffix(member.model, member.thinking)
+    : undefined;
 }
 
 function extractArtifactPath(
@@ -306,6 +429,16 @@ function firstString(...values: readonly unknown[]): string | undefined {
   return undefined;
 }
 
+function recentOutputText(value: unknown): string | undefined {
+  if (
+    !Array.isArray(value) ||
+    !value.every((item) => typeof item === "string")
+  ) {
+    return undefined;
+  }
+  return value.join("\n").trim() || undefined;
+}
+
 function firstNonBlankString(
   ...values: readonly unknown[]
 ): string | undefined {

package/src/run-builder.ts

@@ -1,3 +1,7 @@
+import {
+  PANEL_DECISION_CLOSE,
+  PANEL_DECISION_OPEN,
+} from "./run-observations.js";
 import {
   THINKING_LEVELS,
   type FailedPanelSummary,
@@ -133,7 +137,13 @@ export function buildPanelSpawnParams(
   prompt: string,
 ): PanelSpawnParams {
   return {
-    tasks: profile.panel.map((member) => buildPanelTaskParams(member, prompt)),
+    tasks: profile.panel.map((member) =>
+      buildPanelTaskParams(
+        member,
+        prompt,
+        profile.stopWhenPanelAgrees === true,
+      ),
+    ),
     async: true,
     clarify: false,
     concurrency: profile.concurrency ?? profile.panel.length,
@@ -216,11 +226,12 @@ export function buildJudgeSpawnParams(
 function buildPanelTaskParams(
   member: PanelMemberConfig,
   prompt: string,
+  includeDecisionRecord: boolean,
 ): PanelSubagentTaskParams {
   const model = appendThinkingSuffix(member.model, member.thinking);
   return {
     agent: member.agent,
-    task: buildPanelTask(member, prompt),
+    task: buildPanelTask(member, prompt, includeDecisionRecord),
     output: true,
     outputMode: "inline",
     progress: true,
@@ -237,7 +248,7 @@ function buildPanelChainTaskParams(
   const model = appendThinkingSuffix(member.model, member.thinking);
   return {
     agent: member.agent,
-    task: buildPanelTask(member, "{task}"),
+    task: buildPanelTask(member, "{task}", false),
     as: chainOutputName(member, index),
     label: member.label,
     phase: "Panel",
@@ -250,7 +261,11 @@ function buildPanelChainTaskParams(
   };
 }
 
-function buildPanelTask(member: PanelMemberConfig, prompt: string): string {
+function buildPanelTask(
+  member: PanelMemberConfig,
+  prompt: string,
+  includeDecisionRecord: boolean,
+): string {
   const role = member.role?.trim() || "independent analysis and critique";
   return [
     `Panel member: ${member.label} (${member.id})`,
@@ -269,6 +284,19 @@ function buildPanelTask(member: PanelMemberConfig, prompt: string): string {
     "",
     "Output contract:",
     ...PANEL_OUTPUT_CONTRACT,
+    ...(includeDecisionRecord
+      ? [
+          "",
+          "Decision record:",
+          "- End with exactly one single-line JSON record wrapped in the tags below.",
+          "- Keep the complete human-readable answer in the Markdown sections above the record.",
+          "- recommendation: one short plain-language conclusion.",
+          "- confidence: low, medium, or high.",
+          "- needsMoreEvidence: true when the answer should not be trusted without more investigation.",
+          `- Format: ${PANEL_DECISION_OPEN}{"recommendation":"...","confidence":"high","needsMoreEvidence":false}${PANEL_DECISION_CLOSE}`,
+          "- Do not add Markdown or any other text after the record.",
+        ]
+      : []),
   ].join("\n");
 }