@alexeiled/pi-fusion 0.3.0 → 0.5.0

package/src/run-observations.ts

@@ -0,0 +1,335 @@
+import type {
+  ModelAttempt,
+  PanelConfidence,
+  PanelDecision,
+  PanelOutput,
+  ProviderFailure,
+  RunObservation,
+  RunUsage,
+} from "./types.js";
+import { isFiniteNumber, isRecord } from "./utils.js";
+
+export const PANEL_DECISION_OPEN = "<fusion-panel-decision>";
+export const PANEL_DECISION_CLOSE = "</fusion-panel-decision>";
+
+export function extractPanelDecision(
+  value: unknown,
+): PanelDecision | undefined {
+  if (typeof value === "string") return extractTaggedPanelDecision(value);
+  if (Array.isArray(value) && value.every((item) => typeof item === "string")) {
+    return extractTaggedPanelDecision(value.join("\n"));
+  }
+  return isRecord(value) ? panelDecisionFromRecord(value) : undefined;
+}
+
+function extractTaggedPanelDecision(value: string): PanelDecision | undefined {
+  const openIndex = value.lastIndexOf(PANEL_DECISION_OPEN);
+  if (openIndex < 0) return undefined;
+  const jsonStart = openIndex + PANEL_DECISION_OPEN.length;
+  const closeIndex = value.indexOf(PANEL_DECISION_CLOSE, jsonStart);
+  if (closeIndex < 0) return undefined;
+  const closeEnd = closeIndex + PANEL_DECISION_CLOSE.length;
+  if (value.slice(closeEnd).trim()) return undefined;
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(value.slice(jsonStart, closeIndex));
+  } catch {
+    return undefined;
+  }
+  if (!isRecord(parsed)) return undefined;
+
+  const answerMarkdown = value.slice(0, openIndex).trim();
+  return panelDecisionFromRecord(parsed, answerMarkdown);
+}
+
+function panelDecisionFromRecord(
+  value: Record<string, unknown>,
+  fallbackAnswerMarkdown?: string,
+): PanelDecision | undefined {
+  const recommendation = firstNonBlankString(value.recommendation);
+  const answerMarkdown =
+    firstNonBlankString(value.answerMarkdown) ?? fallbackAnswerMarkdown;
+  const confidence = value.confidence;
+  if (
+    !recommendation ||
+    !answerMarkdown ||
+    !isPanelConfidence(confidence) ||
+    typeof value.needsMoreEvidence !== "boolean"
+  ) {
+    return undefined;
+  }
+  return {
+    recommendation,
+    confidence,
+    needsMoreEvidence: value.needsMoreEvidence,
+    answerMarkdown,
+  };
+}
+
+export function normalizeRecommendation(value: string): string {
+  return value
+    .toLocaleLowerCase()
+    .replace(/[^\p{L}\p{N}]+/gu, " ")
+    .trim();
+}
+
+export function hasStrongPanelAgreement(
+  outputs: readonly PanelOutput[],
+  completedPanelCount: number,
+  totalPanelCount: number,
+): boolean {
+  if (outputs.length < 2 || completedPanelCount >= totalPanelCount) {
+    return false;
+  }
+  const decisions = outputs.map((output) => output.decision);
+  if (decisions.some((decision) => !decision)) return false;
+  if (
+    decisions.some(
+      (decision) =>
+        decision?.confidence !== "high" || decision.needsMoreEvidence,
+    )
+  ) {
+    return false;
+  }
+  const recommendation = normalizeRecommendation(
+    decisions[0]?.recommendation ?? "",
+  );
+  return Boolean(
+    recommendation &&
+    decisions.every(
+      (decision) =>
+        decision &&
+        normalizeRecommendation(decision.recommendation) === recommendation,
+    ),
+  );
+}
+
+export function mergeRunObservations(
+  base: RunObservation | undefined,
+  latest: RunObservation | undefined,
+): RunObservation {
+  if (!base) return latest ? cloneObservation(latest) : {};
+  if (!latest) return cloneObservation(base);
+  const failures = summarizeProviderFailures(
+    latest.providerFailures ?? base.providerFailures ?? [],
+  ).map(({ count, ...failure }) =>
+    count && count > 1 ? { ...failure, count } : failure,
+  );
+  return {
+    ...((latest.model ?? base.model)
+      ? { model: latest.model ?? base.model }
+      : {}),
+    ...((latest.durationMs ?? base.durationMs) !== undefined
+      ? { durationMs: latest.durationMs ?? base.durationMs }
+      : {}),
+    ...mergeUsage(base.usage, latest.usage),
+    ...((latest.attempts ?? base.attempts)
+      ? {
+          attempts: [...(latest.attempts ?? base.attempts ?? [])].map(
+            (attempt) => ({ ...attempt }),
+          ),
+        }
+      : {}),
+    ...(failures.length > 0 ? { providerFailures: failures } : {}),
+  };
+}
+
+export function extractRunObservation(value: unknown): RunObservation {
+  if (!isRecord(value)) return {};
+
+  const model = firstString(value.model);
+  const durationMs = extractDuration(value);
+  const usage = extractUsage(value);
+  const attempts = extractAttempts(value.modelAttempts);
+  const attemptFailures = attempts.flatMap((attempt) =>
+    attempt.success || !attempt.error
+      ? []
+      : [providerFailureFromAttempt(attempt)],
+  );
+  const rawError = firstNonBlankString(value.error);
+  const providerFailures = summarizeProviderFailures(
+    rawError &&
+      attemptFailures.length === 0 &&
+      (value.success === false || value.state === "failed")
+      ? [
+          {
+            provider: model ? providerFromModel(model) : "unknown provider",
+            ...(model ? { model } : {}),
+            message: rawError,
+          },
+        ]
+      : attemptFailures,
+  ).map(({ count, ...failure }) =>
+    count && count > 1 ? { ...failure, count } : failure,
+  );
+
+  return {
+    ...(model ? { model } : {}),
+    ...(durationMs !== undefined ? { durationMs } : {}),
+    ...(usage ? { usage } : {}),
+    ...(attempts.length > 0 ? { attempts } : {}),
+    ...(providerFailures.length > 0 ? { providerFailures } : {}),
+  };
+}
+
+export function summarizeProviderFailures(
+  failures: readonly ProviderFailure[],
+): ProviderFailure[] {
+  const grouped = new Map<string, ProviderFailure & { count: number }>();
+  for (const failure of failures) {
+    const message = failure.message.trim();
+    if (!message) continue;
+    const key = `${failure.provider}\u0000${failure.model ?? ""}\u0000${message}`;
+    const existing = grouped.get(key);
+    if (existing) {
+      existing.count += failure.count ?? 1;
+      continue;
+    }
+    grouped.set(key, {
+      ...failure,
+      message,
+      count: failure.count ?? 1,
+    });
+  }
+
+  return [...grouped.values()]
+    .sort((left, right) =>
+      `${left.provider}\u0000${left.model ?? ""}\u0000${left.message}`.localeCompare(
+        `${right.provider}\u0000${right.model ?? ""}\u0000${right.message}`,
+      ),
+    )
+    .map(({ count, ...failure }) => ({ ...failure, count }));
+}
+
+function mergeUsage(
+  base: RunUsage | undefined,
+  latest: RunUsage | undefined,
+): { usage?: RunUsage } {
+  if (!base && !latest) return {};
+  const inputTokens = latest?.inputTokens ?? base?.inputTokens;
+  const outputTokens = latest?.outputTokens ?? base?.outputTokens;
+  const costUsd = latest?.costUsd ?? base?.costUsd;
+  const usage: RunUsage = {
+    ...(inputTokens !== undefined ? { inputTokens } : {}),
+    ...(outputTokens !== undefined ? { outputTokens } : {}),
+    ...(costUsd !== undefined ? { costUsd } : {}),
+  };
+  return { usage };
+}
+
+function cloneObservation(observation: RunObservation): RunObservation {
+  return {
+    ...observation,
+    ...(observation.usage ? { usage: { ...observation.usage } } : {}),
+    ...(observation.attempts
+      ? { attempts: observation.attempts.map((attempt) => ({ ...attempt })) }
+      : {}),
+    ...(observation.providerFailures
+      ? {
+          providerFailures: observation.providerFailures.map((failure) => ({
+            ...failure,
+          })),
+        }
+      : {}),
+  };
+}
+
+function extractDuration(value: Record<string, unknown>): number | undefined {
+  if (isFiniteNumber(value.durationMs) && value.durationMs >= 0) {
+    return value.durationMs;
+  }
+  if (
+    isFiniteNumber(value.startedAt) &&
+    isFiniteNumber(value.endedAt) &&
+    value.endedAt >= value.startedAt
+  ) {
+    return value.endedAt - value.startedAt;
+  }
+  return undefined;
+}
+
+function extractUsage(value: Record<string, unknown>): RunUsage | undefined {
+  const totalCost = isRecord(value.totalCost) ? value.totalCost : undefined;
+  const rawUsage = isRecord(value.usage) ? value.usage : undefined;
+  const inputTokens = firstFinite(
+    totalCost?.inputTokens,
+    rawUsage?.inputTokens,
+    rawUsage?.input,
+  );
+  const outputTokens = firstFinite(
+    totalCost?.outputTokens,
+    rawUsage?.outputTokens,
+    rawUsage?.output,
+  );
+  const costUsd = firstFinite(
+    totalCost?.costUsd,
+    rawUsage?.costUsd,
+    isRecord(rawUsage?.cost) ? rawUsage.cost.total : undefined,
+  );
+
+  if (
+    inputTokens === undefined &&
+    outputTokens === undefined &&
+    costUsd === undefined
+  ) {
+    return undefined;
+  }
+  return {
+    ...(inputTokens !== undefined ? { inputTokens } : {}),
+    ...(outputTokens !== undefined ? { outputTokens } : {}),
+    ...(costUsd !== undefined ? { costUsd } : {}),
+  };
+}
+
+function extractAttempts(value: unknown): ModelAttempt[] {
+  if (!Array.isArray(value)) return [];
+  const attempts: ModelAttempt[] = [];
+  for (const item of value) {
+    if (!isRecord(item)) continue;
+    const model = firstString(item.model);
+    if (!model || typeof item.success !== "boolean") continue;
+    const error = firstString(item.error);
+    attempts.push({
+      model,
+      success: item.success,
+      ...(error ? { error } : {}),
+    });
+  }
+  return attempts;
+}
+
+function providerFailureFromAttempt(attempt: ModelAttempt): ProviderFailure {
+  return {
+    provider: providerFromModel(attempt.model),
+    model: attempt.model,
+    message: attempt.error ?? "model attempt failed",
+  };
+}
+
+function providerFromModel(model: string): string {
+  return model.split("/", 1)[0] || "unknown provider";
+}
+
+function firstFinite(...values: readonly unknown[]): number | undefined {
+  return values.find(isFiniteNumber);
+}
+
+function isPanelConfidence(value: unknown): value is PanelConfidence {
+  return value === "low" || value === "medium" || value === "high";
+}
+
+function firstNonBlankString(
+  ...values: readonly unknown[]
+): string | undefined {
+  for (const value of values) {
+    if (typeof value !== "string") continue;
+    const trimmed = value.trim();
+    if (trimmed) return trimmed;
+  }
+  return undefined;
+}
+
+function firstString(...values: readonly unknown[]): string | undefined {
+  return values.find((value): value is string => typeof value === "string");
+}

package/src/run-store.ts

@@ -1,5 +1,13 @@
 import { randomUUID } from "node:crypto";
-import type { FusionPhase, FusionRun } from "./types.js";
+import type {
+  FusionPhase,
+  FusionRun,
+  ModelAttempt,
+  PanelDecision,
+  ProviderFailure,
+  RunObservation,
+  RunUsage,
+} from "./types.js";
 import { isFiniteNumber, isNonEmptyString, isRecord } from "./utils.js";
 
 export const FUSION_RUN_ENTRY_TYPE = "fusion-run";
@@ -40,8 +48,12 @@ export interface FusionRunPatch {
   chainRunId?: string;
   chainAsyncDir?: string;
   panelRunId?: string;
+  panelAsyncDir?: string;
+  panelStopReason?: FusionRun["panelStopReason"];
+  panelStoppedIndices?: FusionRun["panelStoppedIndices"];
   judgeRunId?: string;
   judgeAsyncDir?: string;
+  judgeObservation?: FusionRun["judgeObservation"];
   panelOutputs?: FusionRun["panelOutputs"];
   panelFailures?: FusionRun["panelFailures"];
   report?: string;
@@ -260,10 +272,22 @@ function applyPatch(
     updated.chainAsyncDir = patch.chainAsyncDir;
   }
   if (patch.panelRunId !== undefined) updated.panelRunId = patch.panelRunId;
+  if (patch.panelAsyncDir !== undefined) {
+    updated.panelAsyncDir = patch.panelAsyncDir;
+  }
+  if (patch.panelStopReason !== undefined) {
+    updated.panelStopReason = patch.panelStopReason;
+  }
+  if (patch.panelStoppedIndices !== undefined) {
+    updated.panelStoppedIndices = [...patch.panelStoppedIndices];
+  }
   if (patch.judgeRunId !== undefined) updated.judgeRunId = patch.judgeRunId;
   if (patch.judgeAsyncDir !== undefined) {
     updated.judgeAsyncDir = patch.judgeAsyncDir;
   }
+  if (patch.judgeObservation !== undefined) {
+    updated.judgeObservation = cloneObservation(patch.judgeObservation);
+  }
   if (patch.panelOutputs !== undefined) {
     updated.panelOutputs = clonePanelOutputs(patch.panelOutputs);
   }
@@ -325,10 +349,22 @@ function cloneRun(run: FusionRun): FusionRun {
       ? { chainAsyncDir: run.chainAsyncDir }
       : {}),
     ...(run.panelRunId !== undefined ? { panelRunId: run.panelRunId } : {}),
+    ...(run.panelAsyncDir !== undefined
+      ? { panelAsyncDir: run.panelAsyncDir }
+      : {}),
+    ...(run.panelStopReason !== undefined
+      ? { panelStopReason: run.panelStopReason }
+      : {}),
+    ...(run.panelStoppedIndices !== undefined
+      ? { panelStoppedIndices: [...run.panelStoppedIndices] }
+      : {}),
     ...(run.judgeRunId !== undefined ? { judgeRunId: run.judgeRunId } : {}),
     ...(run.judgeAsyncDir !== undefined
       ? { judgeAsyncDir: run.judgeAsyncDir }
       : {}),
+    ...(run.judgeObservation !== undefined
+      ? { judgeObservation: cloneObservation(run.judgeObservation) }
+      : {}),
     ...(run.panelOutputs !== undefined
       ? { panelOutputs: clonePanelOutputs(run.panelOutputs) }
       : {}),
@@ -375,6 +411,28 @@ function isFusionRunState(value: unknown): value is FusionRun {
   if (value.panelRunId !== undefined && typeof value.panelRunId !== "string") {
     return false;
   }
+  if (
+    value.panelAsyncDir !== undefined &&
+    typeof value.panelAsyncDir !== "string"
+  ) {
+    return false;
+  }
+  if (
+    value.panelStopReason !== undefined &&
+    value.panelStopReason !== "agreement"
+  ) {
+    return false;
+  }
+  if (
+    value.panelStoppedIndices !== undefined &&
+    (!Array.isArray(value.panelStoppedIndices) ||
+      !value.panelStoppedIndices.every(
+        (index: unknown) =>
+          typeof index === "number" && Number.isInteger(index) && index >= 0,
+      ))
+  ) {
+    return false;
+  }
   if (value.judgeRunId !== undefined && typeof value.judgeRunId !== "string") {
     return false;
   }
@@ -384,6 +442,12 @@ function isFusionRunState(value: unknown): value is FusionRun {
   ) {
     return false;
   }
+  if (
+    value.judgeObservation !== undefined &&
+    !isRunObservation(value.judgeObservation)
+  ) {
+    return false;
+  }
   if (
     value.panelOutputs !== undefined &&
     !isPanelOutputArray(value.panelOutputs)
@@ -440,9 +504,14 @@ function isPanelOutput(
     typeof value.output === "string" &&
     (value.id === undefined || typeof value.id === "string") &&
     (value.label === undefined || typeof value.label === "string") &&
+    (value.configuredModel === undefined ||
+      typeof value.configuredModel === "string") &&
     (value.artifactPath === undefined ||
       typeof value.artifactPath === "string") &&
-    (value.sessionPath === undefined || typeof value.sessionPath === "string")
+    (value.sessionPath === undefined ||
+      typeof value.sessionPath === "string") &&
+    (value.decision === undefined || isPanelDecision(value.decision)) &&
+    (value.observation === undefined || isRunObservation(value.observation))
   );
 }
 
@@ -462,20 +531,134 @@ function isPanelFailure(
     typeof value.summary === "string" &&
     (value.id === undefined || typeof value.id === "string") &&
     (value.label === undefined || typeof value.label === "string") &&
+    (value.configuredModel === undefined ||
+      typeof value.configuredModel === "string") &&
     (value.artifactPath === undefined ||
       typeof value.artifactPath === "string") &&
-    (value.sessionPath === undefined || typeof value.sessionPath === "string")
+    (value.sessionPath === undefined ||
+      typeof value.sessionPath === "string") &&
+    (value.reason === undefined || isPanelFailureReason(value.reason)) &&
+    (value.observation === undefined || isRunObservation(value.observation))
   );
 }
 
 function clonePanelOutputs(
   outputs: NonNullable<FusionRun["panelOutputs"]>,
 ): NonNullable<FusionRun["panelOutputs"]> {
-  return outputs.map((output) => ({ ...output }));
+  return outputs.map((output) => ({
+    ...output,
+    ...(output.decision
+      ? { decision: clonePanelDecision(output.decision) }
+      : {}),
+    ...(output.observation
+      ? { observation: cloneObservation(output.observation) }
+      : {}),
+  }));
 }
 
 function clonePanelFailures(
   failures: NonNullable<FusionRun["panelFailures"]>,
 ): NonNullable<FusionRun["panelFailures"]> {
-  return failures.map((failure) => ({ ...failure }));
+  return failures.map((failure) => ({
+    ...failure,
+    ...(failure.observation
+      ? { observation: cloneObservation(failure.observation) }
+      : {}),
+  }));
+}
+
+function cloneObservation(observation: RunObservation): RunObservation {
+  return {
+    ...(observation.model ? { model: observation.model } : {}),
+    ...(observation.durationMs !== undefined
+      ? { durationMs: observation.durationMs }
+      : {}),
+    ...(observation.usage ? { usage: { ...observation.usage } } : {}),
+    ...(observation.attempts
+      ? { attempts: observation.attempts.map((attempt) => ({ ...attempt })) }
+      : {}),
+    ...(observation.providerFailures
+      ? {
+          providerFailures: observation.providerFailures.map((failure) => ({
+            ...failure,
+          })),
+        }
+      : {}),
+  };
+}
+
+function clonePanelDecision(decision: PanelDecision): PanelDecision {
+  return { ...decision };
+}
+
+function isRunObservation(value: unknown): value is RunObservation {
+  if (!isRecord(value)) return false;
+  if (value.model !== undefined && typeof value.model !== "string")
+    return false;
+  if (value.durationMs !== undefined && !isFiniteNumber(value.durationMs)) {
+    return false;
+  }
+  if (value.usage !== undefined && !isRunUsage(value.usage)) return false;
+  if (value.attempts !== undefined && !isModelAttemptArray(value.attempts)) {
+    return false;
+  }
+  return (
+    value.providerFailures === undefined ||
+    (Array.isArray(value.providerFailures) &&
+      value.providerFailures.every(isProviderFailure))
+  );
+}
+
+function isRunUsage(value: unknown): value is RunUsage {
+  if (!isRecord(value)) return false;
+  return (
+    (value.inputTokens === undefined || isFiniteNumber(value.inputTokens)) &&
+    (value.outputTokens === undefined || isFiniteNumber(value.outputTokens)) &&
+    (value.costUsd === undefined || isFiniteNumber(value.costUsd))
+  );
+}
+
+function isModelAttemptArray(value: unknown): value is ModelAttempt[] {
+  return (
+    Array.isArray(value) &&
+    value.every((item) => {
+      if (!isRecord(item)) return false;
+      return (
+        typeof item.model === "string" &&
+        typeof item.success === "boolean" &&
+        (item.error === undefined || typeof item.error === "string")
+      );
+    })
+  );
+}
+
+function isProviderFailure(value: unknown): value is ProviderFailure {
+  if (!isRecord(value)) return false;
+  return (
+    typeof value.provider === "string" &&
+    typeof value.message === "string" &&
+    (value.model === undefined || typeof value.model === "string") &&
+    (value.count === undefined || isFiniteNumber(value.count))
+  );
+}
+
+function isPanelDecision(value: unknown): value is PanelDecision {
+  if (!isRecord(value)) return false;
+  return (
+    typeof value.recommendation === "string" &&
+    (value.confidence === "low" ||
+      value.confidence === "medium" ||
+      value.confidence === "high") &&
+    typeof value.needsMoreEvidence === "boolean" &&
+    typeof value.answerMarkdown === "string"
+  );
+}
+
+function isPanelFailureReason(value: unknown): boolean {
+  return (
+    value === "provider" ||
+    value === "timeout" ||
+    value === "interrupted" ||
+    value === "stopped-after-agreement"
+  );
 }

package/src/types.ts

@@ -31,6 +31,43 @@ export interface FusionProfile {
   concurrency?: number;
   timeoutMs?: number;
   context?: FusionContextMode;
+  stopWhenPanelAgrees?: boolean;
+}
+
+export type PanelConfidence = "low" | "medium" | "high";
+
+export interface PanelDecision {
+  recommendation: string;
+  confidence: PanelConfidence;
+  needsMoreEvidence: boolean;
+  answerMarkdown: string;
+}
+
+export interface RunUsage {
+  inputTokens?: number;
+  outputTokens?: number;
+  costUsd?: number;
+}
+
+export interface ModelAttempt {
+  model: string;
+  success: boolean;
+  error?: string;
+}
+
+export interface ProviderFailure {
+  provider: string;
+  model?: string;
+  message: string;
+  count?: number;
+}
+
+export interface RunObservation {
+  model?: string;
+  durationMs?: number;
+  usage?: RunUsage;
+  attempts?: ModelAttempt[];
+  providerFailures?: ProviderFailure[];
 }
 
 export interface FusionConfig {
@@ -51,10 +88,16 @@ export interface PanelOutput {
   label?: string;
   role?: string;
   model?: string;
+  configuredModel?: string;
+  decision?: PanelDecision;
+  observation?: RunObservation;
   artifactPath?: string;
   sessionPath?: string;
 }
 
+export type PanelFailureReason =
+  "provider" | "timeout" | "interrupted" | "stopped-after-agreement";
+
 export interface FailedPanelSummary {
   index: number;
   agent: string;
@@ -63,6 +106,9 @@ export interface FailedPanelSummary {
   label?: string;
   role?: string;
   model?: string;
+  configuredModel?: string;
+  reason?: PanelFailureReason;
+  observation?: RunObservation;
   artifactPath?: string;
   sessionPath?: string;
 }
@@ -80,8 +126,12 @@ export interface FusionRun {
   chainRunId?: string;
   chainAsyncDir?: string;
   panelRunId?: string;
+  panelAsyncDir?: string;
+  panelStopReason?: "agreement";
+  panelStoppedIndices?: number[];
   judgeRunId?: string;
   judgeAsyncDir?: string;
+  judgeObservation?: RunObservation;
   panelOutputs?: PanelOutput[];
   panelFailures?: FailedPanelSummary[];
   report?: string;