@wichayutdew/pi-workflows 2.5.1 → 2.6.0

package/src/integrations/subagents/failure-correlation.ts

@@ -1,285 +0,0 @@
-import { comparableDiagnosticFragments } from './diagnostic-text.ts';
-import type {
-  RecordedCompletion,
-  RecordedToolCall,
-  RecordedToolFailure,
-  RecordedTranscriptWarning,
-  ToolFailureDiagnostic,
-} from './diagnostic-types.ts';
-import { parseFailureTranscript } from './failure-transcript.ts';
-import { reproduceHiddenBashFalsePositive } from './hidden-bash-failure.ts';
-import { isFailureTranscriptReplaySafe } from './replay-safety.ts';
-
-const diagnosticMatchesTerminalError = (
-  diagnostic: ToolFailureDiagnostic,
-  terminalError: string,
-): boolean => {
-  if (!diagnostic.output) return false;
-  const detail =
-    terminalError.match(
-      /\b[a-z][\w-]* failed(?:\s*\([^)]*\))?\s*:\s*([\s\S]+)/i,
-    )?.[1] ?? terminalError;
-  const outputFragments = comparableDiagnosticFragments(diagnostic.output);
-  const errorFragments = comparableDiagnosticFragments(
-    `${terminalError}\n${detail}`,
-  );
-  return outputFragments.some((output) =>
-    errorFragments.some(
-      (error) => output.includes(error) || error.includes(output),
-    ),
-  );
-};
-
-const latestMatching = (
-  diagnostics: ReadonlyArray<RecordedToolFailure>,
-  predicate: (diagnostic: RecordedToolFailure) => boolean,
-): RecordedToolFailure | undefined => {
-  for (let index = diagnostics.length - 1; index >= 0; index -= 1) {
-    const diagnostic = diagnostics[index];
-    if (diagnostic && predicate(diagnostic)) return diagnostic;
-  }
-  return undefined;
-};
-
-const latestWarningAfter = (
-  warnings: ReadonlyArray<RecordedTranscriptWarning>,
-  order: number,
-): RecordedTranscriptWarning | undefined => {
-  for (let index = warnings.length - 1; index >= 0; index -= 1) {
-    const warning = warnings[index];
-    if (warning && warning.order > order) return warning;
-  }
-  return undefined;
-};
-
-const finalCompletion = (
-  completions: ReadonlyArray<RecordedCompletion>,
-  latestFailureOrder: number,
-  lastInteractionOrder: number,
-  allowCompletionProof: boolean,
-): RecordedCompletion | undefined => {
-  if (!allowCompletionProof || completions.length !== 1) return undefined;
-  const completion = completions[0];
-  return completion &&
-    completion.order > latestFailureOrder &&
-    completion.order === lastInteractionOrder
-    ? completion
-    : undefined;
-};
-
-const publicDiagnostic = (
-  diagnostic: RecordedToolFailure,
-  diagnostics: ReadonlyArray<RecordedToolFailure>,
-  recordedCalls: ReadonlyArray<RecordedToolCall>,
-  successfulCompletions: ReadonlyArray<RecordedCompletion>,
-  postCompletionWarning: RecordedTranscriptWarning | undefined,
-  lastInteractionOrder: number,
-  allowCompletionProof: boolean,
-  correlation?: ToolFailureDiagnostic['correlation'],
-): ToolFailureDiagnostic => {
-  const result: ToolFailureDiagnostic = {
-    tool: diagnostic.tool,
-    ...(diagnostic.call ? { call: diagnostic.call } : {}),
-    ...(diagnostic.output ? { output: diagnostic.output } : {}),
-  };
-  const latestFailureOrder = diagnostics.at(-1)?.order ?? diagnostic.order;
-  const completion = finalCompletion(
-    successfulCompletions,
-    latestFailureOrder,
-    lastInteractionOrder,
-    allowCompletionProof,
-  );
-  return {
-    ...result,
-    ...(postCompletionWarning
-      ? { postCompletionWarning: postCompletionWarning.content }
-      : {}),
-    ...(isFailureTranscriptReplaySafe({
-      calls: recordedCalls,
-      diagnostics,
-      isCompleteTranscript: allowCompletionProof,
-    })
-      ? { replaySafe: true as const }
-      : {}),
-    ...(completion
-      ? {
-          completionAfterFailure: true as const,
-          completionValue: completion.value,
-        }
-      : {}),
-    ...(correlation ? { correlation } : {}),
-  };
-};
-
-const hiddenFalsePositiveDiagnostic = (
-  expectedTool: string | undefined,
-  terminalError: string,
-  allowCompletionProof: boolean,
-  transcript: ReturnType<typeof parseFailureTranscript>,
-): ToolFailureDiagnostic | undefined => {
-  const {
-    diagnostics,
-    hasValidFalsePositiveProof,
-    lastInteractionOrder,
-    recordedCalls,
-    recordedMessages,
-    resultCallIds,
-    successfulCompletions,
-    successfulResults,
-    transcriptWarnings,
-  } = transcript;
-  const completionOrder = successfulCompletions.at(-1)?.order ?? 0;
-  const postCompletionWarning = latestWarningAfter(
-    transcriptWarnings,
-    completionOrder,
-  );
-
-  const falsePositive = reproduceHiddenBashFalsePositive(
-    recordedMessages,
-    successfulResults,
-  );
-  const hasLaterFailure =
-    falsePositive !== undefined &&
-    diagnostics.some(
-      (diagnostic) => diagnostic.order > falsePositive.result.order,
-    );
-  const completion = falsePositive
-    ? finalCompletion(
-        successfulCompletions,
-        Math.max(
-          falsePositive.result.order,
-          diagnostics.at(-1)?.order ?? falsePositive.result.order,
-        ),
-        lastInteractionOrder,
-        allowCompletionProof,
-      )
-    : undefined;
-  if (
-    !hasValidFalsePositiveProof ||
-    recordedCalls.length !== resultCallIds.size ||
-    !recordedCalls.every((call) => resultCallIds.has(call.id)) ||
-    recordedCalls.filter((call) => call.tool === 'structured_output').length !==
-      1 ||
-    expectedTool?.toLowerCase() !== 'bash' ||
-    !falsePositive ||
-    hasLaterFailure ||
-    postCompletionWarning !== undefined ||
-    terminalError !== falsePositive.terminalError ||
-    !completion
-  ) {
-    return undefined;
-  }
-
-  const successfulOutput = falsePositive.result;
-  return {
-    tool: successfulOutput.tool,
-    ...(successfulOutput.call ? { call: successfulOutput.call } : {}),
-    ...(successfulOutput.output ? { output: successfulOutput.output } : {}),
-    completionAfterFailure: true,
-    completionValue: completion.value,
-    transcriptToolCount: recordedCalls.length,
-    transcriptTurnCount: recordedMessages.filter(
-      ({ value }) => value.role === 'assistant',
-    ).length,
-    correlation: 'successful-output-before-completion',
-  };
-};
-
-/**
- * Correlates a terminal tool failure with trusted evidence from a child
- * transcript.
- */
-export const parseToolFailureDiagnostic = (
-  transcript: string,
-  expectedTool?: string,
-  terminalError?: string,
-  allowCompletionProof = true,
-): ToolFailureDiagnostic | undefined => {
-  const parsed = parseFailureTranscript(transcript);
-  const {
-    diagnostics,
-    lastInteractionOrder,
-    recordedCalls,
-    successfulCompletions,
-    transcriptWarnings,
-  } = parsed;
-  const completionOrder = successfulCompletions.at(-1)?.order ?? 0;
-  const postCompletionWarning = latestWarningAfter(
-    transcriptWarnings,
-    completionOrder,
-  );
-  const matchesExpectedTool = (
-    diagnostic: Pick<RecordedToolFailure, 'tool'>,
-  ): boolean =>
-    expectedTool === undefined ||
-    diagnostic.tool.toLowerCase() === expectedTool.toLowerCase();
-
-  let selected: RecordedToolFailure | undefined;
-  if (terminalError) {
-    selected = latestMatching(
-      diagnostics,
-      (diagnostic) =>
-        matchesExpectedTool(diagnostic) &&
-        diagnosticMatchesTerminalError(diagnostic, terminalError),
-    );
-    if (!selected) {
-      const fallback = latestMatching(diagnostics, matchesExpectedTool);
-      const latestFailureOrder = diagnostics.at(-1)?.order;
-      if (
-        fallback &&
-        latestFailureOrder !== undefined &&
-        finalCompletion(
-          successfulCompletions,
-          latestFailureOrder,
-          lastInteractionOrder,
-          allowCompletionProof,
-        )
-      ) {
-        return publicDiagnostic(
-          fallback,
-          diagnostics,
-          recordedCalls,
-          successfulCompletions,
-          postCompletionWarning,
-          lastInteractionOrder,
-          allowCompletionProof,
-          'latest-before-completion',
-        );
-      }
-    }
-    if (!selected) {
-      const falsePositive = hiddenFalsePositiveDiagnostic(
-        expectedTool,
-        terminalError,
-        allowCompletionProof,
-        parsed,
-      );
-      if (falsePositive) return falsePositive;
-    }
-  } else {
-    selected = latestMatching(diagnostics, matchesExpectedTool);
-  }
-
-  if (!selected && postCompletionWarning) {
-    selected =
-      latestMatching(diagnostics, matchesExpectedTool) ?? diagnostics.at(-1);
-    if (!selected) {
-      return {
-        tool: expectedTool ?? 'subagent',
-        postCompletionWarning: postCompletionWarning.content,
-      };
-    }
-  }
-
-  return selected
-    ? publicDiagnostic(
-        selected,
-        diagnostics,
-        recordedCalls,
-        successfulCompletions,
-        postCompletionWarning,
-        lastInteractionOrder,
-        allowCompletionProof,
-      )
-    : undefined;
-};

package/src/integrations/subagents/failure-transcript.ts

@@ -1,252 +0,0 @@
-import {
-  diagnosticTextContent,
-  diagnosticToolCallText,
-  firstDiagnosticTextContent,
-  isDiagnosticRecord,
-  structuredCompletionValue,
-} from './diagnostic-text.ts';
-import type {
-  ParsedFailureTranscript,
-  RecordedCompletion,
-  RecordedMessage,
-  RecordedToolCall,
-  RecordedToolFailure,
-  RecordedToolSuccess,
-  RecordedTranscriptWarning,
-} from './diagnostic-types.ts';
-
-const WATCHDOG_WARNING_TYPE = 'subagent_watchdog_warning';
-const MAX_WARNING_FIELD_CHARS = 600;
-
-const boundedWarningField = (value: string): string => {
-  const normalized = value.trim().replaceAll(/\s+/g, ' ');
-  return normalized.length <= MAX_WARNING_FIELD_CHARS
-    ? normalized
-    : `${normalized.slice(0, MAX_WARNING_FIELD_CHARS - 1)}…`;
-};
-
-const transcriptWarningContent = (
-  entry: Readonly<Record<string, unknown>>,
-): string | undefined => {
-  if (
-    entry.type !== 'custom_message' ||
-    entry.customType !== WATCHDOG_WARNING_TYPE
-  ) {
-    return undefined;
-  }
-  const details = entry.details;
-  if (isDiagnosticRecord(details)) {
-    const fields = [
-      typeof details.summary === 'string' ? details.summary : undefined,
-      typeof details.evidence === 'string' ? details.evidence : undefined,
-      typeof details.recommendedAction === 'string'
-        ? `Recommended action: ${details.recommendedAction}`
-        : undefined,
-    ].filter((field): field is string => Boolean(field?.trim()));
-    if (fields.length > 0) return boundedWarningField(fields.join(' '));
-  }
-  return typeof entry.content === 'string' && entry.content.trim()
-    ? boundedWarningField(entry.content)
-    : 'The child emitted an unresolved watchdog warning after completion.';
-};
-
-const isBenignTerminalAssistant = (
-  message: Readonly<Record<string, unknown>>,
-): boolean =>
-  message.stopReason === 'stop' &&
-  message.errorMessage === undefined &&
-  Array.isArray(message.content) &&
-  message.content.length === 1 &&
-  isDiagnosticRecord(message.content[0]) &&
-  message.content[0].type === 'text' &&
-  message.content[0].text === '';
-
-/**
- * Parses the tool calls, results, completions, and structural evidence needed
- * to correlate a terminal failure.
- */
-export const parseFailureTranscript = (
-  transcript: string,
-): ParsedFailureTranscript => {
-  const calls = new Map<string, RecordedToolCall>();
-  const recordedCalls: Array<RecordedToolCall> = [];
-  const diagnostics: Array<RecordedToolFailure> = [];
-  const successfulResults: Array<RecordedToolSuccess> = [];
-  const successfulCompletions: Array<RecordedCompletion> = [];
-  const transcriptWarnings: Array<RecordedTranscriptWarning> = [];
-  const recordedMessages: Array<RecordedMessage> = [];
-  const resultCallIds = new Set<string>();
-  let hasValidFalsePositiveProof = true;
-  let lastInteractionOrder = 0;
-  let order = 0;
-
-  for (const line of transcript.split('\n')) {
-    order += 1;
-    if (!line.trim()) continue;
-
-    let entry: unknown;
-    try {
-      entry = JSON.parse(line);
-    } catch {
-      hasValidFalsePositiveProof = false;
-      continue;
-    }
-    if (isDiagnosticRecord(entry)) {
-      const warning = transcriptWarningContent(entry);
-      if (warning) {
-        transcriptWarnings.push({ order, content: warning });
-        lastInteractionOrder = order;
-        continue;
-      }
-    }
-    if (!isDiagnosticRecord(entry) || entry.type !== 'message') continue;
-    const message = entry.message;
-    if (!isDiagnosticRecord(message)) {
-      hasValidFalsePositiveProof = false;
-      continue;
-    }
-    recordedMessages.push({ order, value: message });
-    if (
-      message.role === 'assistant' &&
-      ((typeof message.errorMessage === 'string' &&
-        message.errorMessage.trim().length > 0) ||
-        message.stopReason === 'error' ||
-        message.stopReason === 'aborted')
-    ) {
-      hasValidFalsePositiveProof = false;
-    }
-
-    if (message.role === 'assistant') {
-      if (!Array.isArray(message.content)) {
-        hasValidFalsePositiveProof = false;
-        continue;
-      }
-      if (!isBenignTerminalAssistant(message)) {
-        lastInteractionOrder = order;
-      }
-      const toolCalls = message.content.filter(
-        (item): item is Record<string, unknown> =>
-          isDiagnosticRecord(item) &&
-          item.type === 'toolCall' &&
-          typeof item.id === 'string' &&
-          typeof item.name === 'string',
-      );
-      for (const item of message.content) {
-        if (
-          !isDiagnosticRecord(item) ||
-          item.type !== 'toolCall' ||
-          typeof item.id !== 'string' ||
-          typeof item.name !== 'string'
-        ) {
-          if (isDiagnosticRecord(item) && item.type === 'toolCall') {
-            hasValidFalsePositiveProof = false;
-          }
-          continue;
-        }
-        if (calls.has(item.id)) hasValidFalsePositiveProof = false;
-
-        const call = diagnosticToolCallText({
-          tool: item.name,
-          argumentsValue: item.arguments,
-        });
-        const isExclusiveCompletion =
-          toolCalls.length === 1 &&
-          message.content.every(
-            (contentItem) =>
-              isDiagnosticRecord(contentItem) &&
-              (contentItem.type === 'thinking' ||
-                contentItem.type === 'text' ||
-                contentItem.type === 'toolCall'),
-          );
-        const completionValue =
-          item.name === 'structured_output' && isExclusiveCompletion
-            ? structuredCompletionValue(item.arguments)
-            : undefined;
-        const recordedCall: RecordedToolCall = {
-          id: item.id,
-          order,
-          tool: item.name,
-          ...(call ? { call } : {}),
-          ...(completionValue ? { completionValue } : {}),
-        };
-        calls.set(item.id, recordedCall);
-        recordedCalls.push(recordedCall);
-      }
-      continue;
-    }
-
-    if (message.role !== 'toolResult' || typeof message.toolName !== 'string') {
-      if (message.role === 'toolResult') hasValidFalsePositiveProof = false;
-      continue;
-    }
-    lastInteractionOrder = order;
-    if (
-      typeof message.toolCallId !== 'string' ||
-      typeof message.isError !== 'boolean' ||
-      !Array.isArray(message.content) ||
-      resultCallIds.has(message.toolCallId)
-    ) {
-      hasValidFalsePositiveProof = false;
-    }
-    if (typeof message.toolCallId === 'string') {
-      resultCallIds.add(message.toolCallId);
-    }
-    const recorded =
-      typeof message.toolCallId === 'string'
-        ? calls.get(message.toolCallId)
-        : undefined;
-    const doesCallMatchResult = recorded?.tool === message.toolName;
-    if (!doesCallMatchResult) hasValidFalsePositiveProof = false;
-
-    if (
-      message.toolName === 'structured_output' &&
-      message.isError === false &&
-      doesCallMatchResult &&
-      recorded.completionValue
-    ) {
-      successfulCompletions.push({
-        order,
-        value: recorded.completionValue,
-      });
-    }
-    if (
-      message.isError === false &&
-      message.toolName !== 'structured_output' &&
-      doesCallMatchResult
-    ) {
-      const output = diagnosticTextContent(message.content);
-      const detectorOutput = firstDiagnosticTextContent(message.content);
-      successfulResults.push({
-        order,
-        tool: message.toolName,
-        ...(recorded.call ? { call: recorded.call } : {}),
-        ...(output ? { output } : {}),
-        ...(detectorOutput !== undefined ? { detectorOutput } : {}),
-      });
-    }
-    if (message.isError !== true) continue;
-
-    const output = diagnosticTextContent(message.content);
-    diagnostics.push({
-      tool: message.toolName,
-      ...(doesCallMatchResult && recorded.call ? { call: recorded.call } : {}),
-      ...(output ? { output } : {}),
-      ...(doesCallMatchResult && typeof message.toolCallId === 'string'
-        ? { callId: message.toolCallId }
-        : {}),
-      order,
-    });
-  }
-
-  return {
-    recordedCalls,
-    diagnostics,
-    successfulResults,
-    successfulCompletions,
-    transcriptWarnings,
-    recordedMessages,
-    resultCallIds,
-    hasValidFalsePositiveProof,
-    lastInteractionOrder,
-  };
-};

package/src/integrations/subagents/hidden-bash-failure.ts

@@ -1,98 +0,0 @@
-import {
-  firstDiagnosticTextContent,
-  isDiagnosticRecord,
-} from './diagnostic-text.ts';
-import type {
-  RecordedMessage,
-  RecordedToolSuccess,
-} from './diagnostic-types.ts';
-
-const HIDDEN_BASH_FATAL_PATTERNS = [
-  /command not found/i,
-  /permission denied/i,
-  /no such file or directory/i,
-  /segmentation fault/i,
-  /killed|terminated/i,
-  /out of memory/i,
-  /connection refused/i,
-  /timeout/i,
-] as const;
-const HIDDEN_BASH_EXIT_PATTERN =
-  /exit(?:ed)?\s*(?:with\s*)?(?:code|status)?\s*[:\s]?\s*(\d+)/i;
-
-export type HiddenBashFalsePositive = {
-  readonly result: RecordedToolSuccess;
-  readonly terminalError: string;
-};
-
-const lastAssistantTextIndex = (
-  messages: ReadonlyArray<RecordedMessage>,
-): number => {
-  for (let index = messages.length - 1; index >= 0; index -= 1) {
-    const message = messages[index]?.value;
-    const hasAssistantText =
-      message?.role === 'assistant' &&
-      Array.isArray(message.content) &&
-      message.content.some(
-        (item) =>
-          isDiagnosticRecord(item) &&
-          item.type === 'text' &&
-          typeof item.text === 'string' &&
-          item.text.trim().length > 0,
-      );
-    if (hasAssistantText) return index;
-  }
-  return -1;
-};
-
-const hiddenExitCode = (output: string): number | undefined => {
-  const exitCodeText = output.match(HIDDEN_BASH_EXIT_PATTERN)?.[1];
-  const exitCode =
-    exitCodeText === undefined ? undefined : Number.parseInt(exitCodeText, 10);
-  if (exitCode !== undefined && exitCode !== 0) return exitCode;
-  return HIDDEN_BASH_FATAL_PATTERNS.some((pattern) => pattern.test(output))
-    ? 1
-    : undefined;
-};
-
-/**
- * Reproduces pi-subagents' hidden Bash failure detection from successful tool
- * output.
- */
-export const reproduceHiddenBashFalsePositive = (
-  messages: ReadonlyArray<RecordedMessage>,
-  successfulResults: ReadonlyArray<RecordedToolSuccess>,
-): HiddenBashFalsePositive | undefined => {
-  const assistantTextIndex = lastAssistantTextIndex(messages);
-  const scanStart = assistantTextIndex >= 0 ? assistantTextIndex + 1 : 0;
-
-  for (let index = messages.length - 1; index >= scanStart; index -= 1) {
-    const recordedMessage = messages[index];
-    const message = recordedMessage?.value;
-    if (
-      !recordedMessage ||
-      message?.role !== 'toolResult' ||
-      message.toolName !== 'bash' ||
-      message.isError !== false
-    ) {
-      continue;
-    }
-    const output = firstDiagnosticTextContent(message.content);
-    if (output === undefined) continue;
-
-    const detectedExitCode = hiddenExitCode(output);
-    if (detectedExitCode === undefined) continue;
-    const result = successfulResults.find(
-      (candidate) =>
-        candidate.order === recordedMessage.order &&
-        candidate.tool === 'bash' &&
-        candidate.detectorOutput === output,
-    );
-    if (!result) return undefined;
-    return {
-      result,
-      terminalError: `bash failed (exit ${detectedExitCode}): ${output.slice(0, 200)}`,
-    };
-  }
-  return undefined;
-};