@bastani/atomic 0.8.26-alpha.5 → 0.8.26-alpha.7

package/dist/builtin/workflows/src/runs/foreground/stage-runner.ts

@@ -241,6 +241,59 @@ function lastAssistantTextFromMessages(messages: AgentSession["messages"]): stri
   return undefined;
 }
 
+function messageStopReason(message: AgentSession["messages"][number]): string | undefined {
+  const record = message as { readonly stopReason?: unknown };
+  return typeof record.stopReason === "string" ? record.stopReason : undefined;
+}
+
+function normalizedStopReason(stopReason: string | undefined): string | undefined {
+  return stopReason?.toLowerCase().replace(/[_-]+/g, "");
+}
+
+function isTerminalAssistantFailureStopReason(stopReason: string | undefined): boolean {
+  const normalized = normalizedStopReason(stopReason);
+  return normalized === "error" || normalized === "aborted";
+}
+
+function isCleanAssistantStopReason(stopReason: string | undefined): boolean {
+  const normalized = normalizedStopReason(stopReason);
+  return normalized === "stop" || normalized === "tooluse" || normalized === "length";
+}
+
+function assistantErrorMessage(message: AgentSession["messages"][number]): string | undefined {
+  const record = message as { readonly errorMessage?: unknown };
+  return typeof record.errorMessage === "string" && record.errorMessage.trim().length > 0
+    ? record.errorMessage
+    : undefined;
+}
+
+function latestTerminalAssistantFailureSince(
+  messages: AgentSession["messages"],
+  startIndex: number,
+): AgentSession["messages"][number] | undefined {
+  for (let index = messages.length - 1; index >= startIndex; index -= 1) {
+    const message = messages[index];
+    if (!message || message.role !== "assistant") continue;
+    const stopReason = messageStopReason(message);
+    if (isTerminalAssistantFailureStopReason(stopReason)) return message;
+    if (isCleanAssistantStopReason(stopReason)) return undefined;
+    if (assistantErrorMessage(message) === undefined && extractMessageText(message).trim().length > 0) {
+      return undefined;
+    }
+  }
+  return undefined;
+}
+
+class WorkflowPromptModelFailure extends Error {
+  override readonly cause: unknown;
+
+  constructor(cause: unknown) {
+    super(errorMessage(cause));
+    this.name = "WorkflowPromptModelFailure";
+    this.cause = cause;
+  }
+}
+
 /**
  * When an agent turn ends on a tool that returned `terminate: true`, control
  * returns with the tool result as the final conversational message and no
@@ -734,7 +787,7 @@ export function createStageContext(opts: StageRunnerOpts): InternalStageContext
     activeSession: StageSessionRuntime,
     initialText: string,
     sdkOptions: PromptOptions | undefined,
-  ): Promise<void> {
+  ): Promise<{ readonly terminalScanStartIndex: number }> {
     // Pause/resume loop: when a controlled pause aborts the SDK call,
     // swallow the resulting abort, suspend on `pauseRequest.deferred`,
     // and either re-issue with the user's resume message or return the
@@ -745,28 +798,32 @@ export function createStageContext(opts: StageRunnerOpts): InternalStageContext
       if (pendingPauseBeforePrompt) {
         const { message } = await pendingPauseBeforePrompt.deferred.promise;
         nextText = message;
-        if (nextText === undefined) return;
+        if (nextText === undefined) return { terminalScanStartIndex: activeSession.messages.length };
         continue;
       }
+      const promptStartIndex = activeSession.messages.length;
       try {
         await activeSession.prompt(nextText, sdkOptions);
         const pendingPauseAfterPrompt = pauseRequest;
         if (pendingPauseAfterPrompt) {
           const { message } = await pendingPauseAfterPrompt.deferred.promise;
           nextText = message;
-          if (nextText === undefined) return;
+          if (nextText === undefined) return { terminalScanStartIndex: activeSession.messages.length };
           continue;
         }
-        nextText = undefined;
+        return { terminalScanStartIndex: promptStartIndex };
       } catch (err) {
-        if (pauseRequest) {
-          const { message } = await pauseRequest.deferred.promise;
+        const pendingPauseAfterThrow = pauseRequest;
+        if (pendingPauseAfterThrow) {
+          const { message } = await pendingPauseAfterThrow.deferred.promise;
           nextText = message;
+          if (nextText === undefined) return { terminalScanStartIndex: activeSession.messages.length };
           continue;
         }
         throw err;
       }
     }
+    return { terminalScanStartIndex: activeSession.messages.length };
   }
 
   async function promptWithFallback(
@@ -795,14 +852,18 @@ export function createStageContext(opts: StageRunnerOpts): InternalStageContext
       selectedModel = candidate.id;
       notifyModelFallbackMetaChange();
       try {
-        await promptWithPauseResume(activeSession, text, sdkOptions);
+        const { terminalScanStartIndex } = await promptWithPauseResume(activeSession, text, sdkOptions);
+        const terminalFailure = latestTerminalAssistantFailureSince(activeSession.messages, terminalScanStartIndex);
+        if (terminalFailure !== undefined) {
+          throw new WorkflowPromptModelFailure(terminalFailure);
+        }
         modelAttempts.push({ model: candidate.id, success: true, ...modelAttemptReasoning(candidate) });
         pendingFallbackWarnings.length = 0;
         return;
       } catch (err) {
         const message = errorMessage(err);
         modelAttempts.push({ model: candidate.id, success: false, ...modelAttemptReasoning(candidate), error: message });
-        if (signal?.aborted || !isRetryableModelFailure(message) || index === candidates.length - 1) {
+        if (signal?.aborted || !isRetryableModelFailure(err) || index === candidates.length - 1) {
           modelWarnings.push(...pendingFallbackWarnings);
           pendingFallbackWarnings.length = 0;
           notifyModelFallbackMetaChange();

package/dist/builtin/workflows/src/runs/shared/model-fallback.ts

@@ -313,9 +313,12 @@ const RETRYABLE_MODEL_FAILURE_PATTERNS: readonly RegExp[] = [
   /billing/i,
   /credit/i,
   /auth(?:entication|orization)?/i,
+  /unauthori[sz]ed/i,
+  /\b40[13]\b/,
   /api\s*key/i,
   /token\s*expired/i,
   /forbidden/i,
+  /invalid\s*key/i,
   /model.*(?:unavailable|disabled|not\s*found|unknown)/i,
   /(?:unavailable|disabled|not\s*found|unknown).*model/i,
   /overloaded/i,
@@ -324,10 +327,11 @@ const RETRYABLE_MODEL_FAILURE_PATTERNS: readonly RegExp[] = [
   /network/i,
   /fetch/i,
   /socket/i,
+  /connection\s*refused/i,
   /upstream/i,
   /timeout/i,
   /timed\s*out/i,
-  /\b50[234]\b/,
+  /\b50[0-4]\b/,
 ];
 
 const NON_RETRYABLE_FAILURE_PATTERNS: readonly RegExp[] = [
@@ -342,15 +346,405 @@ const NON_RETRYABLE_FAILURE_PATTERNS: readonly RegExp[] = [
   /interrupted/i,
 ];
 
+const CANCELLED_FAILURE_PATTERNS: readonly RegExp[] = [
+  /cancel/i,
+  /abort/i,
+  /interrupted/i,
+];
+
+export type ModelFallbackFailureKind =
+  | "auth_on_candidate_provider"
+  | "rate_limit"
+  | "provider_unavailable"
+  | "network_timeout"
+  | "model_unavailable"
+  | "cancelled"
+  | "task_failure"
+  | "unknown";
+
+export type ModelFallbackFailureSource =
+  | "assistant_message"
+  | "diagnostic"
+  | "throw"
+  | "structured"
+  | "string_fallback";
+
+export interface ModelFallbackFailureSignal {
+  readonly kind: ModelFallbackFailureKind;
+  readonly message: string;
+  readonly source: ModelFallbackFailureSource;
+  readonly stopReason?: string;
+  readonly status?: number;
+  readonly code?: string | number;
+  readonly name?: string;
+}
+
+const FALLBACKABLE_FAILURE_KINDS: ReadonlySet<ModelFallbackFailureKind> = new Set([
+  "auth_on_candidate_provider",
+  "rate_limit",
+  "provider_unavailable",
+  "network_timeout",
+  "model_unavailable",
+]);
+
+function asRecord(value: unknown): Record<string, unknown> | undefined {
+  return value !== null && typeof value === "object" ? value as Record<string, unknown> : undefined;
+}
+
+function field(value: unknown, key: string): unknown {
+  return asRecord(value)?.[key];
+}
+
+function stringField(value: unknown, key: string): string | undefined {
+  const raw = field(value, key);
+  return typeof raw === "string" && raw.trim().length > 0 ? raw : undefined;
+}
+
+function errorName(value: unknown): string | undefined {
+  return value instanceof Error ? value.name : stringField(value, "name");
+}
+
+function directMessageFrom(value: unknown): string | undefined {
+  return stringField(value, "errorMessage")
+    ?? stringField(value, "message")
+    ?? stringField(value, "statusText");
+}
+
+function integerFrom(value: unknown): number | undefined {
+  if (typeof value === "number" && Number.isInteger(value)) return value;
+  if (typeof value !== "string" || value.trim().length === 0) return undefined;
+  const parsed = Number(value.trim());
+  return Number.isInteger(parsed) ? parsed : undefined;
+}
+
+function statusFrom(value: unknown): number | undefined {
+  return integerFrom(field(value, "status"))
+    ?? integerFrom(field(value, "statusCode"))
+    ?? integerFrom(field(value, "httpStatus"));
+}
+
+function codeFrom(value: unknown): string | number | undefined {
+  const rawCode = field(value, "code");
+  return typeof rawCode === "string" || typeof rawCode === "number" ? rawCode : undefined;
+}
+
+function stopReasonFrom(value: unknown): string | undefined {
+  return stringField(value, "stopReason");
+}
+
+function finishReasonFrom(value: unknown): string | undefined {
+  return stringField(value, "finish_reason") ?? stringField(value, "finishReason");
+}
+
+function causeOf(value: unknown): unknown {
+  return value instanceof Error ? value.cause : field(value, "cause");
+}
+
+function diagnosticErrors(value: unknown): readonly unknown[] {
+  const diagnostics = field(value, "diagnostics");
+  if (!Array.isArray(diagnostics)) return [];
+  const errors: unknown[] = [];
+  for (const diagnostic of diagnostics) {
+    const diagnosticError = field(diagnostic, "error");
+    errors.push(diagnosticError ?? diagnostic);
+  }
+  return errors;
+}
+
+function normalizeCode(value: string | number | undefined): string | undefined {
+  if (value === undefined) return undefined;
+  const normalized = String(value)
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "_")
+    .replace(/^_+|_+$/g, "");
+  return normalized.length > 0 ? normalized : undefined;
+}
+
+function kindFromStatus(status: number | undefined): ModelFallbackFailureKind | undefined {
+  switch (status) {
+    case 401:
+    case 403:
+      return "auth_on_candidate_provider";
+    case 408:
+      return "network_timeout";
+    case 404:
+      return "model_unavailable";
+    case 429:
+      return "rate_limit";
+    default:
+      if (status !== undefined && status >= 500 && status <= 599) return "provider_unavailable";
+      return undefined;
+  }
+}
+
+function refusalKindFromCode(code: string | number | undefined): ModelFallbackFailureKind | undefined {
+  const normalizedCode = normalizeCode(code);
+  if (normalizedCode === undefined) return undefined;
+  if (normalizedCode.includes("content_filter") || normalizedCode.includes("contentfilter")) return "task_failure";
+  if (normalizedCode.includes("safety") || normalizedCode.includes("policy")) return "task_failure";
+  switch (normalizedCode) {
+    case "blocked":
+    case "blocked_by_provider":
+    case "blocked_by_safety":
+    case "blocked_by_policy":
+    case "provider_refusal":
+    case "refusal":
+    case "tool_refusal":
+    case "tool_call_refusal":
+    case "tool_use_refusal":
+      return "task_failure";
+    default:
+      return undefined;
+  }
+}
+
+function kindFromCode(code: string | number | undefined): ModelFallbackFailureKind | undefined {
+  const normalizedCode = normalizeCode(code);
+  if (normalizedCode === undefined) return undefined;
+  const refusalKind = refusalKindFromCode(code);
+  if (refusalKind !== undefined) return refusalKind;
+  const httpStatusKind = kindFromStatus(integerFrom(code));
+  if (httpStatusKind !== undefined) return httpStatusKind;
+
+  switch (normalizedCode) {
+    case "auth":
+    case "auth_required":
+    case "authentication_required":
+    case "unauthorized":
+    case "forbidden":
+    case "invalid_api_key":
+    case "missing_api_key":
+    case "invalid_key":
+      return "auth_on_candidate_provider";
+    case "etimedout":
+    case "econnreset":
+    case "econnrefused":
+    case "enotfound":
+    case "eai_again":
+    case "fetch_failed":
+    case "network_error":
+    case "timeout":
+    case "timeout_error":
+    case "und_err_connect_timeout":
+      return "network_timeout";
+    case "rate_limit":
+    case "rate_limit_exceeded":
+    case "too_many_requests":
+    case "quota_exceeded":
+      return "rate_limit";
+    case "aborterror":
+    case "aborted":
+    case "cancelled":
+    case "canceled":
+      return "cancelled";
+    case "model_not_found":
+    case "model_unavailable":
+    case "model_disabled":
+    case "unknown_model":
+      return "model_unavailable";
+    case "provider_error":
+    case "api_error":
+    case "service_unavailable":
+    case "temporarily_unavailable":
+    case "overloaded":
+      return "provider_unavailable";
+    default:
+      return undefined;
+  }
+}
+
+const PROVIDER_REFUSAL_FAILURE_PATTERNS: readonly RegExp[] = [
+  /\bfinish[_\s-]?reason\b[^\n]*\bcontent[_\s-]?filter\b/i,
+  /\bcontent[_\s-]?filter(?:ed|ing)?\b/i,
+  /\b(?:safety|policy)\b[^\n]*\b(?:refus(?:e|al|ed|es|ing)?|block(?:ed|ing)?|filter(?:ed|ing)?|violat(?:e|ion|ed|ing)?|disallow(?:ed|ing)?|reject(?:ed|ion|ing)?)\b/i,
+  /\b(?:refus(?:e|al|ed|es|ing)?|block(?:ed|ing)?|filter(?:ed|ing)?|violat(?:e|ion|ed|ing)?|disallow(?:ed|ing)?|reject(?:ed|ion|ing)?)\b[^\n]*\b(?:safety|policy)\b/i,
+  /\btool[_\s-]?(?:call|use)?[_\s-]?refus(?:e|al|ed|es|ing)?\b/i,
+  /\btool(?:\s+call|\s+use)?\b[^\n]*\brefus(?:e|al|ed|es|ing)?\b/i,
+  /\brefus(?:e|al|ed|es|ing)?\b[^\n]*\btool(?:\s+call|\s+use)?\b/i,
+  /\bprovider[_\s-]?refus(?:e|al|ed|es|ing)?\b/i,
+  /\bprovider\b[^\n]*\brefus(?:e|al|ed|es|ing)?\b[^\n]*\b(?:prompt|request|content|policy|safety)\b/i,
+];
+
+function refusalKindFromMessage(message: string): ModelFallbackFailureKind | undefined {
+  if (CANCELLED_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "cancelled";
+  if (NON_RETRYABLE_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "task_failure";
+  if (PROVIDER_REFUSAL_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return "task_failure";
+  return undefined;
+}
+
+function fallbackKindFromMessage(message: string, name: string | undefined): ModelFallbackFailureKind | undefined {
+  const refusalKind = refusalKindFromMessage(message);
+  if (refusalKind !== undefined) return refusalKind;
+  const nameKind = kindFromCode(name);
+  if (nameKind !== undefined) return nameKind;
+  if (!RETRYABLE_MODEL_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return undefined;
+  if (/rate\s*limit|too\s*many\s*requests|\b429\b|quota|billing|credit/i.test(message)) return "rate_limit";
+  if (/auth|unauthori[sz]ed|\b40[13]\b|api\s*key|token\s*expired|forbidden|invalid\s*key/i.test(message)) return "auth_on_candidate_provider";
+  if (/model.*(?:unavailable|disabled|not\s*found|unknown)|(?:unavailable|disabled|not\s*found|unknown).*model/i.test(message)) return "model_unavailable";
+  if (/network|fetch|socket|connection\s*refused|timeout|timed\s*out/i.test(message)) return "network_timeout";
+  return "provider_unavailable";
+}
+
+function signalSource(value: unknown, fallback: ModelFallbackFailureSource | undefined): ModelFallbackFailureSource {
+  if (fallback !== undefined) return fallback;
+  if (stopReasonFrom(value) !== undefined || diagnosticErrors(value).length > 0) return "assistant_message";
+  if (value instanceof Error) return "throw";
+  return "structured";
+}
+
+function makeSignal(
+  kind: ModelFallbackFailureKind,
+  value: unknown,
+  source: ModelFallbackFailureSource | undefined,
+): ModelFallbackFailureSignal {
+  const status = statusFrom(value);
+  const code = codeFrom(value);
+  const name = errorName(value);
+  const stopReason = stopReasonFrom(value);
+  return {
+    kind,
+    message: errorMessage(value),
+    source: signalSource(value, source),
+    ...(stopReason !== undefined ? { stopReason } : {}),
+    ...(status !== undefined ? { status } : {}),
+    ...(code !== undefined ? { code } : {}),
+    ...(name !== undefined ? { name } : {}),
+  };
+}
+
+function fallbackSignalFromMessage(
+  value: unknown,
+  source: ModelFallbackFailureSource | undefined,
+): ModelFallbackFailureSignal | undefined {
+  const message = errorMessage(value);
+  if (!message.trim()) return undefined;
+  const kind = fallbackKindFromMessage(message, errorName(value));
+  return kind === undefined ? undefined : makeSignal(kind, value, source);
+}
+
+function classifyAssistantRefusalSignal(
+  value: unknown,
+  source: ModelFallbackFailureSource | undefined,
+): ModelFallbackFailureSignal | undefined {
+  const codeRefusalKind = refusalKindFromCode(codeFrom(value))
+    ?? refusalKindFromCode(errorName(value))
+    ?? refusalKindFromCode(finishReasonFrom(value));
+  if (codeRefusalKind !== undefined) return makeSignal(codeRefusalKind, value, source);
+
+  const messageRefusalKind = refusalKindFromMessage(directMessageFrom(value) ?? "");
+  return messageRefusalKind === undefined ? undefined : makeSignal(messageRefusalKind, value, source);
+}
+
+function isRefusalSignal(signal: ModelFallbackFailureSignal): boolean {
+  return signal.kind === "cancelled" || signal.kind === "task_failure";
+}
+
+function structuredSignal(
+  value: unknown,
+  seen: Set<unknown>,
+  source?: ModelFallbackFailureSource,
+): ModelFallbackFailureSignal | undefined {
+  if (value === undefined || value === null || seen.has(value)) return undefined;
+  if (typeof value === "object") seen.add(value);
+
+  const stopReason = stopReasonFrom(value)?.toLowerCase();
+  if (stopReason === "aborted") return makeSignal("cancelled", value, source);
+
+  const directRefusalSignal = classifyAssistantRefusalSignal(value, source);
+  if (directRefusalSignal !== undefined) return directRefusalSignal;
+
+  const codeKind = kindFromCode(codeFrom(value));
+  const nameKind = kindFromCode(errorName(value));
+  if (codeKind === "cancelled" || nameKind === "cancelled") return makeSignal("cancelled", value, source);
+
+  let firstNestedFallbackSignal: ModelFallbackFailureSignal | undefined;
+  const nestedSeen = new Set(seen);
+  for (const diagnosticError of diagnosticErrors(value)) {
+    const diagnosticSignal = structuredSignal(diagnosticError, nestedSeen, "diagnostic")
+      ?? fallbackSignalFromMessage(diagnosticError, "diagnostic");
+    if (diagnosticSignal === undefined) continue;
+    if (isRefusalSignal(diagnosticSignal)) return diagnosticSignal;
+    firstNestedFallbackSignal ??= diagnosticSignal;
+  }
+
+  const cause = causeOf(value);
+  const causeSignal = structuredSignal(cause, nestedSeen, source)
+    ?? fallbackSignalFromMessage(cause, source);
+  if (causeSignal !== undefined) {
+    if (isRefusalSignal(causeSignal)) return causeSignal;
+    firstNestedFallbackSignal ??= causeSignal;
+  }
+
+  const statusKind = kindFromStatus(statusFrom(value));
+  if (statusKind !== undefined) return makeSignal(statusKind, value, source);
+  if (codeKind !== undefined) return makeSignal(codeKind, value, source);
+  if (nameKind !== undefined) return makeSignal(nameKind, value, source);
+
+  if (firstNestedFallbackSignal !== undefined) return firstNestedFallbackSignal;
+
+  if (stopReason === "error") return makeSignal("provider_unavailable", value, source);
+
+  return undefined;
+}
+
+function messageFromUnknown(value: unknown, seen: Set<unknown>): string | undefined {
+  if (value === undefined || value === null || seen.has(value)) return undefined;
+  if (typeof value === "string") return value.trim().length > 0 ? value : undefined;
+  if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") return String(value);
+  if (typeof value === "symbol" || typeof value === "function") return undefined;
+  seen.add(value);
+
+  if (value instanceof Error && value.message.trim().length > 0) return value.message;
+  const directMessage = directMessageFrom(value);
+  if (directMessage !== undefined) return directMessage;
+
+  for (const diagnosticError of diagnosticErrors(value)) {
+    const diagnosticMessage = messageFromUnknown(diagnosticError, seen);
+    if (diagnosticMessage !== undefined) return diagnosticMessage;
+  }
+
+  const causeMessage = messageFromUnknown(causeOf(value), seen);
+  if (causeMessage !== undefined) return causeMessage;
+
+  const stopReason = stopReasonFrom(value);
+  if (stopReason !== undefined) return `Assistant message ended with stopReason:${stopReason}`;
+  const finishReason = finishReasonFrom(value);
+  if (finishReason !== undefined) return `Model request finished with finish_reason:${finishReason}`;
+  const status = statusFrom(value);
+  if (status !== undefined) return `Model request failed with status ${status}`;
+  const code = codeFrom(value);
+  if (code !== undefined) return `Model request failed with code ${String(code)}`;
+
+  return undefined;
+}
+
 export function errorMessage(error: unknown): string {
-  if (error instanceof Error) return error.message;
-  return String(error);
+  const structuredMessage = messageFromUnknown(error, new Set());
+  if (structuredMessage !== undefined) return structuredMessage;
+  const rendered = String(error);
+  return rendered === "[object Object]" ? "Model request failed" : rendered;
+}
+
+export function normalizeModelFailureSignal(error: unknown): ModelFallbackFailureSignal {
+  const structured = structuredSignal(error, new Set());
+  if (structured !== undefined) return structured;
+
+  const message = errorMessage(error);
+  const name = errorName(error);
+  const fallbackKind = message.trim().length > 0
+    ? fallbackKindFromMessage(message, name)
+    : undefined;
+  return {
+    kind: fallbackKind ?? "unknown",
+    message,
+    source: "string_fallback",
+    ...(name !== undefined ? { name } : {}),
+  };
 }
 
-export function isRetryableModelFailure(error: string | Error | undefined): boolean {
+export function isRetryableModelFailure(error: unknown): boolean {
   if (error === undefined) return false;
-  const message = typeof error === "string" ? error : error.message;
-  if (!message.trim()) return false;
-  if (NON_RETRYABLE_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) return false;
-  return RETRYABLE_MODEL_FAILURE_PATTERNS.some((pattern) => pattern.test(message));
+  const signal = normalizeModelFailureSignal(error);
+  return FALLBACKABLE_FAILURE_KINDS.has(signal.kind);
 }