@flowcodex/core 0.3.0

package/tests/provider-runner.test.ts

@@ -0,0 +1,535 @@
+import { describe, expect, it } from 'vitest';
+import { runProviderWithRetry } from '../src/agent/provider-runner.js';
+import type { ModelInfo } from '../src/infrastructure/catalog-parser.js';
+import { DefaultCatalogParser } from '../src/infrastructure/catalog-parser.js';
+import { DefaultRetryPolicy } from '../src/infrastructure/retry-policy.js';
+import { EventBus } from '../src/kernel/events.js';
+import type { LLMEvent, LLMRequest, Provider } from '../src/types/provider.js';
+
+function makeErrorProvider(opts: {
+  failTimes: number;
+  status: number;
+  thenSucceedWith?: LLMEvent[];
+  modelName?: string;
+}): { provider: Provider; get callCount(): number; get modelsSeen(): string[] } {
+  const state = { callCount: 0, modelsSeen: [] as string[] };
+  const provider: Provider = {
+    name: opts.modelName ?? 'mock',
+    async *stream(req: LLMRequest): AsyncIterable<LLMEvent> {
+      state.callCount++;
+      state.modelsSeen.push(req.model);
+      if (state.callCount <= opts.failTimes) {
+        const err = new Error(`HTTP ${opts.status}`);
+        (err as { status?: number }).status = opts.status;
+        (err as { retryable?: boolean }).retryable = true;
+        throw err;
+      }
+      const events = opts.thenSucceedWith ?? [
+        { type: 'text_delta', text: 'ok' },
+        { type: 'finish', usage: { input: 10, output: 5 }, stopReason: 'end_turn' },
+      ];
+      for (const ev of events) yield ev;
+    },
+    async complete(): Promise<never> {
+      throw new Error('not impl');
+    },
+  };
+  return {
+    provider,
+    get callCount() {
+      return state.callCount;
+    },
+    get modelsSeen() {
+      return state.modelsSeen;
+    },
+  };
+}
+
+function makeCatalogWithModels(models: ModelInfo[]): DefaultCatalogParser {
+  const parser = new DefaultCatalogParser({ disableNetwork: true });
+  (
+    parser as unknown as { cached: { models: ModelInfo[]; fetchedAt: number; source: string } }
+  ).cached = {
+    models,
+    fetchedAt: Date.now(),
+    source: 'snapshot',
+  };
+  return parser;
+}
+
+const CHEAP_MODELS: ModelInfo[] = [
+  {
+    id: 'claude-haiku-4-5',
+    name: 'Haiku',
+    provider: 'mock',
+    reasoning: false,
+    limit: { context: 200_000, output: 8192 },
+    cost: { input: 0.8, output: 4 },
+    tool_call: true,
+    modalities: { input: ['text'], output: ['text'] },
+  },
+];
+
+const SIGNAL = new AbortController().signal;
+
+describe('runProviderWithRetry — fallback', () => {
+  it('falls back to explicit fallbackModel after retry exhaustion on 429', async () => {
+    const { provider, modelsSeen } = makeErrorProvider({
+      failTimes: 10,
+      status: 429,
+      thenSucceedWith: [
+        { type: 'text_delta', text: 'fallback ok' },
+        { type: 'finish', usage: { input: 5, output: 2 }, stopReason: 'end_turn' },
+      ],
+    });
+
+    const events = new EventBus();
+    const request: LLMRequest = {
+      model: 'claude-sonnet-4-6',
+      system: [],
+      messages: [],
+      max_tokens: 8192,
+    };
+
+    const result = await runProviderWithRetry({
+      provider,
+      request,
+      signal: SIGNAL,
+      events,
+      retry: new DefaultRetryPolicy(),
+      fallbackModel: 'claude-haiku-4-5',
+    });
+
+    expect(result.stopReason).toBe('end_turn');
+    expect(modelsSeen[0]).toBe('claude-sonnet-4-6');
+    expect(modelsSeen[modelsSeen.length - 1]).toBe('claude-haiku-4-5');
+  });
+
+  it('emits provider.fallback event with correct payload', async () => {
+    const { provider } = makeErrorProvider({
+      failTimes: 10,
+      status: 429,
+      thenSucceedWith: [
+        { type: 'text_delta', text: 'ok' },
+        { type: 'finish', usage: { input: 1, output: 1 }, stopReason: 'end_turn' },
+      ],
+    });
+
+    const events = new EventBus();
+    let fallbackEvent: { from: string; to: string; reason: string; status: number } | undefined;
+
+    events.on('provider.fallback', (e) => {
+      fallbackEvent = e;
+    });
+
+    const request: LLMRequest = {
+      model: 'sonnet',
+      system: [],
+      messages: [],
+      max_tokens: 8192,
+    };
+
+    await runProviderWithRetry({
+      provider,
+      request,
+      signal: SIGNAL,
+      events,
+      retry: new DefaultRetryPolicy(),
+      fallbackModel: 'haiku',
+    });
+
+    expect(fallbackEvent).toBeDefined();
+    expect(fallbackEvent!.from).toBe('mock/sonnet');
+    expect(fallbackEvent!.to).toBe('mock/haiku');
+    expect(fallbackEvent!.reason).toBe('rate_limit_exhausted');
+    expect(fallbackEvent!.status).toBe(429);
+  });
+
+  it('throws original error when no fallback configured', async () => {
+    const { provider } = makeErrorProvider({
+      failTimes: 10,
+      status: 429,
+    });
+
+    const events = new EventBus();
+    const request: LLMRequest = {
+      model: 'sonnet',
+      system: [],
+      messages: [],
+      max_tokens: 8192,
+    };
+
+    await expect(
+      runProviderWithRetry({
+        provider,
+        request,
+        signal: SIGNAL,
+        events,
+        retry: new DefaultRetryPolicy(),
+      }),
+    ).rejects.toThrow();
+  });
+
+  it('throws when fallback model also exhausts retries', async () => {
+    const { provider } = makeErrorProvider({
+      failTimes: 100,
+      status: 429,
+    });
+
+    const events = new EventBus();
+    const request: LLMRequest = {
+      model: 'sonnet',
+      system: [],
+      messages: [],
+      max_tokens: 8192,
+    };
+
+    await expect(
+      runProviderWithRetry({
+        provider,
+        request,
+        signal: SIGNAL,
+        events,
+        retry: new DefaultRetryPolicy(),
+        fallbackModel: 'haiku',
+      }),
+    ).rejects.toThrow();
+  }, 120_000);
+
+  it('does not fall back on 500 errors', async () => {
+    const { provider, modelsSeen } = makeErrorProvider({
+      failTimes: 10,
+      status: 500,
+    });
+
+    const events = new EventBus();
+    let fallbackEmitted = false;
+    events.on('provider.fallback', () => {
+      fallbackEmitted = true;
+    });
+
+    const request: LLMRequest = {
+      model: 'sonnet',
+      system: [],
+      messages: [],
+      max_tokens: 8192,
+    };
+
+    await expect(
+      runProviderWithRetry({
+        provider,
+        request,
+        signal: SIGNAL,
+        events,
+        retry: new DefaultRetryPolicy(),
+        fallbackModel: 'haiku',
+      }),
+    ).rejects.toThrow();
+
+    expect(fallbackEmitted).toBe(false);
+    for (const m of modelsSeen) expect(m).toBe('sonnet');
+  });
+
+  it('falls back via catalog auto-select when no explicit fallbackModel', async () => {
+    const { provider, modelsSeen } = makeErrorProvider({
+      failTimes: 10,
+      status: 429,
+      thenSucceedWith: [
+        { type: 'text_delta', text: 'ok' },
+        { type: 'finish', usage: { input: 1, output: 1 }, stopReason: 'end_turn' },
+      ],
+    });
+
+    const catalog = makeCatalogWithModels(CHEAP_MODELS);
+    const events = new EventBus();
+    const request: LLMRequest = {
+      model: 'claude-sonnet-4-6',
+      system: [],
+      messages: [],
+      max_tokens: 8192,
+    };
+
+    const result = await runProviderWithRetry({
+      provider,
+      request,
+      signal: SIGNAL,
+      events,
+      retry: new DefaultRetryPolicy(),
+      catalog,
+    });
+
+    expect(result.stopReason).toBe('end_turn');
+    expect(modelsSeen[modelsSeen.length - 1]).toBe('claude-haiku-4-5');
+  });
+
+  it('does not fall back to the same model as the current request', async () => {
+    const { provider } = makeErrorProvider({
+      failTimes: 10,
+      status: 429,
+    });
+
+    const catalog = makeCatalogWithModels([
+      {
+        id: 'sonnet',
+        name: 'Sonnet',
+        provider: 'mock',
+        reasoning: false,
+        limit: { context: 200_000, output: 8192 },
+        cost: { input: 3, output: 15 },
+        tool_call: true,
+        modalities: { input: ['text'], output: ['text'] },
+      },
+    ]);
+
+    const events = new EventBus();
+    const request: LLMRequest = {
+      model: 'sonnet',
+      system: [],
+      messages: [],
+      max_tokens: 8192,
+    };
+
+    await expect(
+      runProviderWithRetry({
+        provider,
+        request,
+        signal: SIGNAL,
+        events,
+        retry: new DefaultRetryPolicy(),
+        catalog,
+      }),
+    ).rejects.toThrow();
+  });
+
+  it('does not fall back when aborted', async () => {
+    const ctrl = new AbortController();
+    const { provider } = makeErrorProvider({
+      failTimes: 10,
+      status: 429,
+    });
+
+    const events = new EventBus();
+    let fallbackEmitted = false;
+    events.on('provider.fallback', () => {
+      fallbackEmitted = true;
+    });
+
+    const request: LLMRequest = {
+      model: 'sonnet',
+      system: [],
+      messages: [],
+      max_tokens: 8192,
+    };
+
+    ctrl.abort();
+
+    await expect(
+      runProviderWithRetry({
+        provider,
+        request,
+        signal: ctrl.signal,
+        events,
+        retry: new DefaultRetryPolicy(),
+        fallbackModel: 'haiku',
+      }),
+    ).rejects.toThrow();
+
+    expect(fallbackEmitted).toBe(false);
+  });
+});
+
+function makeKeylessProvider(name: string): Provider {
+  return {
+    name,
+    // biome-ignore lint/correctness/useYield: error-only stub
+    async *stream(): AsyncIterable<LLMEvent> {
+      throw new Error('should not be called');
+    },
+    async complete() {
+      return { content: [], usage: { input: 0, output: 0 }, stopReason: 'end_turn' };
+    },
+  };
+}
+
+describe('runProviderWithRetry — cross-provider fallback chain', () => {
+  it('walks fallback chain on 429 exhaustion and succeeds on the second provider', async () => {
+    const calls: string[] = [];
+    const primary: Provider = {
+      name: 'anthropic',
+      // biome-ignore lint/correctness/useYield: error-only stub
+      async *stream(req) {
+        calls.push(`anthropic:${req.model}`);
+        const err = new Error('HTTP 429');
+        (err as { status?: number }).status = 429;
+        (err as { retryable?: boolean }).retryable = true;
+        throw err;
+      },
+      async complete() {
+        throw new Error('not impl');
+      },
+    };
+
+    const secondary: Provider = {
+      name: 'openai',
+      async *stream(req) {
+        calls.push(`openai:${req.model}`);
+        yield { type: 'text_delta', text: 'ok from openai' };
+        yield { type: 'finish', usage: { input: 5, output: 3 }, stopReason: 'end_turn' };
+      },
+      async complete() {
+        throw new Error('not impl');
+      },
+    };
+
+    const events = new EventBus();
+    const fallbackEvents: Array<{ from: string; to: string; reason: string }> = [];
+    events.on('provider.fallback', (e) => fallbackEvents.push(e));
+
+    const result = await runProviderWithRetry({
+      provider: primary,
+      request: { model: 'claude-sonnet-4-6', system: [], messages: [], max_tokens: 8192 },
+      signal: SIGNAL,
+      events,
+      retry: new DefaultRetryPolicy(),
+      fallback: [{ providerId: 'openai', model: 'gpt-4o-mini', providerFactory: () => secondary }],
+    });
+
+    expect(result.stopReason).toBe('end_turn');
+    expect(calls).toContain('anthropic:claude-sonnet-4-6');
+    expect(calls).toContain('openai:gpt-4o-mini');
+    expect(fallbackEvents[0]?.from).toBe('anthropic/claude-sonnet-4-6');
+    expect(fallbackEvents[0]?.to).toBe('openai/gpt-4o-mini');
+  });
+
+  it('skips a chain entry whose factory returns undefined (no api key) and continues', async () => {
+    const calls: string[] = [];
+    const primary: Provider = {
+      name: 'anthropic',
+      // biome-ignore lint/correctness/useYield: error-only stub
+      async *stream(req) {
+        calls.push(`anthropic:${req.model}`);
+        const err = new Error('HTTP 429');
+        (err as { status?: number }).status = 429;
+        (err as { retryable?: boolean }).retryable = true;
+        throw err;
+      },
+      async complete() {
+        throw new Error('not impl');
+      },
+    };
+    const tertiary: Provider = {
+      name: 'deepseek',
+      async *stream(req) {
+        calls.push(`deepseek:${req.model}`);
+        yield { type: 'text_delta', text: 'ok' };
+        yield { type: 'finish', usage: { input: 1, output: 1 }, stopReason: 'end_turn' };
+      },
+      async complete() {
+        throw new Error('not impl');
+      },
+    };
+    const events = new EventBus();
+    const skipped: Array<{ providerId: string; reason: string }> = [];
+    events.on('provider.fallback_skipped', (e) => skipped.push(e));
+
+    const result = await runProviderWithRetry({
+      provider: primary,
+      request: { model: 'sonnet', system: [], messages: [], max_tokens: 8192 },
+      signal: SIGNAL,
+      events,
+      retry: new DefaultRetryPolicy(),
+      fallback: [
+        { providerId: 'openai', model: 'gpt-4o-mini', providerFactory: () => undefined },
+        { providerId: 'deepseek', model: 'deepseek-chat', providerFactory: () => tertiary },
+      ],
+    });
+
+    expect(result.stopReason).toBe('end_turn');
+    expect(skipped[0]?.providerId).toBe('openai');
+    expect(skipped[0]?.reason).toBe('no_api_key');
+    expect(calls).toContain('deepseek:deepseek-chat');
+  });
+
+  it('throws when the entire chain is exhausted', async () => {
+    const primary: Provider = {
+      name: 'anthropic',
+      // biome-ignore lint/correctness/useYield: error-only stub
+      async *stream() {
+        const err = new Error('HTTP 429');
+        (err as { status?: number }).status = 429;
+        (err as { retryable?: boolean }).retryable = true;
+        throw err;
+      },
+      async complete() {
+        throw new Error('not impl');
+      },
+    };
+    const secondary: Provider = {
+      name: 'openai',
+      // biome-ignore lint/correctness/useYield: error-only stub
+      async *stream() {
+        const err = new Error('HTTP 429');
+        (err as { status?: number }).status = 429;
+        (err as { retryable?: boolean }).retryable = true;
+        throw err;
+      },
+      async complete() {
+        throw new Error('not impl');
+      },
+    };
+    const events = new EventBus();
+
+    await expect(
+      runProviderWithRetry({
+        provider: primary,
+        request: { model: 'sonnet', system: [], messages: [], max_tokens: 8192 },
+        signal: SIGNAL,
+        events,
+        retry: new DefaultRetryPolicy(),
+        fallback: [
+          { providerId: 'openai', model: 'gpt-4o-mini', providerFactory: () => secondary },
+        ],
+      }),
+    ).rejects.toThrow();
+  }, 120_000);
+
+  it('does not walk the chain on 500 errors', async () => {
+    const primary: Provider = {
+      name: 'anthropic',
+      // biome-ignore lint/correctness/useYield: error-only stub
+      async *stream() {
+        const err = new Error('HTTP 500');
+        (err as { status?: number }).status = 500;
+        (err as { retryable?: boolean }).retryable = true;
+        throw err;
+      },
+      async complete() {
+        throw new Error('not impl');
+      },
+    };
+    const events = new EventBus();
+    let walked = false;
+    events.on('provider.fallback', () => {
+      walked = true;
+    });
+
+    await expect(
+      runProviderWithRetry({
+        provider: primary,
+        request: { model: 'sonnet', system: [], messages: [], max_tokens: 8192 },
+        signal: SIGNAL,
+        events,
+        retry: new DefaultRetryPolicy(),
+        fallback: [
+          {
+            providerId: 'openai',
+            model: 'gpt-4o-mini',
+            providerFactory: () => makeKeylessProvider('openai'),
+          },
+        ],
+      }),
+    ).rejects.toThrow();
+
+    expect(walked).toBe(false);
+  });
+});

package/tests/retry-policy.test.ts

@@ -0,0 +1,104 @@
+import { describe, it, expect } from 'vitest';
+import { DefaultRetryPolicy, parseRetryAfter, type ProviderError } from '../src/infrastructure/retry-policy.js';
+
+describe('DefaultRetryPolicy', () => {
+  const policy = new DefaultRetryPolicy();
+
+  describe('shouldRetry', () => {
+    it('retries 429 (rate limited)', () => {
+      const err: ProviderError = { status: 429, message: 'rate limited', retryable: true };
+      expect(policy.shouldRetry(err, 0)).toBe(true);
+      expect(policy.shouldRetry(err, 4)).toBe(true);
+      expect(policy.shouldRetry(err, 5)).toBe(false);
+    });
+
+    it('retries 529 (overloaded)', () => {
+      const err: ProviderError = { status: 529, message: 'overloaded', retryable: true };
+      expect(policy.shouldRetry(err, 0)).toBe(true);
+      expect(policy.shouldRetry(err, 3)).toBe(false);
+    });
+
+    it('retries 500/502/503/504', () => {
+      for (const status of [500, 502, 503, 504]) {
+        const err: ProviderError = { status, message: 'server error', retryable: true };
+        expect(policy.shouldRetry(err, 0)).toBe(true);
+      }
+    });
+
+    it('retries 599 (stream hang)', () => {
+      const err: ProviderError = { status: 599, message: 'stream hang', retryable: true };
+      expect(policy.shouldRetry(err, 0)).toBe(true);
+      expect(policy.shouldRetry(err, 2)).toBe(false);
+    });
+
+    it('does not retry 400 (bad request)', () => {
+      const err: ProviderError = { status: 400, message: 'bad request', retryable: false };
+      expect(policy.shouldRetry(err, 0)).toBe(false);
+    });
+
+    it('does not retry 401 (auth failed)', () => {
+      const err: ProviderError = { status: 401, message: 'auth', retryable: false };
+      expect(policy.shouldRetry(err, 0)).toBe(false);
+    });
+
+    it('does not retry 403 (forbidden)', () => {
+      const err: ProviderError = { status: 403, message: 'forbidden', retryable: false };
+      expect(policy.shouldRetry(err, 0)).toBe(false);
+    });
+
+    it('does not retry 422 (validation)', () => {
+      const err: ProviderError = { status: 422, message: 'validation', retryable: false };
+      expect(policy.shouldRetry(err, 0)).toBe(false);
+    });
+  });
+
+  describe('maxAttempts', () => {
+    it('returns 5 for 429', () => {
+      expect(policy.maxAttempts({ status: 429, message: '', retryable: true })).toBe(5);
+    });
+
+    it('returns 3 for 529', () => {
+      expect(policy.maxAttempts({ status: 529, message: '', retryable: true })).toBe(3);
+    });
+
+    it('returns 3 for unknown 5xx', () => {
+      expect(policy.maxAttempts({ status: 599, message: '', retryable: true })).toBe(2);
+    });
+  });
+
+  describe('delayMs', () => {
+    it('returns exponential backoff with jitter', () => {
+      const delay0 = policy.delayMs(0, { status: 429, message: '', retryable: true });
+      const delay1 = policy.delayMs(1, { status: 429, message: '', retryable: true });
+      const delay2 = policy.delayMs(2, { status: 429, message: '', retryable: true });
+      expect(delay0).toBeGreaterThanOrEqual(1000);
+      expect(delay0).toBeLessThanOrEqual(2000);
+      expect(delay1).toBeGreaterThanOrEqual(2000);
+      expect(delay1).toBeLessThanOrEqual(3000);
+      expect(delay2).toBeGreaterThanOrEqual(4000);
+    });
+
+    it('caps at 30 seconds', () => {
+      const delay = policy.delayMs(10, { status: 429, message: '', retryable: true });
+      expect(delay).toBeLessThanOrEqual(31_000);
+    });
+  });
+});
+
+describe('parseRetryAfter', () => {
+  it('parses seconds value', () => {
+    expect(parseRetryAfter('5')).toBe(5_000);
+    expect(parseRetryAfter('120')).toBe(120_000);
+  });
+
+  it('clamps to [1s, 120s]', () => {
+    expect(parseRetryAfter('0')).toBe(1_000);
+    expect(parseRetryAfter('999')).toBe(120_000);
+  });
+
+  it('returns undefined for invalid', () => {
+    expect(parseRetryAfter(undefined)).toBeUndefined();
+    expect(parseRetryAfter('')).toBeUndefined();
+    expect(parseRetryAfter('abc')).toBeUndefined();
+  });
+});