minimal-agent 0.12.0 → 0.14.0

package/plugins/workflow-runner/test/apiCapabilities.test.ts

@@ -0,0 +1,614 @@
+/**
+ * plugins/workflow-runner/test/apiCapabilities.test.ts —— 本轮新增的四项能力
+ *
+ *   ① schema     结构化输出：校验 + 有界修复，返回对象而不是字符串
+ *   ② pipeline   无屏障流水线：扇出与断点续跑兼得
+ *   ③ model      按模型 id 换 provider，解析不到回落基座并告警
+ *   ④ isolated   隔离执行：不看前文、也不污染主线 history
+ *
+ * 用 DI 注入 chat stub（不用 mock.module —— 后者全局泄漏、污染同批其它文件）。
+ */
+
+import { afterEach, describe, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import type { LoopEvent, Message } from '../../../src/plugin-sdk.js';
+import { createWorkflowContext } from '../src/runtime/context.js';
+import { Journal } from '../src/journal.js';
+import type { RunContext } from '../src/types.js';
+import { WorkflowState } from '../src/workflowState.js';
+
+const dirs: string[] = [];
+afterEach(() => {
+  for (const d of dirs.splice(0)) {
+    try {
+      rmSync(d, { recursive: true, force: true, maxRetries: 3 });
+    } catch {
+      /* Windows 偶发 EBUSY */
+    }
+  }
+});
+
+/** 按调用次序依次吐出预设文本的 chat stub，并记录每次收到的参数。 */
+function scriptedChat(texts: string[]) {
+  const calls: {
+    responseFormat: unknown;
+    model: string;
+    messageCount: number;
+    maxTokens: number | null;
+  }[] = [];
+  let n = 0;
+  const fn = async function* (a: {
+    provider: { model: string; apiFormat?: string; maxTokens?: number };
+    messages: Message[];
+    responseFormat?: unknown;
+  }) {
+    calls.push({
+      responseFormat: a.responseFormat ?? null,
+      model: a.provider.model,
+      messageCount: a.messages.length,
+      maxTokens: a.provider.maxTokens ?? null,
+    });
+    yield {
+      type: 'text_delta' as const,
+      delta: texts[Math.min(n++, texts.length - 1)],
+    };
+  };
+  return { fn, calls };
+}
+
+async function harness(
+  chat: unknown,
+  opts: {
+    apiFormat?: 'openai' | 'anthropic';
+    supportsJsonSchema?: boolean;
+    supportsVision?: boolean;
+  } = {},
+) {
+  const wd = mkdtempSync(join(tmpdir(), 'wf-cap-'));
+  dirs.push(wd);
+  const state = new WorkflowState(wd, 'demo', 's1');
+  await state.init({ goal: '测试目标' }, 'hash-v1', []);
+  const journal = new Journal(state.journalPath, { bundleHash: 'test', rootRunId: 'test-root' });
+  await journal.load();
+
+  const events: LoopEvent[] = [];
+  const history: Message[] = [{ role: 'system', content: 'sys' }];
+  const ctx = {
+    provider: {
+      name: 'test',
+      model: 'base-model',
+      baseURL: 'x',
+      apiKey: 'k',
+      contextWindow: 128000,
+      supportsVision: opts.supportsVision ?? true,
+      ...(opts.apiFormat ? { apiFormat: opts.apiFormat } : {}),
+      ...(opts.supportsJsonSchema !== undefined
+        ? { supportsJsonSchema: opts.supportsJsonSchema }
+        : {}),
+    },
+    history,
+    cwd: wd,
+    tools: [],
+    chat,
+  } as unknown as RunContext;
+
+  const globals = createWorkflowContext({
+    ctx,
+    state,
+    journal,
+    emit: (ev) => events.push(ev),
+    args: {},
+    workflowName: 'demo',
+    session: 's1',
+    task: state.taskId,
+  });
+
+  return { globals, events, history, journal, journalPath: state.journalPath };
+}
+
+const NUMS = {
+  type: 'object',
+  required: ['items'],
+  properties: {
+    items: { type: 'array', minItems: 1, items: { type: 'number' } },
+  },
+} as const;
+
+describe('① schema 结构化输出', () => {
+  test('★ 返回的是校验过的对象，不是字符串', async () => {
+    const chat = scriptedChat(['{"items":[1,2,3]}']);
+    const h = await harness(chat.fn);
+    const r = await h.globals.model.generate('给我三个数', { schema: NUMS });
+    expect(r).toEqual({ items: [1, 2, 3] });
+  });
+
+  test('★ 剥 markdown 围栏与 <think> 后再校验', async () => {
+    const chat = scriptedChat([
+      '<think>先想想</think>\n```json\n{"items":[7]}\n```',
+    ]);
+    const h = await harness(chat.fn);
+    expect(await h.globals.model.generate('x', { schema: NUMS })).toEqual({ items: [7] });
+  });
+
+  test('★ 不合格 → 把逐条违规回喂模型做有界修复', async () => {
+    // 第一次给错类型，第二次修对
+    const chat = scriptedChat(['{"items":["a"]}', '{"items":[42]}']);
+    const h = await harness(chat.fn);
+    const r = await h.globals.model.generate('x', { schema: NUMS });
+    expect(r).toEqual({ items: [42] });
+
+    const warn = h.events.find(
+      (e) =>
+        e.type === 'warning' && String(e.message).includes('第 1/2 次修复'),
+    );
+    expect(warn).toBeDefined();
+    // 违规必须点名路径与实际类型，否则模型无从改起
+    expect(String((warn as { message: string }).message)).toContain('items[0]');
+  });
+
+  test('★ 修复用尽仍不合格 → 抛错，且错误里带最后一次原始输出', async () => {
+    const chat = scriptedChat(['不是 JSON', '还不是', '仍然不是']);
+    const h = await harness(chat.fn);
+    await expect(h.globals.model.generate('x', { schema: NUMS })).rejects.toThrow(
+      /2 次修复后仍不符合 schema/,
+    );
+  });
+
+  test('★ 修复循环只记一条账，且记的是**对象**（重放不必再解析一遍）', async () => {
+    // 记成多条会让车道序号随修复次数漂移 —— resume 时必然错位。
+    // 记原始文本则会让重放路径重演一次解析失败。
+    const chat = scriptedChat(['{"items":["a"]}', '{"items":[9]}']);
+    const h = await harness(chat.fn);
+    const first = await h.globals.model.generate('x', { schema: NUMS });
+    expect(first).toEqual({ items: [9] });
+    expect(chat.calls.length).toBe(2); // 确实修复过一次
+
+    const reloaded = new Journal(h.journalPath, { bundleHash: 'test', rootRunId: 'test-root' });
+    await reloaded.load();
+    expect(reloaded.loadedCount).toBe(1); // 两次 chat，一条账
+
+    // 同一份 prompt + schema 重来 → 命中重放，且拿到的是对象
+    const chat2 = scriptedChat(['绝不该被调用']);
+    const h2 = await harness(chat2.fn);
+    // 换掉 h2 的账本为刚才那份：直接在同一个 state 目录上再跑一次更真实，
+    // 这里用 Journal 层验证「结果是对象」这一点就够。
+    const replayed = await reloaded.step(
+      'llm',
+      {
+        prompt: 'x',
+        allowedTools: null,
+        uses: null,
+        images: null,
+        contextFiles: null,
+        maxTurns: 1,
+        model: null,
+        schema: NUMS,
+        nativeSchema: true,
+        isolated: false,
+        timeoutMs: null,
+        maxTokens: null,
+      },
+      async (): Promise<unknown> => {
+        throw new Error('不该实跑');
+      },
+    );
+    expect(replayed).toEqual({ items: [9] });
+    expect(chat2.calls.length).toBe(0);
+  });
+
+  test('★ OpenAI 格式带原生 response_format；Anthropic 格式不带（否则 chat 会抛）', async () => {
+    const a = scriptedChat(['{"items":[1]}']);
+    const ha = await harness(a.fn, { apiFormat: 'openai' });
+    await ha.globals.model.generate('x', { schema: NUMS });
+    expect(a.calls[0].responseFormat).not.toBeNull();
+
+    const b = scriptedChat(['{"items":[1]}']);
+    const hb = await harness(b.fn, { apiFormat: 'anthropic' });
+    await hb.globals.model.generate('x', { schema: NUMS }); // 不得抛
+    expect(b.calls[0].responseFormat).toBeNull();
+  });
+
+  test('显式声明不支持原生 JSON Schema 时退回引擎校验，不向端点发送 response_format', async () => {
+    const chat = scriptedChat(['{"items":[1]}']);
+    const h = await harness(chat.fn, {
+      apiFormat: 'openai',
+      supportsJsonSchema: false,
+    });
+    expect(await h.globals.model.generate('x', { schema: NUMS })).toEqual({ items: [1] });
+    expect(chat.calls[0].responseFormat).toBeNull();
+  });
+
+  test('调用方可仅对不稳定步骤关闭原生 JSON Schema，仍保留引擎校验', async () => {
+    const chat = scriptedChat(['{"items":[1]}']);
+    const h = await harness(chat.fn, { apiFormat: 'openai' });
+    expect(
+      await h.globals.model.generate('x', { schema: NUMS, nativeSchema: false }),
+    ).toEqual({ items: [1] });
+    expect(chat.calls[0].responseFormat).toBeNull();
+  });
+});
+
+describe('① schema —— agent() 路径（带工具跑完再整形）', () => {
+  test('★ agent 的工具循环里不得开原生 response_format（会与 tool_calls 互斥）', async () => {
+    const chat = scriptedChat(['{"items":[5]}']);
+    const h = await harness(chat.fn, { apiFormat: 'openai' });
+    const r = await h.globals.model.agent('干活并给结论', { schema: NUMS });
+    expect(r).toEqual({ items: [5] });
+    // 第一次是带工具的正式调用 —— 必须不带原生约束
+    expect(chat.calls[0].responseFormat).toBeNull();
+  });
+
+  test('★ agent 首次输出不合格 → 零工具修复轮，此时才开原生约束', async () => {
+    const chat = scriptedChat(['我已经完成了，结果很好', '{"items":[8]}']);
+    const h = await harness(chat.fn, { apiFormat: 'openai' });
+    expect(await h.globals.model.agent('干活', { schema: NUMS })).toEqual({
+      items: [8],
+    });
+    expect(chat.calls[0].responseFormat).toBeNull(); // 工具轮
+    expect(chat.calls[1].responseFormat).not.toBeNull(); // 修复轮
+  });
+});
+
+describe('② pipeline 无屏障流水线', () => {
+  test('★ 每项独立走完全部阶段，(prev, item, index) 都对', async () => {
+    const h = await harness(scriptedChat(['ignored']).fn);
+    const seen: string[] = [];
+    const out = await h.globals.flow.pipeline(
+      ['a', 'b', 'c'],
+      { stages: [(prev: unknown, item: unknown, i: number) => {
+        seen.push(`s1:${item}:${i}`);
+        return `${prev}1`;
+      },
+      (prev: unknown, item: unknown, i: number) => {
+        seen.push(`s2:${prev}:${item}:${i}`);
+        return `${prev}2`;
+      }] },
+    );
+    expect(out.results.map((entry: any) => entry.value)).toEqual(['a12', 'b12', 'c12']);
+    expect(seen).toContain('s2:b1:b:1'); // prev 是上阶段结果，item/index 仍是原始项
+  });
+
+  test('★ 无屏障：慢项还在第 1 阶段时，快项已经进到第 2 阶段', async () => {
+    const h = await harness(scriptedChat(['x']).fn);
+    const timeline: string[] = [];
+    await h.globals.flow.pipeline(
+      [0, 1],
+      { stages: [async (_p: unknown, item: number) => {
+        // 第 0 项慢
+        await new Promise((r) => setTimeout(r, item === 0 ? 60 : 1));
+        timeline.push(`s1-done:${item}`);
+        return item;
+      },
+      async (_p: unknown, item: number) => {
+        timeline.push(`s2-start:${item}`);
+        return item;
+      }] },
+    );
+    // 有屏障的话，s2-start:1 必然排在 s1-done:0 之后
+    expect(timeline.indexOf('s2-start:1')).toBeLessThan(
+      timeline.indexOf('s1-done:0'),
+    );
+  });
+
+  test('★ 某项抛错 → 返回结构化失败并跳过其余阶段，其它项照跑', async () => {
+    const h = await harness(scriptedChat(['x']).fn);
+    const reached: number[] = [];
+    const out = await h.globals.flow.pipeline(
+      [0, 1, 2],
+      { errorMode: 'collect', stages: [(_p: unknown, item: number) => {
+        if (item === 1) throw new Error('boom');
+        return item;
+      },
+      (_p: unknown, item: number) => {
+        reached.push(item);
+        return item * 10;
+      }] },
+    );
+    expect(out.results[0]).toMatchObject({ ok: true, value: 0 });
+    expect(out.results[1]).toMatchObject({ ok: false, error: { message: 'boom' } });
+    expect(out.results[2]).toMatchObject({ ok: true, value: 20 });
+    expect(reached).toEqual(expect.arrayContaining([0, 2]));
+    expect(reached).not.toContain(1);
+    expect(out.failed).toBe(1);
+  });
+
+  test('零阶段原样返回；非数组抛错', async () => {
+    const h = await harness(scriptedChat(['x']).fn);
+    expect(await h.globals.flow.pipeline([1, 2], { stages: [] })).toMatchObject({
+      succeeded: 2,
+      failed: 0,
+    });
+    await expect(h.globals.flow.pipeline('nope' as never, { stages: [] })).rejects.toThrow(
+      /必须是数组/,
+    );
+  });
+
+  test('fail-fast 停止未开始项，等待已开始项后抛 AggregateError', async () => {
+    const h = await harness(scriptedChat(['x']).fn);
+    const started: number[] = [];
+    await expect(
+      h.globals.flow.pipeline([0, 1, 2, 3], {
+        label: 'fail-fast-test',
+        concurrency: 2,
+        errorMode: 'fail-fast',
+        stages: [async (_prev: unknown, item: number) => {
+          started.push(item);
+          if (item === 0) throw new Error('first failure');
+          await new Promise((resolve) => setTimeout(resolve, 10));
+          return item;
+        }],
+      }),
+    ).rejects.toBeInstanceOf(AggregateError);
+    expect(started).not.toContain(2);
+    expect(started).not.toContain(3);
+  });
+
+  test('★ 裸 Promise.all 并发记账 → 引擎必须告警（提示改用 pipeline）', async () => {
+    const chat = scriptedChat(['ok']);
+    const h = await harness(chat.fn);
+    await Promise.all([h.globals.model.generate('a'), h.globals.model.generate('b')]);
+    expect(
+      h.events.some(
+        (e) =>
+          e.type === 'warning' &&
+          String(e.message).includes('断点续跑随之失效'),
+      ),
+    ).toBe(true);
+  });
+});
+
+describe('①+② 联动：pipeline 扇出后仍能断点续跑（这是整轮改动的验收点）', () => {
+  test('★ 12 项（> 并发上限 8）扇出，第二次全部重放、一次模型都不调', async () => {
+    const wd = mkdtempSync(join(tmpdir(), 'wf-cap-resume-'));
+    dirs.push(wd);
+    const N = 12;
+
+    const build = async (chatFn: unknown) => {
+      const state = new WorkflowState(wd, 'demo', 's1');
+      await state.init({ goal: 'g' }, 'hash-v1', []);
+      const journal = new Journal(state.journalPath, { bundleHash: 'test', rootRunId: 'test-root' });
+      await journal.load();
+      const events: LoopEvent[] = [];
+      const ctx = {
+        provider: {
+          name: 't',
+          model: 'base-model',
+          baseURL: 'x',
+          apiKey: 'k',
+          contextWindow: 128000,
+        },
+        history: [{ role: 'system', content: 'sys' }] as Message[],
+        cwd: wd,
+        tools: [],
+        chat: chatFn,
+      } as unknown as RunContext;
+      return {
+        globals: createWorkflowContext({
+          ctx,
+          state,
+          journal,
+          emit: (ev) => events.push(ev),
+          args: {},
+          workflowName: 'demo',
+          session: 's1',
+          task: state.taskId,
+        }),
+        journal,
+        events,
+      };
+    };
+
+    // 让完成顺序与发起顺序相反 —— 序号若按「谁先返回」分配，第二次必然错位
+    const slowChat = (texts: string[]) => {
+      let n = 0;
+      return async function* () {
+        const i = n++;
+        await new Promise((r) => setTimeout(r, (texts.length - i) * 3));
+        yield { type: 'text_delta' as const, delta: texts[i] ?? 'x' };
+      };
+    };
+    const texts = Array.from({ length: N }, (_, i) => `{"items":[${i}]}`);
+
+    const run1 = await build(slowChat(texts));
+    const out1 = await run1.globals.flow.pipeline(
+      Array.from({ length: N }, (_, i) => i),
+      { stages: [(_p: unknown, item: number) =>
+        run1.globals.model.generate(`处理 ${item}`, { schema: NUMS, isolated: true })] },
+    );
+    expect(out1.results).toHaveLength(N);
+
+    // 第二次：任何一次模型调用都算失败
+    const run2 = await build(async function* () {
+      throw new Error('第二次不该调模型');
+      // biome-ignore lint/correctness/noUnreachable: deliberate throwing stub
+      yield { type: 'text_delta' as const, delta: '' };
+    });
+    const out2 = await run2.globals.flow.pipeline(
+      Array.from({ length: N }, (_, i) => i),
+      { stages: [(_p: unknown, item: number) =>
+        run2.globals.model.generate(`处理 ${item}`, { schema: NUMS, isolated: true })] },
+    );
+
+    expect(out2.results).toEqual(out1.results);
+    expect(run2.journal.replayedCount).toBe(N);
+    // 车道并发告警不该出现 —— pipeline 给的是各自独立的车道
+    expect(
+      run2.events.some(
+        (e) =>
+          e.type === 'warning' &&
+          String(e.message).includes('断点续跑随之失效'),
+      ),
+    ).toBe(false);
+  });
+});
+
+describe('③ model 覆盖', () => {
+  test('schema repairs share the logical-call deadline rather than resetting it', async () => {
+    const signals: AbortSignal[] = [];
+    const chat = async function* (a: { signal: AbortSignal }) {
+      signals.push(a.signal);
+      if (signals.length === 1) {
+        yield { type: 'text_delta' as const, delta: 'invalid JSON' };
+      } else {
+        await new Promise<void>((resolve) => {
+          if (a.signal.aborted) resolve();
+          else
+            a.signal.addEventListener('abort', () => resolve(), { once: true });
+        });
+      }
+    };
+    const h = await harness(chat);
+    await expect(
+      h.globals.model.generate('repair', { schema: NUMS, timeoutMs: 500, isolated: true }),
+    ).rejects.toThrow('步骤时限');
+    expect(signals).toHaveLength(2);
+    expect(signals[0]).toBe(signals[1]);
+  });
+
+  test('queued calls expire without contacting provider and do not leak gate slots', async () => {
+    let started = 0;
+    const release: (() => void)[] = [];
+    const chat = async function* () {
+      started++;
+      if (started <= 8)
+        await new Promise<void>((resolve) => release.push(resolve));
+      yield { type: 'text_delta' as const, delta: 'ok' };
+    };
+    const h = await harness(chat);
+    const running = h.globals.flow.pipeline(
+      Array.from({ length: 8 }, (_, i) => i),
+      { concurrency: 8, stages: [(i: number) => h.globals.model.generate(`hold ${i}`, { isolated: true })] },
+    );
+    while (started < 8) await new Promise((resolve) => setTimeout(resolve, 1));
+    try {
+      await expect(h.globals.model.generate('queued', { timeoutMs: 20 })).rejects.toThrow(
+        '步骤时限',
+      );
+      expect(started).toBe(8);
+    } finally {
+      for (const resolve of release) resolve();
+      await running;
+    }
+    expect(await h.globals.model.generate('after', { timeoutMs: 200 })).toBe('ok');
+  });
+
+  test('未映射工作流角色时用基座模型做能力预检', async () => {
+    const chat = scriptedChat(['unused']);
+    const h = await harness(chat.fn);
+    const capability = (await h.globals.model.require(
+      'capability_probe_unmapped',
+      {
+        vision: true,
+      },
+    )) as {
+      role: string;
+      model: string;
+      supportsVision: boolean;
+      supportsJsonSchema: boolean;
+    };
+    expect(capability).toEqual({
+      role: 'capability_probe_unmapped',
+      model: 'base-model',
+      supportsVision: true,
+      supportsJsonSchema: false,
+    });
+  });
+
+  test('单步可以限制输出 token，且不超过 provider 自身上限', async () => {
+    const chat = scriptedChat(['ok']);
+    const h = await harness(chat.fn);
+    await h.globals.model.generate('短回答', { maxTokens: 1234 });
+    expect(chat.calls[0]?.maxTokens).toBe(1234);
+  });
+
+  test('单步超过总时限会中止，而不是无限等待流式推理', async () => {
+    const chat = async function* (a: { signal?: AbortSignal }) {
+      await new Promise<void>((resolve) => {
+        if (a.signal?.aborted) resolve();
+        else
+          a.signal?.addEventListener('abort', () => resolve(), { once: true });
+      });
+      if (!a.signal?.aborted) {
+        yield { type: 'text_delta' as const, delta: '不应到达' };
+      }
+    };
+    const h = await harness(chat);
+    await expect(
+      h.globals.model.generate('模拟持续推理', { timeoutMs: 10 }),
+    ).rejects.toThrow(/超过工作流步骤时限 10ms/);
+  });
+
+  test('★ 解析不到 → 回落基座 + 响亮告警，绝不静默换模型', async () => {
+    const chat = scriptedChat(['ok']);
+    const h = await harness(chat.fn);
+    await h.globals.model.generate('x', { model: '不存在的模型' });
+
+    expect(chat.calls[0].model).toBe('base-model'); // 真的回落了
+    expect(
+      h.events.some(
+        (e) =>
+          e.type === 'warning' &&
+          String(e.message).includes('找不到模型「不存在的模型」'),
+      ),
+    ).toBe(true);
+  });
+
+  test('同一个坏 id 只告警一次（循环里不刷屏）', async () => {
+    const h = await harness(scriptedChat(['ok']).fn);
+    await h.globals.model.generate('x', { model: 'bad' });
+    await h.globals.model.generate('y', { model: 'bad' });
+    const n = h.events.filter(
+      (e) => e.type === 'warning' && String(e.message).includes('找不到模型'),
+    ).length;
+    expect(n).toBe(1);
+  });
+});
+
+describe('④ isolated 隔离执行', () => {
+  test('agent schema repairs remain isolated as well as the initial call', async () => {
+    const chat = scriptedChat(['invalid', '{"items":[1]}']);
+    const h = await harness(chat.fn);
+    expect(
+      await h.globals.model.agent('x', {
+        isolated: true,
+        allowedTools: [],
+        schema: NUMS,
+      }),
+    ).toEqual({ items: [1] });
+    expect(h.history).toHaveLength(1);
+    expect(
+      h.events.some(
+        (e) =>
+          e.type === 'assistant_message' || e.type === 'user_message_committed',
+      ),
+    ).toBe(false);
+  });
+  test('★ 主线 history 一个字节都不变长', async () => {
+    const chat = scriptedChat(['隔离产出']);
+    const h = await harness(chat.fn);
+    const before = h.history.length;
+    const r = await h.globals.model.generate('打个分', { isolated: true });
+    expect(r).toBe('隔离产出');
+    expect(h.history.length).toBe(before);
+  });
+
+  test('★ 隔离调用看不到主线积累的上下文（只带系统提示）', async () => {
+    const chat = scriptedChat(['一', '二']);
+    const h = await harness(chat.fn);
+    await h.globals.model.generate('第一次'); // 非隔离：会把 user+assistant 追加进主线
+    const sharedCount = chat.calls[0].messageCount;
+    await h.globals.model.generate('第二次', { isolated: true });
+    // 非隔离的第二次会比第一次多；隔离的必须回到与第一次同量级
+    expect(chat.calls[1].messageCount).toBeLessThanOrEqual(sharedCount);
+  });
+
+  test('默认（不传 isolated）仍走单一上下文 —— 主线会变长', async () => {
+    const h = await harness(scriptedChat(['a', 'b']).fn);
+    const before = h.history.length;
+    await h.globals.model.generate('x');
+    expect(h.history.length).toBeGreaterThan(before);
+  });
+});