@capybara-hq/testing 0.0.0-stage → 0.1.0-rc.0

package/dist/model-conformance.js

@@ -0,0 +1,287 @@
+import { assertValidDeclarations, EFFORT_VALUES, isModelError, sanitizeParams, validateParams, } from "@capybara-hq/plugin-sdk";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { pdfBase64, redPngBase64 } from "./media-samples.js";
+function streamOf(t, req, signal) {
+    const { params, ...rest } = req;
+    if (t.contract === 2)
+        return t.provider.stream({ ...rest, params: params ?? {} }, { signal });
+    return t.provider.stream(rest, { signal });
+}
+/**
+ * `@2` 参数声明检查：每个模型的 `parameters` 须通过 `assertValidDeclarations`，
+ * 每个声明的默认值须通过 `validateParams`。返回问题列表（形如 `<模型 id>: <原因>`），无问题为空。
+ */
+export function checkDeclaredParameters(plugin, models) {
+    const problems = [];
+    for (const m of models) {
+        try {
+            assertValidDeclarations(plugin, m.parameters);
+        }
+        catch (e) {
+            problems.push(`${m.id}: ${e instanceof Error ? e.message : String(e)}`);
+            continue;
+        }
+        for (const d of m.parameters) {
+            if (d.default === undefined)
+                continue;
+            const r = validateParams(m.parameters, { [d.key]: d.default });
+            if (!r.ok)
+                problems.push(`${m.id}: default of ${r.issues.map((i) => i.message).join("; ")}`);
+        }
+    }
+    return problems;
+}
+const enumOf = (decl, key) => {
+    const s = decl.find((d) => d.key === key)?.schema;
+    return s?.type === "string" && "enum" in s ? s.enum : [];
+};
+/**
+ * thinking replay 用例的 `@2` 参数：按模型的声明取最高一档 effort、thinking 取 `summarized`（否则 `on`），
+ * 未声明的不设置；其余已声明参数补默认值。目的是让思考尽量稳定出现（adaptive 思考在简单问题上会跳过思考）。
+ */
+export function thinkingReplayParams(decl) {
+    const want = {};
+    const efforts = enumOf(decl, "effort");
+    const top = [...EFFORT_VALUES].reverse().find((e) => efforts.includes(e));
+    if (top !== undefined)
+        want.effort = top;
+    const thinking = enumOf(decl, "thinking");
+    const mode = ["summarized", "on"].find((m) => thinking.includes(m));
+    if (mode !== undefined)
+        want.thinking = mode;
+    return sanitizeParams(decl, want).value;
+}
+/** thinking replay 的提问：需要多步推理的短应用题（简单算式可能让 adaptive 思考直接跳过）。 */
+const THINKING_PROMPT = "A shop sells pencils at 3 for $2. Maria buys 27 pencils and pays with a $50 bill. " +
+    "How much change does she get? Work it out, then answer with just the amount.";
+async function collect(t, req, signal = new AbortController().signal) {
+    const events = [];
+    for await (const ev of streamOf(t, req, signal))
+        events.push(ev);
+    const completes = events.filter((e) => e.type === "message_complete");
+    expect(completes).toHaveLength(1);
+    expect(events[events.length - 1]?.type).toBe("message_complete");
+    return { events, done: completes[0] };
+}
+/**
+ * 媒体用例的样本与期望答案。夹具服务按原生格式认出这些样本后回答对应的词，所以夹具模式也能证明
+ * 插件把媒体块映射成了供应商认得的形状。
+ */
+export const MEDIA_SAMPLES = {
+    /** 64×64 纯红 PNG；期望回答含 "red"。 */
+    image: { mediaType: "image/png", name: "red.png", answer: "red" },
+    /** 一页 PDF，只印这个词。 */
+    pdf: { mediaType: "application/pdf", name: "fruit.pdf", word: "PINEAPPLE" },
+    /** 纯文本文档，内容是 {@link MEDIA_SAMPLES.text.content}。 */
+    text: {
+        mediaType: "text/plain",
+        name: "note.txt",
+        content: "The secret word is MANGO.\n",
+        word: "MANGO",
+    },
+};
+const textOf = (content) => content.map((b) => (b.type === "text" ? b.text : "")).join("");
+/** model-provider 一致性套件（`@1` 与 `@2`）：每个供应商插件都应通过。 */
+export function describeModelProviderConformance(name, setup) {
+    describe(`model-provider conformance: ${name}`, () => {
+        let t;
+        beforeAll(async () => {
+            t = await setup();
+        });
+        afterAll(async () => {
+            await t?.teardown();
+        });
+        const base = (over = {}) => ({
+            model: t.model,
+            system: [{ text: "You are a concise assistant.", cache: true }],
+            messages: [{ role: "user", content: [{ type: "text", text: "Say hello." }] }],
+            tools: [],
+            maxOutputTokens: 1024,
+            ...over,
+        });
+        const weather = {
+            name: "get_weather",
+            description: "Get the current weather for a city.",
+            inputSchema: {
+                type: "object",
+                properties: { city: { type: "string" } },
+                required: ["city"],
+            },
+        };
+        it("streams text", async () => {
+            const { events, done } = await collect(t, base());
+            const deltas = events.flatMap((e) => (e.type === "text_delta" ? [e.text] : []));
+            expect(deltas.length).toBeGreaterThan(0);
+            expect(deltas.join("")).toBe(textOf(done.content));
+            expect(done.stopReason).toBe("end_turn");
+        });
+        it("tool round trip", async () => {
+            const first = base({
+                tools: [weather],
+                messages: [
+                    { role: "user", content: [{ type: "text", text: "What is the weather in Paris?" }] },
+                ],
+            });
+            const { done } = await collect(t, first);
+            expect(done.stopReason).toBe("tool_use");
+            const call = done.content.find((b) => b.type === "tool_call");
+            expect(call).toBeDefined();
+            if (call?.type !== "tool_call")
+                return;
+            expect(typeof call.input).toBe("object");
+            expect(call.input).not.toBeNull();
+            const second = await collect(t, base({
+                tools: [weather],
+                messages: [
+                    ...first.messages,
+                    { role: "assistant", content: done.content },
+                    {
+                        role: "user",
+                        content: [
+                            { type: "tool_result", callId: call.id, content: "Sunny, 22C", isError: false },
+                        ],
+                    },
+                ],
+            }));
+            expect(second.done.content.some((b) => b.type === "text")).toBe(true);
+        });
+        /** 思考用例的参数：`@1` 发 `effort: "high"`；`@2` 见 {@link thinkingReplayParams}。 */
+        const thinkingOver = async () => {
+            if (t.contract !== 2)
+                return { effort: "high" };
+            const info = (await t.provider.listModels()).find((m) => m.id === t.model);
+            return { params: thinkingReplayParams(info?.parameters ?? []) };
+        };
+        it("declares valid parameters", async (ctx) => {
+            if (t.contract !== 2)
+                return ctx.skip();
+            expect(checkDeclaredParameters(t.plugin, await t.provider.listModels())).toEqual([]);
+        });
+        it("thinking replay", async (ctx) => {
+            if (!t.expect.thinking)
+                return ctx.skip();
+            // 最高一档 effort 下推理 token 也计入输出上限：1024 不够（gpt-5-nano 第二轮会 max_tokens），放宽
+            const over = { ...(await thinkingOver()), maxOutputTokens: 8192 };
+            const first = base({
+                ...over,
+                messages: [{ role: "user", content: [{ type: "text", text: THINKING_PROMPT }] }],
+            });
+            const { done } = await collect(t, first);
+            expect(done.content.some((b) => b.type === "thinking")).toBe(true);
+            const second = await collect(t, base({
+                ...over,
+                messages: [
+                    ...first.messages,
+                    { role: "assistant", content: done.content },
+                    { role: "user", content: [{ type: "text", text: "Now double that amount." }] },
+                ],
+            }));
+            expect(second.done.stopReason).toBe("end_turn");
+        });
+        it("caching", async (ctx) => {
+            if (!t.expect.caching)
+                return ctx.skip();
+            const req = base({
+                system: [
+                    {
+                        text: `Stable prefix. ${"Reference material. ".repeat(Math.ceil((t.cachePrefixTokens ?? 5000) / 4))}`,
+                        cache: true,
+                    },
+                ],
+            });
+            await collect(t, req);
+            const second = await collect(t, req);
+            expect(second.done.usage.cacheReadTokens).toBeGreaterThan(0);
+        });
+        it("usage", async () => {
+            const { done } = await collect(t, base());
+            // inputTokens 只是未缓存部分：缓存命中时可能为 0，所以看三类输入之和
+            const u = done.usage;
+            expect(u.inputTokens + u.cacheReadTokens + u.cacheWriteTokens).toBeGreaterThan(0);
+            expect(u.outputTokens).toBeGreaterThan(0);
+        });
+        it("mid-conversation system", async () => {
+            const { done } = await collect(t, base({
+                messages: [
+                    { role: "user", content: [{ type: "text", text: "Say hello." }] },
+                    { role: "assistant", content: [{ type: "text", text: "Hello." }] },
+                    { role: "system", content: "Reminder: keep answers under ten words." },
+                    { role: "user", content: [{ type: "text", text: "Say goodbye." }] },
+                ],
+            }));
+            expect(done.content.length).toBeGreaterThan(0);
+        });
+        /** `@2` 模型的能力声明；`@1` 返回 null（媒体用例只对 `@2` 跑）。 */
+        const caps = async () => {
+            if (t.contract !== 2)
+                return null;
+            const info = (await t.provider.listModels()).find((m) => m.id === t.model);
+            return info?.capabilities ?? null;
+        };
+        const ask = (block, question) => base({
+            messages: [{ role: "user", content: [block, { type: "text", text: question }] }],
+        });
+        it("user message with image", async (ctx) => {
+            if ((await caps())?.vision !== true)
+                return ctx.skip();
+            const s = MEDIA_SAMPLES.image;
+            const { done } = await collect(t, ask({
+                type: "image",
+                mediaType: s.mediaType,
+                name: s.name,
+                source: { kind: "base64", data: redPngBase64() },
+            }, "What is the color of this image? Answer with one word."));
+            expect(done.stopReason).toBe("end_turn");
+            expect(textOf(done.content).toLowerCase()).toContain(s.answer);
+        });
+        it("user message with PDF document", async (ctx) => {
+            if ((await caps())?.documents !== true)
+                return ctx.skip();
+            const s = MEDIA_SAMPLES.pdf;
+            const { done } = await collect(t, ask({
+                type: "document",
+                mediaType: s.mediaType,
+                name: s.name,
+                source: { kind: "base64", data: pdfBase64(s.word) },
+            }, "What single word is printed in this PDF? Answer with just the word."));
+            expect(done.stopReason).toBe("end_turn");
+            expect(textOf(done.content).toUpperCase()).toContain(s.word);
+        });
+        it("user message with text document", async (ctx) => {
+            if ((await caps())?.documents !== true)
+                return ctx.skip();
+            const s = MEDIA_SAMPLES.text;
+            const { done } = await collect(t, ask({
+                type: "document",
+                mediaType: s.mediaType,
+                name: s.name,
+                source: { kind: "base64", data: Buffer.from(s.content, "utf8").toString("base64") },
+            }, "What is the secret word in the attached file? Answer with just the word."));
+            expect(done.stopReason).toBe("end_turn");
+            expect(textOf(done.content).toUpperCase()).toContain(s.word);
+        });
+        it("abort", async () => {
+            const ac = new AbortController();
+            const it = streamOf(t, base(), ac.signal)[Symbol.asyncIterator]();
+            const firstEv = await it.next();
+            expect(firstEv.done).toBe(false);
+            ac.abort();
+            let err;
+            try {
+                // 事件不再继续产出：中止后最多再得到已缓冲的事件，随后必须抛错而不是正常结束
+                for (;;) {
+                    const r = await it.next();
+                    if (r.done)
+                        break;
+                    expect(r.value.type).not.toBe("message_complete");
+                }
+            }
+            catch (e) {
+                err = e;
+            }
+            expect(isModelError(err)).toBe(true);
+            if (isModelError(err))
+                expect(err.kind).toBe("aborted");
+        });
+    });
+}

package/dist/replica-runner.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/replica-runner.js

@@ -0,0 +1,59 @@
+import { createKernel, KernelConfigSchema } from "@capybara-hq/kernel";
+let kernel;
+const isPlainObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
+/** 深合并：普通对象逐键递归，其余（含数组）由 `over` 整体替换。 */
+function deepMerge(base, over) {
+    const out = { ...base };
+    for (const [k, v] of Object.entries(over)) {
+        const b = out[k];
+        out[k] = isPlainObject(b) && isPlainObject(v) ? deepMerge(b, v) : v;
+    }
+    return out;
+}
+/** 启动内核并回报实际监听端口。 */
+async function startKernel(c) {
+    const base = {
+        db: { url: c.dbUrl, poolMax: 3 },
+        redis: { url: c.redisUrl },
+        masterKey: c.masterKey,
+        http: { port: 0, replicaId: c.replicaId },
+        roles: c.roles,
+        extensions: c.extensions,
+        extensionConfig: c.extensionConfig,
+        // 测试默认自动批准插件权限（生产默认关闭）；调用方可经 config 覆盖为 false 验证待批准行为
+        extensionPermissions: { autoApprove: true },
+        // The official marketplace is on by default and on the internet: tests turn it on explicitly.
+        marketplaces: { official: { enabled: false } },
+        log: { level: "error" },
+    };
+    const config = KernelConfigSchema.parse(deepMerge(base, c.overrides));
+    kernel = await createKernel(config, {
+        resolveFrom: c.resolveFrom,
+        ...(c.bootstrapDir !== undefined && { bootstrapDir: c.bootstrapDir }),
+        ...(c.bootstrapManifest !== undefined && { bootstrapManifest: c.bootstrapManifest }),
+    });
+    await kernel.start();
+    process.send?.({ type: "ready", port: kernel.port });
+}
+let exiting = false;
+/** 停内核后退出；内核停机卡住时 10 秒后强制退出。 */
+function shutdown() {
+    if (exiting)
+        return;
+    exiting = true;
+    setTimeout(() => process.exit(1), 10_000).unref();
+    (kernel?.stop() ?? Promise.resolve()).finally(() => process.exit(0));
+}
+// 父进程（含被 SIGKILL）消失后 IPC 通道断开，副本自行退出，不留孤儿
+process.on("disconnect", shutdown);
+process.on("message", (msg) => {
+    if (msg.type === "start") {
+        startKernel(msg.config).catch((e) => {
+            // 只回报错误信息，绝不回显配置
+            process.send?.({ type: "failed", error: e instanceof Error ? e.message : String(e) }, () => process.exit(1));
+        });
+    }
+    else if (msg.type === "stop") {
+        shutdown();
+    }
+});

package/dist/sse.d.ts

@@ -0,0 +1,20 @@
+/** One parsed SSE message; `comment` is a line starting with `:` (a heartbeat). */
+export interface SseMessage {
+    id?: string;
+    event?: string;
+    data?: string;
+    comment?: string;
+}
+/** An SSE stream read with fetch: messages parsed as they arrive, `until` waits for a condition. */
+export interface SseClient {
+    status: number;
+    headers: Headers;
+    messages: SseMessage[];
+    until(pred: (msgs: SseMessage[]) => boolean, timeoutMs?: number, label?: string): Promise<void>;
+    close(): void;
+}
+/**
+ * Opens an SSE connection (a real port). A non-200 answer is returned too, with no `messages`;
+ * `close()` aborts the request.
+ */
+export declare function openSse(url: string, headers: Record<string, string>): Promise<SseClient>;

package/dist/sse.js

@@ -0,0 +1,87 @@
+/**
+ * Opens an SSE connection (a real port). A non-200 answer is returned too, with no `messages`;
+ * `close()` aborts the request.
+ */
+export async function openSse(url, headers) {
+    const ac = new AbortController();
+    const res = await fetch(url, { headers, signal: ac.signal });
+    const messages = [];
+    const waiters = new Set();
+    const notify = () => {
+        for (const w of [...waiters])
+            w();
+    };
+    let ended = false;
+    void (async () => {
+        if (!res.body)
+            return;
+        const reader = res.body.getReader();
+        const dec = new TextDecoder();
+        let buf = "";
+        try {
+            for (;;) {
+                const { done, value } = await reader.read();
+                if (done)
+                    break;
+                buf += dec.decode(value, { stream: true });
+                for (let i = buf.indexOf("\n\n"); i >= 0; i = buf.indexOf("\n\n")) {
+                    const block = buf.slice(0, i);
+                    buf = buf.slice(i + 2);
+                    const m = {};
+                    for (const line of block.split("\n")) {
+                        if (line.startsWith(":"))
+                            m.comment = line.slice(1).trim();
+                        else {
+                            const c = line.indexOf(":");
+                            const key = line.slice(0, c);
+                            const v = line.slice(c + 1).replace(/^ /, "");
+                            if (key === "id" || key === "event")
+                                m[key] = v;
+                            else if (key === "data")
+                                m.data = m.data === undefined ? v : `${m.data}\n${v}`;
+                        }
+                    }
+                    messages.push(m);
+                }
+                notify();
+            }
+        }
+        catch {
+            // aborted by close()
+        }
+        finally {
+            ended = true;
+            notify();
+        }
+    })();
+    return {
+        status: res.status,
+        headers: res.headers,
+        messages,
+        until(pred, timeoutMs = 10_000, label = "sse condition") {
+            return new Promise((resolve, reject) => {
+                const check = () => {
+                    if (pred(messages)) {
+                        cleanup();
+                        resolve();
+                    }
+                    else if (ended) {
+                        cleanup();
+                        reject(new Error(`stream ended before ${label}`));
+                    }
+                };
+                const timer = setTimeout(() => {
+                    cleanup();
+                    reject(new Error(`timed out waiting for ${label}`));
+                }, timeoutMs);
+                const cleanup = () => {
+                    clearTimeout(timer);
+                    waiters.delete(check);
+                };
+                waiters.add(check);
+                check();
+            });
+        },
+        close: () => ac.abort(),
+    };
+}

package/fixtures/extensions/fake-bootstrap-a/.capybara-plugin/plugin.json

@@ -0,0 +1,7 @@
+{
+  "schemaVersion": 1,
+  "name": "fake-bootstrap-a",
+  "version": "0.1.0",
+  "publisher": "Capybara",
+  "description": "Test fake for bootstrap: a model-provider@2 extension that needs deployment config"
+}

package/fixtures/extensions/fake-bootstrap-a/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "@capybara-hq/test-plugin-fake-bootstrap-a",
+  "version": "0.1.0",
+  "private": true,
+  "description": "Test fake for bootstrap: a model-provider@2 extension that needs deployment config",
+  "license": "Apache-2.0",
+  "type": "module",
+  "dependencies": {
+    "@capybara-hq/plugin-sdk": "^0.1.0-rc.0",
+    "zod": "^4.6.7"
+  }
+}

package/fixtures/extensions/fake-bootstrap-a/src/index.ts

@@ -0,0 +1,62 @@
+import {
+  defineExtension,
+  MODEL_PROVIDER_EXTENSION_POINT_V2,
+  type ModelProvider,
+  secret,
+} from "@capybara-hq/plugin-sdk";
+import { z } from "zod";
+
+/**
+ * Test fake for the bootstrap step: a model provider (`model-provider@2`) whose deployment config
+ * has a required secret `apiKey`, so an unconfigured deployment leaves it pending configuration
+ * (as the image's model provider is). Once configured it offers one model `fake-a-1` that answers
+ * every request with `fake-bootstrap-a`; it never calls a network service.
+ */
+
+const DeploymentConfig = z.object({
+  apiKey: secret(z.string().min(1)),
+  baseUrl: z.string().url().optional(),
+});
+
+const REPLY = "fake-bootstrap-a";
+
+const provider: ModelProvider = {
+  async listModels() {
+    return [
+      {
+        id: "fake-a-1",
+        displayName: "Fake A 1",
+        contextWindow: 8192,
+        maxOutputTokens: 1024,
+        capabilities: {
+          tools: false,
+          vision: false,
+          reasoning: false,
+          caching: false,
+          midConversationSystem: false,
+        },
+        parameters: [],
+      },
+    ];
+  },
+  async *stream() {
+    yield { type: "text_delta", text: REPLY };
+    yield {
+      type: "message_complete",
+      content: [{ type: "text", text: REPLY }],
+      stopReason: "end_turn",
+      usage: { inputTokens: 1, outputTokens: 1, cacheReadTokens: 0, cacheWriteTokens: 0 },
+    };
+  },
+};
+
+export default defineExtension({
+  name: "fake-bootstrap-a",
+  version: "0.1.0",
+  sdk: "^0.1.0-rc.0",
+  provides: [MODEL_PROVIDER_EXTENSION_POINT_V2],
+  config: { deployment: DeploymentConfig },
+  setup(ctx) {
+    ctx.provide(MODEL_PROVIDER_EXTENSION_POINT_V2, provider);
+  },
+});

package/fixtures/extensions/fake-bootstrap-b/.capybara-plugin/plugin.json

@@ -0,0 +1,7 @@
+{
+  "schemaVersion": 1,
+  "name": "fake-bootstrap-b",
+  "version": "0.1.0",
+  "publisher": "Capybara",
+  "description": "Test fake for bootstrap: an object-store@1 extension that keeps objects in its PostgreSQL schema"
+}

package/fixtures/extensions/fake-bootstrap-b/migrations/0001_init.sql

@@ -0,0 +1,10 @@
+-- Runs with search_path "plugin_fake_bootstrap_b, public": table names carry no schema prefix.
+-- The objects of the fake store, per configured bucket.
+CREATE TABLE IF NOT EXISTS objects (
+  bucket text NOT NULL,
+  key text NOT NULL,
+  content_type text NOT NULL,
+  data bytea NOT NULL,
+  updated_at timestamptz NOT NULL DEFAULT now(),
+  PRIMARY KEY (bucket, key)
+);

package/fixtures/extensions/fake-bootstrap-b/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "@capybara-hq/test-plugin-fake-bootstrap-b",
+  "version": "0.1.0",
+  "private": true,
+  "description": "Test fake for bootstrap: an object-store@1 extension that keeps objects in its PostgreSQL schema",
+  "license": "Apache-2.0",
+  "type": "module",
+  "dependencies": {
+    "@capybara-hq/plugin-sdk": "^0.1.0-rc.0",
+    "zod": "^4.6.7"
+  }
+}