@capybara-hq/testing 0.0.0-stage → 0.1.0-rc.0

package/dist/kernel-helpers.js

@@ -0,0 +1,336 @@
+import { execFile } from "node:child_process";
+import { readdir, readFile, readlink } from "node:fs/promises";
+import { promisify } from "node:util";
+import { AccountsService, createKernel, createLogger, KernelConfigSchema, MARKETPLACE_OFFICIAL_URL_DEFAULT, } from "@capybara-hq/kernel";
+import { sql } from "kysely";
+/**
+ * Helpers for extension tests that run a real kernel: in process (`startTestKernel` +
+ * `http.inject`) or as two replicas (`createDualReplicaHarness`). They only use the kernel's public
+ * API and HTTP routes, so a plugin's tests can use them without reaching into kernel sources.
+ */
+/** The password `createLocalAccount` gives accounts (and `login` / `joinOrg` use by default). */
+export const TEST_PASSWORD = "correct horse battery";
+export const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/** Polls `fn` until it returns a value other than null / undefined / false; throws on timeout. */
+export async function waitFor(fn, timeoutMs = 10_000, label = "condition", stepMs = 25) {
+    const deadline = Date.now() + timeoutMs;
+    for (;;) {
+        const v = await fn();
+        if (v !== null && v !== undefined && v !== false)
+            return v;
+        if (Date.now() > deadline)
+            throw new Error(`timed out waiting for ${label}`);
+        await sleep(stepMs);
+    }
+}
+/**
+ * A kernel config for tests: the given PG / Redis, a fixed master key, an ephemeral HTTP port,
+ * auto-approved extension permissions, short run-lock timings and the official marketplace off
+ * (it is on by default and its address is on the internet). `overrides` are top-level config keys
+ * and win; `runtime` is merged into the defaults, and `marketplaces` too, so other marketplace
+ * settings keep the official one off unless `marketplaces.official` turns it on.
+ */
+export function testKernelConfig(urls, overrides) {
+    const { runtime, marketplaces, ...rest } = overrides;
+    return KernelConfigSchema.parse({
+        db: { url: urls.pg, poolMax: 5 },
+        redis: { url: urls.redis },
+        masterKey: Buffer.alloc(32, 7).toString("base64"),
+        http: { port: 0 },
+        extensionPermissions: { autoApprove: true },
+        runtime: { jobLockMs: 2000, stalledIntervalMs: 1000, ...runtime },
+        marketplaces: {
+            ...marketplaces,
+            official: { enabled: false, ...marketplaces?.official },
+        },
+        ...rest,
+    });
+}
+/**
+ * Throws when `config` would make a test kernel fetch the real official marketplace (enabled, at its
+ * default address on the internet): tests turn it off or point it at a fake. A kernel helper calls
+ * it before creating anything.
+ */
+export function assertOfflineOfficialMarketplace(config, allow = false) {
+    const o = config.marketplaces.official;
+    if (allow || !o.enabled || o.url !== MARKETPLACE_OFFICIAL_URL_DEFAULT)
+        return;
+    throw new Error("test kernel config would fetch the real official marketplace " +
+        `(${MARKETPLACE_OFFICIAL_URL_DEFAULT}): set marketplaces.official.enabled to false, point ` +
+        "marketplaces.official.url at a fake, or build the config with testKernelConfig");
+}
+/**
+ * Creates and starts an in-process kernel. Logs are discarded unless `log` is given (JSON lines at
+ * `log.level` and above go to `log.write`, e.g. to assert a secret never reaches the log).
+ * Extensions named by package resolve from `resolveFrom` (default `process.cwd()`). Stop it with
+ * `stop()`.
+ */
+export async function startTestKernel(config, opts = {}) {
+    assertOfflineOfficialMarketplace(config, opts.allowOfficialMarketplace);
+    const k = await createKernel(config, {
+        env: {},
+        logger: createLogger(opts.log?.level ?? "error", { write: opts.log?.write ?? (() => { }) }),
+        ...(opts.resolveFrom !== undefined && { resolveFrom: opts.resolveFrom }),
+    });
+    await k.start();
+    return k;
+}
+/** A request function with its own cookie jar (signed out until a response sets cookies). */
+export function cookieRequest(http) {
+    const jar = new Map();
+    return async ({ csrf = true, ...opts }) => {
+        const cookie = [...jar].map(([k, v]) => `${k}=${encodeURIComponent(v)}`).join("; ");
+        const res = await http.inject({
+            ...opts,
+            headers: {
+                ...(cookie !== "" && { cookie }),
+                ...(csrf && { "x-capybara-csrf": "1" }),
+                ...opts.headers,
+            },
+        });
+        const set = res.headers["set-cookie"];
+        for (const raw of set === undefined ? [] : Array.isArray(set) ? set : [String(set)]) {
+            const [pair = "", ...attrs] = raw.split(";");
+            const eq = pair.indexOf("=");
+            const name = pair.slice(0, eq).trim();
+            const value = decodeURIComponent(pair.slice(eq + 1).trim());
+            const gone = value === "" || attrs.some((a) => /^\s*max-age=0\s*$/i.test(a));
+            if (gone)
+                jar.delete(name);
+            else
+                jar.set(name, value);
+        }
+        return res;
+    };
+}
+function expectStatus(res, status, what) {
+    if (res.statusCode !== status) {
+        throw new Error(`${what}: expected HTTP ${status}, got ${res.statusCode}: ${res.body}`);
+    }
+}
+/** Creates a local account with {@link TEST_PASSWORD}; returns its id. */
+export async function createLocalAccount(db, login, displayName = login) {
+    return (await new AccountsService(db).createLocal({ login, password: TEST_PASSWORD, displayName })).id;
+}
+/** Signs in over HTTP and returns the user's request function. */
+export async function login(k, name, password = TEST_PASSWORD) {
+    const req = cookieRequest(k.http);
+    const res = await req({
+        method: "POST",
+        url: "/api/v1/auth/login",
+        payload: { login: name, password },
+    });
+    expectStatus(res, 200, `login ${name}`);
+    return req;
+}
+/** Creates an org as `req`'s user (who becomes its owner); returns its id. */
+export async function createOrg(req, name = "Acme") {
+    const res = await req({ method: "POST", url: "/api/v1/orgs", payload: { name } });
+    expectStatus(res, 201, "create org");
+    return res.json().org.id;
+}
+/**
+ * A new account joins the org through an invitation (created by `owner`, accepted with `login`
+ * and {@link TEST_PASSWORD}); returns the account id and its signed-in request function.
+ */
+export async function joinOrg(k, owner, orgId, name, role = "member") {
+    const inv = await owner({
+        method: "POST",
+        url: `/api/v1/orgs/${orgId}/invitations`,
+        payload: { role },
+    });
+    expectStatus(inv, 201, "create invitation");
+    const req = cookieRequest(k.http);
+    const accepted = await req({
+        method: "POST",
+        url: `/api/v1/invitations/${inv.json().token}/accept`,
+        payload: { login: name, password: TEST_PASSWORD, displayName: name },
+    });
+    expectStatus(accepted, 200, "accept invitation");
+    return { accountId: accepted.json().account.id, req };
+}
+/**
+ * Creates an org-visible agent; returns its id. Without `tools` the agent has none (model-fake then
+ * answers ordinary messages instead of calling the first tool).
+ */
+export async function createAgent(req, orgId, opts = {}) {
+    const res = await req({
+        method: "POST",
+        url: `/api/v1/orgs/${orgId}/agents`,
+        payload: {
+            definition: {
+                name: "Helper",
+                instructions: "Be helpful.",
+                model: opts.model ?? { provider: "model-fake", model: "fake-1" },
+                tools: opts.tools ?? [],
+            },
+            visibility: "org",
+        },
+    });
+    expectStatus(res, 201, "create agent");
+    return res.json().agent.id;
+}
+/** Creates a conversation with the agent; returns its id. */
+export async function createConversation(req, orgId, agentId) {
+    const res = await req({
+        method: "POST",
+        url: `/api/v1/orgs/${orgId}/conversations`,
+        payload: { agentId },
+    });
+    expectStatus(res, 201, "create conversation");
+    return res.json().conversation.id;
+}
+/** Posts a user message; returns its event seq. */
+export async function postMessage(req, orgId, conversationId, text) {
+    const res = await req({
+        method: "POST",
+        url: `/api/v1/orgs/${orgId}/conversations/${conversationId}/messages`,
+        payload: { text },
+    });
+    expectStatus(res, 202, "post message");
+    return res.json().event.seq;
+}
+/** Every event of a conversation, in seq order (read from the database). */
+export async function eventsOf(db, conversationId) {
+    const rows = await db
+        .selectFrom("conversation_events")
+        .select(["conversation_id", "seq", "type", "run_id", "payload", "created_at"])
+        .where("conversation_id", "=", conversationId)
+        .orderBy("seq")
+        .execute();
+    return rows.map((r) => ({
+        conversationId: r.conversation_id,
+        seq: Number(r.seq),
+        type: r.type,
+        runId: r.run_id,
+        payload: r.payload,
+        createdAt: r.created_at,
+    }));
+}
+/** Waits for the first event matching `pred` and returns it. */
+export function waitForEvent(db, conversationId, pred, timeoutMs = 10_000, label = "event") {
+    return waitFor(async () => (await eventsOf(db, conversationId)).find(pred), timeoutMs, label);
+}
+/** Waits for the conversation's `n`-th terminal event (run_finished / run_failed / run_aborted). */
+export function waitForTerminal(db, conversationId, n = 1, timeoutMs = 15_000) {
+    return waitFor(async () => {
+        const t = (await eventsOf(db, conversationId)).filter((e) => ["run_finished", "run_failed", "run_aborted"].includes(e.type));
+        return t.length >= n ? t : null;
+    }, timeoutMs, `${n} terminal event(s)`);
+}
+/**
+ * Waits until an IM card's message id is recorded (`channel_card_messages`). The kernel writes it in
+ * the transaction after the card is sent; a click before that is treated as "not a card of this
+ * conversation", so a test waits for this before it clicks, not only for the platform to get it.
+ */
+export async function cardRecorded(db, card, timeoutMs = 20_000) {
+    await waitFor(async () => {
+        const q = db.selectFrom("channel_card_messages").select("card_id");
+        const row = await ("messageId" in card
+            ? q.where("message_id", "=", card.messageId)
+            : q.where("card_kind", "=", card.kind)).executeTakeFirst();
+        return row !== undefined;
+    }, timeoutMs, "card message row");
+}
+/** PG deadlock (40P01) and lock timeout (55P03). */
+const LOCK_CONFLICTS = new Set(["40P01", "55P03"]);
+const TRUNCATE_ATTEMPTS = 8;
+/**
+ * `TRUNCATE <tables> CASCADE` between tests, retried on deadlock / lock timeout: a kernel with the
+ * worker role keeps scanning in the background (channel supervisor, deliveries, maintenance), and a
+ * background join holding one table while waiting for another deadlocks with TRUNCATE's ordered
+ * ACCESS EXCLUSIVE locks. TRUNCATE is atomic, so a retry is safe. `tables` is inlined into the SQL:
+ * pass only table names written in the test.
+ */
+export async function truncateTables(db, tables) {
+    for (let attempt = 1;; attempt++) {
+        try {
+            await db.transaction().execute(async (t) => {
+                await sql `SET LOCAL lock_timeout = '5s'`.execute(t);
+                await sql.raw(`TRUNCATE ${tables} CASCADE`).execute(t);
+            });
+            return;
+        }
+        catch (e) {
+            const code = typeof e === "object" && e !== null ? (e.code ?? "") : "";
+            if (!LOCK_CONFLICTS.has(code) || attempt >= TRUNCATE_ATTEMPTS)
+                throw e;
+            console.warn(`truncateTables: ${code} on attempt ${attempt}/${TRUNCATE_ATTEMPTS}, retrying`);
+            await sleep(50 * attempt + Math.floor(Math.random() * 50));
+        }
+    }
+}
+const run = promisify(execFile);
+/**
+ * Finds which of `pids` owns the local TCP port `port` (the client side of a connection). Dual
+ * replica tests use it to map a fake platform's long-connection peer port (`socketPeerPorts()`) to
+ * a replica process: a real extension connecting to a fake platform carries no pid marker.
+ *
+ * Linux reads `/proc/net/tcp{,6}` for the socket inode and looks for it under `/proc/<pid>/fd`,
+ * falling back to `ss -tnp`, then `lsof`; other platforms (macOS) use `lsof`. Undefined when not
+ * found.
+ */
+export async function pidOwningLocalPort(port, pids) {
+    if (process.platform === "linux") {
+        return (await viaProc(port, pids)) ?? (await viaSs(port, pids)) ?? (await viaLsof(port, pids));
+    }
+    return viaLsof(port, pids);
+}
+async function viaProc(port, pids) {
+    const hexPort = port.toString(16).toUpperCase().padStart(4, "0");
+    const inodes = new Set();
+    for (const file of ["/proc/net/tcp", "/proc/net/tcp6"]) {
+        const text = await readFile(file, "utf8").catch(() => "");
+        for (const line of text.split("\n").slice(1)) {
+            const cols = line.trim().split(/\s+/);
+            const local = cols[1];
+            const inode = cols[9];
+            if (local?.endsWith(`:${hexPort}`) && inode !== undefined && inode !== "0")
+                inodes.add(inode);
+        }
+    }
+    if (inodes.size === 0)
+        return undefined;
+    for (const pid of pids) {
+        const fds = await readdir(`/proc/${pid}/fd`).catch(() => []);
+        for (const fd of fds) {
+            const target = await readlink(`/proc/${pid}/fd/${fd}`).catch(() => "");
+            const m = /^socket:\[(\d+)\]$/.exec(target);
+            if (m?.[1] !== undefined && inodes.has(m[1]))
+                return pid;
+        }
+    }
+    return undefined;
+}
+async function viaSs(port, pids) {
+    // Lines like `ESTAB 0 0 127.0.0.1:54321 127.0.0.1:8080 users:(("node",pid=1234,fd=20))`
+    const out = await run("ss", ["-tnpH", "state", "established"]).then((r) => r.stdout, () => "");
+    for (const line of out.split("\n")) {
+        const cols = line.trim().split(/\s+/);
+        const local = cols.find((c, i) => i < 5 && /:\d+$/.test(c));
+        if (!local?.endsWith(`:${port}`))
+            continue;
+        for (const m of line.matchAll(/pid=(\d+)/g)) {
+            const pid = Number(m[1]);
+            if (pids.includes(pid))
+                return pid;
+        }
+    }
+    return undefined;
+}
+async function viaLsof(port, pids) {
+    // -a: AND the conditions; -iTCP:<port> matches the local or the peer port, and the peer is the
+    // fake platform (this test process, not in pids)
+    const args = ["-nP", "-a", "-p", pids.join(","), `-iTCP:${port}`, "-sTCP:ESTABLISHED", "-Fp"];
+    const out = await run("lsof", args).then((r) => r.stdout, 
+    // lsof exits with 1 when nothing matches
+    (e) => e.stdout ?? "");
+    for (const line of out.split("\n")) {
+        if (!line.startsWith("p"))
+            continue;
+        const pid = Number(line.slice(1));
+        if (pids.includes(pid))
+            return pid;
+    }
+    return undefined;
+}

package/dist/media-samples.d.ts

@@ -0,0 +1,10 @@
+/** 纯色 PNG（8 位 RGB），返回 base64。 */
+export declare function solidPngBase64(rgb: [number, number, number], size?: number): string;
+/** 64×64 纯红 PNG 的 base64。 */
+export declare const redPngBase64: () => string;
+/** 一页 PDF，页面上只印 `word`（大写字母与数字），返回 base64。 */
+export declare function pdfBase64(word: string): string;
+/** 夹具用：base64 是否为 PNG（看文件头）。 */
+export declare function isPngBase64(data: unknown): boolean;
+/** 夹具用：从 {@link pdfBase64} 生成的 PDF 里读出印的词；不是这样的 PDF 时返回 null。 */
+export declare function pdfSampleWord(data: unknown): string | null;

package/dist/media-samples.js

@@ -0,0 +1,76 @@
+import { crc32, deflateSync } from "node:zlib";
+/**
+ * 一致性套件与夹具用的媒体样本：运行时生成，不在仓库里放二进制文件。
+ * - {@link redPngBase64}：64×64 纯红 PNG（真实模型能稳定答出颜色）；
+ * - {@link pdfBase64}：一页、只印一个单词的最小 PDF（Helvetica，偏移表按实际字节计算）。
+ */
+function pngChunk(type, data) {
+    const len = Buffer.alloc(4);
+    len.writeUInt32BE(data.length);
+    const body = Buffer.concat([Buffer.from(type, "ascii"), data]);
+    const crc = Buffer.alloc(4);
+    crc.writeUInt32BE(crc32(body) >>> 0);
+    return Buffer.concat([len, body, crc]);
+}
+/** 纯色 PNG（8 位 RGB），返回 base64。 */
+export function solidPngBase64(rgb, size = 64) {
+    const ihdr = Buffer.alloc(13);
+    ihdr.writeUInt32BE(size, 0);
+    ihdr.writeUInt32BE(size, 4);
+    ihdr[8] = 8; // 位深
+    ihdr[9] = 2; // 真彩色
+    const row = 1 + size * 3;
+    const raw = Buffer.alloc(size * row);
+    for (let y = 0; y < size; y++) {
+        for (let x = 0; x < size; x++)
+            raw.set(rgb, y * row + 1 + x * 3);
+    }
+    return Buffer.concat([
+        Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]),
+        pngChunk("IHDR", ihdr),
+        pngChunk("IDAT", deflateSync(raw)),
+        pngChunk("IEND", Buffer.alloc(0)),
+    ]).toString("base64");
+}
+/** 64×64 纯红 PNG 的 base64。 */
+export const redPngBase64 = () => solidPngBase64([255, 0, 0]);
+/** 一页 PDF，页面上只印 `word`（大写字母与数字），返回 base64。 */
+export function pdfBase64(word) {
+    if (!/^[A-Z0-9]+$/.test(word))
+        throw new Error("pdfBase64: word must be [A-Z0-9]+");
+    const stream = `BT /F1 36 Tf 72 700 Td (${word}) Tj ET`;
+    const objects = [
+        "<< /Type /Catalog /Pages 2 0 R >>",
+        "<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+        "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " +
+            "/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>",
+        "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+        `<< /Length ${stream.length} >>\nstream\n${stream}\nendstream`,
+    ];
+    let pdf = "%PDF-1.4\n";
+    const offsets = [];
+    objects.forEach((o, i) => {
+        offsets.push(pdf.length);
+        pdf += `${i + 1} 0 obj\n${o}\nendobj\n`;
+    });
+    const xref = pdf.length;
+    pdf += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
+    pdf += offsets.map((o) => `${String(o).padStart(10, "0")} 00000 n \n`).join("");
+    pdf += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`;
+    return Buffer.from(pdf, "latin1").toString("base64");
+}
+/** 夹具用：base64 是否为 PNG（看文件头）。 */
+export function isPngBase64(data) {
+    if (typeof data !== "string")
+        return false;
+    return Buffer.from(data, "base64").subarray(0, 4).toString("latin1") === "\x89PNG";
+}
+/** 夹具用：从 {@link pdfBase64} 生成的 PDF 里读出印的词；不是这样的 PDF 时返回 null。 */
+export function pdfSampleWord(data) {
+    if (typeof data !== "string")
+        return null;
+    const pdf = Buffer.from(data, "base64").toString("latin1");
+    if (!pdf.startsWith("%PDF-"))
+        return null;
+    return /\(([A-Z0-9]+)\) Tj/.exec(pdf)?.[1] ?? null;
+}

package/dist/model-conformance.d.ts

@@ -0,0 +1,61 @@
+import { type ModelInfo, type ModelParameter, type ModelProvider, type ModelProviderV1, type ParamValue } from "@capybara-hq/plugin-sdk";
+interface TargetCommon {
+    model: string;
+    expect: {
+        thinking: boolean;
+        caching: boolean;
+    };
+    /** caching 用例的稳定前缀大小（约 token 数），默认 5000。 */
+    cachePrefixTokens?: number;
+    teardown(): Promise<void>;
+}
+/**
+ * 被测供应商：`contract` 缺省为 1（`model-provider@1`）；`@2` 插件须给 `contract: 2` 与插件名
+ * （用于校验厂商专有参数的键名前缀）。
+ */
+export type ConformanceTarget = TargetCommon & ({
+    contract?: 1;
+    provider: ModelProviderV1;
+} | {
+    contract: 2;
+    plugin: string;
+    provider: ModelProvider;
+});
+/**
+ * `@2` 参数声明检查：每个模型的 `parameters` 须通过 `assertValidDeclarations`，
+ * 每个声明的默认值须通过 `validateParams`。返回问题列表（形如 `<模型 id>: <原因>`），无问题为空。
+ */
+export declare function checkDeclaredParameters(plugin: string, models: ModelInfo[]): string[];
+/**
+ * thinking replay 用例的 `@2` 参数：按模型的声明取最高一档 effort、thinking 取 `summarized`（否则 `on`），
+ * 未声明的不设置；其余已声明参数补默认值。目的是让思考尽量稳定出现（adaptive 思考在简单问题上会跳过思考）。
+ */
+export declare function thinkingReplayParams(decl: ModelParameter[]): Record<string, ParamValue>;
+/**
+ * 媒体用例的样本与期望答案。夹具服务按原生格式认出这些样本后回答对应的词，所以夹具模式也能证明
+ * 插件把媒体块映射成了供应商认得的形状。
+ */
+export declare const MEDIA_SAMPLES: {
+    /** 64×64 纯红 PNG；期望回答含 "red"。 */
+    readonly image: {
+        readonly mediaType: "image/png";
+        readonly name: "red.png";
+        readonly answer: "red";
+    };
+    /** 一页 PDF，只印这个词。 */
+    readonly pdf: {
+        readonly mediaType: "application/pdf";
+        readonly name: "fruit.pdf";
+        readonly word: "PINEAPPLE";
+    };
+    /** 纯文本文档，内容是 {@link MEDIA_SAMPLES.text.content}。 */
+    readonly text: {
+        readonly mediaType: "text/plain";
+        readonly name: "note.txt";
+        readonly content: "The secret word is MANGO.\n";
+        readonly word: "MANGO";
+    };
+};
+/** model-provider 一致性套件（`@1` 与 `@2`）：每个供应商插件都应通过。 */
+export declare function describeModelProviderConformance(name: string, setup: () => Promise<ConformanceTarget>): void;
+export {};