@vostride/agent-qa-plugin-kit 0.0.1 → 0.3.1

package/dist/testing.js

@@ -0,0 +1,969 @@
+import { spawn } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
+import { dirname, isAbsolute, join, relative, sep } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { isDeepStrictEqual } from "node:util";
+import { z } from "zod";
+import { createServer } from "node:http";
+// src/protocol/messages.ts
+const OPTIONAL_PLUGIN_METHODS = [
+	"check",
+	"uploadApp",
+	"started",
+	"end"
+];
+const PLUGIN_SERVES = [
+	"playwright",
+	"cdp",
+	"appium"
+];
+const PLUGIN_RUN_FACTS = [
+	"testName",
+	"runId",
+	"failure"
+];
+const PLUGIN_FAILURE_KINDS = [
+	"unreachable",
+	"auth",
+	"capacity",
+	"timeout",
+	"version",
+	"refused",
+	"ended",
+	"lost"
+];
+const PLUGIN_ERROR_CODES = {
+	parse: -32700,
+	invalidRequest: -32600,
+	methodNotFound: -32601,
+	invalidParams: -32602,
+	plugin: -32e3,
+	cancelled: -32800
+};
+const CANCEL_METHOD = "$/cancel";
+const MAX_MESSAGE_BYTES = 1048576;
+const JsonObjectSchema = z.record(z.string(), z.unknown());
+const HandleSchema = z.json();
+const RequestIdSchema = z.union([z.string(), z.number()]);
+z.object({
+	protocolVersions: z.array(z.number().int()),
+	host: z.object({
+		pid: z.number().int(),
+		agentQaVersion: z.string(),
+		playwrightVersion: z.string(),
+		webdriverioVersion: z.string()
+	}),
+	entry: z.string(),
+	options: JsonObjectSchema,
+	runFacts: z.object({ runId: z.string().optional() }).optional()
+});
+const InitializeResultSchema = z.object({
+	protocolVersion: z.number().int(),
+	plugin: z.object({
+		name: z.string(),
+		version: z.string()
+	}),
+	serves: z.array(z.enum(PLUGIN_SERVES)),
+	methods: z.array(z.enum(OPTIONAL_PLUGIN_METHODS))
+});
+z.object({ remotes: z.array(z.object({
+	name: z.string(),
+	settings: JsonObjectSchema
+})) });
+z.object({
+	state: z.enum(["ok", "warn"]),
+	lines: z.array(z.string())
+});
+const WebSessionSchema = z.object({
+	platform: z.literal("web"),
+	engine: z.enum([
+		"chromium",
+		"firefox",
+		"webkit"
+	]),
+	context: z.enum(["new", "reuse"])
+});
+const DeviceSessionSchema = z.object({
+	platform: z.enum(["android", "ios"]),
+	capabilities: JsonObjectSchema
+});
+z.object({
+	remote: z.string(),
+	settings: JsonObjectSchema,
+	session: z.discriminatedUnion("platform", [WebSessionSchema, DeviceSessionSchema]),
+	run: z.object({ testName: z.string().optional() }).optional()
+});
+const AcquireResultSchema = z.object({
+	handle: HandleSchema,
+	protocol: z.enum(PLUGIN_SERVES),
+	url: z.string(),
+	headers: z.record(z.string(), z.string()).optional(),
+	capabilities: JsonObjectSchema.optional(),
+	mask: z.array(z.string()).optional(),
+	link: z.string().optional()
+});
+z.object({
+	remote: z.string(),
+	settings: JsonObjectSchema,
+	platform: z.enum(["android", "ios"]),
+	path: z.string(),
+	sha256: z.string(),
+	size: z.number().int().nonnegative()
+});
+z.object({ ref: z.string() });
+z.object({
+	handle: HandleSchema,
+	sessionId: z.string().optional()
+});
+z.object({
+	link: z.string().optional(),
+	appiumVersion: z.string().optional()
+});
+z.object({
+	handle: HandleSchema,
+	outcome: z.enum([
+		"passed",
+		"failed",
+		"error",
+		"cancelled"
+	]),
+	reason: z.string().optional()
+});
+z.object({ handle: HandleSchema });
+z.object({ id: RequestIdSchema });
+z.object({
+	id: RequestIdSchema,
+	message: z.string(),
+	queued: z.number().int().nonnegative().optional(),
+	percent: z.number().min(0).max(100).optional()
+});
+const PluginErrorDataSchema = z.object({
+	kind: z.enum(PLUGIN_FAILURE_KINDS),
+	path: z.string().optional(),
+	handle: HandleSchema.optional(),
+	supported: z.array(z.number().int()).optional(),
+	requested: z.array(z.number().int()).optional()
+});
+// src/protocol/manifest.ts
+const PLUGIN_MANIFEST_FIELD = "agent-qa";
+const PackagePathSchema = z.string().min(1);
+const RemotePluginManifestSchema = z.strictObject({
+	kind: z.literal("remote"),
+	runs: z.literal("process"),
+	protocol: z.number().int().positive(),
+	entry: PackagePathSchema,
+	serves: z.array(z.enum(PLUGIN_SERVES)).nonempty(),
+	methods: z.array(z.enum(OPTIONAL_PLUGIN_METHODS)),
+	options: PackagePathSchema,
+	settings: PackagePathSchema,
+	runFacts: z.array(z.enum(PLUGIN_RUN_FACTS)).optional(),
+	skill: PackagePathSchema.optional()
+});
+const ModelAuthManifestSchema = z.strictObject({
+	kind: z.literal("model-auth"),
+	runs: z.literal("in-process"),
+	entry: PackagePathSchema,
+	options: PackagePathSchema.optional()
+});
+const ModelProviderManifestSchema = z.strictObject({
+	kind: z.literal("model-provider"),
+	runs: z.literal("in-process"),
+	entry: PackagePathSchema,
+	options: PackagePathSchema.optional()
+});
+const PluginManifestSchema = z.discriminatedUnion("kind", [
+	RemotePluginManifestSchema,
+	ModelAuthManifestSchema,
+	ModelProviderManifestSchema
+]);
+// src/emit.ts
+const OPTIONS_SCHEMA_PATH = "./dist/options.schema.json";
+const SETTINGS_SCHEMA_PATH = "./dist/settings.schema.json";
+function pluginManifest(definition) {
+	const manifest = PluginManifestSchema.parse({
+		protocol: 1,
+		kind: "remote",
+		runs: "process",
+		entry: "./dist/main.js",
+		serves: [...definition.serves],
+		methods: [...definition.methods],
+		options: OPTIONS_SCHEMA_PATH,
+		settings: SETTINGS_SCHEMA_PATH,
+		...definition.runFacts === void 0 ? {} : { runFacts: [...definition.runFacts] },
+		...definition.skill === void 0 ? {} : { skill: definition.skill }
+	});
+	if (manifest.kind !== "remote") throw new Error("a plugin built with the kit is remote");
+	return manifest;
+}
+// src/package-root.ts
+function findPackageRoot(fromDir) {
+	let dir = fromDir;
+	for (;;) {
+		if (existsSync(join(dir, "package.json"))) return dir;
+		const parent = dirname(dir);
+		if (parent === dir) return void 0;
+		dir = parent;
+	}
+}
+function readPackageJson(packageRoot) {
+	const parsed = JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8"));
+	return typeof parsed === "object" && parsed !== null ? parsed : {};
+}
+// src/protocol/environment.ts
+const PROXY_NAMES = [
+	"HTTP_PROXY",
+	"HTTPS_PROXY",
+	"NO_PROXY",
+	"ALL_PROXY"
+];
+const PLUGIN_ENVIRONMENT_NAMES = [
+	"PATH",
+	"HOME",
+	"TMPDIR",
+	"TEMP",
+	"TMP",
+	"LANG",
+	"TZ",
+	...PROXY_NAMES,
+	...PROXY_NAMES.map((name) => name.toLowerCase()),
+	"NODE_EXTRA_CA_CERTS",
+	"SSL_CERT_FILE"
+];
+const PLUGIN_WINDOWS_ENVIRONMENT_NAMES = [
+	"SystemRoot",
+	"ComSpec",
+	"PATHEXT",
+	"USERPROFILE",
+	"APPDATA",
+	"LOCALAPPDATA"
+];
+function pluginEnvironment(env, platform) {
+	const windows = platform === "win32";
+	const names = new Set(PLUGIN_ENVIRONMENT_NAMES);
+	if (windows) for (const name of PLUGIN_WINDOWS_ENVIRONMENT_NAMES) names.add(name.toUpperCase());
+	const passes = (key) => {
+		if (key.startsWith("LC_")) return true;
+		if (names.has(key)) return true;
+		return windows && names.has(key.toUpperCase());
+	};
+	const out = {};
+	for (const [key, value] of Object.entries(env)) {
+		if (value === void 0 || key.toUpperCase() === "NODE_OPTIONS") continue;
+		if (passes(key)) out[key] = value;
+	}
+	return out;
+}
+// src/protocol/framing.ts
+function encodeMessage(message) {
+	return `${JSON.stringify(message)}\n`;
+}
+const NEWLINE = 10;
+function createLineDecoder({ maxLineBytes, onLine, onOverflow }) {
+	const encoder = new TextEncoder();
+	const decoder = new TextDecoder();
+	let parts = [];
+	let size = 0;
+	let discarding = false;
+	const append = (segment) => {
+		if (discarding || segment.length === 0) return;
+		if (size + segment.length > maxLineBytes) {
+			discarding = true;
+			parts = [];
+			size = 0;
+			onOverflow();
+			return;
+		}
+		parts.push(new Uint8Array(segment));
+		size += segment.length;
+	};
+	const finishLine = () => {
+		const wasDiscarding = discarding;
+		const lineParts = parts;
+		const lineSize = size;
+		parts = [];
+		size = 0;
+		discarding = false;
+		if (wasDiscarding || lineSize === 0) return;
+		const bytes = new Uint8Array(lineSize);
+		let offset = 0;
+		for (const part of lineParts) {
+			bytes.set(part, offset);
+			offset += part.length;
+		}
+		const line = decoder.decode(bytes).replace(/\r$/, "");
+		if (line.trim() !== "") onLine(line);
+	};
+	return {
+		push(chunk) {
+			const bytes = typeof chunk === "string" ? encoder.encode(chunk) : chunk;
+			let start = 0;
+			while (start <= bytes.length) {
+				const newline = bytes.indexOf(NEWLINE, start);
+				if (newline === -1) {
+					append(bytes.subarray(start));
+					return;
+				}
+				append(bytes.subarray(start, newline));
+				finishLine();
+				start = newline + 1;
+			}
+		},
+		end() {
+			finishLine();
+		}
+	};
+}
+// src/testing/conformance.ts
+const DEFAULT_CALL_TIMEOUT_MS = 1e4;
+const CANCEL_ANSWER_WINDOW_MS = 5e3;
+const EXIT_WINDOW_MS = 5e3;
+const STDIO_DRAIN_MS = 1e3;
+const DRAFT_2020_12 = /^https:\/\/json-schema\.org\/draft\/2020-12\/schema#?$/;
+const DEFAULT_SESSION = {
+	platform: "web",
+	engine: "chromium",
+	context: "new"
+};
+function isRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function errorText(error) {
+	return error instanceof Error ? error.message : String(error);
+}
+function kitVersion() {
+	const root = findPackageRoot(dirname(fileURLToPath(import.meta.url)));
+	const version = root === void 0 ? void 0 : readPackageJson(root).version;
+	return typeof version === "string" ? version : "unknown";
+}
+function hostIdentity(host) {
+	return {
+		pid: process.pid,
+		agentQaVersion: kitVersion(),
+		playwrightVersion: "unknown",
+		webdriverioVersion: "unknown",
+		...host
+	};
+}
+function readManifestField(packageDir) {
+	const parsed = JSON.parse(readFileSync(join(packageDir, "package.json"), "utf8"));
+	const field = isRecord(parsed) ? parsed[PLUGIN_MANIFEST_FIELD] : void 0;
+	return isRecord(field) ? field.plugin : void 0;
+}
+function insidePackage(packageRoot, path) {
+	const target = join(packageRoot, path);
+	if (!existsSync(target)) return void 0;
+	const real = realpathSync(target);
+	const fromRoot = relative(packageRoot, real);
+	if (fromRoot === ".." || fromRoot.startsWith(`..${sep}`) || isAbsolute(fromRoot)) return void 0;
+	return real;
+}
+function checkManifest(packageDir) {
+	const failed = (detail) => ({ check: {
+		name: "manifest",
+		ok: false,
+		detail
+	} });
+	let packageRoot;
+	let field;
+	try {
+		packageRoot = realpathSync(packageDir);
+		field = readManifestField(packageRoot);
+	} catch (error) {
+		return failed(`package.json could not be read: ${errorText(error)}`);
+	}
+	if (field === void 0) return failed(`package.json has no ${PLUGIN_MANIFEST_FIELD}.plugin`);
+	const parsed = PluginManifestSchema.safeParse(field);
+	if (!parsed.success) return failed(parsed.error.issues.map((issue) => `${[
+		PLUGIN_MANIFEST_FIELD,
+		"plugin",
+		...issue.path.map(String)
+	].join(".")}: ${issue.message}`).join("; "));
+	const manifest = parsed.data;
+	if (manifest.kind !== "remote") return failed(`a conformance run checks a remote plugin, and this one is ${manifest.kind}`);
+	const problems = [];
+	const resolved = (key, path) => {
+		const real = insidePackage(packageRoot, path);
+		if (real === void 0) problems.push(`${key} ${path} does not resolve inside the package`);
+		return real;
+	};
+	const entry = resolved("entry", manifest.entry);
+	for (const key of ["options", "settings"]) {
+		const file = resolved(key, manifest[key]);
+		if (file === void 0) continue;
+		try {
+			const schema = JSON.parse(readFileSync(file, "utf8"));
+			if (!isRecord(schema) || typeof schema.$schema !== "string") problems.push(`${key} ${manifest[key]} names no $schema`);
+			else if (!DRAFT_2020_12.test(schema.$schema)) problems.push(`${key} ${manifest[key]} is ${schema.$schema}, not JSON Schema draft 2020-12`);
+		} catch (error) {
+			problems.push(`${key} ${manifest[key]} is not JSON: ${errorText(error)}`);
+		}
+	}
+	if (manifest.skill !== void 0) resolved("skill", manifest.skill);
+	if (problems.length > 0) return failed(problems.join("; "));
+	return {
+		check: {
+			name: "manifest",
+			ok: true
+		},
+		manifest,
+		...entry ? { entry } : {}
+	};
+}
+function within(promise, ms, what) {
+	let timer;
+	const timeout = new Promise((_, reject) => {
+		timer = setTimeout(() => reject(/* @__PURE__ */ new Error(`${what} got no answer within ${ms} ms`)), ms);
+	});
+	return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
+}
+function startPlugin(entry, cwd) {
+	const child = spawn(process.execPath, [entry], {
+		cwd,
+		env: pluginEnvironment(process.env, process.platform),
+		stdio: [
+			"pipe",
+			"pipe",
+			"pipe"
+		]
+	});
+	const waiting = /* @__PURE__ */ new Map();
+	const problems = [];
+	let stderr = "";
+	let exitCode = null;
+	let exited = false;
+	child.stdin.on("error", () => {});
+	child.stderr.on("data", (chunk) => {
+		stderr += chunk.toString();
+	});
+	const decoder = createLineDecoder({
+		maxLineBytes: MAX_MESSAGE_BYTES,
+		onLine: (line) => {
+			let message;
+			try {
+				message = JSON.parse(line);
+			} catch {
+				problems.push(line);
+				return;
+			}
+			if (!isRecord(message) || message.jsonrpc !== "2.0") {
+				problems.push(line);
+				return;
+			}
+			const resolve = typeof message.id === "number" ? waiting.get(message.id) : void 0;
+			if (resolve === void 0 || typeof message.id !== "number") return;
+			waiting.delete(message.id);
+			resolve(message);
+		},
+		onOverflow: () => problems.push(`a line longer than ${MAX_MESSAGE_BYTES} bytes`)
+	});
+	child.stdout.on("data", (chunk) => decoder.push(chunk));
+	const exit = new Promise((resolve) => child.on("exit", (code) => {
+		exited = true;
+		exitCode = code;
+		resolve(code);
+	}));
+	const closed = new Promise((resolve) => child.on("close", () => resolve()));
+	let next = 0;
+	const plugin = {
+		send(method, params) {
+			next += 1;
+			const id = next;
+			const answered = new Promise((resolve, reject) => {
+				waiting.set(id, resolve);
+				exit.then((code) => {
+					const reason = stderr.trim().split("\n")[0] ?? "";
+					reject(/* @__PURE__ */ new Error(`the plugin exited with ${code} before answering ${method} ${reason}`));
+				});
+			});
+			answered.catch(() => {});
+			child.stdin.write(encodeMessage({
+				jsonrpc: "2.0",
+				id,
+				method,
+				params
+			}));
+			return {
+				id,
+				answered
+			};
+		},
+		request(method, params, timeoutMs) {
+			return within(plugin.send(method, params).answered, timeoutMs, method);
+		},
+		notify(method, params) {
+			child.stdin.write(encodeMessage({
+				jsonrpc: "2.0",
+				method,
+				params
+			}));
+		},
+		stdoutProblems() {
+			return [...problems];
+		},
+		closeStdin() {
+			child.stdin.end();
+		},
+		async exitedWithin(ms) {
+			if (exited) return {
+				exited: true,
+				code: exitCode
+			};
+			try {
+				return {
+					exited: true,
+					code: await within(exit, ms, "exit")
+				};
+			} catch {
+				return {
+					exited: false,
+					code: null
+				};
+			}
+		},
+		async drained(ms) {
+			await within(closed, ms, "stdout").catch(() => {});
+			decoder.end();
+		},
+		hasExited() {
+			return exited;
+		},
+		kill() {
+			if (!exited) child.kill("SIGKILL");
+		}
+	};
+	return plugin;
+}
+function describeError(answer) {
+	if (answer.error === void 0) return `answered ${JSON.stringify(answer.result)}`;
+	const data = PluginErrorDataSchema.safeParse(answer.error.data);
+	const kind = data.success ? ` (kind ${data.data.kind})` : "";
+	return `answered ${answer.error.code}: ${answer.error.message}${kind}`;
+}
+function errorHandle(answer) {
+	const data = answer.error?.data;
+	return isRecord(data) ? data.handle : void 0;
+}
+async function checkHandshake(plugin, manifest, input, timeoutMs) {
+	const name = "handshake";
+	let answer;
+	try {
+		answer = await plugin.request("initialize", {
+			protocolVersions: [1],
+			host: hostIdentity(input.host),
+			entry: "conformance",
+			options: input.options
+		}, timeoutMs);
+	} catch (error) {
+		return {
+			name,
+			ok: false,
+			detail: errorText(error)
+		};
+	}
+	if (answer.error !== void 0) {
+		const refusal = answer.error.code === PLUGIN_ERROR_CODES.plugin && typeof answer.error.message === "string" && answer.error.message !== "" && PluginErrorDataSchema.safeParse(answer.error.data).success;
+		if (input.quick === true && refusal) return {
+			name,
+			ok: true,
+			detail: `initialize refused: ${describeError(answer)}`
+		};
+		return {
+			name,
+			ok: false,
+			detail: `initialize ${describeError(answer)}`
+		};
+	}
+	const result = InitializeResultSchema.safeParse(answer.result);
+	if (!result.success) return {
+		name,
+		ok: false,
+		detail: `initialize answered a malformed result: ${result.error.message}`
+	};
+	const sorted = (values) => [...values].sort();
+	const problems = [];
+	if (result.data.protocolVersion !== 1) problems.push(`protocol ${result.data.protocolVersion}, not 1`);
+	if (!isDeepStrictEqual(sorted(result.data.serves), sorted(manifest.serves))) problems.push(`serves ${result.data.serves.join(", ")}, and the manifest says ${manifest.serves.join(", ")}`);
+	if (!isDeepStrictEqual(sorted(result.data.methods), sorted(manifest.methods))) problems.push(`methods ${result.data.methods.join(", ") || "none"}, and the manifest says ${manifest.methods.join(", ") || "none"}`);
+	return problems.length === 0 ? {
+		name,
+		ok: true
+	} : {
+		name,
+		ok: false,
+		detail: `initialize answered ${problems.join("; ")}`
+	};
+}
+async function checkUnknownMethod(plugin, timeoutMs) {
+	const name = "unknown-method";
+	try {
+		const answer = await plugin.request("agent-qa/none", {}, timeoutMs);
+		return answer.error?.code === PLUGIN_ERROR_CODES.methodNotFound ? {
+			name,
+			ok: true
+		} : {
+			name,
+			ok: false,
+			detail: `agent-qa/none ${describeError(answer)}, not ${PLUGIN_ERROR_CODES.methodNotFound}`
+		};
+	} catch (error) {
+		return {
+			name,
+			ok: false,
+			detail: errorText(error)
+		};
+	}
+}
+async function release(plugin, handle, timeoutMs) {
+	return plugin.request("release", { handle }, timeoutMs);
+}
+async function checkIdempotentRelease(plugin, acquireParams, timeoutMs) {
+	const name = "idempotent-release";
+	try {
+		const acquired = await plugin.request("acquire", acquireParams, timeoutMs);
+		if (acquired.error !== void 0) {
+			const handle = errorHandle(acquired);
+			if (handle !== void 0) await release(plugin, handle, timeoutMs).catch(() => {});
+			return {
+				name,
+				ok: false,
+				detail: `acquire ${describeError(acquired)}`
+			};
+		}
+		const result = AcquireResultSchema.safeParse(acquired.result);
+		if (!result.success) return {
+			name,
+			ok: false,
+			detail: `acquire answered a malformed result: ${result.error.message}`
+		};
+		for (const attempt of ["first", "second"]) {
+			const released = await release(plugin, result.data.handle, timeoutMs);
+			if (released.error !== void 0 || !isDeepStrictEqual(released.result, {})) return {
+				name,
+				ok: false,
+				detail: `the ${attempt} release ${describeError(released)}`
+			};
+		}
+		return {
+			name,
+			ok: true
+		};
+	} catch (error) {
+		return {
+			name,
+			ok: false,
+			detail: errorText(error)
+		};
+	}
+}
+async function checkAnsweredCancel(plugin, acquireParams, timeoutMs) {
+	const name = "answered-cancel";
+	try {
+		const acquire = plugin.send("acquire", acquireParams);
+		plugin.notify(CANCEL_METHOD, { id: acquire.id });
+		const answer = await within(acquire.answered, CANCEL_ANSWER_WINDOW_MS, "a cancelled acquire");
+		if (answer.error === void 0) {
+			const result = AcquireResultSchema.safeParse(answer.result);
+			if (result.success) await release(plugin, result.data.handle, timeoutMs).catch(() => {});
+			return result.success ? {
+				name,
+				ok: true
+			} : {
+				name,
+				ok: false,
+				detail: "a cancelled acquire answered a malformed result"
+			};
+		}
+		const handle = errorHandle(answer);
+		if (handle !== void 0) await release(plugin, handle, timeoutMs).catch(() => {});
+		return answer.error.code === PLUGIN_ERROR_CODES.cancelled ? {
+			name,
+			ok: true
+		} : {
+			name,
+			ok: false,
+			detail: `a cancelled acquire ${describeError(answer)}, not a result or ${PLUGIN_ERROR_CODES.cancelled}`
+		};
+	} catch (error) {
+		return {
+			name,
+			ok: false,
+			detail: errorText(error)
+		};
+	}
+}
+async function checkExitOnStdinClose(plugin) {
+	const name = "exit-on-stdin-close";
+	if (plugin.hasExited()) return {
+		name,
+		ok: false,
+		detail: "the plugin exited before its stdin closed"
+	};
+	plugin.closeStdin();
+	const { exited } = await plugin.exitedWithin(EXIT_WINDOW_MS);
+	return exited ? {
+		name,
+		ok: true
+	} : {
+		name,
+		ok: false,
+		detail: `the plugin was still running ${EXIT_WINDOW_MS} ms after stdin closed`
+	};
+}
+async function runConformance(input) {
+	const timeoutMs = input.timeoutMs ?? DEFAULT_CALL_TIMEOUT_MS;
+	const checked = checkManifest(input.packageDir);
+	const checks = [checked.check];
+	if (checked.manifest === void 0 || checked.entry === void 0) return {
+		ok: false,
+		checks
+	};
+	const plugin = startPlugin(checked.entry, input.packageDir);
+	try {
+		checks.push(await checkHandshake(plugin, checked.manifest, input, timeoutMs));
+		checks.push(await checkUnknownMethod(plugin, timeoutMs));
+		if (input.quick !== true) {
+			const acquireParams = {
+				remote: "conformance",
+				settings: input.settings ?? {},
+				session: input.session ?? DEFAULT_SESSION
+			};
+			checks.push(await checkIdempotentRelease(plugin, acquireParams, timeoutMs));
+			checks.push(await checkAnsweredCancel(plugin, acquireParams, timeoutMs));
+		}
+		const exitCheck = await checkExitOnStdinClose(plugin);
+		if (exitCheck.ok) await plugin.drained(STDIO_DRAIN_MS);
+		const unclean = plugin.stdoutProblems();
+		checks.push(unclean.length === 0 ? {
+			name: "clean-stdout",
+			ok: true
+		} : {
+			name: "clean-stdout",
+			ok: false,
+			detail: `stdout carried ${unclean.length} line(s) that are not JSON-RPC, the first: ${unclean[0]?.slice(0, 200)}`
+		});
+		checks.push(exitCheck);
+	} finally {
+		plugin.kill();
+	}
+	return {
+		ok: checks.every((check) => check.ok),
+		checks
+	};
+}
+function expectManifestCurrent(packageDir, definition) {
+	const expected = { ...pluginManifest(definition) };
+	const field = readManifestField(packageDir);
+	const actual = isRecord(field) ? field : {};
+	const differing = [.../* @__PURE__ */ new Set([...Object.keys(expected), ...Object.keys(actual)])].sort().filter((key) => !isDeepStrictEqual(actual[key], expected[key]));
+	if (differing.length > 0) throw new Error(`package.json's ${PLUGIN_MANIFEST_FIELD}.plugin differs from the definition in ${differing.join(", ")}; run the build's emit step`);
+}
+// src/testing/loopback-fake.ts
+function splitPath(path) {
+	return path.split("/").filter((segment) => segment !== "");
+}
+function compile(key, route) {
+	const match = /^([A-Z]+) (\/\S*)$/.exec(key);
+	if (match?.[1] === void 0 || match[2] === void 0) throw new Error(`a loopback route is 'METHOD /path', not '${key}'`);
+	if (Array.isArray(route) && route.length === 0) throw new Error(`the loopback route '${key}' has no replies`);
+	return {
+		key,
+		method: match[1],
+		segments: splitPath(match[2]),
+		route,
+		calls: []
+	};
+}
+function decodeSegment(segment) {
+	try {
+		return decodeURIComponent(segment);
+	} catch {
+		return segment;
+	}
+}
+function matchParams(route, method, path) {
+	if (route.method !== method) return void 0;
+	const segments = splitPath(path);
+	if (segments.length !== route.segments.length) return void 0;
+	const params = {};
+	for (const [index, expected] of route.segments.entries()) {
+		const actual = segments[index] ?? "";
+		if (expected.startsWith(":")) params[expected.slice(1)] = decodeSegment(actual);
+		else if (expected !== actual) return void 0;
+	}
+	return params;
+}
+function lowerCasedHeaders(request) {
+	const headers = {};
+	for (const [name, value] of Object.entries(request.headers)) {
+		if (value === void 0) continue;
+		headers[name.toLowerCase()] = Array.isArray(value) ? value.join(", ") : value;
+	}
+	return headers;
+}
+function parseBody(text) {
+	if (text.length === 0) return void 0;
+	try {
+		return JSON.parse(text);
+	} catch {
+		return text;
+	}
+}
+function send(response, reply) {
+	const [contentType, text] = reply.json !== void 0 ? ["application/json; charset=utf-8", JSON.stringify(reply.json)] : ["text/plain; charset=utf-8", reply.text ?? ""];
+	response.writeHead(reply.status, {
+		"content-type": contentType,
+		"content-length": Buffer.byteLength(text),
+		...reply.headers
+	});
+	response.end(text);
+}
+async function startLoopbackFake(input) {
+	const routes = Object.entries(input.routes).map(([key, route]) => compile(key, route));
+	const requests = [];
+	const sockets = /* @__PURE__ */ new Set();
+	let routeFailure;
+	const answer = (record, response) => {
+		for (const route of routes) {
+			const params = matchParams(route, record.method, record.path);
+			if (params === void 0) continue;
+			const request = {
+				...record,
+				params
+			};
+			requests.push(request);
+			route.calls.push(request);
+			const reply = typeof route.route === "function" ? route.route(request) : route.route[Math.min(route.calls.length, route.route.length) - 1] ?? "hold";
+			if (reply !== "hold") send(response, reply);
+			return;
+		}
+		requests.push(record);
+		send(response, {
+			status: 404,
+			json: { message: "no route" }
+		});
+	};
+	const server = createServer((request, response) => {
+		const chunks = [];
+		request.on("data", (chunk) => chunks.push(chunk));
+		request.on("end", () => {
+			const url = new URL(request.url ?? "/", "http://127.0.0.1");
+			const record = {
+				method: request.method ?? "GET",
+				path: url.pathname,
+				query: Object.fromEntries(url.searchParams),
+				params: {},
+				headers: lowerCasedHeaders(request),
+				body: parseBody(Buffer.concat(chunks).toString("utf8"))
+			};
+			try {
+				answer(record, response);
+			} catch (error) {
+				routeFailure ??= error;
+				if (!response.headersSent) send(response, {
+					status: 500,
+					json: { message: `the route threw: ${String(error)}` }
+				});
+			}
+		});
+	});
+	server.on("connection", (socket) => {
+		sockets.add(socket);
+		socket.on("close", () => sockets.delete(socket));
+	});
+	await new Promise((ready, failed) => {
+		server.once("error", failed);
+		server.listen(0, "127.0.0.1", () => ready());
+	});
+	const { port } = server.address();
+	const origin = `http://127.0.0.1:${String(port)}`;
+	let closing;
+	return {
+		url: origin,
+		origin,
+		requests,
+		routeCalls(key) {
+			return [...routes.find((route) => route.key === key)?.calls ?? []];
+		},
+		close() {
+			closing ??= new Promise((closed, failed) => {
+				for (const socket of sockets) socket.destroy();
+				server.close(() => routeFailure === void 0 ? closed() : failed(routeFailure));
+			});
+			return closing;
+		}
+	};
+}
+// src/testing/scripted-plugin.ts
+const OPEN_OBJECT_SCHEMA = {
+	$schema: "https://json-schema.org/draft/2020-12/schema",
+	type: "object"
+};
+const SKILL = `---
+name: scripted
+description: A scripted agent-qa plugin that answers each call as its options say.
+---
+
+# Scripted plugin
+
+Each call follows \`options.script[method]\`: \`ok\`, \`hold\`, \`ignoreCancel\`, \`crash\`,
+\`uncaught\` (a Node deprecation warning, then an exception nothing catches), \`noise\`,
+\`consoleLog\`, \`{ delayMs }\`, \`{ error }\` or \`{ errorAfterCreate }\`. A list of
+steps gives each call the next one, and the last repeats. \`options.stderr[method]\` is written
+to stderr before the step. Acquire answers \`options.answer\`, or the JSON in the file
+\`options.answerFile\` names.
+With \`options.recordProcess\`, the initialize ledger line also records the process, its
+environment's names, its options, the host and the run facts. With \`options.recordParams\`, the
+acquire line also records the session and run, the uploadApp line the settings, platform, path,
+sha256 and size, the started line the session id, and the end line the outcome and reason.
+Acquire and uploadApp send \`options.progress\`, or \`options.progressByRemote[remote]\`, as
+progress before their step, and with \`options.progressEveryMs\` send them again, one each
+interval, until they answer. uploadApp answers \`options.upload\`.
+`;
+function scriptedMainFile() {
+	const kitRoot = findPackageRoot(dirname(fileURLToPath(import.meta.url)));
+	const file = kitRoot === void 0 ? void 0 : join(kitRoot, "dist", "scripted-plugin.js");
+	if (file === void 0 || !existsSync(file)) throw new Error("the scripted plugin needs the kit built: run pnpm build in the plugin kit");
+	return file;
+}
+function writeScriptedPlugin(dir, input = {}) {
+	const manifest = PluginManifestSchema.parse({
+		kind: "remote",
+		runs: "process",
+		protocol: 1,
+		entry: "./main.mjs",
+		serves: [...input.serves ?? PLUGIN_SERVES],
+		methods: [...input.methods ?? OPTIONAL_PLUGIN_METHODS],
+		options: "./options.schema.json",
+		settings: "./settings.schema.json",
+		...input.runFacts === void 0 ? {} : { runFacts: [...input.runFacts] },
+		skill: "./skill"
+	});
+	const packageJson = {
+		name: input.packageName ?? "@acme/agent-qa-plugin-scripted",
+		version: input.version ?? "1.0.0",
+		private: true,
+		type: "module",
+		keywords: ["agent-qa-plugin"],
+		exports: { "./package.json": "./package.json" },
+		[PLUGIN_MANIFEST_FIELD]: { plugin: manifest }
+	};
+	const mainUrl = pathToFileURL(scriptedMainFile()).href;
+	mkdirSync(join(dir, "skill"), { recursive: true });
+	writeFileSync(join(dir, "package.json"), `${JSON.stringify(packageJson, null, 2)}\n`);
+	const openSchema = `${JSON.stringify(OPEN_OBJECT_SCHEMA, null, 2)}\n`;
+	writeFileSync(join(dir, "options.schema.json"), openSchema);
+	writeFileSync(join(dir, "settings.schema.json"), openSchema);
+	writeFileSync(join(dir, "skill", "SKILL.md"), SKILL);
+	const timing = {
+		...input.timing?.cancelAnswerGraceMs === void 0 ? {} : { cancelAnswerGraceMs: input.timing.cancelAnswerGraceMs },
+		...input.timing?.hostPidPollMs === void 0 ? {} : { hostPidPollMs: input.timing.hostPidPollMs }
+	};
+	writeFileSync(join(dir, "main.mjs"), `import { startScriptedPlugin } from ${JSON.stringify(mainUrl)}\n\nstartScriptedPlugin(${JSON.stringify(timing)})\n`);
+	return {
+		dir,
+		entry: join(dir, "main.mjs")
+	};
+}
+export { expectManifestCurrent, runConformance, startLoopbackFake, writeScriptedPlugin };
+
+//# sourceMappingURL=testing.js.map