@executablemd/test-agent 0.0.0-bootstrap.0 → 0.5.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Taras Mankovski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/esm/mod.js

@@ -0,0 +1,21 @@
+/**
+ * @module
+ * Build reliable ACP integration tests with deterministic,
+ * document-driven agent behavior. In place of a probabilistic coding
+ * agent, the test agent answers ACP prompts by advancing through a
+ * Markdown behavior document, so an integration can be tested against
+ * scripted, repeatable responses (specs/test-agent-spec.md).
+ *
+ * This entry point exposes the `<TestAgent>` vocabulary, the
+ * behavior-document engine, and the controller that registers scenarios
+ * and serves them to workers over the wire protocol.
+ */
+export { parseTemplate, matchPrompt } from "./src/template.js";
+export { createLineSplitter, encodeMessage, formatRoute, parseControllerMessage, parseRoute, parseWorkerMessage, PROBE_INSTANCE, } from "./src/protocol.js";
+export { useTestAgentController } from "./src/controller.js";
+export { installTestAgentVocabulary } from "./src/vocabulary.js";
+export { createMemorySessionStore, useTestAgentAcpx } from "./src/state.js";
+export { runTestAgentWorker } from "./src/worker/run.js";
+export { installWhenPromptVocabulary } from "./src/worker/when-prompt.js";
+export { installWorkerProfile } from "./src/worker/profile.js";
+export { collectTurn, createTurnBridge } from "./src/worker/bridge.js";

package/esm/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "module"
+}

package/esm/src/controller.js

@@ -0,0 +1,295 @@
+/**
+ * The test-agent controller (specs/test-agent-spec.md §Controller and
+ * worker): a localhost line-protocol server owned by the `<TestAgent>`
+ * scope. It serves behavior documents, Markdown dependencies (reads
+ * restricted to Markdown files whose canonical path stays inside the
+ * scenario root), and behavior journals to workers, and records journal
+ * appends and turn-failure diagnostics per scenario instance.
+ *
+ * Each instance admits one worker connection at a time. Unregistering an
+ * instance — or tearing the controller down — revokes and awaits its active
+ * connection before discarding state, so a revoked worker can no longer
+ * append, report failures, or read.
+ */
+import { each, ensure, race, resource, until, withResolvers } from "effection";
+import { randomUUID } from "node:crypto";
+import { isAbsolute, relative, resolve, sep } from "node:path";
+// @effectionx/fs has no realpath, so canonical symlink resolution uses the
+// node:fs/promises primitive directly.
+import { realpath } from "node:fs/promises";
+import { readTextFile, stat } from "@executablemd/runtime";
+import { encodeMessage, formatRoute, parseWorkerMessage, PROBE_INSTANCE } from "./protocol.js";
+import { useLineServer } from "./net.js";
+function send(connection, message) {
+    connection.send(encodeMessage(message));
+}
+/**
+ * Map a worker's virtual path onto the scenario directory. Workers see a
+ * virtual root at the scenario directory; anything resolving outside it
+ * lexically is answered as missing rather than surfaced as an error, so
+ * component fallback continues normally.
+ */
+function scenarioPath(instance, path) {
+    const virtual = isAbsolute(path) ? relative("/", path) : path;
+    const real = resolve(instance.scenarioDir, virtual);
+    if (real !== instance.scenarioDir && !real.startsWith(instance.scenarioDir + sep)) {
+        return undefined;
+    }
+    return real;
+}
+export function useTestAgentController() {
+    return resource(function* (provide) {
+        const token = randomUUID();
+        const instances = new Map();
+        const active = new Map();
+        const canonicalRoots = new Map();
+        // The canonical scenario root, resolving symlinks, memoized per
+        // instance. A dependency read/stat is served only when its canonical
+        // path stays inside this root.
+        function* canonicalRoot(instance) {
+            const cached = canonicalRoots.get(instance.id);
+            if (cached !== undefined) {
+                return cached;
+            }
+            let root;
+            try {
+                root = yield* until(realpath(instance.scenarioDir));
+            }
+            catch {
+                root = instance.scenarioDir;
+            }
+            canonicalRoots.set(instance.id, root);
+            return root;
+        }
+        function* resolveContained(instance, path) {
+            const real = scenarioPath(instance, path);
+            if (real === undefined) {
+                return undefined;
+            }
+            const root = yield* canonicalRoot(instance);
+            let canonical;
+            try {
+                canonical = yield* until(realpath(real));
+            }
+            catch {
+                // A path that does not exist has no symlink to escape through, so
+                // the lexical containment check above is sufficient.
+                return real;
+            }
+            if (canonical !== root && !canonical.startsWith(root + sep)) {
+                return undefined;
+            }
+            return canonical;
+        }
+        // The controller never spawns or touches a raw socket: it hands this
+        // per-connection handler to the server adapter, which owns the accept
+        // loop and the socket lifecycle.
+        function* handleConnection(connection) {
+            const revoke = withResolvers();
+            // Revocation cancels the serve loop; connection.closed covers an
+            // abrupt worker disconnect that never ends the line stream. Either
+            // way the server task's teardown then destroys the socket.
+            yield* race([revoke.operation, connection.closed, serve(connection, revoke)]);
+        }
+        function* serve(connection, revoke) {
+            let attached;
+            for (const line of yield* each(connection.lines)) {
+                const parsed = parseWorkerMessage(line);
+                if (!parsed.ok) {
+                    send(connection, { t: "error", message: parsed.error });
+                    connection.end();
+                    return;
+                }
+                const message = parsed.message;
+                if (attached === undefined) {
+                    if (message.t !== "attach" || message.token !== token) {
+                        send(connection, { t: "error", message: "unauthorized or out-of-order attach" });
+                        connection.end();
+                        return;
+                    }
+                    if (message.instance === PROBE_INSTANCE) {
+                        attached = "probe";
+                        send(connection, { t: "config", mode: "probe" });
+                    }
+                    else {
+                        const instance = instances.get(message.instance);
+                        if (!instance) {
+                            send(connection, { t: "error", message: `unknown instance "${message.instance}"` });
+                            connection.end();
+                            return;
+                        }
+                        // One worker per instance: a second concurrent attach is
+                        // refused so two workers never mutate the same journal.
+                        if (active.has(instance.id)) {
+                            send(connection, {
+                                t: "error",
+                                message: `instance "${instance.id}" already has an active connection`,
+                            });
+                            connection.end();
+                            return;
+                        }
+                        const ended = withResolvers();
+                        const registration = {
+                            revoke: revoke.resolve,
+                            closed: ended.operation,
+                        };
+                        active.set(instance.id, registration);
+                        yield* ensure(() => {
+                            if (active.get(instance.id) === registration) {
+                                active.delete(instance.id);
+                            }
+                            ended.resolve();
+                        });
+                        attached = instance;
+                        send(connection, {
+                            t: "config",
+                            mode: "scenario",
+                            doc: instance.doc,
+                            journal: instance.journal,
+                        });
+                    }
+                    yield* each.next();
+                    continue;
+                }
+                yield* handleMessage(connection, attached, message);
+                yield* each.next();
+            }
+        }
+        function* handleMessage(connection, attached, message) {
+            if (attached === "probe") {
+                send(connection, { t: "error", message: `probe workers may not send "${message.t}"` });
+                connection.end();
+                return;
+            }
+            // A worker whose instance was unregistered mid-connection is cut off
+            // here even before its socket finishes closing: nothing it sends
+            // reaches the discarded journal, failure, or filesystem.
+            if (instances.get(attached.id) !== attached) {
+                send(connection, { t: "error", message: "instance is no longer registered" });
+                connection.end();
+                return;
+            }
+            switch (message.t) {
+                case "journal": {
+                    if (message.seq !== attached.journal.length) {
+                        send(connection, {
+                            t: "error",
+                            message: `journal out of order: expected seq ${attached.journal.length}, got ${message.seq}`,
+                        });
+                        connection.end();
+                        return;
+                    }
+                    attached.journal.push(message.event);
+                    send(connection, { t: "ack", seq: message.seq });
+                    return;
+                }
+                case "read": {
+                    // Reads serve Markdown dependencies only. A .ts candidate is
+                    // never read — its existence is surfaced through stat so the
+                    // worker can emit the unsupported-TypeScript diagnostic.
+                    if (!message.path.endsWith(".md")) {
+                        send(connection, { t: "read", path: message.path, missing: true });
+                        return;
+                    }
+                    const real = yield* resolveContained(attached, message.path);
+                    if (real === undefined) {
+                        send(connection, { t: "read", path: message.path, missing: true });
+                        return;
+                    }
+                    const existing = yield* stat(real);
+                    if (!existing.exists || !existing.isFile) {
+                        send(connection, { t: "read", path: message.path, missing: true });
+                        return;
+                    }
+                    const source = yield* readTextFile(real);
+                    send(connection, { t: "read", path: message.path, source, missing: false });
+                    return;
+                }
+                case "stat": {
+                    const real = yield* resolveContained(attached, message.path);
+                    if (real === undefined) {
+                        send(connection, { t: "stat", path: message.path, exists: false, isFile: false });
+                        return;
+                    }
+                    const existing = yield* stat(real);
+                    send(connection, {
+                        t: "stat",
+                        path: message.path,
+                        exists: existing.exists,
+                        isFile: existing.isFile,
+                    });
+                    return;
+                }
+                case "turn-failure": {
+                    const failure = { kind: message.kind, actual: message.actual };
+                    if (message.expected !== undefined) {
+                        failure.expected = message.expected;
+                    }
+                    // Record before acknowledging: the worker awaits this ack before
+                    // surfacing the ACP error, so the diagnostic is already observable
+                    // once that error is seen.
+                    attached.failure = failure;
+                    send(connection, { t: "recorded" });
+                    return;
+                }
+                case "fatal": {
+                    attached.fatal = message.message;
+                    send(connection, { t: "recorded" });
+                    return;
+                }
+                case "attach": {
+                    send(connection, { t: "error", message: "duplicate attach" });
+                    connection.end();
+                    return;
+                }
+            }
+        }
+        const server = yield* useLineServer("127.0.0.1", handleConnection);
+        const port = server.port;
+        // A scenario instance is a resource: setup adds it to the routing index;
+        // its finalizer removes it from the index, revokes and awaits its active
+        // worker, and always clears its state. The index maps are lookups only —
+        // membership owns no lifecycle, so ending an instance's scope is the only
+        // way it is torn down.
+        function useInstance(config) {
+            return resource(function* (provide) {
+                const id = randomUUID();
+                const instance = {
+                    id,
+                    route: formatRoute({ host: "127.0.0.1", port, token, instance: id }),
+                    scenarioDir: resolve(config.scenarioDir),
+                    doc: config.doc,
+                    journal: [],
+                };
+                instances.set(id, instance);
+                yield* ensure(function* () {
+                    // Remove from the routing index first so in-flight messages are
+                    // rejected and no new worker can attach.
+                    instances.delete(id);
+                    try {
+                        const registration = active.get(id);
+                        if (registration) {
+                            registration.revoke();
+                            yield* registration.closed;
+                        }
+                    }
+                    finally {
+                        // Always clear state — even if worker revocation failed.
+                        instance.journal.length = 0;
+                        instance.failure = undefined;
+                        instance.fatal = undefined;
+                        canonicalRoots.delete(id);
+                    }
+                });
+                yield* provide(instance);
+            });
+        }
+        yield* provide({
+            probeRoute: formatRoute({ host: "127.0.0.1", port, token, instance: PROBE_INSTANCE }),
+            useInstance,
+            instance(id) {
+                return instances.get(id);
+            },
+        });
+    });
+}

package/esm/src/net.js

@@ -0,0 +1,140 @@
+/**
+ * The Node↔Effection boundary for line-framed TCP (specs/test-agent-spec.md
+ * §Controller and worker). A server that owns its listener and dispatches
+ * connections, a socket whose inbound lines are an Effection stream, and a
+ * client that shares the same socket adapter — so callers work with
+ * operations and streams, never raw `.on`/`.once`/Promise plumbing.
+ */
+import { createChannel, each, ensure, race, resource, spawn, withResolvers } from "effection";
+import { fromReadable, on } from "@effectionx/node";
+import { lines } from "@effectionx/stream-helpers";
+import { connect, createServer } from "node:net";
+/** Adapt a connected socket: inbound bytes become a line stream; the socket
+ * is destroyed on teardown. */
+export function useLineSocket(socket) {
+    return resource(function* (provide) {
+        const closed = withResolvers();
+        socket.once("close", () => closed.resolve());
+        yield* ensure(() => {
+            socket.destroy();
+            closed.resolve(); // settle on cancellation even if no 'close' follows
+        });
+        yield* provide({
+            lines: lines()(fromReadable(socket)),
+            send(line) {
+                socket.write(line);
+            },
+            end() {
+                socket.end();
+            },
+            closed: closed.operation,
+        });
+    });
+}
+/**
+ * A localhost line-protocol server as a resource. It listens, then accepts
+ * each connection, adapts it with {@link useLineSocket}, and runs
+ * `onConnection` for it in its own task — so a caller never spawns or
+ * touches a raw socket. The per-connection task is what lets simultaneous
+ * connections run concurrently (`each` itself is sequential).
+ */
+export function useLineServer(host, onConnection) {
+    return resource(function* (provide) {
+        const server = createServer();
+        let acceptor;
+        // Installed before listen so a setup failure is torn down too. The
+        // acceptor's spawned per-connection handlers are its children, so halting
+        // it awaits their full teardown — each socket destroyed by useLineSocket's
+        // ensure — and nothing holds the listener open when we close it.
+        yield* ensure(function* () {
+            if (acceptor) {
+                yield* acceptor.halt();
+            }
+            // Attach the close listener before close() (no missed-event race), and
+            // only when the server actually came up, so a never-started or
+            // already-closed server leaves no pending operation.
+            if (server.listening) {
+                const closed = withResolvers();
+                server.once("close", () => closed.resolve());
+                server.close();
+                yield* closed.operation;
+            }
+        });
+        // Attach the readiness listeners before listen, so the event is never
+        // missed by a later subscription.
+        const listening = withResolvers();
+        server.once("listening", () => listening.resolve());
+        server.once("error", (error) => {
+            listening.reject(error instanceof Error ? error : new Error(String(error)));
+        });
+        server.listen(0, host);
+        yield* listening.operation;
+        const address = server.address();
+        if (!address || typeof address !== "object") {
+            throw new Error("useLineServer: unexpected server address");
+        }
+        const port = address.port;
+        acceptor = yield* spawn(function* () {
+            for (const [socket] of yield* each(on(server, "connection"))) {
+                yield* spawn(function* () {
+                    const connection = yield* useLineSocket(socket);
+                    yield* onConnection(connection);
+                });
+                yield* each.next();
+            }
+        });
+        yield* provide({ port });
+    });
+}
+/**
+ * Connect to a line-protocol server. Shares {@link useLineSocket}, adds the
+ * connect handshake and a parsed inbound queue behind `next()` — all as
+ * operations. `parse` returning `undefined` drops an unparseable line.
+ */
+export function useLineClient(host, port, parse) {
+    return resource(function* (provide) {
+        const socket = connect(port, host);
+        // Attach connect/error listeners before yielding, so neither is missed.
+        const connected = withResolvers();
+        socket.once("connect", () => connected.resolve());
+        socket.once("error", (error) => {
+            connected.reject(error instanceof Error ? error : new Error(String(error)));
+        });
+        const connection = yield* useLineSocket(socket);
+        yield* connected.operation;
+        const inbound = createChannel();
+        const subscription = yield* inbound; // subscribe before the pump: no loss
+        yield* spawn(function* () {
+            // Close inbound on every exit — EOF, abrupt socket close, or
+            // cancellation — so a pending next() always settles.
+            yield* ensure(function* () {
+                yield* inbound.close();
+            });
+            yield* race([
+                connection.closed,
+                (function* () {
+                    for (const line of yield* each(connection.lines)) {
+                        const message = parse(line);
+                        if (message !== undefined) {
+                            yield* inbound.send(message);
+                        }
+                        yield* each.next();
+                    }
+                })(),
+            ]);
+        });
+        yield* provide({
+            send(line) {
+                connection.send(line);
+            },
+            closed: connection.closed,
+            *next() {
+                const result = yield* subscription.next();
+                if (result.done) {
+                    throw new Error("connection closed before a reply");
+                }
+                return result.value;
+            },
+        });
+    });
+}

package/esm/src/protocol.js

@@ -0,0 +1,144 @@
+/**
+ * Controller ↔ worker wire protocol (specs/test-agent-spec.md
+ * §Controller and worker): newline-delimited JSON over localhost TCP.
+ * Every inbound line is parsed into a validated type — malformed,
+ * unknown, and directionally invalid messages are rejected, never cast.
+ */
+import { z } from "zod";
+const json = z.lazy(() => z.union([
+    z.null(),
+    z.boolean(),
+    z.number(),
+    z.string(),
+    z.array(json),
+    z.record(z.string(), json),
+]));
+const durableResult = z.union([
+    z.object({ status: z.literal("ok"), value: json.optional() }),
+    z.object({
+        status: z.literal("err"),
+        error: z.object({
+            message: z.string(),
+            name: z.string().optional(),
+            stack: z.string().optional(),
+        }),
+    }),
+    z.object({ status: z.literal("cancelled") }),
+]);
+const durableEvent = z.union([
+    z
+        .object({
+        type: z.literal("yield"),
+        coroutineId: z.string(),
+        description: z.object({ type: z.string(), name: z.string() }).catchall(json),
+        result: durableResult,
+    })
+        .catchall(json),
+    z
+        .object({
+        type: z.literal("close"),
+        coroutineId: z.string(),
+        result: durableResult,
+    })
+        .catchall(json),
+]);
+const workerMessage = z.discriminatedUnion("t", [
+    z.object({ t: z.literal("attach"), token: z.string(), instance: z.string() }),
+    z.object({ t: z.literal("journal"), seq: z.number().int().nonnegative(), event: durableEvent }),
+    z.object({ t: z.literal("read"), path: z.string() }),
+    z.object({ t: z.literal("stat"), path: z.string() }),
+    z.object({
+        t: z.literal("turn-failure"),
+        kind: z.enum(["mismatch", "exhausted", "config"]),
+        expected: z.string().optional(),
+        actual: z.string(),
+    }),
+    z.object({ t: z.literal("fatal"), message: z.string() }),
+]);
+const configMessage = z.discriminatedUnion("mode", [
+    z.object({ t: z.literal("config"), mode: z.literal("probe") }),
+    z.object({
+        t: z.literal("config"),
+        mode: z.literal("scenario"),
+        doc: z.object({ path: z.string(), source: z.string() }),
+        journal: z.array(durableEvent),
+    }),
+]);
+const controllerMessage = z.union([
+    configMessage,
+    z.object({ t: z.literal("ack"), seq: z.number().int().nonnegative() }),
+    z.object({ t: z.literal("recorded") }),
+    z.object({
+        t: z.literal("read"),
+        path: z.string(),
+        source: z.string().optional(),
+        missing: z.boolean(),
+    }),
+    z.object({
+        t: z.literal("stat"),
+        path: z.string(),
+        exists: z.boolean(),
+        isFile: z.boolean(),
+    }),
+    z.object({ t: z.literal("error"), message: z.string() }),
+]);
+export function encodeMessage(message) {
+    return JSON.stringify(message) + "\n";
+}
+function parseLine(schema, line, direction) {
+    let value;
+    try {
+        value = JSON.parse(line);
+    }
+    catch (error) {
+        return {
+            ok: false,
+            error: `malformed ${direction} message (invalid JSON): ${error instanceof Error ? error.message : String(error)}`,
+        };
+    }
+    const parsed = schema.safeParse(value);
+    if (!parsed.success) {
+        return { ok: false, error: `invalid ${direction} message: ${parsed.error.message}` };
+    }
+    return { ok: true, message: parsed.data };
+}
+export function parseWorkerMessage(line) {
+    return parseLine(workerMessage, line, "worker");
+}
+export function parseControllerMessage(line) {
+    return parseLine(controllerMessage, line, "controller");
+}
+/** Incremental newline splitter: feed chunks, receive complete lines. */
+export function createLineSplitter() {
+    let buffered = "";
+    return {
+        feed(chunk) {
+            buffered += chunk;
+            const lines = buffered.split("\n");
+            buffered = lines.pop() ?? "";
+            return lines.filter((line) => line.trim().length > 0);
+        },
+    };
+}
+/**
+ * Routes are opaque to ACPX and workers alike; this is the one place
+ * that knows the encoding: host:port/token/instance.
+ */
+export function formatRoute(route) {
+    return `${route.host}:${route.port}/${route.token}/${route.instance}`;
+}
+export function parseRoute(value) {
+    const match = /^([^:/]+):(\d+)\/([^/]+)\/(.+)$/.exec(value);
+    if (!match) {
+        return { ok: false, error: `malformed controller route: ${value}` };
+    }
+    const port = Number.parseInt(match[2], 10);
+    if (!Number.isInteger(port) || port <= 0 || port > 65_535) {
+        return { ok: false, error: `malformed controller route port: ${value}` };
+    }
+    return {
+        ok: true,
+        message: { host: match[1], port, token: match[3], instance: match[4] },
+    };
+}
+export const PROBE_INSTANCE = "probe";

package/esm/src/state.js

@@ -0,0 +1,57 @@
+/**
+ * Per-boundary ACPX state for `<TestAgent>` (specs/test-agent-spec.md
+ * §Scenario instances): the production provider state composed with an
+ * in-memory session store and a dynamic registry whose resolve() embeds
+ * the pending instance route into the worker command. Routing flows
+ * through the provider's `sessionRouting` seam: the global route slot is
+ * held only across the provider's registry-dependent work (preparation
+ * and ensure/session validation + turn start), released while the
+ * provider waits on the per-session queue and during turn consumption.
+ */
+import { useAcpxProviderState, useSerialQueues } from "@executablemd/acp";
+export function createMemorySessionStore() {
+    const records = new Map();
+    return {
+        load(sessionId) {
+            return Promise.resolve(records.get(sessionId));
+        },
+        save(record) {
+            records.set(record.acpxRecordId, record);
+            return Promise.resolve();
+        },
+    };
+}
+export function* useTestAgentAcpx(options) {
+    let pendingRoute;
+    const routeQueue = yield* useSerialQueues();
+    // ACPX tokenizes the command on whitespace with quote support, so
+    // command segments containing spaces (e.g. a binary path) are quoted.
+    const quote = (segment) => (/\s/.test(segment) ? `"${segment}"` : segment);
+    const registry = {
+        resolve() {
+            const route = pendingRoute ?? options.probeRoute;
+            return [...options.workerCommand.map(quote), "--connect", route].join(" ");
+        },
+        list() {
+            return options.agents;
+        },
+    };
+    const state = yield* useAcpxProviderState({ defaultAgent: options.defaultAgent, permissionMode: "deny-all" }, {
+        sessionStore: createMemorySessionStore(),
+        agentRegistry: registry,
+        // withSlot bounds the route mutex to the seam's op without a scope
+        // of its own — op's acquisitions (turn resources) belong to the
+        // provider's subscriber scope and outlive the critical section.
+        sessionRouting: (context, op) => routeQueue.withSlot("route", function* () {
+            pendingRoute = options.routeFor(context);
+            try {
+                return yield* op();
+            }
+            finally {
+                pendingRoute = undefined;
+            }
+        }),
+        ...(options.seams?.createRuntime ? { createRuntime: options.seams.createRuntime } : {}),
+    });
+    return { state };
+}