@executablemd/test-agent 0.0.0-bootstrap.0 → 0.5.1

package/esm/src/worker/profile.js

@@ -0,0 +1,86 @@
+/**
+ * Deterministic worker profile (specs/test-agent-spec.md §Deterministic
+ * runtime): a capability policy, not a security sandbox. Process and
+ * Fetch are denied, env() is undefined, cwd() is the virtual scenario
+ * root, and the filesystem is limited to controller-backed reads and
+ * stats. Reading a .ts component candidate raises the explicit
+ * unsupported-TypeScript error before it could ever be materialized or
+ * imported; stats stay honest so an earlier Name.md wins and a missing
+ * Name.ts falls through to Name/index.md. Eval blocks are inline-only:
+ * core strips static imports into options.imports before compiling, so
+ * a non-empty list is rejected, and the transformed source is parsed
+ * with acorn to reject dynamic import expressions before compilation.
+ */
+import { parse } from "acorn";
+import { API } from "@executablemd/runtime";
+function hasDynamicImport(node) {
+    if (Array.isArray(node)) {
+        return node.some(hasDynamicImport);
+    }
+    if (typeof node !== "object" || node === null) {
+        return false;
+    }
+    if ("type" in node && node.type === "ImportExpression") {
+        return true;
+    }
+    return Object.values(node).some(hasDynamicImport);
+}
+export function* installWorkerProfile(filesystem) {
+    yield* API.Process.around({
+        // deno-lint-ignore require-yield
+        *exec() {
+            throw new Error("process access is denied in behavior documents");
+        },
+    });
+    yield* API.Fetch.around({
+        // deno-lint-ignore require-yield
+        *fetch() {
+            throw new Error("network access is denied in behavior documents");
+        },
+    });
+    yield* API.Env.around({
+        // deno-lint-ignore require-yield
+        *cwd() {
+            return "/";
+        },
+        // deno-lint-ignore require-yield
+        *env() {
+            return undefined;
+        },
+    });
+    yield* API.Fs.around({
+        *readTextFile([path]) {
+            if (path.endsWith(".ts")) {
+                throw new Error(`TypeScript components are not supported in behavior documents: ${path}`);
+            }
+            const source = yield* filesystem.read(path);
+            if (source === undefined) {
+                throw new Error(`ENOENT: no such file in the scenario filesystem: ${path}`);
+            }
+            return source;
+        },
+        *stat([path]) {
+            return yield* filesystem.stat(path);
+        },
+        // deno-lint-ignore require-yield
+        *glob() {
+            throw new Error("filesystem globbing is denied in behavior documents");
+        },
+    });
+    yield* API.Compiler.around({
+        *compile([source, options], next) {
+            const imports = options?.imports ?? [];
+            if (imports.length > 0) {
+                throw new Error("eval blocks in behavior documents are inline-only — static imports are not allowed");
+            }
+            const program = parse(`function* __evalBlock__() {${source}\n}`, {
+                ecmaVersion: "latest",
+                sourceType: "script",
+            });
+            if (hasDynamicImport(program.body)) {
+                throw new Error("eval blocks in behavior documents are inline-only — dynamic import() is not allowed");
+            }
+            return yield* next(source, options);
+        },
+    });
+}

package/esm/src/worker/run.js

@@ -0,0 +1,291 @@
+/**
+ * `xmd test-agent` worker runtime (specs/test-agent-spec.md §Controller
+ * and worker). The worker attaches to its controller, rehydrates the
+ * behavior document and journal, and serves ACP on stdio.
+ *
+ * A stage's buffered durable-event suffix — everything through the next
+ * suspension point or the final root Close — is forwarded and
+ * acknowledged before the ACP result is returned; a crash inside that
+ * final delivery window is outside the supported recovery guarantee.
+ *
+ * Cancellation is transactional from the controller journal's
+ * perspective: a cancelled turn commits nothing — the scenario runtime
+ * is halted and rebuilt from the last acknowledged journal before
+ * another turn is accepted, so the next prompt re-enters the same
+ * stage deterministically.
+ */
+import { race, scoped, spawn, suspend, useScope, withResolvers } from "effection";
+import { RequestError } from "@agentclientprotocol/sdk";
+import { Component, DocumentOutput, execute } from "@executablemd/core";
+import { InMemoryStream } from "@executablemd/durable-streams";
+import { encodeMessage, parseControllerMessage, parseRoute, PROBE_INSTANCE } from "../protocol.js";
+import { createTurnBridge, collectTurn } from "./bridge.js";
+import { useLineClient } from "../net.js";
+import { installWhenPromptVocabulary } from "./when-prompt.js";
+import { installWorkerProfile } from "./profile.js";
+import { serveAcp, useProcessStdio } from "./acp-server.js";
+/**
+ * Connect to the controller over the shared line-socket adapter. A socket
+ * close ends the inbound stream, so a pending next() throws rather than
+ * hanging on a dead controller.
+ */
+function* useControllerClient(host, port) {
+    const client = yield* useLineClient(host, port, (line) => {
+        const parsed = parseControllerMessage(line);
+        return parsed.ok ? parsed.message : undefined;
+    });
+    return {
+        send: (message) => client.send(encodeMessage(message)),
+        next: () => client.next(),
+    };
+}
+export function* runTestAgentWorker(options) {
+    const route = parseRoute(options.connect);
+    if (!route.ok) {
+        throw new Error(route.error);
+    }
+    const { host, port, token, instance } = route.message;
+    const client = yield* useControllerClient(host, port);
+    client.send({ t: "attach", token, instance });
+    const config = yield* client.next();
+    if (config.t === "error") {
+        throw new Error(`controller rejected this worker: ${config.message}`);
+    }
+    if (config.t !== "config") {
+        throw new Error(`unexpected controller message "${config.t}" before config`);
+    }
+    if (instance === PROBE_INSTANCE || config.mode !== "scenario") {
+        yield* serveAcp({
+            // deno-lint-ignore require-yield
+            *ready() {
+                throw new Error("probe workers only initialize; they never start a behavior document");
+            },
+            // deno-lint-ignore require-yield
+            *runTurn() {
+                throw new Error("probe workers do not serve prompts");
+            },
+            cancel() { },
+        }, yield* useProcessStdio());
+        return;
+    }
+    const scenario = config;
+    // The worker-side mirror of the controller's journal: only events the
+    // controller has acknowledged. Cancellation rebuilds from exactly
+    // this list.
+    const acked = [...scenario.journal];
+    const workerScope = yield* useScope();
+    // Report a failure diagnostic and wait for the controller to record it. The
+    // ACP error is only surfaced after this returns, so the diagnostic is always
+    // observable on the controller by the time the error is seen.
+    function* report(message) {
+        client.send(message);
+        const reply = yield* client.next();
+        if (reply.t !== "recorded") {
+            throw new Error(`controller did not record diagnostic "${message.t}"`);
+        }
+    }
+    function* startRuntime() {
+        const snapshot = acked.slice();
+        const bridge = createTurnBridge();
+        // The subscription must live in the runtime task's scope — restart
+        // is invoked from short-lived ACP request tasks, and a subscription
+        // created there would die with its request and silently drop
+        // events.
+        const subscription = withResolvers();
+        const readiness = withResolvers();
+        const pending = [];
+        let exhausted = false;
+        function* flush() {
+            while (pending.length > 0) {
+                const event = pending.shift();
+                const seq = acked.length;
+                client.send({ t: "journal", seq, event });
+                const reply = yield* client.next();
+                if (reply.t !== "ack" || reply.seq !== seq) {
+                    throw new Error(`controller did not acknowledge journal event ${seq}`);
+                }
+                acked.push(event);
+            }
+        }
+        const task = yield* workerScope.spawn(() => scoped(function* () {
+            const turnEvents = yield* bridge.events;
+            subscription.resolve(turnEvents);
+            const stream = new InMemoryStream();
+            for (const event of snapshot) {
+                yield* stream.append(event);
+            }
+            stream.onAppend = (event) => {
+                pending.push(event);
+            };
+            // The root behavior document executes from the source snapshot
+            // captured at scenario declaration — changing or removing the
+            // file afterwards never affects an already-declared scenario.
+            // Markdown dependencies still resolve on demand through the
+            // controller.
+            yield* installWorkerProfile({
+                *read(path) {
+                    if (path === scenario.doc.path) {
+                        return scenario.doc.source;
+                    }
+                    client.send({ t: "read", path });
+                    const reply = yield* client.next();
+                    if (reply.t !== "read") {
+                        throw new Error(`unexpected controller reply "${reply.t}" to read`);
+                    }
+                    return reply.missing ? undefined : reply.source;
+                },
+                *stat(path) {
+                    if (path === scenario.doc.path) {
+                        return { exists: true, isFile: true, isDirectory: false };
+                    }
+                    client.send({ t: "stat", path });
+                    const reply = yield* client.next();
+                    if (reply.t !== "stat") {
+                        throw new Error(`unexpected controller reply "${reply.t}" to stat`);
+                    }
+                    return { exists: reply.exists, isFile: reply.isFile, isDirectory: false };
+                },
+            });
+            yield* installWhenPromptVocabulary(bridge);
+            // Behavior-document errors — eval preflight rejections,
+            // unsupported components, matcher configuration — must fail the
+            // scenario, never render as comments and let the turn succeed.
+            yield* Component.around({
+                // deno-lint-ignore require-yield
+                *raise([segment]) {
+                    throw new Error(segment.message);
+                },
+            });
+            yield* DocumentOutput.around({
+                *output([text], next) {
+                    yield* bridge.events.send({ kind: "output", text });
+                    yield* next(text);
+                },
+            });
+            const execution = yield* execute({ docPath: scenario.doc.path, stream });
+            yield* spawn(function* () {
+                const result = yield* execution;
+                if (result.ok) {
+                    yield* bridge.events.send({ kind: "eof" });
+                }
+                else {
+                    yield* bridge.events.send({ kind: "failed", error: result.error.message });
+                }
+            });
+            // Initialization: replay/expand to the first live matcher,
+            // persist the pre-matcher events, and only then report
+            // readiness.
+            yield* spawn(function* () {
+                const initial = yield* collectTurn(turnEvents);
+                if (initial.end === "failed") {
+                    yield* report({ t: "fatal", message: initial.error ?? "behavior document failed" });
+                    readiness.reject(new RequestError(-32603, initial.error ?? "behavior document failed"));
+                    return;
+                }
+                // Rehydrated runtimes replay completed stages, so their
+                // re-rendered output reaches the init collector; the fresh
+                // run already validated the pre-matcher region.
+                const rehydrated = snapshot.length > 0;
+                if (!rehydrated && initial.text.trim().length > 0) {
+                    const message = "behavior documents must not render non-whitespace output before the first <WhenPrompt>";
+                    yield* report({ t: "turn-failure", kind: "config", actual: initial.text });
+                    readiness.reject(new RequestError(-32603, message));
+                    return;
+                }
+                if (initial.end === "eof") {
+                    exhausted = true;
+                }
+                yield* flush();
+                readiness.resolve();
+            });
+            yield* suspend();
+        }));
+        const turnEvents = yield* subscription.operation;
+        return {
+            task,
+            turnEvents,
+            ready: readiness.operation,
+            offer: (text) => bridge.offer(text),
+            flush,
+            isExhausted: () => exhausted,
+            markExhausted: () => {
+                exhausted = true;
+            },
+        };
+    }
+    let current = yield* startRuntime();
+    let cancelRequested;
+    function* restart() {
+        yield* current.task.halt();
+        current = yield* startRuntime();
+        yield* current.ready;
+    }
+    yield* serveAcp({
+        *ready() {
+            yield* current.ready;
+        },
+        *runTurn(text) {
+            if (current.isExhausted()) {
+                yield* report({ t: "turn-failure", kind: "exhausted", actual: text });
+                throw new RequestError(-32603, `scenario exhausted: no stage remains for prompt: ${text}`);
+            }
+            const runtime = current;
+            const cancellation = withResolvers();
+            cancelRequested = () => cancellation.resolve({ cancelled: true });
+            try {
+                const outcome = yield* race([
+                    cancellation.operation,
+                    (function* () {
+                        const match = yield* runtime.offer(text);
+                        if (!match.ok) {
+                            const failure = {
+                                t: "turn-failure",
+                                kind: match.kind === "config" ? "config" : "mismatch",
+                                actual: match.actual,
+                            };
+                            if (match.kind === "mismatch") {
+                                failure.expected = match.expected;
+                            }
+                            // Terminal outcome: close cancellation before the diagnostic
+                            // round-trip so a later cancel cannot strand the recorded ack
+                            // in the shared controller response queue.
+                            cancelRequested = undefined;
+                            yield* report(failure);
+                            throw new RequestError(-32603, match.message);
+                        }
+                        const collected = yield* collectTurn(runtime.turnEvents);
+                        // Commit is transactional: once the turn reaches its terminal
+                        // bridge event cancellation can no longer win, so a later
+                        // session/cancel is ignored and neither the diagnostic nor the
+                        // journal ack below can be stranded.
+                        cancelRequested = undefined;
+                        if (collected.end === "failed") {
+                            yield* report({
+                                t: "fatal",
+                                message: collected.error ?? "behavior document failed",
+                            });
+                            throw new RequestError(-32603, collected.error ?? "behavior document failed");
+                        }
+                        if (collected.end === "eof") {
+                            runtime.markExhausted();
+                        }
+                        yield* runtime.flush();
+                        return { cancelled: false, text: collected.text };
+                    })(),
+                ]);
+                if (outcome.cancelled) {
+                    yield* restart();
+                }
+                return outcome;
+            }
+            finally {
+                cancelRequested = undefined;
+            }
+        },
+        cancel() {
+            if (cancelRequested) {
+                cancelRequested();
+            }
+        },
+    }, yield* useProcessStdio());
+}

package/esm/src/worker/when-prompt.js

@@ -0,0 +1,147 @@
+/**
+ * The `<WhenPrompt>` vocabulary (specs/test-agent-spec.md §Behavior
+ * documents). Each matcher signals the bridge that the previous stage's
+ * rendering is complete, then suspends until an offered prompt matches.
+ * A match is one durable `when_prompt` operation whose record restores
+ * the matched prompt and captures on replay, so a rehydrated worker
+ * advances to the active matcher without re-matching.
+ */
+import { scoped } from "effection";
+import { createDurableOperation } from "@executablemd/durable-streams";
+import { Component, env, renderSegments, validateBindingName } from "@executablemd/core";
+import { matchPrompt, parseTemplate } from "../template.js";
+const WHEN_PROMPT = "when_prompt";
+function parseStageRecord(value) {
+    if (typeof value !== "object" || value === null || Array.isArray(value)) {
+        return undefined;
+    }
+    if (!("prompt" in value) || !("captures" in value)) {
+        return undefined;
+    }
+    const { prompt, captures } = value;
+    if (typeof prompt !== "string") {
+        return undefined;
+    }
+    if (typeof captures !== "object" || captures === null || Array.isArray(captures)) {
+        return undefined;
+    }
+    const parsed = {};
+    for (const [name, text] of Object.entries(captures)) {
+        if (typeof text !== "string") {
+            return undefined;
+        }
+        parsed[name] = text;
+    }
+    return { prompt, captures: parsed };
+}
+function configError(message) {
+    return { type: "error", message: `<WhenPrompt> ${message}`, source: "WhenPrompt" };
+}
+function formatLocation(invocation) {
+    const position = invocation.position;
+    if (!position) {
+        return "unknown";
+    }
+    const at = `${position.line}:${position.column}`;
+    return position.path ? `${position.path}:${at}` : at;
+}
+/**
+ * Waits for a matching prompt, answering every mismatch through its
+ * offer so the ACP turn fails while the stage stays active.
+ */
+function* awaitMatch(bridge, template, bindings) {
+    while (true) {
+        const offer = yield* bridge.nextOffer();
+        const outcome = matchPrompt(template, offer.text, bindings);
+        offer.respond(outcome);
+        if (outcome.ok) {
+            return { prompt: offer.text, captures: outcome.captures };
+        }
+    }
+}
+function* persistStage(identity, live) {
+    const stored = yield createDurableOperation({ type: WHEN_PROMPT, name: identity.name, input: identity.input }, function* () {
+        const record = yield* live();
+        return { prompt: record.prompt, captures: record.captures };
+    });
+    const parsed = parseStageRecord(stored);
+    if (!parsed) {
+        throw new Error(`journaled when_prompt "${identity.name}" has an unexpected shape`);
+    }
+    return parsed;
+}
+export function* installWhenPromptVocabulary(bridge) {
+    const ordinals = new Map();
+    function* expandWhenPrompt(invocation, ctx) {
+        for (const name of Object.keys({ ...invocation.props, ...invocation.expressions })) {
+            if (name !== "template" && name !== "as") {
+                return [configError(`does not accept a "${name}" prop (allowed: template, as).`)];
+            }
+        }
+        const templateProp = invocation.props.template;
+        const hasChildren = !invocation.selfClosing && invocation.children.length > 0;
+        if (typeof templateProp === "string" && hasChildren) {
+            return [configError("accepts either a template prop or children, not both.")];
+        }
+        let source;
+        if (typeof templateProp === "string") {
+            source = templateProp;
+        }
+        else if (hasChildren) {
+            source = renderSegments(yield* ctx.expand(invocation.children)).trim();
+        }
+        else {
+            return [configError("requires a template prop or template children.")];
+        }
+        const parsed = parseTemplate(source);
+        if (!parsed.ok) {
+            return [configError(parsed.error)];
+        }
+        const binding = invocation.props.as;
+        if (parsed.template.captureNames.length > 0 && typeof binding !== "string") {
+            return [configError('captures require an "as" prop.')];
+        }
+        let bindingName;
+        if (binding !== undefined) {
+            const validated = validateBindingName(binding);
+            if (!validated.ok || validated.value === undefined) {
+                return [configError('the "as" prop must be a valid binding name.')];
+            }
+            bindingName = validated.value;
+        }
+        const currentEnv = yield* env;
+        if (!currentEnv) {
+            return [configError("requires an eval scope in context.")];
+        }
+        const location = formatLocation(invocation);
+        const ordinal = ordinals.get(location) ?? 0;
+        ordinals.set(location, ordinal + 1);
+        // The suspension signal lives inside the durable closure, so ONLY a
+        // live matcher emits it: replayed stages resolve from the journal
+        // and their re-rendered output never reaches a turn collector. For
+        // a live matcher the signal completes the previous stage — it
+        // follows all of that stage's output through one ordered channel,
+        // so the collector never loses the final chunk.
+        const record = yield* persistStage({ name: `when:${location}#${ordinal}`, input: source }, () => scoped(function* () {
+            yield* bridge.events.send({ kind: "suspended", stage: source });
+            return yield* awaitMatch(bridge, parsed.template, currentEnv.values);
+        }));
+        if (bindingName !== undefined) {
+            const existing = currentEnv.values[bindingName];
+            const merged = typeof existing === "object" && existing !== null && !Array.isArray(existing)
+                ? { ...existing }
+                : {};
+            Object.assign(merged, record.captures);
+            currentEnv.values[bindingName] = merged;
+        }
+        return [];
+    }
+    yield* Component.around({
+        *expandInvocation([invocation, ctx], next) {
+            if (invocation.name === "WhenPrompt") {
+                return { segments: yield* expandWhenPrompt(invocation, ctx) };
+            }
+            return yield* next(invocation, ctx);
+        },
+    });
+}

package/package.json

@@ -1,14 +1,41 @@
 {
   "name": "@executablemd/test-agent",
-  "version": "0.0.0-bootstrap.0",
-  "description": "Bootstrap reservation for Build reliable ACP integration tests with deterministic, document-driven agent behavior..",
-  "license": "MIT",
+  "version": "0.5.1",
+  "description": "Build reliable ACP integration tests with deterministic, document-driven agent behavior.",
+  "homepage": "https://executable.md",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/taras/executable.md.git"
   },
-  "homepage": "https://executable.md",
-  "files": [
-    "README.md"
-  ]
-}
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/taras/executable.md/issues"
+  },
+  "module": "./esm/mod.js",
+  "types": "./types/mod.d.ts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./types/mod.d.ts",
+        "default": "./esm/mod.js"
+      }
+    }
+  },
+  "scripts": {},
+  "dependencies": {
+    "@agentclientprotocol/sdk": "1.3.0",
+    "@effectionx/node": "0.2.4",
+    "@effectionx/scope-eval": "0.1.3",
+    "@effectionx/stream-helpers": "0.8.3",
+    "@executablemd/acp": "^0.5.1",
+    "@executablemd/core": "^0.5.1",
+    "@executablemd/durable-streams": "^0.5.1",
+    "@executablemd/runtime": "^0.5.1",
+    "@executablemd/testing": "^0.5.1",
+    "acorn": "^8.16.0",
+    "acpx": "0.12.0",
+    "effection": "4.1.0-alpha.7",
+    "zod": "^4.3.6"
+  },
+  "_generatedBy": "dnt@dev"
+}

package/types/mod.d.ts

@@ -0,0 +1,28 @@
+/**
+ * @module
+ * Build reliable ACP integration tests with deterministic,
+ * document-driven agent behavior. In place of a probabilistic coding
+ * agent, the test agent answers ACP prompts by advancing through a
+ * Markdown behavior document, so an integration can be tested against
+ * scripted, repeatable responses (specs/test-agent-spec.md).
+ *
+ * This entry point exposes the `<TestAgent>` vocabulary, the
+ * behavior-document engine, and the controller that registers scenarios
+ * and serves them to workers over the wire protocol.
+ */
+export { parseTemplate, matchPrompt } from "./src/template.js";
+export type { ParsedTemplate, TemplateMatchResult, TemplateParseResult, TemplateToken, } from "./src/template.js";
+export { createLineSplitter, encodeMessage, formatRoute, parseControllerMessage, parseRoute, parseWorkerMessage, PROBE_INSTANCE, } from "./src/protocol.js";
+export type { ControllerMessage, ParsedRoute, ParseResult, WireDurableEvent, WorkerMessage, } from "./src/protocol.js";
+export { useTestAgentController } from "./src/controller.js";
+export type { InstanceFailure, ScenarioInstance, TestAgentController } from "./src/controller.js";
+export { installTestAgentVocabulary } from "./src/vocabulary.js";
+export type { TestAgentVocabularyOptions } from "./src/vocabulary.js";
+export { createMemorySessionStore, useTestAgentAcpx } from "./src/state.js";
+export type { TestAgentAcpx, TestAgentAcpxOptions } from "./src/state.js";
+export { runTestAgentWorker } from "./src/worker/run.js";
+export { installWhenPromptVocabulary } from "./src/worker/when-prompt.js";
+export { installWorkerProfile } from "./src/worker/profile.js";
+export type { WorkerFilesystem } from "./src/worker/profile.js";
+export { collectTurn, createTurnBridge } from "./src/worker/bridge.js";
+export type { BridgeEvent, PromptOffer, TurnBridge } from "./src/worker/bridge.js";

package/types/src/controller.d.ts

@@ -0,0 +1,50 @@
+/**
+ * The test-agent controller (specs/test-agent-spec.md §Controller and
+ * worker): a localhost line-protocol server owned by the `<TestAgent>`
+ * scope. It serves behavior documents, Markdown dependencies (reads
+ * restricted to Markdown files whose canonical path stays inside the
+ * scenario root), and behavior journals to workers, and records journal
+ * appends and turn-failure diagnostics per scenario instance.
+ *
+ * Each instance admits one worker connection at a time. Unregistering an
+ * instance — or tearing the controller down — revokes and awaits its active
+ * connection before discarding state, so a revoked worker can no longer
+ * append, report failures, or read.
+ */
+import type { Operation } from "effection";
+import type { DurableEvent } from "@executablemd/durable-streams";
+export interface InstanceFailure {
+    kind: "mismatch" | "exhausted" | "config";
+    expected?: string;
+    actual: string;
+}
+export interface ScenarioInstance {
+    id: string;
+    route: string;
+    /** The real directory Markdown dependencies are served from. */
+    scenarioDir: string;
+    doc: {
+        path: string;
+        source: string;
+    };
+    journal: DurableEvent[];
+    failure?: InstanceFailure;
+    fatal?: string;
+}
+export interface TestAgentController {
+    probeRoute: string;
+    /**
+     * Register a scenario instance as a resource. Its finalizer removes it from
+     * the routing index, revokes and awaits any active worker, and clears its
+     * journal and diagnostics — so ending the instance's scope tears it down.
+     */
+    useInstance(config: {
+        doc: {
+            path: string;
+            source: string;
+        };
+        scenarioDir: string;
+    }): Operation<ScenarioInstance>;
+    instance(id: string): ScenarioInstance | undefined;
+}
+export declare function useTestAgentController(): Operation<TestAgentController>;