@ryanfw/prompt-orchestration-pipeline 1.3.0 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/docs/pop-task-guide.md +45 -0
- package/package.json +3 -3
- package/src/core/__tests__/agent-step.test.ts +402 -35
- package/src/core/__tests__/task-runner.test.ts +104 -0
- package/src/core/agent-step.ts +295 -41
- package/src/core/agent-types.ts +61 -0
- package/src/core/file-io.ts +8 -74
- package/src/core/orchestrator.ts +2 -1
- package/src/core/pipeline-definition.ts +1 -1
- package/src/core/pipeline-runner.ts +54 -3
- package/src/core/status-writer.ts +44 -26
- package/src/core/task-runner.ts +44 -1
- package/src/core/validation.ts +1 -1
- package/src/harness/__tests__/discovery.test.ts +235 -0
- package/src/harness/discovery.ts +109 -0
- package/src/harness/index.ts +22 -0
- package/src/harness/mcp-io-server.ts +1 -1
- package/src/ui/client/hooks/useJobListWithUpdates.ts +16 -1
- package/src/ui/dist/assets/{index-D7hzshSS.js → index--RH3sAt3.js} +129 -1
- package/src/ui/dist/assets/index--RH3sAt3.js.map +1 -0
- package/src/ui/dist/assets/style-CSSKuMOe.css +2 -0
- package/src/ui/dist/index.html +2 -2
- package/src/ui/embedded-assets.js +6 -6
- package/src/ui/pages/Code.tsx +135 -0
- package/src/ui/server/__tests__/gate-endpoints.test.ts +90 -0
- package/src/ui/server/__tests__/job-control-endpoints.test.ts +188 -0
- package/src/ui/server/__tests__/path-containment.test.ts +54 -0
- package/src/ui/server/__tests__/status-corruption.test.ts +55 -0
- package/src/ui/server/config-bridge.ts +1 -0
- package/src/ui/server/endpoints/__tests__/upload-endpoints.test.ts +77 -0
- package/src/ui/server/endpoints/gate-endpoints.ts +17 -1
- package/src/ui/server/endpoints/job-control-endpoints.ts +36 -2
- package/src/ui/server/endpoints/upload-endpoints.ts +13 -3
- package/src/ui/server/utils/http-utils.ts +6 -1
- package/src/ui/server/utils/path-containment.ts +18 -0
- package/src/ui/server/utils/status-corruption.ts +27 -0
- package/src/harness/__tests__/descriptors.test.ts +0 -378
- package/src/harness/__tests__/executor.test.ts +0 -193
- package/src/harness/__tests__/resolve.test.ts +0 -200
- package/src/harness/__tests__/types.test.ts +0 -297
- package/src/harness/descriptors/claude.ts +0 -132
- package/src/harness/descriptors/codex.ts +0 -126
- package/src/harness/descriptors/index.ts +0 -10
- package/src/harness/descriptors/opencode.ts +0 -147
- package/src/harness/executor.ts +0 -128
- package/src/harness/resolve.ts +0 -176
- package/src/harness/types.ts +0 -100
- package/src/ui/dist/assets/index-D7hzshSS.js.map +0 -1
- package/src/ui/dist/assets/style-BUFg3Sth.css +0 -2
package/README.md
CHANGED
|
@@ -50,6 +50,7 @@ Switch models globally or per-task without rewriting your logic.
|
|
|
50
50
|
* **Moonshot** (Kimi)
|
|
51
51
|
* **Zhipu** (GLM-4)
|
|
52
52
|
* **Claude Code** (CLI integration)
|
|
53
|
+
* **CLI Agents**: Tasks can also drive tool-using CLI coding agents (Claude, Codex, OpenCode) via the injected `runAgent()` helper — for file-aware, multi-turn work alongside single LLM calls. See the [Task Development Guide](docs/pop-task-guide.md#agent-api).
|
|
53
54
|
|
|
54
55
|
---
|
|
55
56
|
|
package/docs/pop-task-guide.md
CHANGED
|
@@ -192,6 +192,50 @@ const response = await llm.deepseek.chat({
|
|
|
192
192
|
|
|
193
193
|
---
|
|
194
194
|
|
|
195
|
+
## Agent API
|
|
196
|
+
|
|
197
|
+
Available via the `runAgent` function passed to stages. It runs a CLI coding
|
|
198
|
+
agent (the harness adapter) from inside a standard JavaScript task — the same
|
|
199
|
+
machinery behind pipeline `agent:` entries, but callable mid-task with a prompt
|
|
200
|
+
you build programmatically from upstream data.
|
|
201
|
+
|
|
202
|
+
```js
|
|
203
|
+
export const inference = async ({ runAgent, io, data, flags }) => {
|
|
204
|
+
const result = await runAgent({
|
|
205
|
+
harness: "claude", // "claude" | "codex" | "opencode"
|
|
206
|
+
prompt: "Read 'context.md', then write a summary to 'summary.md'.",
|
|
207
|
+
// model?: string // optional, passed through to the CLI
|
|
208
|
+
// io?: boolean // default true: bridge POP read/write artifacts
|
|
209
|
+
// timeoutMs?: number // optional wall-clock cap
|
|
210
|
+
// idleTimeoutMs?: number // optional no-output cap
|
|
211
|
+
// captureDiff?: boolean // capture a git diff as 'agent.patch'
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
if (!result.ok) {
|
|
215
|
+
throw new Error(`Agent failed: ${result.error}`);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// result: { ok, finalMessage, artifactsWritten, usage?, costUsd?, sessionId? }
|
|
219
|
+
const summary = await io.readArtifact("summary.md");
|
|
220
|
+
return { output: { summary }, flags };
|
|
221
|
+
};
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
By default (`io` is `true`) the agent shares the task's file I/O: it can call the
|
|
225
|
+
`read_artifact` / `write_artifact` tools to read and write the same artifacts the
|
|
226
|
+
task sees, and its `agent-result.md` is written automatically. Token usage and
|
|
227
|
+
cost flow into the job status like any other LLM call.
|
|
228
|
+
|
|
229
|
+
**`runAgent` vs `llm`**: use `llm.<provider>.chat()` for a single request/response
|
|
230
|
+
LLM call; use `runAgent()` when you need a tool-using CLI agent that reads and
|
|
231
|
+
writes files over multiple turns.
|
|
232
|
+
|
|
233
|
+
**`runAgent` vs an `agent:` pipeline entry**: an `agent:` entry takes a static
|
|
234
|
+
prompt from `pipeline.json`. `runAgent()` lets a JavaScript task compose the
|
|
235
|
+
prompt from seed/stage data and post-process the result in later stages.
|
|
236
|
+
|
|
237
|
+
---
|
|
238
|
+
|
|
195
239
|
## Validation API
|
|
196
240
|
|
|
197
241
|
Available via `validators` object in stages that need schema validation.
|
|
@@ -320,6 +364,7 @@ Each stage receives:
|
|
|
320
364
|
{
|
|
321
365
|
io, // File I/O (may be null)
|
|
322
366
|
llm, // LLM client
|
|
367
|
+
runAgent, // Run a CLI agent harness (see Agent API)
|
|
323
368
|
validators, // { validateWithSchema }
|
|
324
369
|
flags, // Control flags
|
|
325
370
|
meta: { taskName, workDir, jobId },
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ryanfw/prompt-orchestration-pipeline",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.2",
|
|
4
4
|
"description": "A Prompt-orchestration pipeline (POP) is a framework for building, running, and experimenting with complex chains of LLM tasks.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/ui/server/index.ts",
|
|
@@ -64,7 +64,6 @@
|
|
|
64
64
|
"react": "^19.2.0",
|
|
65
65
|
"react-dom": "^19.2.0",
|
|
66
66
|
"react-markdown": "^10.1.0",
|
|
67
|
-
"react-mentions": "^4.4.10",
|
|
68
67
|
"react-router-dom": "^7.9.4",
|
|
69
68
|
"react-syntax-highlighter": "^15.6.1",
|
|
70
69
|
"rehype-highlight": "^7.0.2",
|
|
@@ -72,7 +71,8 @@
|
|
|
72
71
|
"tslib": "^2.8.1",
|
|
73
72
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
74
73
|
"@opencode-ai/sdk": "^1.17.4",
|
|
75
|
-
"zod": "^3.25.0"
|
|
74
|
+
"zod": "^3.25.0",
|
|
75
|
+
"local-llm-cli-adapter": "github:ryan-mahoney/local-llm-cli-adapter#2ea1aa2d8e8dbe43eb845eb4730b08a02618f476"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
78
|
"@eslint/js": "^9.37.0",
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
import { describe, it, expect, mock, beforeEach, afterEach } from "bun:test";
|
|
2
|
-
import { mkdtempSync, rmSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
|
|
2
|
+
import { existsSync, mkdtempSync, rmSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
import type { WriteOptions, TaskFileIO } from "../file-io.ts";
|
|
6
6
|
import type { McpIoServerHandle } from "../../harness/mcp-io-server.ts";
|
|
7
7
|
import type {
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
RunResult,
|
|
9
|
+
HarnessRun,
|
|
10
10
|
HarnessEvent,
|
|
11
|
-
} from "../../harness/
|
|
12
|
-
import { runAgentStep } from "../agent-step.ts";
|
|
11
|
+
} from "../../harness/index.ts";
|
|
12
|
+
import { runAgentStep, executeAgent } from "../agent-step.ts";
|
|
13
13
|
import { createTaskFileIO } from "../file-io.ts";
|
|
14
14
|
|
|
15
15
|
function createFakeIO(): TaskFileIO & { calls: string[] } {
|
|
@@ -53,10 +53,11 @@ function createFakeIO(): TaskFileIO & { calls: string[] } {
|
|
|
53
53
|
|
|
54
54
|
function createFakeMcpHandle(
|
|
55
55
|
artifacts: string[] = [],
|
|
56
|
+
connection = { url: "http://127.0.0.1:9999/mcp", token: "fake-token" },
|
|
56
57
|
): McpIoServerHandle & { closeSpy: ReturnType<typeof mock> } {
|
|
57
58
|
const closeSpy = mock(async () => {});
|
|
58
59
|
return {
|
|
59
|
-
connection
|
|
60
|
+
connection,
|
|
60
61
|
artifactsWritten() {
|
|
61
62
|
return [...artifacts];
|
|
62
63
|
},
|
|
@@ -84,26 +85,53 @@ function makeArgs(overrides?: {
|
|
|
84
85
|
};
|
|
85
86
|
}
|
|
86
87
|
|
|
87
|
-
function
|
|
88
|
+
function makeFakeHarnessRun(result: RunResult | Error): HarnessRun {
|
|
89
|
+
const resolved =
|
|
90
|
+
result instanceof Error
|
|
91
|
+
? Promise.reject(result)
|
|
92
|
+
: Promise.resolve(result);
|
|
93
|
+
return {
|
|
94
|
+
result: resolved,
|
|
95
|
+
sessionId: Promise.resolve("sess-1"),
|
|
96
|
+
abort() {},
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function makeSuccessResult(overrides?: Partial<RunResult>): RunResult {
|
|
88
101
|
return {
|
|
89
102
|
finalMessage: "task complete",
|
|
90
|
-
events: [],
|
|
91
103
|
exitCode: 0,
|
|
92
104
|
usage: { inputTokens: 100, outputTokens: 50, totalTokens: 150 },
|
|
93
|
-
costUsd: 0.12,
|
|
94
105
|
sessionId: "sess-1",
|
|
95
106
|
...overrides,
|
|
96
107
|
};
|
|
97
108
|
}
|
|
98
109
|
|
|
99
|
-
function
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
110
|
+
function deferred<T>(): { promise: Promise<T>; resolve: (value: T) => void; reject: (err: unknown) => void } {
|
|
111
|
+
let resolve!: (value: T) => void;
|
|
112
|
+
let reject!: (err: unknown) => void;
|
|
113
|
+
const promise = new Promise<T>((res, rej) => {
|
|
114
|
+
resolve = res;
|
|
115
|
+
reject = rej;
|
|
103
116
|
});
|
|
117
|
+
return { promise, resolve, reject };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
async function waitFor(predicate: () => boolean, message: string): Promise<void> {
|
|
121
|
+
const deadline = Date.now() + 1_000;
|
|
122
|
+
while (!predicate()) {
|
|
123
|
+
if (Date.now() > deadline) {
|
|
124
|
+
throw new Error(message);
|
|
125
|
+
}
|
|
126
|
+
await new Promise((resolve) => setTimeout(resolve, 1));
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function makeDeps(result: RunResult | Error) {
|
|
131
|
+
const run = mock(() => makeFakeHarnessRun(result));
|
|
104
132
|
const startMcpIoServer = mock(async () => createFakeMcpHandle());
|
|
105
133
|
const createTaskFileIO = mock(() => createFakeIO());
|
|
106
|
-
return {
|
|
134
|
+
return { run, startMcpIoServer, createTaskFileIO };
|
|
107
135
|
}
|
|
108
136
|
|
|
109
137
|
function gitSync(args: string[], cwd: string): string {
|
|
@@ -119,13 +147,335 @@ function makeCaptureDeps(workDir: string) {
|
|
|
119
147
|
statusPath: join(workDir, "tasks-status.json"),
|
|
120
148
|
});
|
|
121
149
|
return {
|
|
122
|
-
|
|
150
|
+
run: mock(() => makeFakeHarnessRun(makeSuccessResult())),
|
|
123
151
|
startMcpIoServer: mock(async () => createFakeMcpHandle()),
|
|
124
152
|
createTaskFileIO: mock(() => io),
|
|
125
153
|
io,
|
|
126
154
|
};
|
|
127
155
|
}
|
|
128
156
|
|
|
157
|
+
describe("executeAgent", () => {
|
|
158
|
+
it("runs against a provided io and returns ok:true with merged artifacts", async () => {
|
|
159
|
+
const io = createFakeIO();
|
|
160
|
+
const run = mock(() => makeFakeHarnessRun(makeSuccessResult()));
|
|
161
|
+
const startMcpIoServer = mock(async () => createFakeMcpHandle(["explainer.md"]));
|
|
162
|
+
|
|
163
|
+
const result = await executeAgent(
|
|
164
|
+
{ io, entry: { name: "agent-explainer", harness: "claude", prompt: "do it" } },
|
|
165
|
+
{ run, startMcpIoServer },
|
|
166
|
+
);
|
|
167
|
+
|
|
168
|
+
expect(result.ok).toBe(true);
|
|
169
|
+
expect(result.finalMessage).toBe("task complete");
|
|
170
|
+
expect(result.artifactsWritten).toContain("explainer.md");
|
|
171
|
+
expect(result.artifactsWritten).toContain("agent-result.md");
|
|
172
|
+
expect(io.calls).toContain("writeArtifact:agent-result.md");
|
|
173
|
+
expect(startMcpIoServer).toHaveBeenCalled();
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it("returns ok:false when the harness run rejects", async () => {
|
|
177
|
+
const io = createFakeIO();
|
|
178
|
+
const run = mock(() => makeFakeHarnessRun(new Error("spawn failed")));
|
|
179
|
+
const startMcpIoServer = mock(async () => createFakeMcpHandle());
|
|
180
|
+
|
|
181
|
+
const result = await executeAgent(
|
|
182
|
+
{ io, entry: { name: "agent-explainer", harness: "codex", prompt: "do it" } },
|
|
183
|
+
{ run, startMcpIoServer },
|
|
184
|
+
);
|
|
185
|
+
|
|
186
|
+
expect(result.ok).toBe(false);
|
|
187
|
+
expect(result.error).toBe("spawn failed");
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
it("skips the MCP server when io is false", async () => {
|
|
191
|
+
const io = createFakeIO();
|
|
192
|
+
const run = mock(() => makeFakeHarnessRun(makeSuccessResult()));
|
|
193
|
+
const startMcpIoServer = mock(async () => createFakeMcpHandle());
|
|
194
|
+
|
|
195
|
+
await executeAgent(
|
|
196
|
+
{ io, entry: { name: "a", harness: "opencode", prompt: "p", io: false } },
|
|
197
|
+
{ run, startMcpIoServer },
|
|
198
|
+
);
|
|
199
|
+
|
|
200
|
+
expect(startMcpIoServer).not.toHaveBeenCalled();
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
it("passes adapter-native MCP servers for claude", async () => {
|
|
204
|
+
const io = createFakeIO();
|
|
205
|
+
let capturedMcpServers: unknown;
|
|
206
|
+
const run = mock((opts: { mcpServers?: unknown }) => {
|
|
207
|
+
capturedMcpServers = opts.mcpServers;
|
|
208
|
+
return makeFakeHarnessRun(makeSuccessResult());
|
|
209
|
+
});
|
|
210
|
+
const startMcpIoServer = mock(async () => createFakeMcpHandle());
|
|
211
|
+
|
|
212
|
+
await executeAgent(
|
|
213
|
+
{ io, entry: { name: "a", harness: "claude", prompt: "p" } },
|
|
214
|
+
{ run, startMcpIoServer },
|
|
215
|
+
);
|
|
216
|
+
|
|
217
|
+
expect(capturedMcpServers).toEqual([{
|
|
218
|
+
name: "popio",
|
|
219
|
+
url: "http://127.0.0.1:9999/mcp",
|
|
220
|
+
headers: { Authorization: "Bearer fake-token" },
|
|
221
|
+
}]);
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
it("bridges MCP for codex through a temporary binary wrapper", async () => {
|
|
225
|
+
const io = createFakeIO();
|
|
226
|
+
const previousPopBin = process.env.POP_CODEX_BIN;
|
|
227
|
+
const previousAdapterBin = process.env.LLM_ADAPTER_CODEX_BIN;
|
|
228
|
+
const previousRealBin = process.env.POP_REAL_CODEX_BIN;
|
|
229
|
+
const previousMcpUrlConfig = process.env.POP_MCP_URL_CONFIG;
|
|
230
|
+
const previousMcpToken = process.env.POP_MCP_TOKEN;
|
|
231
|
+
let wrapperPath = "";
|
|
232
|
+
let capturedMcpServers: unknown;
|
|
233
|
+
try {
|
|
234
|
+
process.env.POP_CODEX_BIN = "/custom/codex";
|
|
235
|
+
delete process.env.LLM_ADAPTER_CODEX_BIN;
|
|
236
|
+
delete process.env.POP_REAL_CODEX_BIN;
|
|
237
|
+
delete process.env.POP_MCP_URL_CONFIG;
|
|
238
|
+
delete process.env.POP_MCP_TOKEN;
|
|
239
|
+
const run = mock((opts: { mcpServers?: unknown; onEvent?: (event: HarnessEvent) => void }) => {
|
|
240
|
+
capturedMcpServers = opts.mcpServers;
|
|
241
|
+
wrapperPath = process.env.LLM_ADAPTER_CODEX_BIN ?? "";
|
|
242
|
+
expect(wrapperPath).toContain("codex-mcp-");
|
|
243
|
+
expect(readFileSync(wrapperPath, "utf8")).toContain("POP_MCP_URL_CONFIG");
|
|
244
|
+
expect(process.env.POP_REAL_CODEX_BIN).toBe("/custom/codex");
|
|
245
|
+
expect(process.env.POP_MCP_URL_CONFIG).toBe(
|
|
246
|
+
'mcp_servers.popio.url="http://127.0.0.1:9999/mcp"',
|
|
247
|
+
);
|
|
248
|
+
expect(process.env.POP_MCP_TOKEN).toBe("fake-token");
|
|
249
|
+
opts.onEvent?.({
|
|
250
|
+
harness: "codex",
|
|
251
|
+
seq: 0,
|
|
252
|
+
at: Date.now(),
|
|
253
|
+
raw: { type: "result", total_cost_usd: 0.42 },
|
|
254
|
+
type: "raw",
|
|
255
|
+
});
|
|
256
|
+
return makeFakeHarnessRun(makeSuccessResult());
|
|
257
|
+
});
|
|
258
|
+
const startMcpIoServer = mock(async () => createFakeMcpHandle());
|
|
259
|
+
|
|
260
|
+
const result = await executeAgent(
|
|
261
|
+
{ io, entry: { name: "a", harness: "codex", prompt: "p" } },
|
|
262
|
+
{ run, startMcpIoServer },
|
|
263
|
+
);
|
|
264
|
+
|
|
265
|
+
expect(capturedMcpServers).toBeUndefined();
|
|
266
|
+
expect(result.costUsd).toBe(0.42);
|
|
267
|
+
expect(process.env.LLM_ADAPTER_CODEX_BIN).toBeUndefined();
|
|
268
|
+
expect(process.env.POP_REAL_CODEX_BIN).toBeUndefined();
|
|
269
|
+
expect(process.env.POP_MCP_URL_CONFIG).toBeUndefined();
|
|
270
|
+
expect(process.env.POP_MCP_TOKEN).toBeUndefined();
|
|
271
|
+
expect(existsSync(wrapperPath)).toBe(false);
|
|
272
|
+
} finally {
|
|
273
|
+
if (previousPopBin === undefined) {
|
|
274
|
+
delete process.env.POP_CODEX_BIN;
|
|
275
|
+
} else {
|
|
276
|
+
process.env.POP_CODEX_BIN = previousPopBin;
|
|
277
|
+
}
|
|
278
|
+
if (previousAdapterBin === undefined) {
|
|
279
|
+
delete process.env.LLM_ADAPTER_CODEX_BIN;
|
|
280
|
+
} else {
|
|
281
|
+
process.env.LLM_ADAPTER_CODEX_BIN = previousAdapterBin;
|
|
282
|
+
}
|
|
283
|
+
if (previousRealBin === undefined) {
|
|
284
|
+
delete process.env.POP_REAL_CODEX_BIN;
|
|
285
|
+
} else {
|
|
286
|
+
process.env.POP_REAL_CODEX_BIN = previousRealBin;
|
|
287
|
+
}
|
|
288
|
+
if (previousMcpUrlConfig === undefined) {
|
|
289
|
+
delete process.env.POP_MCP_URL_CONFIG;
|
|
290
|
+
} else {
|
|
291
|
+
process.env.POP_MCP_URL_CONFIG = previousMcpUrlConfig;
|
|
292
|
+
}
|
|
293
|
+
if (previousMcpToken === undefined) {
|
|
294
|
+
delete process.env.POP_MCP_TOKEN;
|
|
295
|
+
} else {
|
|
296
|
+
process.env.POP_MCP_TOKEN = previousMcpToken;
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
});
|
|
300
|
+
|
|
301
|
+
it("serializes env-backed MCP compatibility for concurrent codex runs", async () => {
|
|
302
|
+
const previousPopBin = process.env.POP_CODEX_BIN;
|
|
303
|
+
const previousAdapterBin = process.env.LLM_ADAPTER_CODEX_BIN;
|
|
304
|
+
const previousRealBin = process.env.POP_REAL_CODEX_BIN;
|
|
305
|
+
const previousMcpUrlConfig = process.env.POP_MCP_URL_CONFIG;
|
|
306
|
+
const previousMcpToken = process.env.POP_MCP_TOKEN;
|
|
307
|
+
const firstRunResult = deferred<RunResult>();
|
|
308
|
+
const secondRunResult = deferred<RunResult>();
|
|
309
|
+
const observedTokens: Array<string | undefined> = [];
|
|
310
|
+
let runCount = 0;
|
|
311
|
+
let startCount = 0;
|
|
312
|
+
try {
|
|
313
|
+
process.env.POP_CODEX_BIN = "/custom/codex";
|
|
314
|
+
delete process.env.LLM_ADAPTER_CODEX_BIN;
|
|
315
|
+
delete process.env.POP_REAL_CODEX_BIN;
|
|
316
|
+
delete process.env.POP_MCP_URL_CONFIG;
|
|
317
|
+
delete process.env.POP_MCP_TOKEN;
|
|
318
|
+
const run = mock(() => {
|
|
319
|
+
runCount += 1;
|
|
320
|
+
observedTokens.push(process.env.POP_MCP_TOKEN);
|
|
321
|
+
return {
|
|
322
|
+
result: runCount === 1 ? firstRunResult.promise : secondRunResult.promise,
|
|
323
|
+
sessionId: Promise.resolve(`sess-${runCount}`),
|
|
324
|
+
abort() {},
|
|
325
|
+
};
|
|
326
|
+
});
|
|
327
|
+
const startMcpIoServer = mock(async () => {
|
|
328
|
+
startCount += 1;
|
|
329
|
+
return createFakeMcpHandle([], {
|
|
330
|
+
url: `http://127.0.0.1:999${startCount}/mcp`,
|
|
331
|
+
token: `token-${startCount}`,
|
|
332
|
+
});
|
|
333
|
+
});
|
|
334
|
+
|
|
335
|
+
const first = executeAgent(
|
|
336
|
+
{ io: createFakeIO(), entry: { name: "a", harness: "codex", prompt: "p" } },
|
|
337
|
+
{ run, startMcpIoServer },
|
|
338
|
+
);
|
|
339
|
+
const second = executeAgent(
|
|
340
|
+
{ io: createFakeIO(), entry: { name: "b", harness: "codex", prompt: "p" } },
|
|
341
|
+
{ run, startMcpIoServer },
|
|
342
|
+
);
|
|
343
|
+
|
|
344
|
+
await waitFor(() => runCount === 1, "first codex run did not start");
|
|
345
|
+
expect(runCount).toBe(1);
|
|
346
|
+
expect(observedTokens).toHaveLength(1);
|
|
347
|
+
const firstObservedToken = observedTokens[0];
|
|
348
|
+
if (firstObservedToken === undefined) {
|
|
349
|
+
throw new Error("first codex run did not receive an MCP token");
|
|
350
|
+
}
|
|
351
|
+
expect(["token-1", "token-2"]).toContain(firstObservedToken);
|
|
352
|
+
|
|
353
|
+
firstRunResult.resolve(makeSuccessResult({ finalMessage: "run-1" }));
|
|
354
|
+
await waitFor(() => runCount === 2, "second codex run did not start");
|
|
355
|
+
expect(runCount).toBe(2);
|
|
356
|
+
expect(new Set(observedTokens)).toEqual(new Set(["token-1", "token-2"]));
|
|
357
|
+
|
|
358
|
+
secondRunResult.resolve(makeSuccessResult({ finalMessage: "run-2" }));
|
|
359
|
+
const results = await Promise.all([first, second]);
|
|
360
|
+
expect(new Set(results.map((r) => r.finalMessage))).toEqual(new Set(["run-1", "run-2"]));
|
|
361
|
+
expect(process.env.POP_MCP_TOKEN).toBeUndefined();
|
|
362
|
+
expect(process.env.POP_MCP_URL_CONFIG).toBeUndefined();
|
|
363
|
+
} finally {
|
|
364
|
+
firstRunResult.resolve(makeSuccessResult());
|
|
365
|
+
secondRunResult.resolve(makeSuccessResult());
|
|
366
|
+
if (previousPopBin === undefined) {
|
|
367
|
+
delete process.env.POP_CODEX_BIN;
|
|
368
|
+
} else {
|
|
369
|
+
process.env.POP_CODEX_BIN = previousPopBin;
|
|
370
|
+
}
|
|
371
|
+
if (previousAdapterBin === undefined) {
|
|
372
|
+
delete process.env.LLM_ADAPTER_CODEX_BIN;
|
|
373
|
+
} else {
|
|
374
|
+
process.env.LLM_ADAPTER_CODEX_BIN = previousAdapterBin;
|
|
375
|
+
}
|
|
376
|
+
if (previousRealBin === undefined) {
|
|
377
|
+
delete process.env.POP_REAL_CODEX_BIN;
|
|
378
|
+
} else {
|
|
379
|
+
process.env.POP_REAL_CODEX_BIN = previousRealBin;
|
|
380
|
+
}
|
|
381
|
+
if (previousMcpUrlConfig === undefined) {
|
|
382
|
+
delete process.env.POP_MCP_URL_CONFIG;
|
|
383
|
+
} else {
|
|
384
|
+
process.env.POP_MCP_URL_CONFIG = previousMcpUrlConfig;
|
|
385
|
+
}
|
|
386
|
+
if (previousMcpToken === undefined) {
|
|
387
|
+
delete process.env.POP_MCP_TOKEN;
|
|
388
|
+
} else {
|
|
389
|
+
process.env.POP_MCP_TOKEN = previousMcpToken;
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
});
|
|
393
|
+
|
|
394
|
+
it("bridges MCP for opencode through OPENCODE_CONFIG_DIR and extracts raw cost", async () => {
|
|
395
|
+
const io = createFakeIO();
|
|
396
|
+
const previousConfigDir = process.env.OPENCODE_CONFIG_DIR;
|
|
397
|
+
const sourceConfigDir = mkdtempSync(join(tmpdir(), "opencode-existing-"));
|
|
398
|
+
let configDir = "";
|
|
399
|
+
let capturedMcpServers: unknown;
|
|
400
|
+
try {
|
|
401
|
+
writeFileSync(
|
|
402
|
+
join(sourceConfigDir, "opencode.json"),
|
|
403
|
+
JSON.stringify({
|
|
404
|
+
provider: "kept",
|
|
405
|
+
mcp: {
|
|
406
|
+
existing: { type: "local", command: ["tool"] },
|
|
407
|
+
},
|
|
408
|
+
}),
|
|
409
|
+
);
|
|
410
|
+
process.env.OPENCODE_CONFIG_DIR = sourceConfigDir;
|
|
411
|
+
const run = mock((opts: { mcpServers?: unknown; onEvent?: (event: HarnessEvent) => void }) => {
|
|
412
|
+
capturedMcpServers = opts.mcpServers;
|
|
413
|
+
configDir = process.env.OPENCODE_CONFIG_DIR ?? "";
|
|
414
|
+
const config = JSON.parse(readFileSync(join(configDir, "opencode.json"), "utf8")) as Record<string, unknown>;
|
|
415
|
+
expect(config).toMatchObject({
|
|
416
|
+
provider: "kept",
|
|
417
|
+
mcp: {
|
|
418
|
+
existing: { type: "local", command: ["tool"] },
|
|
419
|
+
popio: {
|
|
420
|
+
type: "remote",
|
|
421
|
+
url: "http://127.0.0.1:9999/mcp",
|
|
422
|
+
headers: { Authorization: "Bearer fake-token" },
|
|
423
|
+
enabled: true,
|
|
424
|
+
},
|
|
425
|
+
},
|
|
426
|
+
});
|
|
427
|
+
opts.onEvent?.({
|
|
428
|
+
harness: "opencode",
|
|
429
|
+
seq: 0,
|
|
430
|
+
at: Date.now(),
|
|
431
|
+
raw: { type: "result", info: { costUsd: 0.13 } },
|
|
432
|
+
type: "raw",
|
|
433
|
+
});
|
|
434
|
+
return makeFakeHarnessRun(makeSuccessResult());
|
|
435
|
+
});
|
|
436
|
+
const startMcpIoServer = mock(async () => createFakeMcpHandle());
|
|
437
|
+
|
|
438
|
+
const result = await executeAgent(
|
|
439
|
+
{ io, entry: { name: "a", harness: "opencode", prompt: "p" } },
|
|
440
|
+
{ run, startMcpIoServer },
|
|
441
|
+
);
|
|
442
|
+
|
|
443
|
+
expect(capturedMcpServers).toBeUndefined();
|
|
444
|
+
expect(result.costUsd).toBe(0.13);
|
|
445
|
+
expect(process.env.OPENCODE_CONFIG_DIR).toBe(sourceConfigDir);
|
|
446
|
+
expect(existsSync(configDir)).toBe(false);
|
|
447
|
+
} finally {
|
|
448
|
+
rmSync(sourceConfigDir, { recursive: true, force: true });
|
|
449
|
+
if (previousConfigDir === undefined) {
|
|
450
|
+
delete process.env.OPENCODE_CONFIG_DIR;
|
|
451
|
+
} else {
|
|
452
|
+
process.env.OPENCODE_CONFIG_DIR = previousConfigDir;
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
});
|
|
456
|
+
|
|
457
|
+
it("preserves usage and cost when artifact writes fail after the harness run", async () => {
|
|
458
|
+
const io = createFakeIO();
|
|
459
|
+
io.writeArtifact = async () => {
|
|
460
|
+
throw new Error("disk full");
|
|
461
|
+
};
|
|
462
|
+
const run = mock(() => makeFakeHarnessRun(makeSuccessResult({ costUsd: 0.21 })));
|
|
463
|
+
const startMcpIoServer = mock(async () => createFakeMcpHandle(["agent-output.md"]));
|
|
464
|
+
|
|
465
|
+
const result = await executeAgent(
|
|
466
|
+
{ io, entry: { name: "a", harness: "claude", prompt: "p" } },
|
|
467
|
+
{ run, startMcpIoServer },
|
|
468
|
+
);
|
|
469
|
+
|
|
470
|
+
expect(result.ok).toBe(false);
|
|
471
|
+
expect(result.error).toBe("disk full");
|
|
472
|
+
expect(result.usage).toEqual({ inputTokens: 100, outputTokens: 50, totalTokens: 150 });
|
|
473
|
+
expect(result.costUsd).toBe(0.21);
|
|
474
|
+
expect(result.sessionId).toBe("sess-1");
|
|
475
|
+
expect(result.artifactsWritten).toContain("agent-output.md");
|
|
476
|
+
});
|
|
477
|
+
});
|
|
478
|
+
|
|
129
479
|
describe("runAgentStep", () => {
|
|
130
480
|
it("success writes event log and agent-result.md and returns ok:true with usage/cost", async () => {
|
|
131
481
|
const deps = makeDeps(makeSuccessResult());
|
|
@@ -135,7 +485,7 @@ describe("runAgentStep", () => {
|
|
|
135
485
|
expect(result.ok).toBe(true);
|
|
136
486
|
expect(result.finalMessage).toBe("task complete");
|
|
137
487
|
expect(result.usage).toEqual({ inputTokens: 100, outputTokens: 50, totalTokens: 150 });
|
|
138
|
-
expect(result.costUsd).
|
|
488
|
+
expect(result.costUsd).toBeUndefined();
|
|
139
489
|
expect(result.sessionId).toBe("sess-1");
|
|
140
490
|
expect(result.artifactsWritten).toContain("agent-result.md");
|
|
141
491
|
expect(deps.startMcpIoServer).toHaveBeenCalled();
|
|
@@ -150,18 +500,18 @@ describe("runAgentStep", () => {
|
|
|
150
500
|
});
|
|
151
501
|
|
|
152
502
|
const events: HarnessEvent[] = [
|
|
153
|
-
{
|
|
154
|
-
{
|
|
503
|
+
{ harness: "claude", seq: 0, at: Date.now(), raw: { text: "hello" }, type: "assistant_message", text: "hello" },
|
|
504
|
+
{ harness: "claude", seq: 1, at: Date.now(), raw: { message: "done" }, type: "run_completed", result: makeSuccessResult() },
|
|
155
505
|
];
|
|
156
|
-
const
|
|
506
|
+
const run = mock((opts: { onEvent?: (event: HarnessEvent) => void }) => {
|
|
157
507
|
for (const event of events) {
|
|
158
508
|
opts.onEvent?.(event);
|
|
159
509
|
}
|
|
160
|
-
return makeSuccessResult();
|
|
510
|
+
return makeFakeHarnessRun(makeSuccessResult());
|
|
161
511
|
});
|
|
162
512
|
const startMcpIoServer = mock(async () => createFakeMcpHandle());
|
|
163
513
|
|
|
164
|
-
await runAgentStep(makeArgs(), {
|
|
514
|
+
await runAgentStep(makeArgs(), { run, startMcpIoServer, createTaskFileIO });
|
|
165
515
|
|
|
166
516
|
const logCalls = capturedIO!.calls.filter((c) => c.startsWith("writeLog:"));
|
|
167
517
|
expect(logCalls).toHaveLength(2);
|
|
@@ -177,15 +527,15 @@ describe("runAgentStep", () => {
|
|
|
177
527
|
});
|
|
178
528
|
|
|
179
529
|
let capturedPrompt: string | undefined;
|
|
180
|
-
const
|
|
530
|
+
const run = mock((opts: { prompt: string }) => {
|
|
181
531
|
capturedPrompt = opts.prompt;
|
|
182
|
-
return makeSuccessResult();
|
|
532
|
+
return makeFakeHarnessRun(makeSuccessResult());
|
|
183
533
|
});
|
|
184
534
|
const startMcpIoServer = mock(async () => createFakeMcpHandle());
|
|
185
535
|
|
|
186
536
|
const result = await runAgentStep(
|
|
187
537
|
makeArgs({ entry: { prompt: undefined, promptFrom: "my-prompt.md" } }),
|
|
188
|
-
{
|
|
538
|
+
{ run, startMcpIoServer, createTaskFileIO },
|
|
189
539
|
);
|
|
190
540
|
|
|
191
541
|
expect(result.ok).toBe(true);
|
|
@@ -197,9 +547,7 @@ describe("runAgentStep", () => {
|
|
|
197
547
|
const mcpHandle = createFakeMcpHandle();
|
|
198
548
|
const createTaskFileIO = mock(() => createFakeIO());
|
|
199
549
|
const deps = {
|
|
200
|
-
|
|
201
|
-
throw new Error("boom");
|
|
202
|
-
}),
|
|
550
|
+
run: mock(() => makeFakeHarnessRun(new Error("boom"))),
|
|
203
551
|
startMcpIoServer: mock(async () => mcpHandle),
|
|
204
552
|
createTaskFileIO,
|
|
205
553
|
};
|
|
@@ -216,7 +564,7 @@ describe("runAgentStep", () => {
|
|
|
216
564
|
const mcpHandle = createFakeMcpHandle();
|
|
217
565
|
const createTaskFileIO = mock(() => createFakeIO());
|
|
218
566
|
const deps = {
|
|
219
|
-
|
|
567
|
+
run: mock(() => makeFakeHarnessRun(makeSuccessResult())),
|
|
220
568
|
startMcpIoServer: mock(async () => mcpHandle),
|
|
221
569
|
createTaskFileIO,
|
|
222
570
|
};
|
|
@@ -229,9 +577,7 @@ describe("runAgentStep", () => {
|
|
|
229
577
|
const mcpHandle = createFakeMcpHandle();
|
|
230
578
|
const createTaskFileIO = mock(() => createFakeIO());
|
|
231
579
|
const deps = {
|
|
232
|
-
|
|
233
|
-
throw new Error("fail");
|
|
234
|
-
}),
|
|
580
|
+
run: mock(() => makeFakeHarnessRun(new Error("fail"))),
|
|
235
581
|
startMcpIoServer: mock(async () => mcpHandle),
|
|
236
582
|
createTaskFileIO,
|
|
237
583
|
};
|
|
@@ -243,22 +589,43 @@ describe("runAgentStep", () => {
|
|
|
243
589
|
it("MCP server is closed on timeout path", async () => {
|
|
244
590
|
const mcpHandle = createFakeMcpHandle();
|
|
245
591
|
const createTaskFileIO = mock(() => createFakeIO());
|
|
592
|
+
let capturedTimeoutMs: number | undefined;
|
|
593
|
+
let capturedIdleTimeoutMs: number | undefined;
|
|
246
594
|
const deps = {
|
|
247
|
-
|
|
248
|
-
|
|
595
|
+
run: mock((opts) => {
|
|
596
|
+
capturedTimeoutMs = opts.timeoutMs;
|
|
597
|
+
capturedIdleTimeoutMs = opts.idleTimeoutMs;
|
|
598
|
+
return makeFakeHarnessRun(new Error('Harness "claude" timed out after 100ms'));
|
|
249
599
|
}),
|
|
250
600
|
startMcpIoServer: mock(async () => mcpHandle),
|
|
251
601
|
createTaskFileIO,
|
|
252
602
|
};
|
|
253
603
|
|
|
254
604
|
await runAgentStep(makeArgs({ entry: { timeoutMs: 100 } }), deps);
|
|
605
|
+
expect(capturedTimeoutMs).toBe(100);
|
|
606
|
+
expect(capturedIdleTimeoutMs).toBeUndefined();
|
|
255
607
|
expect(mcpHandle.closeSpy).toHaveBeenCalled();
|
|
256
608
|
});
|
|
257
609
|
|
|
610
|
+
it("passes an explicit idle timeout through to the adapter", async () => {
|
|
611
|
+
let capturedIdleTimeoutMs: number | undefined;
|
|
612
|
+
const deps = {
|
|
613
|
+
run: mock((opts) => {
|
|
614
|
+
capturedIdleTimeoutMs = opts.idleTimeoutMs;
|
|
615
|
+
return makeFakeHarnessRun(makeSuccessResult());
|
|
616
|
+
}),
|
|
617
|
+
startMcpIoServer: mock(async () => createFakeMcpHandle()),
|
|
618
|
+
createTaskFileIO: mock(() => createFakeIO()),
|
|
619
|
+
};
|
|
620
|
+
|
|
621
|
+
await runAgentStep(makeArgs({ entry: { idleTimeoutMs: 250 } }), deps);
|
|
622
|
+
expect(capturedIdleTimeoutMs).toBe(250);
|
|
623
|
+
});
|
|
624
|
+
|
|
258
625
|
it("does not start MCP server when io is false", async () => {
|
|
259
626
|
const createTaskFileIO = mock(() => createFakeIO());
|
|
260
627
|
const deps = {
|
|
261
|
-
|
|
628
|
+
run: mock(() => makeFakeHarnessRun(makeSuccessResult())),
|
|
262
629
|
startMcpIoServer: mock(async () => createFakeMcpHandle()),
|
|
263
630
|
createTaskFileIO,
|
|
264
631
|
};
|
|
@@ -284,7 +651,7 @@ describe("runAgentStep", () => {
|
|
|
284
651
|
const mcpHandle = createFakeMcpHandle(["custom-artifact.md", "agent-result.md"]);
|
|
285
652
|
const createTaskFileIO = mock(() => createFakeIO());
|
|
286
653
|
const deps = {
|
|
287
|
-
|
|
654
|
+
run: mock(() => makeFakeHarnessRun(makeSuccessResult())),
|
|
288
655
|
startMcpIoServer: mock(async () => mcpHandle),
|
|
289
656
|
createTaskFileIO,
|
|
290
657
|
};
|