@prismatic-io/lux 0.0.2-preview.25 → 0.0.2-preview.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/answerers/claude-code/index.d.ts +3 -3
- package/lib/answerers/claude-code/index.d.ts.map +1 -1
- package/lib/answerers/claude-code/index.js +3 -2
- package/lib/answerers/claude-code/index.js.map +1 -1
- package/lib/answerers/persona/index.d.ts +3 -3
- package/lib/answerers/persona/index.d.ts.map +1 -1
- package/lib/answerers/persona/index.js +3 -2
- package/lib/answerers/persona/index.js.map +1 -1
- package/lib/answerers/terminal/index.d.ts +3 -3
- package/lib/answerers/terminal/index.d.ts.map +1 -1
- package/lib/answerers/terminal/index.js +3 -2
- package/lib/answerers/terminal/index.js.map +1 -1
- package/lib/assertions/core/command-exits-zero.d.ts +1 -1
- package/lib/assertions/core/command-exits-zero.d.ts.map +1 -1
- package/lib/assertions/core/command-exits-zero.js +4 -3
- package/lib/assertions/core/command-exits-zero.js.map +1 -1
- package/lib/assertions/rubric/index.d.ts +2 -2
- package/lib/assertions/rubric/index.d.ts.map +1 -1
- package/lib/assertions/rubric/index.js +4 -3
- package/lib/assertions/rubric/index.js.map +1 -1
- package/lib/authoring.d.ts +1 -0
- package/lib/authoring.d.ts.map +1 -1
- package/lib/authoring.js +1 -0
- package/lib/authoring.js.map +1 -1
- package/lib/cli/output-schemas.d.ts +2 -2
- package/lib/core/duration.d.ts +5 -0
- package/lib/core/duration.d.ts.map +1 -0
- package/lib/core/duration.js +49 -0
- package/lib/core/duration.js.map +1 -0
- package/lib/core/experiment.d.ts +2 -2
- package/lib/core/experiment.d.ts.map +1 -1
- package/lib/core/experiment.js +2 -1
- package/lib/core/experiment.js.map +1 -1
- package/lib/core/index.d.ts +1 -0
- package/lib/core/index.d.ts.map +1 -1
- package/lib/core/index.js +1 -0
- package/lib/core/index.js.map +1 -1
- package/lib/drivers/antigravity/config.d.ts +3 -3
- package/lib/drivers/antigravity/config.d.ts.map +1 -1
- package/lib/drivers/antigravity/config.js +2 -1
- package/lib/drivers/antigravity/config.js.map +1 -1
- package/lib/drivers/antigravity/index.d.ts +4 -4
- package/lib/drivers/antigravity/index.js +1 -1
- package/lib/drivers/antigravity/index.js.map +1 -1
- package/lib/drivers/claude-code/index.d.ts +14 -14
- package/lib/drivers/claude-code/index.d.ts.map +1 -1
- package/lib/drivers/claude-code/index.js +10 -9
- package/lib/drivers/claude-code/index.js.map +1 -1
- package/lib/drivers/codex/config.d.ts +3 -3
- package/lib/drivers/codex/config.d.ts.map +1 -1
- package/lib/drivers/codex/config.js +2 -1
- package/lib/drivers/codex/config.js.map +1 -1
- package/lib/drivers/codex/index.d.ts +4 -4
- package/lib/drivers/codex/index.d.ts.map +1 -1
- package/lib/drivers/codex/index.js +4 -3
- package/lib/drivers/codex/index.js.map +1 -1
- package/lib/drivers/copilot/config.d.ts +6 -6
- package/lib/drivers/copilot/config.d.ts.map +1 -1
- package/lib/drivers/copilot/config.js +3 -2
- package/lib/drivers/copilot/config.js.map +1 -1
- package/lib/drivers/copilot/index.d.ts +8 -8
- package/lib/drivers/copilot/index.d.ts.map +1 -1
- package/lib/drivers/copilot/index.js +3 -2
- package/lib/drivers/copilot/index.js.map +1 -1
- package/lib/drivers/cursor/config.d.ts +2 -2
- package/lib/drivers/cursor/config.d.ts.map +1 -1
- package/lib/drivers/cursor/config.js +3 -2
- package/lib/drivers/cursor/config.js.map +1 -1
- package/lib/drivers/cursor/index.d.ts +4 -4
- package/lib/drivers/cursor/index.d.ts.map +1 -1
- package/lib/drivers/cursor/index.js +3 -2
- package/lib/drivers/cursor/index.js.map +1 -1
- package/lib/drivers/grok-build/config.d.ts +9 -9
- package/lib/drivers/grok-build/config.d.ts.map +1 -1
- package/lib/drivers/grok-build/config.js +4 -3
- package/lib/drivers/grok-build/config.js.map +1 -1
- package/lib/drivers/grok-build/index.d.ts +12 -12
- package/lib/drivers/grok-build/index.d.ts.map +1 -1
- package/lib/drivers/grok-build/index.js +5 -4
- package/lib/drivers/grok-build/index.js.map +1 -1
- package/lib/drivers/mcp/index.d.ts +6 -6
- package/lib/drivers/mcp/index.d.ts.map +1 -1
- package/lib/drivers/mcp/index.js +9 -8
- package/lib/drivers/mcp/index.js.map +1 -1
- package/lib/drivers/subprocess/index.d.ts +3 -3
- package/lib/drivers/subprocess/index.d.ts.map +1 -1
- package/lib/drivers/subprocess/index.js +4 -3
- package/lib/drivers/subprocess/index.js.map +1 -1
- package/lib/index.d.ts +1 -0
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -0
- package/lib/index.js.map +1 -1
- package/lib/optimization/reflective.d.ts +1 -1
- package/lib/optimization/reflective.d.ts.map +1 -1
- package/lib/optimization/reflective.js +3 -2
- package/lib/optimization/reflective.js.map +1 -1
- package/lib/orchestrator/candidates.d.ts.map +1 -1
- package/lib/orchestrator/candidates.js +3 -2
- package/lib/orchestrator/candidates.js.map +1 -1
- package/lib/orchestrator/experiment-contracts.d.ts +1 -1
- package/lib/orchestrator/experiment-loader.d.ts +1 -1
- package/lib/orchestrator/experiment-report.d.ts +1 -1
- package/lib/orchestrator/experiment-runs.d.ts +1 -1
- package/lib/orchestrator/grading.js +1 -1
- package/lib/orchestrator/grading.js.map +1 -1
- package/lib/orchestrator/rubric-loader.js +1 -1
- package/lib/orchestrator/rubric-loader.js.map +1 -1
- package/package.json +1 -1
- package/skills/lux-answerer/SKILL.md +2 -2
- package/src/answerers/claude-code/index.ts +3 -2
- package/src/answerers/persona/index.ts +3 -2
- package/src/answerers/terminal/index.ts +3 -2
- package/src/assertions/core/command-exits-zero.ts +4 -3
- package/src/assertions/rubric/index.ts +4 -3
- package/src/authoring.ts +1 -0
- package/src/core/duration.ts +52 -0
- package/src/core/experiment.ts +2 -1
- package/src/core/index.ts +1 -0
- package/src/drivers/antigravity/config.ts +2 -1
- package/src/drivers/antigravity/index.ts +1 -1
- package/src/drivers/claude-code/index.ts +10 -9
- package/src/drivers/codex/config.ts +2 -1
- package/src/drivers/codex/index.ts +4 -3
- package/src/drivers/copilot/config.ts +3 -2
- package/src/drivers/copilot/index.ts +4 -3
- package/src/drivers/cursor/config.ts +3 -2
- package/src/drivers/cursor/index.ts +4 -3
- package/src/drivers/grok-build/README.md +2 -2
- package/src/drivers/grok-build/config.ts +4 -3
- package/src/drivers/grok-build/index.ts +6 -5
- package/src/drivers/mcp/index.ts +9 -8
- package/src/drivers/subprocess/index.ts +4 -3
- package/src/index.ts +1 -0
- package/src/optimization/reflective.ts +3 -2
- package/src/orchestrator/candidates.ts +3 -2
- package/src/orchestrator/grading.ts +1 -1
- package/src/orchestrator/rubric-loader.ts +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rubric-loader.js","sourceRoot":"","sources":["../../src/orchestrator/rubric-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,eAAe,EAAyB,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAElG,MAAM,0BAA0B,GAAG;IACjC,SAAS;IACT,UAAU;IACV,OAAO;IACP,iBAAiB;IACjB,
|
|
1
|
+
{"version":3,"file":"rubric-loader.js","sourceRoot":"","sources":["../../src/orchestrator/rubric-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,eAAe,EAAyB,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAElG,MAAM,0BAA0B,GAAG;IACjC,SAAS;IACT,UAAU;IACV,OAAO;IACP,iBAAiB;IACjB,SAAS;IACT,gBAAgB;CACR,CAAC;AACX,MAAM,yBAAyB,GAAG,CAAC,CAAC,WAAW,CAAC;IAC9C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;CACjC,CAAC,CAAC;AACH,MAAM,wBAAwB,GAAG,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;AAEnD;;;GAGG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,KAAK,EAAE,QAAgB,EAA6B,EAAE;IACxF,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/B,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,KAAK,OAAO,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,8DAA8D,IAAI,EAAE,CAAC,CAAC;IACxF,CAAC;IACD,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,OAAO,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,yCAAyC,CAAC,CAAC;IAC1F,CAAC;IACD,MAAM,GAAG,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,sBAAsB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACrD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,KAAK,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CACb,yBAAyB,IAAI,kDAAkD,CAChF,CAAC;IACJ,CAAC;IACD,MAAM,aAAa,GAAG,yBAAyB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC;IACtE,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QAClE,IAAI,SAAS,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CACb,yBAAyB,IAAI,+DAA+D,SAAS,CAAC,IAAI,GAAG,CAC9G,CAAC;QACJ,CAAC;QACD,MAAM,YAAY,GAAG,wBAAwB,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;QACnF,MAAM,KAAK,GAAG,YAAY;YACxB,CAAC,CAAC,0BAA0B,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,YAAY,CAAC;YAC/D,CAAC,CAAC,SAAS,CAAC;QACd,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CACb,yBAAyB,IAAI,2CAA2C,KAAK,GAAG,CACjF,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,KAAK,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACrF,CAAC;QACD,IACE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,MAAM;YACpC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,GAAG;YACpC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,GAAG,OAAO;YAC5C,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,GAAG,OAAO;YAC3C,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,GAAG,OAAO;YAC/C,MAAM,CAAC,IAAI,CAAC,kBAAkB,GAAG,MAAM,EACvC,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,4CAA4C,CAAC,CAAC;QAC7F,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@prismatic-io/lux",
|
|
3
|
-
"version": "0.0.2-preview.
|
|
3
|
+
"version": "0.0.2-preview.26",
|
|
4
4
|
"description": "Coding-agent evaluation and improvement with deterministic assertions, optional LLM judges, and first-class human-in-the-loop runs.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"homepage": "https://github.com/prismatic-io/lux",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: lux-answerer
|
|
3
|
-
version: 0.0.2-preview.
|
|
3
|
+
version: 0.0.2-preview.26
|
|
4
4
|
description: Play the persona for a running Lux orchestrator that uses the claude-code answerer. Read structured events on stdout, decide answers from persona + context, write JSON to its answer channel.
|
|
5
5
|
user-invocable: false
|
|
6
6
|
allowed-tools: Bash, Read
|
|
@@ -128,7 +128,7 @@ The persona is the single source of truth. When deciding an answer:
|
|
|
128
128
|
### Answer wait timed out (no event is emitted)
|
|
129
129
|
|
|
130
130
|
If no reply lands on the answer channel within the answerer's wait budget
|
|
131
|
-
(`
|
|
131
|
+
(`maxAnswerWait`, default 10 minutes), the pending answer is
|
|
132
132
|
rejected and the run fails — there is no further stdout event to
|
|
133
133
|
watch for. Separately, the driver can hit its own idle window if the
|
|
134
134
|
agent stalls mid-LLM-call. Either way, inspect the run dir
|
|
@@ -13,6 +13,7 @@ import { join } from "node:path";
|
|
|
13
13
|
import { createInterface } from "node:readline";
|
|
14
14
|
import { temporaryDirectory } from "tempy";
|
|
15
15
|
import { z } from "zod";
|
|
16
|
+
import { DurationSchema, durationMs } from "../../core/duration.js";
|
|
16
17
|
import {
|
|
17
18
|
type Answer,
|
|
18
19
|
type Answerer,
|
|
@@ -33,7 +34,7 @@ export const ClaudeCodeAnswererConfigSchema = z
|
|
|
33
34
|
/** Where to put the FIFO. Defaults to a fresh tmp dir per run. */
|
|
34
35
|
fifoDir: z.string().optional(),
|
|
35
36
|
/** How long to wait for a single answer to come back, in ms. */
|
|
36
|
-
|
|
37
|
+
maxAnswerWait: DurationSchema.default("10 minutes"),
|
|
37
38
|
/** Inject a custom emitter for tests; defaults to process.stdout. */
|
|
38
39
|
emit: z.custom<AnswererEmitter>(isFunction).optional(),
|
|
39
40
|
/**
|
|
@@ -118,7 +119,7 @@ class ClaudeCodeAnswererImpl implements Answerer {
|
|
|
118
119
|
return await read(
|
|
119
120
|
this.fifoPath,
|
|
120
121
|
interrupt,
|
|
121
|
-
this.config.
|
|
122
|
+
durationMs(this.config.maxAnswerWait),
|
|
122
123
|
this.pendingAborts,
|
|
123
124
|
ctx.abortSignal,
|
|
124
125
|
);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { z } from "zod";
|
|
3
|
+
import { DurationSchema, durationMs } from "../../core/duration.js";
|
|
3
4
|
import {
|
|
4
5
|
type Answer,
|
|
5
6
|
type Answerer,
|
|
@@ -35,7 +36,7 @@ export const PersonaAnswererConfigSchema = z
|
|
|
35
36
|
/** Reasoning effort for providers that support it. Codex defaults to low. */
|
|
36
37
|
reasoningEffort: ReasoningEffortSchema.optional(),
|
|
37
38
|
/** Hard timeout per answer. Default 120s. */
|
|
38
|
-
|
|
39
|
+
timeout: DurationSchema.default("2 minutes"),
|
|
39
40
|
/** Replace the generated persona system prompt entirely. */
|
|
40
41
|
systemPromptOverride: z.string().optional(),
|
|
41
42
|
})
|
|
@@ -284,7 +285,7 @@ class PersonaAnswererImpl implements Answerer {
|
|
|
284
285
|
command: this.config.command,
|
|
285
286
|
model: this.config.model,
|
|
286
287
|
reasoningEffort: this.config.reasoningEffort,
|
|
287
|
-
timeoutMs: this.config.
|
|
288
|
+
timeoutMs: durationMs(this.config.timeout),
|
|
288
289
|
system,
|
|
289
290
|
prompt,
|
|
290
291
|
signal: ctx.abortSignal,
|
|
@@ -2,12 +2,13 @@ import { stdin, stdout } from "node:process";
|
|
|
2
2
|
import { createInterface, type Interface } from "node:readline/promises";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { defineAnswerer } from "../../core/answerer.js";
|
|
5
|
+
import { DurationSchema, durationMs } from "../../core/duration.js";
|
|
5
6
|
import type { Answer, Answerer, AnswererContext, Interrupt } from "../../core/index.js";
|
|
6
7
|
import { approveAnswer } from "../shared.js";
|
|
7
8
|
|
|
8
9
|
export const TerminalAnswererConfigSchema = z
|
|
9
10
|
.object({
|
|
10
|
-
|
|
11
|
+
maxAnswerWait: DurationSchema.default("10 minutes"),
|
|
11
12
|
})
|
|
12
13
|
.strict();
|
|
13
14
|
|
|
@@ -58,7 +59,7 @@ class TerminalAnswererImpl implements Answerer {
|
|
|
58
59
|
private readonly maxAnswerWaitMs: number;
|
|
59
60
|
|
|
60
61
|
constructor(config: TerminalAnswererConfig) {
|
|
61
|
-
this.maxAnswerWaitMs = config.
|
|
62
|
+
this.maxAnswerWaitMs = durationMs(config.maxAnswerWait);
|
|
62
63
|
}
|
|
63
64
|
|
|
64
65
|
async prepare(): Promise<void> {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { resolve } from "node:path";
|
|
2
2
|
import { z } from "zod";
|
|
3
|
+
import { DurationSchema, durationMs } from "../../core/duration.js";
|
|
3
4
|
import { type AssertionImpl, exec, hasRunArtifactEvidenceBoundary } from "../../core/index.js";
|
|
4
5
|
import {
|
|
5
6
|
baseAssertionFields,
|
|
@@ -22,7 +23,7 @@ export const CommandExitsZeroConfigSchema = z
|
|
|
22
23
|
* environment, including credentials.
|
|
23
24
|
*/
|
|
24
25
|
inheritEnv: z.union([z.array(z.string().min(1)), z.literal(true)]).default([]),
|
|
25
|
-
|
|
26
|
+
timeout: DurationSchema.default("2 minutes"),
|
|
26
27
|
...baseAssertionFields,
|
|
27
28
|
})
|
|
28
29
|
.strict();
|
|
@@ -57,7 +58,7 @@ export const commandExitsZeroAssertion: AssertionImpl<CommandExitsZeroConfig> =
|
|
|
57
58
|
cwd: root,
|
|
58
59
|
env: { ...parentEnv, ...config.env },
|
|
59
60
|
shell: true,
|
|
60
|
-
timeoutMs: config.
|
|
61
|
+
timeoutMs: durationMs(config.timeout),
|
|
61
62
|
maxBuffer: 4_000,
|
|
62
63
|
...(ctx?.abortSignal ? { signal: ctx.abortSignal } : {}),
|
|
63
64
|
});
|
|
@@ -73,7 +74,7 @@ export const commandExitsZeroAssertion: AssertionImpl<CommandExitsZeroConfig> =
|
|
|
73
74
|
}
|
|
74
75
|
const passed = result.exitCode === 0 && !result.timedOut;
|
|
75
76
|
const comment = result.timedOut
|
|
76
|
-
? `command timed out after ${config.
|
|
77
|
+
? `command timed out after ${durationMs(config.timeout)}ms`
|
|
77
78
|
: `'${config.command}' exited ${result.exitCode}${result.signal ? ` (signal ${result.signal})` : ""}`;
|
|
78
79
|
return {
|
|
79
80
|
passed,
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { DurationSchema, durationMs } from "../../core/duration.js";
|
|
2
3
|
import {
|
|
3
4
|
type AssertionImpl,
|
|
4
5
|
type AssertionResult,
|
|
@@ -72,7 +73,7 @@ export const RubricConfigSchema = z
|
|
|
72
73
|
/** Reasoning effort for providers that support it. Codex defaults to low. */
|
|
73
74
|
reasoningEffort: ReasoningEffortSchema.optional(),
|
|
74
75
|
/** Hard timeout for the judge call. Default 120s. */
|
|
75
|
-
|
|
76
|
+
timeout: DurationSchema.default("2 minutes"),
|
|
76
77
|
/** Override the default judge prompt template. */
|
|
77
78
|
promptOverride: z.string().max(120_000).optional(),
|
|
78
79
|
/** Cap on file content included in the judge prompt. Default 4000. */
|
|
@@ -232,7 +233,7 @@ const judgeRubric = async (
|
|
|
232
233
|
command: runtime.command,
|
|
233
234
|
model: runtime.model,
|
|
234
235
|
reasoningEffort: runtime.reasoningEffort,
|
|
235
|
-
timeoutMs: config.
|
|
236
|
+
timeoutMs: durationMs(config.timeout),
|
|
236
237
|
system: SYSTEM_PROMPT,
|
|
237
238
|
prompt,
|
|
238
239
|
signal: context?.abortSignal,
|
|
@@ -255,7 +256,7 @@ export const createRubricAssertion = (
|
|
|
255
256
|
evaluate: async (rawConfig, run, ctx) => {
|
|
256
257
|
// Grading passes a validated copy, but direct programmatic callers may
|
|
257
258
|
// hand evaluate() a raw config — re-parse so the schema defaults
|
|
258
|
-
// (model, command,
|
|
259
|
+
// (model, command, timeout, ...) apply either way.
|
|
259
260
|
const config = RubricConfigSchema.parse(rawConfig);
|
|
260
261
|
const prompt = await buildJudgePrompt(config, run);
|
|
261
262
|
return judgeRubric(config, prompt, rubricRuntime(config, harness), ctx);
|
package/src/authoring.ts
CHANGED
|
@@ -133,6 +133,7 @@ export {
|
|
|
133
133
|
type ReportedDriverMetrics,
|
|
134
134
|
ReportedDriverMetricsSchema,
|
|
135
135
|
} from "./core/driver-metrics.js";
|
|
136
|
+
export { DurationSchema, durationMs } from "./core/duration.js";
|
|
136
137
|
export { type EventHandler, TypedEmitter } from "./core/emitter.js";
|
|
137
138
|
export { type EnvironmentDeclaration, EnvironmentDeclarationSchema } from "./core/environment.js";
|
|
138
139
|
export {
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
|
|
3
|
+
const UNITS: Record<string, number> = {
|
|
4
|
+
ms: 1,
|
|
5
|
+
millisecond: 1,
|
|
6
|
+
milliseconds: 1,
|
|
7
|
+
s: 1000,
|
|
8
|
+
second: 1000,
|
|
9
|
+
seconds: 1000,
|
|
10
|
+
m: 60_000,
|
|
11
|
+
minute: 60_000,
|
|
12
|
+
minutes: 60_000,
|
|
13
|
+
h: 3_600_000,
|
|
14
|
+
hour: 3_600_000,
|
|
15
|
+
hours: 3_600_000,
|
|
16
|
+
d: 86_400_000,
|
|
17
|
+
day: 86_400_000,
|
|
18
|
+
days: 86_400_000,
|
|
19
|
+
};
|
|
20
|
+
const MAX_TIMER_MS = 2_147_483_647;
|
|
21
|
+
const INVALID_DURATION =
|
|
22
|
+
'Expected a positive duration such as "20 minutes", "1.5 hours", or "500 ms", up to 2147483647 milliseconds';
|
|
23
|
+
|
|
24
|
+
/** Explicit units prevent ambiguous numeric configuration and Node timer overflow. */
|
|
25
|
+
export const durationMs = (value: string): number => {
|
|
26
|
+
if (value.length > 80) throw new Error(INVALID_DURATION);
|
|
27
|
+
const match =
|
|
28
|
+
/^(\d+(?:\.\d+)?)\s*(ms|milliseconds?|s|seconds?|m|minutes?|h|hours?|d|days?)$/i.exec(
|
|
29
|
+
value.trim(),
|
|
30
|
+
);
|
|
31
|
+
const amount = match?.[1];
|
|
32
|
+
const multiplier = match?.[2] ? UNITS[match[2].toLowerCase()] : undefined;
|
|
33
|
+
if (!amount || !multiplier) throw new Error(INVALID_DURATION);
|
|
34
|
+
const [whole, fraction = ""] = amount.split(".");
|
|
35
|
+
const scale = 10n ** BigInt(fraction.length);
|
|
36
|
+
const numerator = BigInt(`${whole}${fraction}`) * BigInt(multiplier);
|
|
37
|
+
const milliseconds = numerator / scale;
|
|
38
|
+
if (numerator % scale !== 0n || milliseconds <= 0n || milliseconds > BigInt(MAX_TIMER_MS)) {
|
|
39
|
+
throw new Error(INVALID_DURATION);
|
|
40
|
+
}
|
|
41
|
+
return Number(milliseconds);
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
// Keep strings in validated configs: schema validation may run again during
|
|
45
|
+
// experiment preparation or replay. Conversion belongs at the timer boundary.
|
|
46
|
+
export const DurationSchema = z.string().superRefine((value, ctx) => {
|
|
47
|
+
try {
|
|
48
|
+
durationMs(value);
|
|
49
|
+
} catch {
|
|
50
|
+
ctx.addIssue({ code: "custom", message: INVALID_DURATION });
|
|
51
|
+
}
|
|
52
|
+
});
|
package/src/core/experiment.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { DriverMetricDescriptorSchema } from "./driver-metrics.js";
|
|
3
|
+
import { DurationSchema } from "./duration.js";
|
|
3
4
|
import { ConfidenceConfigSchema } from "./statistics.js";
|
|
4
5
|
import { TokenUsageSchema } from "./usage.js";
|
|
5
6
|
import { formatZodIssues } from "./zod.js";
|
|
@@ -46,7 +47,7 @@ export const FileSetSubjectSchema = z
|
|
|
46
47
|
* External paths and URLs are rejected by experiment preflight.
|
|
47
48
|
*/
|
|
48
49
|
args: z.array(z.string()).default([]),
|
|
49
|
-
|
|
50
|
+
timeout: DurationSchema.default("2 minutes"),
|
|
50
51
|
/**
|
|
51
52
|
* Validation launches a host process against model-generated source.
|
|
52
53
|
* Authors must explicitly acknowledge that trust boundary. Commands
|
package/src/core/index.ts
CHANGED
|
@@ -10,6 +10,7 @@ export * from "./case.js";
|
|
|
10
10
|
export * from "./conversation-turn.js";
|
|
11
11
|
export * from "./driver.js";
|
|
12
12
|
export * from "./driver-metrics.js";
|
|
13
|
+
export * from "./duration.js";
|
|
13
14
|
export * from "./emitter.js";
|
|
14
15
|
export * from "./environment.js";
|
|
15
16
|
export * from "./events.js";
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { DurationSchema } from "../../core/duration.js";
|
|
2
3
|
import { antigravityCatalog, applyCatalogEffort } from "../../core/index.js";
|
|
3
4
|
|
|
4
5
|
export const AntigravityDriverConfigSchema = z
|
|
@@ -23,7 +24,7 @@ export const AntigravityDriverConfigSchema = z
|
|
|
23
24
|
cwd: z.string().optional(),
|
|
24
25
|
env: z.record(z.string(), z.string()).optional(),
|
|
25
26
|
extraArgs: z.array(z.string()).optional(),
|
|
26
|
-
|
|
27
|
+
idleTimeout: DurationSchema.default("10 minutes"),
|
|
27
28
|
maxLineBytes: z.number().positive().default(10_000_000),
|
|
28
29
|
})
|
|
29
30
|
.strict()
|
|
@@ -58,7 +58,7 @@ class AntigravityDriver implements AgentDriver {
|
|
|
58
58
|
`The task workspace is ${JSON.stringify(this.cwd)}. Treat that directory as the current directory for this task and create task artifacts there.\n\n${prompt}`,
|
|
59
59
|
),
|
|
60
60
|
closeStdinAfterPrompt: true,
|
|
61
|
-
|
|
61
|
+
idleTimeout: this.config.idleTimeout,
|
|
62
62
|
maxLineBytes: this.config.maxLineBytes,
|
|
63
63
|
maxInterrupts: 1,
|
|
64
64
|
});
|
|
@@ -3,6 +3,7 @@ import { homedir } from "node:os";
|
|
|
3
3
|
import { dirname, join, resolve, sep } from "node:path";
|
|
4
4
|
import { temporaryDirectory } from "tempy";
|
|
5
5
|
import { z } from "zod";
|
|
6
|
+
import { DurationSchema, durationMs } from "../../core/duration.js";
|
|
6
7
|
import {
|
|
7
8
|
type AgentDriver,
|
|
8
9
|
type Answer,
|
|
@@ -108,19 +109,19 @@ export const ClaudeCodeDriverConfigSchema = z
|
|
|
108
109
|
*/
|
|
109
110
|
textQuestionFallback: z.boolean().default(false),
|
|
110
111
|
/** Maximum silence between recognized protocol events; excludes answerer wait time. */
|
|
111
|
-
|
|
112
|
+
idleTimeout: DurationSchema.default("10 minutes"),
|
|
112
113
|
/** Optional elapsed-time cap per defer-resume CLI invocation, regardless of activity. */
|
|
113
|
-
|
|
114
|
+
turnTimeout: DurationSchema.optional(),
|
|
114
115
|
/** Cap on outstanding interrupts before the driver aborts. */
|
|
115
116
|
maxInterrupts: z.number().positive().default(20),
|
|
116
117
|
})
|
|
117
118
|
.strict()
|
|
118
119
|
.superRefine((config, ctx) => {
|
|
119
|
-
if (config.
|
|
120
|
+
if (config.turnTimeout !== undefined && config.interactionMode !== "defer-resume") {
|
|
120
121
|
ctx.addIssue({
|
|
121
122
|
code: "custom",
|
|
122
|
-
path: ["
|
|
123
|
-
message: "
|
|
123
|
+
path: ["turnTimeout"],
|
|
124
|
+
message: "turnTimeout requires defer-resume mode",
|
|
124
125
|
});
|
|
125
126
|
}
|
|
126
127
|
if (config.interactionMode === "defer-resume") {
|
|
@@ -596,7 +597,7 @@ class ClaudeCodeDriverImpl implements AgentDriver {
|
|
|
596
597
|
parseEvent: makeParseEvent(this.config, cliVersion),
|
|
597
598
|
formatAnswer,
|
|
598
599
|
formatPrompt,
|
|
599
|
-
|
|
600
|
+
idleTimeout: this.config.idleTimeout,
|
|
600
601
|
maxInterrupts: this.config.maxInterrupts,
|
|
601
602
|
});
|
|
602
603
|
for (const queued of this.pendingAttach) queued.attach(inner);
|
|
@@ -752,10 +753,10 @@ class ClaudeCodeDriverImpl implements AgentDriver {
|
|
|
752
753
|
...(answer ? { LUX_CLAUDE_DEFERRED_ANSWER: JSON.stringify(answer) } : {}),
|
|
753
754
|
},
|
|
754
755
|
...input,
|
|
755
|
-
idleTimeoutMs: this.config.
|
|
756
|
-
...(this.config.
|
|
756
|
+
idleTimeoutMs: durationMs(this.config.idleTimeout),
|
|
757
|
+
...(this.config.turnTimeout === undefined
|
|
757
758
|
? {}
|
|
758
|
-
: { turnTimeoutMs: this.config.
|
|
759
|
+
: { turnTimeoutMs: durationMs(this.config.turnTimeout) }),
|
|
759
760
|
signal: AbortSignal.any([ctx.abortSignal, this.deferredAbort.signal]),
|
|
760
761
|
parse: (line) => {
|
|
761
762
|
try {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { DurationSchema } from "../../core/duration.js";
|
|
2
3
|
import { applyCatalogEffort, codexCatalog, DEFAULT_CODEX_MODEL } from "../../core/index.js";
|
|
3
4
|
|
|
4
5
|
export type CodexConfigValue =
|
|
@@ -51,7 +52,7 @@ export const CodexDriverConfigSchema = z
|
|
|
51
52
|
isolation: IsolationSchema.optional(),
|
|
52
53
|
maxInterrupts: z.number().positive().default(20),
|
|
53
54
|
maxLineBytes: z.number().positive().default(10_000_000),
|
|
54
|
-
|
|
55
|
+
idleTimeout: DurationSchema.default("10 minutes"),
|
|
55
56
|
})
|
|
56
57
|
.strict()
|
|
57
58
|
.superRefine((config, ctx) => {
|
|
@@ -3,6 +3,7 @@ import { homedir } from "node:os";
|
|
|
3
3
|
import { dirname, join } from "node:path";
|
|
4
4
|
import { createInterface } from "node:readline";
|
|
5
5
|
import { temporaryDirectory } from "tempy";
|
|
6
|
+
import { durationMs } from "../../core/duration.js";
|
|
6
7
|
import {
|
|
7
8
|
type AgentDriver,
|
|
8
9
|
type Answer,
|
|
@@ -654,8 +655,8 @@ class CodexAppServerDriverImpl implements AgentDriver {
|
|
|
654
655
|
this.idleTimer = setTimeout(() => {
|
|
655
656
|
this.idleTimer = null;
|
|
656
657
|
if (this.interrupts.size > 0) return;
|
|
657
|
-
this.fail(`no event for ${this.config.
|
|
658
|
-
}, this.config.
|
|
658
|
+
this.fail(`no event for ${durationMs(this.config.idleTimeout)}ms`, "idle-timeout");
|
|
659
|
+
}, durationMs(this.config.idleTimeout));
|
|
659
660
|
}
|
|
660
661
|
|
|
661
662
|
private clearIdleTimer(): void {
|
|
@@ -718,7 +719,7 @@ class CodexExecDriverImpl implements AgentDriver {
|
|
|
718
719
|
parseEvent: makeParseEvent(this.config, cliVersion),
|
|
719
720
|
formatPrompt: (prompt) => prompt,
|
|
720
721
|
closeStdinAfterPrompt: true,
|
|
721
|
-
|
|
722
|
+
idleTimeout: this.config.idleTimeout,
|
|
722
723
|
maxLineBytes: this.config.maxLineBytes,
|
|
723
724
|
maxInterrupts: 1,
|
|
724
725
|
});
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { DurationSchema } from "../../core/duration.js";
|
|
2
3
|
import { applyCatalogEffort, copilotCatalog } from "../../core/index.js";
|
|
3
4
|
|
|
4
5
|
export const CopilotDriverConfigSchema = z
|
|
@@ -23,8 +24,8 @@ export const CopilotDriverConfigSchema = z
|
|
|
23
24
|
extraArgs: z.array(z.string()).optional(),
|
|
24
25
|
cwd: z.string().optional(),
|
|
25
26
|
env: z.record(z.string(), z.string()).optional(),
|
|
26
|
-
|
|
27
|
-
|
|
27
|
+
startupTimeout: DurationSchema.default("30 seconds"),
|
|
28
|
+
idleTimeout: DurationSchema.default("10 minutes"),
|
|
28
29
|
maxInterrupts: z.number().int().positive().default(20),
|
|
29
30
|
maxLineBytes: z.number().int().positive().default(10_000_000),
|
|
30
31
|
maxQueuedBytes: z.number().int().positive().default(20_000_000),
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { realpath } from "node:fs/promises";
|
|
2
|
+
import { durationMs } from "../../core/duration.js";
|
|
2
3
|
import {
|
|
3
4
|
type AgentDriver,
|
|
4
5
|
type Answer,
|
|
@@ -130,7 +131,7 @@ class CopilotDriver implements AgentDriver {
|
|
|
130
131
|
|
|
131
132
|
private request(method: string, params: RpcRecord): Promise<RpcRecord> {
|
|
132
133
|
if (!this.transport) throw new Error("Copilot ACP has not started");
|
|
133
|
-
return this.transport.request(method, params, this.config.
|
|
134
|
+
return this.transport.request(method, params, durationMs(this.config.startupTimeout));
|
|
134
135
|
}
|
|
135
136
|
|
|
136
137
|
private async run(prompt: string): Promise<void> {
|
|
@@ -209,10 +210,10 @@ class CopilotDriver implements AgentDriver {
|
|
|
209
210
|
this.idleTimer = setTimeout(
|
|
210
211
|
() =>
|
|
211
212
|
this.fail(
|
|
212
|
-
new Error(`Copilot emitted no event for ${this.config.
|
|
213
|
+
new Error(`Copilot emitted no event for ${durationMs(this.config.idleTimeout)}ms`),
|
|
213
214
|
"idle-timeout",
|
|
214
215
|
),
|
|
215
|
-
this.config.
|
|
216
|
+
durationMs(this.config.idleTimeout),
|
|
216
217
|
);
|
|
217
218
|
}
|
|
218
219
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { DurationSchema } from "../../core/duration.js";
|
|
2
3
|
import { cursorCatalog } from "../../core/index.js";
|
|
3
4
|
|
|
4
5
|
export const CursorDriverConfigSchema = z
|
|
@@ -9,8 +10,8 @@ export const CursorDriverConfigSchema = z
|
|
|
9
10
|
mode: z.enum(["agent", "plan", "ask"]).default("agent"),
|
|
10
11
|
cwd: z.string().optional(),
|
|
11
12
|
env: z.record(z.string(), z.string()).optional(),
|
|
12
|
-
|
|
13
|
-
|
|
13
|
+
startupTimeout: DurationSchema.default("30 seconds"),
|
|
14
|
+
idleTimeout: DurationSchema.default("10 minutes"),
|
|
14
15
|
maxInterrupts: z.number().int().positive().default(20),
|
|
15
16
|
maxLineBytes: z.number().int().positive().default(10_000_000),
|
|
16
17
|
maxQueuedBytes: z.number().int().positive().default(20_000_000),
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { realpath } from "node:fs/promises";
|
|
2
|
+
import { durationMs } from "../../core/duration.js";
|
|
2
3
|
import {
|
|
3
4
|
type AgentDriver,
|
|
4
5
|
type Answer,
|
|
@@ -136,7 +137,7 @@ class CursorDriver implements AgentDriver {
|
|
|
136
137
|
|
|
137
138
|
private request(method: string, params: RpcRecord): Promise<RpcRecord> {
|
|
138
139
|
if (!this.transport) throw new Error("Cursor ACP has not started");
|
|
139
|
-
return this.transport.request(method, params, this.config.
|
|
140
|
+
return this.transport.request(method, params, durationMs(this.config.startupTimeout));
|
|
140
141
|
}
|
|
141
142
|
|
|
142
143
|
private async run(prompt: string): Promise<void> {
|
|
@@ -237,10 +238,10 @@ class CursorDriver implements AgentDriver {
|
|
|
237
238
|
this.idleTimer = setTimeout(
|
|
238
239
|
() =>
|
|
239
240
|
this.fail(
|
|
240
|
-
new Error(`Cursor emitted no event for ${this.config.
|
|
241
|
+
new Error(`Cursor emitted no event for ${durationMs(this.config.idleTimeout)}ms`),
|
|
241
242
|
"idle-timeout",
|
|
242
243
|
),
|
|
243
|
-
this.config.
|
|
244
|
+
durationMs(this.config.idleTimeout),
|
|
244
245
|
);
|
|
245
246
|
}
|
|
246
247
|
|
|
@@ -64,8 +64,8 @@ text and invalid fields still deny the operation. If the agent offers no matchin
|
|
|
64
64
|
one-operation option, Lux cancels and reports an error rather than widening the
|
|
65
65
|
grant. Native auto decisions are handled inside Grok and are not Lux interrupts.
|
|
66
66
|
|
|
67
|
-
Permission reviews have a separate `
|
|
68
|
-
default), while ordinary inactivity uses `
|
|
67
|
+
Permission reviews have a separate `permissionTimeout` (150 seconds by
|
|
68
|
+
default), while ordinary inactivity uses `idleTimeout` (10 minutes). Startup
|
|
69
69
|
RPCs time out after 30 seconds. Cancelled turns, process failures, unanswered
|
|
70
70
|
reviews, and turns ending with pending approvals cannot count as successful runs.
|
|
71
71
|
A completed turn can still contain denied tool calls: assert the expected files
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { DurationSchema } from "../../core/duration.js";
|
|
2
3
|
import { applyCatalogEffort, grokBuildCatalog } from "../../core/index.js";
|
|
3
4
|
|
|
4
5
|
export const GrokBuildDriverConfigSchema = z
|
|
@@ -11,9 +12,9 @@ export const GrokBuildDriverConfigSchema = z
|
|
|
11
12
|
sandbox: z.enum(["workspace", "read-only", "strict"]).default("workspace"),
|
|
12
13
|
cwd: z.string().optional(),
|
|
13
14
|
env: z.record(z.string(), z.string()).optional(),
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
startupTimeout: DurationSchema.default("30 seconds"),
|
|
16
|
+
idleTimeout: DurationSchema.default("10 minutes"),
|
|
17
|
+
permissionTimeout: DurationSchema.default("150 seconds"),
|
|
17
18
|
maxInterrupts: z.number().int().positive().default(20),
|
|
18
19
|
maxLineBytes: z.number().int().positive().default(10_000_000),
|
|
19
20
|
maxQueuedBytes: z.number().int().positive().default(20_000_000),
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { realpath } from "node:fs/promises";
|
|
2
|
+
import { durationMs } from "../../core/duration.js";
|
|
2
3
|
import {
|
|
3
4
|
type AgentDriver,
|
|
4
5
|
type Answer,
|
|
@@ -143,7 +144,7 @@ class GrokBuildDriver implements AgentDriver {
|
|
|
143
144
|
|
|
144
145
|
private request(method: string, params: RpcRecord): Promise<RpcRecord> {
|
|
145
146
|
if (!this.transport) throw new Error("Grok.build ACP has not started");
|
|
146
|
-
return this.transport.request(method, params, this.config.
|
|
147
|
+
return this.transport.request(method, params, durationMs(this.config.startupTimeout));
|
|
147
148
|
}
|
|
148
149
|
|
|
149
150
|
private async run(prompt: string): Promise<void> {
|
|
@@ -198,10 +199,10 @@ class GrokBuildDriver implements AgentDriver {
|
|
|
198
199
|
this.interrupts.delete(interruptId);
|
|
199
200
|
this.fail(
|
|
200
201
|
new Error(
|
|
201
|
-
`Grok.build permission review timed out after ${this.config.
|
|
202
|
+
`Grok.build permission review timed out after ${durationMs(this.config.permissionTimeout)}ms`,
|
|
202
203
|
),
|
|
203
204
|
);
|
|
204
|
-
}, this.config.
|
|
205
|
+
}, durationMs(this.config.permissionTimeout));
|
|
205
206
|
this.interrupts.set(interruptId, { id, params, timer });
|
|
206
207
|
clearTimeout(this.idleTimer);
|
|
207
208
|
void this.emitter
|
|
@@ -243,10 +244,10 @@ class GrokBuildDriver implements AgentDriver {
|
|
|
243
244
|
this.idleTimer = setTimeout(
|
|
244
245
|
() =>
|
|
245
246
|
this.fail(
|
|
246
|
-
new Error(`Grok.build emitted no event for ${this.config.
|
|
247
|
+
new Error(`Grok.build emitted no event for ${durationMs(this.config.idleTimeout)}ms`),
|
|
247
248
|
"idle-timeout",
|
|
248
249
|
),
|
|
249
|
-
this.config.
|
|
250
|
+
durationMs(this.config.idleTimeout),
|
|
250
251
|
);
|
|
251
252
|
}
|
|
252
253
|
|
package/src/drivers/mcp/index.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
2
2
|
import { getDefaultEnvironment } from "@modelcontextprotocol/sdk/client/stdio.js";
|
|
3
3
|
import { z } from "zod";
|
|
4
|
+
import { DurationSchema, durationMs as parseDuration } from "../../core/duration.js";
|
|
4
5
|
import {
|
|
5
6
|
type AgentDriver,
|
|
6
7
|
type Answer,
|
|
@@ -72,9 +73,9 @@ export const McpProbeDriverConfigSchema = z
|
|
|
72
73
|
/** The steps to run. Defaults to a single `tools/list` capture. */
|
|
73
74
|
probe: z.array(ProbeStepSchema).default([{ op: "list-tools" }]),
|
|
74
75
|
/** Timeout for the initialize handshake. */
|
|
75
|
-
|
|
76
|
+
startupTimeout: DurationSchema.default("30 seconds"),
|
|
76
77
|
/** Timeout for each `tools/list` / `tools/call` request. */
|
|
77
|
-
|
|
78
|
+
stepTimeout: DurationSchema.default("1 minute"),
|
|
78
79
|
/** Safety cap for a paginated `tools/list` operation. */
|
|
79
80
|
maxListPages: z.number().int().positive().default(100),
|
|
80
81
|
})
|
|
@@ -211,8 +212,8 @@ class McpProbeDriverImpl implements AgentDriver {
|
|
|
211
212
|
|
|
212
213
|
try {
|
|
213
214
|
await client.connect(transport, {
|
|
214
|
-
timeout: this.config.
|
|
215
|
-
maxTotalTimeout: this.config.
|
|
215
|
+
timeout: parseDuration(this.config.startupTimeout),
|
|
216
|
+
maxTotalTimeout: parseDuration(this.config.startupTimeout),
|
|
216
217
|
signal: ctx.abortSignal,
|
|
217
218
|
});
|
|
218
219
|
} catch (err) {
|
|
@@ -261,8 +262,8 @@ class McpProbeDriverImpl implements AgentDriver {
|
|
|
261
262
|
throw new Error(`tools/list exceeded ${this.config.maxListPages} pages`);
|
|
262
263
|
}
|
|
263
264
|
const result = await client.listTools(cursor ? { cursor } : undefined, {
|
|
264
|
-
timeout: this.config.
|
|
265
|
-
maxTotalTimeout: this.config.
|
|
265
|
+
timeout: parseDuration(this.config.stepTimeout),
|
|
266
|
+
maxTotalTimeout: parseDuration(this.config.stepTimeout),
|
|
266
267
|
...(this.abortSignal ? { signal: this.abortSignal } : {}),
|
|
267
268
|
});
|
|
268
269
|
tools.push(...(result.tools as ListedTool[]));
|
|
@@ -285,8 +286,8 @@ class McpProbeDriverImpl implements AgentDriver {
|
|
|
285
286
|
{ name: step.tool, arguments: step.args },
|
|
286
287
|
undefined,
|
|
287
288
|
{
|
|
288
|
-
timeout: this.config.
|
|
289
|
-
maxTotalTimeout: this.config.
|
|
289
|
+
timeout: parseDuration(this.config.stepTimeout),
|
|
290
|
+
maxTotalTimeout: parseDuration(this.config.stepTimeout),
|
|
290
291
|
...(this.abortSignal ? { signal: this.abortSignal } : {}),
|
|
291
292
|
},
|
|
292
293
|
);
|