@dailephd/my-dev-kit-lab 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +26 -10
- package/benchmarks/contracts/benchmark-project-profiles.json +14 -14
- package/benchmarks/contracts/warm-index-benchmark-cases.json +764 -0
- package/dist/src/agents/adapters/claudeAdapter.js +47 -4
- package/dist/src/agents/adapters/codexAdapter.js +64 -5
- package/dist/src/cli/help.js +22 -1
- package/dist/src/commands/runExperimentDescribeCommand.js +6 -1
- package/dist/src/commands/runExperimentRunCommand.js +87 -5
- package/dist/src/commands/runWarmIndexCampaignPresentation.js +64 -0
- package/dist/src/core/runMeasuredCommand.js +49 -10
- package/dist/src/evaluation/benchmarkMetadata.js +191 -1
- package/dist/src/evaluation/projectComplexity.js +1 -1
- package/dist/src/evaluation/readEvaluationCases.js +7 -1
- package/dist/src/evaluation/types.js +1 -1
- package/dist/src/experiments/plugins/warmIndexReuse/agentEvaluation.js +233 -0
- package/dist/src/experiments/plugins/warmIndexReuse/campaignPresets.js +83 -0
- package/dist/src/experiments/plugins/warmIndexReuse/config.js +51 -0
- package/dist/src/experiments/plugins/warmIndexReuse/fakeAgentEvaluation.js +6 -118
- package/dist/src/experiments/plugins/warmIndexReuse/index.js +3 -1
- package/dist/src/experiments/plugins/warmIndexReuse/metrics.js +35 -11
- package/dist/src/experiments/plugins/warmIndexReuse/plugin.js +50 -14
- package/dist/src/experiments/plugins/warmIndexReuse/realAgentPrompt.js +52 -0
- package/dist/src/experiments/plugins/warmIndexReuse/selection.js +35 -0
- package/dist/src/gallery/index.js +1 -0
- package/dist/src/gallery/writeGalleryManifest.js +5 -1
- package/dist/src/gallery/writeWarmIndexCampaignGallery.js +116 -0
- package/dist/src/plots/buildWarmIndexPlotData.js +21 -2
- package/dist/src/report/experiments/buildPluginExperimentReport.js +35 -8
- package/dist/src/report/experiments/buildWarmIndexReuseReport.js +138 -12
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +35 -12
- package/dist/src/report/experiments/renderWarmIndexReuseHtml.js +45 -14
- package/docs/METRICS.md +33 -3
- package/package.json +1 -1
|
@@ -2,6 +2,20 @@ import { runCliAgent } from "./codexAdapter.js";
|
|
|
2
2
|
import { applyPromptToCommandTemplate } from "../runAgentPrompt.js";
|
|
3
3
|
import { parseAgentTokenUsage } from "../parseAgentTokenUsage.js";
|
|
4
4
|
import { runMeasuredCommand } from "../../core/runMeasuredCommand.js";
|
|
5
|
+
// v0.5.2 Batch 2 -- frozen stdin invocation for the real-agent evaluation transport. Every flag
|
|
6
|
+
// here is planner-owned; --bare is intentionally not used (it changes auth behavior and still
|
|
7
|
+
// exposes Bash/read/edit tools unless separately constrained).
|
|
8
|
+
const CLAUDE_STDIN_ARGS = [
|
|
9
|
+
"--restricted",
|
|
10
|
+
"-p",
|
|
11
|
+
"--output-format",
|
|
12
|
+
"json",
|
|
13
|
+
"--no-session-persistence",
|
|
14
|
+
"--tools",
|
|
15
|
+
"",
|
|
16
|
+
"--disallowedTools",
|
|
17
|
+
"mcp__*"
|
|
18
|
+
];
|
|
5
19
|
export const claudeAdapter = {
|
|
6
20
|
id: "claude",
|
|
7
21
|
displayName: "Claude",
|
|
@@ -21,17 +35,46 @@ export const claudeAdapter = {
|
|
|
21
35
|
return check.ok;
|
|
22
36
|
},
|
|
23
37
|
buildCommand(request) {
|
|
38
|
+
if (request.promptTransport === "stdin" && request.commandTemplate) {
|
|
39
|
+
throw new Error("claude agent: stdin prompt transport cannot be combined with a command template.");
|
|
40
|
+
}
|
|
24
41
|
if (request.commandTemplate) {
|
|
25
42
|
return applyPromptToCommandTemplate(request.commandTemplate, request.promptText);
|
|
26
43
|
}
|
|
44
|
+
if (request.promptTransport === "stdin") {
|
|
45
|
+
return { command: "claude", args: [...CLAUDE_STDIN_ARGS], stdinText: request.promptText };
|
|
46
|
+
}
|
|
27
47
|
return { command: "claude", args: ["-p", request.promptText] };
|
|
28
48
|
},
|
|
29
49
|
async runPrompt(request) {
|
|
30
50
|
return runCliAgent(request, this);
|
|
31
51
|
},
|
|
32
52
|
parseTokenUsage: parseAgentTokenUsage,
|
|
33
|
-
parseFinalAnswer
|
|
34
|
-
const trimmed = text.trim();
|
|
35
|
-
return { finalAnswerText: trimmed, finalAnswerParseStatus: trimmed ? "parsed" : "empty" };
|
|
36
|
-
}
|
|
53
|
+
parseFinalAnswer: parseClaudeFinalAnswer
|
|
37
54
|
};
|
|
55
|
+
/**
|
|
56
|
+
* Parses a single JSON result envelope (`{ "result": "..." }`) when stdout is valid JSON, never
|
|
57
|
+
* the raw envelope itself. Falls back to the legacy plain-text behavior when stdout is not valid
|
|
58
|
+
* JSON, so existing argv-mode text output keeps working unchanged.
|
|
59
|
+
*/
|
|
60
|
+
export function parseClaudeFinalAnswer(text) {
|
|
61
|
+
const trimmedInput = text.trim();
|
|
62
|
+
if (trimmedInput) {
|
|
63
|
+
try {
|
|
64
|
+
const parsed = JSON.parse(trimmedInput);
|
|
65
|
+
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
66
|
+
const payload = parsed;
|
|
67
|
+
if (typeof payload.result === "string") {
|
|
68
|
+
const resultText = payload.result.trim();
|
|
69
|
+
return { finalAnswerText: resultText, finalAnswerParseStatus: resultText ? "parsed" : "empty" };
|
|
70
|
+
}
|
|
71
|
+
// Valid JSON object without a string result: empty, never the raw envelope.
|
|
72
|
+
return { finalAnswerText: "", finalAnswerParseStatus: "empty" };
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
// Not valid JSON; fall through to the legacy plain-text behavior below.
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return { finalAnswerText: trimmedInput, finalAnswerParseStatus: trimmedInput ? "parsed" : "empty" };
|
|
80
|
+
}
|
|
@@ -1,6 +1,17 @@
|
|
|
1
1
|
import { runMeasuredCommand } from "../../core/runMeasuredCommand.js";
|
|
2
2
|
import { applyPromptToCommandTemplate } from "../runAgentPrompt.js";
|
|
3
3
|
import { parseAgentTokenUsage } from "../parseAgentTokenUsage.js";
|
|
4
|
+
// v0.5.2 Batch 2 -- frozen stdin invocation for the real-agent evaluation transport. Every flag
|
|
5
|
+
// here is planner-owned; do not add sandbox/model/approval-policy flags in this batch.
|
|
6
|
+
const CODEX_STDIN_ARGS = [
|
|
7
|
+
"exec",
|
|
8
|
+
"--json",
|
|
9
|
+
"--ephemeral",
|
|
10
|
+
"--skip-git-repo-check",
|
|
11
|
+
"--ignore-user-config",
|
|
12
|
+
"--ignore-rules",
|
|
13
|
+
"-"
|
|
14
|
+
];
|
|
4
15
|
export const codexAdapter = {
|
|
5
16
|
id: "codex",
|
|
6
17
|
displayName: "Codex",
|
|
@@ -20,20 +31,67 @@ export const codexAdapter = {
|
|
|
20
31
|
return check.ok;
|
|
21
32
|
},
|
|
22
33
|
buildCommand(request) {
|
|
34
|
+
if (request.promptTransport === "stdin" && request.commandTemplate) {
|
|
35
|
+
throw new Error("codex agent: stdin prompt transport cannot be combined with a command template.");
|
|
36
|
+
}
|
|
23
37
|
if (request.commandTemplate) {
|
|
24
38
|
return applyPromptToCommandTemplate(request.commandTemplate, request.promptText);
|
|
25
39
|
}
|
|
40
|
+
if (request.promptTransport === "stdin") {
|
|
41
|
+
return { command: "codex", args: [...CODEX_STDIN_ARGS], stdinText: request.promptText };
|
|
42
|
+
}
|
|
26
43
|
return { command: "codex", args: ["exec", "--json", request.promptText] };
|
|
27
44
|
},
|
|
28
45
|
async runPrompt(request) {
|
|
29
46
|
return runCliAgent(request, this);
|
|
30
47
|
},
|
|
31
48
|
parseTokenUsage: parseAgentTokenUsage,
|
|
32
|
-
parseFinalAnswer
|
|
33
|
-
const trimmed = text.trim();
|
|
34
|
-
return { finalAnswerText: trimmed, finalAnswerParseStatus: trimmed ? "parsed" : "empty" };
|
|
35
|
-
}
|
|
49
|
+
parseFinalAnswer: parseCodexFinalAnswer
|
|
36
50
|
};
|
|
51
|
+
/**
|
|
52
|
+
* Recognizes a Codex JSONL event stream (at least one line parses to an object with a string
|
|
53
|
+
* `type`) and, when recognized, uses the LAST `item.completed` `agent_message` text as the final
|
|
54
|
+
* answer -- never the raw JSONL. Falls back to the legacy plain-text behavior when no event-shaped
|
|
55
|
+
* line is found, so non-JSONL Codex output (and existing tests) keep working unchanged.
|
|
56
|
+
*/
|
|
57
|
+
export function parseCodexFinalAnswer(text) {
|
|
58
|
+
let sawEventLine = false;
|
|
59
|
+
let lastAgentMessageText = null;
|
|
60
|
+
for (const line of text.split(/\r?\n/)) {
|
|
61
|
+
const trimmedLine = line.trim();
|
|
62
|
+
if (!trimmedLine)
|
|
63
|
+
continue;
|
|
64
|
+
let parsed;
|
|
65
|
+
try {
|
|
66
|
+
parsed = JSON.parse(trimmedLine);
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
// Agent output can mix JSONL with incidental non-JSON noise; ignore malformed lines.
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
|
|
73
|
+
continue;
|
|
74
|
+
const event = parsed;
|
|
75
|
+
if (typeof event.type !== "string")
|
|
76
|
+
continue;
|
|
77
|
+
sawEventLine = true;
|
|
78
|
+
if (event.type === "item.completed") {
|
|
79
|
+
const item = event.item;
|
|
80
|
+
if (item && typeof item === "object" && item.type === "agent_message") {
|
|
81
|
+
const itemText = item.text;
|
|
82
|
+
if (typeof itemText === "string") {
|
|
83
|
+
lastAgentMessageText = itemText;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
if (sawEventLine) {
|
|
89
|
+
const finalAnswerText = lastAgentMessageText ?? "";
|
|
90
|
+
return { finalAnswerText, finalAnswerParseStatus: finalAnswerText ? "parsed" : "empty" };
|
|
91
|
+
}
|
|
92
|
+
const trimmed = text.trim();
|
|
93
|
+
return { finalAnswerText: trimmed, finalAnswerParseStatus: trimmed ? "parsed" : "empty" };
|
|
94
|
+
}
|
|
37
95
|
export async function runCliAgent(request, adapter) {
|
|
38
96
|
const started = Date.now();
|
|
39
97
|
const command = adapter.buildCommand(request);
|
|
@@ -74,7 +132,8 @@ export async function runCliAgent(request, adapter) {
|
|
|
74
132
|
cwd: request.commandTemplate?.cwd ?? request.cwd,
|
|
75
133
|
outDir: request.outDir,
|
|
76
134
|
env: request.env,
|
|
77
|
-
timeoutMs: request.timeoutMs
|
|
135
|
+
timeoutMs: request.timeoutMs,
|
|
136
|
+
stdinText: command.stdinText
|
|
78
137
|
});
|
|
79
138
|
const ended = Date.now();
|
|
80
139
|
const combinedOutput = `${measured.stdout}\n${measured.stderr}`;
|
package/dist/src/cli/help.js
CHANGED
|
@@ -157,8 +157,29 @@ export function renderExperimentRunHelp() {
|
|
|
157
157
|
" --kit-command <command> my-dev-kit command used to build one index per benchmark",
|
|
158
158
|
" project and retrieve per task",
|
|
159
159
|
" (default: npx @dailephd/my-dev-kit@latest)",
|
|
160
|
-
"
|
|
160
|
+
" Legacy/default mode uses the deterministic fake agent only;",
|
|
161
161
|
" agent options below are not accepted by this plugin.",
|
|
162
|
+
" --campaign <preset> Runs the selected real-agent campaign preset: codex-full,",
|
|
163
|
+
" claude-full, codex-timeout-isolation. Selects Codex or",
|
|
164
|
+
" Claude as the preset's single provider and requires",
|
|
165
|
+
" --include-real-agents. The preset owns the bundled",
|
|
166
|
+
" production corpus and project profiles; explicit",
|
|
167
|
+
" --cases/--project-profiles and --target are rejected in",
|
|
168
|
+
" campaign mode. --case and --benchmark-project narrow the",
|
|
169
|
+
" preset's case set. --kit-command and --timeout-ms are",
|
|
170
|
+
" accepted. The report distinguishes warm-index",
|
|
171
|
+
" infrastructure status from agent/provider outcome status;",
|
|
172
|
+
" provider token telemetry may be unavailable and is never",
|
|
173
|
+
" replaced by a context-size estimate. A campaign run",
|
|
174
|
+
" automatically produces the report JSON/text/HTML, the",
|
|
175
|
+
" four warm-index SVG plots, a best-effort report PNG",
|
|
176
|
+
" screenshot, and a gallery manifest/index. A missing",
|
|
177
|
+
" Playwright/browser runtime makes screenshot capture skip",
|
|
178
|
+
" without invalidating the campaign; an unexpected",
|
|
179
|
+
" screenshot failure makes the command unsuccessful after",
|
|
180
|
+
" preserving the report, plots, and gallery. Legacy",
|
|
181
|
+
" non-campaign warm-index runs retain their existing",
|
|
182
|
+
" report-only behavior.",
|
|
162
183
|
"",
|
|
163
184
|
"context-strategy-comparison only:",
|
|
164
185
|
" --agents <list> Comma-separated agent ids (default: fake-agent)",
|
|
@@ -145,11 +145,16 @@ function describeExamples(plugin) {
|
|
|
145
145
|
// Only advertise agent-matrix options for plugins that declare them.
|
|
146
146
|
const agentOptions = hasConfigField(plugin, "agents") ? " --agents fake-agent --complexities short" : "";
|
|
147
147
|
const screenshotOption = hasConfigField(plugin, "agents") ? " --no-screenshot" : "";
|
|
148
|
-
|
|
148
|
+
const examples = [
|
|
149
149
|
`my-dev-kit-lab experiment describe --experiment ${plugin.metadata.id}`,
|
|
150
150
|
`my-dev-kit-lab experiment run --experiment ${plugin.metadata.id}${agentOptions}`,
|
|
151
151
|
`my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --target "Z:\\Users\\newuser\\Projects\\my-dev-kit-v1"${agentOptions}${screenshotOption}`
|
|
152
152
|
];
|
|
153
|
+
// Only advertise campaign examples for plugins that declare the campaignPreset config field.
|
|
154
|
+
if (hasConfigField(plugin, "campaignPreset")) {
|
|
155
|
+
examples.push(`my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --campaign codex-full --include-real-agents --out <dir>`, `my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --campaign claude-full --include-real-agents --case warm-medium-complete-idempotent --out <dir>`);
|
|
156
|
+
}
|
|
157
|
+
return examples;
|
|
153
158
|
}
|
|
154
159
|
function hasConfigField(plugin, name) {
|
|
155
160
|
return (plugin.configDefinition?.fields ?? []).some((field) => field.name === name);
|
|
@@ -2,11 +2,12 @@ import path from "node:path";
|
|
|
2
2
|
import { parseAgentCommandTemplate } from "../agents/index.js";
|
|
3
3
|
import { parseAgentId } from "../agents/agentRegistry.js";
|
|
4
4
|
import { readBenchmarkProjectProfiles, readEvaluationCases } from "../evaluation/index.js";
|
|
5
|
-
import { contextStrategyComparisonPlugin, createDefaultExperimentPluginRegistry, resolveExperimentTarget, runExperiment, warmIndexReusePlugin } from "../experiments/index.js";
|
|
5
|
+
import { contextStrategyComparisonPlugin, createDefaultExperimentPluginRegistry, getWarmIndexCampaignPreset, parseWarmIndexCampaignPresetId, resolveExperimentTarget, runExperiment, warmIndexReusePlugin } from "../experiments/index.js";
|
|
6
6
|
import { buildDefaultExperimentOutputRoot } from "../experiments/outputPaths.js";
|
|
7
7
|
import { parsePromptComplexityLevel, parsePromptStrategy } from "../prompts/index.js";
|
|
8
8
|
import { writePluginExperimentReports } from "../report/index.js";
|
|
9
9
|
import { createLabExecutionContext, resolvePackageResource } from "../runtime/index.js";
|
|
10
|
+
import { runWarmIndexCampaignPresentation } from "./runWarmIndexCampaignPresentation.js";
|
|
10
11
|
// ---------------------------------------------------------------------------
|
|
11
12
|
// v0.4.6 Batch 4 -- reusable experiment-run command owner.
|
|
12
13
|
//
|
|
@@ -61,7 +62,7 @@ export async function runExperimentRunCommandFromArgs(argv, options = {}) {
|
|
|
61
62
|
plugin: registry.describe(result.pluginId),
|
|
62
63
|
outputRoot: String(result.metadata?.outputRoot ?? "")
|
|
63
64
|
});
|
|
64
|
-
|
|
65
|
+
const outputLines = [
|
|
65
66
|
`Experiment: ${result.pluginId}`,
|
|
66
67
|
`Run ID: ${result.runId}`,
|
|
67
68
|
`Status: ${result.status}`,
|
|
@@ -71,8 +72,51 @@ export async function runExperimentRunCommandFromArgs(argv, options = {}) {
|
|
|
71
72
|
`Output: ${String(result.metadata?.outputRoot ?? "")}`,
|
|
72
73
|
`Report JSON: ${reports.outputPaths.jsonPath}`,
|
|
73
74
|
`Report HTML: ${reports.outputPaths.htmlPath}`
|
|
74
|
-
]
|
|
75
|
-
|
|
75
|
+
];
|
|
76
|
+
// Automatic presentation (report already produced above) applies only to warm-index-reuse
|
|
77
|
+
// campaign runs; legacy fake-agent warm-index runs and context-strategy-comparison stay
|
|
78
|
+
// report-only, unchanged.
|
|
79
|
+
let screenshotFailed = false;
|
|
80
|
+
if (args.experimentId === warmIndexReusePlugin.metadata.id && args.config.campaignPreset) {
|
|
81
|
+
const executionArtifactPath = readMetadataString(result.metadata?.executionArtifactPath);
|
|
82
|
+
const campaignAgentId = readMetadataString(result.metadata?.campaignAgentId);
|
|
83
|
+
if (!executionArtifactPath) {
|
|
84
|
+
throw new Error("Warm-index campaign presentation requires an execution artifact path on the completed run.");
|
|
85
|
+
}
|
|
86
|
+
if (campaignAgentId !== "codex" && campaignAgentId !== "claude") {
|
|
87
|
+
throw new Error(`Warm-index campaign presentation requires a codex or claude campaign agent id on the completed run; received ${String(campaignAgentId)}.`);
|
|
88
|
+
}
|
|
89
|
+
const presentation = await runWarmIndexCampaignPresentation({
|
|
90
|
+
outputRoot: String(result.metadata?.outputRoot ?? ""),
|
|
91
|
+
reportPaths: reports.outputPaths,
|
|
92
|
+
executionArtifactPath,
|
|
93
|
+
campaignPreset: args.config.campaignPreset,
|
|
94
|
+
agentId: campaignAgentId,
|
|
95
|
+
captureScreenshot: options.presentation?.captureScreenshot
|
|
96
|
+
});
|
|
97
|
+
outputLines.push(`Plots: ${presentation.plots.artifactPaths.summaryPath}`);
|
|
98
|
+
if (presentation.screenshot.status === "captured") {
|
|
99
|
+
outputLines.push(`Screenshot: ${presentation.screenshot.pngPath}`);
|
|
100
|
+
}
|
|
101
|
+
else if (presentation.screenshot.status === "skipped") {
|
|
102
|
+
outputLines.push("Screenshot: skipped");
|
|
103
|
+
if (presentation.screenshot.warning)
|
|
104
|
+
outputLines.push(presentation.screenshot.warning);
|
|
105
|
+
}
|
|
106
|
+
else {
|
|
107
|
+
outputLines.push("Screenshot: failed");
|
|
108
|
+
outputLines.push(presentation.screenshot.error ?? "Screenshot capture failed.");
|
|
109
|
+
screenshotFailed = true;
|
|
110
|
+
}
|
|
111
|
+
outputLines.push(`Gallery manifest: ${presentation.gallery.manifestPath}`);
|
|
112
|
+
outputLines.push(`Gallery index: ${presentation.gallery.indexPath}`);
|
|
113
|
+
}
|
|
114
|
+
console.log(outputLines.join("\n"));
|
|
115
|
+
if (result.status === "failed")
|
|
116
|
+
return 1;
|
|
117
|
+
if (screenshotFailed)
|
|
118
|
+
return 1;
|
|
119
|
+
return 0;
|
|
76
120
|
}
|
|
77
121
|
catch (error) {
|
|
78
122
|
if (process.env.DEBUG) {
|
|
@@ -101,6 +145,7 @@ export function parseRunExperimentArgs(argv) {
|
|
|
101
145
|
let requireAgents;
|
|
102
146
|
let includeRealAgents;
|
|
103
147
|
let kitCommand;
|
|
148
|
+
let campaignPreset;
|
|
104
149
|
const commandTemplates = {};
|
|
105
150
|
for (let index = 0; index < argv.length; index += 1) {
|
|
106
151
|
const arg = argv[index];
|
|
@@ -161,6 +206,9 @@ export function parseRunExperimentArgs(argv) {
|
|
|
161
206
|
else if (arg === "--kit-command") {
|
|
162
207
|
kitCommand = readRequiredValue(argv, ++index, "--kit-command");
|
|
163
208
|
}
|
|
209
|
+
else if (arg === "--campaign") {
|
|
210
|
+
campaignPreset = parseWarmIndexCampaignPresetId(readRequiredValue(argv, ++index, "--campaign"));
|
|
211
|
+
}
|
|
164
212
|
else if (arg === "--no-screenshot") {
|
|
165
213
|
// The plugin-aware report path does not capture screenshots yet; accept this
|
|
166
214
|
// flag so smoke commands can share the legacy demo option set.
|
|
@@ -175,6 +223,20 @@ export function parseRunExperimentArgs(argv) {
|
|
|
175
223
|
if (kitCommand !== undefined && experimentId !== warmIndexReusePlugin.metadata.id) {
|
|
176
224
|
throw new Error(`--kit-command is only supported for --experiment ${warmIndexReusePlugin.metadata.id}.`);
|
|
177
225
|
}
|
|
226
|
+
if (campaignPreset !== undefined) {
|
|
227
|
+
if (experimentId !== warmIndexReusePlugin.metadata.id) {
|
|
228
|
+
throw new Error(`--campaign is only supported for --experiment ${warmIndexReusePlugin.metadata.id}.`);
|
|
229
|
+
}
|
|
230
|
+
if (targetPath !== undefined) {
|
|
231
|
+
throw new Error("--campaign cannot be combined with --target; campaigns run only against bundled synthetic benchmark projects.");
|
|
232
|
+
}
|
|
233
|
+
if (casesPath !== undefined) {
|
|
234
|
+
throw new Error("--campaign cannot be combined with --cases; the selected preset owns the production corpus.");
|
|
235
|
+
}
|
|
236
|
+
if (projectProfilesPath !== undefined) {
|
|
237
|
+
throw new Error("--campaign cannot be combined with --project-profiles; the selected preset owns the project profiles.");
|
|
238
|
+
}
|
|
239
|
+
}
|
|
178
240
|
return {
|
|
179
241
|
experimentId,
|
|
180
242
|
targetPath,
|
|
@@ -193,7 +255,8 @@ export function parseRunExperimentArgs(argv) {
|
|
|
193
255
|
requireAgents,
|
|
194
256
|
includeRealAgents,
|
|
195
257
|
commandTemplates: Object.keys(commandTemplates).length > 0 ? commandTemplates : undefined,
|
|
196
|
-
kitCommand
|
|
258
|
+
kitCommand,
|
|
259
|
+
campaignPreset
|
|
197
260
|
})
|
|
198
261
|
};
|
|
199
262
|
}
|
|
@@ -232,6 +295,9 @@ async function loadPluginInputs(args, toolRoot, context) {
|
|
|
232
295
|
if (!validation.valid || !validation.config) {
|
|
233
296
|
throw new Error(`Invalid warm index reuse config: ${validation.errors.join("; ")}`);
|
|
234
297
|
}
|
|
298
|
+
if (args.config.campaignPreset) {
|
|
299
|
+
return loadCampaignCasesAndProjectProfiles(args.config.campaignPreset, toolRoot, context);
|
|
300
|
+
}
|
|
235
301
|
return loadCasesAndProjectProfiles(args, toolRoot, context);
|
|
236
302
|
}
|
|
237
303
|
return undefined;
|
|
@@ -252,6 +318,19 @@ async function loadCasesAndProjectProfiles(args, toolRoot, context) {
|
|
|
252
318
|
});
|
|
253
319
|
return { cases, projectProfiles, env: process.env };
|
|
254
320
|
}
|
|
321
|
+
// Campaign resource resolution (v0.5.2 Batch 1): the preset owns both bundled resource paths;
|
|
322
|
+
// explicit --cases/--project-profiles are already rejected for campaign mode by argument parsing.
|
|
323
|
+
async function loadCampaignCasesAndProjectProfiles(campaignPresetId, toolRoot, context) {
|
|
324
|
+
const preset = getWarmIndexCampaignPreset(campaignPresetId);
|
|
325
|
+
const projectProfilesPath = resolvePackageResource(context, preset.projectProfilesResourcePath);
|
|
326
|
+
const casesPath = resolvePackageResource(context, preset.casesResourcePath);
|
|
327
|
+
const projectProfiles = await readBenchmarkProjectProfiles(projectProfilesPath, toolRoot);
|
|
328
|
+
const cases = await readEvaluationCases(casesPath, toolRoot, {
|
|
329
|
+
projectProfiles,
|
|
330
|
+
requireProjectProfileRef: true
|
|
331
|
+
});
|
|
332
|
+
return { cases, projectProfiles, env: process.env };
|
|
333
|
+
}
|
|
255
334
|
function splitList(value) {
|
|
256
335
|
return value
|
|
257
336
|
.split(",")
|
|
@@ -272,6 +351,9 @@ function parsePositiveInteger(label, value) {
|
|
|
272
351
|
}
|
|
273
352
|
return parsed;
|
|
274
353
|
}
|
|
354
|
+
function readMetadataString(value) {
|
|
355
|
+
return typeof value === "string" && value.trim() ? value : undefined;
|
|
356
|
+
}
|
|
275
357
|
function withoutUndefined(value) {
|
|
276
358
|
return Object.fromEntries(Object.entries(value).filter(([, entryValue]) => entryValue !== undefined));
|
|
277
359
|
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { writeWarmIndexCampaignGallery } from "../gallery/writeWarmIndexCampaignGallery.js";
|
|
3
|
+
import { captureReportScreenshot } from "../screenshot/captureReportScreenshot.js";
|
|
4
|
+
import { writePlotArtifacts } from "../plots/writePlotArtifacts.js";
|
|
5
|
+
// ---------------------------------------------------------------------------
|
|
6
|
+
// v0.5.2 Batch 5 -- command-level warm-index campaign presentation owner.
|
|
7
|
+
//
|
|
8
|
+
// This is the single orchestrator for: existing report -> existing four warm-
|
|
9
|
+
// index plots -> best-effort report screenshot -> narrow warm-index campaign
|
|
10
|
+
// gallery. It reuses writePlotArtifacts, captureReportScreenshot, and
|
|
11
|
+
// writeWarmIndexCampaignGallery unchanged; it builds no new plot, screenshot,
|
|
12
|
+
// or gallery mechanism of its own. The warm-index-reuse plugin remains free of
|
|
13
|
+
// this orchestration -- it is invoked only by the command layer, and only for
|
|
14
|
+
// campaign runs.
|
|
15
|
+
// ---------------------------------------------------------------------------
|
|
16
|
+
const EXPECTED_WARM_INDEX_CHART_IDS = [
|
|
17
|
+
"warm-index-amortized-index-cost",
|
|
18
|
+
"warm-index-context-size",
|
|
19
|
+
"warm-index-correctness",
|
|
20
|
+
"warm-index-cumulative-token-usage",
|
|
21
|
+
];
|
|
22
|
+
export async function runWarmIndexCampaignPresentation(args) {
|
|
23
|
+
const plotsDir = path.join(args.outputRoot, "plots");
|
|
24
|
+
const plots = await writePlotArtifacts({ experimentDir: args.outputRoot, outDir: plotsDir });
|
|
25
|
+
const chartIds = Object.keys(plots.artifactPaths.charts);
|
|
26
|
+
const hasExactlyExpectedCharts = chartIds.length === EXPECTED_WARM_INDEX_CHART_IDS.length &&
|
|
27
|
+
EXPECTED_WARM_INDEX_CHART_IDS.every((id) => chartIds.includes(id));
|
|
28
|
+
if (!hasExactlyExpectedCharts) {
|
|
29
|
+
throw new Error(`Warm-index campaign presentation requires exactly the four warm-index plot charts (${EXPECTED_WARM_INDEX_CHART_IDS.join(", ")}); got: ${chartIds.join(", ") || "none"}.`);
|
|
30
|
+
}
|
|
31
|
+
const capture = args.captureScreenshot ?? captureReportScreenshot;
|
|
32
|
+
const pngPath = path.join(args.outputRoot, "report.png");
|
|
33
|
+
// Non-fatal by design: a missing Playwright/browser runtime is a normal, expected environment,
|
|
34
|
+
// never a reason to fail the campaign (see captureReportScreenshot's captured/skipped/failed contract).
|
|
35
|
+
const screenshot = await capture(args.reportPaths.htmlPath, pngPath);
|
|
36
|
+
const gallery = await writeWarmIndexCampaignGallery({
|
|
37
|
+
outDir: path.join(args.outputRoot, "gallery"),
|
|
38
|
+
campaignOutputRoot: args.outputRoot,
|
|
39
|
+
reportPaths: {
|
|
40
|
+
jsonPath: args.reportPaths.jsonPath,
|
|
41
|
+
htmlPath: args.reportPaths.htmlPath,
|
|
42
|
+
textPath: args.reportPaths.textPath,
|
|
43
|
+
},
|
|
44
|
+
executionArtifactPath: args.executionArtifactPath,
|
|
45
|
+
plotArtifacts: plots,
|
|
46
|
+
screenshot,
|
|
47
|
+
campaignPreset: args.campaignPreset,
|
|
48
|
+
agentId: args.agentId,
|
|
49
|
+
});
|
|
50
|
+
const warnings = [];
|
|
51
|
+
if (screenshot.status === "skipped" && screenshot.warning) {
|
|
52
|
+
warnings.push(screenshot.warning);
|
|
53
|
+
}
|
|
54
|
+
else if (screenshot.status === "failed") {
|
|
55
|
+
warnings.push(`Report screenshot capture failed: ${screenshot.error ?? "unknown error"}.`);
|
|
56
|
+
}
|
|
57
|
+
warnings.push(...plots.summary.warnings);
|
|
58
|
+
return {
|
|
59
|
+
plots,
|
|
60
|
+
screenshot,
|
|
61
|
+
gallery: { manifest: gallery.manifest, manifestPath: gallery.manifestPath, indexPath: gallery.indexPath },
|
|
62
|
+
warnings,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
@@ -41,13 +41,15 @@ export async function runMeasuredCommand(options) {
|
|
|
41
41
|
let spawnError;
|
|
42
42
|
let timedOut = false;
|
|
43
43
|
let timeout;
|
|
44
|
+
let killGraceTimeout;
|
|
45
|
+
let settled = false;
|
|
44
46
|
let child;
|
|
45
47
|
try {
|
|
46
48
|
child = spawn(executable, args, {
|
|
47
49
|
cwd: options.cwd,
|
|
48
50
|
env: { ...process.env, ...options.env },
|
|
49
51
|
shell: false,
|
|
50
|
-
stdio: ["ignore", "pipe", "pipe"]
|
|
52
|
+
stdio: options.stdinText !== undefined ? ["pipe", "pipe", "pipe"] : ["ignore", "pipe", "pipe"]
|
|
51
53
|
});
|
|
52
54
|
}
|
|
53
55
|
catch (error) {
|
|
@@ -80,6 +82,7 @@ export async function runMeasuredCommand(options) {
|
|
|
80
82
|
]).then(() => resolve(measured));
|
|
81
83
|
return;
|
|
82
84
|
}
|
|
85
|
+
// stdout/stderr are always piped in both stdio branches above.
|
|
83
86
|
child.stdout.on("data", (chunk) => {
|
|
84
87
|
stdout += String(chunk);
|
|
85
88
|
});
|
|
@@ -89,17 +92,40 @@ export async function runMeasuredCommand(options) {
|
|
|
89
92
|
child.on("error", (error) => {
|
|
90
93
|
spawnError = error.message;
|
|
91
94
|
});
|
|
92
|
-
if (options.
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
spawnError = `
|
|
96
|
-
|
|
97
|
-
|
|
95
|
+
if (options.stdinText !== undefined) {
|
|
96
|
+
// A child that exits or closes stdin before we write must not crash the process (EPIPE).
|
|
97
|
+
child.stdin?.on("error", (error) => {
|
|
98
|
+
spawnError = spawnError ?? `Failed to write to command stdin: ${error.message}`;
|
|
99
|
+
});
|
|
100
|
+
child.stdin?.write(options.stdinText, "utf8");
|
|
101
|
+
child.stdin?.end();
|
|
98
102
|
}
|
|
99
|
-
|
|
100
|
-
|
|
103
|
+
// forceTerminateProcess is fire-and-forget (it never confirms the target process tree actually
|
|
104
|
+
// exited: on Windows it spawns a detached `taskkill /T /F` and does not await or check its
|
|
105
|
+
// result). That is normally sufficient, but a grandchild process spawned through an
|
|
106
|
+
// intermediate shim (for example a Windows .cmd launcher) can occasionally outlive an
|
|
107
|
+
// already-killed intermediate process. Without a bounded fallback, a caller whose kill silently
|
|
108
|
+
// failed would await this promise forever instead of receiving the already-known timeout error.
|
|
109
|
+
const KILL_GRACE_MS = 3000;
|
|
110
|
+
const settle = async (exitCode) => {
|
|
111
|
+
if (settled)
|
|
112
|
+
return;
|
|
113
|
+
settled = true;
|
|
114
|
+
if (timeout)
|
|
101
115
|
clearTimeout(timeout);
|
|
102
|
-
|
|
116
|
+
if (killGraceTimeout)
|
|
117
|
+
clearTimeout(killGraceTimeout);
|
|
118
|
+
// The caller's `scripts/cli.ts` entry point sets `process.exitCode` rather than calling
|
|
119
|
+
// `process.exit()`, so the process only exits once Node's event loop actually drains. A
|
|
120
|
+
// still-open child stdio pipe -- the same situation the kill-grace fallback above exists for
|
|
121
|
+
// -- would otherwise keep this process alive forever even though this promise has already
|
|
122
|
+
// resolved. Detaching is safe to do unconditionally: once "close" has fired the streams are
|
|
123
|
+
// already ended, so this is a harmless no-op on the normal (non-fallback) path.
|
|
124
|
+
child.stdout?.destroy();
|
|
125
|
+
child.stderr?.destroy();
|
|
126
|
+
child.stdin?.destroy();
|
|
127
|
+
child.removeAllListeners();
|
|
128
|
+
child.unref();
|
|
103
129
|
const endedAt = new Date().toISOString();
|
|
104
130
|
const durationMs = Date.now() - started;
|
|
105
131
|
await writeArtifact(stdoutPath, stdout);
|
|
@@ -125,6 +151,19 @@ export async function runMeasuredCommand(options) {
|
|
|
125
151
|
};
|
|
126
152
|
await writeArtifact(telemetryPath, JSON.stringify(measured, null, 2));
|
|
127
153
|
resolve(measured);
|
|
154
|
+
};
|
|
155
|
+
if (options.timeoutMs !== undefined) {
|
|
156
|
+
timeout = setTimeout(() => {
|
|
157
|
+
timedOut = true;
|
|
158
|
+
spawnError = `Command timed out after ${options.timeoutMs}ms.`;
|
|
159
|
+
forceTerminateProcess(child.pid);
|
|
160
|
+
killGraceTimeout = setTimeout(() => {
|
|
161
|
+
void settle(null);
|
|
162
|
+
}, KILL_GRACE_MS);
|
|
163
|
+
}, options.timeoutMs);
|
|
164
|
+
}
|
|
165
|
+
child.on("close", (exitCode) => {
|
|
166
|
+
void settle(exitCode);
|
|
128
167
|
});
|
|
129
168
|
});
|
|
130
169
|
return result;
|