@dailephd/my-dev-kit-lab 0.5.0 → 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/CHANGELOG.md +26 -0
  2. package/README.md +26 -10
  3. package/benchmarks/contracts/benchmark-project-profiles.json +14 -14
  4. package/benchmarks/contracts/warm-index-benchmark-cases.json +764 -0
  5. package/dist/src/agents/adapters/claudeAdapter.js +47 -4
  6. package/dist/src/agents/adapters/codexAdapter.js +64 -5
  7. package/dist/src/cli/help.js +22 -1
  8. package/dist/src/commands/runExperimentDescribeCommand.js +6 -1
  9. package/dist/src/commands/runExperimentRunCommand.js +87 -5
  10. package/dist/src/commands/runWarmIndexCampaignPresentation.js +64 -0
  11. package/dist/src/core/runMeasuredCommand.js +49 -10
  12. package/dist/src/evaluation/benchmarkMetadata.js +191 -1
  13. package/dist/src/evaluation/projectComplexity.js +1 -1
  14. package/dist/src/evaluation/readEvaluationCases.js +7 -1
  15. package/dist/src/evaluation/types.js +1 -1
  16. package/dist/src/experiments/plugins/warmIndexReuse/agentEvaluation.js +233 -0
  17. package/dist/src/experiments/plugins/warmIndexReuse/campaignPresets.js +83 -0
  18. package/dist/src/experiments/plugins/warmIndexReuse/config.js +51 -0
  19. package/dist/src/experiments/plugins/warmIndexReuse/fakeAgentEvaluation.js +6 -118
  20. package/dist/src/experiments/plugins/warmIndexReuse/index.js +3 -1
  21. package/dist/src/experiments/plugins/warmIndexReuse/metrics.js +35 -11
  22. package/dist/src/experiments/plugins/warmIndexReuse/plugin.js +50 -14
  23. package/dist/src/experiments/plugins/warmIndexReuse/realAgentPrompt.js +52 -0
  24. package/dist/src/experiments/plugins/warmIndexReuse/selection.js +35 -0
  25. package/dist/src/gallery/index.js +1 -0
  26. package/dist/src/gallery/writeGalleryManifest.js +5 -1
  27. package/dist/src/gallery/writeWarmIndexCampaignGallery.js +116 -0
  28. package/dist/src/plots/buildWarmIndexPlotData.js +21 -2
  29. package/dist/src/report/experiments/buildPluginExperimentReport.js +35 -8
  30. package/dist/src/report/experiments/buildWarmIndexReuseReport.js +138 -12
  31. package/dist/src/report/experiments/renderPluginExperimentReportText.js +35 -12
  32. package/dist/src/report/experiments/renderWarmIndexReuseHtml.js +45 -14
  33. package/docs/METRICS.md +33 -3
  34. package/package.json +1 -1
@@ -2,6 +2,20 @@ import { runCliAgent } from "./codexAdapter.js";
2
2
  import { applyPromptToCommandTemplate } from "../runAgentPrompt.js";
3
3
  import { parseAgentTokenUsage } from "../parseAgentTokenUsage.js";
4
4
  import { runMeasuredCommand } from "../../core/runMeasuredCommand.js";
5
+ // v0.5.2 Batch 2 -- frozen stdin invocation for the real-agent evaluation transport. Every flag
6
+ // here is planner-owned; --bare is intentionally not used (it changes auth behavior and still
7
+ // exposes Bash/read/edit tools unless separately constrained).
8
+ const CLAUDE_STDIN_ARGS = [
9
+ "--restricted",
10
+ "-p",
11
+ "--output-format",
12
+ "json",
13
+ "--no-session-persistence",
14
+ "--tools",
15
+ "",
16
+ "--disallowedTools",
17
+ "mcp__*"
18
+ ];
5
19
  export const claudeAdapter = {
6
20
  id: "claude",
7
21
  displayName: "Claude",
@@ -21,17 +35,46 @@ export const claudeAdapter = {
21
35
  return check.ok;
22
36
  },
23
37
  buildCommand(request) {
38
+ if (request.promptTransport === "stdin" && request.commandTemplate) {
39
+ throw new Error("claude agent: stdin prompt transport cannot be combined with a command template.");
40
+ }
24
41
  if (request.commandTemplate) {
25
42
  return applyPromptToCommandTemplate(request.commandTemplate, request.promptText);
26
43
  }
44
+ if (request.promptTransport === "stdin") {
45
+ return { command: "claude", args: [...CLAUDE_STDIN_ARGS], stdinText: request.promptText };
46
+ }
27
47
  return { command: "claude", args: ["-p", request.promptText] };
28
48
  },
29
49
  async runPrompt(request) {
30
50
  return runCliAgent(request, this);
31
51
  },
32
52
  parseTokenUsage: parseAgentTokenUsage,
33
- parseFinalAnswer(text) {
34
- const trimmed = text.trim();
35
- return { finalAnswerText: trimmed, finalAnswerParseStatus: trimmed ? "parsed" : "empty" };
36
- }
53
+ parseFinalAnswer: parseClaudeFinalAnswer
37
54
  };
55
+ /**
56
+ * Parses a single JSON result envelope (`{ "result": "..." }`) when stdout is valid JSON, never
57
+ * the raw envelope itself. Falls back to the legacy plain-text behavior when stdout is not valid
58
+ * JSON, so existing argv-mode text output keeps working unchanged.
59
+ */
60
+ export function parseClaudeFinalAnswer(text) {
61
+ const trimmedInput = text.trim();
62
+ if (trimmedInput) {
63
+ try {
64
+ const parsed = JSON.parse(trimmedInput);
65
+ if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
66
+ const payload = parsed;
67
+ if (typeof payload.result === "string") {
68
+ const resultText = payload.result.trim();
69
+ return { finalAnswerText: resultText, finalAnswerParseStatus: resultText ? "parsed" : "empty" };
70
+ }
71
+ // Valid JSON object without a string result: empty, never the raw envelope.
72
+ return { finalAnswerText: "", finalAnswerParseStatus: "empty" };
73
+ }
74
+ }
75
+ catch {
76
+ // Not valid JSON; fall through to the legacy plain-text behavior below.
77
+ }
78
+ }
79
+ return { finalAnswerText: trimmedInput, finalAnswerParseStatus: trimmedInput ? "parsed" : "empty" };
80
+ }
@@ -1,6 +1,17 @@
1
1
  import { runMeasuredCommand } from "../../core/runMeasuredCommand.js";
2
2
  import { applyPromptToCommandTemplate } from "../runAgentPrompt.js";
3
3
  import { parseAgentTokenUsage } from "../parseAgentTokenUsage.js";
4
+ // v0.5.2 Batch 2 -- frozen stdin invocation for the real-agent evaluation transport. Every flag
5
+ // here is planner-owned; do not add sandbox/model/approval-policy flags in this batch.
6
+ const CODEX_STDIN_ARGS = [
7
+ "exec",
8
+ "--json",
9
+ "--ephemeral",
10
+ "--skip-git-repo-check",
11
+ "--ignore-user-config",
12
+ "--ignore-rules",
13
+ "-"
14
+ ];
4
15
  export const codexAdapter = {
5
16
  id: "codex",
6
17
  displayName: "Codex",
@@ -20,20 +31,67 @@ export const codexAdapter = {
20
31
  return check.ok;
21
32
  },
22
33
  buildCommand(request) {
34
+ if (request.promptTransport === "stdin" && request.commandTemplate) {
35
+ throw new Error("codex agent: stdin prompt transport cannot be combined with a command template.");
36
+ }
23
37
  if (request.commandTemplate) {
24
38
  return applyPromptToCommandTemplate(request.commandTemplate, request.promptText);
25
39
  }
40
+ if (request.promptTransport === "stdin") {
41
+ return { command: "codex", args: [...CODEX_STDIN_ARGS], stdinText: request.promptText };
42
+ }
26
43
  return { command: "codex", args: ["exec", "--json", request.promptText] };
27
44
  },
28
45
  async runPrompt(request) {
29
46
  return runCliAgent(request, this);
30
47
  },
31
48
  parseTokenUsage: parseAgentTokenUsage,
32
- parseFinalAnswer(text) {
33
- const trimmed = text.trim();
34
- return { finalAnswerText: trimmed, finalAnswerParseStatus: trimmed ? "parsed" : "empty" };
35
- }
49
+ parseFinalAnswer: parseCodexFinalAnswer
36
50
  };
51
+ /**
52
+ * Recognizes a Codex JSONL event stream (at least one line parses to an object with a string
53
+ * `type`) and, when recognized, uses the LAST `item.completed` `agent_message` text as the final
54
+ * answer -- never the raw JSONL. Falls back to the legacy plain-text behavior when no event-shaped
55
+ * line is found, so non-JSONL Codex output (and existing tests) keep working unchanged.
56
+ */
57
+ export function parseCodexFinalAnswer(text) {
58
+ let sawEventLine = false;
59
+ let lastAgentMessageText = null;
60
+ for (const line of text.split(/\r?\n/)) {
61
+ const trimmedLine = line.trim();
62
+ if (!trimmedLine)
63
+ continue;
64
+ let parsed;
65
+ try {
66
+ parsed = JSON.parse(trimmedLine);
67
+ }
68
+ catch {
69
+ // Agent output can mix JSONL with incidental non-JSON noise; ignore malformed lines.
70
+ continue;
71
+ }
72
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
73
+ continue;
74
+ const event = parsed;
75
+ if (typeof event.type !== "string")
76
+ continue;
77
+ sawEventLine = true;
78
+ if (event.type === "item.completed") {
79
+ const item = event.item;
80
+ if (item && typeof item === "object" && item.type === "agent_message") {
81
+ const itemText = item.text;
82
+ if (typeof itemText === "string") {
83
+ lastAgentMessageText = itemText;
84
+ }
85
+ }
86
+ }
87
+ }
88
+ if (sawEventLine) {
89
+ const finalAnswerText = lastAgentMessageText ?? "";
90
+ return { finalAnswerText, finalAnswerParseStatus: finalAnswerText ? "parsed" : "empty" };
91
+ }
92
+ const trimmed = text.trim();
93
+ return { finalAnswerText: trimmed, finalAnswerParseStatus: trimmed ? "parsed" : "empty" };
94
+ }
37
95
  export async function runCliAgent(request, adapter) {
38
96
  const started = Date.now();
39
97
  const command = adapter.buildCommand(request);
@@ -74,7 +132,8 @@ export async function runCliAgent(request, adapter) {
74
132
  cwd: request.commandTemplate?.cwd ?? request.cwd,
75
133
  outDir: request.outDir,
76
134
  env: request.env,
77
- timeoutMs: request.timeoutMs
135
+ timeoutMs: request.timeoutMs,
136
+ stdinText: command.stdinText
78
137
  });
79
138
  const ended = Date.now();
80
139
  const combinedOutput = `${measured.stdout}\n${measured.stderr}`;
@@ -157,8 +157,29 @@ export function renderExperimentRunHelp() {
157
157
  " --kit-command <command> my-dev-kit command used to build one index per benchmark",
158
158
  " project and retrieve per task",
159
159
  " (default: npx @dailephd/my-dev-kit@latest)",
160
- " Agent evidence uses the deterministic fake agent only;",
160
+ " Legacy/default mode uses the deterministic fake agent only;",
161
161
  " agent options below are not accepted by this plugin.",
162
+ " --campaign <preset> Runs the selected real-agent campaign preset: codex-full,",
163
+ " claude-full, codex-timeout-isolation. Selects Codex or",
164
+ " Claude as the preset's single provider and requires",
165
+ " --include-real-agents. The preset owns the bundled",
166
+ " production corpus and project profiles; explicit",
167
+ " --cases/--project-profiles and --target are rejected in",
168
+ " campaign mode. --case and --benchmark-project narrow the",
169
+ " preset's case set. --kit-command and --timeout-ms are",
170
+ " accepted. The report distinguishes warm-index",
171
+ " infrastructure status from agent/provider outcome status;",
172
+ " provider token telemetry may be unavailable and is never",
173
+ " replaced by a context-size estimate. A campaign run",
174
+ " automatically produces the report JSON/text/HTML, the",
175
+ " four warm-index SVG plots, a best-effort report PNG",
176
+ " screenshot, and a gallery manifest/index. A missing",
177
+ " Playwright/browser runtime makes screenshot capture skip",
178
+ " without invalidating the campaign; an unexpected",
179
+ " screenshot failure makes the command unsuccessful after",
180
+ " preserving the report, plots, and gallery. Legacy",
181
+ " non-campaign warm-index runs retain their existing",
182
+ " report-only behavior.",
162
183
  "",
163
184
  "context-strategy-comparison only:",
164
185
  " --agents <list> Comma-separated agent ids (default: fake-agent)",
@@ -145,11 +145,16 @@ function describeExamples(plugin) {
145
145
  // Only advertise agent-matrix options for plugins that declare them.
146
146
  const agentOptions = hasConfigField(plugin, "agents") ? " --agents fake-agent --complexities short" : "";
147
147
  const screenshotOption = hasConfigField(plugin, "agents") ? " --no-screenshot" : "";
148
- return [
148
+ const examples = [
149
149
  `my-dev-kit-lab experiment describe --experiment ${plugin.metadata.id}`,
150
150
  `my-dev-kit-lab experiment run --experiment ${plugin.metadata.id}${agentOptions}`,
151
151
  `my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --target "Z:\\Users\\newuser\\Projects\\my-dev-kit-v1"${agentOptions}${screenshotOption}`
152
152
  ];
153
+ // Only advertise campaign examples for plugins that declare the campaignPreset config field.
154
+ if (hasConfigField(plugin, "campaignPreset")) {
155
+ examples.push(`my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --campaign codex-full --include-real-agents --out <dir>`, `my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --campaign claude-full --include-real-agents --case warm-medium-complete-idempotent --out <dir>`);
156
+ }
157
+ return examples;
153
158
  }
154
159
  function hasConfigField(plugin, name) {
155
160
  return (plugin.configDefinition?.fields ?? []).some((field) => field.name === name);
@@ -2,11 +2,12 @@ import path from "node:path";
2
2
  import { parseAgentCommandTemplate } from "../agents/index.js";
3
3
  import { parseAgentId } from "../agents/agentRegistry.js";
4
4
  import { readBenchmarkProjectProfiles, readEvaluationCases } from "../evaluation/index.js";
5
- import { contextStrategyComparisonPlugin, createDefaultExperimentPluginRegistry, resolveExperimentTarget, runExperiment, warmIndexReusePlugin } from "../experiments/index.js";
5
+ import { contextStrategyComparisonPlugin, createDefaultExperimentPluginRegistry, getWarmIndexCampaignPreset, parseWarmIndexCampaignPresetId, resolveExperimentTarget, runExperiment, warmIndexReusePlugin } from "../experiments/index.js";
6
6
  import { buildDefaultExperimentOutputRoot } from "../experiments/outputPaths.js";
7
7
  import { parsePromptComplexityLevel, parsePromptStrategy } from "../prompts/index.js";
8
8
  import { writePluginExperimentReports } from "../report/index.js";
9
9
  import { createLabExecutionContext, resolvePackageResource } from "../runtime/index.js";
10
+ import { runWarmIndexCampaignPresentation } from "./runWarmIndexCampaignPresentation.js";
10
11
  // ---------------------------------------------------------------------------
11
12
  // v0.4.6 Batch 4 -- reusable experiment-run command owner.
12
13
  //
@@ -61,7 +62,7 @@ export async function runExperimentRunCommandFromArgs(argv, options = {}) {
61
62
  plugin: registry.describe(result.pluginId),
62
63
  outputRoot: String(result.metadata?.outputRoot ?? "")
63
64
  });
64
- console.log([
65
+ const outputLines = [
65
66
  `Experiment: ${result.pluginId}`,
66
67
  `Run ID: ${result.runId}`,
67
68
  `Status: ${result.status}`,
@@ -71,8 +72,51 @@ export async function runExperimentRunCommandFromArgs(argv, options = {}) {
71
72
  `Output: ${String(result.metadata?.outputRoot ?? "")}`,
72
73
  `Report JSON: ${reports.outputPaths.jsonPath}`,
73
74
  `Report HTML: ${reports.outputPaths.htmlPath}`
74
- ].join("\n"));
75
- return result.status === "failed" ? 1 : 0;
75
+ ];
76
+ // Automatic presentation (report already produced above) applies only to warm-index-reuse
77
+ // campaign runs; legacy fake-agent warm-index runs and context-strategy-comparison stay
78
+ // report-only, unchanged.
79
+ let screenshotFailed = false;
80
+ if (args.experimentId === warmIndexReusePlugin.metadata.id && args.config.campaignPreset) {
81
+ const executionArtifactPath = readMetadataString(result.metadata?.executionArtifactPath);
82
+ const campaignAgentId = readMetadataString(result.metadata?.campaignAgentId);
83
+ if (!executionArtifactPath) {
84
+ throw new Error("Warm-index campaign presentation requires an execution artifact path on the completed run.");
85
+ }
86
+ if (campaignAgentId !== "codex" && campaignAgentId !== "claude") {
87
+ throw new Error(`Warm-index campaign presentation requires a codex or claude campaign agent id on the completed run; received ${String(campaignAgentId)}.`);
88
+ }
89
+ const presentation = await runWarmIndexCampaignPresentation({
90
+ outputRoot: String(result.metadata?.outputRoot ?? ""),
91
+ reportPaths: reports.outputPaths,
92
+ executionArtifactPath,
93
+ campaignPreset: args.config.campaignPreset,
94
+ agentId: campaignAgentId,
95
+ captureScreenshot: options.presentation?.captureScreenshot
96
+ });
97
+ outputLines.push(`Plots: ${presentation.plots.artifactPaths.summaryPath}`);
98
+ if (presentation.screenshot.status === "captured") {
99
+ outputLines.push(`Screenshot: ${presentation.screenshot.pngPath}`);
100
+ }
101
+ else if (presentation.screenshot.status === "skipped") {
102
+ outputLines.push("Screenshot: skipped");
103
+ if (presentation.screenshot.warning)
104
+ outputLines.push(presentation.screenshot.warning);
105
+ }
106
+ else {
107
+ outputLines.push("Screenshot: failed");
108
+ outputLines.push(presentation.screenshot.error ?? "Screenshot capture failed.");
109
+ screenshotFailed = true;
110
+ }
111
+ outputLines.push(`Gallery manifest: ${presentation.gallery.manifestPath}`);
112
+ outputLines.push(`Gallery index: ${presentation.gallery.indexPath}`);
113
+ }
114
+ console.log(outputLines.join("\n"));
115
+ if (result.status === "failed")
116
+ return 1;
117
+ if (screenshotFailed)
118
+ return 1;
119
+ return 0;
76
120
  }
77
121
  catch (error) {
78
122
  if (process.env.DEBUG) {
@@ -101,6 +145,7 @@ export function parseRunExperimentArgs(argv) {
101
145
  let requireAgents;
102
146
  let includeRealAgents;
103
147
  let kitCommand;
148
+ let campaignPreset;
104
149
  const commandTemplates = {};
105
150
  for (let index = 0; index < argv.length; index += 1) {
106
151
  const arg = argv[index];
@@ -161,6 +206,9 @@ export function parseRunExperimentArgs(argv) {
161
206
  else if (arg === "--kit-command") {
162
207
  kitCommand = readRequiredValue(argv, ++index, "--kit-command");
163
208
  }
209
+ else if (arg === "--campaign") {
210
+ campaignPreset = parseWarmIndexCampaignPresetId(readRequiredValue(argv, ++index, "--campaign"));
211
+ }
164
212
  else if (arg === "--no-screenshot") {
165
213
  // The plugin-aware report path does not capture screenshots yet; accept this
166
214
  // flag so smoke commands can share the legacy demo option set.
@@ -175,6 +223,20 @@ export function parseRunExperimentArgs(argv) {
175
223
  if (kitCommand !== undefined && experimentId !== warmIndexReusePlugin.metadata.id) {
176
224
  throw new Error(`--kit-command is only supported for --experiment ${warmIndexReusePlugin.metadata.id}.`);
177
225
  }
226
+ if (campaignPreset !== undefined) {
227
+ if (experimentId !== warmIndexReusePlugin.metadata.id) {
228
+ throw new Error(`--campaign is only supported for --experiment ${warmIndexReusePlugin.metadata.id}.`);
229
+ }
230
+ if (targetPath !== undefined) {
231
+ throw new Error("--campaign cannot be combined with --target; campaigns run only against bundled synthetic benchmark projects.");
232
+ }
233
+ if (casesPath !== undefined) {
234
+ throw new Error("--campaign cannot be combined with --cases; the selected preset owns the production corpus.");
235
+ }
236
+ if (projectProfilesPath !== undefined) {
237
+ throw new Error("--campaign cannot be combined with --project-profiles; the selected preset owns the project profiles.");
238
+ }
239
+ }
178
240
  return {
179
241
  experimentId,
180
242
  targetPath,
@@ -193,7 +255,8 @@ export function parseRunExperimentArgs(argv) {
193
255
  requireAgents,
194
256
  includeRealAgents,
195
257
  commandTemplates: Object.keys(commandTemplates).length > 0 ? commandTemplates : undefined,
196
- kitCommand
258
+ kitCommand,
259
+ campaignPreset
197
260
  })
198
261
  };
199
262
  }
@@ -232,6 +295,9 @@ async function loadPluginInputs(args, toolRoot, context) {
232
295
  if (!validation.valid || !validation.config) {
233
296
  throw new Error(`Invalid warm index reuse config: ${validation.errors.join("; ")}`);
234
297
  }
298
+ if (args.config.campaignPreset) {
299
+ return loadCampaignCasesAndProjectProfiles(args.config.campaignPreset, toolRoot, context);
300
+ }
235
301
  return loadCasesAndProjectProfiles(args, toolRoot, context);
236
302
  }
237
303
  return undefined;
@@ -252,6 +318,19 @@ async function loadCasesAndProjectProfiles(args, toolRoot, context) {
252
318
  });
253
319
  return { cases, projectProfiles, env: process.env };
254
320
  }
321
+ // Campaign resource resolution (v0.5.2 Batch 1): the preset owns both bundled resource paths;
322
+ // explicit --cases/--project-profiles are already rejected for campaign mode by argument parsing.
323
+ async function loadCampaignCasesAndProjectProfiles(campaignPresetId, toolRoot, context) {
324
+ const preset = getWarmIndexCampaignPreset(campaignPresetId);
325
+ const projectProfilesPath = resolvePackageResource(context, preset.projectProfilesResourcePath);
326
+ const casesPath = resolvePackageResource(context, preset.casesResourcePath);
327
+ const projectProfiles = await readBenchmarkProjectProfiles(projectProfilesPath, toolRoot);
328
+ const cases = await readEvaluationCases(casesPath, toolRoot, {
329
+ projectProfiles,
330
+ requireProjectProfileRef: true
331
+ });
332
+ return { cases, projectProfiles, env: process.env };
333
+ }
255
334
  function splitList(value) {
256
335
  return value
257
336
  .split(",")
@@ -272,6 +351,9 @@ function parsePositiveInteger(label, value) {
272
351
  }
273
352
  return parsed;
274
353
  }
354
+ function readMetadataString(value) {
355
+ return typeof value === "string" && value.trim() ? value : undefined;
356
+ }
275
357
  function withoutUndefined(value) {
276
358
  return Object.fromEntries(Object.entries(value).filter(([, entryValue]) => entryValue !== undefined));
277
359
  }
@@ -0,0 +1,64 @@
1
+ import path from "node:path";
2
+ import { writeWarmIndexCampaignGallery } from "../gallery/writeWarmIndexCampaignGallery.js";
3
+ import { captureReportScreenshot } from "../screenshot/captureReportScreenshot.js";
4
+ import { writePlotArtifacts } from "../plots/writePlotArtifacts.js";
5
+ // ---------------------------------------------------------------------------
6
+ // v0.5.2 Batch 5 -- command-level warm-index campaign presentation owner.
7
+ //
8
+ // This is the single orchestrator for: existing report -> existing four warm-
9
+ // index plots -> best-effort report screenshot -> narrow warm-index campaign
10
+ // gallery. It reuses writePlotArtifacts, captureReportScreenshot, and
11
+ // writeWarmIndexCampaignGallery unchanged; it builds no new plot, screenshot,
12
+ // or gallery mechanism of its own. The warm-index-reuse plugin remains free of
13
+ // this orchestration -- it is invoked only by the command layer, and only for
14
+ // campaign runs.
15
+ // ---------------------------------------------------------------------------
16
+ const EXPECTED_WARM_INDEX_CHART_IDS = [
17
+ "warm-index-amortized-index-cost",
18
+ "warm-index-context-size",
19
+ "warm-index-correctness",
20
+ "warm-index-cumulative-token-usage",
21
+ ];
22
+ export async function runWarmIndexCampaignPresentation(args) {
23
+ const plotsDir = path.join(args.outputRoot, "plots");
24
+ const plots = await writePlotArtifacts({ experimentDir: args.outputRoot, outDir: plotsDir });
25
+ const chartIds = Object.keys(plots.artifactPaths.charts);
26
+ const hasExactlyExpectedCharts = chartIds.length === EXPECTED_WARM_INDEX_CHART_IDS.length &&
27
+ EXPECTED_WARM_INDEX_CHART_IDS.every((id) => chartIds.includes(id));
28
+ if (!hasExactlyExpectedCharts) {
29
+ throw new Error(`Warm-index campaign presentation requires exactly the four warm-index plot charts (${EXPECTED_WARM_INDEX_CHART_IDS.join(", ")}); got: ${chartIds.join(", ") || "none"}.`);
30
+ }
31
+ const capture = args.captureScreenshot ?? captureReportScreenshot;
32
+ const pngPath = path.join(args.outputRoot, "report.png");
33
+ // Non-fatal by design: a missing Playwright/browser runtime is a normal, expected environment,
34
+ // never a reason to fail the campaign (see captureReportScreenshot's captured/skipped/failed contract).
35
+ const screenshot = await capture(args.reportPaths.htmlPath, pngPath);
36
+ const gallery = await writeWarmIndexCampaignGallery({
37
+ outDir: path.join(args.outputRoot, "gallery"),
38
+ campaignOutputRoot: args.outputRoot,
39
+ reportPaths: {
40
+ jsonPath: args.reportPaths.jsonPath,
41
+ htmlPath: args.reportPaths.htmlPath,
42
+ textPath: args.reportPaths.textPath,
43
+ },
44
+ executionArtifactPath: args.executionArtifactPath,
45
+ plotArtifacts: plots,
46
+ screenshot,
47
+ campaignPreset: args.campaignPreset,
48
+ agentId: args.agentId,
49
+ });
50
+ const warnings = [];
51
+ if (screenshot.status === "skipped" && screenshot.warning) {
52
+ warnings.push(screenshot.warning);
53
+ }
54
+ else if (screenshot.status === "failed") {
55
+ warnings.push(`Report screenshot capture failed: ${screenshot.error ?? "unknown error"}.`);
56
+ }
57
+ warnings.push(...plots.summary.warnings);
58
+ return {
59
+ plots,
60
+ screenshot,
61
+ gallery: { manifest: gallery.manifest, manifestPath: gallery.manifestPath, indexPath: gallery.indexPath },
62
+ warnings,
63
+ };
64
+ }
@@ -41,13 +41,15 @@ export async function runMeasuredCommand(options) {
41
41
  let spawnError;
42
42
  let timedOut = false;
43
43
  let timeout;
44
+ let killGraceTimeout;
45
+ let settled = false;
44
46
  let child;
45
47
  try {
46
48
  child = spawn(executable, args, {
47
49
  cwd: options.cwd,
48
50
  env: { ...process.env, ...options.env },
49
51
  shell: false,
50
- stdio: ["ignore", "pipe", "pipe"]
52
+ stdio: options.stdinText !== undefined ? ["pipe", "pipe", "pipe"] : ["ignore", "pipe", "pipe"]
51
53
  });
52
54
  }
53
55
  catch (error) {
@@ -80,6 +82,7 @@ export async function runMeasuredCommand(options) {
80
82
  ]).then(() => resolve(measured));
81
83
  return;
82
84
  }
85
+ // stdout/stderr are always piped in both stdio branches above.
83
86
  child.stdout.on("data", (chunk) => {
84
87
  stdout += String(chunk);
85
88
  });
@@ -89,17 +92,40 @@ export async function runMeasuredCommand(options) {
89
92
  child.on("error", (error) => {
90
93
  spawnError = error.message;
91
94
  });
92
- if (options.timeoutMs !== undefined) {
93
- timeout = setTimeout(() => {
94
- timedOut = true;
95
- spawnError = `Command timed out after ${options.timeoutMs}ms.`;
96
- forceTerminateProcess(child.pid);
97
- }, options.timeoutMs);
95
+ if (options.stdinText !== undefined) {
96
+ // A child that exits or closes stdin before we write must not crash the process (EPIPE).
97
+ child.stdin?.on("error", (error) => {
98
+ spawnError = spawnError ?? `Failed to write to command stdin: ${error.message}`;
99
+ });
100
+ child.stdin?.write(options.stdinText, "utf8");
101
+ child.stdin?.end();
98
102
  }
99
- child.on("close", async (exitCode) => {
100
- if (timeout) {
103
+ // forceTerminateProcess is fire-and-forget (it never confirms the target process tree actually
104
+ // exited: on Windows it spawns a detached `taskkill /T /F` and does not await or check its
105
+ // result). That is normally sufficient, but a grandchild process spawned through an
106
+ // intermediate shim (for example a Windows .cmd launcher) can occasionally outlive an
107
+ // already-killed intermediate process. Without a bounded fallback, a caller whose kill silently
108
+ // failed would await this promise forever instead of receiving the already-known timeout error.
109
+ const KILL_GRACE_MS = 3000;
110
+ const settle = async (exitCode) => {
111
+ if (settled)
112
+ return;
113
+ settled = true;
114
+ if (timeout)
101
115
  clearTimeout(timeout);
102
- }
116
+ if (killGraceTimeout)
117
+ clearTimeout(killGraceTimeout);
118
+ // The caller's `scripts/cli.ts` entry point sets `process.exitCode` rather than calling
119
+ // `process.exit()`, so the process only exits once Node's event loop actually drains. A
120
+ // still-open child stdio pipe -- the same situation the kill-grace fallback above exists for
121
+ // -- would otherwise keep this process alive forever even though this promise has already
122
+ // resolved. Detaching is safe to do unconditionally: once "close" has fired the streams are
123
+ // already ended, so this is a harmless no-op on the normal (non-fallback) path.
124
+ child.stdout?.destroy();
125
+ child.stderr?.destroy();
126
+ child.stdin?.destroy();
127
+ child.removeAllListeners();
128
+ child.unref();
103
129
  const endedAt = new Date().toISOString();
104
130
  const durationMs = Date.now() - started;
105
131
  await writeArtifact(stdoutPath, stdout);
@@ -125,6 +151,19 @@ export async function runMeasuredCommand(options) {
125
151
  };
126
152
  await writeArtifact(telemetryPath, JSON.stringify(measured, null, 2));
127
153
  resolve(measured);
154
+ };
155
+ if (options.timeoutMs !== undefined) {
156
+ timeout = setTimeout(() => {
157
+ timedOut = true;
158
+ spawnError = `Command timed out after ${options.timeoutMs}ms.`;
159
+ forceTerminateProcess(child.pid);
160
+ killGraceTimeout = setTimeout(() => {
161
+ void settle(null);
162
+ }, KILL_GRACE_MS);
163
+ }, options.timeoutMs);
164
+ }
165
+ child.on("close", (exitCode) => {
166
+ void settle(exitCode);
128
167
  });
129
168
  });
130
169
  return result;