@dailephd/my-dev-kit-lab 0.8.2 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +19 -3
- package/benchmarks/contracts/agent-success-rate-project-profiles.json +33 -0
- package/benchmarks/contracts/agent-success-rate-tasks.json +671 -0
- package/benchmarks/projects/agent-success-inventory-node/.gitattributes +2 -0
- package/benchmarks/projects/agent-success-inventory-node/README.md +74 -0
- package/benchmarks/projects/agent-success-inventory-node/package.json +14 -0
- package/benchmarks/projects/agent-success-inventory-node/src/fulfillmentReport.js +38 -0
- package/benchmarks/projects/agent-success-inventory-node/src/index.js +26 -0
- package/benchmarks/projects/agent-success-inventory-node/src/inventoryStore.js +114 -0
- package/benchmarks/projects/agent-success-inventory-node/src/quantity.js +19 -0
- package/benchmarks/projects/agent-success-inventory-node/src/reservationService.js +45 -0
- package/benchmarks/projects/agent-success-inventory-node/tests/quantity-edge.check.mjs +51 -0
- package/benchmarks/projects/agent-success-inventory-node/tests/quantity-primary.check.mjs +46 -0
- package/benchmarks/projects/agent-success-inventory-node/tests/regression.check.mjs +115 -0
- package/benchmarks/projects/agent-success-inventory-node/tests/report-edge.check.mjs +109 -0
- package/benchmarks/projects/agent-success-inventory-node/tests/report-primary.check.mjs +47 -0
- package/benchmarks/projects/agent-success-inventory-node/tests/reservation-edge.check.mjs +63 -0
- package/benchmarks/projects/agent-success-inventory-node/tests/reservation-primary.check.mjs +44 -0
- package/benchmarks/projects/agent-success-task-board-node/.gitattributes +2 -0
- package/benchmarks/projects/agent-success-task-board-node/README.md +77 -0
- package/benchmarks/projects/agent-success-task-board-node/package.json +14 -0
- package/benchmarks/projects/agent-success-task-board-node/src/index.js +24 -0
- package/benchmarks/projects/agent-success-task-board-node/src/projectSummary.js +15 -0
- package/benchmarks/projects/agent-success-task-board-node/src/taskService.js +58 -0
- package/benchmarks/projects/agent-success-task-board-node/src/taskStore.js +58 -0
- package/benchmarks/projects/agent-success-task-board-node/src/validation.js +31 -0
- package/benchmarks/projects/agent-success-task-board-node/tests/import-edge.check.mjs +65 -0
- package/benchmarks/projects/agent-success-task-board-node/tests/import-primary.check.mjs +31 -0
- package/benchmarks/projects/agent-success-task-board-node/tests/regression.check.mjs +107 -0
- package/benchmarks/projects/agent-success-task-board-node/tests/summary-edge.check.mjs +67 -0
- package/benchmarks/projects/agent-success-task-board-node/tests/summary-primary.check.mjs +54 -0
- package/benchmarks/projects/agent-success-task-board-node/tests/title-edge.check.mjs +39 -0
- package/benchmarks/projects/agent-success-task-board-node/tests/title-primary.check.mjs +29 -0
- package/dist/src/agents/adapters/claudeAdapter.js +6 -3
- package/dist/src/agents/adapters/codexAdapter.js +51 -10
- package/dist/src/agents/index.js +1 -0
- package/dist/src/agents/normalizeAgentTelemetry.js +50 -0
- package/dist/src/agents/parseAgentTokenUsage.js +71 -49
- package/dist/src/agents/providerBuildObservation.js +97 -0
- package/dist/src/audits/core/projectInventory.js +1 -0
- package/dist/src/cli/help.js +44 -1
- package/dist/src/commands/runExperimentDescribeCommand.js +83 -1
- package/dist/src/commands/runExperimentRunCommand.js +309 -6
- package/dist/src/core/processTree.js +55 -0
- package/dist/src/core/runMeasuredCommand.js +108 -22
- package/dist/src/evaluation/agentSuccess/applyPatchToSandbox.js +37 -0
- package/dist/src/evaluation/agentSuccess/assessBaseline.js +23 -0
- package/dist/src/evaluation/agentSuccess/corpusTypes.js +74 -0
- package/dist/src/evaluation/agentSuccess/extractPatchCandidate.js +46 -0
- package/dist/src/evaluation/agentSuccess/index.js +14 -0
- package/dist/src/evaluation/agentSuccess/parseUnifiedDiff.js +210 -0
- package/dist/src/evaluation/agentSuccess/patchTypes.js +5 -0
- package/dist/src/evaluation/agentSuccess/readAgentSuccessCorpus.js +183 -0
- package/dist/src/evaluation/agentSuccess/runVerificationCheck.js +100 -0
- package/dist/src/evaluation/agentSuccess/taskPaths.js +34 -0
- package/dist/src/evaluation/agentSuccess/taskTypes.js +14 -0
- package/dist/src/evaluation/agentSuccess/validateAgentSuccessCorpus.js +307 -0
- package/dist/src/evaluation/agentSuccess/validateAgentSuccessTask.js +297 -0
- package/dist/src/evaluation/agentSuccess/validatePatchPolicy.js +66 -0
- package/dist/src/evaluation/agentSuccess/verificationTypes.js +1 -0
- package/dist/src/evaluation/benchmarkSandbox/createBenchmarkSandbox.js +127 -0
- package/dist/src/evaluation/benchmarkSandbox/errors.js +11 -0
- package/dist/src/evaluation/benchmarkSandbox/gitExecutor.js +62 -0
- package/dist/src/evaluation/benchmarkSandbox/index.js +8 -0
- package/dist/src/evaluation/benchmarkSandbox/minimalHostEnv.js +37 -0
- package/dist/src/evaluation/benchmarkSandbox/pathPolicy.js +49 -0
- package/dist/src/evaluation/benchmarkSandbox/removeBenchmarkSandbox.js +74 -0
- package/dist/src/evaluation/benchmarkSandbox/treeSnapshot.js +57 -0
- package/dist/src/evaluation/benchmarkSandbox/types.js +3 -0
- package/dist/src/evaluation/changeSet/captureChangeSet.js +98 -0
- package/dist/src/evaluation/changeSet/index.js +2 -0
- package/dist/src/evaluation/changeSet/types.js +9 -0
- package/dist/src/experiments/defaultRegistry.js +2 -0
- package/dist/src/experiments/plugins/agentSuccessRate/agentTaskProjection.js +12 -0
- package/dist/src/experiments/plugins/agentSuccessRate/analysis.js +425 -0
- package/dist/src/experiments/plugins/agentSuccessRate/analysisArtifact.js +155 -0
- package/dist/src/experiments/plugins/agentSuccessRate/analysisTypes.js +65 -0
- package/dist/src/experiments/plugins/agentSuccessRate/attemptEvidence.js +7 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignCheckpoint.js +216 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignCheckpointStore.js +588 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignEvidenceQuality.js +77 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignExecution.js +314 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignIdentity.js +111 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignProgress.js +173 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignQueue.js +11 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignRecovery.js +256 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignResumableExecution.js +207 -0
- package/dist/src/experiments/plugins/agentSuccessRate/config.js +123 -0
- package/dist/src/experiments/plugins/agentSuccessRate/contextGeneration.js +497 -0
- package/dist/src/experiments/plugins/agentSuccessRate/execution.js +441 -0
- package/dist/src/experiments/plugins/agentSuccessRate/executionArtifact.js +126 -0
- package/dist/src/experiments/plugins/agentSuccessRate/executionTypes.js +2 -0
- package/dist/src/experiments/plugins/agentSuccessRate/index.js +26 -0
- package/dist/src/experiments/plugins/agentSuccessRate/metadata.js +28 -0
- package/dist/src/experiments/plugins/agentSuccessRate/metrics.js +76 -0
- package/dist/src/experiments/plugins/agentSuccessRate/plugin.js +517 -0
- package/dist/src/experiments/plugins/agentSuccessRate/realAgentExecution.js +272 -0
- package/dist/src/experiments/plugins/agentSuccessRate/realAgentPrompt.js +43 -0
- package/dist/src/experiments/plugins/agentSuccessRate/repairAnalysis.js +148 -0
- package/dist/src/experiments/plugins/agentSuccessRate/repairExecution.js +103 -0
- package/dist/src/experiments/plugins/agentSuccessRate/repairFeedback.js +91 -0
- package/dist/src/experiments/plugins/agentSuccessRate/repairPolicy.js +88 -0
- package/dist/src/experiments/plugins/agentSuccessRate/timeoutPresets.js +9 -0
- package/dist/src/experiments/plugins/agentSuccessRate/types.js +1 -0
- package/dist/src/experiments/plugins/warmIndexReuse/plugin.js +2 -3
- package/dist/src/report/experiments/agentSuccessCampaignProgressReportModel.js +1 -0
- package/dist/src/report/experiments/agentSuccessRateReportModel.js +6 -0
- package/dist/src/report/experiments/buildAgentSuccessCampaignProgressReport.js +6 -0
- package/dist/src/report/experiments/buildAgentSuccessRateReport.js +266 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +27 -2
- package/dist/src/report/experiments/index.js +4 -0
- package/dist/src/report/experiments/renderAgentSuccessCampaignProgressHtml.js +6 -0
- package/dist/src/report/experiments/renderAgentSuccessCampaignProgressText.js +25 -0
- package/dist/src/report/experiments/renderAgentSuccessRateHtml.js +171 -0
- package/dist/src/report/experiments/renderAgentSuccessRateText.js +146 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +3 -0
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +5 -0
- package/dist/src/report/experiments/writeAgentSuccessCampaignProgressReport.js +135 -0
- package/dist/src/report/experiments/writePluginExperimentReports.js +4 -3
- package/docs/METRICS.md +79 -0
- package/package.json +1 -1
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import test from "node:test";
|
|
3
|
+
import { INVALID_PROJECT, createTaskBoard } from "../src/index.js";
|
|
4
|
+
|
|
5
|
+
test("a project's completed count never exceeds its total", () => {
|
|
6
|
+
const { service, summarize } = createTaskBoard();
|
|
7
|
+
service.createTask({ title: "Solo", projectId: "alpha" });
|
|
8
|
+
for (const title of ["x", "y", "z"]) {
|
|
9
|
+
const task = service.createTask({ title, projectId: "beta" });
|
|
10
|
+
service.completeTask(task.id);
|
|
11
|
+
}
|
|
12
|
+
const summary = summarize("alpha");
|
|
13
|
+
assert.equal(summary.total, 1);
|
|
14
|
+
assert.equal(summary.completed, 0);
|
|
15
|
+
assert.ok(summary.completed <= summary.total);
|
|
16
|
+
assert.equal(summary.open, 1);
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
test("unrelated tasks in the store do not change a project's summary", () => {
|
|
20
|
+
const { service, summarize } = createTaskBoard();
|
|
21
|
+
const a = service.createTask({ title: "A", projectId: "core" });
|
|
22
|
+
service.createTask({ title: "B", projectId: "core" });
|
|
23
|
+
service.completeTask(a.id);
|
|
24
|
+
const before = summarize("core");
|
|
25
|
+
const noise = service.createTask({ title: "Noise", projectId: "core-extras" });
|
|
26
|
+
service.createTask({ title: "More noise", projectId: "CORE2" });
|
|
27
|
+
service.completeTask(noise.id);
|
|
28
|
+
service.importTasks([{ externalId: "n-1", title: "Imported noise", projectId: "c" }]);
|
|
29
|
+
assert.deepEqual(summarize("core"), before);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
test("an unknown project has an empty, consistent summary", () => {
|
|
33
|
+
const { service, summarize } = createTaskBoard();
|
|
34
|
+
service.createTask({ title: "A", projectId: "known" });
|
|
35
|
+
assert.deepEqual(summarize("missing"), { projectId: "missing", total: 0, completed: 0, open: 0, taskIds: [] });
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test("blank project identities are rejected consistently", () => {
|
|
39
|
+
const { service, store, summarize } = createTaskBoard();
|
|
40
|
+
assert.throws(() => service.createTask({ title: "Task", projectId: " " }), { code: INVALID_PROJECT });
|
|
41
|
+
assert.throws(() => service.createTask({ title: "Task", projectId: "" }), { code: INVALID_PROJECT });
|
|
42
|
+
assert.throws(
|
|
43
|
+
() => service.importTasks([{ externalId: "ext-1", title: "Task", projectId: " " }]),
|
|
44
|
+
{ code: INVALID_PROJECT }
|
|
45
|
+
);
|
|
46
|
+
assert.throws(() => summarize(" "), { code: INVALID_PROJECT });
|
|
47
|
+
assert.equal(store.list().length, 0);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test("summaries are deterministic and list ids in creation order", () => {
|
|
51
|
+
const build = () => {
|
|
52
|
+
const { service, summarize } = createTaskBoard();
|
|
53
|
+
service.createTask({ title: "1", projectId: "p" });
|
|
54
|
+
service.createTask({ title: "2", projectId: "q" });
|
|
55
|
+
service.createTask({ title: "3", projectId: "P" });
|
|
56
|
+
service.importTasks([{ externalId: "e-1", title: "4", projectId: "p " }]);
|
|
57
|
+
return summarize("p");
|
|
58
|
+
};
|
|
59
|
+
assert.deepEqual(build(), build());
|
|
60
|
+
assert.deepEqual(build().taskIds, ["task-1", "task-3", "task-4"]);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test("the default project is used for tasks created without one", () => {
|
|
64
|
+
const { service, summarize } = createTaskBoard();
|
|
65
|
+
service.createTask({ title: "No project given" });
|
|
66
|
+
assert.equal(summarize("inbox").total, 1);
|
|
67
|
+
});
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import test from "node:test";
|
|
3
|
+
import { createTaskBoard } from "../src/index.js";
|
|
4
|
+
|
|
5
|
+
test("tasks created with differently written project names share one project identity", () => {
|
|
6
|
+
const { service, store, summarize } = createTaskBoard();
|
|
7
|
+
service.createTask({ title: "One", projectId: "web" });
|
|
8
|
+
service.createTask({ title: "Two", projectId: " WEB " });
|
|
9
|
+
service.createTask({ title: "Three", projectId: "Web" });
|
|
10
|
+
assert.deepEqual(
|
|
11
|
+
store.list().map((task) => task.projectId),
|
|
12
|
+
["web", "web", "web"]
|
|
13
|
+
);
|
|
14
|
+
const summary = summarize("web");
|
|
15
|
+
assert.equal(summary.projectId, "web");
|
|
16
|
+
assert.equal(summary.total, 3);
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
test("a summary only counts tasks of the requested project", () => {
|
|
20
|
+
const { service, summarize } = createTaskBoard();
|
|
21
|
+
const web = service.createTask({ title: "Web task", projectId: "web" });
|
|
22
|
+
service.createTask({ title: "Web app task", projectId: "web-app" });
|
|
23
|
+
service.createTask({ title: "Webhook task", projectId: "webhooks" });
|
|
24
|
+
const other = service.createTask({ title: "Docs task", projectId: "docs" });
|
|
25
|
+
service.completeTask(other.id);
|
|
26
|
+
const summary = summarize("web");
|
|
27
|
+
assert.equal(summary.total, 1);
|
|
28
|
+
assert.equal(summary.completed, 0);
|
|
29
|
+
assert.equal(summary.open, 1);
|
|
30
|
+
assert.deepEqual(summary.taskIds, [web.id]);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test("completed and open counts are correct for the requested project", () => {
|
|
34
|
+
const { service, summarize } = createTaskBoard();
|
|
35
|
+
const a = service.createTask({ title: "A", projectId: "ops" });
|
|
36
|
+
service.createTask({ title: "B", projectId: "ops" });
|
|
37
|
+
service.createTask({ title: "C", projectId: "ops" });
|
|
38
|
+
service.createTask({ title: "Elsewhere", projectId: "labs" });
|
|
39
|
+
service.completeTask(a.id);
|
|
40
|
+
assert.deepEqual(summarize(" OPS "), {
|
|
41
|
+
projectId: "ops",
|
|
42
|
+
total: 3,
|
|
43
|
+
completed: 1,
|
|
44
|
+
open: 2,
|
|
45
|
+
taskIds: ["task-1", "task-2", "task-3"]
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test("imported tasks join the same normalized project identity", () => {
|
|
50
|
+
const { service, summarize } = createTaskBoard();
|
|
51
|
+
service.createTask({ title: "Manual", projectId: "ops" });
|
|
52
|
+
service.importTasks([{ externalId: "ext-1", title: "Imported", projectId: " OPS" }]);
|
|
53
|
+
assert.equal(summarize("ops").total, 2);
|
|
54
|
+
});
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import test from "node:test";
|
|
3
|
+
import { INVALID_TITLE, ValidationError, createTaskBoard, validateTitle } from "../src/index.js";
|
|
4
|
+
|
|
5
|
+
test("rejected titles leave the store and id sequence untouched", () => {
|
|
6
|
+
const { service, store } = createTaskBoard();
|
|
7
|
+
assert.throws(() => service.createTask({ title: " \t " }), { code: INVALID_TITLE });
|
|
8
|
+
assert.throws(() => service.createTask({ title: "" }), { code: INVALID_TITLE });
|
|
9
|
+
assert.equal(store.list().length, 0);
|
|
10
|
+
const task = service.createTask({ title: "First" });
|
|
11
|
+
assert.equal(task.id, "task-1");
|
|
12
|
+
assert.equal(store.list().length, 1);
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
test("interior whitespace is preserved while surrounding whitespace is removed", () => {
|
|
16
|
+
const { service } = createTaskBoard();
|
|
17
|
+
assert.equal(service.createTask({ title: " Fix the bug " }).title, "Fix the bug");
|
|
18
|
+
assert.equal(service.createTask({ title: "\n\tPlan sprint\t\n" }).title, "Plan sprint");
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
test("already-trimmed titles keep their meaning", () => {
|
|
22
|
+
const { service } = createTaskBoard();
|
|
23
|
+
assert.equal(service.createTask({ title: "Ship it" }).title, "Ship it");
|
|
24
|
+
assert.equal(service.createTask({ title: "a" }).title, "a");
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("validateTitle returns the normalized title and throws the documented error", () => {
|
|
28
|
+
assert.equal(validateTitle(" Triage "), "Triage");
|
|
29
|
+
assert.throws(
|
|
30
|
+
() => validateTitle(" "),
|
|
31
|
+
(error) => error instanceof ValidationError && error.code === INVALID_TITLE
|
|
32
|
+
);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test("imported task titles are normalized the same way", () => {
|
|
36
|
+
const { service } = createTaskBoard();
|
|
37
|
+
const { imported } = service.importTasks([{ externalId: "ext-1", title: " Imported item " }]);
|
|
38
|
+
assert.equal(imported[0].title, "Imported item");
|
|
39
|
+
});
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import test from "node:test";
|
|
3
|
+
import { INVALID_TITLE, ValidationError, createTaskBoard } from "../src/index.js";
|
|
4
|
+
|
|
5
|
+
test("createTask trims surrounding whitespace from the title", () => {
|
|
6
|
+
const { service } = createTaskBoard();
|
|
7
|
+
const task = service.createTask({ title: " Write release notes " });
|
|
8
|
+
assert.equal(task.title, "Write release notes");
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
test("the stored task retains the normalized title", () => {
|
|
12
|
+
const { service, store } = createTaskBoard();
|
|
13
|
+
const task = service.createTask({ title: "\tReview backlog " });
|
|
14
|
+
assert.equal(store.get(task.id).title, "Review backlog");
|
|
15
|
+
assert.deepEqual(
|
|
16
|
+
store.list().map((entry) => entry.title),
|
|
17
|
+
["Review backlog"]
|
|
18
|
+
);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
test("a whitespace-only title is rejected with the invalid-title error", () => {
|
|
22
|
+
const { service } = createTaskBoard();
|
|
23
|
+
for (const title of [" ", "\t", "\n \r\n"]) {
|
|
24
|
+
assert.throws(
|
|
25
|
+
() => service.createTask({ title }),
|
|
26
|
+
(error) => error instanceof ValidationError && error.code === INVALID_TITLE
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
});
|
|
@@ -54,8 +54,8 @@ export const claudeAdapter = {
|
|
|
54
54
|
};
|
|
55
55
|
/**
|
|
56
56
|
* Parses a single JSON result envelope (`{ "result": "..." }`) when stdout is valid JSON, never
|
|
57
|
-
* the raw envelope itself.
|
|
58
|
-
*
|
|
57
|
+
* the raw envelope itself. Malformed JSON-shaped output stays empty; legacy plain text output
|
|
58
|
+
* retains its fallback.
|
|
59
59
|
*/
|
|
60
60
|
export function parseClaudeFinalAnswer(text) {
|
|
61
61
|
const trimmedInput = text.trim();
|
|
@@ -71,9 +71,12 @@ export function parseClaudeFinalAnswer(text) {
|
|
|
71
71
|
// Valid JSON object without a string result: empty, never the raw envelope.
|
|
72
72
|
return { finalAnswerText: "", finalAnswerParseStatus: "empty" };
|
|
73
73
|
}
|
|
74
|
+
return { finalAnswerText: "", finalAnswerParseStatus: "empty" };
|
|
74
75
|
}
|
|
75
76
|
catch {
|
|
76
|
-
//
|
|
77
|
+
// A malformed envelope is not a legacy text answer.
|
|
78
|
+
if (/^[\[{]/.test(trimmedInput))
|
|
79
|
+
return { finalAnswerText: "", finalAnswerParseStatus: "empty" };
|
|
77
80
|
}
|
|
78
81
|
}
|
|
79
82
|
return { finalAnswerText: trimmedInput, finalAnswerParseStatus: trimmedInput ? "parsed" : "empty" };
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { runMeasuredCommand } from "../../core/runMeasuredCommand.js";
|
|
2
2
|
import { applyPromptToCommandTemplate } from "../runAgentPrompt.js";
|
|
3
3
|
import { parseAgentTokenUsage } from "../parseAgentTokenUsage.js";
|
|
4
|
+
import { trustedProviderLaunch } from "../providerBuildObservation.js";
|
|
4
5
|
// v0.5.2 Batch 2 -- frozen stdin invocation for the real-agent evaluation transport. Every flag
|
|
5
6
|
// here is planner-owned; do not add sandbox/model/approval-policy flags in this batch.
|
|
6
7
|
const CODEX_STDIN_ARGS = [
|
|
@@ -51,13 +52,21 @@ export const codexAdapter = {
|
|
|
51
52
|
/**
|
|
52
53
|
* Recognizes a Codex JSONL event stream (at least one line parses to an object with a string
|
|
53
54
|
* `type`) and, when recognized, uses the LAST `item.completed` `agent_message` text as the final
|
|
54
|
-
* answer -- never the raw JSONL.
|
|
55
|
-
*
|
|
55
|
+
* answer -- never the raw JSONL. JSON-shaped output without an answer stays empty. Plain text
|
|
56
|
+
* retains the legacy fallback. A single formatted JSON event is supported as well.
|
|
56
57
|
*/
|
|
57
58
|
export function parseCodexFinalAnswer(text) {
|
|
58
|
-
let sawEventLine =
|
|
59
|
+
let sawEventLine = /^[\s]*[\[{]/.test(text);
|
|
59
60
|
let lastAgentMessageText = null;
|
|
60
|
-
|
|
61
|
+
const candidates = function* () {
|
|
62
|
+
const trimmed = text.trim();
|
|
63
|
+
if (trimmed.startsWith("{") || trimmed.startsWith("["))
|
|
64
|
+
yield trimmed;
|
|
65
|
+
for (const match of text.matchAll(/[^\r\n]+/g))
|
|
66
|
+
if (match[0].trim() !== trimmed)
|
|
67
|
+
yield match[0];
|
|
68
|
+
};
|
|
69
|
+
for (const line of candidates()) {
|
|
61
70
|
const trimmedLine = line.trim();
|
|
62
71
|
if (!trimmedLine)
|
|
63
72
|
continue;
|
|
@@ -69,7 +78,10 @@ export function parseCodexFinalAnswer(text) {
|
|
|
69
78
|
// Agent output can mix JSONL with incidental non-JSON noise; ignore malformed lines.
|
|
70
79
|
continue;
|
|
71
80
|
}
|
|
72
|
-
if (!parsed || typeof parsed !== "object"
|
|
81
|
+
if (!parsed || typeof parsed !== "object")
|
|
82
|
+
continue;
|
|
83
|
+
sawEventLine = true;
|
|
84
|
+
if (Array.isArray(parsed))
|
|
73
85
|
continue;
|
|
74
86
|
const event = parsed;
|
|
75
87
|
if (typeof event.type !== "string")
|
|
@@ -95,7 +107,10 @@ export function parseCodexFinalAnswer(text) {
|
|
|
95
107
|
export async function runCliAgent(request, adapter) {
|
|
96
108
|
const started = Date.now();
|
|
97
109
|
const command = adapter.buildCommand(request);
|
|
98
|
-
|
|
110
|
+
if (request.trustedExecutable && (request.commandTemplate || request.promptTransport !== "stdin" || adapter.id === "fake-agent"))
|
|
111
|
+
throw new Error("INVALID_TRUSTED_PROVIDER_REQUEST");
|
|
112
|
+
const launch = request.trustedExecutable ? await trustedProviderLaunch(request.trustedExecutable, adapter.id) : null;
|
|
113
|
+
const available = launch !== null || await adapter.isAvailable(request);
|
|
99
114
|
if (!available) {
|
|
100
115
|
const ended = Date.now();
|
|
101
116
|
const status = request.requireAvailable ? "failed" : "skipped";
|
|
@@ -127,18 +142,20 @@ export async function runCliAgent(request, adapter) {
|
|
|
127
142
|
}
|
|
128
143
|
const measured = await runMeasuredCommand({
|
|
129
144
|
commandId: `${adapter.id}-agent-run`,
|
|
130
|
-
commandString: command.command,
|
|
131
|
-
extraArgs: command.args,
|
|
145
|
+
...(launch ? { executable: launch.executable, args: [...launch.prefix, ...command.args], resolveCommand: false } : { commandString: command.command, extraArgs: command.args }),
|
|
132
146
|
cwd: request.commandTemplate?.cwd ?? request.cwd,
|
|
133
147
|
outDir: request.outDir,
|
|
134
148
|
env: request.env,
|
|
135
149
|
timeoutMs: request.timeoutMs,
|
|
136
|
-
stdinText: command.stdinText
|
|
150
|
+
stdinText: command.stdinText,
|
|
151
|
+
...(launch ? { stdoutMaxBytes: 8 * 1024 * 1024, stderrMaxBytes: 1024 * 1024, terminateProcessTree: true } : {})
|
|
137
152
|
});
|
|
138
153
|
const ended = Date.now();
|
|
139
154
|
const combinedOutput = `${measured.stdout}\n${measured.stderr}`;
|
|
140
155
|
const parsedAnswer = adapter.parseFinalAnswer(measured.stdout || measured.stderr);
|
|
141
156
|
const parsedUsage = adapter.parseTokenUsage(combinedOutput);
|
|
157
|
+
// Bound campaigns require process-channel diagnostics. Never search final agent message/patch bodies.
|
|
158
|
+
const condition = launch ? trustedProcessCondition(measured.stdout, measured.stderr, measured.timedOut === true) : null;
|
|
142
159
|
return {
|
|
143
160
|
runId: request.runId,
|
|
144
161
|
agentId: adapter.id,
|
|
@@ -164,6 +181,30 @@ export async function runCliAgent(request, adapter) {
|
|
|
164
181
|
tokenUsageSource: parsedUsage.tokenUsageSource,
|
|
165
182
|
tokenUsageReliability: parsedUsage.tokenUsageReliability,
|
|
166
183
|
warnings: parsedUsage.warnings,
|
|
167
|
-
errors: measured.ok ? [] : [measured.error ?? "Agent command failed."]
|
|
184
|
+
errors: [...(measured.ok ? [] : [measured.error ?? "Agent command failed."]), ...(condition === "limit" ? ["Provider rate limit reached."] : condition === "timeout" ? ["Provider command timed out."] : [])]
|
|
168
185
|
};
|
|
169
186
|
}
|
|
187
|
+
function trustedProcessCondition(stdout, stderr, timedOut) {
|
|
188
|
+
if (timedOut)
|
|
189
|
+
return "timeout";
|
|
190
|
+
const diagnostics = [];
|
|
191
|
+
const isPatch = (text) => /^(?:diff --git |@@ |\+\+\+ |--- )/m.test(text);
|
|
192
|
+
// Non-interactive CLI diagnostic lines, not arbitrary substrings or diff additions.
|
|
193
|
+
if (!isPatch(stderr))
|
|
194
|
+
diagnostics.push(...stderr.split(/\r?\n/).filter(line => /^(?:error:\s*)?(?:rate limit (?:exceeded|reached)|usage limit (?:exceeded|reached)|quota (?:exceeded|exhausted)|too many requests|you(?:'ve| have) (?:hit|reached) (?:your |the )?(?:usage |session |rate )?limit)/i.test(line.trim())));
|
|
195
|
+
for (const line of stdout.split(/\r?\n/).slice(0, 1000)) {
|
|
196
|
+
try {
|
|
197
|
+
const event = JSON.parse(line);
|
|
198
|
+
if (!event || typeof event !== "object")
|
|
199
|
+
continue;
|
|
200
|
+
if (event.type !== "error" && event.type !== "turn.failed" && event.is_error !== true)
|
|
201
|
+
continue;
|
|
202
|
+
const error = event.error;
|
|
203
|
+
const message = typeof error === "string" ? error : error && typeof error === "object" ? error.message : event.message;
|
|
204
|
+
if (typeof message === "string" && message.length <= 2048 && !isPatch(message))
|
|
205
|
+
diagnostics.push(message);
|
|
206
|
+
}
|
|
207
|
+
catch { /* Not a structured error event; agent output is deliberately not diagnostic evidence. */ }
|
|
208
|
+
}
|
|
209
|
+
return diagnostics.some(text => /\b(?:rate\s+limit|usage\s+limit|quota|too many requests|session\s+limit)\b/i.test(text)) ? "limit" : null;
|
|
210
|
+
}
|
package/dist/src/agents/index.js
CHANGED
|
@@ -2,6 +2,7 @@ export * from "./types.js";
|
|
|
2
2
|
export * from "./agentRegistry.js";
|
|
3
3
|
export * from "./runAgentPrompt.js";
|
|
4
4
|
export * from "./parseAgentTokenUsage.js";
|
|
5
|
+
export { normalizeAgentTokenUsageV091, normalizeAgentDurationV091 } from "./normalizeAgentTelemetry.js";
|
|
5
6
|
export * from "./adapters/fakeAgentAdapter.js";
|
|
6
7
|
export * from "./adapters/codexAdapter.js";
|
|
7
8
|
export * from "./adapters/claudeAdapter.js";
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
const EVIDENCE_LEVEL = {
|
|
2
|
+
"provider-reported": "provider-reported",
|
|
3
|
+
"cli-json": "cli-reported",
|
|
4
|
+
"agent-reported": "parsed-from-output",
|
|
5
|
+
"estimated-from-text": "estimated",
|
|
6
|
+
otel: "unavailable",
|
|
7
|
+
unavailable: "unavailable"
|
|
8
|
+
};
|
|
9
|
+
function normalizeCount(value) {
|
|
10
|
+
return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value) && value >= 0 ? value : null;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Classifies the legacy evidence channel, not the origin of each individual field.
|
|
14
|
+
* Parser-derived totals are preserved; missing totals are never calculated here.
|
|
15
|
+
* The existing contract has no verified OTel attribution, so its channel remains unavailable.
|
|
16
|
+
*/
|
|
17
|
+
export function normalizeAgentTokenUsageV091(usage, reliability) {
|
|
18
|
+
const counts = {
|
|
19
|
+
totalTokens: normalizeCount(usage.totalTokens),
|
|
20
|
+
inputTokens: normalizeCount(usage.inputTokens),
|
|
21
|
+
outputTokens: normalizeCount(usage.outputTokens),
|
|
22
|
+
cachedInputTokens: normalizeCount(usage.cachedInputTokens),
|
|
23
|
+
reasoningTokens: normalizeCount(usage.reasoningTokens)
|
|
24
|
+
};
|
|
25
|
+
const numericReasons = {};
|
|
26
|
+
for (const key of Object.keys(counts)) {
|
|
27
|
+
numericReasons[key] = counts[key] !== null ? null : usage[key] == null ? "missing-token-count" : "invalid-token-count";
|
|
28
|
+
}
|
|
29
|
+
const hasCount = Object.values(counts).some((count) => count !== null);
|
|
30
|
+
return {
|
|
31
|
+
source: usage.source,
|
|
32
|
+
reliability,
|
|
33
|
+
evidenceLevel: EVIDENCE_LEVEL[usage.source],
|
|
34
|
+
classificationReason: usage.source === "otel" ? "unattributed-source" : usage.source === "unavailable" ? "source-unavailable" : null,
|
|
35
|
+
numericAvailability: hasCount ? "available" : "unavailable",
|
|
36
|
+
condition: hasCount ? "token-available" : "token-unavailable",
|
|
37
|
+
...counts,
|
|
38
|
+
numericReasons
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
/** Accepts the current Lab wall-clock measurement, never provider-reported timing. */
|
|
42
|
+
export function normalizeAgentDurationV091(durationMs) {
|
|
43
|
+
if (durationMs == null) {
|
|
44
|
+
return { durationMs: null, source: "unavailable", reason: "missing-duration" };
|
|
45
|
+
}
|
|
46
|
+
if (!Number.isFinite(durationMs) || durationMs < 0) {
|
|
47
|
+
return { durationMs: null, source: "unavailable", reason: "invalid-duration" };
|
|
48
|
+
}
|
|
49
|
+
return { durationMs, source: "lab-wall-clock", reason: null };
|
|
50
|
+
}
|
|
@@ -13,11 +13,11 @@ const numberFields = {
|
|
|
13
13
|
};
|
|
14
14
|
export function parseAgentTokenUsage(text) {
|
|
15
15
|
const warnings = [];
|
|
16
|
-
const jsonUsage = parseJsonUsage(text);
|
|
16
|
+
const jsonUsage = parseJsonUsage(text, warnings);
|
|
17
17
|
if (jsonUsage.found) {
|
|
18
18
|
return buildResult(jsonUsage.usage, "cli-json", text, warnings);
|
|
19
19
|
}
|
|
20
|
-
const plainTextUsage = parsePlainTextUsage(text);
|
|
20
|
+
const plainTextUsage = jsonUsage.structured ? { found: false, usage: {} } : parsePlainTextUsage(text);
|
|
21
21
|
if (plainTextUsage.found) {
|
|
22
22
|
return buildResult(plainTextUsage.usage, "agent-reported", text, warnings);
|
|
23
23
|
}
|
|
@@ -29,57 +29,84 @@ export function parseAgentTokenUsage(text) {
|
|
|
29
29
|
warnings
|
|
30
30
|
};
|
|
31
31
|
}
|
|
32
|
-
function parseJsonUsage(text) {
|
|
33
|
-
|
|
32
|
+
function parseJsonUsage(text, warnings) {
|
|
33
|
+
let usage;
|
|
34
|
+
let structured = false;
|
|
35
|
+
let visited = 0;
|
|
36
|
+
const warn = (message) => { if (!warnings.includes(message))
|
|
37
|
+
warnings.push(message); };
|
|
38
|
+
const visit = (value, depth) => {
|
|
39
|
+
if (!value || typeof value !== "object")
|
|
40
|
+
return;
|
|
41
|
+
if (++visited > 2048 || depth > 16) {
|
|
42
|
+
warn("Token usage traversal limit reached; additional envelopes were ignored.");
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
if (Array.isArray(value)) {
|
|
46
|
+
for (const child of value) {
|
|
47
|
+
if (visited > 2048)
|
|
48
|
+
break;
|
|
49
|
+
visit(child, depth + 1);
|
|
50
|
+
}
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
const record = value;
|
|
54
|
+
const local = {};
|
|
55
|
+
for (const [key, aliases] of Object.entries(numberFields)) {
|
|
56
|
+
for (const alias of aliases) {
|
|
57
|
+
const count = record[alias];
|
|
58
|
+
if (count === undefined)
|
|
59
|
+
continue;
|
|
60
|
+
if (typeof count !== "number" || !Number.isFinite(count) || !Number.isInteger(count) || count < 0) {
|
|
61
|
+
warn("Invalid token count ignored in agent output.");
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
if (local[key] !== undefined && local[key] !== count)
|
|
65
|
+
warn("Conflicting token aliases; legacy maximum precedence retained within one usage record.");
|
|
66
|
+
local[key] = Math.max(local[key] ?? 0, count);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
if (hasAnyUsage(local)) {
|
|
70
|
+
if (usage === undefined)
|
|
71
|
+
usage = local;
|
|
72
|
+
else if (JSON.stringify(usage) !== JSON.stringify(local))
|
|
73
|
+
warn("Multiple token usage records; first valid record retained without combining attempts.");
|
|
74
|
+
}
|
|
75
|
+
// Supported envelope traversal only: patch/tool payload objects are not usage metadata.
|
|
76
|
+
for (const key of ["usage", "token_usage", "tokenUsage", "response", "message", "data", "event", "result"]) {
|
|
77
|
+
if (visited > 2048)
|
|
78
|
+
break;
|
|
79
|
+
visit(record[key], depth + 1);
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
let candidates = 0;
|
|
34
83
|
for (const candidate of collectJsonCandidates(text)) {
|
|
84
|
+
if (++candidates > 4096) {
|
|
85
|
+
warn("Token usage candidate limit reached; additional lines were ignored.");
|
|
86
|
+
break;
|
|
87
|
+
}
|
|
35
88
|
try {
|
|
36
|
-
|
|
89
|
+
const value = JSON.parse(candidate);
|
|
90
|
+
structured = true;
|
|
91
|
+
visit(value, 0);
|
|
37
92
|
}
|
|
38
93
|
catch {
|
|
39
94
|
// Agent output is often mixed JSONL plus text; malformed lines are ignored.
|
|
40
95
|
}
|
|
41
96
|
}
|
|
42
|
-
return { found:
|
|
97
|
+
return { found: usage !== undefined, usage: usage ?? {}, structured };
|
|
43
98
|
}
|
|
44
|
-
function collectJsonCandidates(text) {
|
|
99
|
+
function* collectJsonCandidates(text) {
|
|
45
100
|
const trimmed = text.trim();
|
|
46
|
-
const candidates = [];
|
|
47
101
|
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
|
|
48
|
-
|
|
102
|
+
yield trimmed;
|
|
49
103
|
}
|
|
50
|
-
for (const
|
|
51
|
-
const lineTrimmed =
|
|
52
|
-
if (lineTrimmed.startsWith("{") || lineTrimmed.startsWith("[")) {
|
|
53
|
-
|
|
104
|
+
for (const match of text.matchAll(/[^\r\n]+/g)) {
|
|
105
|
+
const lineTrimmed = match[0].trim();
|
|
106
|
+
if (lineTrimmed !== trimmed && (lineTrimmed.startsWith("{") || lineTrimmed.startsWith("["))) {
|
|
107
|
+
yield lineTrimmed;
|
|
54
108
|
}
|
|
55
109
|
}
|
|
56
|
-
return [...new Set(candidates)];
|
|
57
|
-
}
|
|
58
|
-
function extractUsage(value) {
|
|
59
|
-
const usage = {};
|
|
60
|
-
visit(value, usage);
|
|
61
|
-
return usage;
|
|
62
|
-
}
|
|
63
|
-
function visit(value, usage) {
|
|
64
|
-
if (!value || typeof value !== "object") {
|
|
65
|
-
return;
|
|
66
|
-
}
|
|
67
|
-
if (Array.isArray(value)) {
|
|
68
|
-
value.forEach((item) => visit(item, usage));
|
|
69
|
-
return;
|
|
70
|
-
}
|
|
71
|
-
const record = value;
|
|
72
|
-
for (const [usageKey, fieldNames] of Object.entries(numberFields)) {
|
|
73
|
-
for (const fieldName of fieldNames) {
|
|
74
|
-
const fieldValue = record[fieldName];
|
|
75
|
-
if (typeof fieldValue === "number" && Number.isFinite(fieldValue)) {
|
|
76
|
-
usage[usageKey] = Math.max(usage[usageKey] ?? 0, fieldValue);
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
for (const child of Object.values(record)) {
|
|
81
|
-
visit(child, usage);
|
|
82
|
-
}
|
|
83
110
|
}
|
|
84
111
|
function parsePlainTextUsage(text) {
|
|
85
112
|
const usage = {};
|
|
@@ -91,20 +118,15 @@ function parsePlainTextUsage(text) {
|
|
|
91
118
|
["totalTokens", /total\s+tokens?\s*[:=]\s*(\d+)/i]
|
|
92
119
|
];
|
|
93
120
|
for (const [key, pattern] of patterns) {
|
|
94
|
-
const match = text.match(pattern);
|
|
121
|
+
const match = text.match(new RegExp(pattern.source.replace("(\\d+)", "(\\d+)(?![\\d.eE])"), pattern.flags));
|
|
95
122
|
if (match?.[1]) {
|
|
96
|
-
|
|
123
|
+
const count = Number(match[1]);
|
|
124
|
+
if (Number.isFinite(count) && Number.isInteger(count))
|
|
125
|
+
usage[key] = count;
|
|
97
126
|
}
|
|
98
127
|
}
|
|
99
128
|
return { found: hasAnyUsage(usage), usage };
|
|
100
129
|
}
|
|
101
|
-
function mergeUsage(target, source) {
|
|
102
|
-
for (const key of Object.keys(numberFields)) {
|
|
103
|
-
if (source[key] !== undefined) {
|
|
104
|
-
target[key] = Math.max(target[key] ?? 0, source[key]);
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
130
|
function hasAnyUsage(usage) {
|
|
109
131
|
return Object.values(usage).some((value) => typeof value === "number");
|
|
110
132
|
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { lstat, mkdtemp, readFile, realpath, rm } from "node:fs/promises";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { resolveCommand } from "../core/resolveCommand.js";
|
|
6
|
+
import { runMeasuredCommand } from "../core/runMeasuredCommand.js";
|
|
7
|
+
const launches = new WeakMap();
|
|
8
|
+
const hash = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
9
|
+
const fail = () => { throw new Error("PROVIDER_BUILD_UNVERIFIABLE: use a native executable or an attributable node entrypoint; unsupported wrappers cannot authorize a campaign."); };
|
|
10
|
+
async function inspect(alias) {
|
|
11
|
+
const actual = await realpath(alias);
|
|
12
|
+
const stat = await lstat(actual);
|
|
13
|
+
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 256 * 1024 * 1024 || stat.size === 0)
|
|
14
|
+
return fail();
|
|
15
|
+
const bytes = await readFile(actual);
|
|
16
|
+
if (bytes.length !== stat.size || await realpath(alias) !== actual)
|
|
17
|
+
return fail();
|
|
18
|
+
return { proof: { alias: path.resolve(alias), actual, digest: hash(bytes) }, bytes };
|
|
19
|
+
}
|
|
20
|
+
function native(bytes) {
|
|
21
|
+
return bytes.subarray(0, 2).toString() === "MZ" || bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46])) || ["feedface", "feedfacf", "cefaedfe", "cffaedfe", "cafebabe"].includes(bytes.subarray(0, 4).toString("hex"));
|
|
22
|
+
}
|
|
23
|
+
/** Read-only task-free observation. All temporary version diagnostics are removed. No shell templates. */
|
|
24
|
+
export async function observeProviderBuild(providerId, options = {}) {
|
|
25
|
+
if (providerId !== "codex" && providerId !== "claude")
|
|
26
|
+
return fail();
|
|
27
|
+
const cwd = await mkdtemp(path.join(os.tmpdir(), "lab-provider-build-"));
|
|
28
|
+
try {
|
|
29
|
+
const env = { ...process.env, ...options.env };
|
|
30
|
+
const resolved = resolveCommand(providerId, { cwd, env, allowPowerShellShim: false });
|
|
31
|
+
if (!resolved.resolvedPath || resolved.resolutionKind === "unavailable")
|
|
32
|
+
return fail();
|
|
33
|
+
const selected = await inspect(resolved.resolvedPath);
|
|
34
|
+
let executable = selected.proof.actual;
|
|
35
|
+
const prefix = [];
|
|
36
|
+
const files = [selected.proof];
|
|
37
|
+
let limitation = "Observed launch bytes only; transitive dependencies are not attested.";
|
|
38
|
+
if (!native(selected.bytes)) {
|
|
39
|
+
const text = selected.bytes.toString("utf8");
|
|
40
|
+
// A deliberately closed, attributable wrapper subset. Never execute arbitrary wrapper text.
|
|
41
|
+
const cmd = /^@echo off\r?\n(?:node|"([^"\r\n]+)") "([^"\r\n]+)" %\*\r?\n?$/i.exec(text);
|
|
42
|
+
const shebang = /^#!\/usr\/bin\/env node\r?\n/.test(text);
|
|
43
|
+
if (!cmd && !shebang)
|
|
44
|
+
return fail();
|
|
45
|
+
const node = resolveCommand(cmd?.[1] ?? "node", { cwd, env, allowPowerShellShim: false });
|
|
46
|
+
if (!node.resolvedPath || node.resolutionKind === "unavailable")
|
|
47
|
+
return fail();
|
|
48
|
+
const interpreter = await inspect(node.resolvedPath);
|
|
49
|
+
if (!native(interpreter.bytes))
|
|
50
|
+
return fail();
|
|
51
|
+
files.push(interpreter.proof);
|
|
52
|
+
executable = interpreter.proof.actual;
|
|
53
|
+
if (cmd) {
|
|
54
|
+
if (!path.isAbsolute(cmd[2]))
|
|
55
|
+
return fail();
|
|
56
|
+
const entry = await inspect(cmd[2]);
|
|
57
|
+
files.push(entry.proof);
|
|
58
|
+
prefix.push(entry.proof.actual);
|
|
59
|
+
}
|
|
60
|
+
else
|
|
61
|
+
prefix.push(selected.proof.actual);
|
|
62
|
+
limitation = "Restricted node wrapper, interpreter and entrypoint bytes observed; transitive dependencies are not attested.";
|
|
63
|
+
}
|
|
64
|
+
const version = await runMeasuredCommand({ commandId: `${providerId}-build-version`, executable, args: [...prefix, "--version"], resolveCommand: false, cwd, outDir: cwd, env, timeoutMs: 5000, stdoutMaxBytes: 4096, stderrMaxBytes: 4096, terminateProcessTree: true });
|
|
65
|
+
const observedVersion = version.stdout.trim();
|
|
66
|
+
if (!version.ok || version.timedOut || version.outputLimit || !/^[\x20-\x7e]{1,512}$/.test(observedVersion) || !/\d+\.\d+/.test(observedVersion))
|
|
67
|
+
return fail();
|
|
68
|
+
const capability = Object.freeze({ providerId });
|
|
69
|
+
const bytes = Buffer.from(JSON.stringify({ contract: "local-provider-launch-v1", providerId, origin: "inspected-local-executable", executable, prefix, files, version: observedVersion, adapterContract: `${providerId}-fixed-stdin-v1`, limitation }));
|
|
70
|
+
launches.set(capability, { executable, prefix, files, bytes });
|
|
71
|
+
await trustedProviderObservation(capability, providerId);
|
|
72
|
+
return capability;
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
return fail();
|
|
76
|
+
}
|
|
77
|
+
finally {
|
|
78
|
+
await rm(cwd, { recursive: true, force: true });
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
export async function trustedProviderObservation(capability, providerId) {
|
|
82
|
+
const launch = launches.get(capability);
|
|
83
|
+
if (!launch || capability.providerId !== providerId)
|
|
84
|
+
return fail();
|
|
85
|
+
for (const file of launch.files) {
|
|
86
|
+
const current = await inspect(file.alias);
|
|
87
|
+
if (current.proof.actual !== file.actual || current.proof.digest !== file.digest)
|
|
88
|
+
throw new Error("PROVIDER_BUILD_CHANGED");
|
|
89
|
+
}
|
|
90
|
+
return Buffer.from(launch.bytes);
|
|
91
|
+
}
|
|
92
|
+
/** Fixed adapter args are appended by the adapter owner; no PATH lookup occurs on this launch. */
|
|
93
|
+
export async function trustedProviderLaunch(capability, providerId) {
|
|
94
|
+
await trustedProviderObservation(capability, providerId);
|
|
95
|
+
const launch = launches.get(capability);
|
|
96
|
+
return { executable: launch.executable, prefix: [...launch.prefix] };
|
|
97
|
+
}
|