@cassiomc1/forgeloop 1.6.4 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT_COMPATIBILITY.md +11 -0
- package/DOCS_INDEX.md +1 -0
- package/GUIDE_ROUTER.md +26 -1
- package/LOOP_ENGINEERING.md +48 -0
- package/ORCHESTRATOR_INTEGRATION.md +38 -0
- package/PROTOCOL_INTEGRATION.md +62 -0
- package/README.md +25 -0
- package/benchmarks/execution-profiles/README.md +44 -0
- package/benchmarks/execution-profiles/api-feature.json +18 -0
- package/benchmarks/execution-profiles/authentication-change.json +18 -0
- package/benchmarks/execution-profiles/documentation-correction.json +18 -0
- package/benchmarks/execution-profiles/infrastructure-release.json +18 -0
- package/benchmarks/execution-profiles/novatask-saas-landing-page.json +36 -0
- package/benchmarks/execution-profiles/small-bug-fix.json +18 -0
- package/benchmarks/execution-profiles/static-landing-page.json +18 -0
- package/completions/_forgeloop +6 -4
- package/completions/forgeloop.bash +8 -4
- package/completions/forgeloop.fish +24 -1
- package/docs/AGENT_PROTOCOL_SUMMARY.md +45 -1
- package/docs/ARTIFACT_REFERENCE.md +53 -7
- package/docs/CLI_REFERENCE.md +83 -1
- package/docs/EXECUTION_PROFILE_BENCHMARKS.md +208 -0
- package/docs/GETTING_STARTED.md +28 -0
- package/docs/MCP.md +6 -0
- package/docs/RELEASE_CHECKLIST.md +4 -0
- package/docs/TROUBLESHOOTING.md +6 -0
- package/docs/UNIVERSAL_INTEGRATION.md +58 -0
- package/package.json +14 -2
- package/schemas/config.schema.json +1 -0
- package/schemas/execution-profile-benchmark-aggregate.schema.json +43 -0
- package/schemas/execution-profile-benchmark-run.schema.json +106 -0
- package/schemas/execution-profile-benchmark-scenario.schema.json +66 -0
- package/schemas/routing-result.schema.json +12 -0
- package/schemas/usage.schema.json +30 -0
- package/scripts/check-efficiency-regression.mjs +99 -0
- package/scripts/generate-agent-protocol-summary.mjs +35 -0
- package/scripts/lib/execution-profile-benchmark-io.mjs +67 -0
- package/scripts/run-execution-profile-benchmarks.mjs +265 -0
- package/scripts/summarize-execution-profile-benchmarks.mjs +84 -0
- package/scripts/validate-execution-profile-benchmarks.mjs +120 -0
- package/src/cli.js +16 -4
- package/src/commands/efficiency.js +12 -0
- package/src/commands/eval.js +8 -2
- package/src/commands/metrics.js +2 -2
- package/src/commands/next.js +26 -2
- package/src/commands/route.js +20 -2
- package/src/commands/task-show.js +31 -2
- package/src/commands/usage-record.js +61 -0
- package/src/core/artifact-registry.js +12 -0
- package/src/core/cli-command-definitions.js +27 -0
- package/src/core/command-executors.js +29 -5
- package/src/core/command-input.js +34 -0
- package/src/core/config.js +9 -0
- package/src/core/efficiency.js +197 -0
- package/src/core/error-codes.js +18 -0
- package/src/core/execution-profile-benchmarks.js +674 -0
- package/src/core/execution-profile-context.js +177 -0
- package/src/core/execution-profile.js +248 -0
- package/src/core/integration-invocation-policy.js +43 -0
- package/src/core/integration-resources.js +22 -1
- package/src/core/protocol-info.js +42 -0
- package/src/core/resumability.js +27 -1
- package/src/core/router.js +23 -1
- package/src/core/runtime-context.js +11 -0
- package/src/core/schema-validation.js +4 -0
- package/src/core/task-paths.js +2 -0
- package/src/core/templates.js +4 -0
- package/src/core/trace.js +1 -0
- package/src/core/trajectory-evaluation.js +2 -2
- package/src/core/trajectory-metrics.js +18 -2
- package/src/core/usage.js +137 -0
- package/src/integration.d.ts +84 -0
- package/src/integration.js +14 -0
package/src/commands/next.js
CHANGED
|
@@ -1,15 +1,35 @@
|
|
|
1
1
|
import { getNextAction } from "../core/next-action.js";
|
|
2
2
|
import { withResolvedTask } from "../core/task-command.js";
|
|
3
|
+
import { readPersistedRoute } from "../core/route-artifact.js";
|
|
4
|
+
import { projectExecutionProfile } from "../core/execution-profile.js";
|
|
3
5
|
|
|
4
|
-
export async function runNext({ target, packageRoot, taskId, task, authorityContext, runtimeContext }) {
|
|
6
|
+
export async function runNext({ target, packageRoot, taskId, task, authorityContext, runtimeContext, compact = false }) {
|
|
5
7
|
return withResolvedTask(target, { taskId: taskId ?? task, packageRoot }, async (ctx) => {
|
|
6
|
-
|
|
8
|
+
const result = await getNextAction({
|
|
7
9
|
target,
|
|
8
10
|
packageRoot,
|
|
9
11
|
taskId: ctx?.taskId ?? null,
|
|
10
12
|
authorityContext,
|
|
11
13
|
runtimeContext,
|
|
12
14
|
});
|
|
15
|
+
if (!compact) return result;
|
|
16
|
+
const compactTaskId = ctx?.taskId ?? (result.taskId && result.taskId !== "unknown" ? result.taskId : null);
|
|
17
|
+
let profile = null;
|
|
18
|
+
try {
|
|
19
|
+
const route = await readPersistedRoute(target, packageRoot, { taskId: compactTaskId });
|
|
20
|
+
profile = projectExecutionProfile(route.value);
|
|
21
|
+
} catch {
|
|
22
|
+
// Legacy and incomplete tasks remain readable without a profile.
|
|
23
|
+
}
|
|
24
|
+
return {
|
|
25
|
+
taskId: compactTaskId ?? result.taskId,
|
|
26
|
+
phase: result.currentPhase,
|
|
27
|
+
profile,
|
|
28
|
+
nextAction: result.nextAction,
|
|
29
|
+
command: result.commandSpecs?.[0]?.argv ?? [],
|
|
30
|
+
terminal: result.terminal,
|
|
31
|
+
errors: result.reasonCodes ?? result.reasons?.map((reason) => reason.code) ?? [],
|
|
32
|
+
};
|
|
13
33
|
});
|
|
14
34
|
}
|
|
15
35
|
|
|
@@ -50,3 +70,7 @@ export function formatNextActionResult(result) {
|
|
|
50
70
|
if (result.terminal) lines.push("STATE: TERMINAL");
|
|
51
71
|
return `${lines.join("\n")}\n`;
|
|
52
72
|
}
|
|
73
|
+
|
|
74
|
+
export function formatCompactNextActionResult(result) {
|
|
75
|
+
return `${JSON.stringify(result)}\n`;
|
|
76
|
+
}
|
package/src/commands/route.js
CHANGED
|
@@ -1,11 +1,24 @@
|
|
|
1
1
|
import { evaluateRoute } from "../core/router.js";
|
|
2
2
|
import { persistRoute } from "../core/route-artifact.js";
|
|
3
3
|
import { readContract } from "../core/contract.js";
|
|
4
|
+
import { readConfig } from "../core/config.js";
|
|
4
5
|
import { withTaskMutation } from "../core/task-command.js";
|
|
5
6
|
|
|
6
|
-
export async function runRoute({ target, packageRoot, workType, surfaces, risks, platforms, behaviorChange, executableChange, taskId, task }) {
|
|
7
|
+
export async function runRoute({ target, packageRoot, workType, surfaces, risks, platforms, behaviorChange, executableChange, executionProfile = null, taskId, task }) {
|
|
7
8
|
return withTaskMutation(target, { taskId: taskId ?? task, packageRoot }, "route", async (ctx) => {
|
|
8
9
|
const effectiveTaskId = ctx?.taskId ?? null;
|
|
10
|
+
let contract = null;
|
|
11
|
+
try {
|
|
12
|
+
contract = (await readContract(target, packageRoot, { taskId: effectiveTaskId })).value;
|
|
13
|
+
} catch (error) {
|
|
14
|
+
if (error.code !== "ARTIFACT_MISSING") throw error;
|
|
15
|
+
}
|
|
16
|
+
let configuredProfile = "auto";
|
|
17
|
+
try {
|
|
18
|
+
configuredProfile = (await readConfig(target, packageRoot)).executionProfile ?? "auto";
|
|
19
|
+
} catch (error) {
|
|
20
|
+
if (error.code !== "ARTIFACT_MISSING") throw error;
|
|
21
|
+
}
|
|
9
22
|
const route = evaluateRoute({
|
|
10
23
|
workType,
|
|
11
24
|
surfaces,
|
|
@@ -13,6 +26,11 @@ export async function runRoute({ target, packageRoot, workType, surfaces, risks,
|
|
|
13
26
|
platforms,
|
|
14
27
|
behaviorChange,
|
|
15
28
|
executableChange,
|
|
29
|
+
}, {
|
|
30
|
+
contract,
|
|
31
|
+
taskDescriptor: ctx?.descriptor ?? null,
|
|
32
|
+
configuredProfile,
|
|
33
|
+
requestedProfile: executionProfile,
|
|
16
34
|
});
|
|
17
35
|
if (target && packageRoot) {
|
|
18
36
|
let contractFingerprint;
|
|
@@ -28,7 +46,7 @@ export async function runRoute({ target, packageRoot, workType, surfaces, risks,
|
|
|
28
46
|
}
|
|
29
47
|
|
|
30
48
|
export function formatRouteResult(result) {
|
|
31
|
-
const lines = ["Selected:"];
|
|
49
|
+
const lines = [`Execution profile: ${result.executionProfile?.resolved ?? "legacy"}`, "Selected:"];
|
|
32
50
|
if (result.guides.length === 0) {
|
|
33
51
|
lines.push("- none (use the relevant domain guide for this documentation task)");
|
|
34
52
|
} else {
|
|
@@ -7,6 +7,8 @@ import { readContract } from "../core/contract.js";
|
|
|
7
7
|
import { fileExists, ensureWithin } from "../core/filesystem.js";
|
|
8
8
|
import { E_TASK_NOT_FOUND } from "../core/error-codes.js";
|
|
9
9
|
import { resolveTaskClaimState } from "../core/task-claim-state.js";
|
|
10
|
+
import { readPersistedRoute } from "../core/route-artifact.js";
|
|
11
|
+
import { projectExecutionProfile } from "../core/execution-profile.js";
|
|
10
12
|
|
|
11
13
|
function taskError(code, message, artifacts = []) {
|
|
12
14
|
const error = new Error(message);
|
|
@@ -15,7 +17,7 @@ function taskError(code, message, artifacts = []) {
|
|
|
15
17
|
return error;
|
|
16
18
|
}
|
|
17
19
|
|
|
18
|
-
export async function runTaskShow({ target, packageRoot, taskId } = {}) {
|
|
20
|
+
export async function runTaskShow({ target, packageRoot, taskId, compact = false } = {}) {
|
|
19
21
|
const context = await resolveTaskContext(target, { taskId, packageRoot, explicitRequired: true, selectionMode: TASK_SELECTION_MODES.READ });
|
|
20
22
|
const effectiveTaskId = context.taskId;
|
|
21
23
|
|
|
@@ -56,7 +58,7 @@ export async function runTaskShow({ target, packageRoot, taskId } = {}) {
|
|
|
56
58
|
};
|
|
57
59
|
}
|
|
58
60
|
|
|
59
|
-
|
|
61
|
+
const result = {
|
|
60
62
|
taskId: effectiveTaskId,
|
|
61
63
|
taskKey: context.taskKey,
|
|
62
64
|
directory: taskDirectory(effectiveTaskId),
|
|
@@ -69,6 +71,29 @@ export async function runTaskShow({ target, packageRoot, taskId } = {}) {
|
|
|
69
71
|
createdAt: descriptor.createdAt,
|
|
70
72
|
updatedAt: descriptor.updatedAt,
|
|
71
73
|
};
|
|
74
|
+
if (!compact) return result;
|
|
75
|
+
let profile = null;
|
|
76
|
+
try {
|
|
77
|
+
const route = await readPersistedRoute(target, packageRoot, { taskId: effectiveTaskId });
|
|
78
|
+
profile = projectExecutionProfile(route.value);
|
|
79
|
+
} catch {
|
|
80
|
+
// Legacy routes remain readable without adaptive profile metadata.
|
|
81
|
+
}
|
|
82
|
+
return {
|
|
83
|
+
taskId: result.taskId,
|
|
84
|
+
phase: result.phase,
|
|
85
|
+
profile,
|
|
86
|
+
claimState: result.claimState,
|
|
87
|
+
mutationAllowed: result.mutationAllowed,
|
|
88
|
+
recoveryStatus: result.recoveryStatus,
|
|
89
|
+
lock: result.lock?.classification?.status ?? null,
|
|
90
|
+
artifacts: Object.fromEntries(Object.entries(result.artifacts).map(([name, artifact]) => [name, artifact.exists])),
|
|
91
|
+
errors: [...new Set([
|
|
92
|
+
...(result.reasonCodes ?? []),
|
|
93
|
+
...(result.errors ?? []).flatMap((error) => [error.code, error.causeCode]).filter(Boolean),
|
|
94
|
+
...(result.ownershipErrors ?? []).flatMap((error) => [error.code, error.causeCode]).filter(Boolean),
|
|
95
|
+
])],
|
|
96
|
+
};
|
|
72
97
|
}
|
|
73
98
|
|
|
74
99
|
export function formatTaskShowResult(result) {
|
|
@@ -93,3 +118,7 @@ export function formatTaskShowResult(result) {
|
|
|
93
118
|
];
|
|
94
119
|
return `${lines.join("\n")}\n`;
|
|
95
120
|
}
|
|
121
|
+
|
|
122
|
+
export function formatCompactTaskShowResult(result) {
|
|
123
|
+
return `${JSON.stringify(result)}\n`;
|
|
124
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { readWorkState } from "../core/work-state.js";
|
|
2
|
+
import { normalizeUsage, writeTaskUsage } from "../core/usage.js";
|
|
3
|
+
import { appendProtocolEvent } from "../core/events.js";
|
|
4
|
+
import { withTaskMutation } from "../core/task-command.js";
|
|
5
|
+
|
|
6
|
+
export async function runUsageRecord({
|
|
7
|
+
target,
|
|
8
|
+
packageRoot,
|
|
9
|
+
taskId,
|
|
10
|
+
provider = null,
|
|
11
|
+
model = null,
|
|
12
|
+
inputTokens = null,
|
|
13
|
+
outputTokens = null,
|
|
14
|
+
cacheReadTokens = null,
|
|
15
|
+
cacheWriteTokens = null,
|
|
16
|
+
totalTokens = null,
|
|
17
|
+
costUsd = null,
|
|
18
|
+
source = "ACTOR_REPORTED",
|
|
19
|
+
} = {}) {
|
|
20
|
+
if (source !== "ACTOR_REPORTED") {
|
|
21
|
+
const error = new Error("usage-record accepts only ACTOR_REPORTED; provider and host reports must cross the trusted integration boundary");
|
|
22
|
+
error.code = "E_USAGE_SOURCE_INVALID";
|
|
23
|
+
throw error;
|
|
24
|
+
}
|
|
25
|
+
return withTaskMutation(target, { taskId, packageRoot }, "usage-record", async (ctx) => {
|
|
26
|
+
const effectiveTaskId = ctx.taskId;
|
|
27
|
+
const usage = normalizeUsage({
|
|
28
|
+
provider,
|
|
29
|
+
model,
|
|
30
|
+
inputTokens,
|
|
31
|
+
outputTokens,
|
|
32
|
+
cacheReadTokens,
|
|
33
|
+
cacheWriteTokens,
|
|
34
|
+
totalTokens,
|
|
35
|
+
costUsd,
|
|
36
|
+
source,
|
|
37
|
+
}, { allowedSources: ["ACTOR_REPORTED"] });
|
|
38
|
+
const state = await readWorkState(target, { packageRoot, taskId: effectiveTaskId });
|
|
39
|
+
const artifact = await writeTaskUsage(target, packageRoot, {
|
|
40
|
+
taskId: effectiveTaskId,
|
|
41
|
+
usage,
|
|
42
|
+
recordedAt: new Date().toISOString(),
|
|
43
|
+
});
|
|
44
|
+
await appendProtocolEvent(target, {
|
|
45
|
+
taskId: effectiveTaskId,
|
|
46
|
+
event: "USAGE_RECORDED",
|
|
47
|
+
details: {
|
|
48
|
+
source: usage.source,
|
|
49
|
+
fields: Object.keys(usage).filter((key) => usage[key] !== null && key !== "source").sort(),
|
|
50
|
+
verificationCycle: state?.verificationCycle ?? null,
|
|
51
|
+
usageArtifact: artifact.path,
|
|
52
|
+
},
|
|
53
|
+
}, packageRoot, { taskId: effectiveTaskId });
|
|
54
|
+
return { taskId: effectiveTaskId, path: artifact.path, usage: artifact.value.usage, recordedAt: artifact.value.recordedAt };
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function formatUsageRecordResult(result) {
|
|
59
|
+
return `Usage recorded: ${result.path}\nSource: ${result.usage.source}\nTotal tokens: ${result.usage.totalTokens ?? "unknown"}\n`;
|
|
60
|
+
}
|
|
61
|
+
|
|
@@ -283,6 +283,18 @@ export const ARTIFACT_REGISTRY = Object.freeze({
|
|
|
283
283
|
isPersisted: true,
|
|
284
284
|
description: "Immutable trajectory evaluation results compiled from the canonical trace against a local reference scenario.",
|
|
285
285
|
}),
|
|
286
|
+
usage: Object.freeze({
|
|
287
|
+
key: "usage",
|
|
288
|
+
scope: "TASK",
|
|
289
|
+
path: `${TASK_STATE_ROOT}/<task-key>/${TASK_ARTIFACT_FILES.usage}`,
|
|
290
|
+
schema: "usage",
|
|
291
|
+
owner: "ACTOR_OR_TRUSTED_HOST",
|
|
292
|
+
mutability: "OVERWRITTEN_ON_USAGE_RECORD",
|
|
293
|
+
trustRole: "INFORMATIONAL_USAGE_TELEMETRY",
|
|
294
|
+
isPublic: true,
|
|
295
|
+
isPersisted: true,
|
|
296
|
+
description: "Task-scoped token, cost, model, and provider telemetry; never verification or completion evidence.",
|
|
297
|
+
}),
|
|
286
298
|
workspaceBinding: Object.freeze({
|
|
287
299
|
key: "workspaceBinding",
|
|
288
300
|
scope: "TASK",
|
|
@@ -139,6 +139,7 @@ export const CLI_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
139
139
|
"--platform": Object.freeze({ targetKey: "platforms", parseType: "string", takesValue: true, valueName: "value", repeatable: true, missingValueMessage: "--platform requires a value", description: "affected platform" }),
|
|
140
140
|
"--behavior-change": Object.freeze({ targetKey: "behaviorChange", parseType: "boolean", takesValue: false, description: "declare behavior change" }),
|
|
141
141
|
"--executable-change": Object.freeze({ targetKey: "executableChange", parseType: "boolean", takesValue: false, description: "declare executable/configuration change" }),
|
|
142
|
+
"--execution-profile": Object.freeze({ targetKey: "executionProfile", parseType: "string", takesValue: true, valueName: "profile", missingValueMessage: "--execution-profile requires auto, light, balanced, or full", description: "requested execution profile; safety floors always win" }),
|
|
142
143
|
"--json": Object.freeze({ targetKey: "json", parseType: "boolean", takesValue: false, description: "emit route result as JSON" }),
|
|
143
144
|
}),
|
|
144
145
|
writes: [".forgeloop/task-state/<taskKey>/routing-result.json"],
|
|
@@ -183,6 +184,7 @@ export const CLI_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
183
184
|
options: Object.freeze({
|
|
184
185
|
...CLI_COMMON_OPTIONS,
|
|
185
186
|
...CLI_TASK_OPTION,
|
|
187
|
+
"--compact": Object.freeze({ targetKey: "compact", parseType: "boolean", takesValue: false, description: "emit a bounded next-action projection" }),
|
|
186
188
|
"--json": Object.freeze({ targetKey: "json", parseType: "boolean", takesValue: false, description: "emit structured output as JSON" }),
|
|
187
189
|
}),
|
|
188
190
|
writes: [],
|
|
@@ -381,6 +383,30 @@ export const CLI_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
381
383
|
}), writes: [], removes: [], mayExecuteExternalProcess: false,
|
|
382
384
|
description: "Projects trajectory, action, execution, timing, and known usage metrics without mutating state.",
|
|
383
385
|
}),
|
|
386
|
+
"usage-record": Object.freeze({
|
|
387
|
+
name: "usage-record", category: "diagnostics", mutation: "MUTATING",
|
|
388
|
+
options: Object.freeze({ ...CLI_COMMON_OPTIONS, ...CLI_TASK_OPTION,
|
|
389
|
+
"--provider": Object.freeze({ targetKey: "usageProvider", parseType: "string", takesValue: true, valueName: "name", missingValueMessage: "--provider requires a name", description: "provider name reported by the actor" }),
|
|
390
|
+
"--model": Object.freeze({ targetKey: "usageModel", parseType: "string", takesValue: true, valueName: "name", missingValueMessage: "--model requires a name", description: "model name reported by the actor" }),
|
|
391
|
+
"--input-tokens": Object.freeze({ targetKey: "usageInputTokens", parseType: "non-negative-integer", takesValue: true, valueName: "number", missingValueMessage: "--input-tokens requires a non-negative integer", description: "provider-reported input token count" }),
|
|
392
|
+
"--output-tokens": Object.freeze({ targetKey: "usageOutputTokens", parseType: "non-negative-integer", takesValue: true, valueName: "number", missingValueMessage: "--output-tokens requires a non-negative integer", description: "provider-reported output token count" }),
|
|
393
|
+
"--cache-read-tokens": Object.freeze({ targetKey: "usageCacheReadTokens", parseType: "non-negative-integer", takesValue: true, valueName: "number", missingValueMessage: "--cache-read-tokens requires a non-negative integer", description: "provider-reported cache-read token count" }),
|
|
394
|
+
"--cache-write-tokens": Object.freeze({ targetKey: "usageCacheWriteTokens", parseType: "non-negative-integer", takesValue: true, valueName: "number", missingValueMessage: "--cache-write-tokens requires a non-negative integer", description: "provider-reported cache-write token count" }),
|
|
395
|
+
"--total-tokens": Object.freeze({ targetKey: "usageTotalTokens", parseType: "non-negative-integer", takesValue: true, valueName: "number", missingValueMessage: "--total-tokens requires a non-negative integer", description: "provider-reported total token count; never estimated" }),
|
|
396
|
+
"--cost-usd": Object.freeze({ targetKey: "usageCostUsd", parseType: "string", takesValue: true, valueName: "amount", missingValueMessage: "--cost-usd requires a non-negative amount", description: "provider-reported cost in USD; never estimated" }),
|
|
397
|
+
"--source": Object.freeze({ targetKey: "usageSource", parseType: "string", takesValue: true, valueName: "kind", missingValueMessage: "--source requires ACTOR_REPORTED", description: "usage source; CLI fallback accepts only ACTOR_REPORTED" }),
|
|
398
|
+
"--json": Object.freeze({ targetKey: "json", parseType: "boolean", takesValue: false, description: "emit usage telemetry as JSON" }),
|
|
399
|
+
}), writes: [".forgeloop/task-state/<taskKey>/usage.json", ".forgeloop/task-state/<taskKey>/events.ndjson"], removes: [], mayExecuteExternalProcess: false,
|
|
400
|
+
description: "Records actor-reported usage telemetry without treating it as verification evidence.",
|
|
401
|
+
}),
|
|
402
|
+
efficiency: Object.freeze({
|
|
403
|
+
name: "efficiency", category: "diagnostics", mutation: "READ_ONLY",
|
|
404
|
+
options: Object.freeze({ ...CLI_COMMON_OPTIONS, ...CLI_TASK_OPTION,
|
|
405
|
+
"--baseline": Object.freeze({ targetKey: "baselinePath", parseType: "string", takesValue: true, valueName: "path", missingValueMessage: "--baseline requires a project-local path", description: "optional comparable efficiency baseline JSON" }),
|
|
406
|
+
"--json": Object.freeze({ targetKey: "json", parseType: "boolean", takesValue: false, description: "emit efficiency metrics as JSON" }),
|
|
407
|
+
}), writes: [], removes: [], mayExecuteExternalProcess: false,
|
|
408
|
+
description: "Projects usage and timing efficiency, comparing only against a metadata-compatible local baseline.",
|
|
409
|
+
}),
|
|
384
410
|
eval: Object.freeze({
|
|
385
411
|
name: "eval", category: "diagnostics", mutation: "MUTATING",
|
|
386
412
|
options: Object.freeze({ ...CLI_COMMON_OPTIONS, ...CLI_TASK_OPTION,
|
|
@@ -900,6 +926,7 @@ export const CLI_COMMAND_DEFINITIONS = Object.freeze({
|
|
|
900
926
|
options: Object.freeze({
|
|
901
927
|
...CLI_COMMON_OPTIONS,
|
|
902
928
|
...CLI_TASK_OPTION,
|
|
929
|
+
"--compact": Object.freeze({ targetKey: "compact", parseType: "boolean", takesValue: false, description: "emit a bounded task-status projection" }),
|
|
903
930
|
"--json": Object.freeze({ targetKey: "json", parseType: "boolean", takesValue: false, description: "emit structured output as JSON" }),
|
|
904
931
|
}),
|
|
905
932
|
writes: [],
|
|
@@ -33,6 +33,8 @@ import { runActionAuthorize } from "../commands/action-authorize.js";
|
|
|
33
33
|
import { runActionVerify } from "../commands/action-verify.js";
|
|
34
34
|
import { runActionReconcile } from "../commands/action-reconcile.js";
|
|
35
35
|
import { runMetrics } from "../commands/metrics.js";
|
|
36
|
+
import { runUsageRecord } from "../commands/usage-record.js";
|
|
37
|
+
import { runEfficiency } from "../commands/efficiency.js";
|
|
36
38
|
import { runEval } from "../commands/eval.js";
|
|
37
39
|
import { runApprovalRequest } from "../commands/approval-request.js";
|
|
38
40
|
import { runApprovalResolve } from "../commands/approval-resolve.js";
|
|
@@ -117,6 +119,7 @@ export const COMMAND_EXECUTORS = {
|
|
|
117
119
|
platforms: options.platforms,
|
|
118
120
|
behaviorChange: options.behaviorChange,
|
|
119
121
|
executableChange: options.executableChange,
|
|
122
|
+
executionProfile: options.executionProfile,
|
|
120
123
|
taskId: options.taskId,
|
|
121
124
|
}),
|
|
122
125
|
exitCode: 0,
|
|
@@ -141,7 +144,7 @@ export const COMMAND_EXECUTORS = {
|
|
|
141
144
|
exitCode: 0,
|
|
142
145
|
}),
|
|
143
146
|
next: async ({ target, packageRoot, options, authorityContext, runtimeContext }) => ({
|
|
144
|
-
result: await runNext({ target, packageRoot, taskId: options.taskId, authorityContext, runtimeContext }),
|
|
147
|
+
result: await runNext({ target, packageRoot, taskId: options.taskId, authorityContext, runtimeContext, compact: options.compact }),
|
|
145
148
|
exitCode: 0,
|
|
146
149
|
}),
|
|
147
150
|
continuity: async ({ target, packageRoot, options }) => ({
|
|
@@ -305,9 +308,30 @@ export const COMMAND_EXECUTORS = {
|
|
|
305
308
|
// executor parameter; actor input can never supply it.
|
|
306
309
|
authorityContext,
|
|
307
310
|
}), exitCode: 0 }),
|
|
308
|
-
metrics: async ({ target, packageRoot, options }) => ({ result: await runMetrics({ target, packageRoot, taskId: options.taskId }), exitCode: 0 }),
|
|
309
|
-
|
|
310
|
-
|
|
311
|
+
metrics: async ({ target, packageRoot, options, runtimeContext }) => ({ result: await runMetrics({ target, packageRoot, taskId: options.taskId, runtimeContext }), exitCode: 0 }),
|
|
312
|
+
"usage-record": async ({ target, packageRoot, options }) => ({
|
|
313
|
+
result: await runUsageRecord({
|
|
314
|
+
target,
|
|
315
|
+
packageRoot,
|
|
316
|
+
taskId: options.taskId,
|
|
317
|
+
provider: options.usageProvider,
|
|
318
|
+
model: options.usageModel,
|
|
319
|
+
inputTokens: options.usageInputTokens,
|
|
320
|
+
outputTokens: options.usageOutputTokens,
|
|
321
|
+
cacheReadTokens: options.usageCacheReadTokens,
|
|
322
|
+
cacheWriteTokens: options.usageCacheWriteTokens,
|
|
323
|
+
totalTokens: options.usageTotalTokens,
|
|
324
|
+
costUsd: options.usageCostUsd,
|
|
325
|
+
source: options.usageSource ?? "ACTOR_REPORTED",
|
|
326
|
+
}),
|
|
327
|
+
exitCode: 0,
|
|
328
|
+
}),
|
|
329
|
+
efficiency: async ({ target, packageRoot, options, runtimeContext }) => ({
|
|
330
|
+
result: await runEfficiency({ target, packageRoot, taskId: options.taskId, baselinePath: options.baselinePath, runtimeContext }),
|
|
331
|
+
exitCode: 0,
|
|
332
|
+
}),
|
|
333
|
+
eval: async ({ target, packageRoot, options, runtimeContext }) => {
|
|
334
|
+
const result = await runEval({ target, packageRoot, taskId: options.taskId, scenarioPath: options.scenarioPath, runtimeContext });
|
|
311
335
|
return { result, exitCode: result.result === "PASS" ? 0 : 1 };
|
|
312
336
|
},
|
|
313
337
|
"approval-request": async ({ target, packageRoot, options }) => ({ result: await runApprovalRequest({ target, packageRoot, taskId: options.taskId, approvalId: options.approvalId, actionId: options.actionId, reason: options.reason }), exitCode: 0 }),
|
|
@@ -547,7 +571,7 @@ export const COMMAND_EXECUTORS = {
|
|
|
547
571
|
exitCode: 0,
|
|
548
572
|
}),
|
|
549
573
|
"task-show": async ({ target, packageRoot, options }) => ({
|
|
550
|
-
result: await runTaskShow({ target, packageRoot, taskId: options.taskId }),
|
|
574
|
+
result: await runTaskShow({ target, packageRoot, taskId: options.taskId, compact: options.compact }),
|
|
551
575
|
exitCode: 0,
|
|
552
576
|
}),
|
|
553
577
|
"task-lock-status": async ({ target, packageRoot, options }) => ({
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { continuityOptionDefaults, validateContinuityOptions } from "./continuity-cli-options.js";
|
|
2
2
|
import { E_CLI_INVOCATION_INVALID } from "./error-codes.js";
|
|
3
|
+
import { EXECUTION_PROFILE_REQUESTS } from "./execution-profile.js";
|
|
3
4
|
|
|
4
5
|
function inputError(message) {
|
|
5
6
|
const error = new Error(message);
|
|
@@ -17,6 +18,7 @@ export function defaultCommandInputValues() {
|
|
|
17
18
|
path: ".",
|
|
18
19
|
dryRun: false,
|
|
19
20
|
json: false,
|
|
21
|
+
compact: false,
|
|
20
22
|
strict: false,
|
|
21
23
|
fix: false,
|
|
22
24
|
adopt: [],
|
|
@@ -26,6 +28,17 @@ export function defaultCommandInputValues() {
|
|
|
26
28
|
platforms: [],
|
|
27
29
|
behaviorChange: false,
|
|
28
30
|
executableChange: false,
|
|
31
|
+
executionProfile: null,
|
|
32
|
+
usageProvider: null,
|
|
33
|
+
usageModel: null,
|
|
34
|
+
usageInputTokens: null,
|
|
35
|
+
usageOutputTokens: null,
|
|
36
|
+
usageCacheReadTokens: null,
|
|
37
|
+
usageCacheWriteTokens: null,
|
|
38
|
+
usageTotalTokens: null,
|
|
39
|
+
usageCostUsd: null,
|
|
40
|
+
usageSource: "ACTOR_REPORTED",
|
|
41
|
+
baselinePath: null,
|
|
29
42
|
to: null,
|
|
30
43
|
file: null,
|
|
31
44
|
contractFile: null,
|
|
@@ -102,6 +115,27 @@ export function validateForgeLoopCommandInput({ command, input, help = false } =
|
|
|
102
115
|
if (command === "task-create" && !options.taskId) {
|
|
103
116
|
throw inputError("task-create requires --task");
|
|
104
117
|
}
|
|
118
|
+
if (options.executionProfile !== null && options.executionProfile !== undefined) {
|
|
119
|
+
if (command !== "route") throw inputError(`executionProfile is not valid for ${command}`);
|
|
120
|
+
if (!EXECUTION_PROFILE_REQUESTS.includes(options.executionProfile)) {
|
|
121
|
+
throw inputError(`route --execution-profile must be one of ${EXECUTION_PROFILE_REQUESTS.join(", ")}`);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if (command === "usage-record" && !help) {
|
|
125
|
+
if (!options.taskId) throw inputError("usage-record requires --task");
|
|
126
|
+
if ((options.usageSource ?? "ACTOR_REPORTED") !== "ACTOR_REPORTED") {
|
|
127
|
+
throw inputError("usage-record accepts only --source ACTOR_REPORTED");
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
if (command === "efficiency" && !help && !options.taskId) {
|
|
131
|
+
throw inputError("efficiency requires --task");
|
|
132
|
+
}
|
|
133
|
+
if (command !== "usage-record" && options.usageSource !== undefined && options.usageSource !== "ACTOR_REPORTED") {
|
|
134
|
+
throw inputError(`usageSource is not valid for ${command}`);
|
|
135
|
+
}
|
|
136
|
+
if (options.compact === true && !["next", "task-show"].includes(command)) {
|
|
137
|
+
throw inputError(`compact output is not valid for ${command}`);
|
|
138
|
+
}
|
|
105
139
|
if (["workspace-bind", "workspace-status", "handoff-create", "handoff-list", "handoff-show", "responsibility-set", "responsibility-status", "verify-scope", "attestation-create", "attestation-status", "attestation-verify"].includes(command)
|
|
106
140
|
&& !options.taskId) {
|
|
107
141
|
throw inputError(`${command} requires --task`);
|
package/src/core/config.js
CHANGED
|
@@ -2,6 +2,7 @@ import { PROTOCOL_VERSION } from "./protocol.js";
|
|
|
2
2
|
import { ARTIFACT_PATHS, readJsonArtifact, writeJsonArtifact } from "./artifacts.js";
|
|
3
3
|
import { E_ATTESTATION_CONFIGURATION_INVALID } from "./error-codes.js";
|
|
4
4
|
import { normalizeVerificationConfiguration } from "./verification-scope-capability.js";
|
|
5
|
+
import { EXECUTION_PROFILE_REQUESTS } from "./execution-profile.js";
|
|
5
6
|
|
|
6
7
|
export const CONFIG_SCHEMA_VERSION = 1;
|
|
7
8
|
export const COMPLIANCE_MODES = Object.freeze(["advisory", "standard", "strict"]);
|
|
@@ -72,6 +73,13 @@ export function createConfig(input = {}) {
|
|
|
72
73
|
throw configurationError(error.message);
|
|
73
74
|
}
|
|
74
75
|
}
|
|
76
|
+
let executionProfile;
|
|
77
|
+
if (input.executionProfile !== undefined) {
|
|
78
|
+
if (!EXECUTION_PROFILE_REQUESTS.includes(input.executionProfile)) {
|
|
79
|
+
throw configurationError(`Unknown execution profile: ${input.executionProfile}`);
|
|
80
|
+
}
|
|
81
|
+
executionProfile = input.executionProfile;
|
|
82
|
+
}
|
|
75
83
|
return {
|
|
76
84
|
schemaVersion: CONFIG_SCHEMA_VERSION,
|
|
77
85
|
protocolVersion: PROTOCOL_VERSION,
|
|
@@ -81,6 +89,7 @@ export function createConfig(input = {}) {
|
|
|
81
89
|
...(input.requiredEvidence !== undefined ? { requiredEvidence: stringArray(input.requiredEvidence, "requiredEvidence") } : {}),
|
|
82
90
|
...(verification ? { verification } : {}),
|
|
83
91
|
...(attestation ? { attestation } : {}),
|
|
92
|
+
...(executionProfile ? { executionProfile } : {}),
|
|
84
93
|
};
|
|
85
94
|
}
|
|
86
95
|
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { assertSafePath, ensureWithin } from "./filesystem.js";
|
|
4
|
+
import { readContract } from "./contract.js";
|
|
5
|
+
import { currentRepositoryFingerprint } from "./repository.js";
|
|
6
|
+
import { buildTrajectoryMetrics } from "./trajectory-metrics.js";
|
|
7
|
+
|
|
8
|
+
function efficiencyError(code, message) {
|
|
9
|
+
const error = new Error(message);
|
|
10
|
+
error.code = code;
|
|
11
|
+
return error;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function environmentClass() {
|
|
15
|
+
return `${process.platform}-node${process.versions.node.split(".")[0]}`;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function validNumber(value) {
|
|
19
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function validateBaseline(value) {
|
|
23
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
24
|
+
throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", "efficiency baseline must be a JSON object");
|
|
25
|
+
}
|
|
26
|
+
if (!Number.isInteger(value.comparableSteps) || value.comparableSteps < 0) {
|
|
27
|
+
throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", "efficiency baseline comparableSteps must be a non-negative integer");
|
|
28
|
+
}
|
|
29
|
+
if (value.metadata !== undefined && (!value.metadata || typeof value.metadata !== "object" || Array.isArray(value.metadata))) {
|
|
30
|
+
throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", "efficiency baseline metadata must be an object");
|
|
31
|
+
}
|
|
32
|
+
if (value.usage !== undefined && (!value.usage || typeof value.usage !== "object" || Array.isArray(value.usage))) {
|
|
33
|
+
throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", "efficiency baseline usage must be an object");
|
|
34
|
+
}
|
|
35
|
+
if (value.timing !== undefined && (!value.timing || typeof value.timing !== "object" || Array.isArray(value.timing))) {
|
|
36
|
+
throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", "efficiency baseline timing must be an object");
|
|
37
|
+
}
|
|
38
|
+
if (value.usage?.totalTokens !== undefined && value.usage.totalTokens !== null && !validNumber(value.usage.totalTokens)) {
|
|
39
|
+
throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", "efficiency baseline usage.totalTokens must be a non-negative number or null");
|
|
40
|
+
}
|
|
41
|
+
if (value.timing?.wallClockMs !== undefined && value.timing.wallClockMs !== null && !validNumber(value.timing.wallClockMs)) {
|
|
42
|
+
throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", "efficiency baseline timing.wallClockMs must be a non-negative number or null");
|
|
43
|
+
}
|
|
44
|
+
return value;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function compareMetadata(actual, baseline) {
|
|
48
|
+
const expected = baseline.metadata ?? {};
|
|
49
|
+
const fields = [
|
|
50
|
+
"taskId",
|
|
51
|
+
"scenarioId",
|
|
52
|
+
"model",
|
|
53
|
+
"provider",
|
|
54
|
+
"promptSpecFingerprint",
|
|
55
|
+
"projectRevision",
|
|
56
|
+
"benchmarkVersion",
|
|
57
|
+
"environmentClass",
|
|
58
|
+
"usageSource",
|
|
59
|
+
];
|
|
60
|
+
const mismatches = [];
|
|
61
|
+
for (const field of fields) {
|
|
62
|
+
if (!Object.prototype.hasOwnProperty.call(expected, field)) continue;
|
|
63
|
+
if (actual[field] === undefined || actual[field] === null || actual[field] !== expected[field]) {
|
|
64
|
+
mismatches.push({ field, expected: expected[field] ?? null, actual: actual[field] ?? null });
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
const required = ["taskId", "model", "provider", "projectRevision", "environmentClass"];
|
|
68
|
+
for (const field of required) {
|
|
69
|
+
if (!Object.prototype.hasOwnProperty.call(expected, field)) {
|
|
70
|
+
mismatches.push({ field, expected: "required", actual: actual[field] ?? null });
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return { comparable: mismatches.length === 0, mismatches };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function ratio(actual, baseline) {
|
|
77
|
+
if (!validNumber(actual) || !validNumber(baseline) || baseline === 0) return null;
|
|
78
|
+
return Number(((actual - baseline) / baseline * 100).toFixed(4));
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export async function readEfficiencyBaseline(target, baselinePath) {
|
|
82
|
+
if (typeof baselinePath !== "string" || !baselinePath.trim() || path.isAbsolute(baselinePath)) {
|
|
83
|
+
throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", "baseline path must be a relative project-local file");
|
|
84
|
+
}
|
|
85
|
+
try {
|
|
86
|
+
await assertSafePath(target, baselinePath);
|
|
87
|
+
const parsed = JSON.parse(await readFile(ensureWithin(target, baselinePath), "utf8"));
|
|
88
|
+
return validateBaseline(parsed);
|
|
89
|
+
} catch (error) {
|
|
90
|
+
if (error.code === "E_EFFICIENCY_BASELINE_INVALID") throw error;
|
|
91
|
+
throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", `unable to read baseline safely: ${error.message}`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export async function buildEfficiencyReport({ target, packageRoot, taskId, baselinePath = null, runtimeContext = null } = {}) {
|
|
96
|
+
const metrics = await buildTrajectoryMetrics({ target, packageRoot, taskId, runtimeContext });
|
|
97
|
+
const repository = await currentRepositoryFingerprint(target);
|
|
98
|
+
let promptSpecFingerprint = null;
|
|
99
|
+
try {
|
|
100
|
+
promptSpecFingerprint = (await readContract(target, packageRoot, { taskId })).fingerprint;
|
|
101
|
+
} catch {
|
|
102
|
+
// A task without a contract is not comparable to a contract-bound baseline.
|
|
103
|
+
}
|
|
104
|
+
const actualMetadata = {
|
|
105
|
+
taskId,
|
|
106
|
+
scenarioId: taskId,
|
|
107
|
+
model: metrics.usage.model,
|
|
108
|
+
provider: metrics.usage.provider,
|
|
109
|
+
promptSpecFingerprint,
|
|
110
|
+
projectRevision: repository.head ?? null,
|
|
111
|
+
benchmarkVersion: null,
|
|
112
|
+
environmentClass: environmentClass(),
|
|
113
|
+
usageSource: metrics.usage.source,
|
|
114
|
+
};
|
|
115
|
+
const base = {
|
|
116
|
+
taskId,
|
|
117
|
+
executionProfile: metrics.executionProfile,
|
|
118
|
+
usage: metrics.usage,
|
|
119
|
+
usageSource: metrics.usage.source,
|
|
120
|
+
timing: metrics.timing,
|
|
121
|
+
tokensPerComparableStep: validNumber(metrics.usage.totalTokens) && metrics.comparableSteps > 0
|
|
122
|
+
? metrics.usage.totalTokens / metrics.comparableSteps
|
|
123
|
+
: null,
|
|
124
|
+
};
|
|
125
|
+
if (!baselinePath) {
|
|
126
|
+
return {
|
|
127
|
+
...base,
|
|
128
|
+
comparison: {
|
|
129
|
+
status: "NOT_COMPARABLE",
|
|
130
|
+
comparable: false,
|
|
131
|
+
usageSource: metrics.usage.source,
|
|
132
|
+
reason: "No baseline was supplied.",
|
|
133
|
+
metadata: actualMetadata,
|
|
134
|
+
baseline: null,
|
|
135
|
+
ratios: { comparableStepsPercent: null, totalTokensPercent: null, wallClockMsPercent: null },
|
|
136
|
+
tokenOverheadRatio: null,
|
|
137
|
+
timeOverheadRatio: null,
|
|
138
|
+
tokenOverheadPercent: null,
|
|
139
|
+
timeOverheadPercent: null,
|
|
140
|
+
},
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const baseline = await readEfficiencyBaseline(target, baselinePath);
|
|
145
|
+
const metadata = compareMetadata(actualMetadata, baseline);
|
|
146
|
+
if (!metadata.comparable) {
|
|
147
|
+
return {
|
|
148
|
+
...base,
|
|
149
|
+
comparison: {
|
|
150
|
+
status: "NOT_COMPARABLE",
|
|
151
|
+
comparable: false,
|
|
152
|
+
usageSource: metrics.usage.source,
|
|
153
|
+
reason: "Baseline metadata does not match the current task execution.",
|
|
154
|
+
metadata: actualMetadata,
|
|
155
|
+
mismatches: metadata.mismatches,
|
|
156
|
+
baseline: { path: baselinePath, comparableSteps: baseline.comparableSteps },
|
|
157
|
+
ratios: { comparableStepsPercent: null, totalTokensPercent: null, wallClockMsPercent: null },
|
|
158
|
+
tokenOverheadRatio: null,
|
|
159
|
+
timeOverheadRatio: null,
|
|
160
|
+
tokenOverheadPercent: null,
|
|
161
|
+
timeOverheadPercent: null,
|
|
162
|
+
},
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
return {
|
|
167
|
+
...base,
|
|
168
|
+
comparison: {
|
|
169
|
+
status: "COMPARABLE",
|
|
170
|
+
comparable: true,
|
|
171
|
+
usageSource: metrics.usage.source,
|
|
172
|
+
reason: "Baseline metadata matches the current task execution.",
|
|
173
|
+
metadata: actualMetadata,
|
|
174
|
+
baseline: {
|
|
175
|
+
path: baselinePath,
|
|
176
|
+
comparableSteps: baseline.comparableSteps,
|
|
177
|
+
usage: baseline.usage ?? null,
|
|
178
|
+
timing: baseline.timing ?? null,
|
|
179
|
+
},
|
|
180
|
+
ratios: {
|
|
181
|
+
comparableStepsPercent: ratio(metrics.comparableSteps, baseline.comparableSteps),
|
|
182
|
+
totalTokensPercent: ratio(metrics.usage.totalTokens, baseline.usage?.totalTokens),
|
|
183
|
+
wallClockMsPercent: ratio(metrics.timing.wallClockMs, baseline.timing?.wallClockMs),
|
|
184
|
+
},
|
|
185
|
+
tokenOverheadRatio: validNumber(metrics.usage.totalTokens) && validNumber(baseline.usage?.totalTokens)
|
|
186
|
+
&& baseline.usage.totalTokens !== 0
|
|
187
|
+
? Number((metrics.usage.totalTokens / baseline.usage.totalTokens).toFixed(4))
|
|
188
|
+
: null,
|
|
189
|
+
timeOverheadRatio: validNumber(metrics.timing.wallClockMs) && validNumber(baseline.timing?.wallClockMs)
|
|
190
|
+
&& baseline.timing.wallClockMs !== 0
|
|
191
|
+
? Number((metrics.timing.wallClockMs / baseline.timing.wallClockMs).toFixed(4))
|
|
192
|
+
: null,
|
|
193
|
+
tokenOverheadPercent: ratio(metrics.usage.totalTokens, baseline.usage?.totalTokens),
|
|
194
|
+
timeOverheadPercent: ratio(metrics.timing.wallClockMs, baseline.timing?.wallClockMs),
|
|
195
|
+
},
|
|
196
|
+
};
|
|
197
|
+
}
|