@tea-agent/loop-agent 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +59 -11
- package/README.md +1 -1
- package/dist/application/evaluation/alias.js +184 -0
- package/dist/application/evaluation/budget.js +192 -0
- package/dist/application/evaluation/campaign-hash.js +47 -0
- package/dist/application/evaluation/campaign-matrix.js +372 -0
- package/dist/application/evaluation/campaign-scorecard.js +135 -0
- package/dist/application/evaluation/campaign.js +370 -0
- package/dist/application/evaluation/candidate.js +23 -6
- package/dist/application/evaluation/corpus-hash.js +38 -0
- package/dist/application/evaluation/corpus.js +56 -0
- package/dist/application/evaluation/experiment.js +294 -0
- package/dist/application/evaluation/ignition.js +198 -0
- package/dist/application/evaluation/integrity-audit.js +162 -0
- package/dist/application/evaluation/outer-loop.js +132 -0
- package/dist/application/evaluation/pi-cell-executor.js +39 -0
- package/dist/application/evaluation/private-verifier.js +46 -0
- package/dist/application/evaluation/promotion-policy.js +151 -0
- package/dist/application/evaluation/proposer.js +98 -0
- package/dist/application/evaluation/types.js +522 -0
- package/dist/cli/command-definitions.js +19 -3
- package/dist/commands/eval.js +1176 -13
- package/dist/commands/init.js +4 -1
- package/dist/infrastructure/evaluation/alias-store.js +199 -0
- package/dist/infrastructure/evaluation/campaign-store.js +154 -0
- package/dist/infrastructure/evaluation/corpus-store.js +181 -0
- package/dist/infrastructure/evaluation/experiment-store.js +124 -0
- package/dist/infrastructure/evaluation/ignition-store.js +82 -0
- package/dist/infrastructure/evaluation/private-verifier-store.js +145 -0
- package/dist/infrastructure/evaluation/proposer-store.js +78 -0
- package/dist/worker/cli.js +6 -3
- package/dist/worker/delivery/final-verification.js +96 -8
- package/dist/worker/delivery/package.js +23 -4
- package/dist/worker/delivery/verification-bundle.js +521 -0
- package/dist/worker/feature/fullstack-validate.js +337 -0
- package/dist/worker/feature/profile-schema.js +44 -0
- package/dist/worker/feature/ready-plan-projection.js +1 -0
- package/dist/worker/feature/reducer.js +2 -0
- package/dist/worker/feature/review.js +106 -11
- package/dist/worker/materialize/harness-task-materializer.js +5 -0
- package/dist/worker/observability/read-model.js +7 -0
- package/dist/worker/observe/static/views/task.js +1 -0
- package/dist/worker/outcomes/adapters.js +144 -0
- package/dist/worker/outcomes/evidence-tokens.js +29 -0
- package/dist/worker/outcomes/gate.js +40 -0
- package/dist/worker/outcomes/projector.js +185 -0
- package/dist/worker/outcomes/registry.js +1 -0
- package/dist/worker/outcomes/store.js +131 -0
- package/dist/worker/outcomes/types.js +79 -0
- package/dist/worker/report/morning-report.js +4 -3
- package/dist/worker/run-task/run-task.js +85 -2
- package/dist/worker/runner/run-ready.js +32 -1
- package/dist/worker/task-graph/acceptance-schema.js +12 -0
- package/dist/worker/task-graph/ready-planner.js +131 -0
- package/dist/worker/task-graph/task-graph-schema.js +31 -0
- package/dist/worker/task-graph/validate.js +44 -4
- package/dist/worker/task-spec/schema.js +9 -0
- package/dist/worker/task-spec/validate.js +39 -0
- package/dist/worker/task-spec/workflow-routing.js +149 -0
- package/dist/workflows/dag/budget-enforcement.js +67 -0
- package/dist/workflows/dag/context-policy.js +137 -0
- package/dist/workflows/dag/knowledge-curator.js +3 -0
- package/dist/workflows/dag/node-execution.js +11 -4
- package/dist/workflows/dag/prompt.js +1 -1
- package/dist/workflows/dag/runner.js +43 -16
- package/dist/workflows/dag/skill-snapshot.js +11 -7
- package/dist/workflows/dag/types.js +18 -0
- package/docs/init-surface.manifest.json +3 -0
- package/docs/templates/evaluation/campaign-budget-v1.json +12 -0
- package/docs/templates/evaluation/campaign-dogfood-v0.json +24 -0
- package/docs/templates/evaluation/campaign-evidence-v1.json +44 -0
- package/docs/templates/evaluation/context-policy-baseline-v1.json +17 -0
- package/docs/templates/evaluation/context-policy-role-specialized-v1.json +28 -0
- package/docs/templates/evaluation/corpus-dogfood-v0.manifest.json +118 -0
- package/docs/templates/evaluation/matrix-dag-dry-run-v1.json +21 -0
- package/docs/templates/evaluation/matrix-fixture-v1.json +10 -0
- package/docs/templates/evaluation/private-verifier-dogfood-v0.json +16 -0
- package/docs/templates/product-line/AGENTS.md +1 -0
- package/docs/templates/product-line/README.md +17 -0
- package/docs/templates/product-line/acceptance.yaml +9 -0
- package/docs/templates/product-line/feature.yaml +11 -0
- package/docs/templates/product-line/task-graph.yaml +8 -0
- package/docs/templates/product-line/task.yaml +4 -0
- package/harness.json +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
/**
|
|
3
|
+
* Explicit runtime workflow that routes a TaskSpec to a controller DAG.
|
|
4
|
+
*
|
|
5
|
+
* The workflow is orthogonal to the business {@link TaskSpec.type} and the
|
|
6
|
+
* governance `loop_agent` profile. It is the only field the materializer uses
|
|
7
|
+
* to derive {@link TaskKind} (written to `task.json.taskKind`) which the
|
|
8
|
+
* published controller turns into a concrete DAG.
|
|
9
|
+
*/
|
|
10
|
+
export const workflowSchema = z.enum([
|
|
11
|
+
"agent-dag",
|
|
12
|
+
"frontend-implementation",
|
|
13
|
+
"backend-test",
|
|
14
|
+
"frontend-test",
|
|
15
|
+
]);
|
|
16
|
+
/**
|
|
17
|
+
* Deterministic workflow → taskKind mapping.
|
|
18
|
+
*
|
|
19
|
+
* - `agent-dag` → `standard` (the default agent DAG).
|
|
20
|
+
* - The three remaining workflows map 1:1 to their same-named taskKind.
|
|
21
|
+
*/
|
|
22
|
+
export const WORKFLOW_TASK_KIND = {
|
|
23
|
+
"agent-dag": "standard",
|
|
24
|
+
"frontend-implementation": "frontend-implementation",
|
|
25
|
+
"backend-test": "backend-test",
|
|
26
|
+
"frontend-test": "frontend-test",
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* Legacy QA business types that have no deterministic backend/frontend split.
|
|
30
|
+
* Without an explicit {@link Workflow} they MUST surface migration guidance
|
|
31
|
+
* rather than silently guessing `backend-test` or `frontend-test`.
|
|
32
|
+
*/
|
|
33
|
+
export const LEGACY_QA_TYPES = new Set([
|
|
34
|
+
"qa-casegen",
|
|
35
|
+
"qa-testcode",
|
|
36
|
+
"qa-execute",
|
|
37
|
+
]);
|
|
38
|
+
/**
|
|
39
|
+
* Legacy business types that deterministically route to `frontend-implementation`.
|
|
40
|
+
*/
|
|
41
|
+
const FRONTEND_FEATURE_TYPES = new Set(["frontend-feature"]);
|
|
42
|
+
/**
|
|
43
|
+
* Legacy business types that deterministically route to the `agent-dag`/`standard`
|
|
44
|
+
* DAG. Anything that is not a QA type and not a frontend-feature lands here.
|
|
45
|
+
*/
|
|
46
|
+
export function legacyDefaultWorkflow(type) {
|
|
47
|
+
return FRONTEND_FEATURE_TYPES.has(type) ? "frontend-implementation" : "agent-dag";
|
|
48
|
+
}
|
|
49
|
+
/** Migration guidance surfaced whenever a legacy QA type lacks an explicit workflow. */
|
|
50
|
+
export const LEGACY_QA_MIGRATION_GUIDANCE = "Legacy qa-casegen/qa-testcode/qa-execute tasks no longer auto-select a backend or frontend workflow. " +
|
|
51
|
+
"Add `execution.workflow` with one of: agent-dag, frontend-implementation, backend-test, frontend-test. " +
|
|
52
|
+
"Use backend-test for backend-only test generation/execution, frontend-test for browser/UI tests, " +
|
|
53
|
+
"or agent-dag when the QA task is a generic agent study.";
|
|
54
|
+
export class WorkflowRoutingError extends Error {
|
|
55
|
+
code;
|
|
56
|
+
migrationGuidance;
|
|
57
|
+
constructor(code, message, migrationGuidance) {
|
|
58
|
+
super(message);
|
|
59
|
+
this.name = "WorkflowRoutingError";
|
|
60
|
+
this.code = code;
|
|
61
|
+
this.migrationGuidance = migrationGuidance;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Resolve the runtime {@link Workflow} and {@link TaskKind} for a TaskSpec.
|
|
66
|
+
*
|
|
67
|
+
* Resolution order:
|
|
68
|
+
* 1. Explicit `taskSpec.execution?.workflow` (source: `explicit`).
|
|
69
|
+
* 2. Legacy `frontend-feature` → `frontend-implementation`.
|
|
70
|
+
* 3. Legacy `backend-feature` (and any non-QA legacy type) → `agent-dag`.
|
|
71
|
+
* 4. Legacy QA types ({@link LEGACY_QA_TYPES}) without an explicit workflow →
|
|
72
|
+
* preserve their historic default `standard` execution as `agent-dag` and
|
|
73
|
+
* emit migration guidance during validation. We never guess backend vs frontend.
|
|
74
|
+
*
|
|
75
|
+
* Compatibility between an explicit workflow and the business `type` must be
|
|
76
|
+
* validated first via {@link validateWorkflowCompatibility}; this function
|
|
77
|
+
* assumes the combination is already known-valid.
|
|
78
|
+
*/
|
|
79
|
+
export function resolveWorkflow(taskSpec) {
|
|
80
|
+
const explicit = taskSpec.execution?.workflow;
|
|
81
|
+
if (explicit) {
|
|
82
|
+
return {
|
|
83
|
+
workflow: explicit,
|
|
84
|
+
taskKind: WORKFLOW_TASK_KIND[explicit],
|
|
85
|
+
source: "explicit",
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
if (LEGACY_QA_TYPES.has(taskSpec.type)) {
|
|
89
|
+
return {
|
|
90
|
+
workflow: "agent-dag",
|
|
91
|
+
taskKind: WORKFLOW_TASK_KIND["agent-dag"],
|
|
92
|
+
source: "legacy-warning",
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
const workflow = legacyDefaultWorkflow(taskSpec.type);
|
|
96
|
+
return {
|
|
97
|
+
workflow,
|
|
98
|
+
taskKind: WORKFLOW_TASK_KIND[workflow],
|
|
99
|
+
source: "legacy-deterministic",
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Deterministic compatibility validation between an explicit workflow and the
|
|
104
|
+
* business {@link TaskSpec.type}. Legacy QA types without an explicit workflow
|
|
105
|
+
* surface a single migration-guidance issue instead of a routing decision.
|
|
106
|
+
*
|
|
107
|
+
* Incompatible combinations (e.g. `frontend-feature` + `backend-test`,
|
|
108
|
+
* `backend-feature` + `frontend-implementation`, or a `qa-*` type paired with
|
|
109
|
+
* an implementation/test workflow from the wrong surface) are reported so the
|
|
110
|
+
* validator can fail fast rather than silently materializing a wrong DAG.
|
|
111
|
+
*/
|
|
112
|
+
export function validateWorkflowCompatibility(taskSpec) {
|
|
113
|
+
const explicit = taskSpec.execution?.workflow;
|
|
114
|
+
if (!explicit) {
|
|
115
|
+
if (LEGACY_QA_TYPES.has(taskSpec.type)) {
|
|
116
|
+
return { issues: [], migrationGuidance: LEGACY_QA_MIGRATION_GUIDANCE };
|
|
117
|
+
}
|
|
118
|
+
// Legacy backend/frontend feature and other non-QA types route deterministically.
|
|
119
|
+
return { issues: [] };
|
|
120
|
+
}
|
|
121
|
+
const issues = [];
|
|
122
|
+
const type = taskSpec.type;
|
|
123
|
+
if (type === "backend-feature" && explicit !== "agent-dag" && explicit !== "backend-test") {
|
|
124
|
+
issues.push({
|
|
125
|
+
code: "workflow-type-incompatible",
|
|
126
|
+
message: `backend-feature is incompatible with workflow "${explicit}" (expected agent-dag or backend-test)`,
|
|
127
|
+
path: "execution.workflow",
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
if (type === "frontend-feature" && explicit !== "frontend-implementation" && explicit !== "frontend-test") {
|
|
131
|
+
issues.push({
|
|
132
|
+
code: "workflow-type-incompatible",
|
|
133
|
+
message: `frontend-feature is incompatible with workflow "${explicit}" (expected frontend-implementation or frontend-test)`,
|
|
134
|
+
path: "execution.workflow",
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
if (LEGACY_QA_TYPES.has(type)) {
|
|
138
|
+
// qa-* types are allowed to opt into a concrete test/implementation workflow
|
|
139
|
+
// but never into a feature implementation surface that does not match QA intent.
|
|
140
|
+
if (explicit === "frontend-implementation") {
|
|
141
|
+
issues.push({
|
|
142
|
+
code: "workflow-type-incompatible",
|
|
143
|
+
message: `${type} is a QA type and cannot route to frontend-implementation; use frontend-test, backend-test, or agent-dag`,
|
|
144
|
+
path: "execution.workflow",
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return { issues };
|
|
149
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { writeFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { checkBudgetPreNode, createBudgetLedger, formatBudgetReportMarkdown, isHardBudgetBreached, recordNodeBudgetSample, skipPendingNodesForBudgetBreach, } from "../../application/evaluation/budget.js";
|
|
4
|
+
export function initRunBudgetLedger(state, budget) {
|
|
5
|
+
if (!budget)
|
|
6
|
+
return;
|
|
7
|
+
state.budget = structuredClone(budget);
|
|
8
|
+
state.budgetLedger = createBudgetLedger(budget);
|
|
9
|
+
}
|
|
10
|
+
function wallTimeMs(state, now = Date.now()) {
|
|
11
|
+
return Math.max(0, now - new Date(state.startedAt).getTime());
|
|
12
|
+
}
|
|
13
|
+
function repairPasses(state) {
|
|
14
|
+
return state.convergence?.currentPass ?? 0;
|
|
15
|
+
}
|
|
16
|
+
export function applyHardBudgetBreach(state, breach) {
|
|
17
|
+
if (!state.budgetLedger || state.budgetLedger.mode !== "hard")
|
|
18
|
+
return;
|
|
19
|
+
skipPendingNodesForBudgetBreach(state.nodes, breach);
|
|
20
|
+
state.failureCategory = "budget_breach";
|
|
21
|
+
}
|
|
22
|
+
export function preflightBudgetOrBreach(state) {
|
|
23
|
+
const ledger = state.budgetLedger;
|
|
24
|
+
if (!ledger)
|
|
25
|
+
return undefined;
|
|
26
|
+
const breach = checkBudgetPreNode(ledger, {
|
|
27
|
+
wallTimeMs: wallTimeMs(state),
|
|
28
|
+
repairPasses: repairPasses(state),
|
|
29
|
+
});
|
|
30
|
+
if (breach && isHardBudgetBreached(ledger)) {
|
|
31
|
+
applyHardBudgetBreach(state, breach);
|
|
32
|
+
}
|
|
33
|
+
return breach;
|
|
34
|
+
}
|
|
35
|
+
export function recordFinishedNodeBudget(state, node, contextChars) {
|
|
36
|
+
const ledger = state.budgetLedger;
|
|
37
|
+
if (!ledger)
|
|
38
|
+
return undefined;
|
|
39
|
+
const breach = recordNodeBudgetSample(ledger, {
|
|
40
|
+
nodeId: node.id,
|
|
41
|
+
tokensUsed: node.tokensUsed,
|
|
42
|
+
contextChars,
|
|
43
|
+
wallTimeMs: wallTimeMs(state),
|
|
44
|
+
repairPasses: repairPasses(state),
|
|
45
|
+
});
|
|
46
|
+
if (breach && isHardBudgetBreached(ledger)) {
|
|
47
|
+
applyHardBudgetBreach(state, breach);
|
|
48
|
+
}
|
|
49
|
+
return breach;
|
|
50
|
+
}
|
|
51
|
+
export async function writeBudgetLedgerArtifacts(runDir, ledger) {
|
|
52
|
+
if (!ledger)
|
|
53
|
+
return;
|
|
54
|
+
const jsonPath = path.join(runDir, "budget-ledger.json");
|
|
55
|
+
const mdPath = path.join(runDir, "budget-report.md");
|
|
56
|
+
await writeFile(jsonPath, `${JSON.stringify(ledger, null, 2)}\n`, "utf8");
|
|
57
|
+
await writeFile(mdPath, formatBudgetReportMarkdown(ledger), "utf8");
|
|
58
|
+
}
|
|
59
|
+
export function assertFrozenBudget(specBudget, stateBudget, runId) {
|
|
60
|
+
if (!specBudget && !stateBudget)
|
|
61
|
+
return;
|
|
62
|
+
if (!specBudget ||
|
|
63
|
+
!stateBudget ||
|
|
64
|
+
JSON.stringify(specBudget) !== JSON.stringify(stateBudget)) {
|
|
65
|
+
throw new Error(`dag run ${runId} budget drifted between run.json and state.json; refuse to continue`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { DEFAULT_SKILL_INSTRUCTION_MAX_CHARS, DEFAULT_SKILL_INSTRUCTION_TOTAL_MAX_CHARS, } from "./skill-instructions.js";
|
|
2
|
+
import { MAX_UPSTREAM_CHARS } from "./prompt.js";
|
|
3
|
+
import { resolveDagNodeSkills } from "./skills.js";
|
|
4
|
+
export const CONTEXT_POLICY_IDS = [
|
|
5
|
+
"baseline-v1",
|
|
6
|
+
"role-specialized-v1",
|
|
7
|
+
];
|
|
8
|
+
export const DEFAULT_CONTEXT_POLICY_ID = "baseline-v1";
|
|
9
|
+
function roleOrUndefined(task) {
|
|
10
|
+
return task.role;
|
|
11
|
+
}
|
|
12
|
+
function pickByRole(table, role) {
|
|
13
|
+
if (role && table[role] !== undefined)
|
|
14
|
+
return table[role];
|
|
15
|
+
return table.default;
|
|
16
|
+
}
|
|
17
|
+
class BaselineContextPolicy {
|
|
18
|
+
id = "baseline-v1";
|
|
19
|
+
description = "Current DAG context assembly: shared upstream char budget, role skill defaults, learned patterns only for implementer.";
|
|
20
|
+
resolveSkills(spec, task) {
|
|
21
|
+
return resolveDagNodeSkills(spec, task);
|
|
22
|
+
}
|
|
23
|
+
resolveSkillInstructionBudget(task) {
|
|
24
|
+
return {
|
|
25
|
+
includeLearnedPatterns: task.role === "implementer",
|
|
26
|
+
perSkillMaxChars: DEFAULT_SKILL_INSTRUCTION_MAX_CHARS,
|
|
27
|
+
totalMaxChars: DEFAULT_SKILL_INSTRUCTION_TOTAL_MAX_CHARS,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
resolveMaxUpstreamChars(_task) {
|
|
31
|
+
return MAX_UPSTREAM_CHARS;
|
|
32
|
+
}
|
|
33
|
+
toManifest() {
|
|
34
|
+
return {
|
|
35
|
+
schemaVersion: 1,
|
|
36
|
+
policyId: this.id,
|
|
37
|
+
description: this.description,
|
|
38
|
+
knobs: {
|
|
39
|
+
maxUpstreamCharsByRole: { default: MAX_UPSTREAM_CHARS },
|
|
40
|
+
includeLearnedPatternsRoles: ["implementer"],
|
|
41
|
+
perSkillMaxCharsByRole: {
|
|
42
|
+
default: DEFAULT_SKILL_INSTRUCTION_MAX_CHARS,
|
|
43
|
+
},
|
|
44
|
+
totalMaxCharsByRole: {
|
|
45
|
+
default: DEFAULT_SKILL_INSTRUCTION_TOTAL_MAX_CHARS,
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* First A/B challenger: keep skill name resolution identical to baseline, but
|
|
53
|
+
* specialize upstream / skill-instruction budgets by role so scouts see less
|
|
54
|
+
* noise and implementers retain more upstream + learned patterns surface.
|
|
55
|
+
*/
|
|
56
|
+
class RoleSpecializedContextPolicy {
|
|
57
|
+
id = "role-specialized-v1";
|
|
58
|
+
description = "Role-specialized upstream and skill-instruction budgets; skill names still resolve via baseline merge order.";
|
|
59
|
+
maxUpstreamCharsByRole = {
|
|
60
|
+
default: MAX_UPSTREAM_CHARS,
|
|
61
|
+
scout: 1_200,
|
|
62
|
+
reviewer: 1_200,
|
|
63
|
+
implementer: 3_000,
|
|
64
|
+
verifier: 1_600,
|
|
65
|
+
closeout: 1_600,
|
|
66
|
+
planner: MAX_UPSTREAM_CHARS,
|
|
67
|
+
supervisor: MAX_UPSTREAM_CHARS,
|
|
68
|
+
};
|
|
69
|
+
perSkillMaxCharsByRole = {
|
|
70
|
+
default: DEFAULT_SKILL_INSTRUCTION_MAX_CHARS,
|
|
71
|
+
scout: 2_500,
|
|
72
|
+
implementer: 3_500,
|
|
73
|
+
};
|
|
74
|
+
totalMaxCharsByRole = {
|
|
75
|
+
default: DEFAULT_SKILL_INSTRUCTION_TOTAL_MAX_CHARS,
|
|
76
|
+
scout: 10_000,
|
|
77
|
+
implementer: 14_000,
|
|
78
|
+
};
|
|
79
|
+
learnedPatternRoles = new Set([
|
|
80
|
+
"implementer",
|
|
81
|
+
"closeout",
|
|
82
|
+
]);
|
|
83
|
+
resolveSkills(spec, task) {
|
|
84
|
+
return resolveDagNodeSkills(spec, task);
|
|
85
|
+
}
|
|
86
|
+
resolveSkillInstructionBudget(task) {
|
|
87
|
+
const role = roleOrUndefined(task);
|
|
88
|
+
return {
|
|
89
|
+
includeLearnedPatterns: role
|
|
90
|
+
? this.learnedPatternRoles.has(role)
|
|
91
|
+
: false,
|
|
92
|
+
perSkillMaxChars: pickByRole(this.perSkillMaxCharsByRole, role),
|
|
93
|
+
totalMaxChars: pickByRole(this.totalMaxCharsByRole, role),
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
resolveMaxUpstreamChars(task) {
|
|
97
|
+
return pickByRole(this.maxUpstreamCharsByRole, roleOrUndefined(task));
|
|
98
|
+
}
|
|
99
|
+
toManifest() {
|
|
100
|
+
return {
|
|
101
|
+
schemaVersion: 1,
|
|
102
|
+
policyId: this.id,
|
|
103
|
+
description: this.description,
|
|
104
|
+
knobs: {
|
|
105
|
+
maxUpstreamCharsByRole: this.maxUpstreamCharsByRole,
|
|
106
|
+
includeLearnedPatternsRoles: [...this.learnedPatternRoles],
|
|
107
|
+
perSkillMaxCharsByRole: this.perSkillMaxCharsByRole,
|
|
108
|
+
totalMaxCharsByRole: this.totalMaxCharsByRole,
|
|
109
|
+
},
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
const POLICIES = {
|
|
114
|
+
"baseline-v1": new BaselineContextPolicy(),
|
|
115
|
+
"role-specialized-v1": new RoleSpecializedContextPolicy(),
|
|
116
|
+
};
|
|
117
|
+
export function isContextPolicyId(value) {
|
|
118
|
+
return CONTEXT_POLICY_IDS.includes(value);
|
|
119
|
+
}
|
|
120
|
+
export function getContextPolicy(policyId) {
|
|
121
|
+
return POLICIES[policyId];
|
|
122
|
+
}
|
|
123
|
+
export function listContextPolicies() {
|
|
124
|
+
return CONTEXT_POLICY_IDS.map((id) => POLICIES[id]);
|
|
125
|
+
}
|
|
126
|
+
export function resolveContextPolicyId(spec) {
|
|
127
|
+
const raw = spec.defaults?.contextPolicyId;
|
|
128
|
+
if (!raw)
|
|
129
|
+
return DEFAULT_CONTEXT_POLICY_ID;
|
|
130
|
+
if (!isContextPolicyId(raw)) {
|
|
131
|
+
throw new Error(`unknown contextPolicyId "${raw}"; expected one of ${CONTEXT_POLICY_IDS.join(", ")}`);
|
|
132
|
+
}
|
|
133
|
+
return raw;
|
|
134
|
+
}
|
|
135
|
+
export function resolveContextPolicy(spec) {
|
|
136
|
+
return getContextPolicy(resolveContextPolicyId(spec));
|
|
137
|
+
}
|
|
@@ -103,6 +103,7 @@ export async function curateKnowledgePatterns(input) {
|
|
|
103
103
|
ok: true,
|
|
104
104
|
patternsPath,
|
|
105
105
|
patternCount: 0,
|
|
106
|
+
patterns: [],
|
|
106
107
|
safetyFindings: [],
|
|
107
108
|
message: "no patterns.jsonl found; no proposal generated",
|
|
108
109
|
};
|
|
@@ -139,6 +140,7 @@ export async function curateKnowledgePatterns(input) {
|
|
|
139
140
|
patternsPath,
|
|
140
141
|
outputPath,
|
|
141
142
|
patternCount: patterns.length,
|
|
143
|
+
patterns,
|
|
142
144
|
proposalMarkdown,
|
|
143
145
|
safetyFindings,
|
|
144
146
|
message: "proposal failed skill safety audit",
|
|
@@ -153,6 +155,7 @@ export async function curateKnowledgePatterns(input) {
|
|
|
153
155
|
patternsPath,
|
|
154
156
|
outputPath,
|
|
155
157
|
patternCount: patterns.length,
|
|
158
|
+
patterns,
|
|
156
159
|
proposalMarkdown,
|
|
157
160
|
safetyFindings,
|
|
158
161
|
message: patterns.length === 0
|
|
@@ -3,29 +3,35 @@ import { readFile } from "node:fs/promises";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { recordDecisionEnvelopeForNode, shouldPauseOnHumanEscalation, writeHumanEscalationArtifacts, } from "./decision-envelope.js";
|
|
5
5
|
import { writeNodeRecord, writeNodeSkillArtifacts } from "./run-store.js";
|
|
6
|
+
import { resolveContextPolicy } from "./context-policy.js";
|
|
6
7
|
import { buildDagNodePromptEnvelope } from "./prompt.js";
|
|
7
8
|
import { persistLongNodeOutputArtifacts } from "./upstream-artifacts.js";
|
|
8
9
|
import { computeBackoffDelayMs, isRetryablePiFailureCategory, isSafeReadOnlyPiRetryCandidate, } from "./retry-policy.js";
|
|
9
10
|
import { writeDagNodeJsonArtifact } from "../../infrastructure/harness/artifact-store.js";
|
|
10
11
|
import { assertSkillSnapshotCoversSpec, buildNodePromptFromSnapshot, isDagSkillSnapshotIntegrityError, readSkillSnapshot, } from "./skill-snapshot.js";
|
|
11
12
|
import { resolveDagSkillInstructions, skillInstructionMetadata, } from "./skill-instructions.js";
|
|
12
|
-
import { resolveDagNodeSkills } from "./skills.js";
|
|
13
13
|
import { parseRepairArtifactFromText, resolveRepairTaskForGate, validateRepairArtifactScope, } from "./repair-artifact.js";
|
|
14
14
|
import { resolveModelForTask, } from "./types.js";
|
|
15
15
|
export function buildNodePrompt(spec, task, upstream) {
|
|
16
|
+
const policy = resolveContextPolicy(spec);
|
|
16
17
|
return buildDagNodePromptEnvelope({
|
|
17
18
|
spec,
|
|
18
19
|
task,
|
|
19
20
|
upstream,
|
|
20
|
-
resolvedSkills:
|
|
21
|
+
resolvedSkills: policy.resolveSkills(spec, task),
|
|
22
|
+
maxUpstreamChars: policy.resolveMaxUpstreamChars(task),
|
|
21
23
|
});
|
|
22
24
|
}
|
|
23
25
|
export async function buildNodePromptWithResolvedSkillInstructions(spec, task, upstream, cwd) {
|
|
24
|
-
const
|
|
26
|
+
const policy = resolveContextPolicy(spec);
|
|
27
|
+
const skillNames = policy.resolveSkills(spec, task);
|
|
28
|
+
const budget = policy.resolveSkillInstructionBudget(task);
|
|
25
29
|
const resolvedSkillInstructions = task.executor === "pi"
|
|
26
30
|
? await resolveDagSkillInstructions(skillNames, {
|
|
27
31
|
cwd,
|
|
28
|
-
includeLearnedPatterns:
|
|
32
|
+
includeLearnedPatterns: budget.includeLearnedPatterns,
|
|
33
|
+
perSkillMaxChars: budget.perSkillMaxChars,
|
|
34
|
+
totalMaxChars: budget.totalMaxChars,
|
|
29
35
|
})
|
|
30
36
|
: [];
|
|
31
37
|
return {
|
|
@@ -35,6 +41,7 @@ export async function buildNodePromptWithResolvedSkillInstructions(spec, task, u
|
|
|
35
41
|
upstream,
|
|
36
42
|
resolvedSkills: skillNames,
|
|
37
43
|
resolvedSkillInstructions,
|
|
44
|
+
maxUpstreamChars: policy.resolveMaxUpstreamChars(task),
|
|
38
45
|
}),
|
|
39
46
|
resolvedSkills: skillInstructionMetadata(resolvedSkillInstructions),
|
|
40
47
|
};
|
|
@@ -6,7 +6,7 @@ export const DAG_AUTHORING_GUIDANCE = [
|
|
|
6
6
|
"Prefer same-rank parallel read-only scouts over unnecessary serial depends_on chains.",
|
|
7
7
|
"Add depends_on only when a child truly needs upstream output; default to independent ranks.",
|
|
8
8
|
"Every task must explicitly declare executor; defaults.executor is schema-only, not a runtime fallback.",
|
|
9
|
-
"
|
|
9
|
+
"Pi is the only governed Agent DAG writer; cursor-prompt is an explicit manual one-shot sidecar and must not enter Loop auto-execute or Delegate writers.",
|
|
10
10
|
"exclusive nodes require narrow, concrete, disjoint writeSet paths; never use ** or repo root.",
|
|
11
11
|
"Read-only nodes must not write repository files, including root artifacts/**; return findings in node output only.",
|
|
12
12
|
"If the DAG is a single linear chain, challenge whether read-only work can run in parallel ranks.",
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { readdir, readFile } from "node:fs/promises";
|
|
2
2
|
import { hostname } from "node:os";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { isHardBudgetBreached, resolveEffectiveMaxConcurrent, } from "../../application/evaluation/budget.js";
|
|
4
5
|
import { readCandidateRecord } from "../../infrastructure/evaluation/candidate-store.js";
|
|
5
6
|
import { CANONICAL_TASK_ID_PATTERN, formatLocalCompactDate, } from "../../task/runtime.js";
|
|
7
|
+
import { assertFrozenBudget, initRunBudgetLedger, preflightBudgetOrBreach, recordFinishedNodeBudget, writeBudgetLedgerArtifacts, } from "./budget-enforcement.js";
|
|
6
8
|
import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readHumanApprovalArtifact, requireActiveDagRun, } from "./lifecycle.js";
|
|
7
9
|
import { moveToCompletedRunDir, moveToPausedRunDir, prepareActiveRunDir, writeRunSpec, writeRunState, } from "./run-store.js";
|
|
8
10
|
import { createDagNodeExecutor } from "./executor-registry.js";
|
|
@@ -120,7 +122,7 @@ export function createInitialRunState(spec, opts, ranks, runId = opts.runId ?? "
|
|
|
120
122
|
: {}),
|
|
121
123
|
};
|
|
122
124
|
}
|
|
123
|
-
|
|
125
|
+
const state = {
|
|
124
126
|
version: 1,
|
|
125
127
|
title: spec.title,
|
|
126
128
|
runId,
|
|
@@ -147,6 +149,8 @@ export function createInitialRunState(spec, opts, ranks, runId = opts.runId ?? "
|
|
|
147
149
|
}
|
|
148
150
|
: {}),
|
|
149
151
|
};
|
|
152
|
+
initRunBudgetLedger(state, spec.budget);
|
|
153
|
+
return state;
|
|
150
154
|
}
|
|
151
155
|
export function assertFrozenEvaluationBinding(spec, state) {
|
|
152
156
|
const declared = spec.evaluation;
|
|
@@ -183,7 +187,7 @@ export async function runDag(spec, opts) {
|
|
|
183
187
|
controllerVersion: runningIdentity.packageVersion,
|
|
184
188
|
});
|
|
185
189
|
const { ranks } = topoSortToRanks(spec);
|
|
186
|
-
const maxConcurrent = Math.max(1, opts.maxConcurrent ?? 4);
|
|
190
|
+
const { maxConcurrent } = resolveEffectiveMaxConcurrent(Math.max(1, opts.maxConcurrent ?? 4), spec.budget);
|
|
187
191
|
let runId = opts.runId;
|
|
188
192
|
if (runId) {
|
|
189
193
|
for (const warning of validateDagRunIdFormat(runId).warnings) {
|
|
@@ -268,6 +272,7 @@ export async function resumeDagRun(opts) {
|
|
|
268
272
|
throw new Error(`dag run ${opts.runId} is already terminal (status=${state.status})`);
|
|
269
273
|
}
|
|
270
274
|
assertFrozenEvaluationBinding(spec, state);
|
|
275
|
+
assertFrozenBudget(spec.budget, state.budget, state.runId);
|
|
271
276
|
// Runtime contract + controller identity must be re-verified before executing
|
|
272
277
|
// any remaining node on resume; drift fails closed.
|
|
273
278
|
const runningIdentity = resolveRunningControllerIdentity();
|
|
@@ -311,7 +316,7 @@ export async function resumeDagRun(opts) {
|
|
|
311
316
|
catch (error) {
|
|
312
317
|
throw new Error(`skill snapshot validation failed on resume: ${error instanceof Error ? error.message : String(error)}`);
|
|
313
318
|
}
|
|
314
|
-
const maxConcurrent = Math.max(1, opts.maxConcurrent ?? 4);
|
|
319
|
+
const { maxConcurrent } = resolveEffectiveMaxConcurrent(Math.max(1, opts.maxConcurrent ?? 4), spec.budget);
|
|
315
320
|
state.status = "running";
|
|
316
321
|
const resumedAt = new Date().toISOString();
|
|
317
322
|
state.runner = {
|
|
@@ -410,23 +415,43 @@ async function executeDagCheckpoint(input) {
|
|
|
410
415
|
tasksById,
|
|
411
416
|
meta: { runDir, runId: state.runId, spec },
|
|
412
417
|
}),
|
|
413
|
-
executeScheduledNode: (nodeId, executeNode, onPause) =>
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
state
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
418
|
+
executeScheduledNode: async (nodeId, executeNode, onPause) => {
|
|
419
|
+
if (isHardBudgetBreached(state.budgetLedger))
|
|
420
|
+
return;
|
|
421
|
+
const preBreach = preflightBudgetOrBreach(state);
|
|
422
|
+
if (preBreach && isHardBudgetBreached(state.budgetLedger)) {
|
|
423
|
+
await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
|
|
424
|
+
await persistState();
|
|
425
|
+
return;
|
|
426
|
+
}
|
|
427
|
+
await executeDagNode({
|
|
428
|
+
nodeId,
|
|
429
|
+
tasksById,
|
|
430
|
+
state,
|
|
431
|
+
spec,
|
|
432
|
+
cwd,
|
|
433
|
+
runDir,
|
|
434
|
+
executeNode,
|
|
435
|
+
executeDynamicNode,
|
|
436
|
+
observer: input.observer,
|
|
437
|
+
persistState,
|
|
438
|
+
onPause,
|
|
439
|
+
});
|
|
440
|
+
const node = state.nodes[nodeId];
|
|
441
|
+
if (node &&
|
|
442
|
+
(node.status === "FINISHED" || node.status === "ERROR")) {
|
|
443
|
+
recordFinishedNodeBudget(state, node);
|
|
444
|
+
await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
|
|
445
|
+
await persistState();
|
|
446
|
+
}
|
|
447
|
+
},
|
|
426
448
|
});
|
|
427
449
|
if (pausedByNodeId) {
|
|
428
450
|
break;
|
|
429
451
|
}
|
|
452
|
+
if (isHardBudgetBreached(state.budgetLedger)) {
|
|
453
|
+
break;
|
|
454
|
+
}
|
|
430
455
|
const convergenceDecision = await runConvergencePassController({
|
|
431
456
|
spec,
|
|
432
457
|
state,
|
|
@@ -446,6 +471,7 @@ async function executeDagCheckpoint(input) {
|
|
|
446
471
|
}
|
|
447
472
|
state.finishedAt = new Date().toISOString();
|
|
448
473
|
const runDirBeforeTransfer = runDir;
|
|
474
|
+
await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
|
|
449
475
|
if (pausedByNodeId) {
|
|
450
476
|
state.status = "paused";
|
|
451
477
|
await persistState();
|
|
@@ -466,6 +492,7 @@ async function executeDagCheckpoint(input) {
|
|
|
466
492
|
if (state.convergence) {
|
|
467
493
|
relocateConvergenceArtifactPaths(state.convergence, runDirBeforeTransfer, runDir);
|
|
468
494
|
}
|
|
495
|
+
await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
|
|
469
496
|
await persistState({ allowCompletedFactsWrite: true });
|
|
470
497
|
return {
|
|
471
498
|
title: spec.title,
|
|
@@ -3,9 +3,9 @@ import { readFile } from "node:fs/promises";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
import { writeTextAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
6
|
+
import { resolveContextPolicy } from "./context-policy.js";
|
|
6
7
|
import { buildDagNodePromptEnvelope } from "./prompt.js";
|
|
7
|
-
import {
|
|
8
|
-
import { resolveDagNodeSkills } from "./skills.js";
|
|
8
|
+
import { resolveDagSkillInstructions, } from "./skill-instructions.js";
|
|
9
9
|
export const SKILL_SNAPSHOT_SCHEMA_VERSION = 1;
|
|
10
10
|
export const SKILL_SNAPSHOT_RESOLVER_VERSION = 1;
|
|
11
11
|
export const SKILL_SNAPSHOT_REL_PATH = ".runtime/skill-snapshot.json";
|
|
@@ -117,11 +117,13 @@ function profileId(input) {
|
|
|
117
117
|
export function buildDagSkillSnapshotProfileRequest(spec, task) {
|
|
118
118
|
if (task.executor !== "pi")
|
|
119
119
|
return undefined;
|
|
120
|
+
const policy = resolveContextPolicy(spec);
|
|
121
|
+
const budget = policy.resolveSkillInstructionBudget(task);
|
|
120
122
|
const request = {
|
|
121
|
-
skills: [...
|
|
122
|
-
includeLearnedPatterns:
|
|
123
|
-
perSkillMaxChars:
|
|
124
|
-
totalMaxChars:
|
|
123
|
+
skills: [...policy.resolveSkills(spec, task)],
|
|
124
|
+
includeLearnedPatterns: budget.includeLearnedPatterns,
|
|
125
|
+
perSkillMaxChars: budget.perSkillMaxChars,
|
|
126
|
+
totalMaxChars: budget.totalMaxChars,
|
|
125
127
|
};
|
|
126
128
|
return { id: profileId(request), ...request };
|
|
127
129
|
}
|
|
@@ -511,7 +513,8 @@ function stripPromptText(instruction) {
|
|
|
511
513
|
return metadata;
|
|
512
514
|
}
|
|
513
515
|
export function buildNodePromptFromSnapshot(input) {
|
|
514
|
-
const
|
|
516
|
+
const policy = resolveContextPolicy(input.spec);
|
|
517
|
+
const skillNames = policy.resolveSkills(input.spec, input.task);
|
|
515
518
|
const resolvedSkillInstructions = resolveNodeSkillsFromSnapshot(input.snapshot, input.spec, input.task);
|
|
516
519
|
return {
|
|
517
520
|
prompt: buildDagNodePromptEnvelope({
|
|
@@ -520,6 +523,7 @@ export function buildNodePromptFromSnapshot(input) {
|
|
|
520
523
|
upstream: input.upstream,
|
|
521
524
|
resolvedSkills: skillNames,
|
|
522
525
|
resolvedSkillInstructions,
|
|
526
|
+
maxUpstreamChars: policy.resolveMaxUpstreamChars(input.task),
|
|
523
527
|
}),
|
|
524
528
|
resolvedSkills: resolvedSkillInstructions.map(stripPromptText),
|
|
525
529
|
};
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { campaignBudgetSchema, } from "../../application/evaluation/budget.js";
|
|
2
3
|
import { assertDagPromptSourceRule } from "./prompt-source.js";
|
|
3
4
|
import { dagRetryPolicySchema } from "./retry-policy.js";
|
|
4
5
|
export const dagComplexitySchema = z.enum(["HIGH", "MED", "LOW"]);
|
|
@@ -105,12 +106,18 @@ export const dagWritePolicySchema = z.enum([
|
|
|
105
106
|
"exclusive",
|
|
106
107
|
"none",
|
|
107
108
|
]);
|
|
109
|
+
export const contextPolicyIdSchema = z.enum([
|
|
110
|
+
"baseline-v1",
|
|
111
|
+
"role-specialized-v1",
|
|
112
|
+
]);
|
|
108
113
|
export const dagDefaultsSchema = z
|
|
109
114
|
.object({
|
|
110
115
|
executor: dagNodeExecutorSchema.optional(),
|
|
111
116
|
model: z.string().optional(),
|
|
112
117
|
piBackend: z.enum(["sdk-first", "cli-only"]).optional(),
|
|
113
118
|
contextProfile: z.string().optional(),
|
|
119
|
+
/** Eval Lab ContextPolicy seam; default baseline-v1 when omitted. */
|
|
120
|
+
contextPolicyId: contextPolicyIdSchema.optional(),
|
|
114
121
|
skills: z.array(z.string()).optional(),
|
|
115
122
|
writePolicy: dagWritePolicySchema.optional(),
|
|
116
123
|
})
|
|
@@ -300,6 +307,8 @@ export const dagEvaluationBindingSchema = z
|
|
|
300
307
|
taskRef: z.string().min(1).optional(),
|
|
301
308
|
})
|
|
302
309
|
.strict();
|
|
310
|
+
/** Eval Lab Campaign Budget (W3.4); enforced by runner ledger when mode=hard. */
|
|
311
|
+
export const dagBudgetSchema = campaignBudgetSchema;
|
|
303
312
|
export const dagSourceBindingSchema = z.object({
|
|
304
313
|
schemaVersion: z.literal(1),
|
|
305
314
|
taskId: z.string().min(1),
|
|
@@ -320,6 +329,8 @@ export const dagSpecSchema = z
|
|
|
320
329
|
title: z.string().min(1),
|
|
321
330
|
runtimeContract: dagRuntimeContractSchema.optional(),
|
|
322
331
|
evaluation: dagEvaluationBindingSchema.optional(),
|
|
332
|
+
/** Optional hard/record-only budget; requires version 3. */
|
|
333
|
+
budget: dagBudgetSchema.optional(),
|
|
323
334
|
sourceBinding: dagSourceBindingSchema.optional(),
|
|
324
335
|
outputLanguage: dagOutputLanguageSchema.optional(),
|
|
325
336
|
objective: z.string().optional(),
|
|
@@ -340,6 +351,13 @@ export const dagSpecSchema = z
|
|
|
340
351
|
path: ["evaluation"],
|
|
341
352
|
});
|
|
342
353
|
}
|
|
354
|
+
if (spec.budget && spec.version !== 3) {
|
|
355
|
+
ctx.addIssue({
|
|
356
|
+
code: z.ZodIssueCode.custom,
|
|
357
|
+
message: "budget requires DagSpec version 3",
|
|
358
|
+
path: ["budget"],
|
|
359
|
+
});
|
|
360
|
+
}
|
|
343
361
|
if (spec.runtimeContract && spec.version !== 3) {
|
|
344
362
|
ctx.addIssue({
|
|
345
363
|
code: z.ZodIssueCode.custom,
|