@voybio/ace-swarm 2.4.4 → 2.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.md +72 -532
- package/assets/eval-battery/battery.json +85 -0
- package/assets/eval-battery/fixtures/edit-target.js +3 -0
- package/assets/scripts/render-mcp-configs.sh +12 -0
- package/dist/ace-context.d.ts +18 -0
- package/dist/ace-context.js +161 -16
- package/dist/ace-server-instructions.js +36 -9
- package/dist/bridge-trace.d.ts +17 -0
- package/dist/bridge-trace.js +23 -0
- package/dist/cli.d.ts +9 -1
- package/dist/cli.js +619 -10
- package/dist/discovery-runtime-wrappers.d.ts +16 -1
- package/dist/discovery-runtime-wrappers.js +65 -9
- package/dist/eval-harness.d.ts +110 -0
- package/dist/eval-harness.js +254 -0
- package/dist/experience-reflector.d.ts +13 -0
- package/dist/experience-reflector.js +62 -0
- package/dist/goal-engine.d.ts +155 -0
- package/dist/goal-engine.js +795 -0
- package/dist/helpers/store-resolution.js +40 -39
- package/dist/hermes/session-manager.d.ts +10 -0
- package/dist/hermes/session-manager.js +130 -27
- package/dist/job-scheduler.d.ts +58 -0
- package/dist/job-scheduler.js +202 -2
- package/dist/job-worker.d.ts +87 -0
- package/dist/job-worker.js +182 -0
- package/dist/local-model-policy.d.ts +8 -2
- package/dist/local-model-policy.js +31 -17
- package/dist/local-model-runtime.d.ts +6 -0
- package/dist/local-model-runtime.js +130 -2
- package/dist/measured-classing.d.ts +66 -0
- package/dist/measured-classing.js +165 -0
- package/dist/model-bridge.d.ts +45 -0
- package/dist/model-bridge.js +502 -44
- package/dist/orchestrator-supervisor.js +1 -1
- package/dist/plan-proposal.d.ts +7 -0
- package/dist/plan-proposal.js +74 -4
- package/dist/playbook-adaptation.d.ts +37 -0
- package/dist/playbook-adaptation.js +159 -0
- package/dist/playbook-curator.d.ts +16 -0
- package/dist/playbook-curator.js +106 -0
- package/dist/playbook-llm-reflector.d.ts +26 -0
- package/dist/playbook-llm-reflector.js +157 -0
- package/dist/public-surface.d.ts +2 -0
- package/dist/public-surface.js +26 -14
- package/dist/safe-edit.d.ts +20 -0
- package/dist/safe-edit.js +355 -78
- package/dist/server.d.ts +11 -2
- package/dist/server.js +55 -5
- package/dist/shared.d.ts +12 -0
- package/dist/shared.js +13 -1
- package/dist/source-mutation-coordinator.d.ts +54 -0
- package/dist/source-mutation-coordinator.js +259 -0
- package/dist/status-events.d.ts +4 -1
- package/dist/status-events.js +41 -6
- package/dist/store/ace-packed-store.d.ts +28 -9
- package/dist/store/ace-packed-store.js +324 -57
- package/dist/store/bgzf.d.ts +4 -0
- package/dist/store/bgzf.js +94 -0
- package/dist/store/bootstrap-store.d.ts +4 -1
- package/dist/store/bootstrap-store.js +57 -3
- package/dist/store/cache-workspace.d.ts +8 -0
- package/dist/store/cache-workspace.js +67 -1
- package/dist/store/crc32.d.ts +2 -0
- package/dist/store/crc32.js +16 -0
- package/dist/store/fsck.d.ts +16 -0
- package/dist/store/fsck.js +240 -0
- package/dist/store/materializers/context-snapshot-materializer.d.ts +1 -0
- package/dist/store/materializers/context-snapshot-materializer.js +4 -1
- package/dist/store/materializers/projection-manager.d.ts +7 -0
- package/dist/store/materializers/projection-manager.js +37 -0
- package/dist/store/materializers/scheduler-projection-materializer.js +10 -3
- package/dist/store/repositories/goal-repository.d.ts +112 -0
- package/dist/store/repositories/goal-repository.js +157 -0
- package/dist/store/repositories/playbook-repository.d.ts +107 -0
- package/dist/store/repositories/playbook-repository.js +314 -0
- package/dist/store/store-artifacts.js +44 -2
- package/dist/store/store-authority-audit.js +1 -0
- package/dist/store/store-cli.d.ts +30 -0
- package/dist/store/store-cli.js +242 -0
- package/dist/store/store-snapshot.d.ts +12 -0
- package/dist/store/store-snapshot.js +74 -11
- package/dist/store/types.d.ts +1 -0
- package/dist/store/write-coordinator.d.ts +7 -0
- package/dist/store/write-coordinator.js +11 -0
- package/dist/todo-state.d.ts +21 -0
- package/dist/todo-state.js +150 -53
- package/dist/tools-agent.js +86 -22
- package/dist/tools-discovery.js +5 -3
- package/dist/tools-files.js +39 -18
- package/dist/tools-framework.js +31 -3
- package/dist/tools-goal.d.ts +11 -0
- package/dist/tools-goal.js +108 -0
- package/dist/tools-handoff.js +7 -11
- package/dist/tools-lifecycle.js +2 -0
- package/dist/tools-scheduler.js +44 -1
- package/dist/tools-skills.js +3 -2
- package/dist/tools-todo.js +64 -3
- package/dist/tools.js +2 -0
- package/dist/tui/chat.js +4 -0
- package/dist/tui/index.js +4 -1
- package/dist/tui/ollama.d.ts +2 -0
- package/dist/tui/openai-compatible.d.ts +14 -0
- package/dist/tui/openai-compatible.js +15 -2
- package/dist/tui/provider-discovery.d.ts +1 -0
- package/dist/tui/provider-discovery.js +2 -2
- package/docs/cli-reference.md +125 -0
- package/docs/development.md +46 -0
- package/docs/evaluation-safety.md +27 -0
- package/docs/experience-playbook.md +35 -0
- package/docs/hermes.md +129 -0
- package/docs/library-api.md +31 -0
- package/docs/mcp-tool-reference.md +290 -0
- package/docs/provider-security.md +85 -0
- package/docs/state-model.md +113 -0
- package/docs/troubleshooting.md +95 -0
- package/docs/tui-operator-guide.md +75 -0
- package/package.json +3 -1
- package/scripts/dev/run_hermes_crispr_live.mjs +212 -0
|
@@ -116,7 +116,7 @@ export function markTaskStepStatus(plan, stepId, status, patch = {}) {
|
|
|
116
116
|
export function applyBridgeResultToStep(plan, stepId, result) {
|
|
117
117
|
const nextStatus = result.status === "completed"
|
|
118
118
|
? "done"
|
|
119
|
-
: result.status === "needs_input" || result.status === "max_turns"
|
|
119
|
+
: result.status === "needs_input" || result.status === "max_turns" || result.status === "blocked"
|
|
120
120
|
? "blocked"
|
|
121
121
|
: "failed";
|
|
122
122
|
return withDerivedStatus({
|
package/dist/plan-proposal.d.ts
CHANGED
|
@@ -9,6 +9,13 @@ export interface ContractArtifactExpectation {
|
|
|
9
9
|
path: string;
|
|
10
10
|
required?: boolean;
|
|
11
11
|
evidence_ref_kind?: "artifact" | "diff" | "hash" | "test" | "gate";
|
|
12
|
+
/**
|
|
13
|
+
* True when the deterministic scaffold invented this path to satisfy
|
|
14
|
+
* `validatePlan`'s `contract_artifacts` rule, rather than a planner naming
|
|
15
|
+
* evidence it commits to producing. Scaffolded expectations must never become
|
|
16
|
+
* success-contract obligations — see `compileSuccessContract` in goal-engine.ts.
|
|
17
|
+
*/
|
|
18
|
+
scaffolded?: boolean;
|
|
12
19
|
}
|
|
13
20
|
export interface PlanProposalStep {
|
|
14
21
|
role: string;
|
package/dist/plan-proposal.js
CHANGED
|
@@ -50,17 +50,31 @@ function enrichContractStep(step) {
|
|
|
50
50
|
(inferExpectedOutputClass(step) === "structural_edit_plan" ? true : undefined),
|
|
51
51
|
};
|
|
52
52
|
}
|
|
53
|
+
/**
|
|
54
|
+
* Placeholder evidence for the deterministic scaffold. `validatePlan`'s
|
|
55
|
+
* `contract_artifacts` rule requires every code/structural step to name its
|
|
56
|
+
* evidence before execution, and that rule backs the fallback planner behind
|
|
57
|
+
* `run_orchestrator` — returning `undefined` here makes ACE emit plans its own
|
|
58
|
+
* validator rejects.
|
|
59
|
+
*
|
|
60
|
+
* The scaffold cannot know what a step will produce, so every path it invents is
|
|
61
|
+
* marked `scaffolded`. That flag is what keeps them out of the success contract:
|
|
62
|
+
* a fixed, workspace-shared path like `agent-state/SPEC_CONTRACT.json` is checked
|
|
63
|
+
* with `existsSync`, so once any goal writes it, a later goal's contract would be
|
|
64
|
+
* satisfied by a file that goal never touched. See F1 in
|
|
65
|
+
* REVIEW_MITIGATIONS_CLOSEOUT_2026-07-10.md.
|
|
66
|
+
*/
|
|
53
67
|
function scaffoldArtifactExpectations(step) {
|
|
54
68
|
const expected = inferExpectedOutputClass(step);
|
|
55
69
|
if (expected !== "code_artifact" && expected !== "structural_edit_plan")
|
|
56
70
|
return undefined;
|
|
57
71
|
if (step.role === "spec") {
|
|
58
|
-
return [{ path: "agent-state/SPEC_CONTRACT.json", required: false, evidence_ref_kind: "artifact" }];
|
|
72
|
+
return [{ path: "agent-state/SPEC_CONTRACT.json", required: false, evidence_ref_kind: "artifact", scaffolded: true }];
|
|
59
73
|
}
|
|
60
74
|
if (step.role === "docs") {
|
|
61
|
-
return [{ path: "agent-state", required: false, evidence_ref_kind: "artifact" }];
|
|
75
|
+
return [{ path: "agent-state", required: false, evidence_ref_kind: "artifact", scaffolded: true }];
|
|
62
76
|
}
|
|
63
|
-
return [{ path: "workspace-artifact", required: false, evidence_ref_kind: "artifact" }];
|
|
77
|
+
return [{ path: "workspace-artifact", required: false, evidence_ref_kind: "artifact", scaffolded: true }];
|
|
64
78
|
}
|
|
65
79
|
function isCodeMutatingStep(step) {
|
|
66
80
|
if (step.role === "coders" || step.role === "builder" || step.role === "astgrep")
|
|
@@ -481,7 +495,12 @@ function compileGoalScaffold(task, plan_id) {
|
|
|
481
495
|
? "implementation"
|
|
482
496
|
: rawKind;
|
|
483
497
|
const role = roleForClauseKind(kind);
|
|
484
|
-
|
|
498
|
+
// Namespaced by position, not by role: `goal-${role}-${index+1}` collides with
|
|
499
|
+
// the ship-boundary phase ids below (a `release` clause at index 0 produced
|
|
500
|
+
// `goal-release-1`, the same id as the Phase 7 release step). `publicStepLabels`
|
|
501
|
+
// is keyed by id, so the later push silently re-pointed this step's dependants
|
|
502
|
+
// at the wrong label and closed a cycle in the plan DAG.
|
|
503
|
+
const stepId = `goal-exec-${index + 1}`;
|
|
485
504
|
const dependency = index === 0 ? specId : executionStepIds[index - 1];
|
|
486
505
|
scaffoldSteps.push({
|
|
487
506
|
id: stepId,
|
|
@@ -626,6 +645,15 @@ function compileGoalScaffold(task, plan_id) {
|
|
|
626
645
|
}),
|
|
627
646
|
});
|
|
628
647
|
}
|
|
648
|
+
// Ids are the only key `publicStepLabels` has; a duplicate silently rewrites a
|
|
649
|
+
// dependency to point at the wrong step, which is how the ship-boundary scaffold
|
|
650
|
+
// grew a cycle. Fail loudly here rather than emit an unrunnable plan.
|
|
651
|
+
const duplicateIds = scaffoldSteps
|
|
652
|
+
.map((step) => step.id)
|
|
653
|
+
.filter((id, index, all) => all.indexOf(id) !== index);
|
|
654
|
+
if (duplicateIds.length > 0) {
|
|
655
|
+
throw new Error(`compileGoalScaffold produced duplicate step ids: ${[...new Set(duplicateIds)].join(", ")}`);
|
|
656
|
+
}
|
|
629
657
|
const publicStepLabels = new Map();
|
|
630
658
|
scaffoldSteps.forEach((step, index) => {
|
|
631
659
|
publicStepLabels.set(step.id, `step-${index + 1}`);
|
|
@@ -806,6 +834,48 @@ function validatePlanShape(plan_id, proposal) {
|
|
|
806
834
|
softFindings.push(`stop_conditions: step(s) ${missingStopConditions.join(", ")} have no stop conditions`);
|
|
807
835
|
}
|
|
808
836
|
}
|
|
837
|
+
// 5b. The step graph must be a DAG over known labels. A dangling or cyclic
|
|
838
|
+
// `depends_on` yields a plan where some step is never ready, and the goal engine
|
|
839
|
+
// reads "no ready step" as "waiting on external state" and backs off forever.
|
|
840
|
+
{
|
|
841
|
+
const labels = proposal.steps.map((_, index) => `step-${index + 1}`);
|
|
842
|
+
const indexByLabel = new Map(labels.map((label, index) => [label, index]));
|
|
843
|
+
const unknown = [];
|
|
844
|
+
for (const [index, step] of proposal.steps.entries()) {
|
|
845
|
+
for (const dependency of step.depends_on ?? []) {
|
|
846
|
+
if (!indexByLabel.has(dependency))
|
|
847
|
+
unknown.push(`${labels[index]}→${dependency}`);
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
if (unknown.length > 0) {
|
|
851
|
+
blockingFindings.push(`plan_dag: step(s) depend on unknown labels ${unknown.join(", ")}`);
|
|
852
|
+
}
|
|
853
|
+
// Iterative Kahn's algorithm; any step left with unresolved deps sits on a cycle.
|
|
854
|
+
const resolved = new Set();
|
|
855
|
+
for (let pass = 0; pass < proposal.steps.length; pass += 1) {
|
|
856
|
+
let advanced = false;
|
|
857
|
+
for (const [index, step] of proposal.steps.entries()) {
|
|
858
|
+
if (resolved.has(index))
|
|
859
|
+
continue;
|
|
860
|
+
const ready = (step.depends_on ?? []).every((dependency) => {
|
|
861
|
+
const target = indexByLabel.get(dependency);
|
|
862
|
+
return target === undefined || resolved.has(target);
|
|
863
|
+
});
|
|
864
|
+
if (ready) {
|
|
865
|
+
resolved.add(index);
|
|
866
|
+
advanced = true;
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
if (!advanced)
|
|
870
|
+
break;
|
|
871
|
+
}
|
|
872
|
+
const cyclic = labels.filter((_, index) => !resolved.has(index));
|
|
873
|
+
if (cyclic.length > 0) {
|
|
874
|
+
blockingFindings.push(`plan_dag: dependency cycle among step(s) ${cyclic.join(", ")}`);
|
|
875
|
+
}
|
|
876
|
+
// Deliberately does not touch `passingChecks`: `score` is `passingChecks / 6`,
|
|
877
|
+
// and a structural invariant is not a quality point.
|
|
878
|
+
}
|
|
809
879
|
// 6. Contract-class artifact expectations: code and structural edits must say
|
|
810
880
|
// what artifact evidence can prove them before execution.
|
|
811
881
|
{
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { type EvalRunner, type ModelScorecard } from "./eval-harness.js";
|
|
2
|
+
export interface AdaptationReport {
|
|
3
|
+
model: string;
|
|
4
|
+
generated_at: string;
|
|
5
|
+
epochs: number;
|
|
6
|
+
baseline: ModelScorecard;
|
|
7
|
+
adapted: ModelScorecard;
|
|
8
|
+
/** One scorecard per epoch, each measured against the playbook as of that epoch. */
|
|
9
|
+
epoch_scorecards: ModelScorecard[];
|
|
10
|
+
band_deltas: Array<{
|
|
11
|
+
band: string;
|
|
12
|
+
baseline_passed: number;
|
|
13
|
+
baseline_total: number;
|
|
14
|
+
adapted_passed: number;
|
|
15
|
+
adapted_total: number;
|
|
16
|
+
delta: number;
|
|
17
|
+
}>;
|
|
18
|
+
playbook_stats: {
|
|
19
|
+
added: number;
|
|
20
|
+
merged: number;
|
|
21
|
+
tagged: number;
|
|
22
|
+
quarantined: number;
|
|
23
|
+
pruned: number;
|
|
24
|
+
};
|
|
25
|
+
report_path: string;
|
|
26
|
+
}
|
|
27
|
+
export interface RunAdaptationCycleInput {
|
|
28
|
+
model: string;
|
|
29
|
+
batteryPath: string;
|
|
30
|
+
workspaceRoot: string;
|
|
31
|
+
epochs?: number;
|
|
32
|
+
modelClass?: string;
|
|
33
|
+
runner?: EvalRunner;
|
|
34
|
+
outputDir?: string;
|
|
35
|
+
}
|
|
36
|
+
export declare function runAdaptationCycle(input: RunAdaptationCycleInput): Promise<AdaptationReport>;
|
|
37
|
+
//# sourceMappingURL=playbook-adaptation.d.ts.map
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { readBridgeTrace } from "./bridge-trace.js";
|
|
4
|
+
import { loadBattery, makeLiveRunner, runBattery, scorecardToMarkdown, } from "./eval-harness.js";
|
|
5
|
+
import { reflectDeterministic } from "./experience-reflector.js";
|
|
6
|
+
import { curate } from "./playbook-curator.js";
|
|
7
|
+
import { PlaybookRepository } from "./store/repositories/playbook-repository.js";
|
|
8
|
+
function roleForTask(task) {
|
|
9
|
+
if (task.band === "orchestrated")
|
|
10
|
+
return "orchestrator";
|
|
11
|
+
return "coders";
|
|
12
|
+
}
|
|
13
|
+
function withEnvs(vars, fn) {
|
|
14
|
+
const previous = new Map(Object.keys(vars).map((k) => [k, process.env[k]]));
|
|
15
|
+
Object.assign(process.env, vars);
|
|
16
|
+
return fn().finally(() => {
|
|
17
|
+
for (const [k, v] of previous) {
|
|
18
|
+
if (v === undefined)
|
|
19
|
+
delete process.env[k];
|
|
20
|
+
else
|
|
21
|
+
process.env[k] = v;
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
// Every measured run has implicit per-run curation disabled: the harness's own
|
|
26
|
+
// reflect loop is the sole writer, so `playbook_stats` is exact and the baseline
|
|
27
|
+
// is a real control. See local-model-runtime.ts `reflectRunIntoPlaybook`.
|
|
28
|
+
const MEASURE_OFF = { ACE_PLAYBOOK: "0", ACE_PLAYBOOK_REFLECT: "0" };
|
|
29
|
+
const MEASURE_ON = { ACE_PLAYBOOK: "1", ACE_PLAYBOOK_REFLECT: "0" };
|
|
30
|
+
function captureRunner(runner, sink) {
|
|
31
|
+
return async (task, model, workspaceRoot) => {
|
|
32
|
+
const outcome = await runner(task, model, workspaceRoot);
|
|
33
|
+
sink.set(task.id, outcome);
|
|
34
|
+
return outcome;
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
function bandDeltas(baseline, adapted) {
|
|
38
|
+
const bands = [...new Set([...baseline.tasks.map((t) => t.band), ...adapted.tasks.map((t) => t.band)])].sort();
|
|
39
|
+
return bands.map((band) => {
|
|
40
|
+
const before = baseline.tasks.filter((task) => task.band === band);
|
|
41
|
+
const after = adapted.tasks.filter((task) => task.band === band);
|
|
42
|
+
const baselinePassed = before.filter((task) => task.success).length;
|
|
43
|
+
const adaptedPassed = after.filter((task) => task.success).length;
|
|
44
|
+
return {
|
|
45
|
+
band,
|
|
46
|
+
baseline_passed: baselinePassed,
|
|
47
|
+
baseline_total: before.length,
|
|
48
|
+
adapted_passed: adaptedPassed,
|
|
49
|
+
adapted_total: after.length,
|
|
50
|
+
delta: adaptedPassed - baselinePassed,
|
|
51
|
+
};
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
function renderAdaptationMarkdown(report) {
|
|
55
|
+
return [
|
|
56
|
+
`# Playbook Adaptation Report — ${report.model}`,
|
|
57
|
+
"",
|
|
58
|
+
`- generated: ${report.generated_at}`,
|
|
59
|
+
`- epochs: ${report.epochs}`,
|
|
60
|
+
`- baseline: ${report.baseline.summary.passed}/${report.baseline.summary.total} (${(report.baseline.summary.success_rate * 100).toFixed(1)}%)`,
|
|
61
|
+
`- adapted: ${report.adapted.summary.passed}/${report.adapted.summary.total} (${(report.adapted.summary.success_rate * 100).toFixed(1)}%)`,
|
|
62
|
+
`- playbook: added=${report.playbook_stats.added} merged=${report.playbook_stats.merged} tagged=${report.playbook_stats.tagged} quarantined=${report.playbook_stats.quarantined} pruned=${report.playbook_stats.pruned}`,
|
|
63
|
+
"",
|
|
64
|
+
"## Epochs",
|
|
65
|
+
"",
|
|
66
|
+
"| epoch | passed | total | success rate |",
|
|
67
|
+
"| --- | ---: | ---: | ---: |",
|
|
68
|
+
`| baseline | ${report.baseline.summary.passed} | ${report.baseline.summary.total} | ${(report.baseline.summary.success_rate * 100).toFixed(1)}% |`,
|
|
69
|
+
...report.epoch_scorecards.map((card, index) => `| ${index + 1} | ${card.summary.passed} | ${card.summary.total} | ${(card.summary.success_rate * 100).toFixed(1)}% |`),
|
|
70
|
+
"",
|
|
71
|
+
"## Band Deltas",
|
|
72
|
+
"",
|
|
73
|
+
"| band | baseline | adapted | delta |",
|
|
74
|
+
"| --- | ---: | ---: | ---: |",
|
|
75
|
+
...report.band_deltas.map((row) => `| ${row.band} | ${row.baseline_passed}/${row.baseline_total} | ${row.adapted_passed}/${row.adapted_total} | ${row.delta} |`),
|
|
76
|
+
"",
|
|
77
|
+
"## Baseline",
|
|
78
|
+
"",
|
|
79
|
+
scorecardToMarkdown(report.baseline).trimEnd(),
|
|
80
|
+
"",
|
|
81
|
+
"## Adapted",
|
|
82
|
+
"",
|
|
83
|
+
scorecardToMarkdown(report.adapted).trimEnd(),
|
|
84
|
+
"",
|
|
85
|
+
].join("\n");
|
|
86
|
+
}
|
|
87
|
+
export async function runAdaptationCycle(input) {
|
|
88
|
+
const battery = loadBattery(input.batteryPath);
|
|
89
|
+
const epochs = Math.max(1, Math.min(input.epochs ?? 1, 3));
|
|
90
|
+
const runner = input.runner ?? makeLiveRunner(input.workspaceRoot);
|
|
91
|
+
const baselineOutcomes = new Map();
|
|
92
|
+
const baseline = await withEnvs(MEASURE_OFF, () => runBattery({
|
|
93
|
+
battery,
|
|
94
|
+
model: input.model,
|
|
95
|
+
runner: captureRunner(runner, baselineOutcomes),
|
|
96
|
+
workspaceRoot: input.workspaceRoot,
|
|
97
|
+
}));
|
|
98
|
+
const repo = new PlaybookRepository(input.workspaceRoot);
|
|
99
|
+
const stats = { added: 0, merged: 0, tagged: 0, quarantined: 0, pruned: 0 };
|
|
100
|
+
const epochScorecards = [];
|
|
101
|
+
let current = baseline;
|
|
102
|
+
let currentOutcomes = baselineOutcomes;
|
|
103
|
+
for (let epoch = 0; epoch < epochs; epoch += 1) {
|
|
104
|
+
// 1. Learn from the most recent run's failures and traps.
|
|
105
|
+
for (const score of current.tasks.filter((s) => !s.success || s.trap)) {
|
|
106
|
+
const task = battery.tasks.find((candidate) => candidate.id === score.task_id);
|
|
107
|
+
const outcome = currentOutcomes.get(score.task_id);
|
|
108
|
+
if (!task || !outcome)
|
|
109
|
+
continue;
|
|
110
|
+
const trace = outcome.trace.length > 0
|
|
111
|
+
? outcome.trace
|
|
112
|
+
: readBridgeTrace(input.workspaceRoot, outcome.result.bridge_id);
|
|
113
|
+
const ops = reflectDeterministic({
|
|
114
|
+
trace,
|
|
115
|
+
result: outcome.result,
|
|
116
|
+
role: roleForTask(task),
|
|
117
|
+
model: input.model,
|
|
118
|
+
modelClass: input.modelClass ?? "small_local",
|
|
119
|
+
});
|
|
120
|
+
const cur = await curate(repo, ops, { workspaceRoot: input.workspaceRoot });
|
|
121
|
+
stats.added += cur.added;
|
|
122
|
+
stats.merged += cur.merged;
|
|
123
|
+
stats.tagged += cur.tagged;
|
|
124
|
+
stats.quarantined += cur.quarantined;
|
|
125
|
+
stats.pruned += cur.pruned;
|
|
126
|
+
}
|
|
127
|
+
// 2. Re-measure against the updated playbook. This is what makes an epoch an
|
|
128
|
+
// epoch: without it, `--epochs N` just re-reflects one cached trace N times
|
|
129
|
+
// and silently biases pruneOverflow's utility sort via observed_count.
|
|
130
|
+
const epochOutcomes = new Map();
|
|
131
|
+
current = await withEnvs(MEASURE_ON, () => runBattery({
|
|
132
|
+
battery,
|
|
133
|
+
model: input.model,
|
|
134
|
+
runner: captureRunner(runner, epochOutcomes),
|
|
135
|
+
workspaceRoot: input.workspaceRoot,
|
|
136
|
+
}));
|
|
137
|
+
currentOutcomes = epochOutcomes;
|
|
138
|
+
epochScorecards.push(current);
|
|
139
|
+
}
|
|
140
|
+
const adapted = current;
|
|
141
|
+
const generated = new Date().toISOString();
|
|
142
|
+
const reportBase = {
|
|
143
|
+
model: input.model,
|
|
144
|
+
generated_at: generated,
|
|
145
|
+
epochs,
|
|
146
|
+
baseline,
|
|
147
|
+
adapted,
|
|
148
|
+
epoch_scorecards: epochScorecards,
|
|
149
|
+
band_deltas: bandDeltas(baseline, adapted),
|
|
150
|
+
playbook_stats: stats,
|
|
151
|
+
};
|
|
152
|
+
const outDir = input.outputDir ?? join(input.workspaceRoot, "agent-state", "eval");
|
|
153
|
+
mkdirSync(outDir, { recursive: true });
|
|
154
|
+
const safeModel = input.model.replace(/[^a-zA-Z0-9_.-]/g, "_");
|
|
155
|
+
const path = join(outDir, `adaptation-${safeModel}-${generated.replace(/[:.]/g, "-")}.md`);
|
|
156
|
+
writeFileSync(path, renderAdaptationMarkdown(reportBase));
|
|
157
|
+
return { ...reportBase, report_path: path };
|
|
158
|
+
}
|
|
159
|
+
//# sourceMappingURL=playbook-adaptation.js.map
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { PlaybookConfig, PlaybookDeltaOp, PlaybookRepository } from "./store/repositories/playbook-repository.js";
|
|
2
|
+
export interface CuratorConfig extends PlaybookConfig {
|
|
3
|
+
dedup_jaccard?: number;
|
|
4
|
+
quarantine_harmful_min?: number;
|
|
5
|
+
quarantine_retire_after_days?: number;
|
|
6
|
+
workspaceRoot?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface CurateResult {
|
|
9
|
+
added: number;
|
|
10
|
+
merged: number;
|
|
11
|
+
tagged: number;
|
|
12
|
+
quarantined: number;
|
|
13
|
+
pruned: number;
|
|
14
|
+
}
|
|
15
|
+
export declare function curate(repo: PlaybookRepository, ops: PlaybookDeltaOp[], config?: CuratorConfig): Promise<CurateResult>;
|
|
16
|
+
//# sourceMappingURL=playbook-curator.d.ts.map
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { calculateJaccard, getShingles } from "./semantic-hash.js";
|
|
2
|
+
import { appendStatusEventSafe } from "./status-events.js";
|
|
3
|
+
function scopeEqualOrBroader(candidate, incoming) {
|
|
4
|
+
return Object.entries(candidate).every(([key, value]) => value === undefined || incoming[key] === value);
|
|
5
|
+
}
|
|
6
|
+
function mergeEvidence(a, b) {
|
|
7
|
+
return [...new Set([...a, ...b])].slice(0, 5);
|
|
8
|
+
}
|
|
9
|
+
function applyLifecycle(bullet, config, now = new Date()) {
|
|
10
|
+
let next = bullet;
|
|
11
|
+
let quarantined = false;
|
|
12
|
+
const harmfulMin = config.quarantine_harmful_min ?? 3;
|
|
13
|
+
if (next.status === "active" && next.harmful_count >= harmfulMin && next.harmful_count > next.helpful_count) {
|
|
14
|
+
next = { ...next, status: "quarantined" };
|
|
15
|
+
quarantined = true;
|
|
16
|
+
}
|
|
17
|
+
if (next.status === "quarantined" && config.quarantine_retire_after_days !== undefined) {
|
|
18
|
+
const ageMs = now.getTime() - new Date(next.updated_at).getTime();
|
|
19
|
+
if (ageMs >= config.quarantine_retire_after_days * 24 * 60 * 60 * 1000) {
|
|
20
|
+
next = { ...next, status: "retired" };
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
return { bullet: next, quarantined };
|
|
24
|
+
}
|
|
25
|
+
export async function curate(repo, ops, config = {}) {
|
|
26
|
+
const result = { added: 0, merged: 0, tagged: 0, quarantined: 0, pruned: 0 };
|
|
27
|
+
const threshold = config.dedup_jaccard ?? 0.85;
|
|
28
|
+
for (const op of ops) {
|
|
29
|
+
if (op.type === "TAG") {
|
|
30
|
+
const bullet = repo.getBullet(op.bullet_id);
|
|
31
|
+
if (!bullet)
|
|
32
|
+
continue;
|
|
33
|
+
const applied = applyLifecycle({
|
|
34
|
+
...bullet,
|
|
35
|
+
helpful_count: bullet.helpful_count + (op.tag === "helpful" ? 1 : 0),
|
|
36
|
+
harmful_count: bullet.harmful_count + (op.tag === "harmful" ? 1 : 0),
|
|
37
|
+
}, config);
|
|
38
|
+
repo.saveBullet(applied.bullet);
|
|
39
|
+
result.tagged += 1;
|
|
40
|
+
if (applied.quarantined)
|
|
41
|
+
result.quarantined += 1;
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
const incomingShingles = getShingles(op.content, 3);
|
|
45
|
+
const match = repo
|
|
46
|
+
.listBullets({ section: op.section, status: "active" })
|
|
47
|
+
.filter((bullet) => scopeEqualOrBroader(bullet.scope, op.scope))
|
|
48
|
+
.map((bullet) => ({
|
|
49
|
+
bullet,
|
|
50
|
+
score: calculateJaccard(getShingles(bullet.content, 3), incomingShingles),
|
|
51
|
+
}))
|
|
52
|
+
.sort((a, b) => b.score - a.score)[0];
|
|
53
|
+
if (match && match.score >= threshold) {
|
|
54
|
+
repo.saveBullet({
|
|
55
|
+
...match.bullet,
|
|
56
|
+
observed_count: match.bullet.observed_count + 1,
|
|
57
|
+
evidence_refs: mergeEvidence(match.bullet.evidence_refs, op.evidence_refs),
|
|
58
|
+
});
|
|
59
|
+
result.merged += 1;
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
const add = repo.applyDeltaOps([op], {
|
|
63
|
+
...config,
|
|
64
|
+
max_active_total: Number.MAX_SAFE_INTEGER,
|
|
65
|
+
max_active_per_section: Number.MAX_SAFE_INTEGER,
|
|
66
|
+
});
|
|
67
|
+
result.added += add.added;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
result.pruned += pruneOverflow(repo, config);
|
|
71
|
+
const eventWorkspaceRoot = config.workspaceRoot ?? repo.root;
|
|
72
|
+
await appendStatusEventSafe({
|
|
73
|
+
event_type: "PLAYBOOK_CURATED",
|
|
74
|
+
source_module: "capability-build",
|
|
75
|
+
status: "done",
|
|
76
|
+
summary: `Playbook curated: added=${result.added} merged=${result.merged} tagged=${result.tagged} quarantined=${result.quarantined} pruned=${result.pruned}`,
|
|
77
|
+
payload: { ...result },
|
|
78
|
+
}, eventWorkspaceRoot ? { workspaceRoot: eventWorkspaceRoot } : {});
|
|
79
|
+
return result;
|
|
80
|
+
}
|
|
81
|
+
function pruneOverflow(repo, config) {
|
|
82
|
+
const maxTotal = config.max_active_total ?? 200;
|
|
83
|
+
const maxPerSection = config.max_active_per_section ?? 64;
|
|
84
|
+
const active = repo.listBullets({ status: "active" });
|
|
85
|
+
const toRetire = new Set();
|
|
86
|
+
const utilitySort = (a, b) => (a.helpful_count - a.harmful_count) - (b.helpful_count - b.harmful_count) ||
|
|
87
|
+
a.observed_count - b.observed_count ||
|
|
88
|
+
(a.updated_at < b.updated_at ? -1 : 1);
|
|
89
|
+
for (const section of ["strategies", "tool_usage", "failure_modes", "verification", "env_facts"]) {
|
|
90
|
+
const sectionActive = active.filter((bullet) => bullet.section === section).sort(utilitySort);
|
|
91
|
+
for (const bullet of sectionActive.slice(0, Math.max(0, sectionActive.length - maxPerSection))) {
|
|
92
|
+
toRetire.add(bullet.bullet_id);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const remaining = active.filter((bullet) => !toRetire.has(bullet.bullet_id)).sort(utilitySort);
|
|
96
|
+
for (const bullet of remaining.slice(0, Math.max(0, remaining.length - maxTotal))) {
|
|
97
|
+
toRetire.add(bullet.bullet_id);
|
|
98
|
+
}
|
|
99
|
+
for (const bulletId of toRetire) {
|
|
100
|
+
const bullet = repo.getBullet(bulletId);
|
|
101
|
+
if (bullet)
|
|
102
|
+
repo.saveBullet({ ...bullet, status: "retired" });
|
|
103
|
+
}
|
|
104
|
+
return toRetire.size;
|
|
105
|
+
}
|
|
106
|
+
//# sourceMappingURL=playbook-curator.js.map
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { type RunLocalModelTaskOptions } from "./local-model-runtime.js";
|
|
2
|
+
import type { BridgeResult } from "./model-bridge.js";
|
|
3
|
+
import { type PlaybookDeltaOp } from "./store/repositories/playbook-repository.js";
|
|
4
|
+
import { type JobHandler } from "./job-worker.js";
|
|
5
|
+
export declare const PLAYBOOK_DELTA_JSON_SCHEMA: Record<string, unknown>;
|
|
6
|
+
export interface PlaybookReflectPayload {
|
|
7
|
+
run_id: string;
|
|
8
|
+
role: string;
|
|
9
|
+
model: string;
|
|
10
|
+
model_class: string;
|
|
11
|
+
result?: BridgeResult;
|
|
12
|
+
max_ops_per_run?: number;
|
|
13
|
+
}
|
|
14
|
+
export declare function validateLlmDeltaOps(input: {
|
|
15
|
+
parsed: unknown;
|
|
16
|
+
payload: PlaybookReflectPayload;
|
|
17
|
+
maxOpsPerRun?: number;
|
|
18
|
+
}): PlaybookDeltaOp[];
|
|
19
|
+
export declare function createPlaybookReflectJobHandler(options: {
|
|
20
|
+
workspaceRoot: string;
|
|
21
|
+
runTask?: (options: RunLocalModelTaskOptions) => Promise<{
|
|
22
|
+
result: BridgeResult;
|
|
23
|
+
}>;
|
|
24
|
+
}): JobHandler;
|
|
25
|
+
export declare function registerPlaybookReflectJobHandler(workspaceRoot: string): void;
|
|
26
|
+
//# sourceMappingURL=playbook-llm-reflector.d.ts.map
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import { readBridgeTrace } from "./bridge-trace.js";
|
|
2
|
+
import { appendStatusEventSafe } from "./status-events.js";
|
|
3
|
+
import { curate } from "./playbook-curator.js";
|
|
4
|
+
import { runLocalModelTask } from "./local-model-runtime.js";
|
|
5
|
+
import { PlaybookRepository, } from "./store/repositories/playbook-repository.js";
|
|
6
|
+
import { registerJobHandler, } from "./job-worker.js";
|
|
7
|
+
export const PLAYBOOK_DELTA_JSON_SCHEMA = {
|
|
8
|
+
type: "object",
|
|
9
|
+
properties: {
|
|
10
|
+
reasoning: { type: "string" },
|
|
11
|
+
operations: {
|
|
12
|
+
type: "array",
|
|
13
|
+
items: {
|
|
14
|
+
type: "object",
|
|
15
|
+
properties: {
|
|
16
|
+
type: { type: "string", enum: ["ADD"] },
|
|
17
|
+
section: {
|
|
18
|
+
type: "string",
|
|
19
|
+
enum: ["strategies", "tool_usage", "failure_modes", "verification", "env_facts"],
|
|
20
|
+
},
|
|
21
|
+
content: { type: "string", maxLength: 400 },
|
|
22
|
+
},
|
|
23
|
+
required: ["type", "section", "content"],
|
|
24
|
+
},
|
|
25
|
+
},
|
|
26
|
+
},
|
|
27
|
+
required: ["operations"],
|
|
28
|
+
};
|
|
29
|
+
const PLAYBOOK_SECTIONS = new Set([
|
|
30
|
+
"strategies",
|
|
31
|
+
"tool_usage",
|
|
32
|
+
"failure_modes",
|
|
33
|
+
"verification",
|
|
34
|
+
"env_facts",
|
|
35
|
+
]);
|
|
36
|
+
function parsePayload(value) {
|
|
37
|
+
const runId = typeof value.run_id === "string" ? value.run_id.trim() : "";
|
|
38
|
+
const role = typeof value.role === "string" ? value.role.trim() : "";
|
|
39
|
+
const model = typeof value.model === "string" ? value.model.trim() : "";
|
|
40
|
+
const modelClass = typeof value.model_class === "string" ? value.model_class.trim() : "";
|
|
41
|
+
if (!runId || !role || !model || !modelClass) {
|
|
42
|
+
throw new Error("playbook.reflect requires run_id, role, model, and model_class.");
|
|
43
|
+
}
|
|
44
|
+
return {
|
|
45
|
+
run_id: runId,
|
|
46
|
+
role,
|
|
47
|
+
model,
|
|
48
|
+
model_class: modelClass,
|
|
49
|
+
result: value.result && typeof value.result === "object" ? value.result : undefined,
|
|
50
|
+
max_ops_per_run: typeof value.max_ops_per_run === "number" && Number.isFinite(value.max_ops_per_run)
|
|
51
|
+
? Math.max(0, Math.floor(value.max_ops_per_run))
|
|
52
|
+
: 3,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
function hasExecutionEvidence(trace, result) {
|
|
56
|
+
return trace.some((line) => (line.tool_calls ?? []).length > 0) || Boolean(result?.reason_code);
|
|
57
|
+
}
|
|
58
|
+
function parseReflectorJson(text) {
|
|
59
|
+
const trimmed = text.trim();
|
|
60
|
+
const start = trimmed.indexOf("{");
|
|
61
|
+
const end = trimmed.lastIndexOf("}");
|
|
62
|
+
if (start === -1 || end < start)
|
|
63
|
+
throw new Error("reflector output did not contain JSON.");
|
|
64
|
+
return JSON.parse(trimmed.slice(start, end + 1));
|
|
65
|
+
}
|
|
66
|
+
export function validateLlmDeltaOps(input) {
|
|
67
|
+
const record = input.parsed && typeof input.parsed === "object" ? input.parsed : {};
|
|
68
|
+
const operations = Array.isArray(record.operations) ? record.operations : [];
|
|
69
|
+
const max = input.maxOpsPerRun ?? input.payload.max_ops_per_run ?? 3;
|
|
70
|
+
const ops = [];
|
|
71
|
+
for (const raw of operations) {
|
|
72
|
+
if (!raw || typeof raw !== "object")
|
|
73
|
+
continue;
|
|
74
|
+
const op = raw;
|
|
75
|
+
if (op.type !== "ADD")
|
|
76
|
+
continue;
|
|
77
|
+
if (typeof op.section !== "string" || !PLAYBOOK_SECTIONS.has(op.section))
|
|
78
|
+
continue;
|
|
79
|
+
const content = typeof op.content === "string" ? op.content.replace(/\s+/g, " ").trim() : "";
|
|
80
|
+
if (!content || content.length > 400)
|
|
81
|
+
continue;
|
|
82
|
+
ops.push({
|
|
83
|
+
type: "ADD",
|
|
84
|
+
section: op.section,
|
|
85
|
+
scope: { role: input.payload.role, model_class: input.payload.model_class, model: input.payload.model },
|
|
86
|
+
content,
|
|
87
|
+
evidence_refs: [`agent-state/bridge-traces/${input.payload.run_id}.ndjson`],
|
|
88
|
+
provenance: { source: "llm_reflector", run_id: input.payload.run_id },
|
|
89
|
+
});
|
|
90
|
+
if (ops.length >= max)
|
|
91
|
+
break;
|
|
92
|
+
}
|
|
93
|
+
return ops;
|
|
94
|
+
}
|
|
95
|
+
function buildReflectorPrompt(payload, traceLines) {
|
|
96
|
+
return [
|
|
97
|
+
"Reflect on this ACE run and propose only durable playbook ADD operations.",
|
|
98
|
+
"Return JSON only with shape: {\"reasoning\":\"short\",\"operations\":[{\"type\":\"ADD\",\"section\":\"verification\",\"content\":\"...\"}]}",
|
|
99
|
+
"Do not alter gates, contracts, files, or source. Content must be imperative and <=400 chars.",
|
|
100
|
+
"",
|
|
101
|
+
`run_id: ${payload.run_id}`,
|
|
102
|
+
`role: ${payload.role}`,
|
|
103
|
+
`model: ${payload.model}`,
|
|
104
|
+
`model_class: ${payload.model_class}`,
|
|
105
|
+
"",
|
|
106
|
+
"result:",
|
|
107
|
+
JSON.stringify(payload.result ?? {}, null, 2),
|
|
108
|
+
"",
|
|
109
|
+
"trace:",
|
|
110
|
+
JSON.stringify(traceLines.slice(-8), null, 2),
|
|
111
|
+
].join("\n");
|
|
112
|
+
}
|
|
113
|
+
export function createPlaybookReflectJobHandler(options) {
|
|
114
|
+
return async ({ job }) => {
|
|
115
|
+
let payload;
|
|
116
|
+
try {
|
|
117
|
+
payload = parsePayload(job.payload ?? {});
|
|
118
|
+
const trace = readBridgeTrace(options.workspaceRoot, payload.run_id);
|
|
119
|
+
if (!hasExecutionEvidence(trace, payload.result)) {
|
|
120
|
+
return { success: true, evidence_ref: `agent-state/bridge-traces/${payload.run_id}.ndjson` };
|
|
121
|
+
}
|
|
122
|
+
const runTask = options.runTask ?? runLocalModelTask;
|
|
123
|
+
const reflection = await runTask({
|
|
124
|
+
task: buildReflectorPrompt(payload, trace),
|
|
125
|
+
role: "qa",
|
|
126
|
+
workspaceRoot: options.workspaceRoot,
|
|
127
|
+
model: payload.model,
|
|
128
|
+
modelClass: payload.model_class,
|
|
129
|
+
maxTurns: 1,
|
|
130
|
+
toolScope: [],
|
|
131
|
+
});
|
|
132
|
+
const parsed = parseReflectorJson(reflection.result.summary);
|
|
133
|
+
const ops = validateLlmDeltaOps({ parsed, payload });
|
|
134
|
+
if (ops.length > 0) {
|
|
135
|
+
await curate(new PlaybookRepository(options.workspaceRoot), ops);
|
|
136
|
+
}
|
|
137
|
+
return { success: true, evidence_ref: `agent-state/bridge-traces/${payload.run_id}.ndjson` };
|
|
138
|
+
}
|
|
139
|
+
catch (error) {
|
|
140
|
+
await appendStatusEventSafe({
|
|
141
|
+
source_module: "capability-build",
|
|
142
|
+
event_type: "PLAYBOOK_REFLECT_FAILED",
|
|
143
|
+
status: "fail",
|
|
144
|
+
summary: `LLM playbook reflector failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
145
|
+
payload: {
|
|
146
|
+
job_id: job.job_id,
|
|
147
|
+
kind: job.kind,
|
|
148
|
+
},
|
|
149
|
+
}).catch(() => undefined);
|
|
150
|
+
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
|
151
|
+
}
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
export function registerPlaybookReflectJobHandler(workspaceRoot) {
|
|
155
|
+
registerJobHandler("playbook.reflect", createPlaybookReflectJobHandler({ workspaceRoot }));
|
|
156
|
+
}
|
|
157
|
+
//# sourceMappingURL=playbook-llm-reflector.js.map
|
package/dist/public-surface.d.ts
CHANGED
|
@@ -21,6 +21,8 @@ export interface PublicSurfaceAuditResult {
|
|
|
21
21
|
surfaces: PublicSurfaceEntry[];
|
|
22
22
|
emitted_events: string[];
|
|
23
23
|
report_path: string;
|
|
24
|
+
artifact_path: string;
|
|
25
|
+
written_path?: string;
|
|
24
26
|
}
|
|
25
27
|
export declare function auditPublicSurface(options?: {
|
|
26
28
|
write_artifact?: boolean;
|