@tangle-network/agent-runtime 0.103.1 → 0.105.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +177 -39
- package/dist/agent.d.ts +1 -1
- package/dist/agent.js +5 -5
- package/dist/analyst-loop.d.ts +1 -1
- package/dist/candidate-execution/index.d.ts +2 -2
- package/dist/candidate-execution/index.js +4 -4
- package/dist/{chunk-TUSOOIDV.js → chunk-C5RTIBNZ.js} +2 -2
- package/dist/{chunk-SNSMRT6H.js → chunk-CTRA64LY.js} +3 -3
- package/dist/{chunk-ZXICDSAK.js → chunk-H5QPIZNX.js} +2 -2
- package/dist/{chunk-6WZZXQV5.js → chunk-HLKC4UYB.js} +1 -13
- package/dist/chunk-HLKC4UYB.js.map +1 -0
- package/dist/{chunk-AAN2MB2X.js → chunk-HNP72PNU.js} +3 -10
- package/dist/chunk-HNP72PNU.js.map +1 -0
- package/dist/{chunk-M6MD6JBS.js → chunk-KRBFHMV6.js} +429 -1
- package/dist/chunk-KRBFHMV6.js.map +1 -0
- package/dist/chunk-OPVWXJ2H.js +75 -0
- package/dist/chunk-OPVWXJ2H.js.map +1 -0
- package/dist/{chunk-AYU35OTU.js → chunk-PZZKQVQV.js} +1 -1
- package/dist/chunk-PZZKQVQV.js.map +1 -0
- package/dist/{chunk-B7K7V22Y.js → chunk-RDOAVVHY.js} +2 -2
- package/dist/{chunk-L5DST3QC.js → chunk-SMQXZGLZ.js} +1 -1
- package/dist/chunk-SMQXZGLZ.js.map +1 -0
- package/dist/{chunk-LFM4JBRW.js → chunk-UHEZW5BU.js} +1084 -527
- package/dist/chunk-UHEZW5BU.js.map +1 -0
- package/dist/{chunk-5AITUUHO.js → chunk-VISA6CI3.js} +3 -3
- package/dist/{chunk-ZOYN3JR5.js → chunk-WMTCUOQL.js} +5 -5
- package/dist/{chunk-3LJF5XSE.js → chunk-WRTOVNN4.js} +6 -6
- package/dist/chunk-WRTOVNN4.js.map +1 -0
- package/dist/{chunk-SBTWKPVR.js → chunk-WSTRQZYQ.js} +1 -1
- package/dist/chunk-WSTRQZYQ.js.map +1 -0
- package/dist/{chunk-EAQ5YRRY.js → chunk-XBG2W2VW.js} +50 -23
- package/dist/chunk-XBG2W2VW.js.map +1 -0
- package/dist/{chunk-QYCKIV6C.js → chunk-YJZA2BIK.js} +2 -2
- package/dist/{chunk-QYCKIV6C.js.map → chunk-YJZA2BIK.js.map} +1 -1
- package/dist/{completion-gate-DLINnrkM.d.ts → completion-gate-BMy5LGoP.d.ts} +3 -3
- package/dist/{coordination-DTehA977.d.ts → coordination-BZZSVYpZ.d.ts} +28 -28
- package/dist/environment-provider.d.ts +18 -7
- package/dist/environment-provider.js +3 -1
- package/dist/index.d.ts +106 -182
- package/dist/index.js +374 -236
- package/dist/index.js.map +1 -1
- package/dist/intelligence.d.ts +93 -47
- package/dist/intelligence.js +127 -73
- package/dist/intelligence.js.map +1 -1
- package/dist/knowledge.d.ts +6 -6
- package/dist/knowledge.js +9 -9
- package/dist/{local-harness-CtK9dSny.d.ts → local-harness-BDNcl6jI.d.ts} +1 -1
- package/dist/{loop-runner-bin-DhIXsLAd.d.ts → loop-runner-bin-68aoM9-7.d.ts} +5 -13
- package/dist/loop-runner-bin.d.ts +6 -8
- package/dist/loop-runner-bin.js +6 -6
- package/dist/loops.d.ts +67 -42
- package/dist/loops.js +10 -6
- package/dist/mcp/bin.js +4 -4
- package/dist/mcp/index.d.ts +9 -10
- package/dist/mcp/index.js +11 -11
- package/dist/mcp/index.js.map +1 -1
- package/dist/mcp/memory-bin.js +1 -1
- package/dist/primeintellect/index.d.ts +3 -4
- package/dist/primeintellect/index.js +5 -6
- package/dist/primeintellect/index.js.map +1 -1
- package/dist/profiles.d.ts +6 -6
- package/dist/profiles.js.map +1 -1
- package/dist/{protected-model-port-DzkUY3HQ.d.ts → protected-model-port-B4t-OQjL.d.ts} +1 -1
- package/dist/{improve-B40dRu0X.d.ts → redact-BuFjxBUJ.d.ts} +201 -154
- package/dist/{structural-rollout-BFPIy3iw.d.ts → structural-rollout-DEh6CEsa.d.ts} +4 -4
- package/dist/{supervise-Cp8bNcJi.d.ts → supervise-4_48wwvy.d.ts} +3 -3
- package/dist/testing.js +9 -9
- package/dist/testing.js.map +1 -1
- package/dist/{types-DK11_O4L.d.ts → types-BCoemcXU.d.ts} +4 -4
- package/dist/{types-lZTE_LBd.d.ts → types-CvNwMzQt.d.ts} +4 -4
- package/dist/{worktree-fanout-DiiJWjJO.d.ts → worktree-fanout-DxhAWr5Z.d.ts} +6 -6
- package/package.json +8 -7
- package/skills/build-with-agent-runtime/SKILL.md +15 -9
- package/skills/loop-writer/SKILL.md +1 -1
- package/dist/chunk-3LJF5XSE.js.map +0 -1
- package/dist/chunk-6WZZXQV5.js.map +0 -1
- package/dist/chunk-AAN2MB2X.js.map +0 -1
- package/dist/chunk-AYU35OTU.js.map +0 -1
- package/dist/chunk-EAQ5YRRY.js.map +0 -1
- package/dist/chunk-L5DST3QC.js.map +0 -1
- package/dist/chunk-LFM4JBRW.js.map +0 -1
- package/dist/chunk-M6MD6JBS.js.map +0 -1
- package/dist/chunk-SBTWKPVR.js.map +0 -1
- /package/dist/{chunk-TUSOOIDV.js.map → chunk-C5RTIBNZ.js.map} +0 -0
- /package/dist/{chunk-SNSMRT6H.js.map → chunk-CTRA64LY.js.map} +0 -0
- /package/dist/{chunk-ZXICDSAK.js.map → chunk-H5QPIZNX.js.map} +0 -0
- /package/dist/{chunk-B7K7V22Y.js.map → chunk-RDOAVVHY.js.map} +0 -0
- /package/dist/{chunk-5AITUUHO.js.map → chunk-VISA6CI3.js.map} +0 -0
- /package/dist/{chunk-ZOYN3JR5.js.map → chunk-WMTCUOQL.js.map} +0 -0
|
@@ -16,17 +16,16 @@ import {
|
|
|
16
16
|
verifiedResourceTextByDigest,
|
|
17
17
|
verifyAgentCandidateBundle,
|
|
18
18
|
verifyCanonicalCandidateDocument
|
|
19
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-WSTRQZYQ.js";
|
|
20
20
|
import {
|
|
21
21
|
defaultStructuralRolloutPolicy,
|
|
22
22
|
optimizerMethod
|
|
23
|
-
} from "./chunk-
|
|
23
|
+
} from "./chunk-YJZA2BIK.js";
|
|
24
24
|
import {
|
|
25
|
-
assertModelAllowed,
|
|
26
25
|
harnessInvocation,
|
|
27
26
|
runLocalHarness,
|
|
28
27
|
runSettledCommand
|
|
29
|
-
} from "./chunk-
|
|
28
|
+
} from "./chunk-HLKC4UYB.js";
|
|
30
29
|
import {
|
|
31
30
|
runAnalystLoop
|
|
32
31
|
} from "./chunk-FD2MBMOH.js";
|
|
@@ -70,7 +69,7 @@ function agenticGenerator(opts = {}) {
|
|
|
70
69
|
kind: `agentic:${harness}`,
|
|
71
70
|
// The seed repo + (in rawTraceContext mode) the raw-trace filesystem context
|
|
72
71
|
// are the change signal — an agentic coder proposes from them even when the
|
|
73
|
-
// distiller yielded zero findings. Without this, the
|
|
72
|
+
// distiller yielded zero findings. Without this, the code candidate driver's
|
|
74
73
|
// empty-findings guard short-circuits and generates ZERO candidates on the
|
|
75
74
|
// first (and, for a single-generation run, only) proposal round.
|
|
76
75
|
proposesWithoutFindings: true,
|
|
@@ -758,148 +757,63 @@ function toAnalystFindings(findings, opts = {}) {
|
|
|
758
757
|
return out;
|
|
759
758
|
}
|
|
760
759
|
|
|
761
|
-
// src/improvement/
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
try {
|
|
772
|
-
await cleanup();
|
|
773
|
-
} catch (cleanupCause) {
|
|
774
|
-
cleanupErrors.push(cleanupCause);
|
|
775
|
-
continue;
|
|
776
|
-
}
|
|
777
|
-
if (cleanupErrors.length === 0) throw cause;
|
|
778
|
-
throw new AggregateError([cause, ...cleanupErrors], `${context}; cleanup retry succeeded`);
|
|
760
|
+
// src/improvement/rollout-policy.ts
|
|
761
|
+
var ROLLOUT_POLICY_EXTENSION = "structural-rollout";
|
|
762
|
+
var isBoundedInt = (v, min) => typeof v === "number" && Number.isInteger(v) && v >= min;
|
|
763
|
+
function parseRolloutPolicy(surface) {
|
|
764
|
+
if (typeof surface !== "string" || surface.trim().length === 0) return void 0;
|
|
765
|
+
let raw;
|
|
766
|
+
try {
|
|
767
|
+
raw = JSON.parse(surface);
|
|
768
|
+
} catch {
|
|
769
|
+
return void 0;
|
|
779
770
|
}
|
|
780
|
-
|
|
771
|
+
return normalizeRolloutPolicy(raw);
|
|
781
772
|
}
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
const
|
|
786
|
-
const
|
|
773
|
+
function normalizeRolloutPolicy(raw) {
|
|
774
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return void 0;
|
|
775
|
+
const bag = raw;
|
|
776
|
+
const k = bag.k ?? defaultStructuralRolloutPolicy.k;
|
|
777
|
+
const repairRounds = bag.repairRounds ?? defaultStructuralRolloutPolicy.repairRounds;
|
|
778
|
+
const testgen = bag.testgen ?? defaultStructuralRolloutPolicy.testgen;
|
|
779
|
+
if (!isBoundedInt(k, 1) || !isBoundedInt(repairRounds, 0) || !isBoundedInt(testgen, 0)) {
|
|
780
|
+
return void 0;
|
|
781
|
+
}
|
|
787
782
|
return {
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
}
|
|
794
|
-
const surfaces = [];
|
|
795
|
-
const incumbent = verifiedCodeIncumbent(ctx.currentSurface);
|
|
796
|
-
const proposalBaseRef = incumbent?.baseCommit ?? baseRef;
|
|
797
|
-
for (let i = 0; i < ctx.populationSize; i++) {
|
|
798
|
-
if (ctx.signal.aborted) break;
|
|
799
|
-
const wt = await opts.worktree.create({
|
|
800
|
-
baseRef: proposalBaseRef,
|
|
801
|
-
label: `${opts.generator.kind}-gen${ctx.generation}-cand${i}`
|
|
802
|
-
});
|
|
803
|
-
owned.set(wt.path, wt);
|
|
804
|
-
try {
|
|
805
|
-
if (incumbent) advanceToIncumbent(wt, incumbent);
|
|
806
|
-
const { applied, summary, label, rationale } = await opts.generator.generate({
|
|
807
|
-
worktreePath: wt.path,
|
|
808
|
-
report: ctx.report,
|
|
809
|
-
findings,
|
|
810
|
-
dataset: ctx.dataset,
|
|
811
|
-
maxShots: ctx.maxImprovementShots ?? 1,
|
|
812
|
-
signal: ctx.signal,
|
|
813
|
-
generation: ctx.generation,
|
|
814
|
-
candidateIndex: i,
|
|
815
|
-
...ctx.costLedger ? { costLedger: ctx.costLedger } : {},
|
|
816
|
-
...ctx.costPhase ? { costPhase: ctx.costPhase } : {}
|
|
817
|
-
});
|
|
818
|
-
if (!applied) {
|
|
819
|
-
await opts.worktree.discard(wt);
|
|
820
|
-
owned.delete(wt.path);
|
|
821
|
-
continue;
|
|
822
|
-
}
|
|
823
|
-
const surface = await opts.worktree.finalize(wt, summary);
|
|
824
|
-
surfaces.push(
|
|
825
|
-
label || rationale ? { surface, label: label ?? "", rationale: rationale ?? "" } : surface
|
|
826
|
-
);
|
|
827
|
-
owned.delete(wt.path);
|
|
828
|
-
owned.set(surface.worktreeRef, wt);
|
|
829
|
-
} catch (err) {
|
|
830
|
-
const failure = err instanceof Error ? err.message : String(err);
|
|
831
|
-
return rethrowAfterCleanup(
|
|
832
|
-
err,
|
|
833
|
-
async () => {
|
|
834
|
-
await opts.worktree.discard(wt);
|
|
835
|
-
owned.delete(wt.path);
|
|
836
|
-
},
|
|
837
|
-
`improvementDriver: ${failure}`
|
|
838
|
-
);
|
|
839
|
-
}
|
|
840
|
-
}
|
|
841
|
-
return surfaces;
|
|
842
|
-
},
|
|
843
|
-
async cleanup(retainWorktreeRefs = []) {
|
|
844
|
-
const retained = new Set(retainWorktreeRefs);
|
|
845
|
-
const errors = [];
|
|
846
|
-
for (const [worktreeRef, worktree] of owned) {
|
|
847
|
-
if (retained.has(worktreeRef)) continue;
|
|
848
|
-
try {
|
|
849
|
-
await opts.worktree.discard(worktree);
|
|
850
|
-
owned.delete(worktreeRef);
|
|
851
|
-
} catch (cause) {
|
|
852
|
-
errors.push(cause);
|
|
853
|
-
}
|
|
854
|
-
}
|
|
855
|
-
if (errors.length > 0) {
|
|
856
|
-
throw new AggregateError(errors, "improvementDriver: failed to discard candidate worktrees");
|
|
857
|
-
}
|
|
858
|
-
}
|
|
783
|
+
k,
|
|
784
|
+
repairRounds,
|
|
785
|
+
testgen,
|
|
786
|
+
...typeof bag.diverse === "boolean" ? { diverse: bag.diverse } : {},
|
|
787
|
+
...typeof bag.temperature === "number" ? { temperature: bag.temperature } : {}
|
|
859
788
|
};
|
|
860
789
|
}
|
|
861
|
-
function
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
throw new Error("improvementDriver: candidate worktree does not match incumbent base identity");
|
|
869
|
-
}
|
|
870
|
-
if (worktree.baseCommit === incumbent.candidateCommit) return;
|
|
871
|
-
const merge = spawnSync2("git", ["merge", "--ff-only", incumbent.candidateCommit], {
|
|
872
|
-
cwd: worktree.path,
|
|
873
|
-
encoding: "utf8"
|
|
874
|
-
});
|
|
875
|
-
if (merge.error) {
|
|
876
|
-
throw new Error(
|
|
877
|
-
`improvementDriver: failed to start candidate from incumbent: ${merge.error.message}`
|
|
878
|
-
);
|
|
879
|
-
}
|
|
880
|
-
if (merge.status !== 0) {
|
|
881
|
-
throw new Error(
|
|
882
|
-
`improvementDriver: could not fast-forward candidate to incumbent ${incumbent.candidateCommit}: ${merge.stderr.trim()}`
|
|
883
|
-
);
|
|
884
|
-
}
|
|
885
|
-
const head = spawnSync2("git", ["rev-parse", "--verify", "HEAD"], {
|
|
886
|
-
cwd: worktree.path,
|
|
887
|
-
encoding: "utf8"
|
|
790
|
+
function serializeRolloutPolicy(policy) {
|
|
791
|
+
return JSON.stringify({
|
|
792
|
+
k: policy.k,
|
|
793
|
+
repairRounds: policy.repairRounds,
|
|
794
|
+
testgen: policy.testgen,
|
|
795
|
+
...policy.diverse !== void 0 ? { diverse: policy.diverse } : {},
|
|
796
|
+
...policy.temperature !== void 0 ? { temperature: policy.temperature } : {}
|
|
888
797
|
});
|
|
889
|
-
if (head.error || head.status !== 0 || head.stdout.trim() !== incumbent.candidateCommit) {
|
|
890
|
-
throw new Error("improvementDriver: candidate worktree did not reach the incumbent commit");
|
|
891
|
-
}
|
|
892
798
|
}
|
|
893
|
-
function
|
|
894
|
-
const
|
|
895
|
-
if (
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
799
|
+
function structuralRolloutPolicyFromProfile(profile) {
|
|
800
|
+
const bag = profile.extensions?.[ROLLOUT_POLICY_EXTENSION];
|
|
801
|
+
if (bag === void 0) return void 0;
|
|
802
|
+
return normalizeRolloutPolicy(bag);
|
|
803
|
+
}
|
|
804
|
+
function applyRolloutPolicyToProfile(profile, policy) {
|
|
805
|
+
const candidate = structuredClone(profile);
|
|
806
|
+
const bag = {
|
|
807
|
+
k: policy.k,
|
|
808
|
+
repairRounds: policy.repairRounds,
|
|
809
|
+
testgen: policy.testgen,
|
|
810
|
+
...policy.diverse !== void 0 ? { diverse: policy.diverse } : {},
|
|
811
|
+
...policy.temperature !== void 0 ? { temperature: policy.temperature } : {}
|
|
812
|
+
};
|
|
813
|
+
return {
|
|
814
|
+
...candidate,
|
|
815
|
+
extensions: { ...candidate.extensions, [ROLLOUT_POLICY_EXTENSION]: bag }
|
|
816
|
+
};
|
|
903
817
|
}
|
|
904
818
|
|
|
905
819
|
// src/improvement/raw-trace-distiller.ts
|
|
@@ -1074,175 +988,221 @@ function truncate2(s, n) {
|
|
|
1074
988
|
return s.length <= n ? s : `${s.slice(0, n - 1)}\u2026`;
|
|
1075
989
|
}
|
|
1076
990
|
|
|
1077
|
-
// src/improvement/rollout-policy.ts
|
|
1078
|
-
var ROLLOUT_POLICY_EXTENSION = "structural-rollout";
|
|
1079
|
-
var ROLLOUT_POLICY_BOUNDS = {
|
|
1080
|
-
k: { min: 1, max: 10, step: 2 },
|
|
1081
|
-
repairRounds: { min: 0, max: 3, step: 1 },
|
|
1082
|
-
testgen: { min: 0, max: 10, step: 3 }
|
|
1083
|
-
};
|
|
1084
|
-
var MAX_CANDIDATES_PER_GENERATION = 4;
|
|
1085
|
-
var clamp = (v, min, max) => Math.min(max, Math.max(min, v));
|
|
1086
|
-
var isBoundedInt = (v, min) => typeof v === "number" && Number.isInteger(v) && v >= min;
|
|
1087
|
-
function parseRolloutPolicy(surface) {
|
|
1088
|
-
if (typeof surface !== "string" || surface.trim().length === 0) return void 0;
|
|
1089
|
-
let raw;
|
|
1090
|
-
try {
|
|
1091
|
-
raw = JSON.parse(surface);
|
|
1092
|
-
} catch {
|
|
1093
|
-
return void 0;
|
|
1094
|
-
}
|
|
1095
|
-
return normalizeRolloutPolicy(raw);
|
|
1096
|
-
}
|
|
1097
|
-
function normalizeRolloutPolicy(raw) {
|
|
1098
|
-
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return void 0;
|
|
1099
|
-
const bag = raw;
|
|
1100
|
-
const k = bag.k ?? defaultStructuralRolloutPolicy.k;
|
|
1101
|
-
const repairRounds = bag.repairRounds ?? defaultStructuralRolloutPolicy.repairRounds;
|
|
1102
|
-
const testgen = bag.testgen ?? defaultStructuralRolloutPolicy.testgen;
|
|
1103
|
-
if (!isBoundedInt(k, 1) || !isBoundedInt(repairRounds, 0) || !isBoundedInt(testgen, 0)) {
|
|
1104
|
-
return void 0;
|
|
1105
|
-
}
|
|
1106
|
-
return {
|
|
1107
|
-
k,
|
|
1108
|
-
repairRounds,
|
|
1109
|
-
testgen,
|
|
1110
|
-
...typeof bag.diverse === "boolean" ? { diverse: bag.diverse } : {},
|
|
1111
|
-
...typeof bag.temperature === "number" ? { temperature: bag.temperature } : {}
|
|
1112
|
-
};
|
|
1113
|
-
}
|
|
1114
|
-
function serializeRolloutPolicy(policy) {
|
|
1115
|
-
return JSON.stringify({
|
|
1116
|
-
k: policy.k,
|
|
1117
|
-
repairRounds: policy.repairRounds,
|
|
1118
|
-
testgen: policy.testgen,
|
|
1119
|
-
...policy.diverse !== void 0 ? { diverse: policy.diverse } : {},
|
|
1120
|
-
...policy.temperature !== void 0 ? { temperature: policy.temperature } : {}
|
|
1121
|
-
});
|
|
1122
|
-
}
|
|
1123
|
-
function structuralRolloutPolicyFromProfile(profile) {
|
|
1124
|
-
const bag = profile.extensions?.[ROLLOUT_POLICY_EXTENSION];
|
|
1125
|
-
if (bag === void 0) return void 0;
|
|
1126
|
-
return normalizeRolloutPolicy(bag);
|
|
1127
|
-
}
|
|
1128
|
-
function applyRolloutPolicyToProfile(profile, policy) {
|
|
1129
|
-
const bag = {
|
|
1130
|
-
k: policy.k,
|
|
1131
|
-
repairRounds: policy.repairRounds,
|
|
1132
|
-
testgen: policy.testgen,
|
|
1133
|
-
...policy.diverse !== void 0 ? { diverse: policy.diverse } : {},
|
|
1134
|
-
...policy.temperature !== void 0 ? { temperature: policy.temperature } : {}
|
|
1135
|
-
};
|
|
1136
|
-
return {
|
|
1137
|
-
...profile,
|
|
1138
|
-
extensions: { ...profile.extensions, [ROLLOUT_POLICY_EXTENSION]: bag }
|
|
1139
|
-
};
|
|
1140
|
-
}
|
|
1141
|
-
function enumerateNeighborPolicies(policy) {
|
|
1142
|
-
const moves = [
|
|
1143
|
-
{ dial: "k", delta: 1 },
|
|
1144
|
-
{ dial: "k", delta: -1 },
|
|
1145
|
-
{ dial: "repairRounds", delta: 1 },
|
|
1146
|
-
{ dial: "repairRounds", delta: -1 },
|
|
1147
|
-
{ dial: "testgen", delta: 1 },
|
|
1148
|
-
{ dial: "testgen", delta: -1 }
|
|
1149
|
-
];
|
|
1150
|
-
const seen = /* @__PURE__ */ new Set([serializeRolloutPolicy(policy)]);
|
|
1151
|
-
const neighbors = [];
|
|
1152
|
-
for (const move of moves) {
|
|
1153
|
-
const bounds = ROLLOUT_POLICY_BOUNDS[move.dial];
|
|
1154
|
-
const next = clamp(policy[move.dial] + move.delta * bounds.step, bounds.min, bounds.max);
|
|
1155
|
-
const candidate = { ...policy, [move.dial]: next };
|
|
1156
|
-
const key = serializeRolloutPolicy(candidate);
|
|
1157
|
-
if (seen.has(key)) continue;
|
|
1158
|
-
seen.add(key);
|
|
1159
|
-
neighbors.push(candidate);
|
|
1160
|
-
}
|
|
1161
|
-
return neighbors;
|
|
1162
|
-
}
|
|
1163
|
-
function candidateLabel(base, next) {
|
|
1164
|
-
for (const dial of ["k", "repairRounds", "testgen"]) {
|
|
1165
|
-
if (next[dial] !== base[dial]) return `${dial} ${base[dial]}\u2192${next[dial]}`;
|
|
1166
|
-
}
|
|
1167
|
-
return "unchanged";
|
|
1168
|
-
}
|
|
1169
|
-
function rolloutPolicyProposer() {
|
|
1170
|
-
return {
|
|
1171
|
-
kind: "rollout-policy",
|
|
1172
|
-
async propose(ctx) {
|
|
1173
|
-
const policy = parseRolloutPolicy(ctx.currentSurface);
|
|
1174
|
-
if (!policy) return [];
|
|
1175
|
-
const neighbors = enumerateNeighborPolicies(policy);
|
|
1176
|
-
if (neighbors.length === 0) return [];
|
|
1177
|
-
const cap = Math.max(1, Math.min(ctx.populationSize, MAX_CANDIDATES_PER_GENERATION));
|
|
1178
|
-
const start = ctx.generation * cap % neighbors.length;
|
|
1179
|
-
const window = [];
|
|
1180
|
-
for (let i = 0; i < Math.min(cap, neighbors.length); i += 1) {
|
|
1181
|
-
window.push(neighbors[(start + i) % neighbors.length]);
|
|
1182
|
-
}
|
|
1183
|
-
return window.map((candidate) => ({
|
|
1184
|
-
surface: serializeRolloutPolicy(candidate),
|
|
1185
|
-
label: candidateLabel(policy, candidate),
|
|
1186
|
-
rationale: "bounded single-dial neighbor of the current structuralRollout policy; the held-out gate decides (deterministic enumeration \u2014 the dial space is tiny and prompt-style reflective proposals are a measured zero here)"
|
|
1187
|
-
}));
|
|
1188
|
-
}
|
|
1189
|
-
};
|
|
1190
|
-
}
|
|
1191
|
-
|
|
1192
991
|
// src/improvement/improve.ts
|
|
1193
|
-
import {
|
|
992
|
+
import { agentProfileSchema as agentProfileSchema2 } from "@tangle-network/agent-interface";
|
|
993
|
+
|
|
994
|
+
// src/improvement/code-execution.ts
|
|
995
|
+
import { makeFinding as makeFinding3 } from "@tangle-network/agent-eval";
|
|
1194
996
|
import {
|
|
1195
|
-
|
|
1196
|
-
gitWorktreeAdapter,
|
|
1197
|
-
memoryCurationProposer,
|
|
1198
|
-
skillOptProposer
|
|
997
|
+
gitWorktreeAdapter
|
|
1199
998
|
} from "@tangle-network/agent-eval/campaign";
|
|
1200
999
|
import {
|
|
1201
1000
|
selfImprove
|
|
1202
1001
|
} from "@tangle-network/agent-eval/contract";
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1002
|
+
|
|
1003
|
+
// src/improvement/cleanup.ts
|
|
1004
|
+
async function rethrowAfterCleanup(cause, cleanup, context) {
|
|
1005
|
+
const cleanupErrors = [];
|
|
1006
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
1007
|
+
try {
|
|
1008
|
+
await cleanup();
|
|
1009
|
+
} catch (cleanupCause) {
|
|
1010
|
+
cleanupErrors.push(cleanupCause);
|
|
1011
|
+
continue;
|
|
1012
|
+
}
|
|
1013
|
+
if (cleanupErrors.length === 0) throw cause;
|
|
1014
|
+
throw new AggregateError([cause, ...cleanupErrors], `${context}; cleanup retry succeeded`);
|
|
1015
|
+
}
|
|
1016
|
+
throw new AggregateError([cause, ...cleanupErrors], `${context}; cleanup failed`);
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
// src/improvement/improve-result.ts
|
|
1020
|
+
function copyImproveCost(cost) {
|
|
1021
|
+
return {
|
|
1022
|
+
totalCostUsd: cost.totalCostUsd,
|
|
1023
|
+
accountingComplete: cost.accountingComplete,
|
|
1024
|
+
incompleteReasons: [...cost.incompleteReasons]
|
|
1025
|
+
};
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
// src/improvement/improvement-driver.ts
|
|
1029
|
+
import { spawnSync as spawnSync2 } from "child_process";
|
|
1030
|
+
import {
|
|
1031
|
+
verifyCodeSurface
|
|
1032
|
+
} from "@tangle-network/agent-eval/campaign";
|
|
1033
|
+
function improvementDriver(opts) {
|
|
1034
|
+
const baseRef = opts.baseRef ?? "main";
|
|
1035
|
+
const owned = /* @__PURE__ */ new Map();
|
|
1036
|
+
return {
|
|
1037
|
+
kind: `improvement:${opts.generator.kind}`,
|
|
1038
|
+
async propose(ctx) {
|
|
1039
|
+
const findings = resolveFindings(ctx);
|
|
1040
|
+
if (findings.length === 0 && ctx.report === void 0 && !opts.generator.proposesWithoutFindings) {
|
|
1041
|
+
return [];
|
|
1042
|
+
}
|
|
1043
|
+
const surfaces = [];
|
|
1044
|
+
const incumbent = verifiedCodeIncumbent(ctx.currentSurface);
|
|
1045
|
+
const proposalBaseRef = incumbent?.baseCommit ?? baseRef;
|
|
1046
|
+
for (let i = 0; i < ctx.populationSize; i++) {
|
|
1047
|
+
if (ctx.signal.aborted) break;
|
|
1048
|
+
const wt = await opts.worktree.create({
|
|
1049
|
+
baseRef: proposalBaseRef,
|
|
1050
|
+
label: `${opts.generator.kind}-gen${ctx.generation}-cand${i}`
|
|
1051
|
+
});
|
|
1052
|
+
owned.set(wt.path, wt);
|
|
1053
|
+
try {
|
|
1054
|
+
if (incumbent) advanceToIncumbent(wt, incumbent);
|
|
1055
|
+
const { applied, summary, label, rationale } = await opts.generator.generate({
|
|
1056
|
+
worktreePath: wt.path,
|
|
1057
|
+
report: ctx.report,
|
|
1058
|
+
findings,
|
|
1059
|
+
dataset: ctx.dataset,
|
|
1060
|
+
maxShots: ctx.maxImprovementShots ?? 1,
|
|
1061
|
+
signal: ctx.signal,
|
|
1062
|
+
generation: ctx.generation,
|
|
1063
|
+
candidateIndex: i,
|
|
1064
|
+
...ctx.costLedger ? { costLedger: ctx.costLedger } : {},
|
|
1065
|
+
...ctx.costPhase ? { costPhase: ctx.costPhase } : {}
|
|
1066
|
+
});
|
|
1067
|
+
if (!applied) {
|
|
1068
|
+
await opts.worktree.discard(wt);
|
|
1069
|
+
owned.delete(wt.path);
|
|
1070
|
+
continue;
|
|
1071
|
+
}
|
|
1072
|
+
const surface = await opts.worktree.finalize(wt, summary);
|
|
1073
|
+
surfaces.push(
|
|
1074
|
+
label || rationale ? { surface, label: label ?? "", rationale: rationale ?? "" } : surface
|
|
1075
|
+
);
|
|
1076
|
+
owned.delete(wt.path);
|
|
1077
|
+
owned.set(surface.worktreeRef, wt);
|
|
1078
|
+
} catch (err) {
|
|
1079
|
+
const failure = err instanceof Error ? err.message : String(err);
|
|
1080
|
+
return rethrowAfterCleanup(
|
|
1081
|
+
err,
|
|
1082
|
+
async () => {
|
|
1083
|
+
await opts.worktree.discard(wt);
|
|
1084
|
+
owned.delete(wt.path);
|
|
1085
|
+
},
|
|
1086
|
+
`improvementDriver: ${failure}`
|
|
1087
|
+
);
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
return surfaces;
|
|
1091
|
+
},
|
|
1092
|
+
async cleanup(retainWorktreeRefs = []) {
|
|
1093
|
+
const retained = new Set(retainWorktreeRefs);
|
|
1094
|
+
const errors = [];
|
|
1095
|
+
for (const [worktreeRef, worktree] of owned) {
|
|
1096
|
+
if (retained.has(worktreeRef)) continue;
|
|
1097
|
+
try {
|
|
1098
|
+
await opts.worktree.discard(worktree);
|
|
1099
|
+
owned.delete(worktreeRef);
|
|
1100
|
+
} catch (cause) {
|
|
1101
|
+
errors.push(cause);
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
if (errors.length > 0) {
|
|
1105
|
+
throw new AggregateError(errors, "improvementDriver: failed to discard candidate worktrees");
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
};
|
|
1109
|
+
}
|
|
1110
|
+
function verifiedCodeIncumbent(surface) {
|
|
1111
|
+
if (typeof surface !== "object" || surface.kind !== "code") return void 0;
|
|
1112
|
+
verifyCodeSurface(surface);
|
|
1113
|
+
return surface;
|
|
1224
1114
|
}
|
|
1225
|
-
function
|
|
1115
|
+
function advanceToIncumbent(worktree, incumbent) {
|
|
1116
|
+
if (worktree.baseCommit !== incumbent.baseCommit || worktree.baseTree !== incumbent.baseTree) {
|
|
1117
|
+
throw new Error("improvementDriver: candidate worktree does not match incumbent base identity");
|
|
1118
|
+
}
|
|
1119
|
+
if (worktree.baseCommit === incumbent.candidateCommit) return;
|
|
1120
|
+
const merge = spawnSync2("git", ["merge", "--ff-only", incumbent.candidateCommit], {
|
|
1121
|
+
cwd: worktree.path,
|
|
1122
|
+
encoding: "utf8"
|
|
1123
|
+
});
|
|
1124
|
+
if (merge.error) {
|
|
1125
|
+
throw new Error(
|
|
1126
|
+
`improvementDriver: failed to start candidate from incumbent: ${merge.error.message}`
|
|
1127
|
+
);
|
|
1128
|
+
}
|
|
1129
|
+
if (merge.status !== 0) {
|
|
1130
|
+
throw new Error(
|
|
1131
|
+
`improvementDriver: could not fast-forward candidate to incumbent ${incumbent.candidateCommit}: ${merge.stderr.trim()}`
|
|
1132
|
+
);
|
|
1133
|
+
}
|
|
1134
|
+
const head = spawnSync2("git", ["rev-parse", "--verify", "HEAD"], {
|
|
1135
|
+
cwd: worktree.path,
|
|
1136
|
+
encoding: "utf8"
|
|
1137
|
+
});
|
|
1138
|
+
if (head.error || head.status !== 0 || head.stdout.trim() !== incumbent.candidateCommit) {
|
|
1139
|
+
throw new Error("improvementDriver: candidate worktree did not reach the incumbent commit");
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1142
|
+
function resolveFindings(ctx) {
|
|
1143
|
+
const report = ctx.report;
|
|
1144
|
+
if (report && typeof report === "object" && "findings" in report) {
|
|
1145
|
+
const f = report.findings;
|
|
1146
|
+
if (Array.isArray(f) && f.length > 0) {
|
|
1147
|
+
const lifted = toAnalystFindings(f, { analystId: "report-findings", area: "report" });
|
|
1148
|
+
if (lifted.length > 0) return lifted;
|
|
1149
|
+
}
|
|
1150
|
+
}
|
|
1151
|
+
return toAnalystFindings(ctx.findings ?? [], { analystId: "loop-context", area: "seed" });
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
// src/improvement/profile-surface.ts
|
|
1155
|
+
import { canonicalJson } from "@tangle-network/agent-eval";
|
|
1156
|
+
import {
|
|
1157
|
+
agentProfileSchema
|
|
1158
|
+
} from "@tangle-network/agent-interface";
|
|
1159
|
+
function prepareProfileSurface(profile, surface, skills, profileComponents) {
|
|
1226
1160
|
switch (surface) {
|
|
1227
1161
|
case "prompt":
|
|
1228
|
-
return
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
return
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
return
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
return
|
|
1162
|
+
return {
|
|
1163
|
+
surface: profile.prompt?.systemPrompt ?? "",
|
|
1164
|
+
value: profile.prompt?.systemPrompt ?? ""
|
|
1165
|
+
};
|
|
1166
|
+
case "skills": {
|
|
1167
|
+
const value = inlineSkill(profile, skills).content;
|
|
1168
|
+
return { surface: value, value };
|
|
1169
|
+
}
|
|
1170
|
+
case "tools": {
|
|
1171
|
+
const value = profile.tools ?? {};
|
|
1172
|
+
return { surface: canonicalJson(value), value };
|
|
1173
|
+
}
|
|
1174
|
+
case "mcp": {
|
|
1175
|
+
const value = profile.mcp ?? {};
|
|
1176
|
+
return { surface: canonicalJson(value), value };
|
|
1177
|
+
}
|
|
1178
|
+
case "hooks": {
|
|
1179
|
+
const value = profile.hooks ?? {};
|
|
1180
|
+
return { surface: canonicalJson(value), value };
|
|
1181
|
+
}
|
|
1182
|
+
case "subagents": {
|
|
1183
|
+
const value = profile.subagents ?? {};
|
|
1184
|
+
return { surface: canonicalJson(value), value };
|
|
1185
|
+
}
|
|
1186
|
+
case "agent-profile": {
|
|
1187
|
+
if (profileComponents) {
|
|
1188
|
+
const value = profileComponents.read(profile);
|
|
1189
|
+
return {
|
|
1190
|
+
surface: componentSurface(value, "profileComponents.read"),
|
|
1191
|
+
value
|
|
1192
|
+
};
|
|
1193
|
+
}
|
|
1194
|
+
return { surface: canonicalJson(profile), value: profile };
|
|
1195
|
+
}
|
|
1196
|
+
case "memory": {
|
|
1197
|
+
const value = profileInstructions(profile);
|
|
1198
|
+
return { surface: value, value };
|
|
1199
|
+
}
|
|
1243
1200
|
case "rollout-policy": {
|
|
1244
1201
|
const policy = structuralRolloutPolicyFromProfile(profile);
|
|
1245
|
-
return
|
|
1202
|
+
return {
|
|
1203
|
+
surface: policy ? serializeRolloutPolicy(policy) : "",
|
|
1204
|
+
value: policy ?? null
|
|
1205
|
+
};
|
|
1246
1206
|
}
|
|
1247
1207
|
case "code":
|
|
1248
1208
|
throw new ConfigError(
|
|
@@ -1250,139 +1210,35 @@ function baselineSurfaceFor(profile, surface, skills) {
|
|
|
1250
1210
|
);
|
|
1251
1211
|
}
|
|
1252
1212
|
}
|
|
1253
|
-
var DISTILLED_NOTES_MAX_CHARS = 1500;
|
|
1254
|
-
var DISTILLED_ERROR_MAX_CHARS = 500;
|
|
1255
|
-
function generationFailureDistiller(staticFindings) {
|
|
1256
|
-
const CAP = 12;
|
|
1257
|
-
return async (input) => {
|
|
1258
|
-
const failures = [];
|
|
1259
|
-
for (const candidate of input.candidates) {
|
|
1260
|
-
for (const rawCell of candidate.campaign.cells) {
|
|
1261
|
-
const cell = rawCell;
|
|
1262
|
-
const scenario = String(cell.scenarioId ?? "unknown");
|
|
1263
|
-
const error = typeof cell.error === "string" ? cell.error : void 0;
|
|
1264
|
-
const judgeScores = cell.judgeScores && typeof cell.judgeScores === "object" ? Object.values(
|
|
1265
|
-
cell.judgeScores
|
|
1266
|
-
) : [];
|
|
1267
|
-
const composite = judgeScores.length === 0 ? 0 : judgeScores.reduce((sum, j) => sum + (j.composite ?? 0), 0) / judgeScores.length;
|
|
1268
|
-
if (!error && composite >= 0.999) continue;
|
|
1269
|
-
const notes = judgeScores.map((j) => j.notes).filter((n) => typeof n === "string" && n.length > 0).join("; ").slice(0, DISTILLED_NOTES_MAX_CHARS);
|
|
1270
|
-
const claim = notes || (error ? `Scenario ${scenario} failed: ${error.slice(0, DISTILLED_ERROR_MAX_CHARS)}` : "");
|
|
1271
|
-
failures.push({
|
|
1272
|
-
scenario,
|
|
1273
|
-
composite: Number(composite.toFixed(3)),
|
|
1274
|
-
notes,
|
|
1275
|
-
...claim ? { claim } : {},
|
|
1276
|
-
...error ? { error: error.slice(0, DISTILLED_ERROR_MAX_CHARS) } : {}
|
|
1277
|
-
});
|
|
1278
|
-
}
|
|
1279
|
-
}
|
|
1280
|
-
if (failures.length === 0) return staticFindings;
|
|
1281
|
-
failures.sort((a, b) => a.composite - b.composite);
|
|
1282
|
-
return failures.slice(0, CAP).map(
|
|
1283
|
-
(f) => makeFinding3({
|
|
1284
|
-
analyst_id: "generation-failure-distiller",
|
|
1285
|
-
severity: f.error !== void 0 || f.composite < 0.5 ? "high" : "medium",
|
|
1286
|
-
area: "generation-failure",
|
|
1287
|
-
confidence: 1,
|
|
1288
|
-
subject: f.scenario,
|
|
1289
|
-
claim: `Scenario ${f.scenario} scored composite ${f.composite}${f.notes ? `: ${f.notes}` : ""}${f.error ? ` (error: ${f.error})` : ""}`,
|
|
1290
|
-
evidence_refs: [],
|
|
1291
|
-
metadata: {
|
|
1292
|
-
scenario: f.scenario,
|
|
1293
|
-
composite: f.composite,
|
|
1294
|
-
...f.notes ? { notes: f.notes } : {},
|
|
1295
|
-
...f.error !== void 0 ? { error: f.error } : {}
|
|
1296
|
-
}
|
|
1297
|
-
})
|
|
1298
|
-
);
|
|
1299
|
-
};
|
|
1300
|
-
}
|
|
1301
|
-
function memoryGenerationDistiller(staticFindings) {
|
|
1302
|
-
const distillFailures = generationFailureDistiller(staticFindings);
|
|
1303
|
-
return async (input) => {
|
|
1304
|
-
const fresh = await distillFailures(input);
|
|
1305
|
-
return fresh === staticFindings ? staticFindings : [...staticFindings, ...fresh];
|
|
1306
|
-
};
|
|
1307
|
-
}
|
|
1308
|
-
function defaultDistillerFor(opts, findings) {
|
|
1309
|
-
const surface = opts.surface ?? "prompt";
|
|
1310
|
-
const durableRun = opts.runDir !== void 0 && !opts.runDir.startsWith("mem://");
|
|
1311
|
-
const useRawTraces = opts.rawTraceContext ?? (surface === "memory" ? false : durableRun);
|
|
1312
|
-
if (useRawTraces) return rawTraceDistiller({ fallbackFindings: findings });
|
|
1313
|
-
if (surface === "memory") return memoryGenerationDistiller(findings);
|
|
1314
|
-
return generationFailureDistiller(findings);
|
|
1315
|
-
}
|
|
1316
|
-
async function discardPreparedBaseline(worktree, baselineWorktree, cause) {
|
|
1317
|
-
return rethrowAfterCleanup(
|
|
1318
|
-
cause,
|
|
1319
|
-
() => worktree.discard(baselineWorktree),
|
|
1320
|
-
"improve(): code preparation failed"
|
|
1321
|
-
);
|
|
1322
|
-
}
|
|
1323
1213
|
function isCodeSurface(surface) {
|
|
1324
1214
|
return typeof surface === "object" && surface !== null && surface.kind === "code";
|
|
1325
1215
|
}
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1216
|
+
function isComponentSurface(surface) {
|
|
1217
|
+
return typeof surface === "object" && surface !== null && surface.kind === "components";
|
|
1218
|
+
}
|
|
1219
|
+
function componentSurface(components, source) {
|
|
1220
|
+
const entries = validateComponents(components, source);
|
|
1221
|
+
return immutableCandidateValue({
|
|
1222
|
+
kind: "components",
|
|
1223
|
+
components: Object.fromEntries(entries)
|
|
1331
1224
|
});
|
|
1332
|
-
const baselineWorktree = await worktree.create({ baseRef, label: "incumbent-baseline" });
|
|
1333
|
-
try {
|
|
1334
|
-
const baseline = await worktree.finalize(baselineWorktree, "Incumbent code checkout");
|
|
1335
|
-
let baselineDiscarded = false;
|
|
1336
|
-
const generator = code.generator ?? agenticGenerator({
|
|
1337
|
-
...code.harness ? { harness: code.harness } : {},
|
|
1338
|
-
...code.verify ? { verify: code.verify } : {},
|
|
1339
|
-
...code.timeoutMs ? { timeoutMs: code.timeoutMs } : {}
|
|
1340
|
-
});
|
|
1341
|
-
const managed = improvementDriver({ worktree, generator, baseRef });
|
|
1342
|
-
return {
|
|
1343
|
-
baseline,
|
|
1344
|
-
proposer: managed,
|
|
1345
|
-
async cleanup(retainedWinner) {
|
|
1346
|
-
const errors = [];
|
|
1347
|
-
const retainedWorktreeRef = isCodeSurface(retainedWinner) ? retainedWinner.worktreeRef : void 0;
|
|
1348
|
-
try {
|
|
1349
|
-
await managed?.cleanup(retainedWorktreeRef ? [retainedWorktreeRef] : []);
|
|
1350
|
-
} catch (cause) {
|
|
1351
|
-
errors.push(cause);
|
|
1352
|
-
}
|
|
1353
|
-
if (!baselineDiscarded && retainedWorktreeRef !== baseline.worktreeRef) {
|
|
1354
|
-
try {
|
|
1355
|
-
await worktree.discard(baselineWorktree);
|
|
1356
|
-
baselineDiscarded = true;
|
|
1357
|
-
} catch (cause) {
|
|
1358
|
-
errors.push(cause);
|
|
1359
|
-
}
|
|
1360
|
-
}
|
|
1361
|
-
if (errors.length > 0) {
|
|
1362
|
-
throw new AggregateError(errors, "improve(): failed to clean code improvement worktrees");
|
|
1363
|
-
}
|
|
1364
|
-
}
|
|
1365
|
-
};
|
|
1366
|
-
} catch (cause) {
|
|
1367
|
-
return discardPreparedBaseline(worktree, baselineWorktree, cause);
|
|
1368
|
-
}
|
|
1369
1225
|
}
|
|
1370
|
-
function
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
inFlight = void 0;
|
|
1226
|
+
function validateComponents(components, source) {
|
|
1227
|
+
if (typeof components !== "object" || components === null || Array.isArray(components)) {
|
|
1228
|
+
throw new ConfigError(`improve(): ${source} must return a component record`);
|
|
1229
|
+
}
|
|
1230
|
+
const entries = Object.entries(components);
|
|
1231
|
+
if (entries.length === 0) {
|
|
1232
|
+
throw new ConfigError(`improve(): ${source} must return at least one component`);
|
|
1233
|
+
}
|
|
1234
|
+
for (const [name, value] of entries) {
|
|
1235
|
+
if (!name || name.trim() !== name || typeof value !== "string") {
|
|
1236
|
+
throw new ConfigError(
|
|
1237
|
+
`improve(): ${source} must return trimmed component names with string values`
|
|
1238
|
+
);
|
|
1384
1239
|
}
|
|
1385
|
-
}
|
|
1240
|
+
}
|
|
1241
|
+
return entries;
|
|
1386
1242
|
}
|
|
1387
1243
|
function parseWinnerJson(winner, surface) {
|
|
1388
1244
|
try {
|
|
@@ -1393,16 +1249,58 @@ function parseWinnerJson(winner, surface) {
|
|
|
1393
1249
|
);
|
|
1394
1250
|
}
|
|
1395
1251
|
}
|
|
1396
|
-
function assertCandidateSurfaceKind(surface, winner) {
|
|
1397
|
-
if (surface === "code"
|
|
1252
|
+
function assertCandidateSurfaceKind(surface, baseline, winner) {
|
|
1253
|
+
if (surface === "code") {
|
|
1254
|
+
if (isCodeSurface(winner)) return;
|
|
1255
|
+
throw new ConfigError(
|
|
1256
|
+
`improve(): the '${surface}' candidate returned an incompatible surface value`
|
|
1257
|
+
);
|
|
1258
|
+
}
|
|
1259
|
+
if (typeof baseline === "string") {
|
|
1260
|
+
if (typeof winner === "string") return;
|
|
1261
|
+
throw new ConfigError(
|
|
1262
|
+
`improve(): the '${surface}' candidate changed from a text surface to an incompatible surface value`
|
|
1263
|
+
);
|
|
1264
|
+
}
|
|
1265
|
+
if (!isComponentSurface(baseline) || !isComponentSurface(winner)) {
|
|
1398
1266
|
throw new ConfigError(
|
|
1399
1267
|
`improve(): the '${surface}' candidate returned an incompatible surface value`
|
|
1400
1268
|
);
|
|
1401
1269
|
}
|
|
1270
|
+
validateComponents(winner.components, `the '${surface}' candidate`);
|
|
1271
|
+
const baselineNames = Object.keys(baseline.components).sort();
|
|
1272
|
+
const winnerNames = Object.keys(winner.components).sort();
|
|
1273
|
+
if (baselineNames.length !== winnerNames.length || baselineNames.some((name, index) => name !== winnerNames[index])) {
|
|
1274
|
+
throw new ConfigError(
|
|
1275
|
+
`improve(): the '${surface}' candidate must preserve the exact component names`
|
|
1276
|
+
);
|
|
1277
|
+
}
|
|
1402
1278
|
}
|
|
1403
|
-
function materializeImprovementProfileCandidate(profile, surface, winner, skills) {
|
|
1404
|
-
if (typeof winner !== "string") return void 0;
|
|
1279
|
+
function materializeImprovementProfileCandidate(profile, surface, winner, skills, profileComponents) {
|
|
1405
1280
|
let candidate;
|
|
1281
|
+
if (isComponentSurface(winner)) {
|
|
1282
|
+
if (surface !== "agent-profile" || !profileComponents) {
|
|
1283
|
+
throw new ConfigError(
|
|
1284
|
+
`improve(): the '${surface}' candidate has no profile component mapping`
|
|
1285
|
+
);
|
|
1286
|
+
}
|
|
1287
|
+
const winnerComponents = immutableCandidateValue({ ...winner.components });
|
|
1288
|
+
const applied = profileComponents.apply(profile, winnerComponents);
|
|
1289
|
+
const validated = validateProfileCandidate(applied, surface);
|
|
1290
|
+
const materializedComponents = Object.fromEntries(
|
|
1291
|
+
validateComponents(profileComponents.read(validated), "profileComponents.read after apply")
|
|
1292
|
+
);
|
|
1293
|
+
const names = Object.keys(winnerComponents);
|
|
1294
|
+
if (names.length !== Object.keys(materializedComponents).length || names.some((name) => materializedComponents[name] !== winnerComponents[name])) {
|
|
1295
|
+
throw new ConfigError(
|
|
1296
|
+
"improve(): profileComponents.apply must round-trip every winning component exactly"
|
|
1297
|
+
);
|
|
1298
|
+
}
|
|
1299
|
+
return validated;
|
|
1300
|
+
}
|
|
1301
|
+
if (typeof winner !== "string") {
|
|
1302
|
+
throw new ConfigError(`improve(): the '${surface}' candidate cannot form an AgentProfile`);
|
|
1303
|
+
}
|
|
1406
1304
|
switch (surface) {
|
|
1407
1305
|
case "prompt":
|
|
1408
1306
|
candidate = { ...profile, prompt: { ...profile.prompt, systemPrompt: winner } };
|
|
@@ -1445,7 +1343,6 @@ function materializeImprovementProfileCandidate(profile, surface, winner, skills
|
|
|
1445
1343
|
};
|
|
1446
1344
|
break;
|
|
1447
1345
|
case "rollout-policy": {
|
|
1448
|
-
if (winner === "") return void 0;
|
|
1449
1346
|
const policy = normalizeRolloutPolicy(parseWinnerJson(winner, surface));
|
|
1450
1347
|
if (!policy) {
|
|
1451
1348
|
throw new ConfigError(
|
|
@@ -1455,9 +1352,10 @@ function materializeImprovementProfileCandidate(profile, surface, winner, skills
|
|
|
1455
1352
|
candidate = applyRolloutPolicyToProfile(profile, policy);
|
|
1456
1353
|
break;
|
|
1457
1354
|
}
|
|
1458
|
-
case "code":
|
|
1459
|
-
return void 0;
|
|
1460
1355
|
}
|
|
1356
|
+
return validateProfileCandidate(candidate, surface);
|
|
1357
|
+
}
|
|
1358
|
+
function validateProfileCandidate(candidate, surface) {
|
|
1461
1359
|
const parsed = agentProfileSchema.safeParse(candidate);
|
|
1462
1360
|
if (!parsed.success) {
|
|
1463
1361
|
throw new ConfigError(
|
|
@@ -1466,6 +1364,39 @@ function materializeImprovementProfileCandidate(profile, surface, winner, skills
|
|
|
1466
1364
|
}
|
|
1467
1365
|
return immutableCandidateValue(parsed.data);
|
|
1468
1366
|
}
|
|
1367
|
+
function createProfileCandidateMaterializer(profile, surface, baselineSurface, skills, profileComponents) {
|
|
1368
|
+
const baselineDigest = canonicalCandidateDigest(baselineSurface);
|
|
1369
|
+
if (profileComponents) {
|
|
1370
|
+
const reappliedBaseline = materializeImprovementProfileCandidate(
|
|
1371
|
+
profile,
|
|
1372
|
+
surface,
|
|
1373
|
+
baselineSurface,
|
|
1374
|
+
skills,
|
|
1375
|
+
profileComponents
|
|
1376
|
+
);
|
|
1377
|
+
if (canonicalCandidateDigest(reappliedBaseline) !== canonicalCandidateDigest(profile)) {
|
|
1378
|
+
throw new ConfigError(
|
|
1379
|
+
"improve(): profileComponents.apply(profile, profileComponents.read(profile)) must reproduce the complete baseline profile exactly"
|
|
1380
|
+
);
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
const candidates = /* @__PURE__ */ new Map([[baselineDigest, profile]]);
|
|
1384
|
+
return (candidateSurface) => {
|
|
1385
|
+
assertCandidateSurfaceKind(surface, baselineSurface, candidateSurface);
|
|
1386
|
+
const digest = canonicalCandidateDigest(candidateSurface);
|
|
1387
|
+
const existing = candidates.get(digest);
|
|
1388
|
+
if (existing) return existing;
|
|
1389
|
+
const candidate = materializeImprovementProfileCandidate(
|
|
1390
|
+
profile,
|
|
1391
|
+
surface,
|
|
1392
|
+
immutableCandidateValue(candidateSurface),
|
|
1393
|
+
skills,
|
|
1394
|
+
profileComponents
|
|
1395
|
+
);
|
|
1396
|
+
candidates.set(digest, candidate);
|
|
1397
|
+
return candidate;
|
|
1398
|
+
};
|
|
1399
|
+
}
|
|
1469
1400
|
function inlineSkill(profile, options) {
|
|
1470
1401
|
const resourceName = options?.resourceName.trim();
|
|
1471
1402
|
if (!resourceName) {
|
|
@@ -1494,73 +1425,167 @@ function profileInstructions(profile) {
|
|
|
1494
1425
|
"improve(): surface 'memory' requires inline profile instructions so candidate bytes are exact"
|
|
1495
1426
|
);
|
|
1496
1427
|
}
|
|
1497
|
-
function replaceProfileInstructions(profile, content) {
|
|
1498
|
-
const instructions = profile.resources?.instructions;
|
|
1499
|
-
if (typeof instructions !== "object") return content;
|
|
1500
|
-
if (instructions.kind !== "inline") {
|
|
1501
|
-
throw new ConfigError(
|
|
1502
|
-
"improve(): surface 'memory' requires inline profile instructions so candidate bytes are exact"
|
|
1503
|
-
);
|
|
1428
|
+
function replaceProfileInstructions(profile, content) {
|
|
1429
|
+
const instructions = profile.resources?.instructions;
|
|
1430
|
+
if (typeof instructions !== "object") return content;
|
|
1431
|
+
if (instructions.kind !== "inline") {
|
|
1432
|
+
throw new ConfigError(
|
|
1433
|
+
"improve(): surface 'memory' requires inline profile instructions so candidate bytes are exact"
|
|
1434
|
+
);
|
|
1435
|
+
}
|
|
1436
|
+
return { ...instructions, content };
|
|
1437
|
+
}
|
|
1438
|
+
function assertFailClosedResources(profile, surface) {
|
|
1439
|
+
if (profile.resources?.failOnError !== true) {
|
|
1440
|
+
throw new ConfigError(
|
|
1441
|
+
`improve(): surface '${surface}' requires profile.resources.failOnError: true`
|
|
1442
|
+
);
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
// src/improvement/code-execution.ts
|
|
1447
|
+
var distilledNotesMaxChars = 1500;
|
|
1448
|
+
var distilledErrorMaxChars = 500;
|
|
1449
|
+
function generationFailureDistiller(staticFindings) {
|
|
1450
|
+
const CAP = 12;
|
|
1451
|
+
return async (input) => {
|
|
1452
|
+
const failures = [];
|
|
1453
|
+
for (const candidate of input.candidates) {
|
|
1454
|
+
for (const rawCell of candidate.campaign.cells) {
|
|
1455
|
+
const cell = rawCell;
|
|
1456
|
+
const scenario = String(cell.scenarioId ?? "unknown");
|
|
1457
|
+
const error = typeof cell.error === "string" ? cell.error : void 0;
|
|
1458
|
+
const judgeScores = cell.judgeScores && typeof cell.judgeScores === "object" ? Object.values(
|
|
1459
|
+
cell.judgeScores
|
|
1460
|
+
) : [];
|
|
1461
|
+
const composite = judgeScores.length === 0 ? 0 : judgeScores.reduce((sum, judge) => sum + (judge.composite ?? 0), 0) / judgeScores.length;
|
|
1462
|
+
if (!error && composite >= 0.999) continue;
|
|
1463
|
+
const notes = judgeScores.map((judge) => judge.notes).filter((note) => typeof note === "string" && note.length > 0).join("; ").slice(0, distilledNotesMaxChars);
|
|
1464
|
+
const claim = notes || (error ? `Scenario ${scenario} failed: ${error.slice(0, distilledErrorMaxChars)}` : "");
|
|
1465
|
+
failures.push({
|
|
1466
|
+
scenario,
|
|
1467
|
+
composite: Number(composite.toFixed(3)),
|
|
1468
|
+
notes,
|
|
1469
|
+
...claim ? { claim } : {},
|
|
1470
|
+
...error ? { error: error.slice(0, distilledErrorMaxChars) } : {}
|
|
1471
|
+
});
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
if (failures.length === 0) return staticFindings;
|
|
1475
|
+
failures.sort((left, right) => left.composite - right.composite);
|
|
1476
|
+
return failures.slice(0, CAP).map(
|
|
1477
|
+
(failure) => makeFinding3({
|
|
1478
|
+
analyst_id: "generation-failure-distiller",
|
|
1479
|
+
severity: failure.error !== void 0 || failure.composite < 0.5 ? "high" : "medium",
|
|
1480
|
+
area: "generation-failure",
|
|
1481
|
+
confidence: 1,
|
|
1482
|
+
subject: failure.scenario,
|
|
1483
|
+
claim: `Scenario ${failure.scenario} scored composite ${failure.composite}${failure.notes ? `: ${failure.notes}` : ""}${failure.error ? ` (error: ${failure.error})` : ""}`,
|
|
1484
|
+
evidence_refs: [],
|
|
1485
|
+
metadata: {
|
|
1486
|
+
scenario: failure.scenario,
|
|
1487
|
+
composite: failure.composite,
|
|
1488
|
+
...failure.notes ? { notes: failure.notes } : {},
|
|
1489
|
+
...failure.error !== void 0 ? { error: failure.error } : {}
|
|
1490
|
+
}
|
|
1491
|
+
})
|
|
1492
|
+
);
|
|
1493
|
+
};
|
|
1494
|
+
}
|
|
1495
|
+
function defaultDistillerFor(opts, findings) {
|
|
1496
|
+
const durableRun = opts.runDir !== void 0 && !opts.runDir.startsWith("mem://");
|
|
1497
|
+
const useRawTraces = opts.rawTraceContext ?? durableRun;
|
|
1498
|
+
if (useRawTraces) return rawTraceDistiller({ fallbackFindings: findings });
|
|
1499
|
+
return generationFailureDistiller(findings);
|
|
1500
|
+
}
|
|
1501
|
+
async function discardPreparedBaseline(worktree, baselineWorktree, cause) {
|
|
1502
|
+
return rethrowAfterCleanup(
|
|
1503
|
+
cause,
|
|
1504
|
+
() => worktree.discard(baselineWorktree),
|
|
1505
|
+
"improve(): code preparation failed"
|
|
1506
|
+
);
|
|
1507
|
+
}
|
|
1508
|
+
async function prepareCodeRun(code) {
|
|
1509
|
+
const baseRef = code.baseRef ?? "main";
|
|
1510
|
+
const worktree = code.worktree ?? gitWorktreeAdapter({
|
|
1511
|
+
repoRoot: code.repoRoot,
|
|
1512
|
+
...code.worktreeDir ? { worktreeDir: code.worktreeDir } : {}
|
|
1513
|
+
});
|
|
1514
|
+
const baselineWorktree = await worktree.create({ baseRef, label: "incumbent-baseline" });
|
|
1515
|
+
try {
|
|
1516
|
+
const baseline = await worktree.finalize(baselineWorktree, "Incumbent code checkout");
|
|
1517
|
+
let baselineDiscarded = false;
|
|
1518
|
+
const generator = code.generator ?? agenticGenerator({
|
|
1519
|
+
...code.harness ? { harness: code.harness } : {},
|
|
1520
|
+
...code.verify ? { verify: code.verify } : {},
|
|
1521
|
+
...code.timeoutMs ? { timeoutMs: code.timeoutMs } : {}
|
|
1522
|
+
});
|
|
1523
|
+
const managed = improvementDriver({ worktree, generator, baseRef });
|
|
1524
|
+
return {
|
|
1525
|
+
baseline,
|
|
1526
|
+
proposer: managed,
|
|
1527
|
+
async cleanup(retainedWinner) {
|
|
1528
|
+
const errors = [];
|
|
1529
|
+
const retainedWorktreeRef = isCodeSurface(retainedWinner) ? retainedWinner.worktreeRef : void 0;
|
|
1530
|
+
try {
|
|
1531
|
+
await managed?.cleanup(retainedWorktreeRef ? [retainedWorktreeRef] : []);
|
|
1532
|
+
} catch (cause) {
|
|
1533
|
+
errors.push(cause);
|
|
1534
|
+
}
|
|
1535
|
+
if (!baselineDiscarded && retainedWorktreeRef !== baseline.worktreeRef) {
|
|
1536
|
+
try {
|
|
1537
|
+
await worktree.discard(baselineWorktree);
|
|
1538
|
+
baselineDiscarded = true;
|
|
1539
|
+
} catch (cause) {
|
|
1540
|
+
errors.push(cause);
|
|
1541
|
+
}
|
|
1542
|
+
}
|
|
1543
|
+
if (errors.length > 0) {
|
|
1544
|
+
throw new AggregateError(errors, "improve(): failed to clean code improvement worktrees");
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
1547
|
+
};
|
|
1548
|
+
} catch (cause) {
|
|
1549
|
+
return discardPreparedBaseline(worktree, baselineWorktree, cause);
|
|
1504
1550
|
}
|
|
1505
|
-
return { ...instructions, content };
|
|
1506
1551
|
}
|
|
1507
|
-
function
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
);
|
|
1512
|
-
|
|
1552
|
+
function idempotentDispose(dispose) {
|
|
1553
|
+
let disposed = false;
|
|
1554
|
+
let inFlight;
|
|
1555
|
+
return async () => {
|
|
1556
|
+
if (disposed) return;
|
|
1557
|
+
if (inFlight) return inFlight;
|
|
1558
|
+
inFlight = (async () => {
|
|
1559
|
+
await dispose();
|
|
1560
|
+
disposed = true;
|
|
1561
|
+
})();
|
|
1562
|
+
try {
|
|
1563
|
+
await inFlight;
|
|
1564
|
+
} finally {
|
|
1565
|
+
inFlight = void 0;
|
|
1566
|
+
}
|
|
1567
|
+
};
|
|
1513
1568
|
}
|
|
1514
|
-
async function
|
|
1569
|
+
async function runCodeImprovement(opts) {
|
|
1515
1570
|
const {
|
|
1516
|
-
surface = "prompt",
|
|
1517
1571
|
gate = "holdout",
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
rawTraceContext,
|
|
1572
|
+
findings: inputFindings = [],
|
|
1573
|
+
rawTraceContext: _rawTraceContext,
|
|
1521
1574
|
code,
|
|
1522
|
-
skills,
|
|
1523
1575
|
promotionGate,
|
|
1524
1576
|
analyzeGeneration,
|
|
1577
|
+
surface: _surface,
|
|
1525
1578
|
...sharedOptions
|
|
1526
1579
|
} = opts;
|
|
1527
|
-
const
|
|
1528
|
-
|
|
1529
|
-
throw new ConfigError(
|
|
1530
|
-
`improve(): input is not a valid AgentProfile: ${parsedProfile.error.message}`
|
|
1531
|
-
);
|
|
1532
|
-
}
|
|
1533
|
-
if (surface === "code" && generator) {
|
|
1534
|
-
throw new ConfigError(
|
|
1535
|
-
"improve(): surface 'code' forbids opts.generator because an external SurfaceProposer cannot transfer checkout ownership; pass opts.code.generator instead"
|
|
1536
|
-
);
|
|
1537
|
-
}
|
|
1538
|
-
const usesReflectionModel = !generator && (surface === "prompt" || surface === "skills");
|
|
1539
|
-
if (usesReflectionModel) {
|
|
1540
|
-
assertModelAllowed(sharedOptions.llm?.model ?? defaultReflectionModel, allowedModels);
|
|
1541
|
-
}
|
|
1542
|
-
let preparedCode;
|
|
1543
|
-
if (surface === "code") {
|
|
1544
|
-
if (!code) {
|
|
1545
|
-
throw new ConfigError(
|
|
1546
|
-
"improve(): surface 'code' requires opts.code.repoRoot so the incumbent can run from an isolated checkout"
|
|
1547
|
-
);
|
|
1548
|
-
}
|
|
1549
|
-
preparedCode = await prepareCodeRun(code);
|
|
1550
|
-
}
|
|
1551
|
-
const proposer = preparedCode?.proposer ?? generator ?? defaultGeneratorFor(surface, sharedOptions.llm);
|
|
1552
|
-
if (!proposer) {
|
|
1553
|
-
throw new ConfigError(
|
|
1554
|
-
`improve(): surface '${surface}' has no default generator \u2014 pass opts.generator (a SurfaceProposer) explicitly`
|
|
1555
|
-
);
|
|
1556
|
-
}
|
|
1580
|
+
const findings = [...inputFindings];
|
|
1581
|
+
const preparedCode = await prepareCodeRun(code);
|
|
1557
1582
|
const budget = gate === "none" ? { ...sharedOptions.budget, generations: 0 } : { ...sharedOptions.budget };
|
|
1558
1583
|
let raw;
|
|
1559
1584
|
try {
|
|
1560
1585
|
raw = await selfImprove({
|
|
1561
1586
|
...sharedOptions,
|
|
1562
|
-
baselineSurface: preparedCode
|
|
1563
|
-
proposer,
|
|
1587
|
+
baselineSurface: preparedCode.baseline,
|
|
1588
|
+
proposer: preparedCode.proposer,
|
|
1564
1589
|
budget,
|
|
1565
1590
|
findings,
|
|
1566
1591
|
...promotionGate !== void 0 ? { gate: promotionGate } : {},
|
|
@@ -1569,7 +1594,6 @@ async function improve(profile, findings, opts) {
|
|
|
1569
1594
|
}
|
|
1570
1595
|
});
|
|
1571
1596
|
} catch (cause) {
|
|
1572
|
-
if (!preparedCode) throw cause;
|
|
1573
1597
|
return rethrowAfterCleanup(
|
|
1574
1598
|
cause,
|
|
1575
1599
|
() => preparedCode.cleanup(),
|
|
@@ -1577,50 +1601,423 @@ async function improve(profile, findings, opts) {
|
|
|
1577
1601
|
);
|
|
1578
1602
|
}
|
|
1579
1603
|
const winnerSurface = raw.winner.surface;
|
|
1580
|
-
assertCandidateSurfaceKind(
|
|
1581
|
-
|
|
1604
|
+
assertCandidateSurfaceKind("code", preparedCode.baseline, winnerSurface);
|
|
1605
|
+
try {
|
|
1606
|
+
await preparedCode.cleanup(winnerSurface);
|
|
1607
|
+
} catch (cleanupCause) {
|
|
1582
1608
|
try {
|
|
1583
|
-
await preparedCode.cleanup(
|
|
1584
|
-
} catch (
|
|
1585
|
-
try {
|
|
1586
|
-
await preparedCode.cleanup();
|
|
1587
|
-
} catch (finalCleanupCause) {
|
|
1588
|
-
throw new AggregateError(
|
|
1589
|
-
[cleanupCause, finalCleanupCause],
|
|
1590
|
-
"improve(): code result cleanup failed, including the final all-worktree retry"
|
|
1591
|
-
);
|
|
1592
|
-
}
|
|
1609
|
+
await preparedCode.cleanup();
|
|
1610
|
+
} catch (finalCleanupCause) {
|
|
1593
1611
|
throw new AggregateError(
|
|
1594
|
-
[cleanupCause],
|
|
1595
|
-
"improve(): code result cleanup failed
|
|
1612
|
+
[cleanupCause, finalCleanupCause],
|
|
1613
|
+
"improve(): code result cleanup failed, including the final all-worktree retry"
|
|
1596
1614
|
);
|
|
1597
1615
|
}
|
|
1616
|
+
throw new AggregateError(
|
|
1617
|
+
[cleanupCause],
|
|
1618
|
+
"improve(): code result cleanup failed; the final all-worktree retry succeeded"
|
|
1619
|
+
);
|
|
1598
1620
|
}
|
|
1599
|
-
const dispose = idempotentDispose(async () => preparedCode
|
|
1600
|
-
const candidateProfile = materializeImprovementProfileCandidate(
|
|
1601
|
-
parsedProfile.data,
|
|
1602
|
-
surface,
|
|
1603
|
-
winnerSurface,
|
|
1604
|
-
skills
|
|
1605
|
-
);
|
|
1621
|
+
const dispose = idempotentDispose(async () => preparedCode.cleanup());
|
|
1606
1622
|
const candidate = immutableCandidateValue({
|
|
1607
|
-
surface,
|
|
1608
|
-
value: winnerSurface
|
|
1609
|
-
...candidateProfile ? { profile: candidateProfile } : {}
|
|
1623
|
+
surface: "code",
|
|
1624
|
+
value: winnerSurface
|
|
1610
1625
|
});
|
|
1611
1626
|
return {
|
|
1627
|
+
mode: "code",
|
|
1612
1628
|
candidate,
|
|
1613
1629
|
decision: raw.gateDecision,
|
|
1614
1630
|
...raw.lift !== void 0 ? { lift: raw.lift } : {},
|
|
1631
|
+
cost: copyImproveCost(raw.cost),
|
|
1632
|
+
durationMs: raw.durationMs,
|
|
1633
|
+
lineage: Object.freeze({
|
|
1634
|
+
invocationId: raw.provenance.runId,
|
|
1635
|
+
runId: raw.provenance.runId,
|
|
1636
|
+
developmentSplitDigest: raw.provenance.evidence.search.splitDigest
|
|
1637
|
+
}),
|
|
1638
|
+
generationsExplored: raw.generationsExplored,
|
|
1615
1639
|
raw,
|
|
1616
1640
|
dispose
|
|
1617
1641
|
};
|
|
1618
1642
|
}
|
|
1619
1643
|
|
|
1644
|
+
// src/improvement/method-execution.ts
|
|
1645
|
+
import { randomUUID } from "crypto";
|
|
1646
|
+
import {
|
|
1647
|
+
compareOptimizationMethods
|
|
1648
|
+
} from "@tangle-network/agent-eval/campaign";
|
|
1649
|
+
import {
|
|
1650
|
+
sha256DigestSchema
|
|
1651
|
+
} from "@tangle-network/agent-interface";
|
|
1652
|
+
|
|
1653
|
+
// src/improvement/method-controls.ts
|
|
1654
|
+
var methodRuntimeControls = /* @__PURE__ */ Symbol("agent-runtime.improvement.method-runtime-controls");
|
|
1655
|
+
function withMethodRuntimeControls(method, controls) {
|
|
1656
|
+
return Object.freeze({
|
|
1657
|
+
...method,
|
|
1658
|
+
[methodRuntimeControls]: Object.freeze({ ...controls })
|
|
1659
|
+
});
|
|
1660
|
+
}
|
|
1661
|
+
function methodRuntimeControlsOf(method) {
|
|
1662
|
+
return method[methodRuntimeControls];
|
|
1663
|
+
}
|
|
1664
|
+
|
|
1665
|
+
// src/improvement/method-cost.ts
|
|
1666
|
+
var EVALUATION_TAG = "runtimeEvaluationRef";
|
|
1667
|
+
var INVOCATION_TAG = "runtimeInvocationId";
|
|
1668
|
+
function methodInputWithScopedCost(input, scope, costAttribution = "invocation") {
|
|
1669
|
+
return Object.freeze({
|
|
1670
|
+
...input,
|
|
1671
|
+
costLedger: scopedCostLedger(input.costLedger, scope, costAttribution)
|
|
1672
|
+
});
|
|
1673
|
+
}
|
|
1674
|
+
function methodInvocationCostLedger(ledger, scope) {
|
|
1675
|
+
return scopedCostLedger(ledger, scope, "invocation");
|
|
1676
|
+
}
|
|
1677
|
+
function assertMethodCostRecorded(methodName, result, compatibleLedger, invocationLedger, costCeiling, costAttribution = "invocation") {
|
|
1678
|
+
const { cost } = result;
|
|
1679
|
+
const observed = costAttribution === "optimizer-run" && result.provenance?.runId ? compatibleLedger.summary({ tags: { optimizerRun: result.provenance.runId } }) : invocationLedger.summary();
|
|
1680
|
+
if (costCeiling !== void 0 && !cost.accountingComplete) {
|
|
1681
|
+
throw new ConfigError(
|
|
1682
|
+
`improve(): method '${methodName}' returned incomplete cost accounting under costCeiling; refusing final scoring`
|
|
1683
|
+
);
|
|
1684
|
+
}
|
|
1685
|
+
if (costCeiling !== void 0 && exceeds(cost.totalCostUsd, costCeiling)) {
|
|
1686
|
+
throw new ConfigError(
|
|
1687
|
+
`improve(): method '${methodName}' reported cost $${cost.totalCostUsd} above costCeiling $${costCeiling}; refusing final scoring`
|
|
1688
|
+
);
|
|
1689
|
+
}
|
|
1690
|
+
if (cost.accountingComplete && !observed.accountingComplete) {
|
|
1691
|
+
throw new ConfigError(
|
|
1692
|
+
`improve(): method '${methodName}' reported complete cost accounting but its shared cost receipts are incomplete`
|
|
1693
|
+
);
|
|
1694
|
+
}
|
|
1695
|
+
if (cost.accountingComplete && !approximatelyEqual(cost.totalCostUsd, observed.totalCostUsd)) {
|
|
1696
|
+
throw new ConfigError(
|
|
1697
|
+
`improve(): method '${methodName}' reported $${cost.totalCostUsd} but recorded $${observed.totalCostUsd} through input.costLedger`
|
|
1698
|
+
);
|
|
1699
|
+
}
|
|
1700
|
+
}
|
|
1701
|
+
function scopedCostLedger(parent, scope, costAttribution) {
|
|
1702
|
+
const evaluationTags = {
|
|
1703
|
+
[EVALUATION_TAG]: scope.evaluationRef
|
|
1704
|
+
};
|
|
1705
|
+
const invocationTags = {
|
|
1706
|
+
...evaluationTags,
|
|
1707
|
+
[INVOCATION_TAG]: scope.invocationId
|
|
1708
|
+
};
|
|
1709
|
+
const readTags = (filter) => costAttribution === "optimizer-run" && hasOptimizerRunFilter(filter) ? evaluationTags : invocationTags;
|
|
1710
|
+
return {
|
|
1711
|
+
costCeilingUsd: parent.costCeilingUsd,
|
|
1712
|
+
runPaidCall: (input) => parent.runPaidCall({
|
|
1713
|
+
...input,
|
|
1714
|
+
tags: { ...input.tags ?? {}, ...invocationTags }
|
|
1715
|
+
}),
|
|
1716
|
+
reconcile: (...args) => parent.reconcile(...args),
|
|
1717
|
+
list: (filter) => parent.list(withTags(filter, readTags(filter))),
|
|
1718
|
+
listPending: (filter) => parent.listPending?.(withTags(filter, readTags(filter))) ?? [],
|
|
1719
|
+
summary: (filter) => parent.summary(withTags(filter, readTags(filter))),
|
|
1720
|
+
waitForIdle: (options = {}) => parent.waitForIdle?.({
|
|
1721
|
+
...options,
|
|
1722
|
+
filter: withTags(options.filter, readTags(options.filter))
|
|
1723
|
+
}) ?? Promise.resolve(
|
|
1724
|
+
parent.summary(withTags(options.filter, readTags(options.filter))).pendingCalls === 0
|
|
1725
|
+
),
|
|
1726
|
+
markCompleted: (count) => parent.markCompleted(count),
|
|
1727
|
+
costPerCompletedTask: () => parent.costPerCompletedTask()
|
|
1728
|
+
};
|
|
1729
|
+
}
|
|
1730
|
+
function hasOptimizerRunFilter(filter) {
|
|
1731
|
+
return typeof filter?.tags?.optimizerRun === "string" && filter.tags.optimizerRun.length > 0;
|
|
1732
|
+
}
|
|
1733
|
+
function withTags(filter, tags) {
|
|
1734
|
+
return {
|
|
1735
|
+
...filter ?? {},
|
|
1736
|
+
tags: { ...filter?.tags ?? {}, ...tags }
|
|
1737
|
+
};
|
|
1738
|
+
}
|
|
1739
|
+
function approximatelyEqual(left, right) {
|
|
1740
|
+
const tolerance = Number.EPSILON * Math.max(1, Math.abs(left), Math.abs(right)) * 8;
|
|
1741
|
+
return Math.abs(left - right) <= tolerance;
|
|
1742
|
+
}
|
|
1743
|
+
function exceeds(value, limit) {
|
|
1744
|
+
const tolerance = Number.EPSILON * Math.max(1, Math.abs(value), Math.abs(limit)) * 8;
|
|
1745
|
+
return value - limit > tolerance;
|
|
1746
|
+
}
|
|
1747
|
+
|
|
1748
|
+
// src/improvement/method-identity.ts
|
|
1749
|
+
import {
|
|
1750
|
+
campaignSplitDigest
|
|
1751
|
+
} from "@tangle-network/agent-eval/campaign";
|
|
1752
|
+
function buildMethodEvaluationIdentity(input) {
|
|
1753
|
+
const optimizationReps = input.optimizationRunOptions?.reps ?? 1;
|
|
1754
|
+
const developmentSplitDigest = canonicalCandidateDigest({
|
|
1755
|
+
train: campaignSplitDigest(input.trainScenarios, optimizationReps),
|
|
1756
|
+
selection: campaignSplitDigest(input.selectionScenarios, optimizationReps)
|
|
1757
|
+
});
|
|
1758
|
+
const judgeDescriptors = input.judges.map(judgeDescriptor);
|
|
1759
|
+
const evaluationRef = canonicalCandidateDigest({
|
|
1760
|
+
executionRef: input.executionRef,
|
|
1761
|
+
baselineProfileDigest: input.baselineProfileDigest,
|
|
1762
|
+
coordinate: profileCoordinate(input.surface, input.baselineSurface, input.skills),
|
|
1763
|
+
candidateValidation: input.validateCandidate ? Function.prototype.toString.call(input.validateCandidate) : null,
|
|
1764
|
+
developmentSplitDigest,
|
|
1765
|
+
findings: input.findings,
|
|
1766
|
+
judges: judgeDescriptors,
|
|
1767
|
+
run: {
|
|
1768
|
+
seed: input.seed ?? 42,
|
|
1769
|
+
finalReps: input.reps ?? 1,
|
|
1770
|
+
optimizationReps,
|
|
1771
|
+
costCeiling: input.costCeiling ?? null,
|
|
1772
|
+
resumable: input.optimizationRunOptions?.resumable ?? true,
|
|
1773
|
+
maxConcurrency: input.optimizationRunOptions?.maxConcurrency ?? 2,
|
|
1774
|
+
abortOnCellError: input.optimizationRunOptions?.abortOnCellError ?? false,
|
|
1775
|
+
dispatchTimeoutMs: input.optimizationRunOptions?.dispatchTimeoutMs ?? null,
|
|
1776
|
+
dispatchShutdownTimeoutMs: input.optimizationRunOptions?.dispatchShutdownTimeoutMs ?? 5e3,
|
|
1777
|
+
tracing: input.optimizationRunOptions?.tracing ?? "on",
|
|
1778
|
+
expectUsage: input.optimizationRunOptions?.expectUsage ?? "warn",
|
|
1779
|
+
captureSource: input.optimizationRunOptions?.captureSource ?? null,
|
|
1780
|
+
captureSourceVersionHash: input.optimizationRunOptions?.captureSourceVersionHash ?? null
|
|
1781
|
+
}
|
|
1782
|
+
});
|
|
1783
|
+
return {
|
|
1784
|
+
evaluationRef,
|
|
1785
|
+
developmentSplitDigest,
|
|
1786
|
+
judgeDescriptors
|
|
1787
|
+
};
|
|
1788
|
+
}
|
|
1789
|
+
function profileCoordinate(surface, baselineSurface, skills) {
|
|
1790
|
+
return {
|
|
1791
|
+
surface,
|
|
1792
|
+
resourceName: surface === "skills" ? skills?.resourceName.trim() ?? null : null,
|
|
1793
|
+
componentNames: typeof baselineSurface === "object" && baselineSurface !== null && baselineSurface.kind === "components" ? Object.keys(baselineSurface.components).sort() : []
|
|
1794
|
+
};
|
|
1795
|
+
}
|
|
1796
|
+
function judgeDescriptor(judge) {
|
|
1797
|
+
return {
|
|
1798
|
+
name: judge.name,
|
|
1799
|
+
dimensions: judge.dimensions.map((dimension) => ({ ...dimension })),
|
|
1800
|
+
declaredVersion: judge.judgeVersion ?? null,
|
|
1801
|
+
scoreImplementation: Function.prototype.toString.call(judge.score),
|
|
1802
|
+
appliesToImplementation: judge.appliesTo ? Function.prototype.toString.call(judge.appliesTo) : null
|
|
1803
|
+
};
|
|
1804
|
+
}
|
|
1805
|
+
|
|
1806
|
+
// src/improvement/method-execution.ts
|
|
1807
|
+
function resolveOptimizationMethod(source, context) {
|
|
1808
|
+
const method = typeof source === "function" ? source(context) : source;
|
|
1809
|
+
if (!method || typeof method !== "object" || typeof method.name !== "string" || method.name.trim() !== method.name || method.name.length === 0 || typeof method.optimize !== "function") {
|
|
1810
|
+
throw new ConfigError(
|
|
1811
|
+
"improve(): method must be a complete OptimizationMethod with a trimmed name and optimize(input)"
|
|
1812
|
+
);
|
|
1813
|
+
}
|
|
1814
|
+
return method;
|
|
1815
|
+
}
|
|
1816
|
+
function copyProvenance(provenance) {
|
|
1817
|
+
return immutableCandidateValue(provenance);
|
|
1818
|
+
}
|
|
1819
|
+
function validateExecutionRef(value) {
|
|
1820
|
+
const parsed = sha256DigestSchema.safeParse(value);
|
|
1821
|
+
if (!parsed.success) {
|
|
1822
|
+
throw new ConfigError("improve(): executionRef must be a lowercase sha256:<64 hex> digest");
|
|
1823
|
+
}
|
|
1824
|
+
return parsed.data;
|
|
1825
|
+
}
|
|
1826
|
+
async function runMethodImprovement(profile, opts) {
|
|
1827
|
+
const {
|
|
1828
|
+
surface = "prompt",
|
|
1829
|
+
executionRef: inputExecutionRef,
|
|
1830
|
+
method: methodSource,
|
|
1831
|
+
agent,
|
|
1832
|
+
validateCandidate,
|
|
1833
|
+
findings: inputFindings = [],
|
|
1834
|
+
skills,
|
|
1835
|
+
profileComponents,
|
|
1836
|
+
optimizationRunOptions,
|
|
1837
|
+
minimumLift = 0,
|
|
1838
|
+
...comparisonOptions
|
|
1839
|
+
} = opts;
|
|
1840
|
+
if (!Number.isFinite(minimumLift) || minimumLift < 0) {
|
|
1841
|
+
throw new ConfigError(
|
|
1842
|
+
"improve(): minimumLift must be a finite number greater than or equal to 0"
|
|
1843
|
+
);
|
|
1844
|
+
}
|
|
1845
|
+
if (profileComponents && surface !== "agent-profile") {
|
|
1846
|
+
throw new ConfigError("improve(): profileComponents is valid only with surface 'agent-profile'");
|
|
1847
|
+
}
|
|
1848
|
+
const executionRef = validateExecutionRef(inputExecutionRef);
|
|
1849
|
+
const findings = [...inputFindings];
|
|
1850
|
+
const preparedSurface = prepareProfileSurface(profile, surface, skills, profileComponents);
|
|
1851
|
+
const baselineSurface = preparedSurface.surface;
|
|
1852
|
+
const baselineValue = immutableCandidateValue(preparedSurface.value);
|
|
1853
|
+
const baselineProfileDigest = canonicalCandidateDigest(profile);
|
|
1854
|
+
const identity = buildMethodEvaluationIdentity({
|
|
1855
|
+
executionRef,
|
|
1856
|
+
baselineProfileDigest,
|
|
1857
|
+
baselineSurface,
|
|
1858
|
+
surface,
|
|
1859
|
+
skills,
|
|
1860
|
+
validateCandidate,
|
|
1861
|
+
findings,
|
|
1862
|
+
trainScenarios: comparisonOptions.trainScenarios,
|
|
1863
|
+
selectionScenarios: comparisonOptions.selectionScenarios,
|
|
1864
|
+
judges: comparisonOptions.judges,
|
|
1865
|
+
seed: comparisonOptions.seed,
|
|
1866
|
+
reps: comparisonOptions.reps,
|
|
1867
|
+
costCeiling: comparisonOptions.costCeiling,
|
|
1868
|
+
optimizationRunOptions
|
|
1869
|
+
});
|
|
1870
|
+
const { evaluationRef, developmentSplitDigest } = identity;
|
|
1871
|
+
const dispatchRef = `improve:${evaluationRef}`;
|
|
1872
|
+
const identifiedJudges = comparisonOptions.judges.map(
|
|
1873
|
+
(judge, index) => Object.freeze({
|
|
1874
|
+
...judge,
|
|
1875
|
+
judgeVersion: canonicalCandidateDigest({
|
|
1876
|
+
evaluationRef,
|
|
1877
|
+
descriptor: identity.judgeDescriptors[index]
|
|
1878
|
+
})
|
|
1879
|
+
})
|
|
1880
|
+
);
|
|
1881
|
+
const rawMaterializeProfile = createProfileCandidateMaterializer(
|
|
1882
|
+
profile,
|
|
1883
|
+
surface,
|
|
1884
|
+
baselineSurface,
|
|
1885
|
+
skills,
|
|
1886
|
+
profileComponents
|
|
1887
|
+
);
|
|
1888
|
+
const runtimeInvocationId = `runtime-optimization:${randomUUID()}`;
|
|
1889
|
+
const method = resolveOptimizationMethod(methodSource, {
|
|
1890
|
+
profile,
|
|
1891
|
+
evaluationRef,
|
|
1892
|
+
surface,
|
|
1893
|
+
baselineSurface,
|
|
1894
|
+
baselineValue,
|
|
1895
|
+
findings
|
|
1896
|
+
});
|
|
1897
|
+
const methodControls = methodRuntimeControlsOf(method);
|
|
1898
|
+
const baselineSurfaceDigest = canonicalCandidateDigest(baselineSurface);
|
|
1899
|
+
const validatedCandidates = /* @__PURE__ */ new Set();
|
|
1900
|
+
const materializeProfile2 = (candidateSurface) => {
|
|
1901
|
+
const candidate2 = rawMaterializeProfile(candidateSurface);
|
|
1902
|
+
const candidateDigest = canonicalCandidateDigest(candidateSurface);
|
|
1903
|
+
if (!validatedCandidates.has(candidateDigest)) {
|
|
1904
|
+
const prepared = prepareProfileSurface(candidate2, surface, skills, profileComponents);
|
|
1905
|
+
const validationInput = Object.freeze({
|
|
1906
|
+
profile: candidate2,
|
|
1907
|
+
surface,
|
|
1908
|
+
candidateSurface: immutableCandidateValue(candidateSurface),
|
|
1909
|
+
value: immutableCandidateValue(prepared.value),
|
|
1910
|
+
isBaseline: candidateDigest === baselineSurfaceDigest
|
|
1911
|
+
});
|
|
1912
|
+
methodControls?.validateCandidate(validationInput);
|
|
1913
|
+
validateCandidate?.(validationInput);
|
|
1914
|
+
validatedCandidates.add(candidateDigest);
|
|
1915
|
+
}
|
|
1916
|
+
return candidate2;
|
|
1917
|
+
};
|
|
1918
|
+
materializeProfile2(baselineSurface);
|
|
1919
|
+
const measuredMethod = {
|
|
1920
|
+
...method,
|
|
1921
|
+
async optimize(input) {
|
|
1922
|
+
const costScope = {
|
|
1923
|
+
evaluationRef,
|
|
1924
|
+
invocationId: runtimeInvocationId
|
|
1925
|
+
};
|
|
1926
|
+
const scopedInput = methodInputWithScopedCost(
|
|
1927
|
+
input,
|
|
1928
|
+
costScope,
|
|
1929
|
+
methodControls?.costAttribution
|
|
1930
|
+
);
|
|
1931
|
+
const invocationLedger = methodInvocationCostLedger(input.costLedger, costScope);
|
|
1932
|
+
const result = await method.optimize(scopedInput);
|
|
1933
|
+
assertMethodCostRecorded(
|
|
1934
|
+
method.name,
|
|
1935
|
+
result,
|
|
1936
|
+
scopedInput.costLedger,
|
|
1937
|
+
invocationLedger,
|
|
1938
|
+
comparisonOptions.costCeiling,
|
|
1939
|
+
methodControls?.costAttribution
|
|
1940
|
+
);
|
|
1941
|
+
materializeProfile2(result.winnerSurface);
|
|
1942
|
+
return result;
|
|
1943
|
+
}
|
|
1944
|
+
};
|
|
1945
|
+
const startedAt = Date.now();
|
|
1946
|
+
const raw = await compareOptimizationMethods({
|
|
1947
|
+
...comparisonOptions,
|
|
1948
|
+
judges: identifiedJudges,
|
|
1949
|
+
dispatchRef,
|
|
1950
|
+
optimizationRunOptions: {
|
|
1951
|
+
...optimizationRunOptions ?? {},
|
|
1952
|
+
dispatchRef
|
|
1953
|
+
},
|
|
1954
|
+
methods: [measuredMethod],
|
|
1955
|
+
baselineSurface,
|
|
1956
|
+
dispatchWithSurface: (candidateSurface, scenario, ctx) => agent(materializeProfile2(candidateSurface), scenario, ctx)
|
|
1957
|
+
});
|
|
1958
|
+
if (comparisonOptions.costCeiling !== void 0 && raw.totalCost.totalCostUsd > comparisonOptions.costCeiling) {
|
|
1959
|
+
throw new ConfigError(
|
|
1960
|
+
`improve(): reported total cost $${raw.totalCost.totalCostUsd} exceeds costCeiling $${comparisonOptions.costCeiling}`
|
|
1961
|
+
);
|
|
1962
|
+
}
|
|
1963
|
+
const score = raw.best;
|
|
1964
|
+
const winnerSurface = immutableCandidateValue(score.winnerSurface);
|
|
1965
|
+
assertCandidateSurfaceKind(surface, baselineSurface, winnerSurface);
|
|
1966
|
+
const candidateProfile = materializeProfile2(winnerSurface);
|
|
1967
|
+
const candidate = Object.freeze({
|
|
1968
|
+
surface,
|
|
1969
|
+
value: winnerSurface,
|
|
1970
|
+
profile: candidateProfile
|
|
1971
|
+
});
|
|
1972
|
+
const cost = copyImproveCost(raw.totalCost);
|
|
1973
|
+
return {
|
|
1974
|
+
mode: "method",
|
|
1975
|
+
method: method.name,
|
|
1976
|
+
...score.provenance ? { provenance: copyProvenance(score.provenance) } : {},
|
|
1977
|
+
candidate,
|
|
1978
|
+
decision: cost.accountingComplete && score.liftCi.low > minimumLift ? "ship" : "hold",
|
|
1979
|
+
lift: score.lift,
|
|
1980
|
+
liftInterval: { ...score.liftCi },
|
|
1981
|
+
cost,
|
|
1982
|
+
durationMs: Date.now() - startedAt,
|
|
1983
|
+
lineage: Object.freeze({
|
|
1984
|
+
invocationId: runtimeInvocationId,
|
|
1985
|
+
runId: score.provenance?.runId ?? runtimeInvocationId,
|
|
1986
|
+
developmentSplitDigest,
|
|
1987
|
+
executionRef,
|
|
1988
|
+
baselineProfileDigest
|
|
1989
|
+
}),
|
|
1990
|
+
raw,
|
|
1991
|
+
async dispose() {
|
|
1992
|
+
}
|
|
1993
|
+
};
|
|
1994
|
+
}
|
|
1995
|
+
|
|
1996
|
+
// src/improvement/improve.ts
|
|
1997
|
+
async function improve(profileOrCode, opts) {
|
|
1998
|
+
if (opts === void 0) {
|
|
1999
|
+
const code = profileOrCode;
|
|
2000
|
+
if (!code || code.surface !== "code") {
|
|
2001
|
+
throw new ConfigError("improve(): the one-argument form requires { surface: 'code', ... }");
|
|
2002
|
+
}
|
|
2003
|
+
return runCodeImprovement(code);
|
|
2004
|
+
}
|
|
2005
|
+
if (opts.surface === "code") {
|
|
2006
|
+
throw new ConfigError("improve(): code takes one argument: improve({ surface: 'code', ... })");
|
|
2007
|
+
}
|
|
2008
|
+
const parsedProfile = agentProfileSchema2.safeParse(profileOrCode);
|
|
2009
|
+
if (!parsedProfile.success) {
|
|
2010
|
+
throw new ConfigError(
|
|
2011
|
+
`improve(): input is not a valid AgentProfile: ${parsedProfile.error.message}`
|
|
2012
|
+
);
|
|
2013
|
+
}
|
|
2014
|
+
return runMethodImprovement(immutableCandidateValue(parsedProfile.data), opts);
|
|
2015
|
+
}
|
|
2016
|
+
|
|
1620
2017
|
// src/intelligence/improvement-surfaces.ts
|
|
1621
2018
|
import {
|
|
1622
2019
|
agentProfileDiffSchema,
|
|
1623
|
-
agentProfileSchema as
|
|
2020
|
+
agentProfileSchema as agentProfileSchema3,
|
|
1624
2021
|
defineAgentProfileDiff
|
|
1625
2022
|
} from "@tangle-network/agent-interface";
|
|
1626
2023
|
var changedSurfaceOrder = [
|
|
@@ -1823,7 +2220,7 @@ function exactObject(value, keys, label) {
|
|
|
1823
2220
|
return record;
|
|
1824
2221
|
}
|
|
1825
2222
|
function parseProfileSet(value) {
|
|
1826
|
-
return
|
|
2223
|
+
return agentProfileSchema3.parse(value);
|
|
1827
2224
|
}
|
|
1828
2225
|
function assertKnowledgeCandidatePair(baselineBundle, candidateBundle) {
|
|
1829
2226
|
const baseline = baselineBundle.knowledge;
|
|
@@ -1893,6 +2290,160 @@ function opaqueProfileSlice(profile) {
|
|
|
1893
2290
|
};
|
|
1894
2291
|
}
|
|
1895
2292
|
|
|
2293
|
+
// src/intelligence/optimization-receipt.ts
|
|
2294
|
+
import {
|
|
2295
|
+
agentProfileModelHintsSchema
|
|
2296
|
+
} from "@tangle-network/agent-interface";
|
|
2297
|
+
var optimizationReceiptMetadataKey = "optimizationReceipt";
|
|
2298
|
+
function createOptimizationActivationReceipt(improvement) {
|
|
2299
|
+
const provenance = improvement.provenance;
|
|
2300
|
+
if (!provenance) return void 0;
|
|
2301
|
+
const optimizerModel = provenance.optimizerModel;
|
|
2302
|
+
const candidateModel = improvement.candidate.profile.model;
|
|
2303
|
+
return canonicalCandidateDocument({
|
|
2304
|
+
kind: "optimization-activation-receipt",
|
|
2305
|
+
method: improvement.method,
|
|
2306
|
+
source: provenance.source,
|
|
2307
|
+
...provenance.bridge ? { bridge: provenance.bridge } : {},
|
|
2308
|
+
...provenance.modules ? { modules: provenance.modules } : {},
|
|
2309
|
+
...provenance.python ? { python: provenance.python } : {},
|
|
2310
|
+
...candidateModel || optimizerModel ? {
|
|
2311
|
+
models: {
|
|
2312
|
+
...candidateModel ? { candidate: candidateModel } : {},
|
|
2313
|
+
...optimizerModel ? { optimizer: optimizerModel } : {}
|
|
2314
|
+
}
|
|
2315
|
+
} : {},
|
|
2316
|
+
usage: {
|
|
2317
|
+
optimizerEvaluations: provenance.evaluationCount,
|
|
2318
|
+
...provenance.tokenUsage ? { optimizerTokens: provenance.tokenUsage } : {}
|
|
2319
|
+
},
|
|
2320
|
+
cost: {
|
|
2321
|
+
optimization: receiptCost(improvement.raw.optimizationCost),
|
|
2322
|
+
finalTest: receiptCost(improvement.raw.testCost),
|
|
2323
|
+
total: receiptCost(improvement.raw.totalCost)
|
|
2324
|
+
},
|
|
2325
|
+
invocation: {
|
|
2326
|
+
runtimeInvocationId: improvement.lineage.invocationId,
|
|
2327
|
+
optimizerRunId: provenance.runId,
|
|
2328
|
+
...provenance.compatibleRunId ? { compatibleOptimizerRunId: provenance.compatibleRunId } : {},
|
|
2329
|
+
resumed: provenance.resumed,
|
|
2330
|
+
artifactDir: provenance.artifactDir
|
|
2331
|
+
},
|
|
2332
|
+
developmentDataDigest: improvement.lineage.developmentSplitDigest
|
|
2333
|
+
}).value;
|
|
2334
|
+
}
|
|
2335
|
+
function attachOptimizationActivationReceipt(metadata, receipt) {
|
|
2336
|
+
assertNoCallerOptimizationReceipt(metadata);
|
|
2337
|
+
return immutableCandidateValue({
|
|
2338
|
+
...metadata ?? {},
|
|
2339
|
+
[optimizationReceiptMetadataKey]: receipt
|
|
2340
|
+
});
|
|
2341
|
+
}
|
|
2342
|
+
function assertNoCallerOptimizationReceipt(metadata) {
|
|
2343
|
+
if (metadata && Object.hasOwn(metadata, optimizationReceiptMetadataKey)) {
|
|
2344
|
+
throw new Error(`candidate metadata reserves '${optimizationReceiptMetadataKey}' for Runtime`);
|
|
2345
|
+
}
|
|
2346
|
+
}
|
|
2347
|
+
function optimizationActivationReceiptFromMetadata(metadata) {
|
|
2348
|
+
const value = metadata?.[optimizationReceiptMetadataKey];
|
|
2349
|
+
if (value === void 0) return void 0;
|
|
2350
|
+
return immutableCandidateValue(parseOptimizationActivationReceipt(value));
|
|
2351
|
+
}
|
|
2352
|
+
function parseOptimizationActivationReceipt(value) {
|
|
2353
|
+
if (!isRecord(value) || value.kind !== "optimization-activation-receipt") {
|
|
2354
|
+
throw new Error("optimization receipt must be an optimization-activation-receipt");
|
|
2355
|
+
}
|
|
2356
|
+
if (!isNonEmptyString(value.method) || !isPackageSource(value.source) || value.bridge !== void 0 && !isPackageSource(value.bridge) || !isModules(value.modules) || !isPythonRuntime(value.python) || !isModels(value.models) || !isUsage(value.usage) || !isCost(value.cost) || !isInvocation(value.invocation) || !isSha256Digest(value.developmentDataDigest) || !isSha256Digest(value.digest)) {
|
|
2357
|
+
throw new Error("optimization receipt contains invalid evidence");
|
|
2358
|
+
}
|
|
2359
|
+
const receipt = value;
|
|
2360
|
+
if (canonicalCandidateDigest(omitTopLevelDigest(receipt)) !== receipt.digest) {
|
|
2361
|
+
throw new Error("optimization receipt digest does not match its evidence");
|
|
2362
|
+
}
|
|
2363
|
+
return receipt;
|
|
2364
|
+
}
|
|
2365
|
+
function isPackageSource(value) {
|
|
2366
|
+
if (!isRecord(value) || value.kind !== "package" || value.evidence !== "observed" && value.evidence !== "declared" || !isNonEmptyString(value.package) || !isNonEmptyString(value.version)) {
|
|
2367
|
+
return false;
|
|
2368
|
+
}
|
|
2369
|
+
return isOptionalNonEmptyString(value.sourceUrl) && isOptionalNonEmptyString(value.revision) && (value.sourceSha256 === void 0 || typeof value.sourceSha256 === "string" && /^[0-9a-f]{64}$/.test(value.sourceSha256));
|
|
2370
|
+
}
|
|
2371
|
+
function isModules(value) {
|
|
2372
|
+
return value === void 0 || Array.isArray(value) && value.every(
|
|
2373
|
+
(module) => isRecord(module) && isNonEmptyString(module.module) && typeof module.sourceSha256 === "string" && /^[0-9a-f]{64}$/.test(module.sourceSha256)
|
|
2374
|
+
);
|
|
2375
|
+
}
|
|
2376
|
+
function isPythonRuntime(value) {
|
|
2377
|
+
return value === void 0 || isRecord(value) && isNonEmptyString(value.implementation) && isNonEmptyString(value.version);
|
|
2378
|
+
}
|
|
2379
|
+
function isModels(value) {
|
|
2380
|
+
return value === void 0 || isRecord(value) && (value.candidate === void 0 || agentProfileModelHintsSchema.safeParse(value.candidate).success) && isOptionalNonEmptyString(value.optimizer) && (value.candidate !== void 0 || value.optimizer !== void 0);
|
|
2381
|
+
}
|
|
2382
|
+
function isUsage(value) {
|
|
2383
|
+
if (!isRecord(value) || !isNonNegativeInteger(value.optimizerEvaluations)) return false;
|
|
2384
|
+
if (value.optimizerTokens === void 0) return true;
|
|
2385
|
+
const tokens = value.optimizerTokens;
|
|
2386
|
+
if (!isRecord(tokens)) return false;
|
|
2387
|
+
const inputTokens = tokens.inputTokens;
|
|
2388
|
+
const outputTokens = tokens.outputTokens;
|
|
2389
|
+
const totalTokens = tokens.totalTokens;
|
|
2390
|
+
const calls = tokens.calls;
|
|
2391
|
+
if (!isNonNegativeInteger(inputTokens) || !isNonNegativeInteger(outputTokens) || !isNonNegativeInteger(totalTokens) || !isNonNegativeInteger(calls)) {
|
|
2392
|
+
return false;
|
|
2393
|
+
}
|
|
2394
|
+
if (!isOptionalNonNegativeInteger(tokens.cachedInputTokens) || !isOptionalNonNegativeInteger(tokens.cacheWriteInputTokens) || !isOptionalNonNegativeInteger(tokens.reasoningTokens)) {
|
|
2395
|
+
return false;
|
|
2396
|
+
}
|
|
2397
|
+
const cachedInputTokens = typeof tokens.cachedInputTokens === "number" ? tokens.cachedInputTokens : 0;
|
|
2398
|
+
const cacheWriteInputTokens = typeof tokens.cacheWriteInputTokens === "number" ? tokens.cacheWriteInputTokens : 0;
|
|
2399
|
+
const reasoningTokens = typeof tokens.reasoningTokens === "number" ? tokens.reasoningTokens : 0;
|
|
2400
|
+
return totalTokens === inputTokens + outputTokens && cachedInputTokens + cacheWriteInputTokens <= inputTokens && reasoningTokens <= outputTokens && (totalTokens === 0 || calls > 0);
|
|
2401
|
+
}
|
|
2402
|
+
function isCost(value) {
|
|
2403
|
+
if (!isRecord(value) || !isCostPart(value.optimization) || !isCostPart(value.finalTest) || !isCostPart(value.total)) {
|
|
2404
|
+
return false;
|
|
2405
|
+
}
|
|
2406
|
+
const optimization = value.optimization;
|
|
2407
|
+
const finalTest = value.finalTest;
|
|
2408
|
+
const total = value.total;
|
|
2409
|
+
return approximatelyEqual2(total.totalUsd, optimization.totalUsd + finalTest.totalUsd) && total.accountingComplete === (optimization.accountingComplete && finalTest.accountingComplete);
|
|
2410
|
+
}
|
|
2411
|
+
function isCostPart(value) {
|
|
2412
|
+
return isRecord(value) && typeof value.totalUsd === "number" && Number.isFinite(value.totalUsd) && value.totalUsd >= 0 && typeof value.accountingComplete === "boolean" && Array.isArray(value.incompleteReasons) && value.incompleteReasons.every((reason) => typeof reason === "string");
|
|
2413
|
+
}
|
|
2414
|
+
function receiptCost(value) {
|
|
2415
|
+
return {
|
|
2416
|
+
totalUsd: value.totalCostUsd,
|
|
2417
|
+
accountingComplete: value.accountingComplete,
|
|
2418
|
+
incompleteReasons: [...value.incompleteReasons]
|
|
2419
|
+
};
|
|
2420
|
+
}
|
|
2421
|
+
function approximatelyEqual2(left, right) {
|
|
2422
|
+
const tolerance = Number.EPSILON * Math.max(1, Math.abs(left), Math.abs(right)) * 8;
|
|
2423
|
+
return Math.abs(left - right) <= tolerance;
|
|
2424
|
+
}
|
|
2425
|
+
function isInvocation(value) {
|
|
2426
|
+
return isRecord(value) && isNonEmptyString(value.runtimeInvocationId) && isNonEmptyString(value.optimizerRunId) && isOptionalNonEmptyString(value.compatibleOptimizerRunId) && typeof value.resumed === "boolean" && isNonEmptyString(value.artifactDir);
|
|
2427
|
+
}
|
|
2428
|
+
function isRecord(value) {
|
|
2429
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
2430
|
+
}
|
|
2431
|
+
function isNonEmptyString(value) {
|
|
2432
|
+
return typeof value === "string" && value.trim().length > 0;
|
|
2433
|
+
}
|
|
2434
|
+
function isOptionalNonEmptyString(value) {
|
|
2435
|
+
return value === void 0 || isNonEmptyString(value);
|
|
2436
|
+
}
|
|
2437
|
+
function isNonNegativeInteger(value) {
|
|
2438
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
2439
|
+
}
|
|
2440
|
+
function isOptionalNonNegativeInteger(value) {
|
|
2441
|
+
return value === void 0 || isNonNegativeInteger(value);
|
|
2442
|
+
}
|
|
2443
|
+
function isSha256Digest(value) {
|
|
2444
|
+
return typeof value === "string" && /^sha256:[0-9a-f]{64}$/.test(value);
|
|
2445
|
+
}
|
|
2446
|
+
|
|
1896
2447
|
// src/intelligence/improvement-cycle.ts
|
|
1897
2448
|
import { assertNoJudgeVerdict } from "@tangle-network/agent-eval/analyst";
|
|
1898
2449
|
import {
|
|
@@ -1924,8 +2475,8 @@ function sealAgentImprovementExperiment(material, improvement) {
|
|
|
1924
2475
|
const candidateLineage = {
|
|
1925
2476
|
source: "optimizer",
|
|
1926
2477
|
parentDigests: [material.baseline.digest],
|
|
1927
|
-
runIds: [improvement.
|
|
1928
|
-
developmentSplitDigest: improvement.
|
|
2478
|
+
runIds: [improvement.lineage.runId],
|
|
2479
|
+
developmentSplitDigest: improvement.lineage.developmentSplitDigest
|
|
1929
2480
|
};
|
|
1930
2481
|
return sealCandidateExperiment({ ...material, candidateLineage });
|
|
1931
2482
|
}
|
|
@@ -2019,16 +2570,22 @@ function createAgentImprovementMeasuredComparison(options) {
|
|
|
2019
2570
|
return verifyCandidateExperimentComparison(measuredComparisonFromCandidateExperiment(options));
|
|
2020
2571
|
}
|
|
2021
2572
|
async function proposeAgentImprovement(options) {
|
|
2573
|
+
assertNoCallerOptimizationReceipt(options.metadata);
|
|
2022
2574
|
const analysis = await runAnalystLoop({ ...options.analysis, runId: options.runId });
|
|
2023
2575
|
const findings = assertNoJudgeVerdict(
|
|
2024
2576
|
analysis.analystResult.findings,
|
|
2025
2577
|
"proposeAgentImprovement findings"
|
|
2026
2578
|
);
|
|
2027
|
-
const
|
|
2579
|
+
const improvementInput = {
|
|
2580
|
+
...options.improvement,
|
|
2581
|
+
findings: [...options.improvement.findings ?? [], ...findings]
|
|
2582
|
+
};
|
|
2583
|
+
const improvement = improvementInput.surface === "code" ? await improve(improvementInput) : await improve(options.profile, improvementInput);
|
|
2028
2584
|
try {
|
|
2029
2585
|
if (improvement.decision !== "ship") {
|
|
2030
2586
|
throw new Error("agent improvement search did not produce a promotable candidate");
|
|
2031
2587
|
}
|
|
2588
|
+
const optimizationReceipt = improvement.mode === "method" ? createOptimizationActivationReceipt(improvement) : void 0;
|
|
2032
2589
|
const experiment = sealAgentImprovementExperiment(
|
|
2033
2590
|
await options.buildExperiment({ analysis, improvement }),
|
|
2034
2591
|
improvement
|
|
@@ -2042,10 +2599,12 @@ async function proposeAgentImprovement(options) {
|
|
|
2042
2599
|
...options.maxConcurrency === void 0 ? {} : { maxConcurrency: options.maxConcurrency },
|
|
2043
2600
|
...options.signal ? { signal: options.signal } : {},
|
|
2044
2601
|
...options.candidate ? { candidate: options.candidate } : {},
|
|
2045
|
-
...
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2602
|
+
...optimizationReceipt ? {
|
|
2603
|
+
metadata: attachOptimizationActivationReceipt(options.metadata, optimizationReceipt)
|
|
2604
|
+
} : options.metadata ? { metadata: options.metadata } : {},
|
|
2605
|
+
...improvement.generationsExplored === void 0 ? {} : { generationsExplored: improvement.generationsExplored },
|
|
2606
|
+
searchDurationMs: improvement.durationMs,
|
|
2607
|
+
searchCostUsd: improvement.cost.totalCostUsd
|
|
2049
2608
|
});
|
|
2050
2609
|
const proposal = createAgentImprovementProposal({
|
|
2051
2610
|
runId: options.runId,
|
|
@@ -2067,9 +2626,6 @@ async function proposeAgentImprovement(options) {
|
|
|
2067
2626
|
function assertImprovementCandidateBinding(improvement, experiment) {
|
|
2068
2627
|
const candidate = improvement.candidate;
|
|
2069
2628
|
if (candidate.surface !== "code") {
|
|
2070
|
-
if (!candidate.profile) {
|
|
2071
|
-
throw new Error(`improvement surface '${candidate.surface}' did not produce an exact profile`);
|
|
2072
|
-
}
|
|
2073
2629
|
try {
|
|
2074
2630
|
assertCandidateProfileBinding(candidate.profile, experiment.candidate.profile);
|
|
2075
2631
|
} catch (cause) {
|
|
@@ -2092,6 +2648,7 @@ function createAgentImprovementProposal(options) {
|
|
|
2092
2648
|
"createAgentImprovementProposal findings"
|
|
2093
2649
|
);
|
|
2094
2650
|
const evaluation = verifyCandidateExperimentComparison(options.evaluation);
|
|
2651
|
+
optimizationActivationReceiptFromMetadata(evaluation.metadata);
|
|
2095
2652
|
if (evaluation.decision.outcome !== "ship") {
|
|
2096
2653
|
throw new Error("agent improvement proposal requires a passing experiment");
|
|
2097
2654
|
}
|
|
@@ -2178,6 +2735,7 @@ function verifyAgentImprovementProposal(input) {
|
|
|
2178
2735
|
"agent improvement proposal"
|
|
2179
2736
|
);
|
|
2180
2737
|
const evaluation = verifyCandidateExperimentComparison(proposal.evaluation);
|
|
2738
|
+
optimizationActivationReceiptFromMetadata(evaluation.metadata);
|
|
2181
2739
|
if (evaluation.decision.outcome !== "ship") {
|
|
2182
2740
|
throw new Error("agent improvement proposal does not contain a passing experiment");
|
|
2183
2741
|
}
|
|
@@ -2325,17 +2883,14 @@ export {
|
|
|
2325
2883
|
LIFTED_FINDING_ANALYST_ID,
|
|
2326
2884
|
isAnalystFinding,
|
|
2327
2885
|
toAnalystFindings,
|
|
2328
|
-
improvementDriver,
|
|
2329
|
-
rawTraceDistiller,
|
|
2330
2886
|
ROLLOUT_POLICY_EXTENSION,
|
|
2331
|
-
ROLLOUT_POLICY_BOUNDS,
|
|
2332
2887
|
parseRolloutPolicy,
|
|
2333
2888
|
normalizeRolloutPolicy,
|
|
2334
2889
|
serializeRolloutPolicy,
|
|
2335
2890
|
structuralRolloutPolicyFromProfile,
|
|
2336
2891
|
applyRolloutPolicyToProfile,
|
|
2337
|
-
|
|
2338
|
-
|
|
2892
|
+
rawTraceDistiller,
|
|
2893
|
+
withMethodRuntimeControls,
|
|
2339
2894
|
improve,
|
|
2340
2895
|
AGENT_IMPROVEMENT_PROFILE_SURFACES,
|
|
2341
2896
|
buildAgentImprovementActivationTargets,
|
|
@@ -2345,6 +2900,8 @@ export {
|
|
|
2345
2900
|
agentImprovementProfileSurfaceInput,
|
|
2346
2901
|
agentImprovementProfileSurfaceDigest,
|
|
2347
2902
|
agentImprovementTargetProfileDiffs,
|
|
2903
|
+
createOptimizationActivationReceipt,
|
|
2904
|
+
optimizationActivationReceiptFromMetadata,
|
|
2348
2905
|
AgentCandidateExperimentCellExecutionError,
|
|
2349
2906
|
runAgentCandidateExperiment,
|
|
2350
2907
|
executeAgentCandidateExperimentCell,
|
|
@@ -2358,4 +2915,4 @@ export {
|
|
|
2358
2915
|
verifyAgentImprovementActivation,
|
|
2359
2916
|
verifyCandidateExecutionEvidence
|
|
2360
2917
|
};
|
|
2361
|
-
//# sourceMappingURL=chunk-
|
|
2918
|
+
//# sourceMappingURL=chunk-UHEZW5BU.js.map
|