@tangle-network/agent-bench 0.11.3 → 0.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/HARNESS.md +6 -2
- package/README.md +1 -4
- package/package.json +5 -5
- package/scripts/run-package-tests.mjs +2 -2
- package/src/quant-arena/README.md +0 -144
- package/src/quant-arena/backtest.test.mts +0 -135
- package/src/quant-arena/backtest.ts +0 -218
- package/src/quant-arena/data.test.mts +0 -44
- package/src/quant-arena/data.ts +0 -141
- package/src/quant-arena/driver.test.mts +0 -253
- package/src/quant-arena/driver.ts +0 -219
- package/src/quant-arena/fixtures/data/PROVENANCE.md +0 -26
- package/src/quant-arena/fixtures/data/holdout/IDX.csv +0 -523
- package/src/quant-arena/fixtures/data/holdout/S01.csv +0 -523
- package/src/quant-arena/fixtures/data/holdout/S02.csv +0 -523
- package/src/quant-arena/fixtures/data/holdout/S03.csv +0 -523
- package/src/quant-arena/fixtures/data/holdout/S04.csv +0 -523
- package/src/quant-arena/fixtures/data/holdout/S05.csv +0 -523
- package/src/quant-arena/fixtures/data/holdout/S06.csv +0 -523
- package/src/quant-arena/fixtures/data/holdout/S07.csv +0 -523
- package/src/quant-arena/fixtures/data/holdout/S08.csv +0 -523
- package/src/quant-arena/fixtures/data/holdout/S09.csv +0 -523
- package/src/quant-arena/fixtures/data/holdout/S10.csv +0 -523
- package/src/quant-arena/fixtures/data/insample/IDX.csv +0 -2087
- package/src/quant-arena/fixtures/data/insample/S01.csv +0 -2087
- package/src/quant-arena/fixtures/data/insample/S02.csv +0 -2087
- package/src/quant-arena/fixtures/data/insample/S03.csv +0 -2087
- package/src/quant-arena/fixtures/data/insample/S04.csv +0 -2087
- package/src/quant-arena/fixtures/data/insample/S05.csv +0 -2087
- package/src/quant-arena/fixtures/data/insample/S06.csv +0 -2087
- package/src/quant-arena/fixtures/data/insample/S07.csv +0 -2087
- package/src/quant-arena/fixtures/data/insample/S08.csv +0 -2087
- package/src/quant-arena/fixtures/data/insample/S09.csv +0 -2087
- package/src/quant-arena/fixtures/data/insample/S10.csv +0 -2087
- package/src/quant-arena/fixtures/demo-campaign/cost-ledger.jsonl +0 -16
- package/src/quant-arena/fixtures/demo-campaign/notebook.jsonl +0 -5
- package/src/quant-arena/fixtures/demo-campaign/rollout-manifest.json +0 -171
- package/src/quant-arena/fixtures/demo-campaign/strategies/cand-001-default-author/strategy.ts +0 -119
- package/src/quant-arena/fixtures/demo-campaign/strategies/cand-002-default-author/strategy.ts +0 -119
- package/src/quant-arena/fixtures/demo-campaign/strategies/cand-003-quant-researcher/strategy.ts +0 -105
- package/src/quant-arena/fixtures/demo-campaign/strategies/cand-004-quant-researcher/strategy.ts +0 -102
- package/src/quant-arena/fixtures/demo-campaign-v2/cost-ledger.jsonl +0 -4
- package/src/quant-arena/fixtures/demo-campaign-v2/notebook.jsonl +0 -2
- package/src/quant-arena/fixtures/demo-campaign-v2/rollout-manifest.json +0 -84
- package/src/quant-arena/fixtures/demo-campaign-v2/strategies/cand-001-quant-researcher/strategy.ts +0 -117
- package/src/quant-arena/holdout-certify.mts +0 -206
- package/src/quant-arena/holdout-certify.test.mts +0 -82
- package/src/quant-arena/leak-audit.test.mts +0 -79
- package/src/quant-arena/leak-audit.ts +0 -95
- package/src/quant-arena/make-fixtures.mts +0 -161
- package/src/quant-arena/multiplicity.test.mts +0 -68
- package/src/quant-arena/multiplicity.ts +0 -87
- package/src/quant-arena/nautilus-certify.ts +0 -31
- package/src/quant-arena/oms.ts +0 -90
- package/src/quant-arena/profiles/quant-researcher.profile.json +0 -12
- package/src/quant-arena/python/pyproject.toml +0 -8
- package/src/quant-arena/python/uv.lock +0 -1297
- package/src/quant-arena/python/vbt-worker.py +0 -192
- package/src/quant-arena/quant-loop.mts +0 -840
- package/src/quant-arena/quant-loop.test.mts +0 -75
- package/src/quant-arena/strategies/buy-hold-index/strategy.ts +0 -11
- package/src/quant-arena/strategies/equal-weight/strategy.ts +0 -20
- package/src/quant-arena/strategies/sma-crossover/strategy.ts +0 -42
- package/src/quant-arena/types.ts +0 -133
- package/src/quant-arena/vbt-client.ts +0 -321
- package/src/quant-arena/vbt-parity.test.mts +0 -183
- package/src/quant-arena/windows.test.mts +0 -45
- package/src/quant-arena/windows.ts +0 -54
- package/src/rollout-ledger/backfill-swe-arena.mts +0 -610
- package/src/rollout-ledger/backfill-swe-arena.test.mts +0 -347
- package/src/rollout-ledger/settle-capture.mts +0 -448
- package/src/rollout-ledger/settle-capture.test.mts +0 -270
- package/src/swe-arena/activation.mts +0 -225
- package/src/swe-arena/activation.test.mts +0 -300
- package/src/swe-arena/analyze.ts +0 -211
- package/src/swe-arena/arms.ts +0 -862
- package/src/swe-arena/bootstrap-meta.mts +0 -188
- package/src/swe-arena/bootstrap-meta.test.mts +0 -51
- package/src/swe-arena/briefing.mts +0 -217
- package/src/swe-arena/briefing.test.mts +0 -179
- package/src/swe-arena/calibrate.ts +0 -217
- package/src/swe-arena/capabilities.mts +0 -76
- package/src/swe-arena/capabilities.test.mts +0 -57
- package/src/swe-arena/capacity.ts +0 -198
- package/src/swe-arena/cell-evidence.mts +0 -437
- package/src/swe-arena/cell-evidence.test.mts +0 -248
- package/src/swe-arena/diagnosis-ensemble.test.mts +0 -210
- package/src/swe-arena/diagnosis-ensemble.ts +0 -523
- package/src/swe-arena/execution.test.mts +0 -1171
- package/src/swe-arena/factory-command-container.ts +0 -284
- package/src/swe-arena/factory-judge-child.mts +0 -228
- package/src/swe-arena/factory.test.mts +0 -645
- package/src/swe-arena/fixtures/analyze.py +0 -80
- package/src/swe-arena/fixtures/excludes.txt +0 -8
- package/src/swe-arena/fixtures/factory/agent-eval-309/calibration.md +0 -51
- package/src/swe-arena/fixtures/factory/agent-eval-309/manifest.json +0 -29
- package/src/swe-arena/fixtures/factory/agent-eval-309/spec.md +0 -64
- package/src/swe-arena/fixtures/factory/agent-runtime-232/calibration.md +0 -48
- package/src/swe-arena/fixtures/factory/agent-runtime-232/manifest.json +0 -29
- package/src/swe-arena/fixtures/factory/agent-runtime-232/spec.md +0 -48
- package/src/swe-arena/fixtures/factory/loops-28/calibration.md +0 -47
- package/src/swe-arena/fixtures/factory/loops-28/manifest.json +0 -30
- package/src/swe-arena/fixtures/factory/loops-28/spec.md +0 -50
- package/src/swe-arena/fixtures/gen1-salvage/README.md +0 -45
- package/src/swe-arena/fixtures/gen1-salvage/cand0-e6d7361.diff +0 -116
- package/src/swe-arena/fixtures/gen1-salvage/cand1-76a8590.diff +0 -293
- package/src/swe-arena/fixtures/holdout-preregister.log +0 -12
- package/src/swe-arena/fixtures/holdout.json +0 -44
- package/src/swe-arena/fixtures/instances.json +0 -146
- package/src/swe-arena/fixtures/ledger.jsonl +0 -12
- package/src/swe-arena/fixtures/patches/pallets__flask-5014.solo.patch +0 -36
- package/src/swe-arena/fixtures/patches/pydata__xarray-4687.sup.patch +0 -33
- package/src/swe-arena/fixtures/rejudge.jsonl +0 -15
- package/src/swe-arena/fixtures/rematch.jsonl +0 -3
- package/src/swe-arena/fixtures/rematch2.jsonl +0 -3
- package/src/swe-arena/fixtures/rematch3.jsonl +0 -3
- package/src/swe-arena/fixtures/run-report/README.md +0 -43
- package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.json +0 -173
- package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.md +0 -100
- package/src/swe-arena/fixtures/run-report/gen3-rollup.json +0 -551
- package/src/swe-arena/fixtures/run-report/gen3-rollup.md +0 -64
- package/src/swe-arena/fixtures/sup-journal-true.json +0 -19
- package/src/swe-arena/fixtures/verify/astropy__astropy-13033.sh +0 -48
- package/src/swe-arena/fixtures/verify/django__django-11532.sh +0 -50
- package/src/swe-arena/fixtures/verify/matplotlib__matplotlib-20826.sh +0 -76
- package/src/swe-arena/fixtures/verify/pydata__xarray-4687.sh +0 -44
- package/src/swe-arena/fixtures/verify/pytest-dev__pytest-6197.sh +0 -32
- package/src/swe-arena/fixtures/verify/sphinx-doc__sphinx-9658.sh +0 -51
- package/src/swe-arena/fixtures/worker-tokens.json +0 -42
- package/src/swe-arena/fixtures.ts +0 -237
- package/src/swe-arena/gepa-seat.mts +0 -886
- package/src/swe-arena/gepa-seat.test.mts +0 -1136
- package/src/swe-arena/holdout-certify.mts +0 -408
- package/src/swe-arena/holdout-certify.test.mts +0 -160
- package/src/swe-arena/implementation-ref.test.mts +0 -64
- package/src/swe-arena/implementation-ref.ts +0 -62
- package/src/swe-arena/judge-child.mts +0 -37
- package/src/swe-arena/ledger-orphans.mts +0 -77
- package/src/swe-arena/ledger-orphans.test.mts +0 -149
- package/src/swe-arena/manifest.mts +0 -293
- package/src/swe-arena/manifest.test.mts +0 -169
- package/src/swe-arena/materialize.ts +0 -142
- package/src/swe-arena/outer-loop.mts +0 -2854
- package/src/swe-arena/outer-loop.test.mts +0 -714
- package/src/swe-arena/parity.test.mts +0 -87
- package/src/swe-arena/premeasured-from-cells.mts +0 -296
- package/src/swe-arena/premeasured-from-cells.test.mts +0 -201
- package/src/swe-arena/proc.test.mts +0 -172
- package/src/swe-arena/proc.ts +0 -260
- package/src/swe-arena/profiles/deepseek-author.profile.json +0 -12
- package/src/swe-arena/profiles/default-author.profile.json +0 -12
- package/src/swe-arena/proposer-fanout.mts +0 -736
- package/src/swe-arena/proposer-fanout.test.mts +0 -660
- package/src/swe-arena/proposer-provenance.mts +0 -176
- package/src/swe-arena/proposer-provenance.test.mts +0 -106
- package/src/swe-arena/reconcile.ts +0 -0
- package/src/swe-arena/replay.mts +0 -183
- package/src/swe-arena/replay.test.mts +0 -300
- package/src/swe-arena/run-experiment.mts +0 -729
- package/src/swe-arena/run-report.mts +0 -75
- package/src/swe-arena/run-supervisor.mjs +0 -297
- package/src/swe-arena/run-supervisor.test.mts +0 -539
- package/src/swe-arena/score-split.mts +0 -140
- package/src/swe-arena/score-split.test.mts +0 -123
- package/src/swe-arena/scratch-worktree-serialization.test.mts +0 -72
- package/src/swe-arena/scratch-worktree.test.mts +0 -56
- package/src/swe-arena/scratch-worktree.ts +0 -64
- package/src/swe-arena/serialized-judge.ts +0 -414
- package/src/swe-arena/types.ts +0 -218
|
@@ -1,736 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Gen-3 proposer fan-out — N configured proposers author candidates
|
|
3
|
-
* CONCURRENTLY, each as an `AgentProfile`-pinned harness invocation in its own
|
|
4
|
-
* scratch loops worktree, followed by a CHEAP PRE-FILTER so only survivors
|
|
5
|
-
* reach the expensive full evaluation (reps × instances arm cells).
|
|
6
|
-
*
|
|
7
|
-
* Shape (gen3_design, supervisor-lab/.evolve/state.json):
|
|
8
|
-
* stage 1 — parallel authoring. The improvement driver calls
|
|
9
|
-
* `generate()` once per candidate index, serially. The FIRST call of a
|
|
10
|
-
* generation triggers the fan-out barrier: every proposer authors at once
|
|
11
|
-
* (Promise.all) in scratch worktrees created from the incumbent commit;
|
|
12
|
-
* later calls consume the finished outcomes and apply each proposer's
|
|
13
|
-
* patch onto the driver-owned candidate worktree. Authoring parallelism is
|
|
14
|
-
* free; eval bandwidth stays the binding resource.
|
|
15
|
-
* stage 2 — pre-filter per candidate BEFORE arm spend: change-space check +
|
|
16
|
-
* `tsc` run inside the authoring loop's verifier (existing), plus ONE
|
|
17
|
-
* smoke cell (cheapest instance, 1 rep) through the injected
|
|
18
|
-
* `SmokeRunner`. Killed candidates never become surfaces (the driver
|
|
19
|
-
* discards their worktree — zero arm cells); each kill is recorded and
|
|
20
|
-
* drained into a staircase dot with verdict `rejected-prefilter`.
|
|
21
|
-
* stage 3 — survivors proceed through the unchanged pinned-baseline
|
|
22
|
-
* reps-fail-closed gate.
|
|
23
|
-
*
|
|
24
|
-
* Proposer identity: each harness-authored proposer supplies one complete
|
|
25
|
-
* `AgentProfile`. Runtime validates and materializes that exact profile, then
|
|
26
|
-
* runs it through the CLI bridge in the candidate worktree.
|
|
27
|
-
*
|
|
28
|
-
* NOTE on the import cycle with outer-loop.mts: outer-loop imports this
|
|
29
|
-
* module's generator factory; this module imports outer-loop's change-space +
|
|
30
|
-
* verifier primitives. Every cross-reference is inside a function body (never
|
|
31
|
-
* at module-evaluation time), which ESM resolves correctly.
|
|
32
|
-
*/
|
|
33
|
-
|
|
34
|
-
import { createHash } from 'node:crypto'
|
|
35
|
-
import { readFileSync } from 'node:fs'
|
|
36
|
-
import { mkdir, readFile as readFilePromise, rm, writeFile } from 'node:fs/promises'
|
|
37
|
-
import { join } from 'node:path'
|
|
38
|
-
import { fileURLToPath } from 'node:url'
|
|
39
|
-
import {
|
|
40
|
-
agenticGenerator,
|
|
41
|
-
type CandidateGenerator,
|
|
42
|
-
type AgenticGeneratorShotExecution,
|
|
43
|
-
type AgenticGeneratorShotReceipt,
|
|
44
|
-
} from '@tangle-network/agent-runtime'
|
|
45
|
-
import { agentProfileSchema, type AgentProfile } from '@tangle-network/agent-interface'
|
|
46
|
-
import type { CostLedgerHandle, ProposalFinding } from '@tangle-network/agent-eval'
|
|
47
|
-
import {
|
|
48
|
-
changeSpaceViolations,
|
|
49
|
-
loopsCandidateVerifier,
|
|
50
|
-
purgeIgnoredArtifacts,
|
|
51
|
-
round4BuildPrompt,
|
|
52
|
-
type OuterLoopConfig,
|
|
53
|
-
} from './outer-loop.mts'
|
|
54
|
-
import {
|
|
55
|
-
ACTIVATION_PREDICATE_RELPATH,
|
|
56
|
-
activationPredicateInstruction,
|
|
57
|
-
parseActivationPredicate,
|
|
58
|
-
} from './activation.mts'
|
|
59
|
-
import { briefingPromptSection, type BriefingContext } from './briefing.mts'
|
|
60
|
-
import {
|
|
61
|
-
gepaSeatAuthor,
|
|
62
|
-
isGepaSeat,
|
|
63
|
-
validateGepaSeat,
|
|
64
|
-
type GepaMethodFactory,
|
|
65
|
-
type GepaSeatDeps,
|
|
66
|
-
} from './gepa-seat.mts'
|
|
67
|
-
import type { ScoreSplit } from './score-split.mts'
|
|
68
|
-
import { run, runOk } from './proc.ts'
|
|
69
|
-
import {
|
|
70
|
-
createDetachedWorktree,
|
|
71
|
-
pruneDetachedWorktrees,
|
|
72
|
-
removeDetachedWorktree,
|
|
73
|
-
} from './scratch-worktree.ts'
|
|
74
|
-
|
|
75
|
-
// ---------------------------------------------------------------------------
|
|
76
|
-
// Config types.
|
|
77
|
-
// ---------------------------------------------------------------------------
|
|
78
|
-
|
|
79
|
-
export interface ProposerSpec {
|
|
80
|
-
/** Unique short name — becomes the candidate's staircase `label`. */
|
|
81
|
-
name: string
|
|
82
|
-
/** Path to the complete `AgentProfile` JSON. Absolute, or relative to this
|
|
83
|
-
* module's `profiles/` directory. Required by the Runtime author path. */
|
|
84
|
-
profile?: string
|
|
85
|
-
/** Optional provenance mirror of `profile.harness`; a mismatch is rejected. */
|
|
86
|
-
harness?: 'claude-code' | 'codex' | 'opencode' | 'pi'
|
|
87
|
-
/** Optional provenance mirror of `profile.model.default`; a mismatch is rejected. */
|
|
88
|
-
model?: string
|
|
89
|
-
/** GEN-4 merge seat: this proposer's task is to MERGE the configured
|
|
90
|
-
* Pareto parents' diffs into one coherent surface (see
|
|
91
|
-
* `mergeAuthorPrompt`). Requires ≥2 materialized parents — enforced at
|
|
92
|
-
* generator construction. */
|
|
93
|
-
merge?: boolean
|
|
94
|
-
/** Free-text authoring lens appended to the task prompt (e.g. a mechanics
|
|
95
|
-
* or reviewer stance). Never overrides the protocol change-space text. */
|
|
96
|
-
lens?: string
|
|
97
|
-
/** Which diagnosis findings this proposer sees. Protocol/steering and
|
|
98
|
-
* raw-trace-context findings always pass through. Default 'all'. */
|
|
99
|
-
diagnosisSlice?: 'all' | 'mechanics' | 'prompts'
|
|
100
|
-
/** GEPA seat: this seat is an engine invocation (agent-eval's
|
|
101
|
-
* external-GEPA adapter), not a harness CLI. `gepa` = one bounded engine
|
|
102
|
-
* run; `omni` = GEPA's official Omni recipe. Validation +
|
|
103
|
-
* authoring live in gepa-seat.mts. */
|
|
104
|
-
engine?: 'gepa' | 'omni'
|
|
105
|
-
/** GEPA seat: the one repo-relative change-space file GEPA optimizes
|
|
106
|
-
* as a string; the rest of the loops repo stays at the incumbent commit. */
|
|
107
|
-
surface?: string
|
|
108
|
-
/** GEPA seat: total inner-evaluation budget (each inner call is one
|
|
109
|
-
* real smoke arm cell). Default 10. */
|
|
110
|
-
maxMetricCalls?: number
|
|
111
|
-
/** GEPA seat: hard cap for metered optimizer-model spend. */
|
|
112
|
-
maxProposerCostUsd?: number
|
|
113
|
-
/** GEPA seat: Python executable for the bridge. Default 'python3'. */
|
|
114
|
-
python?: string
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
export interface PrefilterConfig {
|
|
118
|
-
enabled: boolean
|
|
119
|
-
/** Instance for the single smoke cell: an explicit iid (e.g.
|
|
120
|
-
* 'pallets__flask-5014' once its verify fixture is committed) or
|
|
121
|
-
* 'cheapest-of-set' — the improvement-set instance with the smallest
|
|
122
|
-
* measured baseline wall time. */
|
|
123
|
-
smokeInstance: string
|
|
124
|
-
/** When true the smoke cell must RESOLVE to survive (right for a designated
|
|
125
|
-
* easy instance). Default false: the mechanism bar — the arm must complete,
|
|
126
|
-
* deliver a non-empty patch, and draw a conclusive judge verdict — so a
|
|
127
|
-
* hard cheapest-of-set instance cannot kill every candidate. */
|
|
128
|
-
requireResolved?: boolean
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
export interface SmokeVerdict {
|
|
132
|
-
iid: string
|
|
133
|
-
pass: boolean
|
|
134
|
-
reason: string
|
|
135
|
-
resolved: boolean | null
|
|
136
|
-
patchLines: number
|
|
137
|
-
wallS: number
|
|
138
|
-
/** Committed verify fixture passed, used as the GEPA seat's inner-score
|
|
139
|
-
* tiebreak. Absent on errored smokes and older records. */
|
|
140
|
-
verifyPass?: boolean
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
export type SmokeRunner = (args: {
|
|
144
|
-
scratchPath: string
|
|
145
|
-
generation: number
|
|
146
|
-
proposer: ProposerSpec
|
|
147
|
-
evaluationKey: string
|
|
148
|
-
costLedger?: CostLedgerHandle
|
|
149
|
-
}) => Promise<SmokeVerdict>
|
|
150
|
-
|
|
151
|
-
export interface PrefilterKill {
|
|
152
|
-
generation: number
|
|
153
|
-
candidateIndex: number
|
|
154
|
-
proposer: string
|
|
155
|
-
harness: ProposerSpec['harness']
|
|
156
|
-
stage: 'change-space' | 'activation-predicate' | 'smoke'
|
|
157
|
-
reason: string
|
|
158
|
-
diffSha256: string | null
|
|
159
|
-
/** Persisted kill-forensics patch under `<outDir>/proposer-patches/`. */
|
|
160
|
-
patchPath: string | null
|
|
161
|
-
smoke?: SmokeVerdict
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
// ---------------------------------------------------------------------------
|
|
165
|
-
// Author profiles.
|
|
166
|
-
// ---------------------------------------------------------------------------
|
|
167
|
-
|
|
168
|
-
export const PROFILES_DIR = fileURLToPath(new URL('./profiles', import.meta.url))
|
|
169
|
-
|
|
170
|
-
/** Load and schema-validate a proposer's exact `AgentProfile`. */
|
|
171
|
-
export function loadAuthorProfile(spec: ProposerSpec): AgentProfile | undefined {
|
|
172
|
-
if (!spec.profile) return undefined
|
|
173
|
-
const path = spec.profile.startsWith('/') ? spec.profile : join(PROFILES_DIR, spec.profile)
|
|
174
|
-
return agentProfileSchema.parse(JSON.parse(readFileSync(path, 'utf8')))
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
/** Default author: Pi with the exact GLM profile committed beside this module. */
|
|
178
|
-
export function defaultProposers(): ProposerSpec[] {
|
|
179
|
-
return [
|
|
180
|
-
{
|
|
181
|
-
name: 'default-author',
|
|
182
|
-
profile: 'default-author.profile.json',
|
|
183
|
-
harness: 'pi',
|
|
184
|
-
model: 'glm-5.2',
|
|
185
|
-
},
|
|
186
|
-
]
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
/** Resolve the one exact profile; duplicated provenance fields may only agree. */
|
|
190
|
-
export function resolveAuthorProfile(spec: ProposerSpec): AgentProfile | undefined {
|
|
191
|
-
const profile = loadAuthorProfile(spec)
|
|
192
|
-
if (!profile) return undefined
|
|
193
|
-
if (spec.harness !== undefined && profile.harness !== spec.harness) {
|
|
194
|
-
throw new Error(
|
|
195
|
-
`proposer ${spec.name}: harness ${spec.harness} conflicts with profile.harness ${profile.harness}`,
|
|
196
|
-
)
|
|
197
|
-
}
|
|
198
|
-
if (spec.model !== undefined && profile.model?.default !== spec.model) {
|
|
199
|
-
throw new Error(
|
|
200
|
-
`proposer ${spec.name}: model ${spec.model} conflicts with profile.model.default ${profile.model?.default ?? 'missing'}`,
|
|
201
|
-
)
|
|
202
|
-
}
|
|
203
|
-
return profile
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
// ---------------------------------------------------------------------------
|
|
207
|
-
// GEN-4 Pareto parents — cross-run seeding at the buildPrompt seam.
|
|
208
|
-
//
|
|
209
|
-
// The LIB's own `ctx.paretoParents` (runOptimization → SurfaceProposer) only
|
|
210
|
-
// accumulates surfaces scored WITHIN one run: the frontier starts from this
|
|
211
|
-
// run's baseline and generations, and a prior campaign's surfaces cannot be
|
|
212
|
-
// injected without replaying that campaign's runDir + cost ledger (the cached
|
|
213
|
-
// cells refuse to load without their ledger receipts). Gen-4 runs in a fresh
|
|
214
|
-
// outDir, so the gen-3 winners are seeded HERE — their diffs + per-instance
|
|
215
|
-
// results enter every author's task prompt, and the merge seat gets both
|
|
216
|
-
// diffs as its explicit merge input. This is our seam, not the lib path;
|
|
217
|
-
// noted in the gen-4 config docs.
|
|
218
|
-
// ---------------------------------------------------------------------------
|
|
219
|
-
|
|
220
|
-
/** One prior-run frontier member, as configured (commits live in loopsRepo). */
|
|
221
|
-
export interface ParetoParentSeed {
|
|
222
|
-
/** Loops commit of the parent candidate (must exist in `loopsRepo`). */
|
|
223
|
-
commit: string
|
|
224
|
-
/** Staircase label, e.g. 'default-author'. */
|
|
225
|
-
label: string
|
|
226
|
-
/** Instances the parent resolved under the fail-closed all-reps rule. */
|
|
227
|
-
resolvedInstances: string[]
|
|
228
|
-
/** Free-text evidence note (discordant replicates, mechanism summary). */
|
|
229
|
-
note?: string
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
/** A seed materialized against the loops repo: the parent's full diff. */
|
|
233
|
-
export interface ParetoParentContext extends ParetoParentSeed {
|
|
234
|
-
diff: string
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
export const PARENT_DIFF_MAX_CHARS = 60_000
|
|
238
|
-
|
|
239
|
-
/** Materialize parent seeds: verify each commit exists in `loopsRepo` and
|
|
240
|
-
* capture its full diff (`git show`). Fails loud on a missing commit — a
|
|
241
|
-
* silently absent parent would turn the merge seat into a no-op. */
|
|
242
|
-
export async function materializeParetoParents(
|
|
243
|
-
loopsRepo: string,
|
|
244
|
-
seeds: ParetoParentSeed[],
|
|
245
|
-
maxDiffChars = PARENT_DIFF_MAX_CHARS,
|
|
246
|
-
): Promise<ParetoParentContext[]> {
|
|
247
|
-
const parents: ParetoParentContext[] = []
|
|
248
|
-
for (const seed of seeds) {
|
|
249
|
-
const exists = await run('git', ['-C', loopsRepo, 'cat-file', '-e', `${seed.commit}^{commit}`])
|
|
250
|
-
if (exists.code !== 0) {
|
|
251
|
-
throw new Error(`pareto parent ${seed.label}: commit ${seed.commit} not found in ${loopsRepo}`)
|
|
252
|
-
}
|
|
253
|
-
const show = await runOk('git', ['-C', loopsRepo, 'show', '--no-color', seed.commit])
|
|
254
|
-
const diff =
|
|
255
|
-
show.stdout.length > maxDiffChars
|
|
256
|
-
? `${show.stdout.slice(0, maxDiffChars)}\n[... diff truncated at ${maxDiffChars} chars ...]`
|
|
257
|
-
: show.stdout
|
|
258
|
-
parents.push({ ...seed, diff })
|
|
259
|
-
}
|
|
260
|
-
return parents
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
function parentEvidenceLine(parent: ParetoParentContext): string {
|
|
264
|
-
const resolved = parent.resolvedInstances.length > 0 ? parent.resolvedInstances.join(', ') : 'none'
|
|
265
|
-
return `${parent.label} (${parent.commit.slice(0, 10)}) — resolved: ${resolved}${parent.note ? `; ${parent.note}` : ''}`
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
/** The parents section appended to every NON-merge author's prompt: measured
|
|
269
|
-
* evidence of what worked, never an instruction to copy. */
|
|
270
|
-
export function parentsPromptSection(parents: ParetoParentContext[]): string {
|
|
271
|
-
const lines: string[] = [
|
|
272
|
-
'PARETO PARENTS — the prior generation’s frontier candidates, measured on the SAME 6-instance set.',
|
|
273
|
-
'Each beat the baseline on different instances; treat their diffs as measured evidence of what works.',
|
|
274
|
-
'You may build on either (or both), but your candidate is measured on the full set — do not blindly copy.',
|
|
275
|
-
]
|
|
276
|
-
for (const parent of parents) {
|
|
277
|
-
lines.push('', `--- parent: ${parentEvidenceLine(parent)}`, parent.diff.trimEnd())
|
|
278
|
-
}
|
|
279
|
-
return lines.join('\n')
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
/** The merge seat's task prompt: the change-space contract + findings context
|
|
283
|
-
* stay (round4BuildPrompt), and the TASK is replaced with an explicit
|
|
284
|
-
* coherent-union merge of the parents' diffs. */
|
|
285
|
-
export function mergeAuthorPrompt(
|
|
286
|
-
args: { findings: ReadonlyArray<ProposalFinding> },
|
|
287
|
-
spec: ProposerSpec,
|
|
288
|
-
parents: ParetoParentContext[],
|
|
289
|
-
): string {
|
|
290
|
-
if (parents.length < 2) {
|
|
291
|
-
throw new Error(`merge proposer ${spec.name}: needs >=2 materialized pareto parents, got ${parents.length}`)
|
|
292
|
-
}
|
|
293
|
-
const lines: string[] = [
|
|
294
|
-
round4BuildPrompt(args),
|
|
295
|
-
'',
|
|
296
|
-
`YOUR TASK (${spec.name} — MERGE SEAT):`,
|
|
297
|
-
'The prior generation produced the frontier candidates below. Each beat the baseline on DIFFERENT',
|
|
298
|
-
'instances, so their lessons are complementary. Produce ONE coherent surface that is the UNION of the',
|
|
299
|
-
'parent diffs:',
|
|
300
|
-
...parents.map((p) => ` - ${parentEvidenceLine(p)}`),
|
|
301
|
-
'',
|
|
302
|
-
'Merge rules:',
|
|
303
|
-
'- Apply BOTH parents’ behaviors. Where the diffs touch the same file/section (e.g. the supervisor',
|
|
304
|
-
' reviewer bullet or the worker self-test rules), resolve the conflict by KEEPING BOTH BEHAVIORS —',
|
|
305
|
-
' write one merged passage that carries every constraint from each side, never by dropping one side.',
|
|
306
|
-
'- Keep each parent’s mechanical changes intact (code paths, exported helpers, threading) alongside the',
|
|
307
|
-
' other parent’s prompt/goal-authoring changes.',
|
|
308
|
-
'- Do NOT invent new mechanisms beyond the union; the smallest coherent merge wins.',
|
|
309
|
-
'- The merged surface must typecheck and stay inside the declared change-space above.',
|
|
310
|
-
'',
|
|
311
|
-
'Parent diffs (full):',
|
|
312
|
-
]
|
|
313
|
-
for (const parent of parents) {
|
|
314
|
-
lines.push('', `=== parent ${parent.label} (${parent.commit.slice(0, 10)}) ===`, parent.diff.trimEnd())
|
|
315
|
-
}
|
|
316
|
-
return lines.join('\n')
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
// ---------------------------------------------------------------------------
|
|
320
|
-
// Diagnosis slicing + prompt lens.
|
|
321
|
-
// ---------------------------------------------------------------------------
|
|
322
|
-
|
|
323
|
-
const PROMPTY = /prompt|instruction|wording|message text|phrasing/i
|
|
324
|
-
|
|
325
|
-
/** Findings that must reach EVERY proposer regardless of slice: the protocol
|
|
326
|
-
* steering finding and the raw-trace-context findings (they are contract +
|
|
327
|
-
* grounding, not diagnosis). */
|
|
328
|
-
function isSteeringOrRawTrace(f: Record<string, unknown>): boolean {
|
|
329
|
-
return (
|
|
330
|
-
f.analyst_id === 'round4-protocol' ||
|
|
331
|
-
f.analyst_id === 'raw-trace-distiller' ||
|
|
332
|
-
f.area === 'raw-trace-context' ||
|
|
333
|
-
f.area === 'constraint'
|
|
334
|
-
)
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
/** Slice the diagnosis findings for one proposer. Pure. */
|
|
338
|
-
export function sliceFindings(
|
|
339
|
-
findings: ReadonlyArray<ProposalFinding>,
|
|
340
|
-
slice: ProposerSpec['diagnosisSlice'],
|
|
341
|
-
): ProposalFinding[] {
|
|
342
|
-
if (slice === undefined || slice === 'all') return [...findings]
|
|
343
|
-
return findings.filter((finding) => {
|
|
344
|
-
const f = finding as unknown as Record<string, unknown>
|
|
345
|
-
if (isSteeringOrRawTrace(f)) return true
|
|
346
|
-
const text = `${String(f.area ?? '')} ${String(f.subject ?? '')} ${String(f.claim ?? '')} ${String(f.recommended_action ?? '')}`
|
|
347
|
-
return slice === 'prompts' ? PROMPTY.test(text) : !PROMPTY.test(text)
|
|
348
|
-
})
|
|
349
|
-
}
|
|
350
|
-
|
|
351
|
-
/** GEN-5 prompt extensions shared by every seat (merge included): the
|
|
352
|
-
* MAP+TOOLBOX+PERMISSION briefing (briefing.mts) and the activation-predicate
|
|
353
|
-
* deliverable contract (activation.mts). */
|
|
354
|
-
export interface Gen5PromptExtras {
|
|
355
|
-
briefing?: BriefingContext
|
|
356
|
-
activationGate?: boolean
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
function appendGen5Sections(prompt: string, extras: Gen5PromptExtras): string {
|
|
360
|
-
let out = prompt
|
|
361
|
-
if (extras.briefing) out = `${out}\n\n${briefingPromptSection(extras.briefing)}`
|
|
362
|
-
if (extras.activationGate) out = `${out}\n\n${activationPredicateInstruction()}`
|
|
363
|
-
return out
|
|
364
|
-
}
|
|
365
|
-
|
|
366
|
-
/** The proposer's task prompt: the shared round prompt plus this proposer's
|
|
367
|
-
* authoring lens (appended so the protocol change-space text stays intact),
|
|
368
|
-
* plus the gen-4 Pareto-parents section when parents are seeded, plus the
|
|
369
|
-
* gen-5 briefing/activation sections when configured. A merge-seat spec gets
|
|
370
|
-
* the dedicated merge prompt instead (gen-5 sections still apply). */
|
|
371
|
-
export function proposerBuildPrompt(
|
|
372
|
-
args: { findings: ReadonlyArray<ProposalFinding> },
|
|
373
|
-
spec: ProposerSpec,
|
|
374
|
-
parents: ParetoParentContext[] = [],
|
|
375
|
-
extras: Gen5PromptExtras = {},
|
|
376
|
-
): string {
|
|
377
|
-
if (spec.merge) return appendGen5Sections(mergeAuthorPrompt(args, spec, parents), extras)
|
|
378
|
-
let prompt = round4BuildPrompt(args)
|
|
379
|
-
if (spec.lens) prompt = `${prompt}\n\nYOUR AUTHORING LENS (${spec.name}):\n${spec.lens}`
|
|
380
|
-
if (parents.length > 0) prompt = `${prompt}\n\n${parentsPromptSection(parents)}`
|
|
381
|
-
return appendGen5Sections(prompt, extras)
|
|
382
|
-
}
|
|
383
|
-
|
|
384
|
-
// ---------------------------------------------------------------------------
|
|
385
|
-
// Shared shot persistence + spend settlement (used by the fan-out authors AND
|
|
386
|
-
// the legacy single-proposer generator in outer-loop.mts — one implementation).
|
|
387
|
-
// ---------------------------------------------------------------------------
|
|
388
|
-
|
|
389
|
-
export function proposerShotHooks(opts: {
|
|
390
|
-
shotDir: string
|
|
391
|
-
}): (receipt: AgenticGeneratorShotReceipt, execution: AgenticGeneratorShotExecution | null) => Promise<void> {
|
|
392
|
-
return async (receipt, execution) => {
|
|
393
|
-
await mkdir(opts.shotDir, { recursive: true })
|
|
394
|
-
const name = `gen${receipt.generation ?? 'x'}-cand${receipt.candidateIndex ?? 'x'}-shot${receipt.shot}.json`
|
|
395
|
-
await writeFile(
|
|
396
|
-
join(opts.shotDir, name),
|
|
397
|
-
JSON.stringify({ receipt, execution }, null, 2),
|
|
398
|
-
)
|
|
399
|
-
}
|
|
400
|
-
}
|
|
401
|
-
|
|
402
|
-
// ---------------------------------------------------------------------------
|
|
403
|
-
// Scratch authoring worktrees (no node_modules symlink — the verifier and the
|
|
404
|
-
// smoke runner link one in temporarily when they need it).
|
|
405
|
-
// ---------------------------------------------------------------------------
|
|
406
|
-
|
|
407
|
-
const sanitize = (s: string): string => s.replace(/[^a-zA-Z0-9_-]/g, '_')
|
|
408
|
-
|
|
409
|
-
// ---------------------------------------------------------------------------
|
|
410
|
-
// The fan-out generator.
|
|
411
|
-
// ---------------------------------------------------------------------------
|
|
412
|
-
|
|
413
|
-
/** One proposer's authoring + pre-filter outcome for a generation. */
|
|
414
|
-
interface AuthorOutcome {
|
|
415
|
-
proposer: ProposerSpec
|
|
416
|
-
applied: boolean
|
|
417
|
-
summary: string
|
|
418
|
-
/** Full staged diff (tracked edits + untracked deliverables) vs the
|
|
419
|
-
* incumbent commit. Null when not applied or killed. */
|
|
420
|
-
patch: string | null
|
|
421
|
-
kill: PrefilterKill | null
|
|
422
|
-
}
|
|
423
|
-
|
|
424
|
-
/** Authoring seam — test-injectable. The default runs the proposer's
|
|
425
|
-
* `agenticGenerator` (profile-pinned harness invocation) in the scratch
|
|
426
|
-
* worktree. */
|
|
427
|
-
export type AuthorFn = (
|
|
428
|
-
proposer: ProposerSpec,
|
|
429
|
-
args: Parameters<CandidateGenerator['generate']>[0],
|
|
430
|
-
) => Promise<{ applied: boolean; summary: string }>
|
|
431
|
-
|
|
432
|
-
export interface FanOutDeps {
|
|
433
|
-
author?: AuthorFn
|
|
434
|
-
smokeRunner?: SmokeRunner
|
|
435
|
-
/** Immutable digest of the smoke runner plus every captured execution input. */
|
|
436
|
-
runnerImplementationRef?: string
|
|
437
|
-
/** Immutable digest of the official judge implementation and configuration. */
|
|
438
|
-
judgeImplementationRef?: string
|
|
439
|
-
/** GEN-4: materialized Pareto parents seeded into every author's prompt (and
|
|
440
|
-
* the merge seat's explicit merge input). Empty/omitted = gen-3 behavior. */
|
|
441
|
-
parents?: ParetoParentContext[]
|
|
442
|
-
/** GEN-5: MAP+TOOLBOX briefing context threaded into every author prompt. */
|
|
443
|
-
briefing?: BriefingContext
|
|
444
|
-
/** The resolved public smoke instance used by the GEPA seat's inner
|
|
445
|
-
* evaluator target. Required (with `smokeRunner`) when a gepa seat is
|
|
446
|
-
* configured and no custom `author` is injected. */
|
|
447
|
-
smokeInstanceId?: string
|
|
448
|
-
/** The score split; private instance ids never reach the GEPA
|
|
449
|
-
* bridge (gepa-seat.mts asserts, fail-closed). Null/omitted = no split. */
|
|
450
|
-
scoreSplit?: Pick<ScoreSplit, 'privateInstances'> | null
|
|
451
|
-
/** GEPA method override for tests. Default: agent-eval's official GEPA method. */
|
|
452
|
-
gepaMethodFactory?: GepaMethodFactory
|
|
453
|
-
/** Explicit GEPA optimizer model override, primarily for isolated tests. */
|
|
454
|
-
gepaOptimizer?: GepaSeatDeps['optimizer']
|
|
455
|
-
log?: (msg: string) => void
|
|
456
|
-
}
|
|
457
|
-
|
|
458
|
-
export interface FanOutGenerator extends CandidateGenerator {
|
|
459
|
-
/** Prefilter kills recorded so far; draining clears the buffer. The caller
|
|
460
|
-
* turns each into a `rejected-prefilter` staircase dot. */
|
|
461
|
-
drainPrefilterKills(): PrefilterKill[]
|
|
462
|
-
}
|
|
463
|
-
|
|
464
|
-
function defaultAuthor(config: OuterLoopConfig, deps: FanOutDeps): AuthorFn {
|
|
465
|
-
// One agenticGenerator per proposer, created lazily and cached: each carries
|
|
466
|
-
// its own profile, lens prompt, and shot-receipt home.
|
|
467
|
-
const inners = new Map<string, CandidateGenerator>()
|
|
468
|
-
// The GEPA seat authors through the agent-eval adapter, not a
|
|
469
|
-
// harness CLI. Its inner evaluator is the SAME injected smoke runner the
|
|
470
|
-
// pre-filter uses (presence enforced at generator construction).
|
|
471
|
-
let gepaAuthor: AuthorFn | undefined
|
|
472
|
-
return (proposer, args) => {
|
|
473
|
-
if (isGepaSeat(proposer)) {
|
|
474
|
-
gepaAuthor ??= gepaSeatAuthor(config, {
|
|
475
|
-
smokeRunner: deps.smokeRunner!,
|
|
476
|
-
runnerImplementationRef: deps.runnerImplementationRef!,
|
|
477
|
-
judgeImplementationRef: deps.judgeImplementationRef!,
|
|
478
|
-
smokeInstanceId: deps.smokeInstanceId!,
|
|
479
|
-
scoreSplit: deps.scoreSplit ?? null,
|
|
480
|
-
...(deps.gepaMethodFactory ? { methodFactory: deps.gepaMethodFactory } : {}),
|
|
481
|
-
...(deps.gepaOptimizer ? { optimizer: deps.gepaOptimizer } : {}),
|
|
482
|
-
...(deps.log ? { log: deps.log } : {}),
|
|
483
|
-
})
|
|
484
|
-
return gepaAuthor(proposer, args)
|
|
485
|
-
}
|
|
486
|
-
let inner = inners.get(proposer.name)
|
|
487
|
-
if (!inner) {
|
|
488
|
-
const profile = resolveAuthorProfile(proposer)
|
|
489
|
-
if (!profile) {
|
|
490
|
-
throw new Error(`proposer ${proposer.name}: profile is required for a Runtime author seat`)
|
|
491
|
-
}
|
|
492
|
-
const bridgeUrl = process.env.CLI_BRIDGE_URL ?? process.env.BRIDGE_URL
|
|
493
|
-
const bridgeBearer = process.env.CLI_BRIDGE_BEARER ?? process.env.BRIDGE_BEARER
|
|
494
|
-
if (!bridgeUrl || !bridgeBearer) {
|
|
495
|
-
throw new Error(
|
|
496
|
-
'proposer authoring requires CLI_BRIDGE_URL/BRIDGE_URL and CLI_BRIDGE_BEARER/BRIDGE_BEARER',
|
|
497
|
-
)
|
|
498
|
-
}
|
|
499
|
-
inner = agenticGenerator({
|
|
500
|
-
profile,
|
|
501
|
-
executorForWorktree: (cwd) => ({
|
|
502
|
-
backend: 'bridge',
|
|
503
|
-
bridgeUrl,
|
|
504
|
-
bridgeBearer,
|
|
505
|
-
cwd,
|
|
506
|
-
}),
|
|
507
|
-
timeoutMs: config.proposerTimeoutMs,
|
|
508
|
-
buildPrompt: (args) =>
|
|
509
|
-
proposerBuildPrompt(args, proposer, deps.parents ?? [], {
|
|
510
|
-
...(deps.briefing ? { briefing: deps.briefing } : {}),
|
|
511
|
-
...(config.activationGate === true ? { activationGate: true } : {}),
|
|
512
|
-
}),
|
|
513
|
-
verify: loopsCandidateVerifier(config.loopsRepo),
|
|
514
|
-
onShotCompleted: proposerShotHooks({
|
|
515
|
-
shotDir: join(config.outDir, 'proposer-shots', sanitize(proposer.name)),
|
|
516
|
-
}),
|
|
517
|
-
})
|
|
518
|
-
inners.set(proposer.name, inner)
|
|
519
|
-
}
|
|
520
|
-
return inner.generate(args)
|
|
521
|
-
}
|
|
522
|
-
}
|
|
523
|
-
|
|
524
|
-
/**
|
|
525
|
-
* Build the fan-out `CandidateGenerator` for `config.proposers`. The driver
|
|
526
|
-
* still owns every candidate worktree; this generator authors in its OWN
|
|
527
|
-
* scratch worktrees (in parallel, one per proposer) and applies each
|
|
528
|
-
* surviving proposer's patch onto the driver worktree for that candidate
|
|
529
|
-
* index. `populationSize` must equal `proposers.length` — enforced by the
|
|
530
|
-
* caller (runRound) and re-checked per call here.
|
|
531
|
-
*/
|
|
532
|
-
export function fanOutLoopsGenerator(config: OuterLoopConfig, deps: FanOutDeps = {}): FanOutGenerator {
|
|
533
|
-
const proposers = config.proposers ?? []
|
|
534
|
-
if (proposers.length === 0) throw new Error('fanOutLoopsGenerator: config.proposers is empty')
|
|
535
|
-
const names = new Set(proposers.map((p) => p.name))
|
|
536
|
-
if (names.size !== proposers.length) throw new Error('fanOutLoopsGenerator: duplicate proposer names')
|
|
537
|
-
const mergeSeats = proposers.filter((p) => p.merge === true)
|
|
538
|
-
if (mergeSeats.length > 0 && (deps.parents ?? []).length < 2) {
|
|
539
|
-
throw new Error(
|
|
540
|
-
`fanOutLoopsGenerator: merge proposer(s) ${mergeSeats.map((p) => p.name).join(', ')} configured but ` +
|
|
541
|
-
`only ${(deps.parents ?? []).length} pareto parent(s) materialized — a merge seat needs >=2`,
|
|
542
|
-
)
|
|
543
|
-
}
|
|
544
|
-
// Engine seats are validated at construction, and the
|
|
545
|
-
// default author path requires the pre-filter smoke runner — it IS the GEPA
|
|
546
|
-
// seat's inner evaluator (spec: score = smoke resolve + verify-pass
|
|
547
|
-
// tiebreak). A custom injected `author` owns its own evaluator.
|
|
548
|
-
const gepaSeats = proposers.filter(isGepaSeat)
|
|
549
|
-
for (const seat of gepaSeats) validateGepaSeat(seat)
|
|
550
|
-
for (const seat of proposers) {
|
|
551
|
-
if (!isGepaSeat(seat) && seat.harness === undefined) {
|
|
552
|
-
throw new Error(`fanOutLoopsGenerator: proposer ${seat.name} has neither a harness nor an engine`)
|
|
553
|
-
}
|
|
554
|
-
}
|
|
555
|
-
if (
|
|
556
|
-
gepaSeats.length > 0 &&
|
|
557
|
-
deps.author === undefined &&
|
|
558
|
-
(
|
|
559
|
-
deps.smokeRunner === undefined ||
|
|
560
|
-
deps.smokeInstanceId === undefined ||
|
|
561
|
-
deps.runnerImplementationRef === undefined ||
|
|
562
|
-
deps.judgeImplementationRef === undefined
|
|
563
|
-
)
|
|
564
|
-
) {
|
|
565
|
-
throw new Error(
|
|
566
|
-
`fanOutLoopsGenerator: gepa seat(s) ${gepaSeats.map((p) => p.name).join(', ')} need the pre-filter smoke ` +
|
|
567
|
-
'cell plus immutable runner and judge references as their inner evaluator',
|
|
568
|
-
)
|
|
569
|
-
}
|
|
570
|
-
const author = deps.author ?? defaultAuthor(config, deps)
|
|
571
|
-
const log = deps.log ?? (() => {})
|
|
572
|
-
const kills: PrefilterKill[] = []
|
|
573
|
-
const byGeneration = new Map<number, Promise<AuthorOutcome[]>>()
|
|
574
|
-
|
|
575
|
-
const runFanOut = async (
|
|
576
|
-
generation: number,
|
|
577
|
-
args: Parameters<CandidateGenerator['generate']>[0],
|
|
578
|
-
): Promise<AuthorOutcome[]> => {
|
|
579
|
-
const head = await runOk('git', ['-C', args.worktreePath, 'rev-parse', 'HEAD'])
|
|
580
|
-
const baseCommit = head.stdout.trim()
|
|
581
|
-
const patchesDir = join(config.outDir, 'proposer-patches')
|
|
582
|
-
await mkdir(patchesDir, { recursive: true })
|
|
583
|
-
await pruneDetachedWorktrees(config.loopsRepo)
|
|
584
|
-
log(`fan-out gen ${generation}: ${proposers.length} proposer(s) authoring in parallel from ${baseCommit.slice(0, 10)}`)
|
|
585
|
-
return Promise.all(
|
|
586
|
-
proposers.map(async (proposer, index): Promise<AuthorOutcome> => {
|
|
587
|
-
const scratch = join(config.outDir, 'proposer-wt', `gen${generation}-cand${index}-${sanitize(proposer.name)}`)
|
|
588
|
-
await createDetachedWorktree(config.loopsRepo, baseCommit, scratch)
|
|
589
|
-
try {
|
|
590
|
-
const authored = await author(proposer, {
|
|
591
|
-
...args,
|
|
592
|
-
worktreePath: scratch,
|
|
593
|
-
findings: sliceFindings(args.findings, proposer.diagnosisSlice),
|
|
594
|
-
candidateIndex: index,
|
|
595
|
-
})
|
|
596
|
-
if (!authored.applied) {
|
|
597
|
-
log(`fan-out gen ${generation} ${proposer.name}: no verified candidate (applied=false)`)
|
|
598
|
-
return { proposer, applied: false, summary: authored.summary, patch: null, kill: null }
|
|
599
|
-
}
|
|
600
|
-
// Anti-tamper purge BEFORE the diff: ignored artifacts (dep installs)
|
|
601
|
-
// must never reach the candidate worktree or the change-space check.
|
|
602
|
-
await purgeIgnoredArtifacts(scratch)
|
|
603
|
-
await runOk('git', ['-C', scratch, 'add', '-A'])
|
|
604
|
-
const diff = (await runOk('git', ['-C', scratch, 'diff', '--cached'])).stdout
|
|
605
|
-
if (diff.trim().length === 0) {
|
|
606
|
-
return { proposer, applied: false, summary: authored.summary, patch: null, kill: null }
|
|
607
|
-
}
|
|
608
|
-
const namesOut = await runOk('git', ['-C', scratch, 'diff', '--cached', '--name-only'])
|
|
609
|
-
const changed = namesOut.stdout.split('\n').map((s) => s.trim()).filter(Boolean)
|
|
610
|
-
const diffSha256 = `sha256:${createHash('sha256').update(diff).digest('hex')}`
|
|
611
|
-
const patchPath = join(patchesDir, `gen${generation}-cand${index}-${sanitize(proposer.name)}.patch`)
|
|
612
|
-
await writeFile(patchPath, diff)
|
|
613
|
-
|
|
614
|
-
// Pre-filter stage A — change-space (defense-in-depth: the authoring
|
|
615
|
-
// verifier already enforced it shot-by-shot; the final diff rules).
|
|
616
|
-
const violations = changeSpaceViolations(changed)
|
|
617
|
-
if (violations.length > 0) {
|
|
618
|
-
const kill: PrefilterKill = {
|
|
619
|
-
generation,
|
|
620
|
-
candidateIndex: index,
|
|
621
|
-
proposer: proposer.name,
|
|
622
|
-
harness: proposer.harness,
|
|
623
|
-
stage: 'change-space',
|
|
624
|
-
reason: `out-of-space paths: ${violations.join(', ')}`,
|
|
625
|
-
diffSha256,
|
|
626
|
-
patchPath,
|
|
627
|
-
}
|
|
628
|
-
kills.push(kill)
|
|
629
|
-
log(`prefilter KILL gen ${generation} ${proposer.name}: ${kill.reason}`)
|
|
630
|
-
return { proposer, applied: false, summary: authored.summary, patch: null, kill }
|
|
631
|
-
}
|
|
632
|
-
|
|
633
|
-
// Pre-filter stage A.5 — GEN-5 activation gate: a candidate without
|
|
634
|
-
// a parseable machine-checkable predicate is killed before any
|
|
635
|
-
// evaluation spend (the post-eval quarantine needs it to exist).
|
|
636
|
-
if (config.activationGate === true) {
|
|
637
|
-
const raw = await readFilePromise(join(scratch, ACTIVATION_PREDICATE_RELPATH), 'utf8').catch(() => null)
|
|
638
|
-
const parsed = raw === null ? null : parseActivationPredicate(raw)
|
|
639
|
-
if (parsed === null || !parsed.ok) {
|
|
640
|
-
const kill: PrefilterKill = {
|
|
641
|
-
generation,
|
|
642
|
-
candidateIndex: index,
|
|
643
|
-
proposer: proposer.name,
|
|
644
|
-
harness: proposer.harness,
|
|
645
|
-
stage: 'activation-predicate',
|
|
646
|
-
reason:
|
|
647
|
-
parsed === null
|
|
648
|
-
? `missing ${ACTIVATION_PREDICATE_RELPATH} (required activation predicate)`
|
|
649
|
-
: `invalid ${ACTIVATION_PREDICATE_RELPATH}: ${parsed.error}`,
|
|
650
|
-
diffSha256,
|
|
651
|
-
patchPath,
|
|
652
|
-
}
|
|
653
|
-
kills.push(kill)
|
|
654
|
-
log(`prefilter KILL gen ${generation} ${proposer.name}: ${kill.reason}`)
|
|
655
|
-
return { proposer, applied: false, summary: authored.summary, patch: null, kill }
|
|
656
|
-
}
|
|
657
|
-
}
|
|
658
|
-
|
|
659
|
-
// Pre-filter stage B — one smoke cell before any full-arm spend.
|
|
660
|
-
if (config.prefilter?.enabled && deps.smokeRunner) {
|
|
661
|
-
const smoke = await deps.smokeRunner({
|
|
662
|
-
scratchPath: scratch,
|
|
663
|
-
generation,
|
|
664
|
-
proposer,
|
|
665
|
-
evaluationKey: `candidate-${index}`,
|
|
666
|
-
...(args.costLedger ? { costLedger: args.costLedger } : {}),
|
|
667
|
-
})
|
|
668
|
-
if (!smoke.pass) {
|
|
669
|
-
const kill: PrefilterKill = {
|
|
670
|
-
generation,
|
|
671
|
-
candidateIndex: index,
|
|
672
|
-
proposer: proposer.name,
|
|
673
|
-
harness: proposer.harness,
|
|
674
|
-
stage: 'smoke',
|
|
675
|
-
reason: smoke.reason,
|
|
676
|
-
diffSha256,
|
|
677
|
-
patchPath,
|
|
678
|
-
smoke,
|
|
679
|
-
}
|
|
680
|
-
kills.push(kill)
|
|
681
|
-
log(`prefilter KILL gen ${generation} ${proposer.name}: ${kill.reason}`)
|
|
682
|
-
return { proposer, applied: false, summary: authored.summary, patch: null, kill }
|
|
683
|
-
}
|
|
684
|
-
log(`prefilter pass gen ${generation} ${proposer.name}: ${smoke.reason}`)
|
|
685
|
-
}
|
|
686
|
-
return { proposer, applied: true, summary: authored.summary, patch: diff, kill: null }
|
|
687
|
-
} finally {
|
|
688
|
-
await removeDetachedWorktree(config.loopsRepo, scratch)
|
|
689
|
-
}
|
|
690
|
-
}),
|
|
691
|
-
)
|
|
692
|
-
}
|
|
693
|
-
|
|
694
|
-
return {
|
|
695
|
-
kind: `gen3-fanout:${proposers.map((p) => `${p.name}@${p.engine ? `engine:${p.engine}` : p.harness}${p.model ? `:${p.model}` : ''}${p.merge ? ':merge' : ''}`).join('+')}`,
|
|
696
|
-
proposesWithoutFindings: true,
|
|
697
|
-
drainPrefilterKills() {
|
|
698
|
-
return kills.splice(0, kills.length)
|
|
699
|
-
},
|
|
700
|
-
async generate(args) {
|
|
701
|
-
const generation = args.generation ?? 0
|
|
702
|
-
const index = args.candidateIndex ?? 0
|
|
703
|
-
if (index >= proposers.length) {
|
|
704
|
-
throw new Error(
|
|
705
|
-
`fan-out: candidate index ${index} has no proposer (proposers=${proposers.length}) — ` +
|
|
706
|
-
'config.populationSize must equal config.proposers.length',
|
|
707
|
-
)
|
|
708
|
-
}
|
|
709
|
-
let pending = byGeneration.get(generation)
|
|
710
|
-
if (!pending) {
|
|
711
|
-
pending = runFanOut(generation, args)
|
|
712
|
-
byGeneration.set(generation, pending)
|
|
713
|
-
}
|
|
714
|
-
const outcomes = await pending
|
|
715
|
-
const outcome = outcomes[index]!
|
|
716
|
-
if (!outcome.applied || outcome.patch === null) return { applied: false, summary: outcome.summary }
|
|
717
|
-
|
|
718
|
-
// Apply the survivor's patch onto the driver-owned candidate worktree.
|
|
719
|
-
const patchFile = join(config.outDir, 'proposer-patches', `.apply-gen${generation}-cand${index}.patch`)
|
|
720
|
-
await writeFile(patchFile, outcome.patch)
|
|
721
|
-
await runOk('git', ['-C', args.worktreePath, 'apply', '--whitespace=nowarn', patchFile])
|
|
722
|
-
await rm(patchFile, { force: true })
|
|
723
|
-
await purgeIgnoredArtifacts(args.worktreePath)
|
|
724
|
-
return {
|
|
725
|
-
applied: true,
|
|
726
|
-
summary: outcome.summary,
|
|
727
|
-
label: outcome.proposer.name,
|
|
728
|
-
rationale:
|
|
729
|
-
`proposer ${outcome.proposer.name} (${outcome.proposer.engine ? `engine ${outcome.proposer.engine}` : outcome.proposer.harness}` +
|
|
730
|
-
`${outcome.proposer.profile ? `, profile ${outcome.proposer.profile}` : ''}` +
|
|
731
|
-
`${outcome.proposer.diagnosisSlice && outcome.proposer.diagnosisSlice !== 'all' ? `, slice ${outcome.proposer.diagnosisSlice}` : ''}` +
|
|
732
|
-
`): ${outcome.summary || 'authored change'}`,
|
|
733
|
-
}
|
|
734
|
-
},
|
|
735
|
-
}
|
|
736
|
-
}
|