dsh-continual-evolve 0.10.6 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. package/README.md +35 -122
  2. package/README.zh.md +26 -97
  3. package/cordis.patch.yml +2 -2
  4. package/lib/index.d.ts +44 -241
  5. package/lib/index.js +30 -253
  6. package/lib/memory-guide.d.ts +13 -14
  7. package/lib/memory-guide.js +32 -21
  8. package/lib/memory-section.d.ts +77 -0
  9. package/lib/memory-section.js +189 -0
  10. package/package.json +2 -17
  11. package/lib/apply.d.ts +0 -22
  12. package/lib/apply.js +0 -157
  13. package/lib/approval.d.ts +0 -72
  14. package/lib/approval.js +0 -127
  15. package/lib/auto.d.ts +0 -236
  16. package/lib/auto.js +0 -773
  17. package/lib/autocase.d.ts +0 -38
  18. package/lib/autocase.js +0 -76
  19. package/lib/benchmark-command.d.ts +0 -9
  20. package/lib/benchmark-command.js +0 -357
  21. package/lib/benchmark.d.ts +0 -151
  22. package/lib/benchmark.js +0 -250
  23. package/lib/command.d.ts +0 -49
  24. package/lib/command.js +0 -568
  25. package/lib/consolidate.d.ts +0 -90
  26. package/lib/consolidate.js +0 -180
  27. package/lib/copy.d.ts +0 -74
  28. package/lib/copy.js +0 -160
  29. package/lib/declines.d.ts +0 -101
  30. package/lib/declines.js +0 -175
  31. package/lib/evaluate.d.ts +0 -74
  32. package/lib/evaluate.js +0 -271
  33. package/lib/evolve-event.d.ts +0 -40
  34. package/lib/evolve-event.js +0 -59
  35. package/lib/failures.d.ts +0 -42
  36. package/lib/failures.js +0 -171
  37. package/lib/fate.d.ts +0 -130
  38. package/lib/fate.js +0 -350
  39. package/lib/goal-command.d.ts +0 -7
  40. package/lib/goal-command.js +0 -37
  41. package/lib/goal.d.ts +0 -72
  42. package/lib/goal.js +0 -72
  43. package/lib/inject.d.ts +0 -196
  44. package/lib/inject.js +0 -412
  45. package/lib/llm-text.d.ts +0 -104
  46. package/lib/llm-text.js +0 -140
  47. package/lib/logfile.d.ts +0 -71
  48. package/lib/logfile.js +0 -159
  49. package/lib/memory-agent.d.ts +0 -129
  50. package/lib/memory-agent.js +0 -568
  51. package/lib/memory-benchmark.d.ts +0 -76
  52. package/lib/memory-benchmark.js +0 -170
  53. package/lib/memory-index.d.ts +0 -114
  54. package/lib/memory-index.js +0 -197
  55. package/lib/message-source.d.ts +0 -23
  56. package/lib/message-source.js +0 -12
  57. package/lib/mount-command.d.ts +0 -10
  58. package/lib/mount-command.js +0 -48
  59. package/lib/mount.d.ts +0 -42
  60. package/lib/mount.js +0 -212
  61. package/lib/notify.d.ts +0 -59
  62. package/lib/notify.js +0 -97
  63. package/lib/plan.d.ts +0 -16
  64. package/lib/plan.js +0 -126
  65. package/lib/planner.d.ts +0 -61
  66. package/lib/planner.js +0 -144
  67. package/lib/pool.d.ts +0 -7
  68. package/lib/pool.js +0 -25
  69. package/lib/prefix-cache.d.ts +0 -78
  70. package/lib/prefix-cache.js +0 -168
  71. package/lib/project.d.ts +0 -28
  72. package/lib/project.js +0 -66
  73. package/lib/projection.d.ts +0 -22
  74. package/lib/projection.js +0 -112
  75. package/lib/promotion.d.ts +0 -94
  76. package/lib/promotion.js +0 -171
  77. package/lib/recall.d.ts +0 -85
  78. package/lib/recall.js +0 -142
  79. package/lib/record-language.d.ts +0 -85
  80. package/lib/record-language.js +0 -152
  81. package/lib/render.d.ts +0 -13
  82. package/lib/render.js +0 -80
  83. package/lib/review-scheduler.d.ts +0 -38
  84. package/lib/review-scheduler.js +0 -191
  85. package/lib/review.d.ts +0 -60
  86. package/lib/review.js +0 -145
  87. package/lib/rollback.d.ts +0 -9
  88. package/lib/rollback.js +0 -68
  89. package/lib/rubric.d.ts +0 -29
  90. package/lib/rubric.js +0 -119
  91. package/lib/runtime.d.ts +0 -24
  92. package/lib/runtime.js +0 -87
  93. package/lib/score.d.ts +0 -64
  94. package/lib/score.js +0 -191
  95. package/lib/search.d.ts +0 -83
  96. package/lib/search.js +0 -136
  97. package/lib/service.d.ts +0 -49
  98. package/lib/service.js +0 -180
  99. package/lib/skill-render.d.ts +0 -23
  100. package/lib/skill-render.js +0 -69
  101. package/lib/skill.d.ts +0 -15
  102. package/lib/skill.js +0 -80
  103. package/lib/skillquality.d.ts +0 -85
  104. package/lib/skillquality.js +0 -320
  105. package/lib/source.d.ts +0 -30
  106. package/lib/source.js +0 -43
  107. package/lib/state.d.ts +0 -26
  108. package/lib/state.js +0 -151
  109. package/lib/store.d.ts +0 -62
  110. package/lib/store.js +0 -164
  111. package/lib/token-usage.d.ts +0 -68
  112. package/lib/token-usage.js +0 -211
  113. package/lib/tool.d.ts +0 -34
  114. package/lib/tool.js +0 -320
  115. package/lib/turn-snapshot.d.ts +0 -72
  116. package/lib/turn-snapshot.js +0 -238
  117. package/lib/types.d.ts +0 -208
  118. package/lib/types.js +0 -127
  119. package/lib/usage.d.ts +0 -36
  120. package/lib/usage.js +0 -98
  121. package/lib/validate.d.ts +0 -28
  122. package/lib/validate.js +0 -181
  123. package/lib/wrapup-command.d.ts +0 -10
  124. package/lib/wrapup-command.js +0 -243
  125. package/lib/wrapup.d.ts +0 -260
  126. package/lib/wrapup.js +0 -539
package/lib/declines.js DELETED
@@ -1,175 +0,0 @@
1
- /**
2
- * Cross-session ledger of declined memory proposals (the decline loop).
3
- *
4
- * In-memory `GateState.memoryDecisions` dies on restart and its decision
5
- * cursor is per-checkpoint, so a declined project/global batch is re-asked
6
- * in the next session. This file persists batch fingerprints plus content
7
- * tokens under `<baseDir>/evolve/declined-memory.json`: exact batches are
8
- * suppressed at approval time, and checkpoints overlapping declined content
9
- * are suppressed before the agent is even invoked. Token matching is
10
- * containment (entry ⊆ checkpoint) with a conservative floor; rephrasing
11
- * below the floor still asks (documented limitation, not silent loss).
12
- */
13
- import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
14
- import { createHash } from "node:crypto";
15
- import { join } from "node:path";
16
- import { secretLeakReason } from "./promotion.js";
17
- import { tokenize } from "./search.js";
18
- import { EVOLVE_DIR } from "./store.js";
19
- /** Cap on remembered declines; oldest-first pruning keeps the file small. */
20
- export const MAX_DECLINED_MEMORY_LEDGER = 50;
21
- /** Cap on stored content tokens per entry; enough for coverage, never prose. */
22
- export const MAX_DECLINED_ENTRY_TOKENS = 128;
23
- /** Minimum coverage of an entry's tokens by the checkpoint to pre-suppress. */
24
- export const DECLINED_PRECHECK_COVERAGE = 0.85;
25
- /** Minimum overlapping tokens; tiny entries must never match vacuously. */
26
- export const DECLINED_PRECHECK_MIN_TOKENS = 5;
27
- const DECLINED_MEMORY_FILE = "declined-memory.json";
28
- /** Full path of the decline ledger file. */
29
- export function declinedMemoryPath(baseDir) {
30
- return join(baseDir, EVOLVE_DIR, DECLINED_MEMORY_FILE);
31
- }
32
- /**
33
- * Fingerprint one scope batch. Normalization is whitespace-only on purpose:
34
- * case folding could conflate distinct facts (e.g. env var names), and a
35
- * missed match only costs one more popup (fail-closed).
36
- *
37
- * @param scope - persistent scope the batch targets.
38
- * @param edits - the scope batch in proposal order (order is significant).
39
- * @returns 16-char hex fingerprint.
40
- */
41
- export function fingerprintMemoryBatch(scope, edits) {
42
- const parts = [scope];
43
- for (const edit of edits) {
44
- parts.push([edit.action, edit.id ?? "", edit.title ?? "", edit.content ?? ""].join("|"));
45
- }
46
- const material = parts.join("\n").replace(/\s+/g, " ").trim();
47
- return createHash("sha256").update(material, "utf8").digest("hex").slice(0, 16);
48
- }
49
- /**
50
- * Load the decline ledger. Missing or malformed files read as empty so a
51
- * broken ledger degrades to the status quo (ask anyway), never to a crash
52
- * on the approval path.
53
- *
54
- * @param baseDir - engine base directory.
55
- * @returns validated ledger entries (possibly empty).
56
- */
57
- export function loadDeclinedMemory(baseDir) {
58
- const path = declinedMemoryPath(baseDir);
59
- if (!existsSync(path))
60
- return [];
61
- try {
62
- const raw = JSON.parse(readFileSync(path, "utf8"));
63
- if (!Array.isArray(raw))
64
- return [];
65
- return raw
66
- .filter((entry) => {
67
- if (typeof entry !== "object" || entry === null)
68
- return false;
69
- const candidate = entry;
70
- return ((candidate.scope === "project" || candidate.scope === "global") &&
71
- typeof candidate.fingerprint === "string");
72
- })
73
- .map((entry) => ({
74
- ...entry,
75
- tokens: Array.isArray(entry.tokens) ? entry.tokens.filter((token) => typeof token === "string") : [],
76
- }));
77
- }
78
- catch {
79
- return [];
80
- }
81
- }
82
- /**
83
- * Whether this exact batch was declined before.
84
- *
85
- * @param entries - loaded ledger.
86
- * @param scope - persistent scope of the pending batch.
87
- * @param fingerprint - fingerprint of the pending batch.
88
- */
89
- export function isDeclinedRepeat(entries, scope, fingerprint) {
90
- return entries.some((entry) => entry.scope === scope && entry.fingerprint === fingerprint);
91
- }
92
- /**
93
- * Record one human decline, pruning oldest-first past the cap. The batch
94
- * edits supply fingerprint material, match tokens, and the secret screen:
95
- * a secret-shaped decline is still declined, but never lands in the ledger
96
- * (it was never engine-screened, so it must not be persisted anywhere).
97
- *
98
- * @param baseDir - engine base directory.
99
- * @param scope - persistent scope that was declined.
100
- * @param edits - the declined scope batch in proposal order.
101
- * @param title - first edit title, kept for audit readability.
102
- * @returns the ledger file path, or undefined when skipped by the secret screen.
103
- * @throws On filesystem write failure; callers on the approval path should
104
- * contain it (suppression is an optimization, never load-bearing).
105
- */
106
- export function recordDeclinedMemory(baseDir, scope, edits, title) {
107
- const screenable = edits.map((edit) => `${edit.title ?? ""}\n${edit.content ?? ""}`).join("\n");
108
- if (secretLeakReason(screenable) !== undefined)
109
- return undefined;
110
- const entries = loadDeclinedMemory(baseDir);
111
- entries.push({
112
- scope,
113
- fingerprint: fingerprintMemoryBatch(scope, edits),
114
- title,
115
- tokens: declinedContentTokens(edits),
116
- declinedAt: new Date().toISOString(),
117
- });
118
- entries.sort((left, right) => (left.declinedAt < right.declinedAt ? -1 : 1));
119
- while (entries.length > MAX_DECLINED_MEMORY_LEDGER)
120
- entries.shift();
121
- const path = declinedMemoryPath(baseDir);
122
- mkdirSync(join(baseDir, EVOLVE_DIR), { recursive: true });
123
- writeFileSync(path, `${JSON.stringify(entries, null, 2)}\n`, "utf8");
124
- return path;
125
- }
126
- /**
127
- * Deduplicated content tokens of one batch, capped. Shared CJK-aware
128
- * tokenizer with the manifest ranking, so both sides of the containment
129
- * check speak the same token language.
130
- *
131
- * @param edits - batch edits.
132
- * @returns at most MAX_DECLINED_ENTRY_TOKENS distinct tokens.
133
- */
134
- export function declinedContentTokens(edits) {
135
- const seen = new Set();
136
- for (const edit of edits) {
137
- for (const token of tokenize(`${edit.title ?? ""} ${edit.content ?? ""}`)) {
138
- seen.add(token);
139
- if (seen.size >= MAX_DECLINED_ENTRY_TOKENS)
140
- return [...seen];
141
- }
142
- }
143
- return [...seen];
144
- }
145
- /**
146
- * Match a pending checkpoint against declined content (entry ⊆ checkpoint).
147
- * Conservative by design: high coverage plus an absolute token floor, so
148
- * tiny or vague declines can never suppress anything.
149
- *
150
- * @param entries - loaded ledger.
151
- * @param checkpointTokens - tokens of the pending checkpoint trajectory.
152
- * @returns the strongest hit, or undefined when nothing clears the floor.
153
- */
154
- export function matchDeclinedCheckpoint(entries, checkpointTokens) {
155
- const pending = new Set(checkpointTokens);
156
- let best;
157
- for (const entry of entries) {
158
- if (entry.tokens.length === 0)
159
- continue;
160
- let hits = 0;
161
- for (const token of entry.tokens) {
162
- if (pending.has(token))
163
- hits += 1;
164
- }
165
- if (hits < DECLINED_PRECHECK_MIN_TOKENS)
166
- continue;
167
- const coverage = hits / entry.tokens.length;
168
- if (coverage < DECLINED_PRECHECK_COVERAGE)
169
- continue;
170
- if (best === undefined || coverage > best.coverage)
171
- best = { entry, coverage, hits };
172
- }
173
- return best;
174
- }
175
- //# sourceMappingURL=declines.js.map
package/lib/evaluate.d.ts DELETED
@@ -1,74 +0,0 @@
1
- /**
2
- * The evaluation matrix runner: executes every case × run as a TWO-STAGE
3
- * structured-output subagent pair, with the provider/model frozen to the
4
- * calling agent's own route.
5
- *
6
- * Stage 1 — executor: a fresh child performs the case task with its tools
7
- * and records CONCRETE EVIDENCE of what it did and found. It NEVER sees the
8
- * rubric (gap A1, evaluator/scorer separation): the agent under test cannot
9
- * optimize its behavior toward the grading criteria, and cannot grade its
10
- * own execution.
11
- *
12
- * Stage 2 — reviewer: an INDEPENDENT child grades the executor's evidence
13
- * strictly against the rubric. The rubric plaintext is decrypted in the host
14
- * and flows ONLY into this reviewer branch; the executor branch never
15
- * touches it. A separate model instance grading someone else's evidence
16
- * removes the self-serving bias of self-scoring.
17
- *
18
- * Failure-cell protocol (gap A2): a unit that cannot produce a score
19
- * (decrypt error, executor crash, reviewer crash, protocol error) returns a
20
- * cell with `status: "failed"` — NEVER a zero — so aggregation can exclude
21
- * it and the acceptance rule can reject rounds with too many failures instead
22
- * of silently averaging a 0 into the mean.
23
- *
24
- * Each cell carries the executor child's session id (gap A4): the score can
25
- * be drilled back to the exact session transcript that produced the evidence.
26
- *
27
- * Uses the host-plane `subagents` service (available in every profile) with
28
- * the native `outputSchema` structured-output seam: the provider validates
29
- * each child's reply against its cell schema, so the host never parses model
30
- * text for evaluations. (The workflow engine was rejected because the web
31
- * profile keeps it in a per-agent isolated realm a host plugin cannot
32
- * resolve.)
33
- */
34
- import type { Context } from "@deepseek-ai/cordis";
35
- import type { Agent } from "@deepseek-ai/dsh-agent";
36
- import type { BenchmarkCase, CellScore } from "./benchmark.js";
37
- export interface EvaluateOptions {
38
- cases: readonly BenchmarkCase[];
39
- runs: number;
40
- passThreshold: number;
41
- /** Serialized harness state under test (the candidate's guidance). */
42
- harnessOverview: string;
43
- label: string;
44
- /** AES-256 key for the encrypted rubric envelopes (see src/rubric.ts). */
45
- rubricKey?: Buffer;
46
- signal?: AbortSignal;
47
- }
48
- export interface EvaluationOutcome {
49
- label: string;
50
- cells: CellScore[];
51
- stopReason: string;
52
- }
53
- /** How many evaluation units may run concurrently (bounded subagent fan-out). */
54
- export declare const DEFAULT_EVALUATION_CONCURRENCY = 4;
55
- /** Evidence handed to the reviewer is capped so the grading call stays bounded. */
56
- export declare const MAX_EVIDENCE_CHARS = 8000;
57
- export declare function evaluateState(ctx: Context, agent: Agent, options: EvaluateOptions): Promise<EvaluationOutcome>;
58
- export interface ExecutorResult {
59
- caseId: string;
60
- run: number;
61
- evidence: string;
62
- }
63
- /**
64
- * Deterministic hash of a case's statement + rubric envelope (gap A3):
65
- * a 16-char SHA-256 prefix, hex-encoded. Used to detect material changes
66
- * between reference and candidate evaluation runs (see
67
- * `score.flagMaterialDrift`).
68
- */
69
- export declare function caseHash(caseItem: BenchmarkCase): string;
70
- /** Validate a provider-validated executor result; returns undefined when malformed. */
71
- export declare function normalizeExecutor(value: unknown, caseId: string, run: number): ExecutorResult | undefined;
72
- /** Validate a provider-validated structured cell; returns undefined when malformed. */
73
- export declare function normalizeCell(value: unknown, caseId: string, run: number, passThreshold: number): CellScore | undefined;
74
- //# sourceMappingURL=evaluate.d.ts.map
package/lib/evaluate.js DELETED
@@ -1,271 +0,0 @@
1
- import { createHash } from "node:crypto";
2
- import { mapPool } from "./pool.js";
3
- import { decryptRubric, deriveKey, DEV_RUBRIC_KEY } from "./rubric.js";
4
- /** Key used when the caller did not resolve one: mirrors resolveRubricKey's last-resort dev fallback. */
5
- function devRubricKey() {
6
- return deriveKey(DEV_RUBRIC_KEY);
7
- }
8
- /** Stage 1: the agent under test — sees the task, never the rubric. */
9
- const EXECUTOR_SYSTEM_PROMPT = `You are one evaluation unit in a benchmark matrix.
10
-
11
- You are the agent under evaluation. Perform the case task using your tools.
12
- You are NOT asked to grade yourself: a separate evaluator will judge your
13
- work against criteria you do not see. Instead, record CONCRETE, VERIFIABLE
14
- EVIDENCE of what you did and found — the actual commands/reads you performed,
15
- what the harness state contains, the exact text you produced. Evidence is
16
- what your work will be scored from; vague self-assessment earns nothing.
17
-
18
- The harness guidance attached to the state under test is included in the task.
19
- Your reply is structured (see the requested output schema): caseId, run, and
20
- evidence (the concrete artifact/report of your execution).`;
21
- /** Stage 2: the independent grader — sees the rubric, never executes the task. */
22
- const REVIEWER_SYSTEM_PROMPT = `You are an independent evaluator in a benchmark matrix.
23
-
24
- You did NOT perform the task and you cannot interact with the runtime: grade
25
- strictly from the EVIDENCE produced by the agent under evaluation, against
26
- the rubric. The harness guidance under test is included for context so you
27
- can judge whether the evidence genuinely reflects the state under test
28
- (e.g. whether the agent actually inspected the harness store).
29
-
30
- Score YOUR JUDGMENT of the evidence against the rubric criteria: a
31
- 0-100 score, and passed (true iff score >= the stated threshold). Cite in
32
- notes the concrete rubric criteria and the evidence that supports the score.
33
- Do not inflate: grade the evidence as presented, not what the agent might
34
- have meant.`;
35
- const EXECUTOR_SCHEMA = {
36
- type: "object",
37
- additionalProperties: false,
38
- properties: {
39
- caseId: { type: "string" },
40
- run: { type: "number" },
41
- evidence: { type: "string" },
42
- },
43
- required: ["caseId", "run", "evidence"],
44
- };
45
- const CELL_SCHEMA = {
46
- type: "object",
47
- additionalProperties: false,
48
- properties: {
49
- caseId: { type: "string" },
50
- run: { type: "number" },
51
- score: { type: "number" },
52
- passed: { type: "boolean" },
53
- notes: { type: "string" },
54
- },
55
- required: ["caseId", "run", "score", "passed", "notes"],
56
- };
57
- /** How many evaluation units may run concurrently (bounded subagent fan-out). */
58
- export const DEFAULT_EVALUATION_CONCURRENCY = 4;
59
- /** Evidence handed to the reviewer is capped so the grading call stays bounded. */
60
- export const MAX_EVIDENCE_CHARS = 8000;
61
- export async function evaluateState(ctx, agent, options) {
62
- if (!agent.options.provider || !agent.options.model) {
63
- throw new Error("evolve: benchmark evaluation requires a provider/model route");
64
- }
65
- const subagents = ctx.subagents;
66
- if (!subagents) {
67
- throw new Error("evolve: benchmark evaluation requires the subagents service");
68
- }
69
- const units = [];
70
- for (const c of options.cases) {
71
- for (let run = 1; run <= options.runs; run += 1) {
72
- units.push({ case: c, run });
73
- }
74
- }
75
- const cells = await mapPool(units, DEFAULT_EVALUATION_CONCURRENCY, (unit) => runUnit(subagents, agent, options, unit.case, unit.run));
76
- return { label: options.label, cells, stopReason: "completed" };
77
- }
78
- /** A cell that could not be produced: never a zero, excluded by aggregation. */
79
- function failedCell(caseId, run, message) {
80
- return { caseId, run, status: "failed", score: 0, passed: false, notes: message };
81
- }
82
- /**
83
- * Deterministic hash of a case's statement + rubric envelope (gap A3):
84
- * a 16-char SHA-256 prefix, hex-encoded. Used to detect material changes
85
- * between reference and candidate evaluation runs (see
86
- * `score.flagMaterialDrift`).
87
- */
88
- export function caseHash(caseItem) {
89
- const material = `${caseItem.statement}\n${caseItem.rubric}`;
90
- return createHash("sha256").update(material).digest("hex").slice(0, 16);
91
- }
92
- async function runUnit(subagents, agent, options, c, run) {
93
- // Gap C3: track wall-clock duration of the entire cell evaluation.
94
- const cellStart = Date.now();
95
- // The ONLY rubric decryption point: the envelope is opened here, in the
96
- // host, and the plaintext goes ONLY into the reviewer prompt — the
97
- // executor branch never touches it (gap A1).
98
- let rubric;
99
- try {
100
- rubric = decryptRubric(c.rubric, options.rubricKey ?? devRubricKey());
101
- }
102
- catch (cause) {
103
- return { ...failedCell(c.id, run, `rubric decrypt failed: ${cause instanceof Error ? cause.message : String(cause)}`), durationMs: Date.now() - cellStart };
104
- }
105
- // Runtime evidence (gap A3): record actual provider/model from the host,
106
- // and compute a material hash of the case for change detection.
107
- const actualProvider = agent.options.provider ?? "unknown";
108
- const actualModel = agent.options.model ?? "unknown";
109
- const hash = caseHash(c);
110
- // Stage 1: executor — task + evidence, NO rubric.
111
- let evidence;
112
- let sessionId;
113
- try {
114
- const executorRun = await subagents.start("spawn", {
115
- label: `${c.id} r${run} execute`,
116
- prompt: [
117
- {
118
- type: "text",
119
- text: [
120
- EXECUTOR_SYSTEM_PROMPT,
121
- "---",
122
- "Your harness guidance (state under test):",
123
- `<harness_overview>\n${options.harnessOverview}\n</harness_overview>`,
124
- `Case ${c.id} — task (statement):\n${c.statement}`,
125
- `Run ${run} of ${options.runs}.`,
126
- "Execute the task with your tools, then produce the structured evidence.",
127
- ].join("\n\n"),
128
- },
129
- ],
130
- parent: agent,
131
- signal: options.signal ?? new AbortController().signal,
132
- outputSchema: EXECUTOR_SCHEMA,
133
- });
134
- try {
135
- const settled = await executorRun.result;
136
- if (settled.stopReason !== "completed") {
137
- throw new Error(`executor stopped: ${settled.stopReason ?? "unknown"}`);
138
- }
139
- const parsed = normalizeExecutor(settled.structured, c.id, run) ?? fromEvidenceText(settled.output, c.id, run);
140
- if (!parsed) {
141
- throw new Error("executor returned neither a structured value nor usable text");
142
- }
143
- evidence = parsed;
144
- sessionId = executorRun.id || undefined;
145
- }
146
- finally {
147
- executorRun.dispose();
148
- }
149
- }
150
- catch (cause) {
151
- return { ...failedCell(c.id, run, `executor failed: ${cause instanceof Error ? cause.message : String(cause)}`), provider: actualProvider, model: actualModel, caseHash: hash, durationMs: Date.now() - cellStart };
152
- }
153
- // Stage 2: independent reviewer — rubric + evidence, NO task execution.
154
- try {
155
- const reviewerRun = await subagents.start("spawn", {
156
- label: `${c.id} r${run} grade`,
157
- prompt: [
158
- {
159
- type: "text",
160
- text: [
161
- REVIEWER_SYSTEM_PROMPT,
162
- "---",
163
- "Your harness guidance (state under test):",
164
- `<harness_overview>\n${options.harnessOverview}\n</harness_overview>`,
165
- `Case ${c.id} — task (statement):\n${c.statement}`,
166
- `Rubric — grade the evidence strictly against these criteria:\n${rubric}`,
167
- `Evidence produced by the agent under evaluation:\n<evidence>\n${trimEvidence(evidence.evidence)}\n</evidence>`,
168
- `Run ${run} of ${options.runs}. passThreshold = ${options.passThreshold}.`,
169
- "Produce the structured score.",
170
- ].join("\n\n"),
171
- },
172
- ],
173
- parent: agent,
174
- signal: options.signal ?? new AbortController().signal,
175
- outputSchema: CELL_SCHEMA,
176
- });
177
- try {
178
- const settled = await reviewerRun.result;
179
- if (settled.stopReason !== "completed") {
180
- throw new Error(`reviewer stopped: ${settled.stopReason ?? "unknown"}`);
181
- }
182
- const cell = normalizeCell(settled.structured, c.id, run, options.passThreshold) ??
183
- fromOutputText(settled.output, c.id, run, options.passThreshold);
184
- if (!cell) {
185
- throw new Error("reviewer returned neither a structured value nor usable text");
186
- }
187
- return { ...cell, ...(sessionId !== undefined ? { sessionId } : {}), provider: actualProvider, model: actualModel, caseHash: hash, durationMs: Date.now() - cellStart };
188
- }
189
- finally {
190
- reviewerRun.dispose();
191
- }
192
- }
193
- catch (cause) {
194
- return { ...failedCell(c.id, run, `reviewer failed: ${cause instanceof Error ? cause.message : String(cause)}`), provider: actualProvider, model: actualModel, caseHash: hash, durationMs: Date.now() - cellStart };
195
- }
196
- }
197
- /** Cap the evidence handed to the reviewer so the grading call stays bounded. */
198
- function trimEvidence(text) {
199
- if (text.length <= MAX_EVIDENCE_CHARS)
200
- return text;
201
- return `${text.slice(0, MAX_EVIDENCE_CHARS)}\n…[evidence truncated at ${MAX_EVIDENCE_CHARS} chars]`;
202
- }
203
- /** Validate a provider-validated executor result; returns undefined when malformed. */
204
- export function normalizeExecutor(value, caseId, run) {
205
- if (typeof value !== "object" || value === null || Array.isArray(value))
206
- return undefined;
207
- const record = value;
208
- const evidence = typeof record["evidence"] === "string" ? record["evidence"] : "";
209
- if (evidence.trim().length === 0)
210
- return undefined;
211
- return {
212
- caseId: typeof record["caseId"] === "string" && record["caseId"].length > 0 ? record["caseId"] : caseId,
213
- run: typeof record["run"] === "number" && Number.isFinite(record["run"]) ? Math.trunc(record["run"]) : run,
214
- evidence,
215
- };
216
- }
217
- /** Fallback: recover the executor result from its text blocks when no structured value arrived. */
218
- function fromEvidenceText(blocks, caseId, run) {
219
- if (!Array.isArray(blocks))
220
- return undefined;
221
- const text = blocks
222
- .filter((block) => block.type === "text")
223
- .map((block) => block.text ?? "")
224
- .join("\n");
225
- const trimmed = text.trim();
226
- if (trimmed.length === 0)
227
- return undefined;
228
- try {
229
- return normalizeExecutor(JSON.parse(trimmed), caseId, run);
230
- }
231
- catch {
232
- // Not JSON — keep the raw text as the evidence.
233
- return { caseId, run, evidence: trimmed };
234
- }
235
- }
236
- /** Validate a provider-validated structured cell; returns undefined when malformed. */
237
- export function normalizeCell(value, caseId, run, passThreshold) {
238
- if (typeof value !== "object" || value === null || Array.isArray(value))
239
- return undefined;
240
- const record = value;
241
- const score = Number(record["score"]);
242
- if (!Number.isFinite(score))
243
- return undefined;
244
- return {
245
- caseId: typeof record["caseId"] === "string" && record["caseId"].length > 0 ? record["caseId"] : caseId,
246
- run: typeof record["run"] === "number" && Number.isFinite(record["run"]) ? Math.trunc(record["run"]) : run,
247
- status: "ok",
248
- score: Math.min(100, Math.max(0, score)),
249
- passed: record["passed"] === true || score >= passThreshold,
250
- notes: typeof record["notes"] === "string" ? record["notes"] : "",
251
- };
252
- }
253
- /** Fallback: recover a cell from the child's text blocks when no structured value arrived. */
254
- function fromOutputText(blocks, caseId, run, passThreshold) {
255
- if (!Array.isArray(blocks))
256
- return undefined;
257
- const text = blocks
258
- .filter((block) => block.type === "text")
259
- .map((block) => block.text ?? "")
260
- .join("\n");
261
- if (text.length === 0)
262
- return undefined;
263
- try {
264
- const value = JSON.parse(text);
265
- return normalizeCell(value, caseId, run, passThreshold);
266
- }
267
- catch {
268
- return undefined;
269
- }
270
- }
271
- //# sourceMappingURL=evaluate.js.map
@@ -1,40 +0,0 @@
1
- import type { RefinementResult } from "./types.js";
2
- /** The structured event payload emitted after every successful refinement. */
3
- export interface EvolveCompleteEvent {
4
- /** Event discriminator for consumers. */
5
- type: "evolve_complete";
6
- /** The refinement id (same as the result). */
7
- refinementId: string;
8
- /** One-line summary of the refinement. */
9
- summary: string;
10
- /** Number of edits that were actually applied. */
11
- appliedEdits: number;
12
- /** Number of edits that failed to apply. */
13
- failedEdits: number;
14
- /** Scope of the refinement ("local" or "global"). */
15
- scope: string;
16
- /** What triggered this refinement (e.g. "auto_review", "manual_plan", "manual_tool"). */
17
- trigger: string;
18
- /** Session id that owns the refinement (auto or manual). */
19
- sessionId: string;
20
- /** ISO timestamp. */
21
- timestamp: string;
22
- /** Per-edit summaries (kind + id + action) for consumers that want detail. */
23
- edits: {
24
- action: string;
25
- kind: string;
26
- id: string;
27
- applied: boolean;
28
- }[];
29
- }
30
- /** Build a structured evolve-complete event from a refinement result. */
31
- export declare function buildEvolveCompleteEvent(result: RefinementResult, trigger: string, sessionId: string): EvolveCompleteEvent;
32
- /**
33
- * Emit an evolve_complete event to the reviews.jsonl audit trail. The event
34
- * is JSONL-formatted (one line) so consumers can tail and parse it. This is
35
- * a best-effort write — failure never blocks the refinement path. Storage
36
- * hygiene (#20): the trail is truncated to its tail after the append so the
37
- * audit file cannot grow without bound; readers only need a recent window.
38
- */
39
- export declare function emitEvolveComplete(baseDir: string, event: EvolveCompleteEvent, reviewsRetain?: number): void;
40
- //# sourceMappingURL=evolve-event.d.ts.map
@@ -1,59 +0,0 @@
1
- /**
2
- * Structured "evolve complete" events (gap C4): a durable, machine-readable
3
- * record emitted after every successful refinement application — whether
4
- * auto-gated or manual. Third-party consumers can observe these events via
5
- * the plugin log (JSONL) and the reviews.jsonl audit trail.
6
- *
7
- * Design: prime-agent `/refine` emits `refine_complete{id,summary,
8
- * appliedEdits,scope}` extension events. We follow the same pattern with
9
- * added provenance (trigger, source) so consumers know WHY the refinement
10
- * happened and WHO initiated it.
11
- */
12
- import { appendFileSync, mkdirSync } from "node:fs";
13
- import { join } from "node:path";
14
- import { DEFAULT_REVIEWS_RETAIN, pruneJsonlFile, reviewsPath } from "./store.js";
15
- /** Build a structured evolve-complete event from a refinement result. */
16
- export function buildEvolveCompleteEvent(result, trigger, sessionId) {
17
- return {
18
- type: "evolve_complete",
19
- refinementId: result.id,
20
- summary: result.summary,
21
- appliedEdits: result.appliedEdits.filter((e) => e.applied).length,
22
- failedEdits: result.appliedEdits.filter((e) => !e.applied).length,
23
- scope: result.scope ?? "local",
24
- trigger,
25
- sessionId,
26
- timestamp: new Date().toISOString(),
27
- edits: result.appliedEdits.map((e) => ({
28
- action: e.action,
29
- kind: e.kind,
30
- id: e.id,
31
- applied: e.applied,
32
- })),
33
- };
34
- }
35
- /**
36
- * Emit an evolve_complete event to the reviews.jsonl audit trail. The event
37
- * is JSONL-formatted (one line) so consumers can tail and parse it. This is
38
- * a best-effort write — failure never blocks the refinement path. Storage
39
- * hygiene (#20): the trail is truncated to its tail after the append so the
40
- * audit file cannot grow without bound; readers only need a recent window.
41
- */
42
- export function emitEvolveComplete(baseDir, event, reviewsRetain = DEFAULT_REVIEWS_RETAIN) {
43
- try {
44
- const dir = join(baseDir, "evolve");
45
- mkdirSync(dir, { recursive: true });
46
- appendFileSync(join(dir, "reviews.jsonl"), `${JSON.stringify(event)}\n`, "utf8");
47
- }
48
- catch {
49
- // Event emission is diagnostic; never interrupt the refinement path.
50
- return;
51
- }
52
- try {
53
- pruneJsonlFile(reviewsPath(baseDir), reviewsRetain);
54
- }
55
- catch {
56
- // ignored — the next emission retries
57
- }
58
- }
59
- //# sourceMappingURL=evolve-event.js.map
package/lib/failures.d.ts DELETED
@@ -1,42 +0,0 @@
1
- export interface FailureRecord {
2
- /** When the failure was recorded (ISO). Unknown for aggregated benchmark cells without timestamps. */
3
- timestamp?: string;
4
- /** Where the failure came from: "review-gate" | "benchmark:<bid>:<caseId>". */
5
- source: string;
6
- /** Failure class (see classifyFailure). */
7
- kind: string;
8
- /** The original failure text (notes or rationale). */
9
- message: string;
10
- }
11
- export interface FailureSummary {
12
- total: number;
13
- /** Count per failure class, sorted descending (most frequent first). */
14
- byKind: Record<string, number>;
15
- /** Count per source (gate vs benchmark:bid). */
16
- bySource: Record<string, number>;
17
- }
18
- /**
19
- * Classify a failure message by prefix rules. Deterministic and additive:
20
- * unknown text falls into "other" so the summary never drops a failure.
21
- */
22
- export declare function classifyFailure(message: string): string;
23
- /** Aggregate a record list into counts. Empty input yields an all-zero summary. */
24
- export declare function summarizeFailures(records: readonly FailureRecord[]): FailureSummary;
25
- /**
26
- * Read failed review-gate records from `<baseDir>/evolve/reviews.jsonl`
27
- * (outcome === "failed"). Tolerant of a missing/corrupt file.
28
- */
29
- export declare function readReviewFailures(baseDir: string): FailureRecord[];
30
- /**
31
- * Read failed cells from every benchmark scoreboard under
32
- * `<baseDir>/evolve/benchmarks/<bid>/`. Tolerant of missing/corrupt data.
33
- */
34
- export declare function readBenchmarkFailures(baseDir: string): FailureRecord[];
35
- /** Combine both sources into one summary. */
36
- export declare function collectFailureSummary(baseDir: string): {
37
- summary: FailureSummary;
38
- records: FailureRecord[];
39
- };
40
- /** Human-readable report for the command line. */
41
- export declare function formatFailureSummary(summary: FailureSummary): string;
42
- //# sourceMappingURL=failures.d.ts.map