dsh-continual-evolve 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +290 -0
- package/README.zh.md +240 -0
- package/cordis.patch.yml +9 -0
- package/lib/apply.d.ts +24 -0
- package/lib/apply.js +131 -0
- package/lib/approval.d.ts +14 -0
- package/lib/approval.js +27 -0
- package/lib/auto.d.ts +34 -0
- package/lib/auto.js +217 -0
- package/lib/benchmark.d.ts +72 -0
- package/lib/benchmark.js +167 -0
- package/lib/command.d.ts +36 -0
- package/lib/command.js +549 -0
- package/lib/evaluate.d.ts +38 -0
- package/lib/evaluate.js +142 -0
- package/lib/goal.d.ts +72 -0
- package/lib/goal.js +72 -0
- package/lib/index.d.ts +93 -0
- package/lib/index.js +116 -0
- package/lib/inject.d.ts +124 -0
- package/lib/inject.js +231 -0
- package/lib/logfile.d.ts +71 -0
- package/lib/logfile.js +159 -0
- package/lib/mount.d.ts +42 -0
- package/lib/mount.js +198 -0
- package/lib/notify.d.ts +31 -0
- package/lib/notify.js +42 -0
- package/lib/plan.d.ts +16 -0
- package/lib/plan.js +121 -0
- package/lib/planner.d.ts +30 -0
- package/lib/planner.js +110 -0
- package/lib/pool.d.ts +7 -0
- package/lib/pool.js +25 -0
- package/lib/render.d.ts +15 -0
- package/lib/render.js +83 -0
- package/lib/review.d.ts +37 -0
- package/lib/review.js +127 -0
- package/lib/rollback.d.ts +11 -0
- package/lib/rollback.js +69 -0
- package/lib/rubric.d.ts +29 -0
- package/lib/rubric.js +119 -0
- package/lib/score.d.ts +31 -0
- package/lib/score.js +81 -0
- package/lib/service.d.ts +30 -0
- package/lib/service.js +42 -0
- package/lib/skill.d.ts +10 -0
- package/lib/skill.js +75 -0
- package/lib/source.d.ts +29 -0
- package/lib/source.js +42 -0
- package/lib/state.d.ts +34 -0
- package/lib/state.js +154 -0
- package/lib/store.d.ts +20 -0
- package/lib/store.js +74 -0
- package/lib/tool.d.ts +15 -0
- package/lib/tool.js +163 -0
- package/lib/types.d.ts +137 -0
- package/lib/types.js +62 -0
- package/lib/validate.d.ts +11 -0
- package/lib/validate.js +55 -0
- package/package.json +67 -0
package/lib/render.js
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { SOURCE_SESSION_KEY, SOURCE_SEQS_KEY, isArchived } from "./types.js";
|
|
2
|
+
const DEFAULT_MAX_ENTRIES_PER_KIND = 6;
|
|
3
|
+
const DEFAULT_MAX_REFINEMENTS = 5;
|
|
4
|
+
const DEFAULT_MAX_CONTENT_LENGTH = 180;
|
|
5
|
+
export function compactText(text, maxLength) {
|
|
6
|
+
const normalized = text.replace(/\s+/g, " ").trim();
|
|
7
|
+
if (normalized.length <= maxLength) {
|
|
8
|
+
return normalized;
|
|
9
|
+
}
|
|
10
|
+
return `${normalized.slice(0, Math.max(0, maxLength - 3))}...`;
|
|
11
|
+
}
|
|
12
|
+
export function entryLine(entry, maxContentLength) {
|
|
13
|
+
const argumentsText = entry.kind === "skill" && Object.keys(entry.arguments).length > 0
|
|
14
|
+
? ` args=${compactText(JSON.stringify(entry.arguments), maxContentLength)}`
|
|
15
|
+
: "";
|
|
16
|
+
const referenceText = entry.kind === "skill" && Object.keys(entry.reference).length > 0
|
|
17
|
+
? ` ref=${compactText(JSON.stringify(entry.reference), maxContentLength)}`
|
|
18
|
+
: "";
|
|
19
|
+
const citationText = citationSuffix(entry);
|
|
20
|
+
const archivedText = isArchived(entry) ? " [archived]" : "";
|
|
21
|
+
return `- [${entry.scope}:${entry.id}] ${entry.title} (${entry.path}, v${entry.version})${archivedText}${referenceText}${argumentsText}${citationText}: ${compactText(entry.content, maxContentLength)}`;
|
|
22
|
+
}
|
|
23
|
+
/** Trajectory citation suffix (` src=sessionId:1,2`), empty when uncited. */
|
|
24
|
+
function citationSuffix(entry) {
|
|
25
|
+
const sessionId = entry.metadata[SOURCE_SESSION_KEY];
|
|
26
|
+
if (typeof sessionId !== "string" || sessionId.length === 0) {
|
|
27
|
+
return "";
|
|
28
|
+
}
|
|
29
|
+
const seqs = entry.metadata[SOURCE_SEQS_KEY];
|
|
30
|
+
const seqText = Array.isArray(seqs) && seqs.length > 0 ? `:${seqs.join(",")}` : "";
|
|
31
|
+
return ` src=${sessionId}${seqText}`;
|
|
32
|
+
}
|
|
33
|
+
/** Render the full merged state as a bounded overview for the system prompt. */
|
|
34
|
+
export function formatHarnessStateForPrompt(state) {
|
|
35
|
+
const lines = [
|
|
36
|
+
"# Continual Harness State",
|
|
37
|
+
"",
|
|
38
|
+
"Local entries belong to this session. Global entries persist across sessions.",
|
|
39
|
+
"The base system prompt is immutable; prompt entries are supplemental notes only.",
|
|
40
|
+
"",
|
|
41
|
+
];
|
|
42
|
+
let total = 0;
|
|
43
|
+
for (const kind of Object.keys(state.entries)) {
|
|
44
|
+
const entries = Object.values(state.entries[kind]).sort((a, b) => [a.path, a.title, a.id].join("\0").localeCompare([b.path, b.title, b.id].join("\0")));
|
|
45
|
+
total += entries.length;
|
|
46
|
+
lines.push(`${kind}: ${entries.length}`);
|
|
47
|
+
for (const entry of entries.slice(0, DEFAULT_MAX_ENTRIES_PER_KIND)) {
|
|
48
|
+
lines.push(entryLine(entry, DEFAULT_MAX_CONTENT_LENGTH));
|
|
49
|
+
}
|
|
50
|
+
const overflow = entries.length - Math.min(entries.length, DEFAULT_MAX_ENTRIES_PER_KIND);
|
|
51
|
+
if (overflow > 0) {
|
|
52
|
+
lines.push(`- +${overflow} more ${kind} entries`);
|
|
53
|
+
}
|
|
54
|
+
lines.push("");
|
|
55
|
+
}
|
|
56
|
+
if (total === 0) {
|
|
57
|
+
lines.push("No saved harness entries yet.", "");
|
|
58
|
+
}
|
|
59
|
+
lines.push(`recent refinements: ${state.refinements.length}`);
|
|
60
|
+
for (const event of state.refinements.slice(-DEFAULT_MAX_REFINEMENTS)) {
|
|
61
|
+
lines.push(`- [${event.id}] ${compactText(event.trigger, DEFAULT_MAX_CONTENT_LENGTH)}: ${event.changes.join(", ") || "no applied edits"}`);
|
|
62
|
+
}
|
|
63
|
+
return lines.join("\n").trim();
|
|
64
|
+
}
|
|
65
|
+
/** Render recent refinement results for the planner. */
|
|
66
|
+
export function historyForPrompt(history) {
|
|
67
|
+
if (history.length === 0) {
|
|
68
|
+
return "No prior refinement history.";
|
|
69
|
+
}
|
|
70
|
+
return history
|
|
71
|
+
.slice(-20)
|
|
72
|
+
.map((item) => {
|
|
73
|
+
const edits = item.appliedEdits.map((e) => `${e.applied ? "applied" : "failed"} ${e.action} ${e.kind}:${e.id}`).join(", ");
|
|
74
|
+
const rollback = item.rollbackOf ? ` rollbackOf=${item.rollbackOf}` : "";
|
|
75
|
+
return `[${item.id}]${rollback} ${item.summary}\n${edits}\nExpected outcome: ${item.expectedOutcome}`;
|
|
76
|
+
})
|
|
77
|
+
.join("\n\n");
|
|
78
|
+
}
|
|
79
|
+
/** Serialize a refinement event for persistence (lightweight). */
|
|
80
|
+
export function eventToLine(event) {
|
|
81
|
+
return JSON.stringify(event);
|
|
82
|
+
}
|
|
83
|
+
//# sourceMappingURL=render.js.map
|
package/lib/review.d.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The automatic /evolve review gate: a cheap model call that decides whether
|
|
3
|
+
* the current trajectory justifies running the planner. Runs on a turn
|
|
4
|
+
* interval (and, in a later step, at compaction). The gate is deliberately
|
|
5
|
+
* small (bounded input, small output budget) — it only answers
|
|
6
|
+
* "should we refine?", never "what should we edit?".
|
|
7
|
+
*/
|
|
8
|
+
import type { Context } from "@deepseek-ai/cordis";
|
|
9
|
+
import type { Agent } from "@deepseek-ai/dsh-agent";
|
|
10
|
+
import type { HarnessState, RefinementResult } from "./types.js";
|
|
11
|
+
export interface AutoRefineReview {
|
|
12
|
+
shouldRefine: boolean;
|
|
13
|
+
rationale: string;
|
|
14
|
+
instructions?: string;
|
|
15
|
+
}
|
|
16
|
+
export type AutoRefineReason = "turn_interval" | "compact";
|
|
17
|
+
export interface AutoRefineReviewContext {
|
|
18
|
+
reason: AutoRefineReason;
|
|
19
|
+
turnsSinceLastReview: number;
|
|
20
|
+
}
|
|
21
|
+
export interface ReviewOptions {
|
|
22
|
+
agent: Agent;
|
|
23
|
+
state: HarnessState;
|
|
24
|
+
history: readonly RefinementResult[];
|
|
25
|
+
context: AutoRefineReviewContext;
|
|
26
|
+
/** Serialized trajectory text; when absent the gate is skipped by the caller. */
|
|
27
|
+
trajectory?: string;
|
|
28
|
+
signal?: AbortSignal;
|
|
29
|
+
budgetTokens?: number;
|
|
30
|
+
}
|
|
31
|
+
export declare const AUTO_REVIEW_SYSTEM_PROMPT = "You are the automatic /evolve review gate.\n\nDecide whether this checkpoint should run /evolve. Auto /evolve writes local\nharness state by default, so approve when the trajectory contains evidence\nuseful to this session's future turns: a repeated failure, a reusable tactic,\na repeated delegation role, a durable fact or preference, a user correction\nthat should persist, or a narrow behavioral policy.\n\nThe current harness state below includes GLOBAL entries (scope=global) plus\nthis session's local entries (scope=local). When a topic is already covered\nby a global entry, do NOT approve a local duplicate of it \u2014 decline and say\nin the rationale that the topic is already covered globally.\n\nReject one-off noise, unsupported hypotheses, transient tool outputs, and\nrequests that carry no reusable content.\n\nStale local entries (superseded, long-unused, obsolete facts) are a valid\nrefine target: approve with instructions naming the entry ids, and tell the\nplanner to archive them (archive hides from injection, data stays restorable)\nrather than delete.\n\nReturn JSON only:\n{\n \"shouldRefine\": true|false,\n \"rationale\": \"short reason\",\n \"instructions\": \"optional concise instructions for /evolve if shouldRefine is true\"\n}";
|
|
32
|
+
/** Parse the gate's JSON reply. */
|
|
33
|
+
export declare function parseAutoRefineReview(text: string): AutoRefineReview;
|
|
34
|
+
/** Serialize surface events to bounded role-prefixed text. */
|
|
35
|
+
export declare function serializeSurface(events: readonly unknown[], maxChars: number): string;
|
|
36
|
+
export declare function reviewAutoRefine(ctx: Context, options: ReviewOptions): Promise<AutoRefineReview>;
|
|
37
|
+
//# sourceMappingURL=review.d.ts.map
|
package/lib/review.js
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { BlockAssembler, createUserMessage, ReasoningEffortId } from "@deepseek-ai/dsh-llm";
|
|
2
|
+
import { extractJsonObject } from "./plan.js";
|
|
3
|
+
import { formatHarnessStateForPrompt, historyForPrompt } from "./render.js";
|
|
4
|
+
export const AUTO_REVIEW_SYSTEM_PROMPT = `You are the automatic /evolve review gate.
|
|
5
|
+
|
|
6
|
+
Decide whether this checkpoint should run /evolve. Auto /evolve writes local
|
|
7
|
+
harness state by default, so approve when the trajectory contains evidence
|
|
8
|
+
useful to this session's future turns: a repeated failure, a reusable tactic,
|
|
9
|
+
a repeated delegation role, a durable fact or preference, a user correction
|
|
10
|
+
that should persist, or a narrow behavioral policy.
|
|
11
|
+
|
|
12
|
+
The current harness state below includes GLOBAL entries (scope=global) plus
|
|
13
|
+
this session's local entries (scope=local). When a topic is already covered
|
|
14
|
+
by a global entry, do NOT approve a local duplicate of it — decline and say
|
|
15
|
+
in the rationale that the topic is already covered globally.
|
|
16
|
+
|
|
17
|
+
Reject one-off noise, unsupported hypotheses, transient tool outputs, and
|
|
18
|
+
requests that carry no reusable content.
|
|
19
|
+
|
|
20
|
+
Stale local entries (superseded, long-unused, obsolete facts) are a valid
|
|
21
|
+
refine target: approve with instructions naming the entry ids, and tell the
|
|
22
|
+
planner to archive them (archive hides from injection, data stays restorable)
|
|
23
|
+
rather than delete.
|
|
24
|
+
|
|
25
|
+
Return JSON only:
|
|
26
|
+
{
|
|
27
|
+
"shouldRefine": true|false,
|
|
28
|
+
"rationale": "short reason",
|
|
29
|
+
"instructions": "optional concise instructions for /evolve if shouldRefine is true"
|
|
30
|
+
}`;
|
|
31
|
+
/** Parse the gate's JSON reply. */
|
|
32
|
+
export function parseAutoRefineReview(text) {
|
|
33
|
+
const value = extractJsonObject(text);
|
|
34
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
35
|
+
throw new Error("auto-refine review JSON must be an object");
|
|
36
|
+
}
|
|
37
|
+
const record = value;
|
|
38
|
+
const review = {
|
|
39
|
+
shouldRefine: record["shouldRefine"] === true,
|
|
40
|
+
rationale: typeof record["rationale"] === "string" ? record["rationale"] : "No rationale provided.",
|
|
41
|
+
};
|
|
42
|
+
if (typeof record["instructions"] === "string" && record["instructions"].length > 0) {
|
|
43
|
+
review.instructions = record["instructions"];
|
|
44
|
+
}
|
|
45
|
+
return review;
|
|
46
|
+
}
|
|
47
|
+
/** Serialize surface events to bounded role-prefixed text. */
|
|
48
|
+
export function serializeSurface(events, maxChars) {
|
|
49
|
+
const lines = [];
|
|
50
|
+
for (const raw of events) {
|
|
51
|
+
if (typeof raw !== "object" || raw === null)
|
|
52
|
+
continue;
|
|
53
|
+
const event = raw;
|
|
54
|
+
const role = event.type === "user/message" ? "user" : event.type === "assistant/message" ? "assistant" : null;
|
|
55
|
+
if (role === null)
|
|
56
|
+
continue;
|
|
57
|
+
const content = event.data?.content;
|
|
58
|
+
if (!Array.isArray(content))
|
|
59
|
+
continue;
|
|
60
|
+
const text = content
|
|
61
|
+
.filter((block) => typeof block === "object" && block !== null && block.type === "text")
|
|
62
|
+
.map((block) => block.text ?? "")
|
|
63
|
+
.filter(Boolean)
|
|
64
|
+
.join(" ");
|
|
65
|
+
if (text.length > 0) {
|
|
66
|
+
lines.push(`${role}: ${text}`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const joined = lines.join("\n");
|
|
70
|
+
return joined.length <= maxChars ? joined : joined.slice(-maxChars);
|
|
71
|
+
}
|
|
72
|
+
export async function reviewAutoRefine(ctx, options) {
|
|
73
|
+
const { agent, state, history } = options;
|
|
74
|
+
if (!agent.options.provider || !agent.options.model) {
|
|
75
|
+
throw new Error("evolve: no provider/model route for the review gate");
|
|
76
|
+
}
|
|
77
|
+
if (!options.trajectory || options.trajectory.length === 0) {
|
|
78
|
+
throw new Error("evolve: review gate has no trajectory to judge");
|
|
79
|
+
}
|
|
80
|
+
const userPrompt = [
|
|
81
|
+
`<trigger>\n${options.context.reason}; ${options.context.turnsSinceLastReview} turns since the last review\n</trigger>`,
|
|
82
|
+
`<current_harness_state>\n${formatHarnessStateForPrompt(state)}\n</current_harness_state>`,
|
|
83
|
+
`<refinement_history>\n${historyForPrompt(history)}\n</refinement_history>`,
|
|
84
|
+
`<conversation>\n${options.trajectory}\n</conversation>`,
|
|
85
|
+
"Return shouldRefine=true when the trajectory contains evidence useful to this session's future turns. Prefer local edits; do not ask for global refinement here.",
|
|
86
|
+
].join("\n\n");
|
|
87
|
+
const assembler = new BlockAssembler();
|
|
88
|
+
for await (const chunk of ctx.llm.stream({
|
|
89
|
+
provider: agent.options.provider,
|
|
90
|
+
model: agent.options.model,
|
|
91
|
+
system: AUTO_REVIEW_SYSTEM_PROMPT,
|
|
92
|
+
messages: [
|
|
93
|
+
createUserMessage({
|
|
94
|
+
content: [{ type: "text", text: userPrompt }],
|
|
95
|
+
source: { kind: "plugin", plugin: "dsh-continual-evolve" },
|
|
96
|
+
}),
|
|
97
|
+
],
|
|
98
|
+
// Force non-reasoning output so the model spends its budget on the JSON
|
|
99
|
+
// answer, not on visible thinking (reasoning models otherwise produce
|
|
100
|
+
// zero text blocks — the exact failure recorded in reviews.jsonl).
|
|
101
|
+
reasoningEffort: ReasoningEffortId("off"),
|
|
102
|
+
maxTokens: options.budgetTokens ?? 8000,
|
|
103
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
104
|
+
})) {
|
|
105
|
+
assembler.push(chunk);
|
|
106
|
+
}
|
|
107
|
+
const finish = assembler.finish;
|
|
108
|
+
if (finish.kind === "error") {
|
|
109
|
+
throw new Error(`evolve: review gate call failed: ${finish.failure?.message ?? "unknown"}`);
|
|
110
|
+
}
|
|
111
|
+
if (finish.kind === "aborted") {
|
|
112
|
+
throw new Error("evolve: review gate call aborted");
|
|
113
|
+
}
|
|
114
|
+
if (finish.kind === "max-tokens") {
|
|
115
|
+
throw new Error("evolve: review gate output budget exhausted (max-tokens)");
|
|
116
|
+
}
|
|
117
|
+
const text = assembler
|
|
118
|
+
.blocks()
|
|
119
|
+
.filter((block) => block.type === "text")
|
|
120
|
+
.map((block) => block.text)
|
|
121
|
+
.join("\n");
|
|
122
|
+
if (text.length === 0) {
|
|
123
|
+
throw new Error("evolve: review gate produced no text");
|
|
124
|
+
}
|
|
125
|
+
return parseAutoRefineReview(text);
|
|
126
|
+
}
|
|
127
|
+
//# sourceMappingURL=review.js.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic rollback: rebuild the inverse edit list from the applied
|
|
3
|
+
* result, in reverse order. Rollback is pure data transformation — no LLM
|
|
4
|
+
* is asked to "guess" the previous state.
|
|
5
|
+
*/
|
|
6
|
+
import type { HarnessEntry, RefinementProposal, RefinementResult } from "./types.js";
|
|
7
|
+
/** Build the inverse proposal for an applied refinement. */
|
|
8
|
+
export declare function rollbackProposal(target: RefinementResult): RefinementProposal;
|
|
9
|
+
/** Recreate an entry from a prior snapshot (used when an inverse edit is an update). */
|
|
10
|
+
export declare function restoreEntry(prior: HarnessEntry): HarnessEntry;
|
|
11
|
+
//# sourceMappingURL=rollback.d.ts.map
|
package/lib/rollback.js
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/** Build the inverse proposal for an applied refinement. */
|
|
2
|
+
export function rollbackProposal(target) {
|
|
3
|
+
const edits = [];
|
|
4
|
+
for (const edit of [...target.appliedEdits].reverse()) {
|
|
5
|
+
if (!edit.applied)
|
|
6
|
+
continue;
|
|
7
|
+
const inverse = inverseEdit(edit, target.id);
|
|
8
|
+
if (inverse) {
|
|
9
|
+
edits.push(inverse);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
return {
|
|
13
|
+
summary: `Rollback refinement ${target.id}`,
|
|
14
|
+
rationale: `Restores harness state to the snapshot recorded before refinement ${target.id}.`,
|
|
15
|
+
expectedOutcome: "Faulty refinement edits are reverted.",
|
|
16
|
+
edits,
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
function inverseEdit(edit, refinementId) {
|
|
20
|
+
if (edit.before && edit.after) {
|
|
21
|
+
// Forward action was an update: restore the before snapshot.
|
|
22
|
+
return {
|
|
23
|
+
action: "update",
|
|
24
|
+
kind: edit.kind,
|
|
25
|
+
id: edit.id,
|
|
26
|
+
title: edit.before.title,
|
|
27
|
+
content: edit.before.content,
|
|
28
|
+
path: edit.before.path,
|
|
29
|
+
reference: edit.before.reference,
|
|
30
|
+
arguments: edit.before.arguments,
|
|
31
|
+
metadata: edit.before.metadata,
|
|
32
|
+
reason: `Rollback ${refinementId}`,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
if (edit.before) {
|
|
36
|
+
// Forward action was a delete: re-create the entry from the snapshot.
|
|
37
|
+
return {
|
|
38
|
+
action: "create",
|
|
39
|
+
kind: edit.kind,
|
|
40
|
+
id: edit.id,
|
|
41
|
+
title: edit.before.title,
|
|
42
|
+
content: edit.before.content,
|
|
43
|
+
path: edit.before.path,
|
|
44
|
+
reference: edit.before.reference,
|
|
45
|
+
arguments: edit.before.arguments,
|
|
46
|
+
metadata: edit.before.metadata,
|
|
47
|
+
reason: `Rollback ${refinementId}`,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
if (edit.after) {
|
|
51
|
+
// Forward action was a create: delete the created entry.
|
|
52
|
+
return {
|
|
53
|
+
action: "delete",
|
|
54
|
+
kind: edit.kind,
|
|
55
|
+
id: edit.id,
|
|
56
|
+
reason: `Rollback ${refinementId}`,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
return undefined;
|
|
60
|
+
}
|
|
61
|
+
/** Recreate an entry from a prior snapshot (used when an inverse edit is an update). */
|
|
62
|
+
export function restoreEntry(prior) {
|
|
63
|
+
return {
|
|
64
|
+
...prior,
|
|
65
|
+
updated_at: new Date().toISOString(),
|
|
66
|
+
version: prior.version + 1,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=rollback.js.map
|
package/lib/rubric.d.ts
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/** The development fallback key — reachable only when the local key file is unusable. */
|
|
2
|
+
export declare const DEV_RUBRIC_KEY = "dsh-continual-evolve-dev-key";
|
|
3
|
+
/** Name of the per-installation key file under `<baseDir>/evolve/`. */
|
|
4
|
+
export declare const RUBRIC_KEY_FILE_NAME = "rubric.key";
|
|
5
|
+
/** Full path of the per-installation rubric key file. */
|
|
6
|
+
export declare function rubricKeyFilePath(baseDir: string): string;
|
|
7
|
+
export interface RubricCipher {
|
|
8
|
+
iv: string;
|
|
9
|
+
tag: string;
|
|
10
|
+
data: string;
|
|
11
|
+
}
|
|
12
|
+
export declare const RUBRIC_PREFIX = "v1:";
|
|
13
|
+
/** Derive a 32-byte AES-256 key from any passphrase. */
|
|
14
|
+
export declare function deriveKey(passphrase: string): Buffer;
|
|
15
|
+
/** Resolve the effective rubric key with the documented precedence. */
|
|
16
|
+
export declare function resolveRubricKey(baseDir: string, configKey: string | undefined, env?: NodeJS.ProcessEnv, warn?: (message: string) => void): Buffer;
|
|
17
|
+
/** Encrypt rubric plaintext into the `v1:` envelope (never written raw). */
|
|
18
|
+
export declare function encryptRubric(plaintext: string, key: Buffer): string;
|
|
19
|
+
/**
|
|
20
|
+
* Decrypt a `v1:` envelope. Legacy plaintext (no prefix) passes through
|
|
21
|
+
* unchanged so pre-ACL benchmark files keep working. Throws on tampered
|
|
22
|
+
* data, a wrong key, or a malformed envelope.
|
|
23
|
+
*/
|
|
24
|
+
export declare function decryptRubric(payload: string, key: Buffer): string;
|
|
25
|
+
/** True when a stored rubric is an encrypted envelope rather than legacy plaintext. */
|
|
26
|
+
export declare function isEncryptedRubric(payload: string): boolean;
|
|
27
|
+
/** Shape helper for tests and callers that inspect envelopes. */
|
|
28
|
+
export declare function parseEnvelope(payload: string): RubricCipher | undefined;
|
|
29
|
+
//# sourceMappingURL=rubric.d.ts.map
|
package/lib/rubric.js
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rubric ACL: rubric plaintext never touches the disk. The evaluation runner
|
|
3
|
+
* is the ONLY consumer that decrypts — the optimizer (planner) and the model
|
|
4
|
+
* (with its bash tools) can read benchmark files and see ciphertext only, so
|
|
5
|
+
* rubric isolation is enforced by code, not by prompt construction.
|
|
6
|
+
*
|
|
7
|
+
* Format: `v1:<base64(iv) | base64(tag) | base64(ciphertext)>` — each part is
|
|
8
|
+
* URL-safe base64 without padding, joined by `|`. A value without the `v1:`
|
|
9
|
+
* prefix is treated as legacy plaintext (pre-ACL files) and passes through.
|
|
10
|
+
*
|
|
11
|
+
* Key resolution (first match wins):
|
|
12
|
+
* 1. plugin config `rubricKey`
|
|
13
|
+
* 2. environment `DSH_EVOLVE_RUBRIC_KEY`
|
|
14
|
+
* 3. a per-installation local key file at `<baseDir>/evolve/rubric.key`
|
|
15
|
+
* (auto-generated with 0600 permissions on first use — every install
|
|
16
|
+
* gets its own random key, so no user setup is needed and no publicly
|
|
17
|
+
* known key protects anyone's rubrics)
|
|
18
|
+
* 4. a fixed development key as a last-resort fallback when the key file
|
|
19
|
+
* can neither be read nor written (warns; only reachable in
|
|
20
|
+
* pathological environments, since the plugin's stores live under the
|
|
21
|
+
* same directory)
|
|
22
|
+
* The key string is derived to 32 bytes with SHA-256, so any passphrase works.
|
|
23
|
+
*/
|
|
24
|
+
import { createCipheriv, createDecipheriv, createHash, randomBytes } from "node:crypto";
|
|
25
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
26
|
+
import { dirname, join } from "node:path";
|
|
27
|
+
/** The development fallback key — reachable only when the local key file is unusable. */
|
|
28
|
+
export const DEV_RUBRIC_KEY = "dsh-continual-evolve-dev-key";
|
|
29
|
+
/** Name of the per-installation key file under `<baseDir>/evolve/`. */
|
|
30
|
+
export const RUBRIC_KEY_FILE_NAME = "rubric.key";
|
|
31
|
+
/** Full path of the per-installation rubric key file. */
|
|
32
|
+
export function rubricKeyFilePath(baseDir) {
|
|
33
|
+
return join(baseDir, "evolve", RUBRIC_KEY_FILE_NAME);
|
|
34
|
+
}
|
|
35
|
+
export const RUBRIC_PREFIX = "v1:";
|
|
36
|
+
/** Derive a 32-byte AES-256 key from any passphrase. */
|
|
37
|
+
export function deriveKey(passphrase) {
|
|
38
|
+
return createHash("sha256").update(passphrase, "utf8").digest();
|
|
39
|
+
}
|
|
40
|
+
/** Resolve the effective rubric key with the documented precedence. */
|
|
41
|
+
export function resolveRubricKey(baseDir, configKey, env = process.env, warn) {
|
|
42
|
+
if (configKey && configKey.length > 0) {
|
|
43
|
+
return deriveKey(configKey);
|
|
44
|
+
}
|
|
45
|
+
const envKey = env["DSH_EVOLVE_RUBRIC_KEY"];
|
|
46
|
+
if (envKey && envKey.length > 0) {
|
|
47
|
+
return deriveKey(envKey);
|
|
48
|
+
}
|
|
49
|
+
return loadOrCreateLocalKey(baseDir, warn);
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Load the per-installation key file, generating a fresh random key (0600)
|
|
53
|
+
* on first use. Falls back to the development key with a warning when the
|
|
54
|
+
* file can neither be read nor written.
|
|
55
|
+
*/
|
|
56
|
+
function loadOrCreateLocalKey(baseDir, warn) {
|
|
57
|
+
const path = rubricKeyFilePath(baseDir);
|
|
58
|
+
try {
|
|
59
|
+
if (existsSync(path)) {
|
|
60
|
+
const content = readFileSync(path, "utf8").trim();
|
|
61
|
+
if (content.length > 0) {
|
|
62
|
+
return deriveKey(content);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
const key = randomBytes(32).toString("hex");
|
|
66
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
67
|
+
writeFileSync(path, `${key}\n`, { encoding: "utf8", mode: 0o600 });
|
|
68
|
+
return deriveKey(key);
|
|
69
|
+
}
|
|
70
|
+
catch (cause) {
|
|
71
|
+
warn?.(`rubric encryption: cannot read/write the local key file (${path}): ${cause instanceof Error ? cause.message : String(cause)} — using the development key`);
|
|
72
|
+
return deriveKey(DEV_RUBRIC_KEY);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/** Encrypt rubric plaintext into the `v1:` envelope (never written raw). */
|
|
76
|
+
export function encryptRubric(plaintext, key) {
|
|
77
|
+
const iv = randomBytes(12);
|
|
78
|
+
const cipher = createCipheriv("aes-256-gcm", key, iv);
|
|
79
|
+
const data = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]);
|
|
80
|
+
const tag = cipher.getAuthTag();
|
|
81
|
+
return `${RUBRIC_PREFIX}${[iv, tag, data].map((part) => part.toString("base64url")).join("|")}`;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Decrypt a `v1:` envelope. Legacy plaintext (no prefix) passes through
|
|
85
|
+
* unchanged so pre-ACL benchmark files keep working. Throws on tampered
|
|
86
|
+
* data, a wrong key, or a malformed envelope.
|
|
87
|
+
*/
|
|
88
|
+
export function decryptRubric(payload, key) {
|
|
89
|
+
if (!payload.startsWith(RUBRIC_PREFIX)) {
|
|
90
|
+
return payload; // legacy plaintext file
|
|
91
|
+
}
|
|
92
|
+
const raw = payload.slice(RUBRIC_PREFIX.length);
|
|
93
|
+
const parts = raw.split("|");
|
|
94
|
+
if (parts.length !== 3) {
|
|
95
|
+
throw new Error("rubric: malformed encrypted envelope");
|
|
96
|
+
}
|
|
97
|
+
const [ivText, tagText, dataText] = parts;
|
|
98
|
+
const iv = Buffer.from(ivText, "base64url");
|
|
99
|
+
const tag = Buffer.from(tagText, "base64url");
|
|
100
|
+
const data = Buffer.from(dataText, "base64url");
|
|
101
|
+
const decipher = createDecipheriv("aes-256-gcm", key, iv);
|
|
102
|
+
decipher.setAuthTag(tag);
|
|
103
|
+
return Buffer.concat([decipher.update(data), decipher.final()]).toString("utf8");
|
|
104
|
+
}
|
|
105
|
+
/** True when a stored rubric is an encrypted envelope rather than legacy plaintext. */
|
|
106
|
+
export function isEncryptedRubric(payload) {
|
|
107
|
+
return payload.startsWith(RUBRIC_PREFIX);
|
|
108
|
+
}
|
|
109
|
+
/** Shape helper for tests and callers that inspect envelopes. */
|
|
110
|
+
export function parseEnvelope(payload) {
|
|
111
|
+
if (!isEncryptedRubric(payload))
|
|
112
|
+
return undefined;
|
|
113
|
+
const parts = payload.slice(RUBRIC_PREFIX.length).split("|");
|
|
114
|
+
if (parts.length !== 3)
|
|
115
|
+
return undefined;
|
|
116
|
+
const [iv, tag, data] = parts;
|
|
117
|
+
return { iv, tag, data };
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=rubric.js.map
|
package/lib/score.d.ts
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Code-owned scoring: aggregation and acceptance decisions. The model only
|
|
3
|
+
* produces per-cell raw scores; every average and every accept/reject call
|
|
4
|
+
* happens here, in deterministic code.
|
|
5
|
+
*/
|
|
6
|
+
import type { CellScore, EvaluationEntry } from "./benchmark.js";
|
|
7
|
+
export interface AggregateOptions {
|
|
8
|
+
/** A cell is "passed" if its raw score is at least this threshold. */
|
|
9
|
+
passThreshold: number;
|
|
10
|
+
/** Per-case regression tolerance: candidate may drop below reference by at most this much. */
|
|
11
|
+
regressionTolerance: number;
|
|
12
|
+
}
|
|
13
|
+
export declare const DEFAULT_AGGREGATE: AggregateOptions;
|
|
14
|
+
/** Aggregate raw cells into code-owned per-case means + overall mean. */
|
|
15
|
+
export declare function aggregate(cells: readonly CellScore[]): Record<string, number | null> & {
|
|
16
|
+
overall: number | null;
|
|
17
|
+
};
|
|
18
|
+
export declare function entryFromCells(label: string, cells: readonly CellScore[], refinementId?: string): EvaluationEntry;
|
|
19
|
+
export interface Decision {
|
|
20
|
+
accepted: boolean;
|
|
21
|
+
reasons: string[];
|
|
22
|
+
}
|
|
23
|
+
/** Human-readable decision report with per-case before → after deltas. */
|
|
24
|
+
export declare function decisionReport(reference: EvaluationEntry, candidate: EvaluationEntry, decision: Decision): string[];
|
|
25
|
+
/**
|
|
26
|
+
* Non-regressive acceptance rule (Self-Harness style):
|
|
27
|
+
* the candidate is accepted iff its overall mean is STRICTLY higher than the
|
|
28
|
+
* reference AND no case regresses by more than `regressionTolerance` points.
|
|
29
|
+
*/
|
|
30
|
+
export declare function decide(reference: EvaluationEntry, candidate: EvaluationEntry, opts: AggregateOptions): Decision;
|
|
31
|
+
//# sourceMappingURL=score.d.ts.map
|
package/lib/score.js
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
export const DEFAULT_AGGREGATE = { passThreshold: 60, regressionTolerance: 0 };
|
|
2
|
+
/** Aggregate raw cells into code-owned per-case means + overall mean. */
|
|
3
|
+
export function aggregate(cells) {
|
|
4
|
+
const byCase = new Map();
|
|
5
|
+
for (const cell of cells) {
|
|
6
|
+
const list = byCase.get(cell.caseId) ?? [];
|
|
7
|
+
list.push(clampScore(cell.score));
|
|
8
|
+
byCase.set(cell.caseId, list);
|
|
9
|
+
}
|
|
10
|
+
const perCase = {};
|
|
11
|
+
for (const [caseId, scores] of byCase) {
|
|
12
|
+
perCase[caseId] = mean(scores);
|
|
13
|
+
}
|
|
14
|
+
const all = [...byCase.values()].flat();
|
|
15
|
+
return { ...perCase, overall: all.length > 0 ? mean(all) : null };
|
|
16
|
+
}
|
|
17
|
+
export function entryFromCells(label, cells, refinementId) {
|
|
18
|
+
const aggr = aggregate(cells);
|
|
19
|
+
return {
|
|
20
|
+
label,
|
|
21
|
+
...(refinementId ? { refinementId } : {}),
|
|
22
|
+
createdAt: new Date().toISOString(),
|
|
23
|
+
cells: [...cells],
|
|
24
|
+
aggregate: aggr,
|
|
25
|
+
overall: aggr.overall,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
/** Human-readable decision report with per-case before → after deltas. */
|
|
29
|
+
export function decisionReport(reference, candidate, decision) {
|
|
30
|
+
const lines = [`overall: ${reference.overall ?? "?"} → ${candidate.overall ?? "?"}`];
|
|
31
|
+
for (const [caseId, refScore] of Object.entries(reference.aggregate)) {
|
|
32
|
+
if (caseId === "overall" || refScore === null)
|
|
33
|
+
continue;
|
|
34
|
+
const candScore = candidate.aggregate[caseId];
|
|
35
|
+
lines.push(` ${caseId}: ${refScore} → ${candScore ?? "?"}`);
|
|
36
|
+
}
|
|
37
|
+
lines.push(decision.accepted
|
|
38
|
+
? "DECISION: ACCEPTED — overall improved, no regression"
|
|
39
|
+
: `DECISION: REJECTED — ${decision.reasons.join("; ")}`);
|
|
40
|
+
return lines;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Non-regressive acceptance rule (Self-Harness style):
|
|
44
|
+
* the candidate is accepted iff its overall mean is STRICTLY higher than the
|
|
45
|
+
* reference AND no case regresses by more than `regressionTolerance` points.
|
|
46
|
+
*/
|
|
47
|
+
export function decide(reference, candidate, opts) {
|
|
48
|
+
const reasons = [];
|
|
49
|
+
if (reference.overall === null || candidate.overall === null) {
|
|
50
|
+
return { accepted: false, reasons: ["reference or candidate evaluation is incomplete"] };
|
|
51
|
+
}
|
|
52
|
+
if (candidate.overall <= reference.overall) {
|
|
53
|
+
reasons.push(`overall not improved: ${candidate.overall} <= ${reference.overall}`);
|
|
54
|
+
}
|
|
55
|
+
for (const [caseId, refScore] of Object.entries(reference.aggregate)) {
|
|
56
|
+
if (caseId === "overall" || refScore === null)
|
|
57
|
+
continue;
|
|
58
|
+
const candScore = candidate.aggregate[caseId];
|
|
59
|
+
if (candScore === null || candScore === undefined) {
|
|
60
|
+
reasons.push(`candidate missing case ${caseId}`);
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
if (candScore < refScore - opts.regressionTolerance) {
|
|
64
|
+
reasons.push(`case ${caseId} regressed: ${candScore} < ${refScore} - ${opts.regressionTolerance}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return { accepted: reasons.length === 0, reasons };
|
|
68
|
+
}
|
|
69
|
+
function mean(values) {
|
|
70
|
+
const sum = values.reduce((acc, value) => acc + value, 0);
|
|
71
|
+
return round2(sum / values.length);
|
|
72
|
+
}
|
|
73
|
+
function clampScore(score) {
|
|
74
|
+
if (!Number.isFinite(score))
|
|
75
|
+
return 0;
|
|
76
|
+
return Math.min(100, Math.max(0, score));
|
|
77
|
+
}
|
|
78
|
+
function round2(value) {
|
|
79
|
+
return Math.round(value * 100) / 100;
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=score.js.map
|
package/lib/service.d.ts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The evolution engine: the only entry point that mutates harness state.
|
|
3
|
+
* Every mutation path goes through here so snapshot-before-write, apply
|
|
4
|
+
* accounting, persistence, and result history are enforced in one place.
|
|
5
|
+
*/
|
|
6
|
+
import type { EntrySource, HarnessScope, RefinementProposal, RefinementResult } from "./types.js";
|
|
7
|
+
import { applyRefinementProposal } from "./apply.js";
|
|
8
|
+
import { storePaths } from "./store.js";
|
|
9
|
+
export interface ApplyContext {
|
|
10
|
+
scope: HarnessScope;
|
|
11
|
+
sessionId?: string;
|
|
12
|
+
/** When set, optimistic-concurrency checks reject edits whose entries changed since this baseline. */
|
|
13
|
+
baselineState?: Parameters<typeof applyRefinementProposal>[0];
|
|
14
|
+
/** Trajectory citation stamped into newly created entries (see apply.ts). */
|
|
15
|
+
source?: EntrySource | undefined;
|
|
16
|
+
}
|
|
17
|
+
export interface EvolutionHooks {
|
|
18
|
+
/** Called after every applied refinement (side-effect boundary: skills sync, etc.). */
|
|
19
|
+
onApplied?: (result: RefinementResult) => void;
|
|
20
|
+
}
|
|
21
|
+
export declare function createEvolutionEngine(baseDir: string, hooks?: EvolutionHooks): {
|
|
22
|
+
load: (scope: HarnessScope, sessionId: string | undefined) => import("./types.js").HarnessState;
|
|
23
|
+
apply: (scope: HarnessScope, sessionId: string | undefined, proposal: RefinementProposal, context?: ApplyContext) => RefinementResult;
|
|
24
|
+
rollback: (scope: HarnessScope, sessionId: string | undefined, refinementId: string) => RefinementResult;
|
|
25
|
+
history: (scope: HarnessScope, sessionId: string | undefined) => RefinementResult[];
|
|
26
|
+
baseDir: string;
|
|
27
|
+
};
|
|
28
|
+
export type EvolutionEngine = ReturnType<typeof createEvolutionEngine>;
|
|
29
|
+
export { storePaths };
|
|
30
|
+
//# sourceMappingURL=service.d.ts.map
|
package/lib/service.js
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { applyRefinementProposal } from "./apply.js";
|
|
2
|
+
import { rollbackProposal } from "./rollback.js";
|
|
3
|
+
import { loadHarnessState, saveHarnessState } from "./state.js";
|
|
4
|
+
import { appendResult, loadResults, snapshotBefore, storePaths } from "./store.js";
|
|
5
|
+
export function createEvolutionEngine(baseDir, hooks = {}) {
|
|
6
|
+
function load(scope, sessionId) {
|
|
7
|
+
return loadHarnessState(storePaths(baseDir, scope, sessionId).stateDir, scope);
|
|
8
|
+
}
|
|
9
|
+
function apply(scope, sessionId, proposal, context) {
|
|
10
|
+
const paths = storePaths(baseDir, scope, sessionId);
|
|
11
|
+
const state = context?.baselineState ?? load(scope, sessionId);
|
|
12
|
+
const id = `evolve_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
|
|
13
|
+
// Code-enforced snapshot: runs before any mutation, cannot be skipped by the model.
|
|
14
|
+
snapshotBefore(paths, id);
|
|
15
|
+
const result = applyRefinementProposal(state, proposal, {
|
|
16
|
+
id,
|
|
17
|
+
scope,
|
|
18
|
+
...(context?.source ? { source: context.source } : {}),
|
|
19
|
+
...(context?.baselineState ? { baselineState: context.baselineState } : {}),
|
|
20
|
+
});
|
|
21
|
+
saveHarnessState(paths.stateDir, state);
|
|
22
|
+
appendResult(paths, result);
|
|
23
|
+
hooks.onApplied?.(result);
|
|
24
|
+
return result;
|
|
25
|
+
}
|
|
26
|
+
function rollback(scope, sessionId, refinementId) {
|
|
27
|
+
const paths = storePaths(baseDir, scope, sessionId);
|
|
28
|
+
const history = loadResults(paths);
|
|
29
|
+
const target = history.find((item) => item.id === refinementId);
|
|
30
|
+
if (!target) {
|
|
31
|
+
throw new Error(`Refinement ${refinementId} not found in ${scope} history`);
|
|
32
|
+
}
|
|
33
|
+
const proposal = rollbackProposal(target);
|
|
34
|
+
return apply(scope, sessionId, proposal);
|
|
35
|
+
}
|
|
36
|
+
function history(scope, sessionId) {
|
|
37
|
+
return loadResults(storePaths(baseDir, scope, sessionId));
|
|
38
|
+
}
|
|
39
|
+
return { load, apply, rollback, history, baseDir };
|
|
40
|
+
}
|
|
41
|
+
export { storePaths };
|
|
42
|
+
//# sourceMappingURL=service.js.map
|