@hexdecimal16/nexo 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +75 -0
- package/README.md +75 -2
- package/cli/src/bin.js +12 -0
- package/cli/src/common.js +77 -0
- package/cli/src/doctor.js +27 -0
- package/cli/src/eval-cases.js +65 -0
- package/cli/src/eval.js +67 -0
- package/cli/src/init.js +186 -0
- package/cli/src/interactive.js +171 -0
- package/cli/src/learn.js +178 -0
- package/cli/src/main.js +261 -0
- package/cli/src/optimize.js +289 -0
- package/cli/src/prompter.js +47 -0
- package/cli/src/review.js +120 -0
- package/cli/src/runtime.js +2 -0
- package/cli/src/sessions.js +34 -0
- package/cli/src/settings.js +169 -0
- package/cli/src/ui.js +112 -0
- package/core/src/approval/classifier.js +152 -0
- package/core/src/config/load.js +283 -0
- package/core/src/config/schema.js +186 -0
- package/core/src/cost/admission.js +90 -0
- package/core/src/cost/budget.js +67 -0
- package/core/src/cost/pricing.js +53 -0
- package/core/src/engine/charged-completion.js +79 -0
- package/core/src/engine/errors.js +77 -0
- package/core/src/engine/mcp.js +11 -0
- package/core/src/engine/messages.js +86 -0
- package/core/src/engine/prompts.js +32 -0
- package/core/src/engine/provider-gate.js +33 -0
- package/core/src/engine/recorder.js +61 -0
- package/core/src/engine/run.js +1115 -0
- package/core/src/engine/semaphore.js +45 -0
- package/core/src/engine/spawn-tool.js +119 -0
- package/core/src/engine/task-graph.js +153 -0
- package/core/src/index.js +32 -0
- package/core/src/learn/caller.js +171 -0
- package/core/src/learn/context.js +112 -0
- package/core/src/learn/embedder.js +216 -0
- package/core/src/learn/job.js +196 -0
- package/core/src/learn/wiki.js +116 -0
- package/core/src/learn/worker.js +141 -0
- package/core/src/optimize/adapter.js +250 -0
- package/core/src/optimize/climb.js +338 -0
- package/core/src/optimize/mine.js +95 -0
- package/core/src/optimize/optimize.js +377 -0
- package/core/src/optimize/optimizer-run.js +67 -0
- package/core/src/optimize/reflect.js +100 -0
- package/core/src/optimize/scratchpad-wiki.js +69 -0
- package/core/src/providers/claude-p.js +363 -0
- package/core/src/providers/kiro-p.js +408 -0
- package/core/src/providers/models.js +10 -0
- package/core/src/scoring/calibration.js +31 -0
- package/core/src/scoring/continuation.js +24 -0
- package/core/src/scoring/runner.js +322 -0
- package/core/src/scoring/sources.js +88 -0
- package/core/src/session/lock.js +88 -0
- package/core/src/session/session.js +647 -0
- package/core/src/session/shell.js +26 -0
- package/core/src/session/transcript.js +86 -0
- package/core/src/settings/config-edit.js +472 -0
- package/core/src/settings/credentials.js +120 -0
- package/core/src/settings/doctor.js +103 -0
- package/core/src/settings/settings.js +265 -0
- package/mcp/src/hub.js +184 -0
- package/mcp/src/index.js +1 -0
- package/memory/src/curate.js +121 -0
- package/memory/src/index.js +4 -0
- package/memory/src/prompts.js +143 -0
- package/memory/src/retrieval.js +91 -0
- package/memory/src/similarity.js +47 -0
- package/optimize/src/engine.js +206 -0
- package/optimize/src/index.js +8 -0
- package/optimize/src/merge.js +73 -0
- package/optimize/src/novelty.js +37 -0
- package/optimize/src/pareto.js +83 -0
- package/optimize/src/proposer.js +148 -0
- package/optimize/src/report.js +107 -0
- package/optimize/src/rng.js +34 -0
- package/optimize/src/target.js +34 -0
- package/package.json +47 -4
- package/router/src/effort.js +5 -0
- package/router/src/escalation.js +171 -0
- package/router/src/index.js +4 -0
- package/router/src/registry.js +21 -0
- package/router/src/resolve.js +117 -0
- package/scorers/src/deterministic.js +194 -0
- package/scorers/src/family.js +48 -0
- package/scorers/src/index.js +6 -0
- package/scorers/src/judge.js +284 -0
- package/scorers/src/regex-worker.js +3 -0
- package/scorers/src/spec.js +147 -0
- package/scorers/src/trajectory.js +22 -0
- package/scorers/src/types.js +1 -0
- package/sdk/src/index.js +3 -0
- package/sdk/src/nexo.js +94 -0
- package/sdk/src/runtime.js +39 -0
- package/skills/bundled/browser/SKILL.md +57 -0
- package/skills/bundled/wiki/SKILL.md +73 -0
- package/skills/src/frontmatter.js +25 -0
- package/skills/src/index.js +3 -0
- package/skills/src/load-skill-tool.js +41 -0
- package/skills/src/loader.js +95 -0
- package/store/src/artifacts.js +124 -0
- package/store/src/budget-ledger.js +18 -0
- package/store/src/cost-report.js +46 -0
- package/store/src/cursor.js +35 -0
- package/store/src/db.js +83 -0
- package/store/src/errors.js +9 -0
- package/store/src/eval-cases.js +255 -0
- package/store/src/feedback.js +154 -0
- package/store/src/goldens.js +61 -0
- package/store/src/ids.js +6 -0
- package/store/src/index.js +25 -0
- package/store/src/judge-calibration.js +182 -0
- package/store/src/learn-jobs.js +145 -0
- package/store/src/memories.js +261 -0
- package/store/src/migrations/001_init.sql +81 -0
- package/store/src/migrations/002_feedback.sql +9 -0
- package/store/src/migrations/003_sessions.sql +19 -0
- package/store/src/migrations/004_scoring.sql +22 -0
- package/store/src/migrations/005_judge_calibration.sql +27 -0
- package/store/src/migrations/006_learning.sql +77 -0
- package/store/src/migrations/007_optimize.sql +55 -0
- package/store/src/migrations/008_mined_cases_and_climb.sql +46 -0
- package/store/src/mine.js +51 -0
- package/store/src/optim.js +291 -0
- package/store/src/paths.js +22 -0
- package/store/src/playbooks.js +199 -0
- package/store/src/reindex.js +143 -0
- package/store/src/run-dir.js +151 -0
- package/store/src/runs.js +310 -0
- package/store/src/scores.js +92 -0
- package/store/src/sessions.js +304 -0
- package/store/src/span-tree.js +98 -0
- package/store/src/spans.js +84 -0
- package/store/src/sql.js +13 -0
- package/store/src/workspace-copy.js +80 -0
- package/telemetry/src/errors.js +32 -0
- package/telemetry/src/index.js +3 -0
- package/telemetry/src/logger.js +39 -0
- package/telemetry/src/redact.js +106 -0
- package/tools/src/builtin.js +269 -0
- package/tools/src/change-tracker.js +57 -0
- package/tools/src/confine.js +101 -0
- package/tools/src/index.js +7 -0
- package/tools/src/leases.js +57 -0
- package/tools/src/policy.js +162 -0
- package/tools/src/read-only.js +338 -0
- package/tools/src/session-rules.js +140 -0
- package/tui/src/app.js +948 -0
- package/tui/src/approvals.js +96 -0
- package/tui/src/commands.js +38 -0
- package/tui/src/components.js +248 -0
- package/tui/src/index.js +5 -0
- package/tui/src/picker.js +78 -0
- package/tui/src/session-picker.js +100 -0
- package/tui/src/settings-screen.js +346 -0
- package/tui/src/text.js +70 -0
- package/tui/src/theme.js +44 -0
- package/web/public/cost.html +25 -0
- package/web/public/fonts/GeistMono-400.woff2 +0 -0
- package/web/public/fonts/GeistMono-600.woff2 +0 -0
- package/web/public/fonts/InstrumentSans-400.woff2 +0 -0
- package/web/public/fonts/InstrumentSans-600.woff2 +0 -0
- package/web/public/index.html +39 -0
- package/web/public/judge.html +21 -0
- package/web/public/learning.html +46 -0
- package/web/public/lib/api-types.ts +616 -0
- package/web/public/lib/common-view.ts +47 -0
- package/web/public/lib/cost-view.ts +35 -0
- package/web/public/lib/files-view.ts +90 -0
- package/web/public/lib/format.ts +35 -0
- package/web/public/lib/html.ts +51 -0
- package/web/public/lib/judge-view.ts +62 -0
- package/web/public/lib/learning-view.ts +102 -0
- package/web/public/lib/optim-view.ts +148 -0
- package/web/public/lib/pager.ts +56 -0
- package/web/public/lib/poller.ts +79 -0
- package/web/public/lib/run-view.ts +61 -0
- package/web/public/lib/runs-view.ts +87 -0
- package/web/public/lib/score-view.ts +94 -0
- package/web/public/lib/sessions-view.ts +28 -0
- package/web/public/lib/span-view.ts +90 -0
- package/web/public/lib/theme.ts +60 -0
- package/web/public/lib/timeline.ts +97 -0
- package/web/public/lib/wikis-view.ts +257 -0
- package/web/public/optim.html +30 -0
- package/web/public/pages/api.ts +60 -0
- package/web/public/pages/cost.ts +36 -0
- package/web/public/pages/judge.ts +15 -0
- package/web/public/pages/learning.ts +92 -0
- package/web/public/pages/optim.ts +68 -0
- package/web/public/pages/paging.ts +18 -0
- package/web/public/pages/run.ts +221 -0
- package/web/public/pages/runs.ts +91 -0
- package/web/public/pages/sessions.ts +25 -0
- package/web/public/pages/wikis.ts +103 -0
- package/web/public/run.html +36 -0
- package/web/public/sessions.html +28 -0
- package/web/public/style.css +288 -0
- package/web/public/wikis.html +25 -0
- package/web/src/api.js +682 -0
- package/web/src/http-errors.js +60 -0
- package/web/src/index.js +5 -0
- package/web/src/server.js +292 -0
- package/web/src/static.js +86 -0
- package/web/src/strip.js +22 -0
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
import { NexoError } from "../../telemetry/src/index.js";
|
|
2
|
+
import { transaction } from "./db.js";
|
|
3
|
+
import { checkLimit, checkOffset, decodeCursor, pageOf } from "./cursor.js";
|
|
4
|
+
import { invalidRequest } from "./errors.js";
|
|
5
|
+
import { newId } from "./ids.js";
|
|
6
|
+
export const MEMORY_TYPES = ["fact", "preference", "lesson", "workflow"];
|
|
7
|
+
/** Longest memory content, in characters. */
|
|
8
|
+
export const MAX_MEMORY_CHARS = 1000;
|
|
9
|
+
/** Highest importance a curator-written (`ace`) memory may have; human and user memories may reach 1. */
|
|
10
|
+
export const MAX_ACE_IMPORTANCE = 0.7;
|
|
11
|
+
const SCOPE = /^(user|global|skill:[a-z0-9]+(-[a-z0-9]+)*)$/;
|
|
12
|
+
export function isMemoryScope(value) {
|
|
13
|
+
return typeof value === "string" && SCOPE.test(value);
|
|
14
|
+
}
|
|
15
|
+
export function isMemoryType(value) {
|
|
16
|
+
return typeof value === "string" && MEMORY_TYPES.includes(value);
|
|
17
|
+
}
|
|
18
|
+
function decode(row) {
|
|
19
|
+
const nullable = (v) => (v === null || v === undefined ? null : String(v));
|
|
20
|
+
return {
|
|
21
|
+
id: String(row.id),
|
|
22
|
+
scope: String(row.scope),
|
|
23
|
+
type: row.type,
|
|
24
|
+
content: String(row.content),
|
|
25
|
+
importance: Number(row.importance),
|
|
26
|
+
source: row.source,
|
|
27
|
+
sourceRunId: nullable(row.source_run_id),
|
|
28
|
+
supersededBy: nullable(row.superseded_by),
|
|
29
|
+
retiredAt: nullable(row.retired_at),
|
|
30
|
+
createdAt: String(row.created_at),
|
|
31
|
+
lastUsedAt: nullable(row.last_used_at),
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
/** Validates a new memory and returns it redacted, with `ace` importance clamped to MAX_ACE_IMPORTANCE. Throws INVALID_REQUEST. */
|
|
35
|
+
function prepare(input, redact) {
|
|
36
|
+
if (!isMemoryScope(input.scope))
|
|
37
|
+
throw invalidRequest(`scope must be user, global or skill:<name>`, { scope: input.scope });
|
|
38
|
+
if (!isMemoryType(input.type))
|
|
39
|
+
throw invalidRequest(`type must be one of ${MEMORY_TYPES.join(", ")}`, { type: input.type });
|
|
40
|
+
const content = typeof input.content === "string" ? redact(input.content.trim()) : "";
|
|
41
|
+
if (!content)
|
|
42
|
+
throw invalidRequest("memory content is empty");
|
|
43
|
+
if (content.length > MAX_MEMORY_CHARS)
|
|
44
|
+
throw invalidRequest(`memory content is longer than ${MAX_MEMORY_CHARS} characters`, { length: content.length });
|
|
45
|
+
if (!Number.isFinite(input.importance) || input.importance < 0 || input.importance > 1) {
|
|
46
|
+
throw invalidRequest("importance must be a number from 0 to 1", { importance: input.importance });
|
|
47
|
+
}
|
|
48
|
+
const importance = input.source === "ace" ? Math.min(input.importance, MAX_ACE_IMPORTANCE) : input.importance;
|
|
49
|
+
return { ...input, content, importance };
|
|
50
|
+
}
|
|
51
|
+
function insert(db, memory, nowIso) {
|
|
52
|
+
const row = {
|
|
53
|
+
id: newId(),
|
|
54
|
+
scope: memory.scope,
|
|
55
|
+
type: memory.type,
|
|
56
|
+
content: memory.content,
|
|
57
|
+
importance: memory.importance,
|
|
58
|
+
source: memory.source,
|
|
59
|
+
sourceRunId: memory.sourceRunId ?? null,
|
|
60
|
+
supersededBy: null,
|
|
61
|
+
retiredAt: null,
|
|
62
|
+
createdAt: nowIso,
|
|
63
|
+
lastUsedAt: null,
|
|
64
|
+
};
|
|
65
|
+
db.prepare("INSERT INTO memories (id, scope, type, content, importance, source, source_run_id, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)").run(row.id, row.scope, row.type, row.content, row.importance, row.source, row.sourceRunId, nowIso);
|
|
66
|
+
return row;
|
|
67
|
+
}
|
|
68
|
+
/** Stores a memory (redacted, validated, `ace` importance clamped). Throws INVALID_REQUEST. */
|
|
69
|
+
export function addMemory(db, input, redact, nowIso) {
|
|
70
|
+
return insert(db, prepare(input, redact), nowIso);
|
|
71
|
+
}
|
|
72
|
+
export function getMemory(db, id) {
|
|
73
|
+
const row = db.prepare("SELECT * FROM memories WHERE id = ?").get(id);
|
|
74
|
+
return row ? decode(row) : undefined;
|
|
75
|
+
}
|
|
76
|
+
function memoryNotFound(id) {
|
|
77
|
+
return new NexoError({ code: "NOT_FOUND", message: `Memory ${id} does not exist`, context: { id } });
|
|
78
|
+
}
|
|
79
|
+
/** The curator may change only what it wrote: human and user memories are changed by a person alone. */
|
|
80
|
+
function assertActorMayChange(actor, source, what, context) {
|
|
81
|
+
if (actor === "ace" && source !== "ace") {
|
|
82
|
+
throw new NexoError({
|
|
83
|
+
code: "FORBIDDEN",
|
|
84
|
+
message: `The curator cannot change ${what} written by a ${source === "human" ? "human verdict" : "user"}`,
|
|
85
|
+
hint: "Only a user action (the Learning page or the nexo CLI) can change it",
|
|
86
|
+
context,
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Replaces a live memory with a new one (Mem0 UPDATE): the new row is stored and the old one gets `superseded_by`;
|
|
92
|
+
* nothing is deleted. The replacement keeps the old scope, type and source unless given. Throws NOT_FOUND, CONFLICT
|
|
93
|
+
* (already superseded or retired), FORBIDDEN (the curator replacing a human or user memory) and INVALID_REQUEST.
|
|
94
|
+
*/
|
|
95
|
+
export function supersedeMemory(db, id, replacement, actor, redact, nowIso) {
|
|
96
|
+
return transaction(db, () => {
|
|
97
|
+
const old = getMemory(db, id);
|
|
98
|
+
if (!old)
|
|
99
|
+
throw memoryNotFound(id);
|
|
100
|
+
assertActorMayChange(actor, old.source, "a memory", { id, source: old.source });
|
|
101
|
+
if (old.supersededBy || old.retiredAt) {
|
|
102
|
+
throw new NexoError({ code: "CONFLICT", message: `Memory ${id} is no longer active`, context: { id } });
|
|
103
|
+
}
|
|
104
|
+
const next = insert(db, prepare({
|
|
105
|
+
scope: replacement.scope ?? old.scope,
|
|
106
|
+
type: replacement.type ?? old.type,
|
|
107
|
+
content: replacement.content,
|
|
108
|
+
importance: replacement.importance ?? old.importance,
|
|
109
|
+
source: actor === "ace" ? "ace" : (replacement.source ?? "user"),
|
|
110
|
+
sourceRunId: replacement.sourceRunId ?? null,
|
|
111
|
+
}, redact), nowIso);
|
|
112
|
+
db.prepare("UPDATE memories SET superseded_by = ? WHERE id = ?").run(next.id, id);
|
|
113
|
+
return next;
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
/** Retires a live memory (kept for history, never retrieved again). Throws NOT_FOUND, CONFLICT and FORBIDDEN as `supersedeMemory`. */
|
|
117
|
+
export function retireMemory(db, id, actor, nowIso) {
|
|
118
|
+
return transaction(db, () => {
|
|
119
|
+
const old = getMemory(db, id);
|
|
120
|
+
if (!old)
|
|
121
|
+
throw memoryNotFound(id);
|
|
122
|
+
assertActorMayChange(actor, old.source, "a memory", { id, source: old.source });
|
|
123
|
+
if (old.supersededBy || old.retiredAt) {
|
|
124
|
+
throw new NexoError({ code: "CONFLICT", message: `Memory ${id} is no longer active`, context: { id } });
|
|
125
|
+
}
|
|
126
|
+
db.prepare("UPDATE memories SET retired_at = ? WHERE id = ?").run(nowIso, id);
|
|
127
|
+
return { ...old, retiredAt: nowIso };
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
/** Memories by scope and type, newest first. Throws INVALID_REQUEST for a bad scope, type or limit. */
|
|
131
|
+
export function listMemories(db, filter = {}) {
|
|
132
|
+
return queryMemories(db, filter).memories;
|
|
133
|
+
}
|
|
134
|
+
/** One page of `listMemories`, newest first, with the cursor of the next page when there is one. */
|
|
135
|
+
export function queryMemories(db, filter = {}) {
|
|
136
|
+
if (filter.scope !== undefined && !isMemoryScope(filter.scope))
|
|
137
|
+
throw invalidRequest("scope must be user, global or skill:<name>", { scope: filter.scope });
|
|
138
|
+
if (filter.type !== undefined && !isMemoryType(filter.type))
|
|
139
|
+
throw invalidRequest(`type must be one of ${MEMORY_TYPES.join(", ")}`, { type: filter.type });
|
|
140
|
+
const limit = checkLimit(filter.limit ?? 200, 1000);
|
|
141
|
+
const where = [];
|
|
142
|
+
const args = [];
|
|
143
|
+
if (!filter.includeInactive)
|
|
144
|
+
where.push("superseded_by IS NULL AND retired_at IS NULL");
|
|
145
|
+
if (filter.scope !== undefined) {
|
|
146
|
+
where.push("scope = ?");
|
|
147
|
+
args.push(filter.scope);
|
|
148
|
+
}
|
|
149
|
+
if (filter.type !== undefined) {
|
|
150
|
+
where.push("type = ?");
|
|
151
|
+
args.push(filter.type);
|
|
152
|
+
}
|
|
153
|
+
const offset = checkOffset(filter.offset);
|
|
154
|
+
const total = Number(db.prepare(`SELECT COUNT(*) AS n FROM memories ${where.length ? `WHERE ${where.join(" AND ")}` : ""}`).get(...args).n);
|
|
155
|
+
if (filter.cursor !== undefined) {
|
|
156
|
+
const after = decodeCursor(filter.cursor);
|
|
157
|
+
where.push("(created_at < ? OR (created_at = ? AND id < ?))");
|
|
158
|
+
args.push(after.key, after.key, after.id);
|
|
159
|
+
}
|
|
160
|
+
const sql = `SELECT * FROM memories ${where.length ? `WHERE ${where.join(" AND ")}` : ""} ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?`;
|
|
161
|
+
const { items, nextCursor } = pageOf(db.prepare(sql).all(...args, limit + 1, offset).map(decode), limit, (m) => ({ key: m.createdAt, id: m.id }));
|
|
162
|
+
return { memories: items, nextCursor, total };
|
|
163
|
+
}
|
|
164
|
+
/** Live memories in any of `scopes` (retrieval candidates). */
|
|
165
|
+
export function liveMemories(db, scopes) {
|
|
166
|
+
if (scopes.length === 0)
|
|
167
|
+
return [];
|
|
168
|
+
const marks = scopes.map(() => "?").join(", ");
|
|
169
|
+
return db
|
|
170
|
+
.prepare(`SELECT * FROM memories WHERE scope IN (${marks}) AND superseded_by IS NULL AND retired_at IS NULL ORDER BY created_at, id`)
|
|
171
|
+
.all(...scopes)
|
|
172
|
+
.map(decode);
|
|
173
|
+
}
|
|
174
|
+
/** True when any of `scopes` has a live memory. */
|
|
175
|
+
export function hasLiveMemories(db, scopes) {
|
|
176
|
+
if (scopes.length === 0)
|
|
177
|
+
return false;
|
|
178
|
+
const marks = scopes.map(() => "?").join(", ");
|
|
179
|
+
return db.prepare(`SELECT 1 FROM memories WHERE scope IN (${marks}) AND superseded_by IS NULL AND retired_at IS NULL LIMIT 1`).get(...scopes) !== undefined;
|
|
180
|
+
}
|
|
181
|
+
/** The chain of memories this one replaced, newest first (its history). */
|
|
182
|
+
export function memoryHistory(db, id) {
|
|
183
|
+
const out = [];
|
|
184
|
+
let current = id;
|
|
185
|
+
for (let guard = 0; current && guard < 100; guard++) {
|
|
186
|
+
const prev = db.prepare("SELECT * FROM memories WHERE superseded_by = ?").get(current);
|
|
187
|
+
if (!prev)
|
|
188
|
+
break;
|
|
189
|
+
const row = decode(prev);
|
|
190
|
+
out.push(row);
|
|
191
|
+
current = row.id;
|
|
192
|
+
}
|
|
193
|
+
return out;
|
|
194
|
+
}
|
|
195
|
+
/** English function words left out of FTS queries: rare in short notes, they would outrank the words that matter. */
|
|
196
|
+
const STOPWORDS = new Set("a an and are as at be but by can could did do does for from had has have how i if in into is it its me my no not of on or our please should so than that the their them then there these they this to us was we were what when where which who why will with would you your".split(" "));
|
|
197
|
+
/**
|
|
198
|
+
* Words of `text` as an FTS5 query: unique lowercase letter/digit runs of two or more characters that are not
|
|
199
|
+
* STOPWORDS, each quoted, joined with OR. Quoting keeps FTS5 operators in the text from being parsed. Empty when the
|
|
200
|
+
* text has no such word.
|
|
201
|
+
*/
|
|
202
|
+
export function ftsQuery(text, maxTerms = 32) {
|
|
203
|
+
const words = [...new Set((text.toLowerCase().match(/[\p{L}\p{N}]{2,}/gu) ?? []))].filter((w) => !STOPWORDS.has(w)).slice(0, maxTerms);
|
|
204
|
+
return words.map((w) => `"${w}"`).join(" OR ");
|
|
205
|
+
}
|
|
206
|
+
/** BM25 matches of `text` among live memories in `scopes`: memory id → BM25 score (higher is more relevant, > 0). */
|
|
207
|
+
export function ftsScores(db, text, scopes) {
|
|
208
|
+
const query = ftsQuery(text);
|
|
209
|
+
if (!query || scopes.length === 0)
|
|
210
|
+
return new Map();
|
|
211
|
+
const marks = scopes.map(() => "?").join(", ");
|
|
212
|
+
const rows = db
|
|
213
|
+
.prepare(`SELECT m.id AS id, -bm25(memories_fts) AS score FROM memories_fts JOIN memories m ON m.rowid = memories_fts.rowid
|
|
214
|
+
WHERE memories_fts MATCH ? AND m.scope IN (${marks}) AND m.superseded_by IS NULL AND m.retired_at IS NULL`)
|
|
215
|
+
.all(query, ...scopes);
|
|
216
|
+
return new Map(rows.map((r) => [String(r.id), Math.max(Number(r.score), 0)]));
|
|
217
|
+
}
|
|
218
|
+
export function touchMemories(db, ids, nowIso) {
|
|
219
|
+
const stmt = db.prepare("UPDATE memories SET last_used_at = ? WHERE id = ?");
|
|
220
|
+
for (const id of ids)
|
|
221
|
+
stmt.run(nowIso, id);
|
|
222
|
+
}
|
|
223
|
+
/** Encodes a vector as float32 little-endian bytes. */
|
|
224
|
+
export function encodeVector(vector) {
|
|
225
|
+
const bytes = new Uint8Array(vector.length * 4);
|
|
226
|
+
const view = new DataView(bytes.buffer);
|
|
227
|
+
for (let i = 0; i < vector.length; i++)
|
|
228
|
+
view.setFloat32(i * 4, Number(vector[i]), true);
|
|
229
|
+
return bytes;
|
|
230
|
+
}
|
|
231
|
+
export function decodeVector(bytes) {
|
|
232
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
233
|
+
const out = new Float32Array(Math.floor(bytes.byteLength / 4));
|
|
234
|
+
for (let i = 0; i < out.length; i++)
|
|
235
|
+
out[i] = view.getFloat32(i * 4, true);
|
|
236
|
+
return out;
|
|
237
|
+
}
|
|
238
|
+
export function putEmbedding(db, memoryId, model, vector, nowIso) {
|
|
239
|
+
if (vector.length === 0)
|
|
240
|
+
throw invalidRequest("an embedding needs at least one dimension", { memoryId });
|
|
241
|
+
db.prepare("INSERT OR REPLACE INTO memory_embeddings (memory_id, model, dim, vector, created_at) VALUES (?, ?, ?, ?, ?)").run(memoryId, model, vector.length, encodeVector(vector), nowIso);
|
|
242
|
+
}
|
|
243
|
+
/** Vectors of the given memories made by `model`; memories without one (or with another model's) are absent. */
|
|
244
|
+
export function getEmbeddings(db, memoryIds, model) {
|
|
245
|
+
const out = new Map();
|
|
246
|
+
const stmt = db.prepare("SELECT vector FROM memory_embeddings WHERE memory_id = ? AND model = ?");
|
|
247
|
+
for (const id of memoryIds) {
|
|
248
|
+
const row = stmt.get(id, model);
|
|
249
|
+
if (row)
|
|
250
|
+
out.set(id, decodeVector(row.vector));
|
|
251
|
+
}
|
|
252
|
+
return out;
|
|
253
|
+
}
|
|
254
|
+
/** Live memories without an embedding from `model`, oldest first. */
|
|
255
|
+
export function memoriesWithoutEmbedding(db, model, limit) {
|
|
256
|
+
return db
|
|
257
|
+
.prepare(`SELECT m.* FROM memories m LEFT JOIN memory_embeddings e ON e.memory_id = m.id AND e.model = ?
|
|
258
|
+
WHERE e.memory_id IS NULL AND m.superseded_by IS NULL AND m.retired_at IS NULL ORDER BY m.created_at, m.id LIMIT ?`)
|
|
259
|
+
.all(model, limit)
|
|
260
|
+
.map(decode);
|
|
261
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
CREATE TABLE runs (
|
|
2
|
+
id TEXT PRIMARY KEY,
|
|
3
|
+
parent_run_id TEXT REFERENCES runs(id),
|
|
4
|
+
root_run_id TEXT NOT NULL,
|
|
5
|
+
dir_path TEXT NOT NULL,
|
|
6
|
+
instruction TEXT NOT NULL,
|
|
7
|
+
skill TEXT,
|
|
8
|
+
status TEXT NOT NULL CHECK (status IN ('running','succeeded','failed','aborted','budget_exceeded')),
|
|
9
|
+
config_snapshot_hash TEXT NOT NULL,
|
|
10
|
+
artifact_set_id TEXT,
|
|
11
|
+
model_final TEXT,
|
|
12
|
+
tier_final TEXT,
|
|
13
|
+
escalations INTEGER NOT NULL DEFAULT 0,
|
|
14
|
+
started_at TEXT NOT NULL,
|
|
15
|
+
ended_at TEXT,
|
|
16
|
+
cost_usd REAL NOT NULL DEFAULT 0,
|
|
17
|
+
cost_unknown INTEGER NOT NULL DEFAULT 0,
|
|
18
|
+
tokens_in INTEGER NOT NULL DEFAULT 0,
|
|
19
|
+
tokens_out INTEGER NOT NULL DEFAULT 0,
|
|
20
|
+
pid INTEGER,
|
|
21
|
+
error_json TEXT
|
|
22
|
+
);
|
|
23
|
+
CREATE INDEX runs_root ON runs(root_run_id);
|
|
24
|
+
CREATE INDEX runs_parent ON runs(parent_run_id);
|
|
25
|
+
CREATE INDEX runs_started ON runs(started_at);
|
|
26
|
+
CREATE INDEX runs_status ON runs(status);
|
|
27
|
+
|
|
28
|
+
CREATE TABLE spans (
|
|
29
|
+
id TEXT PRIMARY KEY,
|
|
30
|
+
run_id TEXT NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
|
|
31
|
+
parent_span_id TEXT,
|
|
32
|
+
kind TEXT NOT NULL,
|
|
33
|
+
name TEXT NOT NULL,
|
|
34
|
+
status TEXT NOT NULL CHECK (status IN ('running','ok','error')),
|
|
35
|
+
started_at TEXT NOT NULL,
|
|
36
|
+
ended_at TEXT,
|
|
37
|
+
queued_ms INTEGER,
|
|
38
|
+
model TEXT,
|
|
39
|
+
provider TEXT,
|
|
40
|
+
tier TEXT,
|
|
41
|
+
effort TEXT,
|
|
42
|
+
model_source TEXT,
|
|
43
|
+
tokens_in INTEGER NOT NULL DEFAULT 0,
|
|
44
|
+
tokens_out INTEGER NOT NULL DEFAULT 0,
|
|
45
|
+
cache_read INTEGER NOT NULL DEFAULT 0,
|
|
46
|
+
cache_write INTEGER NOT NULL DEFAULT 0,
|
|
47
|
+
cost_usd REAL NOT NULL DEFAULT 0,
|
|
48
|
+
cost_unknown INTEGER NOT NULL DEFAULT 0,
|
|
49
|
+
payload_ref TEXT,
|
|
50
|
+
attrs_json TEXT NOT NULL DEFAULT '{}'
|
|
51
|
+
);
|
|
52
|
+
CREATE INDEX spans_run ON spans(run_id);
|
|
53
|
+
CREATE INDEX spans_kind_started ON spans(kind, started_at);
|
|
54
|
+
|
|
55
|
+
CREATE TABLE artifacts (
|
|
56
|
+
id TEXT PRIMARY KEY,
|
|
57
|
+
kind TEXT NOT NULL CHECK (kind IN ('system_prompt','skill','playbook','router_policy','judge_rubric','planner_prompt')),
|
|
58
|
+
name TEXT NOT NULL,
|
|
59
|
+
version INTEGER NOT NULL,
|
|
60
|
+
content TEXT NOT NULL,
|
|
61
|
+
content_hash TEXT NOT NULL,
|
|
62
|
+
parent_version_id TEXT REFERENCES artifacts(id),
|
|
63
|
+
origin TEXT NOT NULL CHECK (origin IN ('manual','ace','gepa')),
|
|
64
|
+
status TEXT NOT NULL CHECK (status IN ('active','candidate','retired')),
|
|
65
|
+
created_at TEXT NOT NULL,
|
|
66
|
+
UNIQUE (kind, name, version)
|
|
67
|
+
);
|
|
68
|
+
CREATE UNIQUE INDEX artifacts_one_active ON artifacts(kind, name) WHERE status = 'active';
|
|
69
|
+
|
|
70
|
+
CREATE TABLE artifact_sets (
|
|
71
|
+
id TEXT PRIMARY KEY,
|
|
72
|
+
members_json TEXT NOT NULL,
|
|
73
|
+
created_at TEXT NOT NULL
|
|
74
|
+
);
|
|
75
|
+
|
|
76
|
+
CREATE TABLE budget_ledger (
|
|
77
|
+
scope TEXT PRIMARY KEY,
|
|
78
|
+
reserved_usd REAL NOT NULL DEFAULT 0,
|
|
79
|
+
spent_usd REAL NOT NULL DEFAULT 0,
|
|
80
|
+
updated_at TEXT NOT NULL
|
|
81
|
+
);
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
-- run_id has no foreign key: reindex deletes and rebuilds the runs table, and human feedback must survive that.
|
|
2
|
+
CREATE TABLE feedback (
|
|
3
|
+
id TEXT PRIMARY KEY,
|
|
4
|
+
run_id TEXT NOT NULL,
|
|
5
|
+
verdict TEXT NOT NULL CHECK (verdict IN ('done','not_done')),
|
|
6
|
+
comment TEXT NOT NULL DEFAULT '',
|
|
7
|
+
created_at TEXT NOT NULL
|
|
8
|
+
);
|
|
9
|
+
CREATE INDEX feedback_run ON feedback(run_id, created_at);
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
-- Interactive sessions. A session's turns are root runs linked by session_id and turn_index; session_id has no
|
|
2
|
+
-- foreign key because reindex rebuilds both tables from their directories.
|
|
3
|
+
CREATE TABLE sessions (
|
|
4
|
+
id TEXT PRIMARY KEY,
|
|
5
|
+
title TEXT NOT NULL DEFAULT '',
|
|
6
|
+
cwd TEXT NOT NULL,
|
|
7
|
+
status TEXT NOT NULL CHECK (status IN ('active','closed')),
|
|
8
|
+
budget_usd REAL NOT NULL,
|
|
9
|
+
cost_usd REAL NOT NULL DEFAULT 0,
|
|
10
|
+
turns INTEGER NOT NULL DEFAULT 0,
|
|
11
|
+
created_at TEXT NOT NULL,
|
|
12
|
+
updated_at TEXT NOT NULL,
|
|
13
|
+
dir_path TEXT NOT NULL
|
|
14
|
+
);
|
|
15
|
+
CREATE INDEX sessions_updated ON sessions(updated_at);
|
|
16
|
+
|
|
17
|
+
ALTER TABLE runs ADD COLUMN session_id TEXT;
|
|
18
|
+
ALTER TABLE runs ADD COLUMN turn_index INTEGER;
|
|
19
|
+
CREATE INDEX runs_session ON runs(session_id, turn_index);
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
-- Scores of a run's scorers, one row per scorer per scoring round (round 0 is the first scoring, each S2 continuation
|
|
2
|
+
-- adds a round). run_id has no foreign key: reindex rebuilds the runs table and restores scores from run.json.
|
|
3
|
+
CREATE TABLE scores (
|
|
4
|
+
id TEXT PRIMARY KEY,
|
|
5
|
+
run_id TEXT NOT NULL,
|
|
6
|
+
round INTEGER NOT NULL DEFAULT 0,
|
|
7
|
+
scorer_kind TEXT NOT NULL CHECK (scorer_kind IN ('deterministic','judge','human')),
|
|
8
|
+
scorer_type TEXT NOT NULL,
|
|
9
|
+
scorer_name TEXT NOT NULL,
|
|
10
|
+
status TEXT NOT NULL CHECK (status IN ('ok','failed')),
|
|
11
|
+
value REAL,
|
|
12
|
+
passed INTEGER,
|
|
13
|
+
rationale TEXT NOT NULL DEFAULT '',
|
|
14
|
+
criteria_json TEXT,
|
|
15
|
+
details_json TEXT NOT NULL DEFAULT '{}',
|
|
16
|
+
error_json TEXT,
|
|
17
|
+
model TEXT,
|
|
18
|
+
cost_usd REAL NOT NULL DEFAULT 0,
|
|
19
|
+
cost_unknown INTEGER NOT NULL DEFAULT 0,
|
|
20
|
+
created_at TEXT NOT NULL
|
|
21
|
+
);
|
|
22
|
+
CREATE INDEX scores_run ON scores(run_id, round);
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
-- Eval cases (spec §5.2). Phase 3 adds one per run whose judge verdict disagrees with the human verdict, with target
|
|
2
|
+
-- 'judge_rubric'; the optimizer reads the same table later. source_run_id has no foreign key, like feedback.run_id.
|
|
3
|
+
CREATE TABLE eval_cases (
|
|
4
|
+
id TEXT PRIMARY KEY,
|
|
5
|
+
target TEXT NOT NULL,
|
|
6
|
+
source_run_id TEXT,
|
|
7
|
+
input TEXT NOT NULL,
|
|
8
|
+
expected TEXT,
|
|
9
|
+
reference_feedback TEXT,
|
|
10
|
+
scorer_spec_json TEXT,
|
|
11
|
+
split TEXT NOT NULL CHECK (split IN ('feedback','pareto','holdout')),
|
|
12
|
+
created_at TEXT NOT NULL
|
|
13
|
+
);
|
|
14
|
+
CREATE UNIQUE INDEX eval_cases_source ON eval_cases(target, source_run_id);
|
|
15
|
+
|
|
16
|
+
-- Results of `nexo eval judge`: judge verdicts compared with human verdicts.
|
|
17
|
+
CREATE TABLE judge_calibrations (
|
|
18
|
+
id TEXT PRIMARY KEY,
|
|
19
|
+
judge_key TEXT NOT NULL,
|
|
20
|
+
cases INTEGER NOT NULL,
|
|
21
|
+
agreements INTEGER NOT NULL,
|
|
22
|
+
accuracy REAL,
|
|
23
|
+
kappa REAL,
|
|
24
|
+
matrix_json TEXT NOT NULL,
|
|
25
|
+
created_at TEXT NOT NULL
|
|
26
|
+
);
|
|
27
|
+
CREATE INDEX judge_calibrations_created ON judge_calibrations(created_at);
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
-- Learning state (spec §5.2, §8): memories with FTS and embeddings, playbook items with their history, learn jobs.
|
|
2
|
+
-- run ids have no foreign key, like scores and feedback: reindex rebuilds the runs table and keeps these rows.
|
|
3
|
+
CREATE TABLE memories (
|
|
4
|
+
id TEXT PRIMARY KEY,
|
|
5
|
+
scope TEXT NOT NULL CHECK (scope IN ('user', 'global') OR (scope GLOB 'skill:?*')),
|
|
6
|
+
type TEXT NOT NULL CHECK (type IN ('fact', 'preference', 'lesson', 'workflow')),
|
|
7
|
+
content TEXT NOT NULL,
|
|
8
|
+
importance REAL NOT NULL CHECK (importance >= 0 AND importance <= 1),
|
|
9
|
+
source TEXT NOT NULL CHECK (source IN ('ace', 'human', 'user')),
|
|
10
|
+
source_run_id TEXT,
|
|
11
|
+
superseded_by TEXT REFERENCES memories(id),
|
|
12
|
+
retired_at TEXT,
|
|
13
|
+
created_at TEXT NOT NULL,
|
|
14
|
+
last_used_at TEXT
|
|
15
|
+
);
|
|
16
|
+
CREATE INDEX memories_live ON memories(scope) WHERE superseded_by IS NULL AND retired_at IS NULL;
|
|
17
|
+
CREATE INDEX memories_source_run ON memories(source_run_id);
|
|
18
|
+
|
|
19
|
+
-- Full-text index over memories.content; its rowid is the memories rowid. Content never changes after insert
|
|
20
|
+
-- (a replacement is a new row), so only inserts are mirrored.
|
|
21
|
+
CREATE VIRTUAL TABLE memories_fts USING fts5(content, tokenize = 'porter unicode61');
|
|
22
|
+
CREATE TRIGGER memories_fts_insert AFTER INSERT ON memories BEGIN
|
|
23
|
+
INSERT INTO memories_fts (rowid, content) VALUES (new.rowid, new.content);
|
|
24
|
+
END;
|
|
25
|
+
|
|
26
|
+
-- One embedding per memory: float32 little-endian, `dim` values, from `model` (`provider/id`).
|
|
27
|
+
CREATE TABLE memory_embeddings (
|
|
28
|
+
memory_id TEXT PRIMARY KEY REFERENCES memories(id),
|
|
29
|
+
model TEXT NOT NULL,
|
|
30
|
+
dim INTEGER NOT NULL CHECK (dim > 0),
|
|
31
|
+
vector BLOB NOT NULL,
|
|
32
|
+
created_at TEXT NOT NULL
|
|
33
|
+
);
|
|
34
|
+
|
|
35
|
+
CREATE TABLE playbook_items (
|
|
36
|
+
id TEXT PRIMARY KEY,
|
|
37
|
+
artifact_name TEXT NOT NULL,
|
|
38
|
+
item_key TEXT NOT NULL,
|
|
39
|
+
content TEXT NOT NULL,
|
|
40
|
+
helpful INTEGER NOT NULL DEFAULT 0 CHECK (helpful >= 0),
|
|
41
|
+
harmful INTEGER NOT NULL DEFAULT 0 CHECK (harmful >= 0),
|
|
42
|
+
source TEXT NOT NULL CHECK (source IN ('ace', 'human')),
|
|
43
|
+
status TEXT NOT NULL CHECK (status IN ('active', 'retired')),
|
|
44
|
+
source_run_id TEXT,
|
|
45
|
+
created_at TEXT NOT NULL,
|
|
46
|
+
updated_at TEXT NOT NULL,
|
|
47
|
+
UNIQUE (artifact_name, item_key)
|
|
48
|
+
);
|
|
49
|
+
|
|
50
|
+
CREATE TABLE playbook_item_history (
|
|
51
|
+
id TEXT PRIMARY KEY,
|
|
52
|
+
item_id TEXT NOT NULL REFERENCES playbook_items(id),
|
|
53
|
+
action TEXT NOT NULL CHECK (action IN ('add', 'update', 'retire', 'helpful', 'harmful', 'prune')),
|
|
54
|
+
content TEXT,
|
|
55
|
+
actor TEXT NOT NULL CHECK (actor IN ('ace', 'human', 'user')),
|
|
56
|
+
run_id TEXT,
|
|
57
|
+
created_at TEXT NOT NULL
|
|
58
|
+
);
|
|
59
|
+
CREATE INDEX playbook_item_history_item ON playbook_item_history(item_id, created_at);
|
|
60
|
+
|
|
61
|
+
CREATE TABLE learn_jobs (
|
|
62
|
+
id TEXT PRIMARY KEY,
|
|
63
|
+
run_id TEXT NOT NULL,
|
|
64
|
+
kind TEXT NOT NULL CHECK (kind IN ('reflect', 'curate', 'score')),
|
|
65
|
+
trigger TEXT NOT NULL CHECK (trigger IN ('run_end', 'feedback')),
|
|
66
|
+
status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'succeeded', 'failed')),
|
|
67
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
68
|
+
last_error TEXT,
|
|
69
|
+
claimed_by TEXT,
|
|
70
|
+
claimed_at TEXT,
|
|
71
|
+
created_at TEXT NOT NULL,
|
|
72
|
+
updated_at TEXT NOT NULL
|
|
73
|
+
);
|
|
74
|
+
-- At most one pending job per run and kind: a verdict that arrives while the run's job is still pending is picked
|
|
75
|
+
-- up by that job, which reads the latest verdict when it runs.
|
|
76
|
+
CREATE UNIQUE INDEX learn_jobs_one_pending ON learn_jobs(run_id, kind) WHERE status = 'pending';
|
|
77
|
+
CREATE INDEX learn_jobs_status ON learn_jobs(status, created_at);
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
-- GEPA optimization (spec §5.2, §8.2). A rollout is a root run with optim_run_id set; so is the optimizer's own run row
|
|
2
|
+
-- (its id is the optimization id), which holds the reflection and judge-rubric evaluation spans. Run lists and the
|
|
3
|
+
-- review queue leave these runs out. optim_runs and optim_candidates are kept by reindex, like scores and feedback.
|
|
4
|
+
ALTER TABLE runs ADD COLUMN optim_run_id TEXT;
|
|
5
|
+
CREATE INDEX runs_optim ON runs(optim_run_id);
|
|
6
|
+
|
|
7
|
+
-- Eval cases gain the skill the source run used (rollouts run with it) and where the case came from.
|
|
8
|
+
ALTER TABLE eval_cases ADD COLUMN skill TEXT;
|
|
9
|
+
ALTER TABLE eval_cases ADD COLUMN origin TEXT NOT NULL DEFAULT 'run' CHECK (origin IN ('run', 'manual'));
|
|
10
|
+
|
|
11
|
+
CREATE TABLE optim_runs (
|
|
12
|
+
id TEXT PRIMARY KEY,
|
|
13
|
+
target TEXT NOT NULL,
|
|
14
|
+
status TEXT NOT NULL CHECK (status IN ('running', 'succeeded', 'failed', 'aborted')),
|
|
15
|
+
stop_reason TEXT,
|
|
16
|
+
budget_usd REAL NOT NULL CHECK (budget_usd > 0),
|
|
17
|
+
budget_rollouts INTEGER,
|
|
18
|
+
spent_usd REAL NOT NULL DEFAULT 0,
|
|
19
|
+
rollouts INTEGER NOT NULL DEFAULT 0,
|
|
20
|
+
merge INTEGER NOT NULL DEFAULT 0,
|
|
21
|
+
seed INTEGER NOT NULL,
|
|
22
|
+
base_artifact_id TEXT NOT NULL REFERENCES artifacts(id),
|
|
23
|
+
best_artifact_id TEXT REFERENCES artifacts(id),
|
|
24
|
+
dir_path TEXT NOT NULL,
|
|
25
|
+
report_ref TEXT,
|
|
26
|
+
judge_key TEXT,
|
|
27
|
+
judge_only INTEGER,
|
|
28
|
+
holdout_before REAL,
|
|
29
|
+
holdout_after REAL,
|
|
30
|
+
promotion TEXT CHECK (promotion IN ('approved', 'rolled_back')),
|
|
31
|
+
replaced_artifact_id TEXT REFERENCES artifacts(id),
|
|
32
|
+
promoted_at TEXT,
|
|
33
|
+
pid INTEGER,
|
|
34
|
+
error_json TEXT,
|
|
35
|
+
created_at TEXT NOT NULL,
|
|
36
|
+
ended_at TEXT
|
|
37
|
+
);
|
|
38
|
+
CREATE INDEX optim_runs_created ON optim_runs(created_at);
|
|
39
|
+
|
|
40
|
+
CREATE TABLE optim_candidates (
|
|
41
|
+
id TEXT PRIMARY KEY,
|
|
42
|
+
optim_run_id TEXT NOT NULL REFERENCES optim_runs(id),
|
|
43
|
+
artifact_id TEXT NOT NULL REFERENCES artifacts(id),
|
|
44
|
+
parent_ids_json TEXT NOT NULL DEFAULT '[]',
|
|
45
|
+
iteration INTEGER NOT NULL,
|
|
46
|
+
origin TEXT NOT NULL CHECK (origin IN ('seed', 'reflect', 'merge')),
|
|
47
|
+
component TEXT,
|
|
48
|
+
minibatch_json TEXT,
|
|
49
|
+
per_case_json TEXT NOT NULL DEFAULT '{}',
|
|
50
|
+
mean_score REAL,
|
|
51
|
+
mean_cost_usd REAL,
|
|
52
|
+
accepted INTEGER NOT NULL DEFAULT 0,
|
|
53
|
+
created_at TEXT NOT NULL
|
|
54
|
+
);
|
|
55
|
+
CREATE INDEX optim_candidates_run ON optim_candidates(optim_run_id, created_at);
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
-- Mined eval cases and climb (2026-10-16 design §2). eval_cases is rebuilt because SQLite cannot widen a CHECK in place:
|
|
2
|
+
-- origin gains 'mined', and every case gains a status (only 'active' cases enter optimizations), the source of its
|
|
3
|
+
-- golden (`expected`) and, for a mined case, the run ids of its cluster.
|
|
4
|
+
CREATE TABLE eval_cases_new (
|
|
5
|
+
id TEXT PRIMARY KEY,
|
|
6
|
+
target TEXT NOT NULL,
|
|
7
|
+
source_run_id TEXT,
|
|
8
|
+
input TEXT NOT NULL,
|
|
9
|
+
expected TEXT,
|
|
10
|
+
reference_feedback TEXT,
|
|
11
|
+
scorer_spec_json TEXT,
|
|
12
|
+
split TEXT NOT NULL CHECK (split IN ('feedback','pareto','holdout')),
|
|
13
|
+
created_at TEXT NOT NULL,
|
|
14
|
+
skill TEXT,
|
|
15
|
+
origin TEXT NOT NULL DEFAULT 'run' CHECK (origin IN ('run','manual','mined')),
|
|
16
|
+
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('proposed','active','dismissed')),
|
|
17
|
+
golden_source TEXT CHECK (golden_source IN ('run','manual','climb')),
|
|
18
|
+
mined_from_json TEXT
|
|
19
|
+
);
|
|
20
|
+
INSERT INTO eval_cases_new (id, target, source_run_id, input, expected, reference_feedback, scorer_spec_json, split, created_at, skill, origin, status, golden_source)
|
|
21
|
+
SELECT id, target, source_run_id, input, expected, reference_feedback, scorer_spec_json, split, created_at, skill, origin, 'active',
|
|
22
|
+
CASE WHEN expected IS NULL OR target = 'judge_rubric' THEN NULL WHEN origin = 'manual' THEN 'manual' ELSE 'run' END
|
|
23
|
+
FROM eval_cases;
|
|
24
|
+
DROP TABLE eval_cases;
|
|
25
|
+
ALTER TABLE eval_cases_new RENAME TO eval_cases;
|
|
26
|
+
CREATE UNIQUE INDEX eval_cases_source ON eval_cases(target, source_run_id);
|
|
27
|
+
CREATE INDEX eval_cases_target_status ON eval_cases(target, status);
|
|
28
|
+
|
|
29
|
+
-- Every value a case's golden has held, so a climb raise can be audited and reverted.
|
|
30
|
+
CREATE TABLE eval_case_goldens (
|
|
31
|
+
id TEXT PRIMARY KEY,
|
|
32
|
+
case_id TEXT NOT NULL REFERENCES eval_cases(id) ON DELETE CASCADE,
|
|
33
|
+
expected TEXT NOT NULL,
|
|
34
|
+
source TEXT NOT NULL CHECK (source IN ('run','manual','climb')),
|
|
35
|
+
source_run_id TEXT,
|
|
36
|
+
optim_run_id TEXT,
|
|
37
|
+
created_at TEXT NOT NULL
|
|
38
|
+
);
|
|
39
|
+
CREATE INDEX eval_case_goldens_case ON eval_case_goldens(case_id, created_at);
|
|
40
|
+
INSERT INTO eval_case_goldens (id, case_id, expected, source, source_run_id, optim_run_id, created_at)
|
|
41
|
+
SELECT 'g' || id, id, expected, golden_source, source_run_id, NULL, created_at FROM eval_cases WHERE golden_source IS NOT NULL;
|
|
42
|
+
|
|
43
|
+
-- Climb runs share optim_runs with GEPA runs.
|
|
44
|
+
ALTER TABLE optim_runs ADD COLUMN mode TEXT NOT NULL DEFAULT 'gepa' CHECK (mode IN ('gepa','climb'));
|
|
45
|
+
ALTER TABLE optim_runs ADD COLUMN case_id TEXT;
|
|
46
|
+
ALTER TABLE optim_runs ADD COLUMN climb_outcome TEXT CHECK (climb_outcome IN ('improved','golden_raised','no_signal','not_improved','guard_regressed'));
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { transaction } from "./db.js";
|
|
2
|
+
import { caseSplit, MAX_EVAL_INPUT_CHARS, targetRunFilter } from "./eval-cases.js";
|
|
3
|
+
import { newId } from "./ids.js";
|
|
4
|
+
import { getEvalCase } from "./judge-calibration.js";
|
|
5
|
+
/**
|
|
6
|
+
* Runs mining may turn into proposed cases (design §3.2): root runs of `target` (see `targetRunFilter`) with no
|
|
7
|
+
* feedback, not already the source of a case for `target`, started at or after `sinceIso`; the newest `maxRuns`, newest first.
|
|
8
|
+
*/
|
|
9
|
+
export function mineCandidateRuns(db, target, options) {
|
|
10
|
+
const filter = targetRunFilter(target);
|
|
11
|
+
return db
|
|
12
|
+
.prepare(`SELECT r.id, r.instruction, r.skill, r.started_at FROM runs r WHERE ${filter.sql}
|
|
13
|
+
AND NOT EXISTS (SELECT 1 FROM feedback f WHERE f.run_id = r.id)
|
|
14
|
+
AND NOT EXISTS (SELECT 1 FROM eval_cases c WHERE c.target = ? AND c.source_run_id = r.id)
|
|
15
|
+
AND r.started_at >= ?
|
|
16
|
+
ORDER BY r.started_at DESC, r.id DESC LIMIT ?`)
|
|
17
|
+
.all(...filter.args, target.key, options.sinceIso, options.maxRuns)
|
|
18
|
+
.map((row) => ({ runId: String(row.id), instruction: String(row.instruction), skill: row.skill ?? null, startedAt: String(row.started_at) }));
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Greedy leader clustering (design §3.3). `items` are newest first; each joins the first cluster whose leader (first
|
|
22
|
+
* member) it is `similar` to, or leads a new one. Clusters smaller than `minCluster`, and clusters whose leader is
|
|
23
|
+
* `nearExisting`, are dropped; the rest are ordered by size, then by leader position, and capped at `maxNew`.
|
|
24
|
+
*/
|
|
25
|
+
export function proposeClusters(items, options) {
|
|
26
|
+
const clusters = [];
|
|
27
|
+
for (const item of items) {
|
|
28
|
+
const home = clusters.find((c) => options.similar(c[0], item));
|
|
29
|
+
if (home)
|
|
30
|
+
home.push(item);
|
|
31
|
+
else
|
|
32
|
+
clusters.push([item]);
|
|
33
|
+
}
|
|
34
|
+
return clusters
|
|
35
|
+
.map((members, order) => ({ members, order }))
|
|
36
|
+
.filter((c) => c.members.length >= options.minCluster && !options.nearExisting(c.members[0]))
|
|
37
|
+
.sort((a, b) => b.members.length - a.members.length || a.order - b.order)
|
|
38
|
+
.slice(0, options.maxNew)
|
|
39
|
+
.map((c) => c.members);
|
|
40
|
+
}
|
|
41
|
+
/** Stores one proposed case per cluster, led by its newest run (design §3.3), in one transaction. */
|
|
42
|
+
export function insertMinedCases(db, target, clusters, redact, nowIso) {
|
|
43
|
+
return transaction(db, () => clusters.map((members) => {
|
|
44
|
+
const leader = members[0];
|
|
45
|
+
const text = redact(leader.instruction.trim());
|
|
46
|
+
const input = text.length > MAX_EVAL_INPUT_CHARS ? `${text.slice(0, MAX_EVAL_INPUT_CHARS - 1)}…` : text;
|
|
47
|
+
const id = newId();
|
|
48
|
+
db.prepare("INSERT INTO eval_cases (id, target, source_run_id, input, expected, reference_feedback, scorer_spec_json, split, created_at, skill, origin, status, golden_source, mined_from_json) VALUES (?, ?, ?, ?, NULL, NULL, NULL, ?, ?, ?, 'mined', 'proposed', NULL, ?)").run(id, target.key, leader.runId, input, caseSplit(input), nowIso, target.kind === "skill" ? target.name : leader.skill, JSON.stringify(members.map((m) => m.runId)));
|
|
49
|
+
return getEvalCase(db, id);
|
|
50
|
+
}));
|
|
51
|
+
}
|