@hexdecimal16/nexo 0.0.0-stage → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (209) hide show
  1. package/LICENSE +75 -0
  2. package/README.md +75 -2
  3. package/cli/src/bin.js +12 -0
  4. package/cli/src/common.js +78 -0
  5. package/cli/src/doctor.js +27 -0
  6. package/cli/src/eval-cases.js +65 -0
  7. package/cli/src/eval.js +67 -0
  8. package/cli/src/init.js +186 -0
  9. package/cli/src/interactive.js +171 -0
  10. package/cli/src/learn.js +178 -0
  11. package/cli/src/main.js +266 -0
  12. package/cli/src/optimize.js +289 -0
  13. package/cli/src/prompter.js +47 -0
  14. package/cli/src/review.js +120 -0
  15. package/cli/src/runtime.js +2 -0
  16. package/cli/src/sessions.js +34 -0
  17. package/cli/src/settings.js +169 -0
  18. package/cli/src/ui.js +112 -0
  19. package/cli/src/version.js +19 -0
  20. package/core/src/approval/classifier.js +152 -0
  21. package/core/src/config/load.js +283 -0
  22. package/core/src/config/schema.js +186 -0
  23. package/core/src/cost/admission.js +90 -0
  24. package/core/src/cost/budget.js +67 -0
  25. package/core/src/cost/pricing.js +53 -0
  26. package/core/src/engine/charged-completion.js +79 -0
  27. package/core/src/engine/errors.js +77 -0
  28. package/core/src/engine/mcp.js +11 -0
  29. package/core/src/engine/messages.js +86 -0
  30. package/core/src/engine/prompts.js +32 -0
  31. package/core/src/engine/provider-gate.js +33 -0
  32. package/core/src/engine/recorder.js +61 -0
  33. package/core/src/engine/run.js +1115 -0
  34. package/core/src/engine/semaphore.js +45 -0
  35. package/core/src/engine/spawn-tool.js +119 -0
  36. package/core/src/engine/task-graph.js +153 -0
  37. package/core/src/index.js +32 -0
  38. package/core/src/learn/caller.js +171 -0
  39. package/core/src/learn/context.js +112 -0
  40. package/core/src/learn/embedder.js +216 -0
  41. package/core/src/learn/job.js +196 -0
  42. package/core/src/learn/wiki.js +116 -0
  43. package/core/src/learn/worker.js +141 -0
  44. package/core/src/optimize/adapter.js +250 -0
  45. package/core/src/optimize/climb.js +338 -0
  46. package/core/src/optimize/mine.js +95 -0
  47. package/core/src/optimize/optimize.js +377 -0
  48. package/core/src/optimize/optimizer-run.js +67 -0
  49. package/core/src/optimize/reflect.js +100 -0
  50. package/core/src/optimize/scratchpad-wiki.js +69 -0
  51. package/core/src/providers/claude-p.js +363 -0
  52. package/core/src/providers/kiro-p.js +408 -0
  53. package/core/src/providers/models.js +10 -0
  54. package/core/src/scoring/calibration.js +31 -0
  55. package/core/src/scoring/continuation.js +24 -0
  56. package/core/src/scoring/runner.js +322 -0
  57. package/core/src/scoring/sources.js +88 -0
  58. package/core/src/session/lock.js +88 -0
  59. package/core/src/session/session.js +647 -0
  60. package/core/src/session/shell.js +26 -0
  61. package/core/src/session/transcript.js +86 -0
  62. package/core/src/settings/config-edit.js +472 -0
  63. package/core/src/settings/credentials.js +120 -0
  64. package/core/src/settings/doctor.js +103 -0
  65. package/core/src/settings/settings.js +265 -0
  66. package/mcp/src/hub.js +184 -0
  67. package/mcp/src/index.js +1 -0
  68. package/memory/src/curate.js +121 -0
  69. package/memory/src/index.js +4 -0
  70. package/memory/src/prompts.js +143 -0
  71. package/memory/src/retrieval.js +91 -0
  72. package/memory/src/similarity.js +47 -0
  73. package/optimize/src/engine.js +206 -0
  74. package/optimize/src/index.js +8 -0
  75. package/optimize/src/merge.js +73 -0
  76. package/optimize/src/novelty.js +37 -0
  77. package/optimize/src/pareto.js +83 -0
  78. package/optimize/src/proposer.js +148 -0
  79. package/optimize/src/report.js +107 -0
  80. package/optimize/src/rng.js +34 -0
  81. package/optimize/src/target.js +34 -0
  82. package/package.json +47 -4
  83. package/router/src/effort.js +5 -0
  84. package/router/src/escalation.js +171 -0
  85. package/router/src/index.js +4 -0
  86. package/router/src/registry.js +21 -0
  87. package/router/src/resolve.js +117 -0
  88. package/scorers/src/deterministic.js +194 -0
  89. package/scorers/src/family.js +48 -0
  90. package/scorers/src/index.js +6 -0
  91. package/scorers/src/judge.js +284 -0
  92. package/scorers/src/regex-worker.js +3 -0
  93. package/scorers/src/spec.js +147 -0
  94. package/scorers/src/trajectory.js +22 -0
  95. package/scorers/src/types.js +1 -0
  96. package/sdk/src/index.js +3 -0
  97. package/sdk/src/nexo.js +94 -0
  98. package/sdk/src/runtime.js +39 -0
  99. package/skills/bundled/browser/SKILL.md +57 -0
  100. package/skills/bundled/wiki/SKILL.md +73 -0
  101. package/skills/src/frontmatter.js +25 -0
  102. package/skills/src/index.js +3 -0
  103. package/skills/src/load-skill-tool.js +41 -0
  104. package/skills/src/loader.js +95 -0
  105. package/store/src/artifacts.js +124 -0
  106. package/store/src/budget-ledger.js +18 -0
  107. package/store/src/cost-report.js +46 -0
  108. package/store/src/cursor.js +35 -0
  109. package/store/src/db.js +83 -0
  110. package/store/src/errors.js +9 -0
  111. package/store/src/eval-cases.js +255 -0
  112. package/store/src/feedback.js +154 -0
  113. package/store/src/goldens.js +61 -0
  114. package/store/src/ids.js +6 -0
  115. package/store/src/index.js +25 -0
  116. package/store/src/judge-calibration.js +182 -0
  117. package/store/src/learn-jobs.js +145 -0
  118. package/store/src/memories.js +261 -0
  119. package/store/src/migrations/001_init.sql +81 -0
  120. package/store/src/migrations/002_feedback.sql +9 -0
  121. package/store/src/migrations/003_sessions.sql +19 -0
  122. package/store/src/migrations/004_scoring.sql +22 -0
  123. package/store/src/migrations/005_judge_calibration.sql +27 -0
  124. package/store/src/migrations/006_learning.sql +77 -0
  125. package/store/src/migrations/007_optimize.sql +55 -0
  126. package/store/src/migrations/008_mined_cases_and_climb.sql +46 -0
  127. package/store/src/mine.js +51 -0
  128. package/store/src/optim.js +291 -0
  129. package/store/src/paths.js +22 -0
  130. package/store/src/playbooks.js +199 -0
  131. package/store/src/reindex.js +143 -0
  132. package/store/src/run-dir.js +151 -0
  133. package/store/src/runs.js +310 -0
  134. package/store/src/scores.js +92 -0
  135. package/store/src/sessions.js +304 -0
  136. package/store/src/span-tree.js +98 -0
  137. package/store/src/spans.js +84 -0
  138. package/store/src/sql.js +13 -0
  139. package/store/src/workspace-copy.js +80 -0
  140. package/telemetry/src/errors.js +32 -0
  141. package/telemetry/src/index.js +3 -0
  142. package/telemetry/src/logger.js +39 -0
  143. package/telemetry/src/redact.js +106 -0
  144. package/tools/src/builtin.js +269 -0
  145. package/tools/src/change-tracker.js +57 -0
  146. package/tools/src/confine.js +101 -0
  147. package/tools/src/index.js +7 -0
  148. package/tools/src/leases.js +57 -0
  149. package/tools/src/policy.js +162 -0
  150. package/tools/src/read-only.js +338 -0
  151. package/tools/src/session-rules.js +140 -0
  152. package/tui/src/app.js +965 -0
  153. package/tui/src/approvals.js +96 -0
  154. package/tui/src/commands.js +38 -0
  155. package/tui/src/components.js +248 -0
  156. package/tui/src/index.js +5 -0
  157. package/tui/src/picker.js +78 -0
  158. package/tui/src/session-picker.js +100 -0
  159. package/tui/src/settings-screen.js +346 -0
  160. package/tui/src/text.js +70 -0
  161. package/tui/src/theme.js +44 -0
  162. package/web/public/cost.html +25 -0
  163. package/web/public/fonts/GeistMono-400.woff2 +0 -0
  164. package/web/public/fonts/GeistMono-600.woff2 +0 -0
  165. package/web/public/fonts/InstrumentSans-400.woff2 +0 -0
  166. package/web/public/fonts/InstrumentSans-600.woff2 +0 -0
  167. package/web/public/index.html +39 -0
  168. package/web/public/judge.html +21 -0
  169. package/web/public/learning.html +46 -0
  170. package/web/public/lib/api-types.ts +616 -0
  171. package/web/public/lib/common-view.ts +47 -0
  172. package/web/public/lib/cost-view.ts +35 -0
  173. package/web/public/lib/files-view.ts +90 -0
  174. package/web/public/lib/format.ts +35 -0
  175. package/web/public/lib/html.ts +51 -0
  176. package/web/public/lib/judge-view.ts +62 -0
  177. package/web/public/lib/learning-view.ts +102 -0
  178. package/web/public/lib/optim-view.ts +148 -0
  179. package/web/public/lib/pager.ts +56 -0
  180. package/web/public/lib/poller.ts +79 -0
  181. package/web/public/lib/run-view.ts +61 -0
  182. package/web/public/lib/runs-view.ts +87 -0
  183. package/web/public/lib/score-view.ts +94 -0
  184. package/web/public/lib/sessions-view.ts +28 -0
  185. package/web/public/lib/span-view.ts +90 -0
  186. package/web/public/lib/theme.ts +60 -0
  187. package/web/public/lib/timeline.ts +97 -0
  188. package/web/public/lib/wikis-view.ts +257 -0
  189. package/web/public/optim.html +30 -0
  190. package/web/public/pages/api.ts +60 -0
  191. package/web/public/pages/cost.ts +36 -0
  192. package/web/public/pages/judge.ts +15 -0
  193. package/web/public/pages/learning.ts +92 -0
  194. package/web/public/pages/optim.ts +68 -0
  195. package/web/public/pages/paging.ts +18 -0
  196. package/web/public/pages/run.ts +221 -0
  197. package/web/public/pages/runs.ts +91 -0
  198. package/web/public/pages/sessions.ts +25 -0
  199. package/web/public/pages/wikis.ts +103 -0
  200. package/web/public/run.html +36 -0
  201. package/web/public/sessions.html +28 -0
  202. package/web/public/style.css +288 -0
  203. package/web/public/wikis.html +25 -0
  204. package/web/src/api.js +682 -0
  205. package/web/src/http-errors.js +60 -0
  206. package/web/src/index.js +5 -0
  207. package/web/src/server.js +292 -0
  208. package/web/src/static.js +86 -0
  209. package/web/src/strip.js +22 -0
@@ -0,0 +1,124 @@
1
+ import { createHash } from "node:crypto";
2
+ import { transaction } from "./db.js";
3
+ import { newId } from "./ids.js";
4
+ function sha256(text) {
5
+ return createHash("sha256").update(text).digest("hex");
6
+ }
7
+ function decode(row) {
8
+ return {
9
+ id: String(row.id),
10
+ kind: row.kind,
11
+ name: String(row.name),
12
+ version: Number(row.version),
13
+ content: String(row.content),
14
+ contentHash: String(row.content_hash),
15
+ parentVersionId: row.parent_version_id ? String(row.parent_version_id) : null,
16
+ origin: row.origin,
17
+ status: row.status,
18
+ createdAt: String(row.created_at),
19
+ };
20
+ }
21
+ export function getArtifact(db, id) {
22
+ const row = db.prepare("SELECT * FROM artifacts WHERE id = ?").get(id);
23
+ return row ? decode(row) : undefined;
24
+ }
25
+ export function getActiveArtifact(db, kind, name) {
26
+ const row = db.prepare("SELECT * FROM artifacts WHERE kind = ? AND name = ? AND status = 'active' ORDER BY version DESC LIMIT 1").get(kind, name);
27
+ return row ? decode(row) : undefined;
28
+ }
29
+ /** Returns the active version when its content matches; otherwise stores a new active manual version and retires the old one. */
30
+ export function ensureManualArtifact(db, kind, name, content, nowIso) {
31
+ return appendArtifactVersion(db, kind, name, content, "manual", nowIso);
32
+ }
33
+ /**
34
+ * Returns the active version of a learnable artifact (router policy, judge rubric), creating version 1 from `seed` when
35
+ * none exists. Unlike `ensureManualArtifact` it never replaces an active version, so a promoted version stays active.
36
+ */
37
+ export function ensureSeedArtifact(db, kind, name, seed, nowIso) {
38
+ const active = getActiveArtifact(db, kind, name);
39
+ if (active)
40
+ return active;
41
+ return ensureManualArtifact(db, kind, name, seed, nowIso);
42
+ }
43
+ /**
44
+ * Stores `content` as a new active version with the given origin and retires the previous active version; returns the
45
+ * active version unchanged when its content already matches.
46
+ */
47
+ export function appendArtifactVersion(db, kind, name, content, origin, nowIso) {
48
+ const hash = sha256(content);
49
+ return transaction(db, () => {
50
+ const active = getActiveArtifact(db, kind, name);
51
+ if (active && active.contentHash === hash)
52
+ return active;
53
+ const max = db.prepare("SELECT COALESCE(MAX(version), 0) AS v FROM artifacts WHERE kind = ? AND name = ?").get(kind, name);
54
+ const row = {
55
+ id: newId(),
56
+ kind,
57
+ name,
58
+ version: Number(max?.v ?? 0) + 1,
59
+ content,
60
+ contentHash: hash,
61
+ parentVersionId: active?.id ?? null,
62
+ origin,
63
+ status: "active",
64
+ createdAt: nowIso,
65
+ };
66
+ if (active)
67
+ db.prepare("UPDATE artifacts SET status = 'retired' WHERE id = ?").run(active.id);
68
+ db.prepare("INSERT INTO artifacts (id, kind, name, version, content, content_hash, parent_version_id, origin, status, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(row.id, kind, name, row.version, content, hash, row.parentVersionId, origin, "active", nowIso);
69
+ return row;
70
+ });
71
+ }
72
+ /**
73
+ * The version a run uses for an artifact whose text comes from a source (SKILL.md, a built-in prompt, a `file:`
74
+ * rubric): a promoted (`gepa`) active version while the newest `manual` version still matches `source`, so a promotion
75
+ * stays in use until the source is edited; otherwise `source` as the active manual version (`ensureManualArtifact`).
76
+ */
77
+ export function resolveSourcedArtifact(db, kind, name, source, nowIso) {
78
+ return transaction(db, () => {
79
+ const active = getActiveArtifact(db, kind, name);
80
+ if (active?.origin === "gepa") {
81
+ const manual = db
82
+ .prepare("SELECT content_hash FROM artifacts WHERE kind = ? AND name = ? AND origin = 'manual' ORDER BY version DESC LIMIT 1")
83
+ .get(kind, name);
84
+ if (manual && String(manual.content_hash) === sha256(source))
85
+ return active;
86
+ }
87
+ return appendArtifactVersion(db, kind, name, source, "manual", nowIso);
88
+ });
89
+ }
90
+ /**
91
+ * Stores an optimization candidate: a new version with status `candidate` and origin `gepa` whose parent is the
92
+ * candidate it was derived from. The active version is not touched.
93
+ */
94
+ export function insertCandidateArtifact(db, kind, name, content, parentId, nowIso) {
95
+ return transaction(db, () => {
96
+ const max = db.prepare("SELECT COALESCE(MAX(version), 0) AS v FROM artifacts WHERE kind = ? AND name = ?").get(kind, name);
97
+ const row = {
98
+ id: newId(),
99
+ kind,
100
+ name,
101
+ version: Number(max?.v ?? 0) + 1,
102
+ content,
103
+ contentHash: sha256(content),
104
+ parentVersionId: parentId,
105
+ origin: "gepa",
106
+ status: "candidate",
107
+ createdAt: nowIso,
108
+ };
109
+ db.prepare("INSERT INTO artifacts (id, kind, name, version, content, content_hash, parent_version_id, origin, status, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, 'gepa', 'candidate', ?)").run(row.id, kind, name, row.version, content, row.contentHash, parentId, nowIso);
110
+ return row;
111
+ });
112
+ }
113
+ /** Every version of an artifact, newest first. */
114
+ export function listArtifactVersions(db, kind, name) {
115
+ return db.prepare("SELECT * FROM artifacts WHERE kind = ? AND name = ? ORDER BY version DESC").all(kind, name).map(decode);
116
+ }
117
+ /** Stores an artifact set keyed by the hash of its sorted members, so identical sets share one id. */
118
+ export function createArtifactSet(db, members, nowIso) {
119
+ const sorted = [...members].sort((a, b) => `${a.kind}/${a.name}`.localeCompare(`${b.kind}/${b.name}`));
120
+ const json = JSON.stringify(sorted);
121
+ const id = sha256(json).slice(0, 32);
122
+ db.prepare("INSERT OR IGNORE INTO artifact_sets (id, members_json, created_at) VALUES (?, ?, ?)").run(id, json, nowIso);
123
+ return id;
124
+ }
@@ -0,0 +1,18 @@
1
+ export function dayScope(date) {
2
+ return `day:${date.toISOString().slice(0, 10)}`;
3
+ }
4
+ export function monthScope(date) {
5
+ return `month:${date.toISOString().slice(0, 7)}`;
6
+ }
7
+ export function addSpend(db, scopes, usd, nowIso) {
8
+ if (usd <= 0)
9
+ return;
10
+ const stmt = db.prepare("INSERT INTO budget_ledger (scope, spent_usd, updated_at) VALUES (?, ?, ?) " +
11
+ "ON CONFLICT(scope) DO UPDATE SET spent_usd = spent_usd + excluded.spent_usd, updated_at = excluded.updated_at");
12
+ for (const scope of scopes)
13
+ stmt.run(scope, usd, nowIso);
14
+ }
15
+ export function getSpent(db, scope) {
16
+ const row = db.prepare("SELECT spent_usd FROM budget_ledger WHERE scope = ?").get(scope);
17
+ return row ? Number(row.spent_usd) : 0;
18
+ }
@@ -0,0 +1,46 @@
1
+ import { invalidRequest } from "./errors.js";
2
+ /** The one list of cost dimensions; the CLI, the API and the cost page all take it from here. */
3
+ export const COST_DIMENSIONS = ["day", "model", "skill", "tier", "run"];
4
+ /** `since` defaults to this many days before now. */
5
+ export const COST_DEFAULT_DAYS = 30;
6
+ const KEY_SQL = {
7
+ model: "COALESCE(s.provider, '?') || '/' || COALESCE(s.model, '?')",
8
+ tier: "COALESCE(s.tier, 'unknown')",
9
+ day: "substr(s.started_at, 1, 10)",
10
+ run: "COALESCE('optim:' || r.optim_run_id, r.root_run_id)",
11
+ skill: "COALESCE(r.skill, '(none)')",
12
+ };
13
+ export function isCostDimension(value) {
14
+ return typeof value === "string" && COST_DIMENSIONS.includes(value);
15
+ }
16
+ /**
17
+ * LLM-call spend grouped by one dimension since a date, most expensive first, with totals. Shared by `nexo cost` and
18
+ * the web UI; both pass raw user input, and this function owns the defaults and the validation.
19
+ */
20
+ export function costReport(db, options) {
21
+ const by = options.by;
22
+ if (!isCostDimension(by)) {
23
+ throw invalidRequest(`Unknown cost dimension "${String(by)}"`, { by: by ?? null }, `Use one of ${COST_DIMENSIONS.join(", ")}`);
24
+ }
25
+ const now = options.now ?? new Date();
26
+ const since = options.since ?? new Date(now.getTime() - COST_DEFAULT_DAYS * 86_400_000).toISOString().slice(0, 10);
27
+ const ms = /^\d{4}-\d{2}-\d{2}$/.test(since) ? Date.parse(since) : Number.NaN;
28
+ if (!Number.isFinite(ms) || new Date(ms).toISOString().slice(0, 10) !== since) {
29
+ throw invalidRequest("since must be a date in the form YYYY-MM-DD", { since });
30
+ }
31
+ const sql = `SELECT ${KEY_SQL[by]} AS key, SUM(s.cost_usd) AS cost, COUNT(*) AS calls, SUM(s.cost_unknown) AS unknown
32
+ FROM spans s JOIN runs r ON r.id = s.run_id
33
+ WHERE s.kind IN ('llm_call', 'compaction', 'judge_call', 'self_check', 'learn_call', 'embedding_call', 'gepa_call', 'approval_call') AND s.started_at >= ?
34
+ GROUP BY key ORDER BY cost DESC, key`;
35
+ const rows = db
36
+ .prepare(sql)
37
+ .all(since)
38
+ .map((row) => ({ key: String(row.key), costUsd: Number(row.cost), calls: Number(row.calls), unknownCalls: Number(row.unknown) }));
39
+ return {
40
+ by,
41
+ since,
42
+ rows,
43
+ totalUsd: rows.reduce((sum, row) => sum + row.costUsd, 0),
44
+ unknownCalls: rows.reduce((sum, row) => sum + row.unknownCalls, 0),
45
+ };
46
+ }
@@ -0,0 +1,35 @@
1
+ import { invalidRequest } from "./errors.js";
2
+ /** An opaque keyset cursor: the sort key and id of the last row of a page, so the next page starts after it. */
3
+ export function encodeCursor(key, id) {
4
+ return Buffer.from(JSON.stringify([key, id])).toString("base64url");
5
+ }
6
+ export function decodeCursor(cursor) {
7
+ try {
8
+ const value = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
9
+ if (Array.isArray(value) && value.length === 2 && typeof value[0] === "string" && typeof value[1] === "string")
10
+ return { key: value[0], id: value[1] };
11
+ }
12
+ catch {
13
+ // Falls through to the error below.
14
+ }
15
+ throw invalidRequest("cursor is not a cursor returned by this API", { cursor });
16
+ }
17
+ /** The first `limit` of `limit + 1` fetched rows, with the cursor for the next page when the extra row showed there is one. */
18
+ export function pageOf(rows, limit, keyOf) {
19
+ const items = rows.slice(0, limit);
20
+ const last = items.at(-1);
21
+ return { items, nextCursor: rows.length > limit && last ? encodeCursor(keyOf(last).key, keyOf(last).id) : null };
22
+ }
23
+ /** `offset` (absent means 0) as an integer from 0 up, or INVALID_REQUEST. */
24
+ export function checkOffset(offset) {
25
+ const value = offset ?? 0;
26
+ if (!Number.isInteger(value) || value < 0)
27
+ throw invalidRequest("offset must be an integer from 0 up", { offset: value });
28
+ return value;
29
+ }
30
+ /** `limit` as an integer from 1 to `max`, or INVALID_REQUEST. */
31
+ export function checkLimit(limit, max) {
32
+ if (!Number.isInteger(limit) || limit < 1 || limit > max)
33
+ throw invalidRequest(`limit must be an integer from 1 to ${max}`, { limit });
34
+ return limit;
35
+ }
@@ -0,0 +1,83 @@
1
+ import { mkdirSync, readdirSync, readFileSync } from "node:fs";
2
+ import { dirname, join } from "node:path";
3
+ import { DatabaseSync } from "node:sqlite";
4
+ import { fileURLToPath } from "node:url";
5
+ import { NexoError } from "../../telemetry/src/index.js";
6
+ const MIGRATIONS_DIR = fileURLToPath(new URL("./migrations/", import.meta.url));
7
+ export function openDb(path) {
8
+ try {
9
+ if (path !== ":memory:")
10
+ mkdirSync(dirname(path), { recursive: true });
11
+ const db = new DatabaseSync(path);
12
+ try {
13
+ db.exec("PRAGMA busy_timeout = 5000; PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON;");
14
+ migrate(db);
15
+ }
16
+ catch (error) {
17
+ db.close();
18
+ throw error;
19
+ }
20
+ return db;
21
+ }
22
+ catch (cause) {
23
+ if (cause instanceof NexoError)
24
+ throw cause;
25
+ throw new NexoError({
26
+ code: "STORE_IO",
27
+ message: `Cannot open database at ${path}`,
28
+ hint: "Check that NEXO_HOME points to a writable directory",
29
+ context: { path },
30
+ cause,
31
+ });
32
+ }
33
+ }
34
+ /** Applies pending `NNN_name.sql` migrations in order, each in its own transaction. Returns applied versions. */
35
+ export function migrate(db) {
36
+ db.exec("CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at TEXT NOT NULL)");
37
+ const files = readdirSync(MIGRATIONS_DIR)
38
+ .filter((f) => /^\d{3}_.+\.sql$/.test(f))
39
+ .sort();
40
+ const applied = [];
41
+ for (const file of files) {
42
+ const version = Number(file.slice(0, 3));
43
+ const sql = readFileSync(join(MIGRATIONS_DIR, file), "utf8");
44
+ const ran = transaction(db, () => {
45
+ const current = Number(db.prepare("SELECT COALESCE(MAX(version), 0) AS v FROM schema_migrations").get()?.v ?? 0);
46
+ if (version <= current)
47
+ return false;
48
+ db.exec(sql);
49
+ db.prepare("INSERT INTO schema_migrations (version, name, applied_at) VALUES (?, ?, ?)").run(version, file, new Date().toISOString());
50
+ return true;
51
+ });
52
+ if (ran)
53
+ applied.push(version);
54
+ }
55
+ return applied;
56
+ }
57
+ let savepoints = 0;
58
+ /** Runs `fn` in a transaction (BEGIN IMMEDIATE); inside an open transaction it uses a savepoint, so calls nest. */
59
+ export function transaction(db, fn) {
60
+ if (db.isTransaction) {
61
+ const name = `nexo_sp_${++savepoints}`;
62
+ db.exec(`SAVEPOINT ${name}`);
63
+ try {
64
+ const result = fn();
65
+ db.exec(`RELEASE ${name}`);
66
+ return result;
67
+ }
68
+ catch (error) {
69
+ db.exec(`ROLLBACK TO ${name}; RELEASE ${name}`);
70
+ throw error;
71
+ }
72
+ }
73
+ db.exec("BEGIN IMMEDIATE");
74
+ try {
75
+ const result = fn();
76
+ db.exec("COMMIT");
77
+ return result;
78
+ }
79
+ catch (error) {
80
+ db.exec("ROLLBACK");
81
+ throw error;
82
+ }
83
+ }
@@ -0,0 +1,9 @@
1
+ import { NexoError } from "../../telemetry/src/index.js";
2
+ /** A request the caller can fix: bad filter, bad body field, bad path. Maps to HTTP 400. */
3
+ export function invalidRequest(message, context = {}, hint) {
4
+ return new NexoError({ code: "INVALID_REQUEST", message, context, ...(hint ? { hint } : {}) });
5
+ }
6
+ /** The one NOT_FOUND error for an unknown run id, shared by the store and the web API. */
7
+ export function runNotFound(runId) {
8
+ return new NexoError({ code: "NOT_FOUND", message: `Run ${runId} does not exist`, context: { runId } });
9
+ }
@@ -0,0 +1,255 @@
1
+ import { NexoError } from "../../telemetry/src/index.js";
2
+ import { transaction } from "./db.js";
3
+ import { invalidRequest } from "./errors.js";
4
+ import { latestFeedback } from "./feedback.js";
5
+ import { recordGolden } from "./goldens.js";
6
+ import { newId } from "./ids.js";
7
+ import { evalSplitFor, getEvalCase, listActiveEvalCases, listEvalCases } from "./judge-calibration.js";
8
+ import { readRunJson } from "./run-dir.js";
9
+ import { getRun } from "./runs.js";
10
+ /** Longest eval-case input, expected text or reference feedback, in characters. */
11
+ export const MAX_EVAL_INPUT_CHARS = 20_000;
12
+ /** Most runs listed when an eval set is too small or a promotion needs more human verdicts. */
13
+ export const RUNS_TO_REVIEW = 10;
14
+ const SKILL_TARGET = /^skill:([a-z0-9]+(-[a-z0-9]+)*)$/;
15
+ /**
16
+ * Parses an optimization target. `router_policy` is refused: it is a JSON policy of numeric thresholds, not prose.
17
+ * Throws INVALID_REQUEST.
18
+ */
19
+ export function parseOptimTarget(text) {
20
+ const skill = SKILL_TARGET.exec(text);
21
+ if (skill && skill[1].length <= 64)
22
+ return { key: text, kind: "skill", name: skill[1] };
23
+ if (text === "system_prompt" || text === "planner_prompt" || text === "judge_rubric")
24
+ return { key: text, kind: text, name: "default" };
25
+ if (text === "router_policy") {
26
+ throw invalidRequest("router_policy is not an optimization target yet: it is a JSON policy of numeric thresholds, not prose, and evaluating one needs S2 escalation rollouts on every case", { target: text }, "Optimize skill:<name>, system_prompt, planner_prompt or judge_rubric; edit router thresholds with router.policy = file:<path>");
27
+ }
28
+ throw invalidRequest(`Unknown optimization target "${text}"`, { target: text }, "Use skill:<name>, system_prompt, planner_prompt or judge_rubric");
29
+ }
30
+ /** The text a case's split is derived from: trimmed, whitespace collapsed, lower-cased. */
31
+ export function normalizeCaseInput(input) {
32
+ return input.trim().replace(/\s+/g, " ").toLowerCase();
33
+ }
34
+ /** The split of a case: a stable hash of its normalized input, so the same instruction always lands in the same split. */
35
+ export function caseSplit(input) {
36
+ return evalSplitFor(normalizeCaseInput(input));
37
+ }
38
+ /** Root runs that can be eval cases of `target`: finished, not optimization runs, not a session turn after the first. */
39
+ export function targetRunFilter(target) {
40
+ const base = "r.parent_run_id IS NULL AND r.optim_run_id IS NULL AND r.status != 'running' AND (r.session_id IS NULL OR r.turn_index = 1)";
41
+ if (target.kind === "skill") {
42
+ return {
43
+ sql: `${base} AND (r.skill = ? OR EXISTS (SELECT 1 FROM spans s WHERE s.run_id = r.id AND s.kind = 'skill_load' AND s.status = 'ok' AND json_extract(s.attrs_json, '$.args.name') = ?))`,
44
+ args: [target.name, target.name],
45
+ };
46
+ }
47
+ if (target.kind === "judge_rubric") {
48
+ return { sql: `${base} AND EXISTS (SELECT 1 FROM scores s WHERE s.run_id = r.id AND s.scorer_type = 'llm_judge' AND s.status = 'ok')`, args: [] };
49
+ }
50
+ return { sql: base, args: [] };
51
+ }
52
+ function clip(text) {
53
+ return text.length > MAX_EVAL_INPUT_CHARS ? `${text.slice(0, MAX_EVAL_INPUT_CHARS - 1)}…` : text;
54
+ }
55
+ /**
56
+ * Turns every reviewed run of `target` into an eval case (one per run), refreshing cases whose verdict changed, and
57
+ * recomputes every case's split from its input. Agent targets: `expected` is the run's output when the verdict is
58
+ * done, `reference_feedback` the comment. `judge_rubric`: `expected` is the verdict.
59
+ */
60
+ export function syncRunEvalCases(db, target, nowIso) {
61
+ const filter = targetRunFilter(target);
62
+ const ids = db
63
+ .prepare(`SELECT r.id FROM runs r WHERE ${filter.sql} AND EXISTS (SELECT 1 FROM feedback f WHERE f.run_id = r.id) ORDER BY r.started_at, r.id`)
64
+ .all(...filter.args)
65
+ .map((row) => String(row.id));
66
+ let added = 0;
67
+ let updated = 0;
68
+ transaction(db, () => {
69
+ for (const id of ids) {
70
+ const run = getRun(db, id);
71
+ const feedback = latestFeedback(db, id);
72
+ if (!run || !feedback)
73
+ continue;
74
+ let expected;
75
+ if (target.kind === "judge_rubric") {
76
+ expected = feedback.verdict;
77
+ }
78
+ else {
79
+ let output = "";
80
+ try {
81
+ output = readRunJson(run.dirPath).output ?? "";
82
+ }
83
+ catch {
84
+ output = "";
85
+ }
86
+ expected = feedback.verdict === "done" && output.trim() ? clip(output) : null;
87
+ }
88
+ const reference = feedback.comment.trim() ? clip(feedback.comment) : null;
89
+ const rubric = target.kind === "judge_rubric";
90
+ const existing = db.prepare("SELECT id, expected, golden_source FROM eval_cases WHERE target = ? AND source_run_id = ?").get(target.key, id);
91
+ if (existing) {
92
+ const caseId = String(existing.id);
93
+ if (existing.golden_source === "climb" || existing.golden_source === "manual") {
94
+ // Goldens from climb or a human are not replaced by the run's output. Only a proposed case becomes active.
95
+ db.prepare("UPDATE eval_cases SET reference_feedback = ?, skill = ?, status = CASE WHEN status = 'proposed' THEN 'active' ELSE status END WHERE id = ?").run(reference, run.skill, caseId);
96
+ }
97
+ else {
98
+ db.prepare("UPDATE eval_cases SET expected = ?, reference_feedback = ?, skill = ?, status = CASE WHEN status = 'proposed' THEN 'active' ELSE status END, golden_source = ? WHERE id = ?").run(expected, reference, run.skill, !rubric && expected !== null ? "run" : null, caseId);
99
+ if (!rubric && expected !== null && expected !== existing.expected)
100
+ recordGolden(db, { caseId, expected, source: "run", sourceRunId: id, nowIso });
101
+ }
102
+ updated++;
103
+ }
104
+ else {
105
+ const caseId = newId();
106
+ db.prepare("INSERT INTO eval_cases (id, target, source_run_id, input, expected, reference_feedback, scorer_spec_json, split, created_at, skill, origin, status, golden_source) VALUES (?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, 'run', 'active', ?)").run(caseId, target.key, id, run.instruction, expected, reference, caseSplit(run.instruction), nowIso, run.skill, !rubric && expected !== null ? "run" : null);
107
+ if (!rubric && expected !== null)
108
+ recordGolden(db, { caseId, expected, source: "run", sourceRunId: id, nowIso });
109
+ added++;
110
+ }
111
+ }
112
+ const update = db.prepare("UPDATE eval_cases SET split = ? WHERE id = ? AND split != ?");
113
+ for (const c of listEvalCases(db, target.key)) {
114
+ const split = caseSplit(c.input);
115
+ update.run(split, c.id, split);
116
+ }
117
+ });
118
+ return { added, updated };
119
+ }
120
+ /** Stores a hand-written eval case (`origin = 'manual'`), redacted. Throws INVALID_REQUEST. */
121
+ export function addEvalCase(db, input, redact, nowIso) {
122
+ const target = parseOptimTarget(input.target);
123
+ if (target.kind === "judge_rubric") {
124
+ throw invalidRequest("judge_rubric cases come from reviewed runs that the judge scored", { target: target.key }, "Review judged runs in the UI or with /done and /notdone; they become judge_rubric cases");
125
+ }
126
+ const text = input.input.trim();
127
+ if (!text)
128
+ throw invalidRequest("The case input is empty");
129
+ const check = (name, value) => {
130
+ if (value && value.length > MAX_EVAL_INPUT_CHARS)
131
+ throw invalidRequest(`${name} is longer than ${MAX_EVAL_INPUT_CHARS} characters`, { length: value.length });
132
+ };
133
+ check("input", text);
134
+ check("expected", input.expected);
135
+ check("feedback", input.referenceFeedback);
136
+ const skill = target.kind === "skill" ? target.name : (input.skill ?? null);
137
+ if (target.kind === "skill" && input.skill && input.skill !== target.name) {
138
+ throw invalidRequest(`A ${target.key} case always runs with skill ${target.name}`, { skill: input.skill });
139
+ }
140
+ if (skill !== null && !/^[a-z0-9]+(-[a-z0-9]+)*$/.test(skill))
141
+ throw invalidRequest("skill must be a skill name", { skill });
142
+ const row = {
143
+ id: newId(),
144
+ target: target.key,
145
+ sourceRunId: null,
146
+ input: redact(text),
147
+ expected: input.expected?.trim() ? redact(input.expected.trim()) : null,
148
+ referenceFeedback: input.referenceFeedback?.trim() ? redact(input.referenceFeedback.trim()) : null,
149
+ scorerSpec: null,
150
+ split: caseSplit(redact(text)),
151
+ createdAt: nowIso,
152
+ skill,
153
+ origin: "manual",
154
+ status: "active",
155
+ goldenSource: input.expected?.trim() ? "manual" : null,
156
+ minedFrom: null,
157
+ };
158
+ transaction(db, () => {
159
+ db.prepare("INSERT INTO eval_cases (id, target, source_run_id, input, expected, reference_feedback, scorer_spec_json, split, created_at, skill, origin, status, golden_source) VALUES (?, ?, NULL, ?, ?, ?, NULL, ?, ?, ?, 'manual', 'active', ?)").run(row.id, row.target, row.input, row.expected, row.referenceFeedback, row.split, nowIso, skill, row.goldenSource);
160
+ if (row.expected !== null)
161
+ recordGolden(db, { caseId: row.id, expected: row.expected, source: "manual", nowIso });
162
+ });
163
+ return row;
164
+ }
165
+ /** Finished, unreviewed runs of `target` that a verdict would turn into cases: lowest score first, unscored next, newest first. */
166
+ export function runsNeedingReview(db, target, limit = RUNS_TO_REVIEW) {
167
+ const filter = targetRunFilter(target);
168
+ const rows = db
169
+ .prepare(`SELECT r.id, r.instruction, r.started_at,
170
+ (SELECT AVG(s.value) FROM scores s WHERE s.run_id = r.id AND s.status = 'ok' AND s.round = (SELECT MAX(s2.round) FROM scores s2 WHERE s2.run_id = r.id)) AS score
171
+ FROM runs r WHERE ${filter.sql} AND NOT EXISTS (SELECT 1 FROM feedback f WHERE f.run_id = r.id)
172
+ ORDER BY (score IS NULL), score ASC, r.started_at DESC, r.id DESC LIMIT ?`)
173
+ .all(...filter.args, limit);
174
+ return rows.map((row) => {
175
+ const first = String(row.instruction).split("\n")[0] ?? "";
176
+ return {
177
+ runId: String(row.id),
178
+ instruction: first.length > 120 ? `${first.slice(0, 119)}…` : first,
179
+ score: row.score === null ? null : Number(row.score),
180
+ startedAt: String(row.started_at),
181
+ };
182
+ });
183
+ }
184
+ /**
185
+ * The user's decision on a proposed (mined) case: `accept` makes it active, optionally with a golden; `dismiss` keeps it
186
+ * out of optimizations and stops its cluster from being proposed again. Throws NOT_FOUND, CONFLICT (not proposed) or
187
+ * INVALID_REQUEST (golden too long).
188
+ */
189
+ export function reviewProposedCase(db, id, decision, options) {
190
+ const golden = options.expected?.trim() ? options.redact(options.expected.trim()) : null;
191
+ if (golden && golden.length > MAX_EVAL_INPUT_CHARS)
192
+ throw invalidRequest(`expected is longer than ${MAX_EVAL_INPUT_CHARS} characters`, { length: golden.length });
193
+ return transaction(db, () => {
194
+ const c = getEvalCase(db, id);
195
+ if (!c)
196
+ throw new NexoError({ code: "NOT_FOUND", message: `Eval case ${id} does not exist`, context: { caseId: id } });
197
+ if (c.status !== "proposed")
198
+ throw new NexoError({ code: "CONFLICT", message: `Eval case ${id} is ${c.status}, not proposed`, context: { caseId: id, status: c.status } });
199
+ if (decision === "dismiss") {
200
+ db.prepare("UPDATE eval_cases SET status = 'dismissed' WHERE id = ?").run(id);
201
+ }
202
+ else if (golden) {
203
+ db.prepare("UPDATE eval_cases SET status = 'active', expected = ?, golden_source = 'manual' WHERE id = ?").run(golden, id);
204
+ recordGolden(db, { caseId: id, expected: golden, source: "manual", nowIso: options.nowIso });
205
+ }
206
+ else {
207
+ db.prepare("UPDATE eval_cases SET status = 'active' WHERE id = ?").run(id);
208
+ }
209
+ return getEvalCase(db, id);
210
+ });
211
+ }
212
+ /**
213
+ * The case a climb runs on (design §4.2). With `caseId`: that case, which must be an active case of `target` with a
214
+ * golden. Without: the newest such case with no succeeded climb against `baseArtifactId`. Throws NOT_FOUND,
215
+ * INVALID_REQUEST, EVAL_SET_TOO_SMALL (no case has a golden) or CONFLICT (every case was climbed against this base).
216
+ */
217
+ export function chooseClimbCase(db, target, baseArtifactId, caseId) {
218
+ if (caseId !== undefined) {
219
+ const c = getEvalCase(db, caseId);
220
+ if (!c)
221
+ throw new NexoError({ code: "NOT_FOUND", message: `Eval case ${caseId} does not exist`, context: { caseId }, hint: `List cases with \`nexo eval list --target ${target.key}\`` });
222
+ if (c.target !== target.key)
223
+ throw invalidRequest(`Eval case ${caseId} belongs to ${c.target}, not ${target.key}`, { caseId });
224
+ if (c.status !== "active")
225
+ throw invalidRequest(`Eval case ${caseId} is ${c.status}; climb needs an active case`, { caseId }, `Accept it with \`nexo eval accept ${caseId}\``);
226
+ if (c.expected === null)
227
+ throw invalidRequest(`Eval case ${caseId} has no golden (expected output)`, { caseId }, "Mark a run of this instruction done, or accept it with --expected");
228
+ return c;
229
+ }
230
+ const withGolden = listActiveEvalCases(db, target.key).filter((c) => c.expected !== null);
231
+ if (withGolden.length === 0) {
232
+ const runsToReview = runsNeedingReview(db, target).map((r) => r.runId);
233
+ const proposed = listEvalCases(db, target.key).filter((c) => c.status === "proposed").map((c) => c.id);
234
+ throw new NexoError({
235
+ code: "EVAL_SET_TOO_SMALL",
236
+ message: `Climbing ${target.key} needs an active eval case with a golden (a run marked done); it has none`,
237
+ hint: runsToReview.length > 0 ? `Mark one of these runs done: ${runsToReview.join(", ")}` : `Add one with \`nexo eval add --target ${target.key} --input "<instruction>" --expected "<answer>"\``,
238
+ context: { target: target.key, runsToReview, proposedCases: proposed },
239
+ });
240
+ }
241
+ const climbed = new Set(db
242
+ .prepare("SELECT case_id FROM optim_runs WHERE mode = 'climb' AND status = 'succeeded' AND base_artifact_id = ? AND case_id IS NOT NULL")
243
+ .all(baseArtifactId)
244
+ .map((row) => String(row.case_id)));
245
+ const next = [...withGolden].reverse().find((c) => !climbed.has(c.id));
246
+ if (!next) {
247
+ throw new NexoError({
248
+ code: "CONFLICT",
249
+ message: `Every eval case of ${target.key} with a golden was already climbed against the current version`,
250
+ hint: "Pass --case <id> to climb one again, or add cases",
251
+ context: { target: target.key, baseArtifactId },
252
+ });
253
+ }
254
+ return next;
255
+ }