@hexdecimal16/nexo 0.0.0-stage → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (209) hide show
  1. package/LICENSE +75 -0
  2. package/README.md +75 -2
  3. package/cli/src/bin.js +12 -0
  4. package/cli/src/common.js +78 -0
  5. package/cli/src/doctor.js +27 -0
  6. package/cli/src/eval-cases.js +65 -0
  7. package/cli/src/eval.js +67 -0
  8. package/cli/src/init.js +186 -0
  9. package/cli/src/interactive.js +171 -0
  10. package/cli/src/learn.js +178 -0
  11. package/cli/src/main.js +266 -0
  12. package/cli/src/optimize.js +289 -0
  13. package/cli/src/prompter.js +47 -0
  14. package/cli/src/review.js +120 -0
  15. package/cli/src/runtime.js +2 -0
  16. package/cli/src/sessions.js +34 -0
  17. package/cli/src/settings.js +169 -0
  18. package/cli/src/ui.js +112 -0
  19. package/cli/src/version.js +19 -0
  20. package/core/src/approval/classifier.js +152 -0
  21. package/core/src/config/load.js +283 -0
  22. package/core/src/config/schema.js +186 -0
  23. package/core/src/cost/admission.js +90 -0
  24. package/core/src/cost/budget.js +67 -0
  25. package/core/src/cost/pricing.js +53 -0
  26. package/core/src/engine/charged-completion.js +79 -0
  27. package/core/src/engine/errors.js +77 -0
  28. package/core/src/engine/mcp.js +11 -0
  29. package/core/src/engine/messages.js +86 -0
  30. package/core/src/engine/prompts.js +32 -0
  31. package/core/src/engine/provider-gate.js +33 -0
  32. package/core/src/engine/recorder.js +61 -0
  33. package/core/src/engine/run.js +1115 -0
  34. package/core/src/engine/semaphore.js +45 -0
  35. package/core/src/engine/spawn-tool.js +119 -0
  36. package/core/src/engine/task-graph.js +153 -0
  37. package/core/src/index.js +32 -0
  38. package/core/src/learn/caller.js +171 -0
  39. package/core/src/learn/context.js +112 -0
  40. package/core/src/learn/embedder.js +216 -0
  41. package/core/src/learn/job.js +196 -0
  42. package/core/src/learn/wiki.js +116 -0
  43. package/core/src/learn/worker.js +141 -0
  44. package/core/src/optimize/adapter.js +250 -0
  45. package/core/src/optimize/climb.js +338 -0
  46. package/core/src/optimize/mine.js +95 -0
  47. package/core/src/optimize/optimize.js +377 -0
  48. package/core/src/optimize/optimizer-run.js +67 -0
  49. package/core/src/optimize/reflect.js +100 -0
  50. package/core/src/optimize/scratchpad-wiki.js +69 -0
  51. package/core/src/providers/claude-p.js +363 -0
  52. package/core/src/providers/kiro-p.js +408 -0
  53. package/core/src/providers/models.js +10 -0
  54. package/core/src/scoring/calibration.js +31 -0
  55. package/core/src/scoring/continuation.js +24 -0
  56. package/core/src/scoring/runner.js +322 -0
  57. package/core/src/scoring/sources.js +88 -0
  58. package/core/src/session/lock.js +88 -0
  59. package/core/src/session/session.js +647 -0
  60. package/core/src/session/shell.js +26 -0
  61. package/core/src/session/transcript.js +86 -0
  62. package/core/src/settings/config-edit.js +472 -0
  63. package/core/src/settings/credentials.js +120 -0
  64. package/core/src/settings/doctor.js +103 -0
  65. package/core/src/settings/settings.js +265 -0
  66. package/mcp/src/hub.js +184 -0
  67. package/mcp/src/index.js +1 -0
  68. package/memory/src/curate.js +121 -0
  69. package/memory/src/index.js +4 -0
  70. package/memory/src/prompts.js +143 -0
  71. package/memory/src/retrieval.js +91 -0
  72. package/memory/src/similarity.js +47 -0
  73. package/optimize/src/engine.js +206 -0
  74. package/optimize/src/index.js +8 -0
  75. package/optimize/src/merge.js +73 -0
  76. package/optimize/src/novelty.js +37 -0
  77. package/optimize/src/pareto.js +83 -0
  78. package/optimize/src/proposer.js +148 -0
  79. package/optimize/src/report.js +107 -0
  80. package/optimize/src/rng.js +34 -0
  81. package/optimize/src/target.js +34 -0
  82. package/package.json +47 -4
  83. package/router/src/effort.js +5 -0
  84. package/router/src/escalation.js +171 -0
  85. package/router/src/index.js +4 -0
  86. package/router/src/registry.js +21 -0
  87. package/router/src/resolve.js +117 -0
  88. package/scorers/src/deterministic.js +194 -0
  89. package/scorers/src/family.js +48 -0
  90. package/scorers/src/index.js +6 -0
  91. package/scorers/src/judge.js +284 -0
  92. package/scorers/src/regex-worker.js +3 -0
  93. package/scorers/src/spec.js +147 -0
  94. package/scorers/src/trajectory.js +22 -0
  95. package/scorers/src/types.js +1 -0
  96. package/sdk/src/index.js +3 -0
  97. package/sdk/src/nexo.js +94 -0
  98. package/sdk/src/runtime.js +39 -0
  99. package/skills/bundled/browser/SKILL.md +57 -0
  100. package/skills/bundled/wiki/SKILL.md +73 -0
  101. package/skills/src/frontmatter.js +25 -0
  102. package/skills/src/index.js +3 -0
  103. package/skills/src/load-skill-tool.js +41 -0
  104. package/skills/src/loader.js +95 -0
  105. package/store/src/artifacts.js +124 -0
  106. package/store/src/budget-ledger.js +18 -0
  107. package/store/src/cost-report.js +46 -0
  108. package/store/src/cursor.js +35 -0
  109. package/store/src/db.js +83 -0
  110. package/store/src/errors.js +9 -0
  111. package/store/src/eval-cases.js +255 -0
  112. package/store/src/feedback.js +154 -0
  113. package/store/src/goldens.js +61 -0
  114. package/store/src/ids.js +6 -0
  115. package/store/src/index.js +25 -0
  116. package/store/src/judge-calibration.js +182 -0
  117. package/store/src/learn-jobs.js +145 -0
  118. package/store/src/memories.js +261 -0
  119. package/store/src/migrations/001_init.sql +81 -0
  120. package/store/src/migrations/002_feedback.sql +9 -0
  121. package/store/src/migrations/003_sessions.sql +19 -0
  122. package/store/src/migrations/004_scoring.sql +22 -0
  123. package/store/src/migrations/005_judge_calibration.sql +27 -0
  124. package/store/src/migrations/006_learning.sql +77 -0
  125. package/store/src/migrations/007_optimize.sql +55 -0
  126. package/store/src/migrations/008_mined_cases_and_climb.sql +46 -0
  127. package/store/src/mine.js +51 -0
  128. package/store/src/optim.js +291 -0
  129. package/store/src/paths.js +22 -0
  130. package/store/src/playbooks.js +199 -0
  131. package/store/src/reindex.js +143 -0
  132. package/store/src/run-dir.js +151 -0
  133. package/store/src/runs.js +310 -0
  134. package/store/src/scores.js +92 -0
  135. package/store/src/sessions.js +304 -0
  136. package/store/src/span-tree.js +98 -0
  137. package/store/src/spans.js +84 -0
  138. package/store/src/sql.js +13 -0
  139. package/store/src/workspace-copy.js +80 -0
  140. package/telemetry/src/errors.js +32 -0
  141. package/telemetry/src/index.js +3 -0
  142. package/telemetry/src/logger.js +39 -0
  143. package/telemetry/src/redact.js +106 -0
  144. package/tools/src/builtin.js +269 -0
  145. package/tools/src/change-tracker.js +57 -0
  146. package/tools/src/confine.js +101 -0
  147. package/tools/src/index.js +7 -0
  148. package/tools/src/leases.js +57 -0
  149. package/tools/src/policy.js +162 -0
  150. package/tools/src/read-only.js +338 -0
  151. package/tools/src/session-rules.js +140 -0
  152. package/tui/src/app.js +965 -0
  153. package/tui/src/approvals.js +96 -0
  154. package/tui/src/commands.js +38 -0
  155. package/tui/src/components.js +248 -0
  156. package/tui/src/index.js +5 -0
  157. package/tui/src/picker.js +78 -0
  158. package/tui/src/session-picker.js +100 -0
  159. package/tui/src/settings-screen.js +346 -0
  160. package/tui/src/text.js +70 -0
  161. package/tui/src/theme.js +44 -0
  162. package/web/public/cost.html +25 -0
  163. package/web/public/fonts/GeistMono-400.woff2 +0 -0
  164. package/web/public/fonts/GeistMono-600.woff2 +0 -0
  165. package/web/public/fonts/InstrumentSans-400.woff2 +0 -0
  166. package/web/public/fonts/InstrumentSans-600.woff2 +0 -0
  167. package/web/public/index.html +39 -0
  168. package/web/public/judge.html +21 -0
  169. package/web/public/learning.html +46 -0
  170. package/web/public/lib/api-types.ts +616 -0
  171. package/web/public/lib/common-view.ts +47 -0
  172. package/web/public/lib/cost-view.ts +35 -0
  173. package/web/public/lib/files-view.ts +90 -0
  174. package/web/public/lib/format.ts +35 -0
  175. package/web/public/lib/html.ts +51 -0
  176. package/web/public/lib/judge-view.ts +62 -0
  177. package/web/public/lib/learning-view.ts +102 -0
  178. package/web/public/lib/optim-view.ts +148 -0
  179. package/web/public/lib/pager.ts +56 -0
  180. package/web/public/lib/poller.ts +79 -0
  181. package/web/public/lib/run-view.ts +61 -0
  182. package/web/public/lib/runs-view.ts +87 -0
  183. package/web/public/lib/score-view.ts +94 -0
  184. package/web/public/lib/sessions-view.ts +28 -0
  185. package/web/public/lib/span-view.ts +90 -0
  186. package/web/public/lib/theme.ts +60 -0
  187. package/web/public/lib/timeline.ts +97 -0
  188. package/web/public/lib/wikis-view.ts +257 -0
  189. package/web/public/optim.html +30 -0
  190. package/web/public/pages/api.ts +60 -0
  191. package/web/public/pages/cost.ts +36 -0
  192. package/web/public/pages/judge.ts +15 -0
  193. package/web/public/pages/learning.ts +92 -0
  194. package/web/public/pages/optim.ts +68 -0
  195. package/web/public/pages/paging.ts +18 -0
  196. package/web/public/pages/run.ts +221 -0
  197. package/web/public/pages/runs.ts +91 -0
  198. package/web/public/pages/sessions.ts +25 -0
  199. package/web/public/pages/wikis.ts +103 -0
  200. package/web/public/run.html +36 -0
  201. package/web/public/sessions.html +28 -0
  202. package/web/public/style.css +288 -0
  203. package/web/public/wikis.html +25 -0
  204. package/web/src/api.js +682 -0
  205. package/web/src/http-errors.js +60 -0
  206. package/web/src/index.js +5 -0
  207. package/web/src/server.js +292 -0
  208. package/web/src/static.js +86 -0
  209. package/web/src/strip.js +22 -0
@@ -0,0 +1,291 @@
1
+ import { NexoError } from "../../telemetry/src/index.js";
2
+ import { getActiveArtifact, getArtifact } from "./artifacts.js";
3
+ import { checkLimit, checkOffset, decodeCursor, pageOf } from "./cursor.js";
4
+ import { transaction } from "./db.js";
5
+ import { RUNS_TO_REVIEW } from "./eval-cases.js";
6
+ import { agreementStats, confusionMatrix, judgeHumanPairs } from "./judge-calibration.js";
7
+ import { insertRow, updateById } from "./sql.js";
8
+ const RUN_COLUMNS = {
9
+ id: "id",
10
+ target: "target",
11
+ status: "status",
12
+ stopReason: "stop_reason",
13
+ budgetUsd: "budget_usd",
14
+ budgetRollouts: "budget_rollouts",
15
+ spentUsd: "spent_usd",
16
+ rollouts: "rollouts",
17
+ merge: "merge",
18
+ seed: "seed",
19
+ baseArtifactId: "base_artifact_id",
20
+ bestArtifactId: "best_artifact_id",
21
+ dirPath: "dir_path",
22
+ reportRef: "report_ref",
23
+ judgeKey: "judge_key",
24
+ judgeOnly: "judge_only",
25
+ holdoutBefore: "holdout_before",
26
+ holdoutAfter: "holdout_after",
27
+ promotion: "promotion",
28
+ replacedArtifactId: "replaced_artifact_id",
29
+ promotedAt: "promoted_at",
30
+ pid: "pid",
31
+ error: "error_json",
32
+ createdAt: "created_at",
33
+ endedAt: "ended_at",
34
+ mode: "mode",
35
+ caseId: "case_id",
36
+ climbOutcome: "climb_outcome",
37
+ };
38
+ const CANDIDATE_COLUMNS = {
39
+ id: "id",
40
+ optimRunId: "optim_run_id",
41
+ artifactId: "artifact_id",
42
+ parentIds: "parent_ids_json",
43
+ iteration: "iteration",
44
+ origin: "origin",
45
+ component: "component",
46
+ minibatch: "minibatch_json",
47
+ perCase: "per_case_json",
48
+ meanScore: "mean_score",
49
+ meanCostUsd: "mean_cost_usd",
50
+ accepted: "accepted",
51
+ createdAt: "created_at",
52
+ };
53
+ function encode(key, value) {
54
+ if (key === "merge" || key === "accepted")
55
+ return value ? 1 : 0;
56
+ if (key === "judgeOnly")
57
+ return value === null || value === undefined ? null : value ? 1 : 0;
58
+ if (key === "error" || key === "minibatch")
59
+ return value === null || value === undefined ? null : JSON.stringify(value);
60
+ if (key === "parentIds" || key === "perCase")
61
+ return JSON.stringify(value ?? (key === "parentIds" ? [] : {}));
62
+ return (value ?? null);
63
+ }
64
+ const num = (v) => (v === null || v === undefined ? null : Number(v));
65
+ const str = (v) => (v === null || v === undefined ? null : String(v));
66
+ function decodeRun(row) {
67
+ return {
68
+ id: String(row.id),
69
+ target: String(row.target),
70
+ status: row.status,
71
+ stopReason: row.stop_reason ?? null,
72
+ budgetUsd: Number(row.budget_usd),
73
+ budgetRollouts: num(row.budget_rollouts),
74
+ spentUsd: Number(row.spent_usd),
75
+ rollouts: Number(row.rollouts),
76
+ merge: Number(row.merge) === 1,
77
+ seed: Number(row.seed),
78
+ baseArtifactId: String(row.base_artifact_id),
79
+ bestArtifactId: str(row.best_artifact_id),
80
+ dirPath: String(row.dir_path),
81
+ reportRef: str(row.report_ref),
82
+ judgeKey: str(row.judge_key),
83
+ judgeOnly: row.judge_only === null || row.judge_only === undefined ? null : Number(row.judge_only) === 1,
84
+ holdoutBefore: num(row.holdout_before),
85
+ holdoutAfter: num(row.holdout_after),
86
+ promotion: row.promotion ?? null,
87
+ replacedArtifactId: str(row.replaced_artifact_id),
88
+ promotedAt: str(row.promoted_at),
89
+ pid: num(row.pid),
90
+ error: row.error_json ? JSON.parse(String(row.error_json)) : null,
91
+ createdAt: String(row.created_at),
92
+ endedAt: str(row.ended_at),
93
+ mode: row.mode === "climb" ? "climb" : "gepa",
94
+ caseId: str(row.case_id),
95
+ climbOutcome: row.climb_outcome ?? null,
96
+ };
97
+ }
98
+ function decodeCandidate(row) {
99
+ return {
100
+ id: String(row.id),
101
+ optimRunId: String(row.optim_run_id),
102
+ artifactId: String(row.artifact_id),
103
+ parentIds: JSON.parse(String(row.parent_ids_json)),
104
+ iteration: Number(row.iteration),
105
+ origin: row.origin,
106
+ component: str(row.component),
107
+ minibatch: row.minibatch_json ? JSON.parse(String(row.minibatch_json)) : null,
108
+ perCase: JSON.parse(String(row.per_case_json)),
109
+ meanScore: num(row.mean_score),
110
+ meanCostUsd: num(row.mean_cost_usd),
111
+ accepted: Number(row.accepted) === 1,
112
+ createdAt: String(row.created_at),
113
+ };
114
+ }
115
+ export function insertOptimRun(db, row) {
116
+ insertRow(db, "optim_runs", RUN_COLUMNS, { ...row }, encode);
117
+ }
118
+ export function updateOptimRun(db, id, patch) {
119
+ updateById(db, "optim_runs", RUN_COLUMNS, id, patch, encode);
120
+ }
121
+ export function getOptimRun(db, id) {
122
+ const row = db.prepare("SELECT * FROM optim_runs WHERE id = ?").get(id);
123
+ return row ? decodeRun(row) : undefined;
124
+ }
125
+ /** Optimizations, newest first. */
126
+ export function listOptimRuns(db, options = {}) {
127
+ return queryOptimRuns(db, options).runs;
128
+ }
129
+ /** One page of optimizations, newest first; `cursor` (from a previous page) continues after its last run. */
130
+ export function queryOptimRuns(db, options = {}) {
131
+ const limit = checkLimit(options.limit ?? 50, 200);
132
+ const offset = checkOffset(options.offset);
133
+ const after = options.cursor === undefined ? null : decodeCursor(options.cursor);
134
+ const total = Number(db.prepare("SELECT COUNT(*) AS n FROM optim_runs").get().n);
135
+ const rows = db
136
+ .prepare(`SELECT * FROM optim_runs ${after ? "WHERE (created_at < ? OR (created_at = ? AND id < ?)) " : ""}ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?`)
137
+ .all(...(after ? [after.key, after.key, after.id] : []), limit + 1, offset)
138
+ .map(decodeRun);
139
+ const { items, nextCursor } = pageOf(rows, limit, (r) => ({ key: r.createdAt, id: r.id }));
140
+ return { runs: items, nextCursor, total };
141
+ }
142
+ export function insertOptimCandidate(db, row) {
143
+ insertRow(db, "optim_candidates", CANDIDATE_COLUMNS, { ...row }, encode);
144
+ }
145
+ export function updateOptimCandidate(db, id, patch) {
146
+ updateById(db, "optim_candidates", CANDIDATE_COLUMNS, id, patch, encode);
147
+ }
148
+ /** An optimization's candidates, oldest first. */
149
+ export function listOptimCandidates(db, optimRunId) {
150
+ return db.prepare("SELECT * FROM optim_candidates WHERE optim_run_id = ? ORDER BY created_at, rowid").all(optimRunId).map(decodeCandidate);
151
+ }
152
+ /** Retires the optimization's candidate artifacts that are still `candidate`, except `keepArtifactId`. Returns how many. */
153
+ export function retireOptimCandidates(db, optimRunId, keepArtifactId) {
154
+ const result = db
155
+ .prepare(`UPDATE artifacts SET status = 'retired' WHERE status = 'candidate' AND id != ?
156
+ AND id IN (SELECT artifact_id FROM optim_candidates WHERE optim_run_id = ?)`)
157
+ .run(keepArtifactId ?? "", optimRunId);
158
+ return Number(result.changes);
159
+ }
160
+ /**
161
+ * Marks optimizations left `running` by processes that no longer exist as `aborted` (stop reason `process_exited`) and
162
+ * retires their candidates. Optimizations without a recorded pid are left alone. Returns the affected ids.
163
+ */
164
+ export function recoverCrashedOptimRuns(db, isAlive, nowIso) {
165
+ const ids = [];
166
+ for (const row of db.prepare("SELECT id, pid FROM optim_runs WHERE status = 'running' AND pid IS NOT NULL").all()) {
167
+ const pid = Number(row.pid);
168
+ if (isAlive(pid))
169
+ continue;
170
+ const id = String(row.id);
171
+ transaction(db, () => {
172
+ updateOptimRun(db, id, {
173
+ status: "aborted",
174
+ stopReason: "process_exited",
175
+ endedAt: nowIso,
176
+ error: { code: "INTERNAL", message: `The nexo process (pid ${pid}) exited while this optimization was running`, retryable: false },
177
+ });
178
+ retireOptimCandidates(db, id, null);
179
+ });
180
+ ids.push(id);
181
+ }
182
+ return ids;
183
+ }
184
+ /** Root runs scored by `judgeKey` that have no human verdict yet, newest first; reviewing them calibrates the judge. */
185
+ export function judgedRunsToReview(db, judgeKey, limit = RUNS_TO_REVIEW) {
186
+ return db
187
+ .prepare(`SELECT r.id FROM runs r WHERE r.parent_run_id IS NULL AND r.optim_run_id IS NULL
188
+ AND EXISTS (SELECT 1 FROM scores s WHERE s.run_id = r.id AND s.scorer_type = 'llm_judge' AND s.status = 'ok' AND s.model = ?)
189
+ AND NOT EXISTS (SELECT 1 FROM feedback f WHERE f.run_id = r.id)
190
+ ORDER BY r.started_at DESC, r.id DESC LIMIT ?`)
191
+ .all(judgeKey, limit)
192
+ .map((row) => String(row.id));
193
+ }
194
+ /** Cohen's κ of `judgeKey` against the current human verdicts; null without pairs or variation. */
195
+ export function liveJudgeKappa(db, judgeKey) {
196
+ return agreementStats(confusionMatrix(judgeHumanPairs(db, judgeKey))).kappa;
197
+ }
198
+ function refusal(code, message, hint, context) {
199
+ return new NexoError({ code, message, hint, context }).toPayload();
200
+ }
201
+ /**
202
+ * Whether an optimization's best candidate may be promoted (spec §8.2.6): the optimization succeeded, found a candidate
203
+ * other than the base that is still a candidate, is not approved yet, whose target is still on the base version, and did not get worse on the holdout split; and,
204
+ * when the holdout evidence is judge-only, the judge's live κ is at least `minJudgeKappa`.
205
+ */
206
+ export function checkPromotion(db, run, minJudgeKappa) {
207
+ const deny = (error, kappa = null) => ({ allowed: false, error, kappa });
208
+ const ctx = { optimRunId: run.id };
209
+ if (run.status !== "succeeded")
210
+ return deny(refusal("CONFLICT", `Optimization ${run.id} is ${run.status}, not succeeded`, "Only a finished optimization with a report can be approved", ctx));
211
+ if (run.promotion === "approved")
212
+ return deny(refusal("CONFLICT", `Optimization ${run.id} is already approved`, `Roll it back with \`nexo optimize rollback ${run.id}\``, ctx));
213
+ if (!run.bestArtifactId || run.bestArtifactId === run.baseArtifactId) {
214
+ return deny(refusal("CONFLICT", `Optimization ${run.id} found nothing better than the version it started from`, "Run another optimization with a larger budget or more eval cases", ctx));
215
+ }
216
+ const best = getArtifact(db, run.bestArtifactId);
217
+ if (!best || best.status !== "candidate") {
218
+ return deny(refusal("CONFLICT", `The best candidate of optimization ${run.id} is ${best?.status ?? "missing"}, not a candidate`, "It was approved or retired already", ctx));
219
+ }
220
+ const active = getActiveArtifact(db, best.kind, best.name);
221
+ if (active?.id !== run.baseArtifactId) {
222
+ return deny(refusal("CONFLICT", `The target ${run.target} changed since the optimization started: its active version is no longer the one optimization ${run.id} optimized, so approving would override that change`, "Run a new optimization against the current version", { ...ctx, baseArtifactId: run.baseArtifactId, activeArtifactId: active?.id ?? null }));
223
+ }
224
+ if (run.holdoutBefore !== null && run.holdoutAfter !== null && run.holdoutAfter < run.holdoutBefore) {
225
+ return deny(refusal("CONFLICT", `The best candidate scored worse on the holdout split (${run.holdoutAfter.toFixed(3)}) than the version it started from (${run.holdoutBefore.toFixed(3)})`, "It overfits the training cases; do not promote it", { ...ctx, holdoutBefore: run.holdoutBefore, holdoutAfter: run.holdoutAfter }));
226
+ }
227
+ if (!run.judgeOnly)
228
+ return { allowed: true, error: null, kappa: null };
229
+ const key = run.judgeKey ?? "(no judge)";
230
+ const kappa = run.judgeKey ? liveJudgeKappa(db, run.judgeKey) : null;
231
+ if (kappa !== null && kappa >= minJudgeKappa)
232
+ return { allowed: true, error: null, kappa };
233
+ const runsToReview = run.judgeKey ? judgedRunsToReview(db, run.judgeKey) : [];
234
+ const why = kappa === null
235
+ ? `the judge ${key} has no Cohen's κ against your verdicts yet`
236
+ : `the judge ${key} agrees with your verdicts at Cohen's κ ${kappa.toFixed(2)}, below optimize.minJudgeKappa (${minJudgeKappa})`;
237
+ return deny(refusal("FORBIDDEN", `Promotion refused: the holdout evidence comes only from the judge, and ${why}`, runsToReview.length > 0
238
+ ? `Review these judged runs (done / not done) to calibrate the judge, then approve again: ${runsToReview.join(", ")}`
239
+ : "Review more judged runs (done / not done) to calibrate the judge, then approve again", { ...ctx, judgeKey: run.judgeKey, kappa, minJudgeKappa, runsToReview }), kappa);
240
+ }
241
+ /**
242
+ * Promotes an optimization's best candidate (a user action): in one transaction the target's active version is retired
243
+ * and the candidate becomes active, and the replaced version is recorded for rollback. Throws NOT_FOUND, or the
244
+ * CONFLICT / FORBIDDEN refusal of `checkPromotion`.
245
+ */
246
+ export function approveOptimRun(db, id, options) {
247
+ return transaction(db, () => {
248
+ const run = getOptimRun(db, id);
249
+ if (!run)
250
+ throw new NexoError({ code: "NOT_FOUND", message: `Optimization ${id} does not exist`, context: { optimRunId: id } });
251
+ const check = checkPromotion(db, run, options.minJudgeKappa);
252
+ if (check.error)
253
+ throw new NexoError({ ...check.error });
254
+ const best = getArtifact(db, run.bestArtifactId);
255
+ const active = getActiveArtifact(db, best.kind, best.name);
256
+ if (active)
257
+ db.prepare("UPDATE artifacts SET status = 'retired' WHERE id = ?").run(active.id);
258
+ db.prepare("UPDATE artifacts SET status = 'active' WHERE id = ?").run(best.id);
259
+ updateOptimRun(db, id, { promotion: "approved", replacedArtifactId: active?.id ?? null, promotedAt: options.nowIso });
260
+ return { promoted: { ...best, status: "active" }, replaced: active ? { ...active, status: "retired" } : null };
261
+ });
262
+ }
263
+ /**
264
+ * Undoes an approval (a user action): the promoted version is retired and the version it replaced is active again.
265
+ * Throws NOT_FOUND, or CONFLICT when the optimization was not approved or its promoted version is no longer active.
266
+ */
267
+ export function rollbackOptimRun(db, id, nowIso) {
268
+ return transaction(db, () => {
269
+ const run = getOptimRun(db, id);
270
+ if (!run)
271
+ throw new NexoError({ code: "NOT_FOUND", message: `Optimization ${id} does not exist`, context: { optimRunId: id } });
272
+ if (run.promotion !== "approved" || !run.bestArtifactId) {
273
+ throw new NexoError({ code: "CONFLICT", message: `Optimization ${id} is not approved, so there is nothing to roll back`, context: { optimRunId: id, promotion: run.promotion } });
274
+ }
275
+ const promoted = getArtifact(db, run.bestArtifactId);
276
+ if (promoted.status !== "active") {
277
+ throw new NexoError({
278
+ code: "CONFLICT",
279
+ message: `The version optimization ${id} promoted (v${promoted.version}) is no longer active`,
280
+ hint: "A newer version replaced it (an edit of its source or another approval); nothing was changed",
281
+ context: { optimRunId: id, artifactId: promoted.id, status: promoted.status },
282
+ });
283
+ }
284
+ db.prepare("UPDATE artifacts SET status = 'retired' WHERE id = ?").run(promoted.id);
285
+ const replaced = run.replacedArtifactId ? getArtifact(db, run.replacedArtifactId) : undefined;
286
+ if (replaced)
287
+ db.prepare("UPDATE artifacts SET status = 'active' WHERE id = ?").run(replaced.id);
288
+ updateOptimRun(db, id, { promotion: "rolled_back", promotedAt: nowIso });
289
+ return { active: replaced ? { ...replaced, status: "active" } : null, retired: { ...promoted, status: "retired" } };
290
+ });
291
+ }
@@ -0,0 +1,22 @@
1
+ import { mkdirSync } from "node:fs";
2
+ import { homedir } from "node:os";
3
+ import { join, resolve } from "node:path";
4
+ export function resolveNexoPaths(env = process.env) {
5
+ const home = env.NEXO_HOME ? resolve(env.NEXO_HOME) : join(homedir(), ".nexo");
6
+ return {
7
+ home,
8
+ db: join(home, "nexo.db"),
9
+ runs: join(home, "runs"),
10
+ sessions: join(home, "sessions"),
11
+ optim: join(home, "optim"),
12
+ logs: join(home, "logs"),
13
+ skills: join(home, "skills"),
14
+ config: join(home, "config.json"),
15
+ credentials: join(home, "credentials.json"),
16
+ wikis: join(home, "wikis"),
17
+ };
18
+ }
19
+ export function ensureNexoDirs(paths) {
20
+ for (const dir of [paths.home, paths.runs, paths.sessions, paths.optim, paths.logs, paths.skills, paths.wikis])
21
+ mkdirSync(dir, { recursive: true });
22
+ }
@@ -0,0 +1,199 @@
1
+ import { NexoError } from "../../telemetry/src/index.js";
2
+ import { appendArtifactVersion, getActiveArtifact } from "./artifacts.js";
3
+ import { transaction } from "./db.js";
4
+ import { invalidRequest } from "./errors.js";
5
+ import { newId } from "./ids.js";
6
+ /** Longest playbook item, in characters. */
7
+ export const MAX_PLAYBOOK_ITEM_CHARS = 500;
8
+ const SKILL_NAME = /^[a-z0-9]+(-[a-z0-9]+)*$/;
9
+ export function isSkillName(value) {
10
+ return typeof value === "string" && value.length <= 64 && SKILL_NAME.test(value);
11
+ }
12
+ function assertSkill(skill) {
13
+ if (!isSkillName(skill))
14
+ throw invalidRequest("skill must be a skill name ([a-z0-9-], at most 64 characters)", { skill });
15
+ }
16
+ function decode(row) {
17
+ return {
18
+ id: String(row.id),
19
+ skill: String(row.artifact_name),
20
+ key: String(row.item_key),
21
+ content: String(row.content),
22
+ helpful: Number(row.helpful),
23
+ harmful: Number(row.harmful),
24
+ source: row.source,
25
+ status: row.status,
26
+ sourceRunId: row.source_run_id === null || row.source_run_id === undefined ? null : String(row.source_run_id),
27
+ createdAt: String(row.created_at),
28
+ updatedAt: String(row.updated_at),
29
+ };
30
+ }
31
+ function history(db, itemId, action, content, actor, runId, nowIso) {
32
+ db.prepare("INSERT INTO playbook_item_history (id, item_id, action, content, actor, run_id, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)").run(newId(), itemId, action, content, actor, runId, nowIso);
33
+ }
34
+ function cleanContent(content, redact) {
35
+ const text = typeof content === "string" ? redact(content.trim().replace(/\s+/g, " ")) : "";
36
+ if (!text)
37
+ throw invalidRequest("playbook item content is empty");
38
+ if (text.length > MAX_PLAYBOOK_ITEM_CHARS)
39
+ throw invalidRequest(`playbook item is longer than ${MAX_PLAYBOOK_ITEM_CHARS} characters`, { length: text.length });
40
+ return text;
41
+ }
42
+ export function getPlaybookItem(db, skill, key) {
43
+ const row = db.prepare("SELECT * FROM playbook_items WHERE artifact_name = ? AND item_key = ?").get(skill, key);
44
+ return row ? decode(row) : undefined;
45
+ }
46
+ function itemNotFound(skill, key) {
47
+ return new NexoError({ code: "NOT_FOUND", message: `Playbook ${skill} has no item ${key}`, context: { skill, key } });
48
+ }
49
+ /** Items of one skill's playbook in key order; retired ones only when asked. */
50
+ export function listPlaybookItems(db, skill, options = {}) {
51
+ assertSkill(skill);
52
+ const where = options.includeRetired ? "" : " AND status = 'active'";
53
+ return db.prepare(`SELECT * FROM playbook_items WHERE artifact_name = ?${where} ORDER BY item_key`).all(skill).map(decode);
54
+ }
55
+ /** Skill names that have a playbook item, in name order. */
56
+ export function listPlaybookSkills(db) {
57
+ return db.prepare("SELECT DISTINCT artifact_name AS name FROM playbook_items ORDER BY artifact_name").all().map((r) => String(r.name));
58
+ }
59
+ /** The next free key of a skill's playbook: k001, k002, … */
60
+ function nextKey(db, skill) {
61
+ const rows = db.prepare("SELECT item_key FROM playbook_items WHERE artifact_name = ?").all(skill);
62
+ let max = 0;
63
+ for (const r of rows) {
64
+ const m = /^k(\d+)$/.exec(String(r.item_key));
65
+ if (m)
66
+ max = Math.max(max, Number(m[1]));
67
+ }
68
+ return `k${String(max + 1).padStart(3, "0")}`;
69
+ }
70
+ /** Adds an item under the next free key (redacted, length-capped). Throws INVALID_REQUEST. */
71
+ export function addPlaybookItem(db, input, redact, nowIso) {
72
+ assertSkill(input.skill);
73
+ const content = cleanContent(input.content, redact);
74
+ return transaction(db, () => {
75
+ const row = {
76
+ id: newId(),
77
+ skill: input.skill,
78
+ key: nextKey(db, input.skill),
79
+ content,
80
+ helpful: 0,
81
+ harmful: 0,
82
+ source: input.source,
83
+ status: "active",
84
+ sourceRunId: input.runId ?? null,
85
+ createdAt: nowIso,
86
+ updatedAt: nowIso,
87
+ };
88
+ db.prepare("INSERT INTO playbook_items (id, artifact_name, item_key, content, helpful, harmful, source, status, source_run_id, created_at, updated_at) VALUES (?, ?, ?, ?, 0, 0, ?, 'active', ?, ?, ?)").run(row.id, row.skill, row.key, content, row.source, row.sourceRunId, nowIso, nowIso);
89
+ history(db, row.id, "add", content, input.source, row.sourceRunId, nowIso);
90
+ return row;
91
+ });
92
+ }
93
+ function requireActive(db, skill, key) {
94
+ assertSkill(skill);
95
+ const item = getPlaybookItem(db, skill, key);
96
+ if (!item)
97
+ throw itemNotFound(skill, key);
98
+ if (item.status !== "active")
99
+ throw new NexoError({ code: "CONFLICT", message: `Playbook item ${skill}/${key} is retired`, context: { skill, key } });
100
+ return item;
101
+ }
102
+ function assertMayChange(actor, item) {
103
+ if (actor === "ace" && item.source === "human") {
104
+ throw new NexoError({
105
+ code: "FORBIDDEN",
106
+ message: `The curator cannot change the human playbook item ${item.skill}/${item.key}`,
107
+ hint: "Only a user action (the Learning page or `nexo playbook retire`) can change it",
108
+ context: { skill: item.skill, key: item.key, source: item.source },
109
+ });
110
+ }
111
+ }
112
+ /** Rewrites an item's content. Throws NOT_FOUND, CONFLICT (retired), FORBIDDEN (the curator on a human item), INVALID_REQUEST. */
113
+ export function updatePlaybookItem(db, skill, key, content, actor, redact, nowIso, runId = null) {
114
+ const text = cleanContent(content, redact);
115
+ return transaction(db, () => {
116
+ const item = requireActive(db, skill, key);
117
+ assertMayChange(actor, item);
118
+ db.prepare("UPDATE playbook_items SET content = ?, updated_at = ? WHERE id = ?").run(text, nowIso, item.id);
119
+ history(db, item.id, "update", text, actor, runId, nowIso);
120
+ return { ...item, content: text, updatedAt: nowIso };
121
+ });
122
+ }
123
+ /** Retires an item (kept with its history). Throws NOT_FOUND, CONFLICT and FORBIDDEN as `updatePlaybookItem`. */
124
+ export function retirePlaybookItem(db, skill, key, actor, nowIso, runId = null) {
125
+ return transaction(db, () => {
126
+ const item = requireActive(db, skill, key);
127
+ assertMayChange(actor, item);
128
+ db.prepare("UPDATE playbook_items SET status = 'retired', updated_at = ? WHERE id = ?").run(nowIso, item.id);
129
+ history(db, item.id, "retire", null, actor, runId, nowIso);
130
+ return { ...item, status: "retired", updatedAt: nowIso };
131
+ });
132
+ }
133
+ /** Counts one helpful or harmful use of an active item (human items too: counters are not content). Unknown or retired keys are ignored; returns whether it counted. */
134
+ export function tagPlaybookItem(db, skill, key, tag, runId, nowIso) {
135
+ const item = getPlaybookItem(db, skill, key);
136
+ if (!item || item.status !== "active")
137
+ return false;
138
+ db.prepare(`UPDATE playbook_items SET ${tag === "helpful" ? "helpful = helpful + 1" : "harmful = harmful + 1"}, updated_at = ? WHERE id = ?`).run(nowIso, item.id);
139
+ history(db, item.id, tag, null, "ace", runId, nowIso);
140
+ return true;
141
+ }
142
+ /** (helpful + 1) / (harmful + 1): the pruning order, lowest first. */
143
+ export function playbookItemRatio(item) {
144
+ return (item.helpful + 1) / (item.harmful + 1);
145
+ }
146
+ /**
147
+ * Keeps at most `max` active items: retires `ace` items with the lowest (helpful+1)/(harmful+1), oldest first on ties.
148
+ * Human items are never pruned, so a playbook with more than `max` human items keeps them all. Returns the pruned keys.
149
+ */
150
+ export function prunePlaybook(db, skill, max, nowIso, runId = null) {
151
+ return transaction(db, () => {
152
+ const items = listPlaybookItems(db, skill);
153
+ const excess = items.length - max;
154
+ if (excess <= 0)
155
+ return [];
156
+ const candidates = items
157
+ .filter((i) => i.source === "ace")
158
+ .sort((a, b) => playbookItemRatio(a) - playbookItemRatio(b) || a.createdAt.localeCompare(b.createdAt) || a.key.localeCompare(b.key))
159
+ .slice(0, excess);
160
+ for (const item of candidates) {
161
+ db.prepare("UPDATE playbook_items SET status = 'retired', updated_at = ? WHERE id = ?").run(nowIso, item.id);
162
+ history(db, item.id, "prune", null, "ace", runId, nowIso);
163
+ }
164
+ return candidates.map((i) => i.key);
165
+ });
166
+ }
167
+ export function playbookItemHistory(db, itemId) {
168
+ return db
169
+ .prepare("SELECT * FROM playbook_item_history WHERE item_id = ? ORDER BY created_at, id")
170
+ .all(itemId)
171
+ .map((r) => ({
172
+ id: String(r.id),
173
+ itemId: String(r.item_id),
174
+ action: r.action,
175
+ content: r.content === null ? null : String(r.content),
176
+ actor: r.actor,
177
+ runId: r.run_id === null ? null : String(r.run_id),
178
+ createdAt: String(r.created_at),
179
+ }));
180
+ }
181
+ /** The playbook text appended by load_skill: one line per active item, `- [key] content`. Empty without items. */
182
+ export function renderPlaybook(items) {
183
+ return items
184
+ .filter((i) => i.status === "active")
185
+ .map((i) => `- [${i.key}] ${i.content}`)
186
+ .join("\n");
187
+ }
188
+ /**
189
+ * Stores the skill's current active items as a new `playbook` artifact version with `origin` when the rendered text
190
+ * changed (an empty playbook too, once one existed). Returns the active version, or undefined when the skill never had
191
+ * a playbook.
192
+ */
193
+ export function publishPlaybook(db, skill, origin, nowIso) {
194
+ const content = renderPlaybook(listPlaybookItems(db, skill));
195
+ const active = getActiveArtifact(db, "playbook", skill);
196
+ if (!active && content === "")
197
+ return undefined;
198
+ return appendArtifactVersion(db, "playbook", skill, content, origin, nowIso);
199
+ }
@@ -0,0 +1,143 @@
1
+ import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ import { transaction } from "./db.js";
4
+ import { restoreFeedback } from "./feedback.js";
5
+ import { readRunJson, readTrajectory } from "./run-dir.js";
6
+ import { insertRun } from "./runs.js";
7
+ import { restoreScores } from "./scores.js";
8
+ import { sessionRowFromJson, upsertSessionRow } from "./sessions.js";
9
+ import { insertSpan } from "./spans.js";
10
+ /**
11
+ * Rebuilds the `runs` and `spans` tables from run directories, and the `sessions` table from session directories when
12
+ * `sessionsRoot` is given. With `optimRoot`, each optimization directory's own run and its `rollouts/` runs are indexed
13
+ * too. Directories are the source of truth; a run's session and optimization links come from its run.json.
14
+ * The feedback and scores tables are kept; a run with no feedback rows gets one back from its run.json `humanVerdict`,
15
+ * and a run with no score rows gets its run.json `scores` back.
16
+ * A run.json still marked running is indexed as aborted, because a reindexed row has no pid for crash recovery to check.
17
+ */
18
+ export function reindex(db, runsRoot, sessionsRoot, optimRoot) {
19
+ const result = { runs: 0, spans: 0, sessions: 0, skipped: [] };
20
+ transaction(db, () => {
21
+ db.exec("DELETE FROM spans; DELETE FROM runs;");
22
+ if (existsSync(runsRoot)) {
23
+ for (const name of readdirSync(runsRoot).sort())
24
+ indexDir(db, join(runsRoot, name), result);
25
+ }
26
+ if (optimRoot !== undefined && existsSync(optimRoot)) {
27
+ for (const name of readdirSync(optimRoot).sort())
28
+ indexOptimDir(db, join(optimRoot, name), result);
29
+ }
30
+ if (sessionsRoot === undefined)
31
+ return;
32
+ db.exec("DELETE FROM sessions;");
33
+ if (!existsSync(sessionsRoot))
34
+ return;
35
+ for (const name of readdirSync(sessionsRoot).sort())
36
+ indexSession(db, join(sessionsRoot, name), result);
37
+ });
38
+ return result;
39
+ }
40
+ /** An optimization directory: its own run (when it has a run.json) and every rollout under `rollouts/`. */
41
+ function indexOptimDir(db, dir, result) {
42
+ try {
43
+ if (!statSync(dir).isDirectory())
44
+ return;
45
+ }
46
+ catch {
47
+ return;
48
+ }
49
+ if (existsSync(join(dir, "run.json")))
50
+ indexDir(db, dir, result);
51
+ const rollouts = join(dir, "rollouts");
52
+ if (existsSync(rollouts)) {
53
+ for (const name of readdirSync(rollouts).sort())
54
+ indexDir(db, join(rollouts, name), result);
55
+ }
56
+ }
57
+ function indexSession(db, dir, result) {
58
+ try {
59
+ if (!statSync(dir).isDirectory())
60
+ return;
61
+ const json = JSON.parse(readFileSync(join(dir, "session.json"), "utf8"));
62
+ if (typeof json !== "object" || json === null || typeof json.id !== "string")
63
+ throw new Error("session.json is not a session");
64
+ upsertSessionRow(db, sessionRowFromJson(json, dir));
65
+ result.sessions++;
66
+ }
67
+ catch {
68
+ result.skipped.push(dir);
69
+ }
70
+ }
71
+ function indexDir(db, dir, result) {
72
+ try {
73
+ if (!statSync(dir).isDirectory())
74
+ return;
75
+ }
76
+ catch {
77
+ return;
78
+ }
79
+ const counts = indexRun(db, dir);
80
+ if (!counts) {
81
+ result.skipped.push(dir);
82
+ return;
83
+ }
84
+ result.runs++;
85
+ result.spans += counts.spans;
86
+ const subagents = join(dir, "subagents");
87
+ if (existsSync(subagents)) {
88
+ for (const name of readdirSync(subagents).sort())
89
+ indexDir(db, join(subagents, name), result);
90
+ }
91
+ }
92
+ /** Inserts one directory's run and spans atomically; returns null (leaving the index untouched) when it cannot be indexed. */
93
+ function indexRun(db, dir) {
94
+ db.exec("SAVEPOINT index_dir");
95
+ try {
96
+ const run = readRunJson(dir);
97
+ if (typeof run !== "object" || run === null || Array.isArray(run))
98
+ throw new Error("run.json is not an object");
99
+ const stillRunning = run.status === "running";
100
+ insertRun(db, {
101
+ id: run.id,
102
+ parentRunId: run.parentRunId,
103
+ rootRunId: run.rootRunId,
104
+ dirPath: dir,
105
+ instruction: run.instruction,
106
+ skill: run.skill,
107
+ status: stillRunning ? "aborted" : run.status,
108
+ configSnapshotHash: run.configSnapshotHash,
109
+ artifactSetId: run.artifactSetId,
110
+ modelFinal: run.model ? `${run.model.provider}/${run.model.id}` : null,
111
+ tierFinal: run.model?.role ?? null,
112
+ escalations: typeof run.escalations === "number" ? run.escalations : 0,
113
+ startedAt: run.startedAt,
114
+ endedAt: run.endedAt,
115
+ costUsd: run.costUsd,
116
+ costUnknown: run.costUnknown,
117
+ tokensIn: run.tokensIn,
118
+ tokensOut: run.tokensOut,
119
+ pid: null,
120
+ sessionId: run.sessionId ?? null,
121
+ turnIndex: run.turnIndex ?? null,
122
+ optimRunId: run.optimRunId ?? null,
123
+ error: stillRunning
124
+ ? { code: "INTERNAL", message: "The run was still marked running when the index was rebuilt", retryable: false }
125
+ : run.error,
126
+ });
127
+ if (run.humanVerdict)
128
+ restoreFeedback(db, run.id, run.humanVerdict);
129
+ if (Array.isArray(run.scores))
130
+ restoreScores(db, run.id, run.scores);
131
+ let spans = 0;
132
+ for (const span of readTrajectory(dir)) {
133
+ insertSpan(db, span);
134
+ spans++;
135
+ }
136
+ db.exec("RELEASE index_dir");
137
+ return { spans };
138
+ }
139
+ catch {
140
+ db.exec("ROLLBACK TO index_dir; RELEASE index_dir");
141
+ return null;
142
+ }
143
+ }