@hexdecimal16/nexo 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (208) hide show
  1. package/LICENSE +75 -0
  2. package/README.md +75 -2
  3. package/cli/src/bin.js +12 -0
  4. package/cli/src/common.js +77 -0
  5. package/cli/src/doctor.js +27 -0
  6. package/cli/src/eval-cases.js +65 -0
  7. package/cli/src/eval.js +67 -0
  8. package/cli/src/init.js +186 -0
  9. package/cli/src/interactive.js +171 -0
  10. package/cli/src/learn.js +178 -0
  11. package/cli/src/main.js +261 -0
  12. package/cli/src/optimize.js +289 -0
  13. package/cli/src/prompter.js +47 -0
  14. package/cli/src/review.js +120 -0
  15. package/cli/src/runtime.js +2 -0
  16. package/cli/src/sessions.js +34 -0
  17. package/cli/src/settings.js +169 -0
  18. package/cli/src/ui.js +112 -0
  19. package/core/src/approval/classifier.js +152 -0
  20. package/core/src/config/load.js +283 -0
  21. package/core/src/config/schema.js +186 -0
  22. package/core/src/cost/admission.js +90 -0
  23. package/core/src/cost/budget.js +67 -0
  24. package/core/src/cost/pricing.js +53 -0
  25. package/core/src/engine/charged-completion.js +79 -0
  26. package/core/src/engine/errors.js +77 -0
  27. package/core/src/engine/mcp.js +11 -0
  28. package/core/src/engine/messages.js +86 -0
  29. package/core/src/engine/prompts.js +32 -0
  30. package/core/src/engine/provider-gate.js +33 -0
  31. package/core/src/engine/recorder.js +61 -0
  32. package/core/src/engine/run.js +1115 -0
  33. package/core/src/engine/semaphore.js +45 -0
  34. package/core/src/engine/spawn-tool.js +119 -0
  35. package/core/src/engine/task-graph.js +153 -0
  36. package/core/src/index.js +32 -0
  37. package/core/src/learn/caller.js +171 -0
  38. package/core/src/learn/context.js +112 -0
  39. package/core/src/learn/embedder.js +216 -0
  40. package/core/src/learn/job.js +196 -0
  41. package/core/src/learn/wiki.js +116 -0
  42. package/core/src/learn/worker.js +141 -0
  43. package/core/src/optimize/adapter.js +250 -0
  44. package/core/src/optimize/climb.js +338 -0
  45. package/core/src/optimize/mine.js +95 -0
  46. package/core/src/optimize/optimize.js +377 -0
  47. package/core/src/optimize/optimizer-run.js +67 -0
  48. package/core/src/optimize/reflect.js +100 -0
  49. package/core/src/optimize/scratchpad-wiki.js +69 -0
  50. package/core/src/providers/claude-p.js +363 -0
  51. package/core/src/providers/kiro-p.js +408 -0
  52. package/core/src/providers/models.js +10 -0
  53. package/core/src/scoring/calibration.js +31 -0
  54. package/core/src/scoring/continuation.js +24 -0
  55. package/core/src/scoring/runner.js +322 -0
  56. package/core/src/scoring/sources.js +88 -0
  57. package/core/src/session/lock.js +88 -0
  58. package/core/src/session/session.js +647 -0
  59. package/core/src/session/shell.js +26 -0
  60. package/core/src/session/transcript.js +86 -0
  61. package/core/src/settings/config-edit.js +472 -0
  62. package/core/src/settings/credentials.js +120 -0
  63. package/core/src/settings/doctor.js +103 -0
  64. package/core/src/settings/settings.js +265 -0
  65. package/mcp/src/hub.js +184 -0
  66. package/mcp/src/index.js +1 -0
  67. package/memory/src/curate.js +121 -0
  68. package/memory/src/index.js +4 -0
  69. package/memory/src/prompts.js +143 -0
  70. package/memory/src/retrieval.js +91 -0
  71. package/memory/src/similarity.js +47 -0
  72. package/optimize/src/engine.js +206 -0
  73. package/optimize/src/index.js +8 -0
  74. package/optimize/src/merge.js +73 -0
  75. package/optimize/src/novelty.js +37 -0
  76. package/optimize/src/pareto.js +83 -0
  77. package/optimize/src/proposer.js +148 -0
  78. package/optimize/src/report.js +107 -0
  79. package/optimize/src/rng.js +34 -0
  80. package/optimize/src/target.js +34 -0
  81. package/package.json +47 -4
  82. package/router/src/effort.js +5 -0
  83. package/router/src/escalation.js +171 -0
  84. package/router/src/index.js +4 -0
  85. package/router/src/registry.js +21 -0
  86. package/router/src/resolve.js +117 -0
  87. package/scorers/src/deterministic.js +194 -0
  88. package/scorers/src/family.js +48 -0
  89. package/scorers/src/index.js +6 -0
  90. package/scorers/src/judge.js +284 -0
  91. package/scorers/src/regex-worker.js +3 -0
  92. package/scorers/src/spec.js +147 -0
  93. package/scorers/src/trajectory.js +22 -0
  94. package/scorers/src/types.js +1 -0
  95. package/sdk/src/index.js +3 -0
  96. package/sdk/src/nexo.js +94 -0
  97. package/sdk/src/runtime.js +39 -0
  98. package/skills/bundled/browser/SKILL.md +57 -0
  99. package/skills/bundled/wiki/SKILL.md +73 -0
  100. package/skills/src/frontmatter.js +25 -0
  101. package/skills/src/index.js +3 -0
  102. package/skills/src/load-skill-tool.js +41 -0
  103. package/skills/src/loader.js +95 -0
  104. package/store/src/artifacts.js +124 -0
  105. package/store/src/budget-ledger.js +18 -0
  106. package/store/src/cost-report.js +46 -0
  107. package/store/src/cursor.js +35 -0
  108. package/store/src/db.js +83 -0
  109. package/store/src/errors.js +9 -0
  110. package/store/src/eval-cases.js +255 -0
  111. package/store/src/feedback.js +154 -0
  112. package/store/src/goldens.js +61 -0
  113. package/store/src/ids.js +6 -0
  114. package/store/src/index.js +25 -0
  115. package/store/src/judge-calibration.js +182 -0
  116. package/store/src/learn-jobs.js +145 -0
  117. package/store/src/memories.js +261 -0
  118. package/store/src/migrations/001_init.sql +81 -0
  119. package/store/src/migrations/002_feedback.sql +9 -0
  120. package/store/src/migrations/003_sessions.sql +19 -0
  121. package/store/src/migrations/004_scoring.sql +22 -0
  122. package/store/src/migrations/005_judge_calibration.sql +27 -0
  123. package/store/src/migrations/006_learning.sql +77 -0
  124. package/store/src/migrations/007_optimize.sql +55 -0
  125. package/store/src/migrations/008_mined_cases_and_climb.sql +46 -0
  126. package/store/src/mine.js +51 -0
  127. package/store/src/optim.js +291 -0
  128. package/store/src/paths.js +22 -0
  129. package/store/src/playbooks.js +199 -0
  130. package/store/src/reindex.js +143 -0
  131. package/store/src/run-dir.js +151 -0
  132. package/store/src/runs.js +310 -0
  133. package/store/src/scores.js +92 -0
  134. package/store/src/sessions.js +304 -0
  135. package/store/src/span-tree.js +98 -0
  136. package/store/src/spans.js +84 -0
  137. package/store/src/sql.js +13 -0
  138. package/store/src/workspace-copy.js +80 -0
  139. package/telemetry/src/errors.js +32 -0
  140. package/telemetry/src/index.js +3 -0
  141. package/telemetry/src/logger.js +39 -0
  142. package/telemetry/src/redact.js +106 -0
  143. package/tools/src/builtin.js +269 -0
  144. package/tools/src/change-tracker.js +57 -0
  145. package/tools/src/confine.js +101 -0
  146. package/tools/src/index.js +7 -0
  147. package/tools/src/leases.js +57 -0
  148. package/tools/src/policy.js +162 -0
  149. package/tools/src/read-only.js +338 -0
  150. package/tools/src/session-rules.js +140 -0
  151. package/tui/src/app.js +948 -0
  152. package/tui/src/approvals.js +96 -0
  153. package/tui/src/commands.js +38 -0
  154. package/tui/src/components.js +248 -0
  155. package/tui/src/index.js +5 -0
  156. package/tui/src/picker.js +78 -0
  157. package/tui/src/session-picker.js +100 -0
  158. package/tui/src/settings-screen.js +346 -0
  159. package/tui/src/text.js +70 -0
  160. package/tui/src/theme.js +44 -0
  161. package/web/public/cost.html +25 -0
  162. package/web/public/fonts/GeistMono-400.woff2 +0 -0
  163. package/web/public/fonts/GeistMono-600.woff2 +0 -0
  164. package/web/public/fonts/InstrumentSans-400.woff2 +0 -0
  165. package/web/public/fonts/InstrumentSans-600.woff2 +0 -0
  166. package/web/public/index.html +39 -0
  167. package/web/public/judge.html +21 -0
  168. package/web/public/learning.html +46 -0
  169. package/web/public/lib/api-types.ts +616 -0
  170. package/web/public/lib/common-view.ts +47 -0
  171. package/web/public/lib/cost-view.ts +35 -0
  172. package/web/public/lib/files-view.ts +90 -0
  173. package/web/public/lib/format.ts +35 -0
  174. package/web/public/lib/html.ts +51 -0
  175. package/web/public/lib/judge-view.ts +62 -0
  176. package/web/public/lib/learning-view.ts +102 -0
  177. package/web/public/lib/optim-view.ts +148 -0
  178. package/web/public/lib/pager.ts +56 -0
  179. package/web/public/lib/poller.ts +79 -0
  180. package/web/public/lib/run-view.ts +61 -0
  181. package/web/public/lib/runs-view.ts +87 -0
  182. package/web/public/lib/score-view.ts +94 -0
  183. package/web/public/lib/sessions-view.ts +28 -0
  184. package/web/public/lib/span-view.ts +90 -0
  185. package/web/public/lib/theme.ts +60 -0
  186. package/web/public/lib/timeline.ts +97 -0
  187. package/web/public/lib/wikis-view.ts +257 -0
  188. package/web/public/optim.html +30 -0
  189. package/web/public/pages/api.ts +60 -0
  190. package/web/public/pages/cost.ts +36 -0
  191. package/web/public/pages/judge.ts +15 -0
  192. package/web/public/pages/learning.ts +92 -0
  193. package/web/public/pages/optim.ts +68 -0
  194. package/web/public/pages/paging.ts +18 -0
  195. package/web/public/pages/run.ts +221 -0
  196. package/web/public/pages/runs.ts +91 -0
  197. package/web/public/pages/sessions.ts +25 -0
  198. package/web/public/pages/wikis.ts +103 -0
  199. package/web/public/run.html +36 -0
  200. package/web/public/sessions.html +28 -0
  201. package/web/public/style.css +288 -0
  202. package/web/public/wikis.html +25 -0
  203. package/web/src/api.js +682 -0
  204. package/web/src/http-errors.js +60 -0
  205. package/web/src/index.js +5 -0
  206. package/web/src/server.js +292 -0
  207. package/web/src/static.js +86 -0
  208. package/web/src/strip.js +22 -0
@@ -0,0 +1,154 @@
1
+ import { existsSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ import { NexoError } from "../../telemetry/src/index.js";
4
+ import { transaction } from "./db.js";
5
+ import { invalidRequest, runNotFound } from "./errors.js";
6
+ import { newId } from "./ids.js";
7
+ import { syncJudgeDisagreement } from "./judge-calibration.js";
8
+ import { enqueueLearnJob } from "./learn-jobs.js";
9
+ import { addMemory, MAX_MEMORY_CHARS } from "./memories.js";
10
+ import { getRun } from "./runs.js";
11
+ export const VERDICTS = ["done", "not_done"];
12
+ /** Longest accepted feedback comment, in characters. */
13
+ export const MAX_COMMENT_CHARS = 10_000;
14
+ /** Instruction text quoted in a human lesson. */
15
+ const LESSON_INSTRUCTION_CHARS = 160;
16
+ /**
17
+ * The human lesson of a `not_done` comment (spec §8.1 human priority): a memory with source `human`, importance 1, scope
18
+ * `skill:<run skill>` or `user`, written without any model call. The same comment on the same run is stored once.
19
+ * Returns the memory, or null when there is no comment or it was already stored.
20
+ */
21
+ export function storeHumanLesson(db, run, comment, redact, nowIso) {
22
+ const text = comment.trim();
23
+ if (!text)
24
+ return null;
25
+ const first = (run.instruction.split("\n")[0] ?? "").trim();
26
+ const quoted = first.length > LESSON_INSTRUCTION_CHARS ? `${first.slice(0, LESSON_INSTRUCTION_CHARS)}…` : first;
27
+ const prefix = `The user marked "${quoted}" not done: `;
28
+ const room = MAX_MEMORY_CHARS - prefix.length;
29
+ const content = prefix + (text.length > room ? `${text.slice(0, room - 1)}…` : text);
30
+ const existing = db
31
+ .prepare("SELECT id FROM memories WHERE source = 'human' AND source_run_id = ? AND content = ?")
32
+ .get(run.id, redact(content));
33
+ if (existing)
34
+ return null;
35
+ return addMemory(db, { scope: run.skill ? `skill:${run.skill}` : "user", type: "lesson", content, importance: 1, source: "human", sourceRunId: run.id }, redact, nowIso);
36
+ }
37
+ function decode(row) {
38
+ return {
39
+ id: String(row.id),
40
+ runId: String(row.run_id),
41
+ verdict: row.verdict,
42
+ comment: String(row.comment),
43
+ createdAt: String(row.created_at),
44
+ };
45
+ }
46
+ export function isVerdict(value) {
47
+ return typeof value === "string" && VERDICTS.includes(value);
48
+ }
49
+ /**
50
+ * Validates and stores a human verdict for a finished run, then mirrors the latest verdict into the run's run.json
51
+ * (`humanVerdict`) when the directory still exists. A verdict that disagrees with the run's judge verdict stores a
52
+ * `judge_rubric` eval case (`syncJudgeDisagreement`). Throws INVALID_REQUEST (bad verdict or comment), NOT_FOUND (unknown
53
+ * run) or CONFLICT (run still running).
54
+ */
55
+ export function recordFeedback(db, input) {
56
+ if (!isVerdict(input.verdict)) {
57
+ throw invalidRequest(`verdict must be "done" or "not_done"`, { verdict: input.verdict ?? null });
58
+ }
59
+ const comment = input.comment ?? "";
60
+ if (typeof comment !== "string")
61
+ throw invalidRequest("comment must be a string");
62
+ if (comment.length > MAX_COMMENT_CHARS) {
63
+ throw invalidRequest(`comment is longer than ${MAX_COMMENT_CHARS} characters`, { length: comment.length });
64
+ }
65
+ const run = getRun(db, input.runId);
66
+ if (!run)
67
+ throw runNotFound(input.runId);
68
+ if (run.status === "running") {
69
+ throw new NexoError({
70
+ code: "CONFLICT",
71
+ message: `Run ${input.runId} is still running`,
72
+ hint: "Review the run after it finishes",
73
+ context: { runId: input.runId },
74
+ });
75
+ }
76
+ const row = {
77
+ id: newId(),
78
+ runId: run.id,
79
+ verdict: input.verdict,
80
+ comment: input.redact(comment.trim()),
81
+ createdAt: input.nowIso,
82
+ };
83
+ // The verdict, its eval case, the human lesson and the learn job are stored together or not at all.
84
+ transaction(db, () => {
85
+ db.prepare("INSERT INTO feedback (id, run_id, verdict, comment, created_at) VALUES (?, ?, ?, ?, ?)").run(row.id, row.runId, row.verdict, row.comment, row.createdAt);
86
+ syncJudgeDisagreement(db, run.id, input.nowIso);
87
+ if (input.learn) {
88
+ if (row.verdict === "not_done")
89
+ storeHumanLesson(db, run, row.comment, input.redact, input.nowIso);
90
+ enqueueLearnJob(db, { runId: run.id, trigger: "feedback" }, input.nowIso);
91
+ }
92
+ });
93
+ const latest = latestFeedback(db, run.id) ?? row;
94
+ writeHumanVerdict(run.dirPath, { verdict: latest.verdict, comment: latest.comment, createdAt: latest.createdAt });
95
+ return row;
96
+ }
97
+ /** All feedback for a run, oldest first. */
98
+ export function listFeedback(db, runId) {
99
+ return db.prepare("SELECT * FROM feedback WHERE run_id = ? ORDER BY created_at, id").all(runId).map(decode);
100
+ }
101
+ export function latestFeedback(db, runId) {
102
+ const row = db.prepare("SELECT * FROM feedback WHERE run_id = ? ORDER BY created_at DESC, id DESC LIMIT 1").get(runId);
103
+ return row ? decode(row) : undefined;
104
+ }
105
+ /** Inserts a feedback row taken from run.json during reindex, unless the run already has feedback. */
106
+ export function restoreFeedback(db, runId, verdict) {
107
+ if (!isVerdict(verdict.verdict) || typeof verdict.createdAt !== "string" || latestFeedback(db, runId))
108
+ return false;
109
+ db.prepare("INSERT INTO feedback (id, run_id, verdict, comment, created_at) VALUES (?, ?, ?, ?, ?)").run(newId(), runId, verdict.verdict, typeof verdict.comment === "string" ? verdict.comment : "", verdict.createdAt);
110
+ return true;
111
+ }
112
+ /**
113
+ * Mirrors `verdict` into run.json as `humanVerdict` with a read-compare-write: a verdict already in the file with a
114
+ * newer `createdAt` (written by another process) is kept. The feedback table stays authoritative; run.json is a copy
115
+ * that `reindex` reads back. A missing run.json is left alone and a corrupt one is rewritten. Read and write failures
116
+ * throw STORE_IO.
117
+ */
118
+ function writeHumanVerdict(dir, verdict) {
119
+ const file = join(dir, "run.json");
120
+ if (!existsSync(file))
121
+ return;
122
+ const tmp = `${file}.${process.pid}.tmp`;
123
+ try {
124
+ const text = readFileSync(file, "utf8");
125
+ let json = {};
126
+ try {
127
+ const parsed = JSON.parse(text);
128
+ if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed))
129
+ json = parsed;
130
+ }
131
+ catch {
132
+ json = {};
133
+ }
134
+ const current = json.humanVerdict;
135
+ if (typeof current?.createdAt === "string" && current.createdAt > verdict.createdAt)
136
+ return;
137
+ writeFileSync(tmp, `${JSON.stringify({ ...json, humanVerdict: verdict }, null, 2)}\n`);
138
+ renameSync(tmp, file);
139
+ }
140
+ catch (error) {
141
+ try {
142
+ unlinkSync(tmp);
143
+ }
144
+ catch {
145
+ // the tmp file may not exist
146
+ }
147
+ throw new NexoError({
148
+ code: "STORE_IO",
149
+ message: `Could not write the verdict to ${file}: ${error instanceof Error ? error.message : String(error)}`,
150
+ retryable: true,
151
+ context: { path: file },
152
+ });
153
+ }
154
+ }
@@ -0,0 +1,61 @@
1
+ import { NexoError } from "../../telemetry/src/index.js";
2
+ import { transaction } from "./db.js";
3
+ import { newId } from "./ids.js";
4
+ import { getEvalCase } from "./judge-calibration.js";
5
+ /** Longest eval-case text, in characters (also `MAX_EVAL_INPUT_CHARS` in eval-cases.ts). */
6
+ const MAX_GOLDEN_CHARS = 20_000;
7
+ function decode(row) {
8
+ return {
9
+ id: String(row.id),
10
+ caseId: String(row.case_id),
11
+ expected: String(row.expected),
12
+ source: row.source,
13
+ sourceRunId: row.source_run_id ?? null,
14
+ optimRunId: row.optim_run_id ?? null,
15
+ createdAt: String(row.created_at),
16
+ };
17
+ }
18
+ function requireCase(db, caseId) {
19
+ const c = getEvalCase(db, caseId);
20
+ if (!c)
21
+ throw new NexoError({ code: "NOT_FOUND", message: `Eval case ${caseId} does not exist`, context: { caseId }, hint: "List cases with `nexo eval list --target <target>`" });
22
+ return c;
23
+ }
24
+ /** Appends one value to a case's golden history. Callers write `eval_cases.expected` in the same transaction. */
25
+ export function recordGolden(db, input) {
26
+ const row = { id: newId(), caseId: input.caseId, expected: input.expected, source: input.source, sourceRunId: input.sourceRunId ?? null, optimRunId: input.optimRunId ?? null, createdAt: input.nowIso };
27
+ db.prepare("INSERT INTO eval_case_goldens (id, case_id, expected, source, source_run_id, optim_run_id, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)").run(row.id, row.caseId, row.expected, row.source, row.sourceRunId, row.optimRunId, row.createdAt);
28
+ return row;
29
+ }
30
+ /** A case's golden history, oldest first. */
31
+ export function listGoldens(db, caseId) {
32
+ return db.prepare("SELECT * FROM eval_case_goldens WHERE case_id = ? ORDER BY created_at, rowid").all(caseId).map(decode);
33
+ }
34
+ /** A climb's baseline beat the golden: it becomes the golden (redacted, clipped). Throws NOT_FOUND. */
35
+ export function raiseGolden(db, input) {
36
+ return transaction(db, () => {
37
+ requireCase(db, input.caseId);
38
+ const text = input.redact(input.expected);
39
+ const expected = text.length > MAX_GOLDEN_CHARS ? `${text.slice(0, MAX_GOLDEN_CHARS - 1)}…` : text;
40
+ db.prepare("UPDATE eval_cases SET expected = ?, golden_source = 'climb' WHERE id = ?").run(expected, input.caseId);
41
+ recordGolden(db, { caseId: input.caseId, expected, source: "climb", sourceRunId: input.sourceRunId, optimRunId: input.optimRunId, nowIso: input.nowIso });
42
+ return requireCase(db, input.caseId);
43
+ });
44
+ }
45
+ /**
46
+ * Undoes climb raises: the golden becomes the newest history value whose source is not `climb` (the last
47
+ * human-approved golden), or none. Throws NOT_FOUND, or CONFLICT when the current golden is not from climb.
48
+ */
49
+ export function revertGolden(db, caseId, nowIso) {
50
+ return transaction(db, () => {
51
+ const c = requireCase(db, caseId);
52
+ if (c.goldenSource !== "climb") {
53
+ throw new NexoError({ code: "CONFLICT", message: `The golden of eval case ${caseId} is not from climb, so there is nothing to revert`, context: { caseId, goldenSource: c.goldenSource } });
54
+ }
55
+ const human = listGoldens(db, caseId).filter((g) => g.source !== "climb").at(-1);
56
+ db.prepare("UPDATE eval_cases SET expected = ?, golden_source = ? WHERE id = ?").run(human?.expected ?? null, human?.source ?? null, caseId);
57
+ if (human)
58
+ recordGolden(db, { caseId, expected: human.expected, source: human.source, sourceRunId: human.sourceRunId, nowIso });
59
+ return requireCase(db, caseId);
60
+ });
61
+ }
@@ -0,0 +1,6 @@
1
+ import { monotonicFactory } from "ulid";
2
+ const nextUlid = monotonicFactory();
3
+ /** Time-sortable unique id (ULID) used for runs, spans and artifacts. */
4
+ export function newId() {
5
+ return nextUlid();
6
+ }
@@ -0,0 +1,25 @@
1
+ export * from "./artifacts.js";
2
+ export * from "./budget-ledger.js";
3
+ export * from "./cost-report.js";
4
+ export * from "./cursor.js";
5
+ export * from "./db.js";
6
+ export * from "./errors.js";
7
+ export * from "./eval-cases.js";
8
+ export * from "./feedback.js";
9
+ export * from "./goldens.js";
10
+ export * from "./ids.js";
11
+ export * from "./judge-calibration.js";
12
+ export * from "./learn-jobs.js";
13
+ export * from "./memories.js";
14
+ export * from "./mine.js";
15
+ export * from "./optim.js";
16
+ export * from "./paths.js";
17
+ export * from "./playbooks.js";
18
+ export * from "./runs.js";
19
+ export * from "./scores.js";
20
+ export * from "./sessions.js";
21
+ export * from "./span-tree.js";
22
+ export * from "./spans.js";
23
+ export * from "./reindex.js";
24
+ export * from "./run-dir.js";
25
+ export * from "./workspace-copy.js";
@@ -0,0 +1,182 @@
1
+ import { createHash } from "node:crypto";
2
+ import { NexoError } from "../../telemetry/src/index.js";
3
+ import { newId } from "./ids.js";
4
+ import { getRun } from "./runs.js";
5
+ /** Eval-case target for judge/human disagreements: they are evidence for the judge rubric. */
6
+ export const JUDGE_CASE_TARGET = "judge_rubric";
7
+ /** Stable 40/40/20 feedback/pareto/holdout split by the hash of `key` (spec §8.2); eval cases pass their normalized instruction. */
8
+ export function evalSplitFor(key) {
9
+ const bucket = createHash("sha256").update(key).digest().readUInt32BE(0) % 10;
10
+ return bucket < 4 ? "feedback" : bucket < 8 ? "pareto" : "holdout";
11
+ }
12
+ /** The latest human verdict and comment of a run. Read here directly, because feedback.ts calls into this module. */
13
+ function latestHuman(db, runId) {
14
+ const row = db.prepare("SELECT verdict, comment FROM feedback WHERE run_id = ? ORDER BY created_at DESC, id DESC LIMIT 1").get(runId);
15
+ return row ? { verdict: row.verdict, comment: String(row.comment) } : undefined;
16
+ }
17
+ /** The latest round's successful llm_judge verdict of a run, or null. */
18
+ export function latestJudgeVerdict(db, runId) {
19
+ const row = db
20
+ .prepare(`SELECT passed, model FROM scores WHERE run_id = ? AND scorer_type = 'llm_judge' AND status = 'ok'
21
+ AND round = (SELECT MAX(round) FROM scores WHERE run_id = ?) ORDER BY rowid LIMIT 1`)
22
+ .get(runId, runId);
23
+ if (!row || row.passed === null)
24
+ return null;
25
+ return { passed: Number(row.passed) === 1, model: row.model ?? null };
26
+ }
27
+ /** True when the run's latest judge verdict and latest human verdict disagree; false when either is missing. */
28
+ export function judgeDisagrees(db, runId) {
29
+ const judge = latestJudgeVerdict(db, runId);
30
+ const human = latestHuman(db, runId);
31
+ return judge !== null && human !== undefined && judge.passed !== (human.verdict === "done");
32
+ }
33
+ function oneOf(value, allowed, column) {
34
+ if (typeof value === "string" && allowed.includes(value))
35
+ return value;
36
+ throw new NexoError({ code: "INTERNAL", message: `eval_cases.${column} holds an unknown value`, context: { column, value } });
37
+ }
38
+ function decodeCase(row) {
39
+ return {
40
+ id: String(row.id),
41
+ target: String(row.target),
42
+ sourceRunId: row.source_run_id ?? null,
43
+ input: String(row.input),
44
+ expected: row.expected ?? null,
45
+ referenceFeedback: row.reference_feedback ?? null,
46
+ scorerSpec: row.scorer_spec_json ? JSON.parse(String(row.scorer_spec_json)) : null,
47
+ split: oneOf(row.split, ["feedback", "pareto", "holdout"], "split"),
48
+ createdAt: String(row.created_at),
49
+ skill: row.skill ?? null,
50
+ origin: oneOf(row.origin, ["run", "manual", "mined"], "origin"),
51
+ status: oneOf(row.status, ["proposed", "active", "dismissed"], "status"),
52
+ goldenSource: row.golden_source === null || row.golden_source === undefined ? null : oneOf(row.golden_source, ["run", "manual", "climb"], "golden_source"),
53
+ minedFrom: row.mined_from_json ? JSON.parse(String(row.mined_from_json)) : null,
54
+ };
55
+ }
56
+ export function listEvalCases(db, target) {
57
+ return db.prepare("SELECT * FROM eval_cases WHERE target = ? ORDER BY created_at, id").all(target).map(decodeCase);
58
+ }
59
+ export function getEvalCase(db, id) {
60
+ const row = db.prepare("SELECT * FROM eval_cases WHERE id = ?").get(id);
61
+ return row ? decodeCase(row) : undefined;
62
+ }
63
+ /** The cases an optimization or climb uses: `active` only. */
64
+ export function listActiveEvalCases(db, target) {
65
+ return db.prepare("SELECT * FROM eval_cases WHERE target = ? AND status = 'active' ORDER BY created_at, id").all(target).map(decodeCase);
66
+ }
67
+ /**
68
+ * After a human verdict: when the run's judge verdict disagrees, stores (or refreshes) a `judge_rubric` eval case with
69
+ * the human verdict as `expected` and the comment as `reference_feedback`. An existing case is refreshed even when the
70
+ * verdicts now agree, so `expected` always holds the latest human verdict. Returns whether the verdicts disagree.
71
+ */
72
+ export function syncJudgeDisagreement(db, runId, nowIso) {
73
+ const disagrees = judgeDisagrees(db, runId);
74
+ const human = latestHuman(db, runId);
75
+ const run = getRun(db, runId);
76
+ if (!human || !run)
77
+ return disagrees;
78
+ const existing = db.prepare("SELECT id FROM eval_cases WHERE target = ? AND source_run_id = ?").get(JUDGE_CASE_TARGET, runId);
79
+ if (existing) {
80
+ db.prepare("UPDATE eval_cases SET expected = ?, reference_feedback = ? WHERE id = ?").run(human.verdict, human.comment, String(existing.id));
81
+ }
82
+ else if (disagrees) {
83
+ db.prepare("INSERT INTO eval_cases (id, target, source_run_id, input, expected, reference_feedback, scorer_spec_json, split, created_at) VALUES (?, ?, ?, ?, ?, ?, NULL, ?, ?)").run(newId(), JUDGE_CASE_TARGET, runId, run.instruction, human.verdict, human.comment, evalSplitFor(runId), nowIso);
84
+ }
85
+ return disagrees;
86
+ }
87
+ /**
88
+ * Runs that have both a successful judge verdict (latest round) and a human verdict (latest feedback). With `judgeKey`,
89
+ * only verdicts by that judge (score `model`) count.
90
+ */
91
+ export function judgeHumanPairs(db, judgeKey) {
92
+ const rows = db
93
+ .prepare(`SELECT s.run_id AS run_id, s.passed AS passed, s.model AS model,
94
+ (SELECT f.verdict FROM feedback f WHERE f.run_id = s.run_id ORDER BY f.created_at DESC, f.id DESC LIMIT 1) AS verdict
95
+ FROM scores s
96
+ WHERE s.scorer_type = 'llm_judge' AND s.status = 'ok' AND s.passed IS NOT NULL
97
+ AND s.round = (SELECT MAX(s2.round) FROM scores s2 WHERE s2.run_id = s.run_id)
98
+ AND s.rowid = (SELECT MIN(s3.rowid) FROM scores s3 WHERE s3.run_id = s.run_id AND s3.round = s.round AND s3.scorer_type = 'llm_judge' AND s3.status = 'ok')
99
+ ORDER BY s.run_id`)
100
+ .all();
101
+ const pairs = [];
102
+ for (const row of rows) {
103
+ if (row.verdict !== "done" && row.verdict !== "not_done")
104
+ continue;
105
+ if (judgeKey !== undefined && row.model !== judgeKey)
106
+ continue;
107
+ pairs.push({ runId: String(row.run_id), judgePassed: Number(row.passed) === 1, verdict: row.verdict });
108
+ }
109
+ return pairs;
110
+ }
111
+ export function confusionMatrix(pairs) {
112
+ const m = { bothPass: 0, judgePassHumanFail: 0, judgeFailHumanPass: 0, bothFail: 0 };
113
+ for (const p of pairs) {
114
+ const done = p.verdict === "done";
115
+ if (p.judgePassed && done)
116
+ m.bothPass++;
117
+ else if (p.judgePassed)
118
+ m.judgePassHumanFail++;
119
+ else if (done)
120
+ m.judgeFailHumanPass++;
121
+ else
122
+ m.bothFail++;
123
+ }
124
+ return m;
125
+ }
126
+ /** Accuracy and Cohen's κ = (pₒ − pₑ) / (1 − pₑ); κ is null when pₑ = 1 (no variation to agree beyond chance). */
127
+ export function agreementStats(m) {
128
+ const cases = m.bothPass + m.judgePassHumanFail + m.judgeFailHumanPass + m.bothFail;
129
+ const agreements = m.bothPass + m.bothFail;
130
+ if (cases === 0)
131
+ return { cases, agreements, accuracy: null, kappa: null };
132
+ const po = agreements / cases;
133
+ const judgePass = (m.bothPass + m.judgePassHumanFail) / cases;
134
+ const humanPass = (m.bothPass + m.judgeFailHumanPass) / cases;
135
+ const pe = judgePass * humanPass + (1 - judgePass) * (1 - humanPass);
136
+ const kappa = Math.abs(1 - pe) < 1e-12 ? null : (po - pe) / (1 - pe);
137
+ return { cases, agreements, accuracy: po, kappa };
138
+ }
139
+ /** Computes and stores a calibration of `judgeKey` from the current judge and human verdicts. */
140
+ export function recordCalibration(db, judgeKey, nowIso) {
141
+ const matrix = confusionMatrix(judgeHumanPairs(db, judgeKey));
142
+ const stats = agreementStats(matrix);
143
+ const row = { id: newId(), judgeKey, ...stats, matrix, createdAt: nowIso };
144
+ db.prepare("INSERT INTO judge_calibrations (id, judge_key, cases, agreements, accuracy, kappa, matrix_json, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)").run(row.id, judgeKey, row.cases, row.agreements, row.accuracy, row.kappa, JSON.stringify(matrix), nowIso);
145
+ return row;
146
+ }
147
+ function decodeCalibration(row) {
148
+ return {
149
+ id: String(row.id),
150
+ judgeKey: String(row.judge_key),
151
+ cases: Number(row.cases),
152
+ agreements: Number(row.agreements),
153
+ accuracy: row.accuracy === null ? null : Number(row.accuracy),
154
+ kappa: row.kappa === null ? null : Number(row.kappa),
155
+ matrix: JSON.parse(String(row.matrix_json)),
156
+ createdAt: String(row.created_at),
157
+ };
158
+ }
159
+ /** Stored calibrations, newest first; with `judgeKey`, only that judge's. */
160
+ export function listCalibrations(db, options = {}) {
161
+ const limit = options.limit ?? 20;
162
+ const rows = options.judgeKey === undefined
163
+ ? db.prepare("SELECT * FROM judge_calibrations ORDER BY created_at DESC, id DESC LIMIT ?").all(limit)
164
+ : db.prepare("SELECT * FROM judge_calibrations WHERE judge_key = ? ORDER BY created_at DESC, id DESC LIMIT ?").all(options.judgeKey, limit);
165
+ return rows.map(decodeCalibration);
166
+ }
167
+ /** Runs whose latest judge verdict disagrees with their latest human verdict (optionally one judge's), newest run first. */
168
+ export function listDisagreements(db, options = {}) {
169
+ const out = [];
170
+ for (const pair of judgeHumanPairs(db, options.judgeKey)) {
171
+ if (pair.judgePassed === (pair.verdict === "done"))
172
+ continue;
173
+ const human = latestHuman(db, pair.runId);
174
+ const instruction = db.prepare("SELECT instruction FROM runs WHERE id = ?").get(pair.runId)?.instruction;
175
+ out.push({ runId: pair.runId, instruction: typeof instruction === "string" ? instruction : "", judgePassed: pair.judgePassed, verdict: pair.verdict, comment: human?.comment ?? "" });
176
+ }
177
+ return out.reverse().slice(0, options.limit ?? 50);
178
+ }
179
+ /** Cases in `status` across targets, newest first. */
180
+ export function listEvalCasesByStatus(db, status, limit) {
181
+ return db.prepare("SELECT * FROM eval_cases WHERE status = ? ORDER BY created_at DESC, id DESC LIMIT ?").all(status, limit).map(decodeCase);
182
+ }
@@ -0,0 +1,145 @@
1
+ import { transaction } from "./db.js";
2
+ import { checkLimit, checkOffset, decodeCursor, pageOf } from "./cursor.js";
3
+ import { invalidRequest } from "./errors.js";
4
+ import { newId } from "./ids.js";
5
+ /** Attempts per job, the first one included (spec §8.1). */
6
+ export const MAX_LEARN_ATTEMPTS = 3;
7
+ export const LEARN_JOB_STATUSES = ["pending", "running", "succeeded", "failed"];
8
+ function decode(row) {
9
+ const nullable = (v) => (v === null || v === undefined ? null : String(v));
10
+ return {
11
+ id: String(row.id),
12
+ runId: String(row.run_id),
13
+ kind: row.kind,
14
+ trigger: row.trigger,
15
+ status: row.status,
16
+ attempts: Number(row.attempts),
17
+ lastError: row.last_error ? JSON.parse(String(row.last_error)) : null,
18
+ claimedBy: nullable(row.claimed_by),
19
+ claimedAt: nullable(row.claimed_at),
20
+ createdAt: String(row.created_at),
21
+ updatedAt: String(row.updated_at),
22
+ };
23
+ }
24
+ /**
25
+ * Adds a pending job for the run, unless one is already pending (that job reads the latest verdict when it runs, so
26
+ * it covers this trigger too). Returns the new job, or null when one was already pending.
27
+ */
28
+ export function enqueueLearnJob(db, input, nowIso) {
29
+ const id = newId();
30
+ const kind = input.kind ?? "reflect";
31
+ const result = db
32
+ .prepare("INSERT OR IGNORE INTO learn_jobs (id, run_id, kind, trigger, status, attempts, created_at, updated_at) VALUES (?, ?, ?, ?, 'pending', 0, ?, ?)")
33
+ .run(id, input.runId, kind, input.trigger, nowIso, nowIso);
34
+ return Number(result.changes) === 1 ? (getLearnJob(db, id) ?? null) : null;
35
+ }
36
+ export function getLearnJob(db, id) {
37
+ const row = db.prepare("SELECT * FROM learn_jobs WHERE id = ?").get(id);
38
+ return row ? decode(row) : undefined;
39
+ }
40
+ /**
41
+ * Atomically takes the oldest pending job for `workerId` (status running, attempts + 1) in one UPDATE … RETURNING, so
42
+ * two processes on the same database never take the same job. Returns undefined when nothing is pending.
43
+ */
44
+ export function claimLearnJob(db, workerId, nowIso) {
45
+ const row = db
46
+ .prepare(`UPDATE learn_jobs SET status = 'running', attempts = attempts + 1, claimed_by = ?, claimed_at = ?, updated_at = ?
47
+ WHERE id = (SELECT id FROM learn_jobs WHERE status = 'pending' ORDER BY created_at, id LIMIT 1) AND status = 'pending'
48
+ RETURNING *`)
49
+ .get(workerId, nowIso, nowIso);
50
+ return row ? decode(row) : undefined;
51
+ }
52
+ /** Marks a job this worker holds as succeeded. Returns false when the worker no longer holds it (it was reclaimed). */
53
+ export function completeLearnJob(db, id, workerId, nowIso) {
54
+ const r = db
55
+ .prepare("UPDATE learn_jobs SET status = 'succeeded', last_error = NULL, updated_at = ? WHERE id = ? AND status = 'running' AND claimed_by = ?")
56
+ .run(nowIso, id, workerId);
57
+ return Number(r.changes) === 1;
58
+ }
59
+ /** Marks a job this worker holds as failed with `error`. Returns false when the worker no longer holds it. */
60
+ export function failLearnJob(db, id, workerId, error, nowIso) {
61
+ const r = db
62
+ .prepare("UPDATE learn_jobs SET status = 'failed', last_error = ?, updated_at = ? WHERE id = ? AND status = 'running' AND claimed_by = ?")
63
+ .run(JSON.stringify(error), nowIso, id, workerId);
64
+ return Number(r.changes) === 1;
65
+ }
66
+ /** Puts a job this worker holds back to pending without counting the attempt (the worker stopped before finishing it). */
67
+ export function releaseLearnJob(db, id, workerId, nowIso) {
68
+ const r = db
69
+ .prepare("UPDATE learn_jobs SET status = 'pending', attempts = MAX(attempts - 1, 0), claimed_by = NULL, claimed_at = NULL, updated_at = ? WHERE id = ? AND status = 'running' AND claimed_by = ?")
70
+ .run(nowIso, id, workerId);
71
+ return Number(r.changes) === 1;
72
+ }
73
+ /**
74
+ * Jobs left running by a crashed process (claimed before `staleBeforeIso`) go back to pending, or to failed once they
75
+ * used every attempt. Returns the ids put back to pending and the ids failed.
76
+ */
77
+ export function reclaimStaleLearnJobs(db, staleBeforeIso, nowIso) {
78
+ return transaction(db, () => {
79
+ const abandoned = { code: "INTERNAL", message: "The process running this job stopped before it finished", retryable: true };
80
+ const failed = db
81
+ .prepare("UPDATE learn_jobs SET status = 'failed', last_error = ?, updated_at = ? WHERE status = 'running' AND claimed_at < ? AND attempts >= ? RETURNING id")
82
+ .all(JSON.stringify(abandoned), nowIso, staleBeforeIso, MAX_LEARN_ATTEMPTS)
83
+ .map((r) => String(r.id));
84
+ const pending = db
85
+ .prepare(
86
+ // A run can have one pending job per kind; a stale job whose run already has a newer pending one is dropped into failed instead.
87
+ `UPDATE OR IGNORE learn_jobs SET status = 'pending', claimed_by = NULL, claimed_at = NULL, updated_at = ?
88
+ WHERE status = 'running' AND claimed_at < ? RETURNING id`)
89
+ .all(nowIso, staleBeforeIso)
90
+ .map((r) => String(r.id));
91
+ // The newer pending job covers the run, so the duplicate is failed out of attempts and `--retry` never revives it.
92
+ const leftover = db
93
+ .prepare("UPDATE learn_jobs SET status = 'failed', attempts = ?, last_error = ?, updated_at = ? WHERE status = 'running' AND claimed_at < ? RETURNING id")
94
+ .all(MAX_LEARN_ATTEMPTS, JSON.stringify(abandoned), nowIso, staleBeforeIso)
95
+ .map((r) => String(r.id));
96
+ return { pending, failed: [...failed, ...leftover] };
97
+ });
98
+ }
99
+ /** Failed jobs with attempts left go back to pending (`nexo learn --retry`). Returns how many. */
100
+ export function retryFailedLearnJobs(db, nowIso) {
101
+ const r = db
102
+ .prepare("UPDATE OR IGNORE learn_jobs SET status = 'pending', claimed_by = NULL, claimed_at = NULL, updated_at = ? WHERE status = 'failed' AND attempts < ?")
103
+ .run(nowIso, MAX_LEARN_ATTEMPTS);
104
+ return Number(r.changes);
105
+ }
106
+ /** Jobs by status (or run), newest first. Throws INVALID_REQUEST for a bad status or limit. */
107
+ export function listLearnJobs(db, filter = {}) {
108
+ return queryLearnJobs(db, filter).jobs;
109
+ }
110
+ /** One page of `listLearnJobs`, newest first; `cursor` (from a previous page) continues after its last job. */
111
+ export function queryLearnJobs(db, filter = {}) {
112
+ if (filter.status !== undefined && !LEARN_JOB_STATUSES.includes(filter.status)) {
113
+ throw invalidRequest(`status must be one of ${LEARN_JOB_STATUSES.join(", ")}`, { status: filter.status });
114
+ }
115
+ const limit = checkLimit(filter.limit ?? 50, 1000);
116
+ const where = [];
117
+ const args = [];
118
+ if (filter.status) {
119
+ where.push("status = ?");
120
+ args.push(filter.status);
121
+ }
122
+ if (filter.runId) {
123
+ where.push("run_id = ?");
124
+ args.push(filter.runId);
125
+ }
126
+ const offset = checkOffset(filter.offset);
127
+ const total = Number(db.prepare(`SELECT COUNT(*) AS n FROM learn_jobs ${where.length ? `WHERE ${where.join(" AND ")}` : ""}`).get(...args).n);
128
+ if (filter.cursor !== undefined) {
129
+ const after = decodeCursor(filter.cursor);
130
+ where.push("(created_at < ? OR (created_at = ? AND id < ?))");
131
+ args.push(after.key, after.key, after.id);
132
+ }
133
+ const rows = db
134
+ .prepare(`SELECT * FROM learn_jobs ${where.length ? `WHERE ${where.join(" AND ")}` : ""} ORDER BY created_at DESC, id DESC LIMIT ? OFFSET ?`)
135
+ .all(...args, limit + 1, offset)
136
+ .map(decode);
137
+ const { items, nextCursor } = pageOf(rows, limit, (j) => ({ key: j.createdAt, id: j.id }));
138
+ return { jobs: items, nextCursor, total };
139
+ }
140
+ export function countLearnJobs(db) {
141
+ const out = { pending: 0, running: 0, succeeded: 0, failed: 0 };
142
+ for (const r of db.prepare("SELECT status, COUNT(*) AS n FROM learn_jobs GROUP BY status").all())
143
+ out[r.status] = Number(r.n);
144
+ return out;
145
+ }