@hexdecimal16/nexo 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (208) hide show
  1. package/LICENSE +75 -0
  2. package/README.md +75 -2
  3. package/cli/src/bin.js +12 -0
  4. package/cli/src/common.js +77 -0
  5. package/cli/src/doctor.js +27 -0
  6. package/cli/src/eval-cases.js +65 -0
  7. package/cli/src/eval.js +67 -0
  8. package/cli/src/init.js +186 -0
  9. package/cli/src/interactive.js +171 -0
  10. package/cli/src/learn.js +178 -0
  11. package/cli/src/main.js +261 -0
  12. package/cli/src/optimize.js +289 -0
  13. package/cli/src/prompter.js +47 -0
  14. package/cli/src/review.js +120 -0
  15. package/cli/src/runtime.js +2 -0
  16. package/cli/src/sessions.js +34 -0
  17. package/cli/src/settings.js +169 -0
  18. package/cli/src/ui.js +112 -0
  19. package/core/src/approval/classifier.js +152 -0
  20. package/core/src/config/load.js +283 -0
  21. package/core/src/config/schema.js +186 -0
  22. package/core/src/cost/admission.js +90 -0
  23. package/core/src/cost/budget.js +67 -0
  24. package/core/src/cost/pricing.js +53 -0
  25. package/core/src/engine/charged-completion.js +79 -0
  26. package/core/src/engine/errors.js +77 -0
  27. package/core/src/engine/mcp.js +11 -0
  28. package/core/src/engine/messages.js +86 -0
  29. package/core/src/engine/prompts.js +32 -0
  30. package/core/src/engine/provider-gate.js +33 -0
  31. package/core/src/engine/recorder.js +61 -0
  32. package/core/src/engine/run.js +1115 -0
  33. package/core/src/engine/semaphore.js +45 -0
  34. package/core/src/engine/spawn-tool.js +119 -0
  35. package/core/src/engine/task-graph.js +153 -0
  36. package/core/src/index.js +32 -0
  37. package/core/src/learn/caller.js +171 -0
  38. package/core/src/learn/context.js +112 -0
  39. package/core/src/learn/embedder.js +216 -0
  40. package/core/src/learn/job.js +196 -0
  41. package/core/src/learn/wiki.js +116 -0
  42. package/core/src/learn/worker.js +141 -0
  43. package/core/src/optimize/adapter.js +250 -0
  44. package/core/src/optimize/climb.js +338 -0
  45. package/core/src/optimize/mine.js +95 -0
  46. package/core/src/optimize/optimize.js +377 -0
  47. package/core/src/optimize/optimizer-run.js +67 -0
  48. package/core/src/optimize/reflect.js +100 -0
  49. package/core/src/optimize/scratchpad-wiki.js +69 -0
  50. package/core/src/providers/claude-p.js +363 -0
  51. package/core/src/providers/kiro-p.js +408 -0
  52. package/core/src/providers/models.js +10 -0
  53. package/core/src/scoring/calibration.js +31 -0
  54. package/core/src/scoring/continuation.js +24 -0
  55. package/core/src/scoring/runner.js +322 -0
  56. package/core/src/scoring/sources.js +88 -0
  57. package/core/src/session/lock.js +88 -0
  58. package/core/src/session/session.js +647 -0
  59. package/core/src/session/shell.js +26 -0
  60. package/core/src/session/transcript.js +86 -0
  61. package/core/src/settings/config-edit.js +472 -0
  62. package/core/src/settings/credentials.js +120 -0
  63. package/core/src/settings/doctor.js +103 -0
  64. package/core/src/settings/settings.js +265 -0
  65. package/mcp/src/hub.js +184 -0
  66. package/mcp/src/index.js +1 -0
  67. package/memory/src/curate.js +121 -0
  68. package/memory/src/index.js +4 -0
  69. package/memory/src/prompts.js +143 -0
  70. package/memory/src/retrieval.js +91 -0
  71. package/memory/src/similarity.js +47 -0
  72. package/optimize/src/engine.js +206 -0
  73. package/optimize/src/index.js +8 -0
  74. package/optimize/src/merge.js +73 -0
  75. package/optimize/src/novelty.js +37 -0
  76. package/optimize/src/pareto.js +83 -0
  77. package/optimize/src/proposer.js +148 -0
  78. package/optimize/src/report.js +107 -0
  79. package/optimize/src/rng.js +34 -0
  80. package/optimize/src/target.js +34 -0
  81. package/package.json +47 -4
  82. package/router/src/effort.js +5 -0
  83. package/router/src/escalation.js +171 -0
  84. package/router/src/index.js +4 -0
  85. package/router/src/registry.js +21 -0
  86. package/router/src/resolve.js +117 -0
  87. package/scorers/src/deterministic.js +194 -0
  88. package/scorers/src/family.js +48 -0
  89. package/scorers/src/index.js +6 -0
  90. package/scorers/src/judge.js +284 -0
  91. package/scorers/src/regex-worker.js +3 -0
  92. package/scorers/src/spec.js +147 -0
  93. package/scorers/src/trajectory.js +22 -0
  94. package/scorers/src/types.js +1 -0
  95. package/sdk/src/index.js +3 -0
  96. package/sdk/src/nexo.js +94 -0
  97. package/sdk/src/runtime.js +39 -0
  98. package/skills/bundled/browser/SKILL.md +57 -0
  99. package/skills/bundled/wiki/SKILL.md +73 -0
  100. package/skills/src/frontmatter.js +25 -0
  101. package/skills/src/index.js +3 -0
  102. package/skills/src/load-skill-tool.js +41 -0
  103. package/skills/src/loader.js +95 -0
  104. package/store/src/artifacts.js +124 -0
  105. package/store/src/budget-ledger.js +18 -0
  106. package/store/src/cost-report.js +46 -0
  107. package/store/src/cursor.js +35 -0
  108. package/store/src/db.js +83 -0
  109. package/store/src/errors.js +9 -0
  110. package/store/src/eval-cases.js +255 -0
  111. package/store/src/feedback.js +154 -0
  112. package/store/src/goldens.js +61 -0
  113. package/store/src/ids.js +6 -0
  114. package/store/src/index.js +25 -0
  115. package/store/src/judge-calibration.js +182 -0
  116. package/store/src/learn-jobs.js +145 -0
  117. package/store/src/memories.js +261 -0
  118. package/store/src/migrations/001_init.sql +81 -0
  119. package/store/src/migrations/002_feedback.sql +9 -0
  120. package/store/src/migrations/003_sessions.sql +19 -0
  121. package/store/src/migrations/004_scoring.sql +22 -0
  122. package/store/src/migrations/005_judge_calibration.sql +27 -0
  123. package/store/src/migrations/006_learning.sql +77 -0
  124. package/store/src/migrations/007_optimize.sql +55 -0
  125. package/store/src/migrations/008_mined_cases_and_climb.sql +46 -0
  126. package/store/src/mine.js +51 -0
  127. package/store/src/optim.js +291 -0
  128. package/store/src/paths.js +22 -0
  129. package/store/src/playbooks.js +199 -0
  130. package/store/src/reindex.js +143 -0
  131. package/store/src/run-dir.js +151 -0
  132. package/store/src/runs.js +310 -0
  133. package/store/src/scores.js +92 -0
  134. package/store/src/sessions.js +304 -0
  135. package/store/src/span-tree.js +98 -0
  136. package/store/src/spans.js +84 -0
  137. package/store/src/sql.js +13 -0
  138. package/store/src/workspace-copy.js +80 -0
  139. package/telemetry/src/errors.js +32 -0
  140. package/telemetry/src/index.js +3 -0
  141. package/telemetry/src/logger.js +39 -0
  142. package/telemetry/src/redact.js +106 -0
  143. package/tools/src/builtin.js +269 -0
  144. package/tools/src/change-tracker.js +57 -0
  145. package/tools/src/confine.js +101 -0
  146. package/tools/src/index.js +7 -0
  147. package/tools/src/leases.js +57 -0
  148. package/tools/src/policy.js +162 -0
  149. package/tools/src/read-only.js +338 -0
  150. package/tools/src/session-rules.js +140 -0
  151. package/tui/src/app.js +948 -0
  152. package/tui/src/approvals.js +96 -0
  153. package/tui/src/commands.js +38 -0
  154. package/tui/src/components.js +248 -0
  155. package/tui/src/index.js +5 -0
  156. package/tui/src/picker.js +78 -0
  157. package/tui/src/session-picker.js +100 -0
  158. package/tui/src/settings-screen.js +346 -0
  159. package/tui/src/text.js +70 -0
  160. package/tui/src/theme.js +44 -0
  161. package/web/public/cost.html +25 -0
  162. package/web/public/fonts/GeistMono-400.woff2 +0 -0
  163. package/web/public/fonts/GeistMono-600.woff2 +0 -0
  164. package/web/public/fonts/InstrumentSans-400.woff2 +0 -0
  165. package/web/public/fonts/InstrumentSans-600.woff2 +0 -0
  166. package/web/public/index.html +39 -0
  167. package/web/public/judge.html +21 -0
  168. package/web/public/learning.html +46 -0
  169. package/web/public/lib/api-types.ts +616 -0
  170. package/web/public/lib/common-view.ts +47 -0
  171. package/web/public/lib/cost-view.ts +35 -0
  172. package/web/public/lib/files-view.ts +90 -0
  173. package/web/public/lib/format.ts +35 -0
  174. package/web/public/lib/html.ts +51 -0
  175. package/web/public/lib/judge-view.ts +62 -0
  176. package/web/public/lib/learning-view.ts +102 -0
  177. package/web/public/lib/optim-view.ts +148 -0
  178. package/web/public/lib/pager.ts +56 -0
  179. package/web/public/lib/poller.ts +79 -0
  180. package/web/public/lib/run-view.ts +61 -0
  181. package/web/public/lib/runs-view.ts +87 -0
  182. package/web/public/lib/score-view.ts +94 -0
  183. package/web/public/lib/sessions-view.ts +28 -0
  184. package/web/public/lib/span-view.ts +90 -0
  185. package/web/public/lib/theme.ts +60 -0
  186. package/web/public/lib/timeline.ts +97 -0
  187. package/web/public/lib/wikis-view.ts +257 -0
  188. package/web/public/optim.html +30 -0
  189. package/web/public/pages/api.ts +60 -0
  190. package/web/public/pages/cost.ts +36 -0
  191. package/web/public/pages/judge.ts +15 -0
  192. package/web/public/pages/learning.ts +92 -0
  193. package/web/public/pages/optim.ts +68 -0
  194. package/web/public/pages/paging.ts +18 -0
  195. package/web/public/pages/run.ts +221 -0
  196. package/web/public/pages/runs.ts +91 -0
  197. package/web/public/pages/sessions.ts +25 -0
  198. package/web/public/pages/wikis.ts +103 -0
  199. package/web/public/run.html +36 -0
  200. package/web/public/sessions.html +28 -0
  201. package/web/public/style.css +288 -0
  202. package/web/public/wikis.html +25 -0
  203. package/web/src/api.js +682 -0
  204. package/web/src/http-errors.js +60 -0
  205. package/web/src/index.js +5 -0
  206. package/web/src/server.js +292 -0
  207. package/web/src/static.js +86 -0
  208. package/web/src/strip.js +22 -0
@@ -0,0 +1,322 @@
1
+ import { realpathSync } from "node:fs";
2
+ import { join, sep } from "node:path";
3
+ import { commandScorer, containsScorer, DATA_BLOCK_NOTE, dataBlock, humanScorer, judgeScorer, jsonSchemaScorer, loadScorerFile, regexScorer, scorerName, } from "../../../scorers/src/index.js";
4
+ import { insertScore, latestFeedback, newId } from "../../../store/src/index.js";
5
+ import { NexoError, toErrorPayload } from "../../../telemetry/src/index.js";
6
+ import { createPolicy, runBash } from "../../../tools/src/index.js";
7
+ import { DEFAULT_CONFIG } from "../config/schema.js";
8
+ import { runChargedCompletion } from "../engine/charged-completion.js";
9
+ import { judgeModels } from "./sources.js";
10
+ /** Longest tool-argument text kept per step of the trajectory summary. */
11
+ const STEP_DETAIL_CHARS = 200;
12
+ /** True when `dir` is `root` or inside it, comparing real paths. */
13
+ function isInside(dir, root) {
14
+ try {
15
+ const d = realpathSync(dir);
16
+ const r = realpathSync(root);
17
+ return d === r || d.startsWith(r + sep);
18
+ }
19
+ catch {
20
+ return false;
21
+ }
22
+ }
23
+ /**
24
+ * The scorers of a run, in priority order: the skill's `nexo-scorers` file, then `scoring.scorers`, then the judge alone
25
+ * when `models.judge` is set. Subagent runs are scored only with `scoring.subagents` or their own skill file. Human
26
+ * specs are left out: human verdicts arrive after the run, through the feedback table. Called before the agent loop
27
+ * starts, so the agent cannot change the run's scorers by editing a scorer file.
28
+ */
29
+ export function planScoring(env, skill, depth, workspace) {
30
+ const { config, paths } = env.deps;
31
+ const none = { specs: [], source: null, specError: null, origin: "user" };
32
+ if (!config.scoring.enabled)
33
+ return none;
34
+ const runnable = (specs) => specs.filter((s) => s.kind !== "human");
35
+ if (skill?.meta.scorers) {
36
+ // A skill inside the run's workspace is project-level even under NEXO_HOME (a session started in the home directory).
37
+ const origin = isInside(skill.dir, paths.home) && !isInside(skill.dir, workspace) ? "user" : "project";
38
+ try {
39
+ return { specs: runnable(loadScorerFile(join(skill.dir, skill.meta.scorers))), source: `skill:${skill.name}`, specError: null, origin };
40
+ }
41
+ catch (error) {
42
+ return { specs: [], source: `skill:${skill.name}`, specError: toErrorPayload(error), origin };
43
+ }
44
+ }
45
+ if (depth > 0 && !config.scoring.subagents)
46
+ return none;
47
+ if (config.scoring.scorers.length > 0) {
48
+ return { specs: runnable(config.scoring.scorers), source: "config", specError: null, origin: config.scoring.scorersOrigin ?? "project" };
49
+ }
50
+ if (judgeModels(config).length > 0)
51
+ return { specs: [{ kind: "llm_judge" }], source: "default-judge", specError: null, origin: "user" };
52
+ return none;
53
+ }
54
+ /** Environment variables that hold provider credentials (API keys, tokens, credential files, cloud access keys). */
55
+ const PROVIDER_KEY_VAR = /(^|_)(API_KEY|AUTH_TOKEN|OAUTH_TOKEN|ACCESS_KEY_ID|SECRET_ACCESS_KEY|SESSION_TOKEN|BEARER_TOKEN(_[A-Z0-9_]+)?|CREDENTIALS|TOKEN_FILE)$|^AWS_CONTAINER_|^(HF_TOKEN|COPILOT_GITHUB_TOKEN|GH_TOKEN|GITHUB_TOKEN)$/;
56
+ /** The environment command scorers run with: the process environment without provider credential variables. */
57
+ export function scorerEnv(env) {
58
+ const out = {};
59
+ for (const [key, value] of Object.entries(env))
60
+ if (!PROVIDER_KEY_VAR.test(key.toUpperCase()))
61
+ out[key] = value;
62
+ return out;
63
+ }
64
+ /** The trajectory summary scorers see: tool, skill and subagent steps with redacted arguments, errors and files. */
65
+ export function summarizeSpans(spans) {
66
+ return spans
67
+ .filter((s) => s.kind === "tool_call" || s.kind === "skill_load" || s.kind === "subagent" || s.kind === "escalation")
68
+ .map((s) => {
69
+ const args = (s.attrs.args ?? {});
70
+ const detail = s.kind === "escalation" ? String(s.attrs.reason ?? "") : JSON.stringify(args);
71
+ const result = typeof s.attrs.result === "string" ? s.attrs.result : "";
72
+ const file = (s.name === "write" || s.name === "edit") && s.status === "ok" && typeof args.path === "string" ? args.path : undefined;
73
+ const status = s.status === "running" ? "running" : s.status;
74
+ return {
75
+ kind: s.kind,
76
+ name: s.name,
77
+ status,
78
+ detail: detail.length > STEP_DETAIL_CHARS ? `${detail.slice(0, STEP_DETAIL_CHARS)}…` : detail,
79
+ ...(status === "error" && result ? { error: result.slice(0, STEP_DETAIL_CHARS) } : {}),
80
+ ...(file ? { file } : {}),
81
+ };
82
+ });
83
+ }
84
+ /**
85
+ * One LLM completion charged to the run and recorded as a span (`judge_call` for judges, `self_check` for self-checks):
86
+ * `runChargedCompletion` on the run budget, charged through `deps.charge` (the run's cost, tokens and the spend ledger).
87
+ * Parallel calls (panel members, both pairwise orders) reserve their estimates from the same budget, so together they
88
+ * never spend more than it. Throws BUDGET_EXCEEDED when the estimate cannot be reserved, a classified provider error, or
89
+ * on abort.
90
+ */
91
+ async function chargedCompletion(deps, model, request, span, signal) {
92
+ const { budget } = deps;
93
+ return runChargedCompletion({
94
+ env: deps.env,
95
+ recorder: deps.recorder,
96
+ budget,
97
+ charge: deps.charge,
98
+ refused: (estimate) => new NexoError({
99
+ code: "BUDGET_EXCEEDED",
100
+ message: `The run budget cannot cover the ${span.kind === "judge_call" ? "judge" : "self-check"} call (estimated $${estimate.toFixed(4)}, $${budget.remainingUsd().toFixed(4)} left)`,
101
+ hint: "Raise --budget or budgets.perRunUsd so scoring fits after the run",
102
+ }),
103
+ abortMessage: "The scoring call was aborted",
104
+ }, { model, system: request.system, user: request.user, effort: request.effort, span }, signal);
105
+ }
106
+ /** A JudgeCaller whose calls are `judge_call` spans (tier `judge`) under `scorerSpanId`, charged to the run. */
107
+ export function createJudgeCaller(deps, scorerSpanId) {
108
+ return {
109
+ complete: async (ref, request, signal) => {
110
+ const model = deps.env.registry.get(ref.provider, ref.id);
111
+ return chargedCompletion(deps, model, { system: request.system, user: request.user, effort: "off" }, { kind: "judge_call", name: `${request.purpose} ${model.id}`, parentSpanId: scorerSpanId(), tier: "judge", attrs: { purpose: request.purpose } }, signal);
112
+ },
113
+ };
114
+ }
115
+ /**
116
+ * How command scorers run: the bash tool's `runBash` (process group killed on timeout, abort and exit) without provider
117
+ * credentials in the environment. A project-level command first passes the user-level bash tool policy (tools.policy,
118
+ * tools.rules and tools.nonInteractiveAsk from the defaults, ~/.nexo/config.json and CLI flags; ./.nexo/config.json does
119
+ * not count): allow, deny, or ask, where ask uses the interactive prompt and otherwise follows tools.nonInteractiveAsk.
120
+ */
121
+ function commandRunner(deps, name, origin) {
122
+ const { env } = deps;
123
+ return async (run) => {
124
+ if (origin === "project") {
125
+ // Only the user-level policy (defaults, ~/.nexo/config.json, CLI flags) applies, so a project cannot approve its own commands.
126
+ const policy = createPolicy(env.deps.config.scoring.scorerTools ?? DEFAULT_CONFIG.tools);
127
+ const auth = await policy.authorize(env.deps.prompter, "bash", { command: run.command }, run.signal, `scorer ${name}`);
128
+ if (!auth.allowed) {
129
+ throw new NexoError({
130
+ code: "SCORER_FAILED",
131
+ message: `Command scorer ${name} was not run: ${auth.error.message}`,
132
+ hint: "Allow the command in tools.policy or tools.rules of ~/.nexo/config.json (project config cannot approve it), or define the scorer in ~/.nexo",
133
+ context: { cause: auth.error.toPayload(), origin },
134
+ });
135
+ }
136
+ }
137
+ return runBash({ ...run, env: scorerEnv(process.env) });
138
+ };
139
+ }
140
+ function buildScorer(deps, spec, caller, origin) {
141
+ const { config } = deps.env.deps;
142
+ const name = scorerName(spec);
143
+ switch (spec.kind) {
144
+ case "command":
145
+ return commandScorer({ name, command: spec.command, timeoutMs: spec.timeoutMs ?? config.scoring.commandTimeoutMs, run: commandRunner(deps, name, origin) });
146
+ case "json_schema":
147
+ return jsonSchemaScorer({ name, schema: spec.schema });
148
+ case "regex":
149
+ return regexScorer({ name, pattern: spec.pattern, flags: spec.flags, negate: spec.negate });
150
+ case "contains":
151
+ return containsScorer({ name, text: spec.text, ignoreCase: spec.ignoreCase, negate: spec.negate });
152
+ case "llm_judge":
153
+ return judgeScorer({
154
+ name,
155
+ caller,
156
+ models: judgeModels(config),
157
+ rubric: deps.rubric,
158
+ extraRubric: spec.rubric,
159
+ passThreshold: config.scoring.passThreshold,
160
+ generators: deps.generators(),
161
+ });
162
+ case "human":
163
+ return humanScorer({ name, readVerdict: (runId) => latestFeedback(deps.env.deps.db, runId) });
164
+ }
165
+ }
166
+ function emitScore(deps, row) {
167
+ const event = {
168
+ type: "score",
169
+ runId: deps.runId,
170
+ depth: deps.depth,
171
+ round: row.round,
172
+ scorer: row.scorerName,
173
+ kind: row.scorerKind,
174
+ status: row.status,
175
+ value: row.value,
176
+ passed: row.passed,
177
+ costUsd: row.costUsd,
178
+ };
179
+ deps.env.deps.onEvent?.(event);
180
+ }
181
+ /**
182
+ * Runs every scorer of the plan once (one round), sequentially, and records each result: a `scorer` span, a scores row,
183
+ * files under `scores/round-<n>/` and a `score` event. A scorer that throws is recorded as failed with its error (spec
184
+ * §12: scorer failures never change the run's status). Returns the round's rows.
185
+ */
186
+ export async function runScoringRound(deps, plan, input, round) {
187
+ const { env, recorder, runDir } = deps;
188
+ const { db, redact } = env.deps;
189
+ const rows = [];
190
+ const record = (row) => {
191
+ const safe = redact(row);
192
+ insertScore(db, safe);
193
+ rows.push(safe);
194
+ emitScore(deps, safe);
195
+ };
196
+ /** Recording a failure must not fail the run (spec §12): a write error is logged and scoring continues. */
197
+ const guarded = (what, fn) => {
198
+ try {
199
+ fn();
200
+ }
201
+ catch (error) {
202
+ env.deps.logger.error(`${what} failed`, { error, run_id: deps.runId });
203
+ }
204
+ };
205
+ if (plan.specError) {
206
+ guarded("recording a failed score", () => record({
207
+ id: newId(),
208
+ runId: deps.runId,
209
+ round,
210
+ scorerKind: "deterministic",
211
+ scorerType: "spec",
212
+ scorerName: "nexo-scorers",
213
+ status: "failed",
214
+ value: null,
215
+ passed: null,
216
+ rationale: "",
217
+ criteria: null,
218
+ details: { source: plan.source },
219
+ error: plan.specError,
220
+ model: null,
221
+ costUsd: 0,
222
+ costUnknown: false,
223
+ createdAt: env.now().toISOString(),
224
+ }));
225
+ }
226
+ const signal = deps.signal ?? new AbortController().signal;
227
+ for (const spec of plan.specs) {
228
+ if (signal.aborted)
229
+ break;
230
+ const name = scorerName(spec);
231
+ const spanId = recorder.start({ kind: "scorer", name, parentSpanId: deps.runSpanId, attrs: { type: spec.kind, round } });
232
+ const caller = createJudgeCaller(deps, () => spanId);
233
+ const base = { id: newId(), runId: deps.runId, round, scorerName: name, scorerType: spec.kind, createdAt: "" };
234
+ let scorer;
235
+ try {
236
+ scorer = buildScorer(deps, spec, caller, plan.origin);
237
+ const result = await scorer.score(input, signal);
238
+ const files = Object.entries(result.files ?? {}).map(([file, content]) => runDir.writeScoreFile(`round-${round}/${file}`, content));
239
+ const row = {
240
+ ...base,
241
+ scorerKind: scorer.kind,
242
+ status: "ok",
243
+ value: Math.min(1, Math.max(0, result.value)),
244
+ passed: result.passed,
245
+ rationale: result.rationale,
246
+ criteria: result.criteria ?? null,
247
+ details: { ...(result.details ?? {}), ...(files.length ? { files } : {}) },
248
+ error: null,
249
+ model: result.model ?? null,
250
+ costUsd: result.costUsd,
251
+ costUnknown: result.costUnknown ?? false,
252
+ createdAt: env.now().toISOString(),
253
+ };
254
+ record(row);
255
+ recorder.finish(spanId, { status: "ok", attrs: { value: row.value, passed: row.passed, costUsd: row.costUsd } });
256
+ }
257
+ catch (caught) {
258
+ if (signal.aborted) {
259
+ // An abort ends the run as aborted: the interrupted scorer gives no verdict and leaves no score row.
260
+ recorder.finish(spanId, { status: "error", attrs: { reason: "aborted" } });
261
+ break;
262
+ }
263
+ const error = caught instanceof NexoError && caught.code === "SCORER_FAILED" ? caught : new NexoError({
264
+ code: "SCORER_FAILED",
265
+ message: `Scorer ${name} failed: ${caught instanceof Error ? caught.message : String(caught)}`,
266
+ cause: caught,
267
+ context: caught instanceof NexoError ? { cause: caught.toPayload() } : {},
268
+ });
269
+ const ctx = error.context ?? {};
270
+ const ctxFiles = (ctx.files ?? {});
271
+ const files = [];
272
+ for (const [file, content] of Object.entries(ctxFiles)) {
273
+ guarded("writing a score file", () => files.push(runDir.writeScoreFile(`round-${round}/${file}`, content)));
274
+ }
275
+ const { files: _f, replies: _r, votes, ...rest } = ctx;
276
+ const payload = { ...error.toPayload(), context: rest };
277
+ guarded("recording a failed score", () => record({
278
+ ...base,
279
+ scorerKind: spec.kind === "llm_judge" ? "judge" : spec.kind === "human" ? "human" : "deterministic",
280
+ status: "failed",
281
+ value: null,
282
+ passed: null,
283
+ rationale: "",
284
+ criteria: null,
285
+ details: { ...(votes ? { votes } : {}), ...(files.length ? { files } : {}) },
286
+ error: payload,
287
+ model: typeof ctx.model === "string" ? ctx.model : null,
288
+ costUsd: typeof ctx.costUsd === "number" ? ctx.costUsd : 0,
289
+ costUnknown: ctx.costUnknown === true,
290
+ createdAt: env.now().toISOString(),
291
+ }));
292
+ guarded("finishing a scorer span", () => recorder.finish(spanId, { status: "error", attrs: { error: redact(payload) } }));
293
+ }
294
+ }
295
+ return rows;
296
+ }
297
+ const SELF_CHECK_SYSTEM = `You review your own answer before it is returned. Rate how completely and correctly the answer carries out the instruction.
298
+ ${DATA_BLOCK_NOTE}
299
+ Reply with only a JSON object: {"score": <number from 0 to 1>, "reason": "<one sentence>"}`;
300
+ /**
301
+ * Asks the run's current model to rate its final answer (the `self_check` trigger). Returns the score, or null when the
302
+ * call fails or the reply has no score in 0..1; a failed self-check never escalates.
303
+ */
304
+ export async function runSelfCheck(deps, model, effort, role, input) {
305
+ const signal = deps.signal ?? new AbortController().signal;
306
+ try {
307
+ const { text } = await chargedCompletion(deps, model, {
308
+ system: SELF_CHECK_SYSTEM,
309
+ user: `# Instruction\n${dataBlock("instruction", input.instruction)}\n\n# Your final answer\n${dataBlock("final_answer", input.finalOutput.slice(-20_000) || "(empty)")}`,
310
+ effort,
311
+ }, { kind: "self_check", name: model.id, parentSpanId: deps.runSpanId, tier: role, attrs: {} }, signal);
312
+ const match = /"score"\s*:\s*(-?\d+(?:\.\d+)?)/.exec(text);
313
+ const score = match ? Number(match[1]) : Number.NaN;
314
+ return Number.isFinite(score) && score >= 0 && score <= 1 ? score : null;
315
+ }
316
+ catch (error) {
317
+ if (signal.aborted)
318
+ throw error;
319
+ deps.env.deps.logger.warn("self-check failed", { run_id: deps.runId, error });
320
+ return null;
321
+ }
322
+ }
@@ -0,0 +1,88 @@
1
+ import { readFileSync, statSync } from "node:fs";
2
+ import { DEFAULT_ROUTER_POLICY, parseRouterPolicy } from "../../../router/src/index.js";
3
+ import { DEFAULT_JUDGE_RUBRIC, modelKeyOf, sameFamilyPairs } from "../../../scorers/src/index.js";
4
+ import { ensureManualArtifact, ensureSeedArtifact, resolveSourcedArtifact } from "../../../store/src/index.js";
5
+ import { NexoError } from "../../../telemetry/src/index.js";
6
+ /** Artifact name of the router policy and judge rubric that `artifact:` sources and `file:` sources both use. */
7
+ export const DEFAULT_ARTIFACT_NAME = "default";
8
+ /** Largest `file:` source read; a router policy or rubric is a few KiB. */
9
+ export const MAX_SOURCE_BYTES = 256 * 1024;
10
+ /** Reads a regular file of at most MAX_SOURCE_BYTES; anything else is CONFIG_INVALID naming the key. */
11
+ function readSource(path, key) {
12
+ const fail = (why, cause) => {
13
+ throw new NexoError({
14
+ code: "CONFIG_INVALID",
15
+ message: `Cannot read ${path} (${key}): ${why}`,
16
+ hint: `Point ${key} at an existing regular file of at most ${MAX_SOURCE_BYTES / 1024} KiB, or set it to its artifact: default`,
17
+ context: { key, path },
18
+ cause,
19
+ });
20
+ };
21
+ try {
22
+ const info = statSync(path);
23
+ if (!info.isFile())
24
+ return fail("not a regular file");
25
+ if (info.size > MAX_SOURCE_BYTES)
26
+ return fail(`larger than ${MAX_SOURCE_BYTES} bytes`);
27
+ return readFileSync(path, "utf8");
28
+ }
29
+ catch (cause) {
30
+ if (cause instanceof NexoError)
31
+ throw cause;
32
+ return fail(cause.code ?? "unreadable", cause);
33
+ }
34
+ }
35
+ /**
36
+ * The run's router policy and the artifact version it came from. `artifact:router_policy` uses the active version,
37
+ * seeding version 1 with DEFAULT_ROUTER_POLICY; `file:<path>` stores the file (redacted) as a new manual version when it
38
+ * changed.
39
+ * Throws CONFIG_INVALID for an unreadable file or an invalid policy.
40
+ */
41
+ export function loadRouterPolicy(db, config, nowIso, redact) {
42
+ const source = config.router.policy;
43
+ let artifact;
44
+ if (source.startsWith("file:")) {
45
+ const text = readSource(source.slice("file:".length), "router.policy");
46
+ parseRouterPolicy(text, source);
47
+ artifact = ensureManualArtifact(db, "router_policy", DEFAULT_ARTIFACT_NAME, redact(text), nowIso);
48
+ }
49
+ else {
50
+ artifact = ensureSeedArtifact(db, "router_policy", DEFAULT_ARTIFACT_NAME, `${JSON.stringify(DEFAULT_ROUTER_POLICY, null, 2)}\n`, nowIso);
51
+ }
52
+ return { policy: parseRouterPolicy(artifact.content, `router_policy v${artifact.version}`), artifact };
53
+ }
54
+ /**
55
+ * The generic judge rubric and its artifact version; `artifact:judge_rubric` seeds DEFAULT_JUDGE_RUBRIC. A `file:`
56
+ * rubric is a sourced artifact: a promoted version stays in use until the file changes.
57
+ */
58
+ export function loadJudgeRubric(db, config, nowIso, redact) {
59
+ const source = config.scoring.rubric;
60
+ const artifact = source.startsWith("file:")
61
+ ? resolveSourcedArtifact(db, "judge_rubric", DEFAULT_ARTIFACT_NAME, redact(readSource(source.slice("file:".length), "scoring.rubric")), nowIso)
62
+ : ensureSeedArtifact(db, "judge_rubric", DEFAULT_ARTIFACT_NAME, `${DEFAULT_JUDGE_RUBRIC}\n`, nowIso);
63
+ if (!artifact.content.trim()) {
64
+ throw new NexoError({ code: "CONFIG_INVALID", message: "The judge rubric is empty", hint: "Write grading criteria into the rubric file", context: { source } });
65
+ }
66
+ return { rubric: artifact.content, artifact };
67
+ }
68
+ /** The configured judge models (`models.judge`); empty when no judge is configured. */
69
+ export function judgeModels(config) {
70
+ return (config.models.judge ?? []).map((m) => ({ provider: m.provider, id: m.id }));
71
+ }
72
+ /** The judge key stored with judge scores and calibrations: model keys joined with `+`; null without a judge. */
73
+ export function judgeKey(config) {
74
+ const models = judgeModels(config);
75
+ return models.length === 0 ? null : models.map(modelKeyOf).join("+");
76
+ }
77
+ /** One warning per judge model from the same family as `models.s1` or `models.s2` (spec §7.4). */
78
+ export function judgeFamilyWarnings(config) {
79
+ const generators = [config.models.s1, config.models.s2].map((m) => ({ provider: m.provider, id: m.id }));
80
+ const byJudge = new Map();
81
+ for (const p of sameFamilyPairs(judgeModels(config), generators)) {
82
+ const entry = byJudge.get(p.judge) ?? { family: p.family, generators: [] };
83
+ if (!entry.generators.includes(p.generator))
84
+ entry.generators.push(p.generator);
85
+ byJudge.set(p.judge, entry);
86
+ }
87
+ return [...byJudge].map(([judge, e]) => `Judge ${judge} is from the same model family (${e.family}) as ${e.generators.join(" and ")}; its verdicts may favour that model. Use a judge from another family in models.judge.`);
88
+ }
@@ -0,0 +1,88 @@
1
+ import { readFileSync, unlinkSync, writeFileSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ import { isPidAlive } from "../../../store/src/index.js";
4
+ import { NexoError } from "../../../telemetry/src/index.js";
5
+ const LOCK_FILE = "session.lock";
6
+ const held = new Set();
7
+ let exitHookInstalled = false;
8
+ function releaseAllOnExit() {
9
+ for (const file of held) {
10
+ try {
11
+ unlinkSync(file);
12
+ }
13
+ catch {
14
+ // already gone
15
+ }
16
+ }
17
+ held.clear();
18
+ }
19
+ /**
20
+ * Exclusive ownership of an open session across processes: `session.lock` in the session directory, created with
21
+ * O_EXCL and holding the owner's pid. A lock whose pid is dead is taken over. Released by `release()` or at process exit.
22
+ */
23
+ export class SessionLock {
24
+ file;
25
+ released = false;
26
+ constructor(file) {
27
+ this.file = file;
28
+ }
29
+ /** Throws CONFLICT when a live process (this one included) holds the lock. */
30
+ static acquire(sessionDir, sessionId) {
31
+ const file = join(sessionDir, LOCK_FILE);
32
+ for (let attempt = 0; attempt < 2; attempt++) {
33
+ try {
34
+ writeFileSync(file, String(process.pid), { flag: "wx" });
35
+ held.add(file);
36
+ if (!exitHookInstalled) {
37
+ process.on("exit", releaseAllOnExit);
38
+ exitHookInstalled = true;
39
+ }
40
+ return new SessionLock(file);
41
+ }
42
+ catch (error) {
43
+ if (error.code !== "EEXIST") {
44
+ throw new NexoError({ code: "STORE_IO", message: `Cannot create ${file}`, cause: error });
45
+ }
46
+ }
47
+ let pid = Number.NaN;
48
+ try {
49
+ pid = Number(readFileSync(file, "utf8").trim());
50
+ }
51
+ catch {
52
+ // unreadable: treated as stale
53
+ }
54
+ if (Number.isInteger(pid) && pid > 0 && isPidAlive(pid)) {
55
+ throw new NexoError({
56
+ code: "CONFLICT",
57
+ message: `Session ${sessionId} is open in another nexo process (pid ${pid})`,
58
+ hint: `Close that session first; if no nexo process is using it, delete ${file}`,
59
+ context: { sessionId, pid },
60
+ });
61
+ }
62
+ try {
63
+ unlinkSync(file);
64
+ }
65
+ catch {
66
+ // another process may have taken it over; the next attempt decides
67
+ }
68
+ }
69
+ throw new NexoError({
70
+ code: "CONFLICT",
71
+ message: `Session ${sessionId} was locked by another process while opening it`,
72
+ hint: "Retry",
73
+ context: { sessionId },
74
+ });
75
+ }
76
+ release() {
77
+ if (this.released)
78
+ return;
79
+ this.released = true;
80
+ held.delete(this.file);
81
+ try {
82
+ unlinkSync(this.file);
83
+ }
84
+ catch {
85
+ // already gone
86
+ }
87
+ }
88
+ }