@hexdecimal16/nexo 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +75 -0
- package/README.md +75 -2
- package/cli/src/bin.js +12 -0
- package/cli/src/common.js +77 -0
- package/cli/src/doctor.js +27 -0
- package/cli/src/eval-cases.js +65 -0
- package/cli/src/eval.js +67 -0
- package/cli/src/init.js +186 -0
- package/cli/src/interactive.js +171 -0
- package/cli/src/learn.js +178 -0
- package/cli/src/main.js +261 -0
- package/cli/src/optimize.js +289 -0
- package/cli/src/prompter.js +47 -0
- package/cli/src/review.js +120 -0
- package/cli/src/runtime.js +2 -0
- package/cli/src/sessions.js +34 -0
- package/cli/src/settings.js +169 -0
- package/cli/src/ui.js +112 -0
- package/core/src/approval/classifier.js +152 -0
- package/core/src/config/load.js +283 -0
- package/core/src/config/schema.js +186 -0
- package/core/src/cost/admission.js +90 -0
- package/core/src/cost/budget.js +67 -0
- package/core/src/cost/pricing.js +53 -0
- package/core/src/engine/charged-completion.js +79 -0
- package/core/src/engine/errors.js +77 -0
- package/core/src/engine/mcp.js +11 -0
- package/core/src/engine/messages.js +86 -0
- package/core/src/engine/prompts.js +32 -0
- package/core/src/engine/provider-gate.js +33 -0
- package/core/src/engine/recorder.js +61 -0
- package/core/src/engine/run.js +1115 -0
- package/core/src/engine/semaphore.js +45 -0
- package/core/src/engine/spawn-tool.js +119 -0
- package/core/src/engine/task-graph.js +153 -0
- package/core/src/index.js +32 -0
- package/core/src/learn/caller.js +171 -0
- package/core/src/learn/context.js +112 -0
- package/core/src/learn/embedder.js +216 -0
- package/core/src/learn/job.js +196 -0
- package/core/src/learn/wiki.js +116 -0
- package/core/src/learn/worker.js +141 -0
- package/core/src/optimize/adapter.js +250 -0
- package/core/src/optimize/climb.js +338 -0
- package/core/src/optimize/mine.js +95 -0
- package/core/src/optimize/optimize.js +377 -0
- package/core/src/optimize/optimizer-run.js +67 -0
- package/core/src/optimize/reflect.js +100 -0
- package/core/src/optimize/scratchpad-wiki.js +69 -0
- package/core/src/providers/claude-p.js +363 -0
- package/core/src/providers/kiro-p.js +408 -0
- package/core/src/providers/models.js +10 -0
- package/core/src/scoring/calibration.js +31 -0
- package/core/src/scoring/continuation.js +24 -0
- package/core/src/scoring/runner.js +322 -0
- package/core/src/scoring/sources.js +88 -0
- package/core/src/session/lock.js +88 -0
- package/core/src/session/session.js +647 -0
- package/core/src/session/shell.js +26 -0
- package/core/src/session/transcript.js +86 -0
- package/core/src/settings/config-edit.js +472 -0
- package/core/src/settings/credentials.js +120 -0
- package/core/src/settings/doctor.js +103 -0
- package/core/src/settings/settings.js +265 -0
- package/mcp/src/hub.js +184 -0
- package/mcp/src/index.js +1 -0
- package/memory/src/curate.js +121 -0
- package/memory/src/index.js +4 -0
- package/memory/src/prompts.js +143 -0
- package/memory/src/retrieval.js +91 -0
- package/memory/src/similarity.js +47 -0
- package/optimize/src/engine.js +206 -0
- package/optimize/src/index.js +8 -0
- package/optimize/src/merge.js +73 -0
- package/optimize/src/novelty.js +37 -0
- package/optimize/src/pareto.js +83 -0
- package/optimize/src/proposer.js +148 -0
- package/optimize/src/report.js +107 -0
- package/optimize/src/rng.js +34 -0
- package/optimize/src/target.js +34 -0
- package/package.json +47 -4
- package/router/src/effort.js +5 -0
- package/router/src/escalation.js +171 -0
- package/router/src/index.js +4 -0
- package/router/src/registry.js +21 -0
- package/router/src/resolve.js +117 -0
- package/scorers/src/deterministic.js +194 -0
- package/scorers/src/family.js +48 -0
- package/scorers/src/index.js +6 -0
- package/scorers/src/judge.js +284 -0
- package/scorers/src/regex-worker.js +3 -0
- package/scorers/src/spec.js +147 -0
- package/scorers/src/trajectory.js +22 -0
- package/scorers/src/types.js +1 -0
- package/sdk/src/index.js +3 -0
- package/sdk/src/nexo.js +94 -0
- package/sdk/src/runtime.js +39 -0
- package/skills/bundled/browser/SKILL.md +57 -0
- package/skills/bundled/wiki/SKILL.md +73 -0
- package/skills/src/frontmatter.js +25 -0
- package/skills/src/index.js +3 -0
- package/skills/src/load-skill-tool.js +41 -0
- package/skills/src/loader.js +95 -0
- package/store/src/artifacts.js +124 -0
- package/store/src/budget-ledger.js +18 -0
- package/store/src/cost-report.js +46 -0
- package/store/src/cursor.js +35 -0
- package/store/src/db.js +83 -0
- package/store/src/errors.js +9 -0
- package/store/src/eval-cases.js +255 -0
- package/store/src/feedback.js +154 -0
- package/store/src/goldens.js +61 -0
- package/store/src/ids.js +6 -0
- package/store/src/index.js +25 -0
- package/store/src/judge-calibration.js +182 -0
- package/store/src/learn-jobs.js +145 -0
- package/store/src/memories.js +261 -0
- package/store/src/migrations/001_init.sql +81 -0
- package/store/src/migrations/002_feedback.sql +9 -0
- package/store/src/migrations/003_sessions.sql +19 -0
- package/store/src/migrations/004_scoring.sql +22 -0
- package/store/src/migrations/005_judge_calibration.sql +27 -0
- package/store/src/migrations/006_learning.sql +77 -0
- package/store/src/migrations/007_optimize.sql +55 -0
- package/store/src/migrations/008_mined_cases_and_climb.sql +46 -0
- package/store/src/mine.js +51 -0
- package/store/src/optim.js +291 -0
- package/store/src/paths.js +22 -0
- package/store/src/playbooks.js +199 -0
- package/store/src/reindex.js +143 -0
- package/store/src/run-dir.js +151 -0
- package/store/src/runs.js +310 -0
- package/store/src/scores.js +92 -0
- package/store/src/sessions.js +304 -0
- package/store/src/span-tree.js +98 -0
- package/store/src/spans.js +84 -0
- package/store/src/sql.js +13 -0
- package/store/src/workspace-copy.js +80 -0
- package/telemetry/src/errors.js +32 -0
- package/telemetry/src/index.js +3 -0
- package/telemetry/src/logger.js +39 -0
- package/telemetry/src/redact.js +106 -0
- package/tools/src/builtin.js +269 -0
- package/tools/src/change-tracker.js +57 -0
- package/tools/src/confine.js +101 -0
- package/tools/src/index.js +7 -0
- package/tools/src/leases.js +57 -0
- package/tools/src/policy.js +162 -0
- package/tools/src/read-only.js +338 -0
- package/tools/src/session-rules.js +140 -0
- package/tui/src/app.js +948 -0
- package/tui/src/approvals.js +96 -0
- package/tui/src/commands.js +38 -0
- package/tui/src/components.js +248 -0
- package/tui/src/index.js +5 -0
- package/tui/src/picker.js +78 -0
- package/tui/src/session-picker.js +100 -0
- package/tui/src/settings-screen.js +346 -0
- package/tui/src/text.js +70 -0
- package/tui/src/theme.js +44 -0
- package/web/public/cost.html +25 -0
- package/web/public/fonts/GeistMono-400.woff2 +0 -0
- package/web/public/fonts/GeistMono-600.woff2 +0 -0
- package/web/public/fonts/InstrumentSans-400.woff2 +0 -0
- package/web/public/fonts/InstrumentSans-600.woff2 +0 -0
- package/web/public/index.html +39 -0
- package/web/public/judge.html +21 -0
- package/web/public/learning.html +46 -0
- package/web/public/lib/api-types.ts +616 -0
- package/web/public/lib/common-view.ts +47 -0
- package/web/public/lib/cost-view.ts +35 -0
- package/web/public/lib/files-view.ts +90 -0
- package/web/public/lib/format.ts +35 -0
- package/web/public/lib/html.ts +51 -0
- package/web/public/lib/judge-view.ts +62 -0
- package/web/public/lib/learning-view.ts +102 -0
- package/web/public/lib/optim-view.ts +148 -0
- package/web/public/lib/pager.ts +56 -0
- package/web/public/lib/poller.ts +79 -0
- package/web/public/lib/run-view.ts +61 -0
- package/web/public/lib/runs-view.ts +87 -0
- package/web/public/lib/score-view.ts +94 -0
- package/web/public/lib/sessions-view.ts +28 -0
- package/web/public/lib/span-view.ts +90 -0
- package/web/public/lib/theme.ts +60 -0
- package/web/public/lib/timeline.ts +97 -0
- package/web/public/lib/wikis-view.ts +257 -0
- package/web/public/optim.html +30 -0
- package/web/public/pages/api.ts +60 -0
- package/web/public/pages/cost.ts +36 -0
- package/web/public/pages/judge.ts +15 -0
- package/web/public/pages/learning.ts +92 -0
- package/web/public/pages/optim.ts +68 -0
- package/web/public/pages/paging.ts +18 -0
- package/web/public/pages/run.ts +221 -0
- package/web/public/pages/runs.ts +91 -0
- package/web/public/pages/sessions.ts +25 -0
- package/web/public/pages/wikis.ts +103 -0
- package/web/public/run.html +36 -0
- package/web/public/sessions.html +28 -0
- package/web/public/style.css +288 -0
- package/web/public/wikis.html +25 -0
- package/web/src/api.js +682 -0
- package/web/src/http-errors.js +60 -0
- package/web/src/index.js +5 -0
- package/web/src/server.js +292 -0
- package/web/src/static.js +86 -0
- package/web/src/strip.js +22 -0
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { cosine, jaccard } from "../../../memory/src/index.js";
|
|
2
|
+
import { addSpend, dayScope, getSpent, insertMinedCases, listEvalCases, mineCandidateRuns, monthScope, normalizeCaseInput, proposeClusters, syncRunEvalCases, } from "../../../store/src/index.js";
|
|
3
|
+
import { NexoError } from "../../../telemetry/src/index.js";
|
|
4
|
+
import { checkAdmission } from "../cost/admission.js";
|
|
5
|
+
import { embeddingCost, embeddingEndpoint, embedTexts, estimateEmbeddingUsd } from "../learn/embedder.js";
|
|
6
|
+
/** Longest text sent to the embedder; clustering needs only the gist. */
|
|
7
|
+
const EMBED_CLIP_CHARS = 2000;
|
|
8
|
+
/** Most texts per embedding request. */
|
|
9
|
+
const EMBED_CHUNK = 100;
|
|
10
|
+
/**
|
|
11
|
+
* Embeds through `models.embedding`, charging the day and month ledgers and `scopes`. Refuses an exhausted ledger
|
|
12
|
+
* (checkAdmission) and throws BUDGET_EXCEEDED when the estimate is more than what is left of the daily or monthly
|
|
13
|
+
* budget. Null without an embedding model.
|
|
14
|
+
*/
|
|
15
|
+
export function ledgerEmbedder(env, scopes) {
|
|
16
|
+
const { config, db } = env.deps;
|
|
17
|
+
const endpoint = embeddingEndpoint(config);
|
|
18
|
+
if (!endpoint)
|
|
19
|
+
return null;
|
|
20
|
+
return async (texts) => {
|
|
21
|
+
const now = env.now();
|
|
22
|
+
checkAdmission(db, config.budgets, now);
|
|
23
|
+
const estimate = estimateEmbeddingUsd(config, endpoint, texts.reduce((n, t) => n + t.length, 0));
|
|
24
|
+
const left = Math.min(config.budgets.dailyUsd - getSpent(db, dayScope(now)), config.budgets.monthlyUsd - getSpent(db, monthScope(now)));
|
|
25
|
+
if (estimate > left) {
|
|
26
|
+
throw new NexoError({ code: "BUDGET_EXCEEDED", message: `Embedding ${texts.length} instructions (estimated $${estimate.toFixed(4)}) is more than the $${Math.max(0, left).toFixed(4)} left today or this month` });
|
|
27
|
+
}
|
|
28
|
+
const result = await embedTexts(endpoint, texts, { apiKey: env.deps.resolveApiKey?.(endpoint.provider) });
|
|
29
|
+
const priced = embeddingCost(config, endpoint, result.tokens);
|
|
30
|
+
addSpend(db, [dayScope(now), monthScope(now), ...scopes], priced.costUsd, now.toISOString());
|
|
31
|
+
return result.vectors;
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* One mining pass (design §3): proposes up to `optimize.mine.maxNew` cases from clusters of similar unreviewed runs.
|
|
36
|
+
* Uses embedding cosine when `embed` (default: `ledgerEmbedder` with scope `mine:<target>`) is available and succeeds,
|
|
37
|
+
* otherwise Jaccard. Throws INVALID_REQUEST for judge_rubric.
|
|
38
|
+
*/
|
|
39
|
+
export async function mineEvalCases(env, target, options = {}) {
|
|
40
|
+
if (target.kind === "judge_rubric") {
|
|
41
|
+
throw new NexoError({ code: "INVALID_REQUEST", message: "judge_rubric cases come from reviewed runs that the judge scored, not from mining", context: { target: target.key } });
|
|
42
|
+
}
|
|
43
|
+
const { db, config, redact, logger } = env.deps;
|
|
44
|
+
syncRunEvalCases(db, target, env.now().toISOString());
|
|
45
|
+
const settings = config.optimize.mine;
|
|
46
|
+
const now = env.now();
|
|
47
|
+
const days = options.days ?? settings.days;
|
|
48
|
+
const items = mineCandidateRuns(db, target, { sinceIso: new Date(now.getTime() - days * 86_400_000).toISOString(), maxRuns: settings.maxRuns });
|
|
49
|
+
const existing = listEvalCases(db, target.key);
|
|
50
|
+
const waitingOf = () => listEvalCases(db, target.key).filter((c) => c.status === "proposed");
|
|
51
|
+
if (items.length < settings.minCluster) {
|
|
52
|
+
logger.info("mined eval cases", { target: target.key, considered: items.length, measure: "jaccard", added: 0 });
|
|
53
|
+
return { added: [], waiting: waitingOf(), measure: "jaccard", considered: items.length, fallbackReason: null };
|
|
54
|
+
}
|
|
55
|
+
const text = (s) => normalizeCaseInput(s);
|
|
56
|
+
let measure = "jaccard";
|
|
57
|
+
let fallbackReason = null;
|
|
58
|
+
let similar = (a, b) => jaccard(text(a.instruction), text(b.instruction)) >= settings.jaccard;
|
|
59
|
+
let nearExisting = (leader) => existing.some((c) => jaccard(text(leader.instruction), text(c.input)) >= settings.jaccard);
|
|
60
|
+
const embed = options.embed === undefined ? ledgerEmbedder(env, [`mine:${target.key}`]) : options.embed;
|
|
61
|
+
if (embed) {
|
|
62
|
+
try {
|
|
63
|
+
const texts = [...items.map((i) => text(i.instruction)), ...existing.map((c) => text(c.input))].map((t) => t.slice(0, EMBED_CLIP_CHARS));
|
|
64
|
+
const vectors = [];
|
|
65
|
+
for (let at = 0; at < texts.length; at += EMBED_CHUNK)
|
|
66
|
+
vectors.push(...(await embed(texts.slice(at, at + EMBED_CHUNK))));
|
|
67
|
+
if (vectors.length !== texts.length)
|
|
68
|
+
throw new Error(`embedder returned ${vectors.length} vectors for ${texts.length} texts`);
|
|
69
|
+
const byRun = new Map(items.map((i, k) => [i.runId, vectors[k]]));
|
|
70
|
+
const caseVectors = existing.map((_, k) => vectors[items.length + k]);
|
|
71
|
+
similar = (a, b) => cosine(byRun.get(a.runId), byRun.get(b.runId)) >= settings.cosine;
|
|
72
|
+
nearExisting = (leader) => caseVectors.some((v) => cosine(byRun.get(leader.runId), v) >= settings.cosine);
|
|
73
|
+
measure = "cosine";
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
fallbackReason = error instanceof Error ? error.message : String(error);
|
|
77
|
+
logger.warn("embedding mined instructions failed; clustering by word overlap instead", { target: target.key, error });
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
const clusters = proposeClusters(items, { similar, nearExisting, minCluster: settings.minCluster, maxNew: settings.maxNew });
|
|
81
|
+
const added = insertMinedCases(db, target, clusters, redact, now.toISOString());
|
|
82
|
+
logger.info("mined eval cases", { target: target.key, considered: items.length, measure, added: added.length });
|
|
83
|
+
return { added, waiting: waitingOf(), measure, considered: items.length, fallbackReason };
|
|
84
|
+
}
|
|
85
|
+
/** Mining before an optimization or climb: never fails it, only logs. */
|
|
86
|
+
export async function mineQuietly(env, target) {
|
|
87
|
+
if (target.kind === "judge_rubric")
|
|
88
|
+
return;
|
|
89
|
+
try {
|
|
90
|
+
await mineEvalCases(env, target);
|
|
91
|
+
}
|
|
92
|
+
catch (error) {
|
|
93
|
+
env.deps.logger.warn("mining eval cases failed; continuing without it", { target: target.key, error });
|
|
94
|
+
}
|
|
95
|
+
}
|
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
import { existsSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { BudgetStop, buildReport, contentOf, MAX_CANDIDATE_CHARS, meanOf, MINIBATCH_SIZE, mulberry32, runGepa, targetComponents, textsOf, } from "../../../optimize/src/index.js";
|
|
4
|
+
import { dayScope, getArtifact, getOptimRun, getRun, getSpent, insertCandidateArtifact, insertOptimCandidate, insertOptimRun, isPidAlive, listActiveEvalCases, listEvalCases, listOptimCandidates, liveJudgeKappa, monthScope, newId, parseOptimTarget, recoverCrashedOptimRuns, resolveSourcedArtifact, retireOptimCandidates, runsNeedingReview, syncRunEvalCases, updateOptimCandidate, updateOptimRun, } from "../../../store/src/index.js";
|
|
5
|
+
import { NexoError, toErrorPayload } from "../../../telemetry/src/index.js";
|
|
6
|
+
import { nonInteractivePrompter } from "../../../tools/src/index.js";
|
|
7
|
+
import { checkAdmission } from "../cost/admission.js";
|
|
8
|
+
import { RunBudget } from "../cost/budget.js";
|
|
9
|
+
import { DEFAULT_PLANNER_PROMPT, DEFAULT_SYSTEM_PROMPT } from "../engine/prompts.js";
|
|
10
|
+
import { createEngineEnv } from "../engine/run.js";
|
|
11
|
+
import { judgeKey, judgeModels, loadJudgeRubric } from "../scoring/sources.js";
|
|
12
|
+
import { mineQuietly } from "./mine.js";
|
|
13
|
+
import { readScratchpad, writeScratchpad } from "./scratchpad-wiki.js";
|
|
14
|
+
import { assertReflectionFits } from "./reflect.js";
|
|
15
|
+
import { createOptimAdapter } from "./adapter.js";
|
|
16
|
+
import { closeOptimizerRun, openOptimizerRun } from "./optimizer-run.js";
|
|
17
|
+
function toEvalCase(c) {
|
|
18
|
+
return { id: c.id, input: c.input, skill: c.skill, expected: c.expected, referenceFeedback: c.referenceFeedback, sourceRunId: c.sourceRunId };
|
|
19
|
+
}
|
|
20
|
+
/** The version the optimization starts from: what the next normal run would use. */
|
|
21
|
+
export function baseVersion(env, target, nowIso) {
|
|
22
|
+
const { db, redact, config, skills } = env.deps;
|
|
23
|
+
switch (target.kind) {
|
|
24
|
+
case "skill": {
|
|
25
|
+
const skill = skills.skills.get(target.name);
|
|
26
|
+
if (!skill) {
|
|
27
|
+
throw new NexoError({ code: "SKILL_NOT_FOUND", message: `Unknown skill "${target.name}"`, hint: `Available: ${[...skills.skills.keys()].join(", ") || "(none)"}` });
|
|
28
|
+
}
|
|
29
|
+
return resolveSourcedArtifact(db, "skill", skill.name, redact(skill.content), nowIso);
|
|
30
|
+
}
|
|
31
|
+
case "system_prompt":
|
|
32
|
+
return resolveSourcedArtifact(db, "system_prompt", "default", redact(DEFAULT_SYSTEM_PROMPT), nowIso);
|
|
33
|
+
case "planner_prompt":
|
|
34
|
+
return resolveSourcedArtifact(db, "planner_prompt", "default", redact(DEFAULT_PLANNER_PROMPT), nowIso);
|
|
35
|
+
case "judge_rubric":
|
|
36
|
+
return loadJudgeRubric(db, config, nowIso, redact).artifact;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/** Refuses targets nothing can score (D5). */
|
|
40
|
+
export function assertEvaluator(env, target) {
|
|
41
|
+
const { config, skills } = env.deps;
|
|
42
|
+
if (judgeModels(config).length === 0 && (target.kind === "judge_rubric" || !config.scoring.enabled || (config.scoring.scorers.length === 0 && !(target.kind === "skill" && skills.skills.get(target.name)?.meta.scorers)))) {
|
|
43
|
+
throw new NexoError({
|
|
44
|
+
code: "CONFIG_INVALID",
|
|
45
|
+
message: target.kind === "judge_rubric" ? "Optimizing judge_rubric needs a judge (models.judge)" : `Nothing can score candidates for ${target.key}: no judge and no scorers are configured`,
|
|
46
|
+
hint: 'Set models.judge, for example [{"provider":"google","id":"<model-id>"}], or scoring.scorers (or the skill\'s nexo-scorers file)',
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
if (target.kind !== "judge_rubric" && !config.scoring.enabled) {
|
|
50
|
+
throw new NexoError({ code: "CONFIG_INVALID", message: "Optimization needs scoring.enabled", hint: "Set scoring.enabled to true" });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** Throws BUDGET_EXCEEDED when the optimization budget is more than what is left of the daily or monthly budget (D10). */
|
|
54
|
+
export function assertBudget(env, budgetUsd) {
|
|
55
|
+
const { db, config } = env.deps;
|
|
56
|
+
const now = env.now();
|
|
57
|
+
checkAdmission(db, config.budgets, now);
|
|
58
|
+
const limits = [
|
|
59
|
+
{ left: config.budgets.dailyUsd - getSpent(db, dayScope(now)), label: "daily", key: "budgets.dailyUsd" },
|
|
60
|
+
{ left: config.budgets.monthlyUsd - getSpent(db, monthScope(now)), label: "monthly", key: "budgets.monthlyUsd" },
|
|
61
|
+
];
|
|
62
|
+
for (const l of limits) {
|
|
63
|
+
if (budgetUsd > l.left + 1e-9) {
|
|
64
|
+
throw new NexoError({
|
|
65
|
+
code: "BUDGET_EXCEEDED",
|
|
66
|
+
message: `The optimization budget of $${budgetUsd.toFixed(2)} is more than what is left of the ${l.label} budget ($${Math.max(0, l.left).toFixed(2)})`,
|
|
67
|
+
hint: `Pass a smaller --budget-usd, or raise ${l.key}`,
|
|
68
|
+
context: { budgetUsd, leftUsd: l.left },
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/** The eval cases, split (D6). Throws EVAL_SET_TOO_SMALL with the runs that most need review. */
|
|
74
|
+
function loadCases(env, target, nowIso) {
|
|
75
|
+
const { db, config } = env.deps;
|
|
76
|
+
syncRunEvalCases(db, target, nowIso);
|
|
77
|
+
let cases = listActiveEvalCases(db, target.key);
|
|
78
|
+
if (target.kind === "judge_rubric") {
|
|
79
|
+
cases = cases.filter((c) => {
|
|
80
|
+
const run = c.sourceRunId ? getRun(db, c.sourceRunId) : undefined;
|
|
81
|
+
return run !== undefined && existsSync(run.dirPath) && (c.expected === "done" || c.expected === "not_done");
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
const split = { feedback: [], pareto: [], holdout: [] };
|
|
85
|
+
for (const c of cases)
|
|
86
|
+
split[c.split].push(toEvalCase(c));
|
|
87
|
+
const empty = ["feedback", "pareto", "holdout"].find((s) => split[s].length === 0);
|
|
88
|
+
if (cases.length < config.optimize.minCases || empty) {
|
|
89
|
+
const runsToReview = runsNeedingReview(db, target).map((r) => r.runId);
|
|
90
|
+
const proposedCases = listEvalCases(db, target.key).filter((c) => c.status === "proposed").map((c) => c.id);
|
|
91
|
+
throw new NexoError({
|
|
92
|
+
code: "EVAL_SET_TOO_SMALL",
|
|
93
|
+
message: cases.length < config.optimize.minCases
|
|
94
|
+
? `Optimizing ${target.key} needs at least ${config.optimize.minCases} eval cases (optimize.minCases); it has ${cases.length}`
|
|
95
|
+
: `Optimizing ${target.key} needs cases in every split; the ${empty} split is empty (${cases.length} cases)`,
|
|
96
|
+
hint: (runsToReview.length > 0 ? `Review these runs (done / not done) so they become eval cases: ${runsToReview.join(", ")}. ` : "") +
|
|
97
|
+
(proposedCases.length > 0 ? `Accept proposed cases with \`nexo eval accept <id>\`: ${proposedCases.join(", ")}. ` : "") +
|
|
98
|
+
(target.kind === "judge_rubric" ? "Judge rubric cases come from reviewed runs that the judge scored." : `Or add cases with \`nexo eval add --target ${target.key} --input "<instruction>"\`.`),
|
|
99
|
+
context: { target: target.key, cases: cases.length, minCases: config.optimize.minCases, runsToReview, proposedCases },
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
return split;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Runs one GEPA optimization in this process (spec §8.2): checks the budget, builds the eval set, evaluates the base on
|
|
106
|
+
* the pareto and holdout splits, runs the loop, rolls the best candidate out on the holdout split and writes the report.
|
|
107
|
+
* Candidates are never made active. Throws (after recording the failure) on errors; an abort returns with status
|
|
108
|
+
* `aborted` and no report.
|
|
109
|
+
*/
|
|
110
|
+
export async function startOptimization(baseEnv, options) {
|
|
111
|
+
// Rollouts never prompt: `ask` follows tools.nonInteractiveAsk (spec §7.6).
|
|
112
|
+
const env = createEngineEnv({ ...baseEnv.deps, prompter: nonInteractivePrompter, onEvent: undefined });
|
|
113
|
+
const { db, config, redact, logger } = env.deps;
|
|
114
|
+
const signal = options.signal ?? new AbortController().signal;
|
|
115
|
+
const nowIso = () => env.now().toISOString();
|
|
116
|
+
const recovered = recoverCrashedOptimRuns(db, isPidAlive, nowIso());
|
|
117
|
+
if (recovered.length > 0)
|
|
118
|
+
logger.warn("marked optimizations of exited processes as aborted", { optim_run_ids: recovered });
|
|
119
|
+
const target = parseOptimTarget(options.target);
|
|
120
|
+
const budgetUsd = options.budgetUsd ?? config.budgets.optimDefaultUsd;
|
|
121
|
+
if (!(Number.isFinite(budgetUsd) && budgetUsd > 0))
|
|
122
|
+
throw new NexoError({ code: "INVALID_REQUEST", message: "--budget-usd must be a positive number of USD", context: { budgetUsd } });
|
|
123
|
+
if (options.maxRollouts !== undefined && !(Number.isInteger(options.maxRollouts) && options.maxRollouts > 0)) {
|
|
124
|
+
throw new NexoError({ code: "INVALID_REQUEST", message: "--max-rollouts must be a positive integer", context: { maxRollouts: options.maxRollouts } });
|
|
125
|
+
}
|
|
126
|
+
assertEvaluator(env, target);
|
|
127
|
+
assertBudget(env, budgetUsd);
|
|
128
|
+
await mineQuietly(env, target);
|
|
129
|
+
const cases = loadCases(env, target, nowIso());
|
|
130
|
+
const minibatch = Math.min(MINIBATCH_SIZE, cases.feedback.length);
|
|
131
|
+
const minRollouts = 2 * (cases.pareto.length + cases.holdout.length) + 2 * minibatch;
|
|
132
|
+
if (options.maxRollouts !== undefined && options.maxRollouts < minRollouts) {
|
|
133
|
+
throw new NexoError({
|
|
134
|
+
code: "INVALID_REQUEST",
|
|
135
|
+
message: `--max-rollouts must be at least ${minRollouts}: the base on the pareto (${cases.pareto.length}) and holdout (${cases.holdout.length}) splits, one iteration (${2 * minibatch} on the minibatch and ${cases.pareto.length} on the pareto split) and the winner on the holdout split (${cases.holdout.length})`,
|
|
136
|
+
context: { maxRollouts: options.maxRollouts, minRollouts },
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
const base = baseVersion(env, target, nowIso());
|
|
140
|
+
const components = targetComponents(target);
|
|
141
|
+
const baseTexts = textsOf(target, base.content);
|
|
142
|
+
if (Object.values(baseTexts).some((t) => t.length > MAX_CANDIDATE_CHARS)) {
|
|
143
|
+
throw new NexoError({ code: "INVALID_REQUEST", message: `The ${target.key} text is longer than ${MAX_CANDIDATE_CHARS} characters, the most an optimization handles`, context: { target: target.key } });
|
|
144
|
+
}
|
|
145
|
+
assertReflectionFits(target, baseTexts);
|
|
146
|
+
const optimRunId = newId();
|
|
147
|
+
const seed = options.seed ?? Math.floor(Math.random() * 2 ** 31);
|
|
148
|
+
const optimizer = openOptimizerRun(env, optimRunId, target);
|
|
149
|
+
insertOptimRun(db, {
|
|
150
|
+
id: optimRunId,
|
|
151
|
+
target: target.key,
|
|
152
|
+
status: "running",
|
|
153
|
+
stopReason: null,
|
|
154
|
+
budgetUsd,
|
|
155
|
+
budgetRollouts: options.maxRollouts ?? null,
|
|
156
|
+
spentUsd: 0,
|
|
157
|
+
rollouts: 0,
|
|
158
|
+
merge: options.merge ?? false,
|
|
159
|
+
seed,
|
|
160
|
+
baseArtifactId: base.id,
|
|
161
|
+
bestArtifactId: null,
|
|
162
|
+
dirPath: optimizer.runDir.root,
|
|
163
|
+
reportRef: null,
|
|
164
|
+
judgeKey: null,
|
|
165
|
+
judgeOnly: null,
|
|
166
|
+
holdoutBefore: null,
|
|
167
|
+
holdoutAfter: null,
|
|
168
|
+
promotion: null,
|
|
169
|
+
replacedArtifactId: null,
|
|
170
|
+
promotedAt: null,
|
|
171
|
+
pid: process.pid,
|
|
172
|
+
error: null,
|
|
173
|
+
createdAt: nowIso(),
|
|
174
|
+
endedAt: null,
|
|
175
|
+
mode: "gepa",
|
|
176
|
+
caseId: null,
|
|
177
|
+
climbOutcome: null,
|
|
178
|
+
});
|
|
179
|
+
const seedRef = { id: newId(), artifactId: base.id, texts: baseTexts };
|
|
180
|
+
insertOptimCandidate(db, { id: seedRef.id, optimRunId, artifactId: base.id, parentIds: [], iteration: 0, origin: "seed", component: null, minibatch: null, perCase: {}, meanScore: null, meanCostUsd: null, accepted: true, createdAt: nowIso() });
|
|
181
|
+
options.onEvent?.({ type: "start", optimRunId, target: target.key, budgetUsd, cases: { feedback: cases.feedback.length, pareto: cases.pareto.length, holdout: cases.holdout.length } });
|
|
182
|
+
let ownCost = 0;
|
|
183
|
+
let ownUnknown = false;
|
|
184
|
+
const ctx = {
|
|
185
|
+
env,
|
|
186
|
+
optimRunId,
|
|
187
|
+
recorder: optimizer.recorder,
|
|
188
|
+
parentSpanId: optimizer.spanId,
|
|
189
|
+
budget: new RunBudget(budgetUsd),
|
|
190
|
+
signal,
|
|
191
|
+
addCost: (usd, unknown) => {
|
|
192
|
+
ownCost += usd;
|
|
193
|
+
ownUnknown ||= unknown;
|
|
194
|
+
},
|
|
195
|
+
target,
|
|
196
|
+
runsRoot: join(optimizer.runDir.root, "rollouts"),
|
|
197
|
+
phase: "setup",
|
|
198
|
+
reserveUsd: 0,
|
|
199
|
+
rolloutsLeft: options.maxRollouts ?? Number.POSITIVE_INFINITY,
|
|
200
|
+
rollouts: 0,
|
|
201
|
+
onEvent: (event) => {
|
|
202
|
+
updateOptimRun(db, optimRunId, { spentUsd: ctx.budget.spentUsd, rollouts: ctx.rollouts });
|
|
203
|
+
options.onEvent?.(event);
|
|
204
|
+
},
|
|
205
|
+
};
|
|
206
|
+
const adapter = createOptimAdapter(ctx);
|
|
207
|
+
const perCaseOf = (batch, result) => Object.fromEntries(batch.map((c, i) => [c.id, result.scores[i] ?? { quality: 0, costUsd: 0 }]));
|
|
208
|
+
const artifactOfCandidate = new Map([[seedRef.id, base.id]]);
|
|
209
|
+
const minibatches = new Map();
|
|
210
|
+
const finish = (patch) => {
|
|
211
|
+
updateOptimRun(db, optimRunId, { spentUsd: ctx.budget.spentUsd, rollouts: ctx.rollouts, endedAt: nowIso(), ...patch });
|
|
212
|
+
const run = getOptimRun(db, optimRunId);
|
|
213
|
+
options.onEvent?.({ type: "done", run });
|
|
214
|
+
return run;
|
|
215
|
+
};
|
|
216
|
+
try {
|
|
217
|
+
options.onEvent?.({ type: "phase", phase: "baseline" });
|
|
218
|
+
let baseline;
|
|
219
|
+
try {
|
|
220
|
+
baseline = { pareto: await adapter.evaluate(cases.pareto, seedRef, false, signal), holdout: await adapter.evaluate(cases.holdout, seedRef, false, signal) };
|
|
221
|
+
}
|
|
222
|
+
catch (error) {
|
|
223
|
+
if (!(error instanceof BudgetStop))
|
|
224
|
+
throw error;
|
|
225
|
+
throw new NexoError({
|
|
226
|
+
code: "BUDGET_EXCEEDED",
|
|
227
|
+
message: `The optimization budget does not cover rolling the current version out on the pareto and holdout splits: ${error.message}`,
|
|
228
|
+
hint: "Pass a larger --budget-usd or --max-rollouts, or lower budgets.perRunUsd",
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
const seedPareto = perCaseOf(cases.pareto, baseline.pareto);
|
|
232
|
+
const seedMean = meanOf(seedPareto, cases.pareto.map((c) => c.id));
|
|
233
|
+
updateOptimCandidate(db, seedRef.id, { perCase: seedPareto, meanScore: seedMean.quality, meanCostUsd: seedMean.costUsd });
|
|
234
|
+
const dearest = Math.max(0, ...baseline.pareto.scores.map((s) => s.costUsd), ...baseline.holdout.scores.map((s) => s.costUsd));
|
|
235
|
+
ctx.reserveUsd = cases.holdout.length * Math.min(config.budgets.perRunUsd, 2 * dearest);
|
|
236
|
+
ctx.phase = "loop";
|
|
237
|
+
options.onEvent?.({ type: "phase", phase: "loop" });
|
|
238
|
+
const wikisDir = env.deps.paths?.wikis;
|
|
239
|
+
const seededPad = wikisDir ? readScratchpad(wikisDir, target.key) : undefined;
|
|
240
|
+
const result = await runGepa({
|
|
241
|
+
adapter,
|
|
242
|
+
seed: seedRef,
|
|
243
|
+
seedPareto,
|
|
244
|
+
components,
|
|
245
|
+
feedback: cases.feedback,
|
|
246
|
+
pareto: cases.pareto,
|
|
247
|
+
patience: config.optimize.patience,
|
|
248
|
+
noveltySimilarity: config.optimize.noveltySimilarity,
|
|
249
|
+
...(seededPad ? { scratchpad: seededPad } : {}),
|
|
250
|
+
offspringAlpha: config.optimize.offspringAlpha,
|
|
251
|
+
merge: options.merge ?? false,
|
|
252
|
+
rng: mulberry32(seed),
|
|
253
|
+
signal,
|
|
254
|
+
callbacks: {
|
|
255
|
+
createCandidate: (input) => {
|
|
256
|
+
const parentArtifact = artifactOfCandidate.get(input.parentIds[0]) ?? base.id;
|
|
257
|
+
const artifact = insertCandidateArtifact(db, base.kind, base.name, redact(contentOf(target, base.content, input.texts)), parentArtifact, nowIso());
|
|
258
|
+
const id = newId();
|
|
259
|
+
artifactOfCandidate.set(id, artifact.id);
|
|
260
|
+
insertOptimCandidate(db, { id, optimRunId, artifactId: artifact.id, parentIds: input.parentIds, iteration: input.iteration, origin: input.origin, component: input.component, minibatch: null, perCase: {}, meanScore: null, meanCostUsd: null, accepted: false, createdAt: nowIso() });
|
|
261
|
+
return { id, artifactId: artifact.id, texts: textsOf(target, artifact.content) };
|
|
262
|
+
},
|
|
263
|
+
recordMinibatch: (id, record, accepted) => {
|
|
264
|
+
minibatches.set(id, record);
|
|
265
|
+
updateOptimCandidate(db, id, { minibatch: record, accepted });
|
|
266
|
+
},
|
|
267
|
+
recordNotes: (id, notes) => {
|
|
268
|
+
const record = minibatches.get(id);
|
|
269
|
+
if (!record)
|
|
270
|
+
return;
|
|
271
|
+
const saved = redact({ rationale: notes.rationale, prediction: notes.prediction });
|
|
272
|
+
updateOptimCandidate(db, id, { minibatch: { ...record, rationale: saved.rationale, ...(saved.prediction ? { prediction: saved.prediction } : {}) } });
|
|
273
|
+
},
|
|
274
|
+
recordPareto: (id, perCase, accepted) => {
|
|
275
|
+
const mean = meanOf(perCase, cases.pareto.map((c) => c.id));
|
|
276
|
+
updateOptimCandidate(db, id, { perCase, meanScore: mean.quality, meanCostUsd: mean.costUsd, accepted });
|
|
277
|
+
},
|
|
278
|
+
canStartIteration: () => {
|
|
279
|
+
if (ctx.rolloutsLeft < 2 * minibatch + cases.pareto.length + cases.holdout.length)
|
|
280
|
+
return "budget_rollouts";
|
|
281
|
+
if (ctx.budget.remainingUsd() - ctx.reserveUsd < config.budgets.perRunUsd)
|
|
282
|
+
return "budget_usd";
|
|
283
|
+
return null;
|
|
284
|
+
},
|
|
285
|
+
onEvent: (event) => options.onEvent?.(event),
|
|
286
|
+
},
|
|
287
|
+
});
|
|
288
|
+
if (result.stopReason === "aborted" || signal.aborted)
|
|
289
|
+
throw new NexoError({ code: "INTERNAL", message: "The optimization was aborted" });
|
|
290
|
+
// The report (spec §8.2.5): the base and the best candidate on the holdout split only.
|
|
291
|
+
ctx.phase = "final";
|
|
292
|
+
options.onEvent?.({ type: "phase", phase: "holdout" });
|
|
293
|
+
const bestArtifactId = artifactOfCandidate.get(result.bestId) ?? base.id;
|
|
294
|
+
const best = getArtifact(db, bestArtifactId);
|
|
295
|
+
const before = perCaseOf(cases.holdout, baseline.holdout);
|
|
296
|
+
let after = before;
|
|
297
|
+
let deterministic = baseline.holdout.deterministic.some(Boolean);
|
|
298
|
+
if (bestArtifactId !== base.id) {
|
|
299
|
+
const bestRef = { id: result.bestId, artifactId: best.id, texts: textsOf(target, best.content) };
|
|
300
|
+
after = {};
|
|
301
|
+
for (const c of cases.holdout) {
|
|
302
|
+
try {
|
|
303
|
+
const one = await adapter.evaluate([c], bestRef, false, signal);
|
|
304
|
+
after[c.id] = one.scores[0] ?? { quality: 0, costUsd: 0 };
|
|
305
|
+
deterministic ||= one.deterministic.some(Boolean);
|
|
306
|
+
}
|
|
307
|
+
catch (error) {
|
|
308
|
+
if (!(error instanceof BudgetStop))
|
|
309
|
+
throw error;
|
|
310
|
+
// A holdout case the budget cannot cover counts against the candidate.
|
|
311
|
+
after[c.id] = { quality: 0, costUsd: 0 };
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
const key = judgeKey(config);
|
|
316
|
+
const judgeOnly = target.kind !== "judge_rubric" && !deterministic;
|
|
317
|
+
const holdoutIds = cases.holdout.map((c) => c.id);
|
|
318
|
+
const report = buildReport({
|
|
319
|
+
optimRunId,
|
|
320
|
+
target: target.key,
|
|
321
|
+
baseArtifactId: base.id,
|
|
322
|
+
bestArtifactId,
|
|
323
|
+
baseText: base.content,
|
|
324
|
+
bestText: best.content,
|
|
325
|
+
holdout: cases.holdout.map((c) => ({ caseId: c.id, input: c.input })),
|
|
326
|
+
before,
|
|
327
|
+
after,
|
|
328
|
+
candidates: listOptimCandidates(db, optimRunId),
|
|
329
|
+
evidence: { judgeOnly, humanScored: target.kind === "judge_rubric", judgeKey: key, kappa: key ? liveJudgeKappa(db, key) : null, minJudgeKappa: config.optimize.minJudgeKappa },
|
|
330
|
+
stopReason: result.stopReason,
|
|
331
|
+
spentUsd: ctx.budget.spentUsd,
|
|
332
|
+
rollouts: ctx.rollouts,
|
|
333
|
+
iterations: result.iterations,
|
|
334
|
+
merges: result.merges,
|
|
335
|
+
mergeNote: options.merge && result.merges === 0 && components.length === 1 ? `${target.key} has one component, so no merge is possible` : null,
|
|
336
|
+
createdAt: nowIso(),
|
|
337
|
+
});
|
|
338
|
+
writeFileSync(join(optimizer.runDir.root, "report.json"), `${JSON.stringify(redact(report), null, 2)}\n`);
|
|
339
|
+
// Lessons are kept only from an optimization whose best candidate beat the base, so one lucky minibatch does not harden into guidance.
|
|
340
|
+
if (wikisDir && report.improved && result.scratchpad.patterns.length + result.scratchpad.antiPatterns.length > 0) {
|
|
341
|
+
try {
|
|
342
|
+
writeScratchpad(wikisDir, target.key, result.scratchpad, { optimRunId, baseVersion: base.version, today: nowIso().slice(0, 10), redact: (text) => String(redact(text)) });
|
|
343
|
+
}
|
|
344
|
+
catch (error) {
|
|
345
|
+
logger.warn("could not write the optimization scratchpad to the wiki", { error });
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
retireOptimCandidates(db, optimRunId, bestArtifactId === base.id ? null : bestArtifactId);
|
|
349
|
+
closeOptimizerRun(env, optimizer, { status: "succeeded", costUsd: ownCost, costUnknown: ownUnknown, error: null, output: `best ${bestArtifactId === base.id ? "is the base version" : `v${best.version}`}` });
|
|
350
|
+
const run = finish({
|
|
351
|
+
status: "succeeded",
|
|
352
|
+
stopReason: result.stopReason,
|
|
353
|
+
bestArtifactId,
|
|
354
|
+
reportRef: "report.json",
|
|
355
|
+
judgeKey: key,
|
|
356
|
+
judgeOnly,
|
|
357
|
+
holdoutBefore: meanOf(before, holdoutIds).quality,
|
|
358
|
+
holdoutAfter: meanOf(after, holdoutIds).quality,
|
|
359
|
+
});
|
|
360
|
+
return { run, report };
|
|
361
|
+
}
|
|
362
|
+
catch (error) {
|
|
363
|
+
const aborted = signal.aborted;
|
|
364
|
+
const payload = redact(toErrorPayload(error));
|
|
365
|
+
retireOptimCandidates(db, optimRunId, null);
|
|
366
|
+
try {
|
|
367
|
+
closeOptimizerRun(env, optimizer, { status: aborted ? "aborted" : "failed", costUsd: ownCost, costUnknown: ownUnknown, error: aborted ? null : payload, output: "" });
|
|
368
|
+
}
|
|
369
|
+
catch (closeError) {
|
|
370
|
+
logger.error("closing the optimizer run failed", { error: closeError });
|
|
371
|
+
}
|
|
372
|
+
const run = finish({ status: aborted ? "aborted" : "failed", stopReason: aborted ? "aborted" : "error", error: aborted ? null : payload });
|
|
373
|
+
if (aborted)
|
|
374
|
+
return { run, report: null };
|
|
375
|
+
throw error;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { insertRun, RunDir, updateRun } from "../../../store/src/index.js";
|
|
2
|
+
import { Recorder } from "../engine/recorder.js";
|
|
3
|
+
function runJson(run, env, status, extra) {
|
|
4
|
+
return {
|
|
5
|
+
id: run.id,
|
|
6
|
+
parentRunId: null,
|
|
7
|
+
rootRunId: run.id,
|
|
8
|
+
instruction: run.instruction,
|
|
9
|
+
skill: null,
|
|
10
|
+
status,
|
|
11
|
+
model: null,
|
|
12
|
+
startedAt: run.startedAt,
|
|
13
|
+
endedAt: status === "running" ? null : env.now().toISOString(),
|
|
14
|
+
costUsd: extra.costUsd,
|
|
15
|
+
costUnknown: extra.costUnknown,
|
|
16
|
+
tokensIn: 0,
|
|
17
|
+
tokensOut: 0,
|
|
18
|
+
output: extra.output,
|
|
19
|
+
error: extra.error,
|
|
20
|
+
artifactSetId: null,
|
|
21
|
+
configSnapshotHash: env.deps.configHash,
|
|
22
|
+
workspace: run.runDir.workspaceDir,
|
|
23
|
+
taskGraphs: [],
|
|
24
|
+
optimRunId: run.id,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
/** Creates the optimization directory (`~/.nexo/optim/<id>`), the optimizer's run row and its `optimize` span. */
|
|
28
|
+
export function openOptimizerRun(env, optimRunId, target) {
|
|
29
|
+
const { db, redact, paths } = env.deps;
|
|
30
|
+
const runDir = RunDir.create(paths.optim, optimRunId, redact);
|
|
31
|
+
const startedAt = env.now().toISOString();
|
|
32
|
+
const instruction = `optimize ${target.key}`;
|
|
33
|
+
insertRun(db, {
|
|
34
|
+
id: optimRunId,
|
|
35
|
+
parentRunId: null,
|
|
36
|
+
rootRunId: optimRunId,
|
|
37
|
+
dirPath: runDir.root,
|
|
38
|
+
instruction,
|
|
39
|
+
skill: null,
|
|
40
|
+
status: "running",
|
|
41
|
+
configSnapshotHash: env.deps.configHash,
|
|
42
|
+
artifactSetId: null,
|
|
43
|
+
modelFinal: null,
|
|
44
|
+
tierFinal: null,
|
|
45
|
+
escalations: 0,
|
|
46
|
+
startedAt,
|
|
47
|
+
endedAt: null,
|
|
48
|
+
costUsd: 0,
|
|
49
|
+
costUnknown: false,
|
|
50
|
+
tokensIn: 0,
|
|
51
|
+
tokensOut: 0,
|
|
52
|
+
pid: process.pid,
|
|
53
|
+
error: null,
|
|
54
|
+
optimRunId,
|
|
55
|
+
});
|
|
56
|
+
const recorder = new Recorder({ db, runDir, runId: optimRunId, now: env.now, redact });
|
|
57
|
+
const run = { id: optimRunId, runDir, recorder, spanId: "", startedAt, instruction };
|
|
58
|
+
run.spanId = recorder.start({ kind: "optimize", name: target.key, parentSpanId: null, attrs: { target: target.key } });
|
|
59
|
+
runDir.writeRunJson(runJson(run, env, "running", { costUsd: 0, costUnknown: false, error: null, output: "" }));
|
|
60
|
+
return run;
|
|
61
|
+
}
|
|
62
|
+
/** Finishes the optimizer's span, run row and run.json; `costUsd` is what the optimizer's own calls cost. */
|
|
63
|
+
export function closeOptimizerRun(env, run, end) {
|
|
64
|
+
run.recorder.finish(run.spanId, { status: end.status === "succeeded" ? "ok" : "error", attrs: { status: end.status } });
|
|
65
|
+
updateRun(env.deps.db, run.id, { status: end.status, endedAt: env.now().toISOString(), costUsd: end.costUsd, costUnknown: end.costUnknown, error: end.error });
|
|
66
|
+
run.runDir.writeRunJson(runJson(run, env, end.status, end));
|
|
67
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { repairPrompt, validateReply } from "../../../memory/src/index.js";
|
|
2
|
+
import { BudgetStop, ProposalSchema, proposalProblem, reflectionPrompt, reflectionSystem } from "../../../optimize/src/index.js";
|
|
3
|
+
import { addSpend, dayScope, monthScope } from "../../../store/src/index.js";
|
|
4
|
+
import { NexoError } from "../../../telemetry/src/index.js";
|
|
5
|
+
import { optimScope } from "../cost/admission.js";
|
|
6
|
+
import { priceUsage } from "../cost/pricing.js";
|
|
7
|
+
import { runChargedCompletion } from "../engine/charged-completion.js";
|
|
8
|
+
/** The most output tokens a reflection reply may be capped at. */
|
|
9
|
+
export const REFLECTION_MAX_OUTPUT_TOKENS = 8192;
|
|
10
|
+
/** Output tokens a reply holding a rewrite of a text of `chars` characters needs: the JSON-escaped text (about 3 characters a token) plus the rationale. */
|
|
11
|
+
export function requiredReflectionTokens(chars) {
|
|
12
|
+
return Math.ceil(chars / 3) + 1024;
|
|
13
|
+
}
|
|
14
|
+
/** Refuses (INVALID_REQUEST) a base text whose rewrite cannot fit in `maxTokens` output tokens, before anything is spent. */
|
|
15
|
+
export function assertReflectionFits(target, texts, maxTokens = REFLECTION_MAX_OUTPUT_TOKENS) {
|
|
16
|
+
for (const [component, text] of Object.entries(texts)) {
|
|
17
|
+
const needed = requiredReflectionTokens(text.length);
|
|
18
|
+
if (needed <= maxTokens)
|
|
19
|
+
continue;
|
|
20
|
+
throw new NexoError({
|
|
21
|
+
code: "INVALID_REQUEST",
|
|
22
|
+
message: `The ${target.key} text (${component}, ${text.length} characters) is too long to optimize: a reflection reply needs about ${needed} output tokens and the limit is ${maxTokens}`,
|
|
23
|
+
hint: "Shorten the text, for example by moving detail into files the agent reads, then run the optimization again",
|
|
24
|
+
context: { target: target.key, component, chars: text.length, neededTokens: needed, maxTokens },
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/** Output tokens a reflection call is capped at and its budget reservation assumes: enough for the current text, at least `budgets.estimateOutputTokens`, at most REFLECTION_MAX_OUTPUT_TOKENS. */
|
|
29
|
+
export function reflectionOutputTokens(estimateOutputTokens, currentTextChars) {
|
|
30
|
+
return Math.min(REFLECTION_MAX_OUTPUT_TOKENS, Math.max(estimateOutputTokens, requiredReflectionTokens(currentTextChars)));
|
|
31
|
+
}
|
|
32
|
+
/** Charges a call to the optimization budget, the day, month and `optim:<id>` ledgers and the optimizer's totals. */
|
|
33
|
+
export function optimCharge(payer, usd, unknown) {
|
|
34
|
+
const now = payer.env.now();
|
|
35
|
+
payer.budget.charge(usd);
|
|
36
|
+
addSpend(payer.env.deps.db, [dayScope(now), monthScope(now), optimScope(payer.optimRunId)], usd, now.toISOString());
|
|
37
|
+
payer.addCost(usd, unknown);
|
|
38
|
+
}
|
|
39
|
+
/** Runs `fn`, turning a refused budget reservation into a BudgetStop that ends the loop. */
|
|
40
|
+
export async function stopOnRefusal(fn) {
|
|
41
|
+
try {
|
|
42
|
+
return await fn();
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
if (error instanceof NexoError && error.code === "BUDGET_EXCEEDED")
|
|
46
|
+
throw new BudgetStop("budget_usd", error.message);
|
|
47
|
+
throw error;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
/** One reflection completion on `models.s2`: a `gepa_call` span (tier `gepa`) charged to the optimization. */
|
|
51
|
+
async function reflectionCompletion(payer, request) {
|
|
52
|
+
const { env, budget } = payer;
|
|
53
|
+
const { config } = env.deps;
|
|
54
|
+
const spec = config.models.s2;
|
|
55
|
+
const model = env.registry.get(spec.provider, spec.id);
|
|
56
|
+
const effort = env.registry.clampEffort(model, spec.effort ?? "off");
|
|
57
|
+
const { text } = await stopOnRefusal(() => runChargedCompletion({
|
|
58
|
+
env,
|
|
59
|
+
recorder: payer.recorder,
|
|
60
|
+
budget,
|
|
61
|
+
...(payer.holdbackUsd ? { holdbackUsd: payer.holdbackUsd } : {}),
|
|
62
|
+
charge: (m, msg) => {
|
|
63
|
+
const priced = priceUsage(m, msg.usage, config.pricingOverrides);
|
|
64
|
+
optimCharge(payer, priced.costUsd, priced.costUnknown);
|
|
65
|
+
return priced;
|
|
66
|
+
},
|
|
67
|
+
refused: (estimate) => new NexoError({
|
|
68
|
+
code: "BUDGET_EXCEEDED",
|
|
69
|
+
message: `The optimization budget cannot cover a reflection call (estimated $${estimate.toFixed(4)}, $${budget.remainingUsd().toFixed(4)} left)`,
|
|
70
|
+
}),
|
|
71
|
+
abortMessage: "The reflection call was aborted",
|
|
72
|
+
}, {
|
|
73
|
+
model,
|
|
74
|
+
system: request.system,
|
|
75
|
+
user: request.user,
|
|
76
|
+
effort,
|
|
77
|
+
outputTokens: reflectionOutputTokens(config.budgets.estimateOutputTokens, request.currentTextChars),
|
|
78
|
+
span: { kind: "gepa_call", name: `${request.purpose} ${model.id}`, parentSpanId: payer.parentSpanId, tier: "gepa", attrs: { purpose: request.purpose } },
|
|
79
|
+
}, payer.signal));
|
|
80
|
+
return text;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Asks the reflection model for a new text of `component` (spec §8.2.2.4). An invalid reply gets one repair request;
|
|
84
|
+
* a second invalid reply, an empty or unchanged text is returned as `{ problem }`. Budget refusals throw BudgetStop.
|
|
85
|
+
*/
|
|
86
|
+
export async function proposeText(payer, input) {
|
|
87
|
+
const system = reflectionSystem(input.target);
|
|
88
|
+
const user = reflectionPrompt(input);
|
|
89
|
+
const currentTextChars = input.currentText.length;
|
|
90
|
+
const first = await reflectionCompletion(payer, { system, user, purpose: "reflect", currentTextChars });
|
|
91
|
+
let checked = validateReply(ProposalSchema, first);
|
|
92
|
+
if (!checked.ok) {
|
|
93
|
+
const second = await reflectionCompletion(payer, { system, user: `${user}\n\n${repairPrompt(first, checked.errors)}`, purpose: "reflect-repair", currentTextChars });
|
|
94
|
+
checked = validateReply(ProposalSchema, second);
|
|
95
|
+
if (!checked.ok)
|
|
96
|
+
return { problem: `the reflection reply did not match its schema after one repair: ${checked.errors.slice(0, 3).join("; ")}` };
|
|
97
|
+
}
|
|
98
|
+
const problem = proposalProblem(checked.value.text, input.currentText);
|
|
99
|
+
return problem ? { problem } : { text: checked.value.text, rationale: checked.value.rationale, ...(checked.value.prediction ? { prediction: checked.value.prediction } : {}) };
|
|
100
|
+
}
|