@hexdecimal16/nexo 0.3.5 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/src/common.js +2 -1
- package/cli/src/main.js +9 -0
- package/cli/src/review-queue.js +31 -0
- package/core/src/config/schema.js +8 -1
- package/core/src/engine/context-trim.js +26 -0
- package/core/src/engine/prompts.js +1 -0
- package/core/src/engine/run.js +58 -1
- package/core/src/engine/spawn-tool.js +5 -1
- package/core/src/optimize/holdout-gate.js +19 -0
- package/core/src/optimize/optimize.js +13 -2
- package/core/src/optimize/reflect.js +1 -1
- package/core/src/settings/config-edit.js +24 -0
- package/package.json +1 -1
- package/sdk/src/nexo.js +5 -1
- package/sdk/src/run-tree.js +6 -0
- package/store/src/index.js +1 -0
- package/store/src/review-queue.js +46 -0
- package/tools/src/builtin.js +12 -2
- package/tui/src/app.js +22 -1
- package/tui/src/commands.js +1 -0
- package/web/public/lib/api-types.ts +18 -0
- package/web/public/lib/run-view.ts +19 -1
- package/web/public/pages/run.ts +3 -1
- package/web/public/run.html +2 -1
- package/web/src/api.js +32 -0
- package/web/src/review-queue-api.js +18 -0
- package/web/src/server.js +7 -0
package/cli/src/common.js
CHANGED
|
@@ -10,12 +10,13 @@ export const USAGE = `Usage:
|
|
|
10
10
|
nexo --version print the installed version
|
|
11
11
|
nexo settings open the settings screen (needs a terminal)
|
|
12
12
|
nexo settings get <key>
|
|
13
|
-
nexo settings set <key> <value> keys: models.s1|s2|judge|reflector|embedding, budgets.*, tools.policy.*, tools.autoApprove, tools.nonInteractiveAsk, models.approver, scoring.judge, learning.*, memory.*, optimize.minCases|patience|minJudgeKappa|noveltySimilarity|offspringAlpha
|
|
13
|
+
nexo settings set <key> <value> keys: models.s1|s2|judge|reflector|embedding, budgets.*, tools.policy.*, tools.autoApprove, tools.nonInteractiveAsk, models.approver, scoring.judge, learning.*, memory.*, optimize.minCases|patience|minJudgeKappa|noveltySimilarity|offspringAlpha|reflectionTier|maxCaseDrop, loop.maxToolCalls|repeatLimit
|
|
14
14
|
nexo settings reset <key> remove it from the user config
|
|
15
15
|
nexo settings set-key <provider> reads the key from a hidden prompt, or from stdin when piped
|
|
16
16
|
nexo run <instruction...> [--skill <name>] [--model <s1|s2|provider/id>] [--effort <level>] [--workspace <dir>] [--budget <usd>] [--no-review]
|
|
17
17
|
nexo cost [--by model|tier|day|run|skill] [--since YYYY-MM-DD]
|
|
18
18
|
nexo reindex
|
|
19
|
+
nexo review-queue [--limit <n>] [--days <n>] recent runs without a verdict, worst first
|
|
19
20
|
nexo eval judge [--judge <provider/id[+provider/id…]>] judge-vs-human agreement and Cohen's κ
|
|
20
21
|
nexo eval add --target <target> --input <text> [--skill <name>] [--expected <text>] [--feedback <text>]
|
|
21
22
|
nexo eval list --target <target>
|
package/cli/src/main.js
CHANGED
|
@@ -9,6 +9,7 @@ import { evalCommand } from "./eval.js";
|
|
|
9
9
|
import { learnCommand, learnEventLine, memoryCommand, playbookCommand } from "./learn.js";
|
|
10
10
|
import { optimizeCommand } from "./optimize.js";
|
|
11
11
|
import { createTtyPrompter } from "./prompter.js";
|
|
12
|
+
import { reviewQueueCommand } from "./review-queue.js";
|
|
12
13
|
import { createTtyLineAsker, isInteractiveTerminal, promptReview } from "./review.js";
|
|
13
14
|
import { createRuntime } from "./runtime.js";
|
|
14
15
|
import { interactiveCommand, settingsScreenCommand } from "./interactive.js";
|
|
@@ -117,6 +118,12 @@ async function runCommand(args, io, options) {
|
|
|
117
118
|
io.stderr.write(`\x1b[2m${" ".repeat(event.depth)} auto-allowed: ${escapeControl(event.reason)}\x1b[22m\n`);
|
|
118
119
|
if (event.type === "escalation" && event.applied)
|
|
119
120
|
io.stderr.write(escalationLine(event.depth, event.model, event.trigger, event.reason));
|
|
121
|
+
if (event.type === "loop_guard") {
|
|
122
|
+
const what = event.kind === "max_tool_calls" ? `tool-call cap reached (loop.maxToolCalls ${event.limit}); asked the agent to report` : `blocked ${escapeControl(event.toolName)}: ${event.count} identical calls in a row (loop.repeatLimit ${event.limit})`;
|
|
123
|
+
io.stderr.write(`${" ".repeat(event.depth)} ⚠ ${what}\n`);
|
|
124
|
+
}
|
|
125
|
+
if (event.type === "context_trim" && event.depth === 0)
|
|
126
|
+
io.stderr.write(` · context nearly full: cleared ${event.cleared} old tool outputs (~${event.tokensBefore} → ~${event.tokensAfter} tokens)\n`);
|
|
120
127
|
},
|
|
121
128
|
onLearnEvent: (event) => {
|
|
122
129
|
const line = learnEventLine(event);
|
|
@@ -223,6 +230,8 @@ export async function runCli(argv, io, options = {}) {
|
|
|
223
230
|
return costCommand(rest, io, options);
|
|
224
231
|
case "reindex":
|
|
225
232
|
return reindexCommand(io, options);
|
|
233
|
+
case "review-queue":
|
|
234
|
+
return reviewQueueCommand(rest, io, options);
|
|
226
235
|
case "eval":
|
|
227
236
|
return await evalCommand(rest, io, options);
|
|
228
237
|
case "optimize":
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { parseArgs } from "node:util";
|
|
2
|
+
import { reviewQueue } from "../../store/src/index.js";
|
|
3
|
+
import { escapeControl } from "../../tui/src/index.js";
|
|
4
|
+
import { usageError } from "./common.js";
|
|
5
|
+
import { openStore } from "./learn.js";
|
|
6
|
+
/** `nexo review-queue [--limit <n>] [--days <n>]`: recent runs without a verdict, worst first. */
|
|
7
|
+
export function reviewQueueCommand(args, io, options) {
|
|
8
|
+
const { values } = parseArgs({ args, options: { limit: { type: "string" }, days: { type: "string" } } });
|
|
9
|
+
const limit = values.limit === undefined ? undefined : Number(values.limit);
|
|
10
|
+
const days = values.days === undefined ? undefined : Number(values.days);
|
|
11
|
+
if (limit !== undefined && !(Number.isInteger(limit) && limit > 0))
|
|
12
|
+
throw usageError("--limit must be a positive integer");
|
|
13
|
+
if (days !== undefined && !(Number.isInteger(days) && days > 0))
|
|
14
|
+
throw usageError("--days must be a positive integer");
|
|
15
|
+
const store = openStore(options);
|
|
16
|
+
try {
|
|
17
|
+
const items = reviewQueue(store.db, { ...(limit ? { limit } : {}), ...(days ? { days } : {}) });
|
|
18
|
+
if (items.length === 0)
|
|
19
|
+
io.stdout.write("No unreviewed runs in this period\n");
|
|
20
|
+
for (const i of items) {
|
|
21
|
+
const score = i.score === null ? "–" : i.score.toFixed(2);
|
|
22
|
+
io.stdout.write(`${i.runId} ${i.failureMode.padEnd(15)} score ${score} $${i.costUsd.toFixed(4)} ${escapeControl(i.instruction)}\n`);
|
|
23
|
+
}
|
|
24
|
+
if (items.length > 0)
|
|
25
|
+
io.stdout.write("Label a run in the UI or with /done and /notdone in a session\n");
|
|
26
|
+
return 0;
|
|
27
|
+
}
|
|
28
|
+
finally {
|
|
29
|
+
store.close();
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -79,6 +79,8 @@ export const ConfigSchema = Type.Object({
|
|
|
79
79
|
/** Also compact when one S2 request with the history would cost more than this share of the turn's budget; 0 turns it off. */
|
|
80
80
|
compactAtBudgetShare: Type.Number({ minimum: 0, maximum: 1 }),
|
|
81
81
|
}, { additionalProperties: false }),
|
|
82
|
+
/** Guards against a run that loops: a cap on tool calls and a limit on identical consecutive calls; 0 turns a guard off. */
|
|
83
|
+
loop: Type.Object({ maxToolCalls: Type.Integer({ minimum: 0 }), repeatLimit: Type.Integer({ minimum: 0 }) }, { additionalProperties: false }),
|
|
82
84
|
spawn: Type.Object({ maxParallelSubagents: Type.Integer({ minimum: 1 }), maxDepth: Type.Integer({ minimum: 0 }) }, { additionalProperties: false }),
|
|
83
85
|
/** Extra instruction files (paths or globs inside the project) loaded into the prompt next to AGENTS.md / CLAUDE.md. */
|
|
84
86
|
instructions: Type.Array(Type.String({ minLength: 1 })),
|
|
@@ -131,6 +133,10 @@ export const ConfigSchema = Type.Object({
|
|
|
131
133
|
patience: Type.Integer({ minimum: 1 }),
|
|
132
134
|
/** Lowest judge Cohen's κ that lets judge-only evidence promote a candidate. */
|
|
133
135
|
minJudgeKappa: Type.Number({ minimum: -1, maximum: 1 }),
|
|
136
|
+
/** Model tier that writes candidate edits. Edit quality barely depends on model size, so s1 is the default. */
|
|
137
|
+
reflectionTier: Type.Union([Type.Literal("s1"), Type.Literal("s2")]),
|
|
138
|
+
/** Largest drop in one holdout case's quality that a promoted candidate may have; a larger drop, or a lower holdout mean, reverts to the base version. */
|
|
139
|
+
maxCaseDrop: Type.Number({ minimum: 0, maximum: 1 }),
|
|
134
140
|
/** Embedding cosine at or above which a reflected text counts as a near-duplicate of a candidate and is skipped before any rollout. */
|
|
135
141
|
noveltySimilarity: Type.Number({ exclusiveMinimum: 0, maximum: 1 }),
|
|
136
142
|
/** How strongly a parent's child count lowers its chance of being sampled again (0 turns the penalty off). */
|
|
@@ -173,6 +179,7 @@ export const DEFAULT_CONFIG = {
|
|
|
173
179
|
estimateOutputTokens: 4096,
|
|
174
180
|
},
|
|
175
181
|
session: { compactAt: 0.8, compactAtBudgetShare: 0.2 },
|
|
182
|
+
loop: { maxToolCalls: 200, repeatLimit: 3 },
|
|
176
183
|
spawn: { maxParallelSubagents: 4, maxDepth: 4 },
|
|
177
184
|
instructions: [],
|
|
178
185
|
skills: { dirs: ["$NEXO_HOME/skills", "./.nexo/skills"], pins: {} },
|
|
@@ -188,5 +195,5 @@ export const DEFAULT_CONFIG = {
|
|
|
188
195
|
redaction: { patterns: [] },
|
|
189
196
|
memory: { enabled: true, maxTokens: 1500, weights: { relevance: 1, importance: 0.5, recency: 0.3 }, tauDays: 14, topK: 20, minRelevance: 0 },
|
|
190
197
|
learning: { enabled: true, maxPlaybookItems: 50, perJobUsd: 0.15, staleMinutes: 10, drainAfterRunMs: 30_000 },
|
|
191
|
-
optimize: { minCases: 10, patience: 5, minJudgeKappa: 0.6, noveltySimilarity: 0.98, offspringAlpha: 0.5, mine: { days: 30, maxRuns: 500, minCluster: 3, maxNew: 3, cosine: 0.8, jaccard: 0.5 }, climb: { guards: 2 } },
|
|
198
|
+
optimize: { minCases: 10, patience: 5, minJudgeKappa: 0.6, reflectionTier: "s1", maxCaseDrop: 0.2, noveltySimilarity: 0.98, offspringAlpha: 0.5, mine: { days: 30, maxRuns: 500, minCluster: 3, maxNew: 3, cosine: 0.8, jaccard: 0.5 }, climb: { guards: 2 } },
|
|
192
199
|
};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export const CLEARED_TOOL_OUTPUT = "[nexo] Earlier tool output cleared to save context. Run the tool again if you need it.";
|
|
2
|
+
/** Tool results shorter than this are left alone: the stub would not save anything. */
|
|
3
|
+
const MIN_CLEARABLE_CHARS = 400;
|
|
4
|
+
function textLength(content) {
|
|
5
|
+
if (typeof content === "string")
|
|
6
|
+
return content.length;
|
|
7
|
+
if (!Array.isArray(content))
|
|
8
|
+
return 0;
|
|
9
|
+
return content.reduce((sum, block) => sum + (block?.type === "text" && typeof block.text === "string" ? block.text.length : 0), 0);
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Replaces the content of every tool result except the newest `keepRecent` with a short stub. Used mid-run when a
|
|
13
|
+
* request nears the context window: it needs no model call, and the call and its arguments stay in the history.
|
|
14
|
+
*/
|
|
15
|
+
export function clearOldToolResults(messages, keepRecent) {
|
|
16
|
+
const resultIndexes = messages.flatMap((m, i) => (m.role === "toolResult" ? [i] : []));
|
|
17
|
+
const candidates = new Set(resultIndexes.slice(0, Math.max(0, resultIndexes.length - keepRecent)));
|
|
18
|
+
let cleared = 0;
|
|
19
|
+
const out = messages.map((m, i) => {
|
|
20
|
+
if (!candidates.has(i) || textLength(m.content) < MIN_CLEARABLE_CHARS)
|
|
21
|
+
return m;
|
|
22
|
+
cleared++;
|
|
23
|
+
return { ...m, content: [{ type: "text", text: CLEARED_TOOL_OUTPUT }] };
|
|
24
|
+
});
|
|
25
|
+
return { messages: cleared > 0 ? out : messages, cleared };
|
|
26
|
+
}
|
|
@@ -7,6 +7,7 @@ Working rules:
|
|
|
7
7
|
- Make independent tool calls in the same turn, and combine related shell steps into one command. To read several files, pass paths to one read call.
|
|
8
8
|
- Scope searches to a directory, never find /. Start a command that may outlive the bash timeout in the background (nohup ... > job.log 2>&1 &) and poll the log.
|
|
9
9
|
- A budget line is appended to your latest input each turn. Prefer fewer, larger steps when it is low. Never say you hit or ran out of budget unless the budget line shows $0.0000 remaining or a tool result says BUDGET_EXCEEDED; otherwise finish the task.
|
|
10
|
+
- Before you finish, check the result against the instruction: run the test, open the file you wrote, or re-read the output. Report what you verified and what you could not verify. If the same call fails twice with the same result, change your approach instead of repeating it.
|
|
10
11
|
- End with a concise final answer that states what you did and where any output files are.`;
|
|
11
12
|
export const DEFAULT_PLANNER_PROMPT = `Parallel work:
|
|
12
13
|
- When the instruction has independent parts, call spawn once with all independent tasks so they run in parallel.
|
package/core/src/engine/run.js
CHANGED
|
@@ -11,6 +11,7 @@ import { createApprovalClassifier } from "../approval/classifier.js";
|
|
|
11
11
|
import { budgetExhaustedError, checkAdmission, recordSpend, runBudgetLimit } from "../cost/admission.js";
|
|
12
12
|
import { budgetLine, RunBudget } from "../cost/budget.js";
|
|
13
13
|
import { estimateCallCostUsd, priceUsage } from "../cost/pricing.js";
|
|
14
|
+
import { clearOldToolResults } from "./context-trim.js";
|
|
14
15
|
import { classifyProviderError, contextOverflowError, providerRetryDelayMs, toolErrorText, withToolErrorCodes } from "./errors.js";
|
|
15
16
|
import { assistantText, contentText, estimatePromptChars, foreignToolCallsAsText, lastAssistant, recordedContent, withBudgetLine } from "./messages.js";
|
|
16
17
|
import { DEFAULT_PLANNER_PROMPT, DEFAULT_SYSTEM_PROMPT, renderSystemPrompt } from "./prompts.js";
|
|
@@ -33,6 +34,12 @@ function toolOutputExcerpt(text, pointer) {
|
|
|
33
34
|
}
|
|
34
35
|
/** Longest tool result text carried by a `tool_end` preview (`output` carries all of it). */
|
|
35
36
|
export const TOOL_END_PREVIEW_CHARS = 2_000;
|
|
37
|
+
/** The tools an explore-mode subagent keeps: read-only research. */
|
|
38
|
+
const EXPLORE_TOOLS = new Set(["read", "web_fetch", "load_skill"]);
|
|
39
|
+
/** JSON with object keys sorted at every level, so the same arguments in another key order compare equal. */
|
|
40
|
+
function stableJson(value) {
|
|
41
|
+
return JSON.stringify(value, (_key, v) => v && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) : v);
|
|
42
|
+
}
|
|
36
43
|
const extraToolFactories = [];
|
|
37
44
|
/** Registers tools that every run receives in addition to the built-ins (used by the spawn tool). Returns an unregister function. */
|
|
38
45
|
export function registerExtraTools(factory) {
|
|
@@ -601,6 +608,7 @@ export async function executeRun(env, input) {
|
|
|
601
608
|
if (deps.mcp)
|
|
602
609
|
baseTools.push(...(await deps.mcp.tools()));
|
|
603
610
|
const tools = baseTools
|
|
611
|
+
.filter((t) => !request.exploreOnly || EXPLORE_TOOLS.has(t.name))
|
|
604
612
|
.map((t) => (toolDescriptionOverrides.has(t.name) ? { ...t, description: toolDescriptionOverrides.get(t.name) } : t))
|
|
605
613
|
.map(withToolErrorCodes);
|
|
606
614
|
if (input.memoryContext) {
|
|
@@ -718,6 +726,13 @@ export async function executeRun(env, input) {
|
|
|
718
726
|
}
|
|
719
727
|
const approvedBy = new Map();
|
|
720
728
|
let classify = null;
|
|
729
|
+
let lastTrimCleared = 0;
|
|
730
|
+
/** completedCalls when the tool-call cap was first passed; tool calls from a later model reply end the run. */
|
|
731
|
+
let loopCapWarnedAt = null;
|
|
732
|
+
let loopLimitHit = false;
|
|
733
|
+
let toolCallCount = 0;
|
|
734
|
+
let lastCallSignature = "";
|
|
735
|
+
let repeatCount = 0;
|
|
721
736
|
const runAgent = new Agent({
|
|
722
737
|
streamFn: env.streamFn,
|
|
723
738
|
initialState: { systemPrompt, model, thinkingLevel: resolved.effort, tools, messages: history },
|
|
@@ -734,12 +749,46 @@ export async function executeRun(env, input) {
|
|
|
734
749
|
return undefined;
|
|
735
750
|
return { model: target, thinkingLevel: active.resolved.effort };
|
|
736
751
|
},
|
|
737
|
-
transformContext: async (messages) =>
|
|
752
|
+
transformContext: async (messages) => {
|
|
753
|
+
// Mid-turn relief: compaction runs only before a turn's first request, so a long tool-heavy turn clears old tool outputs instead.
|
|
754
|
+
let current = messages;
|
|
755
|
+
const tokens = Math.ceil(requestChars(current, tools) / 4);
|
|
756
|
+
if (tokens > config.session.compactAt * active.model.contextWindow) {
|
|
757
|
+
const trimmed = clearOldToolResults(current, 4);
|
|
758
|
+
if (trimmed.cleared > lastTrimCleared) {
|
|
759
|
+
const after = Math.ceil(requestChars(trimmed.messages, tools) / 4);
|
|
760
|
+
deps.onEvent?.({ type: "context_trim", runId, depth, cleared: trimmed.cleared, tokensBefore: tokens, tokensAfter: after });
|
|
761
|
+
}
|
|
762
|
+
lastTrimCleared = Math.max(lastTrimCleared, trimmed.cleared);
|
|
763
|
+
current = trimmed.messages;
|
|
764
|
+
}
|
|
765
|
+
return withBudgetLine(foreignToolCallsAsText(current, active.model), budgetLine(budget));
|
|
766
|
+
},
|
|
738
767
|
beforeToolCall: async ({ toolCall, args }, signal) => {
|
|
739
768
|
if (budget.isExhausted()) {
|
|
740
769
|
budgetExceeded = true;
|
|
741
770
|
return { block: true, reason: "BUDGET_EXCEEDED: the run budget is exhausted", terminate: true };
|
|
742
771
|
}
|
|
772
|
+
const { maxToolCalls, repeatLimit } = config.loop;
|
|
773
|
+
toolCallCount++;
|
|
774
|
+
if (maxToolCalls > 0 && toolCallCount > maxToolCalls) {
|
|
775
|
+
if (loopCapWarnedAt !== null && completedCalls !== loopCapWarnedAt) {
|
|
776
|
+
loopLimitHit = true;
|
|
777
|
+
return { block: true, reason: `LOOP_LIMIT: the run kept calling tools after passing loop.maxToolCalls (${maxToolCalls}); it ends here.`, terminate: true };
|
|
778
|
+
}
|
|
779
|
+
if (loopCapWarnedAt === null) {
|
|
780
|
+
loopCapWarnedAt = completedCalls;
|
|
781
|
+
deps.onEvent?.({ type: "loop_guard", runId, depth, toolCallId: toolCall.id, toolName: toolCall.name, kind: "max_tool_calls", count: toolCallCount - 1, limit: maxToolCalls });
|
|
782
|
+
}
|
|
783
|
+
return { block: true, reason: `LOOP_LIMIT: this run made ${maxToolCalls} tool calls (loop.maxToolCalls). Do not call more tools: reply now with what you have done, what you verified and what remains.` };
|
|
784
|
+
}
|
|
785
|
+
const signature = `${toolCall.name}:${stableJson(args)}`;
|
|
786
|
+
repeatCount = signature === lastCallSignature ? repeatCount + 1 : 1;
|
|
787
|
+
lastCallSignature = signature;
|
|
788
|
+
if (repeatLimit > 0 && repeatCount > repeatLimit) {
|
|
789
|
+
deps.onEvent?.({ type: "loop_guard", runId, depth, toolCallId: toolCall.id, toolName: toolCall.name, kind: "repeat", count: repeatCount, limit: repeatLimit });
|
|
790
|
+
return { block: true, reason: `LOOP_REPEAT: ${toolCall.name} was called ${repeatCount} times in a row with identical arguments (loop.repeatLimit ${repeatLimit}). Change your approach or report the blocker.` };
|
|
791
|
+
}
|
|
743
792
|
const classifier = classify;
|
|
744
793
|
const auth = await env.policy.authorize(deps.prompter, toolCall.name, args, signal, depth > 0 ? `subagent ${runId.slice(-6)}` : undefined, approvals
|
|
745
794
|
? {
|
|
@@ -953,6 +1002,14 @@ export async function executeRun(env, input) {
|
|
|
953
1002
|
status = "budget_exceeded";
|
|
954
1003
|
error = budgetExhaustedError(budget, input.budgetLimit ?? "run", sessionId !== null && !parent).toPayload();
|
|
955
1004
|
}
|
|
1005
|
+
else if (loopLimitHit) {
|
|
1006
|
+
status = "failed";
|
|
1007
|
+
error = new NexoError({
|
|
1008
|
+
code: "LOOP_LIMIT",
|
|
1009
|
+
message: `The run passed loop.maxToolCalls (${config.loop.maxToolCalls}) and kept calling tools after it was asked to report`,
|
|
1010
|
+
hint: "Raise loop.maxToolCalls (0 turns the cap off) or split the task into smaller ones",
|
|
1011
|
+
}).toPayload();
|
|
1012
|
+
}
|
|
956
1013
|
else if (input.signal?.aborted || last?.stopReason === "aborted") {
|
|
957
1014
|
status = "aborted";
|
|
958
1015
|
}
|
|
@@ -12,6 +12,9 @@ const SpawnTaskSchema = Type.Object({
|
|
|
12
12
|
effort: Type.Optional(EffortSchema),
|
|
13
13
|
budgetUsd: Type.Optional(Type.Number({ exclusiveMinimum: 0 })),
|
|
14
14
|
dependsOn: Type.Optional(Type.Array(Type.String(), { description: "Task ids whose outputs this task needs" })),
|
|
15
|
+
mode: Type.Optional(Type.Union([Type.Literal("explore"), Type.Literal("work")], {
|
|
16
|
+
description: '"explore": read-only research (read, web_fetch, load_skill) that returns a short summary and keeps the parent context small; "work" (default): full tools. Run write-capable tasks one at a time.',
|
|
17
|
+
})),
|
|
15
18
|
});
|
|
16
19
|
const SpawnParams = Type.Object({ tasks: Type.Array(SpawnTaskSchema, { minItems: 1 }) });
|
|
17
20
|
function withDependencyOutputs(instruction, outputs) {
|
|
@@ -50,7 +53,8 @@ async function runSpawn(ctx, toolCallId, tasks, signal) {
|
|
|
50
53
|
const startedAt = env.now().toISOString();
|
|
51
54
|
const runChild = (childBudgetUsd) => executeRun(env, {
|
|
52
55
|
request: {
|
|
53
|
-
instruction: withDependencyOutputs(task.instruction, input.dependencyOutputs),
|
|
56
|
+
instruction: task.mode === "explore" ? `${withDependencyOutputs(task.instruction, input.dependencyOutputs)}\n\nYou are an explorer: you cannot change anything. Finish with a summary of at most 300 words: findings, source paths or URLs, and what you could not verify.` : withDependencyOutputs(task.instruction, input.dependencyOutputs),
|
|
57
|
+
...(task.mode === "explore" ? { exploreOnly: true } : {}),
|
|
54
58
|
skill: task.skill,
|
|
55
59
|
model: task.model,
|
|
56
60
|
effort: task.effort,
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/** Float noise below this is not a drop. */
|
|
2
|
+
const EPSILON = 1e-9;
|
|
3
|
+
/**
|
|
4
|
+
* Verify-or-rollback check on the holdout split: a candidate regresses when its holdout mean is lower than the base's,
|
|
5
|
+
* or when any one case drops by more than `maxCaseDrop`. A case with no score after counts as 0.
|
|
6
|
+
*/
|
|
7
|
+
export function holdoutRegression(before, after, ids, maxCaseDrop) {
|
|
8
|
+
if (ids.length === 0)
|
|
9
|
+
return { regressed: false, meanDrop: 0, worstDrop: 0 };
|
|
10
|
+
let sumDrop = 0;
|
|
11
|
+
let worstDrop = 0;
|
|
12
|
+
for (const id of ids) {
|
|
13
|
+
const drop = (before[id]?.quality ?? 0) - (after[id]?.quality ?? 0);
|
|
14
|
+
sumDrop += drop;
|
|
15
|
+
worstDrop = Math.max(worstDrop, drop);
|
|
16
|
+
}
|
|
17
|
+
const meanDrop = sumDrop / ids.length;
|
|
18
|
+
return { regressed: meanDrop > EPSILON || worstDrop > maxCaseDrop + EPSILON, meanDrop, worstDrop };
|
|
19
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { holdoutRegression } from "./holdout-gate.js";
|
|
1
2
|
import { existsSync, writeFileSync } from "node:fs";
|
|
2
3
|
import { join } from "node:path";
|
|
3
4
|
import { BudgetStop, buildReport, contentOf, MAX_CANDIDATE_CHARS, meanOf, MINIBATCH_SIZE, mulberry32, runGepa, targetComponents, textsOf, } from "../../../optimize/src/index.js";
|
|
@@ -298,8 +299,8 @@ export async function startOptimization(baseEnv, options) {
|
|
|
298
299
|
// The report (spec §8.2.5): the base and the best candidate on the holdout split only.
|
|
299
300
|
ctx.phase = "final";
|
|
300
301
|
options.onEvent?.({ type: "phase", phase: "holdout" });
|
|
301
|
-
|
|
302
|
-
|
|
302
|
+
let bestArtifactId = artifactOfCandidate.get(result.bestId) ?? base.id;
|
|
303
|
+
let best = getArtifact(db, bestArtifactId);
|
|
303
304
|
const before = perCaseOf(cases.holdout, baseline.holdout);
|
|
304
305
|
let after = before;
|
|
305
306
|
let deterministic = baseline.holdout.deterministic.some(Boolean);
|
|
@@ -320,6 +321,16 @@ export async function startOptimization(baseEnv, options) {
|
|
|
320
321
|
}
|
|
321
322
|
}
|
|
322
323
|
}
|
|
324
|
+
// Verify-or-rollback: a candidate that lowers the holdout mean, or drops any one case by more than maxCaseDrop, is not promoted.
|
|
325
|
+
if (bestArtifactId !== base.id) {
|
|
326
|
+
const gate = holdoutRegression(before, after, cases.holdout.map((c) => c.id), config.optimize.maxCaseDrop);
|
|
327
|
+
if (gate.regressed) {
|
|
328
|
+
logger.warn("optimization reverted to the base version: the best candidate regressed on the holdout", { optim_run_id: optimRunId, mean_drop: gate.meanDrop, worst_case_drop: gate.worstDrop });
|
|
329
|
+
bestArtifactId = base.id;
|
|
330
|
+
best = base;
|
|
331
|
+
after = before;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
323
334
|
const key = judgeKey(config);
|
|
324
335
|
const judgeOnly = target.kind !== "judge_rubric" && !deterministic;
|
|
325
336
|
const holdoutIds = cases.holdout.map((c) => c.id);
|
|
@@ -51,7 +51,7 @@ export async function stopOnRefusal(fn) {
|
|
|
51
51
|
async function reflectionCompletion(payer, request) {
|
|
52
52
|
const { env, budget } = payer;
|
|
53
53
|
const { config } = env.deps;
|
|
54
|
-
const spec = config.models.
|
|
54
|
+
const spec = config.models[config.optimize.reflectionTier];
|
|
55
55
|
const model = env.registry.get(spec.provider, spec.id);
|
|
56
56
|
const effort = env.registry.clampEffort(model, spec.effort ?? "off");
|
|
57
57
|
const { text } = await stopOnRefusal(() => runChargedCompletion({
|
|
@@ -70,6 +70,30 @@ export const EDITABLE_SETTINGS = [
|
|
|
70
70
|
description: "Also compact when one S2 request with the history would cost more than this share of a turn's budget (0 to 1; 0 = off)",
|
|
71
71
|
type: { kind: "number", integer: false, min: 0, max: 1 },
|
|
72
72
|
},
|
|
73
|
+
{
|
|
74
|
+
key: "optimize.reflectionTier",
|
|
75
|
+
label: "Optimizer edit model",
|
|
76
|
+
description: "Which model tier writes candidate edits in `nexo optimize` (s1 is cheaper; edit quality barely depends on model size)",
|
|
77
|
+
type: { kind: "enum", values: ["s1", "s2"] },
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
key: "optimize.maxCaseDrop",
|
|
81
|
+
label: "Optimizer max case drop",
|
|
82
|
+
description: "Largest quality drop on one holdout case a promoted candidate may have (0 to 1); a larger drop or a lower holdout mean reverts to the base version",
|
|
83
|
+
type: { kind: "number", integer: false, min: 0, max: 1 },
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
key: "loop.maxToolCalls",
|
|
87
|
+
label: "Max tool calls",
|
|
88
|
+
description: "Most tool calls one run may make before it is stopped and told to report (0 = no cap)",
|
|
89
|
+
type: { kind: "number", integer: true, min: 0 },
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
key: "loop.repeatLimit",
|
|
93
|
+
label: "Repeat limit",
|
|
94
|
+
description: "Most identical consecutive tool calls (same tool and arguments) before the next one is blocked (0 = off)",
|
|
95
|
+
type: { kind: "number", integer: true, min: 0 },
|
|
96
|
+
},
|
|
73
97
|
{
|
|
74
98
|
key: "spawn.maxParallelSubagents",
|
|
75
99
|
label: "Parallel subagents",
|
package/package.json
CHANGED
package/sdk/src/nexo.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createSession, createSettings, nexoModels, resumeSession, runDoctor, } from "../../core/src/index.js";
|
|
2
|
-
import { resolveNexoPaths } from "../../store/src/index.js";
|
|
2
|
+
import { resolveNexoPaths, reviewQueue } from "../../store/src/index.js";
|
|
3
3
|
import { NexoError } from "../../telemetry/src/index.js";
|
|
4
4
|
import { nonInteractivePrompter } from "../../tools/src/index.js";
|
|
5
5
|
import { createRuntime } from "./runtime.js";
|
|
@@ -76,6 +76,10 @@ export function createNexo(options = {}) {
|
|
|
76
76
|
live();
|
|
77
77
|
return resumeSession(sessionDeps(), id);
|
|
78
78
|
},
|
|
79
|
+
reviewQueue(queueOptions) {
|
|
80
|
+
live();
|
|
81
|
+
return reviewQueue(runtime.db, queueOptions);
|
|
82
|
+
},
|
|
79
83
|
doctor() {
|
|
80
84
|
return runDoctor({ paths: runtime.paths, settings: runtime.settings, env, cwd });
|
|
81
85
|
},
|
package/sdk/src/run-tree.js
CHANGED
|
@@ -27,6 +27,7 @@ export function trackRunTree(onChange) {
|
|
|
27
27
|
status: "running",
|
|
28
28
|
costUsd: 0,
|
|
29
29
|
toolCalls: 0,
|
|
30
|
+
loopGuards: 0,
|
|
30
31
|
currentTool: null,
|
|
31
32
|
error: null,
|
|
32
33
|
startedAt: Date.now(),
|
|
@@ -48,6 +49,11 @@ export function trackRunTree(onChange) {
|
|
|
48
49
|
n.currentTool = event.toolName;
|
|
49
50
|
});
|
|
50
51
|
break;
|
|
52
|
+
case "loop_guard":
|
|
53
|
+
update(event.runId, (n) => {
|
|
54
|
+
n.loopGuards++;
|
|
55
|
+
});
|
|
56
|
+
break;
|
|
51
57
|
case "tool_end":
|
|
52
58
|
update(event.runId, (n) => {
|
|
53
59
|
n.currentTool = null;
|
package/store/src/index.js
CHANGED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
export const DEFAULT_REVIEW_QUEUE_SIZE = 15;
|
|
2
|
+
const LOW_SCORE = 0.6;
|
|
3
|
+
/**
|
|
4
|
+
* Recent finished root runs that have no human verdict, worst first: failed, over-budget and aborted runs, then low
|
|
5
|
+
* scores, then unscored runs, then the rest by cost. A person reviews the head of this list each week and labels it.
|
|
6
|
+
*/
|
|
7
|
+
export function reviewQueue(db, options = {}) {
|
|
8
|
+
const limit = Math.max(1, Math.min(100, options.limit ?? DEFAULT_REVIEW_QUEUE_SIZE));
|
|
9
|
+
const since = new Date((options.now ?? new Date()).getTime() - (options.days ?? 7) * 86_400_000).toISOString();
|
|
10
|
+
const rows = db
|
|
11
|
+
.prepare(`SELECT r.id, r.instruction, r.status, r.started_at, r.cost_usd, r.error_json,
|
|
12
|
+
(SELECT AVG(s.value) FROM scores s WHERE s.run_id = r.id AND s.status = 'ok' AND s.round = (SELECT MAX(s2.round) FROM scores s2 WHERE s2.run_id = r.id)) AS score
|
|
13
|
+
FROM runs r
|
|
14
|
+
WHERE r.parent_run_id IS NULL AND r.status != 'running' AND r.started_at >= ?
|
|
15
|
+
AND NOT EXISTS (SELECT 1 FROM feedback f WHERE f.run_id = r.id)`)
|
|
16
|
+
.all(since);
|
|
17
|
+
const items = rows.map((row) => {
|
|
18
|
+
const status = String(row.status);
|
|
19
|
+
const score = row.score === null ? null : Number(row.score);
|
|
20
|
+
const failureMode = status === "failed" || status === "budget_exceeded" || status === "aborted" ? status : score === null ? "unscored" : score < LOW_SCORE ? "low_score" : "ok";
|
|
21
|
+
let errorCode = null;
|
|
22
|
+
if (typeof row.error_json === "string") {
|
|
23
|
+
try {
|
|
24
|
+
const code = JSON.parse(row.error_json).code;
|
|
25
|
+
errorCode = typeof code === "string" ? code : null;
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
errorCode = null;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
const first = String(row.instruction).split("\n")[0] ?? "";
|
|
32
|
+
return {
|
|
33
|
+
runId: String(row.id),
|
|
34
|
+
instruction: first.length > 120 ? `${first.slice(0, 119)}…` : first,
|
|
35
|
+
status,
|
|
36
|
+
failureMode,
|
|
37
|
+
errorCode,
|
|
38
|
+
score,
|
|
39
|
+
costUsd: Number(row.cost_usd),
|
|
40
|
+
startedAt: String(row.started_at),
|
|
41
|
+
};
|
|
42
|
+
});
|
|
43
|
+
const rank = { failed: 0, budget_exceeded: 1, aborted: 2, low_score: 3, unscored: 4, ok: 5 };
|
|
44
|
+
items.sort((a, b) => rank[a.failureMode] - rank[b.failureMode] || (a.score ?? 1) - (b.score ?? 1) || b.costUsd - a.costUsd || b.startedAt.localeCompare(a.startedAt));
|
|
45
|
+
return items.slice(0, limit);
|
|
46
|
+
}
|
package/tools/src/builtin.js
CHANGED
|
@@ -43,6 +43,16 @@ function ensureExitHandler() {
|
|
|
43
43
|
exitHandlerRegistered = true;
|
|
44
44
|
process.on("exit", killLiveBashGroups);
|
|
45
45
|
}
|
|
46
|
+
const SECRET_ENV_NAME = /(^|_)(API_?KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIALS?|PRIVATE_KEY|ACCESS_KEY|SESSION_KEY)(_|$)|^(AWS|AZURE|GCP|GOOGLE|OPENAI|ANTHROPIC|GITHUB|GH|NPM|SSH_AUTH)_.*(KEY|TOKEN|SECRET|SOCK)/i;
|
|
47
|
+
/** The process environment without variables whose names look like secrets (API keys, tokens, passwords), for commands the agent runs. */
|
|
48
|
+
export function bashEnv(source = process.env) {
|
|
49
|
+
const out = {};
|
|
50
|
+
for (const [name, value] of Object.entries(source)) {
|
|
51
|
+
if (value !== undefined && !SECRET_ENV_NAME.test(name))
|
|
52
|
+
out[name] = value;
|
|
53
|
+
}
|
|
54
|
+
return out;
|
|
55
|
+
}
|
|
46
56
|
/** Runs `command` with `bash -c` in its own process group, killed on timeout, abort or host exit. Never rejects. */
|
|
47
57
|
export function runBash(run) {
|
|
48
58
|
const { signal } = run;
|
|
@@ -239,11 +249,11 @@ export function createBuiltinTools(ctx) {
|
|
|
239
249
|
const bash = {
|
|
240
250
|
name: "bash",
|
|
241
251
|
label: "Run command",
|
|
242
|
-
description: "Run a bash command in the workspace directory.",
|
|
252
|
+
description: "Run a bash command in the workspace directory (bash -c, no TTY, no stdin). Returns `exit code: N` followed by combined stdout and stderr; output beyond the limit is truncated, so filter with grep, head or tail. Chain related steps with && in one call. A command that may outlive the timeout belongs in the background: nohup <command> > job.log 2>&1 &, then poll job.log. Pass timeoutMs to raise the default. The environment is filtered, so secrets such as API keys are not available.",
|
|
243
253
|
parameters: BashParams,
|
|
244
254
|
execute: async (_id, params, signal) => {
|
|
245
255
|
const timeoutMs = params.timeoutMs ?? ctx.bashTimeoutMs;
|
|
246
|
-
const r = await runBash({ command: params.command, cwd: root, timeoutMs, signal });
|
|
256
|
+
const r = await runBash({ command: params.command, cwd: root, timeoutMs, signal, env: bashEnv() });
|
|
247
257
|
const body = `${r.output}${r.truncated ? "\n[output truncated]" : ""}`;
|
|
248
258
|
switch (r.ended) {
|
|
249
259
|
case "failed":
|
package/tui/src/app.js
CHANGED
|
@@ -2,7 +2,7 @@ import { homedir } from "node:os";
|
|
|
2
2
|
import { sep } from "node:path";
|
|
3
3
|
import { CombinedAutocompleteProvider, Container, Editor, isKeyRelease, isKeyRepeat, Loader, matchesKey, ProcessTerminal, setCapabilityOverrides, TuiMainScreen, } from "@earendil-works/pi-tui";
|
|
4
4
|
import { listSessions, projectRoots } from "../../core/src/index.js";
|
|
5
|
-
import { getSessionRow, sessionNotFound } from "../../store/src/index.js";
|
|
5
|
+
import { getSessionRow, reviewQueue, sessionNotFound } from "../../store/src/index.js";
|
|
6
6
|
import { toErrorPayload } from "../../telemetry/src/index.js";
|
|
7
7
|
import { describeSessionRule } from "../../tools/src/index.js";
|
|
8
8
|
import { createApprovalPrompter } from "./approvals.js";
|
|
@@ -643,6 +643,20 @@ class InteractiveApp {
|
|
|
643
643
|
this.chat.addChild(new Block(`score ${escapeControl(oneLine(event.scorer, 60))}: ${scoreVerdict(event)}`, style.gray, " "));
|
|
644
644
|
break;
|
|
645
645
|
}
|
|
646
|
+
case "loop_guard": {
|
|
647
|
+
this.currentText = null;
|
|
648
|
+
const what = event.kind === "max_tool_calls"
|
|
649
|
+
? `tool-call cap reached (loop.maxToolCalls ${event.limit}); the agent was asked to report`
|
|
650
|
+
: `blocked ${escapeControl(oneLine(event.toolName, 40))}: ${event.count} identical calls in a row (loop.repeatLimit ${event.limit})`;
|
|
651
|
+
this.chat.addChild(new Block(`⚠ ${what}`, style.yellow, " "));
|
|
652
|
+
break;
|
|
653
|
+
}
|
|
654
|
+
case "context_trim":
|
|
655
|
+
if (event.depth !== 0)
|
|
656
|
+
return;
|
|
657
|
+
this.currentText = null;
|
|
658
|
+
this.chat.addChild(new Block(`context nearly full: cleared ${event.cleared} old tool outputs (~${event.tokensBefore} → ~${event.tokensAfter} tokens)`, style.gray, " "));
|
|
659
|
+
break;
|
|
646
660
|
case "compaction_start":
|
|
647
661
|
this.currentText = null;
|
|
648
662
|
this.chat.addChild(new Block("compacting earlier turns…", style.gray, " "));
|
|
@@ -769,6 +783,13 @@ class InteractiveApp {
|
|
|
769
783
|
this.say("── conversation cleared ── (same session; earlier turns stay in its history)");
|
|
770
784
|
this.tui.requestRender(true);
|
|
771
785
|
break;
|
|
786
|
+
case "queue": {
|
|
787
|
+
const items = reviewQueue(this.options.db, { limit: 10 });
|
|
788
|
+
this.say(items.length === 0
|
|
789
|
+
? "no unreviewed runs in the last 7 days"
|
|
790
|
+
: items.map((i) => `${i.runId.slice(-6)} ${i.failureMode} ${i.score === null ? "–" : i.score.toFixed(2)} ${escapeControl(i.instruction)}`).join("\n"));
|
|
791
|
+
break;
|
|
792
|
+
}
|
|
772
793
|
case "sessions":
|
|
773
794
|
this.say(this.sessionsText(session));
|
|
774
795
|
break;
|
package/tui/src/commands.js
CHANGED
|
@@ -17,6 +17,7 @@ export const SLASH_COMMANDS = [
|
|
|
17
17
|
{ name: "done", description: "Mark the last turn done" },
|
|
18
18
|
{ name: "notdone", description: "Mark the last turn not done", argumentHint: "[comment]" },
|
|
19
19
|
{ name: "cost", description: "Show the session's cost and budget" },
|
|
20
|
+
{ name: "queue", description: "List recent runs without a verdict, worst first" },
|
|
20
21
|
{ name: "budget", description: "Set this session's budget cap", argumentHint: "<usd>" },
|
|
21
22
|
{ name: "model", description: "Set this session's model (default clears it)", argumentHint: "<s1|s2|provider/id|default>", getArgumentCompletions: items(["s1", "s2", "default"]) },
|
|
22
23
|
{ name: "effort", description: "Set this session's effort (default clears it)", argumentHint: "<level|default>", getArgumentCompletions: items([...EFFORTS, "default"]) },
|
|
@@ -175,6 +175,24 @@ export interface RunDetailResponse {
|
|
|
175
175
|
escalations: Escalation[];
|
|
176
176
|
/** True when the latest judge verdict and the latest human verdict disagree. */
|
|
177
177
|
judgeDisagrees: boolean;
|
|
178
|
+
/** The memories retrieval chose for this run (or for the session turn whose set it reuses); null when none was recorded. */
|
|
179
|
+
injectedMemories: InjectedMemories | null;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
export interface InjectedMemory {
|
|
183
|
+
id: string;
|
|
184
|
+
score: number | null;
|
|
185
|
+
relevance: number | null;
|
|
186
|
+
/** The stored memory; null when it no longer exists. */
|
|
187
|
+
memory: Memory | null;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
export interface InjectedMemories {
|
|
191
|
+
/** The run whose `memory_read` span recorded the selection (an earlier turn for later turns of a session). */
|
|
192
|
+
fromRunId: string;
|
|
193
|
+
mode: string | null;
|
|
194
|
+
candidates: number | null;
|
|
195
|
+
items: InjectedMemory[];
|
|
178
196
|
}
|
|
179
197
|
|
|
180
198
|
export interface Span {
|
|
@@ -1,8 +1,26 @@
|
|
|
1
|
-
import type { Feedback, ReadOnlyInfo, RunDetailResponse, RunStatus, RunSummary } from "./api-types.ts";
|
|
1
|
+
import type { Feedback, InjectedMemories, ReadOnlyInfo, RunDetailResponse, RunStatus, RunSummary } from "./api-types.ts";
|
|
2
2
|
import { renderError, runLink, sessionLink, stateClass, statusBadge, verdictBadge } from "./common-view.ts";
|
|
3
3
|
import { durationMs, formatDuration, formatTime, formatUsd, shortId, truncate } from "./format.ts";
|
|
4
4
|
import { html, type SafeHtml } from "./html.ts";
|
|
5
5
|
|
|
6
|
+
/** The memories injected into the run, with where the selection was recorded and each memory's rank score. */
|
|
7
|
+
export function renderInjectedMemories(data: InjectedMemories | null, runId: string): SafeHtml {
|
|
8
|
+
if (!data) return html`<p class="muted">No memory retrieval was recorded for this run.</p>`;
|
|
9
|
+
const origin = data.fromRunId === runId ? "" : html` Retrieved at an earlier turn of this session: <a href="${runLink(data.fromRunId)}">run ${shortId(data.fromRunId)}</a>.`;
|
|
10
|
+
const facts = html`<p class="muted">${data.mode ? `Mode ${data.mode}. ` : ""}${data.candidates !== null ? `${data.candidates} candidates. ` : ""}${origin}</p>`;
|
|
11
|
+
if (data.items.length === 0) return html`${facts}<p class="muted">Retrieval selected no memories.</p>`;
|
|
12
|
+
return html`${facts}<table class="runs memories">
|
|
13
|
+
<thead><tr><th>Scope</th><th>Type</th><th>Source</th><th>Memory</th><th class="num">Score</th><th class="num">Relevance</th></tr></thead>
|
|
14
|
+
<tbody>${data.items.map(
|
|
15
|
+
(i) => html`<tr>
|
|
16
|
+
<td class="mono">${i.memory?.scope ?? ""}</td><td>${i.memory?.type ?? ""}</td><td>${i.memory?.source ?? ""}</td>
|
|
17
|
+
<td>${i.memory ? i.memory.content : html`<span class="muted mono">${i.id} (no longer in the store)</span>`}${i.memory?.retiredAt ? html` <span class="muted">retired since</span>` : ""}${i.memory?.supersededBy ? html` <span class="muted">replaced since</span>` : ""}</td>
|
|
18
|
+
<td class="num">${i.score === null ? "" : i.score.toFixed(2)}</td><td class="num">${i.relevance === null ? "" : i.relevance.toFixed(2)}</td>
|
|
19
|
+
</tr>`,
|
|
20
|
+
)}</tbody>
|
|
21
|
+
</table>`;
|
|
22
|
+
}
|
|
23
|
+
|
|
6
24
|
export function shouldPoll(status: RunStatus): boolean {
|
|
7
25
|
return status === "running";
|
|
8
26
|
}
|
package/web/public/pages/run.ts
CHANGED
|
@@ -12,7 +12,7 @@ import { renderError } from "../lib/common-view.ts";
|
|
|
12
12
|
import { filePath, mediaKind, renderFile, renderListing, renderMedia } from "../lib/files-view.ts";
|
|
13
13
|
import { html, type SafeHtml } from "../lib/html.ts";
|
|
14
14
|
import { createPoller } from "../lib/poller.ts";
|
|
15
|
-
import { renderChildren, renderFeedbackForm, renderRunHeader, shouldPoll } from "../lib/run-view.ts";
|
|
15
|
+
import { renderChildren, renderFeedbackForm, renderInjectedMemories, renderRunHeader, shouldPoll } from "../lib/run-view.ts";
|
|
16
16
|
import { renderEscalations, renderScores } from "../lib/score-view.ts";
|
|
17
17
|
import { findSpan, renderSpanDetail, renderSpanTree } from "../lib/span-view.ts";
|
|
18
18
|
import { renderTimeline, timelineRows } from "../lib/timeline.ts";
|
|
@@ -23,6 +23,7 @@ const header = el<HTMLElement>("#header");
|
|
|
23
23
|
const review = el<HTMLElement>("#review");
|
|
24
24
|
const scores = el<HTMLElement>("#scores");
|
|
25
25
|
const escalations = el<HTMLElement>("#escalations");
|
|
26
|
+
const injected = el<HTMLElement>("#injected-memories");
|
|
26
27
|
const children = el<HTMLElement>("#children");
|
|
27
28
|
const timeline = el<HTMLElement>("#timeline");
|
|
28
29
|
const tree = el<HTMLElement>("#tree");
|
|
@@ -140,6 +141,7 @@ async function load(): Promise<boolean> {
|
|
|
140
141
|
setHtml(review, renderFeedbackForm(run.run, run.feedback.at(-1), readOnly ?? null));
|
|
141
142
|
setHtml(scores, renderScores(run.scores, run.run.verdict, run.judgeDisagrees));
|
|
142
143
|
setHtml(escalations, renderEscalations(run.escalations));
|
|
144
|
+
setHtml(injected, renderInjectedMemories(run.injectedMemories, run.run.id));
|
|
143
145
|
setHtml(children, renderChildren(run.children, now, readOnly ?? null, run.childFeedback));
|
|
144
146
|
roots = spans.roots;
|
|
145
147
|
setHtml(timeline, renderTimeline(timelineRows(roots, now), selectedSpanId));
|
package/web/public/run.html
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
<a class="skip" href="#main">Skip to content</a>
|
|
15
15
|
<nav class="top" aria-label="Primary"><a href="/" class="brand">nexo</a><a href="/">Runs</a><a href="/sessions.html">Sessions</a><a href="/cost.html">Cost</a><a href="/learning.html">Learning</a><a href="/judge.html">Judge</a><a href="/optim.html">Optimization</a><a href="/wikis.html">Wikis</a></nav>
|
|
16
16
|
<main id="main" tabindex="-1">
|
|
17
|
-
<nav class="run-nav" aria-label="Run sections"><a href="#sec-timeline">Timeline</a><a href="#sec-spans">Spans</a><a href="#sec-scores">Scores</a><a href="#sec-subagents">Subagents</a><a href="#sec-files">Files</a><a href="#sec-review">Review</a></nav>
|
|
17
|
+
<nav class="run-nav" aria-label="Run sections"><a href="#sec-timeline">Timeline</a><a href="#sec-spans">Spans</a><a href="#sec-scores">Scores</a><a href="#sec-memories">Memories</a><a href="#sec-subagents">Subagents</a><a href="#sec-files">Files</a><a href="#sec-review">Review</a></nav>
|
|
18
18
|
<div id="error"></div>
|
|
19
19
|
<div id="header"></div>
|
|
20
20
|
<section id="sec-timeline"><h2>Timeline</h2><div id="timeline"></div></section>
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
<div><h2>Scores</h2><div id="scores"></div></div>
|
|
27
27
|
<div><h2>Escalations</h2><div id="escalations"></div></div>
|
|
28
28
|
</section>
|
|
29
|
+
<section id="sec-memories"><h2>Memories injected</h2><div id="injected-memories"></div></section>
|
|
29
30
|
<section id="sec-subagents"><h2>Subagents</h2><div id="children"></div></section>
|
|
30
31
|
<section class="split" id="sec-files">
|
|
31
32
|
<div><h2>Files</h2><div id="files"></div></div>
|
package/web/src/api.js
CHANGED
|
@@ -105,8 +105,40 @@ export function runDetailResponse(ctx, id) {
|
|
|
105
105
|
scores: listScores(ctx.db, id).map(({ details: { rubric: _rubric, ...details }, ...score }) => ({ ...score, details })),
|
|
106
106
|
escalations: listEscalations(ctx.db, id),
|
|
107
107
|
judgeDisagrees: judgeDisagrees(ctx.db, id),
|
|
108
|
+
injectedMemories: injectedMemories(ctx, run),
|
|
108
109
|
};
|
|
109
110
|
}
|
|
111
|
+
/**
|
|
112
|
+
* The memories injected into a run. Retrieval runs once per process, at its first turn, and later turns reuse that set,
|
|
113
|
+
* so for a session turn this looks at the turn itself and then at earlier turns for the latest `memory_read` "retrieve" span.
|
|
114
|
+
*/
|
|
115
|
+
function injectedMemories(ctx, run) {
|
|
116
|
+
const turn = run.turnIndex ?? null;
|
|
117
|
+
const candidates = [run.id];
|
|
118
|
+
if (run.sessionId && turn !== null) {
|
|
119
|
+
const earlier = queryRuns(ctx.db, { sessionId: run.sessionId, limit: MAX_RUN_PAGE })
|
|
120
|
+
.runs.filter((r) => r.id !== run.id && r.turnIndex !== null && r.turnIndex <= turn)
|
|
121
|
+
.sort((a, b) => (b.turnIndex ?? 0) - (a.turnIndex ?? 0));
|
|
122
|
+
candidates.push(...earlier.map((r) => r.id));
|
|
123
|
+
}
|
|
124
|
+
const num = (v) => (typeof v === "number" ? v : null);
|
|
125
|
+
for (const runId of candidates) {
|
|
126
|
+
const span = listSpans(ctx.db, runId).find((s) => s.kind === "memory_read" && s.name === "retrieve");
|
|
127
|
+
if (!span)
|
|
128
|
+
continue;
|
|
129
|
+
const selected = Array.isArray(span.attrs.selected) ? span.attrs.selected : [];
|
|
130
|
+
const items = selected.flatMap((entry) => {
|
|
131
|
+
if (typeof entry !== "object" || entry === null)
|
|
132
|
+
return [];
|
|
133
|
+
const e = entry;
|
|
134
|
+
if (typeof e.id !== "string")
|
|
135
|
+
return [];
|
|
136
|
+
return [{ id: e.id, score: num(e.score), relevance: num(e.relevance), memory: getMemory(ctx.db, e.id) ?? null }];
|
|
137
|
+
});
|
|
138
|
+
return { fromRunId: runId, mode: typeof span.attrs.mode === "string" ? span.attrs.mode : null, candidates: num(span.attrs.candidates), items };
|
|
139
|
+
}
|
|
140
|
+
return null;
|
|
141
|
+
}
|
|
110
142
|
/** The run's own `escalation` spans, oldest first. */
|
|
111
143
|
function listEscalations(db, runId) {
|
|
112
144
|
const text = (v) => (typeof v === "string" ? v : null);
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { reviewQueue } from "../../store/src/index.js";
|
|
2
|
+
import { NexoError } from "../../telemetry/src/index.js";
|
|
3
|
+
function positiveInt(params, name) {
|
|
4
|
+
const raw = params.get(name);
|
|
5
|
+
if (raw === null || raw === "")
|
|
6
|
+
return undefined;
|
|
7
|
+
const n = Number(raw);
|
|
8
|
+
if (!Number.isInteger(n) || n < 1)
|
|
9
|
+
throw new NexoError({ code: "INVALID_REQUEST", message: `${name} must be a positive integer`, context: { [name]: raw } });
|
|
10
|
+
return n;
|
|
11
|
+
}
|
|
12
|
+
/** `GET /api/review-queue?limit=&days=`: recent runs without a human verdict, worst first. */
|
|
13
|
+
export function reviewQueueResponse(ctx, params) {
|
|
14
|
+
const limit = positiveInt(params, "limit");
|
|
15
|
+
const days = positiveInt(params, "days");
|
|
16
|
+
const items = reviewQueue(ctx.db, { now: ctx.now(), ...(limit ? { limit } : {}), ...(days ? { days } : {}) });
|
|
17
|
+
return { items: items.map((i) => ({ ...i, instruction: String(ctx.redact(i.instruction)) })) };
|
|
18
|
+
}
|
package/web/src/server.js
CHANGED
|
@@ -6,6 +6,7 @@ import { costResponse, evalCasesResponse, reviewEvalCaseResponse, feedbackRespon
|
|
|
6
6
|
import { sendError, sendJson } from "./http-errors.js";
|
|
7
7
|
import { companionAnswer } from "./companion.js";
|
|
8
8
|
import { createStaticServer } from "./static.js";
|
|
9
|
+
import { reviewQueueResponse } from "./review-queue-api.js";
|
|
9
10
|
export const DEFAULT_PUBLIC_DIR = fileURLToPath(new URL("../public/", import.meta.url));
|
|
10
11
|
/** Largest accepted request body (feedback JSON). */
|
|
11
12
|
export const MAX_BODY_BYTES = 64 * 1024;
|
|
@@ -32,6 +33,7 @@ const ROUTES = {
|
|
|
32
33
|
optim: new RegExp(`^/api/optim/${RUN_ID}$`),
|
|
33
34
|
optimApprove: new RegExp(`^/api/optim/${RUN_ID}/approve$`),
|
|
34
35
|
evalCases: /^\/api\/eval-cases$/,
|
|
36
|
+
reviewQueue: /^\/api\/review-queue$/,
|
|
35
37
|
evalCaseReview: new RegExp(`^/api/eval-cases/${RUN_ID}/(accept|dismiss)$`),
|
|
36
38
|
meta: /^\/api\/meta$/,
|
|
37
39
|
companion: /^\/api\/companion\/ask$/,
|
|
@@ -242,6 +244,11 @@ export async function startWebServer(options) {
|
|
|
242
244
|
assertWritable();
|
|
243
245
|
return sendJson(res, 200, approveOptimResponse(ctx, m[1]));
|
|
244
246
|
}
|
|
247
|
+
if (ROUTES.reviewQueue.test(path)) {
|
|
248
|
+
if (method !== "GET")
|
|
249
|
+
throw methodNotAllowed(method, path, "GET");
|
|
250
|
+
return sendJson(res, 200, reviewQueueResponse(ctx, url.searchParams));
|
|
251
|
+
}
|
|
245
252
|
if (ROUTES.evalCases.test(path)) {
|
|
246
253
|
if (method !== "GET")
|
|
247
254
|
throw methodNotAllowed(method, path, "GET");
|