@foldrun/core 2.4.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/agent-activity.d.ts +5 -0
- package/dist/src/agent-activity.js +31 -0
- package/dist/src/agent-import.js +2 -2
- package/dist/src/agent-tools.js +2 -2
- package/dist/src/agents-md.d.ts +5 -1
- package/dist/src/agents-md.js +42 -18
- package/dist/src/alerts.d.ts +60 -0
- package/dist/src/alerts.js +161 -0
- package/dist/src/browser-pod.js +4 -0
- package/dist/src/capture.d.ts +23 -0
- package/dist/src/capture.js +80 -0
- package/dist/src/compare.d.ts +57 -0
- package/dist/src/compare.js +116 -0
- package/dist/src/completions.js +26 -3
- package/dist/src/deploy.d.ts +36 -1
- package/dist/src/deploy.js +237 -13
- package/dist/src/egress.d.ts +16 -1
- package/dist/src/evals.d.ts +27 -2
- package/dist/src/evals.js +62 -19
- package/dist/src/flow-lint.d.ts +14 -0
- package/dist/src/flow-lint.js +35 -0
- package/dist/src/flow-patterns.js +29 -14
- package/dist/src/frontmatter-fields.d.ts +27 -0
- package/dist/src/frontmatter-fields.js +60 -3
- package/dist/src/frontmatter-guard.d.ts +19 -0
- package/dist/src/frontmatter-guard.js +67 -0
- package/dist/src/frontmatter-schema.d.ts +2 -2
- package/dist/src/frontmatter-schema.js +33 -2
- package/dist/src/frontmatter-text.d.ts +31 -0
- package/dist/src/frontmatter-text.js +102 -0
- package/dist/src/history.d.ts +4 -0
- package/dist/src/history.js +15 -0
- package/dist/src/hooks.js +1 -1
- package/dist/src/insights.d.ts +30 -0
- package/dist/src/insights.js +152 -0
- package/dist/src/issues.d.ts +140 -0
- package/dist/src/issues.js +302 -0
- package/dist/src/judge-verdict.d.ts +19 -0
- package/dist/src/judge-verdict.js +54 -0
- package/dist/src/library.js +1 -1
- package/dist/src/limits.d.ts +8 -0
- package/dist/src/limits.js +36 -5
- package/dist/src/linkable.js +1 -1
- package/dist/src/matter.d.ts +42 -0
- package/dist/src/matter.js +132 -0
- package/dist/src/model-credential.d.ts +14 -0
- package/dist/src/model-credential.js +40 -0
- package/dist/src/network.d.ts +12 -1
- package/dist/src/network.js +35 -8
- package/dist/src/notify.d.ts +16 -0
- package/dist/src/notify.js +17 -7
- package/dist/src/okf.js +1 -1
- package/dist/src/online-checks.d.ts +76 -0
- package/dist/src/online-checks.js +432 -0
- package/dist/src/operator.d.ts +277 -0
- package/dist/src/operator.js +176 -2
- package/dist/src/otel-config.d.ts +18 -0
- package/dist/src/otel-config.js +69 -0
- package/dist/src/otel.d.ts +38 -0
- package/dist/src/otel.js +85 -0
- package/dist/src/outcome.d.ts +40 -0
- package/dist/src/outcome.js +77 -0
- package/dist/src/package.js +9 -2
- package/dist/src/platform.d.ts +1 -1
- package/dist/src/prompt-layers.d.ts +30 -0
- package/dist/src/prompt-layers.js +77 -0
- package/dist/src/provider-health.js +2 -2
- package/dist/src/providers.d.ts +5 -0
- package/dist/src/providers.js +2 -2
- package/dist/src/run-container.d.ts +25 -2
- package/dist/src/run-container.js +61 -6
- package/dist/src/runner.d.ts +53 -2
- package/dist/src/runner.js +471 -102
- package/dist/src/runs-filter.d.ts +48 -0
- package/dist/src/runs-filter.js +73 -0
- package/dist/src/scan.d.ts +30 -1
- package/dist/src/scan.js +145 -14
- package/dist/src/starter.js +27 -2
- package/dist/src/step-exec.d.ts +39 -3
- package/dist/src/step-exec.js +118 -15
- package/dist/src/store.d.ts +144 -3
- package/dist/src/store.js +150 -7
- package/dist/src/tokens.d.ts +38 -0
- package/dist/src/tokens.js +57 -0
- package/dist/src/tool-names.js +1 -1
- package/dist/src/tool-programs.js +1 -1
- package/dist/src/trace-tree.d.ts +40 -0
- package/dist/src/trace-tree.js +156 -0
- package/dist/src/translator.d.ts +8 -0
- package/dist/src/translator.js +7 -2
- package/dist/src/trigger-log.d.ts +1 -1
- package/dist/src/workspace-check.d.ts +18 -0
- package/dist/src/workspace-check.js +34 -0
- package/package.json +2 -2
- package/src/agent-activity.ts +32 -0
- package/src/agent-import.ts +2 -2
- package/src/agent-tools.ts +2 -2
- package/src/agents-md.ts +39 -14
- package/src/alerts.ts +190 -0
- package/src/browser-pod.ts +3 -0
- package/src/capture.ts +88 -0
- package/src/compare.ts +146 -0
- package/src/completions.ts +27 -3
- package/src/deploy.ts +238 -12
- package/src/egress.ts +12 -2
- package/src/evals.ts +64 -19
- package/src/flow-lint.ts +36 -0
- package/src/flow-patterns.ts +26 -13
- package/src/frontmatter-fields.ts +83 -3
- package/src/frontmatter-guard.ts +70 -0
- package/src/frontmatter-schema.ts +33 -4
- package/src/frontmatter-text.ts +109 -0
- package/src/history.ts +13 -0
- package/src/hooks.ts +1 -1
- package/src/insights.ts +174 -0
- package/src/issues.ts +396 -0
- package/src/judge-verdict.ts +52 -0
- package/src/library.ts +1 -1
- package/src/limits.ts +38 -5
- package/src/linkable.ts +1 -1
- package/src/matter.ts +145 -0
- package/src/model-credential.ts +45 -0
- package/src/network.ts +35 -7
- package/src/notify.ts +22 -7
- package/src/okf.ts +1 -1
- package/src/online-checks.ts +450 -0
- package/src/operator.ts +213 -3
- package/src/otel-config.ts +67 -0
- package/src/otel.ts +100 -0
- package/src/outcome.ts +90 -0
- package/src/package.ts +8 -2
- package/src/platform.ts +1 -1
- package/src/prompt-layers.ts +90 -0
- package/src/provider-health.ts +2 -2
- package/src/providers.ts +7 -2
- package/src/run-container.ts +77 -8
- package/src/runner.ts +477 -79
- package/src/runs-filter.ts +88 -0
- package/src/scan.ts +153 -14
- package/src/starter.ts +27 -2
- package/src/step-exec.ts +168 -17
- package/src/store.ts +268 -9
- package/src/tokens.ts +68 -0
- package/src/tool-names.ts +1 -1
- package/src/tool-programs.ts +1 -1
- package/src/trace-tree.ts +187 -0
- package/src/translator.ts +13 -2
- package/src/trigger-log.ts +1 -0
- package/src/workspace-check.ts +41 -0
package/dist/src/step-exec.js
CHANGED
|
@@ -7,18 +7,20 @@
|
|
|
7
7
|
// assembles the system prompt and builds MCP servers, then hands this the
|
|
8
8
|
// results. That is what lets the same function run somewhere the vault, the
|
|
9
9
|
// library and the account do not exist.
|
|
10
|
+
import { judgeVerdict, JUDGE_VERDICT_INSTRUCTION } from "./judge-verdict.js";
|
|
10
11
|
import { runHook, runToolCallHooks, matchesTool, hookLine, resolveHooks } from "./hooks.js";
|
|
11
12
|
import fs from "node:fs";
|
|
12
13
|
import os from "node:os";
|
|
13
14
|
import path from "node:path";
|
|
14
15
|
import { query } from "@anthropic-ai/claude-agent-sdk";
|
|
16
|
+
import { CAPTURE_CAP } from "./store.js";
|
|
15
17
|
import { costOf } from "./catalog.js";
|
|
16
18
|
import { spawn } from "node:child_process";
|
|
17
19
|
import { checkPaths, checkBash, isFilesystemTool, linkWorkspace, workspaceLinkEnv, resolveAgentPath, isWithin } from "./confine.js";
|
|
18
20
|
import { DELEGATE_TOOLS, subagentGuard, toAgentDefinitions } from "./subagents.js";
|
|
19
21
|
import { hostSafeEnv } from "./host-env.js";
|
|
20
22
|
import { validateSchema, describeSchemaErrors, looksLikeSchema } from "./json-schema.js";
|
|
21
|
-
import { CallCounter,
|
|
23
|
+
import { CallCounter, callKeysFor } from "./limits.js";
|
|
22
24
|
import { classifyCall, podAt, readPodEvents } from "./browser-pod.js";
|
|
23
25
|
/** The four counts from one SDK usage object (snake_case, as the API
|
|
24
26
|
* reports it). The API counts fresh input apart from the cache's; here
|
|
@@ -50,6 +52,14 @@ export function verifyBudgetMs(timeoutSec, elapsedMs) {
|
|
|
50
52
|
/** After an interrupt is asked for, how long the model loop gets to wind
|
|
51
53
|
* down on its own before the query is aborted outright. */
|
|
52
54
|
export const INTERRUPT_GRACE_MS = 10_000;
|
|
55
|
+
// The cap plus slack: the runner masks, then cuts to the cap (capture.ts), so
|
|
56
|
+
// a secret on the edge is masked whole.
|
|
57
|
+
const SEND_CAP = CAPTURE_CAP + 4096;
|
|
58
|
+
function capped(text) {
|
|
59
|
+
if (text.length <= SEND_CAP)
|
|
60
|
+
return { text, chars: text.length, truncated: false };
|
|
61
|
+
return { text: text.slice(0, SEND_CAP), chars: text.length, truncated: true };
|
|
62
|
+
}
|
|
53
63
|
/** The tag on every event a failed check writes — `verify:`, `output: json`,
|
|
54
64
|
* a schema — headline and detail alike. The runner's no-retry rule for
|
|
55
65
|
* outward steps reads it (actedThenFailedCheck); reading the wording of the
|
|
@@ -228,6 +238,10 @@ async function executeStepInner(opts, runQuery) {
|
|
|
228
238
|
...(opts.effort ? { effort: opts.effort } : {}),
|
|
229
239
|
systemPrompt: opts.systemPrompt,
|
|
230
240
|
...(opts.maxTurns ? { maxTurns: opts.maxTurns } : {}),
|
|
241
|
+
// The raw stream, only when model calls are recorded: an assistant
|
|
242
|
+
// message carries usage from the call's start (output a few tokens) and
|
|
243
|
+
// no stop reason; the call's message_delta has both (FOL-72).
|
|
244
|
+
...(opts.capture ? { includePartialMessages: true } : {}),
|
|
231
245
|
// Sub-agents: the SDK's own delegation. Every definition carries an
|
|
232
246
|
// explicit tool list (an absent one inherits all of ours).
|
|
233
247
|
...(subagents ? { agents: toAgentDefinitions(subagents) } : {}),
|
|
@@ -313,7 +327,7 @@ async function executeStepInner(opts, runQuery) {
|
|
|
313
327
|
// refused before it runs: the tool, and any paid API behind it,
|
|
314
328
|
// never sees it.
|
|
315
329
|
if (counter.active) {
|
|
316
|
-
const refusal = counter.take(
|
|
330
|
+
const refusal = counter.take(callKeysFor(hookInput.tool_name, hookInput.tool_input, opts.toolOwners));
|
|
317
331
|
if (refusal) {
|
|
318
332
|
emit("info", `${refusal.message} (${hookInput.tool_name} refused)`, via ? { subagent: via } : undefined);
|
|
319
333
|
return { hookSpecificOutput: { hookEventName: "PreToolUse", permissionDecision: "deny", permissionDecisionReason: refusal.message } };
|
|
@@ -330,7 +344,7 @@ async function executeStepInner(opts, runQuery) {
|
|
|
330
344
|
for (const r of v.records)
|
|
331
345
|
record(r, via);
|
|
332
346
|
const refundCall = () => { if (counter.active)
|
|
333
|
-
counter.refund(
|
|
347
|
+
counter.refund(callKeysFor(hookInput.tool_name, hookInput.tool_input, opts.toolOwners)); };
|
|
334
348
|
if (v.decision === "deny") {
|
|
335
349
|
refundCall();
|
|
336
350
|
return { hookSpecificOutput: { hookEventName: "PreToolUse", permissionDecision: "deny", permissionDecisionReason: v.reason } };
|
|
@@ -407,7 +421,7 @@ async function executeStepInner(opts, runQuery) {
|
|
|
407
421
|
const fromGrantedServer = opts.mcpNames.some((n) => toolName.startsWith(`mcp__${n}__`));
|
|
408
422
|
// A call the hook counted and this refuses never ran: give it back.
|
|
409
423
|
const refund = () => { if (counter.active)
|
|
410
|
-
counter.refund(
|
|
424
|
+
counter.refund(callKeysFor(toolName, input, opts.toolOwners)); };
|
|
411
425
|
if (!opts.allowed.includes(toolName) && !fromGrantedServer) {
|
|
412
426
|
refund();
|
|
413
427
|
return {
|
|
@@ -480,6 +494,29 @@ async function executeStepInner(opts, runQuery) {
|
|
|
480
494
|
// ceiling and on the bill.
|
|
481
495
|
const turns = new Map();
|
|
482
496
|
let anonymousTurns = 0;
|
|
497
|
+
// Each model call as a span (FOL-54 step 4), keyed like `turns`: a turn's
|
|
498
|
+
// blocks arrive as several messages under one id, so it is updated in place
|
|
499
|
+
// and sent again; the runner keeps the latest by id.
|
|
500
|
+
const spans = new Map();
|
|
501
|
+
// From the raw stream: which call each agent is in (message_start), and each
|
|
502
|
+
// call's final output tokens and stop reason (message_delta).
|
|
503
|
+
const streaming = new Map();
|
|
504
|
+
let saidResponding = false;
|
|
505
|
+
const finals = new Map();
|
|
506
|
+
const withFinal = (span) => {
|
|
507
|
+
const f = finals.get(span.id);
|
|
508
|
+
return {
|
|
509
|
+
...span,
|
|
510
|
+
...(f?.output !== undefined ? { tokens: { ...span.tokens, output: f.output } } : {}),
|
|
511
|
+
// Without the stream (an older CLI), a call that used tools stopped to use them.
|
|
512
|
+
...(f?.stop ? { stop: f.stop } : !span.stop && span.tools > 0 ? { stop: "tool_use" } : {}),
|
|
513
|
+
};
|
|
514
|
+
};
|
|
515
|
+
// When each agent's next request went out — the step's own, and each
|
|
516
|
+
// sub-agent's apart, so one's tool results do not restart another's clock.
|
|
517
|
+
const lastInputAt = new Map();
|
|
518
|
+
const sentAt = (via) => lastInputAt.get(via ?? "") ?? loopStartedAt;
|
|
519
|
+
let loopStartedAt = Date.now();
|
|
483
520
|
const specByName = new Map((subagents ?? []).map((s) => [s.name, s]));
|
|
484
521
|
const priceOf = (message) => {
|
|
485
522
|
const parent = message.parent_tool_use_id;
|
|
@@ -560,13 +597,65 @@ async function executeStepInner(opts, runQuery) {
|
|
|
560
597
|
};
|
|
561
598
|
try {
|
|
562
599
|
for await (const message of q) {
|
|
600
|
+
// The session is up: the first model call is timed from here, not from
|
|
601
|
+
// before the SDK and its MCP servers started.
|
|
602
|
+
if (message.type === "system" && message.subtype === "init")
|
|
603
|
+
loopStartedAt = Date.now();
|
|
604
|
+
if (message.type === "stream_event") {
|
|
605
|
+
const ev = message.event;
|
|
606
|
+
const agentKey = message.parent_tool_use_id ?? "";
|
|
607
|
+
if (ev?.type === "message_start" && ev.message?.id) {
|
|
608
|
+
streaming.set(agentKey, ev.message.id);
|
|
609
|
+
// The step's own first response has begun: its prompt is in the
|
|
610
|
+
// cache now, so siblings held back to read it may start.
|
|
611
|
+
if (!agentKey && !saidResponding) {
|
|
612
|
+
saidResponding = true;
|
|
613
|
+
emit("info", "", { responding: true });
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
if (ev?.type === "message_delta") {
|
|
617
|
+
const id = streaming.get(agentKey);
|
|
618
|
+
if (id) {
|
|
619
|
+
finals.set(id, {
|
|
620
|
+
...(typeof ev.usage?.output_tokens === "number" ? { output: ev.usage.output_tokens } : {}),
|
|
621
|
+
...(ev.delta?.stop_reason ? { stop: ev.delta.stop_reason } : {}),
|
|
622
|
+
});
|
|
623
|
+
const span = spans.get(id);
|
|
624
|
+
if (span) {
|
|
625
|
+
const final = withFinal(span);
|
|
626
|
+
spans.set(id, final);
|
|
627
|
+
emit("info", "", { turn: final });
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
continue;
|
|
632
|
+
}
|
|
563
633
|
if (ended)
|
|
564
634
|
break;
|
|
565
635
|
if (message.type === "assistant") {
|
|
566
636
|
const m = message.message;
|
|
567
637
|
const u = m.usage;
|
|
638
|
+
const turnKey = m.id ?? `turn-${anonymousTurns++}`;
|
|
568
639
|
if (u)
|
|
569
|
-
turns.set(
|
|
640
|
+
turns.set(turnKey, { u, price: priceOf(message) });
|
|
641
|
+
if (opts.capture && u) {
|
|
642
|
+
const t = usageFromSdk(u);
|
|
643
|
+
const prior = spans.get(turnKey);
|
|
644
|
+
const span = {
|
|
645
|
+
id: turnKey,
|
|
646
|
+
n: prior?.n ?? spans.size + 1,
|
|
647
|
+
at: prior?.at ?? new Date().toISOString(),
|
|
648
|
+
ms: prior?.ms ?? Date.now() - sentAt(subagentOf(message)),
|
|
649
|
+
...(m.model ? { model: m.model } : prior?.model ? { model: prior.model } : {}),
|
|
650
|
+
...(m.stop_reason ? { stop: m.stop_reason } : prior?.stop ? { stop: prior.stop } : {}),
|
|
651
|
+
tokens: { input: t.inputTokens, output: t.outputTokens, cacheRead: t.cacheReadTokens, cacheWrite: t.cacheWriteTokens },
|
|
652
|
+
tools: (prior?.tools ?? 0) + (m.content ?? []).filter((b) => b?.type === "tool_use").length,
|
|
653
|
+
...(subagentOf(message) ? { subagent: subagentOf(message) } : {}),
|
|
654
|
+
};
|
|
655
|
+
const final = withFinal(span);
|
|
656
|
+
spans.set(turnKey, final);
|
|
657
|
+
emit("info", "", { turn: final });
|
|
658
|
+
}
|
|
570
659
|
if (ceiling && u) {
|
|
571
660
|
spentUsd = turnsTotal();
|
|
572
661
|
if (spentUsd >= ceiling) {
|
|
@@ -590,6 +679,8 @@ async function executeStepInner(opts, runQuery) {
|
|
|
590
679
|
const input = block.input;
|
|
591
680
|
const to = String(input?.subagent_type ?? "subagent");
|
|
592
681
|
delegations.set(block.id, to);
|
|
682
|
+
// That sub-agent's first model call is timed from its job going out.
|
|
683
|
+
lastInputAt.set(to, Date.now());
|
|
593
684
|
// One line a person can read: who got the job, and what it was.
|
|
594
685
|
const job = String(input?.description ?? input?.prompt ?? "").replace(/\s+/g, " ").trim().slice(0, 160);
|
|
595
686
|
emit("info", `delegating to ${to}${job ? `: ${job}` : ""}`, { subagent: to });
|
|
@@ -597,7 +688,13 @@ async function executeStepInner(opts, runQuery) {
|
|
|
597
688
|
openCalls.set(block.id, { name: block.name, at: Date.now(), ...(via ? { subagent: via } : {}) });
|
|
598
689
|
if (pod)
|
|
599
690
|
ledger.set(block.id, classifyCall(block.name, block.input, pod));
|
|
600
|
-
|
|
691
|
+
const input = opts.capture === "full" ? capped(JSON.stringify(block.input ?? {})) : null;
|
|
692
|
+
emit("tool", block.name, {
|
|
693
|
+
call: block.id,
|
|
694
|
+
...(opts.capture ? { inCall: turnKey } : {}),
|
|
695
|
+
...(via ? { subagent: via } : {}),
|
|
696
|
+
...(input ? { io: { input: input.text, inputChars: input.chars, ...(input.truncated ? { truncated: true } : {}) } } : {}),
|
|
697
|
+
});
|
|
601
698
|
}
|
|
602
699
|
}
|
|
603
700
|
}
|
|
@@ -613,6 +710,7 @@ async function executeStepInner(opts, runQuery) {
|
|
|
613
710
|
if (!open)
|
|
614
711
|
continue;
|
|
615
712
|
openCalls.delete(block.tool_use_id);
|
|
713
|
+
const output = opts.capture === "full" ? capped(toolResultText(block.content)) : null;
|
|
616
714
|
emit("tool", open.name, {
|
|
617
715
|
call: block.tool_use_id,
|
|
618
716
|
ms: Date.now() - open.at,
|
|
@@ -621,6 +719,7 @@ async function executeStepInner(opts, runQuery) {
|
|
|
621
719
|
bytes: toolResultText(block.content).length,
|
|
622
720
|
...(block.is_error ? { err: true } : {}),
|
|
623
721
|
...(open.subagent ? { subagent: open.subagent } : {}),
|
|
722
|
+
...(output ? { io: { output: output.text, outputChars: output.chars, ...(output.truncated ? { truncated: true } : {}) } } : {}),
|
|
624
723
|
});
|
|
625
724
|
// Why it failed, on the trail. The span above carried only `err`,
|
|
626
725
|
// so a refused call left the agent's own guess as the only account
|
|
@@ -633,6 +732,8 @@ async function executeStepInner(opts, runQuery) {
|
|
|
633
732
|
}
|
|
634
733
|
readPod(block.tool_use_id);
|
|
635
734
|
}
|
|
735
|
+
// That agent's next model call is timed from here: the results went back.
|
|
736
|
+
lastInputAt.set(subagentOf(message) ?? "", Date.now());
|
|
636
737
|
}
|
|
637
738
|
}
|
|
638
739
|
else if (message.type === "result") {
|
|
@@ -801,6 +902,8 @@ async function executeStepInner(opts, runQuery) {
|
|
|
801
902
|
data,
|
|
802
903
|
modelEnv: opts.env,
|
|
803
904
|
signal: halt.signal,
|
|
905
|
+
...(opts.judgeModel ? { judgeModel: opts.judgeModel } : {}),
|
|
906
|
+
...(opts.judgeEffort ? { judgeEffort: opts.judgeEffort } : {}),
|
|
804
907
|
});
|
|
805
908
|
}
|
|
806
909
|
finally {
|
|
@@ -945,17 +1048,17 @@ export async function checkVerify(agentDir, verify, ctx) {
|
|
|
945
1048
|
// GOOD` on a reply that did (fix-desk, run-muwevg72-7mza): the joined
|
|
946
1049
|
// text opened with the narration before it.
|
|
947
1050
|
const reply = ctx.conclusion?.trim() ? ctx.conclusion : output;
|
|
948
|
-
const
|
|
949
|
-
const ok =
|
|
950
|
-
return { ok, headline: ok ? "PASS" : "FAIL", detail
|
|
1051
|
+
const said = await judgeReply(value, reply, output, ctx.modelEnv ?? {}, ctx.signal, ctx.judgeModel, ctx.judgeEffort);
|
|
1052
|
+
const { passed: ok, detail } = judgeVerdict(said);
|
|
1053
|
+
return { ok, headline: ok ? "PASS" : "FAIL", detail };
|
|
951
1054
|
}
|
|
952
1055
|
}
|
|
953
1056
|
return { ok: false, headline: `unknown check "${kind}"`, detail: "" };
|
|
954
1057
|
}
|
|
955
1058
|
/**
|
|
956
1059
|
* A toolless, fast-tier grading call: does the reply satisfy the sentence?
|
|
957
|
-
* Same shape as the eval judge
|
|
958
|
-
*
|
|
1060
|
+
* Same shape as the eval judge: reasoning first, then the verdict alone on
|
|
1061
|
+
* the last line (judge-verdict.ts says why).
|
|
959
1062
|
*/
|
|
960
1063
|
export function judgePrompt(rubric, reply, work) {
|
|
961
1064
|
const shown = reply.slice(0, 20_000);
|
|
@@ -963,12 +1066,11 @@ export function judgePrompt(rubric, reply, work) {
|
|
|
963
1066
|
? `Everything the step wrote while working, for context. "The reply" means only the reply block below, ` +
|
|
964
1067
|
`never this:\n<work>\n${work.slice(-20_000)}\n</work>\n\n`
|
|
965
1068
|
: "";
|
|
966
|
-
return (`You are grading a reply against one requirement.
|
|
967
|
-
`first word, then one sentence of reason.\n\nRequirement: ${rubric}\n\n` +
|
|
1069
|
+
return (`You are grading a reply against one requirement. ${JUDGE_VERDICT_INSTRUCTION}\n\nRequirement: ${rubric}\n\n` +
|
|
968
1070
|
context +
|
|
969
1071
|
`<reply>\n${shown}\n</reply>`);
|
|
970
1072
|
}
|
|
971
|
-
async function judgeReply(rubric, reply, work, env, signal) {
|
|
1073
|
+
async function judgeReply(rubric, reply, work, env, signal, model = "haiku", effort) {
|
|
972
1074
|
const texts = [];
|
|
973
1075
|
const abortController = new AbortController();
|
|
974
1076
|
if (signal?.aborted)
|
|
@@ -978,7 +1080,8 @@ async function judgeReply(rubric, reply, work, env, signal) {
|
|
|
978
1080
|
const q = query({
|
|
979
1081
|
prompt: judgePrompt(rubric, reply, work),
|
|
980
1082
|
options: {
|
|
981
|
-
model
|
|
1083
|
+
model,
|
|
1084
|
+
...(effort ? { effort } : {}),
|
|
982
1085
|
systemPrompt: "You grade text against a stated requirement. Be strict and literal.",
|
|
983
1086
|
tools: [],
|
|
984
1087
|
settingSources: [],
|
package/dist/src/store.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type { BrowserPodOutcome } from "./browser-pod.ts";
|
|
|
3
3
|
import { singleWorkspace } from "./paths.ts";
|
|
4
4
|
import type { TestEffect } from "./test-mode.ts";
|
|
5
5
|
import type { OperatorEvent } from "./operator.ts";
|
|
6
|
+
import { frontmatterErrorText } from "./matter.ts";
|
|
6
7
|
/**
|
|
7
8
|
* The frontmatter of one markdown file, as the runtime reads it. Exported
|
|
8
9
|
* for the CLI: `foldrun probe` used to import gray-matter itself, which is
|
|
@@ -282,6 +283,12 @@ export interface ApiSpec {
|
|
|
282
283
|
export interface AgentInfo {
|
|
283
284
|
name: string;
|
|
284
285
|
description: string;
|
|
286
|
+
/** `owner:` — the person accountable for this agent: its own, else the
|
|
287
|
+
* workspace AGENTS.md's, else the account's. Null when none of the three
|
|
288
|
+
* names one. */
|
|
289
|
+
owner: string | null;
|
|
290
|
+
/** Which of the three `owner` came from. */
|
|
291
|
+
ownerFrom: OwnerSource | null;
|
|
285
292
|
model: string;
|
|
286
293
|
/** The other half of model selection. listAgents has always returned it;
|
|
287
294
|
* the interface did not say so, so every consumer had to re-read the file
|
|
@@ -352,6 +359,10 @@ export interface AgentInfo {
|
|
|
352
359
|
/** Tools this agent declares inline rather than in `tools:` — `apis:`,
|
|
353
360
|
* `scripts:` and `mcpServers:` names — which a limit may also name. */
|
|
354
361
|
inlineTools: string[];
|
|
362
|
+
/** agent.md's frontmatter does not parse (bad YAML): the agent is listed
|
|
363
|
+
* with defaults and this says why, rather than the listing throwing —
|
|
364
|
+
* one broken file once took down every account's worker (20 Oct 2026). */
|
|
365
|
+
frontmatterError?: string;
|
|
355
366
|
}
|
|
356
367
|
export declare const MODEL_TIERS: Record<string, string>;
|
|
357
368
|
export type Tier = keyof typeof MODEL_TIERS;
|
|
@@ -512,6 +523,9 @@ export interface FlowInfo {
|
|
|
512
523
|
* overlap) — a default that changed underfoot would surprise every flow
|
|
513
524
|
* that legitimately runs in parallel. */
|
|
514
525
|
overlap: "skip" | "queue" | null;
|
|
526
|
+
/** `issues: auto-close`: a run that goes fine closes this flow's open
|
|
527
|
+
* issues (FOL-72 step 7). Unset is `stay-open`: a person resolves them. */
|
|
528
|
+
issues?: "auto-close" | "stay-open";
|
|
515
529
|
/** Where this flow's runs stand in the queue against everyone else's:
|
|
516
530
|
* "high" is claimed before the round-robin, "low" after it. Adds to the
|
|
517
531
|
* account's own priority; the account's cap still bounds how many run
|
|
@@ -606,6 +620,12 @@ export interface FlowInfo {
|
|
|
606
620
|
* and the trace. Read here so a documented, scaffolded field means
|
|
607
621
|
* something rather than sitting in every file unread. */
|
|
608
622
|
description: string | null;
|
|
623
|
+
/** `owner:` — the person accountable for this flow: its own, else the
|
|
624
|
+
* workspace AGENTS.md's, else the account's. parseFlow sees only the
|
|
625
|
+
* file, so it fills the flow's own; listFlows and readFlow add the rest. */
|
|
626
|
+
owner: string | null;
|
|
627
|
+
/** Which of the three `owner` came from. */
|
|
628
|
+
ownerFrom: OwnerSource | null;
|
|
609
629
|
/** Flow-level frontmatter values the parser could not read — a bad
|
|
610
630
|
* `overlap:`/`priority:`/`catchup:`/`on:`/`signature:`, or a step line
|
|
611
631
|
* whose `[[target]]` is malformed. Errors, the way a bad step option is,
|
|
@@ -621,6 +641,7 @@ export interface AgentAssets {
|
|
|
621
641
|
}
|
|
622
642
|
export declare function agentAssets(tenant: string, workspace: string, agent: string): AgentAssets;
|
|
623
643
|
export declare function listAgents(tenant: string, workspace: string): AgentInfo[];
|
|
644
|
+
export { frontmatterErrorText };
|
|
624
645
|
/** The option keys a step actually has. An indented `word: rest` that is
|
|
625
646
|
* NOT one of these is a line of the instruction — "Warning: do not
|
|
626
647
|
* publish" is prose that happens to contain a colon, and it used to be
|
|
@@ -748,6 +769,14 @@ export declare const reorderFlowSteps: (raw: string, groups: number[][]) => stri
|
|
|
748
769
|
export declare const addFlowStep: (raw: string, step: Parameters<typeof addFlowStepLF>[1]) => string;
|
|
749
770
|
export declare const removeFlowStep: (raw: string, index: number) => string;
|
|
750
771
|
export declare function listFlows(tenant: string, workspace: string): FlowInfo[];
|
|
772
|
+
export type OwnerSource = "agent" | "flow" | "workspace" | "account";
|
|
773
|
+
/** `owner:` down the cascade: the file's own, the workspace AGENTS.md's, the
|
|
774
|
+
* account's. A blank or non-string value is no value. `own` is labelled
|
|
775
|
+
* "agent" or "flow" by the caller's kind; here it is whichever is given. */
|
|
776
|
+
export declare function resolveOwner(own: unknown, workspace?: unknown, account?: unknown, ownKind?: "agent" | "flow"): {
|
|
777
|
+
owner: string | null;
|
|
778
|
+
ownerFrom: OwnerSource | null;
|
|
779
|
+
};
|
|
751
780
|
/** One flow of a workspace by name, or null. The same read listFlows makes,
|
|
752
781
|
* named so callers that want a single flow — the approval rules, the
|
|
753
782
|
* runner's budget stamp — do not each re-implement the directory walk. */
|
|
@@ -922,6 +951,62 @@ export interface RunEvent {
|
|
|
922
951
|
reason?: string;
|
|
923
952
|
ms: number;
|
|
924
953
|
};
|
|
954
|
+
/** Set on a tool event whose content was recorded (FOL-54 step 4): the
|
|
955
|
+
* call's input, on the call; its output, on the completion. The content is
|
|
956
|
+
* under runs/<run>/calls/<call>.json, secrets masked — readCapturedCall.
|
|
957
|
+
* Absent when the workspace says `capture: metadata`, and before 19 Oct
|
|
958
|
+
* 2026. */
|
|
959
|
+
captured?: boolean;
|
|
960
|
+
/** On a tool call: the id of the model call that asked for it (a
|
|
961
|
+
* ModelCall's `id`), so the trace can nest it. Since 19 Oct 2026. */
|
|
962
|
+
inCall?: string;
|
|
963
|
+
}
|
|
964
|
+
/** One model call in a step (FOL-54 step 4): what the model was sent back
|
|
965
|
+
* for and what came of it. A step's turns, one by one, where the step
|
|
966
|
+
* record had only their sum. */
|
|
967
|
+
export interface ModelCall {
|
|
968
|
+
/** The provider's message id. */
|
|
969
|
+
id: string;
|
|
970
|
+
/** 1, 2, 3 … in the order they came back. */
|
|
971
|
+
n: number;
|
|
972
|
+
/** When the response arrived. */
|
|
973
|
+
at: string;
|
|
974
|
+
/** From the request going out (the step's start, or the last tool result
|
|
975
|
+
* coming back) to the response arriving. */
|
|
976
|
+
ms: number;
|
|
977
|
+
model?: string;
|
|
978
|
+
/** Why the model stopped: end_turn, tool_use, max_tokens, … */
|
|
979
|
+
stop?: string;
|
|
980
|
+
/** All input (fresh + cache), as the step's own `tokens` counts it. */
|
|
981
|
+
tokens: {
|
|
982
|
+
input: number;
|
|
983
|
+
output: number;
|
|
984
|
+
cacheRead: number;
|
|
985
|
+
cacheWrite: number;
|
|
986
|
+
};
|
|
987
|
+
/** How many tools it called in this turn. */
|
|
988
|
+
tools: number;
|
|
989
|
+
/** Set when a sub-agent (`subagents:`) made the call. */
|
|
990
|
+
subagent?: string;
|
|
991
|
+
/** Which attempt of the step made it, from the second on. */
|
|
992
|
+
attempt?: number;
|
|
993
|
+
}
|
|
994
|
+
/** The most of one tool input or output a run records (FOL-54 step 4). */
|
|
995
|
+
export declare const CAPTURE_CAP: number;
|
|
996
|
+
/** What a recorded tool call took and gave, secrets masked, each part capped. */
|
|
997
|
+
export interface CapturedCall {
|
|
998
|
+
id: string;
|
|
999
|
+
tool: string;
|
|
1000
|
+
input?: string;
|
|
1001
|
+
output?: string;
|
|
1002
|
+
/** The whole input's / output's length in characters, before masking and
|
|
1003
|
+
* the cap; `truncated` says whether the text was cut. */
|
|
1004
|
+
inputChars?: number;
|
|
1005
|
+
outputChars?: number;
|
|
1006
|
+
/** Either part cut; and which. */
|
|
1007
|
+
truncated?: boolean;
|
|
1008
|
+
inputCut?: boolean;
|
|
1009
|
+
outputCut?: boolean;
|
|
925
1010
|
}
|
|
926
1011
|
/** A question an agent asked a person while its step ran (tools: [ask]). */
|
|
927
1012
|
export interface StepQuestion {
|
|
@@ -935,12 +1020,18 @@ export interface StepQuestion {
|
|
|
935
1020
|
/** true when the wait ran out with no answer. */
|
|
936
1021
|
unanswered?: boolean;
|
|
937
1022
|
}
|
|
938
|
-
/** A message a person sent into a running step, and when the agent got it
|
|
1023
|
+
/** A message a person sent into a running step, and when the agent got it —
|
|
1024
|
+
* or, with `from`, one a colleague in the same parallel group sent it
|
|
1025
|
+
* (tools: [peers]); with `to`, one this step sent its colleagues. */
|
|
939
1026
|
export interface StepMessage {
|
|
940
1027
|
text: string;
|
|
941
1028
|
by?: string | null;
|
|
942
1029
|
at: string;
|
|
943
1030
|
deliveredAt: string;
|
|
1031
|
+
/** The sibling agent that sent it (`by` is `peer:<agent>` then). */
|
|
1032
|
+
from?: string;
|
|
1033
|
+
/** The sibling agents this step sent it to. */
|
|
1034
|
+
to?: string[];
|
|
944
1035
|
}
|
|
945
1036
|
/** One attempt of a step — see StepRecord.tries. */
|
|
946
1037
|
export interface StepAttempt {
|
|
@@ -975,8 +1066,16 @@ export interface StepAttempt {
|
|
|
975
1066
|
* attempt reported the figure — an in-process step rents no sandbox, and
|
|
976
1067
|
* billing reads null as zero, not as "unknown, charge something". */
|
|
977
1068
|
export declare function sumAttempts(tries: StepAttempt[]): Pick<StepAttempt, "costUsd" | "tokens" | "computeSecs" | "turns">;
|
|
978
|
-
export { type TokenUsage, tokensOf, sumTokens, cacheHitRate, measuredHitRate, runTokens } from "./tokens.ts";
|
|
979
|
-
import { type TokenUsage } from "./tokens.ts";
|
|
1069
|
+
export { type TokenUsage, type PromptLayerRecord, tokensOf, sumTokens, cacheHitRate, measuredHitRate, runTokens, promptLayersChanged, promptCacheNote } from "./tokens.ts";
|
|
1070
|
+
import { type TokenUsage, type PromptLayerRecord } from "./tokens.ts";
|
|
1071
|
+
/** The marker rescueStep puts in a rescue's instruction; how a rescue record
|
|
1072
|
+
* written before `rescueOf` existed is still recognised. */
|
|
1073
|
+
export declare const RESCUE_MARKER = "<rescue>\nYou are taking this step over:";
|
|
1074
|
+
/** Is this record an on-fail rescue rather than a step of the flow? */
|
|
1075
|
+
export declare function isRescueRecord(step: {
|
|
1076
|
+
rescueOf?: string;
|
|
1077
|
+
instruction?: string | null;
|
|
1078
|
+
}): boolean;
|
|
980
1079
|
export interface StepRecord {
|
|
981
1080
|
agent: string;
|
|
982
1081
|
instruction: string;
|
|
@@ -1060,6 +1159,9 @@ export interface StepRecord {
|
|
|
1060
1159
|
* records from before this existed; readers fall back to the events. */
|
|
1061
1160
|
startedAt?: string;
|
|
1062
1161
|
finishedAt?: string;
|
|
1162
|
+
/** Every model call the step made, in order (FOL-54 step 4). Absent before
|
|
1163
|
+
* 19 Oct 2026. */
|
|
1164
|
+
calls?: ModelCall[];
|
|
1063
1165
|
/** Why a step was skipped, shown in the trace. */
|
|
1064
1166
|
skipReason?: string;
|
|
1065
1167
|
/** Evaluator loop, carried from the flow — see FlowStep. `loopRemaining`
|
|
@@ -1133,6 +1235,13 @@ export interface StepRecord {
|
|
|
1133
1235
|
* actually executed in. Carried steps provide context and are never
|
|
1134
1236
|
* billed again — the original run's ledger line already paid for them. */
|
|
1135
1237
|
carriedFrom?: string;
|
|
1238
|
+
/** Set on an on-fail rescue record (runner.ts rescueStep): the agent of the
|
|
1239
|
+
* failed step it took over. A rescue is not a step of the flow: a resumed
|
|
1240
|
+
* drive must not launch it as one of its group, and a re-run must not
|
|
1241
|
+
* copy it as work (21 Oct 2026: one rerun carried 77 fixers as pending
|
|
1242
|
+
* steps). Records written before this field are recognised by their
|
|
1243
|
+
* instruction's `<rescue>` block instead — see isRescueRecord. */
|
|
1244
|
+
rescueOf?: string;
|
|
1136
1245
|
/**
|
|
1137
1246
|
* Sandbox seconds this step rented, and how many of them were cold start.
|
|
1138
1247
|
* Set only on the isolated path — an in-process step rents nothing, so
|
|
@@ -1170,6 +1279,11 @@ export interface StepRecord {
|
|
|
1170
1279
|
* what says where a step's input tokens went. Absent before 18 Oct 2026. */
|
|
1171
1280
|
turns?: number | null;
|
|
1172
1281
|
systemPromptChars?: number;
|
|
1282
|
+
/** The system prompt's layers — name, length and hash of each, stable
|
|
1283
|
+
* ones first, the ones that change between runs marked. Two steps of
|
|
1284
|
+
* one agent compared layer by layer say why the model's cache missed
|
|
1285
|
+
* (prompt-layers.ts). Absent before 20 Oct 2026. */
|
|
1286
|
+
promptLayers?: PromptLayerRecord[];
|
|
1173
1287
|
/** Why the model's last turn ended (end_turn, max_tokens, tool_use…), as
|
|
1174
1288
|
* the SDK's closing result says. */
|
|
1175
1289
|
stopReason?: string | null;
|
|
@@ -1237,6 +1351,21 @@ export interface RunRecord {
|
|
|
1237
1351
|
* the agents decided was worth remembering, listed so a person can review
|
|
1238
1352
|
* the lesson without diffing the folder. */
|
|
1239
1353
|
memoryWrites?: string[];
|
|
1354
|
+
/** The workspace's deployed commit when the run was created — which
|
|
1355
|
+
* version of the files it ran (FOL-54 step 4). Absent for a workspace
|
|
1356
|
+
* never deployed, and before 19 Oct 2026. */
|
|
1357
|
+
revision?: string | null;
|
|
1358
|
+
/** When a worker first picked the run up: startedAt is when it was
|
|
1359
|
+
* created, so the gap is its time in the queue. */
|
|
1360
|
+
drivenAt?: string | null;
|
|
1361
|
+
/** A re-run's source (FOL-72 step 6): which run, the 1-based step it
|
|
1362
|
+
* started from, and the instruction it was given there when a person
|
|
1363
|
+
* edited it for this re-run alone. */
|
|
1364
|
+
rerunOf?: {
|
|
1365
|
+
run: string;
|
|
1366
|
+
step: number;
|
|
1367
|
+
instruction?: string;
|
|
1368
|
+
};
|
|
1240
1369
|
/** Who asked for this run, when a person did: their email address. A
|
|
1241
1370
|
* schedule, a webhook and a chained flow have no one, and leave it unset.
|
|
1242
1371
|
* Kept so a gate can refuse the one approval nobody should be able to
|
|
@@ -1259,6 +1388,11 @@ export interface RunRecord {
|
|
|
1259
1388
|
approvalLinksUsed?: string[];
|
|
1260
1389
|
steps: StepRecord[];
|
|
1261
1390
|
}
|
|
1391
|
+
/** Merge one part (the input, on the call; the output, on its completion)
|
|
1392
|
+
* into the call's file. */
|
|
1393
|
+
export declare function writeCapturedCall(workspaceRoot: string, runId: string, part: CapturedCall): void;
|
|
1394
|
+
/** One recorded call of a run, or null (not recorded, or not a call). */
|
|
1395
|
+
export declare function readCapturedCall(tenant: string, workspace: string, runId: string, callId: string): CapturedCall | null;
|
|
1262
1396
|
export declare function runFilePath(tenant: string, workspace: string, runId: string): string;
|
|
1263
1397
|
/**
|
|
1264
1398
|
* Erase a run from history: its record and the outputs archived under its
|
|
@@ -1346,6 +1480,11 @@ export interface RunSummaryStep {
|
|
|
1346
1480
|
instruction?: string;
|
|
1347
1481
|
/** When a parked step last spoke — the moment the banner sorts by. */
|
|
1348
1482
|
waitingSince?: string;
|
|
1483
|
+
/** For search (FOL-72 step 9): the tools it called, how many of its calls
|
|
1484
|
+
* errored, and its model. Absent on rows written before 19 Oct 2026. */
|
|
1485
|
+
tools?: string[];
|
|
1486
|
+
toolErrors?: number;
|
|
1487
|
+
model?: string;
|
|
1349
1488
|
startedAt?: string;
|
|
1350
1489
|
finishedAt?: string;
|
|
1351
1490
|
}
|
|
@@ -1363,6 +1502,8 @@ export interface RunSummary {
|
|
|
1363
1502
|
tags?: string[];
|
|
1364
1503
|
startedBy?: string | null;
|
|
1365
1504
|
parkedAt?: string | null;
|
|
1505
|
+
/** The revision it ran (RunRecord.revision), for issues' "since". */
|
|
1506
|
+
revision?: string | null;
|
|
1366
1507
|
/** A step is parked for a person — what the approvals banner counts. */
|
|
1367
1508
|
awaitingApproval: boolean;
|
|
1368
1509
|
steps: RunSummaryStep[];
|