@sweny-ai/core 0.1.156 → 0.1.158
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/budget.d.ts +115 -0
- package/dist/budget.js +191 -0
- package/dist/cli/main.d.ts +2 -0
- package/dist/cli/main.js +5 -1
- package/dist/cli/run-output.d.ts +10 -0
- package/dist/cli/run-output.js +20 -1
- package/dist/cli/workflow-input.d.ts +9 -0
- package/dist/cli/workflow-input.js +21 -0
- package/dist/eval/index.js +32 -32
- package/dist/executor.d.ts +12 -0
- package/dist/executor.js +177 -33
- package/dist/harness/acp.js +4 -1
- package/dist/harness/capabilities.d.ts +10 -4
- package/dist/harness/capabilities.js +12 -6
- package/dist/harness/claude-code.js +25 -1
- package/dist/harness/index.d.ts +1 -1
- package/dist/harness/index.js +1 -1
- package/dist/harness/policy.d.ts +13 -0
- package/dist/harness/policy.js +37 -0
- package/dist/harness/types.d.ts +7 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1 -0
- package/dist/schema.d.ts +144 -1
- package/dist/schema.js +61 -1
- package/dist/types.d.ts +35 -0
- package/dist/types.js +2 -0
- package/package.json +1 -1
package/dist/executor.js
CHANGED
|
@@ -23,7 +23,8 @@ import { validateWorkflow } from "./schema.js";
|
|
|
23
23
|
import { resolveAgentAccess } from "./agent-env.js";
|
|
24
24
|
import { fenceUntrusted } from "./untrusted.js";
|
|
25
25
|
import { asClaude } from "./harness/compat.js";
|
|
26
|
-
import { isToolClass, policyGate } from "./harness/policy.js";
|
|
26
|
+
import { budgetGate, isToolClass, policyGate, resolveHarnessPolicy } from "./harness/policy.js";
|
|
27
|
+
import { BudgetGuard, describeOverrun, minLimits, toLimits } from "./budget.js";
|
|
27
28
|
import { buildNodePolicy, resolveNodePermissions } from "./node-policy.js";
|
|
28
29
|
import { applySafeOutputs, createEmitOutputTool, createWriteStageState, resolveActor, safeOutputsInstruction, unresolvedOutputs, SAFE_OUTPUT_SCREEN_INSTRUCTION, } from "./safe-outputs.js";
|
|
29
30
|
/**
|
|
@@ -82,6 +83,10 @@ export async function execute(workflow, input, options) {
|
|
|
82
83
|
const actor = usesOutputs ? resolveActor(runEnv, options.actor) : {};
|
|
83
84
|
// The run input, for `number: { input }` issue pins on outputs.
|
|
84
85
|
const runInput = input && typeof input === "object" ? input : undefined;
|
|
86
|
+
// Spend budgets (#449): the run ceiling is the lower of the workflow's
|
|
87
|
+
// `budget` and the caller's (`--max-tokens`, `--max-cost`).
|
|
88
|
+
const budgetGuard = new BudgetGuard(minLimits(toLimits(workflow.budget), toLimits(options.budget)));
|
|
89
|
+
const harnessPolicy = options.harnessPolicy ?? resolveHarnessPolicy(runEnv);
|
|
85
90
|
// Build an eval-time alias table from the loaded skills. Each skill owns
|
|
86
91
|
// its own mapping between skill-tool names and equivalent MCP names. Core
|
|
87
92
|
// stays vendor-neutral; this call just unions what the skills declare.
|
|
@@ -206,14 +211,22 @@ export async function execute(workflow, input, options) {
|
|
|
206
211
|
// (a Record<name, EvalResult> for downstream lookup like
|
|
207
212
|
// `priorNode.evals.tests_run_clean.pass`). This namespace is reserved
|
|
208
213
|
// for runtime verdicts; an agent-provided `data.evals` never overrides it.
|
|
209
|
-
|
|
214
|
+
// `requires` always reads this full map: it is a deterministic gate, not
|
|
215
|
+
// a prompt, so it costs no tokens and keeps every declared path working.
|
|
216
|
+
const requiresContext = {
|
|
210
217
|
input,
|
|
211
218
|
...Object.fromEntries([...results.entries()].map(([k, v]) => [k, buildPriorNodeContext(v)])),
|
|
212
219
|
};
|
|
220
|
+
// What the model sees (#337): only nodes this one can depend on, and a
|
|
221
|
+
// schema'd node's declared fields instead of its prose. `context_mode:
|
|
222
|
+
// full` keeps the old everything-prior map.
|
|
223
|
+
const context = workflow.context_mode === "full"
|
|
224
|
+
? requiresContext
|
|
225
|
+
: buildBoundedContext(workflow, currentId, results, input, resolvedInstruction);
|
|
213
226
|
// Pre-condition gate: evaluate `requires` against the cross-node context
|
|
214
227
|
// BEFORE invoking the LLM. Failure either marks the node failed (on_fail
|
|
215
228
|
// default) or skipped (on_fail: "skip") and skips execution entirely.
|
|
216
|
-
const requiresError = evaluateRequires(node.requires,
|
|
229
|
+
const requiresError = evaluateRequires(node.requires, requiresContext);
|
|
217
230
|
if (requiresError) {
|
|
218
231
|
const onFail = node.requires?.on_fail ?? "fail";
|
|
219
232
|
const result = onFail === "skip"
|
|
@@ -304,39 +317,98 @@ export async function execute(workflow, input, options) {
|
|
|
304
317
|
const gate = nodePolicy && options.harness
|
|
305
318
|
? policyGate(options.harness.capabilities, { ...nodePolicy, egress: [] })
|
|
306
319
|
: undefined;
|
|
320
|
+
// Spend budgets (#449): this node's own limits plus what is left of the
|
|
321
|
+
// run's. The gate says whether the harness can keep them; a strict policy
|
|
322
|
+
// refuses the node when it cannot report the budgeted unit at all.
|
|
323
|
+
const nodeBudget = budgetGuard.node(toLimits(node.budget));
|
|
324
|
+
const budgetOn = budgetGuard.active(nodeBudget.limits);
|
|
325
|
+
const budgetCheck = budgetOn && options.harness
|
|
326
|
+
? budgetGate(options.harness.capabilities, minLimits(nodeBudget.limits, budgetGuard.runLimits), permissions.strict || harnessPolicy === "strict")
|
|
327
|
+
: undefined;
|
|
328
|
+
// A budget stop: fail_soft and on_fail: continue never apply, the run halts.
|
|
329
|
+
let budgetStop;
|
|
330
|
+
let previous;
|
|
307
331
|
while (true) {
|
|
308
|
-
|
|
332
|
+
const refusal = gate?.refuse ?? budgetCheck?.refuse;
|
|
333
|
+
if (refusal) {
|
|
309
334
|
// Not an agent failure: fail_soft never softens a policy refusal.
|
|
310
|
-
logger.warn(` harness refused the node: ${
|
|
335
|
+
logger.warn(` harness refused the node: ${refusal}`, { node: currentId });
|
|
311
336
|
result = {
|
|
312
337
|
status: "failed",
|
|
313
|
-
data: { error:
|
|
338
|
+
data: { error: refusal, refused: true },
|
|
314
339
|
toolCalls: [],
|
|
315
|
-
degraded: gate
|
|
340
|
+
degraded: [...(gate?.degraded ?? []), ...(budgetCheck?.degraded ?? [])],
|
|
316
341
|
};
|
|
317
342
|
break;
|
|
318
343
|
}
|
|
344
|
+
// No attempt may start once a ceiling is reached (the run's budget spent
|
|
345
|
+
// by earlier nodes, or this node's by an earlier attempt).
|
|
346
|
+
const spent = budgetOn ? nodeBudget.exhausted() : undefined;
|
|
347
|
+
if (spent) {
|
|
348
|
+
const error = describeOverrun(spent, currentId);
|
|
349
|
+
logger.warn(` ${error}; not starting the node`, { node: currentId });
|
|
350
|
+
result = {
|
|
351
|
+
...(previous ?? { toolCalls: [] }),
|
|
352
|
+
status: "failed",
|
|
353
|
+
data: { ...(previous?.data ?? {}), error, budget_exceeded: true },
|
|
354
|
+
budget: spent,
|
|
355
|
+
};
|
|
356
|
+
agentRunFailed = true;
|
|
357
|
+
budgetStop = spent;
|
|
358
|
+
break;
|
|
359
|
+
}
|
|
319
360
|
// Only the final attempt's intents may be applied.
|
|
320
361
|
intents.length = 0;
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
362
|
+
const attemptBudget = budgetOn ? nodeBudget.attempt(signal) : undefined;
|
|
363
|
+
try {
|
|
364
|
+
result = await claude.run({
|
|
365
|
+
instruction: currentInstruction,
|
|
366
|
+
context,
|
|
367
|
+
tools: trackedTools,
|
|
368
|
+
outputSchema: node.output,
|
|
369
|
+
maxTurns: node.max_turns,
|
|
370
|
+
disallowedTools: node.disallowed_tools,
|
|
371
|
+
...(denyClasses.length > 0 ? { deny: denyClasses } : {}),
|
|
372
|
+
model: nodeModel,
|
|
373
|
+
signal: attemptBudget?.signal ?? signal,
|
|
374
|
+
timeoutMs,
|
|
375
|
+
agentAccess,
|
|
376
|
+
...(nodePolicy ? { policy: nodePolicy } : {}),
|
|
377
|
+
...(readOnlyNode ? {} : { mcpServers: skillMcpServers }),
|
|
378
|
+
...(readOnlyNode ? { readOnly: true } : {}),
|
|
379
|
+
...(attemptBudget ? { onUsage: attemptBudget.onUsage } : {}),
|
|
380
|
+
onProgress: (message) => {
|
|
381
|
+
safeObserve(observer, { type: "node:progress", node: currentId, message }, logger);
|
|
382
|
+
},
|
|
383
|
+
});
|
|
384
|
+
}
|
|
385
|
+
finally {
|
|
386
|
+
attemptBudget?.dispose();
|
|
387
|
+
}
|
|
388
|
+
previous = result;
|
|
389
|
+
if (budgetCheck && budgetCheck.degraded.length > 0) {
|
|
390
|
+
result = { ...result, degraded: [...new Set([...(result.degraded ?? []), ...budgetCheck.degraded])] };
|
|
391
|
+
}
|
|
392
|
+
if (attemptBudget) {
|
|
393
|
+
// Commit this attempt's spend; a live stop, or a crossing visible only
|
|
394
|
+
// in the final usage, fails the node whatever else it did.
|
|
395
|
+
const breach = attemptBudget.finish(result.usage);
|
|
396
|
+
if (breach) {
|
|
397
|
+
const error = describeOverrun(breach, currentId);
|
|
398
|
+
logger.warn(` ${error}`, { node: currentId });
|
|
399
|
+
const usage = result.usage ?? attemptBudget.lastUsage;
|
|
400
|
+
result = {
|
|
401
|
+
...result,
|
|
402
|
+
status: "failed",
|
|
403
|
+
data: { ...result.data, error, budget_exceeded: true },
|
|
404
|
+
...(usage ? { usage } : {}),
|
|
405
|
+
budget: breach,
|
|
406
|
+
};
|
|
407
|
+
agentRunFailed = true;
|
|
408
|
+
budgetStop = breach;
|
|
409
|
+
break;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
340
412
|
// Opinions the harness could not honor natively: said once per node, never silently dropped.
|
|
341
413
|
if (!degradedLogged && result.degraded && result.degraded.length > 0) {
|
|
342
414
|
degradedLogged = true;
|
|
@@ -442,7 +514,7 @@ export async function execute(workflow, input, options) {
|
|
|
442
514
|
// observers. Eval failures never take this path (agentRunFailed guards it).
|
|
443
515
|
// A strict-policy refusal (#331) is not an agent failure: fail_soft never softens it.
|
|
444
516
|
const refused = result.data?.refused === true;
|
|
445
|
-
if (agentRunFailed && result.status === "failed" && node.fail_soft === true && !refused) {
|
|
517
|
+
if (agentRunFailed && result.status === "failed" && node.fail_soft === true && !refused && !budgetStop) {
|
|
446
518
|
const failError = result.data?.error;
|
|
447
519
|
logger.warn(` fail_soft: node failed (${String(failError ?? "unknown error")}); continuing with partial output`, {
|
|
448
520
|
node: currentId,
|
|
@@ -508,9 +580,10 @@ export async function execute(workflow, input, options) {
|
|
|
508
580
|
// the back of a failure. The failed result stays in `results`, so the run
|
|
509
581
|
// surfaces as failed to callers (CLI exit code, cloud status). Authors who
|
|
510
582
|
// want the legacy fall-through opt in per-node with `on_fail: "continue"`.
|
|
511
|
-
if (result.status === "failed" && (node.on_fail ?? "halt") === "halt") {
|
|
512
|
-
|
|
513
|
-
|
|
583
|
+
if (result.status === "failed" && ((node.on_fail ?? "halt") === "halt" || budgetStop)) {
|
|
584
|
+
const why = budgetStop ? "budget exceeded" : "on_fail: halt";
|
|
585
|
+
logger.warn(` node failed; halting workflow (${why})`, { node: currentId });
|
|
586
|
+
safeObserve(observer, { type: "route", from: currentId, to: "(end)", reason: `node failed (${why})` }, logger);
|
|
514
587
|
break;
|
|
515
588
|
}
|
|
516
589
|
// Dry run gate + routing — shared with requires path via advanceFromNode helper.
|
|
@@ -653,9 +726,9 @@ function buildPriorNodeContext(result) {
|
|
|
653
726
|
* evaluator. When no schema is declared we fall back to the full data
|
|
654
727
|
* (back-compat for workflows without structured outputs).
|
|
655
728
|
*
|
|
656
|
-
*
|
|
657
|
-
* `
|
|
658
|
-
*
|
|
729
|
+
* Downstream node prompts get a similar projection (see
|
|
730
|
+
* `buildBoundedNodeEntry`), minus the null-fill, unless the workflow sets
|
|
731
|
+
* `context_mode: full`.
|
|
659
732
|
*/
|
|
660
733
|
function buildRouteEvalEntry(result, sourceNode) {
|
|
661
734
|
const fullData = (result.data ?? {});
|
|
@@ -714,6 +787,77 @@ function getDeclaredOutputProperties(output) {
|
|
|
714
787
|
return null;
|
|
715
788
|
return new Set(keys);
|
|
716
789
|
}
|
|
790
|
+
// ─── Bounded node context (#337) ─────────────────────────────────
|
|
791
|
+
//
|
|
792
|
+
// Full mode hands every node the whole results map: node N re-sends nodes
|
|
793
|
+
// 1..N-1 including each one's prose `summary`, so prompt bytes per run grow
|
|
794
|
+
// quadratically and one verbose node inflates every later prompt. Bounded
|
|
795
|
+
// mode (the default) keeps the same keys a node can depend on, and sends a
|
|
796
|
+
// schema'd node's declared fields only.
|
|
797
|
+
/** Keys the executor itself writes into data; kept when a view is projected. */
|
|
798
|
+
const RUNTIME_DATA_KEYS = ["error", "fail_soft", "skipped_reason"];
|
|
799
|
+
/**
|
|
800
|
+
* The prompt-context entry for one prior node in bounded mode. A successful
|
|
801
|
+
* node with a declared `output.properties` block contributes those fields
|
|
802
|
+
* (plus executor-written error/fail_soft keys); everything else, including
|
|
803
|
+
* the free-text `summary`, is dropped. Nodes without a schema, and nodes
|
|
804
|
+
* that did not succeed, keep the full `buildPriorNodeContext` entry. The
|
|
805
|
+
* trusted `evals` and `safe_outputs` namespaces always come from the runtime.
|
|
806
|
+
*/
|
|
807
|
+
function buildBoundedNodeEntry(result, sourceNode) {
|
|
808
|
+
const full = buildPriorNodeContext(result);
|
|
809
|
+
const declared = getDeclaredOutputProperties(sourceNode?.output);
|
|
810
|
+
if (!declared || result.status !== "success")
|
|
811
|
+
return full;
|
|
812
|
+
const keep = new Set([...declared, ...RUNTIME_DATA_KEYS, "evals", "safe_outputs"]);
|
|
813
|
+
return Object.fromEntries(Object.entries(full).filter(([k]) => keep.has(k)));
|
|
814
|
+
}
|
|
815
|
+
/** First segment of a requires path (`any:triage.findings[*].x` -> `triage`). */
|
|
816
|
+
function pathRoot(path) {
|
|
817
|
+
return /^(?:all:|any:)?([^.[\]]+)/.exec(path)?.[1];
|
|
818
|
+
}
|
|
819
|
+
function mentionsNode(text, nodeId) {
|
|
820
|
+
const escaped = nodeId.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
821
|
+
return new RegExp(`(?<![A-Za-z0-9_-])${escaped}(?![A-Za-z0-9_-])`).test(text);
|
|
822
|
+
}
|
|
823
|
+
/**
|
|
824
|
+
* Node ids whose results `nodeId` may read: its graph ancestors (every node
|
|
825
|
+
* with an edge path into it, itself included when it sits on a cycle), any
|
|
826
|
+
* node a `requires` path names, and any node id its instruction mentions.
|
|
827
|
+
*/
|
|
828
|
+
function contextDependencies(workflow, nodeId, instruction = "") {
|
|
829
|
+
const deps = new Set();
|
|
830
|
+
const stack = [nodeId];
|
|
831
|
+
for (let id = stack.pop(); id !== undefined; id = stack.pop()) {
|
|
832
|
+
for (const e of workflow.edges) {
|
|
833
|
+
if (e.to === id && !deps.has(e.from)) {
|
|
834
|
+
deps.add(e.from);
|
|
835
|
+
stack.push(e.from);
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
const requires = workflow.nodes[nodeId]?.requires;
|
|
840
|
+
for (const p of [...(requires?.output_required ?? []), ...(requires?.output_matches ?? []).map((m) => m.path)]) {
|
|
841
|
+
const root = pathRoot(p);
|
|
842
|
+
if (root && root !== "input")
|
|
843
|
+
deps.add(root);
|
|
844
|
+
}
|
|
845
|
+
for (const id of Object.keys(workflow.nodes)) {
|
|
846
|
+
if (id !== nodeId && mentionsNode(instruction, id))
|
|
847
|
+
deps.add(id);
|
|
848
|
+
}
|
|
849
|
+
return deps;
|
|
850
|
+
}
|
|
851
|
+
/** The bounded prompt context for one node: `input` plus its dependencies' entries. */
|
|
852
|
+
function buildBoundedContext(workflow, nodeId, results, input, instruction) {
|
|
853
|
+
const deps = contextDependencies(workflow, nodeId, instruction);
|
|
854
|
+
return {
|
|
855
|
+
input,
|
|
856
|
+
...Object.fromEntries([...results.entries()]
|
|
857
|
+
.filter(([k]) => deps.has(k))
|
|
858
|
+
.map(([k, v]) => [k, buildBoundedNodeEntry(v, workflow.nodes[k])])),
|
|
859
|
+
};
|
|
860
|
+
}
|
|
717
861
|
/**
|
|
718
862
|
* Return the declared `required` field names for a node's output schema,
|
|
719
863
|
* or an empty array when no required block is declared. Tolerant of
|
package/dist/harness/acp.js
CHANGED
|
@@ -437,8 +437,10 @@ export class AcpHarness {
|
|
|
437
437
|
// `used` and `size` are context window occupancy, not billed tokens: not mapped.
|
|
438
438
|
const cost = u.cost;
|
|
439
439
|
if (isObj(cost) && typeof cost.amount === "number" && Number.isFinite(cost.amount)) {
|
|
440
|
-
if (typeof cost.currency === "string" && cost.currency.toUpperCase() === "USD")
|
|
440
|
+
if (typeof cost.currency === "string" && cost.currency.toUpperCase() === "USD") {
|
|
441
441
|
out.costUsd = cost.amount;
|
|
442
|
+
o.onUsage?.({ costUsd: cost.amount });
|
|
443
|
+
}
|
|
442
444
|
else if (!costWarned) {
|
|
443
445
|
costWarned = true;
|
|
444
446
|
this.logger.debug(`[acp] usage_update cost is in ${String(cost.currency)}, not USD: not recorded`);
|
|
@@ -789,6 +791,7 @@ export class AcpHarness {
|
|
|
789
791
|
signal: req.signal,
|
|
790
792
|
maxToolCalls: maxTurns,
|
|
791
793
|
onProgress: req.onProgress,
|
|
794
|
+
onUsage: req.onUsage,
|
|
792
795
|
});
|
|
793
796
|
}
|
|
794
797
|
finally {
|
|
@@ -3,7 +3,12 @@
|
|
|
3
3
|
* them without pulling in an agent SDK.
|
|
4
4
|
*/
|
|
5
5
|
import type { HarnessCapabilities } from "./types.js";
|
|
6
|
-
/**
|
|
6
|
+
/**
|
|
7
|
+
* Claude Code enforces every sweny opinion natively, so `policyGate` never degrades or refuses it.
|
|
8
|
+
* usage: tokens live (per assistant message, a lower bound until the terminal
|
|
9
|
+
* `result` message gives the authoritative total); cost only at the end of the
|
|
10
|
+
* node (the SDK prices the run in `total_cost_usd` on `result`).
|
|
11
|
+
*/
|
|
7
12
|
export declare const CLAUDE_CODE_CAPABILITIES: HarnessCapabilities;
|
|
8
13
|
/**
|
|
9
14
|
* Codex (`codex exec --json`), declared from the openai/codex source at
|
|
@@ -81,9 +86,10 @@ export declare const PI_CAPABILITIES: HarnessCapabilities;
|
|
|
81
86
|
* - readOnly none: modes are agent-defined. `policyGate` counts read-only as
|
|
82
87
|
* enforced only with the wrapper's read-only mount.
|
|
83
88
|
* - turnLimit watchdog: no turn limit in the protocol; sweny counts tool calls.
|
|
84
|
-
* - usage costUsd only: `usage_update {used, size, cost?}` is the stable
|
|
85
|
-
* usage
|
|
86
|
-
*
|
|
89
|
+
* - usage costUsd only, live: `usage_update {used, size, cost?}` is the stable
|
|
90
|
+
* usage and arrives while the turn runs, so a cost budget stops the agent
|
|
91
|
+
* mid-node. `used` and `size` are context window occupancy, not billed
|
|
92
|
+
* tokens, so they are not mapped; `cost` (cumulative, ISO 4217 currency) is, when it
|
|
87
93
|
* is USD. Per-turn token usage is behind an unstable flag.
|
|
88
94
|
* - cancel rpc: `session/cancel`, then kill.
|
|
89
95
|
* - resume false: sweny starts a fresh session per node.
|
|
@@ -2,7 +2,12 @@
|
|
|
2
2
|
* Declared capabilities per adapter. No runtime imports, so the executor can use
|
|
3
3
|
* them without pulling in an agent SDK.
|
|
4
4
|
*/
|
|
5
|
-
/**
|
|
5
|
+
/**
|
|
6
|
+
* Claude Code enforces every sweny opinion natively, so `policyGate` never degrades or refuses it.
|
|
7
|
+
* usage: tokens live (per assistant message, a lower bound until the terminal
|
|
8
|
+
* `result` message gives the authoritative total); cost only at the end of the
|
|
9
|
+
* node (the SDK prices the run in `total_cost_usd` on `result`).
|
|
10
|
+
*/
|
|
6
11
|
export const CLAUDE_CODE_CAPABILITIES = {
|
|
7
12
|
structuredOutput: "native",
|
|
8
13
|
toolTrace: "full",
|
|
@@ -11,7 +16,7 @@ export const CLAUDE_CODE_CAPABILITIES = {
|
|
|
11
16
|
sandbox: { fs: true, network: true },
|
|
12
17
|
readOnly: "native",
|
|
13
18
|
turnLimit: "native",
|
|
14
|
-
usage: { tokens: true, costUsd: true, live:
|
|
19
|
+
usage: { tokens: true, costUsd: true, live: true, liveUnits: ["tokens"] },
|
|
15
20
|
cancel: "signal",
|
|
16
21
|
resume: false,
|
|
17
22
|
};
|
|
@@ -115,9 +120,10 @@ export const PI_CAPABILITIES = {
|
|
|
115
120
|
* - readOnly none: modes are agent-defined. `policyGate` counts read-only as
|
|
116
121
|
* enforced only with the wrapper's read-only mount.
|
|
117
122
|
* - turnLimit watchdog: no turn limit in the protocol; sweny counts tool calls.
|
|
118
|
-
* - usage costUsd only: `usage_update {used, size, cost?}` is the stable
|
|
119
|
-
* usage
|
|
120
|
-
*
|
|
123
|
+
* - usage costUsd only, live: `usage_update {used, size, cost?}` is the stable
|
|
124
|
+
* usage and arrives while the turn runs, so a cost budget stops the agent
|
|
125
|
+
* mid-node. `used` and `size` are context window occupancy, not billed
|
|
126
|
+
* tokens, so they are not mapped; `cost` (cumulative, ISO 4217 currency) is, when it
|
|
121
127
|
* is USD. Per-turn token usage is behind an unstable flag.
|
|
122
128
|
* - cancel rpc: `session/cancel`, then kill.
|
|
123
129
|
* - resume false: sweny starts a fresh session per node.
|
|
@@ -130,7 +136,7 @@ export const ACP_CAPABILITIES = {
|
|
|
130
136
|
sandbox: { fs: false, network: false },
|
|
131
137
|
readOnly: "none",
|
|
132
138
|
turnLimit: "watchdog",
|
|
133
|
-
usage: { tokens: false, costUsd: true, live:
|
|
139
|
+
usage: { tokens: false, costUsd: true, live: true, liveUnits: ["costUsd"] },
|
|
134
140
|
cancel: "rpc",
|
|
135
141
|
resume: false,
|
|
136
142
|
};
|
|
@@ -315,7 +315,7 @@ export class ClaudeCodeHarness {
|
|
|
315
315
|
return { ...result, harness: this.info(), degraded: gate.degraded };
|
|
316
316
|
}
|
|
317
317
|
async runQuery(opts) {
|
|
318
|
-
const { instruction, context, tools, outputSchema, onProgress, maxTurns, model, timeoutMs, signal, readOnly, agentAccess, } = opts;
|
|
318
|
+
const { instruction, context, tools, outputSchema, onProgress, maxTurns, model, timeoutMs, signal, readOnly, agentAccess, onUsage, } = opts;
|
|
319
319
|
// Dry run (#380): external MCP servers cannot be classified per tool, so
|
|
320
320
|
// they are unknown, and unknown means write. Drop them, and disallow the
|
|
321
321
|
// built-ins that can change the workspace or shell out.
|
|
@@ -361,6 +361,11 @@ export class ClaudeCodeHarness {
|
|
|
361
361
|
// why FIFO-by-name is wrong.
|
|
362
362
|
const toolCalls = [];
|
|
363
363
|
const pendingByUseId = new Map();
|
|
364
|
+
// Live token tally (#449). Each assistant message carries its API
|
|
365
|
+
// message's usage; the SDK repeats one API message across its content
|
|
366
|
+
// blocks, so keep the latest per message id and sum. A lower bound until
|
|
367
|
+
// the terminal `result` message, whose total is authoritative.
|
|
368
|
+
const liveByMessage = new Map();
|
|
364
369
|
// Convert core tools to SDK MCP tools. The wrapper invokes the user's
|
|
365
370
|
// handler and returns `content` via the MCP transport. We do not push
|
|
366
371
|
// to toolCalls from the wrapper - the stream's tool_result is the sole
|
|
@@ -468,6 +473,25 @@ export class ClaudeCodeHarness {
|
|
|
468
473
|
// Tool_use blocks start a ToolCall record. Status + output are
|
|
469
474
|
// filled in when the matching user tool_result arrives.
|
|
470
475
|
const am = message;
|
|
476
|
+
if (onUsage) {
|
|
477
|
+
const u = am.message?.usage;
|
|
478
|
+
if (u && typeof u === "object") {
|
|
479
|
+
const key = typeof am.message?.id === "string" ? am.message.id : `#${liveByMessage.size}`;
|
|
480
|
+
const num = (v) => (typeof v === "number" && Number.isFinite(v) ? v : 0);
|
|
481
|
+
const prev = liveByMessage.get(key);
|
|
482
|
+
liveByMessage.set(key, {
|
|
483
|
+
input: Math.max(prev?.input ?? 0, num(u.input_tokens)),
|
|
484
|
+
output: Math.max(prev?.output ?? 0, num(u.output_tokens)),
|
|
485
|
+
});
|
|
486
|
+
let inputTokens = 0;
|
|
487
|
+
let outputTokens = 0;
|
|
488
|
+
for (const v of liveByMessage.values()) {
|
|
489
|
+
inputTokens += v.input;
|
|
490
|
+
outputTokens += v.output;
|
|
491
|
+
}
|
|
492
|
+
onUsage({ inputTokens, outputTokens });
|
|
493
|
+
}
|
|
494
|
+
}
|
|
471
495
|
if (am.message?.content && Array.isArray(am.message.content)) {
|
|
472
496
|
for (const block of am.message.content) {
|
|
473
497
|
if (block.type === "tool_use") {
|
package/dist/harness/index.d.ts
CHANGED
|
@@ -8,7 +8,7 @@ import { AcpHarness, type AcpHarnessOptions } from "./acp.js";
|
|
|
8
8
|
import { PiHarness, type PiHarnessOptions } from "./pi.js";
|
|
9
9
|
export { claudeCompat, asClaude } from "./compat.js";
|
|
10
10
|
export type { AgentHarness, HarnessCapabilities, HarnessCompleteRequest, HarnessId, HarnessInfo, HarnessRunRequest, HarnessRunResult, NodePolicy, PolicyGateResult, PolicyWrappers, ToolClass, } from "./types.js";
|
|
11
|
-
export { policyGate, nativeDenyClasses, resolveHarnessPolicy, isToolClass, TOOL_CLASSES } from "./policy.js";
|
|
11
|
+
export { policyGate, budgetGate, nativeDenyClasses, resolveHarnessPolicy, isToolClass, TOOL_CLASSES, } from "./policy.js";
|
|
12
12
|
export type { HarnessPolicyMode } from "./policy.js";
|
|
13
13
|
export { ask, evaluate, buildAskPrompt, buildEvaluatePrompt } from "./prompts.js";
|
|
14
14
|
export { ClaudeCodeHarness, CLAUDE_CODE_CAPABILITIES } from "./claude-code.js";
|
package/dist/harness/index.js
CHANGED
|
@@ -8,7 +8,7 @@ import { AcpHarness } from "./acp.js";
|
|
|
8
8
|
import { PiHarness } from "./pi.js";
|
|
9
9
|
import { parseAcpAgent, unsupportedAgentError } from "./agents.js";
|
|
10
10
|
export { claudeCompat, asClaude } from "./compat.js";
|
|
11
|
-
export { policyGate, nativeDenyClasses, resolveHarnessPolicy, isToolClass, TOOL_CLASSES } from "./policy.js";
|
|
11
|
+
export { policyGate, budgetGate, nativeDenyClasses, resolveHarnessPolicy, isToolClass, TOOL_CLASSES, } from "./policy.js";
|
|
12
12
|
export { ask, evaluate, buildAskPrompt, buildEvaluatePrompt } from "./prompts.js";
|
|
13
13
|
export { ClaudeCodeHarness, CLAUDE_CODE_CAPABILITIES } from "./claude-code.js";
|
|
14
14
|
export { CodexHarness, CODEX_CAPABILITIES, MIN_CODEX_VERSION } from "./codex.js";
|
package/dist/harness/policy.d.ts
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* - wrapped: sweny keeps the opinion itself, but not natively (for example a
|
|
13
13
|
* watchdog in place of a native turn limit). Reported, never refused.
|
|
14
14
|
*/
|
|
15
|
+
import type { SpendLimits } from "../budget.js";
|
|
15
16
|
import type { HarnessCapabilities, NodePolicy, PolicyGateResult, PolicyWrappers, ToolClass } from "./types.js";
|
|
16
17
|
/** `strict` refuses a node whose opinions the harness cannot honor; `warn` runs it and reports `degraded`. */
|
|
17
18
|
export type HarnessPolicyMode = "strict" | "warn";
|
|
@@ -30,3 +31,15 @@ export declare function isToolClass(name: string): name is ToolClass;
|
|
|
30
31
|
/** The tool classes a harness can deny natively. */
|
|
31
32
|
export declare function nativeDenyClasses(caps: HarnessCapabilities): ToolClass[];
|
|
32
33
|
export declare function policyGate(caps: HarnessCapabilities, policy: NodePolicy, wrappers?: PolicyWrappers): PolicyGateResult;
|
|
34
|
+
/**
|
|
35
|
+
* budgetGate (#449): can this harness keep a spend budget? Pure; the executor
|
|
36
|
+
* calls it once per node that has a token or cost limit (node or run).
|
|
37
|
+
*
|
|
38
|
+
* Per budgeted unit:
|
|
39
|
+
* - the harness cannot report the unit at all (Codex has no cost): nothing can
|
|
40
|
+
* be enforced. Reported as `budget_<unit>`; `strict` refuses the node.
|
|
41
|
+
* - the harness reports it only when the node ends: enforced between nodes, so
|
|
42
|
+
* one node can overrun before it is stopped. Reported as `budget_live`,
|
|
43
|
+
* never refused.
|
|
44
|
+
*/
|
|
45
|
+
export declare function budgetGate(caps: HarnessCapabilities, limits: SpendLimits, strict: boolean): PolicyGateResult;
|
package/dist/harness/policy.js
CHANGED
|
@@ -111,3 +111,40 @@ export function policyGate(caps, policy, wrappers = {}) {
|
|
|
111
111
|
}
|
|
112
112
|
return { degraded };
|
|
113
113
|
}
|
|
114
|
+
/**
|
|
115
|
+
* budgetGate (#449): can this harness keep a spend budget? Pure; the executor
|
|
116
|
+
* calls it once per node that has a token or cost limit (node or run).
|
|
117
|
+
*
|
|
118
|
+
* Per budgeted unit:
|
|
119
|
+
* - the harness cannot report the unit at all (Codex has no cost): nothing can
|
|
120
|
+
* be enforced. Reported as `budget_<unit>`; `strict` refuses the node.
|
|
121
|
+
* - the harness reports it only when the node ends: enforced between nodes, so
|
|
122
|
+
* one node can overrun before it is stopped. Reported as `budget_live`,
|
|
123
|
+
* never refused.
|
|
124
|
+
*/
|
|
125
|
+
export function budgetGate(caps, limits, strict) {
|
|
126
|
+
const unenforced = [];
|
|
127
|
+
const wrapped = [];
|
|
128
|
+
const units = [
|
|
129
|
+
{ unit: "tokens", key: "tokens", limit: limits.tokens, reports: caps.usage.tokens },
|
|
130
|
+
{ unit: "cost_usd", key: "costUsd", limit: limits.costUsd, reports: caps.usage.costUsd },
|
|
131
|
+
];
|
|
132
|
+
const notLive = [];
|
|
133
|
+
for (const u of units) {
|
|
134
|
+
if (u.limit === undefined)
|
|
135
|
+
continue;
|
|
136
|
+
if (!u.reports) {
|
|
137
|
+
unenforced.push(`budget_${u.unit}: harness cannot report ${u.unit === "tokens" ? "token" : "cost"} usage, so this budget is not enforced`);
|
|
138
|
+
}
|
|
139
|
+
else if (!caps.usage.live || (caps.usage.liveUnits !== undefined && !caps.usage.liveUnits.includes(u.key))) {
|
|
140
|
+
notLive.push(u.unit);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
if (notLive.length > 0) {
|
|
144
|
+
wrapped.push(`budget_live: harness reports ${notLive.join(" and ")} only when a node ends; the budget is enforced between nodes, so a node can overrun before it is stopped`);
|
|
145
|
+
}
|
|
146
|
+
const degraded = [...unenforced, ...wrapped];
|
|
147
|
+
if (unenforced.length > 0 && strict)
|
|
148
|
+
return { degraded, refuse: `strict policy: ${unenforced.join("; ")}` };
|
|
149
|
+
return { degraded };
|
|
150
|
+
}
|
package/dist/harness/types.d.ts
CHANGED
|
@@ -36,10 +36,17 @@ export interface HarnessCapabilities {
|
|
|
36
36
|
};
|
|
37
37
|
readOnly: "native" | "tool-allowlist" | "none";
|
|
38
38
|
turnLimit: "native" | "watchdog" | "none";
|
|
39
|
+
/**
|
|
40
|
+
* What the harness reports. `live`: usage arrives while the node runs (through
|
|
41
|
+
* `HarnessRunRequest.onUsage`), so a spend budget stops the agent mid-node.
|
|
42
|
+
* `liveUnits` narrows which of the reported units arrive live; absent means
|
|
43
|
+
* every reported unit. Without `live`, budgets are enforced between nodes.
|
|
44
|
+
*/
|
|
39
45
|
usage: {
|
|
40
46
|
tokens: boolean;
|
|
41
47
|
costUsd: boolean;
|
|
42
48
|
live: boolean;
|
|
49
|
+
liveUnits?: Array<"tokens" | "costUsd">;
|
|
43
50
|
};
|
|
44
51
|
cancel: "signal" | "rpc" | "kill";
|
|
45
52
|
resume: boolean;
|
package/dist/index.d.ts
CHANGED
|
@@ -27,6 +27,8 @@ export { EVALUATOR_KINDS, EVAL_POLICIES, REQUIRES_ON_FAIL, NODE_ON_FAIL, MCP_TRA
|
|
|
27
27
|
export { consoleLogger } from "./types.js";
|
|
28
28
|
export { execute, RouteEvaluationError } from "./executor.js";
|
|
29
29
|
export type { ExecuteOptions } from "./executor.js";
|
|
30
|
+
export { BudgetGuard, describeOverrun, minLimits, toLimits } from "./budget.js";
|
|
31
|
+
export type { Budget, BudgetOverrun, BudgetUnit, SpendLimits } from "./budget.js";
|
|
30
32
|
export { resolveNodePermissions, buildNodePolicy } from "./node-policy.js";
|
|
31
33
|
export type { ResolvedPermissions } from "./node-policy.js";
|
|
32
34
|
export { applySafeOutputs, createEmitOutputTool, resolveActor, EMIT_OUTPUT_TOOL } from "./safe-outputs.js";
|
package/dist/index.js
CHANGED
|
@@ -28,6 +28,7 @@ export { consoleLogger } from "./types.js";
|
|
|
28
28
|
// Executor
|
|
29
29
|
export { execute, RouteEvaluationError } from "./executor.js";
|
|
30
30
|
// Least-privilege nodes and safe outputs (#365)
|
|
31
|
+
export { BudgetGuard, describeOverrun, minLimits, toLimits } from "./budget.js";
|
|
31
32
|
export { resolveNodePermissions, buildNodePolicy } from "./node-policy.js";
|
|
32
33
|
export { applySafeOutputs, createEmitOutputTool, resolveActor, EMIT_OUTPUT_TOOL } from "./safe-outputs.js";
|
|
33
34
|
// Agent harness (the seam) and its adapters: Claude Code, Codex and pi
|