agentfences 0.5.0 → 0.6.0

package/README.md

@@ -78,6 +78,8 @@ if (!approval.granted) return `I didn't issue the refund: ${approval.note}`;
 
 An approved amount raises the run's budget. It never hangs or throws: in local mode, outside a run, or on a server without approvals it's denied right away, and with no answer it's denied after `timeoutS`. The heartbeat's timer never keeps your process alive on its own.
 
+**Waiting at a limit** (from 0.6.0): with `init({ onBreach: "hold" })` (or `governed(fn, { onBreach: "hold" })`), a limit the server says is crossed makes `checkpoint()` wait instead of returning the breach, so whoever is on call can raise it from the dashboard and the agent carries on. A guarded tool denied for budget waits the same way. A stop ends the wait; after `pauseTimeoutS` the agent gets the breach it was held at. Cloud only. Don't hold an agent a person is waiting on.
+
 **Containment hooks** (from 0.5.0) run in your process when a stop or quarantine arrives, to contain what Fences can't reach, such as a credential the agent holds:
 
 ```ts

package/dist/control.d.ts

@@ -34,6 +34,8 @@ export interface Control {
     paused: boolean;
     pauseExpired: boolean;
     quarantined: boolean;
+    /** While waiting at a limit: "breach" or "budget". */
+    held: "breach" | "budget" | null;
     /** approval id -> the answer, or null while waiting */
     approvals: Map<string, Approval | null>;
     awaiting: number;
@@ -66,6 +68,15 @@ export declare function apply(run: Run, commands: Reply[] | undefined): void;
  * its containment, and says so.
  */
 export declare function drain(timeoutMs?: number): Promise<void>;
+/** Breaches a person can lift by raising a limit, so onBreach "hold" waits at them. */
+export declare const HOLDABLE: Set<string>;
+export declare const BUDGET_BREACHES: Set<string>;
+/** Whether this run waits at a limit: onBreach "hold", in cloud mode (someone can raise it), and not stopped. */
+export declare const holds: (run: Run) => boolean;
+/** Pause the run at a limit: "breach" (a checkpoint breached) or "budget" (a call refused for budget). */
+export declare function startHold(run: Run, kind: "breach" | "budget"): void;
+/** After waiting: "released" (an operator raised the limit), "stopped", or "timeout". */
+export declare function endHold(run: Run): "released" | "stopped" | "timeout";
 /** Hold the agent at its checkpoint while its run is paused, up to pauseTimeoutS. */
 export declare function waitIfPaused(run: Run): Promise<void>;
 /**

package/dist/control.js

@@ -6,11 +6,13 @@
  * on as before. Same behaviour as the Python SDK (PROTOCOL.md, "Live control").
  */
 import { randomUUID } from "node:crypto";
+/** Heartbeat statuses whose acks go again with the next heartbeat: rate limited, or a server error. */
+const retryable = (status) => status === undefined || status === 429 || status >= 500;
 const FAST_MS = 2000; // heartbeat interval while a run is paused, waiting for an approval, or stopping
 const HOOK_TIMEOUT_MS = 30_000; // a hook still running after this is reported as timed out
 export const newControl = () => ({
     stopped: false, stoppedAt: 0, stopSeen: false, cancelled: false, onCancel: null, paused: false, pauseExpired: false,
-    quarantined: false, approvals: new Map(), awaiting: 0, acked: new Set(),
+    quarantined: false, held: null, approvals: new Map(), awaiting: 0, acked: new Set(),
 });
 const live = new Map(); // run id -> runs in progress
 let client = null;
@@ -52,7 +54,8 @@ export function unsupported(kind, what, consequence) {
 const LIMITS = { budget_usd: "budgetUsd", max_iterations: "maxIterations", max_duration_ms: "maxDurationMs", max_tokens: "maxTokens" };
 /** Apply the commands addressed to this run (from a heartbeat or a checkpoint reply), and acknowledge them. Each is safe to apply twice. */
 export function apply(run, commands) {
-    for (const c of commands ?? []) {
+    // Limits first: a resume in the same reply may release a call held for budget, which re-checks against them at once
+    for (const c of [...(commands ?? [])].sort((a, b) => Number(a.type !== "limits") - Number(b.type !== "limits"))) {
         if (c.run_id !== undefined && c.run_id !== run.runId)
             continue;
         const ctl = run.control;
@@ -153,7 +156,7 @@ async function sendAcks() {
     const batch = acks;
     acks = [];
     const resp = await client.heartbeat([], batch).catch(() => ({ network_error: "heartbeat failed" }));
-    if (resp.network_error || (resp.ok === false && (resp.status ?? 500) >= 500))
+    if (resp.network_error || (resp.ok === false && retryable(resp.status)))
         acks = [...batch, ...acks];
 }
 // Node exits once nothing keeps it busy: beforeExit lets acks queued by a finished hook still go out.
@@ -201,10 +204,11 @@ async function beat() {
     acks = [];
     try {
         const resp = await client.heartbeat(runs.map((r) => ({ run_id: r.runId, iterations: r.iterations, spent_usd: r.costUsd,
-            tokens_used: r.tokensUsed, paused: r.control.paused, stopped: r.control.stopped })), batch);
+            tokens_used: r.tokensUsed, paused: r.control.paused, stopped: r.control.stopped,
+            ...(r.control.held ? { held: r.control.held } : {}) })), batch);
         if (resp.network_error || (resp.ok === false && !resp.unsupported)) {
-            if (resp.network_error || (resp.status ?? 500) >= 500)
-                acks = [...batch, ...acks]; // at least once; a 4xx won't get better
+            if (resp.network_error || retryable(resp.status))
+                acks = [...batch, ...acks]; // at least once; any other 4xx won't get better
         }
         else if (resp.unsupported) {
             unsupported("heartbeat", "the heartbeat", "a stop from the dashboard applies at the next checkpoint and pause isn't available");
@@ -223,6 +227,32 @@ async function beat() {
     }
     schedule(nextDelay());
 }
+// ── Waiting at a limit ────────────────────────────────────────────────────────
+/** Breaches a person can lift by raising a limit, so onBreach "hold" waits at them. */
+export const HOLDABLE = new Set(["budget_exceeded", "iteration_limit", "time_limit", "token_limit", "key_daily_budget",
+    "key_monthly_budget", "limit_reached"]);
+export const BUDGET_BREACHES = new Set(["budget_exceeded", "key_daily_budget", "key_monthly_budget"]);
+/** Whether this run waits at a limit: onBreach "hold", in cloud mode (someone can raise it), and not stopped. */
+export const holds = (run) => run.onBreach === "hold" && client !== null && !run.control.stopped;
+/** Pause the run at a limit: "breach" (a checkpoint breached) or "budget" (a call refused for budget). */
+export function startHold(run, kind) {
+    run.control.held = kind;
+    if (!run.control.paused) {
+        run.control.paused = true;
+        schedule(0);
+    }
+}
+/** After waiting: "released" (an operator raised the limit), "stopped", or "timeout". */
+export function endHold(run) {
+    run.control.held = null;
+    if (run.control.stopped)
+        return "stopped";
+    if (run.control.pauseExpired) {
+        run.control.pauseExpired = false; // the run ends with the limit it was held at, not "paused"
+        return "timeout";
+    }
+    return "released";
+}
 // ── Pausing ───────────────────────────────────────────────────────────────────
 /** Hold the agent at its checkpoint while its run is paused, up to pauseTimeoutS. */
 export async function waitIfPaused(run) {

package/dist/core.d.ts

@@ -60,6 +60,8 @@ export declare class Run {
     readonly control: control.Control;
     /** Held by guarded tools that are running. */
     reservedUsd: number;
+    /** "hold": wait at a limit for someone to raise it (see InitOptions.onBreach). */
+    onBreach: "hold" | null;
     private readonly abort;
     /**
      * Aborts when Fences halts the run after a stop it didn't reach a checkpoint for (see
@@ -109,6 +111,13 @@ export interface InitOptions {
     onStop?: control.Hook;
     /** The same, when the agent is quarantined (once per quarantine, however many runs it reaches). */
     onQuarantine?: control.Hook;
+    /**
+     * "hold" makes runs wait at a limit instead of ending: when the server says a limit is crossed,
+     * checkpoint() waits (as if paused) until someone raises it and resumes the run, then returns ok
+     * and the agent carries on. A guarded tool denied for budget waits the same way. A stop ends the
+     * wait, and after pauseTimeoutS the breach is returned as usual. Cloud only; override per agent in governed().
+     */
+    onBreach?: "hold" | null;
 }
 /** Call once at startup, before any governed function runs. */
 export declare function init(options?: InitOptions): void;
@@ -127,6 +136,7 @@ export declare const _client: () => FencesClient | null;
  */
 export declare function governed<A extends unknown[], R>(fn: (...args: A) => R | Promise<R>, options?: Limits & {
     name?: string;
+    onBreach?: "hold" | null;
 }): (...args: A) => Promise<Awaited<R>>;
 export interface Extra {
     /** Extra spend for this step in USD, e.g. a paid tool call. Without a response, the step's whole cost. */

package/dist/core.js

@@ -62,6 +62,8 @@ export class Run {
     control = control.newControl();
     /** Held by guarded tools that are running. */
     reservedUsd = 0;
+    /** "hold": wait at a limit for someone to raise it (see InitOptions.onBreach). */
+    onBreach = null;
     abort = new AbortController();
     /**
      * Aborts when Fences halts the run after a stop it didn't reach a checkpoint for (see
@@ -89,12 +91,21 @@ export function getActiveRun() {
 let client = null;
 let localOnly = false;
 let failClosed = false;
+let defaultOnBreach = null;
+function checkOnBreach(value) {
+    if (value === undefined || value === null)
+        return null;
+    if (value !== "hold")
+        throw new TypeError(`onBreach must be "hold" or null, not ${JSON.stringify(value)}`);
+    return value;
+}
 /** Call once at startup, before any governed function runs. */
 export function init(options = {}) {
     setCustomPrices(options.prices);
     events.configure(options.environment, options.release, options.redact);
     localOnly = !!options.localOnly;
     failClosed = !!options.failClosed;
+    defaultOnBreach = checkOnBreach(options.onBreach);
     if (localOnly) {
         client = null;
     }
@@ -137,9 +148,11 @@ function warnUnreachable(run, err) {
  */
 export function governed(fn, options = {}) {
     const name = options.name ?? (fn.name || "agent");
+    checkOnBreach(options.onBreach);
     return async (...args) => {
         requireInit();
         const run = await startRun(name, options);
+        run.onBreach = checkOnBreach(options.onBreach) ?? defaultOnBreach;
         // A stop the agent doesn't reach a checkpoint for within stopGraceS halts the call here
         const halted = new Promise((_, reject) => {
             run.control.onCancel = () => {
@@ -251,6 +264,17 @@ export async function checkpoint(response, extra = {}) {
         cost_delta_usd: stepCost, tokens_used: stepTokens, iterations: run.iterations, duration_ms: run.durationMs
     }) : null;
     const result = decide(run, resp);
+    // onBreach "hold": wait here for someone to raise the limit. Only a breach the server decided: an unreachable one can't be raised
+    if (result.breachType && control.HOLDABLE.has(result.breachType) && control.holds(run) && resp && !resp.network_error) {
+        control.startHold(run, "breach");
+        await control.waitIfPaused(run);
+        const outcome = control.endHold(run);
+        if (outcome === "released") { // an operator raised the limit and resumed the run
+            run.lastBreach = null;
+            return okResult(run.costUsd);
+        }
+        return outcome === "stopped" ? finalResult(run) : result; // timed out: the limit it was held at
+    }
     if (run.control.paused && !result.breached) { // paused from the dashboard: wait here until resumed
         await control.waitIfPaused(run);
         return finalResult(run);

package/dist/tools.js

@@ -36,7 +36,7 @@ async function open(run, name, summary, amount, needsApproval) {
         await control.waitIfPaused(run);
     if (run.control.pauseExpired)
         block(run, name, "the run was paused from the Fences dashboard and not resumed in time", "paused");
-    let approvalId = needsApproval ? await approve(run, name, summary, amount) : null;
+    const approvalId = needsApproval ? await approve(run, name, summary, amount) : null;
     if (run.control.stopped)
         block(run, name, "the run was stopped from the Fences dashboard", "stopped_by_user");
     const client = _client();
@@ -49,6 +49,27 @@ async function open(run, name, summary, amount, needsApproval) {
         run.reservedUsd += amount;
         return randomUUID();
     }
+    const call = { guardId: randomUUID(), approvalId }; // one id per call: the server answers a re-check of a denied id afresh
+    for (;;) {
+        try {
+            return await check(run, client, name, summary, amount, call);
+        }
+        catch (e) {
+            if (!(e instanceof ToolBlocked && control.BUDGET_BREACHES.has(e.breachType ?? "") && control.holds(run)))
+                throw e;
+            control.startHold(run, "budget"); // onBreach "hold": wait for someone to raise the budget, then check again
+            await control.waitIfPaused(run);
+            const outcome = control.endHold(run);
+            if (outcome === "stopped")
+                block(run, name, "the run was stopped from the Fences dashboard", "stopped_by_user");
+            if (outcome !== "released")
+                throw e;
+        }
+    }
+}
+async function check(run, client, name, summary, amount, call) {
+    if (run.control.stopped)
+        block(run, name, "the run was stopped from the Fences dashboard", "stopped_by_user");
     // Check against what this process already holds before asking the server, which can't see it; reserve until it answers
     if (!fits(run, amount))
         block(run, name, `$${amount.toFixed(2)} more could take the run past its $${run.budgetUsd.toFixed(2)} budget`, "budget_exceeded");
@@ -57,15 +78,14 @@ async function open(run, name, summary, amount, needsApproval) {
         const sent = events.redact({ type: "guard", tool: name, arguments: summary }) ?? {}; // the check goes ahead without them
         let resp = {}; // eslint-disable-line @typescript-eslint/no-explicit-any
         for (let attempt = 0; attempt < 2; attempt++) { // once more after an approval the server asked for
-            const guardId = randomUUID();
-            resp = await client.guard(run.runId, { guard_id: guardId, phase: "check", tool: name, arguments: sent.arguments ?? null,
-                amount_usd: amount, approval_id: approvalId });
+            resp = await client.guard(run.runId, { guard_id: call.guardId, phase: "check", tool: name, arguments: sent.arguments ?? null,
+                amount_usd: amount, approval_id: call.approvalId });
             control.apply(run, resp.commands);
             if (resp.allow === true)
-                return guardId;
-            if (!(resp.needs_approval && approvalId === null))
+                return call.guardId;
+            if (!(resp.needs_approval && call.approvalId === null))
                 break;
-            approvalId = await approve(run, name, summary, amount);
+            call.approvalId = await approve(run, name, summary, amount);
         }
         if (resp.unsupported) {
             control.unsupported("guard", "guarded tools", "they are blocked");

package/dist/version.d.ts

@@ -1,2 +1,2 @@
 /** The one place the version lives; sent to Fences on every request. Keep in step with package.json. */
-export declare const VERSION = "0.5.0";
+export declare const VERSION = "0.6.0";

package/dist/version.js

@@ -1,2 +1,2 @@
 /** The one place the version lives; sent to Fences on every request. Keep in step with package.json. */
-export const VERSION = "0.5.0";
+export const VERSION = "0.6.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentfences",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "On-call for AI agents: limits your agents can't raise, cost tracking, a decision trail, and a stop button",
   "license": "MIT",
   "author": "Bhavik Sardar",