agentfences 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -4
- package/dist/approvals.d.ts +24 -0
- package/dist/approvals.js +57 -0
- package/dist/client.d.ts +25 -3
- package/dist/client.js +94 -14
- package/dist/control.d.ts +27 -27
- package/dist/control.js +39 -72
- package/dist/core.d.ts +13 -74
- package/dist/core.js +40 -145
- package/dist/index.d.ts +4 -2
- package/dist/index.js +2 -1
- package/dist/instrument.d.ts +12 -0
- package/dist/instrument.js +124 -0
- package/dist/mcp.js +3 -3
- package/dist/outbox.d.ts +23 -0
- package/dist/outbox.js +35 -0
- package/dist/precheck.d.ts +33 -0
- package/dist/precheck.js +136 -0
- package/dist/pricing.d.ts +1 -0
- package/dist/pricing.js +17 -4
- package/dist/results.d.ts +16 -0
- package/dist/results.js +33 -0
- package/dist/run.d.ts +79 -0
- package/dist/run.js +56 -0
- package/dist/spans.d.ts +11 -1
- package/dist/spans.js +16 -2
- package/dist/tools.d.ts +4 -1
- package/dist/tools.js +23 -12
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +28 -6
package/dist/core.d.ts
CHANGED
|
@@ -2,80 +2,11 @@ import { FencesClient } from "./client.js";
|
|
|
2
2
|
import * as control from "./control.js";
|
|
3
3
|
import * as events from "./events.js";
|
|
4
4
|
import { type CustomPrice } from "./pricing.js";
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
/** First-person text, ready to return to the user. */
|
|
11
|
-
message: string;
|
|
12
|
-
/** An instruction to give the model so it wraps up. */
|
|
13
|
-
systemPrompt: string;
|
|
14
|
-
/** The run's total spend so far. */
|
|
15
|
-
spentUsd: number;
|
|
16
|
-
}
|
|
17
|
-
/** Every limit is optional; see governed(). */
|
|
18
|
-
export interface Limits {
|
|
19
|
-
/** Maximum spend for the run in USD. None anywhere: unlimited spend, with a warning. */
|
|
20
|
-
budgetUsd?: number | null;
|
|
21
|
-
/** Maximum checkpoints (steps). Default 100. */
|
|
22
|
-
maxIterations?: number | null;
|
|
23
|
-
/** Maximum wall-clock time in milliseconds. Default 300000 (5 minutes). */
|
|
24
|
-
maxDurationMs?: number | null;
|
|
25
|
-
/** Maximum tokens (input + output). 0 or unset: no limit. */
|
|
26
|
-
maxTokens?: number | null;
|
|
27
|
-
}
|
|
28
|
-
export declare const MAX_EVENTS_PER_RUN = 1000;
|
|
29
|
-
export interface Span {
|
|
30
|
-
name: string;
|
|
31
|
-
start_ts: number;
|
|
32
|
-
end_ts: number;
|
|
33
|
-
status: "ok" | "error";
|
|
34
|
-
attributes: Record<string, unknown>;
|
|
35
|
-
}
|
|
36
|
-
export declare class Run {
|
|
37
|
-
readonly agentName: string;
|
|
38
|
-
budgetUsd: number | null;
|
|
39
|
-
maxIterations: number;
|
|
40
|
-
maxDurationMs: number;
|
|
41
|
-
maxTokens: number;
|
|
42
|
-
readonly context: Record<string, string>;
|
|
43
|
-
readonly runId: `${string}-${string}-${string}-${string}-${string}`;
|
|
44
|
-
costUsd: number;
|
|
45
|
-
iterations: number;
|
|
46
|
-
tokensUsed: number;
|
|
47
|
-
readonly startedAt: number;
|
|
48
|
-
readonly decisions: {
|
|
49
|
-
timestamp: number;
|
|
50
|
-
iteration: number;
|
|
51
|
-
reasoning: string;
|
|
52
|
-
action: string | null;
|
|
53
|
-
}[];
|
|
54
|
-
/** Tool and model calls as OpenTelemetry-shaped spans, newest 1,000 kept. */
|
|
55
|
-
readonly events: Span[];
|
|
56
|
-
lastBreach: string | null;
|
|
57
|
-
exception?: events.ExceptionInfo;
|
|
58
|
-
warned: boolean;
|
|
59
|
-
/** Live control from the dashboard (see control.ts). */
|
|
60
|
-
readonly control: control.Control;
|
|
61
|
-
/** Held by guarded tools that are running. */
|
|
62
|
-
reservedUsd: number;
|
|
63
|
-
/** "hold": wait at a limit for someone to raise it (see InitOptions.onBreach). */
|
|
64
|
-
onBreach: "hold" | null;
|
|
65
|
-
private readonly abort;
|
|
66
|
-
/**
|
|
67
|
-
* Aborts when Fences halts the run after a stop it didn't reach a checkpoint for (see
|
|
68
|
-
* stopGraceS). Pass it to fetch and other cancellable work so a stuck run really stops:
|
|
69
|
-
* await fetch(url, { signal: getActiveRun()?.signal });
|
|
70
|
-
*/
|
|
71
|
-
get signal(): AbortSignal;
|
|
72
|
-
/** @internal */
|
|
73
|
-
_abort(reason: unknown): void;
|
|
74
|
-
constructor(agentName: string, budgetUsd: number | null, maxIterations: number, maxDurationMs: number, maxTokens: number, context: Record<string, string>);
|
|
75
|
-
get durationMs(): number;
|
|
76
|
-
}
|
|
77
|
-
/** The run in progress in this async context, if any. */
|
|
78
|
-
export declare function getActiveRun(): Run | null;
|
|
5
|
+
import { type ToolArguments } from "./spans.js";
|
|
6
|
+
import { type CheckpointResult } from "./results.js";
|
|
7
|
+
import { Run, type Limits } from "./run.js";
|
|
8
|
+
export type { CheckpointResult } from "./results.js";
|
|
9
|
+
export { Run, getActiveRun, type Limits, type Span } from "./run.js";
|
|
79
10
|
export interface InitOptions {
|
|
80
11
|
/** Your Fences API key (fc_...). Required unless localOnly. */
|
|
81
12
|
apiKey?: string;
|
|
@@ -118,6 +49,12 @@ export interface InitOptions {
|
|
|
118
49
|
* wait, and after pauseTimeoutS the breach is returned as usual. Cloud only; override per agent in governed().
|
|
119
50
|
*/
|
|
120
51
|
onBreach?: "hold" | null;
|
|
52
|
+
/**
|
|
53
|
+
* What a recorded tool call carries of its arguments: "names" (the default: `query, limit`),
|
|
54
|
+
* "full" (the arguments as JSON, capped) or "none". A guarded tool's check and an approval request
|
|
55
|
+
* still send the values, which rules and approvers need; the redact hook can change or drop them.
|
|
56
|
+
*/
|
|
57
|
+
toolArguments?: ToolArguments;
|
|
121
58
|
}
|
|
122
59
|
/** Call once at startup, before any governed function runs. */
|
|
123
60
|
export declare function init(options?: InitOptions): void;
|
|
@@ -156,6 +93,8 @@ export interface Extra {
|
|
|
156
93
|
* Never throws for a breach. Outside a governed run it does nothing.
|
|
157
94
|
*/
|
|
158
95
|
export declare function checkpoint(response?: unknown, extra?: Extra): Promise<CheckpointResult>;
|
|
96
|
+
/** @internal A cost or count must be a finite number, zero or more: a NaN or negative one would switch the budget off. */
|
|
97
|
+
export declare function checkAmount(value: unknown, what: string): number;
|
|
159
98
|
/** @internal The breach the run is in by local rules, if any. */
|
|
160
99
|
export declare function localBreach(run: Run): string | null;
|
|
161
100
|
/**
|
package/dist/core.js
CHANGED
|
@@ -1,93 +1,20 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* init(), governed() and checkpoint(): starting and ending runs, checking every step against
|
|
3
|
+
* its limits (the server decides in cloud mode, the same rules apply here otherwise), and
|
|
4
|
+
* recording decisions and spans.
|
|
5
|
+
*/
|
|
2
6
|
import { randomUUID } from "node:crypto";
|
|
3
7
|
import { FencesClient } from "./client.js";
|
|
4
8
|
import * as control from "./control.js";
|
|
5
9
|
import { AgentQuarantined, FencesStop } from "./errors.js";
|
|
6
10
|
import * as events from "./events.js";
|
|
11
|
+
import * as outbox from "./outbox.js";
|
|
12
|
+
import * as precheck from "./precheck.js";
|
|
7
13
|
import { costOf, setCustomPrices } from "./pricing.js";
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
function breachResult(type, run) {
|
|
13
|
-
const spent = run.costUsd, limit = run.budgetUsd ?? 0;
|
|
14
|
-
const text = {
|
|
15
|
-
budget_exceeded: [`I've reached my budget limit (spent ${money(spent)} of ${money(limit)}). ${SUMMARY}`,
|
|
16
|
-
`You have reached your budget limit (${money(spent)} of ${money(limit)} spent). Stop your current task immediately ` +
|
|
17
|
-
"and provide a clear summary of what you have found or completed so far. Tell the user you stopped due to budget and what you accomplished."],
|
|
18
|
-
iteration_limit: [`I've reached my iteration limit (${run.iterations} steps). ${SUMMARY}`,
|
|
19
|
-
`You have reached your iteration limit (${run.iterations} steps). ${WRAP_UP} Tell the user you stopped due to the iteration limit.`],
|
|
20
|
-
time_limit: [`I've reached my time limit (${run.durationMs}ms). ${SUMMARY}`,
|
|
21
|
-
`You have reached your time limit. ${WRAP_UP} Tell the user you stopped due to the time limit.`],
|
|
22
|
-
token_limit: [`I've reached my token limit (${count(run.tokensUsed)} tokens). ${SUMMARY}`,
|
|
23
|
-
`You have reached your token limit (${count(run.tokensUsed)} tokens used). ${WRAP_UP} Tell the user you stopped due to the token limit.`],
|
|
24
|
-
stopped_by_user: [`I was stopped from the Fences dashboard. ${SUMMARY}`,
|
|
25
|
-
`A person stopped this run from the Fences dashboard. ${WRAP_UP} Tell the user you were stopped by an operator.`],
|
|
26
|
-
key_daily_budget: [`My daily spending cap has been reached. ${SUMMARY}`,
|
|
27
|
-
`Your daily spending cap, across all of your runs, has been reached. ${WRAP_UP} Tell the user you stopped because the daily cap was reached.`],
|
|
28
|
-
key_monthly_budget: [`My monthly spending cap has been reached. ${SUMMARY}`,
|
|
29
|
-
`Your monthly spending cap, across all of your runs, has been reached. ${WRAP_UP} Tell the user you stopped because the monthly cap was reached.`],
|
|
30
|
-
fences_unreachable: [`I can't reach Fences to confirm I'm within my limits, so I'm stopping to be safe. ${SUMMARY}`,
|
|
31
|
-
`Your governance service can't be reached and you are configured to stop when that happens. ${WRAP_UP}`],
|
|
32
|
-
paused: [`I was paused from the Fences dashboard and not resumed in time. ${SUMMARY}`,
|
|
33
|
-
`A person paused this run from the Fences dashboard and it was not resumed in time. Stop your current task and summarize ` +
|
|
34
|
-
"what you have found or completed so far. Tell the user the run was paused by an operator."],
|
|
35
|
-
};
|
|
36
|
-
const [message, systemPrompt] = text[type] ??
|
|
37
|
-
["Governance limit reached.", "A governance limit has been reached. Summarize what you have done so far."];
|
|
38
|
-
return { ok: false, breached: true, breachType: type, message, systemPrompt, spentUsd: run.costUsd };
|
|
39
|
-
}
|
|
40
|
-
const okResult = (spentUsd = 0) => ({ ok: true, breached: false, breachType: null, message: "", systemPrompt: "", spentUsd });
|
|
41
|
-
const DEFAULTS = { maxIterations: 100, maxDurationMs: 300_000, maxTokens: 0 };
|
|
42
|
-
export const MAX_EVENTS_PER_RUN = 1000;
|
|
43
|
-
export class Run {
|
|
44
|
-
agentName;
|
|
45
|
-
budgetUsd;
|
|
46
|
-
maxIterations;
|
|
47
|
-
maxDurationMs;
|
|
48
|
-
maxTokens;
|
|
49
|
-
context;
|
|
50
|
-
runId = randomUUID();
|
|
51
|
-
costUsd = 0;
|
|
52
|
-
iterations = 0;
|
|
53
|
-
tokensUsed = 0;
|
|
54
|
-
startedAt = Date.now();
|
|
55
|
-
decisions = [];
|
|
56
|
-
/** Tool and model calls as OpenTelemetry-shaped spans, newest 1,000 kept. */
|
|
57
|
-
events = [];
|
|
58
|
-
lastBreach = null;
|
|
59
|
-
exception;
|
|
60
|
-
warned = false;
|
|
61
|
-
/** Live control from the dashboard (see control.ts). */
|
|
62
|
-
control = control.newControl();
|
|
63
|
-
/** Held by guarded tools that are running. */
|
|
64
|
-
reservedUsd = 0;
|
|
65
|
-
/** "hold": wait at a limit for someone to raise it (see InitOptions.onBreach). */
|
|
66
|
-
onBreach = null;
|
|
67
|
-
abort = new AbortController();
|
|
68
|
-
/**
|
|
69
|
-
* Aborts when Fences halts the run after a stop it didn't reach a checkpoint for (see
|
|
70
|
-
* stopGraceS). Pass it to fetch and other cancellable work so a stuck run really stops:
|
|
71
|
-
* await fetch(url, { signal: getActiveRun()?.signal });
|
|
72
|
-
*/
|
|
73
|
-
get signal() { return this.abort.signal; }
|
|
74
|
-
/** @internal */
|
|
75
|
-
_abort(reason) { this.abort.abort(reason); }
|
|
76
|
-
constructor(agentName, budgetUsd, maxIterations, maxDurationMs, maxTokens, context) {
|
|
77
|
-
this.agentName = agentName;
|
|
78
|
-
this.budgetUsd = budgetUsd;
|
|
79
|
-
this.maxIterations = maxIterations;
|
|
80
|
-
this.maxDurationMs = maxDurationMs;
|
|
81
|
-
this.maxTokens = maxTokens;
|
|
82
|
-
this.context = context;
|
|
83
|
-
}
|
|
84
|
-
get durationMs() { return Date.now() - this.startedAt; }
|
|
85
|
-
}
|
|
86
|
-
const store = new AsyncLocalStorage();
|
|
87
|
-
/** The run in progress in this async context, if any. */
|
|
88
|
-
export function getActiveRun() {
|
|
89
|
-
return store.getStore() ?? null;
|
|
90
|
-
}
|
|
14
|
+
import { setToolArguments } from "./spans.js";
|
|
15
|
+
import { breachResult, okResult } from "./results.js";
|
|
16
|
+
import { DEFAULTS, MAX_EVENTS_PER_RUN, Run, getActiveRun, store } from "./run.js";
|
|
17
|
+
export { Run, getActiveRun } from "./run.js";
|
|
91
18
|
let client = null;
|
|
92
19
|
let localOnly = false;
|
|
93
20
|
let failClosed = false;
|
|
@@ -101,6 +28,7 @@ function checkOnBreach(value) {
|
|
|
101
28
|
}
|
|
102
29
|
/** Call once at startup, before any governed function runs. */
|
|
103
30
|
export function init(options = {}) {
|
|
31
|
+
setToolArguments(options.toolArguments ?? "names");
|
|
104
32
|
setCustomPrices(options.prices);
|
|
105
33
|
events.configure(options.environment, options.release, options.redact);
|
|
106
34
|
localOnly = !!options.localOnly;
|
|
@@ -130,11 +58,11 @@ function requireInit() {
|
|
|
130
58
|
throw new Error("Call agentfences.init() before using Fences. For local usage: init({ localOnly: true })");
|
|
131
59
|
}
|
|
132
60
|
}
|
|
133
|
-
function warnUnreachable(run, err) {
|
|
61
|
+
function warnUnreachable(run, err, ending = false) {
|
|
134
62
|
if (run.warned)
|
|
135
63
|
return;
|
|
136
64
|
run.warned = true;
|
|
137
|
-
console.warn(`agentfences: Fences backend unreachable for run ${run.runId} (${err}); enforcing limits locally`);
|
|
65
|
+
console.warn(`agentfences: Fences backend unreachable for run ${run.runId} (${err}); ${ending ? "ending the run" : "enforcing limits locally"}`);
|
|
138
66
|
}
|
|
139
67
|
// ── governed ──────────────────────────────────────────────────────────────────
|
|
140
68
|
/**
|
|
@@ -186,7 +114,6 @@ function outcome(run, failed, error) {
|
|
|
186
114
|
const e = run.exception;
|
|
187
115
|
return ["error", e.message ? `${e.type}: ${e.message}` : e.type];
|
|
188
116
|
}
|
|
189
|
-
const LIMIT_KEYS = { budget_usd: "budgetUsd", max_iterations: "maxIterations", max_duration_ms: "maxDurationMs", max_tokens: "maxTokens" };
|
|
190
117
|
async function startRun(agentName, limits) {
|
|
191
118
|
const run = new Run(agentName, limits.budgetUsd ?? null, limits.maxIterations ?? DEFAULTS.maxIterations, limits.maxDurationMs ?? DEFAULTS.maxDurationMs, limits.maxTokens ?? DEFAULTS.maxTokens, events.currentContext());
|
|
192
119
|
if (!client) {
|
|
@@ -206,11 +133,7 @@ async function startRun(agentName, limits) {
|
|
|
206
133
|
if (resp.quarantined)
|
|
207
134
|
throw new AgentQuarantined(agentName, resp.detail);
|
|
208
135
|
// The server caps limits at the agent's ceilings: enforce the same numbers if it becomes unreachable later
|
|
209
|
-
|
|
210
|
-
const v = resp.limits?.[wire];
|
|
211
|
-
if (typeof v === "number" || (wire === "budget_usd" && v === null))
|
|
212
|
-
run[field] = v; // eslint-disable-line @typescript-eslint/no-explicit-any
|
|
213
|
-
}
|
|
136
|
+
control.adopt(run, resp.limits ?? {}, control.LIMITS, true);
|
|
214
137
|
for (const n of resp.notices ?? []) {
|
|
215
138
|
events.warnOnce(`${agentName}:${n.code}`, `${n.message ?? ""}${n.url ? ` ${n.url}` : ""}`);
|
|
216
139
|
}
|
|
@@ -256,13 +179,15 @@ export async function checkpoint(response, extra = {}) {
|
|
|
256
179
|
throw new TypeError("checkpoint() takes the model's response first; pass a cost as checkpoint(null, { costDeltaUsd })");
|
|
257
180
|
}
|
|
258
181
|
const [cost, tokens] = response === undefined || response === null ? [0, 0] : costOf(response);
|
|
259
|
-
const stepCost = cost + (extra.costDeltaUsd ?? 0
|
|
260
|
-
|
|
182
|
+
const stepCost = cost + checkAmount(extra.costDeltaUsd ?? 0, "costDeltaUsd");
|
|
183
|
+
const stepTokens = tokens + checkAmount(extra.tokensUsed ?? 0, "tokensUsed");
|
|
184
|
+
run.costUsd += stepCost; // the actual cost replaces the holds of model calls that finished
|
|
185
|
+
precheck.releaseFinished(run);
|
|
261
186
|
run.tokensUsed += stepTokens;
|
|
262
187
|
run.iterations += 1;
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
188
|
+
// A step_id per step: a retried checkpoint is counted once on servers that support it
|
|
189
|
+
const resp = client ? await client.checkpoint(run.runId, { cost_delta_usd: stepCost, tokens_used: stepTokens,
|
|
190
|
+
iterations: run.iterations, duration_ms: run.durationMs, step_id: randomUUID().replaceAll("-", "") }) : null;
|
|
266
191
|
const result = decide(run, resp);
|
|
267
192
|
// onBreach "hold": wait here for someone to raise the limit. Only a breach the server decided: an unreachable one can't be raised
|
|
268
193
|
if (result.breachType && control.HOLDABLE.has(result.breachType) && control.holds(run) && resp && !resp.network_error) {
|
|
@@ -281,6 +206,14 @@ export async function checkpoint(response, extra = {}) {
|
|
|
281
206
|
}
|
|
282
207
|
return result;
|
|
283
208
|
}
|
|
209
|
+
/** @internal A cost or count must be a finite number, zero or more: a NaN or negative one would switch the budget off. */
|
|
210
|
+
export function checkAmount(value, what) {
|
|
211
|
+
if (typeof value !== "number" || Number.isNaN(value))
|
|
212
|
+
throw new TypeError(`${what} must be a number, not ${String(value)}`);
|
|
213
|
+
if (value < 0 || value === Infinity)
|
|
214
|
+
throw new RangeError(`${what} must be zero or more and finite, not ${value}`);
|
|
215
|
+
return value;
|
|
216
|
+
}
|
|
284
217
|
/** @internal The breach the run is in by local rules, if any. */
|
|
285
218
|
export function localBreach(run) {
|
|
286
219
|
// Limits trip only once exceeded, same rule as the server; round away float drift (0.02*5 != 0.10)
|
|
@@ -297,10 +230,11 @@ export function localBreach(run) {
|
|
|
297
230
|
function decide(run, resp) {
|
|
298
231
|
if (!resp || resp.network_error) {
|
|
299
232
|
if (resp) {
|
|
300
|
-
|
|
301
|
-
|
|
233
|
+
const ending = failClosed || !!resp.auth_rejected; // a key revoked mid-run: the run can't go on unchecked
|
|
234
|
+
warnUnreachable(run, resp.network_error, ending);
|
|
235
|
+
if (ending) {
|
|
302
236
|
run.lastBreach = "fences_unreachable";
|
|
303
|
-
return
|
|
237
|
+
return finalResult(run);
|
|
304
238
|
}
|
|
305
239
|
}
|
|
306
240
|
run.lastBreach = localBreach(run);
|
|
@@ -308,13 +242,8 @@ function decide(run, resp) {
|
|
|
308
242
|
else {
|
|
309
243
|
control.apply(run, resp.commands);
|
|
310
244
|
// The server is authoritative: adopt its totals and (on a breach) its current limits
|
|
311
|
-
run
|
|
312
|
-
run
|
|
313
|
-
run.tokensUsed = resp.tokens_used ?? run.tokensUsed;
|
|
314
|
-
for (const [wire, field] of Object.entries(LIMIT_KEYS)) {
|
|
315
|
-
if (wire in resp)
|
|
316
|
-
run[field] = resp[wire]; // eslint-disable-line @typescript-eslint/no-explicit-any
|
|
317
|
-
}
|
|
245
|
+
control.adopt(run, resp, control.TOTALS);
|
|
246
|
+
control.adopt(run, resp, control.LIMITS, true);
|
|
318
247
|
// A fresh breach, or a 409 because the run is still fenced from an earlier one
|
|
319
248
|
run.lastBreach = resp.ok ? null : (resp.breach ?? run.lastBreach ?? "limit_reached");
|
|
320
249
|
}
|
|
@@ -330,47 +259,13 @@ function finalResult(run) {
|
|
|
330
259
|
return run.lastBreach ? breachResult(run.lastBreach, run) : okResult(run.costUsd);
|
|
331
260
|
}
|
|
332
261
|
// ── Decision trail and spans ──────────────────────────────────────────────────
|
|
333
|
-
const SPAN_BATCH = 50;
|
|
334
|
-
const outbox = [];
|
|
335
|
-
let draining = null;
|
|
336
|
-
let spansSupported = true;
|
|
337
|
-
function enqueue(item) {
|
|
338
|
-
outbox.push(item);
|
|
339
|
-
draining ??= drain().finally(() => { draining = null; });
|
|
340
|
-
}
|
|
341
|
-
/** Decisions and spans go to the server in the background, in order; consecutive spans of a run go together. */
|
|
342
|
-
async function drain() {
|
|
343
|
-
while (outbox.length) {
|
|
344
|
-
const item = outbox.shift();
|
|
345
|
-
const c = client;
|
|
346
|
-
if (!c)
|
|
347
|
-
continue;
|
|
348
|
-
try {
|
|
349
|
-
if (item.kind === "decision") {
|
|
350
|
-
await c.logDecision(item.runId, item.body);
|
|
351
|
-
continue;
|
|
352
|
-
}
|
|
353
|
-
const batch = [item.span];
|
|
354
|
-
while (outbox[0]?.kind === "span" && outbox[0].runId === item.runId && batch.length < SPAN_BATCH) {
|
|
355
|
-
batch.push(outbox.shift().span);
|
|
356
|
-
}
|
|
357
|
-
if (spansSupported && (await c.logSpans(item.runId, batch)).unsupported) {
|
|
358
|
-
spansSupported = false;
|
|
359
|
-
console.warn("agentfences: this Fences server doesn't accept tool and model call spans yet; " +
|
|
360
|
-
"they're kept on the run locally but not sent");
|
|
361
|
-
}
|
|
362
|
-
}
|
|
363
|
-
catch { /* never let delivery break the agent */ }
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
262
|
/**
|
|
367
263
|
* Wait up to timeoutMs for queued decisions and spans to reach the server, and for onStop and
|
|
368
264
|
* onQuarantine hooks to finish. Call before a serverless handler returns.
|
|
369
265
|
*/
|
|
370
266
|
export async function flush(timeoutMs = 5000) {
|
|
371
267
|
const deadline = Date.now() + timeoutMs;
|
|
372
|
-
|
|
373
|
-
await Promise.race([draining, new Promise((r) => setTimeout(r, timeoutMs).unref())]);
|
|
268
|
+
await outbox.wait(timeoutMs);
|
|
374
269
|
await control.drain(Math.max(0, deadline - Date.now()));
|
|
375
270
|
}
|
|
376
271
|
/** Record the agent's reasoning at this step. Does nothing outside a governed run. */
|
|
@@ -383,7 +278,7 @@ export function logDecision(reasoning, action = null) {
|
|
|
383
278
|
return;
|
|
384
279
|
const sent = events.redact({ type: "decision", reasoning, action });
|
|
385
280
|
if (sent && typeof sent.reasoning === "string" && sent.reasoning) {
|
|
386
|
-
enqueue({ kind: "decision", runId: run.runId,
|
|
281
|
+
outbox.enqueue({ kind: "decision", client, runId: run.runId,
|
|
387
282
|
body: { iteration: run.iterations, reasoning: sent.reasoning, action: sent.action ?? null } });
|
|
388
283
|
}
|
|
389
284
|
}
|
|
@@ -412,11 +307,11 @@ export function recordSpan(name, elapsedMs, attributes, error) {
|
|
|
412
307
|
run.events.push(span);
|
|
413
308
|
if (run.events.length > MAX_EVENTS_PER_RUN)
|
|
414
309
|
run.events.shift(); // a runaway agent can't eat memory
|
|
415
|
-
if (!client || !spansSupported)
|
|
310
|
+
if (!client || !outbox.spansSupported)
|
|
416
311
|
return;
|
|
417
312
|
const sent = events.redact({ type: "span", ...span });
|
|
418
313
|
if (sent) {
|
|
419
314
|
const { type: _, ...redacted } = sent; // the type key is for the redact hook only
|
|
420
|
-
enqueue({ kind: "span", runId: run.runId, span: redacted });
|
|
315
|
+
outbox.enqueue({ kind: "span", client, runId: run.runId, span: redacted });
|
|
421
316
|
}
|
|
422
317
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -4,10 +4,12 @@ export { context } from "./events.js";
|
|
|
4
4
|
export type { RedactHook, Event } from "./events.js";
|
|
5
5
|
export { tool } from "./tools.js";
|
|
6
6
|
export type { GuardedToolOptions } from "./tools.js";
|
|
7
|
-
export { requestApproval } from "./
|
|
8
|
-
export type { Approval
|
|
7
|
+
export { requestApproval } from "./approvals.js";
|
|
8
|
+
export type { Approval } from "./approvals.js";
|
|
9
|
+
export type { ControlEvent, Hook } from "./control.js";
|
|
9
10
|
export { priceOf } from "./pricing.js";
|
|
10
11
|
export type { CustomPrice } from "./pricing.js";
|
|
11
12
|
export { FencesError, FencesAuthError, AgentQuarantined, FencesStop, ToolBlocked } from "./errors.js";
|
|
12
13
|
export { VERSION } from "./version.js";
|
|
13
14
|
export * as mcp from "./mcp.js";
|
|
15
|
+
export { instrument } from "./instrument.js";
|
package/dist/index.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
export { init, governed, checkpoint, logDecision, flush, getActiveRun, Run } from "./core.js";
|
|
2
2
|
export { context } from "./events.js";
|
|
3
3
|
export { tool } from "./tools.js";
|
|
4
|
-
export { requestApproval } from "./
|
|
4
|
+
export { requestApproval } from "./approvals.js";
|
|
5
5
|
export { priceOf } from "./pricing.js";
|
|
6
6
|
export { FencesError, FencesAuthError, AgentQuarantined, FencesStop, ToolBlocked } from "./errors.js";
|
|
7
7
|
export { VERSION } from "./version.js";
|
|
8
8
|
export * as mcp from "./mcp.js";
|
|
9
|
+
export { instrument } from "./instrument.js";
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Patch the SDK behind this client so every model call inside a governed run is recorded and,
|
|
3
|
+
* with a budget, prechecked:
|
|
4
|
+
*
|
|
5
|
+
* import OpenAI from "openai";
|
|
6
|
+
* const openai = instrument(new OpenAI());
|
|
7
|
+
*
|
|
8
|
+
* Takes an OpenAI client (chat.completions and responses, including OpenAI-compatible clients)
|
|
9
|
+
* or an Anthropic one (messages). One call covers every client of that SDK; calling it again
|
|
10
|
+
* does nothing. Returns the client.
|
|
11
|
+
*/
|
|
12
|
+
export declare function instrument<C extends object>(client: C): C;
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Record every OpenAI and Anthropic model call made inside a governed run, with no other code
|
|
3
|
+
* changes: model, tokens, cost, latency, and the error if it failed. Calls don't count as
|
|
4
|
+
* checkpoints and don't add to the run's spend (checkpoint(response) does), but in a run with a
|
|
5
|
+
* budget each one is prechecked: refused with FencesStop if its worst case doesn't fit.
|
|
6
|
+
*/
|
|
7
|
+
import * as precheck from "./precheck.js";
|
|
8
|
+
import { getActiveRun, store } from "./run.js";
|
|
9
|
+
import { recordLlmCall } from "./spans.js";
|
|
10
|
+
const PATCHED = Symbol.for("agentfences.instrumented");
|
|
11
|
+
/**
|
|
12
|
+
* Patch the SDK behind this client so every model call inside a governed run is recorded and,
|
|
13
|
+
* with a budget, prechecked:
|
|
14
|
+
*
|
|
15
|
+
* import OpenAI from "openai";
|
|
16
|
+
* const openai = instrument(new OpenAI());
|
|
17
|
+
*
|
|
18
|
+
* Takes an OpenAI client (chat.completions and responses, including OpenAI-compatible clients)
|
|
19
|
+
* or an Anthropic one (messages). One call covers every client of that SDK; calling it again
|
|
20
|
+
* does nothing. Returns the client.
|
|
21
|
+
*/
|
|
22
|
+
export function instrument(client) {
|
|
23
|
+
const c = client;
|
|
24
|
+
const provider = c.chat ? "openai" : "anthropic";
|
|
25
|
+
const resources = [c.chat?.completions, c.responses, c.messages].filter((r) => typeof r?.create === "function");
|
|
26
|
+
if (!resources.length)
|
|
27
|
+
throw new TypeError("instrument() takes an OpenAI or Anthropic client");
|
|
28
|
+
for (const r of resources) {
|
|
29
|
+
const proto = Object.getPrototypeOf(r);
|
|
30
|
+
if (typeof proto?.create === "function" && !proto.create[PATCHED])
|
|
31
|
+
proto.create = wrap(proto.create, provider);
|
|
32
|
+
}
|
|
33
|
+
return client;
|
|
34
|
+
}
|
|
35
|
+
function wrap(create, provider) {
|
|
36
|
+
const wrapped = function (...args) {
|
|
37
|
+
const run = getActiveRun();
|
|
38
|
+
if (!run)
|
|
39
|
+
return create.apply(this, args);
|
|
40
|
+
const body = (args[0] ?? {});
|
|
41
|
+
const prepared = precheck.prepare(body.model, body, body);
|
|
42
|
+
if (!prepared)
|
|
43
|
+
return send(this, create, args, provider, run, null);
|
|
44
|
+
let hold;
|
|
45
|
+
try {
|
|
46
|
+
hold = precheck.holdNow(prepared); // FencesStop if the call might not fit the budget
|
|
47
|
+
}
|
|
48
|
+
catch (e) {
|
|
49
|
+
return refused(e);
|
|
50
|
+
}
|
|
51
|
+
if (hold !== "wait")
|
|
52
|
+
return send(this, create, args, provider, run, hold);
|
|
53
|
+
// onBreach "hold": wait for a raised budget, then send. Boxed, so nothing awaits the SDK's promise before the caller does
|
|
54
|
+
return later(precheck.holdAsync(prepared).then((h) => ({ out: send(this, create, args, provider, run, h) })));
|
|
55
|
+
};
|
|
56
|
+
wrapped[PATCHED] = true;
|
|
57
|
+
return wrapped;
|
|
58
|
+
}
|
|
59
|
+
/** A call sent once a hold is released: awaitable, with the SDK promise's methods, each forwarded once it's sent. */
|
|
60
|
+
function later(sent) {
|
|
61
|
+
const value = () => sent.then(({ out }) => out);
|
|
62
|
+
return {
|
|
63
|
+
then: (ok, ko) => value().then(ok, ko),
|
|
64
|
+
catch: (ko) => value().catch(ko),
|
|
65
|
+
finally: (fn) => value().finally(fn),
|
|
66
|
+
withResponse: () => sent.then(({ out }) => out.withResponse()),
|
|
67
|
+
asResponse: () => sent.then(({ out }) => out.asResponse()),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
/** A refused call rejects like the SDK's own promise would, never throwing synchronously, and keeps its methods. */
|
|
71
|
+
function refused(error) {
|
|
72
|
+
const p = Promise.reject(error);
|
|
73
|
+
p.withResponse = p.asResponse = () => p;
|
|
74
|
+
return p;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Make the call and return the SDK's own promise (an APIPromise, with .withResponse()) unchanged,
|
|
78
|
+
* except that awaiting it, or calling withResponse() or asResponse(), also records the call and
|
|
79
|
+
* settles its hold. Nothing reads the response before the caller does: after asResponse(), whose
|
|
80
|
+
* body the caller reads, the call is recorded without usage and its worst case stays held until
|
|
81
|
+
* the next checkpoint.
|
|
82
|
+
*/
|
|
83
|
+
function send(self, create, args, provider, run, hold) {
|
|
84
|
+
const body = (args[0] ?? {});
|
|
85
|
+
const started = performance.now();
|
|
86
|
+
let done = false;
|
|
87
|
+
const record = (response, error) => {
|
|
88
|
+
if (done)
|
|
89
|
+
return;
|
|
90
|
+
done = true;
|
|
91
|
+
store.run(run, () => {
|
|
92
|
+
if (hold)
|
|
93
|
+
precheck.finish(run, hold, response, error !== undefined);
|
|
94
|
+
recordLlmCall(provider, body.model, started, response, error, { stream: !!body.stream });
|
|
95
|
+
});
|
|
96
|
+
};
|
|
97
|
+
let out;
|
|
98
|
+
try {
|
|
99
|
+
out = create.apply(self, args);
|
|
100
|
+
}
|
|
101
|
+
catch (e) {
|
|
102
|
+
record(undefined, e);
|
|
103
|
+
throw e;
|
|
104
|
+
}
|
|
105
|
+
if (!out || typeof out.then !== "function") {
|
|
106
|
+
record(out);
|
|
107
|
+
return out;
|
|
108
|
+
}
|
|
109
|
+
const then = out.then.bind(out);
|
|
110
|
+
out.then = (ok, ko) => then((v) => { record(v); return ok ? ok(v) : v; }, (e) => { record(undefined, e); if (ko)
|
|
111
|
+
return ko(e); throw e; });
|
|
112
|
+
// APIPromise has its own catch and finally, which skip then: route them through the one above
|
|
113
|
+
out.catch = (ko) => Promise.prototype.catch.call(out, ko);
|
|
114
|
+
out.finally = (fn) => Promise.prototype.finally.call(out, fn);
|
|
115
|
+
if (typeof out.asResponse === "function") { // the caller reads the body: the call is recorded without usage
|
|
116
|
+
const asResponse = out.asResponse.bind(out);
|
|
117
|
+
out.asResponse = () => asResponse().then((r) => { record(); return r; }, (e) => { record(undefined, e); throw e; });
|
|
118
|
+
}
|
|
119
|
+
if (typeof out.withResponse === "function") {
|
|
120
|
+
const withResponse = out.withResponse.bind(out);
|
|
121
|
+
out.withResponse = () => withResponse().then((r) => { record(r?.data); return r; }, (e) => { record(undefined, e); throw e; });
|
|
122
|
+
}
|
|
123
|
+
return out;
|
|
124
|
+
}
|
package/dist/mcp.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getActiveRun } from "./
|
|
1
|
+
import { getActiveRun } from "./run.js";
|
|
2
2
|
import { argsSummary, callAsync } from "./tools.js";
|
|
3
3
|
/** An MCP tool answered with isError: recorded as the call's error, the result still goes back. */
|
|
4
4
|
export class MCPToolError extends Error {
|
|
@@ -29,12 +29,12 @@ export function wrap(client, options = {}) {
|
|
|
29
29
|
const { name } = params, args = params.arguments ?? {}, c = Object.hasOwn(costs, name) ? costs[name] : undefined;
|
|
30
30
|
const guarded = options.guard === true || named.has(name) || c !== undefined || approve.has(name);
|
|
31
31
|
try {
|
|
32
|
-
return await callAsync(run, name, argsSummary([args]), guarded ? () =>
|
|
32
|
+
return await callAsync(run, name, argsSummary([args]), guarded ? () => (typeof c === "function" ? c(args) : c) ?? 0 : null, approve.has(name), async () => {
|
|
33
33
|
const result = await original(params, ...rest);
|
|
34
34
|
if (result?.isError)
|
|
35
35
|
throw new MCPToolError(result);
|
|
36
36
|
return result;
|
|
37
|
-
});
|
|
37
|
+
}, Object.keys(args).join(", "));
|
|
38
38
|
}
|
|
39
39
|
catch (e) {
|
|
40
40
|
if (e instanceof MCPToolError)
|
package/dist/outbox.d.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decisions and spans go to the server in the background, in order, so logDecision() and
|
|
3
|
+
* recorded calls never wait on a network round trip. Consecutive spans of a run go together.
|
|
4
|
+
*/
|
|
5
|
+
import type { FencesClient } from "./client.js";
|
|
6
|
+
import type { Span } from "./run.js";
|
|
7
|
+
type Item = {
|
|
8
|
+
kind: "decision";
|
|
9
|
+
client: FencesClient;
|
|
10
|
+
runId: string;
|
|
11
|
+
body: Record<string, unknown>;
|
|
12
|
+
} | {
|
|
13
|
+
kind: "span";
|
|
14
|
+
client: FencesClient;
|
|
15
|
+
runId: string;
|
|
16
|
+
span: Span;
|
|
17
|
+
};
|
|
18
|
+
/** Off once a server says it has no spans endpoint. */
|
|
19
|
+
export declare let spansSupported: boolean;
|
|
20
|
+
export declare function enqueue(item: Item): void;
|
|
21
|
+
/** Wait up to timeoutMs for everything queued to be sent. */
|
|
22
|
+
export declare function wait(timeoutMs: number): Promise<void>;
|
|
23
|
+
export {};
|
package/dist/outbox.js
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
const SPAN_BATCH = 50;
|
|
2
|
+
const queue = [];
|
|
3
|
+
let draining = null;
|
|
4
|
+
/** Off once a server says it has no spans endpoint. */
|
|
5
|
+
export let spansSupported = true;
|
|
6
|
+
export function enqueue(item) {
|
|
7
|
+
queue.push(item);
|
|
8
|
+
draining ??= drain().finally(() => { draining = null; });
|
|
9
|
+
}
|
|
10
|
+
/** Wait up to timeoutMs for everything queued to be sent. */
|
|
11
|
+
export async function wait(timeoutMs) {
|
|
12
|
+
if (draining)
|
|
13
|
+
await Promise.race([draining, new Promise((r) => setTimeout(r, timeoutMs).unref())]);
|
|
14
|
+
}
|
|
15
|
+
async function drain() {
|
|
16
|
+
while (queue.length) {
|
|
17
|
+
const item = queue.shift();
|
|
18
|
+
try {
|
|
19
|
+
if (item.kind === "decision") {
|
|
20
|
+
await item.client.logDecision(item.runId, item.body);
|
|
21
|
+
continue;
|
|
22
|
+
}
|
|
23
|
+
const batch = [item.span];
|
|
24
|
+
while (queue[0]?.kind === "span" && queue[0].runId === item.runId && batch.length < SPAN_BATCH) {
|
|
25
|
+
batch.push(queue.shift().span);
|
|
26
|
+
}
|
|
27
|
+
if (spansSupported && (await item.client.logSpans(item.runId, batch)).unsupported) {
|
|
28
|
+
spansSupported = false;
|
|
29
|
+
console.warn("agentfences: this Fences server doesn't accept tool and model call spans yet; " +
|
|
30
|
+
"they're kept on the run locally but not sent");
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
catch { /* never let delivery break the agent */ }
|
|
34
|
+
}
|
|
35
|
+
}
|