agentfences 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -3
- package/dist/approvals.d.ts +24 -0
- package/dist/approvals.js +57 -0
- package/dist/client.d.ts +5 -3
- package/dist/client.js +17 -9
- package/dist/control.d.ts +27 -27
- package/dist/control.js +22 -69
- package/dist/core.d.ts +11 -74
- package/dist/core.js +35 -143
- package/dist/index.d.ts +5 -2
- package/dist/index.js +3 -1
- package/dist/instrument.d.ts +12 -0
- package/dist/instrument.js +124 -0
- package/dist/mcp.d.ts +40 -0
- package/dist/mcp.js +46 -0
- package/dist/outbox.d.ts +23 -0
- package/dist/outbox.js +35 -0
- package/dist/precheck.d.ts +33 -0
- package/dist/precheck.js +136 -0
- package/dist/pricing.d.ts +1 -0
- package/dist/pricing.js +17 -4
- package/dist/results.d.ts +16 -0
- package/dist/results.js +33 -0
- package/dist/run.d.ts +79 -0
- package/dist/run.js +56 -0
- package/dist/tools.d.ts +3 -0
- package/dist/tools.js +31 -23
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +28 -6
package/dist/core.js
CHANGED
|
@@ -1,93 +1,18 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
/**
|
|
2
|
+
* init(), governed() and checkpoint(): starting and ending runs, checking every step against
|
|
3
|
+
* its limits (the server decides in cloud mode, the same rules apply here otherwise), and
|
|
4
|
+
* recording decisions and spans.
|
|
5
|
+
*/
|
|
3
6
|
import { FencesClient } from "./client.js";
|
|
4
7
|
import * as control from "./control.js";
|
|
5
8
|
import { AgentQuarantined, FencesStop } from "./errors.js";
|
|
6
9
|
import * as events from "./events.js";
|
|
10
|
+
import * as outbox from "./outbox.js";
|
|
11
|
+
import * as precheck from "./precheck.js";
|
|
7
12
|
import { costOf, setCustomPrices } from "./pricing.js";
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
const count = (n) => n.toLocaleString("en-US");
|
|
12
|
-
function breachResult(type, run) {
|
|
13
|
-
const spent = run.costUsd, limit = run.budgetUsd ?? 0;
|
|
14
|
-
const text = {
|
|
15
|
-
budget_exceeded: [`I've reached my budget limit (spent ${money(spent)} of ${money(limit)}). ${SUMMARY}`,
|
|
16
|
-
`You have reached your budget limit (${money(spent)} of ${money(limit)} spent). Stop your current task immediately ` +
|
|
17
|
-
"and provide a clear summary of what you have found or completed so far. Tell the user you stopped due to budget and what you accomplished."],
|
|
18
|
-
iteration_limit: [`I've reached my iteration limit (${run.iterations} steps). ${SUMMARY}`,
|
|
19
|
-
`You have reached your iteration limit (${run.iterations} steps). ${WRAP_UP} Tell the user you stopped due to the iteration limit.`],
|
|
20
|
-
time_limit: [`I've reached my time limit (${run.durationMs}ms). ${SUMMARY}`,
|
|
21
|
-
`You have reached your time limit. ${WRAP_UP} Tell the user you stopped due to the time limit.`],
|
|
22
|
-
token_limit: [`I've reached my token limit (${count(run.tokensUsed)} tokens). ${SUMMARY}`,
|
|
23
|
-
`You have reached your token limit (${count(run.tokensUsed)} tokens used). ${WRAP_UP} Tell the user you stopped due to the token limit.`],
|
|
24
|
-
stopped_by_user: [`I was stopped from the Fences dashboard. ${SUMMARY}`,
|
|
25
|
-
`A person stopped this run from the Fences dashboard. ${WRAP_UP} Tell the user you were stopped by an operator.`],
|
|
26
|
-
key_daily_budget: [`My daily spending cap has been reached. ${SUMMARY}`,
|
|
27
|
-
`Your daily spending cap, across all of your runs, has been reached. ${WRAP_UP} Tell the user you stopped because the daily cap was reached.`],
|
|
28
|
-
key_monthly_budget: [`My monthly spending cap has been reached. ${SUMMARY}`,
|
|
29
|
-
`Your monthly spending cap, across all of your runs, has been reached. ${WRAP_UP} Tell the user you stopped because the monthly cap was reached.`],
|
|
30
|
-
fences_unreachable: [`I can't reach Fences to confirm I'm within my limits, so I'm stopping to be safe. ${SUMMARY}`,
|
|
31
|
-
`Your governance service can't be reached and you are configured to stop when that happens. ${WRAP_UP}`],
|
|
32
|
-
paused: [`I was paused from the Fences dashboard and not resumed in time. ${SUMMARY}`,
|
|
33
|
-
`A person paused this run from the Fences dashboard and it was not resumed in time. Stop your current task and summarize ` +
|
|
34
|
-
"what you have found or completed so far. Tell the user the run was paused by an operator."],
|
|
35
|
-
};
|
|
36
|
-
const [message, systemPrompt] = text[type] ??
|
|
37
|
-
["Governance limit reached.", "A governance limit has been reached. Summarize what you have done so far."];
|
|
38
|
-
return { ok: false, breached: true, breachType: type, message, systemPrompt, spentUsd: run.costUsd };
|
|
39
|
-
}
|
|
40
|
-
const okResult = (spentUsd = 0) => ({ ok: true, breached: false, breachType: null, message: "", systemPrompt: "", spentUsd });
|
|
41
|
-
const DEFAULTS = { maxIterations: 100, maxDurationMs: 300_000, maxTokens: 0 };
|
|
42
|
-
export const MAX_EVENTS_PER_RUN = 1000;
|
|
43
|
-
export class Run {
|
|
44
|
-
agentName;
|
|
45
|
-
budgetUsd;
|
|
46
|
-
maxIterations;
|
|
47
|
-
maxDurationMs;
|
|
48
|
-
maxTokens;
|
|
49
|
-
context;
|
|
50
|
-
runId = randomUUID();
|
|
51
|
-
costUsd = 0;
|
|
52
|
-
iterations = 0;
|
|
53
|
-
tokensUsed = 0;
|
|
54
|
-
startedAt = Date.now();
|
|
55
|
-
decisions = [];
|
|
56
|
-
/** Tool and model calls as OpenTelemetry-shaped spans, newest 1,000 kept. */
|
|
57
|
-
events = [];
|
|
58
|
-
lastBreach = null;
|
|
59
|
-
exception;
|
|
60
|
-
warned = false;
|
|
61
|
-
/** Live control from the dashboard (see control.ts). */
|
|
62
|
-
control = control.newControl();
|
|
63
|
-
/** Held by guarded tools that are running. */
|
|
64
|
-
reservedUsd = 0;
|
|
65
|
-
/** "hold": wait at a limit for someone to raise it (see InitOptions.onBreach). */
|
|
66
|
-
onBreach = null;
|
|
67
|
-
abort = new AbortController();
|
|
68
|
-
/**
|
|
69
|
-
* Aborts when Fences halts the run after a stop it didn't reach a checkpoint for (see
|
|
70
|
-
* stopGraceS). Pass it to fetch and other cancellable work so a stuck run really stops:
|
|
71
|
-
* await fetch(url, { signal: getActiveRun()?.signal });
|
|
72
|
-
*/
|
|
73
|
-
get signal() { return this.abort.signal; }
|
|
74
|
-
/** @internal */
|
|
75
|
-
_abort(reason) { this.abort.abort(reason); }
|
|
76
|
-
constructor(agentName, budgetUsd, maxIterations, maxDurationMs, maxTokens, context) {
|
|
77
|
-
this.agentName = agentName;
|
|
78
|
-
this.budgetUsd = budgetUsd;
|
|
79
|
-
this.maxIterations = maxIterations;
|
|
80
|
-
this.maxDurationMs = maxDurationMs;
|
|
81
|
-
this.maxTokens = maxTokens;
|
|
82
|
-
this.context = context;
|
|
83
|
-
}
|
|
84
|
-
get durationMs() { return Date.now() - this.startedAt; }
|
|
85
|
-
}
|
|
86
|
-
const store = new AsyncLocalStorage();
|
|
87
|
-
/** The run in progress in this async context, if any. */
|
|
88
|
-
export function getActiveRun() {
|
|
89
|
-
return store.getStore() ?? null;
|
|
90
|
-
}
|
|
13
|
+
import { breachResult, okResult } from "./results.js";
|
|
14
|
+
import { DEFAULTS, MAX_EVENTS_PER_RUN, Run, getActiveRun, store } from "./run.js";
|
|
15
|
+
export { Run, getActiveRun } from "./run.js";
|
|
91
16
|
let client = null;
|
|
92
17
|
let localOnly = false;
|
|
93
18
|
let failClosed = false;
|
|
@@ -130,11 +55,11 @@ function requireInit() {
|
|
|
130
55
|
throw new Error("Call agentfences.init() before using Fences. For local usage: init({ localOnly: true })");
|
|
131
56
|
}
|
|
132
57
|
}
|
|
133
|
-
function warnUnreachable(run, err) {
|
|
58
|
+
function warnUnreachable(run, err, ending = false) {
|
|
134
59
|
if (run.warned)
|
|
135
60
|
return;
|
|
136
61
|
run.warned = true;
|
|
137
|
-
console.warn(`agentfences: Fences backend unreachable for run ${run.runId} (${err}); enforcing limits locally`);
|
|
62
|
+
console.warn(`agentfences: Fences backend unreachable for run ${run.runId} (${err}); ${ending ? "ending the run" : "enforcing limits locally"}`);
|
|
138
63
|
}
|
|
139
64
|
// ── governed ──────────────────────────────────────────────────────────────────
|
|
140
65
|
/**
|
|
@@ -186,7 +111,6 @@ function outcome(run, failed, error) {
|
|
|
186
111
|
const e = run.exception;
|
|
187
112
|
return ["error", e.message ? `${e.type}: ${e.message}` : e.type];
|
|
188
113
|
}
|
|
189
|
-
const LIMIT_KEYS = { budget_usd: "budgetUsd", max_iterations: "maxIterations", max_duration_ms: "maxDurationMs", max_tokens: "maxTokens" };
|
|
190
114
|
async function startRun(agentName, limits) {
|
|
191
115
|
const run = new Run(agentName, limits.budgetUsd ?? null, limits.maxIterations ?? DEFAULTS.maxIterations, limits.maxDurationMs ?? DEFAULTS.maxDurationMs, limits.maxTokens ?? DEFAULTS.maxTokens, events.currentContext());
|
|
192
116
|
if (!client) {
|
|
@@ -206,11 +130,7 @@ async function startRun(agentName, limits) {
|
|
|
206
130
|
if (resp.quarantined)
|
|
207
131
|
throw new AgentQuarantined(agentName, resp.detail);
|
|
208
132
|
// The server caps limits at the agent's ceilings: enforce the same numbers if it becomes unreachable later
|
|
209
|
-
|
|
210
|
-
const v = resp.limits?.[wire];
|
|
211
|
-
if (typeof v === "number" || (wire === "budget_usd" && v === null))
|
|
212
|
-
run[field] = v; // eslint-disable-line @typescript-eslint/no-explicit-any
|
|
213
|
-
}
|
|
133
|
+
control.adopt(run, resp.limits ?? {}, control.LIMITS, true);
|
|
214
134
|
for (const n of resp.notices ?? []) {
|
|
215
135
|
events.warnOnce(`${agentName}:${n.code}`, `${n.message ?? ""}${n.url ? ` ${n.url}` : ""}`);
|
|
216
136
|
}
|
|
@@ -256,8 +176,10 @@ export async function checkpoint(response, extra = {}) {
|
|
|
256
176
|
throw new TypeError("checkpoint() takes the model's response first; pass a cost as checkpoint(null, { costDeltaUsd })");
|
|
257
177
|
}
|
|
258
178
|
const [cost, tokens] = response === undefined || response === null ? [0, 0] : costOf(response);
|
|
259
|
-
const stepCost = cost + (extra.costDeltaUsd ?? 0
|
|
260
|
-
|
|
179
|
+
const stepCost = cost + checkAmount(extra.costDeltaUsd ?? 0, "costDeltaUsd");
|
|
180
|
+
const stepTokens = tokens + checkAmount(extra.tokensUsed ?? 0, "tokensUsed");
|
|
181
|
+
run.costUsd += stepCost; // the actual cost replaces the holds of model calls that finished
|
|
182
|
+
precheck.releaseFinished(run);
|
|
261
183
|
run.tokensUsed += stepTokens;
|
|
262
184
|
run.iterations += 1;
|
|
263
185
|
const resp = client ? await client.checkpoint(run.runId, {
|
|
@@ -281,6 +203,14 @@ export async function checkpoint(response, extra = {}) {
|
|
|
281
203
|
}
|
|
282
204
|
return result;
|
|
283
205
|
}
|
|
206
|
+
/** @internal A cost or count must be a finite number, zero or more: a NaN or negative one would switch the budget off. */
|
|
207
|
+
export function checkAmount(value, what) {
|
|
208
|
+
if (typeof value !== "number" || Number.isNaN(value))
|
|
209
|
+
throw new TypeError(`${what} must be a number, not ${String(value)}`);
|
|
210
|
+
if (value < 0 || value === Infinity)
|
|
211
|
+
throw new RangeError(`${what} must be zero or more and finite, not ${value}`);
|
|
212
|
+
return value;
|
|
213
|
+
}
|
|
284
214
|
/** @internal The breach the run is in by local rules, if any. */
|
|
285
215
|
export function localBreach(run) {
|
|
286
216
|
// Limits trip only once exceeded, same rule as the server; round away float drift (0.02*5 != 0.10)
|
|
@@ -297,10 +227,11 @@ export function localBreach(run) {
|
|
|
297
227
|
function decide(run, resp) {
|
|
298
228
|
if (!resp || resp.network_error) {
|
|
299
229
|
if (resp) {
|
|
300
|
-
|
|
301
|
-
|
|
230
|
+
const ending = failClosed || !!resp.auth_rejected; // a key revoked mid-run: the run can't go on unchecked
|
|
231
|
+
warnUnreachable(run, resp.network_error, ending);
|
|
232
|
+
if (ending) {
|
|
302
233
|
run.lastBreach = "fences_unreachable";
|
|
303
|
-
return
|
|
234
|
+
return finalResult(run);
|
|
304
235
|
}
|
|
305
236
|
}
|
|
306
237
|
run.lastBreach = localBreach(run);
|
|
@@ -308,13 +239,8 @@ function decide(run, resp) {
|
|
|
308
239
|
else {
|
|
309
240
|
control.apply(run, resp.commands);
|
|
310
241
|
// The server is authoritative: adopt its totals and (on a breach) its current limits
|
|
311
|
-
run
|
|
312
|
-
run
|
|
313
|
-
run.tokensUsed = resp.tokens_used ?? run.tokensUsed;
|
|
314
|
-
for (const [wire, field] of Object.entries(LIMIT_KEYS)) {
|
|
315
|
-
if (wire in resp)
|
|
316
|
-
run[field] = resp[wire]; // eslint-disable-line @typescript-eslint/no-explicit-any
|
|
317
|
-
}
|
|
242
|
+
control.adopt(run, resp, control.TOTALS);
|
|
243
|
+
control.adopt(run, resp, control.LIMITS, true);
|
|
318
244
|
// A fresh breach, or a 409 because the run is still fenced from an earlier one
|
|
319
245
|
run.lastBreach = resp.ok ? null : (resp.breach ?? run.lastBreach ?? "limit_reached");
|
|
320
246
|
}
|
|
@@ -330,47 +256,13 @@ function finalResult(run) {
|
|
|
330
256
|
return run.lastBreach ? breachResult(run.lastBreach, run) : okResult(run.costUsd);
|
|
331
257
|
}
|
|
332
258
|
// ── Decision trail and spans ──────────────────────────────────────────────────
|
|
333
|
-
const SPAN_BATCH = 50;
|
|
334
|
-
const outbox = [];
|
|
335
|
-
let draining = null;
|
|
336
|
-
let spansSupported = true;
|
|
337
|
-
function enqueue(item) {
|
|
338
|
-
outbox.push(item);
|
|
339
|
-
draining ??= drain().finally(() => { draining = null; });
|
|
340
|
-
}
|
|
341
|
-
/** Decisions and spans go to the server in the background, in order; consecutive spans of a run go together. */
|
|
342
|
-
async function drain() {
|
|
343
|
-
while (outbox.length) {
|
|
344
|
-
const item = outbox.shift();
|
|
345
|
-
const c = client;
|
|
346
|
-
if (!c)
|
|
347
|
-
continue;
|
|
348
|
-
try {
|
|
349
|
-
if (item.kind === "decision") {
|
|
350
|
-
await c.logDecision(item.runId, item.body);
|
|
351
|
-
continue;
|
|
352
|
-
}
|
|
353
|
-
const batch = [item.span];
|
|
354
|
-
while (outbox[0]?.kind === "span" && outbox[0].runId === item.runId && batch.length < SPAN_BATCH) {
|
|
355
|
-
batch.push(outbox.shift().span);
|
|
356
|
-
}
|
|
357
|
-
if (spansSupported && (await c.logSpans(item.runId, batch)).unsupported) {
|
|
358
|
-
spansSupported = false;
|
|
359
|
-
console.warn("agentfences: this Fences server doesn't accept tool and model call spans yet; " +
|
|
360
|
-
"they're kept on the run locally but not sent");
|
|
361
|
-
}
|
|
362
|
-
}
|
|
363
|
-
catch { /* never let delivery break the agent */ }
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
259
|
/**
|
|
367
260
|
* Wait up to timeoutMs for queued decisions and spans to reach the server, and for onStop and
|
|
368
261
|
* onQuarantine hooks to finish. Call before a serverless handler returns.
|
|
369
262
|
*/
|
|
370
263
|
export async function flush(timeoutMs = 5000) {
|
|
371
264
|
const deadline = Date.now() + timeoutMs;
|
|
372
|
-
|
|
373
|
-
await Promise.race([draining, new Promise((r) => setTimeout(r, timeoutMs).unref())]);
|
|
265
|
+
await outbox.wait(timeoutMs);
|
|
374
266
|
await control.drain(Math.max(0, deadline - Date.now()));
|
|
375
267
|
}
|
|
376
268
|
/** Record the agent's reasoning at this step. Does nothing outside a governed run. */
|
|
@@ -383,7 +275,7 @@ export function logDecision(reasoning, action = null) {
|
|
|
383
275
|
return;
|
|
384
276
|
const sent = events.redact({ type: "decision", reasoning, action });
|
|
385
277
|
if (sent && typeof sent.reasoning === "string" && sent.reasoning) {
|
|
386
|
-
enqueue({ kind: "decision", runId: run.runId,
|
|
278
|
+
outbox.enqueue({ kind: "decision", client, runId: run.runId,
|
|
387
279
|
body: { iteration: run.iterations, reasoning: sent.reasoning, action: sent.action ?? null } });
|
|
388
280
|
}
|
|
389
281
|
}
|
|
@@ -412,11 +304,11 @@ export function recordSpan(name, elapsedMs, attributes, error) {
|
|
|
412
304
|
run.events.push(span);
|
|
413
305
|
if (run.events.length > MAX_EVENTS_PER_RUN)
|
|
414
306
|
run.events.shift(); // a runaway agent can't eat memory
|
|
415
|
-
if (!client || !spansSupported)
|
|
307
|
+
if (!client || !outbox.spansSupported)
|
|
416
308
|
return;
|
|
417
309
|
const sent = events.redact({ type: "span", ...span });
|
|
418
310
|
if (sent) {
|
|
419
311
|
const { type: _, ...redacted } = sent; // the type key is for the redact hook only
|
|
420
|
-
enqueue({ kind: "span", runId: run.runId, span: redacted });
|
|
312
|
+
outbox.enqueue({ kind: "span", client, runId: run.runId, span: redacted });
|
|
421
313
|
}
|
|
422
314
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -4,9 +4,12 @@ export { context } from "./events.js";
|
|
|
4
4
|
export type { RedactHook, Event } from "./events.js";
|
|
5
5
|
export { tool } from "./tools.js";
|
|
6
6
|
export type { GuardedToolOptions } from "./tools.js";
|
|
7
|
-
export { requestApproval } from "./
|
|
8
|
-
export type { Approval
|
|
7
|
+
export { requestApproval } from "./approvals.js";
|
|
8
|
+
export type { Approval } from "./approvals.js";
|
|
9
|
+
export type { ControlEvent, Hook } from "./control.js";
|
|
9
10
|
export { priceOf } from "./pricing.js";
|
|
10
11
|
export type { CustomPrice } from "./pricing.js";
|
|
11
12
|
export { FencesError, FencesAuthError, AgentQuarantined, FencesStop, ToolBlocked } from "./errors.js";
|
|
12
13
|
export { VERSION } from "./version.js";
|
|
14
|
+
export * as mcp from "./mcp.js";
|
|
15
|
+
export { instrument } from "./instrument.js";
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
export { init, governed, checkpoint, logDecision, flush, getActiveRun, Run } from "./core.js";
|
|
2
2
|
export { context } from "./events.js";
|
|
3
3
|
export { tool } from "./tools.js";
|
|
4
|
-
export { requestApproval } from "./
|
|
4
|
+
export { requestApproval } from "./approvals.js";
|
|
5
5
|
export { priceOf } from "./pricing.js";
|
|
6
6
|
export { FencesError, FencesAuthError, AgentQuarantined, FencesStop, ToolBlocked } from "./errors.js";
|
|
7
7
|
export { VERSION } from "./version.js";
|
|
8
|
+
export * as mcp from "./mcp.js";
|
|
9
|
+
export { instrument } from "./instrument.js";
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Patch the SDK behind this client so every model call inside a governed run is recorded and,
|
|
3
|
+
* with a budget, prechecked:
|
|
4
|
+
*
|
|
5
|
+
* import OpenAI from "openai";
|
|
6
|
+
* const openai = instrument(new OpenAI());
|
|
7
|
+
*
|
|
8
|
+
* Takes an OpenAI client (chat.completions and responses, including OpenAI-compatible clients)
|
|
9
|
+
* or an Anthropic one (messages). One call covers every client of that SDK; calling it again
|
|
10
|
+
* does nothing. Returns the client.
|
|
11
|
+
*/
|
|
12
|
+
export declare function instrument<C extends object>(client: C): C;
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Record every OpenAI and Anthropic model call made inside a governed run, with no other code
|
|
3
|
+
* changes: model, tokens, cost, latency, and the error if it failed. Calls don't count as
|
|
4
|
+
* checkpoints and don't add to the run's spend (checkpoint(response) does), but in a run with a
|
|
5
|
+
* budget each one is prechecked: refused with FencesStop if its worst case doesn't fit.
|
|
6
|
+
*/
|
|
7
|
+
import * as precheck from "./precheck.js";
|
|
8
|
+
import { getActiveRun, store } from "./run.js";
|
|
9
|
+
import { recordLlmCall } from "./spans.js";
|
|
10
|
+
const PATCHED = Symbol.for("agentfences.instrumented");
|
|
11
|
+
/**
|
|
12
|
+
* Patch the SDK behind this client so every model call inside a governed run is recorded and,
|
|
13
|
+
* with a budget, prechecked:
|
|
14
|
+
*
|
|
15
|
+
* import OpenAI from "openai";
|
|
16
|
+
* const openai = instrument(new OpenAI());
|
|
17
|
+
*
|
|
18
|
+
* Takes an OpenAI client (chat.completions and responses, including OpenAI-compatible clients)
|
|
19
|
+
* or an Anthropic one (messages). One call covers every client of that SDK; calling it again
|
|
20
|
+
* does nothing. Returns the client.
|
|
21
|
+
*/
|
|
22
|
+
export function instrument(client) {
|
|
23
|
+
const c = client;
|
|
24
|
+
const provider = c.chat ? "openai" : "anthropic";
|
|
25
|
+
const resources = [c.chat?.completions, c.responses, c.messages].filter((r) => typeof r?.create === "function");
|
|
26
|
+
if (!resources.length)
|
|
27
|
+
throw new TypeError("instrument() takes an OpenAI or Anthropic client");
|
|
28
|
+
for (const r of resources) {
|
|
29
|
+
const proto = Object.getPrototypeOf(r);
|
|
30
|
+
if (typeof proto?.create === "function" && !proto.create[PATCHED])
|
|
31
|
+
proto.create = wrap(proto.create, provider);
|
|
32
|
+
}
|
|
33
|
+
return client;
|
|
34
|
+
}
|
|
35
|
+
function wrap(create, provider) {
|
|
36
|
+
const wrapped = function (...args) {
|
|
37
|
+
const run = getActiveRun();
|
|
38
|
+
if (!run)
|
|
39
|
+
return create.apply(this, args);
|
|
40
|
+
const body = (args[0] ?? {});
|
|
41
|
+
const prepared = precheck.prepare(body.model, body, body);
|
|
42
|
+
if (!prepared)
|
|
43
|
+
return send(this, create, args, provider, run, null);
|
|
44
|
+
let hold;
|
|
45
|
+
try {
|
|
46
|
+
hold = precheck.holdNow(prepared); // FencesStop if the call might not fit the budget
|
|
47
|
+
}
|
|
48
|
+
catch (e) {
|
|
49
|
+
return refused(e);
|
|
50
|
+
}
|
|
51
|
+
if (hold !== "wait")
|
|
52
|
+
return send(this, create, args, provider, run, hold);
|
|
53
|
+
// onBreach "hold": wait for a raised budget, then send. Boxed, so nothing awaits the SDK's promise before the caller does
|
|
54
|
+
return later(precheck.holdAsync(prepared).then((h) => ({ out: send(this, create, args, provider, run, h) })));
|
|
55
|
+
};
|
|
56
|
+
wrapped[PATCHED] = true;
|
|
57
|
+
return wrapped;
|
|
58
|
+
}
|
|
59
|
+
/** A call sent once a hold is released: awaitable, with the SDK promise's methods, each forwarded once it's sent. */
|
|
60
|
+
function later(sent) {
|
|
61
|
+
const value = () => sent.then(({ out }) => out);
|
|
62
|
+
return {
|
|
63
|
+
then: (ok, ko) => value().then(ok, ko),
|
|
64
|
+
catch: (ko) => value().catch(ko),
|
|
65
|
+
finally: (fn) => value().finally(fn),
|
|
66
|
+
withResponse: () => sent.then(({ out }) => out.withResponse()),
|
|
67
|
+
asResponse: () => sent.then(({ out }) => out.asResponse()),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
/** A refused call rejects like the SDK's own promise would, never throwing synchronously, and keeps its methods. */
|
|
71
|
+
function refused(error) {
|
|
72
|
+
const p = Promise.reject(error);
|
|
73
|
+
p.withResponse = p.asResponse = () => p;
|
|
74
|
+
return p;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Make the call and return the SDK's own promise (an APIPromise, with .withResponse()) unchanged,
|
|
78
|
+
* except that awaiting it, or calling withResponse() or asResponse(), also records the call and
|
|
79
|
+
* settles its hold. Nothing reads the response before the caller does: after asResponse(), whose
|
|
80
|
+
* body the caller reads, the call is recorded without usage and its worst case stays held until
|
|
81
|
+
* the next checkpoint.
|
|
82
|
+
*/
|
|
83
|
+
function send(self, create, args, provider, run, hold) {
|
|
84
|
+
const body = (args[0] ?? {});
|
|
85
|
+
const started = performance.now();
|
|
86
|
+
let done = false;
|
|
87
|
+
const record = (response, error) => {
|
|
88
|
+
if (done)
|
|
89
|
+
return;
|
|
90
|
+
done = true;
|
|
91
|
+
store.run(run, () => {
|
|
92
|
+
if (hold)
|
|
93
|
+
precheck.finish(run, hold, response, error !== undefined);
|
|
94
|
+
recordLlmCall(provider, body.model, started, response, error, { stream: !!body.stream });
|
|
95
|
+
});
|
|
96
|
+
};
|
|
97
|
+
let out;
|
|
98
|
+
try {
|
|
99
|
+
out = create.apply(self, args);
|
|
100
|
+
}
|
|
101
|
+
catch (e) {
|
|
102
|
+
record(undefined, e);
|
|
103
|
+
throw e;
|
|
104
|
+
}
|
|
105
|
+
if (!out || typeof out.then !== "function") {
|
|
106
|
+
record(out);
|
|
107
|
+
return out;
|
|
108
|
+
}
|
|
109
|
+
const then = out.then.bind(out);
|
|
110
|
+
out.then = (ok, ko) => then((v) => { record(v); return ok ? ok(v) : v; }, (e) => { record(undefined, e); if (ko)
|
|
111
|
+
return ko(e); throw e; });
|
|
112
|
+
// APIPromise has its own catch and finally, which skip then: route them through the one above
|
|
113
|
+
out.catch = (ko) => Promise.prototype.catch.call(out, ko);
|
|
114
|
+
out.finally = (fn) => Promise.prototype.finally.call(out, fn);
|
|
115
|
+
if (typeof out.asResponse === "function") { // the caller reads the body: the call is recorded without usage
|
|
116
|
+
const asResponse = out.asResponse.bind(out);
|
|
117
|
+
out.asResponse = () => asResponse().then((r) => { record(); return r; }, (e) => { record(undefined, e); throw e; });
|
|
118
|
+
}
|
|
119
|
+
if (typeof out.withResponse === "function") {
|
|
120
|
+
const withResponse = out.withResponse.bind(out);
|
|
121
|
+
out.withResponse = () => withResponse().then((r) => { record(r?.data); return r; }, (e) => { record(undefined, e); throw e; });
|
|
122
|
+
}
|
|
123
|
+
return out;
|
|
124
|
+
}
|
package/dist/mcp.d.ts
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
type Args = Record<string, unknown>;
|
|
2
|
+
interface CallToolClient {
|
|
3
|
+
callTool(params: {
|
|
4
|
+
name: string;
|
|
5
|
+
arguments?: Args;
|
|
6
|
+
}, ...rest: any[]): Promise<any>;
|
|
7
|
+
}
|
|
8
|
+
export interface WrapOptions {
|
|
9
|
+
/** Tool names to check with Fences before each call, or true for every tool. */
|
|
10
|
+
guard?: boolean | string[];
|
|
11
|
+
/** Per tool, what a call costs the run: a number or a function of its arguments. Such a tool is guarded. */
|
|
12
|
+
costUsd?: Record<string, number | ((args: Args) => number)>;
|
|
13
|
+
/** Tool names a person approves from the dashboard first. Such a tool is guarded. */
|
|
14
|
+
approve?: string[];
|
|
15
|
+
}
|
|
16
|
+
/** An MCP tool answered with isError: recorded as the call's error, the result still goes back. */
|
|
17
|
+
export declare class MCPToolError extends Error {
|
|
18
|
+
readonly result: {
|
|
19
|
+
content?: {
|
|
20
|
+
text?: string;
|
|
21
|
+
}[];
|
|
22
|
+
};
|
|
23
|
+
constructor(result: {
|
|
24
|
+
content?: {
|
|
25
|
+
text?: string;
|
|
26
|
+
}[];
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Record every tool call an MCP client makes, and guard the ones you name:
|
|
31
|
+
*
|
|
32
|
+
* mcp.wrap(client, { guard: ["refund", "send_email"], costUsd: { refund: (a) => a.amount as number } });
|
|
33
|
+
*
|
|
34
|
+
* Patches client.callTool in place and returns the client. Each call inside a governed run
|
|
35
|
+
* becomes a tool span, with no tool() per tool. A result with isError counts as a failed call.
|
|
36
|
+
* Guarded tools go through the same check and settle as tool(fn, { guard: true }); a refused
|
|
37
|
+
* one throws ToolBlocked and never reaches the MCP server. Outside a governed run calls pass straight through.
|
|
38
|
+
*/
|
|
39
|
+
export declare function wrap<C extends CallToolClient>(client: C, options?: WrapOptions): C;
|
|
40
|
+
export {};
|
package/dist/mcp.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { getActiveRun } from "./run.js";
|
|
2
|
+
import { argsSummary, callAsync } from "./tools.js";
|
|
3
|
+
/** An MCP tool answered with isError: recorded as the call's error, the result still goes back. */
|
|
4
|
+
export class MCPToolError extends Error {
|
|
5
|
+
result;
|
|
6
|
+
constructor(result) {
|
|
7
|
+
super((result.content ?? []).map((c) => c.text ?? "").join(" ").trim() || "isError");
|
|
8
|
+
this.result = result;
|
|
9
|
+
this.name = "MCPToolError";
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Record every tool call an MCP client makes, and guard the ones you name:
|
|
14
|
+
*
|
|
15
|
+
* mcp.wrap(client, { guard: ["refund", "send_email"], costUsd: { refund: (a) => a.amount as number } });
|
|
16
|
+
*
|
|
17
|
+
* Patches client.callTool in place and returns the client. Each call inside a governed run
|
|
18
|
+
* becomes a tool span, with no tool() per tool. A result with isError counts as a failed call.
|
|
19
|
+
* Guarded tools go through the same check and settle as tool(fn, { guard: true }); a refused
|
|
20
|
+
* one throws ToolBlocked and never reaches the MCP server. Outside a governed run calls pass straight through.
|
|
21
|
+
*/
|
|
22
|
+
export function wrap(client, options = {}) {
|
|
23
|
+
const original = client.callTool.bind(client), costs = options.costUsd ?? {}, approve = new Set(options.approve);
|
|
24
|
+
const named = new Set(Array.isArray(options.guard) ? options.guard : []);
|
|
25
|
+
client.callTool = async (params, ...rest) => {
|
|
26
|
+
const run = getActiveRun();
|
|
27
|
+
if (!run)
|
|
28
|
+
return original(params, ...rest);
|
|
29
|
+
const { name } = params, args = params.arguments ?? {}, c = Object.hasOwn(costs, name) ? costs[name] : undefined;
|
|
30
|
+
const guarded = options.guard === true || named.has(name) || c !== undefined || approve.has(name);
|
|
31
|
+
try {
|
|
32
|
+
return await callAsync(run, name, argsSummary([args]), guarded ? () => (typeof c === "function" ? c(args) : c) ?? 0 : null, approve.has(name), async () => {
|
|
33
|
+
const result = await original(params, ...rest);
|
|
34
|
+
if (result?.isError)
|
|
35
|
+
throw new MCPToolError(result);
|
|
36
|
+
return result;
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
catch (e) {
|
|
40
|
+
if (e instanceof MCPToolError)
|
|
41
|
+
return e.result;
|
|
42
|
+
throw e;
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
return client;
|
|
46
|
+
}
|
package/dist/outbox.d.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decisions and spans go to the server in the background, in order, so logDecision() and
|
|
3
|
+
* recorded calls never wait on a network round trip. Consecutive spans of a run go together.
|
|
4
|
+
*/
|
|
5
|
+
import type { FencesClient } from "./client.js";
|
|
6
|
+
import type { Span } from "./run.js";
|
|
7
|
+
type Item = {
|
|
8
|
+
kind: "decision";
|
|
9
|
+
client: FencesClient;
|
|
10
|
+
runId: string;
|
|
11
|
+
body: Record<string, unknown>;
|
|
12
|
+
} | {
|
|
13
|
+
kind: "span";
|
|
14
|
+
client: FencesClient;
|
|
15
|
+
runId: string;
|
|
16
|
+
span: Span;
|
|
17
|
+
};
|
|
18
|
+
/** Off once a server says it has no spans endpoint. */
|
|
19
|
+
export declare let spansSupported: boolean;
|
|
20
|
+
export declare function enqueue(item: Item): void;
|
|
21
|
+
/** Wait up to timeoutMs for everything queued to be sent. */
|
|
22
|
+
export declare function wait(timeoutMs: number): Promise<void>;
|
|
23
|
+
export {};
|
package/dist/outbox.js
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
const SPAN_BATCH = 50;
|
|
2
|
+
const queue = [];
|
|
3
|
+
let draining = null;
|
|
4
|
+
/** Off once a server says it has no spans endpoint. */
|
|
5
|
+
export let spansSupported = true;
|
|
6
|
+
export function enqueue(item) {
|
|
7
|
+
queue.push(item);
|
|
8
|
+
draining ??= drain().finally(() => { draining = null; });
|
|
9
|
+
}
|
|
10
|
+
/** Wait up to timeoutMs for everything queued to be sent. */
|
|
11
|
+
export async function wait(timeoutMs) {
|
|
12
|
+
if (draining)
|
|
13
|
+
await Promise.race([draining, new Promise((r) => setTimeout(r, timeoutMs).unref())]);
|
|
14
|
+
}
|
|
15
|
+
async function drain() {
|
|
16
|
+
while (queue.length) {
|
|
17
|
+
const item = queue.shift();
|
|
18
|
+
try {
|
|
19
|
+
if (item.kind === "decision") {
|
|
20
|
+
await item.client.logDecision(item.runId, item.body);
|
|
21
|
+
continue;
|
|
22
|
+
}
|
|
23
|
+
const batch = [item.span];
|
|
24
|
+
while (queue[0]?.kind === "span" && queue[0].runId === item.runId && batch.length < SPAN_BATCH) {
|
|
25
|
+
batch.push(queue.shift().span);
|
|
26
|
+
}
|
|
27
|
+
if (spansSupported && (await item.client.logSpans(item.runId, batch)).unsupported) {
|
|
28
|
+
spansSupported = false;
|
|
29
|
+
console.warn("agentfences: this Fences server doesn't accept tool and model call spans yet; " +
|
|
30
|
+
"they're kept on the run locally but not sent");
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
catch { /* never let delivery break the agent */ }
|
|
34
|
+
}
|
|
35
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { type Hold, type Run } from "./run.js";
|
|
2
|
+
type Params = Record<string, unknown>;
|
|
3
|
+
/** The output-token cap a request sets, if any. */
|
|
4
|
+
export declare function maxOut(params: Params): number | null;
|
|
5
|
+
/**
|
|
6
|
+
* The most a call could cost: its input, estimated high from the request's size, at the dearest
|
|
7
|
+
* input rate, plus its output cap (or the model's) at the output rate. null for a model with no
|
|
8
|
+
* known price, which can't be prechecked.
|
|
9
|
+
*/
|
|
10
|
+
export declare function worstCaseUsd(model: unknown, request: unknown, outCap?: number | null, n?: number): number | null;
|
|
11
|
+
export interface Prepared {
|
|
12
|
+
run: Run;
|
|
13
|
+
worst: number;
|
|
14
|
+
outCap: number | null;
|
|
15
|
+
}
|
|
16
|
+
/** The call to check, or null when there's nothing to check (no run, no budget, a model with no known price). */
|
|
17
|
+
export declare function prepare(model: unknown, request: unknown, params?: Params): Prepared | null;
|
|
18
|
+
/**
|
|
19
|
+
* Before a model call, without waiting: hold its worst case, or throw FencesStop if it doesn't fit.
|
|
20
|
+
* Returns "wait" when it doesn't fit but the run waits at a limit (onBreach "hold"): then use holdAsync.
|
|
21
|
+
*/
|
|
22
|
+
export declare function holdNow(p: Prepared): Hold | "wait";
|
|
23
|
+
/** The same, waiting (onBreach "hold") for someone to raise the budget while the call doesn't fit. */
|
|
24
|
+
export declare function holdAsync(p: Prepared): Promise<Hold>;
|
|
25
|
+
/**
|
|
26
|
+
* After the call. A failed call isn't billed, so its hold is released now. Otherwise the hold
|
|
27
|
+
* shrinks to the response's actual cost (a stream without usage keeps its worst case), and the
|
|
28
|
+
* next checkpoint, which records that cost, releases it.
|
|
29
|
+
*/
|
|
30
|
+
export declare function finish(run: Run, h: Hold, response?: unknown, failed?: boolean): void;
|
|
31
|
+
/** At a checkpoint, which records the actual cost: release the holds of calls that have finished. */
|
|
32
|
+
export declare function releaseFinished(run: Run): void;
|
|
33
|
+
export {};
|