@zanii/blackbox 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +109 -2
- package/dist/agents/index.d.ts +34 -0
- package/dist/agents/index.js +73 -0
- package/dist/analysis/detectors.d.ts +36 -0
- package/dist/analysis/detectors.js +339 -0
- package/dist/analysis/faults.d.ts +9 -0
- package/dist/analysis/faults.js +210 -0
- package/dist/analysis/index.d.ts +68 -0
- package/dist/analysis/index.js +388 -0
- package/dist/analysis/landing.d.ts +25 -0
- package/dist/analysis/landing.js +225 -0
- package/dist/analysis/memory.d.ts +13 -0
- package/dist/analysis/memory.js +33 -0
- package/dist/analysis/waste.d.ts +29 -0
- package/dist/analysis/waste.js +79 -0
- package/dist/approvals/index.d.ts +11 -0
- package/dist/approvals/index.js +27 -0
- package/dist/approvals/warnings.d.ts +2 -0
- package/dist/approvals/warnings.js +28 -0
- package/dist/attest/index.d.ts +17 -0
- package/dist/attest/index.js +106 -0
- package/dist/authority/index.d.ts +24 -0
- package/dist/authority/index.js +77 -0
- package/dist/billing/index.d.ts +99 -0
- package/dist/billing/index.js +174 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +1019 -0
- package/dist/client/index.d.ts +146 -0
- package/dist/client/index.js +210 -0
- package/dist/compliance/index.d.ts +41 -0
- package/dist/compliance/index.js +96 -0
- package/dist/cost/index.d.ts +133 -0
- package/dist/cost/index.js +293 -0
- package/dist/directives/index.d.ts +35 -0
- package/dist/directives/index.js +80 -0
- package/dist/drills/index.d.ts +43 -0
- package/dist/drills/index.js +101 -0
- package/dist/duty/index.d.ts +21 -0
- package/dist/duty/index.js +68 -0
- package/dist/fleet/index.d.ts +141 -0
- package/dist/fleet/index.js +454 -0
- package/dist/hooks/ai-sdk.d.ts +42 -0
- package/dist/hooks/ai-sdk.js +62 -0
- package/dist/hooks/claude-agent-sdk.d.ts +14 -0
- package/dist/hooks/claude-agent-sdk.js +70 -0
- package/dist/hooks/index.d.ts +7 -0
- package/dist/hooks/index.js +10 -0
- package/dist/hooks/langchain-agent.d.ts +69 -0
- package/dist/hooks/langchain-agent.js +163 -0
- package/dist/hooks/langchain.d.ts +41 -0
- package/dist/hooks/langchain.js +216 -0
- package/dist/hooks/langgraph-checkpoint.d.ts +12 -0
- package/dist/hooks/langgraph-checkpoint.js +73 -0
- package/dist/hooks/memory.d.ts +17 -0
- package/dist/hooks/memory.js +64 -0
- package/dist/hooks/openai-agents.d.ts +6 -0
- package/dist/hooks/openai-agents.js +40 -0
- package/dist/hooks/protect.d.ts +7 -0
- package/dist/hooks/protect.js +39 -0
- package/dist/hooks/providers.d.ts +16 -0
- package/dist/hooks/providers.js +149 -0
- package/dist/hooks/shared.d.ts +11 -0
- package/dist/hooks/shared.js +39 -0
- package/dist/index.d.ts +45 -0
- package/dist/index.js +47 -0
- package/dist/investigate/index.d.ts +66 -0
- package/dist/investigate/index.js +119 -0
- package/dist/mcp-server/index.d.ts +85 -0
- package/dist/mcp-server/index.js +216 -0
- package/dist/mcp-wrap/index.d.ts +17 -0
- package/dist/mcp-wrap/index.js +170 -0
- package/dist/money/index.d.ts +114 -0
- package/dist/money/index.js +622 -0
- package/dist/occurrence/index.d.ts +108 -0
- package/dist/occurrence/index.js +168 -0
- package/dist/ocsf/index.d.ts +22 -0
- package/dist/ocsf/index.js +168 -0
- package/dist/otlp/index.d.ts +24 -0
- package/dist/otlp/index.js +143 -0
- package/dist/packs/index.d.ts +40 -0
- package/dist/packs/index.js +217 -0
- package/dist/policy/delta.d.ts +11 -0
- package/dist/policy/delta.js +39 -0
- package/dist/policy/drafts.d.ts +34 -0
- package/dist/policy/drafts.js +129 -0
- package/dist/policy/index.d.ts +47 -0
- package/dist/policy/index.js +154 -0
- package/dist/precog/index.d.ts +96 -0
- package/dist/precog/index.js +167 -0
- package/dist/precog/intervention.d.ts +22 -0
- package/dist/precog/intervention.js +44 -0
- package/dist/precog/normal.d.ts +31 -0
- package/dist/precog/normal.js +89 -0
- package/dist/preflight/index.d.ts +11 -0
- package/dist/preflight/index.js +19 -0
- package/dist/ratings/index.d.ts +21 -0
- package/dist/ratings/index.js +48 -0
- package/dist/reconcile/claude-code.d.ts +19 -0
- package/dist/reconcile/claude-code.js +220 -0
- package/dist/reconcile/codex.d.ts +5 -0
- package/dist/reconcile/codex.js +191 -0
- package/dist/reconcile/index.d.ts +19 -0
- package/dist/reconcile/index.js +50 -0
- package/dist/reconcile/record.d.ts +49 -0
- package/dist/reconcile/record.js +225 -0
- package/dist/reconcile/shared.d.ts +65 -0
- package/dist/reconcile/shared.js +113 -0
- package/dist/replay/index.d.ts +11 -0
- package/dist/replay/index.js +64 -0
- package/dist/replay/repair.d.ts +10 -0
- package/dist/replay/repair.js +62 -0
- package/dist/session/drain.d.ts +13 -0
- package/dist/session/drain.js +35 -0
- package/dist/session/index.d.ts +256 -0
- package/dist/session/index.js +658 -0
- package/dist/undo/index.d.ts +45 -0
- package/dist/undo/index.js +212 -0
- package/dist/verify/anchor.d.ts +54 -0
- package/dist/verify/anchor.js +77 -0
- package/dist/verify/chain.d.ts +27 -0
- package/dist/verify/chain.js +105 -0
- package/dist/verify/envelope.d.ts +28 -0
- package/dist/verify/envelope.js +55 -0
- package/dist/verify/index.d.ts +3 -0
- package/dist/verify/index.js +3 -0
- package/dist/version.d.ts +1 -0
- package/dist/version.js +2 -0
- package/dist/weather/index.d.ts +24 -0
- package/dist/weather/index.js +45 -0
- package/dist/workspace-receipt/index.d.ts +15 -0
- package/dist/workspace-receipt/index.js +121 -0
- package/package.json +56 -3
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
// The cost engine (spec/cost.md): integer micro-USD from a versioned price table, AED display.
|
|
2
|
+
// Mirrors sdks/python/src/zanii_blackbox/cost.py.
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
const CLASSES = ["input", "output", "cache_read", "cache_write"];
|
|
5
|
+
const EXTRA = ["cache_write_1h", "input_audio", "output_audio"];
|
|
6
|
+
const OPTIONAL_PRICES = [
|
|
7
|
+
"context_window",
|
|
8
|
+
"cache_write_1h",
|
|
9
|
+
"input_audio",
|
|
10
|
+
"output_audio",
|
|
11
|
+
"web_search_per_1k",
|
|
12
|
+
];
|
|
13
|
+
/** Parses and checks a price table; its sha256 is of the exact file bytes. */
|
|
14
|
+
export function loadPrices(bytes) {
|
|
15
|
+
const table = JSON.parse(new TextDecoder().decode(bytes));
|
|
16
|
+
if (table.currency !== "USD" ||
|
|
17
|
+
table.unit !== "micro_usd_per_mtok" ||
|
|
18
|
+
typeof table.models !== "object")
|
|
19
|
+
throw new Error("not a v1 price table (USD, micro_usd_per_mtok)");
|
|
20
|
+
const whole = (model, field, v) => {
|
|
21
|
+
if (!Number.isSafeInteger(v) || v < 0)
|
|
22
|
+
throw new Error(`price table: ${model}.${field} must be an integer ≥ 0`);
|
|
23
|
+
};
|
|
24
|
+
for (const [model, p] of Object.entries(table.models)) {
|
|
25
|
+
for (const c of CLASSES)
|
|
26
|
+
whole(model, c, p[c]);
|
|
27
|
+
for (const c of OPTIONAL_PRICES)
|
|
28
|
+
if (p[c] !== undefined)
|
|
29
|
+
whole(model, c, p[c]);
|
|
30
|
+
for (const [i, t] of (p.context_tiers ?? []).entries()) {
|
|
31
|
+
whole(model, `context_tiers[${i}].above`, t.above);
|
|
32
|
+
for (const c of CLASSES)
|
|
33
|
+
if (t[c] !== undefined)
|
|
34
|
+
whole(model, `context_tiers[${i}].${c}`, t[c]);
|
|
35
|
+
}
|
|
36
|
+
for (const [name, t] of Object.entries(p.service_tiers ?? {}))
|
|
37
|
+
for (const c of CLASSES)
|
|
38
|
+
if (t[c] !== undefined)
|
|
39
|
+
whole(model, `service_tiers.${name}.${c}`, t[c]);
|
|
40
|
+
if (p.status !== undefined && p.status !== "deprecated" && p.status !== "beta")
|
|
41
|
+
throw new Error(`price table: ${model}.status must be deprecated or beta`);
|
|
42
|
+
}
|
|
43
|
+
return { table, sha256: `sha256:${createHash("sha256").update(bytes).digest("hex")}` };
|
|
44
|
+
}
|
|
45
|
+
/** The exact model, else the longest table key the model starts with; null when there's none. */
|
|
46
|
+
export function priceFor(model, table,
|
|
47
|
+
/** The configured provider id: `<provider>/<model>` keys are tried first (spec/cost.md §1). */
|
|
48
|
+
provider) {
|
|
49
|
+
if (provider) {
|
|
50
|
+
const scoped = lookup(`${provider}/${model}`, table);
|
|
51
|
+
if (scoped)
|
|
52
|
+
return scoped;
|
|
53
|
+
}
|
|
54
|
+
const found = lookup(model, table);
|
|
55
|
+
if (found)
|
|
56
|
+
return found;
|
|
57
|
+
// spec/cost.md §1: a cloud's model id (Bedrock's `us.anthropic.claude-…`) as the vendor's own
|
|
58
|
+
const plain = model.replace(GEO, "").replace(VENDOR, "");
|
|
59
|
+
return plain !== model ? lookup(plain, table) : null;
|
|
60
|
+
}
|
|
61
|
+
const GEO = /^(?:us|eu|apac|au|ca|jp|global|us-gov)\./;
|
|
62
|
+
const VENDOR = /^(?:anthropic|amazon|meta|mistral|cohere|ai21|deepseek|openai|qwen|writer|google)\./;
|
|
63
|
+
function lookup(model, table) {
|
|
64
|
+
const exact = table.models[model];
|
|
65
|
+
if (exact)
|
|
66
|
+
return { key: model, price: exact };
|
|
67
|
+
let best = null;
|
|
68
|
+
for (const key of Object.keys(table.models))
|
|
69
|
+
if (model.startsWith(key) && (best === null || key.length > best.length))
|
|
70
|
+
best = key;
|
|
71
|
+
return best === null ? null : { key: best, price: table.models[best] };
|
|
72
|
+
}
|
|
73
|
+
/** Token classes from a recorded, flattened `meta.usage` (spec/cost.md §2). */
|
|
74
|
+
export function tokensOf(usage) {
|
|
75
|
+
const n = (k) => {
|
|
76
|
+
const v = usage[k];
|
|
77
|
+
return typeof v === "number" && Number.isSafeInteger(v) && v > 0 ? v : 0;
|
|
78
|
+
};
|
|
79
|
+
if ("promptTokenCount" in usage) {
|
|
80
|
+
// Gemini (spec/providers.md §5): the prompt count includes the cached part; thoughts are output.
|
|
81
|
+
const cached = n("cachedContentTokenCount");
|
|
82
|
+
const audioIn = n("promptTokensDetails_AUDIO");
|
|
83
|
+
const audioOut = n("candidatesTokensDetails_AUDIO");
|
|
84
|
+
return extras({
|
|
85
|
+
input: Math.max(0, n("promptTokenCount") - cached - audioIn) + n("toolUsePromptTokenCount"),
|
|
86
|
+
output: Math.max(0, n("candidatesTokenCount") - audioOut) + n("thoughtsTokenCount"),
|
|
87
|
+
cache_read: cached,
|
|
88
|
+
cache_write: 0,
|
|
89
|
+
}, { input_audio: audioIn, output_audio: audioOut });
|
|
90
|
+
}
|
|
91
|
+
if ("inputTokens" in usage)
|
|
92
|
+
// Bedrock Converse (spec/providers.md §5)
|
|
93
|
+
return {
|
|
94
|
+
input: n("inputTokens"),
|
|
95
|
+
output: n("outputTokens"),
|
|
96
|
+
cache_read: n("cacheReadInputTokens"),
|
|
97
|
+
cache_write: n("cacheWriteInputTokens"),
|
|
98
|
+
};
|
|
99
|
+
if ("prompt_tokens" in usage) {
|
|
100
|
+
// prompt_tokens includes both the cached part and the part written to the cache (B1)
|
|
101
|
+
const cached = n("prompt_tokens_details_cached_tokens");
|
|
102
|
+
const written = n("prompt_tokens_details_cache_write_tokens");
|
|
103
|
+
const audioIn = n("prompt_tokens_details_audio_tokens");
|
|
104
|
+
const audioOut = n("completion_tokens_details_audio_tokens");
|
|
105
|
+
return extras({
|
|
106
|
+
input: Math.max(0, n("prompt_tokens") - cached - written - audioIn),
|
|
107
|
+
output: Math.max(0, n("completion_tokens") - audioOut),
|
|
108
|
+
cache_read: cached,
|
|
109
|
+
cache_write: written,
|
|
110
|
+
}, { input_audio: audioIn, output_audio: audioOut });
|
|
111
|
+
}
|
|
112
|
+
if (Object.keys(usage).some((k) => k.startsWith("input_tokens_details_") || k.startsWith("output_tokens_details_"))) {
|
|
113
|
+
const cached = n("input_tokens_details_cached_tokens");
|
|
114
|
+
const written = n("input_tokens_details_cache_write_tokens");
|
|
115
|
+
const audioIn = n("input_tokens_details_audio_tokens");
|
|
116
|
+
const audioOut = n("output_tokens_details_audio_tokens");
|
|
117
|
+
return extras({
|
|
118
|
+
input: Math.max(0, n("input_tokens") - cached - written - audioIn),
|
|
119
|
+
output: Math.max(0, n("output_tokens") - audioOut),
|
|
120
|
+
cache_read: cached,
|
|
121
|
+
cache_write: written,
|
|
122
|
+
}, { input_audio: audioIn, output_audio: audioOut });
|
|
123
|
+
}
|
|
124
|
+
// Anthropic: cache_creation_input_tokens is both lifetimes; the 1-hour part is priced apart
|
|
125
|
+
const hour = n("cache_creation_ephemeral_1h_input_tokens");
|
|
126
|
+
return extras({
|
|
127
|
+
input: n("input_tokens"),
|
|
128
|
+
output: n("output_tokens"),
|
|
129
|
+
cache_read: n("cache_read_input_tokens"),
|
|
130
|
+
cache_write: Math.max(0, n("cache_creation_input_tokens") - hour),
|
|
131
|
+
}, { cache_write_1h: hour });
|
|
132
|
+
}
|
|
133
|
+
/** Adds the optional classes that are non-zero, so a record without them prices as before. */
|
|
134
|
+
function extras(base, more) {
|
|
135
|
+
const out = { ...base };
|
|
136
|
+
for (const c of EXTRA)
|
|
137
|
+
if ((more[c] ?? 0) > 0)
|
|
138
|
+
out[c] = more[c];
|
|
139
|
+
return out;
|
|
140
|
+
}
|
|
141
|
+
/** Requests a call made to a provider's server tools (Anthropic web search). */
|
|
142
|
+
export function requestsOf(usage) {
|
|
143
|
+
const v = usage.server_tool_use_web_search_requests;
|
|
144
|
+
return typeof v === "number" && Number.isSafeInteger(v) && v > 0 ? { web_search: v } : undefined;
|
|
145
|
+
}
|
|
146
|
+
/** A call's prompt, in tokens: what a context tier is decided on. */
|
|
147
|
+
export function promptTokens(t) {
|
|
148
|
+
return t.input + t.cache_read + t.cache_write + (t.cache_write_1h ?? 0) + (t.input_audio ?? 0);
|
|
149
|
+
}
|
|
150
|
+
/** The prices a call pays: a context tier when its prompt is over one, then its service tier. */
|
|
151
|
+
export function effectivePrice(price, tokens, serviceTier) {
|
|
152
|
+
let out = price;
|
|
153
|
+
const tier = {};
|
|
154
|
+
const size = promptTokens(tokens);
|
|
155
|
+
const context = [...(price.context_tiers ?? [])]
|
|
156
|
+
.sort((a, b) => b.above - a.above)
|
|
157
|
+
.find((t) => size > t.above);
|
|
158
|
+
if (context) {
|
|
159
|
+
out = { ...out, ...pick(context) };
|
|
160
|
+
tier.context_above = context.above;
|
|
161
|
+
}
|
|
162
|
+
const service = serviceTier ? price.service_tiers?.[serviceTier] : undefined;
|
|
163
|
+
if (service && serviceTier) {
|
|
164
|
+
out = { ...out, ...pick(service) };
|
|
165
|
+
tier.service = serviceTier;
|
|
166
|
+
}
|
|
167
|
+
return Object.keys(tier).length ? { price: out, tier } : { price: out };
|
|
168
|
+
}
|
|
169
|
+
function pick(t) {
|
|
170
|
+
const o = {};
|
|
171
|
+
for (const c of CLASSES)
|
|
172
|
+
if (t[c] !== undefined)
|
|
173
|
+
o[c] = t[c];
|
|
174
|
+
return o;
|
|
175
|
+
}
|
|
176
|
+
/** round_half_up((tokens × price + requests × price per 1k × 1000) / 1e6), integers only. */
|
|
177
|
+
export function callMicroUsd(tokens, price, requests) {
|
|
178
|
+
let sum = 0;
|
|
179
|
+
for (const c of CLASSES)
|
|
180
|
+
sum += tokens[c] * price[c];
|
|
181
|
+
sum += (tokens.cache_write_1h ?? 0) * (price.cache_write_1h ?? price.cache_write);
|
|
182
|
+
sum += (tokens.input_audio ?? 0) * (price.input_audio ?? price.input);
|
|
183
|
+
sum += (tokens.output_audio ?? 0) * (price.output_audio ?? price.output);
|
|
184
|
+
sum += (requests?.web_search ?? 0) * (price.web_search_per_1k ?? 0) * 1000;
|
|
185
|
+
if (!Number.isSafeInteger(sum))
|
|
186
|
+
throw new RangeError("cost overflow: tokens × price beyond 2^53");
|
|
187
|
+
return Math.floor((sum + 500_000) / 1_000_000);
|
|
188
|
+
}
|
|
189
|
+
/** One call's cost as the cost report prices it: the table (provider-scoped, tiered), else the
|
|
190
|
+
* provider's own reported cost, else null. The spend and duty checks use the same. */
|
|
191
|
+
export function priceCall(call, table) {
|
|
192
|
+
const tokens = tokensOf(call.usage);
|
|
193
|
+
const requests = requestsOf(call.usage);
|
|
194
|
+
const match = call.model === null ? null : priceFor(call.model, table, call.provider);
|
|
195
|
+
if (match) {
|
|
196
|
+
const { price, tier } = effectivePrice(match.price, tokens, call.serviceTier);
|
|
197
|
+
return {
|
|
198
|
+
key: match.key,
|
|
199
|
+
tokens,
|
|
200
|
+
micro_usd: callMicroUsd(tokens, price, requests),
|
|
201
|
+
...(requests ? { requests } : {}),
|
|
202
|
+
...(tier ? { tier } : {}),
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
if (typeof call.reportedMicroUsd === "number")
|
|
206
|
+
return {
|
|
207
|
+
key: null,
|
|
208
|
+
tokens,
|
|
209
|
+
micro_usd: call.reportedMicroUsd,
|
|
210
|
+
...(requests ? { requests } : {}),
|
|
211
|
+
source: "reported",
|
|
212
|
+
};
|
|
213
|
+
return { key: null, tokens, micro_usd: null, ...(requests ? { requests } : {}) };
|
|
214
|
+
}
|
|
215
|
+
/** fils at the CBUAE peg (3.6725 AED / USD), rounded half up. */
|
|
216
|
+
export function toAedFils(microUsd) {
|
|
217
|
+
return Math.floor((microUsd * 36725 + 50_000_000) / 100_000_000);
|
|
218
|
+
}
|
|
219
|
+
export function formatAed(fils) {
|
|
220
|
+
const sign = fils < 0 ? "-" : "";
|
|
221
|
+
const abs = Math.abs(fils);
|
|
222
|
+
return `${sign}AED ${Math.floor(abs / 100)}.${String(abs % 100).padStart(2, "0")}`;
|
|
223
|
+
}
|
|
224
|
+
/** The session's cost report from its recorded lines. */
|
|
225
|
+
export function sessionCost(lines, prices) {
|
|
226
|
+
const calls = [];
|
|
227
|
+
const unknown = new Set();
|
|
228
|
+
const total = { input: 0, output: 0, cache_read: 0, cache_write: 0 };
|
|
229
|
+
let micro = 0;
|
|
230
|
+
const replayed = new Set(); // requests served from a cassette: no provider was paid (spec/cost.md)
|
|
231
|
+
const providers = new Map(); // request seq → the configured provider id
|
|
232
|
+
let webSearch = 0;
|
|
233
|
+
let pinnedSha; // session.open's meta.prices_sha256 (L3.1.5)
|
|
234
|
+
for (const line of lines) {
|
|
235
|
+
const e = JSON.parse(line);
|
|
236
|
+
if (e.kind === "session.open")
|
|
237
|
+
pinnedSha = e.meta.prices_sha256;
|
|
238
|
+
if (e.kind === "llm.request") {
|
|
239
|
+
const r = e.meta.replay;
|
|
240
|
+
if (Number.isSafeInteger(r?.take))
|
|
241
|
+
replayed.add(e.seq);
|
|
242
|
+
if (typeof e.meta.provider === "string")
|
|
243
|
+
providers.set(e.seq, e.meta.provider);
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
246
|
+
if (e.kind !== "llm.response" && e.kind !== "llm.incomplete")
|
|
247
|
+
continue;
|
|
248
|
+
if (replayed.has(e.meta.request_seq))
|
|
249
|
+
continue;
|
|
250
|
+
const usage = e.meta.usage;
|
|
251
|
+
if (typeof usage !== "object" || usage === null)
|
|
252
|
+
continue;
|
|
253
|
+
const model = typeof e.meta.model === "string" ? e.meta.model : null;
|
|
254
|
+
const priced = priceCall({
|
|
255
|
+
model,
|
|
256
|
+
usage: usage,
|
|
257
|
+
provider: providers.get(e.meta.request_seq) ?? null,
|
|
258
|
+
serviceTier: typeof e.meta.service_tier === "string" ? e.meta.service_tier : null,
|
|
259
|
+
reportedMicroUsd: Number.isSafeInteger(e.meta.reported_cost_micro_usd)
|
|
260
|
+
? e.meta.reported_cost_micro_usd
|
|
261
|
+
: null,
|
|
262
|
+
}, prices.table);
|
|
263
|
+
const { key: _key, ...call } = priced;
|
|
264
|
+
for (const c of CLASSES)
|
|
265
|
+
total[c] += call.tokens[c];
|
|
266
|
+
for (const c of EXTRA)
|
|
267
|
+
if (call.tokens[c] !== undefined)
|
|
268
|
+
total[c] = (total[c] ?? 0) + call.tokens[c];
|
|
269
|
+
webSearch += call.requests?.web_search ?? 0;
|
|
270
|
+
if (call.micro_usd === null)
|
|
271
|
+
unknown.add(model ?? "(no model)");
|
|
272
|
+
else
|
|
273
|
+
micro += call.micro_usd;
|
|
274
|
+
calls.push({ seq: e.seq, model, ...call });
|
|
275
|
+
}
|
|
276
|
+
return {
|
|
277
|
+
v: 1,
|
|
278
|
+
price_table: {
|
|
279
|
+
version: prices.table.version,
|
|
280
|
+
sha256: prices.sha256,
|
|
281
|
+
...(typeof pinnedSha === "string" ? { pinned: pinnedSha === prices.sha256 } : {}),
|
|
282
|
+
},
|
|
283
|
+
total_micro_usd: micro,
|
|
284
|
+
total_aed_fils: toAedFils(micro),
|
|
285
|
+
calls_priced: calls.filter((c) => c.micro_usd !== null).length,
|
|
286
|
+
calls_unpriced: calls.filter((c) => c.micro_usd === null).length,
|
|
287
|
+
unknown_models: [...unknown].sort(),
|
|
288
|
+
tokens: total,
|
|
289
|
+
calls,
|
|
290
|
+
...(replayed.size ? { calls_replayed: replayed.size } : {}),
|
|
291
|
+
...(webSearch ? { requests: { web_search: webSearch } } : {}),
|
|
292
|
+
};
|
|
293
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { type Summary } from "../fleet/index.ts";
|
|
2
|
+
export interface DirectiveItem {
|
|
3
|
+
tenant: string;
|
|
4
|
+
summary: Summary;
|
|
5
|
+
/** The models the session's calls used, sorted. */
|
|
6
|
+
models: string[];
|
|
7
|
+
}
|
|
8
|
+
export interface Directive {
|
|
9
|
+
id: string;
|
|
10
|
+
applies_to: {
|
|
11
|
+
model: string;
|
|
12
|
+
} | {
|
|
13
|
+
tool: string;
|
|
14
|
+
};
|
|
15
|
+
pattern: string[];
|
|
16
|
+
signature: string;
|
|
17
|
+
sessions: number;
|
|
18
|
+
type_sessions: number;
|
|
19
|
+
affected_ppm: number;
|
|
20
|
+
failure_ppm: number;
|
|
21
|
+
tenants: number;
|
|
22
|
+
first_seen: string;
|
|
23
|
+
}
|
|
24
|
+
export interface DirectiveOptions {
|
|
25
|
+
k?: number;
|
|
26
|
+
minSessions?: number;
|
|
27
|
+
minAffectedPpm?: number;
|
|
28
|
+
}
|
|
29
|
+
/** spec/directives.md §1. */
|
|
30
|
+
export declare function directives(items: readonly DirectiveItem[], { k, minSessions, minAffectedPpm }?: DirectiveOptions): Directive[];
|
|
31
|
+
/** spec/directives.md §2: the directives for the types one fleet flies, with its own sessions. */
|
|
32
|
+
export declare function fleetDirectives(all: readonly Directive[], mine: readonly DirectiveItem[]): Array<Directive & {
|
|
33
|
+
your_sessions: string[];
|
|
34
|
+
your_type_sessions: number;
|
|
35
|
+
}>;
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// Airworthiness directives (spec/directives.md): one failure pattern, on one model or tool, across
|
|
2
|
+
// fleets. Pure; mirrors sdks/python/src/zanii_blackbox/directives.py; pinned by
|
|
3
|
+
// spec/vectors/directives.json.
|
|
4
|
+
import { createHash } from "node:crypto";
|
|
5
|
+
import { signatureOf } from "../fleet/index.js";
|
|
6
|
+
const FAILED = new Set(["failure", "abandoned"]);
|
|
7
|
+
const typesOf = (item) => [
|
|
8
|
+
...[...new Set(item.models)].sort().map((m) => `model:${m}`),
|
|
9
|
+
...Object.keys(item.summary.tools)
|
|
10
|
+
.sort()
|
|
11
|
+
.map((t) => `tool:${t}`),
|
|
12
|
+
];
|
|
13
|
+
/** spec/directives.md §1. */
|
|
14
|
+
export function directives(items, { k = 3, minSessions = 5, minAffectedPpm = 50_000 } = {}) {
|
|
15
|
+
const typeSessions = new Map();
|
|
16
|
+
const groups = new Map();
|
|
17
|
+
for (const item of items) {
|
|
18
|
+
const sig = signatureOf(item.summary);
|
|
19
|
+
for (const type of typesOf(item)) {
|
|
20
|
+
typeSessions.set(type, (typeSessions.get(type) ?? 0) + 1);
|
|
21
|
+
if (!sig)
|
|
22
|
+
continue;
|
|
23
|
+
const key = `${type}|${sig.pattern.join(",")}`;
|
|
24
|
+
const g = groups.get(key) ?? {
|
|
25
|
+
type,
|
|
26
|
+
pattern: sig.pattern,
|
|
27
|
+
signature: sig.signature,
|
|
28
|
+
tenants: new Set(),
|
|
29
|
+
sessions: 0,
|
|
30
|
+
failed: 0,
|
|
31
|
+
first: item.summary.started_at,
|
|
32
|
+
};
|
|
33
|
+
g.tenants.add(item.tenant);
|
|
34
|
+
g.sessions++;
|
|
35
|
+
if (FAILED.has(item.summary.outcome ?? ""))
|
|
36
|
+
g.failed++;
|
|
37
|
+
if (item.summary.started_at < g.first)
|
|
38
|
+
g.first = item.summary.started_at;
|
|
39
|
+
groups.set(key, g);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
const out = [];
|
|
43
|
+
for (const [key, g] of groups) {
|
|
44
|
+
const total = typeSessions.get(g.type) ?? 0;
|
|
45
|
+
const affected = Math.floor((g.sessions * 1_000_000) / total);
|
|
46
|
+
if (g.tenants.size < k || g.sessions < minSessions || affected < minAffectedPpm)
|
|
47
|
+
continue;
|
|
48
|
+
const [kind, ...rest] = g.type.split(":");
|
|
49
|
+
const name = rest.join(":");
|
|
50
|
+
out.push({
|
|
51
|
+
id: `AD-${createHash("sha256").update(key).digest("hex").slice(0, 10)}`,
|
|
52
|
+
applies_to: kind === "model" ? { model: name } : { tool: name },
|
|
53
|
+
pattern: g.pattern,
|
|
54
|
+
signature: g.signature,
|
|
55
|
+
sessions: g.sessions,
|
|
56
|
+
type_sessions: total,
|
|
57
|
+
affected_ppm: affected,
|
|
58
|
+
failure_ppm: Math.floor((g.failed * 1_000_000) / g.sessions),
|
|
59
|
+
tenants: g.tenants.size,
|
|
60
|
+
first_seen: g.first,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
return out.sort((a, b) => b.affected_ppm !== a.affected_ppm ? b.affected_ppm - a.affected_ppm : a.id < b.id ? -1 : 1);
|
|
64
|
+
}
|
|
65
|
+
/** spec/directives.md §2: the directives for the types one fleet flies, with its own sessions. */
|
|
66
|
+
export function fleetDirectives(all, mine) {
|
|
67
|
+
const out = [];
|
|
68
|
+
for (const d of all) {
|
|
69
|
+
const type = "model" in d.applies_to ? `model:${d.applies_to.model}` : `tool:${d.applies_to.tool}`;
|
|
70
|
+
const flying = mine.filter((i) => typesOf(i).includes(type));
|
|
71
|
+
if (flying.length === 0)
|
|
72
|
+
continue;
|
|
73
|
+
const hit = flying
|
|
74
|
+
.filter((i) => signatureOf(i.summary)?.signature === d.signature)
|
|
75
|
+
.map((i) => i.summary.session_id)
|
|
76
|
+
.sort();
|
|
77
|
+
out.push({ ...d, your_sessions: hit, your_type_sessions: flying.length });
|
|
78
|
+
}
|
|
79
|
+
return out;
|
|
80
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
export type Fault = {
|
|
2
|
+
kind: "provider_error";
|
|
3
|
+
at_call: number;
|
|
4
|
+
status: number;
|
|
5
|
+
} | {
|
|
6
|
+
kind: "latency";
|
|
7
|
+
at_call: number;
|
|
8
|
+
ms: number;
|
|
9
|
+
} | {
|
|
10
|
+
kind: "truncate";
|
|
11
|
+
at_call: number;
|
|
12
|
+
} | {
|
|
13
|
+
kind: "tool_error";
|
|
14
|
+
at_tool_call: number;
|
|
15
|
+
message?: string;
|
|
16
|
+
} | {
|
|
17
|
+
kind: "bad_data";
|
|
18
|
+
at_tool_call: number;
|
|
19
|
+
text: string;
|
|
20
|
+
};
|
|
21
|
+
/** spec/drills.md §1: the drill with defaults filled in, or an error message. */
|
|
22
|
+
export declare function checkDrill(value: unknown): {
|
|
23
|
+
faults: Fault[];
|
|
24
|
+
} | string;
|
|
25
|
+
/** spec/drills.md §3. */
|
|
26
|
+
export declare function drillReport(lines: readonly string[]): {
|
|
27
|
+
drill: false;
|
|
28
|
+
planned: never[];
|
|
29
|
+
injected: never[];
|
|
30
|
+
findings_after: never[];
|
|
31
|
+
recovered: boolean;
|
|
32
|
+
outcome: null;
|
|
33
|
+
} | {
|
|
34
|
+
drill: true;
|
|
35
|
+
planned: Fault[];
|
|
36
|
+
injected: {
|
|
37
|
+
seq: number;
|
|
38
|
+
kind: string;
|
|
39
|
+
}[];
|
|
40
|
+
findings_after: string[];
|
|
41
|
+
recovered: boolean;
|
|
42
|
+
outcome: string | null;
|
|
43
|
+
};
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
// Drills (spec/drills.md): scheduled failures, checked, and what the agent did about them. Pure;
|
|
2
|
+
// mirrors sdks/python/src/zanii_blackbox/drills.py; pinned by spec/vectors/drills.json.
|
|
3
|
+
const MODEL_KINDS = new Set(["provider_error", "latency", "truncate"]);
|
|
4
|
+
const TOOL_KINDS = new Set(["tool_error", "bad_data"]);
|
|
5
|
+
const int = (v, min, max) => Number.isSafeInteger(v) && v >= min && v <= max;
|
|
6
|
+
/** spec/drills.md §1: the drill with defaults filled in, or an error message. */
|
|
7
|
+
export function checkDrill(value) {
|
|
8
|
+
const faults = value?.faults;
|
|
9
|
+
if (typeof value !== "object" || value === null || Array.isArray(value) || !Array.isArray(faults))
|
|
10
|
+
return "drill must be {faults: [...]}";
|
|
11
|
+
if (faults.length === 0 || faults.length > 20)
|
|
12
|
+
return "drill.faults must hold 1-20 faults";
|
|
13
|
+
const out = [];
|
|
14
|
+
for (const [i, raw] of faults.entries()) {
|
|
15
|
+
const f = raw;
|
|
16
|
+
const at = `drill.faults[${i}]`;
|
|
17
|
+
if (typeof f !== "object" || f === null)
|
|
18
|
+
return `${at} must be an object`;
|
|
19
|
+
if (MODEL_KINDS.has(f.kind)) {
|
|
20
|
+
if (!int(f.at_call, 1, 1_000_000))
|
|
21
|
+
return `${at}.at_call must be an integer of at least 1`;
|
|
22
|
+
if (f.kind === "provider_error") {
|
|
23
|
+
const status = f.status ?? 500;
|
|
24
|
+
if (!(status === 429 || int(status, 500, 599)))
|
|
25
|
+
return `${at}.status must be 429 or 500-599`;
|
|
26
|
+
out.push({
|
|
27
|
+
kind: "provider_error",
|
|
28
|
+
at_call: f.at_call,
|
|
29
|
+
status: status,
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
else if (f.kind === "latency") {
|
|
33
|
+
if (!int(f.ms, 1, 60_000))
|
|
34
|
+
return `${at}.ms must be 1-60000`;
|
|
35
|
+
out.push({ kind: "latency", at_call: f.at_call, ms: f.ms });
|
|
36
|
+
}
|
|
37
|
+
else
|
|
38
|
+
out.push({ kind: "truncate", at_call: f.at_call });
|
|
39
|
+
}
|
|
40
|
+
else if (TOOL_KINDS.has(f.kind)) {
|
|
41
|
+
if (!int(f.at_tool_call, 1, 1_000_000))
|
|
42
|
+
return `${at}.at_tool_call must be an integer of at least 1`;
|
|
43
|
+
if (f.kind === "tool_error") {
|
|
44
|
+
if (f.message !== undefined && (typeof f.message !== "string" || f.message.length > 500))
|
|
45
|
+
return `${at}.message must be at most 500 characters`;
|
|
46
|
+
out.push({
|
|
47
|
+
kind: "tool_error",
|
|
48
|
+
at_tool_call: f.at_tool_call,
|
|
49
|
+
...(f.message !== undefined ? { message: f.message } : {}),
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
if (typeof f.text !== "string" || f.text.length > 2000)
|
|
54
|
+
return `${at}.text must be a string of at most 2000 characters`;
|
|
55
|
+
out.push({ kind: "bad_data", at_tool_call: f.at_tool_call, text: f.text });
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
else
|
|
59
|
+
return `${at}.kind must be provider_error, latency, truncate, tool_error or bad_data`;
|
|
60
|
+
}
|
|
61
|
+
return { faults: out };
|
|
62
|
+
}
|
|
63
|
+
/** spec/drills.md §3. */
|
|
64
|
+
export function drillReport(lines) {
|
|
65
|
+
const events = lines.map((l) => JSON.parse(l));
|
|
66
|
+
const drill = events[0]?.meta.drill?.faults;
|
|
67
|
+
if (events[0]?.kind !== "session.open" || !drill)
|
|
68
|
+
return {
|
|
69
|
+
drill: false,
|
|
70
|
+
planned: [],
|
|
71
|
+
injected: [],
|
|
72
|
+
findings_after: [],
|
|
73
|
+
recovered: false,
|
|
74
|
+
outcome: null,
|
|
75
|
+
};
|
|
76
|
+
const injected = events
|
|
77
|
+
.filter((e) => (e.kind === "llm.request" || e.kind === "tool.call") &&
|
|
78
|
+
typeof e.meta.injected === "object" &&
|
|
79
|
+
e.meta.injected !== null)
|
|
80
|
+
.map((e) => ({ seq: e.seq, kind: String(e.meta.injected.kind) }));
|
|
81
|
+
const first = injected[0]?.seq ?? Number.POSITIVE_INFINITY;
|
|
82
|
+
const lastInjected = Math.max(-1, ...events.filter((e) => e.meta.injected !== undefined).map((e) => e.seq));
|
|
83
|
+
const codes = [
|
|
84
|
+
...new Set(events.filter((e) => e.kind === "finding" && e.seq > first).map((e) => String(e.meta.code))),
|
|
85
|
+
].sort();
|
|
86
|
+
const recovered = events.some((e) => e.kind === "llm.response" &&
|
|
87
|
+
e.seq > lastInjected &&
|
|
88
|
+
e.meta.status === 200 &&
|
|
89
|
+
e.meta.injected === undefined &&
|
|
90
|
+
injected.length > 0);
|
|
91
|
+
const outcomes = events.filter((e) => e.kind === "outcome");
|
|
92
|
+
const outcome = outcomes.length ? String(outcomes.at(-1)?.meta.outcome) : null;
|
|
93
|
+
return {
|
|
94
|
+
drill: true,
|
|
95
|
+
planned: drill,
|
|
96
|
+
injected,
|
|
97
|
+
findings_after: codes,
|
|
98
|
+
recovered,
|
|
99
|
+
outcome,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { type LoadedPrices } from "../cost/index.ts";
|
|
2
|
+
export interface DutyLimits {
|
|
3
|
+
max_minutes?: number;
|
|
4
|
+
max_llm_calls?: number;
|
|
5
|
+
max_context_pct?: number;
|
|
6
|
+
}
|
|
7
|
+
export interface DutyFinding {
|
|
8
|
+
code: "DUTY_LIMIT";
|
|
9
|
+
source: "duty";
|
|
10
|
+
severity: "warning";
|
|
11
|
+
ref: {
|
|
12
|
+
seq: number;
|
|
13
|
+
limit: "minutes" | "llm_calls" | "context_pct";
|
|
14
|
+
value: number;
|
|
15
|
+
max: number;
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
/** spec/duty.md §2: at most one DUTY_LIMIT per limit, where it was first crossed. */
|
|
19
|
+
export declare function dutyFindings(lines: readonly string[], prices?: LoadedPrices, defaults?: DutyLimits): DutyFinding[];
|
|
20
|
+
/** Checks a flight plan's `duty` (spec/duty.md §1): an error message, or undefined. */
|
|
21
|
+
export declare function checkDuty(duty: unknown): string | undefined;
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// Duty-time limits (spec/duty.md): long sessions and full context windows, detected. Pure; mirrors
|
|
2
|
+
// sdks/python/src/zanii_blackbox/duty.py; pinned by spec/vectors/duty.json.
|
|
3
|
+
import { priceFor, promptTokens, sessionCost } from "../cost/index.js";
|
|
4
|
+
const KEYS = ["max_minutes", "max_llm_calls", "max_context_pct"];
|
|
5
|
+
/** spec/duty.md §1: the flight plan's limits win, limit by limit, over the server's defaults. */
|
|
6
|
+
function limitsOf(open, defaults) {
|
|
7
|
+
const plan = open?.meta.flight_plan?.duty;
|
|
8
|
+
const out = { max_minutes: 0, max_llm_calls: 0, max_context_pct: 0 };
|
|
9
|
+
for (const k of KEYS) {
|
|
10
|
+
const v = plan?.[k] ?? defaults[k];
|
|
11
|
+
if (Number.isSafeInteger(v) && v > 0)
|
|
12
|
+
out[k] = v;
|
|
13
|
+
}
|
|
14
|
+
return out;
|
|
15
|
+
}
|
|
16
|
+
/** spec/duty.md §2: at most one DUTY_LIMIT per limit, where it was first crossed. */
|
|
17
|
+
export function dutyFindings(lines, prices, defaults = {}) {
|
|
18
|
+
const events = lines.map((l) => JSON.parse(l));
|
|
19
|
+
const open = events[0]?.kind === "session.open" ? events[0] : undefined;
|
|
20
|
+
const limits = limitsOf(open, defaults);
|
|
21
|
+
const out = [];
|
|
22
|
+
const finding = (seq, limit, value, max) => out.push({
|
|
23
|
+
code: "DUTY_LIMIT",
|
|
24
|
+
source: "duty",
|
|
25
|
+
severity: "warning",
|
|
26
|
+
ref: { seq, limit, value, max },
|
|
27
|
+
});
|
|
28
|
+
if (limits.max_minutes && open) {
|
|
29
|
+
const start = Date.parse(open.ts);
|
|
30
|
+
const late = events.find((e) => Date.parse(e.ts) - start > limits.max_minutes * 60_000);
|
|
31
|
+
if (late)
|
|
32
|
+
finding(late.seq, "minutes", Math.floor((Date.parse(late.ts) - start) / 60_000), limits.max_minutes);
|
|
33
|
+
}
|
|
34
|
+
if (limits.max_llm_calls) {
|
|
35
|
+
const requests = events.filter((e) => e.kind === "llm.request");
|
|
36
|
+
const over = requests[limits.max_llm_calls];
|
|
37
|
+
if (over)
|
|
38
|
+
finding(over.seq, "llm_calls", limits.max_llm_calls + 1, limits.max_llm_calls);
|
|
39
|
+
}
|
|
40
|
+
if (limits.max_context_pct && prices) {
|
|
41
|
+
for (const c of sessionCost(lines, prices).calls) {
|
|
42
|
+
const window = c.model ? priceFor(c.model, prices.table)?.price.context_window : undefined;
|
|
43
|
+
if (!window)
|
|
44
|
+
continue;
|
|
45
|
+
const prompt = promptTokens(c.tokens);
|
|
46
|
+
const pct = Math.floor((prompt * 100) / window);
|
|
47
|
+
if (pct >= limits.max_context_pct) {
|
|
48
|
+
finding(c.seq, "context_pct", pct, limits.max_context_pct);
|
|
49
|
+
break;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return out;
|
|
54
|
+
}
|
|
55
|
+
/** Checks a flight plan's `duty` (spec/duty.md §1): an error message, or undefined. */
|
|
56
|
+
export function checkDuty(duty) {
|
|
57
|
+
if (typeof duty !== "object" || duty === null || Array.isArray(duty))
|
|
58
|
+
return "flight_plan.duty must be an object";
|
|
59
|
+
for (const [k, v] of Object.entries(duty)) {
|
|
60
|
+
if (!KEYS.includes(k))
|
|
61
|
+
return `flight_plan.duty has an unknown field: ${k}`;
|
|
62
|
+
if (!Number.isSafeInteger(v) ||
|
|
63
|
+
v < 1 ||
|
|
64
|
+
(k === "max_context_pct" && v > 100))
|
|
65
|
+
return `flight_plan.duty.${k} must be a positive integer${k === "max_context_pct" ? " of at most 100" : ""}`;
|
|
66
|
+
}
|
|
67
|
+
return undefined;
|
|
68
|
+
}
|