@zanii/blackbox 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +109 -2
- package/dist/agents/index.d.ts +34 -0
- package/dist/agents/index.js +73 -0
- package/dist/analysis/detectors.d.ts +36 -0
- package/dist/analysis/detectors.js +339 -0
- package/dist/analysis/faults.d.ts +9 -0
- package/dist/analysis/faults.js +250 -0
- package/dist/analysis/index.d.ts +68 -0
- package/dist/analysis/index.js +388 -0
- package/dist/analysis/landing.d.ts +25 -0
- package/dist/analysis/landing.js +225 -0
- package/dist/analysis/memory.d.ts +13 -0
- package/dist/analysis/memory.js +33 -0
- package/dist/analysis/waste.d.ts +29 -0
- package/dist/analysis/waste.js +79 -0
- package/dist/approvals/index.d.ts +11 -0
- package/dist/approvals/index.js +27 -0
- package/dist/approvals/warnings.d.ts +2 -0
- package/dist/approvals/warnings.js +28 -0
- package/dist/attest/index.d.ts +17 -0
- package/dist/attest/index.js +106 -0
- package/dist/authority/index.d.ts +24 -0
- package/dist/authority/index.js +77 -0
- package/dist/billing/index.d.ts +99 -0
- package/dist/billing/index.js +174 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +1057 -0
- package/dist/client/index.d.ts +146 -0
- package/dist/client/index.js +210 -0
- package/dist/compliance/index.d.ts +41 -0
- package/dist/compliance/index.js +96 -0
- package/dist/cost/index.d.ts +133 -0
- package/dist/cost/index.js +293 -0
- package/dist/data/index.d.ts +191 -0
- package/dist/data/index.js +762 -0
- package/dist/directives/index.d.ts +35 -0
- package/dist/directives/index.js +80 -0
- package/dist/drills/index.d.ts +43 -0
- package/dist/drills/index.js +101 -0
- package/dist/duty/index.d.ts +21 -0
- package/dist/duty/index.js +68 -0
- package/dist/fleet/index.d.ts +141 -0
- package/dist/fleet/index.js +454 -0
- package/dist/hooks/ai-sdk.d.ts +42 -0
- package/dist/hooks/ai-sdk.js +62 -0
- package/dist/hooks/claude-agent-sdk.d.ts +14 -0
- package/dist/hooks/claude-agent-sdk.js +70 -0
- package/dist/hooks/index.d.ts +7 -0
- package/dist/hooks/index.js +10 -0
- package/dist/hooks/langchain-agent.d.ts +69 -0
- package/dist/hooks/langchain-agent.js +163 -0
- package/dist/hooks/langchain.d.ts +41 -0
- package/dist/hooks/langchain.js +216 -0
- package/dist/hooks/langgraph-checkpoint.d.ts +12 -0
- package/dist/hooks/langgraph-checkpoint.js +73 -0
- package/dist/hooks/memory.d.ts +17 -0
- package/dist/hooks/memory.js +64 -0
- package/dist/hooks/openai-agents.d.ts +6 -0
- package/dist/hooks/openai-agents.js +40 -0
- package/dist/hooks/protect.d.ts +7 -0
- package/dist/hooks/protect.js +39 -0
- package/dist/hooks/providers.d.ts +16 -0
- package/dist/hooks/providers.js +149 -0
- package/dist/hooks/shared.d.ts +11 -0
- package/dist/hooks/shared.js +39 -0
- package/dist/index.d.ts +46 -0
- package/dist/index.js +48 -0
- package/dist/investigate/index.d.ts +66 -0
- package/dist/investigate/index.js +119 -0
- package/dist/mcp-server/index.d.ts +85 -0
- package/dist/mcp-server/index.js +216 -0
- package/dist/mcp-wrap/index.d.ts +17 -0
- package/dist/mcp-wrap/index.js +170 -0
- package/dist/money/index.d.ts +114 -0
- package/dist/money/index.js +622 -0
- package/dist/occurrence/index.d.ts +108 -0
- package/dist/occurrence/index.js +168 -0
- package/dist/ocsf/index.d.ts +22 -0
- package/dist/ocsf/index.js +168 -0
- package/dist/otlp/index.d.ts +24 -0
- package/dist/otlp/index.js +143 -0
- package/dist/packs/index.d.ts +48 -0
- package/dist/packs/index.js +343 -0
- package/dist/policy/delta.d.ts +11 -0
- package/dist/policy/delta.js +39 -0
- package/dist/policy/drafts.d.ts +34 -0
- package/dist/policy/drafts.js +129 -0
- package/dist/policy/index.d.ts +47 -0
- package/dist/policy/index.js +154 -0
- package/dist/precog/index.d.ts +96 -0
- package/dist/precog/index.js +167 -0
- package/dist/precog/intervention.d.ts +22 -0
- package/dist/precog/intervention.js +44 -0
- package/dist/precog/normal.d.ts +31 -0
- package/dist/precog/normal.js +89 -0
- package/dist/preflight/index.d.ts +11 -0
- package/dist/preflight/index.js +19 -0
- package/dist/ratings/index.d.ts +21 -0
- package/dist/ratings/index.js +48 -0
- package/dist/reconcile/claude-code.d.ts +19 -0
- package/dist/reconcile/claude-code.js +220 -0
- package/dist/reconcile/codex.d.ts +5 -0
- package/dist/reconcile/codex.js +191 -0
- package/dist/reconcile/index.d.ts +19 -0
- package/dist/reconcile/index.js +50 -0
- package/dist/reconcile/record.d.ts +49 -0
- package/dist/reconcile/record.js +225 -0
- package/dist/reconcile/shared.d.ts +65 -0
- package/dist/reconcile/shared.js +113 -0
- package/dist/replay/index.d.ts +11 -0
- package/dist/replay/index.js +64 -0
- package/dist/replay/repair.d.ts +10 -0
- package/dist/replay/repair.js +62 -0
- package/dist/session/drain.d.ts +13 -0
- package/dist/session/drain.js +35 -0
- package/dist/session/index.d.ts +275 -0
- package/dist/session/index.js +681 -0
- package/dist/undo/index.d.ts +45 -0
- package/dist/undo/index.js +212 -0
- package/dist/verify/anchor.d.ts +54 -0
- package/dist/verify/anchor.js +77 -0
- package/dist/verify/chain.d.ts +27 -0
- package/dist/verify/chain.js +105 -0
- package/dist/verify/envelope.d.ts +28 -0
- package/dist/verify/envelope.js +55 -0
- package/dist/verify/index.d.ts +3 -0
- package/dist/verify/index.js +3 -0
- package/dist/version.d.ts +1 -0
- package/dist/version.js +2 -0
- package/dist/weather/index.d.ts +24 -0
- package/dist/weather/index.js +45 -0
- package/dist/workspace-receipt/index.d.ts +15 -0
- package/dist/workspace-receipt/index.js +121 -0
- package/package.json +56 -3
|
@@ -0,0 +1,622 @@
|
|
|
1
|
+
// Money (spec/money.md): spend by model, the provider's statement against the record, cost
|
|
2
|
+
// baselines per task, fuel to destination. Mirrors sdks/python/src/zanii_blackbox/money.py.
|
|
3
|
+
import { sessionCost } from "../cost/index.js";
|
|
4
|
+
const byKey = (m) => Object.fromEntries([...m.keys()].sort().map((k) => [k, m.get(k)]));
|
|
5
|
+
/** `{model: micro_usd}` over the priced calls, sorted by model. */
|
|
6
|
+
export function modelSpend(lines, prices) {
|
|
7
|
+
const m = new Map();
|
|
8
|
+
for (const c of sessionCost(lines, prices).calls)
|
|
9
|
+
if (c.micro_usd !== null && c.model !== null)
|
|
10
|
+
m.set(c.model, (m.get(c.model) ?? 0) + c.micro_usd);
|
|
11
|
+
return byKey(m);
|
|
12
|
+
}
|
|
13
|
+
/** Why a statement is invalid, or undefined. */
|
|
14
|
+
export function checkStatement(s) {
|
|
15
|
+
const lines = s?.lines;
|
|
16
|
+
if (!Array.isArray(lines) ||
|
|
17
|
+
lines.length > 1000 ||
|
|
18
|
+
!lines.every((l) => typeof l === "object" &&
|
|
19
|
+
l !== null &&
|
|
20
|
+
typeof l.model === "string" &&
|
|
21
|
+
l.model.length > 0 &&
|
|
22
|
+
l.model.length <= 128 &&
|
|
23
|
+
Number.isSafeInteger(l.micro_usd) &&
|
|
24
|
+
l.micro_usd >= 0))
|
|
25
|
+
return "statement must be {lines: [{model, micro_usd}]}: at most 1000 lines, integer micro_usd ≥ 0";
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
export function reconcileBilling(ours, statement, tolerancePpm = 10_000) {
|
|
29
|
+
const theirs = new Map();
|
|
30
|
+
for (const l of statement.lines)
|
|
31
|
+
theirs.set(l.model, (theirs.get(l.model) ?? 0) + l.micro_usd);
|
|
32
|
+
const names = [...new Set([...Object.keys(ours), ...theirs.keys()])].sort();
|
|
33
|
+
const models = names.map((model) => {
|
|
34
|
+
const o = ours[model] ?? 0;
|
|
35
|
+
const t = theirs.get(model) ?? 0;
|
|
36
|
+
const diff = t - o;
|
|
37
|
+
const status = Math.abs(diff) * 1_000_000 <= tolerancePpm * Math.max(o, t)
|
|
38
|
+
? "match"
|
|
39
|
+
: diff > 0
|
|
40
|
+
? "provider_higher"
|
|
41
|
+
: "provider_lower";
|
|
42
|
+
return { model, ours: o, theirs: t, diff, status };
|
|
43
|
+
});
|
|
44
|
+
const oursTotal = models.reduce((n, m) => n + m.ours, 0);
|
|
45
|
+
const theirsTotal = models.reduce((n, m) => n + m.theirs, 0);
|
|
46
|
+
return {
|
|
47
|
+
v: 1,
|
|
48
|
+
tolerance_ppm: tolerancePpm,
|
|
49
|
+
ours_micro_usd: oursTotal,
|
|
50
|
+
theirs_micro_usd: theirsTotal,
|
|
51
|
+
diff_micro_usd: theirsTotal - oursTotal,
|
|
52
|
+
ok: models.every((m) => m.status === "match"),
|
|
53
|
+
models,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
/** Nearest rank: the p-th of the sorted values is at index ceil(p·n/100) − 1. */
|
|
57
|
+
function rank(values, p) {
|
|
58
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
59
|
+
return sorted[Math.max(0, Math.ceil((p * sorted.length) / 100) - 1)];
|
|
60
|
+
}
|
|
61
|
+
/** Per label, over successful sessions only (spec/money.md §3). */
|
|
62
|
+
export function baselines(summaries) {
|
|
63
|
+
const by = new Map();
|
|
64
|
+
for (const s of summaries)
|
|
65
|
+
if (s.outcome === "success") {
|
|
66
|
+
const k = s.label ?? "(no label)";
|
|
67
|
+
by.set(k, [...(by.get(k) ?? []), s]);
|
|
68
|
+
}
|
|
69
|
+
return {
|
|
70
|
+
labels: [...by.keys()].sort().map((label) => {
|
|
71
|
+
const ss = by.get(label);
|
|
72
|
+
const spend = ss.map((s) => s.spend_micro_usd);
|
|
73
|
+
return {
|
|
74
|
+
label,
|
|
75
|
+
successes: ss.length,
|
|
76
|
+
median_spend_micro_usd: rank(spend, 50),
|
|
77
|
+
p90_spend_micro_usd: rank(spend, 90),
|
|
78
|
+
median_model_calls: rank(ss.map((s) => s.model_calls), 50),
|
|
79
|
+
median_tool_calls: rank(ss.map((s) => s.tool_calls), 50),
|
|
80
|
+
};
|
|
81
|
+
}),
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
/** Will the remaining budget reach the destination? (spec/money.md §4) */
|
|
85
|
+
export function fuel(summary, budgetMicroUsd, baseline) {
|
|
86
|
+
const spent = summary.spend_micro_usd;
|
|
87
|
+
const expected = baseline ? Math.max(spent, baseline.median_spend_micro_usd) : null;
|
|
88
|
+
const p90 = baseline ? Math.max(spent, baseline.p90_spend_micro_usd) : null;
|
|
89
|
+
const verdict = expected === null || p90 === null
|
|
90
|
+
? "unknown"
|
|
91
|
+
: budgetMicroUsd === null
|
|
92
|
+
? "no_budget"
|
|
93
|
+
: p90 <= budgetMicroUsd
|
|
94
|
+
? "likely"
|
|
95
|
+
: expected <= budgetMicroUsd
|
|
96
|
+
? "at_risk"
|
|
97
|
+
: "unlikely";
|
|
98
|
+
return {
|
|
99
|
+
spent_micro_usd: spent,
|
|
100
|
+
budget_micro_usd: budgetMicroUsd,
|
|
101
|
+
expected_total_micro_usd: expected,
|
|
102
|
+
p90_total_micro_usd: p90,
|
|
103
|
+
verdict,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
// ---------------------------------------------------------------- statements from cost APIs (§2a)
|
|
107
|
+
const DECIMAL = /^(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/;
|
|
108
|
+
/** Per unit: the power of ten to multiply by, and the divisor (AED at the CBUAE peg, 3.6725). */
|
|
109
|
+
const UNITS = {
|
|
110
|
+
usd: [6, 1n],
|
|
111
|
+
cents: [4, 1n],
|
|
112
|
+
micro_usd: [0, 1n],
|
|
113
|
+
aed: [10, 36725n],
|
|
114
|
+
};
|
|
115
|
+
/** spec/money.md §2a-b: an exact decimal amount (string, or a JSON number's text) to integer
|
|
116
|
+
* micro-USD, rounded half up. `unit` is what the amount counts. */
|
|
117
|
+
export function decimalToMicroUsd(value, unit) {
|
|
118
|
+
const text = typeof value === "number" && Number.isFinite(value) ? String(value) : value;
|
|
119
|
+
const m = typeof text === "string" ? DECIMAL.exec(text.trim()) : null;
|
|
120
|
+
const scale = UNITS[unit];
|
|
121
|
+
if (!m || !scale)
|
|
122
|
+
throw new Error(`not a non-negative decimal amount: ${JSON.stringify(value)}`);
|
|
123
|
+
const frac = m[2] ?? "";
|
|
124
|
+
const digits = BigInt(`${m[1]}${frac}`);
|
|
125
|
+
if (digits === 0n)
|
|
126
|
+
return 0;
|
|
127
|
+
const k = Number(m[3] ?? 0) - frac.length + scale[0];
|
|
128
|
+
if (k > 30)
|
|
129
|
+
throw new Error(`amount too large: ${JSON.stringify(value)}`);
|
|
130
|
+
const num = k >= 0 ? digits * 10n ** BigInt(k) : digits;
|
|
131
|
+
const den = scale[1] * (k >= 0 ? 1n : 10n ** BigInt(-k));
|
|
132
|
+
let micro = num / den;
|
|
133
|
+
if ((num % den) * 2n >= den)
|
|
134
|
+
micro += 1n;
|
|
135
|
+
if (micro > BigInt(Number.MAX_SAFE_INTEGER))
|
|
136
|
+
throw new Error(`amount too large: ${JSON.stringify(value)}`);
|
|
137
|
+
return Number(micro);
|
|
138
|
+
}
|
|
139
|
+
const bucketsOf = (pages) => pages.flatMap((p) => p?.data ?? []);
|
|
140
|
+
const resultsOf = (b) => b.results ?? [];
|
|
141
|
+
function toStatement(m) {
|
|
142
|
+
if (m.size > 1000)
|
|
143
|
+
throw new Error("a statement holds at most 1,000 lines");
|
|
144
|
+
const names = [...m.keys()].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
|
|
145
|
+
return { lines: names.map((model) => ({ model, micro_usd: m.get(model) })) };
|
|
146
|
+
}
|
|
147
|
+
const add = (m, model, micro) => {
|
|
148
|
+
const key = model.slice(0, 128);
|
|
149
|
+
m.set(key, (m.get(key) ?? 0) + micro);
|
|
150
|
+
};
|
|
151
|
+
/** spec/money.md §2a: Anthropic's cost report pages (grouped by description) as a statement. */
|
|
152
|
+
export function anthropicStatement(pages) {
|
|
153
|
+
const m = new Map();
|
|
154
|
+
for (const bucket of bucketsOf(pages))
|
|
155
|
+
for (const r of resultsOf(bucket)) {
|
|
156
|
+
if (r.currency !== "USD")
|
|
157
|
+
throw new Error(`unexpected currency ${JSON.stringify(r.currency)}`);
|
|
158
|
+
const model = typeof r.model === "string" && r.model !== ""
|
|
159
|
+
? r.model
|
|
160
|
+
: `anthropic:${typeof r.cost_type === "string" && r.cost_type !== "" ? r.cost_type : "other"}`;
|
|
161
|
+
add(m, model, decimalToMicroUsd(r.amount, "cents"));
|
|
162
|
+
}
|
|
163
|
+
return toStatement(m);
|
|
164
|
+
}
|
|
165
|
+
/** spec/money.md §2a: OpenAI's organization costs pages (grouped by line_item) as a statement. */
|
|
166
|
+
export function openaiStatement(pages) {
|
|
167
|
+
const m = new Map();
|
|
168
|
+
for (const bucket of bucketsOf(pages))
|
|
169
|
+
for (const r of resultsOf(bucket)) {
|
|
170
|
+
const amount = r.amount ?? {};
|
|
171
|
+
if (amount.currency !== "usd")
|
|
172
|
+
throw new Error(`unexpected currency ${JSON.stringify(amount.currency)}`);
|
|
173
|
+
const li = r.line_item;
|
|
174
|
+
const cut = typeof li === "string" ? li.lastIndexOf(", ") : -1;
|
|
175
|
+
const model = typeof li !== "string" || li === ""
|
|
176
|
+
? "openai:unattributed"
|
|
177
|
+
: cut > 0
|
|
178
|
+
? li.slice(0, cut)
|
|
179
|
+
: `openai:${li}`;
|
|
180
|
+
add(m, model, decimalToMicroUsd(amount.value, "usd"));
|
|
181
|
+
}
|
|
182
|
+
return toStatement(m);
|
|
183
|
+
}
|
|
184
|
+
// ---------------------------------------------------------------- any provider (§2b)
|
|
185
|
+
/** spec/money.md §2b: every value at a path (`a.b[].c`, `[]` alone for a list itself). */
|
|
186
|
+
export function resolvePath(value, path) {
|
|
187
|
+
let current = [value];
|
|
188
|
+
for (const seg of path.split(".")) {
|
|
189
|
+
const list = seg.endsWith("[]");
|
|
190
|
+
const key = list ? seg.slice(0, -2) : seg;
|
|
191
|
+
const next = [];
|
|
192
|
+
for (const v of current) {
|
|
193
|
+
const got = key === ""
|
|
194
|
+
? v
|
|
195
|
+
: v !== null && typeof v === "object" && !Array.isArray(v)
|
|
196
|
+
? v[key]
|
|
197
|
+
: undefined;
|
|
198
|
+
if (list) {
|
|
199
|
+
if (Array.isArray(got))
|
|
200
|
+
next.push(...got);
|
|
201
|
+
}
|
|
202
|
+
else if (got !== undefined)
|
|
203
|
+
next.push(got);
|
|
204
|
+
}
|
|
205
|
+
current = next;
|
|
206
|
+
}
|
|
207
|
+
return current;
|
|
208
|
+
}
|
|
209
|
+
/** spec/money.md §2b: a CSV export with named columns as a statement. */
|
|
210
|
+
export function csvStatement(text, opts) {
|
|
211
|
+
const rows = parseCsv(text.replace(/^/, ""), opts.delimiter ?? ",");
|
|
212
|
+
const header = rows.shift();
|
|
213
|
+
if (!header)
|
|
214
|
+
throw new Error("the CSV is empty");
|
|
215
|
+
const mi = header.indexOf(opts.model_column);
|
|
216
|
+
const ai = header.indexOf(opts.amount_column);
|
|
217
|
+
if (mi < 0)
|
|
218
|
+
throw new Error(`no column ${JSON.stringify(opts.model_column)}`);
|
|
219
|
+
if (ai < 0)
|
|
220
|
+
throw new Error(`no column ${JSON.stringify(opts.amount_column)}`);
|
|
221
|
+
const m = new Map();
|
|
222
|
+
rows.forEach((row, i) => {
|
|
223
|
+
if (row.length === 1 && row[0] === "")
|
|
224
|
+
return; // a blank line
|
|
225
|
+
if (row.length <= Math.max(mi, ai))
|
|
226
|
+
throw new Error(`row ${i + 2}: too few fields`);
|
|
227
|
+
let micro;
|
|
228
|
+
try {
|
|
229
|
+
micro = decimalToMicroUsd(row[ai], opts.unit);
|
|
230
|
+
}
|
|
231
|
+
catch {
|
|
232
|
+
throw new Error(`row ${i + 2}: ${JSON.stringify(row[ai])} isn't a plain decimal amount`);
|
|
233
|
+
}
|
|
234
|
+
add(m, row[mi].trim() || "unattributed", micro);
|
|
235
|
+
});
|
|
236
|
+
return toStatement(m);
|
|
237
|
+
}
|
|
238
|
+
/** RFC 4180-style: quoted fields, `""` for a quote, CRLF or LF. */
|
|
239
|
+
function parseCsv(text, delimiter) {
|
|
240
|
+
const rows = [];
|
|
241
|
+
let row = [];
|
|
242
|
+
let field = "";
|
|
243
|
+
let quoted = false;
|
|
244
|
+
for (let i = 0; i < text.length; i++) {
|
|
245
|
+
const c = text[i];
|
|
246
|
+
if (quoted) {
|
|
247
|
+
if (c === '"' && text[i + 1] === '"') {
|
|
248
|
+
field += '"';
|
|
249
|
+
i++;
|
|
250
|
+
}
|
|
251
|
+
else if (c === '"')
|
|
252
|
+
quoted = false;
|
|
253
|
+
else
|
|
254
|
+
field += c;
|
|
255
|
+
}
|
|
256
|
+
else if (c === '"' && field === "")
|
|
257
|
+
quoted = true;
|
|
258
|
+
else if (c === delimiter) {
|
|
259
|
+
row.push(field);
|
|
260
|
+
field = "";
|
|
261
|
+
}
|
|
262
|
+
else if (c === "\n" || c === "\r") {
|
|
263
|
+
if (c === "\r" && text[i + 1] === "\n")
|
|
264
|
+
i++;
|
|
265
|
+
row.push(field);
|
|
266
|
+
rows.push(row);
|
|
267
|
+
row = [];
|
|
268
|
+
field = "";
|
|
269
|
+
}
|
|
270
|
+
else
|
|
271
|
+
field += c;
|
|
272
|
+
}
|
|
273
|
+
if (quoted)
|
|
274
|
+
throw new Error("the CSV ends inside a quoted field");
|
|
275
|
+
if (field !== "" || row.length) {
|
|
276
|
+
row.push(field);
|
|
277
|
+
rows.push(row);
|
|
278
|
+
}
|
|
279
|
+
return rows;
|
|
280
|
+
}
|
|
281
|
+
const NAME = /^[a-z0-9][a-z0-9._-]{0,63}$/;
|
|
282
|
+
const PATH = /^(\[\]|[A-Za-z0-9_$:@#-]+(\[\])?)(\.([A-Za-z0-9_$:@#-]+(\[\])?|\[\]))*$/;
|
|
283
|
+
/** spec/money.md §2b: why a connector file is invalid, or undefined. */
|
|
284
|
+
export function checkConnector(value) {
|
|
285
|
+
const c = value;
|
|
286
|
+
if (!c || typeof c !== "object" || Array.isArray(c))
|
|
287
|
+
return "a connector must be an object";
|
|
288
|
+
if (c.v !== 1)
|
|
289
|
+
return "v must be 1";
|
|
290
|
+
if (typeof c.name !== "string" || !NAME.test(c.name))
|
|
291
|
+
return "name must be 1-64 lowercase letters, digits, '.', '_' or '-'";
|
|
292
|
+
const url = c.request?.url;
|
|
293
|
+
if (typeof url !== "string")
|
|
294
|
+
return "request.url is required";
|
|
295
|
+
let parsed;
|
|
296
|
+
try {
|
|
297
|
+
parsed = new URL(url.replace(/\{(from|to)(_unix)?\}/g, "0"));
|
|
298
|
+
}
|
|
299
|
+
catch {
|
|
300
|
+
return "request.url isn't a URL";
|
|
301
|
+
}
|
|
302
|
+
const local = parsed.hostname === "localhost" || parsed.hostname === "127.0.0.1";
|
|
303
|
+
if (!(parsed.protocol === "https:" || (parsed.protocol === "http:" && local)))
|
|
304
|
+
return "request.url must be https (http only for localhost)";
|
|
305
|
+
if (/\{env:/.test(url))
|
|
306
|
+
return "secrets go in request.headers, never in the URL";
|
|
307
|
+
const headers = c.request?.headers ?? {};
|
|
308
|
+
if (typeof headers !== "object" ||
|
|
309
|
+
Array.isArray(headers) ||
|
|
310
|
+
!Object.values(headers).every((v) => typeof v === "string"))
|
|
311
|
+
return "request.headers must map names to strings";
|
|
312
|
+
for (const k of ["results", "model", "amount"])
|
|
313
|
+
if (typeof c[k] !== "string" || !PATH.test(c[k]))
|
|
314
|
+
return `${k} must be a path`;
|
|
315
|
+
if (!c.unit || !(c.unit in UNITS))
|
|
316
|
+
return "unit must be usd, cents, micro_usd or aed";
|
|
317
|
+
if (c.currency !== undefined &&
|
|
318
|
+
(typeof c.currency?.path !== "string" ||
|
|
319
|
+
!PATH.test(c.currency.path) ||
|
|
320
|
+
typeof c.currency.equals !== "string"))
|
|
321
|
+
return "currency must be {path, equals}";
|
|
322
|
+
const method = c.request?.method ?? "GET";
|
|
323
|
+
if (method !== "GET" && method !== "POST")
|
|
324
|
+
return "request.method must be GET or POST";
|
|
325
|
+
const body = c.request?.body;
|
|
326
|
+
if (method === "GET" && (body !== undefined || c.request?.content_type !== undefined))
|
|
327
|
+
return "request.body and request.content_type are for POST";
|
|
328
|
+
if (method === "POST") {
|
|
329
|
+
if (typeof body !== "string" || Buffer.byteLength(body, "utf8") > 65_536)
|
|
330
|
+
return "a POST needs request.body, at most 64 KiB";
|
|
331
|
+
if (typeof c.request?.content_type !== "string" ||
|
|
332
|
+
!/^[\x21-\x7e][\x20-\x7e]{0,127}$/.test(c.request.content_type))
|
|
333
|
+
return "a POST needs request.content_type";
|
|
334
|
+
if (body.includes("{env:"))
|
|
335
|
+
return "secrets go in request.headers, never in the body";
|
|
336
|
+
}
|
|
337
|
+
if (c.format !== undefined && c.format !== "json" && c.format !== "xml")
|
|
338
|
+
return "format must be json or xml";
|
|
339
|
+
if (c.next !== undefined) {
|
|
340
|
+
const bad = checkNext(c.next);
|
|
341
|
+
if (bad)
|
|
342
|
+
return bad;
|
|
343
|
+
}
|
|
344
|
+
return undefined;
|
|
345
|
+
}
|
|
346
|
+
const PARAM = /^[A-Za-z0-9_[\].-]{1,64}$/;
|
|
347
|
+
const intIn = (v, min, max) => Number.isSafeInteger(v) && v >= min && v <= max;
|
|
348
|
+
function checkNext(next) {
|
|
349
|
+
const n = next;
|
|
350
|
+
const only = (keys) => Object.keys(n ?? {}).every((k) => keys.includes(k));
|
|
351
|
+
const param = (k) => typeof n?.[k] === "string" && PARAM.test(n[k]);
|
|
352
|
+
if (!n || typeof n !== "object" || Array.isArray(n))
|
|
353
|
+
return "next must be an object";
|
|
354
|
+
if ("cursor" in n) {
|
|
355
|
+
if (!only(["cursor", "param"]) ||
|
|
356
|
+
typeof n.cursor !== "string" ||
|
|
357
|
+
!PATH.test(n.cursor) ||
|
|
358
|
+
!param("param"))
|
|
359
|
+
return "next must be {cursor: path, param: query parameter name}";
|
|
360
|
+
return undefined;
|
|
361
|
+
}
|
|
362
|
+
if (n.size_param !== undefined && (!param("size_param") || n.size === undefined))
|
|
363
|
+
return "next.size_param must be a query parameter name, with next.size";
|
|
364
|
+
if (n.size !== undefined && !intIn(n.size, 1, 10_000))
|
|
365
|
+
return "next.size must be 1-10000";
|
|
366
|
+
if ("page_param" in n) {
|
|
367
|
+
if (!only(["page_param", "first_page", "size_param", "size"]) || !param("page_param"))
|
|
368
|
+
return "next must be {page_param, first_page?, size_param?, size?}";
|
|
369
|
+
if (n.first_page !== undefined && !intIn(n.first_page, 0, 1_000_000))
|
|
370
|
+
return "next.first_page must be 0 or more";
|
|
371
|
+
return undefined;
|
|
372
|
+
}
|
|
373
|
+
if ("offset_param" in n) {
|
|
374
|
+
if (!only(["offset_param", "size", "size_param"]) ||
|
|
375
|
+
!param("offset_param") ||
|
|
376
|
+
n.size === undefined)
|
|
377
|
+
return "next must be {offset_param, size, size_param?}";
|
|
378
|
+
return undefined;
|
|
379
|
+
}
|
|
380
|
+
return "next must have cursor, page_param or offset_param";
|
|
381
|
+
}
|
|
382
|
+
/** spec/money.md §2b: pages from a connector's API as a statement. */
|
|
383
|
+
export function customStatement(pages, connector) {
|
|
384
|
+
const bad = checkConnector(connector);
|
|
385
|
+
if (bad)
|
|
386
|
+
throw new Error(`connector: ${bad}`);
|
|
387
|
+
const m = new Map();
|
|
388
|
+
for (const page of pages)
|
|
389
|
+
for (const row of resolvePath(page, connector.results)) {
|
|
390
|
+
if (connector.currency) {
|
|
391
|
+
const cur = resolvePath(row, connector.currency.path)[0];
|
|
392
|
+
if (cur !== connector.currency.equals)
|
|
393
|
+
throw new Error(`unexpected currency ${JSON.stringify(cur)}`);
|
|
394
|
+
}
|
|
395
|
+
const model = resolvePath(row, connector.model)[0];
|
|
396
|
+
add(m, typeof model === "string" && model !== "" ? model : `${connector.name}:unattributed`, decimalToMicroUsd(resolvePath(row, connector.amount)[0], connector.unit));
|
|
397
|
+
}
|
|
398
|
+
return toStatement(m);
|
|
399
|
+
}
|
|
400
|
+
/** spec/money.md §2b: the query parameters for the next request, or null when there are no more. */
|
|
401
|
+
export function pageParams(c, pages) {
|
|
402
|
+
const next = c.next;
|
|
403
|
+
const n = pages.length;
|
|
404
|
+
if (!next)
|
|
405
|
+
return n === 0 ? [] : null;
|
|
406
|
+
if ("cursor" in next) {
|
|
407
|
+
if (n === 0)
|
|
408
|
+
return [];
|
|
409
|
+
const v = resolvePath(pages[n - 1], next.cursor)[0];
|
|
410
|
+
return typeof v === "string" && v !== "" ? [[next.param, v]] : null;
|
|
411
|
+
}
|
|
412
|
+
const size = next.size;
|
|
413
|
+
const rows = n === 0 ? 0 : resolvePath(pages[n - 1], c.results).length;
|
|
414
|
+
if (n > 0 && (rows === 0 || (size !== undefined && rows < size)))
|
|
415
|
+
return null;
|
|
416
|
+
const sized = next.size_param !== undefined && size !== undefined ? [[next.size_param, String(size)]] : [];
|
|
417
|
+
if ("page_param" in next)
|
|
418
|
+
return [[next.page_param, String((next.first_page ?? 1) + n)], ...sized];
|
|
419
|
+
return [[next.offset_param, String(n * next.size)], ...sized];
|
|
420
|
+
}
|
|
421
|
+
// ---------------------------------------------------------------- XML (§2b)
|
|
422
|
+
const XML_WS = " \t\r\n";
|
|
423
|
+
const trimXml = (s) => {
|
|
424
|
+
let a = 0;
|
|
425
|
+
let b = s.length;
|
|
426
|
+
while (a < b && XML_WS.includes(s[a]))
|
|
427
|
+
a++;
|
|
428
|
+
while (b > a && XML_WS.includes(s[b - 1]))
|
|
429
|
+
b--;
|
|
430
|
+
return s.slice(a, b);
|
|
431
|
+
};
|
|
432
|
+
const XML_NAME = /^[A-Za-z_:][A-Za-z0-9_.:-]*/;
|
|
433
|
+
const ENTITIES = { lt: "<", gt: ">", amp: "&", quot: '"', apos: "'" };
|
|
434
|
+
function xmlDecode(s) {
|
|
435
|
+
let out = "";
|
|
436
|
+
let i = 0;
|
|
437
|
+
for (;;) {
|
|
438
|
+
const amp = s.indexOf("&", i);
|
|
439
|
+
if (amp < 0)
|
|
440
|
+
return out + s.slice(i);
|
|
441
|
+
out += s.slice(i, amp);
|
|
442
|
+
const semi = s.indexOf(";", amp);
|
|
443
|
+
const ref = semi < 0 ? "" : s.slice(amp + 1, semi);
|
|
444
|
+
let ch;
|
|
445
|
+
if (/^#x[0-9A-Fa-f]{1,6}$/.test(ref) || /^#[0-9]{1,7}$/.test(ref)) {
|
|
446
|
+
const cp = ref[1] === "x" ? Number.parseInt(ref.slice(2), 16) : Number.parseInt(ref.slice(1), 10);
|
|
447
|
+
const ok = cp === 0x9 ||
|
|
448
|
+
cp === 0xa ||
|
|
449
|
+
cp === 0xd ||
|
|
450
|
+
(cp >= 0x20 && cp <= 0xd7ff) ||
|
|
451
|
+
(cp >= 0xe000 && cp <= 0xfffd) ||
|
|
452
|
+
(cp >= 0x10000 && cp <= 0x10ffff);
|
|
453
|
+
if (ok)
|
|
454
|
+
ch = String.fromCodePoint(cp);
|
|
455
|
+
}
|
|
456
|
+
else
|
|
457
|
+
ch = ENTITIES[ref];
|
|
458
|
+
if (ch === undefined)
|
|
459
|
+
throw new Error(`XML: unknown entity &${ref.slice(0, 16)};`);
|
|
460
|
+
out += ch;
|
|
461
|
+
i = semi + 1;
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
/** spec/money.md §2b: XML as `{root: [value]}`. No DTDs, so no external or expanding entities. */
|
|
465
|
+
export function parseXml(text) {
|
|
466
|
+
let i = text.charCodeAt(0) === 0xfeff ? 1 : 0;
|
|
467
|
+
const stack = [];
|
|
468
|
+
let root = null;
|
|
469
|
+
const fail = (msg) => {
|
|
470
|
+
throw new Error(`XML: ${msg} (at ${i})`);
|
|
471
|
+
};
|
|
472
|
+
const finish = (f) => {
|
|
473
|
+
const value = f.attrs.length === 0 && f.children.size === 0
|
|
474
|
+
? trimXml(f.text)
|
|
475
|
+
: Object.fromEntries([
|
|
476
|
+
...f.attrs.map(([k, v]) => [`@${k}`, v]),
|
|
477
|
+
...(trimXml(f.text) !== "" ? [["#text", trimXml(f.text)]] : []),
|
|
478
|
+
...f.children.entries(),
|
|
479
|
+
]);
|
|
480
|
+
const parent = stack.at(-1);
|
|
481
|
+
if (parent) {
|
|
482
|
+
const list = parent.children.get(f.name);
|
|
483
|
+
if (list)
|
|
484
|
+
list.push(value);
|
|
485
|
+
else
|
|
486
|
+
parent.children.set(f.name, [value]);
|
|
487
|
+
}
|
|
488
|
+
else
|
|
489
|
+
root = [f.name, value];
|
|
490
|
+
};
|
|
491
|
+
const name = () => {
|
|
492
|
+
const m = XML_NAME.exec(text.slice(i, i + 256));
|
|
493
|
+
if (!m)
|
|
494
|
+
fail("a name was expected");
|
|
495
|
+
i += m[0].length;
|
|
496
|
+
return m[0];
|
|
497
|
+
};
|
|
498
|
+
const skipWs = () => {
|
|
499
|
+
while (i < text.length && XML_WS.includes(text[i]))
|
|
500
|
+
i++;
|
|
501
|
+
};
|
|
502
|
+
while (i < text.length) {
|
|
503
|
+
if (text.startsWith("<!--", i)) {
|
|
504
|
+
const end = text.indexOf("-->", i + 4);
|
|
505
|
+
if (end < 0)
|
|
506
|
+
fail("an unclosed comment");
|
|
507
|
+
i = end + 3;
|
|
508
|
+
}
|
|
509
|
+
else if (text.startsWith("<![CDATA[", i)) {
|
|
510
|
+
const end = text.indexOf("]]>", i + 9);
|
|
511
|
+
const top = stack.at(-1);
|
|
512
|
+
if (end < 0 || !top)
|
|
513
|
+
fail("CDATA outside an element, or unclosed");
|
|
514
|
+
top.text += text.slice(i + 9, end);
|
|
515
|
+
i = end + 3;
|
|
516
|
+
}
|
|
517
|
+
else if (text.startsWith("<!", i))
|
|
518
|
+
fail("DTDs and declarations aren't allowed");
|
|
519
|
+
else if (text.startsWith("<?", i)) {
|
|
520
|
+
const end = text.indexOf("?>", i + 2);
|
|
521
|
+
if (end < 0)
|
|
522
|
+
fail("an unclosed processing instruction");
|
|
523
|
+
i = end + 2;
|
|
524
|
+
}
|
|
525
|
+
else if (text.startsWith("</", i)) {
|
|
526
|
+
i += 2;
|
|
527
|
+
const n = name();
|
|
528
|
+
skipWs();
|
|
529
|
+
if (text[i] !== ">")
|
|
530
|
+
fail("'>' was expected");
|
|
531
|
+
i++;
|
|
532
|
+
const f = stack.pop();
|
|
533
|
+
if (!f || f.name !== n)
|
|
534
|
+
fail(`</${n}> doesn't close the open element`);
|
|
535
|
+
finish(f);
|
|
536
|
+
}
|
|
537
|
+
else if (text[i] === "<") {
|
|
538
|
+
i++;
|
|
539
|
+
if (root !== null)
|
|
540
|
+
fail("more than one root element");
|
|
541
|
+
if (stack.length >= 64)
|
|
542
|
+
fail("nesting deeper than 64");
|
|
543
|
+
const f = { name: name(), attrs: [], text: "", children: new Map() };
|
|
544
|
+
for (;;) {
|
|
545
|
+
skipWs();
|
|
546
|
+
if (text.startsWith("/>", i)) {
|
|
547
|
+
i += 2;
|
|
548
|
+
stack.push(f);
|
|
549
|
+
finish(stack.pop());
|
|
550
|
+
break;
|
|
551
|
+
}
|
|
552
|
+
if (text[i] === ">") {
|
|
553
|
+
i++;
|
|
554
|
+
stack.push(f);
|
|
555
|
+
break;
|
|
556
|
+
}
|
|
557
|
+
const a = name();
|
|
558
|
+
skipWs();
|
|
559
|
+
if (text[i] !== "=")
|
|
560
|
+
fail("'=' was expected");
|
|
561
|
+
i++;
|
|
562
|
+
skipWs();
|
|
563
|
+
const q = text[i];
|
|
564
|
+
if (q !== '"' && q !== "'")
|
|
565
|
+
fail("a quoted value was expected");
|
|
566
|
+
const end = text.indexOf(q, i + 1);
|
|
567
|
+
if (end < 0)
|
|
568
|
+
fail("an unclosed attribute value");
|
|
569
|
+
const raw = text.slice(i + 1, end);
|
|
570
|
+
if (raw.includes("<"))
|
|
571
|
+
fail("'<' in an attribute value");
|
|
572
|
+
if (f.attrs.some(([k]) => k === a))
|
|
573
|
+
fail(`attribute ${a} twice`);
|
|
574
|
+
f.attrs.push([a, xmlDecode(raw)]);
|
|
575
|
+
i = end + 1;
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
else {
|
|
579
|
+
const next = text.indexOf("<", i);
|
|
580
|
+
const chunk = text.slice(i, next < 0 ? text.length : next);
|
|
581
|
+
const top = stack.at(-1);
|
|
582
|
+
if (top)
|
|
583
|
+
top.text += xmlDecode(chunk);
|
|
584
|
+
else if (trimXml(chunk) !== "")
|
|
585
|
+
fail("text outside the root element");
|
|
586
|
+
i = next < 0 ? text.length : next;
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
if (stack.length)
|
|
590
|
+
fail(`<${stack.at(-1).name}> isn't closed`);
|
|
591
|
+
if (root === null)
|
|
592
|
+
fail("no root element");
|
|
593
|
+
const [rootName, rootValue] = root;
|
|
594
|
+
return Object.fromEntries([[rootName, [rootValue]]]);
|
|
595
|
+
}
|
|
596
|
+
const xmlEscape = (v) => v.replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c]);
|
|
597
|
+
const jsonEscape = (v) => JSON.stringify(v).slice(1, -1);
|
|
598
|
+
/** spec/money.md §2b: the next request a connector makes, or null when there are no more. */
|
|
599
|
+
export function requestFor(c, pages, from, to) {
|
|
600
|
+
const params = pageParams(c, pages);
|
|
601
|
+
if (params === null)
|
|
602
|
+
return null;
|
|
603
|
+
const unix = (d) => String(Date.parse(`${d}T00:00:00Z`) / 1000);
|
|
604
|
+
const vars = { from, to, from_unix: unix(from), to_unix: unix(to) };
|
|
605
|
+
const url = c.request.url.replace(/\{(from|to|from_unix|to_unix)\}/g, (_, k) => encodeURIComponent(vars[k]));
|
|
606
|
+
if ((c.request.method ?? "GET") === "GET")
|
|
607
|
+
return { method: "GET", url, query: params };
|
|
608
|
+
const type = c.request.content_type.toLowerCase();
|
|
609
|
+
const esc = type.includes("xml")
|
|
610
|
+
? xmlEscape
|
|
611
|
+
: type.includes("json")
|
|
612
|
+
? jsonEscape
|
|
613
|
+
: (v) => v;
|
|
614
|
+
const next = c.next;
|
|
615
|
+
const vals = {
|
|
616
|
+
...vars,
|
|
617
|
+
next: params[0]?.[1] ?? "",
|
|
618
|
+
size: next && "size" in next && next.size !== undefined ? String(next.size) : "",
|
|
619
|
+
};
|
|
620
|
+
const body = c.request.body.replace(/\{(from|to|from_unix|to_unix|next|size)\}/g, (_, k) => esc(vals[k]));
|
|
621
|
+
return { method: "POST", url, query: [], body, content_type: c.request.content_type };
|
|
622
|
+
}
|