@phosphorco/bb-thread-metrics 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRACT.md +318 -0
- package/LICENSE +21 -0
- package/README.md +30 -2
- package/dist/bb.js +197 -0
- package/dist/index.js +304 -0
- package/dist/testing.js +81 -0
- package/dist/types/bb.d.ts +286 -0
- package/dist/types/index.d.ts +940 -0
- package/dist/types/testing.d.ts +20 -0
- package/package.json +22 -4
package/dist/index.js
ADDED
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
// src/index.ts
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { UsageVectorSchema } from "@phosphorco/bb-usage-algebra/vector";
|
|
4
|
+
var LIMITS = Object.freeze({ threadIds: 200, seriesPoints: 288, detailRows: 200, inputBytes: 65536, outputBytes: 2097152, describeBytes: 16384, candidates: 128, listedPlugins: 1e4, concurrency: 4, callMs: 2000, discoveryMs: 5000, familyEdges: 50000, familyMembers: 1e4, familyDepth: 256 });
|
|
5
|
+
var METRIC_BUCKET_MS = 300000;
|
|
6
|
+
var WINDOW_BUCKETS = { m20: 4, h24: 288, d7: 2016, lifetime: 2016 };
|
|
7
|
+
function retainedWindowStart(asOf, window) {
|
|
8
|
+
time.parse(asOf);
|
|
9
|
+
windowSchema.parse(window);
|
|
10
|
+
return Math.max(0, Math.floor(asOf / METRIC_BUCKET_MS) * METRIC_BUCKET_MS - (WINDOW_BUCKETS[window] - 1) * METRIC_BUCKET_MS);
|
|
11
|
+
}
|
|
12
|
+
var METHODS = Object.freeze({ describe: "threadMetricsDescribe", read: "threadMetricsReadV1", detail: "threadMetricsDetailV1" });
|
|
13
|
+
var id = z.string().min(1).max(200).regex(/^[A-Za-z0-9][A-Za-z0-9_.:-]*$/);
|
|
14
|
+
var text = z.string().min(1).max(256);
|
|
15
|
+
var time = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER);
|
|
16
|
+
var money = z.number().finite().nonnegative().nullable();
|
|
17
|
+
var scope = z.enum(["own", "family"]);
|
|
18
|
+
var windowSchema = z.enum(["m20", "h24", "d7", "lifetime"]);
|
|
19
|
+
var coverageSchema = z.enum(["complete", "partial", "unknown"]);
|
|
20
|
+
var coverageReasonSchema = z.enum(["source-gap", "root-unattested", "history-unavailable", "member-cap", "depth-cap", "edge-cap", "missing-thread", "unpriced", "retention", "row-cap", "pagination", "population-incomplete"]);
|
|
21
|
+
var reasons = z.array(coverageReasonSchema).max(16);
|
|
22
|
+
var approximationSchema = z.object({
|
|
23
|
+
approximate: z.boolean(),
|
|
24
|
+
reasons: z.array(z.enum(["closest-model", "provider-default", "model-attribution", "bucket-boundary", "missing-price", "source-gap"])).max(6),
|
|
25
|
+
rates: z.array(z.object({ providerId: id, modelId: text, rateId: text.nullable(), basis: z.enum(["matched", "closest-model", "provider-default", "unpriced"]) }).superRefine((rate, ctx) => {
|
|
26
|
+
if (rate.basis === "unpriced" !== (rate.rateId === null))
|
|
27
|
+
ctx.addIssue({ code: "custom", path: ["rateId"], message: "Priced basis requires rateId; unpriced basis requires null" });
|
|
28
|
+
})).max(32).optional()
|
|
29
|
+
}).strip().superRefine((v, ctx) => {
|
|
30
|
+
if (!v.approximate && (v.reasons.length || v.rates?.some((r) => r.basis !== "matched")))
|
|
31
|
+
ctx.addIssue({ code: "custom", message: "Approximation provenance requires approximate flag" });
|
|
32
|
+
});
|
|
33
|
+
var rank = z.number().finite().min(0).max(1).nullable();
|
|
34
|
+
var ranksSchema = z.object({ m20: rank, h24: rank, d7: rank }).strip();
|
|
35
|
+
var population = z.object({ asOf: time, count: z.number().int().nonnegative(), coverage: coverageSchema }).strip();
|
|
36
|
+
var rankPopulationSchema = z.object({ m20: population, h24: population, d7: population }).strip();
|
|
37
|
+
var windowValueSchema = z.object({ tokens: UsageVectorSchema, amount: money }).strip();
|
|
38
|
+
var windowsSchema = z.object({ m20: windowValueSchema, h24: windowValueSchema, d7: windowValueSchema, lifetime: windowValueSchema }).strip();
|
|
39
|
+
var bucketSchema = z.object({ start: time, amount: money }).strip();
|
|
40
|
+
var metadata = {
|
|
41
|
+
threadId: id,
|
|
42
|
+
scope,
|
|
43
|
+
memberCount: z.number().int().nonnegative().max(LIMITS.familyMembers),
|
|
44
|
+
coverage: coverageSchema,
|
|
45
|
+
coverageReasons: reasons,
|
|
46
|
+
excluded: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).optional(),
|
|
47
|
+
asOf: time,
|
|
48
|
+
revision: text,
|
|
49
|
+
priceVersion: text,
|
|
50
|
+
currency: z.literal("USD"),
|
|
51
|
+
estimated: z.literal(true),
|
|
52
|
+
approximation: approximationSchema,
|
|
53
|
+
ranks: ranksSchema,
|
|
54
|
+
rankPopulation: rankPopulationSchema,
|
|
55
|
+
reported: z.object({ amount: z.number().finite().nonnegative(), currency: z.string().min(1).max(16), asOf: time }).strip().optional()
|
|
56
|
+
};
|
|
57
|
+
var summaryRowSchema = z.discriminatedUnion("changed", [
|
|
58
|
+
z.object({ ...metadata, changed: z.literal(true), windows: windowsSchema, buckets: z.array(bucketSchema).max(LIMITS.seriesPoints).optional() }).strip(),
|
|
59
|
+
z.object({ ...metadata, changed: z.literal(false), windows: z.never().optional(), buckets: z.never().optional() }).strip()
|
|
60
|
+
]);
|
|
61
|
+
var readInputSchema = z.object({
|
|
62
|
+
threadIds: z.array(id).min(1).max(LIMITS.threadIds),
|
|
63
|
+
scope,
|
|
64
|
+
includeArchived: z.boolean().optional(),
|
|
65
|
+
knownRevisions: z.record(id, text).optional(),
|
|
66
|
+
series: z.object({ activeThreadId: id, maxPoints: z.number().int().min(1).max(LIMITS.seriesPoints) }).strict().optional()
|
|
67
|
+
}).strict().superRefine((v, ctx) => {
|
|
68
|
+
if (new Set(v.threadIds).size !== v.threadIds.length)
|
|
69
|
+
ctx.addIssue({ code: "custom", message: "Duplicate thread IDs" });
|
|
70
|
+
if (v.series && !v.threadIds.includes(v.series.activeThreadId))
|
|
71
|
+
ctx.addIssue({ code: "custom", message: "Series requires requested active thread" });
|
|
72
|
+
if (v.knownRevisions && Object.keys(v.knownRevisions).some((k) => !v.threadIds.includes(k)))
|
|
73
|
+
ctx.addIssue({ code: "custom", message: "Unrequested revision key" });
|
|
74
|
+
});
|
|
75
|
+
var readResultSchema = z.object({ version: z.literal(1), rows: z.array(summaryRowSchema).max(LIMITS.threadIds) }).strip();
|
|
76
|
+
var detailInputSchema = z.object({ threadId: id, scope, includeArchived: z.boolean().optional(), axis: z.enum(["model", "time"]), window: windowSchema, cursor: z.string().min(1).max(1024).optional(), limit: z.number().int().min(1).max(LIMITS.detailRows).optional() }).strict();
|
|
77
|
+
var detailBase = { amount: money, tokens: UsageVectorSchema, approximation: approximationSchema };
|
|
78
|
+
var detailRowSchema = z.discriminatedUnion("axis", [
|
|
79
|
+
z.object({ ...detailBase, axis: z.literal("model"), providerId: id, modelId: text }).strip(),
|
|
80
|
+
z.object({ ...detailBase, axis: z.literal("time"), start: time, end: time }).strip().refine((v) => v.end > v.start, { message: "Empty time interval" })
|
|
81
|
+
]);
|
|
82
|
+
var detailResultSchema = z.object({ version: z.literal(1), threadId: id, scope, axis: z.enum(["model", "time"]), window: windowSchema, asOf: time, revision: text, priceVersion: text, currency: z.literal("USD"), estimated: z.literal(true), coverage: coverageSchema, coverageReasons: reasons, excluded: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).optional(), approximation: approximationSchema, total: windowValueSchema, rows: z.array(detailRowSchema).max(LIMITS.detailRows), nextCursor: z.string().min(1).max(1024).optional() }).strip();
|
|
83
|
+
var describeInputSchema = z.object({}).strict();
|
|
84
|
+
var describeEnvelopeSchema = z.object({ protocol: z.literal("bb-thread-metrics"), versions: z.array(z.number().int().positive()).max(16) }).passthrough();
|
|
85
|
+
var capabilitiesSchema = z.object({ summary: z.literal(true), series: z.boolean(), detail: z.boolean() }).strip();
|
|
86
|
+
var describeV1Schema = describeEnvelopeSchema.extend({ capabilities: capabilitiesSchema });
|
|
87
|
+
function parseBounded(schema, value, maxBytes) {
|
|
88
|
+
const json = JSON.stringify(value);
|
|
89
|
+
if (json === undefined || new TextEncoder().encode(json).byteLength > maxBytes) {
|
|
90
|
+
const error = new Error("Payload byte limit");
|
|
91
|
+
error.name = "MetricsCodecError";
|
|
92
|
+
throw error;
|
|
93
|
+
}
|
|
94
|
+
return schema.parse(value);
|
|
95
|
+
}
|
|
96
|
+
function negotiateVersion(value) {
|
|
97
|
+
const env = describeEnvelopeSchema.safeParse(value);
|
|
98
|
+
if (!env.success || !env.data.versions.includes(1))
|
|
99
|
+
return { kind: "incompatible" };
|
|
100
|
+
const v = describeV1Schema.safeParse(value);
|
|
101
|
+
return v.success ? { kind: "ready", version: 1, capabilities: v.data.capabilities } : { kind: "incompatible" };
|
|
102
|
+
}
|
|
103
|
+
function dollarPercentiles(population2) {
|
|
104
|
+
const seen = new Set;
|
|
105
|
+
for (const p of population2) {
|
|
106
|
+
if (seen.has(p.threadId))
|
|
107
|
+
throw new Error("Duplicate population thread");
|
|
108
|
+
seen.add(p.threadId);
|
|
109
|
+
if (p.amount !== null && (!Number.isFinite(p.amount) || p.amount < 0))
|
|
110
|
+
throw new Error("Invalid dollars");
|
|
111
|
+
}
|
|
112
|
+
const active = population2.filter((p) => p.amount !== null && p.amount > 0);
|
|
113
|
+
const sorted = active.map((p) => p.amount).sort((a, b) => a - b);
|
|
114
|
+
const lower = new Map;
|
|
115
|
+
sorted.forEach((x, i) => {
|
|
116
|
+
if (!lower.has(x))
|
|
117
|
+
lower.set(x, i);
|
|
118
|
+
});
|
|
119
|
+
return new Map(population2.map((p) => [p.threadId, p.amount === null || p.amount === 0 ? null : active.length < 2 ? 0 : lower.get(p.amount) / (active.length - 1)]));
|
|
120
|
+
}
|
|
121
|
+
function heat(ranks, gates = { m20: 0.9, h24: 0.9, d7: 0.9 }) {
|
|
122
|
+
for (const w of ["m20", "h24", "d7"])
|
|
123
|
+
if (!Number.isFinite(gates[w]) || gates[w] < 0 || gates[w] > 1)
|
|
124
|
+
throw new Error("Invalid gate");
|
|
125
|
+
return ranks.m20 !== null && ranks.m20 >= gates.m20 ? "on-fire" : ranks.h24 !== null && ranks.h24 >= gates.h24 ? "hot" : ranks.d7 !== null && ranks.d7 >= gates.d7 ? "heavy" : null;
|
|
126
|
+
}
|
|
127
|
+
function familyClosure(edges, roots, policy = {}) {
|
|
128
|
+
const cap = (n, fallback) => {
|
|
129
|
+
const v = n ?? fallback;
|
|
130
|
+
if (!Number.isInteger(v) || v < 1 || v > fallback)
|
|
131
|
+
throw new Error("Invalid family cap");
|
|
132
|
+
return v;
|
|
133
|
+
};
|
|
134
|
+
const maxMembers = cap(policy.maxMembers, LIMITS.familyMembers), maxDepth = cap(policy.maxDepth, LIMITS.familyDepth), maxEdges = cap(policy.maxEdges, LIMITS.familyEdges);
|
|
135
|
+
if (roots.length > LIMITS.threadIds)
|
|
136
|
+
throw new Error("Too many roots");
|
|
137
|
+
const why = new Set, nodes = new Map, children = new Map;
|
|
138
|
+
if (edges.length > maxEdges)
|
|
139
|
+
why.add("edge-cap");
|
|
140
|
+
for (const e of edges.slice(0, maxEdges)) {
|
|
141
|
+
if (nodes.has(e.threadId))
|
|
142
|
+
throw new Error("Duplicate edge thread");
|
|
143
|
+
nodes.set(e.threadId, e);
|
|
144
|
+
for (const p of [e.parentThreadId, e.lifecycleOwnerThreadId])
|
|
145
|
+
if (p) {
|
|
146
|
+
let c = children.get(p);
|
|
147
|
+
if (!c)
|
|
148
|
+
children.set(p, c = new Set);
|
|
149
|
+
c.add(e.threadId);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
const queue = Array.from(new Set(roots), (threadId) => ({ threadId, depth: 0 })), seen = new Set, members = [];
|
|
153
|
+
for (let i = 0;i < queue.length; i++) {
|
|
154
|
+
const { threadId, depth } = queue[i];
|
|
155
|
+
if (seen.has(threadId))
|
|
156
|
+
continue;
|
|
157
|
+
seen.add(threadId);
|
|
158
|
+
const node = nodes.get(threadId);
|
|
159
|
+
if (!node) {
|
|
160
|
+
why.add("missing-thread");
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
if (node.deleted)
|
|
164
|
+
continue;
|
|
165
|
+
if (policy.includeArchived !== false || !node.archived) {
|
|
166
|
+
if (members.length >= maxMembers) {
|
|
167
|
+
why.add("member-cap");
|
|
168
|
+
break;
|
|
169
|
+
}
|
|
170
|
+
members.push(threadId);
|
|
171
|
+
}
|
|
172
|
+
const next = children.get(threadId);
|
|
173
|
+
if (next) {
|
|
174
|
+
if (depth >= maxDepth) {
|
|
175
|
+
if ([...next].some((c) => !seen.has(c)))
|
|
176
|
+
why.add("depth-cap");
|
|
177
|
+
} else
|
|
178
|
+
for (const c of next)
|
|
179
|
+
if (!seen.has(c))
|
|
180
|
+
queue.push({ threadId: c, depth: depth + 1 });
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
const source = coverageSchema.parse(policy.sourceCoverage ?? "complete");
|
|
184
|
+
if (source === "partial")
|
|
185
|
+
why.add("source-gap");
|
|
186
|
+
if (source === "unknown")
|
|
187
|
+
why.add("history-unavailable");
|
|
188
|
+
return { members, coverage: source === "unknown" ? "unknown" : why.size ? "partial" : "complete", coverageReasons: [...why] };
|
|
189
|
+
}
|
|
190
|
+
function validateReadResult(input, value) {
|
|
191
|
+
const out = parseBounded(readResultSchema, value, LIMITS.outputBytes), seen = new Set;
|
|
192
|
+
for (const row of out.rows) {
|
|
193
|
+
if (!input.threadIds.includes(row.threadId) || row.scope !== input.scope || seen.has(row.threadId))
|
|
194
|
+
throw new Error("Unrequested, duplicate or wrong-scope row");
|
|
195
|
+
seen.add(row.threadId);
|
|
196
|
+
if (!row.changed && input.knownRevisions?.[row.threadId] !== row.revision)
|
|
197
|
+
throw new Error("Unmatched unchanged revision");
|
|
198
|
+
if (row.changed && Object.values(row.windows).some((w) => w.amount === null) && !row.approximation.reasons.includes("missing-price"))
|
|
199
|
+
throw new Error("Unknown dollars require missing-price provenance");
|
|
200
|
+
if (row.changed && row.buckets) {
|
|
201
|
+
if (input.series?.activeThreadId !== row.threadId || row.buckets.length > input.series.maxPoints)
|
|
202
|
+
throw new Error("Unrequested series");
|
|
203
|
+
for (let i = 0;i < row.buckets.length; i++) {
|
|
204
|
+
const b = row.buckets[i];
|
|
205
|
+
if (b.amount === null && !row.approximation.reasons.includes("missing-price"))
|
|
206
|
+
throw new Error("Unknown bucket dollars require missing-price provenance");
|
|
207
|
+
if (b.start % METRIC_BUCKET_MS !== 0 || b.start > row.asOf || b.start < retainedWindowStart(row.asOf, "h24") || i > 0 && b.start <= row.buckets[i - 1].start)
|
|
208
|
+
throw new Error("Invalid series range/order");
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
return out;
|
|
213
|
+
}
|
|
214
|
+
function validateDetailResult(input, value) {
|
|
215
|
+
const out = parseBounded(detailResultSchema, value, LIMITS.outputBytes);
|
|
216
|
+
if (out.total.amount === null && !out.approximation.reasons.includes("missing-price"))
|
|
217
|
+
throw new Error("Unknown total dollars require missing-price provenance");
|
|
218
|
+
if (out.nextCursor && out.coverage === "complete")
|
|
219
|
+
throw new Error("Pagination must disclose incomplete coverage");
|
|
220
|
+
if (out.threadId !== input.threadId || out.scope !== input.scope || out.axis !== input.axis || out.window !== input.window || out.rows.length > (input.limit ?? LIMITS.detailRows))
|
|
221
|
+
throw new Error("Detail request mismatch");
|
|
222
|
+
let priorEnd;
|
|
223
|
+
const keys = new Set;
|
|
224
|
+
for (const row of out.rows) {
|
|
225
|
+
if (row.approximation.approximate && (!out.approximation.approximate || row.approximation.reasons.some((reason) => !out.approximation.reasons.includes(reason))))
|
|
226
|
+
throw new Error("Aggregate total must retain contributing row approximation provenance");
|
|
227
|
+
if (row.amount === null && !row.approximation.reasons.includes("missing-price"))
|
|
228
|
+
throw new Error("Unknown dollars require missing-price provenance");
|
|
229
|
+
if (row.axis !== input.axis)
|
|
230
|
+
throw new Error("Detail axis mismatch");
|
|
231
|
+
if (row.axis === "time") {
|
|
232
|
+
if (row.start < retainedWindowStart(out.asOf, input.window) || row.start >= out.asOf || row.end > out.asOf || row.start % METRIC_BUCKET_MS !== 0 || row.end !== out.asOf && row.end % METRIC_BUCKET_MS !== 0 || priorEnd !== undefined && row.start < priorEnd)
|
|
233
|
+
throw new Error("Invalid time detail range/order/overlap");
|
|
234
|
+
priorEnd = row.end;
|
|
235
|
+
}
|
|
236
|
+
const key = row.axis === "model" ? JSON.stringify([row.providerId, row.modelId]) : String(row.start);
|
|
237
|
+
if (keys.has(key))
|
|
238
|
+
throw new Error("Duplicate detail row");
|
|
239
|
+
keys.add(key);
|
|
240
|
+
}
|
|
241
|
+
if (out.coverage === "complete" && !input.cursor && !out.nextCursor) {
|
|
242
|
+
for (const key of ["inputUncached", "cacheRead", "cacheWrite", "output"]) {
|
|
243
|
+
const sum = out.rows.reduce((n, r) => n + r.tokens[key], 0);
|
|
244
|
+
if (!Number.isFinite(sum) || Math.abs(sum - out.total.tokens[key]) > 0.00000001 * Math.max(1, sum, out.total.tokens[key]))
|
|
245
|
+
throw new Error("Detail token total mismatch");
|
|
246
|
+
}
|
|
247
|
+
const hasUnknown = out.rows.some((r) => r.amount === null);
|
|
248
|
+
if (hasUnknown && out.total.amount !== null)
|
|
249
|
+
throw new Error("Unknown detail dollars must remain unknown");
|
|
250
|
+
if (!hasUnknown) {
|
|
251
|
+
const sum = out.rows.reduce((n, r) => n + (r.amount ?? 0), 0);
|
|
252
|
+
if (out.total.amount === null || !Number.isFinite(sum) || Math.abs(sum - out.total.amount) > 0.00000001 * Math.max(1, sum, out.total.amount))
|
|
253
|
+
throw new Error("Detail dollar total mismatch");
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
return out;
|
|
257
|
+
}
|
|
258
|
+
function classifyOwnerError(error) {
|
|
259
|
+
const e = error;
|
|
260
|
+
if (e?.status === 404 || e?.code === "unknown_method")
|
|
261
|
+
return "absent";
|
|
262
|
+
if (e?.status === 503)
|
|
263
|
+
return "disabled";
|
|
264
|
+
if (e?.name === "TimeoutError")
|
|
265
|
+
return "timeout";
|
|
266
|
+
if (e?.name === "AbortError")
|
|
267
|
+
return "aborted";
|
|
268
|
+
if (e?.name === "ZodError" || e?.name === "MetricsCodecError")
|
|
269
|
+
return "malformed";
|
|
270
|
+
return "failed";
|
|
271
|
+
}
|
|
272
|
+
export {
|
|
273
|
+
windowsSchema,
|
|
274
|
+
windowValueSchema,
|
|
275
|
+
windowSchema,
|
|
276
|
+
validateReadResult,
|
|
277
|
+
validateDetailResult,
|
|
278
|
+
summaryRowSchema,
|
|
279
|
+
retainedWindowStart,
|
|
280
|
+
readResultSchema,
|
|
281
|
+
readInputSchema,
|
|
282
|
+
ranksSchema,
|
|
283
|
+
rankPopulationSchema,
|
|
284
|
+
parseBounded,
|
|
285
|
+
negotiateVersion,
|
|
286
|
+
heat,
|
|
287
|
+
familyClosure,
|
|
288
|
+
dollarPercentiles,
|
|
289
|
+
detailRowSchema,
|
|
290
|
+
detailResultSchema,
|
|
291
|
+
detailInputSchema,
|
|
292
|
+
describeV1Schema,
|
|
293
|
+
describeInputSchema,
|
|
294
|
+
describeEnvelopeSchema,
|
|
295
|
+
coverageSchema,
|
|
296
|
+
coverageReasonSchema,
|
|
297
|
+
classifyOwnerError,
|
|
298
|
+
capabilitiesSchema,
|
|
299
|
+
bucketSchema,
|
|
300
|
+
approximationSchema,
|
|
301
|
+
METRIC_BUCKET_MS,
|
|
302
|
+
METHODS,
|
|
303
|
+
LIMITS
|
|
304
|
+
};
|
package/dist/testing.js
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// src/testing.ts
|
|
2
|
+
import { createFakePluginHost } from "@get-bb/plugin-sdk/testing";
|
|
3
|
+
import { METHODS, LIMITS, readInputSchema, detailInputSchema, describeEnvelopeSchema, parseBounded, negotiateVersion, validateReadResult, validateDetailResult } from "./index.js";
|
|
4
|
+
async function runSupplierConformance(options) {
|
|
5
|
+
if (!options.readCases.length)
|
|
6
|
+
throw new Error("At least one real read case required");
|
|
7
|
+
const host = createFakePluginHost({ pluginId: options.pluginId }), checks = [];
|
|
8
|
+
const deadline = options.deadlineMs ?? LIMITS.callMs;
|
|
9
|
+
if (!Number.isInteger(deadline) || deadline < 1 || deadline > LIMITS.discoveryMs)
|
|
10
|
+
throw new Error("Invalid conformance deadline");
|
|
11
|
+
const call = (method, input) => host.harness.behavior.callRpc(method, input);
|
|
12
|
+
async function check(name, fn) {
|
|
13
|
+
let timer;
|
|
14
|
+
try {
|
|
15
|
+
await Promise.race([fn(), new Promise((_, reject) => {
|
|
16
|
+
timer = setTimeout(() => reject(new Error("Conformance deadline")), deadline);
|
|
17
|
+
})]);
|
|
18
|
+
checks.push({ name, passed: true });
|
|
19
|
+
} catch (e) {
|
|
20
|
+
checks.push({ name, passed: false, error: e instanceof Error ? e.message : String(e) });
|
|
21
|
+
} finally {
|
|
22
|
+
clearTimeout(timer);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
function validationError(e) {
|
|
26
|
+
const v = e;
|
|
27
|
+
return v.name === "ZodError" && Array.isArray(v.issues) || v.code === "invalid_input" || v.body?.code === "invalid_input";
|
|
28
|
+
}
|
|
29
|
+
async function rejectsInput(method, input) {
|
|
30
|
+
try {
|
|
31
|
+
await call(method, input);
|
|
32
|
+
} catch (e) {
|
|
33
|
+
if (validationError(e))
|
|
34
|
+
return;
|
|
35
|
+
throw new Error("Expected structured invalid_input or ZodError");
|
|
36
|
+
}
|
|
37
|
+
throw new Error("Invalid input accepted");
|
|
38
|
+
}
|
|
39
|
+
try {
|
|
40
|
+
await options.register(host.bb);
|
|
41
|
+
let detail = false;
|
|
42
|
+
await check("describe-negotiation", async () => {
|
|
43
|
+
const n = negotiateVersion(parseBounded(describeEnvelopeSchema, await call(METHODS.describe, {}), LIMITS.describeBytes));
|
|
44
|
+
if (n.kind !== "ready")
|
|
45
|
+
throw new Error("No compatible v1");
|
|
46
|
+
detail = n.capabilities.detail;
|
|
47
|
+
});
|
|
48
|
+
await check("strict-describe-input", () => rejectsInput(METHODS.describe, { unexpected: true }));
|
|
49
|
+
for (let i = 0;i < options.readCases.length; i++)
|
|
50
|
+
await check(`read-${i}`, async () => {
|
|
51
|
+
const input = parseBounded(readInputSchema, options.readCases[i], LIMITS.inputBytes);
|
|
52
|
+
validateReadResult(input, await call(METHODS.read, input));
|
|
53
|
+
});
|
|
54
|
+
await check("strict-read-input", () => rejectsInput(METHODS.read, { ...options.readCases[0], unexpected: true }));
|
|
55
|
+
await check("oversized-id-count", () => rejectsInput(METHODS.read, { scope: "family", threadIds: Array.from({ length: 201 }, (_, i) => `thr_${i}`) }));
|
|
56
|
+
await check("oversized-input-bytes", async () => {
|
|
57
|
+
const threadIds = Array.from({ length: 200 }, (_, i) => `thr_${i}_`.padEnd(200, "x"));
|
|
58
|
+
const oversized = { scope: "family", threadIds, knownRevisions: Object.fromEntries(threadIds.map((id) => [id, "r".repeat(256)])) };
|
|
59
|
+
readInputSchema.parse(oversized);
|
|
60
|
+
if (new TextEncoder().encode(JSON.stringify(oversized)).byteLength <= LIMITS.inputBytes)
|
|
61
|
+
throw new Error("Byte fixture is not oversized");
|
|
62
|
+
await rejectsInput(METHODS.read, oversized);
|
|
63
|
+
});
|
|
64
|
+
if (detail) {
|
|
65
|
+
if (!options.detailCases?.length)
|
|
66
|
+
checks.push({ name: "detail-cases-required", passed: false, error: "Advertised detail needs real cases" });
|
|
67
|
+
for (let i = 0;i < (options.detailCases?.length ?? 0); i++)
|
|
68
|
+
await check(`detail-${i}`, async () => {
|
|
69
|
+
const input = parseBounded(detailInputSchema, options.detailCases[i], LIMITS.inputBytes);
|
|
70
|
+
validateDetailResult(input, await call(METHODS.detail, input));
|
|
71
|
+
});
|
|
72
|
+
await check("strict-detail-input", () => rejectsInput(METHODS.detail, { ...options.detailCases?.[0], unexpected: true }));
|
|
73
|
+
}
|
|
74
|
+
} finally {
|
|
75
|
+
await host.harness.lifecycle.dispose();
|
|
76
|
+
}
|
|
77
|
+
return { passed: checks.every((c) => c.passed), checks, proof: "source-conformance" };
|
|
78
|
+
}
|
|
79
|
+
export {
|
|
80
|
+
runSupplierConformance
|
|
81
|
+
};
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
import type { BbPluginApi } from '@get-bb/plugin-sdk';
|
|
2
|
+
import type { PluginBrowserBbSdk } from '@get-bb/plugin-sdk/app';
|
|
3
|
+
import type { ReadInput, ReadResult, DetailInput, DetailResult, Capabilities, OwnerErrorKind } from './index.js';
|
|
4
|
+
export type MetricsSdk = {
|
|
5
|
+
readonly plugins: Pick<BbPluginApi['sdk']['plugins'], 'list' | 'callRpc'>;
|
|
6
|
+
} | {
|
|
7
|
+
readonly plugins: Pick<PluginBrowserBbSdk['plugins'], 'list' | 'callRpc'>;
|
|
8
|
+
};
|
|
9
|
+
export interface MetricsPort {
|
|
10
|
+
capabilities: Capabilities;
|
|
11
|
+
read(input: ReadInput): Promise<ReadResult>;
|
|
12
|
+
detail?(input: DetailInput): Promise<DetailResult>;
|
|
13
|
+
}
|
|
14
|
+
export declare function registerThreadMetricsOwner(bb: Pick<BbPluginApi, 'rpc'>, port: MetricsPort): void;
|
|
15
|
+
export declare function createThreadMetricsClient(sdk: MetricsSdk, pluginId: string): {
|
|
16
|
+
pluginId: string;
|
|
17
|
+
describe(signal?: AbortSignal): Promise<{
|
|
18
|
+
kind: "ready";
|
|
19
|
+
version: 1;
|
|
20
|
+
capabilities: Capabilities;
|
|
21
|
+
} | {
|
|
22
|
+
kind: "incompatible";
|
|
23
|
+
}>;
|
|
24
|
+
read(raw: ReadInput, signal?: AbortSignal): Promise<{
|
|
25
|
+
version: 1;
|
|
26
|
+
rows: ({
|
|
27
|
+
changed: true;
|
|
28
|
+
windows: {
|
|
29
|
+
m20: {
|
|
30
|
+
tokens: {
|
|
31
|
+
inputUncached: number;
|
|
32
|
+
cacheRead: number;
|
|
33
|
+
cacheWrite: number;
|
|
34
|
+
output: number;
|
|
35
|
+
};
|
|
36
|
+
amount: number | null;
|
|
37
|
+
};
|
|
38
|
+
h24: {
|
|
39
|
+
tokens: {
|
|
40
|
+
inputUncached: number;
|
|
41
|
+
cacheRead: number;
|
|
42
|
+
cacheWrite: number;
|
|
43
|
+
output: number;
|
|
44
|
+
};
|
|
45
|
+
amount: number | null;
|
|
46
|
+
};
|
|
47
|
+
d7: {
|
|
48
|
+
tokens: {
|
|
49
|
+
inputUncached: number;
|
|
50
|
+
cacheRead: number;
|
|
51
|
+
cacheWrite: number;
|
|
52
|
+
output: number;
|
|
53
|
+
};
|
|
54
|
+
amount: number | null;
|
|
55
|
+
};
|
|
56
|
+
lifetime: {
|
|
57
|
+
tokens: {
|
|
58
|
+
inputUncached: number;
|
|
59
|
+
cacheRead: number;
|
|
60
|
+
cacheWrite: number;
|
|
61
|
+
output: number;
|
|
62
|
+
};
|
|
63
|
+
amount: number | null;
|
|
64
|
+
};
|
|
65
|
+
};
|
|
66
|
+
threadId: string;
|
|
67
|
+
scope: "own" | "family";
|
|
68
|
+
memberCount: number;
|
|
69
|
+
coverage: "complete" | "partial" | "unknown";
|
|
70
|
+
coverageReasons: ("source-gap" | "root-unattested" | "history-unavailable" | "member-cap" | "depth-cap" | "edge-cap" | "missing-thread" | "unpriced" | "retention" | "row-cap" | "pagination" | "population-incomplete")[];
|
|
71
|
+
asOf: number;
|
|
72
|
+
revision: string;
|
|
73
|
+
priceVersion: string;
|
|
74
|
+
currency: "USD";
|
|
75
|
+
estimated: true;
|
|
76
|
+
approximation: {
|
|
77
|
+
approximate: boolean;
|
|
78
|
+
reasons: ("source-gap" | "closest-model" | "provider-default" | "model-attribution" | "bucket-boundary" | "missing-price")[];
|
|
79
|
+
rates?: {
|
|
80
|
+
providerId: string;
|
|
81
|
+
modelId: string;
|
|
82
|
+
rateId: string | null;
|
|
83
|
+
basis: "unpriced" | "closest-model" | "provider-default" | "matched";
|
|
84
|
+
}[] | undefined;
|
|
85
|
+
};
|
|
86
|
+
ranks: {
|
|
87
|
+
m20: number | null;
|
|
88
|
+
h24: number | null;
|
|
89
|
+
d7: number | null;
|
|
90
|
+
};
|
|
91
|
+
rankPopulation: {
|
|
92
|
+
m20: {
|
|
93
|
+
asOf: number;
|
|
94
|
+
count: number;
|
|
95
|
+
coverage: "complete" | "partial" | "unknown";
|
|
96
|
+
};
|
|
97
|
+
h24: {
|
|
98
|
+
asOf: number;
|
|
99
|
+
count: number;
|
|
100
|
+
coverage: "complete" | "partial" | "unknown";
|
|
101
|
+
};
|
|
102
|
+
d7: {
|
|
103
|
+
asOf: number;
|
|
104
|
+
count: number;
|
|
105
|
+
coverage: "complete" | "partial" | "unknown";
|
|
106
|
+
};
|
|
107
|
+
};
|
|
108
|
+
buckets?: {
|
|
109
|
+
start: number;
|
|
110
|
+
amount: number | null;
|
|
111
|
+
}[] | undefined;
|
|
112
|
+
excluded?: number | undefined;
|
|
113
|
+
reported?: {
|
|
114
|
+
amount: number;
|
|
115
|
+
currency: string;
|
|
116
|
+
asOf: number;
|
|
117
|
+
} | undefined;
|
|
118
|
+
} | {
|
|
119
|
+
changed: false;
|
|
120
|
+
threadId: string;
|
|
121
|
+
scope: "own" | "family";
|
|
122
|
+
memberCount: number;
|
|
123
|
+
coverage: "complete" | "partial" | "unknown";
|
|
124
|
+
coverageReasons: ("source-gap" | "root-unattested" | "history-unavailable" | "member-cap" | "depth-cap" | "edge-cap" | "missing-thread" | "unpriced" | "retention" | "row-cap" | "pagination" | "population-incomplete")[];
|
|
125
|
+
asOf: number;
|
|
126
|
+
revision: string;
|
|
127
|
+
priceVersion: string;
|
|
128
|
+
currency: "USD";
|
|
129
|
+
estimated: true;
|
|
130
|
+
approximation: {
|
|
131
|
+
approximate: boolean;
|
|
132
|
+
reasons: ("source-gap" | "closest-model" | "provider-default" | "model-attribution" | "bucket-boundary" | "missing-price")[];
|
|
133
|
+
rates?: {
|
|
134
|
+
providerId: string;
|
|
135
|
+
modelId: string;
|
|
136
|
+
rateId: string | null;
|
|
137
|
+
basis: "unpriced" | "closest-model" | "provider-default" | "matched";
|
|
138
|
+
}[] | undefined;
|
|
139
|
+
};
|
|
140
|
+
ranks: {
|
|
141
|
+
m20: number | null;
|
|
142
|
+
h24: number | null;
|
|
143
|
+
d7: number | null;
|
|
144
|
+
};
|
|
145
|
+
rankPopulation: {
|
|
146
|
+
m20: {
|
|
147
|
+
asOf: number;
|
|
148
|
+
count: number;
|
|
149
|
+
coverage: "complete" | "partial" | "unknown";
|
|
150
|
+
};
|
|
151
|
+
h24: {
|
|
152
|
+
asOf: number;
|
|
153
|
+
count: number;
|
|
154
|
+
coverage: "complete" | "partial" | "unknown";
|
|
155
|
+
};
|
|
156
|
+
d7: {
|
|
157
|
+
asOf: number;
|
|
158
|
+
count: number;
|
|
159
|
+
coverage: "complete" | "partial" | "unknown";
|
|
160
|
+
};
|
|
161
|
+
};
|
|
162
|
+
windows?: undefined;
|
|
163
|
+
buckets?: undefined;
|
|
164
|
+
excluded?: number | undefined;
|
|
165
|
+
reported?: {
|
|
166
|
+
amount: number;
|
|
167
|
+
currency: string;
|
|
168
|
+
asOf: number;
|
|
169
|
+
} | undefined;
|
|
170
|
+
})[];
|
|
171
|
+
}>;
|
|
172
|
+
detail(raw: DetailInput, signal?: AbortSignal): Promise<{
|
|
173
|
+
version: 1;
|
|
174
|
+
threadId: string;
|
|
175
|
+
scope: "own" | "family";
|
|
176
|
+
axis: "model" | "time";
|
|
177
|
+
window: "m20" | "h24" | "d7" | "lifetime";
|
|
178
|
+
asOf: number;
|
|
179
|
+
revision: string;
|
|
180
|
+
priceVersion: string;
|
|
181
|
+
currency: "USD";
|
|
182
|
+
estimated: true;
|
|
183
|
+
coverage: "complete" | "partial" | "unknown";
|
|
184
|
+
coverageReasons: ("source-gap" | "root-unattested" | "history-unavailable" | "member-cap" | "depth-cap" | "edge-cap" | "missing-thread" | "unpriced" | "retention" | "row-cap" | "pagination" | "population-incomplete")[];
|
|
185
|
+
approximation: {
|
|
186
|
+
approximate: boolean;
|
|
187
|
+
reasons: ("source-gap" | "closest-model" | "provider-default" | "model-attribution" | "bucket-boundary" | "missing-price")[];
|
|
188
|
+
rates?: {
|
|
189
|
+
providerId: string;
|
|
190
|
+
modelId: string;
|
|
191
|
+
rateId: string | null;
|
|
192
|
+
basis: "unpriced" | "closest-model" | "provider-default" | "matched";
|
|
193
|
+
}[] | undefined;
|
|
194
|
+
};
|
|
195
|
+
total: {
|
|
196
|
+
tokens: {
|
|
197
|
+
inputUncached: number;
|
|
198
|
+
cacheRead: number;
|
|
199
|
+
cacheWrite: number;
|
|
200
|
+
output: number;
|
|
201
|
+
};
|
|
202
|
+
amount: number | null;
|
|
203
|
+
};
|
|
204
|
+
rows: ({
|
|
205
|
+
axis: "time";
|
|
206
|
+
start: number;
|
|
207
|
+
end: number;
|
|
208
|
+
amount: number | null;
|
|
209
|
+
tokens: {
|
|
210
|
+
inputUncached: number;
|
|
211
|
+
cacheRead: number;
|
|
212
|
+
cacheWrite: number;
|
|
213
|
+
output: number;
|
|
214
|
+
};
|
|
215
|
+
approximation: {
|
|
216
|
+
approximate: boolean;
|
|
217
|
+
reasons: ("source-gap" | "closest-model" | "provider-default" | "model-attribution" | "bucket-boundary" | "missing-price")[];
|
|
218
|
+
rates?: {
|
|
219
|
+
providerId: string;
|
|
220
|
+
modelId: string;
|
|
221
|
+
rateId: string | null;
|
|
222
|
+
basis: "unpriced" | "closest-model" | "provider-default" | "matched";
|
|
223
|
+
}[] | undefined;
|
|
224
|
+
};
|
|
225
|
+
} | {
|
|
226
|
+
axis: "model";
|
|
227
|
+
providerId: string;
|
|
228
|
+
modelId: string;
|
|
229
|
+
amount: number | null;
|
|
230
|
+
tokens: {
|
|
231
|
+
inputUncached: number;
|
|
232
|
+
cacheRead: number;
|
|
233
|
+
cacheWrite: number;
|
|
234
|
+
output: number;
|
|
235
|
+
};
|
|
236
|
+
approximation: {
|
|
237
|
+
approximate: boolean;
|
|
238
|
+
reasons: ("source-gap" | "closest-model" | "provider-default" | "model-attribution" | "bucket-boundary" | "missing-price")[];
|
|
239
|
+
rates?: {
|
|
240
|
+
providerId: string;
|
|
241
|
+
modelId: string;
|
|
242
|
+
rateId: string | null;
|
|
243
|
+
basis: "unpriced" | "closest-model" | "provider-default" | "matched";
|
|
244
|
+
}[] | undefined;
|
|
245
|
+
};
|
|
246
|
+
})[];
|
|
247
|
+
excluded?: number | undefined;
|
|
248
|
+
nextCursor?: string | undefined;
|
|
249
|
+
}>;
|
|
250
|
+
};
|
|
251
|
+
export type DiscoveryRow = {
|
|
252
|
+
pluginId: string;
|
|
253
|
+
state: 'ready';
|
|
254
|
+
version: 1;
|
|
255
|
+
capabilities: Capabilities;
|
|
256
|
+
} | {
|
|
257
|
+
pluginId: string;
|
|
258
|
+
state: 'incompatible';
|
|
259
|
+
} | {
|
|
260
|
+
pluginId: string;
|
|
261
|
+
state: 'error';
|
|
262
|
+
error: OwnerErrorKind;
|
|
263
|
+
};
|
|
264
|
+
export interface DiscoveryResult {
|
|
265
|
+
rows: DiscoveryRow[];
|
|
266
|
+
omittedCount: number;
|
|
267
|
+
complete: boolean;
|
|
268
|
+
listError?: OwnerErrorKind | 'busy';
|
|
269
|
+
}
|
|
270
|
+
export interface DiscoveryOptions {
|
|
271
|
+
sdk: MetricsSdk;
|
|
272
|
+
signal?: AbortSignal;
|
|
273
|
+
maxCandidates?: number;
|
|
274
|
+
concurrency?: number;
|
|
275
|
+
callMs?: number;
|
|
276
|
+
deadlineMs?: number;
|
|
277
|
+
onRow?: (row: DiscoveryRow) => void;
|
|
278
|
+
}
|
|
279
|
+
/** One instance per mounted demand coordinator, retained across refresh/reconnect passes.
|
|
280
|
+
* A completed local deadline does not free underlying reads. Busy passes do no I/O.
|
|
281
|
+
* No queue, retries, timers or global singleton are created by the coordinator itself.
|
|
282
|
+
*/
|
|
283
|
+
export declare function createThreadMetricsDiscovery(sdk: MetricsSdk): {
|
|
284
|
+
readonly outstandingReads: number;
|
|
285
|
+
discover(options?: Omit<DiscoveryOptions, "sdk">): Promise<DiscoveryResult>;
|
|
286
|
+
};
|