nexus-ai-pro 2.4.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/API_STABILITY.md +85 -0
- package/CHANGELOG.md +147 -1
- package/README.md +8 -3
- package/SECURITY.md +78 -9
- package/dist/context-hub/hub.d.ts +53 -0
- package/dist/context-hub/hub.js +108 -3
- package/dist/context-hub/index.d.ts +2 -1
- package/dist/context-hub/index.js +2 -1
- package/dist/context-hub/signing.d.ts +95 -0
- package/dist/context-hub/signing.js +88 -0
- package/dist/doctor/index.d.ts +4 -2
- package/dist/doctor/index.js +29 -0
- package/dist/evaluate/threads.d.ts +187 -0
- package/dist/evaluate/threads.js +187 -0
- package/dist/operations/adapters.d.ts +2 -1
- package/dist/operations/runner.js +2 -0
- package/dist/operations/stats.js +11 -4
- package/dist/postgres/index.d.ts +2 -1
- package/dist/postgres/index.js +4 -0
- package/dist/postgres/operations.js +11 -4
- package/dist/postgres/store.d.ts +7 -0
- package/dist/postgres/store.js +21 -0
- package/dist/postgres/tenancy.d.ts +43 -0
- package/dist/postgres/tenancy.js +71 -0
- package/dist/server/deployments.d.ts +127 -12
- package/dist/server/deployments.js +402 -37
- package/dist/server/metrics.d.ts +2 -0
- package/dist/server/metrics.js +7 -0
- package/dist/server/runs.d.ts +9 -1
- package/dist/server/runs.js +46 -7
- package/dist/server/server.js +4 -1
- package/dist/server/state.d.ts +12 -2
- package/dist/server/state.js +15 -0
- package/dist/sqlite/operations.js +15 -4
- package/dist/sqlite/store.d.ts +7 -0
- package/dist/sqlite/store.js +28 -0
- package/dist/store/memory.d.ts +8 -0
- package/dist/store/memory.js +12 -1
- package/dist/store/redis.d.ts +13 -0
- package/dist/store/redis.js +38 -0
- package/dist/tracing/instrument.js +1 -0
- package/dist/tracing/tracer.d.ts +4 -0
- package/dist/tracing/tracer.js +6 -0
- package/dist/types/operations.d.ts +7 -0
- package/dist/types/server.d.ts +61 -2
- package/dist/types/store.d.ts +9 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist-cjs/context-hub/hub.d.ts +53 -0
- package/dist-cjs/context-hub/hub.js +110 -4
- package/dist-cjs/context-hub/index.d.ts +2 -1
- package/dist-cjs/context-hub/index.js +7 -1
- package/dist-cjs/context-hub/signing.d.ts +95 -0
- package/dist-cjs/context-hub/signing.js +96 -0
- package/dist-cjs/doctor/index.d.ts +4 -2
- package/dist-cjs/doctor/index.js +29 -0
- package/dist-cjs/evaluate/threads.d.ts +187 -0
- package/dist-cjs/evaluate/threads.js +198 -0
- package/dist-cjs/operations/adapters.d.ts +2 -1
- package/dist-cjs/operations/runner.js +2 -0
- package/dist-cjs/operations/stats.js +11 -4
- package/dist-cjs/postgres/index.d.ts +2 -1
- package/dist-cjs/postgres/index.js +8 -1
- package/dist-cjs/postgres/operations.js +11 -4
- package/dist-cjs/postgres/store.d.ts +7 -0
- package/dist-cjs/postgres/store.js +21 -0
- package/dist-cjs/postgres/tenancy.d.ts +43 -0
- package/dist-cjs/postgres/tenancy.js +110 -0
- package/dist-cjs/server/deployments.d.ts +127 -12
- package/dist-cjs/server/deployments.js +402 -37
- package/dist-cjs/server/metrics.d.ts +2 -0
- package/dist-cjs/server/metrics.js +8 -0
- package/dist-cjs/server/runs.d.ts +9 -1
- package/dist-cjs/server/runs.js +45 -6
- package/dist-cjs/server/server.js +4 -1
- package/dist-cjs/server/state.d.ts +12 -2
- package/dist-cjs/server/state.js +15 -0
- package/dist-cjs/sqlite/operations.js +15 -4
- package/dist-cjs/sqlite/store.d.ts +7 -0
- package/dist-cjs/sqlite/store.js +28 -0
- package/dist-cjs/store/memory.d.ts +8 -0
- package/dist-cjs/store/memory.js +12 -1
- package/dist-cjs/store/redis.d.ts +13 -0
- package/dist-cjs/store/redis.js +38 -0
- package/dist-cjs/tracing/instrument.js +1 -0
- package/dist-cjs/tracing/tracer.d.ts +4 -0
- package/dist-cjs/tracing/tracer.js +6 -0
- package/dist-cjs/types/operations.d.ts +7 -0
- package/dist-cjs/types/server.d.ts +61 -2
- package/dist-cjs/types/store.d.ts +9 -0
- package/dist-cjs/version.d.ts +1 -1
- package/dist-cjs/version.js +1 -1
- package/llms.txt +1 -1
- package/package.json +27 -1
|
@@ -3,15 +3,25 @@ import { AssistantCapabilityError, BadRequestError, ServerError } from './errors
|
|
|
3
3
|
import { MemoryServerStore, RUNS_NAMESPACE } from './state.js';
|
|
4
4
|
const DEPLOYMENTS = ['nexus', 'server', 'deployments'];
|
|
5
5
|
const REPLICAS = ['nexus', 'server', 'replicas'];
|
|
6
|
+
const ROLLUPS = ['nexus', 'server', 'rollups'];
|
|
6
7
|
const EPSILON = 1e-9;
|
|
8
|
+
const MAX_CHANGE_ATTEMPTS = 10;
|
|
9
|
+
const LATENCY_GROWTH = 1.05;
|
|
10
|
+
const ROLLUPS_KEPT = 100;
|
|
7
11
|
export class Deployments {
|
|
8
12
|
options;
|
|
9
13
|
state;
|
|
14
|
+
fleet = new Map();
|
|
10
15
|
changing = new Map();
|
|
11
16
|
now;
|
|
12
17
|
registrations = new Map();
|
|
13
18
|
cache = new Map();
|
|
14
19
|
heartbeat;
|
|
20
|
+
pending = new Map();
|
|
21
|
+
written = new Map();
|
|
22
|
+
rollupTimer;
|
|
23
|
+
flushing = Promise.resolve();
|
|
24
|
+
unattached = `replica-${globalThis.crypto.randomUUID().slice(0, 8)}`;
|
|
15
25
|
constructor(options = {}) {
|
|
16
26
|
this.options = options;
|
|
17
27
|
this.state = options.state ?? new MemoryServerStore();
|
|
@@ -22,10 +32,14 @@ export class Deployments {
|
|
|
22
32
|
if (ids.length === 0)
|
|
23
33
|
throw new BadRequestError(`Assistant "${id}" needs at least one revision`, 'NO_REVISIONS');
|
|
24
34
|
const live = options.live ?? ids[ids.length - 1];
|
|
25
|
-
if (!revisions
|
|
35
|
+
if (!own(revisions, live))
|
|
26
36
|
throw new BadRequestError(`Assistant "${id}" has no revision "${live}"`, 'UNKNOWN_REVISION');
|
|
27
|
-
this.registrations.set(id, {
|
|
28
|
-
|
|
37
|
+
this.registrations.set(id, {
|
|
38
|
+
revisions,
|
|
39
|
+
live,
|
|
40
|
+
...(options.bucketStrategy ? { bucketStrategy: checkStrategy(options.bucketStrategy) } : {}),
|
|
41
|
+
});
|
|
42
|
+
const pick = (context) => (context?.revision !== undefined ? own(revisions, context.revision) : undefined) ??
|
|
29
43
|
revisions[live];
|
|
30
44
|
const fallback = revisions[live];
|
|
31
45
|
return {
|
|
@@ -46,7 +60,7 @@ export class Deployments {
|
|
|
46
60
|
...(fallback.restore ? { restore: (threadId, step) => fallback.restore?.(threadId, step) } : {}),
|
|
47
61
|
...(fallback.step ? { step: (threadId) => fallback.step?.(threadId) } : {}),
|
|
48
62
|
route: (request) => this.route(id, request),
|
|
49
|
-
revision: (revision) => revisions
|
|
63
|
+
revision: (revision) => own(revisions, revision),
|
|
50
64
|
};
|
|
51
65
|
}
|
|
52
66
|
async route(assistant, request) {
|
|
@@ -55,19 +69,24 @@ export class Deployments {
|
|
|
55
69
|
return undefined;
|
|
56
70
|
const deployment = await this.current(assistant);
|
|
57
71
|
if (request.requested !== undefined) {
|
|
58
|
-
const
|
|
59
|
-
|
|
72
|
+
const runnable = own(registration.revisions, request.requested) !== undefined ||
|
|
73
|
+
(request.queued === true && (await this.workerRevisions(assistant)).has(request.requested));
|
|
74
|
+
if (!runnable) {
|
|
60
75
|
throw new BadRequestError(`Assistant "${assistant}" has no revision "${request.requested}"`, 'UNKNOWN_REVISION');
|
|
61
76
|
}
|
|
62
|
-
return choice(request.requested,
|
|
77
|
+
return choice(request.requested, registration, deployment, 'requested');
|
|
63
78
|
}
|
|
64
|
-
const
|
|
79
|
+
const fleet = request.queued ? await this.workerRevisions(assistant) : undefined;
|
|
80
|
+
const traffic = servable(deployment.traffic, registration, fleet);
|
|
65
81
|
const threadRevision = request.threadRevision;
|
|
66
82
|
if (threadRevision !== undefined && (traffic[threadRevision] ?? 0) > 0) {
|
|
67
|
-
return choice(threadRevision, registration
|
|
83
|
+
return choice(threadRevision, registration, deployment, 'thread');
|
|
68
84
|
}
|
|
69
|
-
const revision = pickRevision(traffic, deployment.live, bucket(`${assistant}:${request.threadId ?? request.runId}
|
|
70
|
-
return choice(revision, registration
|
|
85
|
+
const revision = pickRevision(traffic, deployment.live, bucket(`${assistant}:${request.threadId ?? request.runId}`, deployment.bucketStrategy));
|
|
86
|
+
return choice(revision, registration, deployment, 'split');
|
|
87
|
+
}
|
|
88
|
+
get atomicChanges() {
|
|
89
|
+
return typeof this.state.putIfVersion === 'function';
|
|
71
90
|
}
|
|
72
91
|
async list() {
|
|
73
92
|
const stored = await this.state.list(DEPLOYMENTS, { limit: 500 });
|
|
@@ -94,7 +113,24 @@ export class Deployments {
|
|
|
94
113
|
}
|
|
95
114
|
}
|
|
96
115
|
async applyChange(assistant, change, by) {
|
|
97
|
-
const
|
|
116
|
+
const versioned = this.state.putIfVersion?.bind(this.state);
|
|
117
|
+
for (let attempt = 1;; attempt += 1) {
|
|
118
|
+
const stored = await this.state.get(DEPLOYMENTS, assistant);
|
|
119
|
+
const next = await this.decide(assistant, change, by, stored);
|
|
120
|
+
if (!versioned)
|
|
121
|
+
await this.state.put(DEPLOYMENTS, assistant, next);
|
|
122
|
+
else if (!(await versioned(DEPLOYMENTS, assistant, next, stored ? stored.version : null))) {
|
|
123
|
+
if (attempt >= MAX_CHANGE_ATTEMPTS) {
|
|
124
|
+
throw new ServerError(`Deployment "${assistant}" changed ${attempt} times while this change was being decided; try again`, 'DEPLOYMENT_CONFLICT', 409);
|
|
125
|
+
}
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
this.cache.set(assistant, { record: next, at: Date.now() });
|
|
129
|
+
return next;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
async decide(assistant, change, by, stored) {
|
|
133
|
+
const current = stored ?? this.initial(assistant);
|
|
98
134
|
if (!current) {
|
|
99
135
|
throw new BadRequestError(`Assistant "${assistant}" has no deployment yet: serve it through Deployments.assistant() first`, 'UNKNOWN_DEPLOYMENT');
|
|
100
136
|
}
|
|
@@ -109,7 +145,12 @@ export class Deployments {
|
|
|
109
145
|
};
|
|
110
146
|
let live = current.live;
|
|
111
147
|
let traffic;
|
|
148
|
+
let strategy = current.bucketStrategy;
|
|
112
149
|
switch (change.action) {
|
|
150
|
+
case 'bucketing':
|
|
151
|
+
strategy = checkStrategy(change.strategy);
|
|
152
|
+
traffic = current.traffic;
|
|
153
|
+
break;
|
|
113
154
|
case 'split':
|
|
114
155
|
for (const revision of Object.keys(change.traffic))
|
|
115
156
|
check(revision);
|
|
@@ -141,7 +182,7 @@ export class Deployments {
|
|
|
141
182
|
break;
|
|
142
183
|
}
|
|
143
184
|
default:
|
|
144
|
-
throw new BadRequestError('A change needs an action: split, canary, promote, or
|
|
185
|
+
throw new BadRequestError('A change needs an action: split, canary, promote, rollback, or bucketing');
|
|
145
186
|
}
|
|
146
187
|
const at = this.now().toISOString();
|
|
147
188
|
const next = {
|
|
@@ -149,6 +190,7 @@ export class Deployments {
|
|
|
149
190
|
version: current.version + 1,
|
|
150
191
|
live,
|
|
151
192
|
traffic,
|
|
193
|
+
...(strategy ? { bucketStrategy: strategy } : {}),
|
|
152
194
|
updatedAt: at,
|
|
153
195
|
...(by ? { updatedBy: by } : {}),
|
|
154
196
|
history: [
|
|
@@ -158,6 +200,7 @@ export class Deployments {
|
|
|
158
200
|
action: change.action,
|
|
159
201
|
live,
|
|
160
202
|
traffic,
|
|
203
|
+
...(change.action === 'bucketing' ? { bucketStrategy: strategy } : {}),
|
|
161
204
|
...(by ? { by } : {}),
|
|
162
205
|
...(change.reason ? { reason: change.reason } : {}),
|
|
163
206
|
},
|
|
@@ -169,8 +212,6 @@ export class Deployments {
|
|
|
169
212
|
const same = canaryRevisions(current).join() === canaries.join();
|
|
170
213
|
next.canarySince = same && current.canarySince ? current.canarySince : at;
|
|
171
214
|
}
|
|
172
|
-
await this.state.put(DEPLOYMENTS, assistant, next);
|
|
173
|
-
this.cache.set(assistant, { record: next, at: Date.now() });
|
|
174
215
|
return next;
|
|
175
216
|
}
|
|
176
217
|
split(assistant, traffic, options = {}) {
|
|
@@ -185,6 +226,9 @@ export class Deployments {
|
|
|
185
226
|
rollback(assistant, options = {}) {
|
|
186
227
|
return this.change(assistant, { action: 'rollback', to: options.to, reason: options.reason }, options.by);
|
|
187
228
|
}
|
|
229
|
+
bucketing(assistant, strategy, options = {}) {
|
|
230
|
+
return this.change(assistant, { action: 'bucketing', strategy, reason: options.reason }, options.by);
|
|
231
|
+
}
|
|
188
232
|
async replicas() {
|
|
189
233
|
const ttl = this.options.replicaTtlMs ?? 3 * (this.options.heartbeatMs ?? 10_000);
|
|
190
234
|
const cutoff = this.now().getTime() - ttl;
|
|
@@ -197,7 +241,10 @@ export class Deployments {
|
|
|
197
241
|
for (const assistant of this.registrations.keys()) {
|
|
198
242
|
const initial = this.initial(assistant);
|
|
199
243
|
if (initial && !(await this.state.get(DEPLOYMENTS, assistant))) {
|
|
200
|
-
|
|
244
|
+
if (this.state.putIfVersion)
|
|
245
|
+
await this.state.putIfVersion(DEPLOYMENTS, assistant, initial, null);
|
|
246
|
+
else
|
|
247
|
+
await this.state.put(DEPLOYMENTS, assistant, initial);
|
|
201
248
|
}
|
|
202
249
|
}
|
|
203
250
|
if (this.heartbeat)
|
|
@@ -219,6 +266,7 @@ export class Deployments {
|
|
|
219
266
|
await beat();
|
|
220
267
|
}
|
|
221
268
|
async detach() {
|
|
269
|
+
await this.flushRollups();
|
|
222
270
|
if (!this.heartbeat)
|
|
223
271
|
return;
|
|
224
272
|
clearInterval(this.heartbeat.timer);
|
|
@@ -230,6 +278,11 @@ export class Deployments {
|
|
|
230
278
|
return [...(await this.knownRevisions(assistant))];
|
|
231
279
|
}
|
|
232
280
|
async stats(assistant, options = {}) {
|
|
281
|
+
if (options.deployment !== undefined) {
|
|
282
|
+
const rollups = await this.rollups(assistant, options.deployment);
|
|
283
|
+
if (rollups.size > 0)
|
|
284
|
+
return [...rollups].map(([revision, rollup]) => rollupStats(revision, rollup));
|
|
285
|
+
}
|
|
233
286
|
const byRevision = new Map();
|
|
234
287
|
for (const run of await this.runs(assistant, options)) {
|
|
235
288
|
const revision = run.revision?.id ?? '';
|
|
@@ -237,6 +290,91 @@ export class Deployments {
|
|
|
237
290
|
}
|
|
238
291
|
return [...byRevision].map(([revision, runs]) => summarize(revision, runs));
|
|
239
292
|
}
|
|
293
|
+
async rollups(assistant, deployment) {
|
|
294
|
+
await this.flushRollups();
|
|
295
|
+
const stored = await this.state.list([...ROLLUPS, assistant, String(deployment)], {
|
|
296
|
+
limit: 10_000,
|
|
297
|
+
});
|
|
298
|
+
const total = new Map();
|
|
299
|
+
for (const rollup of stored) {
|
|
300
|
+
for (const [revision, counts] of Object.entries(rollup.revisions)) {
|
|
301
|
+
total.set(revision, addCounts(total.get(revision), counts));
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
return total;
|
|
305
|
+
}
|
|
306
|
+
recordRun(run) {
|
|
307
|
+
const revision = run.revision;
|
|
308
|
+
if (!revision)
|
|
309
|
+
return;
|
|
310
|
+
const outcome = outcomeOf(run);
|
|
311
|
+
if (!outcome)
|
|
312
|
+
return;
|
|
313
|
+
const key = `${run.assistant}\u0000${revision.deployment}`;
|
|
314
|
+
const rollup = this.pending.get(key) ?? {
|
|
315
|
+
assistant: run.assistant,
|
|
316
|
+
deployment: revision.deployment,
|
|
317
|
+
replica: '',
|
|
318
|
+
updatedAt: '',
|
|
319
|
+
revisions: {},
|
|
320
|
+
};
|
|
321
|
+
rollup.revisions[revision.id] = addCounts(rollup.revisions[revision.id], countsOf(run, outcome));
|
|
322
|
+
this.pending.set(key, rollup);
|
|
323
|
+
if (!this.rollupTimer) {
|
|
324
|
+
this.rollupTimer = setTimeout(() => {
|
|
325
|
+
this.rollupTimer = undefined;
|
|
326
|
+
void this.flushRollups();
|
|
327
|
+
}, this.options.rollupMs ?? 1_000);
|
|
328
|
+
this.rollupTimer.unref?.();
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
flushRollups() {
|
|
332
|
+
if (this.rollupTimer)
|
|
333
|
+
clearTimeout(this.rollupTimer);
|
|
334
|
+
this.rollupTimer = undefined;
|
|
335
|
+
this.flushing = this.flushing.then(() => this.writeRollups());
|
|
336
|
+
return this.flushing;
|
|
337
|
+
}
|
|
338
|
+
async writeRollups() {
|
|
339
|
+
const batch = [...this.pending.values()];
|
|
340
|
+
this.pending.clear();
|
|
341
|
+
const replica = this.heartbeat?.id ?? this.unattached;
|
|
342
|
+
for (const delta of batch) {
|
|
343
|
+
const namespace = [...ROLLUPS, delta.assistant, String(delta.deployment)];
|
|
344
|
+
const key = `${replica}\u0000${delta.assistant}\u0000${delta.deployment}`;
|
|
345
|
+
try {
|
|
346
|
+
const base = this.written.get(key) ?? (await this.state.get(namespace, replica));
|
|
347
|
+
const revisions = { ...base?.revisions };
|
|
348
|
+
for (const [revision, counts] of Object.entries(delta.revisions)) {
|
|
349
|
+
revisions[revision] = addCounts(revisions[revision], counts);
|
|
350
|
+
}
|
|
351
|
+
const next = {
|
|
352
|
+
assistant: delta.assistant,
|
|
353
|
+
deployment: delta.deployment,
|
|
354
|
+
replica,
|
|
355
|
+
updatedAt: this.now().toISOString(),
|
|
356
|
+
revisions,
|
|
357
|
+
};
|
|
358
|
+
await this.state.put(namespace, replica, next);
|
|
359
|
+
this.written.delete(key);
|
|
360
|
+
this.written.set(key, next);
|
|
361
|
+
if (this.written.size > ROLLUPS_KEPT)
|
|
362
|
+
this.written.delete(this.written.keys().next().value);
|
|
363
|
+
}
|
|
364
|
+
catch (error) {
|
|
365
|
+
const pendingKey = `${delta.assistant}\u0000${delta.deployment}`;
|
|
366
|
+
const again = this.pending.get(pendingKey);
|
|
367
|
+
if (again) {
|
|
368
|
+
for (const [revision, counts] of Object.entries(delta.revisions)) {
|
|
369
|
+
again.revisions[revision] = addCounts(again.revisions[revision], counts);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
else
|
|
373
|
+
this.pending.set(pendingKey, delta);
|
|
374
|
+
this.options.onError?.(error);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
}
|
|
240
378
|
async samples(assistant, revision, since, limit) {
|
|
241
379
|
return (await this.runs(assistant, { since, limit }))
|
|
242
380
|
.filter((run) => (run.revision?.id ?? '') === revision)
|
|
@@ -312,6 +450,20 @@ export class Deployments {
|
|
|
312
450
|
this.cache.set(assistant, { record, at: Date.now() });
|
|
313
451
|
return record;
|
|
314
452
|
}
|
|
453
|
+
async workerRevisions(assistant) {
|
|
454
|
+
const cached = this.fleet.get(assistant);
|
|
455
|
+
if (cached && Date.now() - cached.at < (this.options.cacheMs ?? 2_000))
|
|
456
|
+
return cached.revisions;
|
|
457
|
+
const revisions = new Set();
|
|
458
|
+
for (const replica of await this.replicas()) {
|
|
459
|
+
if (!replica.claims || replica.draining)
|
|
460
|
+
continue;
|
|
461
|
+
for (const revision of replica.assistants[assistant] ?? [])
|
|
462
|
+
revisions.add(revision);
|
|
463
|
+
}
|
|
464
|
+
this.fleet.set(assistant, { revisions, at: Date.now() });
|
|
465
|
+
return revisions;
|
|
466
|
+
}
|
|
315
467
|
initial(assistant) {
|
|
316
468
|
const registration = this.registrations.get(assistant);
|
|
317
469
|
if (!registration)
|
|
@@ -321,6 +473,7 @@ export class Deployments {
|
|
|
321
473
|
version: 0,
|
|
322
474
|
live: registration.live,
|
|
323
475
|
traffic: { [registration.live]: 1 },
|
|
476
|
+
...(registration.bucketStrategy ? { bucketStrategy: registration.bucketStrategy } : {}),
|
|
324
477
|
updatedAt: new Date(0).toISOString(),
|
|
325
478
|
history: [],
|
|
326
479
|
};
|
|
@@ -342,29 +495,38 @@ export function watchCanaries(options) {
|
|
|
342
495
|
const deployments = options.deployments;
|
|
343
496
|
const minRuns = options.minRuns ?? 20;
|
|
344
497
|
const by = options.by ?? 'guard';
|
|
498
|
+
const fromRollups = options.from === 'rollups';
|
|
499
|
+
if (fromRollups && (options.samples || options.feedback?.length)) {
|
|
500
|
+
throw new TypeError('A canary guard over rollups judges counts, so it takes neither samples nor feedback');
|
|
501
|
+
}
|
|
345
502
|
const samples = options.samples ??
|
|
346
503
|
((request) => deployments.samples(request.assistant, request.revision, request.since));
|
|
347
|
-
async function
|
|
504
|
+
async function measure(deployment, canary) {
|
|
505
|
+
const group = `${deployment.assistant}@${canary}`;
|
|
506
|
+
const metrics = options.metrics ?? ['error-rate', 'latency'];
|
|
507
|
+
if (fromRollups) {
|
|
508
|
+
const rollups = await deployments.rollups(deployment.assistant, deployment.version);
|
|
509
|
+
const live = rollups.get(deployment.live);
|
|
510
|
+
const candidate = rollups.get(canary);
|
|
511
|
+
const counted = { live: finishedOf(live), canary: finishedOf(candidate) };
|
|
512
|
+
if (counted.live < minRuns || counted.canary < minRuns)
|
|
513
|
+
return { regressions: [], counted };
|
|
514
|
+
return {
|
|
515
|
+
regressions: compareRollups(live, candidate, group, metrics),
|
|
516
|
+
counted,
|
|
517
|
+
};
|
|
518
|
+
}
|
|
348
519
|
const since = deployment.history[0]?.at ?? deployment.canarySince ?? deployment.updatedAt;
|
|
349
520
|
const [liveRuns, canaryRuns] = await Promise.all([
|
|
350
521
|
samples({ assistant: deployment.assistant, revision: deployment.live, since }),
|
|
351
522
|
samples({ assistant: deployment.assistant, revision: canary, since }),
|
|
352
523
|
]);
|
|
353
|
-
const
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
canary,
|
|
357
|
-
live: deployment.live,
|
|
358
|
-
action: 'hold',
|
|
359
|
-
regressions: [],
|
|
360
|
-
samples: { live: liveRuns.length, canary: canaryRuns.length },
|
|
361
|
-
weight,
|
|
362
|
-
};
|
|
363
|
-
if (liveRuns.length < minRuns || canaryRuns.length < minRuns)
|
|
364
|
-
return decision;
|
|
524
|
+
const counted = { live: liveRuns.length, canary: canaryRuns.length };
|
|
525
|
+
if (counted.live < minRuns || counted.canary < minRuns)
|
|
526
|
+
return { regressions: [], counted };
|
|
365
527
|
const regressions = compareRuns(liveRuns, canaryRuns, {
|
|
366
|
-
group
|
|
367
|
-
metrics
|
|
528
|
+
group,
|
|
529
|
+
metrics,
|
|
368
530
|
feedback: options.feedback,
|
|
369
531
|
minRuns,
|
|
370
532
|
latencyIncrease: options.latencyIncrease,
|
|
@@ -379,6 +541,80 @@ export function watchCanaries(options) {
|
|
|
379
541
|
resamples: options.resamples,
|
|
380
542
|
}),
|
|
381
543
|
});
|
|
544
|
+
return { regressions, counted };
|
|
545
|
+
}
|
|
546
|
+
function compareRollups(live, candidate, group, metrics) {
|
|
547
|
+
const regressions = [];
|
|
548
|
+
const samples = { baseline: finishedOf(live), current: finishedOf(candidate) };
|
|
549
|
+
if (metrics.includes('error-rate')) {
|
|
550
|
+
const p1 = live.failed / samples.baseline;
|
|
551
|
+
const p2 = candidate.failed / samples.current;
|
|
552
|
+
const pooled = (live.failed + candidate.failed) / (samples.baseline + samples.current);
|
|
553
|
+
const standardError = Math.sqrt(pooled * (1 - pooled) * (1 / samples.baseline + 1 / samples.current));
|
|
554
|
+
const z = options.confidence === undefined ? 1.96 : normalQuantile((1 + options.confidence) / 2);
|
|
555
|
+
const minimum = options.confidence === undefined ? 0 : (options.minErrorRateIncrease ?? 0.02);
|
|
556
|
+
if (p2 > p1 && standardError > 0 && (p2 - p1) / standardError >= z && p2 - p1 >= minimum) {
|
|
557
|
+
regressions.push({
|
|
558
|
+
group,
|
|
559
|
+
metric: 'error-rate',
|
|
560
|
+
baseline: p1,
|
|
561
|
+
current: p2,
|
|
562
|
+
change: p2 - p1,
|
|
563
|
+
samples,
|
|
564
|
+
summary: `${group}: error rate rose from ${rate(p1)} to ${rate(p2)}`,
|
|
565
|
+
});
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
if (metrics.includes('latency')) {
|
|
569
|
+
const before = latencyPercentile(live.latency, 0.95);
|
|
570
|
+
const after = latencyPercentile(candidate.latency, 0.95);
|
|
571
|
+
if (before !== undefined &&
|
|
572
|
+
after !== undefined &&
|
|
573
|
+
before > 0 &&
|
|
574
|
+
after >= before * (1 + (options.latencyIncrease ?? 0.25)) &&
|
|
575
|
+
after - before >= (options.minLatencyChangeMs ?? 50)) {
|
|
576
|
+
regressions.push({
|
|
577
|
+
group,
|
|
578
|
+
metric: 'latency-p95',
|
|
579
|
+
baseline: before,
|
|
580
|
+
current: after,
|
|
581
|
+
change: after / before - 1,
|
|
582
|
+
samples,
|
|
583
|
+
summary: `${group}: p95 latency rose from ${Math.round(before)}ms to ${Math.round(after)}ms`,
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
if (metrics.includes('cost') && live.costed > 0 && candidate.costed > 0) {
|
|
588
|
+
const before = live.cost / live.costed;
|
|
589
|
+
const after = candidate.cost / candidate.costed;
|
|
590
|
+
if (before > 0 && after >= before * (1 + (options.costIncrease ?? 0.25))) {
|
|
591
|
+
regressions.push({
|
|
592
|
+
group,
|
|
593
|
+
metric: 'cost',
|
|
594
|
+
baseline: before,
|
|
595
|
+
current: after,
|
|
596
|
+
change: after / before - 1,
|
|
597
|
+
samples,
|
|
598
|
+
summary: `${group}: mean cost rose from $${before.toFixed(4)} to $${after.toFixed(4)}`,
|
|
599
|
+
});
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
return regressions;
|
|
603
|
+
}
|
|
604
|
+
async function judge(deployment, canary) {
|
|
605
|
+
const { regressions, counted } = await measure(deployment, canary);
|
|
606
|
+
const weight = deployment.traffic[canary] ?? 0;
|
|
607
|
+
const decision = {
|
|
608
|
+
assistant: deployment.assistant,
|
|
609
|
+
canary,
|
|
610
|
+
live: deployment.live,
|
|
611
|
+
action: 'hold',
|
|
612
|
+
regressions: [],
|
|
613
|
+
samples: counted,
|
|
614
|
+
weight,
|
|
615
|
+
};
|
|
616
|
+
if (counted.live < minRuns || counted.canary < minRuns)
|
|
617
|
+
return decision;
|
|
382
618
|
if (regressions.length > 0) {
|
|
383
619
|
await deployments.change(deployment.assistant, {
|
|
384
620
|
action: 'rollback',
|
|
@@ -393,7 +629,7 @@ export function watchCanaries(options) {
|
|
|
393
629
|
action: 'canary',
|
|
394
630
|
revision: canary,
|
|
395
631
|
weight: next,
|
|
396
|
-
reason: `canary ${canary} held up at ${percent(weight)} over ${
|
|
632
|
+
reason: `canary ${canary} held up at ${percent(weight)} over ${counted.canary} runs`,
|
|
397
633
|
expectedVersion: deployment.version,
|
|
398
634
|
}, by);
|
|
399
635
|
return { ...decision, action: 'advance', weight: next };
|
|
@@ -436,14 +672,129 @@ export function watchCanaries(options) {
|
|
|
436
672
|
timer.unref?.();
|
|
437
673
|
return { check, stop: () => clearInterval(timer) };
|
|
438
674
|
}
|
|
439
|
-
export function bucket(key) {
|
|
675
|
+
export function bucket(key, strategy = 'fnv1a-v1') {
|
|
440
676
|
let hash = 0x811c9dc5;
|
|
441
677
|
for (let index = 0; index < key.length; index += 1) {
|
|
442
678
|
hash ^= key.charCodeAt(index);
|
|
443
679
|
hash = Math.imul(hash, 0x01000193);
|
|
444
680
|
}
|
|
681
|
+
if (strategy === 'hash-v2') {
|
|
682
|
+
hash ^= hash >>> 16;
|
|
683
|
+
hash = Math.imul(hash, 0x85ebca6b);
|
|
684
|
+
hash ^= hash >>> 13;
|
|
685
|
+
hash = Math.imul(hash, 0xc2b2ae35);
|
|
686
|
+
hash ^= hash >>> 16;
|
|
687
|
+
}
|
|
445
688
|
return (hash >>> 0) / 0x1_0000_0000;
|
|
446
689
|
}
|
|
690
|
+
function own(revisions, id) {
|
|
691
|
+
return Object.hasOwn(revisions, id) ? revisions[id] : undefined;
|
|
692
|
+
}
|
|
693
|
+
function checkStrategy(strategy) {
|
|
694
|
+
if (strategy !== 'fnv1a-v1' && strategy !== 'hash-v2') {
|
|
695
|
+
throw new BadRequestError(`Unknown bucket strategy "${String(strategy)}": use fnv1a-v1 or hash-v2`);
|
|
696
|
+
}
|
|
697
|
+
return strategy;
|
|
698
|
+
}
|
|
699
|
+
function outcomeOf(run) {
|
|
700
|
+
if (run.status === 'succeeded' || run.status === 'awaiting_input')
|
|
701
|
+
return 'succeeded';
|
|
702
|
+
if (run.status === 'failed' || run.status === 'expired')
|
|
703
|
+
return 'failed';
|
|
704
|
+
if (run.status === 'cancelled')
|
|
705
|
+
return run.interrupt === undefined ? 'cancelled' : undefined;
|
|
706
|
+
return undefined;
|
|
707
|
+
}
|
|
708
|
+
function countsOf(run, outcome) {
|
|
709
|
+
const finished = outcome !== 'cancelled';
|
|
710
|
+
return {
|
|
711
|
+
runs: 1,
|
|
712
|
+
succeeded: outcome === 'succeeded' ? 1 : 0,
|
|
713
|
+
failed: outcome === 'failed' ? 1 : 0,
|
|
714
|
+
cancelled: outcome === 'cancelled' ? 1 : 0,
|
|
715
|
+
latency: finished && run.durationMs !== undefined ? { [String(latencyIndex(run.durationMs))]: 1 } : {},
|
|
716
|
+
cost: finished && run.cost !== undefined ? run.cost : 0,
|
|
717
|
+
costed: finished && run.cost !== undefined ? 1 : 0,
|
|
718
|
+
};
|
|
719
|
+
}
|
|
720
|
+
function addCounts(a, b) {
|
|
721
|
+
if (!a)
|
|
722
|
+
return { ...b, latency: { ...b.latency } };
|
|
723
|
+
const latency = { ...a.latency };
|
|
724
|
+
for (const [index, count] of Object.entries(b.latency))
|
|
725
|
+
latency[index] = (latency[index] ?? 0) + count;
|
|
726
|
+
return {
|
|
727
|
+
runs: a.runs + b.runs,
|
|
728
|
+
succeeded: a.succeeded + b.succeeded,
|
|
729
|
+
failed: a.failed + b.failed,
|
|
730
|
+
cancelled: a.cancelled + b.cancelled,
|
|
731
|
+
latency,
|
|
732
|
+
cost: a.cost + b.cost,
|
|
733
|
+
costed: a.costed + b.costed,
|
|
734
|
+
};
|
|
735
|
+
}
|
|
736
|
+
function finishedOf(rollup) {
|
|
737
|
+
return rollup ? rollup.succeeded + rollup.failed : 0;
|
|
738
|
+
}
|
|
739
|
+
function latencyIndex(ms) {
|
|
740
|
+
return ms <= 1 ? 0 : Math.ceil(Math.log(ms) / Math.log(LATENCY_GROWTH));
|
|
741
|
+
}
|
|
742
|
+
function latencyPercentile(latency, p) {
|
|
743
|
+
const buckets = Object.entries(latency)
|
|
744
|
+
.map(([index, count]) => [Number(index), count])
|
|
745
|
+
.sort((a, b) => a[0] - b[0]);
|
|
746
|
+
const total = buckets.reduce((sum, [, count]) => sum + count, 0);
|
|
747
|
+
if (total === 0)
|
|
748
|
+
return undefined;
|
|
749
|
+
const rank = Math.max(1, Math.ceil(p * total));
|
|
750
|
+
let seen = 0;
|
|
751
|
+
for (const [index, count] of buckets) {
|
|
752
|
+
seen += count;
|
|
753
|
+
if (seen >= rank)
|
|
754
|
+
return index === 0 ? 1 : LATENCY_GROWTH ** (index - 0.5);
|
|
755
|
+
}
|
|
756
|
+
return undefined;
|
|
757
|
+
}
|
|
758
|
+
function rollupStats(revision, rollup) {
|
|
759
|
+
const finished = finishedOf(rollup);
|
|
760
|
+
const p50 = latencyPercentile(rollup.latency, 0.5);
|
|
761
|
+
const p95 = latencyPercentile(rollup.latency, 0.95);
|
|
762
|
+
return {
|
|
763
|
+
revision,
|
|
764
|
+
runs: rollup.runs,
|
|
765
|
+
succeeded: rollup.succeeded,
|
|
766
|
+
failed: rollup.failed,
|
|
767
|
+
cancelled: rollup.cancelled,
|
|
768
|
+
errorRate: finished === 0 ? 0 : rollup.failed / finished,
|
|
769
|
+
...(p50 !== undefined && p95 !== undefined ? { p50Ms: Math.round(p50), p95Ms: Math.round(p95) } : {}),
|
|
770
|
+
...(rollup.costed > 0 ? { meanCost: rollup.cost / rollup.costed } : {}),
|
|
771
|
+
};
|
|
772
|
+
}
|
|
773
|
+
function normalQuantile(p) {
|
|
774
|
+
const a = [
|
|
775
|
+
-39.69683028665376, 220.9460984245205, -275.9285104469687, 138.357751867269, -30.66479806614716, 2.506628277459239,
|
|
776
|
+
];
|
|
777
|
+
const b = [-54.47609879822406, 161.5858368580409, -155.6989798598866, 66.80131188771972, -13.28068155288572];
|
|
778
|
+
const c = [
|
|
779
|
+
-0.007784894002430293, -0.3223964580411365, -2.400758277161838, -2.549732539343734, 4.374664141464968,
|
|
780
|
+
2.938163982698783,
|
|
781
|
+
];
|
|
782
|
+
const d = [0.007784695709041462, 0.3224671290700398, 2.445134137142996, 3.754408661907416];
|
|
783
|
+
const tail = (q) => (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) /
|
|
784
|
+
((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
|
|
785
|
+
if (p <= 0)
|
|
786
|
+
return -Infinity;
|
|
787
|
+
if (p >= 1)
|
|
788
|
+
return Infinity;
|
|
789
|
+
if (p < 0.02425)
|
|
790
|
+
return tail(Math.sqrt(-2 * Math.log(p)));
|
|
791
|
+
if (p > 1 - 0.02425)
|
|
792
|
+
return -tail(Math.sqrt(-2 * Math.log(1 - p)));
|
|
793
|
+
const q = p - 0.5;
|
|
794
|
+
const r = q * q;
|
|
795
|
+
return (((((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q) /
|
|
796
|
+
(((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1));
|
|
797
|
+
}
|
|
447
798
|
function pickRevision(traffic, live, position) {
|
|
448
799
|
let edge = 0;
|
|
449
800
|
for (const revision of Object.keys(traffic)
|
|
@@ -455,8 +806,8 @@ function pickRevision(traffic, live, position) {
|
|
|
455
806
|
}
|
|
456
807
|
return (traffic[live] ?? 0) > 0 ? live : (Object.keys(traffic)[0] ?? live);
|
|
457
808
|
}
|
|
458
|
-
function servable(traffic, registration) {
|
|
459
|
-
const kept = Object.entries(traffic).filter(([revision, weight]) => weight > 0 && registration.revisions
|
|
809
|
+
function servable(traffic, registration, fleet) {
|
|
810
|
+
const kept = Object.entries(traffic).filter(([revision, weight]) => weight > 0 && (own(registration.revisions, revision) || fleet?.has(revision)));
|
|
460
811
|
const total = kept.reduce((sum, [, weight]) => sum + weight, 0);
|
|
461
812
|
if (total <= 0)
|
|
462
813
|
return { [registration.live]: 1 };
|
|
@@ -490,8 +841,19 @@ function canaryRevisions(deployment) {
|
|
|
490
841
|
function hasCanary(deployment) {
|
|
491
842
|
return canaryRevisions(deployment).length > 0;
|
|
492
843
|
}
|
|
493
|
-
function choice(id,
|
|
494
|
-
|
|
844
|
+
function choice(id, registration, deployment, reason) {
|
|
845
|
+
const here = own(registration.revisions, id);
|
|
846
|
+
const assistant = (here ??
|
|
847
|
+
registration.revisions[registration.live] ??
|
|
848
|
+
Object.values(registration.revisions)[0]);
|
|
849
|
+
return {
|
|
850
|
+
id,
|
|
851
|
+
assistant,
|
|
852
|
+
local: here !== undefined,
|
|
853
|
+
weight: deployment.traffic[id] ?? 0,
|
|
854
|
+
deployment: deployment.version,
|
|
855
|
+
reason,
|
|
856
|
+
};
|
|
495
857
|
}
|
|
496
858
|
function summarize(revision, runs) {
|
|
497
859
|
const succeeded = runs.filter((run) => run.status === 'succeeded' || run.status === 'awaiting_input');
|
|
@@ -517,6 +879,9 @@ function percentile(values, p) {
|
|
|
517
879
|
function round(value) {
|
|
518
880
|
return Math.round(value * 1e6) / 1e6;
|
|
519
881
|
}
|
|
882
|
+
function rate(value) {
|
|
883
|
+
return `${(value * 100).toFixed(1)}%`;
|
|
884
|
+
}
|
|
520
885
|
function percent(value) {
|
|
521
886
|
return `${Math.round(value * 100)}%`;
|
|
522
887
|
}
|
package/dist/server/metrics.d.ts
CHANGED
|
@@ -21,4 +21,6 @@ export declare class ServerMetrics {
|
|
|
21
21
|
}
|
|
22
22
|
/** Appends one gauge with its header. */
|
|
23
23
|
export declare function gauge(lines: string[], name: string, help: string, value: number, labels?: Labels): void;
|
|
24
|
+
/** One gauge with several labelled samples, under one header. Writes nothing without samples. */
|
|
25
|
+
export declare function gauges(lines: string[], name: string, help: string, samples: ReadonlyArray<[Labels, number]>): void;
|
|
24
26
|
export {};
|
package/dist/server/metrics.js
CHANGED
|
@@ -52,6 +52,13 @@ export function gauge(lines, name, help, value, labels = {}) {
|
|
|
52
52
|
header(lines, name, 'gauge', help);
|
|
53
53
|
lines.push(`${name}${labelText(labels)} ${round(value)}`);
|
|
54
54
|
}
|
|
55
|
+
export function gauges(lines, name, help, samples) {
|
|
56
|
+
if (samples.length === 0)
|
|
57
|
+
return;
|
|
58
|
+
header(lines, name, 'gauge', help);
|
|
59
|
+
for (const [labels, value] of samples)
|
|
60
|
+
lines.push(`${name}${labelText(labels)} ${round(value)}`);
|
|
61
|
+
}
|
|
55
62
|
function header(lines, name, type, help) {
|
|
56
63
|
if (help)
|
|
57
64
|
lines.push(`# HELP ${name} ${help}`);
|
package/dist/server/runs.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { OperationRunnerConfig } from '../types/operations.js';
|
|
2
2
|
import type { RunFeedback } from '../types/tracing.js';
|
|
3
|
-
import type { Principal, ReplicaReport, RunEventLog, RunRecord, ScalingSnapshot, ServerAssistant, ServerStateStore, TenantGate, ThreadBusyPolicy, ThreadRecord } from '../types/server.js';
|
|
3
|
+
import type { Principal, ReplicaReport, RunEventLog, RunRecord, ScalingSnapshot, ServerAssistant, ServerDeployments, ServerStateStore, TenantGate, ThreadBusyPolicy, ThreadRecord } from '../types/server.js';
|
|
4
4
|
import { ServerMetrics } from './metrics.js';
|
|
5
5
|
/** What a run needs to start. */
|
|
6
6
|
export interface StartRunOptions {
|
|
@@ -60,6 +60,8 @@ export interface RunManagerOptions {
|
|
|
60
60
|
recoverEveryMs?: number;
|
|
61
61
|
/** Per-tenant limits, enforced when a run is accepted. See `tenantLimits()` in `nexus-ai-pro/server/tenancy`. */
|
|
62
62
|
tenants?: TenantGate;
|
|
63
|
+
/** The deployments registry, which counts each finished run in this replica's rollups. */
|
|
64
|
+
deployments?: Pick<ServerDeployments, 'recordRun'>;
|
|
63
65
|
/** What to do when a thread is already running something. Defaults to `reject`. */
|
|
64
66
|
onBusy?: ThreadBusyPolicy;
|
|
65
67
|
/** How long a run may take before it expires, in milliseconds. */
|
|
@@ -182,6 +184,12 @@ export declare class RunManager {
|
|
|
182
184
|
* a test or a custom scheduler can call it directly. Resolves to the ids it claimed.
|
|
183
185
|
*/
|
|
184
186
|
claim(): Promise<string[]>;
|
|
187
|
+
/**
|
|
188
|
+
* The kinds of run this replica can execute: each assistant's runs without a revision, and those
|
|
189
|
+
* of every revision it carries. A run routed to a revision this image lacks is left for a worker
|
|
190
|
+
* that has it.
|
|
191
|
+
*/
|
|
192
|
+
private claimableKinds;
|
|
185
193
|
/**
|
|
186
194
|
* Starts the worker loop: claiming queued runs while there is room, and taking over runs whose
|
|
187
195
|
* worker stopped. Does nothing without a queue or `recoverEveryMs`. `stopWorking()` ends it.
|