nexus-ai-pro 2.4.0 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/API_STABILITY.md +85 -0
  2. package/CHANGELOG.md +147 -1
  3. package/README.md +8 -3
  4. package/SECURITY.md +78 -9
  5. package/dist/context-hub/hub.d.ts +53 -0
  6. package/dist/context-hub/hub.js +108 -3
  7. package/dist/context-hub/index.d.ts +2 -1
  8. package/dist/context-hub/index.js +2 -1
  9. package/dist/context-hub/signing.d.ts +95 -0
  10. package/dist/context-hub/signing.js +88 -0
  11. package/dist/doctor/index.d.ts +4 -2
  12. package/dist/doctor/index.js +29 -0
  13. package/dist/evaluate/threads.d.ts +187 -0
  14. package/dist/evaluate/threads.js +187 -0
  15. package/dist/operations/adapters.d.ts +2 -1
  16. package/dist/operations/runner.js +2 -0
  17. package/dist/operations/stats.js +11 -4
  18. package/dist/postgres/index.d.ts +2 -1
  19. package/dist/postgres/index.js +4 -0
  20. package/dist/postgres/operations.js +11 -4
  21. package/dist/postgres/store.d.ts +7 -0
  22. package/dist/postgres/store.js +21 -0
  23. package/dist/postgres/tenancy.d.ts +43 -0
  24. package/dist/postgres/tenancy.js +71 -0
  25. package/dist/server/deployments.d.ts +127 -12
  26. package/dist/server/deployments.js +402 -37
  27. package/dist/server/metrics.d.ts +2 -0
  28. package/dist/server/metrics.js +7 -0
  29. package/dist/server/runs.d.ts +9 -1
  30. package/dist/server/runs.js +46 -7
  31. package/dist/server/server.js +4 -1
  32. package/dist/server/state.d.ts +12 -2
  33. package/dist/server/state.js +15 -0
  34. package/dist/sqlite/operations.js +15 -4
  35. package/dist/sqlite/store.d.ts +7 -0
  36. package/dist/sqlite/store.js +28 -0
  37. package/dist/store/memory.d.ts +8 -0
  38. package/dist/store/memory.js +12 -1
  39. package/dist/store/redis.d.ts +13 -0
  40. package/dist/store/redis.js +38 -0
  41. package/dist/tracing/instrument.js +1 -0
  42. package/dist/tracing/tracer.d.ts +4 -0
  43. package/dist/tracing/tracer.js +6 -0
  44. package/dist/types/operations.d.ts +7 -0
  45. package/dist/types/server.d.ts +61 -2
  46. package/dist/types/store.d.ts +9 -0
  47. package/dist/version.d.ts +1 -1
  48. package/dist/version.js +1 -1
  49. package/dist-cjs/context-hub/hub.d.ts +53 -0
  50. package/dist-cjs/context-hub/hub.js +110 -4
  51. package/dist-cjs/context-hub/index.d.ts +2 -1
  52. package/dist-cjs/context-hub/index.js +7 -1
  53. package/dist-cjs/context-hub/signing.d.ts +95 -0
  54. package/dist-cjs/context-hub/signing.js +96 -0
  55. package/dist-cjs/doctor/index.d.ts +4 -2
  56. package/dist-cjs/doctor/index.js +29 -0
  57. package/dist-cjs/evaluate/threads.d.ts +187 -0
  58. package/dist-cjs/evaluate/threads.js +198 -0
  59. package/dist-cjs/operations/adapters.d.ts +2 -1
  60. package/dist-cjs/operations/runner.js +2 -0
  61. package/dist-cjs/operations/stats.js +11 -4
  62. package/dist-cjs/postgres/index.d.ts +2 -1
  63. package/dist-cjs/postgres/index.js +8 -1
  64. package/dist-cjs/postgres/operations.js +11 -4
  65. package/dist-cjs/postgres/store.d.ts +7 -0
  66. package/dist-cjs/postgres/store.js +21 -0
  67. package/dist-cjs/postgres/tenancy.d.ts +43 -0
  68. package/dist-cjs/postgres/tenancy.js +110 -0
  69. package/dist-cjs/server/deployments.d.ts +127 -12
  70. package/dist-cjs/server/deployments.js +402 -37
  71. package/dist-cjs/server/metrics.d.ts +2 -0
  72. package/dist-cjs/server/metrics.js +8 -0
  73. package/dist-cjs/server/runs.d.ts +9 -1
  74. package/dist-cjs/server/runs.js +45 -6
  75. package/dist-cjs/server/server.js +4 -1
  76. package/dist-cjs/server/state.d.ts +12 -2
  77. package/dist-cjs/server/state.js +15 -0
  78. package/dist-cjs/sqlite/operations.js +15 -4
  79. package/dist-cjs/sqlite/store.d.ts +7 -0
  80. package/dist-cjs/sqlite/store.js +28 -0
  81. package/dist-cjs/store/memory.d.ts +8 -0
  82. package/dist-cjs/store/memory.js +12 -1
  83. package/dist-cjs/store/redis.d.ts +13 -0
  84. package/dist-cjs/store/redis.js +38 -0
  85. package/dist-cjs/tracing/instrument.js +1 -0
  86. package/dist-cjs/tracing/tracer.d.ts +4 -0
  87. package/dist-cjs/tracing/tracer.js +6 -0
  88. package/dist-cjs/types/operations.d.ts +7 -0
  89. package/dist-cjs/types/server.d.ts +61 -2
  90. package/dist-cjs/types/store.d.ts +9 -0
  91. package/dist-cjs/version.d.ts +1 -1
  92. package/dist-cjs/version.js +1 -1
  93. package/llms.txt +1 -1
  94. package/package.json +27 -1
@@ -3,15 +3,25 @@ import { AssistantCapabilityError, BadRequestError, ServerError } from './errors
3
3
  import { MemoryServerStore, RUNS_NAMESPACE } from './state.js';
4
4
  const DEPLOYMENTS = ['nexus', 'server', 'deployments'];
5
5
  const REPLICAS = ['nexus', 'server', 'replicas'];
6
+ const ROLLUPS = ['nexus', 'server', 'rollups'];
6
7
  const EPSILON = 1e-9;
8
+ const MAX_CHANGE_ATTEMPTS = 10;
9
+ const LATENCY_GROWTH = 1.05;
10
+ const ROLLUPS_KEPT = 100;
7
11
  export class Deployments {
8
12
  options;
9
13
  state;
14
+ fleet = new Map();
10
15
  changing = new Map();
11
16
  now;
12
17
  registrations = new Map();
13
18
  cache = new Map();
14
19
  heartbeat;
20
+ pending = new Map();
21
+ written = new Map();
22
+ rollupTimer;
23
+ flushing = Promise.resolve();
24
+ unattached = `replica-${globalThis.crypto.randomUUID().slice(0, 8)}`;
15
25
  constructor(options = {}) {
16
26
  this.options = options;
17
27
  this.state = options.state ?? new MemoryServerStore();
@@ -22,10 +32,14 @@ export class Deployments {
22
32
  if (ids.length === 0)
23
33
  throw new BadRequestError(`Assistant "${id}" needs at least one revision`, 'NO_REVISIONS');
24
34
  const live = options.live ?? ids[ids.length - 1];
25
- if (!revisions[live])
35
+ if (!own(revisions, live))
26
36
  throw new BadRequestError(`Assistant "${id}" has no revision "${live}"`, 'UNKNOWN_REVISION');
27
- this.registrations.set(id, { revisions, live });
28
- const pick = (context) => (context?.revision !== undefined ? revisions[context.revision] : undefined) ??
37
+ this.registrations.set(id, {
38
+ revisions,
39
+ live,
40
+ ...(options.bucketStrategy ? { bucketStrategy: checkStrategy(options.bucketStrategy) } : {}),
41
+ });
42
+ const pick = (context) => (context?.revision !== undefined ? own(revisions, context.revision) : undefined) ??
29
43
  revisions[live];
30
44
  const fallback = revisions[live];
31
45
  return {
@@ -46,7 +60,7 @@ export class Deployments {
46
60
  ...(fallback.restore ? { restore: (threadId, step) => fallback.restore?.(threadId, step) } : {}),
47
61
  ...(fallback.step ? { step: (threadId) => fallback.step?.(threadId) } : {}),
48
62
  route: (request) => this.route(id, request),
49
- revision: (revision) => revisions[revision],
63
+ revision: (revision) => own(revisions, revision),
50
64
  };
51
65
  }
52
66
  async route(assistant, request) {
@@ -55,19 +69,24 @@ export class Deployments {
55
69
  return undefined;
56
70
  const deployment = await this.current(assistant);
57
71
  if (request.requested !== undefined) {
58
- const chosen = registration.revisions[request.requested];
59
- if (!chosen) {
72
+ const runnable = own(registration.revisions, request.requested) !== undefined ||
73
+ (request.queued === true && (await this.workerRevisions(assistant)).has(request.requested));
74
+ if (!runnable) {
60
75
  throw new BadRequestError(`Assistant "${assistant}" has no revision "${request.requested}"`, 'UNKNOWN_REVISION');
61
76
  }
62
- return choice(request.requested, chosen, deployment, 'requested');
77
+ return choice(request.requested, registration, deployment, 'requested');
63
78
  }
64
- const traffic = servable(deployment.traffic, registration);
79
+ const fleet = request.queued ? await this.workerRevisions(assistant) : undefined;
80
+ const traffic = servable(deployment.traffic, registration, fleet);
65
81
  const threadRevision = request.threadRevision;
66
82
  if (threadRevision !== undefined && (traffic[threadRevision] ?? 0) > 0) {
67
- return choice(threadRevision, registration.revisions[threadRevision], deployment, 'thread');
83
+ return choice(threadRevision, registration, deployment, 'thread');
68
84
  }
69
- const revision = pickRevision(traffic, deployment.live, bucket(`${assistant}:${request.threadId ?? request.runId}`));
70
- return choice(revision, registration.revisions[revision], deployment, 'split');
85
+ const revision = pickRevision(traffic, deployment.live, bucket(`${assistant}:${request.threadId ?? request.runId}`, deployment.bucketStrategy));
86
+ return choice(revision, registration, deployment, 'split');
87
+ }
88
+ get atomicChanges() {
89
+ return typeof this.state.putIfVersion === 'function';
71
90
  }
72
91
  async list() {
73
92
  const stored = await this.state.list(DEPLOYMENTS, { limit: 500 });
@@ -94,7 +113,24 @@ export class Deployments {
94
113
  }
95
114
  }
96
115
  async applyChange(assistant, change, by) {
97
- const current = await this.get(assistant);
116
+ const versioned = this.state.putIfVersion?.bind(this.state);
117
+ for (let attempt = 1;; attempt += 1) {
118
+ const stored = await this.state.get(DEPLOYMENTS, assistant);
119
+ const next = await this.decide(assistant, change, by, stored);
120
+ if (!versioned)
121
+ await this.state.put(DEPLOYMENTS, assistant, next);
122
+ else if (!(await versioned(DEPLOYMENTS, assistant, next, stored ? stored.version : null))) {
123
+ if (attempt >= MAX_CHANGE_ATTEMPTS) {
124
+ throw new ServerError(`Deployment "${assistant}" changed ${attempt} times while this change was being decided; try again`, 'DEPLOYMENT_CONFLICT', 409);
125
+ }
126
+ continue;
127
+ }
128
+ this.cache.set(assistant, { record: next, at: Date.now() });
129
+ return next;
130
+ }
131
+ }
132
+ async decide(assistant, change, by, stored) {
133
+ const current = stored ?? this.initial(assistant);
98
134
  if (!current) {
99
135
  throw new BadRequestError(`Assistant "${assistant}" has no deployment yet: serve it through Deployments.assistant() first`, 'UNKNOWN_DEPLOYMENT');
100
136
  }
@@ -109,7 +145,12 @@ export class Deployments {
109
145
  };
110
146
  let live = current.live;
111
147
  let traffic;
148
+ let strategy = current.bucketStrategy;
112
149
  switch (change.action) {
150
+ case 'bucketing':
151
+ strategy = checkStrategy(change.strategy);
152
+ traffic = current.traffic;
153
+ break;
113
154
  case 'split':
114
155
  for (const revision of Object.keys(change.traffic))
115
156
  check(revision);
@@ -141,7 +182,7 @@ export class Deployments {
141
182
  break;
142
183
  }
143
184
  default:
144
- throw new BadRequestError('A change needs an action: split, canary, promote, or rollback');
185
+ throw new BadRequestError('A change needs an action: split, canary, promote, rollback, or bucketing');
145
186
  }
146
187
  const at = this.now().toISOString();
147
188
  const next = {
@@ -149,6 +190,7 @@ export class Deployments {
149
190
  version: current.version + 1,
150
191
  live,
151
192
  traffic,
193
+ ...(strategy ? { bucketStrategy: strategy } : {}),
152
194
  updatedAt: at,
153
195
  ...(by ? { updatedBy: by } : {}),
154
196
  history: [
@@ -158,6 +200,7 @@ export class Deployments {
158
200
  action: change.action,
159
201
  live,
160
202
  traffic,
203
+ ...(change.action === 'bucketing' ? { bucketStrategy: strategy } : {}),
161
204
  ...(by ? { by } : {}),
162
205
  ...(change.reason ? { reason: change.reason } : {}),
163
206
  },
@@ -169,8 +212,6 @@ export class Deployments {
169
212
  const same = canaryRevisions(current).join() === canaries.join();
170
213
  next.canarySince = same && current.canarySince ? current.canarySince : at;
171
214
  }
172
- await this.state.put(DEPLOYMENTS, assistant, next);
173
- this.cache.set(assistant, { record: next, at: Date.now() });
174
215
  return next;
175
216
  }
176
217
  split(assistant, traffic, options = {}) {
@@ -185,6 +226,9 @@ export class Deployments {
185
226
  rollback(assistant, options = {}) {
186
227
  return this.change(assistant, { action: 'rollback', to: options.to, reason: options.reason }, options.by);
187
228
  }
229
+ bucketing(assistant, strategy, options = {}) {
230
+ return this.change(assistant, { action: 'bucketing', strategy, reason: options.reason }, options.by);
231
+ }
188
232
  async replicas() {
189
233
  const ttl = this.options.replicaTtlMs ?? 3 * (this.options.heartbeatMs ?? 10_000);
190
234
  const cutoff = this.now().getTime() - ttl;
@@ -197,7 +241,10 @@ export class Deployments {
197
241
  for (const assistant of this.registrations.keys()) {
198
242
  const initial = this.initial(assistant);
199
243
  if (initial && !(await this.state.get(DEPLOYMENTS, assistant))) {
200
- await this.state.put(DEPLOYMENTS, assistant, initial);
244
+ if (this.state.putIfVersion)
245
+ await this.state.putIfVersion(DEPLOYMENTS, assistant, initial, null);
246
+ else
247
+ await this.state.put(DEPLOYMENTS, assistant, initial);
201
248
  }
202
249
  }
203
250
  if (this.heartbeat)
@@ -219,6 +266,7 @@ export class Deployments {
219
266
  await beat();
220
267
  }
221
268
  async detach() {
269
+ await this.flushRollups();
222
270
  if (!this.heartbeat)
223
271
  return;
224
272
  clearInterval(this.heartbeat.timer);
@@ -230,6 +278,11 @@ export class Deployments {
230
278
  return [...(await this.knownRevisions(assistant))];
231
279
  }
232
280
  async stats(assistant, options = {}) {
281
+ if (options.deployment !== undefined) {
282
+ const rollups = await this.rollups(assistant, options.deployment);
283
+ if (rollups.size > 0)
284
+ return [...rollups].map(([revision, rollup]) => rollupStats(revision, rollup));
285
+ }
233
286
  const byRevision = new Map();
234
287
  for (const run of await this.runs(assistant, options)) {
235
288
  const revision = run.revision?.id ?? '';
@@ -237,6 +290,91 @@ export class Deployments {
237
290
  }
238
291
  return [...byRevision].map(([revision, runs]) => summarize(revision, runs));
239
292
  }
293
+ async rollups(assistant, deployment) {
294
+ await this.flushRollups();
295
+ const stored = await this.state.list([...ROLLUPS, assistant, String(deployment)], {
296
+ limit: 10_000,
297
+ });
298
+ const total = new Map();
299
+ for (const rollup of stored) {
300
+ for (const [revision, counts] of Object.entries(rollup.revisions)) {
301
+ total.set(revision, addCounts(total.get(revision), counts));
302
+ }
303
+ }
304
+ return total;
305
+ }
306
+ recordRun(run) {
307
+ const revision = run.revision;
308
+ if (!revision)
309
+ return;
310
+ const outcome = outcomeOf(run);
311
+ if (!outcome)
312
+ return;
313
+ const key = `${run.assistant}\u0000${revision.deployment}`;
314
+ const rollup = this.pending.get(key) ?? {
315
+ assistant: run.assistant,
316
+ deployment: revision.deployment,
317
+ replica: '',
318
+ updatedAt: '',
319
+ revisions: {},
320
+ };
321
+ rollup.revisions[revision.id] = addCounts(rollup.revisions[revision.id], countsOf(run, outcome));
322
+ this.pending.set(key, rollup);
323
+ if (!this.rollupTimer) {
324
+ this.rollupTimer = setTimeout(() => {
325
+ this.rollupTimer = undefined;
326
+ void this.flushRollups();
327
+ }, this.options.rollupMs ?? 1_000);
328
+ this.rollupTimer.unref?.();
329
+ }
330
+ }
331
+ flushRollups() {
332
+ if (this.rollupTimer)
333
+ clearTimeout(this.rollupTimer);
334
+ this.rollupTimer = undefined;
335
+ this.flushing = this.flushing.then(() => this.writeRollups());
336
+ return this.flushing;
337
+ }
338
+ async writeRollups() {
339
+ const batch = [...this.pending.values()];
340
+ this.pending.clear();
341
+ const replica = this.heartbeat?.id ?? this.unattached;
342
+ for (const delta of batch) {
343
+ const namespace = [...ROLLUPS, delta.assistant, String(delta.deployment)];
344
+ const key = `${replica}\u0000${delta.assistant}\u0000${delta.deployment}`;
345
+ try {
346
+ const base = this.written.get(key) ?? (await this.state.get(namespace, replica));
347
+ const revisions = { ...base?.revisions };
348
+ for (const [revision, counts] of Object.entries(delta.revisions)) {
349
+ revisions[revision] = addCounts(revisions[revision], counts);
350
+ }
351
+ const next = {
352
+ assistant: delta.assistant,
353
+ deployment: delta.deployment,
354
+ replica,
355
+ updatedAt: this.now().toISOString(),
356
+ revisions,
357
+ };
358
+ await this.state.put(namespace, replica, next);
359
+ this.written.delete(key);
360
+ this.written.set(key, next);
361
+ if (this.written.size > ROLLUPS_KEPT)
362
+ this.written.delete(this.written.keys().next().value);
363
+ }
364
+ catch (error) {
365
+ const pendingKey = `${delta.assistant}\u0000${delta.deployment}`;
366
+ const again = this.pending.get(pendingKey);
367
+ if (again) {
368
+ for (const [revision, counts] of Object.entries(delta.revisions)) {
369
+ again.revisions[revision] = addCounts(again.revisions[revision], counts);
370
+ }
371
+ }
372
+ else
373
+ this.pending.set(pendingKey, delta);
374
+ this.options.onError?.(error);
375
+ }
376
+ }
377
+ }
240
378
  async samples(assistant, revision, since, limit) {
241
379
  return (await this.runs(assistant, { since, limit }))
242
380
  .filter((run) => (run.revision?.id ?? '') === revision)
@@ -312,6 +450,20 @@ export class Deployments {
312
450
  this.cache.set(assistant, { record, at: Date.now() });
313
451
  return record;
314
452
  }
453
+ async workerRevisions(assistant) {
454
+ const cached = this.fleet.get(assistant);
455
+ if (cached && Date.now() - cached.at < (this.options.cacheMs ?? 2_000))
456
+ return cached.revisions;
457
+ const revisions = new Set();
458
+ for (const replica of await this.replicas()) {
459
+ if (!replica.claims || replica.draining)
460
+ continue;
461
+ for (const revision of replica.assistants[assistant] ?? [])
462
+ revisions.add(revision);
463
+ }
464
+ this.fleet.set(assistant, { revisions, at: Date.now() });
465
+ return revisions;
466
+ }
315
467
  initial(assistant) {
316
468
  const registration = this.registrations.get(assistant);
317
469
  if (!registration)
@@ -321,6 +473,7 @@ export class Deployments {
321
473
  version: 0,
322
474
  live: registration.live,
323
475
  traffic: { [registration.live]: 1 },
476
+ ...(registration.bucketStrategy ? { bucketStrategy: registration.bucketStrategy } : {}),
324
477
  updatedAt: new Date(0).toISOString(),
325
478
  history: [],
326
479
  };
@@ -342,29 +495,38 @@ export function watchCanaries(options) {
342
495
  const deployments = options.deployments;
343
496
  const minRuns = options.minRuns ?? 20;
344
497
  const by = options.by ?? 'guard';
498
+ const fromRollups = options.from === 'rollups';
499
+ if (fromRollups && (options.samples || options.feedback?.length)) {
500
+ throw new TypeError('A canary guard over rollups judges counts, so it takes neither samples nor feedback');
501
+ }
345
502
  const samples = options.samples ??
346
503
  ((request) => deployments.samples(request.assistant, request.revision, request.since));
347
- async function judge(deployment, canary) {
504
+ async function measure(deployment, canary) {
505
+ const group = `${deployment.assistant}@${canary}`;
506
+ const metrics = options.metrics ?? ['error-rate', 'latency'];
507
+ if (fromRollups) {
508
+ const rollups = await deployments.rollups(deployment.assistant, deployment.version);
509
+ const live = rollups.get(deployment.live);
510
+ const candidate = rollups.get(canary);
511
+ const counted = { live: finishedOf(live), canary: finishedOf(candidate) };
512
+ if (counted.live < minRuns || counted.canary < minRuns)
513
+ return { regressions: [], counted };
514
+ return {
515
+ regressions: compareRollups(live, candidate, group, metrics),
516
+ counted,
517
+ };
518
+ }
348
519
  const since = deployment.history[0]?.at ?? deployment.canarySince ?? deployment.updatedAt;
349
520
  const [liveRuns, canaryRuns] = await Promise.all([
350
521
  samples({ assistant: deployment.assistant, revision: deployment.live, since }),
351
522
  samples({ assistant: deployment.assistant, revision: canary, since }),
352
523
  ]);
353
- const weight = deployment.traffic[canary] ?? 0;
354
- const decision = {
355
- assistant: deployment.assistant,
356
- canary,
357
- live: deployment.live,
358
- action: 'hold',
359
- regressions: [],
360
- samples: { live: liveRuns.length, canary: canaryRuns.length },
361
- weight,
362
- };
363
- if (liveRuns.length < minRuns || canaryRuns.length < minRuns)
364
- return decision;
524
+ const counted = { live: liveRuns.length, canary: canaryRuns.length };
525
+ if (counted.live < minRuns || counted.canary < minRuns)
526
+ return { regressions: [], counted };
365
527
  const regressions = compareRuns(liveRuns, canaryRuns, {
366
- group: `${deployment.assistant}@${canary}`,
367
- metrics: options.metrics ?? ['error-rate', 'latency'],
528
+ group,
529
+ metrics,
368
530
  feedback: options.feedback,
369
531
  minRuns,
370
532
  latencyIncrease: options.latencyIncrease,
@@ -379,6 +541,80 @@ export function watchCanaries(options) {
379
541
  resamples: options.resamples,
380
542
  }),
381
543
  });
544
+ return { regressions, counted };
545
+ }
546
+ function compareRollups(live, candidate, group, metrics) {
547
+ const regressions = [];
548
+ const samples = { baseline: finishedOf(live), current: finishedOf(candidate) };
549
+ if (metrics.includes('error-rate')) {
550
+ const p1 = live.failed / samples.baseline;
551
+ const p2 = candidate.failed / samples.current;
552
+ const pooled = (live.failed + candidate.failed) / (samples.baseline + samples.current);
553
+ const standardError = Math.sqrt(pooled * (1 - pooled) * (1 / samples.baseline + 1 / samples.current));
554
+ const z = options.confidence === undefined ? 1.96 : normalQuantile((1 + options.confidence) / 2);
555
+ const minimum = options.confidence === undefined ? 0 : (options.minErrorRateIncrease ?? 0.02);
556
+ if (p2 > p1 && standardError > 0 && (p2 - p1) / standardError >= z && p2 - p1 >= minimum) {
557
+ regressions.push({
558
+ group,
559
+ metric: 'error-rate',
560
+ baseline: p1,
561
+ current: p2,
562
+ change: p2 - p1,
563
+ samples,
564
+ summary: `${group}: error rate rose from ${rate(p1)} to ${rate(p2)}`,
565
+ });
566
+ }
567
+ }
568
+ if (metrics.includes('latency')) {
569
+ const before = latencyPercentile(live.latency, 0.95);
570
+ const after = latencyPercentile(candidate.latency, 0.95);
571
+ if (before !== undefined &&
572
+ after !== undefined &&
573
+ before > 0 &&
574
+ after >= before * (1 + (options.latencyIncrease ?? 0.25)) &&
575
+ after - before >= (options.minLatencyChangeMs ?? 50)) {
576
+ regressions.push({
577
+ group,
578
+ metric: 'latency-p95',
579
+ baseline: before,
580
+ current: after,
581
+ change: after / before - 1,
582
+ samples,
583
+ summary: `${group}: p95 latency rose from ${Math.round(before)}ms to ${Math.round(after)}ms`,
584
+ });
585
+ }
586
+ }
587
+ if (metrics.includes('cost') && live.costed > 0 && candidate.costed > 0) {
588
+ const before = live.cost / live.costed;
589
+ const after = candidate.cost / candidate.costed;
590
+ if (before > 0 && after >= before * (1 + (options.costIncrease ?? 0.25))) {
591
+ regressions.push({
592
+ group,
593
+ metric: 'cost',
594
+ baseline: before,
595
+ current: after,
596
+ change: after / before - 1,
597
+ samples,
598
+ summary: `${group}: mean cost rose from $${before.toFixed(4)} to $${after.toFixed(4)}`,
599
+ });
600
+ }
601
+ }
602
+ return regressions;
603
+ }
604
+ async function judge(deployment, canary) {
605
+ const { regressions, counted } = await measure(deployment, canary);
606
+ const weight = deployment.traffic[canary] ?? 0;
607
+ const decision = {
608
+ assistant: deployment.assistant,
609
+ canary,
610
+ live: deployment.live,
611
+ action: 'hold',
612
+ regressions: [],
613
+ samples: counted,
614
+ weight,
615
+ };
616
+ if (counted.live < minRuns || counted.canary < minRuns)
617
+ return decision;
382
618
  if (regressions.length > 0) {
383
619
  await deployments.change(deployment.assistant, {
384
620
  action: 'rollback',
@@ -393,7 +629,7 @@ export function watchCanaries(options) {
393
629
  action: 'canary',
394
630
  revision: canary,
395
631
  weight: next,
396
- reason: `canary ${canary} held up at ${percent(weight)} over ${canaryRuns.length} runs`,
632
+ reason: `canary ${canary} held up at ${percent(weight)} over ${counted.canary} runs`,
397
633
  expectedVersion: deployment.version,
398
634
  }, by);
399
635
  return { ...decision, action: 'advance', weight: next };
@@ -436,14 +672,129 @@ export function watchCanaries(options) {
436
672
  timer.unref?.();
437
673
  return { check, stop: () => clearInterval(timer) };
438
674
  }
439
- export function bucket(key) {
675
+ export function bucket(key, strategy = 'fnv1a-v1') {
440
676
  let hash = 0x811c9dc5;
441
677
  for (let index = 0; index < key.length; index += 1) {
442
678
  hash ^= key.charCodeAt(index);
443
679
  hash = Math.imul(hash, 0x01000193);
444
680
  }
681
+ if (strategy === 'hash-v2') {
682
+ hash ^= hash >>> 16;
683
+ hash = Math.imul(hash, 0x85ebca6b);
684
+ hash ^= hash >>> 13;
685
+ hash = Math.imul(hash, 0xc2b2ae35);
686
+ hash ^= hash >>> 16;
687
+ }
445
688
  return (hash >>> 0) / 0x1_0000_0000;
446
689
  }
690
+ function own(revisions, id) {
691
+ return Object.hasOwn(revisions, id) ? revisions[id] : undefined;
692
+ }
693
+ function checkStrategy(strategy) {
694
+ if (strategy !== 'fnv1a-v1' && strategy !== 'hash-v2') {
695
+ throw new BadRequestError(`Unknown bucket strategy "${String(strategy)}": use fnv1a-v1 or hash-v2`);
696
+ }
697
+ return strategy;
698
+ }
699
+ function outcomeOf(run) {
700
+ if (run.status === 'succeeded' || run.status === 'awaiting_input')
701
+ return 'succeeded';
702
+ if (run.status === 'failed' || run.status === 'expired')
703
+ return 'failed';
704
+ if (run.status === 'cancelled')
705
+ return run.interrupt === undefined ? 'cancelled' : undefined;
706
+ return undefined;
707
+ }
708
+ function countsOf(run, outcome) {
709
+ const finished = outcome !== 'cancelled';
710
+ return {
711
+ runs: 1,
712
+ succeeded: outcome === 'succeeded' ? 1 : 0,
713
+ failed: outcome === 'failed' ? 1 : 0,
714
+ cancelled: outcome === 'cancelled' ? 1 : 0,
715
+ latency: finished && run.durationMs !== undefined ? { [String(latencyIndex(run.durationMs))]: 1 } : {},
716
+ cost: finished && run.cost !== undefined ? run.cost : 0,
717
+ costed: finished && run.cost !== undefined ? 1 : 0,
718
+ };
719
+ }
720
+ function addCounts(a, b) {
721
+ if (!a)
722
+ return { ...b, latency: { ...b.latency } };
723
+ const latency = { ...a.latency };
724
+ for (const [index, count] of Object.entries(b.latency))
725
+ latency[index] = (latency[index] ?? 0) + count;
726
+ return {
727
+ runs: a.runs + b.runs,
728
+ succeeded: a.succeeded + b.succeeded,
729
+ failed: a.failed + b.failed,
730
+ cancelled: a.cancelled + b.cancelled,
731
+ latency,
732
+ cost: a.cost + b.cost,
733
+ costed: a.costed + b.costed,
734
+ };
735
+ }
736
+ function finishedOf(rollup) {
737
+ return rollup ? rollup.succeeded + rollup.failed : 0;
738
+ }
739
+ function latencyIndex(ms) {
740
+ return ms <= 1 ? 0 : Math.ceil(Math.log(ms) / Math.log(LATENCY_GROWTH));
741
+ }
742
+ function latencyPercentile(latency, p) {
743
+ const buckets = Object.entries(latency)
744
+ .map(([index, count]) => [Number(index), count])
745
+ .sort((a, b) => a[0] - b[0]);
746
+ const total = buckets.reduce((sum, [, count]) => sum + count, 0);
747
+ if (total === 0)
748
+ return undefined;
749
+ const rank = Math.max(1, Math.ceil(p * total));
750
+ let seen = 0;
751
+ for (const [index, count] of buckets) {
752
+ seen += count;
753
+ if (seen >= rank)
754
+ return index === 0 ? 1 : LATENCY_GROWTH ** (index - 0.5);
755
+ }
756
+ return undefined;
757
+ }
758
+ function rollupStats(revision, rollup) {
759
+ const finished = finishedOf(rollup);
760
+ const p50 = latencyPercentile(rollup.latency, 0.5);
761
+ const p95 = latencyPercentile(rollup.latency, 0.95);
762
+ return {
763
+ revision,
764
+ runs: rollup.runs,
765
+ succeeded: rollup.succeeded,
766
+ failed: rollup.failed,
767
+ cancelled: rollup.cancelled,
768
+ errorRate: finished === 0 ? 0 : rollup.failed / finished,
769
+ ...(p50 !== undefined && p95 !== undefined ? { p50Ms: Math.round(p50), p95Ms: Math.round(p95) } : {}),
770
+ ...(rollup.costed > 0 ? { meanCost: rollup.cost / rollup.costed } : {}),
771
+ };
772
+ }
773
+ function normalQuantile(p) {
774
+ const a = [
775
+ -39.69683028665376, 220.9460984245205, -275.9285104469687, 138.357751867269, -30.66479806614716, 2.506628277459239,
776
+ ];
777
+ const b = [-54.47609879822406, 161.5858368580409, -155.6989798598866, 66.80131188771972, -13.28068155288572];
778
+ const c = [
779
+ -0.007784894002430293, -0.3223964580411365, -2.400758277161838, -2.549732539343734, 4.374664141464968,
780
+ 2.938163982698783,
781
+ ];
782
+ const d = [0.007784695709041462, 0.3224671290700398, 2.445134137142996, 3.754408661907416];
783
+ const tail = (q) => (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) /
784
+ ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
785
+ if (p <= 0)
786
+ return -Infinity;
787
+ if (p >= 1)
788
+ return Infinity;
789
+ if (p < 0.02425)
790
+ return tail(Math.sqrt(-2 * Math.log(p)));
791
+ if (p > 1 - 0.02425)
792
+ return -tail(Math.sqrt(-2 * Math.log(1 - p)));
793
+ const q = p - 0.5;
794
+ const r = q * q;
795
+ return (((((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q) /
796
+ (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1));
797
+ }
447
798
  function pickRevision(traffic, live, position) {
448
799
  let edge = 0;
449
800
  for (const revision of Object.keys(traffic)
@@ -455,8 +806,8 @@ function pickRevision(traffic, live, position) {
455
806
  }
456
807
  return (traffic[live] ?? 0) > 0 ? live : (Object.keys(traffic)[0] ?? live);
457
808
  }
458
- function servable(traffic, registration) {
459
- const kept = Object.entries(traffic).filter(([revision, weight]) => weight > 0 && registration.revisions[revision]);
809
+ function servable(traffic, registration, fleet) {
810
+ const kept = Object.entries(traffic).filter(([revision, weight]) => weight > 0 && (own(registration.revisions, revision) || fleet?.has(revision)));
460
811
  const total = kept.reduce((sum, [, weight]) => sum + weight, 0);
461
812
  if (total <= 0)
462
813
  return { [registration.live]: 1 };
@@ -490,8 +841,19 @@ function canaryRevisions(deployment) {
490
841
  function hasCanary(deployment) {
491
842
  return canaryRevisions(deployment).length > 0;
492
843
  }
493
- function choice(id, assistant, deployment, reason) {
494
- return { id, assistant, weight: deployment.traffic[id] ?? 0, deployment: deployment.version, reason };
844
+ function choice(id, registration, deployment, reason) {
845
+ const here = own(registration.revisions, id);
846
+ const assistant = (here ??
847
+ registration.revisions[registration.live] ??
848
+ Object.values(registration.revisions)[0]);
849
+ return {
850
+ id,
851
+ assistant,
852
+ local: here !== undefined,
853
+ weight: deployment.traffic[id] ?? 0,
854
+ deployment: deployment.version,
855
+ reason,
856
+ };
495
857
  }
496
858
  function summarize(revision, runs) {
497
859
  const succeeded = runs.filter((run) => run.status === 'succeeded' || run.status === 'awaiting_input');
@@ -517,6 +879,9 @@ function percentile(values, p) {
517
879
  function round(value) {
518
880
  return Math.round(value * 1e6) / 1e6;
519
881
  }
882
+ function rate(value) {
883
+ return `${(value * 100).toFixed(1)}%`;
884
+ }
520
885
  function percent(value) {
521
886
  return `${Math.round(value * 100)}%`;
522
887
  }
@@ -21,4 +21,6 @@ export declare class ServerMetrics {
21
21
  }
22
22
  /** Appends one gauge with its header. */
23
23
  export declare function gauge(lines: string[], name: string, help: string, value: number, labels?: Labels): void;
24
+ /** One gauge with several labelled samples, under one header. Writes nothing without samples. */
25
+ export declare function gauges(lines: string[], name: string, help: string, samples: ReadonlyArray<[Labels, number]>): void;
24
26
  export {};
@@ -52,6 +52,13 @@ export function gauge(lines, name, help, value, labels = {}) {
52
52
  header(lines, name, 'gauge', help);
53
53
  lines.push(`${name}${labelText(labels)} ${round(value)}`);
54
54
  }
55
+ export function gauges(lines, name, help, samples) {
56
+ if (samples.length === 0)
57
+ return;
58
+ header(lines, name, 'gauge', help);
59
+ for (const [labels, value] of samples)
60
+ lines.push(`${name}${labelText(labels)} ${round(value)}`);
61
+ }
55
62
  function header(lines, name, type, help) {
56
63
  if (help)
57
64
  lines.push(`# HELP ${name} ${help}`);
@@ -1,6 +1,6 @@
1
1
  import type { OperationRunnerConfig } from '../types/operations.js';
2
2
  import type { RunFeedback } from '../types/tracing.js';
3
- import type { Principal, ReplicaReport, RunEventLog, RunRecord, ScalingSnapshot, ServerAssistant, ServerStateStore, TenantGate, ThreadBusyPolicy, ThreadRecord } from '../types/server.js';
3
+ import type { Principal, ReplicaReport, RunEventLog, RunRecord, ScalingSnapshot, ServerAssistant, ServerDeployments, ServerStateStore, TenantGate, ThreadBusyPolicy, ThreadRecord } from '../types/server.js';
4
4
  import { ServerMetrics } from './metrics.js';
5
5
  /** What a run needs to start. */
6
6
  export interface StartRunOptions {
@@ -60,6 +60,8 @@ export interface RunManagerOptions {
60
60
  recoverEveryMs?: number;
61
61
  /** Per-tenant limits, enforced when a run is accepted. See `tenantLimits()` in `nexus-ai-pro/server/tenancy`. */
62
62
  tenants?: TenantGate;
63
+ /** The deployments registry, which counts each finished run in this replica's rollups. */
64
+ deployments?: Pick<ServerDeployments, 'recordRun'>;
63
65
  /** What to do when a thread is already running something. Defaults to `reject`. */
64
66
  onBusy?: ThreadBusyPolicy;
65
67
  /** How long a run may take before it expires, in milliseconds. */
@@ -182,6 +184,12 @@ export declare class RunManager {
182
184
  * a test or a custom scheduler can call it directly. Resolves to the ids it claimed.
183
185
  */
184
186
  claim(): Promise<string[]>;
187
+ /**
188
+ * The kinds of run this replica can execute: each assistant's runs without a revision, and those
189
+ * of every revision it carries. A run routed to a revision this image lacks is left for a worker
190
+ * that has it.
191
+ */
192
+ private claimableKinds;
185
193
  /**
186
194
  * Starts the worker loop: claiming queued runs while there is room, and taking over runs whose
187
195
  * worker stopped. Does nothing without a queue or `recoverEveryMs`. `stopWorking()` ends it.