nexus-ai-pro 2.4.0 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/API_STABILITY.md +85 -0
  2. package/CHANGELOG.md +147 -1
  3. package/README.md +8 -3
  4. package/SECURITY.md +78 -9
  5. package/dist/context-hub/hub.d.ts +53 -0
  6. package/dist/context-hub/hub.js +108 -3
  7. package/dist/context-hub/index.d.ts +2 -1
  8. package/dist/context-hub/index.js +2 -1
  9. package/dist/context-hub/signing.d.ts +95 -0
  10. package/dist/context-hub/signing.js +88 -0
  11. package/dist/doctor/index.d.ts +4 -2
  12. package/dist/doctor/index.js +29 -0
  13. package/dist/evaluate/threads.d.ts +187 -0
  14. package/dist/evaluate/threads.js +187 -0
  15. package/dist/operations/adapters.d.ts +2 -1
  16. package/dist/operations/runner.js +2 -0
  17. package/dist/operations/stats.js +11 -4
  18. package/dist/postgres/index.d.ts +2 -1
  19. package/dist/postgres/index.js +4 -0
  20. package/dist/postgres/operations.js +11 -4
  21. package/dist/postgres/store.d.ts +7 -0
  22. package/dist/postgres/store.js +21 -0
  23. package/dist/postgres/tenancy.d.ts +43 -0
  24. package/dist/postgres/tenancy.js +71 -0
  25. package/dist/server/deployments.d.ts +127 -12
  26. package/dist/server/deployments.js +402 -37
  27. package/dist/server/metrics.d.ts +2 -0
  28. package/dist/server/metrics.js +7 -0
  29. package/dist/server/runs.d.ts +9 -1
  30. package/dist/server/runs.js +46 -7
  31. package/dist/server/server.js +4 -1
  32. package/dist/server/state.d.ts +12 -2
  33. package/dist/server/state.js +15 -0
  34. package/dist/sqlite/operations.js +15 -4
  35. package/dist/sqlite/store.d.ts +7 -0
  36. package/dist/sqlite/store.js +28 -0
  37. package/dist/store/memory.d.ts +8 -0
  38. package/dist/store/memory.js +12 -1
  39. package/dist/store/redis.d.ts +13 -0
  40. package/dist/store/redis.js +38 -0
  41. package/dist/tracing/instrument.js +1 -0
  42. package/dist/tracing/tracer.d.ts +4 -0
  43. package/dist/tracing/tracer.js +6 -0
  44. package/dist/types/operations.d.ts +7 -0
  45. package/dist/types/server.d.ts +61 -2
  46. package/dist/types/store.d.ts +9 -0
  47. package/dist/version.d.ts +1 -1
  48. package/dist/version.js +1 -1
  49. package/dist-cjs/context-hub/hub.d.ts +53 -0
  50. package/dist-cjs/context-hub/hub.js +110 -4
  51. package/dist-cjs/context-hub/index.d.ts +2 -1
  52. package/dist-cjs/context-hub/index.js +7 -1
  53. package/dist-cjs/context-hub/signing.d.ts +95 -0
  54. package/dist-cjs/context-hub/signing.js +96 -0
  55. package/dist-cjs/doctor/index.d.ts +4 -2
  56. package/dist-cjs/doctor/index.js +29 -0
  57. package/dist-cjs/evaluate/threads.d.ts +187 -0
  58. package/dist-cjs/evaluate/threads.js +198 -0
  59. package/dist-cjs/operations/adapters.d.ts +2 -1
  60. package/dist-cjs/operations/runner.js +2 -0
  61. package/dist-cjs/operations/stats.js +11 -4
  62. package/dist-cjs/postgres/index.d.ts +2 -1
  63. package/dist-cjs/postgres/index.js +8 -1
  64. package/dist-cjs/postgres/operations.js +11 -4
  65. package/dist-cjs/postgres/store.d.ts +7 -0
  66. package/dist-cjs/postgres/store.js +21 -0
  67. package/dist-cjs/postgres/tenancy.d.ts +43 -0
  68. package/dist-cjs/postgres/tenancy.js +110 -0
  69. package/dist-cjs/server/deployments.d.ts +127 -12
  70. package/dist-cjs/server/deployments.js +402 -37
  71. package/dist-cjs/server/metrics.d.ts +2 -0
  72. package/dist-cjs/server/metrics.js +8 -0
  73. package/dist-cjs/server/runs.d.ts +9 -1
  74. package/dist-cjs/server/runs.js +45 -6
  75. package/dist-cjs/server/server.js +4 -1
  76. package/dist-cjs/server/state.d.ts +12 -2
  77. package/dist-cjs/server/state.js +15 -0
  78. package/dist-cjs/sqlite/operations.js +15 -4
  79. package/dist-cjs/sqlite/store.d.ts +7 -0
  80. package/dist-cjs/sqlite/store.js +28 -0
  81. package/dist-cjs/store/memory.d.ts +8 -0
  82. package/dist-cjs/store/memory.js +12 -1
  83. package/dist-cjs/store/redis.d.ts +13 -0
  84. package/dist-cjs/store/redis.js +38 -0
  85. package/dist-cjs/tracing/instrument.js +1 -0
  86. package/dist-cjs/tracing/tracer.d.ts +4 -0
  87. package/dist-cjs/tracing/tracer.js +6 -0
  88. package/dist-cjs/types/operations.d.ts +7 -0
  89. package/dist-cjs/types/server.d.ts +61 -2
  90. package/dist-cjs/types/store.d.ts +9 -0
  91. package/dist-cjs/version.d.ts +1 -1
  92. package/dist-cjs/version.js +1 -1
  93. package/llms.txt +1 -1
  94. package/package.json +27 -1
@@ -4,11 +4,28 @@ import { operationStats } from '../operations/stats.js';
4
4
  import { MemoryOperationStore } from '../operations/store.js';
5
5
  import { AssistantCapabilityError, BadRequestError, NotFoundError, TenantLimitError, ThreadBusyError, } from './errors.js';
6
6
  import { MemoryRunEventLog } from './events.js';
7
- import { gauge, ServerMetrics } from './metrics.js';
7
+ import { gauge, gauges, ServerMetrics } from './metrics.js';
8
8
  import { MemoryServerStore, RUNS_NAMESPACE } from './state.js';
9
9
  const THREADS = ['nexus', 'server', 'threads'];
10
10
  const RUNS = RUNS_NAMESPACE;
11
11
  const KIND_PREFIX = 'assistant:';
12
+ function runKind(assistant, revision) {
13
+ return revision === undefined ? `${KIND_PREFIX}${assistant}` : `${KIND_PREFIX}${assistant}@${revision}`;
14
+ }
15
+ function byRevision(queuedByKind) {
16
+ const counts = {};
17
+ for (const [kind, queued] of Object.entries(queuedByKind)) {
18
+ if (!kind.startsWith(KIND_PREFIX))
19
+ continue;
20
+ const rest = kind.slice(KIND_PREFIX.length);
21
+ const at = rest.lastIndexOf('@');
22
+ if (at <= 0)
23
+ continue;
24
+ const assistant = rest.slice(0, at);
25
+ counts[assistant] = { ...counts[assistant], [rest.slice(at + 1)]: queued };
26
+ }
27
+ return counts;
28
+ }
12
29
  const THREAD_CLAIM = 'nexus-thread:';
13
30
  const UNRECORDED_CLAIM_MS = 60_000;
14
31
  const DEFAULT_CONCURRENCY = 10;
@@ -69,7 +86,7 @@ export class RunManager {
69
86
  return this.draining;
70
87
  }
71
88
  assistant(id) {
72
- const found = this.options.assistants[id];
89
+ const found = Object.hasOwn(this.options.assistants, id) ? this.options.assistants[id] : undefined;
73
90
  if (!found)
74
91
  throw new BadRequestError(`Unknown assistant "${id}"`, 'UNKNOWN_ASSISTANT');
75
92
  return found;
@@ -140,6 +157,7 @@ export class RunManager {
140
157
  async startClaimed(runId, base, thread, options) {
141
158
  let assistant = base;
142
159
  let revision;
160
+ let elsewhere = false;
143
161
  if (base.route) {
144
162
  const choice = await base.route({
145
163
  runId,
@@ -147,11 +165,13 @@ export class RunManager {
147
165
  threadRevision: thread?.revision,
148
166
  requested: options.revision,
149
167
  principal: options.principal,
168
+ queued: !this.runsHere(),
150
169
  });
151
170
  if (choice) {
152
- const { assistant: chosen, ...chosenRevision } = choice;
171
+ const { assistant: chosen, local, ...chosenRevision } = choice;
153
172
  assistant = chosen;
154
173
  revision = chosenRevision;
174
+ elsewhere = local === false;
155
175
  }
156
176
  }
157
177
  else if (options.revision !== undefined) {
@@ -170,7 +190,7 @@ export class RunManager {
170
190
  throw error;
171
191
  }
172
192
  try {
173
- return await this.accept(runId, options, assistant, thread, revision);
193
+ return await this.accept(runId, options, assistant, thread, revision, elsewhere);
174
194
  }
175
195
  catch (error) {
176
196
  await this.releaseTenant(runId, options.principal?.tenantId);
@@ -226,6 +246,7 @@ export class RunManager {
226
246
  try {
227
247
  const handles = await this.runner.claimQueued((context) => this.executeStored(context), free, {
228
248
  kindPrefix: KIND_PREFIX,
249
+ kinds: this.claimableKinds(),
229
250
  });
230
251
  for (const handle of handles)
231
252
  this.track(handle);
@@ -235,6 +256,15 @@ export class RunManager {
235
256
  this.claiming -= free;
236
257
  }
237
258
  }
259
+ claimableKinds() {
260
+ const kinds = [];
261
+ for (const [id, assistant] of Object.entries(this.options.assistants)) {
262
+ kinds.push(runKind(id, undefined));
263
+ for (const revision of assistant.revisions ?? [])
264
+ kinds.push(runKind(id, revision));
265
+ }
266
+ return kinds;
267
+ }
238
268
  startWorking() {
239
269
  if (this.working || (!this.options.queue && this.options.recoverEveryMs === undefined))
240
270
  return;
@@ -306,6 +336,8 @@ export class RunManager {
306
336
  : 0,
307
337
  lapsedLeases: stats.lapsedLeases,
308
338
  replica: this.replica(metadata),
339
+ atomicChanges: typeof this.state.putIfVersion === 'function',
340
+ queuedByRevision: byRevision(stats.queuedByKind ?? {}),
309
341
  };
310
342
  }
311
343
  async prometheus(extra = () => undefined) {
@@ -316,6 +348,7 @@ export class RunManager {
316
348
  gauge(lines, 'nexus_server_queue_load', 'Queued plus running runs, across the deployment.', snapshot.load);
317
349
  gauge(lines, 'nexus_server_queue_oldest_seconds', 'Seconds the oldest queued run has waited.', snapshot.oldestQueuedSeconds);
318
350
  gauge(lines, 'nexus_server_leases_lapsed', 'Running runs whose lease lapsed.', snapshot.lapsedLeases);
351
+ gauges(lines, 'nexus_server_runs_queued_by_revision', 'Runs waiting for a worker that carries their revision, across the deployment.', Object.entries(snapshot.queuedByRevision ?? {}).flatMap(([assistant, revisions]) => Object.entries(revisions).map(([revision, queued]) => [{ assistant, revision }, queued])));
319
352
  gauge(lines, 'nexus_server_worker_in_flight', 'Runs this replica is executing.', snapshot.replica.inFlight);
320
353
  if (snapshot.replica.capacity !== undefined) {
321
354
  gauge(lines, 'nexus_server_worker_capacity', 'Runs this replica executes at once.', snapshot.replica.capacity);
@@ -325,7 +358,7 @@ export class RunManager {
325
358
  await extra(lines);
326
359
  return `${lines.join('\n')}\n`;
327
360
  }
328
- async accept(runId, options, assistant, thread, revision) {
361
+ async accept(runId, options, assistant, thread, revision, elsewhere = false) {
329
362
  const at = this.now().toISOString();
330
363
  const startedAtStep = thread && assistant.step ? await assistant.step(thread.id) : undefined;
331
364
  const run = {
@@ -351,7 +384,7 @@ export class RunManager {
351
384
  };
352
385
  const submit = {
353
386
  id: runId,
354
- kind: `${KIND_PREFIX}${options.assistant}`,
387
+ kind: runKind(options.assistant, revision?.id),
355
388
  idempotencyKey: options.idempotencyKey,
356
389
  metadata: submission,
357
390
  ...(this.options.runTimeoutMs === undefined
@@ -359,7 +392,7 @@ export class RunManager {
359
392
  : { expiresAt: new Date(this.now().getTime() + this.options.runTimeoutMs).toISOString() }),
360
393
  };
361
394
  let acceptedId;
362
- if (this.runsHere()) {
395
+ if (this.runsHere() && !elsewhere) {
363
396
  this.slots.add(runId);
364
397
  const plan = {
365
398
  assistantId: options.assistant,
@@ -700,6 +733,12 @@ export class RunManager {
700
733
  if (run.durationMs !== undefined) {
701
734
  this.metrics.observe('nexus_server_run_duration_seconds', labels, run.durationMs / 1000);
702
735
  }
736
+ try {
737
+ this.options.deployments?.recordRun?.(run);
738
+ }
739
+ catch (error) {
740
+ this.report(error, { runId: run.id });
741
+ }
703
742
  }
704
743
  async record(runId, type, data) {
705
744
  try {
@@ -56,7 +56,10 @@ export function createAgentServer(options) {
56
56
  const deployment = await deployments.get(params.assistant);
57
57
  if (!deployment)
58
58
  throw new NotFoundError('Deployment', params.assistant);
59
- const stats = await deployments.stats?.(deployment.assistant, { since: deployment.history[0]?.at });
59
+ const stats = await deployments.stats?.(deployment.assistant, {
60
+ since: deployment.history[0]?.at,
61
+ deployment: deployment.version,
62
+ });
60
63
  return json(stats ? { ...deployment, stats } : deployment);
61
64
  });
62
65
  admin('POST', '/deployments/:assistant', async ({ request, params, principal }) => {
@@ -9,6 +9,10 @@ export declare class MemoryServerStore implements ServerStateStore {
9
9
  get<V>(namespace: readonly string[], key: string): V | undefined;
10
10
  /** Writes a record. */
11
11
  put<V>(namespace: readonly string[], key: string, value: V): void;
12
+ /** Writes a record only when the stored one's `version` is `expected`, or none is stored and `expected` is null. */
13
+ putIfVersion<V extends {
14
+ version: number;
15
+ }>(namespace: readonly string[], key: string, value: V, expected: number | null): boolean;
12
16
  /** Removes a record. */
13
17
  delete(namespace: readonly string[], key: string): void;
14
18
  /** Records under a namespace, newest written first. Defaults to 50. */
@@ -26,6 +30,10 @@ export interface StoreLike {
26
30
  } | undefined> | {
27
31
  value: unknown;
28
32
  } | undefined;
33
+ /** Stores an item only when the stored one's `version` is `expected`. Optional. */
34
+ putIfVersion?(namespace: readonly string[], key: string, value: {
35
+ version: number;
36
+ }, expected: number | null): Promise<boolean> | boolean;
29
37
  /** Deletes an item. */
30
38
  delete(namespace: readonly string[], key: string): Promise<void> | void;
31
39
  /** Items under a namespace prefix, newest first. */
@@ -40,7 +48,9 @@ export interface StoreLike {
40
48
  /**
41
49
  * Server state on top of a long-term store, so threads and runs live wherever memory already does.
42
50
  *
43
- * `MemoryStore`, `RedisStore`, and `PostgresStore` all satisfy `StoreLike`, which is what lets a
44
- * second replica see the first replica's threads without the server growing its own adapters.
51
+ * `MemoryStore`, `RedisStore`, `PostgresStore`, and `SqliteStore` all satisfy `StoreLike`, which is
52
+ * what lets a second replica see the first replica's threads without the server growing its own
53
+ * adapters. A store with `putIfVersion()` passes it on, so deployment changes are atomic across
54
+ * replicas.
45
55
  */
46
56
  export declare function fromStore(store: StoreLike): ServerStateStore;
@@ -12,6 +12,13 @@ export class MemoryServerStore {
12
12
  bucket.set(key, { value, at: this.sequence });
13
13
  this.items.set(id, bucket);
14
14
  }
15
+ putIfVersion(namespace, key, value, expected) {
16
+ const current = this.get(namespace, key);
17
+ if (expected === null ? current !== undefined : current?.version !== expected)
18
+ return false;
19
+ this.put(namespace, key, value);
20
+ return true;
21
+ }
15
22
  delete(namespace, key) {
16
23
  this.items.get(namespace.join('\u0000'))?.delete(key);
17
24
  }
@@ -24,7 +31,15 @@ export class MemoryServerStore {
24
31
  }
25
32
  }
26
33
  export function fromStore(store) {
34
+ const versioned = store.putIfVersion?.bind(store);
27
35
  return {
36
+ ...(versioned
37
+ ? {
38
+ async putIfVersion(namespace, key, value, expected) {
39
+ return versioned(namespace, key, value, expected);
40
+ },
41
+ }
42
+ : {}),
28
43
  async get(namespace, key) {
29
44
  return (await store.get(namespace, key))?.value;
30
45
  },
@@ -132,22 +132,25 @@ export class SqliteOperationStore {
132
132
  async stats(now, filter) {
133
133
  const params = [now];
134
134
  const narrowed = filterSql(filter, params);
135
- const rows = (await this.client.all(`SELECT status, count(*) AS count,
135
+ const rows = (await this.client.all(`SELECT status, kind, count(*) AS count,
136
136
  min(CASE WHEN status = 'queued' THEN created_at END) AS oldest,
137
137
  sum(CASE WHEN status = 'running' AND (lease_expires_at IS NULL OR lease_expires_at <= ?) THEN 1 ELSE 0 END) AS lapsed
138
138
  FROM ${this.table}
139
139
  WHERE status NOT IN (${TERMINAL})${narrowed}
140
- GROUP BY status`, params));
140
+ GROUP BY status, kind`, params));
141
141
  const byStatus = {};
142
+ const queuedByKind = {};
142
143
  let oldestQueuedAt;
143
144
  let lapsedLeases = 0;
144
145
  for (const row of rows) {
145
- byStatus[row.status] = Number(row.count);
146
+ byStatus[row.status] = (byStatus[row.status] ?? 0) + Number(row.count);
147
+ if (row.status === 'queued')
148
+ queuedByKind[row.kind ?? ''] = (queuedByKind[row.kind ?? ''] ?? 0) + Number(row.count);
146
149
  if (row.oldest && (!oldestQueuedAt || row.oldest < oldestQueuedAt))
147
150
  oldestQueuedAt = row.oldest;
148
151
  lapsedLeases += Number(row.lapsed);
149
152
  }
150
- return { byStatus, ...(oldestQueuedAt ? { oldestQueuedAt } : {}), lapsedLeases };
153
+ return { byStatus, ...(oldestQueuedAt ? { oldestQueuedAt } : {}), lapsedLeases, queuedByKind };
151
154
  }
152
155
  async prune(before) {
153
156
  return (await this.client.run(`DELETE FROM ${this.table} WHERE status IN (${TERMINAL}) AND updated_at < ?`, [before])).changes;
@@ -167,6 +170,14 @@ function filterSql(filter, params) {
167
170
  params.push(filter.kindPrefix, filter.kindPrefix);
168
171
  sql += ' AND substr(kind, 1, length(?)) = ?';
169
172
  }
173
+ if (filter?.kinds) {
174
+ if (filter.kinds.length === 0)
175
+ sql += ' AND 0';
176
+ else {
177
+ params.push(...filter.kinds);
178
+ sql += ` AND kind IN (${filter.kinds.map(() => '?').join(', ')})`;
179
+ }
180
+ }
170
181
  if (filter?.tenantId !== undefined) {
171
182
  params.push(filter.tenantId);
172
183
  sql += " AND json_extract(doc, '$.tenantId') = ?";
@@ -36,6 +36,13 @@ export declare class SqliteStore implements Store {
36
36
  migrate(options?: SqliteMigrateOptions): Promise<MigrationResult>;
37
37
  /** Stores an item, keeping its original `createdAt` when it replaces one. */
38
38
  put<V>(namespace: StoreNamespace, key: string, value: V, options?: StorePutOptions): Promise<void>;
39
+ /**
40
+ * Stores an item only when the stored one's `version` is `expected`, or, with `expected` `null`,
41
+ * only when none is stored. One statement, so two processes sharing the file cannot both win.
42
+ */
43
+ putIfVersion<V extends {
44
+ version: number;
45
+ }>(namespace: StoreNamespace, key: string, value: V, expected: number | null): Promise<boolean>;
39
46
  /** Reads an item, or `undefined` when it does not exist or has expired. */
40
47
  get<V>(namespace: StoreNamespace, key: string): Promise<StoreItem<V> | undefined>;
41
48
  /** Deletes an item. */
@@ -66,6 +66,34 @@ export class SqliteStore {
66
66
  vector ? JSON.stringify(vector) : null,
67
67
  ]);
68
68
  }
69
+ async putIfVersion(namespace, key, value, expected) {
70
+ if (!key.trim())
71
+ throw new RangeError('A store key must not be empty');
72
+ const timestamp = this.now().toISOString();
73
+ const vector = await this.embed(value, undefined);
74
+ const embedding = vector ? JSON.stringify(vector) : null;
75
+ const rows = expected === null
76
+ ?
77
+ await this.client.all(`INSERT INTO ${this.table} (ns_key, key, namespace, value, created_at, updated_at, expires_at, embedding)
78
+ VALUES (?, ?, ?, ?, ?, ?, NULL, ?)
79
+ ON CONFLICT (ns_key, key) DO UPDATE SET value = excluded.value, created_at = excluded.created_at,
80
+ updated_at = excluded.updated_at, expires_at = NULL, embedding = excluded.embedding
81
+ WHERE expires_at IS NOT NULL AND expires_at <= excluded.updated_at
82
+ RETURNING key`, [
83
+ namespaceKey(namespace),
84
+ key,
85
+ JSON.stringify([...namespace]),
86
+ JSON.stringify(value),
87
+ timestamp,
88
+ timestamp,
89
+ embedding,
90
+ ])
91
+ : await this.client.all(`UPDATE ${this.table} SET value = ?, updated_at = ?, expires_at = NULL, embedding = ?
92
+ WHERE ns_key = ? AND key = ? AND (expires_at IS NULL OR expires_at > ?)
93
+ AND json_type(value, '$.version') IN ('integer', 'real') AND json_extract(value, '$.version') = ?
94
+ RETURNING key`, [JSON.stringify(value), timestamp, embedding, namespaceKey(namespace), key, timestamp, expected]);
95
+ return rows.length > 0;
96
+ }
69
97
  async get(namespace, key) {
70
98
  const [row] = (await this.client.all(`SELECT namespace, key, value, created_at, updated_at, expires_at FROM ${this.table}
71
99
  WHERE ns_key = ? AND key = ? AND (expires_at IS NULL OR expires_at > ?)`, [namespaceKey(namespace), key, this.now().toISOString()]));
@@ -28,6 +28,14 @@ export declare class MemoryStore implements Store {
28
28
  constructor(options?: MemoryStoreOptions);
29
29
  /** Stores an item, keeping its original `createdAt` when it replaces one. */
30
30
  put<V>(namespace: StoreNamespace, key: string, value: V, options?: StorePutOptions): Promise<void>;
31
+ /**
32
+ * Stores an item only when the stored one's `version` is `expected`, or, with `expected` `null`,
33
+ * only when none is stored. The check and the write run with nothing in between.
34
+ */
35
+ putIfVersion<V extends {
36
+ version: number;
37
+ }>(namespace: StoreNamespace, key: string, value: V, expected: number | null): Promise<boolean>;
38
+ private write;
31
39
  /** Reads an item, or `undefined` when it does not exist or has expired. */
32
40
  get<V>(namespace: StoreNamespace, key: string): StoreItem<V> | undefined;
33
41
  /** Deletes an item. */
@@ -16,10 +16,21 @@ export class MemoryStore {
16
16
  }
17
17
  async put(namespace, key, value, options = {}) {
18
18
  assertKey(key);
19
+ this.write(namespace, key, value, options, await this.embed(value, options.index));
20
+ }
21
+ async putIfVersion(namespace, key, value, expected) {
22
+ assertKey(key);
23
+ const vector = await this.embed(value, undefined);
24
+ const current = this.get(namespace, key);
25
+ if (expected === null ? current !== undefined : current?.value?.version !== expected)
26
+ return false;
27
+ this.write(namespace, key, value, {}, vector);
28
+ return true;
29
+ }
30
+ write(namespace, key, value, options, vector) {
19
31
  const id = idOf(namespace, key);
20
32
  const timestamp = this.now().toISOString();
21
33
  const existing = this.items.get(id);
22
- const vector = await this.embed(value, options.index);
23
34
  if (options.ttlMs !== undefined)
24
35
  this.expiring = true;
25
36
  this.items.delete(id);
@@ -22,6 +22,11 @@ export interface RedisStoreLikeClient {
22
22
  srem(key: string, member: string): Promise<unknown>;
23
23
  /** Reads a set. */
24
24
  smembers(key: string): Promise<string[]>;
25
+ /**
26
+ * Runs a Lua script, ioredis style: the script, the number of keys, then keys and arguments.
27
+ * Optional; with it, the store offers `putIfVersion()`, whose check and write run as one script.
28
+ */
29
+ eval?(script: string, numKeys: number, ...args: string[]): Promise<unknown>;
25
30
  }
26
31
  /** Options for the Redis store. */
27
32
  export interface RedisStoreOptions {
@@ -44,6 +49,13 @@ export declare class RedisStore implements Store {
44
49
  private readonly options;
45
50
  private readonly prefix;
46
51
  private readonly now;
52
+ /**
53
+ * Stores an item only when the stored one's `version` is `expected`, or, with `expected` `null`,
54
+ * only when none is stored, as one script. Present only when the client has `eval`.
55
+ */
56
+ readonly putIfVersion?: <V extends {
57
+ version: number;
58
+ }>(namespace: StoreNamespace, key: string, value: V, expected: number | null) => Promise<boolean>;
47
59
  constructor(client: RedisStoreLikeClient, options?: RedisStoreOptions);
48
60
  /** Stores an item, keeping its original `createdAt` when it replaces one. */
49
61
  put<V>(namespace: StoreNamespace, key: string, value: V, options?: StorePutOptions): Promise<void>;
@@ -62,6 +74,7 @@ export declare class RedisStore implements Store {
62
74
  limit?: number;
63
75
  }): Promise<string[][]>;
64
76
  private namespacesUnder;
77
+ private versionedPut;
65
78
  private read;
66
79
  private embed;
67
80
  private itemKey;
@@ -1,14 +1,29 @@
1
1
  import { cosine, matches, textOf } from './helpers.js';
2
+ const PUT_IF_VERSION = `-- nexus:put-if-version
3
+ local current = redis.call('GET', KEYS[1])
4
+ if current then
5
+ if ARGV[2] == '' then return 0 end
6
+ local ok, version = pcall(function() return cjson.decode(current).value.version end)
7
+ if not ok or type(version) ~= 'number' or version ~= tonumber(ARGV[2]) then return 0 end
8
+ elseif ARGV[2] ~= '' then
9
+ return 0
10
+ end
11
+ redis.call('SET', KEYS[1], ARGV[1])
12
+ return 1`;
2
13
  export class RedisStore {
3
14
  client;
4
15
  options;
5
16
  prefix;
6
17
  now;
18
+ putIfVersion;
7
19
  constructor(client, options = {}) {
8
20
  this.client = client;
9
21
  this.options = options;
10
22
  this.prefix = options.prefix ?? 'nexus:store';
11
23
  this.now = options.now ?? (() => new Date());
24
+ if (typeof client.eval === 'function') {
25
+ this.putIfVersion = (namespace, key, value, expected) => this.versionedPut(namespace, key, value, expected);
26
+ }
12
27
  }
13
28
  async put(namespace, key, value, options = {}) {
14
29
  if (!key.trim())
@@ -83,6 +98,29 @@ export class RedisStore {
83
98
  .map((entry) => entry.split('\u0000'))
84
99
  .filter((namespace) => prefix.every((part, index) => namespace[index] === part));
85
100
  }
101
+ async versionedPut(namespace, key, value, expected) {
102
+ if (!key.trim())
103
+ throw new RangeError('A store key must not be empty');
104
+ const id = this.itemKey(namespace, key);
105
+ const existing = await this.read(id);
106
+ const timestamp = this.now().toISOString();
107
+ const vector = await this.embed(value, undefined);
108
+ const record = {
109
+ namespace: [...namespace],
110
+ key,
111
+ value,
112
+ createdAt: existing?.createdAt ?? timestamp,
113
+ updatedAt: timestamp,
114
+ ...(vector ? { vector } : {}),
115
+ };
116
+ const eval_ = this.client.eval;
117
+ const wrote = Number(await eval_.call(this.client, PUT_IF_VERSION, 1, id, JSON.stringify(record), expected === null ? '' : String(expected)));
118
+ if (wrote !== 1)
119
+ return false;
120
+ await this.client.sadd(this.namespaceKey(namespace), key);
121
+ await this.client.sadd(this.indexKey(), namespace.join('\u0000'));
122
+ return true;
123
+ }
86
124
  async read(id) {
87
125
  const raw = await this.client.get(id);
88
126
  if (!raw)
@@ -140,6 +140,7 @@ export function traceModelClient(client, tracer, options = {}) {
140
140
  model: response.meta?.modelUsed,
141
141
  finishReason: response.finishReason,
142
142
  ...(request.metadata?.prompt === undefined ? {} : { prompt: request.metadata.prompt }),
143
+ ...(request.metadata?.context === undefined ? {} : { context: request.metadata.context }),
143
144
  ...(response.meta?.cache ? { cache: response.meta.cache } : {}),
144
145
  ...(request.tenantId ? { tenantId: request.tenantId } : {}),
145
146
  },
@@ -110,6 +110,10 @@ export declare class Tracer {
110
110
  private readonly writes;
111
111
  private readonly now;
112
112
  constructor(options: TracerOptions);
113
+ /** Where runs are written. */
114
+ get store(): TraceStore;
115
+ /** Whether runs are written as they start and finish, as `incremental` asks. */
116
+ get incremental(): boolean;
113
117
  /** The run currently in scope, if any. */
114
118
  current(): {
115
119
  traceId: string;
@@ -10,6 +10,12 @@ export class Tracer {
10
10
  this.options = options;
11
11
  this.now = options.now ?? (() => new Date());
12
12
  }
13
+ get store() {
14
+ return this.options.store;
15
+ }
16
+ get incremental() {
17
+ return this.options.incremental === true;
18
+ }
13
19
  current() {
14
20
  const active = this.context.getStore();
15
21
  return active ? { traceId: active.traceId, runId: active.runId } : undefined;
@@ -219,6 +219,11 @@ export interface OperationStore<TResult = unknown> {
219
219
  export interface OperationStoreFilter {
220
220
  /** Only records whose `kind` starts with this, such as `assistant:` for the agent server's runs. */
221
221
  kindPrefix?: string;
222
+ /**
223
+ * Only records whose `kind` is one of these, exactly. A worker carrying some revisions of an
224
+ * assistant claims with it, so it never takes a run routed to a revision it does not have.
225
+ */
226
+ kinds?: readonly string[];
222
227
  /** Only records of this tenant. Postgres and SQLite read it through an index on queued work. */
223
228
  tenantId?: string;
224
229
  }
@@ -234,6 +239,8 @@ export interface OperationStoreStats {
234
239
  oldestQueuedAt?: string;
235
240
  /** Records still marked running whose lease has lapsed: work a stopped worker left behind. */
236
241
  lapsedLeases: number;
242
+ /** Queued records per `kind`, for an autoscaler that sizes a pool per kind of work. */
243
+ queuedByKind?: Record<string, number>;
237
244
  }
238
245
  /**
239
246
  * Hands an accepted operation to a worker process.
@@ -110,11 +110,25 @@ export interface RevisionRequest {
110
110
  requested?: string;
111
111
  /** Who asked. */
112
112
  principal?: Principal;
113
+ /**
114
+ * Whether the run will wait for a worker rather than start on this replica. The split then takes in
115
+ * the revisions any fresh worker carries, not only this replica's, so a new revision in a new worker
116
+ * image takes its share before the API replicas carry it.
117
+ */
118
+ queued?: boolean;
113
119
  }
114
120
  /** The revision `route()` chose: where it came from, and its code. */
115
121
  export interface RevisionChoice extends RunRevision {
116
- /** The revision's assistant, which runs this request. */
122
+ /**
123
+ * The revision's assistant, which runs this request. When this replica does not carry the revision,
124
+ * its live revision stands in for the checks made before the run is queued.
125
+ */
117
126
  assistant: ServerAssistant;
127
+ /**
128
+ * Whether this replica carries the revision. When false, the run waits in the queue for a worker that
129
+ * does, and never runs here.
130
+ */
131
+ local?: boolean;
118
132
  }
119
133
  /**
120
134
  * Enforces per-tenant limits at the server: how many runs a tenant may have in flight, how many it may
@@ -186,6 +200,16 @@ export interface ScalingSnapshot {
186
200
  lapsedLeases: number;
187
201
  /** The replica that answered. */
188
202
  replica: ReplicaReport;
203
+ /**
204
+ * Whether deployment changes are atomic across replicas, which they are when the state store has
205
+ * `putIfVersion()`. When false, two replicas changing one deployment at once can lose a change.
206
+ */
207
+ atomicChanges?: boolean;
208
+ /**
209
+ * Queued runs per assistant and revision, such as `{ support: { v2: 12 } }`, so an autoscaler starts
210
+ * the worker pool that carries a revision. Runs without a revision are counted only in `queued`.
211
+ */
212
+ queuedByRevision?: Record<string, Record<string, number>>;
189
213
  }
190
214
  /** How traffic for one assistant is split between its revisions. */
191
215
  export interface DeploymentRecord {
@@ -199,6 +223,11 @@ export interface DeploymentRecord {
199
223
  traffic: Record<string, number>;
200
224
  /** ISO-8601 time the current canary began, when one is taking traffic. */
201
225
  canarySince?: string;
226
+ /**
227
+ * How a new thread or a stateless run is placed in the split. Absent means `fnv1a-v1`, which every
228
+ * deployment recorded before 2.5 uses, so no thread moves through an upgrade.
229
+ */
230
+ bucketStrategy?: BucketStrategy;
202
231
  /** ISO-8601 time of the last change. */
203
232
  updatedAt: string;
204
233
  /** Who made the last change. */
@@ -206,6 +235,16 @@ export interface DeploymentRecord {
206
235
  /** Changes, newest first, up to the last 50. */
207
236
  history: DeploymentChangeRecord[];
208
237
  }
238
+ /**
239
+ * How a thread or run id becomes a position in a traffic split.
240
+ *
241
+ * - `fnv1a-v1` is FNV-1a over the id. It places random ids evenly, but ids that differ only in their
242
+ * last characters, such as `user-1` to `user-500`, cluster, so a canary's share of them drifts from
243
+ * its weight.
244
+ * - `hash-v2` mixes the FNV-1a hash with MurmurHash3's finalizer, so sequential ids spread as evenly
245
+ * as random ones.
246
+ */
247
+ export type BucketStrategy = 'fnv1a-v1' | 'hash-v2';
209
248
  /** A change to a deployment. */
210
249
  export type DeploymentChange = {
211
250
  action: 'split';
@@ -224,6 +263,10 @@ export type DeploymentChange = {
224
263
  action: 'rollback';
225
264
  to?: string;
226
265
  reason?: string;
266
+ } | {
267
+ action: 'bucketing';
268
+ strategy: BucketStrategy;
269
+ reason?: string;
227
270
  };
228
271
  /** One entry in a deployment's history. */
229
272
  export interface DeploymentChangeRecord {
@@ -237,6 +280,8 @@ export interface DeploymentChangeRecord {
237
280
  live: string;
238
281
  /** The split after it. */
239
282
  traffic: Record<string, number>;
283
+ /** The bucket strategy a `bucketing` change chose. */
284
+ bucketStrategy?: BucketStrategy;
240
285
  /** Who did it: a person, or `guard` when a canary guard acted. */
241
286
  by?: string;
242
287
  /** Why, such as the regression that triggered a rollback. */
@@ -256,10 +301,16 @@ export interface ServerDeployments {
256
301
  change(assistant: string, change: DeploymentChange, by?: string): Promise<DeploymentRecord>;
257
302
  /** The replicas whose heartbeats are fresh. */
258
303
  replicas(): Promise<ReplicaReport[]> | ReplicaReport[];
259
- /** What each revision's runs did since a time, for `GET /deployments/:assistant`. */
304
+ /**
305
+ * What each revision's runs did, for `GET /deployments/:assistant`: since a time, or under one
306
+ * deployment version.
307
+ */
260
308
  stats?(assistant: string, options?: {
261
309
  since?: string;
310
+ deployment?: number;
262
311
  }): Promise<unknown[]> | unknown[];
312
+ /** Counts a finished run in this replica's rollups. The server calls it as each run finishes. */
313
+ recordRun?(run: RunRecord): void;
263
314
  /** Starts heartbeating this replica's report. */
264
315
  attach(report: () => ReplicaReport): Promise<void> | void;
265
316
  /** Stops heartbeating, and removes this replica's report. */
@@ -310,6 +361,14 @@ export interface ServerStateStore {
310
361
  get<V>(namespace: readonly string[], key: string): Promise<V | undefined> | V | undefined;
311
362
  /** Writes a record. */
312
363
  put<V>(namespace: readonly string[], key: string, value: V): Promise<void> | void;
364
+ /**
365
+ * Writes a record only when the stored one's `version` field is `expected`, or, with `expected`
366
+ * `null`, only when none is stored, and resolves whether it wrote. Optional: with it, two replicas
367
+ * changing one deployment at once never lose a change; without it, the later write wins.
368
+ */
369
+ putIfVersion?<V extends {
370
+ version: number;
371
+ }>(namespace: readonly string[], key: string, value: V, expected: number | null): Promise<boolean> | boolean;
313
372
  /** Removes a record. */
314
373
  delete(namespace: readonly string[], key: string): Promise<void> | void;
315
374
  /** Records under a namespace, newest first. */
@@ -66,6 +66,15 @@ export interface Store {
66
66
  put<V = unknown>(namespace: StoreNamespace, key: string, value: V, options?: StorePutOptions): Promise<void> | void;
67
67
  /** Reads an item, or `undefined` when it does not exist or has expired. */
68
68
  get<V = unknown>(namespace: StoreNamespace, key: string): Promise<StoreItem<V> | undefined> | StoreItem<V> | undefined;
69
+ /**
70
+ * Stores an item only when the stored one's `version` field is `expected`, or, with `expected`
71
+ * `null`, only when none is stored. Resolves whether it wrote. One atomic step in the store, so two
72
+ * processes deciding on the same version cannot both write: the loser reads again and decides
73
+ * again. Optional; the memory, Redis (with `eval`), Postgres, and SQLite stores have it.
74
+ */
75
+ putIfVersion?<V extends {
76
+ version: number;
77
+ }>(namespace: StoreNamespace, key: string, value: V, expected: number | null): Promise<boolean> | boolean;
69
78
  /** Deletes an item. */
70
79
  delete(namespace: StoreNamespace, key: string): Promise<void> | void;
71
80
  /** Items under a namespace prefix, newest first, or ranked by similarity when a query is given. */
package/dist/version.d.ts CHANGED
@@ -2,4 +2,4 @@
2
2
  * This package's version, recorded where provenance matters, such as on every experiment. A test
3
3
  * keeps it equal to `package.json`.
4
4
  */
5
- export declare const NEXUS_VERSION = "2.4.0";
5
+ export declare const NEXUS_VERSION = "2.5.0";