nexus-ai-pro 2.4.0 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/API_STABILITY.md +85 -0
  2. package/CHANGELOG.md +147 -1
  3. package/README.md +8 -3
  4. package/SECURITY.md +78 -9
  5. package/dist/context-hub/hub.d.ts +53 -0
  6. package/dist/context-hub/hub.js +108 -3
  7. package/dist/context-hub/index.d.ts +2 -1
  8. package/dist/context-hub/index.js +2 -1
  9. package/dist/context-hub/signing.d.ts +95 -0
  10. package/dist/context-hub/signing.js +88 -0
  11. package/dist/doctor/index.d.ts +4 -2
  12. package/dist/doctor/index.js +29 -0
  13. package/dist/evaluate/threads.d.ts +187 -0
  14. package/dist/evaluate/threads.js +187 -0
  15. package/dist/operations/adapters.d.ts +2 -1
  16. package/dist/operations/runner.js +2 -0
  17. package/dist/operations/stats.js +11 -4
  18. package/dist/postgres/index.d.ts +2 -1
  19. package/dist/postgres/index.js +4 -0
  20. package/dist/postgres/operations.js +11 -4
  21. package/dist/postgres/store.d.ts +7 -0
  22. package/dist/postgres/store.js +21 -0
  23. package/dist/postgres/tenancy.d.ts +43 -0
  24. package/dist/postgres/tenancy.js +71 -0
  25. package/dist/server/deployments.d.ts +127 -12
  26. package/dist/server/deployments.js +402 -37
  27. package/dist/server/metrics.d.ts +2 -0
  28. package/dist/server/metrics.js +7 -0
  29. package/dist/server/runs.d.ts +9 -1
  30. package/dist/server/runs.js +46 -7
  31. package/dist/server/server.js +4 -1
  32. package/dist/server/state.d.ts +12 -2
  33. package/dist/server/state.js +15 -0
  34. package/dist/sqlite/operations.js +15 -4
  35. package/dist/sqlite/store.d.ts +7 -0
  36. package/dist/sqlite/store.js +28 -0
  37. package/dist/store/memory.d.ts +8 -0
  38. package/dist/store/memory.js +12 -1
  39. package/dist/store/redis.d.ts +13 -0
  40. package/dist/store/redis.js +38 -0
  41. package/dist/tracing/instrument.js +1 -0
  42. package/dist/tracing/tracer.d.ts +4 -0
  43. package/dist/tracing/tracer.js +6 -0
  44. package/dist/types/operations.d.ts +7 -0
  45. package/dist/types/server.d.ts +61 -2
  46. package/dist/types/store.d.ts +9 -0
  47. package/dist/version.d.ts +1 -1
  48. package/dist/version.js +1 -1
  49. package/dist-cjs/context-hub/hub.d.ts +53 -0
  50. package/dist-cjs/context-hub/hub.js +110 -4
  51. package/dist-cjs/context-hub/index.d.ts +2 -1
  52. package/dist-cjs/context-hub/index.js +7 -1
  53. package/dist-cjs/context-hub/signing.d.ts +95 -0
  54. package/dist-cjs/context-hub/signing.js +96 -0
  55. package/dist-cjs/doctor/index.d.ts +4 -2
  56. package/dist-cjs/doctor/index.js +29 -0
  57. package/dist-cjs/evaluate/threads.d.ts +187 -0
  58. package/dist-cjs/evaluate/threads.js +198 -0
  59. package/dist-cjs/operations/adapters.d.ts +2 -1
  60. package/dist-cjs/operations/runner.js +2 -0
  61. package/dist-cjs/operations/stats.js +11 -4
  62. package/dist-cjs/postgres/index.d.ts +2 -1
  63. package/dist-cjs/postgres/index.js +8 -1
  64. package/dist-cjs/postgres/operations.js +11 -4
  65. package/dist-cjs/postgres/store.d.ts +7 -0
  66. package/dist-cjs/postgres/store.js +21 -0
  67. package/dist-cjs/postgres/tenancy.d.ts +43 -0
  68. package/dist-cjs/postgres/tenancy.js +110 -0
  69. package/dist-cjs/server/deployments.d.ts +127 -12
  70. package/dist-cjs/server/deployments.js +402 -37
  71. package/dist-cjs/server/metrics.d.ts +2 -0
  72. package/dist-cjs/server/metrics.js +8 -0
  73. package/dist-cjs/server/runs.d.ts +9 -1
  74. package/dist-cjs/server/runs.js +45 -6
  75. package/dist-cjs/server/server.js +4 -1
  76. package/dist-cjs/server/state.d.ts +12 -2
  77. package/dist-cjs/server/state.js +15 -0
  78. package/dist-cjs/sqlite/operations.js +15 -4
  79. package/dist-cjs/sqlite/store.d.ts +7 -0
  80. package/dist-cjs/sqlite/store.js +28 -0
  81. package/dist-cjs/store/memory.d.ts +8 -0
  82. package/dist-cjs/store/memory.js +12 -1
  83. package/dist-cjs/store/redis.d.ts +13 -0
  84. package/dist-cjs/store/redis.js +38 -0
  85. package/dist-cjs/tracing/instrument.js +1 -0
  86. package/dist-cjs/tracing/tracer.d.ts +4 -0
  87. package/dist-cjs/tracing/tracer.js +6 -0
  88. package/dist-cjs/types/operations.d.ts +7 -0
  89. package/dist-cjs/types/server.d.ts +61 -2
  90. package/dist-cjs/types/store.d.ts +9 -0
  91. package/dist-cjs/version.d.ts +1 -1
  92. package/dist-cjs/version.js +1 -1
  93. package/llms.txt +1 -1
  94. package/package.json +27 -1
@@ -0,0 +1,187 @@
1
+ /**
2
+ * Evaluation of whole conversations. A simulated user holds a conversation with an agent for each
3
+ * goal in a dataset, and thread evaluators score the conversation: whether the goal was met, each
4
+ * turn, and the turn where it went wrong. A multi-turn agent is then compared between versions as a
5
+ * single-turn one is, with `evaluate()` and `compareExperiments()`.
6
+ */
7
+ import type { LLMJudge } from '../evals/judge.js';
8
+ import type { Evaluator } from '../types/evaluate.js';
9
+ import type { CompletionRequest, Message } from '../types/messages.js';
10
+ import type { EvaluationTarget } from './run.js';
11
+ /** One turn of a conversation. */
12
+ export interface ThreadTurn {
13
+ /** Who spoke. */
14
+ role: 'user' | 'assistant';
15
+ /** What they said. */
16
+ content: string;
17
+ /** For an assistant turn, how long the agent took, in milliseconds. */
18
+ latencyMs?: number;
19
+ /**
20
+ * For an assistant turn, every message the agent added: its tool calls, their results, and the
21
+ * answer. The agent is given them back on later turns, and an evaluator can read its trajectory.
22
+ */
23
+ messages?: Message[];
24
+ }
25
+ /** Why a conversation ended. */
26
+ export type ThreadEnding = 'goal' | 'gave-up' | 'max-turns' | 'error';
27
+ /** A finished conversation: what a conversation target returns, and what thread evaluators read. */
28
+ export interface Thread {
29
+ /** Every turn, in order. */
30
+ turns: ThreadTurn[];
31
+ /**
32
+ * `goal` when the user said its goal was met, `gave-up` when it said it never would be, `max-turns`
33
+ * when the turn limit ended it, and `error` when the agent failed.
34
+ */
35
+ ended: ThreadEnding;
36
+ /** The agent's error, when it failed. */
37
+ error?: string;
38
+ }
39
+ /** What a conversation is about: the user's goal, and optionally who they are and how they open. */
40
+ export interface ThreadInputs {
41
+ /** What the user wants, in their words or the dataset's. */
42
+ goal: string;
43
+ /** Who the user is and how they talk, such as "an impatient customer who writes in short lines". */
44
+ persona?: string;
45
+ /** The user's first message, instead of letting the simulated user write it. */
46
+ opening?: string;
47
+ }
48
+ /** What a simulated user sees when it writes its next message. */
49
+ export interface SimulatedUserContext {
50
+ /** The goal. */
51
+ goal: string;
52
+ /** The persona, when the example gives one. */
53
+ persona?: string;
54
+ /** The conversation so far. Empty for the first message. */
55
+ turns: readonly ThreadTurn[];
56
+ /** Aborted when the evaluation is cancelled or times out. */
57
+ signal?: AbortSignal;
58
+ }
59
+ /** A simulated user's move: its next message, or the end of the conversation and why. */
60
+ export type SimulatedUserReply = {
61
+ message: string;
62
+ } | {
63
+ done: 'goal' | 'gave-up';
64
+ };
65
+ /** Plays the user in a conversation. */
66
+ export interface SimulatedUser {
67
+ /** The next move, given the conversation so far. */
68
+ next(context: SimulatedUserContext): Promise<SimulatedUserReply> | SimulatedUserReply;
69
+ }
70
+ /** A model client, as the judges take one. */
71
+ export interface SimulatedUserClient {
72
+ /** Runs one completion. */
73
+ complete(request: CompletionRequest): Promise<{
74
+ content: string;
75
+ }>;
76
+ }
77
+ /** Options for `simulatedUser()`. */
78
+ export interface SimulatedUserOptions {
79
+ /** The client the user's model is called through. */
80
+ client: SimulatedUserClient;
81
+ /** The model that plays the user. */
82
+ model: string;
83
+ /** A persona for every conversation, when the examples give none. */
84
+ persona?: string;
85
+ /** Sampling temperature. Defaults to 0, so a conversation repeats. */
86
+ temperature?: number;
87
+ /** Output token limit for each message. Defaults to 300. */
88
+ maxTokens?: number;
89
+ /** Replaces the instructions the model is given. `{goal}` and `{persona}` are filled in. */
90
+ instructions?: string;
91
+ }
92
+ /**
93
+ * A simulated user driven by a model: it is told the goal and the persona, sees the conversation
94
+ * with the roles turned around, and writes the next message, or says it is done and why.
95
+ */
96
+ export declare function simulatedUser(options: SimulatedUserOptions): SimulatedUser;
97
+ /**
98
+ * A user that follows a script, for deterministic tests: its messages in order, then `done`. A
99
+ * function decides each move itself.
100
+ */
101
+ export declare function scriptedUser(script: {
102
+ messages: readonly string[];
103
+ done?: 'goal' | 'gave-up';
104
+ } | SimulatedUser['next']): SimulatedUser;
105
+ /** What a thread agent is told besides the conversation. */
106
+ export interface ThreadAgentContext<I extends ThreadInputs = ThreadInputs> {
107
+ /** The example's inputs. */
108
+ inputs: I;
109
+ /** The turn, counted from 1. */
110
+ turn: number;
111
+ /** Names this conversation, the same on every turn and new for every conversation: a thread id. */
112
+ conversationId: string;
113
+ /** Aborted when the evaluation is cancelled or times out. */
114
+ signal?: AbortSignal;
115
+ }
116
+ /**
117
+ * What a thread agent answers: the reply, or the reply and every message the agent added on the way,
118
+ * such as its tool calls and their results. Those are given back to it on later turns.
119
+ */
120
+ export type ThreadAgentReply = string | {
121
+ content: string;
122
+ messages?: Message[];
123
+ };
124
+ /**
125
+ * The agent a conversation target talks to. It is given the conversation so far, with its own earlier
126
+ * turns as it recorded them, and answers the user's newest message.
127
+ */
128
+ export type ThreadAgent<I extends ThreadInputs = ThreadInputs> = (messages: readonly Message[], context: ThreadAgentContext<I>) => Promise<ThreadAgentReply> | ThreadAgentReply;
129
+ /** Options for `conversationTarget()`. */
130
+ export interface ConversationTargetOptions<I extends ThreadInputs = ThreadInputs> {
131
+ /** The user, or a user per example, such as one with the example's own persona. */
132
+ user: SimulatedUser | ((inputs: I) => SimulatedUser);
133
+ /** The most user messages a conversation may have. Defaults to 10. */
134
+ maxTurns?: number;
135
+ }
136
+ /**
137
+ * An `evaluate()` target that holds a conversation per example: the simulated user speaks, the agent
138
+ * answers, until the user says it is done or the turn limit is reached. It returns the `Thread`.
139
+ */
140
+ export declare function conversationTarget<I extends ThreadInputs = ThreadInputs>(agent: ThreadAgent<I>, options: ConversationTargetOptions<I>): EvaluationTarget<I>;
141
+ /** A graph a thread agent can drive: one that takes `{ messages }`, as `createAgent()` builds. */
142
+ export interface ThreadGraph {
143
+ /** Runs one turn. */
144
+ invoke(input: {
145
+ messages: Message[];
146
+ }, options?: {
147
+ threadId?: string;
148
+ signal?: AbortSignal;
149
+ }): Promise<{
150
+ state: unknown;
151
+ }>;
152
+ }
153
+ /**
154
+ * An agent from `createAgent()`, or any graph that takes `{ messages }` and ends with an `answer`, as
155
+ * a thread agent. Each turn runs on the conversation's thread with the transcript so far, and the
156
+ * messages the run added to its `messages` are recorded with the turn. The agent therefore sees its
157
+ * own tool calls on later turns, as it does with a real user, without needing a checkpointer.
158
+ */
159
+ export declare function graphThreadAgent(graph: ThreadGraph): ThreadAgent;
160
+ /** The conversation as text, for a judge or a person. */
161
+ export declare function formatThread(thread: Thread): string;
162
+ /**
163
+ * Whether the user said its goal was met: 1 when the conversation ended on `goal`, otherwise 0, with
164
+ * how it ended as the comment. Key `goal_reached`.
165
+ */
166
+ export declare function goalReached(): Evaluator<ThreadInputs>;
167
+ /**
168
+ * Whether the goal was met, in a judge's opinion of the whole conversation, not the user's. Key
169
+ * `goal_completion`, with the judge's rationale as the comment.
170
+ */
171
+ export declare function goalCompletion(judge: Pick<LLMJudge, 'evaluate'>): Evaluator<ThreadInputs>;
172
+ /** How many times the assistant spoke: fewer is better. Key `turns`; compare with `lowerIsBetter: ['turns']`. */
173
+ export declare function turnCount(): Evaluator<ThreadInputs>;
174
+ /** Options for `turnScores()`. */
175
+ export interface TurnScoresOptions {
176
+ /** The score's key. Defaults to `turn_score`. */
177
+ key?: string;
178
+ /** A turn scoring below this went wrong. Defaults to 0.5. */
179
+ threshold?: number;
180
+ }
181
+ /**
182
+ * Scores every assistant turn with an evaluator of single answers, and the conversation by their
183
+ * mean. The evaluator sees each turn as an example of its own: the user's message as the input, the
184
+ * conversation before it in `metadata.history`, and the turn as the output. The score's metadata holds
185
+ * each turn's score, and `wentWrongAt`, the first turn below the threshold, counted from 1.
186
+ */
187
+ export declare function turnScores<I = unknown>(evaluator: Evaluator<I>, options?: TurnScoresOptions): Evaluator<ThreadInputs>;
@@ -0,0 +1,198 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.simulatedUser = simulatedUser;
4
+ exports.scriptedUser = scriptedUser;
5
+ exports.conversationTarget = conversationTarget;
6
+ exports.graphThreadAgent = graphThreadAgent;
7
+ exports.formatThread = formatThread;
8
+ exports.goalReached = goalReached;
9
+ exports.goalCompletion = goalCompletion;
10
+ exports.turnCount = turnCount;
11
+ exports.turnScores = turnScores;
12
+ const USER_INSTRUCTIONS = `You are playing a user who is talking to an assistant. Stay in character.
13
+ Your goal: {goal}
14
+ {persona}
15
+ Write only your next message to the assistant, as the user would. Give details the assistant asks for when your goal needs them.
16
+ When the assistant has fully achieved your goal, reply with exactly: DONE: GOAL
17
+ When you are sure the assistant will never achieve it, reply with exactly: DONE: GAVE UP`;
18
+ const DONE = /^\s*DONE:\s*(GOAL|GAVE[\s-]*UP)\b/i;
19
+ function simulatedUser(options) {
20
+ return {
21
+ async next(context) {
22
+ const instructions = (options.instructions ?? USER_INSTRUCTIONS)
23
+ .replace('{goal}', context.goal)
24
+ .replace('{persona}', context.persona ?? options.persona ?? '');
25
+ const messages = [
26
+ { role: 'system', content: instructions },
27
+ ...context.turns.map((turn) => ({
28
+ role: turn.role === 'assistant' ? 'user' : 'assistant',
29
+ content: turn.content,
30
+ })),
31
+ ];
32
+ if (context.turns.length === 0)
33
+ messages.push({ role: 'user', content: 'Begin the conversation.' });
34
+ const response = await options.client.complete({
35
+ model: options.model,
36
+ messages,
37
+ temperature: options.temperature ?? 0,
38
+ maxTokens: options.maxTokens ?? 300,
39
+ ...(context.signal ? { signal: context.signal } : {}),
40
+ });
41
+ const done = DONE.exec(response.content ?? '');
42
+ if (done)
43
+ return { done: /^GOAL$/i.test(done[1]) ? 'goal' : 'gave-up' };
44
+ return { message: (response.content ?? '').trim() };
45
+ },
46
+ };
47
+ }
48
+ function scriptedUser(script) {
49
+ if (typeof script === 'function')
50
+ return { next: script };
51
+ return {
52
+ next: ({ turns }) => {
53
+ const said = turns.filter((turn) => turn.role === 'user').length;
54
+ const message = script.messages[said];
55
+ return message === undefined ? { done: script.done ?? 'goal' } : { message };
56
+ },
57
+ };
58
+ }
59
+ function conversationTarget(agent, options) {
60
+ const maxTurns = options.maxTurns ?? 10;
61
+ return async (inputs, context) => {
62
+ const user = typeof options.user === 'function' ? options.user(inputs) : options.user;
63
+ const turns = [];
64
+ const conversationId = `${context.example.id}:${context.run}:${globalThis.crypto.randomUUID()}`;
65
+ const ask = async () => user.next({
66
+ goal: inputs.goal,
67
+ ...(inputs.persona === undefined ? {} : { persona: inputs.persona }),
68
+ turns,
69
+ ...(context.signal ? { signal: context.signal } : {}),
70
+ });
71
+ let move = inputs.opening === undefined ? await ask() : { message: inputs.opening };
72
+ for (let turn = 1;; turn += 1) {
73
+ if ('done' in move)
74
+ return { turns, ended: move.done };
75
+ if (turn > maxTurns)
76
+ return { turns, ended: 'max-turns' };
77
+ turns.push({ role: 'user', content: move.message });
78
+ const started = performance.now();
79
+ try {
80
+ const reply = await agent(transcriptOf(turns), {
81
+ inputs,
82
+ turn,
83
+ conversationId,
84
+ ...(context.signal ? { signal: context.signal } : {}),
85
+ });
86
+ const latencyMs = Math.round(performance.now() - started);
87
+ turns.push(typeof reply === 'string'
88
+ ? { role: 'assistant', content: reply, latencyMs }
89
+ : {
90
+ role: 'assistant',
91
+ content: reply.content,
92
+ latencyMs,
93
+ ...(reply.messages ? { messages: reply.messages } : {}),
94
+ });
95
+ }
96
+ catch (error) {
97
+ return {
98
+ turns,
99
+ ended: 'error',
100
+ error: error instanceof Error ? error.message : String(error),
101
+ };
102
+ }
103
+ move = await ask();
104
+ }
105
+ };
106
+ }
107
+ function transcriptOf(turns) {
108
+ return turns.flatMap((turn) => turn.role === 'assistant' && turn.messages?.length ? turn.messages : [{ role: turn.role, content: turn.content }]);
109
+ }
110
+ function graphThreadAgent(graph) {
111
+ return async (messages, context) => {
112
+ const result = await graph.invoke({ messages: [...messages] }, { threadId: context.conversationId, ...(context.signal ? { signal: context.signal } : {}) });
113
+ const state = result.state;
114
+ const content = typeof state?.answer === 'string' ? state.answer : JSON.stringify(result.state);
115
+ const after = Array.isArray(state?.messages) ? state.messages : undefined;
116
+ return after && after.length > messages.length ? { content, messages: after.slice(messages.length) } : content;
117
+ };
118
+ }
119
+ function threadOf(context) {
120
+ const output = context.output;
121
+ return output && Array.isArray(output.turns) ? output : undefined;
122
+ }
123
+ function formatThread(thread) {
124
+ return thread.turns.map((turn) => `${turn.role === 'user' ? 'User' : 'Assistant'}: ${turn.content}`).join('\n\n');
125
+ }
126
+ function goalReached() {
127
+ return (context) => {
128
+ const thread = threadOf(context);
129
+ const reached = thread?.ended === 'goal';
130
+ return {
131
+ key: 'goal_reached',
132
+ score: reached ? 1 : 0,
133
+ passed: reached,
134
+ comment: thread ? `ended: ${thread.ended}${thread.error ? ` (${thread.error})` : ''}` : 'no conversation',
135
+ };
136
+ };
137
+ }
138
+ function goalCompletion(judge) {
139
+ return async (context) => {
140
+ const thread = threadOf(context);
141
+ if (!thread)
142
+ return { key: 'goal_completion', score: 0, passed: false, comment: 'no conversation' };
143
+ const verdict = await judge.evaluate({
144
+ actual: formatThread(thread),
145
+ query: context.example.inputs.goal,
146
+ rubric: "The text is a conversation between a user and an assistant. Score whether the assistant achieved the user's goal, given as the question, by the end of the conversation.",
147
+ });
148
+ return {
149
+ key: 'goal_completion',
150
+ score: verdict.score,
151
+ passed: verdict.passed,
152
+ ...(verdict.rationale ? { comment: verdict.rationale } : {}),
153
+ };
154
+ };
155
+ }
156
+ function turnCount() {
157
+ return (context) => ({
158
+ key: 'turns',
159
+ score: threadOf(context)?.turns.filter((turn) => turn.role === 'assistant').length ?? 0,
160
+ });
161
+ }
162
+ function turnScores(evaluator, options = {}) {
163
+ const key = options.key ?? 'turn_score';
164
+ const threshold = options.threshold ?? 0.5;
165
+ return async (context) => {
166
+ const thread = threadOf(context);
167
+ const scores = [];
168
+ for (const [index, turn] of (thread?.turns ?? []).entries()) {
169
+ if (turn.role !== 'assistant')
170
+ continue;
171
+ const history = thread.turns.slice(0, index);
172
+ const question = [...history].reverse().find((item) => item.role === 'user')?.content ?? '';
173
+ const example = {
174
+ ...context.example,
175
+ id: `${context.example.id}#${scores.length + 1}`,
176
+ inputs: question,
177
+ metadata: { ...context.example.metadata, history, goal: context.example.inputs.goal },
178
+ };
179
+ scores.push(numeric(await evaluator({ ...context, example, output: turn.content })));
180
+ }
181
+ const mean = scores.length ? scores.reduce((sum, score) => sum + score, 0) / scores.length : 0;
182
+ const wrong = scores.findIndex((score) => score < threshold);
183
+ return {
184
+ key,
185
+ score: mean,
186
+ metadata: { turns: scores, ...(wrong === -1 ? {} : { wentWrongAt: wrong + 1 }) },
187
+ ...(wrong === -1 ? {} : { comment: `turn ${wrong + 1} scored ${scores[wrong].toFixed(2)}` }),
188
+ };
189
+ };
190
+ }
191
+ function numeric(result) {
192
+ if (typeof result === 'number')
193
+ return result;
194
+ if (typeof result === 'boolean')
195
+ return result ? 1 : 0;
196
+ const first = Array.isArray(result) ? result[0] : result;
197
+ return first?.score ?? 0;
198
+ }
@@ -58,7 +58,8 @@ export declare class RedisOperationStore<TResult = unknown> implements Operation
58
58
  private readonly client;
59
59
  private readonly prefix;
60
60
  private readonly useEval;
61
- private readonly indexed;
61
+ /** Whether queued and leased work is kept in sorted sets, as `index: true` asks. */
62
+ readonly indexed: boolean;
62
63
  constructor(client: RedisOperationLikeClient, options?: RedisOperationStoreOptions);
63
64
  /**
64
65
  * Stores a new record and indexes its idempotency key. Throws `OperationDuplicateError` when another
@@ -54,6 +54,8 @@ class OperationRunner {
54
54
  for (const record of candidates) {
55
55
  if (claimed.length >= limit)
56
56
  break;
57
+ if (!(0, stats_js_1.matchesFilter)(record, filter))
58
+ continue;
57
59
  if (record.expiresAt && record.expiresAt <= nowIso) {
58
60
  await this.settleExpired(record);
59
61
  continue;
@@ -11,29 +11,36 @@ function isUnheldQueued(record, now) {
11
11
  function matchesFilter(record, filter) {
12
12
  if (filter?.tenantId !== undefined && record.tenantId !== filter.tenantId)
13
13
  return false;
14
+ if (filter?.kinds && !filter.kinds.includes(record.kind ?? ''))
15
+ return false;
14
16
  return !filter?.kindPrefix || (record.kind ?? '').startsWith(filter.kindPrefix);
15
17
  }
16
18
  function countRecords(records, now, filter) {
17
19
  const byStatus = {};
18
20
  let oldestQueuedAt;
19
21
  let lapsedLeases = 0;
22
+ const queuedByKind = {};
20
23
  for (const record of records) {
21
24
  if ((0, operations_js_1.isTerminalOperationStatus)(record.status) || !matchesFilter(record, filter))
22
25
  continue;
23
26
  byStatus[record.status] = (byStatus[record.status] ?? 0) + 1;
24
- if (record.status === 'queued' && (!oldestQueuedAt || record.createdAt < oldestQueuedAt)) {
25
- oldestQueuedAt = record.createdAt;
27
+ if (record.status === 'queued') {
28
+ if (!oldestQueuedAt || record.createdAt < oldestQueuedAt)
29
+ oldestQueuedAt = record.createdAt;
30
+ const kind = record.kind ?? '';
31
+ queuedByKind[kind] = (queuedByKind[kind] ?? 0) + 1;
26
32
  }
27
33
  if (record.status === 'running' && (!record.lease || record.lease.expiresAt <= now))
28
34
  lapsedLeases += 1;
29
35
  }
30
- return { byStatus, ...(oldestQueuedAt ? { oldestQueuedAt } : {}), lapsedLeases };
36
+ return { byStatus, ...(oldestQueuedAt ? { oldestQueuedAt } : {}), lapsedLeases, queuedByKind };
31
37
  }
32
38
  async function operationStats(store, options = {}) {
33
39
  const now = (options.now ?? new Date()).toISOString();
34
- const filter = options.kindPrefix || options.tenantId !== undefined
40
+ const filter = options.kindPrefix || options.kinds || options.tenantId !== undefined
35
41
  ? {
36
42
  ...(options.kindPrefix ? { kindPrefix: options.kindPrefix } : {}),
43
+ ...(options.kinds ? { kinds: options.kinds } : {}),
37
44
  ...(options.tenantId !== undefined ? { tenantId: options.tenantId } : {}),
38
45
  }
39
46
  : undefined;
@@ -9,9 +9,10 @@ export { PostgresPromptStore, type PostgresPromptStoreOptions, promptStoreMigrat
9
9
  export { PostgresRollupStore, type PostgresRollupStoreOptions, rollupStoreMigration, rollupStoreMigrations, } from './rollups.js';
10
10
  export { PostgresStore, type PostgresStoreOptions, storeMigration, storeMigrations } from './store.js';
11
11
  export { PostgresTraceStore, type PostgresTraceStoreOptions, traceStoreMigration, traceStoreMigrations, } from './traces.js';
12
+ export { PostgresTenantUsage, type PostgresTenantUsageOptions, tenantUsageMigration, tenantUsageMigrations, } from './tenancy.js';
12
13
  export { PostgresVectorStore, type PostgresVectorStoreOptions, vectorStoreMigration, vectorStoreMigrations, } from './vectors.js';
13
14
  /** The Postgres adapters a migration can include. */
14
- export type PostgresAdapter = 'operations' | 'store' | 'traces' | 'evaluation' | 'circuits' | 'prompts' | 'rollups' | 'vectors';
15
+ export type PostgresAdapter = 'operations' | 'store' | 'traces' | 'evaluation' | 'circuits' | 'tenancy' | 'prompts' | 'rollups' | 'vectors';
15
16
  /** Options for `postgresMigrations()` and `postgresMigration()`. */
16
17
  export interface PostgresMigrationOptions {
17
18
  /**
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.POSTGRES_ADAPTERS = exports.vectorStoreMigrations = exports.vectorStoreMigration = exports.PostgresVectorStore = exports.traceStoreMigrations = exports.traceStoreMigration = exports.PostgresTraceStore = exports.storeMigrations = exports.storeMigration = exports.PostgresStore = exports.rollupStoreMigrations = exports.rollupStoreMigration = exports.PostgresRollupStore = exports.promptStoreMigrations = exports.promptStoreMigration = exports.PostgresPromptStore = exports.PostgresOperationStore = exports.operationStoreMigrations = exports.operationStoreMigration = exports.postgresMigrationStatus = exports.applyPostgresMigrations = exports.PostgresExperimentStore = exports.PostgresDatasetStore = exports.evaluationStoreMigrations = exports.evaluationStoreMigration = exports.fromPostgresJs = exports.PostgresCircuitStateStore = exports.circuitStoreMigrations = exports.circuitStoreMigration = exports.SchemaMigrationError = exports.migrationChecksum = void 0;
3
+ exports.POSTGRES_ADAPTERS = exports.vectorStoreMigrations = exports.vectorStoreMigration = exports.PostgresVectorStore = exports.tenantUsageMigrations = exports.tenantUsageMigration = exports.PostgresTenantUsage = exports.traceStoreMigrations = exports.traceStoreMigration = exports.PostgresTraceStore = exports.storeMigrations = exports.storeMigration = exports.PostgresStore = exports.rollupStoreMigrations = exports.rollupStoreMigration = exports.PostgresRollupStore = exports.promptStoreMigrations = exports.promptStoreMigration = exports.PostgresPromptStore = exports.PostgresOperationStore = exports.operationStoreMigrations = exports.operationStoreMigration = exports.postgresMigrationStatus = exports.applyPostgresMigrations = exports.PostgresExperimentStore = exports.PostgresDatasetStore = exports.evaluationStoreMigrations = exports.evaluationStoreMigration = exports.fromPostgresJs = exports.PostgresCircuitStateStore = exports.circuitStoreMigrations = exports.circuitStoreMigration = exports.SchemaMigrationError = exports.migrationChecksum = void 0;
4
4
  exports.postgresMigrations = postgresMigrations;
5
5
  exports.postgresMigration = postgresMigration;
6
6
  const schema_migrations_js_1 = require("../utils/schema-migrations.js");
@@ -12,6 +12,7 @@ const operations_js_1 = require("./operations.js");
12
12
  const prompts_js_1 = require("./prompts.js");
13
13
  const rollups_js_1 = require("./rollups.js");
14
14
  const store_js_1 = require("./store.js");
15
+ const tenancy_js_1 = require("./tenancy.js");
15
16
  const traces_js_1 = require("./traces.js");
16
17
  const vectors_js_1 = require("./vectors.js");
17
18
  var schema_migrations_js_2 = require("../utils/schema-migrations.js");
@@ -51,6 +52,10 @@ var traces_js_2 = require("./traces.js");
51
52
  Object.defineProperty(exports, "PostgresTraceStore", { enumerable: true, get: function () { return traces_js_2.PostgresTraceStore; } });
52
53
  Object.defineProperty(exports, "traceStoreMigration", { enumerable: true, get: function () { return traces_js_2.traceStoreMigration; } });
53
54
  Object.defineProperty(exports, "traceStoreMigrations", { enumerable: true, get: function () { return traces_js_2.traceStoreMigrations; } });
55
+ var tenancy_js_2 = require("./tenancy.js");
56
+ Object.defineProperty(exports, "PostgresTenantUsage", { enumerable: true, get: function () { return tenancy_js_2.PostgresTenantUsage; } });
57
+ Object.defineProperty(exports, "tenantUsageMigration", { enumerable: true, get: function () { return tenancy_js_2.tenantUsageMigration; } });
58
+ Object.defineProperty(exports, "tenantUsageMigrations", { enumerable: true, get: function () { return tenancy_js_2.tenantUsageMigrations; } });
54
59
  var vectors_js_2 = require("./vectors.js");
55
60
  Object.defineProperty(exports, "PostgresVectorStore", { enumerable: true, get: function () { return vectors_js_2.PostgresVectorStore; } });
56
61
  Object.defineProperty(exports, "vectorStoreMigration", { enumerable: true, get: function () { return vectors_js_2.vectorStoreMigration; } });
@@ -61,6 +66,7 @@ exports.POSTGRES_ADAPTERS = [
61
66
  'traces',
62
67
  'evaluation',
63
68
  'circuits',
69
+ 'tenancy',
64
70
  'prompts',
65
71
  'rollups',
66
72
  'vectors',
@@ -74,6 +80,7 @@ function postgresMigrations(options = {}) {
74
80
  traces: () => (0, traces_js_1.traceStoreMigrations)(),
75
81
  evaluation: () => (0, evaluate_js_1.evaluationStoreMigrations)(),
76
82
  circuits: () => (0, circuits_js_1.circuitStoreMigrations)(),
83
+ tenancy: () => (0, tenancy_js_1.tenantUsageMigrations)(),
77
84
  prompts: () => (0, prompts_js_1.promptStoreMigrations)(),
78
85
  rollups: () => (0, rollups_js_1.rollupStoreMigrations)(),
79
86
  vectors: () => {
@@ -169,12 +169,12 @@ class PostgresOperationStore {
169
169
  async stats(now, filter) {
170
170
  const params = [now];
171
171
  const narrowed = filterSql(filter, params);
172
- const { rows } = await this.client.query(`SELECT status, count(*)::int AS count,
172
+ const { rows } = await this.client.query(`SELECT status, kind, count(*)::int AS count,
173
173
  min(created_at) FILTER (WHERE status = 'queued') AS oldest,
174
174
  count(*) FILTER (WHERE status = 'running' AND (lease_expires_at IS NULL OR lease_expires_at <= $1))::int AS lapsed
175
175
  FROM ${this.table}
176
176
  WHERE status NOT IN (${TERMINAL})${narrowed}
177
- GROUP BY status`, params);
177
+ GROUP BY status, kind`, params);
178
178
  return statsFromRows(rows);
179
179
  }
180
180
  async prune(before) {
@@ -202,6 +202,10 @@ function filterSql(filter, params) {
202
202
  params.push(filter.kindPrefix);
203
203
  sql += ` AND substr(kind, 1, char_length($${params.length})) = $${params.length}`;
204
204
  }
205
+ if (filter?.kinds) {
206
+ params.push([...filter.kinds]);
207
+ sql += ` AND kind = ANY($${params.length}::text[])`;
208
+ }
205
209
  if (filter?.tenantId !== undefined) {
206
210
  params.push(filter.tenantId);
207
211
  sql += ` AND doc ->> 'tenantId' = $${params.length}`;
@@ -212,11 +216,14 @@ function statsFromRows(rows) {
212
216
  const byStatus = {};
213
217
  let oldestQueuedAt;
214
218
  let lapsedLeases = 0;
219
+ const queuedByKind = {};
215
220
  for (const row of rows) {
216
- byStatus[row.status] = Number(row.count);
221
+ byStatus[row.status] = (byStatus[row.status] ?? 0) + Number(row.count);
222
+ if (row.status === 'queued')
223
+ queuedByKind[row.kind ?? ''] = (queuedByKind[row.kind ?? ''] ?? 0) + Number(row.count);
217
224
  if (row.oldest && (!oldestQueuedAt || row.oldest < oldestQueuedAt))
218
225
  oldestQueuedAt = row.oldest;
219
226
  lapsedLeases += Number(row.lapsed);
220
227
  }
221
- return { byStatus, ...(oldestQueuedAt ? { oldestQueuedAt } : {}), lapsedLeases };
228
+ return { byStatus, ...(oldestQueuedAt ? { oldestQueuedAt } : {}), lapsedLeases, queuedByKind };
222
229
  }
@@ -44,6 +44,13 @@ export declare class PostgresStore implements Store {
44
44
  migrate(options?: PostgresMigrateOptions): Promise<MigrationResult>;
45
45
  /** Stores an item, keeping its original `createdAt` when it replaces one. */
46
46
  put<V>(namespace: StoreNamespace, key: string, value: V, options?: StorePutOptions): Promise<void>;
47
+ /**
48
+ * Stores an item only when the stored one's `version` is `expected`, or, with `expected` `null`,
49
+ * only when none is stored. One statement, so two writers deciding on one version cannot both win.
50
+ */
51
+ putIfVersion<V extends {
52
+ version: number;
53
+ }>(namespace: StoreNamespace, key: string, value: V, expected: number | null): Promise<boolean>;
47
54
  /** Reads an item, or `undefined` when it does not exist or has expired. */
48
55
  get<V>(namespace: StoreNamespace, key: string): Promise<StoreItem<V> | undefined>;
49
56
  /** Deletes an item. */
@@ -109,6 +109,27 @@ class PostgresStore {
109
109
  vector ? JSON.stringify(vector) : null,
110
110
  ]);
111
111
  }
112
+ async putIfVersion(namespace, key, value, expected) {
113
+ if (!key.trim())
114
+ throw new RangeError('A store key must not be empty');
115
+ const timestamp = this.now().toISOString();
116
+ const vector = await this.embed(value, undefined);
117
+ const embedding = vector ? JSON.stringify(vector) : null;
118
+ const cast = this.options.vectorDimensions ? '::vector' : '::jsonb';
119
+ const { rows } = expected === null
120
+ ?
121
+ await this.client.query(`INSERT INTO ${this.table} (ns_key, key, namespace, value, created_at, updated_at, expires_at, embedding)
122
+ VALUES ($1, $2, $3::jsonb, $4::jsonb, $5, $5, NULL, $6${cast})
123
+ ON CONFLICT (ns_key, key) DO UPDATE SET value = EXCLUDED.value, created_at = EXCLUDED.created_at,
124
+ updated_at = EXCLUDED.updated_at, expires_at = NULL, embedding = EXCLUDED.embedding
125
+ WHERE ${this.table}.expires_at IS NOT NULL AND ${this.table}.expires_at <= $5
126
+ RETURNING key`, [namespaceKey(namespace), key, JSON.stringify([...namespace]), JSON.stringify(value), timestamp, embedding])
127
+ : await this.client.query(`UPDATE ${this.table} SET value = $3::jsonb, updated_at = $4, expires_at = NULL, embedding = $5${cast}
128
+ WHERE ns_key = $1 AND key = $2 AND (expires_at IS NULL OR expires_at > $4)
129
+ AND jsonb_typeof(value -> 'version') = 'number' AND (value ->> 'version')::numeric = $6
130
+ RETURNING key`, [namespaceKey(namespace), key, JSON.stringify(value), timestamp, embedding, expected]);
131
+ return rows.length > 0;
132
+ }
112
133
  async get(namespace, key) {
113
134
  const { rows } = await this.client.query(`SELECT namespace::text AS namespace, key, value::text AS value, created_at, updated_at, expires_at FROM ${this.table}
114
135
  WHERE ns_key = $1 AND key = $2 AND (expires_at IS NULL OR expires_at > $3)`, [namespaceKey(namespace), key, this.now().toISOString()]);
@@ -0,0 +1,43 @@
1
+ import type { TenantUsageStore } from '../server/tenancy.js';
2
+ import type { MigrationResult, SchemaMigration } from '../utils/schema-migrations.js';
3
+ import { type PostgresLikeClient } from './client.js';
4
+ import type { PostgresMigrateOptions } from './migrations.js';
5
+ /** Options for `PostgresTenantUsage`. */
6
+ export interface PostgresTenantUsageOptions {
7
+ /** Table name, optionally schema-qualified. Defaults to `nexus_tenant_usage`. */
8
+ table?: string;
9
+ /** Clock for slot leases and spending periods, in epoch milliseconds. Defaults to `Date.now`. */
10
+ now?: () => number;
11
+ }
12
+ /** The schema, as statements: every migration's, in order. */
13
+ export declare function tenantUsageMigration(options?: Pick<PostgresTenantUsageOptions, 'table'>): string[];
14
+ /** The versioned schema, which `migrate()` and `nexus db migrate` apply. */
15
+ export declare function tenantUsageMigrations(options?: Pick<PostgresTenantUsageOptions, 'table'>): SchemaMigration[];
16
+ /**
17
+ * Tenant usage in Postgres, shared by every replica, for a deployment that already runs Postgres
18
+ * and not Redis. Pass it to `tenantLimits({ usage })`.
19
+ *
20
+ * Each key is one row: a tenant's active slots as a JSON map from run to lease expiry, or its spending
21
+ * in the current period. Every step is a single statement on that row, and Postgres applies
22
+ * concurrent updates of one row one at a time, re-checking each against the newest version. Two
23
+ * replicas admitting runs for one tenant at the same moment therefore cannot both take its last slot.
24
+ */
25
+ export declare class PostgresTenantUsage implements TenantUsageStore {
26
+ private readonly client;
27
+ private readonly options;
28
+ private readonly table;
29
+ private readonly now;
30
+ constructor(client: PostgresLikeClient, options?: PostgresTenantUsageOptions);
31
+ /** Creates the table, or brings it up to date. */
32
+ migrate(options?: PostgresMigrateOptions): Promise<MigrationResult>;
33
+ /** Takes a slot for `member` unless `max` live slots are held. Holding one already counts as taken. */
34
+ acquire(key: string, member: string, max: number, ttlMs: number): Promise<boolean>;
35
+ /** Frees a slot. Freeing one that is not held does nothing. */
36
+ release(key: string, member: string): Promise<void>;
37
+ /** Live slots held under a key. */
38
+ count(key: string): Promise<number>;
39
+ /** Adds to a total that resets at `resetAt`, and resolves to the new total. */
40
+ add(key: string, amount: number, resetAt: number): Promise<number>;
41
+ /** A total, or 0 once its period has reset. */
42
+ total(key: string): Promise<number>;
43
+ }