nexus-ai-pro 2.4.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/API_STABILITY.md +121 -0
- package/CHANGELOG.md +248 -1
- package/README.md +14 -3
- package/SECURITY.md +78 -9
- package/dist/agent/create-agent.d.ts +37 -2
- package/dist/agent/create-agent.js +62 -4
- package/dist/agent/index.d.ts +2 -1
- package/dist/agent/tool.d.ts +19 -4
- package/dist/agent/tool.js +27 -4
- package/dist/context-hub/hub.d.ts +53 -0
- package/dist/context-hub/hub.js +108 -3
- package/dist/context-hub/index.d.ts +2 -1
- package/dist/context-hub/index.js +2 -1
- package/dist/context-hub/signing.d.ts +95 -0
- package/dist/context-hub/signing.js +88 -0
- package/dist/doctor/index.d.ts +4 -2
- package/dist/doctor/index.js +29 -0
- package/dist/evaluate/threads.d.ts +187 -0
- package/dist/evaluate/threads.js +187 -0
- package/dist/graph/event-stream.js +2 -0
- package/dist/graph/graph.js +4 -1
- package/dist/mcp/client.d.ts +9 -1
- package/dist/mcp/client.js +38 -8
- package/dist/mcp/server.js +42 -2
- package/dist/operations/adapters.d.ts +2 -1
- package/dist/operations/runner.js +2 -0
- package/dist/operations/stats.js +11 -4
- package/dist/postgres/index.d.ts +2 -1
- package/dist/postgres/index.js +4 -0
- package/dist/postgres/operations.js +11 -4
- package/dist/postgres/store.d.ts +7 -0
- package/dist/postgres/store.js +21 -0
- package/dist/postgres/tenancy.d.ts +43 -0
- package/dist/postgres/tenancy.js +71 -0
- package/dist/protocols/a2a.d.ts +166 -1
- package/dist/protocols/a2a.js +499 -132
- package/dist/protocols/acp-editor.d.ts +43 -0
- package/dist/protocols/acp-editor.js +164 -0
- package/dist/protocols/acp.d.ts +76 -3
- package/dist/protocols/acp.js +180 -7
- package/dist/protocols/ag-ui.d.ts +23 -0
- package/dist/protocols/ag-ui.js +128 -8
- package/dist/server/deployments.d.ts +127 -12
- package/dist/server/deployments.js +402 -37
- package/dist/server/metrics.d.ts +2 -0
- package/dist/server/metrics.js +7 -0
- package/dist/server/runs.d.ts +9 -1
- package/dist/server/runs.js +46 -7
- package/dist/server/server.js +4 -1
- package/dist/server/state.d.ts +12 -2
- package/dist/server/state.js +15 -0
- package/dist/sqlite/operations.js +15 -4
- package/dist/sqlite/store.d.ts +7 -0
- package/dist/sqlite/store.js +28 -0
- package/dist/store/memory.d.ts +8 -0
- package/dist/store/memory.js +12 -1
- package/dist/store/redis.d.ts +13 -0
- package/dist/store/redis.js +38 -0
- package/dist/tracing/instrument.js +15 -3
- package/dist/tracing/tracer.d.ts +4 -0
- package/dist/tracing/tracer.js +6 -0
- package/dist/types/graph.d.ts +7 -0
- package/dist/types/messages.d.ts +12 -0
- package/dist/types/operations.d.ts +7 -0
- package/dist/types/server.d.ts +61 -2
- package/dist/types/store.d.ts +9 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist-cjs/agent/create-agent.d.ts +37 -2
- package/dist-cjs/agent/create-agent.js +62 -4
- package/dist-cjs/agent/index.d.ts +2 -1
- package/dist-cjs/agent/tool.d.ts +19 -4
- package/dist-cjs/agent/tool.js +27 -4
- package/dist-cjs/context-hub/hub.d.ts +53 -0
- package/dist-cjs/context-hub/hub.js +110 -4
- package/dist-cjs/context-hub/index.d.ts +2 -1
- package/dist-cjs/context-hub/index.js +7 -1
- package/dist-cjs/context-hub/signing.d.ts +95 -0
- package/dist-cjs/context-hub/signing.js +96 -0
- package/dist-cjs/doctor/index.d.ts +4 -2
- package/dist-cjs/doctor/index.js +29 -0
- package/dist-cjs/evaluate/threads.d.ts +187 -0
- package/dist-cjs/evaluate/threads.js +198 -0
- package/dist-cjs/graph/event-stream.js +2 -0
- package/dist-cjs/graph/graph.js +4 -1
- package/dist-cjs/mcp/client.d.ts +9 -1
- package/dist-cjs/mcp/client.js +37 -7
- package/dist-cjs/mcp/server.js +42 -2
- package/dist-cjs/operations/adapters.d.ts +2 -1
- package/dist-cjs/operations/runner.js +2 -0
- package/dist-cjs/operations/stats.js +11 -4
- package/dist-cjs/postgres/index.d.ts +2 -1
- package/dist-cjs/postgres/index.js +8 -1
- package/dist-cjs/postgres/operations.js +11 -4
- package/dist-cjs/postgres/store.d.ts +7 -0
- package/dist-cjs/postgres/store.js +21 -0
- package/dist-cjs/postgres/tenancy.d.ts +43 -0
- package/dist-cjs/postgres/tenancy.js +110 -0
- package/dist-cjs/protocols/a2a.d.ts +166 -1
- package/dist-cjs/protocols/a2a.js +501 -132
- package/dist-cjs/protocols/acp-editor.d.ts +43 -0
- package/dist-cjs/protocols/acp-editor.js +167 -0
- package/dist-cjs/protocols/acp.d.ts +76 -3
- package/dist-cjs/protocols/acp.js +183 -8
- package/dist-cjs/protocols/ag-ui.d.ts +23 -0
- package/dist-cjs/protocols/ag-ui.js +127 -7
- package/dist-cjs/server/deployments.d.ts +127 -12
- package/dist-cjs/server/deployments.js +402 -37
- package/dist-cjs/server/metrics.d.ts +2 -0
- package/dist-cjs/server/metrics.js +8 -0
- package/dist-cjs/server/runs.d.ts +9 -1
- package/dist-cjs/server/runs.js +45 -6
- package/dist-cjs/server/server.js +4 -1
- package/dist-cjs/server/state.d.ts +12 -2
- package/dist-cjs/server/state.js +15 -0
- package/dist-cjs/sqlite/operations.js +15 -4
- package/dist-cjs/sqlite/store.d.ts +7 -0
- package/dist-cjs/sqlite/store.js +28 -0
- package/dist-cjs/store/memory.d.ts +8 -0
- package/dist-cjs/store/memory.js +12 -1
- package/dist-cjs/store/redis.d.ts +13 -0
- package/dist-cjs/store/redis.js +38 -0
- package/dist-cjs/tracing/instrument.js +15 -3
- package/dist-cjs/tracing/tracer.d.ts +4 -0
- package/dist-cjs/tracing/tracer.js +6 -0
- package/dist-cjs/types/graph.d.ts +7 -0
- package/dist-cjs/types/messages.d.ts +12 -0
- package/dist-cjs/types/operations.d.ts +7 -0
- package/dist-cjs/types/server.d.ts +61 -2
- package/dist-cjs/types/store.d.ts +9 -0
- package/dist-cjs/version.d.ts +1 -1
- package/dist-cjs/version.js +1 -1
- package/llms.txt +1 -1
- package/package.json +43 -2
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { canonicalJson } from '../prompts/version.js';
|
|
2
|
+
import { decodeBase64, encodeBase64 } from '../utils/base64.js';
|
|
3
|
+
const subtle = () => globalThis.crypto.subtle;
|
|
4
|
+
const ED25519 = { name: 'Ed25519' };
|
|
5
|
+
export async function generateContextKey(keyId) {
|
|
6
|
+
const pair = await subtle().generateKey(ED25519, true, ['sign', 'verify']);
|
|
7
|
+
const publicKey = await subtle().exportKey('jwk', pair.publicKey);
|
|
8
|
+
const privateKey = await subtle().exportKey('jwk', pair.privateKey);
|
|
9
|
+
const id = keyId ?? (await keyIdOf(publicKey));
|
|
10
|
+
return { keyId: id, signer: await ed25519Signer(id, pair.privateKey), publicKey, privateKey };
|
|
11
|
+
}
|
|
12
|
+
export async function ed25519Signer(keyId, privateKey) {
|
|
13
|
+
const key = isCryptoKey(privateKey)
|
|
14
|
+
? privateKey
|
|
15
|
+
: await subtle().importKey('jwk', privateKey, ED25519, false, ['sign']);
|
|
16
|
+
return { keyId, sign: async (data) => new Uint8Array(await subtle().sign(ED25519, key, data)) };
|
|
17
|
+
}
|
|
18
|
+
export async function ed25519Keyring(keys) {
|
|
19
|
+
const imported = new Map();
|
|
20
|
+
for (const [keyId, key] of Object.entries(keys)) {
|
|
21
|
+
imported.set(keyId, isCryptoKey(key) ? key : await subtle().importKey('jwk', key, ED25519, false, ['verify']));
|
|
22
|
+
}
|
|
23
|
+
return {
|
|
24
|
+
async verify(keyId, data, signature) {
|
|
25
|
+
const key = imported.get(keyId);
|
|
26
|
+
if (!key)
|
|
27
|
+
return false;
|
|
28
|
+
try {
|
|
29
|
+
return await subtle().verify(ED25519, key, signature, data);
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
export async function contextDigest(content) {
|
|
38
|
+
const text = canonicalJson({ format: 'nexus-context-signature', formatVersion: 1, ...content });
|
|
39
|
+
const digest = await subtle().digest('SHA-256', new TextEncoder().encode(text));
|
|
40
|
+
return `sha256:${hexOf(new Uint8Array(digest))}`;
|
|
41
|
+
}
|
|
42
|
+
export async function signDigest(signer, digest, at) {
|
|
43
|
+
const signature = await signer.sign(new TextEncoder().encode(digest));
|
|
44
|
+
return {
|
|
45
|
+
keyId: signer.keyId,
|
|
46
|
+
algorithm: 'Ed25519',
|
|
47
|
+
digest,
|
|
48
|
+
signature: toBase64Url(signature),
|
|
49
|
+
signedAt: at.toISOString(),
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
export async function verifiedBy(keyring, digest, signatures = []) {
|
|
53
|
+
const data = new TextEncoder().encode(digest);
|
|
54
|
+
for (const signature of signatures) {
|
|
55
|
+
if (signature?.algorithm !== 'Ed25519' || signature.digest !== digest)
|
|
56
|
+
continue;
|
|
57
|
+
let bytes;
|
|
58
|
+
try {
|
|
59
|
+
bytes = fromBase64Url(signature.signature);
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
if (await keyring.verify(signature.keyId, data, bytes))
|
|
65
|
+
return signature;
|
|
66
|
+
}
|
|
67
|
+
return undefined;
|
|
68
|
+
}
|
|
69
|
+
async function keyIdOf(publicKey) {
|
|
70
|
+
const digest = await subtle().digest('SHA-256', new TextEncoder().encode(String(publicKey.x)));
|
|
71
|
+
return hexOf(new Uint8Array(digest).subarray(0, 8));
|
|
72
|
+
}
|
|
73
|
+
function hexOf(bytes) {
|
|
74
|
+
let hex = '';
|
|
75
|
+
for (const byte of bytes)
|
|
76
|
+
hex += byte.toString(16).padStart(2, '0');
|
|
77
|
+
return hex;
|
|
78
|
+
}
|
|
79
|
+
function isCryptoKey(value) {
|
|
80
|
+
return typeof value.algorithm === 'object' && Array.isArray(value.usages);
|
|
81
|
+
}
|
|
82
|
+
function toBase64Url(bytes) {
|
|
83
|
+
return encodeBase64(bytes).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
|
|
84
|
+
}
|
|
85
|
+
function fromBase64Url(text) {
|
|
86
|
+
const padded = text.replace(/-/g, '+').replace(/_/g, '/');
|
|
87
|
+
return decodeBase64(padded + '='.repeat((4 - (padded.length % 4)) % 4));
|
|
88
|
+
}
|
package/dist/doctor/index.d.ts
CHANGED
|
@@ -42,8 +42,10 @@ export interface DoctorDeployment {
|
|
|
42
42
|
replicas: number;
|
|
43
43
|
/**
|
|
44
44
|
* The stores and shared state it uses, by any name: a checkpointer, the operation store, the
|
|
45
|
-
* server's state, tenant usage, a rate-limit store, a circuit store, a cache. Each is
|
|
46
|
-
* it keeps its state in process memory, where each replica would see only its own
|
|
45
|
+
* server's state, tenant usage, a rate-limit store, a circuit store, a cache, a tracer. Each is
|
|
46
|
+
* reported when it keeps its state in process memory, where each replica would see only its own,
|
|
47
|
+
* and when it lacks an opt-in setting a deployment needs: a Redis operation store without its
|
|
48
|
+
* dispatch index, or a tracer writing to a persistent store without incremental tracing.
|
|
47
49
|
*/
|
|
48
50
|
stores?: Record<string, unknown>;
|
|
49
51
|
}
|
package/dist/doctor/index.js
CHANGED
|
@@ -54,6 +54,7 @@ export async function diagnose(options = {}) {
|
|
|
54
54
|
await run('queue', 'Operation queue and leases', () => queueCheck(options.operations, now(), options.maxQueueAgeMs));
|
|
55
55
|
await run('health', 'Provider health', () => healthCheck(options.ai));
|
|
56
56
|
await run('topology', 'Settings that only work in one process', () => topologyCheck(options.deployment));
|
|
57
|
+
await run('settings', 'Opt-in settings a deployment needs', () => settingsCheck(options.deployment));
|
|
57
58
|
const failures = checks.filter((check) => check.status === 'fail').length;
|
|
58
59
|
const warnings = checks.filter((check) => check.status === 'warn').length;
|
|
59
60
|
return { checks, ok: failures === 0, failures, warnings };
|
|
@@ -235,3 +236,31 @@ function topologyCheck(deployment) {
|
|
|
235
236
|
hint: 'Use the Redis, Postgres, or SQLite adapter for each, or run one replica.',
|
|
236
237
|
};
|
|
237
238
|
}
|
|
239
|
+
function settingsCheck(deployment) {
|
|
240
|
+
const stores = Object.entries(deployment?.stores ?? {});
|
|
241
|
+
if (stores.length === 0)
|
|
242
|
+
return { status: 'skip', detail: 'No stores described.' };
|
|
243
|
+
const findings = [];
|
|
244
|
+
const hints = new Set();
|
|
245
|
+
for (const [name, value] of stores) {
|
|
246
|
+
const kind = kindOf(value);
|
|
247
|
+
if (kind === 'RedisOperationStore' && value.indexed === false) {
|
|
248
|
+
findings.push(`${name} is a RedisOperationStore without index: true, so finding queued work reads every record and a long queue slows every claim`);
|
|
249
|
+
hints.add('Give the Redis operation store index: true once every worker sharing its prefix runs 2.2 or later, then call reindex() once.');
|
|
250
|
+
}
|
|
251
|
+
if (kind === 'Tracer') {
|
|
252
|
+
const tracer = value;
|
|
253
|
+
const store = kindOf(tracer.store);
|
|
254
|
+
if (tracer.incremental === false && store && store !== 'MemoryTraceStore') {
|
|
255
|
+
findings.push(`${name} writes to a ${store} without incremental, so a trace is written when its root finishes and a replica that dies mid-run loses it`);
|
|
256
|
+
hints.add('Create the Tracer with incremental: true, and close what a dead process left with closeAbandonedRuns().');
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
if (findings.length === 0)
|
|
261
|
+
return { status: 'ok', detail: 'No store lacks a setting a deployment needs.' };
|
|
262
|
+
return { status: 'warn', detail: `${findings.join('; ')}.`, hint: [...hints].join(' ') };
|
|
263
|
+
}
|
|
264
|
+
function kindOf(value) {
|
|
265
|
+
return value?.constructor?.name;
|
|
266
|
+
}
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Evaluation of whole conversations. A simulated user holds a conversation with an agent for each
|
|
3
|
+
* goal in a dataset, and thread evaluators score the conversation: whether the goal was met, each
|
|
4
|
+
* turn, and the turn where it went wrong. A multi-turn agent is then compared between versions as a
|
|
5
|
+
* single-turn one is, with `evaluate()` and `compareExperiments()`.
|
|
6
|
+
*/
|
|
7
|
+
import type { LLMJudge } from '../evals/judge.js';
|
|
8
|
+
import type { Evaluator } from '../types/evaluate.js';
|
|
9
|
+
import type { CompletionRequest, Message } from '../types/messages.js';
|
|
10
|
+
import type { EvaluationTarget } from './run.js';
|
|
11
|
+
/** One turn of a conversation. */
|
|
12
|
+
export interface ThreadTurn {
|
|
13
|
+
/** Who spoke. */
|
|
14
|
+
role: 'user' | 'assistant';
|
|
15
|
+
/** What they said. */
|
|
16
|
+
content: string;
|
|
17
|
+
/** For an assistant turn, how long the agent took, in milliseconds. */
|
|
18
|
+
latencyMs?: number;
|
|
19
|
+
/**
|
|
20
|
+
* For an assistant turn, every message the agent added: its tool calls, their results, and the
|
|
21
|
+
* answer. The agent is given them back on later turns, and an evaluator can read its trajectory.
|
|
22
|
+
*/
|
|
23
|
+
messages?: Message[];
|
|
24
|
+
}
|
|
25
|
+
/** Why a conversation ended. */
|
|
26
|
+
export type ThreadEnding = 'goal' | 'gave-up' | 'max-turns' | 'error';
|
|
27
|
+
/** A finished conversation: what a conversation target returns, and what thread evaluators read. */
|
|
28
|
+
export interface Thread {
|
|
29
|
+
/** Every turn, in order. */
|
|
30
|
+
turns: ThreadTurn[];
|
|
31
|
+
/**
|
|
32
|
+
* `goal` when the user said its goal was met, `gave-up` when it said it never would be, `max-turns`
|
|
33
|
+
* when the turn limit ended it, and `error` when the agent failed.
|
|
34
|
+
*/
|
|
35
|
+
ended: ThreadEnding;
|
|
36
|
+
/** The agent's error, when it failed. */
|
|
37
|
+
error?: string;
|
|
38
|
+
}
|
|
39
|
+
/** What a conversation is about: the user's goal, and optionally who they are and how they open. */
|
|
40
|
+
export interface ThreadInputs {
|
|
41
|
+
/** What the user wants, in their words or the dataset's. */
|
|
42
|
+
goal: string;
|
|
43
|
+
/** Who the user is and how they talk, such as "an impatient customer who writes in short lines". */
|
|
44
|
+
persona?: string;
|
|
45
|
+
/** The user's first message, instead of letting the simulated user write it. */
|
|
46
|
+
opening?: string;
|
|
47
|
+
}
|
|
48
|
+
/** What a simulated user sees when it writes its next message. */
|
|
49
|
+
export interface SimulatedUserContext {
|
|
50
|
+
/** The goal. */
|
|
51
|
+
goal: string;
|
|
52
|
+
/** The persona, when the example gives one. */
|
|
53
|
+
persona?: string;
|
|
54
|
+
/** The conversation so far. Empty for the first message. */
|
|
55
|
+
turns: readonly ThreadTurn[];
|
|
56
|
+
/** Aborted when the evaluation is cancelled or times out. */
|
|
57
|
+
signal?: AbortSignal;
|
|
58
|
+
}
|
|
59
|
+
/** A simulated user's move: its next message, or the end of the conversation and why. */
|
|
60
|
+
export type SimulatedUserReply = {
|
|
61
|
+
message: string;
|
|
62
|
+
} | {
|
|
63
|
+
done: 'goal' | 'gave-up';
|
|
64
|
+
};
|
|
65
|
+
/** Plays the user in a conversation. */
|
|
66
|
+
export interface SimulatedUser {
|
|
67
|
+
/** The next move, given the conversation so far. */
|
|
68
|
+
next(context: SimulatedUserContext): Promise<SimulatedUserReply> | SimulatedUserReply;
|
|
69
|
+
}
|
|
70
|
+
/** A model client, as the judges take one. */
|
|
71
|
+
export interface SimulatedUserClient {
|
|
72
|
+
/** Runs one completion. */
|
|
73
|
+
complete(request: CompletionRequest): Promise<{
|
|
74
|
+
content: string;
|
|
75
|
+
}>;
|
|
76
|
+
}
|
|
77
|
+
/** Options for `simulatedUser()`. */
|
|
78
|
+
export interface SimulatedUserOptions {
|
|
79
|
+
/** The client the user's model is called through. */
|
|
80
|
+
client: SimulatedUserClient;
|
|
81
|
+
/** The model that plays the user. */
|
|
82
|
+
model: string;
|
|
83
|
+
/** A persona for every conversation, when the examples give none. */
|
|
84
|
+
persona?: string;
|
|
85
|
+
/** Sampling temperature. Defaults to 0, so a conversation repeats. */
|
|
86
|
+
temperature?: number;
|
|
87
|
+
/** Output token limit for each message. Defaults to 300. */
|
|
88
|
+
maxTokens?: number;
|
|
89
|
+
/** Replaces the instructions the model is given. `{goal}` and `{persona}` are filled in. */
|
|
90
|
+
instructions?: string;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* A simulated user driven by a model: it is told the goal and the persona, sees the conversation
|
|
94
|
+
* with the roles turned around, and writes the next message, or says it is done and why.
|
|
95
|
+
*/
|
|
96
|
+
export declare function simulatedUser(options: SimulatedUserOptions): SimulatedUser;
|
|
97
|
+
/**
|
|
98
|
+
* A user that follows a script, for deterministic tests: its messages in order, then `done`. A
|
|
99
|
+
* function decides each move itself.
|
|
100
|
+
*/
|
|
101
|
+
export declare function scriptedUser(script: {
|
|
102
|
+
messages: readonly string[];
|
|
103
|
+
done?: 'goal' | 'gave-up';
|
|
104
|
+
} | SimulatedUser['next']): SimulatedUser;
|
|
105
|
+
/** What a thread agent is told besides the conversation. */
|
|
106
|
+
export interface ThreadAgentContext<I extends ThreadInputs = ThreadInputs> {
|
|
107
|
+
/** The example's inputs. */
|
|
108
|
+
inputs: I;
|
|
109
|
+
/** The turn, counted from 1. */
|
|
110
|
+
turn: number;
|
|
111
|
+
/** Names this conversation, the same on every turn and new for every conversation: a thread id. */
|
|
112
|
+
conversationId: string;
|
|
113
|
+
/** Aborted when the evaluation is cancelled or times out. */
|
|
114
|
+
signal?: AbortSignal;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* What a thread agent answers: the reply, or the reply and every message the agent added on the way,
|
|
118
|
+
* such as its tool calls and their results. Those are given back to it on later turns.
|
|
119
|
+
*/
|
|
120
|
+
export type ThreadAgentReply = string | {
|
|
121
|
+
content: string;
|
|
122
|
+
messages?: Message[];
|
|
123
|
+
};
|
|
124
|
+
/**
|
|
125
|
+
* The agent a conversation target talks to. It is given the conversation so far, with its own earlier
|
|
126
|
+
* turns as it recorded them, and answers the user's newest message.
|
|
127
|
+
*/
|
|
128
|
+
export type ThreadAgent<I extends ThreadInputs = ThreadInputs> = (messages: readonly Message[], context: ThreadAgentContext<I>) => Promise<ThreadAgentReply> | ThreadAgentReply;
|
|
129
|
+
/** Options for `conversationTarget()`. */
|
|
130
|
+
export interface ConversationTargetOptions<I extends ThreadInputs = ThreadInputs> {
|
|
131
|
+
/** The user, or a user per example, such as one with the example's own persona. */
|
|
132
|
+
user: SimulatedUser | ((inputs: I) => SimulatedUser);
|
|
133
|
+
/** The most user messages a conversation may have. Defaults to 10. */
|
|
134
|
+
maxTurns?: number;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* An `evaluate()` target that holds a conversation per example: the simulated user speaks, the agent
|
|
138
|
+
* answers, until the user says it is done or the turn limit is reached. It returns the `Thread`.
|
|
139
|
+
*/
|
|
140
|
+
export declare function conversationTarget<I extends ThreadInputs = ThreadInputs>(agent: ThreadAgent<I>, options: ConversationTargetOptions<I>): EvaluationTarget<I>;
|
|
141
|
+
/** A graph a thread agent can drive: one that takes `{ messages }`, as `createAgent()` builds. */
|
|
142
|
+
export interface ThreadGraph {
|
|
143
|
+
/** Runs one turn. */
|
|
144
|
+
invoke(input: {
|
|
145
|
+
messages: Message[];
|
|
146
|
+
}, options?: {
|
|
147
|
+
threadId?: string;
|
|
148
|
+
signal?: AbortSignal;
|
|
149
|
+
}): Promise<{
|
|
150
|
+
state: unknown;
|
|
151
|
+
}>;
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* An agent from `createAgent()`, or any graph that takes `{ messages }` and ends with an `answer`, as
|
|
155
|
+
* a thread agent. Each turn runs on the conversation's thread with the transcript so far, and the
|
|
156
|
+
* messages the run added to its `messages` are recorded with the turn. The agent therefore sees its
|
|
157
|
+
* own tool calls on later turns, as it does with a real user, without needing a checkpointer.
|
|
158
|
+
*/
|
|
159
|
+
export declare function graphThreadAgent(graph: ThreadGraph): ThreadAgent;
|
|
160
|
+
/** The conversation as text, for a judge or a person. */
|
|
161
|
+
export declare function formatThread(thread: Thread): string;
|
|
162
|
+
/**
|
|
163
|
+
* Whether the user said its goal was met: 1 when the conversation ended on `goal`, otherwise 0, with
|
|
164
|
+
* how it ended as the comment. Key `goal_reached`.
|
|
165
|
+
*/
|
|
166
|
+
export declare function goalReached(): Evaluator<ThreadInputs>;
|
|
167
|
+
/**
|
|
168
|
+
* Whether the goal was met, in a judge's opinion of the whole conversation, not the user's. Key
|
|
169
|
+
* `goal_completion`, with the judge's rationale as the comment.
|
|
170
|
+
*/
|
|
171
|
+
export declare function goalCompletion(judge: Pick<LLMJudge, 'evaluate'>): Evaluator<ThreadInputs>;
|
|
172
|
+
/** How many times the assistant spoke: fewer is better. Key `turns`; compare with `lowerIsBetter: ['turns']`. */
|
|
173
|
+
export declare function turnCount(): Evaluator<ThreadInputs>;
|
|
174
|
+
/** Options for `turnScores()`. */
|
|
175
|
+
export interface TurnScoresOptions {
|
|
176
|
+
/** The score's key. Defaults to `turn_score`. */
|
|
177
|
+
key?: string;
|
|
178
|
+
/** A turn scoring below this went wrong. Defaults to 0.5. */
|
|
179
|
+
threshold?: number;
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Scores every assistant turn with an evaluator of single answers, and the conversation by their
|
|
183
|
+
* mean. The evaluator sees each turn as an example of its own: the user's message as the input, the
|
|
184
|
+
* conversation before it in `metadata.history`, and the turn as the output. The score's metadata holds
|
|
185
|
+
* each turn's score, and `wentWrongAt`, the first turn below the threshold, counted from 1.
|
|
186
|
+
*/
|
|
187
|
+
export declare function turnScores<I = unknown>(evaluator: Evaluator<I>, options?: TurnScoresOptions): Evaluator<ThreadInputs>;
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
const USER_INSTRUCTIONS = `You are playing a user who is talking to an assistant. Stay in character.
|
|
2
|
+
Your goal: {goal}
|
|
3
|
+
{persona}
|
|
4
|
+
Write only your next message to the assistant, as the user would. Give details the assistant asks for when your goal needs them.
|
|
5
|
+
When the assistant has fully achieved your goal, reply with exactly: DONE: GOAL
|
|
6
|
+
When you are sure the assistant will never achieve it, reply with exactly: DONE: GAVE UP`;
|
|
7
|
+
const DONE = /^\s*DONE:\s*(GOAL|GAVE[\s-]*UP)\b/i;
|
|
8
|
+
export function simulatedUser(options) {
|
|
9
|
+
return {
|
|
10
|
+
async next(context) {
|
|
11
|
+
const instructions = (options.instructions ?? USER_INSTRUCTIONS)
|
|
12
|
+
.replace('{goal}', context.goal)
|
|
13
|
+
.replace('{persona}', context.persona ?? options.persona ?? '');
|
|
14
|
+
const messages = [
|
|
15
|
+
{ role: 'system', content: instructions },
|
|
16
|
+
...context.turns.map((turn) => ({
|
|
17
|
+
role: turn.role === 'assistant' ? 'user' : 'assistant',
|
|
18
|
+
content: turn.content,
|
|
19
|
+
})),
|
|
20
|
+
];
|
|
21
|
+
if (context.turns.length === 0)
|
|
22
|
+
messages.push({ role: 'user', content: 'Begin the conversation.' });
|
|
23
|
+
const response = await options.client.complete({
|
|
24
|
+
model: options.model,
|
|
25
|
+
messages,
|
|
26
|
+
temperature: options.temperature ?? 0,
|
|
27
|
+
maxTokens: options.maxTokens ?? 300,
|
|
28
|
+
...(context.signal ? { signal: context.signal } : {}),
|
|
29
|
+
});
|
|
30
|
+
const done = DONE.exec(response.content ?? '');
|
|
31
|
+
if (done)
|
|
32
|
+
return { done: /^GOAL$/i.test(done[1]) ? 'goal' : 'gave-up' };
|
|
33
|
+
return { message: (response.content ?? '').trim() };
|
|
34
|
+
},
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
export function scriptedUser(script) {
|
|
38
|
+
if (typeof script === 'function')
|
|
39
|
+
return { next: script };
|
|
40
|
+
return {
|
|
41
|
+
next: ({ turns }) => {
|
|
42
|
+
const said = turns.filter((turn) => turn.role === 'user').length;
|
|
43
|
+
const message = script.messages[said];
|
|
44
|
+
return message === undefined ? { done: script.done ?? 'goal' } : { message };
|
|
45
|
+
},
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
export function conversationTarget(agent, options) {
|
|
49
|
+
const maxTurns = options.maxTurns ?? 10;
|
|
50
|
+
return async (inputs, context) => {
|
|
51
|
+
const user = typeof options.user === 'function' ? options.user(inputs) : options.user;
|
|
52
|
+
const turns = [];
|
|
53
|
+
const conversationId = `${context.example.id}:${context.run}:${globalThis.crypto.randomUUID()}`;
|
|
54
|
+
const ask = async () => user.next({
|
|
55
|
+
goal: inputs.goal,
|
|
56
|
+
...(inputs.persona === undefined ? {} : { persona: inputs.persona }),
|
|
57
|
+
turns,
|
|
58
|
+
...(context.signal ? { signal: context.signal } : {}),
|
|
59
|
+
});
|
|
60
|
+
let move = inputs.opening === undefined ? await ask() : { message: inputs.opening };
|
|
61
|
+
for (let turn = 1;; turn += 1) {
|
|
62
|
+
if ('done' in move)
|
|
63
|
+
return { turns, ended: move.done };
|
|
64
|
+
if (turn > maxTurns)
|
|
65
|
+
return { turns, ended: 'max-turns' };
|
|
66
|
+
turns.push({ role: 'user', content: move.message });
|
|
67
|
+
const started = performance.now();
|
|
68
|
+
try {
|
|
69
|
+
const reply = await agent(transcriptOf(turns), {
|
|
70
|
+
inputs,
|
|
71
|
+
turn,
|
|
72
|
+
conversationId,
|
|
73
|
+
...(context.signal ? { signal: context.signal } : {}),
|
|
74
|
+
});
|
|
75
|
+
const latencyMs = Math.round(performance.now() - started);
|
|
76
|
+
turns.push(typeof reply === 'string'
|
|
77
|
+
? { role: 'assistant', content: reply, latencyMs }
|
|
78
|
+
: {
|
|
79
|
+
role: 'assistant',
|
|
80
|
+
content: reply.content,
|
|
81
|
+
latencyMs,
|
|
82
|
+
...(reply.messages ? { messages: reply.messages } : {}),
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
catch (error) {
|
|
86
|
+
return {
|
|
87
|
+
turns,
|
|
88
|
+
ended: 'error',
|
|
89
|
+
error: error instanceof Error ? error.message : String(error),
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
move = await ask();
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
function transcriptOf(turns) {
|
|
97
|
+
return turns.flatMap((turn) => turn.role === 'assistant' && turn.messages?.length ? turn.messages : [{ role: turn.role, content: turn.content }]);
|
|
98
|
+
}
|
|
99
|
+
export function graphThreadAgent(graph) {
|
|
100
|
+
return async (messages, context) => {
|
|
101
|
+
const result = await graph.invoke({ messages: [...messages] }, { threadId: context.conversationId, ...(context.signal ? { signal: context.signal } : {}) });
|
|
102
|
+
const state = result.state;
|
|
103
|
+
const content = typeof state?.answer === 'string' ? state.answer : JSON.stringify(result.state);
|
|
104
|
+
const after = Array.isArray(state?.messages) ? state.messages : undefined;
|
|
105
|
+
return after && after.length > messages.length ? { content, messages: after.slice(messages.length) } : content;
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
function threadOf(context) {
|
|
109
|
+
const output = context.output;
|
|
110
|
+
return output && Array.isArray(output.turns) ? output : undefined;
|
|
111
|
+
}
|
|
112
|
+
export function formatThread(thread) {
|
|
113
|
+
return thread.turns.map((turn) => `${turn.role === 'user' ? 'User' : 'Assistant'}: ${turn.content}`).join('\n\n');
|
|
114
|
+
}
|
|
115
|
+
export function goalReached() {
|
|
116
|
+
return (context) => {
|
|
117
|
+
const thread = threadOf(context);
|
|
118
|
+
const reached = thread?.ended === 'goal';
|
|
119
|
+
return {
|
|
120
|
+
key: 'goal_reached',
|
|
121
|
+
score: reached ? 1 : 0,
|
|
122
|
+
passed: reached,
|
|
123
|
+
comment: thread ? `ended: ${thread.ended}${thread.error ? ` (${thread.error})` : ''}` : 'no conversation',
|
|
124
|
+
};
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
export function goalCompletion(judge) {
|
|
128
|
+
return async (context) => {
|
|
129
|
+
const thread = threadOf(context);
|
|
130
|
+
if (!thread)
|
|
131
|
+
return { key: 'goal_completion', score: 0, passed: false, comment: 'no conversation' };
|
|
132
|
+
const verdict = await judge.evaluate({
|
|
133
|
+
actual: formatThread(thread),
|
|
134
|
+
query: context.example.inputs.goal,
|
|
135
|
+
rubric: "The text is a conversation between a user and an assistant. Score whether the assistant achieved the user's goal, given as the question, by the end of the conversation.",
|
|
136
|
+
});
|
|
137
|
+
return {
|
|
138
|
+
key: 'goal_completion',
|
|
139
|
+
score: verdict.score,
|
|
140
|
+
passed: verdict.passed,
|
|
141
|
+
...(verdict.rationale ? { comment: verdict.rationale } : {}),
|
|
142
|
+
};
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
export function turnCount() {
|
|
146
|
+
return (context) => ({
|
|
147
|
+
key: 'turns',
|
|
148
|
+
score: threadOf(context)?.turns.filter((turn) => turn.role === 'assistant').length ?? 0,
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
export function turnScores(evaluator, options = {}) {
|
|
152
|
+
const key = options.key ?? 'turn_score';
|
|
153
|
+
const threshold = options.threshold ?? 0.5;
|
|
154
|
+
return async (context) => {
|
|
155
|
+
const thread = threadOf(context);
|
|
156
|
+
const scores = [];
|
|
157
|
+
for (const [index, turn] of (thread?.turns ?? []).entries()) {
|
|
158
|
+
if (turn.role !== 'assistant')
|
|
159
|
+
continue;
|
|
160
|
+
const history = thread.turns.slice(0, index);
|
|
161
|
+
const question = [...history].reverse().find((item) => item.role === 'user')?.content ?? '';
|
|
162
|
+
const example = {
|
|
163
|
+
...context.example,
|
|
164
|
+
id: `${context.example.id}#${scores.length + 1}`,
|
|
165
|
+
inputs: question,
|
|
166
|
+
metadata: { ...context.example.metadata, history, goal: context.example.inputs.goal },
|
|
167
|
+
};
|
|
168
|
+
scores.push(numeric(await evaluator({ ...context, example, output: turn.content })));
|
|
169
|
+
}
|
|
170
|
+
const mean = scores.length ? scores.reduce((sum, score) => sum + score, 0) / scores.length : 0;
|
|
171
|
+
const wrong = scores.findIndex((score) => score < threshold);
|
|
172
|
+
return {
|
|
173
|
+
key,
|
|
174
|
+
score: mean,
|
|
175
|
+
metadata: { turns: scores, ...(wrong === -1 ? {} : { wentWrongAt: wrong + 1 }) },
|
|
176
|
+
...(wrong === -1 ? {} : { comment: `turn ${wrong + 1} scored ${scores[wrong].toFixed(2)}` }),
|
|
177
|
+
};
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
function numeric(result) {
|
|
181
|
+
if (typeof result === 'number')
|
|
182
|
+
return result;
|
|
183
|
+
if (typeof result === 'boolean')
|
|
184
|
+
return result ? 1 : 0;
|
|
185
|
+
const first = Array.isArray(result) ? result[0] : result;
|
|
186
|
+
return first?.score ?? 0;
|
|
187
|
+
}
|
|
@@ -265,6 +265,8 @@ function fromGraphEvent(event) {
|
|
|
265
265
|
return [
|
|
266
266
|
{ type: 'custom', step: event.step, node: event.node, taskId: event.taskId, data: event.data, namespace },
|
|
267
267
|
];
|
|
268
|
+
case 'step':
|
|
269
|
+
return [];
|
|
268
270
|
case 'checkpoint':
|
|
269
271
|
return [{ type: 'checkpoints', step: event.step, status: event.status, next: event.next, namespace }];
|
|
270
272
|
case 'task_end':
|
package/dist/graph/graph.js
CHANGED
|
@@ -477,7 +477,10 @@ export class CompiledGraph {
|
|
|
477
477
|
this.writers.add(writer);
|
|
478
478
|
let failed = false;
|
|
479
479
|
try {
|
|
480
|
-
|
|
480
|
+
for await (const event of this.steps(threadId, runOptions, seed, (written, durable) => writer.persist(written, durable))) {
|
|
481
|
+
notify(runOptions, { type: 'step', step: event.step, status: event.status, state: event.state });
|
|
482
|
+
yield event;
|
|
483
|
+
}
|
|
481
484
|
}
|
|
482
485
|
catch (error) {
|
|
483
486
|
failed = true;
|
package/dist/mcp/client.d.ts
CHANGED
|
@@ -61,6 +61,8 @@ export declare class McpClient {
|
|
|
61
61
|
private readonly options;
|
|
62
62
|
private readonly pending;
|
|
63
63
|
private nextId;
|
|
64
|
+
/** Where each progress token's reports go, while its tool call runs. */
|
|
65
|
+
private readonly progress;
|
|
64
66
|
private initialized;
|
|
65
67
|
private connecting;
|
|
66
68
|
private serverInfo;
|
|
@@ -80,7 +82,13 @@ export declare class McpClient {
|
|
|
80
82
|
* Calls a tool. A tool that fails reports `isError` rather than throwing; a protocol failure
|
|
81
83
|
* throws `McpError`.
|
|
82
84
|
*/
|
|
83
|
-
callTool(name: string, args?: Record<string, unknown
|
|
85
|
+
callTool(name: string, args?: Record<string, unknown>, options?: {
|
|
86
|
+
progress?: (progress: {
|
|
87
|
+
progress: number;
|
|
88
|
+
total?: number;
|
|
89
|
+
message?: string;
|
|
90
|
+
}) => void;
|
|
91
|
+
}): Promise<McpToolResult>;
|
|
84
92
|
/** Lists the server's resources. */
|
|
85
93
|
listResources(): Promise<McpResourceDescriptor[]>;
|
|
86
94
|
/** Reads a resource's content. */
|
package/dist/mcp/client.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import { isRequest, isResponse, LineDecoder, MCP_PROTOCOL_VERSION, McpError, } from './protocol.js';
|
|
1
|
+
import { isNotification, isRequest, isResponse, LineDecoder, MCP_PROTOCOL_VERSION, McpError, } from './protocol.js';
|
|
2
2
|
export class McpClient {
|
|
3
3
|
transport;
|
|
4
4
|
options;
|
|
5
5
|
pending = new Map();
|
|
6
6
|
nextId = 1;
|
|
7
|
+
progress = new Map();
|
|
7
8
|
initialized = false;
|
|
8
9
|
connecting;
|
|
9
10
|
serverInfo = {};
|
|
@@ -38,10 +39,23 @@ export class McpClient {
|
|
|
38
39
|
const result = (await this.request('tools/list'));
|
|
39
40
|
return result?.tools ?? [];
|
|
40
41
|
}
|
|
41
|
-
async callTool(name, args = {}) {
|
|
42
|
+
async callTool(name, args = {}, options = {}) {
|
|
42
43
|
await this.connect();
|
|
43
|
-
const
|
|
44
|
-
|
|
44
|
+
const token = options.progress ? `progress-${this.nextId}` : undefined;
|
|
45
|
+
if (token && options.progress)
|
|
46
|
+
this.progress.set(token, options.progress);
|
|
47
|
+
try {
|
|
48
|
+
const result = (await this.request('tools/call', {
|
|
49
|
+
name,
|
|
50
|
+
arguments: args,
|
|
51
|
+
...(token ? { _meta: { progressToken: token } } : {}),
|
|
52
|
+
}));
|
|
53
|
+
return { content: result?.content ?? [], ...(result?.isError ? { isError: true } : {}) };
|
|
54
|
+
}
|
|
55
|
+
finally {
|
|
56
|
+
if (token)
|
|
57
|
+
this.progress.delete(token);
|
|
58
|
+
}
|
|
45
59
|
}
|
|
46
60
|
async listResources() {
|
|
47
61
|
await this.connect();
|
|
@@ -69,8 +83,9 @@ export class McpClient {
|
|
|
69
83
|
name: options.prefix ? `${options.prefix}_${descriptor.name}` : descriptor.name,
|
|
70
84
|
description: descriptor.description ?? `MCP tool ${descriptor.name}`,
|
|
71
85
|
parameters: descriptor.inputSchema ?? { type: 'object', properties: {} },
|
|
72
|
-
execute: async (args) => {
|
|
73
|
-
const
|
|
86
|
+
execute: async (args, context) => {
|
|
87
|
+
const progress = context?.progress;
|
|
88
|
+
const result = await this.callTool(descriptor.name, args, progress ? { progress } : {});
|
|
74
89
|
const text = result.content
|
|
75
90
|
.map((part) => part.text ?? (part.type === 'text' ? '' : JSON.stringify(part)))
|
|
76
91
|
.filter(Boolean)
|
|
@@ -107,6 +122,16 @@ export class McpClient {
|
|
|
107
122
|
this.settle(message.id, message.result, message.error ? new McpError(message.error.message, message.error.code, message.error.data) : undefined);
|
|
108
123
|
return;
|
|
109
124
|
}
|
|
125
|
+
if (isNotification(message) && message.method === 'notifications/progress') {
|
|
126
|
+
const params = (message.params ?? {});
|
|
127
|
+
const report = this.progress.get(String(params.progressToken));
|
|
128
|
+
report?.({
|
|
129
|
+
progress: typeof params.progress === 'number' ? params.progress : 0,
|
|
130
|
+
...(typeof params.total === 'number' ? { total: params.total } : {}),
|
|
131
|
+
...(typeof params.message === 'string' ? { message: params.message } : {}),
|
|
132
|
+
});
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
110
135
|
if (isRequest(message)) {
|
|
111
136
|
void this.transport.send({
|
|
112
137
|
jsonrpc: '2.0',
|
|
@@ -221,9 +246,14 @@ export function createStdioTransport(options) {
|
|
|
221
246
|
onClose(handler) {
|
|
222
247
|
onClose = handler;
|
|
223
248
|
},
|
|
224
|
-
close() {
|
|
225
|
-
child
|
|
249
|
+
async close() {
|
|
250
|
+
const running = child;
|
|
226
251
|
child = undefined;
|
|
252
|
+
if (!running || running.exitCode !== null || running.signalCode !== null)
|
|
253
|
+
return;
|
|
254
|
+
const exited = new Promise((resolve) => running.once('close', () => resolve()));
|
|
255
|
+
running.kill();
|
|
256
|
+
await Promise.race([exited, new Promise((resolve) => setTimeout(resolve, 2_000).unref?.())]);
|
|
227
257
|
},
|
|
228
258
|
};
|
|
229
259
|
}
|