nexus-ai-pro 2.18.0 → 2.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/API_STABILITY.md +64 -1
- package/CHANGELOG.md +113 -1
- package/README.md +11 -3
- package/dist/agent/capabilities.d.ts +3 -0
- package/dist/agent/openapi.d.ts +139 -0
- package/dist/agent/openapi.js +455 -0
- package/dist/capabilities/negotiate.js +5 -0
- package/dist/decisions/completion.d.ts +48 -0
- package/dist/decisions/completion.js +195 -0
- package/dist/decisions/core.d.ts +525 -0
- package/dist/decisions/core.js +493 -0
- package/dist/decisions/index.d.ts +9 -0
- package/dist/decisions/index.js +3 -0
- package/dist/decisions/integrations.d.ts +142 -0
- package/dist/decisions/integrations.js +170 -0
- package/dist/decisions/mock.d.ts +32 -0
- package/dist/decisions/mock.js +47 -0
- package/dist/decisions/typesafe.d.ts +45 -0
- package/dist/decisions/typesafe.js +144 -0
- package/dist/evaluate/emulate.d.ts +73 -6
- package/dist/evaluate/emulate.js +114 -12
- package/dist/index.d.ts +1 -1
- package/dist/optimizer/densifier.js +20 -11
- package/dist/protocols/a2a-grpc.js +1 -1
- package/dist/protocols/a2a-proto.d.ts +2 -2
- package/dist/protocols/a2a-proto.js +76 -1
- package/dist/protocols/a2a.d.ts +121 -0
- package/dist/protocols/a2a.js +88 -29
- package/dist/providers/google.js +28 -0
- package/dist/providers/ollama.js +23 -0
- package/dist/providers/openai.d.ts +6 -0
- package/dist/providers/openai.js +26 -0
- package/dist/types/lifecycle.d.ts +1 -1
- package/dist/types/messages.d.ts +9 -0
- package/dist/types/providers.d.ts +2 -0
- package/dist/types/response.d.ts +17 -0
- package/dist/types/tracing.d.ts +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/workflow/chains.d.ts +13 -1
- package/dist/workflow/chains.js +40 -0
- package/dist-cjs/agent/capabilities.d.ts +3 -0
- package/dist-cjs/agent/openapi.d.ts +139 -0
- package/dist-cjs/agent/openapi.js +461 -0
- package/dist-cjs/capabilities/negotiate.js +5 -0
- package/dist-cjs/decisions/completion.d.ts +48 -0
- package/dist-cjs/decisions/completion.js +198 -0
- package/dist-cjs/decisions/core.d.ts +525 -0
- package/dist-cjs/decisions/core.js +509 -0
- package/dist-cjs/decisions/index.d.ts +9 -0
- package/dist-cjs/decisions/index.js +27 -0
- package/dist-cjs/decisions/integrations.d.ts +142 -0
- package/dist-cjs/decisions/integrations.js +178 -0
- package/dist-cjs/decisions/mock.d.ts +32 -0
- package/dist-cjs/decisions/mock.js +51 -0
- package/dist-cjs/decisions/typesafe.d.ts +45 -0
- package/dist-cjs/decisions/typesafe.js +148 -0
- package/dist-cjs/evaluate/emulate.d.ts +73 -6
- package/dist-cjs/evaluate/emulate.js +116 -13
- package/dist-cjs/index.d.ts +1 -1
- package/dist-cjs/optimizer/densifier.js +20 -11
- package/dist-cjs/protocols/a2a-grpc.js +1 -1
- package/dist-cjs/protocols/a2a-proto.d.ts +2 -2
- package/dist-cjs/protocols/a2a-proto.js +76 -1
- package/dist-cjs/protocols/a2a.d.ts +121 -0
- package/dist-cjs/protocols/a2a.js +88 -29
- package/dist-cjs/providers/google.js +28 -0
- package/dist-cjs/providers/ollama.js +23 -0
- package/dist-cjs/providers/openai.d.ts +6 -0
- package/dist-cjs/providers/openai.js +26 -0
- package/dist-cjs/types/lifecycle.d.ts +1 -1
- package/dist-cjs/types/messages.d.ts +9 -0
- package/dist-cjs/types/providers.d.ts +2 -0
- package/dist-cjs/types/response.d.ts +17 -0
- package/dist-cjs/types/tracing.d.ts +1 -1
- package/dist-cjs/version.d.ts +1 -1
- package/dist-cjs/version.js +1 -1
- package/dist-cjs/workflow/chains.d.ts +13 -1
- package/dist-cjs/workflow/chains.js +40 -0
- package/llms.txt +2 -1
- package/package.json +54 -2
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
export function decisionRoute(decisions, options) {
|
|
2
|
+
return async (graph) => {
|
|
3
|
+
const { answers } = await decisions.decide({
|
|
4
|
+
...options.request?.(graph),
|
|
5
|
+
state: options.state(graph),
|
|
6
|
+
questions: { route: options.question },
|
|
7
|
+
});
|
|
8
|
+
const answer = answers.route;
|
|
9
|
+
const picked = answer.type === 'choice' ? answer.choice : answer.yes ? 'yes' : 'no';
|
|
10
|
+
const node = answer.confidence >= options.act ? (options.routes?.[picked] ?? picked) : options.review;
|
|
11
|
+
options.onAnswer?.(answer, node);
|
|
12
|
+
return node;
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
export function decisionPermissions(decisions, options) {
|
|
16
|
+
const question = options.question ?? {
|
|
17
|
+
type: 'yes-no',
|
|
18
|
+
question: 'Is this tool call safe to run without a person checking it first?',
|
|
19
|
+
criteria: { yes: 'It does what the tool is for, on what it names', no: 'It could cause harm, loss, or a leak' },
|
|
20
|
+
};
|
|
21
|
+
return {
|
|
22
|
+
async decide(request) {
|
|
23
|
+
const base = await options.base?.decide(request);
|
|
24
|
+
if (base && base.decision !== 'ask')
|
|
25
|
+
return base;
|
|
26
|
+
const tenantId = request.principal?.tenantId;
|
|
27
|
+
const { answers } = await decisions.decide({
|
|
28
|
+
...(tenantId !== undefined ? { tenantId } : {}),
|
|
29
|
+
...options.request?.(request),
|
|
30
|
+
state: options.state?.(request) ?? {
|
|
31
|
+
tool: request.tool,
|
|
32
|
+
arguments: request.args,
|
|
33
|
+
...(request.capabilities ? { capabilities: request.capabilities } : {}),
|
|
34
|
+
},
|
|
35
|
+
questions: { safe: question },
|
|
36
|
+
});
|
|
37
|
+
const answer = answers.safe;
|
|
38
|
+
const odds = `yes at ${answer.probability.toFixed(2)}`;
|
|
39
|
+
if (answer.confidence >= options.act) {
|
|
40
|
+
return answer.yes
|
|
41
|
+
? { decision: 'allow', reason: `A decision allowed it: ${odds}` }
|
|
42
|
+
: { decision: 'deny', reason: `A decision refused it: ${odds}` };
|
|
43
|
+
}
|
|
44
|
+
return {
|
|
45
|
+
decision: 'ask',
|
|
46
|
+
reason: `A decision was unsure, ${odds}; a person decides${base ? `. ${base.reason}` : ''}`,
|
|
47
|
+
};
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
export function decisionGuard(decisions, options) {
|
|
52
|
+
const refusal = options.refusal ?? "I can't help with that.";
|
|
53
|
+
const judge = async (where, question, text, context) => {
|
|
54
|
+
const { answers } = await decisions.decide({
|
|
55
|
+
...(context.tenantId !== undefined ? { tenantId: context.tenantId } : {}),
|
|
56
|
+
...options.request,
|
|
57
|
+
state: text,
|
|
58
|
+
questions: { flagged: question },
|
|
59
|
+
signal: context.signal,
|
|
60
|
+
});
|
|
61
|
+
const answer = answers.flagged;
|
|
62
|
+
if (answer.confidence >= options.act) {
|
|
63
|
+
options.onVerdict?.({ where, outcome: answer.yes ? 'stopped' : 'passed', probability: answer.probability });
|
|
64
|
+
return !answer.yes;
|
|
65
|
+
}
|
|
66
|
+
options.onVerdict?.({ where, outcome: 'reviewed', probability: answer.probability });
|
|
67
|
+
const review = context.interrupt({
|
|
68
|
+
reason: `The guard is unsure whether this ${where === 'input' ? 'message' : 'answer'} may go on`,
|
|
69
|
+
payload: { where, text, question: question.question, probability: answer.probability },
|
|
70
|
+
});
|
|
71
|
+
return review?.allow === true;
|
|
72
|
+
};
|
|
73
|
+
return {
|
|
74
|
+
name: 'decision-guard',
|
|
75
|
+
async wrapModelCall(request, next, context) {
|
|
76
|
+
const last = request.messages.at(-1);
|
|
77
|
+
if (options.input && last?.role === 'user' && !(await judge('input', options.input, textOf(last), context))) {
|
|
78
|
+
return context.stop(refusal);
|
|
79
|
+
}
|
|
80
|
+
const response = await next(request);
|
|
81
|
+
if (options.output && response.content.trim() && !response.toolCalls?.length) {
|
|
82
|
+
if (!(await judge('output', options.output, response.content, context)))
|
|
83
|
+
return { ...response, content: refusal };
|
|
84
|
+
}
|
|
85
|
+
return response;
|
|
86
|
+
},
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
export function decisionModel(decisions, options) {
|
|
90
|
+
const chosen = new Map();
|
|
91
|
+
return {
|
|
92
|
+
name: 'decision-model',
|
|
93
|
+
async beforeModel(context) {
|
|
94
|
+
const message = [...context.request.messages].reverse().find((item) => item.role === 'user');
|
|
95
|
+
if (!message)
|
|
96
|
+
return;
|
|
97
|
+
const text = textOf(message);
|
|
98
|
+
const key = `${context.threadId}\u0000${text}`;
|
|
99
|
+
if (!chosen.has(key)) {
|
|
100
|
+
const { answers } = await decisions.decide({
|
|
101
|
+
...(context.tenantId !== undefined ? { tenantId: context.tenantId } : {}),
|
|
102
|
+
...options.request,
|
|
103
|
+
state: text,
|
|
104
|
+
questions: { kind: options.question },
|
|
105
|
+
signal: context.signal,
|
|
106
|
+
});
|
|
107
|
+
const answer = answers.kind;
|
|
108
|
+
chosen.set(key, answer.confidence >= options.act ? (options.models[answer.choice] ?? null) : null);
|
|
109
|
+
if (chosen.size > 1_000)
|
|
110
|
+
chosen.delete(chosen.keys().next().value);
|
|
111
|
+
}
|
|
112
|
+
const model = chosen.get(key);
|
|
113
|
+
return model ? { ...context.request, model } : undefined;
|
|
114
|
+
},
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
export function decisionEvaluator(decisions, options) {
|
|
118
|
+
return async (context) => {
|
|
119
|
+
const { answers, model } = await decisions.decide({
|
|
120
|
+
...options.request,
|
|
121
|
+
state: options.state?.(context) ?? {
|
|
122
|
+
inputs: context.example.inputs,
|
|
123
|
+
output: context.output,
|
|
124
|
+
...(context.example.expected !== undefined ? { expected: context.example.expected } : {}),
|
|
125
|
+
},
|
|
126
|
+
questions: { judged: options.question },
|
|
127
|
+
});
|
|
128
|
+
const answer = answers.judged;
|
|
129
|
+
const metadata = {
|
|
130
|
+
confidence: answer.confidence,
|
|
131
|
+
probabilities: answer.type === 'yes-no' ? { yes: answer.probability } : answer.probabilities,
|
|
132
|
+
model,
|
|
133
|
+
};
|
|
134
|
+
if (answer.type === 'yes-no') {
|
|
135
|
+
const passWhen = options.passing !== false;
|
|
136
|
+
return {
|
|
137
|
+
key: options.key,
|
|
138
|
+
score: answer.probability,
|
|
139
|
+
passed: answer.yes === passWhen,
|
|
140
|
+
metadata,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
if (answer.type === 'score') {
|
|
144
|
+
const top = answer.probabilities.length - 1;
|
|
145
|
+
const passing = typeof options.passing === 'number' ? options.passing : top;
|
|
146
|
+
return { key: options.key, score: top > 0 ? answer.score / top : 1, passed: answer.level >= passing, metadata };
|
|
147
|
+
}
|
|
148
|
+
const passing = typeof options.passing === 'string' ? options.passing : undefined;
|
|
149
|
+
return {
|
|
150
|
+
key: options.key,
|
|
151
|
+
score: passing !== undefined ? (answer.probabilities[passing] ?? 0) : answer.confidence,
|
|
152
|
+
...(passing !== undefined ? { passed: answer.choice === passing } : {}),
|
|
153
|
+
comment: answer.choice,
|
|
154
|
+
metadata,
|
|
155
|
+
};
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
export function reviewUnsure(threshold, key) {
|
|
159
|
+
return (scores) => scores.some((score) => (key === undefined || score.key === key) &&
|
|
160
|
+
typeof score.metadata?.confidence === 'number' &&
|
|
161
|
+
score.metadata.confidence < threshold);
|
|
162
|
+
}
|
|
163
|
+
function textOf(message) {
|
|
164
|
+
if (typeof message.content === 'string')
|
|
165
|
+
return message.content;
|
|
166
|
+
return message.content
|
|
167
|
+
.map((part) => (part.type === 'text' ? part.text : ''))
|
|
168
|
+
.filter(Boolean)
|
|
169
|
+
.join('\n');
|
|
170
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/** A decision provider for tests: it answers as it is told, and records what it was asked. */
|
|
2
|
+
import type { DecisionCallContext, DecisionDistribution, DecisionProvider, DecisionProviderInfo, DecisionProviderQuestion, DecisionProviderRequest, DecisionProviderResult, DecisionState } from './core.js';
|
|
3
|
+
/** Options for `MockDecisionProvider`. */
|
|
4
|
+
export interface MockDecisionProviderOptions {
|
|
5
|
+
/** Its name in results. Defaults to `mock`. */
|
|
6
|
+
name?: string;
|
|
7
|
+
/** The model it reports. Defaults to `mock-decision`. */
|
|
8
|
+
model?: string;
|
|
9
|
+
/**
|
|
10
|
+
* Answers each question with its probabilities, from the question's name, the question, and the
|
|
11
|
+
* state. Returning `undefined`, or leaving it out, answers with an even spread: confidence 0.
|
|
12
|
+
*/
|
|
13
|
+
answer?: (name: string, question: DecisionProviderQuestion, state: DecisionState) => DecisionDistribution | undefined;
|
|
14
|
+
/** Fails every call with this error, or the error the function returns. */
|
|
15
|
+
fail?: Error | ((request: DecisionProviderRequest) => Error | undefined);
|
|
16
|
+
/** The input tokens it reports per call. Defaults to a quarter of the request's length. */
|
|
17
|
+
inputTokens?: number;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Answers decisions as `answer` says, without a model, and keeps every request it was asked in
|
|
21
|
+
* `requests`. It takes every question type, 255 options, and 10 levels, as TypeSafe AI's Jev does.
|
|
22
|
+
*/
|
|
23
|
+
export declare class MockDecisionProvider implements DecisionProvider {
|
|
24
|
+
private readonly options;
|
|
25
|
+
/** Its name and what it takes. */
|
|
26
|
+
readonly info: DecisionProviderInfo;
|
|
27
|
+
/** Every request it was asked, in order. */
|
|
28
|
+
readonly requests: DecisionProviderRequest[];
|
|
29
|
+
constructor(options?: MockDecisionProviderOptions);
|
|
30
|
+
/** Answers every question as `answer` says, or with an even spread. */
|
|
31
|
+
decide(request: DecisionProviderRequest, context: DecisionCallContext): Promise<DecisionProviderResult>;
|
|
32
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
export class MockDecisionProvider {
|
|
2
|
+
options;
|
|
3
|
+
info;
|
|
4
|
+
requests = [];
|
|
5
|
+
constructor(options = {}) {
|
|
6
|
+
this.options = options;
|
|
7
|
+
this.info = {
|
|
8
|
+
name: options.name ?? 'mock',
|
|
9
|
+
capabilities: {
|
|
10
|
+
types: ['yes-no', 'choice', 'score'],
|
|
11
|
+
maxOptions: 255,
|
|
12
|
+
maxLevels: 10,
|
|
13
|
+
defaultModel: options.model ?? 'mock-decision',
|
|
14
|
+
},
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
async decide(request, context) {
|
|
18
|
+
this.requests.push(request);
|
|
19
|
+
context.signal.throwIfAborted();
|
|
20
|
+
const failure = typeof this.options.fail === 'function' ? this.options.fail(request) : this.options.fail;
|
|
21
|
+
if (failure)
|
|
22
|
+
throw failure;
|
|
23
|
+
const distributions = {};
|
|
24
|
+
for (const [name, question] of Object.entries(request.questions)) {
|
|
25
|
+
distributions[name] = this.options.answer?.(name, question, request.state) ?? evenly(question);
|
|
26
|
+
}
|
|
27
|
+
return {
|
|
28
|
+
model: request.model,
|
|
29
|
+
distributions,
|
|
30
|
+
usage: {
|
|
31
|
+
inputTokens: this.options.inputTokens ?? Math.ceil(JSON.stringify(request).length / 4),
|
|
32
|
+
outputTokens: 0,
|
|
33
|
+
},
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
function evenly(question) {
|
|
38
|
+
if (question.type === 'yes-no')
|
|
39
|
+
return { type: 'yes-no', yes: 0.5 };
|
|
40
|
+
if (question.type === 'choice') {
|
|
41
|
+
return {
|
|
42
|
+
type: 'choice',
|
|
43
|
+
probabilities: Object.fromEntries(question.options.map((option) => [option.name, 1 / question.options.length])),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
return { type: 'score', probabilities: question.levels.map(() => 1 / question.levels.length) };
|
|
47
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { type DecisionCallContext, type DecisionProvider, type DecisionProviderInfo, type DecisionProviderRequest, type DecisionProviderResult } from './core.js';
|
|
2
|
+
/** Options for `TypeSafeDecisionProvider`. */
|
|
3
|
+
export interface TypeSafeDecisionProviderOptions {
|
|
4
|
+
/** A TypeSafe AI API key, from its console. A tenant's resolved credential replaces it per call. */
|
|
5
|
+
apiKey: string;
|
|
6
|
+
/**
|
|
7
|
+
* The API's base URL. Defaults to `https://api.typesafe.ai`; a proxy that passes the API through,
|
|
8
|
+
* such as LiteLLM's at `<proxy>/typesafe`, works as well.
|
|
9
|
+
*/
|
|
10
|
+
baseUrl?: string;
|
|
11
|
+
/** The model asked when a request names none. Defaults to `jev-latest`. */
|
|
12
|
+
model?: string;
|
|
13
|
+
/**
|
|
14
|
+
* Sends `TypeSafe-Danger-Zone-Long-Context: true`, which TypeSafe AI requires for a state longer
|
|
15
|
+
* than its usual context, and marks as experimental.
|
|
16
|
+
*/
|
|
17
|
+
longContext?: boolean;
|
|
18
|
+
/** How long a call may take before it fails, in milliseconds. Defaults to 30 seconds. */
|
|
19
|
+
timeoutMs?: number;
|
|
20
|
+
/** Headers added to every request. */
|
|
21
|
+
headers?: Record<string, string>;
|
|
22
|
+
/** Replaces the global `fetch`. */
|
|
23
|
+
fetch?: typeof fetch;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* TypeSafe AI's System One API (`POST /v1/systemone`): Jev, a model that answers only typed
|
|
27
|
+
* questions, a yes or no ("noul"), a choice among up to 255 options, or a score on up to ten levels,
|
|
28
|
+
* each with its probabilities, and bills for input tokens alone. Every question of a decision goes in
|
|
29
|
+
* one call and is answered in isolation, against the same state.
|
|
30
|
+
*
|
|
31
|
+
* ```ts
|
|
32
|
+
* const decisions = new DecisionManager({ providers: { jev: new TypeSafeDecisionProvider({ apiKey: process.env.TYPESAFE_API_KEY! }) } });
|
|
33
|
+
* ```
|
|
34
|
+
*/
|
|
35
|
+
export declare class TypeSafeDecisionProvider implements DecisionProvider {
|
|
36
|
+
private readonly options;
|
|
37
|
+
/** Its name and what it takes: every question type, 255 options, and 10 levels. */
|
|
38
|
+
readonly info: DecisionProviderInfo;
|
|
39
|
+
constructor(options: TypeSafeDecisionProviderOptions);
|
|
40
|
+
/** Answers every question of a decision in one call. */
|
|
41
|
+
decide(request: DecisionProviderRequest, context: DecisionCallContext): Promise<DecisionProviderResult>;
|
|
42
|
+
/** The models the key may use, from `GET /v1/models`. */
|
|
43
|
+
listModels(context?: Partial<DecisionCallContext>): Promise<string[]>;
|
|
44
|
+
private call;
|
|
45
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { linkSignals } from '../utils/signals.js';
|
|
2
|
+
import { DecisionError, } from './core.js';
|
|
3
|
+
export class TypeSafeDecisionProvider {
|
|
4
|
+
options;
|
|
5
|
+
info;
|
|
6
|
+
constructor(options) {
|
|
7
|
+
this.options = options;
|
|
8
|
+
this.info = {
|
|
9
|
+
name: 'typesafe',
|
|
10
|
+
capabilities: {
|
|
11
|
+
types: ['yes-no', 'choice', 'score'],
|
|
12
|
+
maxOptions: 255,
|
|
13
|
+
maxLevels: 10,
|
|
14
|
+
defaultModel: options.model ?? 'jev-latest',
|
|
15
|
+
},
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
async decide(request, context) {
|
|
19
|
+
const questions = Object.fromEntries(Object.entries(request.questions).map(([name, question]) => [name, wireQuestion(question)]));
|
|
20
|
+
const payload = await this.call('/v1/systemone', { method: 'POST', body: JSON.stringify({ model: request.model, state: request.state, questions }) }, context);
|
|
21
|
+
const distributions = {};
|
|
22
|
+
for (const [name, question] of Object.entries(request.questions)) {
|
|
23
|
+
const answer = payload.answers?.[name];
|
|
24
|
+
if (!answer)
|
|
25
|
+
throw new DecisionError(`TypeSafe AI did not answer "${name}"`, 'DECISION_FAILED');
|
|
26
|
+
distributions[name] = distributionOf(name, question, answer);
|
|
27
|
+
}
|
|
28
|
+
return {
|
|
29
|
+
...(payload.model ? { model: payload.model } : {}),
|
|
30
|
+
distributions,
|
|
31
|
+
...(payload.usage
|
|
32
|
+
? {
|
|
33
|
+
usage: {
|
|
34
|
+
...(payload.usage.input_tokens !== undefined ? { inputTokens: payload.usage.input_tokens } : {}),
|
|
35
|
+
...(payload.usage.output_tokens !== undefined ? { outputTokens: payload.usage.output_tokens } : {}),
|
|
36
|
+
},
|
|
37
|
+
}
|
|
38
|
+
: {}),
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
async listModels(context = {}) {
|
|
42
|
+
const payload = await this.call('/v1/models', { method: 'GET' }, context);
|
|
43
|
+
const listed = Array.isArray(payload)
|
|
44
|
+
? payload
|
|
45
|
+
: (payload?.data ??
|
|
46
|
+
payload?.models ??
|
|
47
|
+
[]);
|
|
48
|
+
return listed
|
|
49
|
+
.map((item) => typeof item === 'string'
|
|
50
|
+
? item
|
|
51
|
+
: String(item?.id ?? item?.name ?? ''))
|
|
52
|
+
.filter(Boolean);
|
|
53
|
+
}
|
|
54
|
+
async call(path, init, context) {
|
|
55
|
+
const credential = await context.credential?.(this.info.name);
|
|
56
|
+
const linked = linkSignals([context.signal, AbortSignal.timeout(this.options.timeoutMs ?? 30_000)]);
|
|
57
|
+
const signal = linked.signal;
|
|
58
|
+
let response;
|
|
59
|
+
let text;
|
|
60
|
+
try {
|
|
61
|
+
response = await (this.options.fetch ?? fetch)(`${(this.options.baseUrl ?? 'https://api.typesafe.ai').replace(/\/+$/, '')}${path}`, {
|
|
62
|
+
...init,
|
|
63
|
+
headers: {
|
|
64
|
+
...(init.body ? { 'content-type': 'application/json' } : {}),
|
|
65
|
+
authorization: `Bearer ${credential?.apiKey ?? this.options.apiKey}`,
|
|
66
|
+
...(this.options.longContext ? { 'TypeSafe-Danger-Zone-Long-Context': 'true' } : {}),
|
|
67
|
+
...this.options.headers,
|
|
68
|
+
...credential?.headers,
|
|
69
|
+
},
|
|
70
|
+
signal,
|
|
71
|
+
});
|
|
72
|
+
text = await response.text();
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
if (context.signal?.aborted)
|
|
76
|
+
throw error;
|
|
77
|
+
throw new DecisionError(`TypeSafe AI could not be reached: ${error instanceof Error ? error.message : String(error)}`, 'DECISION_FAILED', true);
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
linked.dispose();
|
|
81
|
+
}
|
|
82
|
+
if (!response.ok)
|
|
83
|
+
throw failure(response.status, text);
|
|
84
|
+
try {
|
|
85
|
+
return JSON.parse(text);
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
throw new DecisionError(`TypeSafe AI answered with something other than JSON: ${text.slice(0, 200)}`, 'DECISION_FAILED', true, response.status);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
function wireQuestion(question) {
|
|
93
|
+
const instructions = question.question ? { instructions: question.question } : {};
|
|
94
|
+
if (question.type === 'yes-no') {
|
|
95
|
+
const criteria = {
|
|
96
|
+
...(question.criteria?.yes ? { true: question.criteria.yes } : {}),
|
|
97
|
+
...(question.criteria?.no ? { false: question.criteria.no } : {}),
|
|
98
|
+
};
|
|
99
|
+
return { type: 'noul', ...instructions, ...(Object.keys(criteria).length ? { criteria } : {}) };
|
|
100
|
+
}
|
|
101
|
+
if (question.type === 'choice') {
|
|
102
|
+
return {
|
|
103
|
+
type: 'choice',
|
|
104
|
+
...instructions,
|
|
105
|
+
criteria: Object.fromEntries(question.options.map((option) => [option.name, option.description ?? null])),
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
return { type: 'score', ...instructions, criteria: [...question.levels] };
|
|
109
|
+
}
|
|
110
|
+
function distributionOf(name, question, answer) {
|
|
111
|
+
if (question.type === 'yes-no') {
|
|
112
|
+
if (typeof answer.noul !== 'number')
|
|
113
|
+
throw invalidAnswer(name, 'noul', answer);
|
|
114
|
+
return { type: 'yes-no', yes: answer.noul };
|
|
115
|
+
}
|
|
116
|
+
if (question.type === 'choice') {
|
|
117
|
+
if (!answer.probabilities)
|
|
118
|
+
throw invalidAnswer(name, 'choice', answer);
|
|
119
|
+
return { type: 'choice', probabilities: answer.probabilities };
|
|
120
|
+
}
|
|
121
|
+
if (!answer.probabilities)
|
|
122
|
+
throw invalidAnswer(name, 'score', answer);
|
|
123
|
+
const probabilities = answer.probabilities;
|
|
124
|
+
return { type: 'score', probabilities: question.levels.map((_, level) => probabilities[String(level)] ?? 0) };
|
|
125
|
+
}
|
|
126
|
+
function invalidAnswer(name, type, answer) {
|
|
127
|
+
return new DecisionError(`TypeSafe AI answered "${name}" without the ${type} probabilities it asks for: ${JSON.stringify(answer).slice(0, 200)}`, 'DECISION_FAILED');
|
|
128
|
+
}
|
|
129
|
+
function failure(status, text) {
|
|
130
|
+
if (status === 422) {
|
|
131
|
+
let detail = text.slice(0, 300);
|
|
132
|
+
try {
|
|
133
|
+
const parsed = JSON.parse(text);
|
|
134
|
+
if (Array.isArray(parsed.detail)) {
|
|
135
|
+
detail = parsed.detail.map((item) => `${(item.loc ?? []).join('.')}: ${item.msg ?? 'invalid'}`).join('; ');
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
catch {
|
|
139
|
+
}
|
|
140
|
+
return new DecisionError(`TypeSafe AI refused the decision: ${detail}`, 'DECISION_INVALID', false, status);
|
|
141
|
+
}
|
|
142
|
+
const retryable = status === 408 || status === 429 || status >= 500;
|
|
143
|
+
return new DecisionError(`TypeSafe AI answered ${status}: ${text.slice(0, 300)}`, 'DECISION_FAILED', retryable, status);
|
|
144
|
+
}
|
|
@@ -1,8 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Tools emulated by a model, for evaluations that must not touch the world: an agent that would
|
|
3
|
-
* charge a card or write to a production database runs against a model playing each tool instead,
|
|
4
|
-
* with nothing sent, and the emulator's answers recorded so a run replays them exactly.
|
|
5
|
-
*/
|
|
6
1
|
import type { CompletionRequest, ToolDefinition } from '../types/messages.js';
|
|
7
2
|
import type { NexusResponse } from '../types/response.js';
|
|
8
3
|
/** The model that plays the tools: the client, or anything with its `complete()`. */
|
|
@@ -89,6 +84,76 @@ export interface EmulatedToolsOptions {
|
|
|
89
84
|
maxTokens?: number;
|
|
90
85
|
/** Called after each call, with who answered it. */
|
|
91
86
|
onCall?: (call: EmulatedToolCall) => void;
|
|
87
|
+
/**
|
|
88
|
+
* The world the tools act in, held across every run that shares it: each call made in it, and the
|
|
89
|
+
* changes they made, so an order one example refunds is refunded for the next. The emulator is
|
|
90
|
+
* told every change, and the latest `history` calls; its scenario is the tools' when they name
|
|
91
|
+
* none. Share one world among runs made one at a time, in order; or start each run from a
|
|
92
|
+
* `fork()`, or from a `snapshot()` read back.
|
|
93
|
+
*/
|
|
94
|
+
world?: EmulatedWorld;
|
|
95
|
+
/**
|
|
96
|
+
* Tools that change nothing, by name, beside those whose every capability reads: `filesystem:read`,
|
|
97
|
+
* `mcp.read`, or `api.read`, or none at all. A world counts what any other tool did, when it did
|
|
98
|
+
* not fail, as a change.
|
|
99
|
+
*/
|
|
100
|
+
reads?: readonly string[];
|
|
101
|
+
}
|
|
102
|
+
/** One call made in an emulated world. */
|
|
103
|
+
export interface EmulatedWorldCall extends EmulatedResult {
|
|
104
|
+
/** Whether it changed the world: it succeeded, and its tool does more than read. */
|
|
105
|
+
changed: boolean;
|
|
106
|
+
}
|
|
107
|
+
/** An emulated world as plain JSON, to write to a file and start runs from. */
|
|
108
|
+
export interface EmulatedWorldSnapshot {
|
|
109
|
+
/** The world the tools act on, before any call. */
|
|
110
|
+
scenario?: string;
|
|
111
|
+
/** Every call made in it, oldest first. */
|
|
112
|
+
calls: EmulatedWorldCall[];
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* A world emulated tools act in, kept across the runs of an evaluation: the scenario it started
|
|
116
|
+
* from, and every call made in it, so what one run changes is there for the next, as on a staging
|
|
117
|
+
* copy of a service. `snapshot()` writes it as JSON, `new EmulatedWorld(snapshot)` reads it back,
|
|
118
|
+
* and `fork()` starts a world from this one's present whose changes stay its own.
|
|
119
|
+
*
|
|
120
|
+
* ```ts
|
|
121
|
+
* const world = new EmulatedWorld({ scenario: 'Order 1042 is delivered and cost $59.' });
|
|
122
|
+
* await evaluate(
|
|
123
|
+
* (inputs) => agentOver(emulatedTools(shop.tools, { client: ai, results: shop.results, world })).invoke(agentInput(inputs.request)),
|
|
124
|
+
* dataset,
|
|
125
|
+
* evaluators,
|
|
126
|
+
* { concurrency: 1 },
|
|
127
|
+
* );
|
|
128
|
+
* ```
|
|
129
|
+
*/
|
|
130
|
+
export declare class EmulatedWorld {
|
|
131
|
+
/** The world the tools act on, before any call. */
|
|
132
|
+
readonly scenario?: string;
|
|
133
|
+
private readonly log;
|
|
134
|
+
constructor(start?: {
|
|
135
|
+
scenario?: string;
|
|
136
|
+
calls?: readonly EmulatedWorldCall[];
|
|
137
|
+
});
|
|
138
|
+
/** Every call made in it, oldest first. */
|
|
139
|
+
get calls(): readonly EmulatedWorldCall[];
|
|
140
|
+
/** The calls that changed it, oldest first. */
|
|
141
|
+
get changes(): readonly EmulatedWorldCall[];
|
|
142
|
+
/** Records a call made in it, and whether it changed it. */
|
|
143
|
+
record(call: EmulatedResult, changed: boolean): void;
|
|
144
|
+
/**
|
|
145
|
+
* A digest of its scenario and every change made in it, which the keys of its calls' answers carry:
|
|
146
|
+
* two worlds that started alike and changed alike have the same digest, however much was read.
|
|
147
|
+
*/
|
|
148
|
+
digest(): string;
|
|
149
|
+
/** How many calls with these arguments it has seen since its last change. */
|
|
150
|
+
ordinal(tool: string, args: Record<string, unknown>): number;
|
|
151
|
+
/** It as plain JSON, to write to a file and start runs from. */
|
|
152
|
+
snapshot(): EmulatedWorldSnapshot;
|
|
153
|
+
/** A world starting from this one's present, whose changes stay its own. */
|
|
154
|
+
fork(): EmulatedWorld;
|
|
155
|
+
/** The same as `snapshot()`, so `JSON.stringify(world)` writes it. */
|
|
156
|
+
toJSON(): EmulatedWorldSnapshot;
|
|
92
157
|
}
|
|
93
158
|
/** Raised when a call cannot be emulated: no kept answer in `replay`, or no valid answer from the model. */
|
|
94
159
|
export declare class EmulationError extends Error {
|
|
@@ -121,13 +186,15 @@ export declare class MemoryEmulationStore implements EmulationStore {
|
|
|
121
186
|
}
|
|
122
187
|
/**
|
|
123
188
|
* The key of a call: its tool, its arguments, its scenario, and which call with those arguments it
|
|
124
|
-
* is, so an order looked up before and after a refund keeps both answers.
|
|
189
|
+
* is, so an order looked up before and after a refund keeps both answers. In a world, also the
|
|
190
|
+
* world's `digest()`, so the same call answers again only after the same changes.
|
|
125
191
|
*/
|
|
126
192
|
export declare function emulationKey(call: {
|
|
127
193
|
tool: string;
|
|
128
194
|
args: Record<string, unknown>;
|
|
129
195
|
scenario?: string;
|
|
130
196
|
ordinal?: number;
|
|
197
|
+
world?: string;
|
|
131
198
|
}): string;
|
|
132
199
|
/**
|
|
133
200
|
* Tools that look the same to the agent, by name, description, parameters, and capabilities, but are
|