nexus-ai-pro 2.18.0 → 2.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/API_STABILITY.md +64 -1
- package/CHANGELOG.md +113 -1
- package/README.md +11 -3
- package/dist/agent/capabilities.d.ts +3 -0
- package/dist/agent/openapi.d.ts +139 -0
- package/dist/agent/openapi.js +455 -0
- package/dist/capabilities/negotiate.js +5 -0
- package/dist/decisions/completion.d.ts +48 -0
- package/dist/decisions/completion.js +195 -0
- package/dist/decisions/core.d.ts +525 -0
- package/dist/decisions/core.js +493 -0
- package/dist/decisions/index.d.ts +9 -0
- package/dist/decisions/index.js +3 -0
- package/dist/decisions/integrations.d.ts +142 -0
- package/dist/decisions/integrations.js +170 -0
- package/dist/decisions/mock.d.ts +32 -0
- package/dist/decisions/mock.js +47 -0
- package/dist/decisions/typesafe.d.ts +45 -0
- package/dist/decisions/typesafe.js +144 -0
- package/dist/evaluate/emulate.d.ts +73 -6
- package/dist/evaluate/emulate.js +114 -12
- package/dist/index.d.ts +1 -1
- package/dist/optimizer/densifier.js +20 -11
- package/dist/protocols/a2a-grpc.js +1 -1
- package/dist/protocols/a2a-proto.d.ts +2 -2
- package/dist/protocols/a2a-proto.js +76 -1
- package/dist/protocols/a2a.d.ts +121 -0
- package/dist/protocols/a2a.js +88 -29
- package/dist/providers/google.js +28 -0
- package/dist/providers/ollama.js +23 -0
- package/dist/providers/openai.d.ts +6 -0
- package/dist/providers/openai.js +26 -0
- package/dist/types/lifecycle.d.ts +1 -1
- package/dist/types/messages.d.ts +9 -0
- package/dist/types/providers.d.ts +2 -0
- package/dist/types/response.d.ts +17 -0
- package/dist/types/tracing.d.ts +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/workflow/chains.d.ts +13 -1
- package/dist/workflow/chains.js +40 -0
- package/dist-cjs/agent/capabilities.d.ts +3 -0
- package/dist-cjs/agent/openapi.d.ts +139 -0
- package/dist-cjs/agent/openapi.js +461 -0
- package/dist-cjs/capabilities/negotiate.js +5 -0
- package/dist-cjs/decisions/completion.d.ts +48 -0
- package/dist-cjs/decisions/completion.js +198 -0
- package/dist-cjs/decisions/core.d.ts +525 -0
- package/dist-cjs/decisions/core.js +509 -0
- package/dist-cjs/decisions/index.d.ts +9 -0
- package/dist-cjs/decisions/index.js +27 -0
- package/dist-cjs/decisions/integrations.d.ts +142 -0
- package/dist-cjs/decisions/integrations.js +178 -0
- package/dist-cjs/decisions/mock.d.ts +32 -0
- package/dist-cjs/decisions/mock.js +51 -0
- package/dist-cjs/decisions/typesafe.d.ts +45 -0
- package/dist-cjs/decisions/typesafe.js +148 -0
- package/dist-cjs/evaluate/emulate.d.ts +73 -6
- package/dist-cjs/evaluate/emulate.js +116 -13
- package/dist-cjs/index.d.ts +1 -1
- package/dist-cjs/optimizer/densifier.js +20 -11
- package/dist-cjs/protocols/a2a-grpc.js +1 -1
- package/dist-cjs/protocols/a2a-proto.d.ts +2 -2
- package/dist-cjs/protocols/a2a-proto.js +76 -1
- package/dist-cjs/protocols/a2a.d.ts +121 -0
- package/dist-cjs/protocols/a2a.js +88 -29
- package/dist-cjs/providers/google.js +28 -0
- package/dist-cjs/providers/ollama.js +23 -0
- package/dist-cjs/providers/openai.d.ts +6 -0
- package/dist-cjs/providers/openai.js +26 -0
- package/dist-cjs/types/lifecycle.d.ts +1 -1
- package/dist-cjs/types/messages.d.ts +9 -0
- package/dist-cjs/types/providers.d.ts +2 -0
- package/dist-cjs/types/response.d.ts +17 -0
- package/dist-cjs/types/tracing.d.ts +1 -1
- package/dist-cjs/version.d.ts +1 -1
- package/dist-cjs/version.js +1 -1
- package/dist-cjs/workflow/chains.d.ts +13 -1
- package/dist-cjs/workflow/chains.js +40 -0
- package/llms.txt +2 -1
- package/package.json +54 -2
|
@@ -0,0 +1,493 @@
|
|
|
1
|
+
import { createCacheKey } from '../cache/memory-cache.js';
|
|
2
|
+
import { withLifecycle } from '../lifecycle/run.js';
|
|
3
|
+
import { generateRequestId } from '../utils/ids.js';
|
|
4
|
+
import { sha256Hex } from '../utils/sha256.js';
|
|
5
|
+
export class DecisionError extends Error {
|
|
6
|
+
code;
|
|
7
|
+
retryable;
|
|
8
|
+
status;
|
|
9
|
+
constructor(message, code, retryable = false, status) {
|
|
10
|
+
super(message);
|
|
11
|
+
this.code = code;
|
|
12
|
+
this.retryable = retryable;
|
|
13
|
+
this.status = status;
|
|
14
|
+
this.name = 'DecisionError';
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
export function yesNoConfidence(probability) {
|
|
18
|
+
return clamp(Math.abs(2 * probability - 1));
|
|
19
|
+
}
|
|
20
|
+
export function choiceConfidence(probabilities) {
|
|
21
|
+
const n = probabilities.length;
|
|
22
|
+
if (n <= 1)
|
|
23
|
+
return 1;
|
|
24
|
+
const top = Math.max(...probabilities);
|
|
25
|
+
return clamp((top - 1 / n) / (1 - 1 / n));
|
|
26
|
+
}
|
|
27
|
+
export function scoreConfidence(probabilities) {
|
|
28
|
+
const n = probabilities.length;
|
|
29
|
+
if (n <= 1)
|
|
30
|
+
return 1;
|
|
31
|
+
const mode = argmax(probabilities);
|
|
32
|
+
let spread = 0;
|
|
33
|
+
let even = 0;
|
|
34
|
+
for (let index = 0; index < n; index += 1) {
|
|
35
|
+
spread += (probabilities[index] ?? 0) * Math.abs(index - mode);
|
|
36
|
+
even += Math.abs(index - mode) / n;
|
|
37
|
+
}
|
|
38
|
+
return even === 0 ? 1 : clamp(1 - spread / even);
|
|
39
|
+
}
|
|
40
|
+
export function decisionAnswer(question, distribution) {
|
|
41
|
+
if (distribution.type !== question.type) {
|
|
42
|
+
throw new DecisionError(`A ${question.type} question was answered as a ${distribution.type}`, 'DECISION_INVALID');
|
|
43
|
+
}
|
|
44
|
+
if (question.type === 'yes-no' && distribution.type === 'yes-no') {
|
|
45
|
+
const probability = probabilityOf(distribution.yes, 'yes');
|
|
46
|
+
return { type: 'yes-no', yes: probability >= 0.5, probability, confidence: yesNoConfidence(probability) };
|
|
47
|
+
}
|
|
48
|
+
if (question.type === 'choice' && distribution.type === 'choice') {
|
|
49
|
+
const names = question.options.map((option) => option.name);
|
|
50
|
+
for (const name of Object.keys(distribution.probabilities)) {
|
|
51
|
+
if (!names.includes(name))
|
|
52
|
+
throw new DecisionError(`"${name}" is not an option of the question`, 'DECISION_INVALID');
|
|
53
|
+
}
|
|
54
|
+
const weights = normalize(names.map((name) => probabilityOf(distribution.probabilities[name] ?? 0, name)));
|
|
55
|
+
const probabilities = Object.fromEntries(names.map((name, index) => [name, weights[index]]));
|
|
56
|
+
return {
|
|
57
|
+
type: 'choice',
|
|
58
|
+
choice: names[argmax(weights)],
|
|
59
|
+
probabilities,
|
|
60
|
+
confidence: choiceConfidence(weights),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
if (question.type === 'score' && distribution.type === 'score') {
|
|
64
|
+
if (distribution.probabilities.length !== question.levels.length) {
|
|
65
|
+
throw new DecisionError(`A score of ${question.levels.length} levels was answered with ${distribution.probabilities.length} probabilities`, 'DECISION_INVALID');
|
|
66
|
+
}
|
|
67
|
+
const weights = normalize(distribution.probabilities.map((value, index) => probabilityOf(value, `level ${index}`)));
|
|
68
|
+
return {
|
|
69
|
+
type: 'score',
|
|
70
|
+
score: weights.reduce((sum, weight, index) => sum + weight * index, 0),
|
|
71
|
+
level: argmax(weights),
|
|
72
|
+
probabilities: weights,
|
|
73
|
+
confidence: scoreConfidence(weights),
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
throw new DecisionError(`Unknown question type "${question.type}"`, 'DECISION_INVALID');
|
|
77
|
+
}
|
|
78
|
+
export function gate(answer, thresholds) {
|
|
79
|
+
if (answer.confidence >= thresholds.act)
|
|
80
|
+
return 'act';
|
|
81
|
+
return answer.confidence >= (thresholds.review ?? 0) ? 'review' : 'abstain';
|
|
82
|
+
}
|
|
83
|
+
export function answerProbability(answer) {
|
|
84
|
+
if (answer.type === 'yes-no')
|
|
85
|
+
return Math.max(answer.probability, 1 - answer.probability);
|
|
86
|
+
if (answer.type === 'choice')
|
|
87
|
+
return answer.probabilities[answer.choice] ?? 0;
|
|
88
|
+
return answer.probabilities[answer.level] ?? 0;
|
|
89
|
+
}
|
|
90
|
+
export function calibrate(samples, options = {}) {
|
|
91
|
+
const count = Math.max(1, Math.floor(options.bins ?? 10));
|
|
92
|
+
const valid = samples
|
|
93
|
+
.map((sample) => ({
|
|
94
|
+
...sample,
|
|
95
|
+
probability: answerProbability(sample.answer),
|
|
96
|
+
confidence: sample.answer.confidence,
|
|
97
|
+
}))
|
|
98
|
+
.filter((sample) => Number.isFinite(sample.probability) && Number.isFinite(sample.confidence));
|
|
99
|
+
const right = valid.filter((sample) => sample.correct).length;
|
|
100
|
+
const bins = [];
|
|
101
|
+
let error = 0;
|
|
102
|
+
for (let index = 0; index < count; index += 1) {
|
|
103
|
+
const from = index / count;
|
|
104
|
+
const to = (index + 1) / count;
|
|
105
|
+
const inside = valid.filter((sample) => {
|
|
106
|
+
const probability = clamp(sample.probability);
|
|
107
|
+
return probability >= from && (probability < to || (index === count - 1 && probability <= 1));
|
|
108
|
+
});
|
|
109
|
+
if (inside.length === 0)
|
|
110
|
+
continue;
|
|
111
|
+
const probability = inside.reduce((sum, sample) => sum + clamp(sample.probability), 0) / inside.length;
|
|
112
|
+
const accuracy = inside.filter((sample) => sample.correct).length / inside.length;
|
|
113
|
+
error += (inside.length / valid.length) * Math.abs(probability - accuracy);
|
|
114
|
+
bins.push({ from, to, count: inside.length, probability, accuracy });
|
|
115
|
+
}
|
|
116
|
+
const report = {
|
|
117
|
+
samples: valid.length,
|
|
118
|
+
accuracy: valid.length ? right / valid.length : 0,
|
|
119
|
+
expectedCalibrationError: error,
|
|
120
|
+
bins,
|
|
121
|
+
};
|
|
122
|
+
if (options.targetAccuracy === undefined || valid.length === 0)
|
|
123
|
+
return report;
|
|
124
|
+
const sorted = [...valid].sort((a, b) => b.confidence - a.confidence);
|
|
125
|
+
const minSamples = Math.max(1, options.minSamples ?? 10);
|
|
126
|
+
let kept = 0;
|
|
127
|
+
let correct = 0;
|
|
128
|
+
for (let index = 0; index < sorted.length; index += 1) {
|
|
129
|
+
const sample = sorted[index];
|
|
130
|
+
kept += 1;
|
|
131
|
+
if (sample.correct)
|
|
132
|
+
correct += 1;
|
|
133
|
+
if (index + 1 < sorted.length && sorted[index + 1].confidence === sample.confidence)
|
|
134
|
+
continue;
|
|
135
|
+
if (kept >= minSamples && correct / kept >= options.targetAccuracy) {
|
|
136
|
+
report.threshold = sample.confidence;
|
|
137
|
+
report.coverage = kept / valid.length;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return report;
|
|
141
|
+
}
|
|
142
|
+
export function answeredRight(answer, expected, options = {}) {
|
|
143
|
+
if (answer.type === 'yes-no')
|
|
144
|
+
return answer.yes === expected;
|
|
145
|
+
if (answer.type === 'choice')
|
|
146
|
+
return answer.choice === expected;
|
|
147
|
+
if (typeof expected !== 'number')
|
|
148
|
+
return false;
|
|
149
|
+
return options.tolerance === undefined
|
|
150
|
+
? answer.level === expected
|
|
151
|
+
: Math.abs(answer.score - expected) <= options.tolerance + 1e-9;
|
|
152
|
+
}
|
|
153
|
+
export const DECISION_PRICES = {
|
|
154
|
+
jev: { inputPerMillion: 0.042, outputPerMillion: 0 },
|
|
155
|
+
};
|
|
156
|
+
function priceOf(model, prices) {
|
|
157
|
+
if (prices[model])
|
|
158
|
+
return prices[model];
|
|
159
|
+
const prefix = Object.keys(prices)
|
|
160
|
+
.filter((name) => model.startsWith(name))
|
|
161
|
+
.sort((a, b) => b.length - a.length)[0];
|
|
162
|
+
return prefix ? prices[prefix] : undefined;
|
|
163
|
+
}
|
|
164
|
+
export function priceDecision(model, usage, prices = DECISION_PRICES) {
|
|
165
|
+
const price = priceOf(model, prices);
|
|
166
|
+
if (!price || usage?.inputTokens === undefined)
|
|
167
|
+
return undefined;
|
|
168
|
+
const input = (usage.inputTokens / 1_000_000) * price.inputPerMillion;
|
|
169
|
+
const output = ((usage.outputTokens ?? 0) / 1_000_000) * (price.outputPerMillion ?? 0);
|
|
170
|
+
return { amount: input + output, currency: 'USD', basis: 'estimated', input, output };
|
|
171
|
+
}
|
|
172
|
+
export class DecisionManager {
|
|
173
|
+
config;
|
|
174
|
+
runtime;
|
|
175
|
+
providers = new Map();
|
|
176
|
+
constructor(config = {}, runtime = {}) {
|
|
177
|
+
this.config = config;
|
|
178
|
+
this.runtime = runtime;
|
|
179
|
+
for (const [name, provider] of Object.entries(config.providers ?? {}))
|
|
180
|
+
this.registerDecisionProvider(name, provider);
|
|
181
|
+
}
|
|
182
|
+
registerDecisionProvider(name, provider) {
|
|
183
|
+
if (!name.trim())
|
|
184
|
+
throw new DecisionError('A decision provider needs a name', 'DECISION_INVALID');
|
|
185
|
+
this.providers.set(name.trim(), provider);
|
|
186
|
+
return this;
|
|
187
|
+
}
|
|
188
|
+
hasDecisionProvider(name) {
|
|
189
|
+
return this.providers.has(name);
|
|
190
|
+
}
|
|
191
|
+
listDecisionProviders() {
|
|
192
|
+
return [...this.providers.keys()];
|
|
193
|
+
}
|
|
194
|
+
async decide(request) {
|
|
195
|
+
request.signal?.throwIfAborted();
|
|
196
|
+
const questions = normalizeQuestions(request.questions);
|
|
197
|
+
const chosen = request.provider ?? this.config.defaultProvider ?? this.providers.keys().next().value;
|
|
198
|
+
if (chosen === undefined)
|
|
199
|
+
throw new DecisionError('No decision provider is registered', 'DECISION_NO_PROVIDER');
|
|
200
|
+
const order = [chosen, ...(this.config.fallbacks ?? []).filter((name) => name !== chosen)];
|
|
201
|
+
let last;
|
|
202
|
+
for (const [position, name] of order.entries()) {
|
|
203
|
+
const provider = this.providers.get(name);
|
|
204
|
+
if (!provider)
|
|
205
|
+
throw new DecisionError(`No decision provider is registered as "${name}"`, 'DECISION_NO_PROVIDER');
|
|
206
|
+
const unfit = unsupported(provider, questions);
|
|
207
|
+
if (unfit) {
|
|
208
|
+
if (position > 0)
|
|
209
|
+
continue;
|
|
210
|
+
throw new DecisionError(`Decision provider "${name}" ${unfit}`, 'DECISION_UNSUPPORTED');
|
|
211
|
+
}
|
|
212
|
+
try {
|
|
213
|
+
return (await this.ask(name, provider, request, questions));
|
|
214
|
+
}
|
|
215
|
+
catch (error) {
|
|
216
|
+
last = error;
|
|
217
|
+
if (request.signal?.aborted || !retryable(error))
|
|
218
|
+
throw error;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
throw last;
|
|
222
|
+
}
|
|
223
|
+
async ask(name, provider, request, questions) {
|
|
224
|
+
const model = request.model ?? provider.info.capabilities.defaultModel;
|
|
225
|
+
const started = Date.now();
|
|
226
|
+
const cache = request.cache === false ? undefined : this.config.cache;
|
|
227
|
+
const key = cache
|
|
228
|
+
? `nexus:decision:${sha256Hex(createCacheKey({ name, model, tenant: request.tenantId, state: request.state, questions }))}`
|
|
229
|
+
: undefined;
|
|
230
|
+
if (cache && key) {
|
|
231
|
+
const hit = await cache.store.get(key);
|
|
232
|
+
if (hit) {
|
|
233
|
+
return {
|
|
234
|
+
answers: hit.answers,
|
|
235
|
+
provider: hit.provider,
|
|
236
|
+
model: hit.model,
|
|
237
|
+
...(hit.usage ? { usage: hit.usage } : {}),
|
|
238
|
+
latencyMs: Date.now() - started,
|
|
239
|
+
cached: true,
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
const requestId = generateRequestId();
|
|
244
|
+
const charged = provider.info.chargedByItsCalls === true;
|
|
245
|
+
const run = this.runtime.tracer?.startRun({
|
|
246
|
+
name: `decide ${Object.keys(questions).join(', ')}`,
|
|
247
|
+
kind: 'decision',
|
|
248
|
+
inputs: { state: request.state, questions },
|
|
249
|
+
model,
|
|
250
|
+
provider: name,
|
|
251
|
+
...(request.metadata ? { metadata: request.metadata } : {}),
|
|
252
|
+
});
|
|
253
|
+
const call = () => withLifecycle(this.runtime.lifecycle, {
|
|
254
|
+
family: 'decision',
|
|
255
|
+
operation: 'decision.decide',
|
|
256
|
+
requestId,
|
|
257
|
+
provider: name,
|
|
258
|
+
model,
|
|
259
|
+
...(request.tenantId !== undefined ? { tenantId: request.tenantId } : {}),
|
|
260
|
+
metadata: { questions: Object.keys(questions).length, ...request.metadata },
|
|
261
|
+
}, request.signal ? { signal: request.signal } : {}, async (ticket) => {
|
|
262
|
+
const estimate = charged
|
|
263
|
+
? undefined
|
|
264
|
+
: priceDecision(model, { inputTokens: Math.ceil(JSON.stringify({ state: request.state, questions }).length / 4) }, this.prices());
|
|
265
|
+
if (ticket && estimate)
|
|
266
|
+
await ticket.reserve(estimate.amount);
|
|
267
|
+
const answered = await provider.decide({ model, state: request.state, questions }, {
|
|
268
|
+
signal: ticket?.context.signal ?? request.signal ?? new AbortController().signal,
|
|
269
|
+
requestId,
|
|
270
|
+
...(request.tenantId !== undefined ? { tenantId: request.tenantId } : {}),
|
|
271
|
+
...(ticket?.context.credential ? { credential: ticket.context.credential } : {}),
|
|
272
|
+
});
|
|
273
|
+
const answers = {};
|
|
274
|
+
for (const [question, asked] of Object.entries(questions)) {
|
|
275
|
+
const distribution = answered.distributions[question];
|
|
276
|
+
if (!distribution) {
|
|
277
|
+
throw new DecisionError(`Decision provider "${name}" did not answer "${question}"`, 'DECISION_FAILED');
|
|
278
|
+
}
|
|
279
|
+
answers[question] = decisionAnswer(asked, distribution);
|
|
280
|
+
}
|
|
281
|
+
const resolved = answered.model ?? model;
|
|
282
|
+
const cost = answered.cost ?? priceDecision(resolved, answered.usage, this.prices());
|
|
283
|
+
return {
|
|
284
|
+
answers,
|
|
285
|
+
provider: name,
|
|
286
|
+
model: resolved,
|
|
287
|
+
...(answered.usage ? { usage: answered.usage } : {}),
|
|
288
|
+
...(cost ? { cost } : {}),
|
|
289
|
+
latencyMs: Date.now() - started,
|
|
290
|
+
};
|
|
291
|
+
}, (result) => result.cost && !charged ? { cost: result.cost.amount, model: result.model } : { model: result.model });
|
|
292
|
+
try {
|
|
293
|
+
const result = run && this.runtime.tracer ? await this.runtime.tracer.within(run, call) : await call();
|
|
294
|
+
await run?.finish({
|
|
295
|
+
outputs: result.answers,
|
|
296
|
+
model: result.model,
|
|
297
|
+
provider: name,
|
|
298
|
+
...(result.usage ? { usage: numbers(result.usage) } : {}),
|
|
299
|
+
...(result.cost ? { cost: result.cost.amount } : {}),
|
|
300
|
+
});
|
|
301
|
+
if (cache && key) {
|
|
302
|
+
await cache.store.set(key, {
|
|
303
|
+
answers: result.answers,
|
|
304
|
+
provider: result.provider,
|
|
305
|
+
model: result.model,
|
|
306
|
+
...(result.usage ? { usage: result.usage } : {}),
|
|
307
|
+
}, cache.ttlSeconds ?? 86_400);
|
|
308
|
+
}
|
|
309
|
+
return result;
|
|
310
|
+
}
|
|
311
|
+
catch (error) {
|
|
312
|
+
await run?.finish({ error });
|
|
313
|
+
throw error;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
prices() {
|
|
317
|
+
return this.config.prices ? { ...DECISION_PRICES, ...this.config.prices } : DECISION_PRICES;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
export async function decide(provider, request) {
|
|
321
|
+
return new DecisionManager({ providers: { [provider.info.name]: provider } }).decide(request);
|
|
322
|
+
}
|
|
323
|
+
export async function runDecisionConformance(provider, options = {}) {
|
|
324
|
+
const report = { provider: provider.info.name, passed: [], failures: [] };
|
|
325
|
+
const check = async (name, run) => {
|
|
326
|
+
try {
|
|
327
|
+
await run();
|
|
328
|
+
report.passed.push(name);
|
|
329
|
+
}
|
|
330
|
+
catch (error) {
|
|
331
|
+
report.failures.push(`${name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
332
|
+
}
|
|
333
|
+
};
|
|
334
|
+
const capabilities = provider.info.capabilities;
|
|
335
|
+
const ask = (state, questions) => decide(provider, { state, questions, ...(options.model ? { model: options.model } : {}) });
|
|
336
|
+
await check('capabilities', () => {
|
|
337
|
+
if (!provider.info.name.trim())
|
|
338
|
+
throw new Error('the provider has no name');
|
|
339
|
+
if (capabilities.types.length === 0)
|
|
340
|
+
throw new Error('it answers no question type');
|
|
341
|
+
if (capabilities.types.includes('choice') && !(capabilities.maxOptions >= 2))
|
|
342
|
+
throw new Error('a choice needs two options');
|
|
343
|
+
if (capabilities.types.includes('score') && !(capabilities.maxLevels >= 2))
|
|
344
|
+
throw new Error('a score needs two levels');
|
|
345
|
+
if (!capabilities.defaultModel.trim())
|
|
346
|
+
throw new Error('it names no default model');
|
|
347
|
+
});
|
|
348
|
+
const state = 'The invoice INV-77 for $120 was paid in full on 2026-10-01, a week before it was due.';
|
|
349
|
+
const samples = {
|
|
350
|
+
'yes-no': {
|
|
351
|
+
type: 'yes-no',
|
|
352
|
+
question: 'Was the invoice paid?',
|
|
353
|
+
criteria: { yes: 'Paid in full', no: 'Unpaid or partly paid' },
|
|
354
|
+
},
|
|
355
|
+
choice: {
|
|
356
|
+
type: 'choice',
|
|
357
|
+
question: 'When was it paid?',
|
|
358
|
+
options: { early: 'Before it was due', late: 'After it was due' },
|
|
359
|
+
},
|
|
360
|
+
score: { type: 'score', question: 'How much of it was paid?', levels: ['None', 'Some', 'All'] },
|
|
361
|
+
};
|
|
362
|
+
for (const type of capabilities.types) {
|
|
363
|
+
await check(`a ${type} question`, async () => {
|
|
364
|
+
const question = samples[type];
|
|
365
|
+
const { answers } = await ask(state, { asked: question });
|
|
366
|
+
const answer = answers.asked;
|
|
367
|
+
if (answer.type !== type)
|
|
368
|
+
throw new Error(`answered as a ${answer.type}`);
|
|
369
|
+
if (!(answer.confidence >= 0 && answer.confidence <= 1))
|
|
370
|
+
throw new Error(`confidence ${answer.confidence}`);
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
await check('several questions in one decision', async () => {
|
|
374
|
+
const questions = Object.fromEntries(capabilities.types.map((type) => [`q-${type}`, samples[type]]));
|
|
375
|
+
const { answers } = await ask(state, questions);
|
|
376
|
+
for (const name of Object.keys(questions))
|
|
377
|
+
if (!answers[name])
|
|
378
|
+
throw new Error(`"${name}" got no answer`);
|
|
379
|
+
});
|
|
380
|
+
await check('a stopped decision is not answered', async () => {
|
|
381
|
+
const controller = new AbortController();
|
|
382
|
+
controller.abort(new Error('stopped'));
|
|
383
|
+
const question = samples[capabilities.types[0] ?? 'yes-no'];
|
|
384
|
+
const answered = await provider
|
|
385
|
+
.decide({
|
|
386
|
+
model: options.model ?? capabilities.defaultModel,
|
|
387
|
+
state,
|
|
388
|
+
questions: normalizeQuestions({ asked: question }),
|
|
389
|
+
}, { signal: controller.signal, requestId: generateRequestId() })
|
|
390
|
+
.then(() => true, () => false);
|
|
391
|
+
if (answered)
|
|
392
|
+
throw new Error('it answered after the call was stopped');
|
|
393
|
+
});
|
|
394
|
+
for (const [index, item] of (options.cases ?? []).entries()) {
|
|
395
|
+
await check(`case ${index + 1}`, async () => {
|
|
396
|
+
const { answers } = await ask(item.state, { asked: item.question });
|
|
397
|
+
const answer = answers.asked;
|
|
398
|
+
if (!answeredRight(answer, item.expected)) {
|
|
399
|
+
const given = answer.type === 'yes-no' ? answer.yes : answer.type === 'choice' ? answer.choice : answer.level;
|
|
400
|
+
throw new Error(`expected ${JSON.stringify(item.expected)}, answered ${JSON.stringify(given)}`);
|
|
401
|
+
}
|
|
402
|
+
});
|
|
403
|
+
}
|
|
404
|
+
return report;
|
|
405
|
+
}
|
|
406
|
+
function normalizeQuestions(questions) {
|
|
407
|
+
const names = Object.keys(questions ?? {});
|
|
408
|
+
if (names.length === 0)
|
|
409
|
+
throw new DecisionError('A decision needs at least one question', 'DECISION_INVALID');
|
|
410
|
+
const normalized = {};
|
|
411
|
+
for (const name of names) {
|
|
412
|
+
const question = questions[name];
|
|
413
|
+
if (!name.trim())
|
|
414
|
+
throw new DecisionError('A question needs a name', 'DECISION_INVALID');
|
|
415
|
+
if (question?.type === 'yes-no') {
|
|
416
|
+
normalized[name] = question;
|
|
417
|
+
}
|
|
418
|
+
else if (question?.type === 'choice') {
|
|
419
|
+
const options = Array.isArray(question.options)
|
|
420
|
+
? question.options.map((option) => ({ name: option }))
|
|
421
|
+
: Object.entries(question.options).map(([option, description]) => ({
|
|
422
|
+
name: option,
|
|
423
|
+
...(description ? { description } : {}),
|
|
424
|
+
}));
|
|
425
|
+
if (options.length < 2)
|
|
426
|
+
throw new DecisionError(`Choice "${name}" needs at least two options`, 'DECISION_INVALID');
|
|
427
|
+
if (new Set(options.map((option) => option.name)).size !== options.length) {
|
|
428
|
+
throw new DecisionError(`Choice "${name}" names an option twice`, 'DECISION_INVALID');
|
|
429
|
+
}
|
|
430
|
+
if (options.some((option) => !option.name.trim())) {
|
|
431
|
+
throw new DecisionError(`Choice "${name}" has an option without a name`, 'DECISION_INVALID');
|
|
432
|
+
}
|
|
433
|
+
normalized[name] = { type: 'choice', ...(question.question ? { question: question.question } : {}), options };
|
|
434
|
+
}
|
|
435
|
+
else if (question?.type === 'score') {
|
|
436
|
+
if (!Array.isArray(question.levels) || question.levels.length < 2) {
|
|
437
|
+
throw new DecisionError(`Score "${name}" needs at least two levels`, 'DECISION_INVALID');
|
|
438
|
+
}
|
|
439
|
+
normalized[name] = question;
|
|
440
|
+
}
|
|
441
|
+
else {
|
|
442
|
+
throw new DecisionError(`Question "${name}" is a yes-no, a choice, or a score, not ${JSON.stringify(question?.type)}`, 'DECISION_INVALID');
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
return normalized;
|
|
446
|
+
}
|
|
447
|
+
function unsupported(provider, questions) {
|
|
448
|
+
const capabilities = provider.info.capabilities;
|
|
449
|
+
const count = Object.keys(questions).length;
|
|
450
|
+
if (capabilities.maxQuestions !== undefined && count > capabilities.maxQuestions) {
|
|
451
|
+
return `takes at most ${capabilities.maxQuestions} questions, not ${count}`;
|
|
452
|
+
}
|
|
453
|
+
for (const [name, question] of Object.entries(questions)) {
|
|
454
|
+
if (!capabilities.types.includes(question.type))
|
|
455
|
+
return `does not answer ${question.type} questions, such as "${name}"`;
|
|
456
|
+
if (question.type === 'choice' && question.options.length > capabilities.maxOptions) {
|
|
457
|
+
return `takes at most ${capabilities.maxOptions} options, not ${question.options.length} in "${name}"`;
|
|
458
|
+
}
|
|
459
|
+
if (question.type === 'score' && question.levels.length > capabilities.maxLevels) {
|
|
460
|
+
return `takes at most ${capabilities.maxLevels} levels, not ${question.levels.length} in "${name}"`;
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
return undefined;
|
|
464
|
+
}
|
|
465
|
+
function retryable(error) {
|
|
466
|
+
return error instanceof DecisionError ? error.retryable : true;
|
|
467
|
+
}
|
|
468
|
+
function probabilityOf(value, what) {
|
|
469
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
|
470
|
+
throw new DecisionError(`The probability of ${what} is ${String(value)}, not a number from 0 to 1`, 'DECISION_INVALID');
|
|
471
|
+
}
|
|
472
|
+
return value;
|
|
473
|
+
}
|
|
474
|
+
function normalize(weights) {
|
|
475
|
+
const total = weights.reduce((sum, weight) => sum + weight, 0);
|
|
476
|
+
if (total <= 0)
|
|
477
|
+
return weights.map(() => 1 / weights.length);
|
|
478
|
+
return weights.map((weight) => weight / total);
|
|
479
|
+
}
|
|
480
|
+
function argmax(values) {
|
|
481
|
+
let best = 0;
|
|
482
|
+
for (let index = 1; index < values.length; index += 1) {
|
|
483
|
+
if ((values[index] ?? -Infinity) > (values[best] ?? -Infinity))
|
|
484
|
+
best = index;
|
|
485
|
+
}
|
|
486
|
+
return best;
|
|
487
|
+
}
|
|
488
|
+
function clamp(value) {
|
|
489
|
+
return Math.min(1, Math.max(0, value));
|
|
490
|
+
}
|
|
491
|
+
function numbers(usage) {
|
|
492
|
+
return Object.fromEntries(Object.entries(usage).filter((entry) => typeof entry[1] === 'number'));
|
|
493
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typed decisions: questions a model answers with a yes or no, a choice, or a score, each with its
|
|
3
|
+
* probabilities and a confidence read from them, so software acts on an answer without parsing text.
|
|
4
|
+
* The contract, its manager, gates, and calibration; decisions from any completion model; and
|
|
5
|
+
* decisions in graphs, agents, and evaluations.
|
|
6
|
+
*/
|
|
7
|
+
export { type CompletionDecisionMethod, completionDecisions, type CompletionDecisionsOptions, type DecisionCompletionClient, } from './completion.js';
|
|
8
|
+
export { decisionEvaluator, type DecisionEvaluatorOptions, decisionGuard, type DecisionGuardOptions, type DecisionGuardReview, type DecisionManagerLike, decisionModel, type DecisionModelOptions, decisionPermissions, type DecisionPermissionsOptions, type DecisionRequestOptions, decisionRoute, type DecisionRouteOptions, reviewUnsure, } from './integrations.js';
|
|
9
|
+
export { answeredRight, type AnswerOf, answerProbability, type CachedDecision, calibrate, type CalibrateOptions, type CalibrationBin, type CalibrationReport, type CalibrationSample, type ChoiceAnswer, choiceConfidence, type ChoiceQuestion, decide, DECISION_PRICES, decisionAnswer, type DecisionAnswer, type DecisionAnswers, type DecisionCallContext, type DecisionConfig, type DecisionConformanceCase, type DecisionConformanceOptions, type DecisionConformanceReport, type DecisionDistribution, DecisionError, type DecisionErrorCode, DecisionManager, type DecisionPrice, type DecisionProvider, type DecisionProviderCapabilities, type DecisionProviderInfo, type DecisionProviderQuestion, type DecisionProviderRequest, type DecisionProviderResult, type DecisionQuestion, type DecisionQuestions, type DecisionQuestionType, type DecisionRequest, type DecisionResult, type DecisionRunLike, type DecisionRuntime, type DecisionState, type DecisionTracerLike, type DecisionUsage, gate, type GateOutcome, type GateThresholds, priceDecision, runDecisionConformance, type ScoreAnswer, scoreConfidence, type ScoreQuestion, type YesNoAnswer, yesNoConfidence, type YesNoQuestion, } from './core.js';
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
export { completionDecisions, } from './completion.js';
|
|
2
|
+
export { decisionEvaluator, decisionGuard, decisionModel, decisionPermissions, decisionRoute, reviewUnsure, } from './integrations.js';
|
|
3
|
+
export { answeredRight, answerProbability, calibrate, choiceConfidence, decide, DECISION_PRICES, decisionAnswer, DecisionError, DecisionManager, gate, priceDecision, runDecisionConformance, scoreConfidence, yesNoConfidence, } from './core.js';
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decisions where the package already decides: a graph's branch, an agent's approval, a guardrail, a
|
|
3
|
+
* model's choice, a judge's score, and an evaluation sent to people. A confident answer acts, and an
|
|
4
|
+
* uncertain one goes to a person.
|
|
5
|
+
*/
|
|
6
|
+
import type { AgentMiddleware } from '../agent/create-agent.js';
|
|
7
|
+
import type { PermissionPolicyLike, PermissionRequest } from '../agent/permissions.js';
|
|
8
|
+
import type { EvaluationContext, EvaluationScore, Evaluator } from '../types/evaluate.js';
|
|
9
|
+
import type { ChoiceAnswer, ChoiceQuestion, DecisionQuestions, DecisionRequest, DecisionResult, DecisionState, ScoreQuestion, YesNoAnswer, YesNoQuestion } from './core.js';
|
|
10
|
+
/** Anything that answers decisions, such as a `DecisionManager`. */
|
|
11
|
+
export interface DecisionManagerLike {
|
|
12
|
+
/** Answers every question of a decision. */
|
|
13
|
+
decide<const Q extends DecisionQuestions>(request: DecisionRequest<Q>): Promise<DecisionResult<Q>>;
|
|
14
|
+
}
|
|
15
|
+
/** The parts of a decision request an integration takes from the application: whom and what to ask. */
|
|
16
|
+
export type DecisionRequestOptions = Partial<Pick<DecisionRequest, 'provider' | 'model' | 'tenantId' | 'metadata'>>;
|
|
17
|
+
/** Options for `decisionRoute()`. */
|
|
18
|
+
export interface DecisionRouteOptions<S> {
|
|
19
|
+
/** What the question is about, read from the graph's state. */
|
|
20
|
+
state: (graph: S) => DecisionState;
|
|
21
|
+
/** A choice whose options name the nodes to go to, or a yes or no; `routes` maps them otherwise. */
|
|
22
|
+
question: ChoiceQuestion | YesNoQuestion;
|
|
23
|
+
/** Where each answer goes, by option, or by `yes` and `no`, when they are not node names. */
|
|
24
|
+
routes?: Readonly<Record<string, string>>;
|
|
25
|
+
/** At or above this confidence, the answer routes; below it, the run goes to `review`. */
|
|
26
|
+
act: number;
|
|
27
|
+
/** The node an uncertain answer goes to, such as one that asks a person with `interrupt()`. */
|
|
28
|
+
review: string;
|
|
29
|
+
/** Whom and what to ask, from the graph's state, such as its tenant. */
|
|
30
|
+
request?: (graph: S) => DecisionRequestOptions;
|
|
31
|
+
/** Hears each answer and the node it chose, such as to record it. */
|
|
32
|
+
onAnswer?: (answer: ChoiceAnswer | YesNoAnswer, node: string) => void;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* A graph router that asks a decision, for `addConditionalEdges()`: a confident answer goes to its
|
|
36
|
+
* node, and an uncertain one to `review`, where a person can answer instead.
|
|
37
|
+
*
|
|
38
|
+
* ```ts
|
|
39
|
+
* graph.addConditionalEdges('triage', decisionRoute(decisions, {
|
|
40
|
+
* state: (s) => s.ticket,
|
|
41
|
+
* question: { type: 'choice', options: { billing: 'Charges', technical: 'Errors' } },
|
|
42
|
+
* act: 0.8,
|
|
43
|
+
* review: 'ask-a-person',
|
|
44
|
+
* }));
|
|
45
|
+
* ```
|
|
46
|
+
*/
|
|
47
|
+
export declare function decisionRoute<S>(decisions: DecisionManagerLike, options: DecisionRouteOptions<S>): (graph: S) => Promise<string>;
|
|
48
|
+
/** Options for `decisionPermissions()`. */
|
|
49
|
+
export interface DecisionPermissionsOptions {
|
|
50
|
+
/**
|
|
51
|
+
* The policy that decides first: what it allows or refuses stands, and only the calls it would ask a
|
|
52
|
+
* person about are asked as a decision. Without one, every call is.
|
|
53
|
+
*/
|
|
54
|
+
base?: PermissionPolicyLike;
|
|
55
|
+
/** The question asked of a call: yes runs it. Defaults to whether it is safe to run without a person checking. */
|
|
56
|
+
question?: YesNoQuestion;
|
|
57
|
+
/** What the question is about. Defaults to the tool, its arguments, and what it declares. */
|
|
58
|
+
state?: (request: PermissionRequest) => DecisionState;
|
|
59
|
+
/** A yes this sure runs the call, and a no this sure refuses it; anything less asks a person. */
|
|
60
|
+
act: number;
|
|
61
|
+
/** Whom and what to ask. Defaults to the principal's tenant. */
|
|
62
|
+
request?: (request: PermissionRequest) => DecisionRequestOptions;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* A permission policy that asks a decision where a person would otherwise be asked: a confident yes
|
|
66
|
+
* runs the call, a confident no refuses it, and anything less still asks the person.
|
|
67
|
+
*/
|
|
68
|
+
export declare function decisionPermissions(decisions: DecisionManagerLike, options: DecisionPermissionsOptions): PermissionPolicyLike;
|
|
69
|
+
/** What a person answers when a guard is unsure: let it pass, or not. */
|
|
70
|
+
export interface DecisionGuardReview {
|
|
71
|
+
/** True lets the message, or the answer, go on. */
|
|
72
|
+
allow: boolean;
|
|
73
|
+
}
|
|
74
|
+
/** Options for `decisionGuard()`. */
|
|
75
|
+
export interface DecisionGuardOptions {
|
|
76
|
+
/** Asked of each new message from the user, before the model sees it: a yes stops the run. */
|
|
77
|
+
input?: YesNoQuestion;
|
|
78
|
+
/** Asked of each answer, before it is given: a yes withholds it. */
|
|
79
|
+
output?: YesNoQuestion;
|
|
80
|
+
/** A yes this sure stops, and a no this sure passes; anything between asks a person. */
|
|
81
|
+
act: number;
|
|
82
|
+
/** What the agent answers in place of what the guard stopped. Defaults to "I can't help with that." */
|
|
83
|
+
refusal?: string;
|
|
84
|
+
/** Whom and what to ask. Defaults to the run's tenant. */
|
|
85
|
+
request?: DecisionRequestOptions;
|
|
86
|
+
/** Hears each verdict, without the text it judged. */
|
|
87
|
+
onVerdict?: (verdict: {
|
|
88
|
+
where: 'input' | 'output';
|
|
89
|
+
outcome: 'passed' | 'stopped' | 'reviewed';
|
|
90
|
+
probability: number;
|
|
91
|
+
}) => void;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Agent middleware that guards each message and answer with a yes-or-no decision, such as whether a
|
|
95
|
+
* message is a prompt injection or an answer leaks what it must not. A confident yes stops it, a
|
|
96
|
+
* confident no passes it, and in between the run asks a person, who answers a `DecisionGuardReview`.
|
|
97
|
+
*/
|
|
98
|
+
export declare function decisionGuard(decisions: DecisionManagerLike, options: DecisionGuardOptions): AgentMiddleware;
|
|
99
|
+
/** Options for `decisionModel()`. */
|
|
100
|
+
export interface DecisionModelOptions<K extends string = string> {
|
|
101
|
+
/** A choice among kinds of request, asked of each new message from the user. */
|
|
102
|
+
question: ChoiceQuestion<K>;
|
|
103
|
+
/** The model each kind goes to. */
|
|
104
|
+
models: Readonly<Record<K, string>>;
|
|
105
|
+
/** At or above this confidence the request goes to its kind's model; below, it keeps its own. */
|
|
106
|
+
act: number;
|
|
107
|
+
/** Whom and what to ask. Defaults to the run's tenant. */
|
|
108
|
+
request?: DecisionRequestOptions;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Agent middleware that sends each turn to a model by a decision about the user's message: a quick
|
|
112
|
+
* question to a fast model, a hard one to a strong one. Every model call of the turn goes to the model
|
|
113
|
+
* its message chose, and an uncertain answer keeps the agent's own model.
|
|
114
|
+
*/
|
|
115
|
+
export declare function decisionModel<const K extends string>(decisions: DecisionManagerLike, options: DecisionModelOptions<K>): AgentMiddleware;
|
|
116
|
+
/** Options for `decisionEvaluator()`. */
|
|
117
|
+
export interface DecisionEvaluatorOptions<I = unknown, O = unknown> {
|
|
118
|
+
/** The score's key. */
|
|
119
|
+
key: string;
|
|
120
|
+
/** The question: a yes or no, a score, or a choice. */
|
|
121
|
+
question: YesNoQuestion | ScoreQuestion | ChoiceQuestion;
|
|
122
|
+
/** What the question is about. Defaults to the example's inputs, the output, and what was expected. */
|
|
123
|
+
state?: (context: EvaluationContext<I, O>) => DecisionState;
|
|
124
|
+
/**
|
|
125
|
+
* What passes: for a yes or no, `true` (the default) or `false`; for a score, the lowest level that
|
|
126
|
+
* passes (the top one by default); for a choice, the option that does.
|
|
127
|
+
*/
|
|
128
|
+
passing?: boolean | number | string;
|
|
129
|
+
/** Whom and what to ask. */
|
|
130
|
+
request?: DecisionRequestOptions;
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* An evaluator that asks a decision: the score is the probability of yes, the score's level as a
|
|
134
|
+
* share of the top one, or the passing option's probability, and its metadata carries the answer's
|
|
135
|
+
* confidence and probabilities, so an uncertain score can go to people with `reviewUnsure()`.
|
|
136
|
+
*/
|
|
137
|
+
export declare function decisionEvaluator<I = unknown, O = unknown>(decisions: DecisionManagerLike, options: DecisionEvaluatorOptions<I, O>): Evaluator<I, O>;
|
|
138
|
+
/**
|
|
139
|
+
* A `reviewWhen` for online evaluation that sends a run to people when any of its decision scores,
|
|
140
|
+
* or the one `key` names, is less sure than `threshold`.
|
|
141
|
+
*/
|
|
142
|
+
export declare function reviewUnsure(threshold: number, key?: string): (scores: EvaluationScore[]) => boolean;
|