@graphorin/provider 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/README.md +8 -3
- package/dist/adapters/llamacpp-server.d.ts +1 -1
- package/dist/adapters/llamacpp-server.js.map +1 -1
- package/dist/adapters/ollama.d.ts.map +1 -1
- package/dist/adapters/ollama.js +19 -6
- package/dist/adapters/ollama.js.map +1 -1
- package/dist/adapters/vercel.d.ts +1 -1
- package/dist/adapters/vercel.d.ts.map +1 -1
- package/dist/adapters/vercel.js +111 -33
- package/dist/adapters/vercel.js.map +1 -1
- package/dist/errors/errors.d.ts +1 -1
- package/dist/errors/errors.js +1 -1
- package/dist/errors/errors.js.map +1 -1
- package/dist/index.d.ts +1 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -8
- package/dist/index.js.map +1 -1
- package/dist/internal/http.js +111 -7
- package/dist/internal/http.js.map +1 -1
- package/dist/internal/openai-shaped.js +21 -4
- package/dist/internal/openai-shaped.js.map +1 -1
- package/dist/middleware/with-fallback.js +1 -1
- package/dist/middleware/with-rate-limit.d.ts +21 -8
- package/dist/middleware/with-rate-limit.d.ts.map +1 -1
- package/dist/middleware/with-rate-limit.js +65 -12
- package/dist/middleware/with-rate-limit.js.map +1 -1
- package/dist/middleware/with-retry.js +1 -1
- package/dist/package.js +6 -0
- package/dist/package.js.map +1 -0
- package/package.json +18 -16
- package/src/adapters/index.ts +14 -0
- package/src/adapters/llamacpp-server.ts +102 -0
- package/src/adapters/ollama.ts +382 -0
- package/src/adapters/openai-compatible.ts +95 -0
- package/src/adapters/vercel-messages.ts +308 -0
- package/src/adapters/vercel.ts +706 -0
- package/src/counters/anthropic-wire.ts +199 -0
- package/src/counters/anthropic.ts +114 -0
- package/src/counters/bedrock.ts +46 -0
- package/src/counters/dispatcher.ts +127 -0
- package/src/counters/global.ts +39 -0
- package/src/counters/google.ts +46 -0
- package/src/counters/heuristic.ts +107 -0
- package/src/counters/index.ts +35 -0
- package/src/counters/js-tiktoken.ts +135 -0
- package/src/counters/serialize.ts +85 -0
- package/src/errors/errors.ts +316 -0
- package/src/errors/index.ts +20 -0
- package/src/index.ts +42 -0
- package/src/internal/abort.ts +30 -0
- package/src/internal/http.ts +388 -0
- package/src/internal/openai-shaped.ts +555 -0
- package/src/internal/sse.ts +112 -0
- package/src/internal/url-utils.ts +20 -0
- package/src/middleware/compose.ts +213 -0
- package/src/middleware/index.ts +37 -0
- package/src/middleware/production-hook.ts +47 -0
- package/src/middleware/with-cost-limit.ts +131 -0
- package/src/middleware/with-cost-tracking.ts +216 -0
- package/src/middleware/with-fallback.ts +157 -0
- package/src/middleware/with-rate-limit.ts +306 -0
- package/src/middleware/with-redaction.ts +671 -0
- package/src/middleware/with-retry.ts +274 -0
- package/src/middleware/with-tracing.ts +117 -0
- package/src/model-tier/classify.ts +125 -0
- package/src/model-tier/index.ts +11 -0
- package/src/provider.ts +121 -0
- package/src/reasoning/apply-policy.ts +89 -0
- package/src/reasoning/classify-contract.ts +120 -0
- package/src/reasoning/index.ts +21 -0
- package/src/reasoning/retention.ts +64 -0
- package/src/tool-examples.ts +54 -0
- package/src/trust/classify-local-provider.ts +254 -0
- package/src/trust/index.ts +14 -0
- package/dist/adapters/index.js +0 -6
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `withRetry` - retry idempotent failures (5xx, network, timeouts)
|
|
3
|
+
* with exponential backoff + jitter. Lives outside `withRateLimit`
|
|
4
|
+
* per the canonical order so retried requests respect the rate-limit
|
|
5
|
+
* bucket of the next attempt.
|
|
6
|
+
*
|
|
7
|
+
* @packageDocumentation
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { Provider, ProviderEvent, ProviderRequest, ProviderResponse } from '@graphorin/core';
|
|
11
|
+
|
|
12
|
+
import { isAbortError } from '../internal/abort.js';
|
|
13
|
+
import { defineProviderMiddleware } from './compose.js';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Options for {@link withRetry}.
|
|
17
|
+
*
|
|
18
|
+
* @stable
|
|
19
|
+
*/
|
|
20
|
+
export interface WithRetryOptions {
|
|
21
|
+
readonly maxRetries?: number;
|
|
22
|
+
readonly backoff?: 'exponential' | 'linear' | 'constant';
|
|
23
|
+
readonly initialDelayMs?: number;
|
|
24
|
+
readonly maxDelayMs?: number;
|
|
25
|
+
readonly jitter?: boolean;
|
|
26
|
+
readonly retryableErrors?: (err: unknown) => boolean;
|
|
27
|
+
/** Optional sleep override (test fixtures use a synchronous resolver). */
|
|
28
|
+
readonly sleepImpl?: (ms: number, signal?: AbortSignal) => Promise<void>;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* @stable
|
|
33
|
+
*/
|
|
34
|
+
export const withRetry = defineProviderMiddleware<WithRetryOptions>({
|
|
35
|
+
kind: 'withRetry',
|
|
36
|
+
factory: (opts: WithRetryOptions) => {
|
|
37
|
+
const maxRetries = opts.maxRetries ?? 3;
|
|
38
|
+
const backoff = opts.backoff ?? 'exponential';
|
|
39
|
+
const initialDelay = opts.initialDelayMs ?? 250;
|
|
40
|
+
const maxDelay = opts.maxDelayMs ?? 30_000;
|
|
41
|
+
const jitter = opts.jitter ?? true;
|
|
42
|
+
const isRetryable = opts.retryableErrors ?? defaultRetryable;
|
|
43
|
+
const sleep = opts.sleepImpl ?? defaultSleep;
|
|
44
|
+
return (next: Provider): Provider => ({
|
|
45
|
+
name: next.name,
|
|
46
|
+
modelId: next.modelId,
|
|
47
|
+
capabilities: next.capabilities,
|
|
48
|
+
...(next.acceptsSensitivity !== undefined
|
|
49
|
+
? { acceptsSensitivity: next.acceptsSensitivity }
|
|
50
|
+
: {}),
|
|
51
|
+
stream(req) {
|
|
52
|
+
return retryingStream(
|
|
53
|
+
next,
|
|
54
|
+
req,
|
|
55
|
+
maxRetries,
|
|
56
|
+
backoff,
|
|
57
|
+
initialDelay,
|
|
58
|
+
maxDelay,
|
|
59
|
+
jitter,
|
|
60
|
+
isRetryable,
|
|
61
|
+
sleep,
|
|
62
|
+
);
|
|
63
|
+
},
|
|
64
|
+
async generate(req) {
|
|
65
|
+
let attempt = 0;
|
|
66
|
+
for (;;) {
|
|
67
|
+
try {
|
|
68
|
+
return await next.generate(req);
|
|
69
|
+
} catch (err) {
|
|
70
|
+
if (req.signal?.aborted) throw err;
|
|
71
|
+
if (attempt >= maxRetries) throw err;
|
|
72
|
+
if (!isRetryable(err)) throw err;
|
|
73
|
+
const hint = readRetryAfter(err);
|
|
74
|
+
const delay =
|
|
75
|
+
hint !== null
|
|
76
|
+
? Math.min(hint, maxDelay)
|
|
77
|
+
: computeDelay(backoff, initialDelay, attempt, maxDelay, jitter);
|
|
78
|
+
await sleep(delay, req.signal);
|
|
79
|
+
attempt++;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
...(next.countTokens ? { countTokens: next.countTokens.bind(next) } : {}),
|
|
84
|
+
});
|
|
85
|
+
},
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
async function* retryingStream(
|
|
89
|
+
next: Provider,
|
|
90
|
+
req: ProviderRequest,
|
|
91
|
+
maxRetries: number,
|
|
92
|
+
backoff: 'exponential' | 'linear' | 'constant',
|
|
93
|
+
initialDelay: number,
|
|
94
|
+
maxDelay: number,
|
|
95
|
+
jitter: boolean,
|
|
96
|
+
isRetryable: (err: unknown) => boolean,
|
|
97
|
+
sleep: (ms: number, signal?: AbortSignal) => Promise<void>,
|
|
98
|
+
): AsyncIterable<ProviderEvent> {
|
|
99
|
+
let attempt = 0;
|
|
100
|
+
// Stream-level (not per-attempt): once *any* event has reached the
|
|
101
|
+
// consumer, a retryable failure mid-stream must NOT restart the stream -
|
|
102
|
+
// that would replay stream-start + already-delivered text/tool-call events
|
|
103
|
+
// into the same iteration (duplicate output, potential double tool execution).
|
|
104
|
+
// `withFallback` upholds the same invariant. Pre-yield failures still retry.
|
|
105
|
+
let yieldedAny = false;
|
|
106
|
+
for (;;) {
|
|
107
|
+
try {
|
|
108
|
+
for await (const event of next.stream(req)) {
|
|
109
|
+
yieldedAny = true;
|
|
110
|
+
yield event;
|
|
111
|
+
}
|
|
112
|
+
if (yieldedAny) return;
|
|
113
|
+
// Empty stream - surface a finish so callers do not hang.
|
|
114
|
+
yield {
|
|
115
|
+
type: 'finish',
|
|
116
|
+
finishReason: 'stop',
|
|
117
|
+
usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 },
|
|
118
|
+
};
|
|
119
|
+
return;
|
|
120
|
+
} catch (err) {
|
|
121
|
+
if (req.signal?.aborted) throw err;
|
|
122
|
+
// PS-1: never restart a stream that already emitted events.
|
|
123
|
+
if (yieldedAny) throw err;
|
|
124
|
+
if (attempt >= maxRetries) throw err;
|
|
125
|
+
if (!isRetryable(err)) throw err;
|
|
126
|
+
const hint = readRetryAfter(err);
|
|
127
|
+
const delay =
|
|
128
|
+
hint !== null
|
|
129
|
+
? Math.min(hint, maxDelay)
|
|
130
|
+
: computeDelay(backoff, initialDelay, attempt, maxDelay, jitter);
|
|
131
|
+
await sleep(delay, req.signal);
|
|
132
|
+
attempt++;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Read a `Retry-After` hint from a thrown error. Recognises:
|
|
139
|
+
*
|
|
140
|
+
* - `RateLimitExceededError`-shaped errors carrying a `retryAfterMs`
|
|
141
|
+
* field (already milliseconds).
|
|
142
|
+
* - HTTP-shaped errors carrying a `headers['retry-after']` value
|
|
143
|
+
* (numeric seconds; we convert to milliseconds).
|
|
144
|
+
* - HTTP-shaped errors carrying a `retryAfterSeconds` numeric field.
|
|
145
|
+
*
|
|
146
|
+
* Returns the resolved delay in milliseconds or `null` when no hint
|
|
147
|
+
* is available.
|
|
148
|
+
*
|
|
149
|
+
* @internal
|
|
150
|
+
*/
|
|
151
|
+
function readRetryAfter(err: unknown): number | null {
|
|
152
|
+
if (err === null || typeof err !== 'object') return null;
|
|
153
|
+
const e = err as {
|
|
154
|
+
retryAfterMs?: number;
|
|
155
|
+
retryAfterSeconds?: number;
|
|
156
|
+
headers?: Record<string, string | undefined>;
|
|
157
|
+
};
|
|
158
|
+
if (
|
|
159
|
+
typeof e.retryAfterMs === 'number' &&
|
|
160
|
+
Number.isFinite(e.retryAfterMs) &&
|
|
161
|
+
e.retryAfterMs >= 0
|
|
162
|
+
) {
|
|
163
|
+
return e.retryAfterMs;
|
|
164
|
+
}
|
|
165
|
+
if (
|
|
166
|
+
typeof e.retryAfterSeconds === 'number' &&
|
|
167
|
+
Number.isFinite(e.retryAfterSeconds) &&
|
|
168
|
+
e.retryAfterSeconds >= 0
|
|
169
|
+
) {
|
|
170
|
+
return Math.round(e.retryAfterSeconds * 1000);
|
|
171
|
+
}
|
|
172
|
+
const headerValue =
|
|
173
|
+
e.headers?.['retry-after'] ?? e.headers?.['Retry-After'] ?? e.headers?.['RETRY-AFTER'];
|
|
174
|
+
if (typeof headerValue === 'string' && headerValue.length > 0) {
|
|
175
|
+
const seconds = Number(headerValue);
|
|
176
|
+
if (Number.isFinite(seconds) && seconds >= 0) return Math.round(seconds * 1000);
|
|
177
|
+
const httpDateMs = Date.parse(headerValue);
|
|
178
|
+
if (Number.isFinite(httpDateMs)) {
|
|
179
|
+
const delta = httpDateMs - Date.now();
|
|
180
|
+
return delta > 0 ? delta : 0;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
return null;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function defaultRetryable(err: unknown): boolean {
|
|
187
|
+
if (err === null || typeof err !== 'object') return false;
|
|
188
|
+
// An aborted request is never retryable, even when it surfaces as a
|
|
189
|
+
// `status: 0` network error (PS-2). The retry loop also short-circuits on
|
|
190
|
+
// `req.signal?.aborted`, but the predicate must exclude abort independently
|
|
191
|
+
// so an internally-aborted call is not retried.
|
|
192
|
+
if (isAbortError(err)) return false;
|
|
193
|
+
const e = err as { kind?: string; errorKind?: string; status?: number; name?: string };
|
|
194
|
+
// `ProviderHttpError.kind` is always 'provider-http'; the canonical
|
|
195
|
+
// mapped kind rides on `errorKind` - consult both.
|
|
196
|
+
const kinds = [e.kind, e.errorKind];
|
|
197
|
+
if (
|
|
198
|
+
kinds.includes('transient') ||
|
|
199
|
+
kinds.includes('rate-limit') ||
|
|
200
|
+
kinds.includes('rate-limit-exceeded') ||
|
|
201
|
+
kinds.includes('capacity')
|
|
202
|
+
) {
|
|
203
|
+
return true;
|
|
204
|
+
}
|
|
205
|
+
if (
|
|
206
|
+
kinds.includes('unauthorized') ||
|
|
207
|
+
kinds.includes('invalid-request') ||
|
|
208
|
+
kinds.includes('context-length') ||
|
|
209
|
+
kinds.includes('content-filter')
|
|
210
|
+
) {
|
|
211
|
+
return false;
|
|
212
|
+
}
|
|
213
|
+
if (typeof e.status === 'number' && e.status === 429) return true;
|
|
214
|
+
if (typeof e.status === 'number' && e.status >= 500 && e.status < 600) return true;
|
|
215
|
+
// PS-2: a `ProviderHttpError{ status: 0 }` is a fetch-level network failure
|
|
216
|
+
// (ECONNREFUSED, DNS, connection reset) - exactly the transient class
|
|
217
|
+
// `withRetry` documents. Retry it (abort already excluded above).
|
|
218
|
+
if (typeof e.status === 'number' && e.status === 0) return true;
|
|
219
|
+
return false;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
function computeDelay(
|
|
223
|
+
backoff: 'exponential' | 'linear' | 'constant',
|
|
224
|
+
initial: number,
|
|
225
|
+
attempt: number,
|
|
226
|
+
maxDelay: number,
|
|
227
|
+
jitter: boolean,
|
|
228
|
+
): number {
|
|
229
|
+
let delay: number;
|
|
230
|
+
switch (backoff) {
|
|
231
|
+
case 'exponential':
|
|
232
|
+
delay = initial * 2 ** attempt;
|
|
233
|
+
break;
|
|
234
|
+
case 'linear':
|
|
235
|
+
delay = initial * (attempt + 1);
|
|
236
|
+
break;
|
|
237
|
+
default:
|
|
238
|
+
delay = initial;
|
|
239
|
+
break;
|
|
240
|
+
}
|
|
241
|
+
delay = Math.min(delay, maxDelay);
|
|
242
|
+
if (jitter) {
|
|
243
|
+
const factor = 0.5 + Math.random() * 0.5; // 0.5..1.0
|
|
244
|
+
delay = Math.floor(delay * factor);
|
|
245
|
+
}
|
|
246
|
+
return delay;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function defaultSleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
250
|
+
return new Promise<void>((resolve, reject) => {
|
|
251
|
+
if (signal?.aborted) {
|
|
252
|
+
reject(signal.reason ?? new Error('aborted'));
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
const timer = setTimeout(() => {
|
|
256
|
+
cleanup();
|
|
257
|
+
resolve();
|
|
258
|
+
}, ms);
|
|
259
|
+
const onAbort = () => {
|
|
260
|
+
cleanup();
|
|
261
|
+
reject(signal?.reason ?? new Error('aborted'));
|
|
262
|
+
};
|
|
263
|
+
function cleanup(): void {
|
|
264
|
+
clearTimeout(timer);
|
|
265
|
+
signal?.removeEventListener('abort', onAbort);
|
|
266
|
+
}
|
|
267
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// Cast for return-type compatibility with the discriminated union in
|
|
272
|
+
// `Provider.generate(...)` (TypeScript has no concept of "exhaustive
|
|
273
|
+
// retry loop" so the empty exit path is unreachable).
|
|
274
|
+
export type _RetryProviderResponseGuard = ProviderResponse;
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `withTracing` - outermost middleware. Wraps every `provider.stream`
|
|
3
|
+
* / `provider.generate` call in a Graphorin OTel-shaped span so the
|
|
4
|
+
* span captures retry decisions, rate-limit waits, and the underlying
|
|
5
|
+
* provider call as one logical unit.
|
|
6
|
+
*
|
|
7
|
+
* The middleware accepts any object that exposes the
|
|
8
|
+
* {@link Tracer} contract from `@graphorin/core/contracts`. Tests pass
|
|
9
|
+
* a `NOOP_TRACER`-shaped stub; production wires the real Graphorin
|
|
10
|
+
* tracer from `@graphorin/observability`.
|
|
11
|
+
*
|
|
12
|
+
* @packageDocumentation
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import type { Provider, Tracer } from '@graphorin/core';
|
|
16
|
+
|
|
17
|
+
import { defineProviderMiddleware } from './compose.js';
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Options for {@link withTracing}.
|
|
21
|
+
*
|
|
22
|
+
* @stable
|
|
23
|
+
*/
|
|
24
|
+
export interface WithTracingOptions {
|
|
25
|
+
/** Tracer instance. Defaults to a no-op tracer if unset. */
|
|
26
|
+
readonly tracer?: Tracer;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* @stable
|
|
31
|
+
*/
|
|
32
|
+
export const withTracing = defineProviderMiddleware<WithTracingOptions>({
|
|
33
|
+
kind: 'withTracing',
|
|
34
|
+
factory: (opts: WithTracingOptions) => {
|
|
35
|
+
return (next: Provider): Provider => ({
|
|
36
|
+
name: next.name,
|
|
37
|
+
modelId: next.modelId,
|
|
38
|
+
capabilities: next.capabilities,
|
|
39
|
+
...(next.acceptsSensitivity !== undefined
|
|
40
|
+
? { acceptsSensitivity: next.acceptsSensitivity }
|
|
41
|
+
: {}),
|
|
42
|
+
stream(req) {
|
|
43
|
+
if (opts.tracer === undefined) return next.stream(req);
|
|
44
|
+
return tracedStream(opts.tracer, next, req);
|
|
45
|
+
},
|
|
46
|
+
async generate(req) {
|
|
47
|
+
if (opts.tracer === undefined) return next.generate(req);
|
|
48
|
+
return opts.tracer.span(
|
|
49
|
+
{
|
|
50
|
+
type: 'provider.generate' as const,
|
|
51
|
+
attrs: {
|
|
52
|
+
'graphorin.provider.id': next.name,
|
|
53
|
+
'graphorin.provider.model': next.modelId,
|
|
54
|
+
// C7: OTel GenAI semantic conventions.
|
|
55
|
+
'gen_ai.operation.name': 'chat',
|
|
56
|
+
'gen_ai.provider.name': next.name,
|
|
57
|
+
'gen_ai.request.model': next.modelId,
|
|
58
|
+
},
|
|
59
|
+
// C7: parent under the agent step span when the loop supplies one.
|
|
60
|
+
...(req.parentSpan !== undefined ? { parent: req.parentSpan } : {}),
|
|
61
|
+
},
|
|
62
|
+
async (span) => {
|
|
63
|
+
const response = await next.generate(req);
|
|
64
|
+
span.setAttributes({
|
|
65
|
+
'gen_ai.usage.input_tokens': response.usage.promptTokens,
|
|
66
|
+
'gen_ai.usage.output_tokens': response.usage.completionTokens,
|
|
67
|
+
});
|
|
68
|
+
return response;
|
|
69
|
+
},
|
|
70
|
+
);
|
|
71
|
+
},
|
|
72
|
+
...(next.countTokens ? { countTokens: next.countTokens.bind(next) } : {}),
|
|
73
|
+
});
|
|
74
|
+
},
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
async function* tracedStream(
|
|
78
|
+
tracer: Tracer,
|
|
79
|
+
next: Provider,
|
|
80
|
+
req: import('@graphorin/core').ProviderRequest,
|
|
81
|
+
): AsyncIterable<import('@graphorin/core').ProviderEvent> {
|
|
82
|
+
const span = tracer.startSpan({
|
|
83
|
+
type: 'provider.stream' as const,
|
|
84
|
+
attrs: {
|
|
85
|
+
'graphorin.provider.id': next.name,
|
|
86
|
+
'graphorin.provider.model': next.modelId,
|
|
87
|
+
// C7: OTel GenAI semantic conventions.
|
|
88
|
+
'gen_ai.operation.name': 'chat',
|
|
89
|
+
'gen_ai.provider.name': next.name,
|
|
90
|
+
'gen_ai.request.model': next.modelId,
|
|
91
|
+
},
|
|
92
|
+
// C7: parent under the agent step span when the loop supplies one.
|
|
93
|
+
...(req.parentSpan !== undefined ? { parent: req.parentSpan } : {}),
|
|
94
|
+
});
|
|
95
|
+
try {
|
|
96
|
+
for await (const event of next.stream(req)) {
|
|
97
|
+
if (event.type === 'finish') {
|
|
98
|
+
span.setAttributes({
|
|
99
|
+
'gen_ai.usage.input_tokens': event.usage.promptTokens,
|
|
100
|
+
'gen_ai.usage.output_tokens': event.usage.completionTokens,
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
yield event;
|
|
104
|
+
}
|
|
105
|
+
span.setStatus('ok');
|
|
106
|
+
} catch (err) {
|
|
107
|
+
span.recordException(err);
|
|
108
|
+
span.setStatus('error', (err as Error).message);
|
|
109
|
+
throw err;
|
|
110
|
+
} finally {
|
|
111
|
+
// PS-7: a consumer `break`/early-return injects a generator `return` at the
|
|
112
|
+
// `yield`, skipping both the success and catch paths. Ending the span in
|
|
113
|
+
// `finally` guarantees it is closed exactly once on every exit - normal
|
|
114
|
+
// completion, error, or early abort (which leaves the status unset).
|
|
115
|
+
span.end();
|
|
116
|
+
}
|
|
117
|
+
}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-provider model-tier auto-classifier - returns
|
|
3
|
+
* `'fast' | 'balanced' | 'smart' | undefined` for any model id. The
|
|
4
|
+
* classifier is consumed by the agent runtime (Phase 12) to validate
|
|
5
|
+
* operator-supplied tier mappings and to surface tier-not-mapped
|
|
6
|
+
* recommendations.
|
|
7
|
+
*
|
|
8
|
+
* The dispatcher is a pure function; it reads from a small static
|
|
9
|
+
* rule table keyed on regex patterns matched against the lowercased
|
|
10
|
+
* model id. The table can be inspected via {@link CLASSIFIER_RULES}
|
|
11
|
+
* for debugging and downstream tooling (CLI, dashboard, lint rules).
|
|
12
|
+
*
|
|
13
|
+
* @packageDocumentation
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import type { ModelHint, ProviderLike } from '@graphorin/core';
|
|
17
|
+
|
|
18
|
+
import { InvalidProviderError } from '../errors/errors.js';
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Single entry in the classifier rule table.
|
|
22
|
+
*
|
|
23
|
+
* @stable
|
|
24
|
+
*/
|
|
25
|
+
export interface ClassifierRule {
|
|
26
|
+
readonly tier: ModelHint;
|
|
27
|
+
readonly pattern: RegExp;
|
|
28
|
+
/** Human-readable family label (`'anthropic-claude-haiku'`, …). */
|
|
29
|
+
readonly family: string;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The static rule table. Order matters - higher-specificity entries
|
|
34
|
+
* come first (e.g. `claude-haiku` before `claude-`). Tests assert
|
|
35
|
+
* that the table covers the canonical 2026 model families.
|
|
36
|
+
*
|
|
37
|
+
* @stable
|
|
38
|
+
*/
|
|
39
|
+
export const CLASSIFIER_RULES: readonly ClassifierRule[] = Object.freeze([
|
|
40
|
+
// Anthropic - direct. PS-20: the version segment between `claude` and the
|
|
41
|
+
// family word can be multi-part (`claude-3-5-haiku`, `claude-3-7-sonnet`) or
|
|
42
|
+
// absent (`claude-haiku-4-5`), so allow zero-or-more `-<digits/dots>` groups.
|
|
43
|
+
{ tier: 'fast', family: 'anthropic-haiku', pattern: /^claude(?:-[\d.]+)*-?haiku/ },
|
|
44
|
+
{ tier: 'balanced', family: 'anthropic-sonnet', pattern: /^claude(?:-[\d.]+)*-?sonnet/ },
|
|
45
|
+
{ tier: 'smart', family: 'anthropic-opus', pattern: /^claude(?:-[\d.]+)*-?opus/ },
|
|
46
|
+
{ tier: 'smart', family: 'anthropic-fable', pattern: /^claude.*fable/ },
|
|
47
|
+
{ tier: 'smart', family: 'anthropic-mythos', pattern: /^claude.*mythos/ },
|
|
48
|
+
// Anthropic - via Bedrock.
|
|
49
|
+
{
|
|
50
|
+
tier: 'fast',
|
|
51
|
+
family: 'bedrock-claude-haiku',
|
|
52
|
+
pattern: /^anthropic\.claude(?:-[\d.]+)*-?haiku/,
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
tier: 'balanced',
|
|
56
|
+
family: 'bedrock-claude-sonnet',
|
|
57
|
+
pattern: /^anthropic\.claude(?:-[\d.]+)*-?sonnet/,
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
tier: 'smart',
|
|
61
|
+
family: 'bedrock-claude-opus',
|
|
62
|
+
pattern: /^anthropic\.claude(?:-[\d.]+)*-?opus/,
|
|
63
|
+
},
|
|
64
|
+
// OpenAI.
|
|
65
|
+
{ tier: 'fast', family: 'openai-mini', pattern: /^gpt-(\d|\d+\.\d+)-?mini/ },
|
|
66
|
+
{ tier: 'fast', family: 'openai-nano', pattern: /^gpt-(\d|\d+\.\d+)-?nano/ },
|
|
67
|
+
{ tier: 'balanced', family: 'openai-gpt', pattern: /^gpt-(?!.*(?:mini|nano))/ },
|
|
68
|
+
{ tier: 'smart', family: 'openai-reasoning', pattern: /^o[1-9]\b/ },
|
|
69
|
+
{ tier: 'smart', family: 'openai-reasoning-extended', pattern: /^o[1-9][a-z-]/ },
|
|
70
|
+
// Google Gemini.
|
|
71
|
+
{ tier: 'fast', family: 'gemini-flash', pattern: /^gemini.*flash/ },
|
|
72
|
+
{ tier: 'balanced', family: 'gemini-pro', pattern: /^gemini.*pro\b/ },
|
|
73
|
+
{ tier: 'smart', family: 'gemini-ultra', pattern: /^gemini.*ultra/ },
|
|
74
|
+
]);
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Classify a `Provider`'s `modelId` into one of `'fast' | 'balanced' |
|
|
78
|
+
* 'smart'`. Returns `undefined` when the model id matches none of the
|
|
79
|
+
* canonical 2026 mappings (Ollama / OpenAI-compatible / unknown).
|
|
80
|
+
*
|
|
81
|
+
* @stable
|
|
82
|
+
*/
|
|
83
|
+
export function classifyModelTier(provider: ProviderLike): ModelHint | undefined {
|
|
84
|
+
if (provider === null || typeof provider !== 'object') {
|
|
85
|
+
throw new InvalidProviderError('classifyModelTier: argument must be a Provider-shaped object.');
|
|
86
|
+
}
|
|
87
|
+
const modelId = provider.modelId;
|
|
88
|
+
if (typeof modelId !== 'string' || modelId.length === 0) {
|
|
89
|
+
throw new InvalidProviderError(
|
|
90
|
+
'classifyModelTier: provider.modelId must be a non-empty string.',
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
const normalised = stripFamilyPrefix(modelId.toLowerCase());
|
|
94
|
+
for (const rule of CLASSIFIER_RULES) {
|
|
95
|
+
if (rule.pattern.test(normalised)) return rule.tier;
|
|
96
|
+
}
|
|
97
|
+
return undefined;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Bedrock cross-region inference-profile prefix (`us.anthropic.claude-…`).
|
|
102
|
+
* Stripped so region-qualified ids hit the `^anthropic\.claude` rules
|
|
103
|
+
* (core-provider-11).
|
|
104
|
+
*/
|
|
105
|
+
const BEDROCK_REGION_PREFIX = /^(?:us|eu|apac|jp|au|us-gov)\./;
|
|
106
|
+
|
|
107
|
+
function stripFamilyPrefix(model: string): string {
|
|
108
|
+
// Common prefixes used by adapters: `anthropic/...`, `openai/...`,
|
|
109
|
+
// `google/...`, `provider:model`, etc. Strip them so the rule
|
|
110
|
+
// patterns can stay anchored at `^`.
|
|
111
|
+
const slash = model.indexOf('/');
|
|
112
|
+
if (slash !== -1) return model.slice(slash + 1).replace(BEDROCK_REGION_PREFIX, '');
|
|
113
|
+
const deRegioned = model.replace(BEDROCK_REGION_PREFIX, '');
|
|
114
|
+
// Bedrock ids end in ':<version>' (`anthropic.claude-...-v1:0`) - the
|
|
115
|
+
// colon there is a version separator, not a provider/model split, and
|
|
116
|
+
// the rule patterns are prefix-anchored so the suffix is harmless.
|
|
117
|
+
if (deRegioned.startsWith('anthropic.')) return deRegioned;
|
|
118
|
+
const colon = deRegioned.indexOf(':');
|
|
119
|
+
if (colon !== -1 && !deRegioned.startsWith('http')) {
|
|
120
|
+
// Skip URL-like values (`http://localhost:8080`) - `:` there is a
|
|
121
|
+
// port separator, not a provider/model split.
|
|
122
|
+
return deRegioned.slice(colon + 1);
|
|
123
|
+
}
|
|
124
|
+
return deRegioned;
|
|
125
|
+
}
|
package/src/provider.ts
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `createProvider(...)` - wrap an adapter return value in the canonical
|
|
3
|
+
* Graphorin `Provider` shape and apply optional sensitivity / capability
|
|
4
|
+
* overrides.
|
|
5
|
+
*
|
|
6
|
+
* Adapters return a "raw" provider object; `createProvider(...)` gives
|
|
7
|
+
* downstream code a single place to centralise:
|
|
8
|
+
*
|
|
9
|
+
* - per-instance `acceptsSensitivity` declarations,
|
|
10
|
+
* - capability overrides (e.g. forcing `multimodal: false` for a tool
|
|
11
|
+
* that does not need it),
|
|
12
|
+
* - default `reasoningRetention` resolution from the adapter's declared
|
|
13
|
+
* `reasoningContract`,
|
|
14
|
+
* - and the agreed-upon `name` / `modelId` lock-in.
|
|
15
|
+
*
|
|
16
|
+
* @stable
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import type {
|
|
20
|
+
Provider,
|
|
21
|
+
ProviderCapabilities,
|
|
22
|
+
ProviderRequest,
|
|
23
|
+
ReasoningRetention,
|
|
24
|
+
Sensitivity,
|
|
25
|
+
} from '@graphorin/core';
|
|
26
|
+
|
|
27
|
+
import { resolveReasoningRetention } from './reasoning/retention.js';
|
|
28
|
+
import { foldToolExamples } from './tool-examples.js';
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Options accepted by {@link createProvider}.
|
|
32
|
+
*
|
|
33
|
+
* @stable
|
|
34
|
+
*/
|
|
35
|
+
export interface CreateProviderOptions {
|
|
36
|
+
/**
|
|
37
|
+
* Sensitivity tiers this provider is allowed to receive. When
|
|
38
|
+
* unset, the value is forwarded from the wrapped adapter (which is
|
|
39
|
+
* itself populated from the trust class for `baseUrl`-driven
|
|
40
|
+
* adapters).
|
|
41
|
+
*/
|
|
42
|
+
readonly acceptsSensitivity?: ReadonlyArray<Sensitivity>;
|
|
43
|
+
/**
|
|
44
|
+
* Per-request override of the reasoning-retention default. The
|
|
45
|
+
* adapter's `capabilities.reasoningContract` decides the auto-
|
|
46
|
+
* detected default; this option pins a different value for every
|
|
47
|
+
* request the wrapper sees.
|
|
48
|
+
*/
|
|
49
|
+
readonly reasoningRetention?: ReasoningRetention;
|
|
50
|
+
/**
|
|
51
|
+
* Optional capability override. Useful for narrowing what a
|
|
52
|
+
* downstream tool advertises (e.g. setting `multimodal: false`
|
|
53
|
+
* when the consumer's prompt cache is text-only).
|
|
54
|
+
*/
|
|
55
|
+
readonly capabilities?: Partial<ProviderCapabilities>;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Wrap an adapter in the canonical `Provider` shape. Adapters returned
|
|
60
|
+
* by the bundled factories already implement `Provider`; passing them
|
|
61
|
+
* through `createProvider(...)` is the recommended entry point because
|
|
62
|
+
* it keeps the construction site documented and gives downstream
|
|
63
|
+
* middleware a single attachment surface.
|
|
64
|
+
*
|
|
65
|
+
* @example
|
|
66
|
+
* ```ts
|
|
67
|
+
* const provider = createProvider(vercelAdapter(model), {
|
|
68
|
+
* acceptsSensitivity: ['public', 'internal'],
|
|
69
|
+
* });
|
|
70
|
+
* ```
|
|
71
|
+
*
|
|
72
|
+
* @stable
|
|
73
|
+
*/
|
|
74
|
+
export function createProvider(adapter: Provider, options: CreateProviderOptions = {}): Provider {
|
|
75
|
+
const capabilities: ProviderCapabilities = {
|
|
76
|
+
...adapter.capabilities,
|
|
77
|
+
...options.capabilities,
|
|
78
|
+
};
|
|
79
|
+
const acceptsSensitivity = options.acceptsSensitivity ?? adapter.acceptsSensitivity ?? undefined;
|
|
80
|
+
|
|
81
|
+
const wrapped: Provider = {
|
|
82
|
+
name: adapter.name,
|
|
83
|
+
modelId: adapter.modelId,
|
|
84
|
+
capabilities,
|
|
85
|
+
...(acceptsSensitivity ? { acceptsSensitivity } : {}),
|
|
86
|
+
stream(req) {
|
|
87
|
+
return adapter.stream(applyDefaults(req, options, capabilities));
|
|
88
|
+
},
|
|
89
|
+
generate(req) {
|
|
90
|
+
return adapter.generate(applyDefaults(req, options, capabilities));
|
|
91
|
+
},
|
|
92
|
+
...(adapter.countTokens ? { countTokens: adapter.countTokens.bind(adapter) } : {}),
|
|
93
|
+
};
|
|
94
|
+
return wrapped;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function applyDefaults(
|
|
98
|
+
req: ProviderRequest,
|
|
99
|
+
options: CreateProviderOptions,
|
|
100
|
+
capabilities: ProviderCapabilities,
|
|
101
|
+
): ProviderRequest {
|
|
102
|
+
let next = req;
|
|
103
|
+
// A1: fold worked tool examples into the model-facing description so the model
|
|
104
|
+
// actually sees them (the wire contract carries them; adapters never rendered
|
|
105
|
+
// them). Deterministic ⇒ the tool spec stays prompt-cache-stable.
|
|
106
|
+
if (next.tools !== undefined) {
|
|
107
|
+
const foldedTools = foldToolExamples(next.tools);
|
|
108
|
+
if (foldedTools !== next.tools) next = { ...next, tools: foldedTools };
|
|
109
|
+
}
|
|
110
|
+
const effectiveRetention = resolveReasoningRetention({
|
|
111
|
+
...(next.reasoningRetention !== undefined ? { requested: next.reasoningRetention } : {}),
|
|
112
|
+
...(options.reasoningRetention !== undefined ? { overridden: options.reasoningRetention } : {}),
|
|
113
|
+
...(capabilities.reasoningContract !== undefined
|
|
114
|
+
? { contract: capabilities.reasoningContract }
|
|
115
|
+
: {}),
|
|
116
|
+
});
|
|
117
|
+
if (effectiveRetention !== next.reasoningRetention) {
|
|
118
|
+
next = { ...next, reasoningRetention: effectiveRetention };
|
|
119
|
+
}
|
|
120
|
+
return next;
|
|
121
|
+
}
|