@graphorin/provider 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/README.md +8 -3
- package/dist/adapters/llamacpp-server.d.ts +1 -1
- package/dist/adapters/llamacpp-server.js.map +1 -1
- package/dist/adapters/ollama.d.ts.map +1 -1
- package/dist/adapters/ollama.js +19 -6
- package/dist/adapters/ollama.js.map +1 -1
- package/dist/adapters/vercel.d.ts +1 -1
- package/dist/adapters/vercel.d.ts.map +1 -1
- package/dist/adapters/vercel.js +111 -33
- package/dist/adapters/vercel.js.map +1 -1
- package/dist/errors/errors.d.ts +1 -1
- package/dist/errors/errors.js +1 -1
- package/dist/errors/errors.js.map +1 -1
- package/dist/index.d.ts +1 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -8
- package/dist/index.js.map +1 -1
- package/dist/internal/http.js +111 -7
- package/dist/internal/http.js.map +1 -1
- package/dist/internal/openai-shaped.js +21 -4
- package/dist/internal/openai-shaped.js.map +1 -1
- package/dist/middleware/with-fallback.js +1 -1
- package/dist/middleware/with-rate-limit.d.ts +21 -8
- package/dist/middleware/with-rate-limit.d.ts.map +1 -1
- package/dist/middleware/with-rate-limit.js +65 -12
- package/dist/middleware/with-rate-limit.js.map +1 -1
- package/dist/middleware/with-retry.js +1 -1
- package/dist/package.js +6 -0
- package/dist/package.js.map +1 -0
- package/package.json +18 -16
- package/src/adapters/index.ts +14 -0
- package/src/adapters/llamacpp-server.ts +102 -0
- package/src/adapters/ollama.ts +382 -0
- package/src/adapters/openai-compatible.ts +95 -0
- package/src/adapters/vercel-messages.ts +308 -0
- package/src/adapters/vercel.ts +706 -0
- package/src/counters/anthropic-wire.ts +199 -0
- package/src/counters/anthropic.ts +114 -0
- package/src/counters/bedrock.ts +46 -0
- package/src/counters/dispatcher.ts +127 -0
- package/src/counters/global.ts +39 -0
- package/src/counters/google.ts +46 -0
- package/src/counters/heuristic.ts +107 -0
- package/src/counters/index.ts +35 -0
- package/src/counters/js-tiktoken.ts +135 -0
- package/src/counters/serialize.ts +85 -0
- package/src/errors/errors.ts +316 -0
- package/src/errors/index.ts +20 -0
- package/src/index.ts +42 -0
- package/src/internal/abort.ts +30 -0
- package/src/internal/http.ts +388 -0
- package/src/internal/openai-shaped.ts +555 -0
- package/src/internal/sse.ts +112 -0
- package/src/internal/url-utils.ts +20 -0
- package/src/middleware/compose.ts +213 -0
- package/src/middleware/index.ts +37 -0
- package/src/middleware/production-hook.ts +47 -0
- package/src/middleware/with-cost-limit.ts +131 -0
- package/src/middleware/with-cost-tracking.ts +216 -0
- package/src/middleware/with-fallback.ts +157 -0
- package/src/middleware/with-rate-limit.ts +306 -0
- package/src/middleware/with-redaction.ts +671 -0
- package/src/middleware/with-retry.ts +274 -0
- package/src/middleware/with-tracing.ts +117 -0
- package/src/model-tier/classify.ts +125 -0
- package/src/model-tier/index.ts +11 -0
- package/src/provider.ts +121 -0
- package/src/reasoning/apply-policy.ts +89 -0
- package/src/reasoning/classify-contract.ts +120 -0
- package/src/reasoning/index.ts +21 -0
- package/src/reasoning/retention.ts +64 -0
- package/src/tool-examples.ts +54 -0
- package/src/trust/classify-local-provider.ts +254 -0
- package/src/trust/index.ts +14 -0
- package/dist/adapters/index.js +0 -6
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Conversion from Graphorin messages to the Anthropic Messages API
|
|
3
|
+
* wire shape, used by `AnthropicAPICounter` (core-provider-04).
|
|
4
|
+
*
|
|
5
|
+
* The `count_tokens` endpoint accepts the same body as `/v1/messages`:
|
|
6
|
+
* only `user` / `assistant` roles in `messages`, system prompts in the
|
|
7
|
+
* top-level `system` field, assistant tool calls as `tool_use` blocks,
|
|
8
|
+
* tool results as `tool_result` blocks inside a **user** turn, and
|
|
9
|
+
* images as `source` blocks. Posting Graphorin-shaped messages 400s on
|
|
10
|
+
* any transcript containing a system / tool message or `toolCalls`,
|
|
11
|
+
* silently downgrading the "native" counter to tiktoken.
|
|
12
|
+
*
|
|
13
|
+
* @internal
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import type { Message, MessageContent } from '@graphorin/core';
|
|
17
|
+
|
|
18
|
+
/** Anthropic-shaped count_tokens payload. */
|
|
19
|
+
export interface AnthropicCountPayload {
|
|
20
|
+
readonly system?: string;
|
|
21
|
+
readonly messages: ReadonlyArray<{
|
|
22
|
+
readonly role: 'user' | 'assistant';
|
|
23
|
+
readonly content: ReadonlyArray<Readonly<Record<string, unknown>>>;
|
|
24
|
+
}>;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Convert Graphorin messages to the Anthropic wire shape. Returns
|
|
29
|
+
* `null` when nothing countable remains (the caller should use its
|
|
30
|
+
* local fallback instead of paying an HTTP round trip for a 400).
|
|
31
|
+
*
|
|
32
|
+
* Consecutive same-role turns are merged and the transcript is forced
|
|
33
|
+
* to start with a user turn - both hard requirements of the API.
|
|
34
|
+
* Parts with no wire representation for counting purposes (audio,
|
|
35
|
+
* non-PDF files, unsigned reasoning) are skipped; the count is then a
|
|
36
|
+
* slight undercount rather than a guaranteed 400.
|
|
37
|
+
*
|
|
38
|
+
* @internal
|
|
39
|
+
*/
|
|
40
|
+
export function toAnthropicCountPayload(
|
|
41
|
+
messages: ReadonlyArray<Message>,
|
|
42
|
+
): AnthropicCountPayload | null {
|
|
43
|
+
const systemParts: string[] = [];
|
|
44
|
+
const turns: Array<{ role: 'user' | 'assistant'; content: Array<Record<string, unknown>> }> = [];
|
|
45
|
+
|
|
46
|
+
const pushBlocks = (role: 'user' | 'assistant', blocks: Array<Record<string, unknown>>): void => {
|
|
47
|
+
if (blocks.length === 0) return;
|
|
48
|
+
const last = turns[turns.length - 1];
|
|
49
|
+
if (last !== undefined && last.role === role) {
|
|
50
|
+
last.content.push(...blocks);
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
turns.push({ role, content: blocks });
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
for (const msg of messages) {
|
|
57
|
+
switch (msg.role) {
|
|
58
|
+
case 'system':
|
|
59
|
+
if (msg.content.length > 0) systemParts.push(msg.content);
|
|
60
|
+
break;
|
|
61
|
+
case 'user':
|
|
62
|
+
pushBlocks('user', contentToBlocks(msg.content, 'user'));
|
|
63
|
+
break;
|
|
64
|
+
case 'assistant': {
|
|
65
|
+
const blocks = contentToBlocks(msg.content, 'assistant');
|
|
66
|
+
for (const call of msg.toolCalls ?? []) {
|
|
67
|
+
blocks.push({
|
|
68
|
+
type: 'tool_use',
|
|
69
|
+
id: call.toolCallId,
|
|
70
|
+
name: call.toolName,
|
|
71
|
+
input: toInputObject(call.args),
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
pushBlocks('assistant', blocks);
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
case 'tool': {
|
|
78
|
+
// Tool results live in the next USER turn on the Anthropic wire.
|
|
79
|
+
const text = flattenToText(msg.content);
|
|
80
|
+
pushBlocks('user', [
|
|
81
|
+
{
|
|
82
|
+
type: 'tool_result',
|
|
83
|
+
tool_use_id: msg.toolCallId,
|
|
84
|
+
...(text.length > 0 ? { content: [{ type: 'text', text }] } : {}),
|
|
85
|
+
},
|
|
86
|
+
]);
|
|
87
|
+
break;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
if (turns.length === 0 && systemParts.length === 0) return null;
|
|
93
|
+
// The API requires the first message to be a user turn.
|
|
94
|
+
if (turns.length === 0 || turns[0]?.role !== 'user') {
|
|
95
|
+
turns.unshift({ role: 'user', content: [{ type: 'text', text: '.' }] });
|
|
96
|
+
}
|
|
97
|
+
return {
|
|
98
|
+
...(systemParts.length > 0 ? { system: systemParts.join('\n\n') } : {}),
|
|
99
|
+
messages: turns,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function contentToBlocks(
|
|
104
|
+
content: string | ReadonlyArray<MessageContent>,
|
|
105
|
+
role: 'user' | 'assistant',
|
|
106
|
+
): Array<Record<string, unknown>> {
|
|
107
|
+
if (typeof content === 'string') {
|
|
108
|
+
return content.length > 0 ? [{ type: 'text', text: content }] : [];
|
|
109
|
+
}
|
|
110
|
+
const blocks: Array<Record<string, unknown>> = [];
|
|
111
|
+
for (const part of content) {
|
|
112
|
+
switch (part.type) {
|
|
113
|
+
case 'text':
|
|
114
|
+
if (part.text.length > 0) blocks.push({ type: 'text', text: part.text });
|
|
115
|
+
break;
|
|
116
|
+
case 'image': {
|
|
117
|
+
const source = toImageSource(part.image, part.mimeType);
|
|
118
|
+
if (source !== null && role === 'user') blocks.push({ type: 'image', source });
|
|
119
|
+
break;
|
|
120
|
+
}
|
|
121
|
+
case 'file': {
|
|
122
|
+
// PDFs count as document blocks; other file kinds have no
|
|
123
|
+
// countable wire form and are skipped.
|
|
124
|
+
if (part.mimeType === 'application/pdf' && part.file instanceof Uint8Array) {
|
|
125
|
+
blocks.push({
|
|
126
|
+
type: 'document',
|
|
127
|
+
source: {
|
|
128
|
+
type: 'base64',
|
|
129
|
+
media_type: 'application/pdf',
|
|
130
|
+
data: toBase64(part.file),
|
|
131
|
+
},
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
break;
|
|
135
|
+
}
|
|
136
|
+
case 'reasoning': {
|
|
137
|
+
// Only signed thinking blocks round-trip through the API;
|
|
138
|
+
// unsigned reasoning would 400, so it is skipped (the default
|
|
139
|
+
// retention policy strips it from outbound requests anyway).
|
|
140
|
+
if (
|
|
141
|
+
role === 'assistant' &&
|
|
142
|
+
part.meta?.provider === 'anthropic' &&
|
|
143
|
+
typeof part.meta.signature === 'string'
|
|
144
|
+
) {
|
|
145
|
+
blocks.push({ type: 'thinking', thinking: part.text, signature: part.meta.signature });
|
|
146
|
+
}
|
|
147
|
+
break;
|
|
148
|
+
}
|
|
149
|
+
case 'audio':
|
|
150
|
+
// No audio block on the Messages API; skip.
|
|
151
|
+
break;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return blocks;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function toImageSource(
|
|
158
|
+
image: Uint8Array | URL,
|
|
159
|
+
mimeType: string | undefined,
|
|
160
|
+
): Record<string, unknown> | null {
|
|
161
|
+
if (image instanceof URL) {
|
|
162
|
+
return { type: 'url', url: image.toString() };
|
|
163
|
+
}
|
|
164
|
+
if (image instanceof Uint8Array) {
|
|
165
|
+
return {
|
|
166
|
+
type: 'base64',
|
|
167
|
+
media_type: mimeType ?? 'image/png',
|
|
168
|
+
data: toBase64(image),
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
return null;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function toBase64(bytes: Uint8Array): string {
|
|
175
|
+
return Buffer.from(bytes).toString('base64');
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function toInputObject(args: unknown): Record<string, unknown> {
|
|
179
|
+
if (typeof args === 'string') {
|
|
180
|
+
try {
|
|
181
|
+
const parsed: unknown = JSON.parse(args);
|
|
182
|
+
return typeof parsed === 'object' && parsed !== null
|
|
183
|
+
? (parsed as Record<string, unknown>)
|
|
184
|
+
: {};
|
|
185
|
+
} catch {
|
|
186
|
+
return {};
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
return typeof args === 'object' && args !== null ? (args as Record<string, unknown>) : {};
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function flattenToText(content: string | ReadonlyArray<MessageContent>): string {
|
|
193
|
+
if (typeof content === 'string') return content;
|
|
194
|
+
const buffer: string[] = [];
|
|
195
|
+
for (const part of content) {
|
|
196
|
+
if (part.type === 'text' || part.type === 'reasoning') buffer.push(part.text);
|
|
197
|
+
}
|
|
198
|
+
return buffer.join('');
|
|
199
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `AnthropicAPICounter` - uses Anthropic's native count-tokens
|
|
3
|
+
* endpoint when an API key is configured, otherwise falls back to
|
|
4
|
+
* `JsTiktokenCounter`. The native call is opt-in: the user supplies
|
|
5
|
+
* an explicit `apiKey`. The framework never makes the call without
|
|
6
|
+
* one.
|
|
7
|
+
*
|
|
8
|
+
* @packageDocumentation
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { Message, TokenCounter } from '@graphorin/core';
|
|
12
|
+
|
|
13
|
+
import { stripTrailingSlashes } from '../internal/url-utils.js';
|
|
14
|
+
import { toAnthropicCountPayload } from './anthropic-wire.js';
|
|
15
|
+
import { JsTiktokenCounter } from './js-tiktoken.js';
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Options for {@link AnthropicAPICounter}.
|
|
19
|
+
*
|
|
20
|
+
* @stable
|
|
21
|
+
*/
|
|
22
|
+
export interface AnthropicAPICounterOptions {
|
|
23
|
+
readonly modelId: string;
|
|
24
|
+
readonly apiKey?: string;
|
|
25
|
+
readonly baseUrl?: string;
|
|
26
|
+
readonly fetchImpl?: typeof fetch;
|
|
27
|
+
/** Identifier carried on the produced counter. */
|
|
28
|
+
readonly id?: string;
|
|
29
|
+
/**
|
|
30
|
+
* Log sink for the WARN emitted (once per counter) when the native
|
|
31
|
+
* call degrades to the tiktoken fallback. Defaults to `console.warn`.
|
|
32
|
+
*/
|
|
33
|
+
readonly logger?: (message: string, meta?: object) => void;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Counter that talks to `POST /v1/messages/count_tokens` when an API
|
|
38
|
+
* key is configured. Without an API key, the counter delegates to
|
|
39
|
+
* `JsTiktokenCounter('cl100k_base')` - the closest publicly-available
|
|
40
|
+
* proxy for Anthropic's tokenizer.
|
|
41
|
+
*
|
|
42
|
+
* @stable
|
|
43
|
+
*/
|
|
44
|
+
export class AnthropicAPICounter implements TokenCounter {
|
|
45
|
+
readonly id: string;
|
|
46
|
+
readonly version: string;
|
|
47
|
+
readonly #modelId: string;
|
|
48
|
+
readonly #apiKey?: string;
|
|
49
|
+
readonly #baseUrl: string;
|
|
50
|
+
readonly #fetchImpl: typeof fetch;
|
|
51
|
+
readonly #fallback: JsTiktokenCounter;
|
|
52
|
+
readonly #logger: (message: string, meta?: object) => void;
|
|
53
|
+
#fallbackWarned = false;
|
|
54
|
+
|
|
55
|
+
constructor(options: AnthropicAPICounterOptions) {
|
|
56
|
+
this.#modelId = options.modelId;
|
|
57
|
+
if (options.apiKey !== undefined) this.#apiKey = options.apiKey;
|
|
58
|
+
this.#baseUrl = stripTrailingSlashes(options.baseUrl ?? 'https://api.anthropic.com');
|
|
59
|
+
this.#fetchImpl = options.fetchImpl ?? globalThis.fetch.bind(globalThis);
|
|
60
|
+
this.#fallback = new JsTiktokenCounter({ encoding: 'cl100k_base', modelId: options.modelId });
|
|
61
|
+
this.#logger = options.logger ?? ((message, meta) => console.warn(message, meta ?? ''));
|
|
62
|
+
this.id = options.id ?? `anthropic-native@${options.modelId}`;
|
|
63
|
+
this.version = `anthropic-native-${options.modelId}-v1`;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
async count(messages: ReadonlyArray<Message>): Promise<number> {
|
|
67
|
+
if (this.#apiKey === undefined) return this.#fallback.count(messages);
|
|
68
|
+
// core-provider-04: the endpoint accepts only Anthropic wire-shaped
|
|
69
|
+
// bodies. Posting Graphorin messages 400s on any transcript with a
|
|
70
|
+
// system / tool message or `toolCalls`, so convert first.
|
|
71
|
+
const payload = toAnthropicCountPayload(messages);
|
|
72
|
+
if (payload === null) return this.#fallback.count(messages);
|
|
73
|
+
try {
|
|
74
|
+
const resp = await this.#fetchImpl(`${this.#baseUrl}/v1/messages/count_tokens`, {
|
|
75
|
+
method: 'POST',
|
|
76
|
+
headers: {
|
|
77
|
+
'content-type': 'application/json',
|
|
78
|
+
'x-api-key': this.#apiKey,
|
|
79
|
+
'anthropic-version': '2023-06-01',
|
|
80
|
+
},
|
|
81
|
+
body: JSON.stringify({ model: this.#modelId, ...payload }),
|
|
82
|
+
});
|
|
83
|
+
if (!resp.ok) {
|
|
84
|
+
this.#warnFallback(`HTTP ${resp.status}`);
|
|
85
|
+
return this.#fallback.count(messages);
|
|
86
|
+
}
|
|
87
|
+
const json = (await resp.json()) as { input_tokens?: number };
|
|
88
|
+
if (typeof json.input_tokens === 'number') return json.input_tokens;
|
|
89
|
+
this.#warnFallback('response missing input_tokens');
|
|
90
|
+
return this.#fallback.count(messages);
|
|
91
|
+
} catch (cause) {
|
|
92
|
+
this.#warnFallback((cause as Error).message);
|
|
93
|
+
return this.#fallback.count(messages);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async countText(text: string): Promise<number> {
|
|
98
|
+
return this.#fallback.countText(text);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* WARN once per counter when the native path degrades - a silent
|
|
103
|
+
* fallback to cl100k tiktoken undercounts Claude tokens by ~15-20%,
|
|
104
|
+
* so the degradation must be visible.
|
|
105
|
+
*/
|
|
106
|
+
#warnFallback(reason: string): void {
|
|
107
|
+
if (this.#fallbackWarned) return;
|
|
108
|
+
this.#fallbackWarned = true;
|
|
109
|
+
this.#logger(
|
|
110
|
+
`[graphorin/provider] AnthropicAPICounter(${this.#modelId}): native count_tokens failed (${reason}); ` +
|
|
111
|
+
'falling back to cl100k tiktoken (undercounts Claude by ~15-20%). Further fallbacks are silent.',
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `BedrockAPICounter` - placeholder counter that delegates to
|
|
3
|
+
* `JsTiktokenCounter('cl100k_base')` for now (Bedrock's native
|
|
4
|
+
* `count_tokens` API requires AWS Signature v4; full integration
|
|
5
|
+
* lands in Phase 16). The proxy is sufficient for v0.1 within the
|
|
6
|
+
* documented ±5% accuracy envelope.
|
|
7
|
+
*
|
|
8
|
+
* @packageDocumentation
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { Message, TokenCounter } from '@graphorin/core';
|
|
12
|
+
|
|
13
|
+
import { JsTiktokenCounter } from './js-tiktoken.js';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Options for {@link BedrockAPICounter}.
|
|
17
|
+
*
|
|
18
|
+
* @stable
|
|
19
|
+
*/
|
|
20
|
+
export interface BedrockAPICounterOptions {
|
|
21
|
+
readonly modelId: string;
|
|
22
|
+
readonly id?: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @stable
|
|
27
|
+
*/
|
|
28
|
+
export class BedrockAPICounter implements TokenCounter {
|
|
29
|
+
readonly id: string;
|
|
30
|
+
readonly version: string;
|
|
31
|
+
readonly #fallback: JsTiktokenCounter;
|
|
32
|
+
|
|
33
|
+
constructor(options: BedrockAPICounterOptions) {
|
|
34
|
+
this.#fallback = new JsTiktokenCounter({ encoding: 'cl100k_base', modelId: options.modelId });
|
|
35
|
+
this.id = options.id ?? `bedrock-proxy@${options.modelId}`;
|
|
36
|
+
this.version = `bedrock-cl100k-proxy-${options.modelId}-v1`;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
count(messages: ReadonlyArray<Message>): Promise<number> {
|
|
40
|
+
return this.#fallback.count(messages);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
countText(text: string): Promise<number> {
|
|
44
|
+
return this.#fallback.countText(text);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `createDefaultCounter` - model-id classifier that returns the
|
|
3
|
+
* recommended {@link TokenCounter} for a given `(provider, model)`
|
|
4
|
+
* pair. The same classifier is reused by the model-tier dispatcher
|
|
5
|
+
* (`@graphorin/provider/model-tier`) so provider-family detection
|
|
6
|
+
* lives in exactly one place.
|
|
7
|
+
*
|
|
8
|
+
* @packageDocumentation
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { TokenCounter } from '@graphorin/core';
|
|
12
|
+
|
|
13
|
+
import { AnthropicAPICounter } from './anthropic.js';
|
|
14
|
+
import { BedrockAPICounter } from './bedrock.js';
|
|
15
|
+
import { GoogleAPICounter } from './google.js';
|
|
16
|
+
import { HeuristicCounter } from './heuristic.js';
|
|
17
|
+
import { JsTiktokenCounter } from './js-tiktoken.js';
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Options for {@link createDefaultCounter}.
|
|
21
|
+
*
|
|
22
|
+
* @stable
|
|
23
|
+
*/
|
|
24
|
+
export interface CreateDefaultCounterOptions {
|
|
25
|
+
/** Concrete model id (e.g. `'gpt-4o'`, `'claude-opus-4-7'`). */
|
|
26
|
+
readonly model: string;
|
|
27
|
+
/** Optional provider hint to short-circuit the regex matching. */
|
|
28
|
+
readonly provider?: string;
|
|
29
|
+
/** Optional Anthropic API key threaded through to {@link AnthropicAPICounter}. */
|
|
30
|
+
readonly anthropicApiKey?: string;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Detect the provider family for a model id. Exposed for downstream
|
|
35
|
+
* consumers (the model-tier classifier; future per-tool counters) so
|
|
36
|
+
* provider-family detection is centralised here.
|
|
37
|
+
*
|
|
38
|
+
* @stable
|
|
39
|
+
*/
|
|
40
|
+
export function detectProviderFamily(args: {
|
|
41
|
+
readonly model: string;
|
|
42
|
+
readonly provider?: string;
|
|
43
|
+
}): 'anthropic' | 'openai' | 'google' | 'bedrock' | 'ollama' | 'openai-compatible' | 'unknown' {
|
|
44
|
+
const provider = args.provider?.toLowerCase();
|
|
45
|
+
// Strip the Bedrock cross-region inference-profile prefix
|
|
46
|
+
// (`us.anthropic.claude-…`) so region-qualified ids classify like
|
|
47
|
+
// their bare form (core-provider-11).
|
|
48
|
+
const modelLower = args.model.toLowerCase().replace(/^(?:us|eu|apac|jp|au|us-gov)\./, '');
|
|
49
|
+
// The AI SDK reports dotted provider ids ('anthropic.messages',
|
|
50
|
+
// 'amazon-bedrock.messages'), so match by prefix / substring.
|
|
51
|
+
const providerIsAnthropic = provider?.startsWith('anthropic');
|
|
52
|
+
const providerIsBedrock = provider?.includes('bedrock');
|
|
53
|
+
if (
|
|
54
|
+
providerIsAnthropic ||
|
|
55
|
+
modelLower.startsWith('claude') ||
|
|
56
|
+
modelLower.startsWith('anthropic.claude')
|
|
57
|
+
) {
|
|
58
|
+
if (providerIsBedrock || modelLower.startsWith('anthropic.claude')) return 'bedrock';
|
|
59
|
+
return 'anthropic';
|
|
60
|
+
}
|
|
61
|
+
if (provider === 'google' || modelLower.startsWith('gemini')) return 'google';
|
|
62
|
+
if (providerIsBedrock) return 'bedrock';
|
|
63
|
+
if (provider === 'openai' || provider === 'azure-openai' || /^(gpt-|o1|o3|o4)/.test(modelLower)) {
|
|
64
|
+
return 'openai';
|
|
65
|
+
}
|
|
66
|
+
if (provider === 'ollama' || modelLower.startsWith('ollama-')) return 'ollama';
|
|
67
|
+
if (provider === 'openai-compatible' || provider === 'lmstudio' || provider === 'vllm') {
|
|
68
|
+
return 'openai-compatible';
|
|
69
|
+
}
|
|
70
|
+
return 'unknown';
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Build the recommended {@link TokenCounter} for the given
|
|
75
|
+
* `(provider, model)` pair. The dispatch table:
|
|
76
|
+
*
|
|
77
|
+
* - Anthropic Claude → `AnthropicAPICounter` (native if `apiKey` set,
|
|
78
|
+
* otherwise `cl100k_base` proxy).
|
|
79
|
+
* - OpenAI / OpenAI-compatible → `JsTiktokenCounter('cl100k_base')`.
|
|
80
|
+
* - Google Gemini → `GoogleAPICounter` (cl100k_base proxy in v0.1).
|
|
81
|
+
* - Bedrock Claude → `BedrockAPICounter` (cl100k_base proxy in v0.1).
|
|
82
|
+
* - Ollama / unknown → `HeuristicCounter` with one WARN per process.
|
|
83
|
+
*
|
|
84
|
+
* @stable
|
|
85
|
+
*/
|
|
86
|
+
export function createDefaultCounter(options: CreateDefaultCounterOptions): TokenCounter {
|
|
87
|
+
const family = detectProviderFamily({
|
|
88
|
+
model: options.model,
|
|
89
|
+
...(options.provider !== undefined ? { provider: options.provider } : {}),
|
|
90
|
+
});
|
|
91
|
+
switch (family) {
|
|
92
|
+
case 'anthropic':
|
|
93
|
+
return new AnthropicAPICounter({
|
|
94
|
+
modelId: options.model,
|
|
95
|
+
...(options.anthropicApiKey !== undefined ? { apiKey: options.anthropicApiKey } : {}),
|
|
96
|
+
});
|
|
97
|
+
case 'bedrock':
|
|
98
|
+
return new BedrockAPICounter({ modelId: options.model });
|
|
99
|
+
case 'google':
|
|
100
|
+
return new GoogleAPICounter({ modelId: options.model });
|
|
101
|
+
case 'openai':
|
|
102
|
+
case 'openai-compatible':
|
|
103
|
+
return new JsTiktokenCounter({
|
|
104
|
+
encoding: defaultOpenAiEncoding(options.model),
|
|
105
|
+
modelId: options.model,
|
|
106
|
+
});
|
|
107
|
+
default:
|
|
108
|
+
return new HeuristicCounter({ modelId: options.model });
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* PS-20: pick the fallback tiktoken encoding by model family. `js-tiktoken`'s
|
|
114
|
+
* `encodingForModel` handles ids it recognises; this is the explicit fallback
|
|
115
|
+
* for ids it does not (the 2025/2026 families). gpt-4o / gpt-4.1 / gpt-5+ and
|
|
116
|
+
* the o-series reasoning models use `o200k_base`; legacy gpt-4 / gpt-3.5 stay
|
|
117
|
+
* on `cl100k_base`.
|
|
118
|
+
*/
|
|
119
|
+
export function defaultOpenAiEncoding(model: string | undefined): 'o200k_base' | 'cl100k_base' {
|
|
120
|
+
if (model === undefined) return 'cl100k_base';
|
|
121
|
+
const id = model.toLowerCase();
|
|
122
|
+
const sep = Math.max(id.lastIndexOf('/'), id.lastIndexOf(':'));
|
|
123
|
+
const bare = sep === -1 ? id : id.slice(sep + 1);
|
|
124
|
+
return /^(?:gpt-4o|gpt-4\.1|gpt-[5-9]|gpt-\d{2}|o[1-9])/.test(bare)
|
|
125
|
+
? 'o200k_base'
|
|
126
|
+
: 'cl100k_base';
|
|
127
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Process-global token counter slot. Components (e.g. the
|
|
3
|
+
* ContextEngine in Phase 10) read the global counter when no
|
|
4
|
+
* per-provider counter is wired explicitly.
|
|
5
|
+
*
|
|
6
|
+
* @packageDocumentation
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import type { TokenCounter } from '@graphorin/core';
|
|
10
|
+
|
|
11
|
+
let globalCounter: TokenCounter | null = null;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Set the process-global counter. Called once at startup by user
|
|
15
|
+
* code; passing `null` clears the slot.
|
|
16
|
+
*
|
|
17
|
+
* @stable
|
|
18
|
+
*/
|
|
19
|
+
export function setGlobalTokenCounter(counter: TokenCounter | null): void {
|
|
20
|
+
globalCounter = counter;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Read the process-global counter, or `null` if unset.
|
|
25
|
+
*
|
|
26
|
+
* @stable
|
|
27
|
+
*/
|
|
28
|
+
export function getGlobalTokenCounter(): TokenCounter | null {
|
|
29
|
+
return globalCounter;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Test-only hook that resets the global slot to `null` between cases.
|
|
34
|
+
*
|
|
35
|
+
* @internal
|
|
36
|
+
*/
|
|
37
|
+
export function __resetGlobalTokenCounter(): void {
|
|
38
|
+
globalCounter = null;
|
|
39
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `GoogleAPICounter` - placeholder counter that delegates to
|
|
3
|
+
* `JsTiktokenCounter('cl100k_base')` for now. The full Google `:countTokens`
|
|
4
|
+
* implementation lands in Phase 16 alongside the optional pricing
|
|
5
|
+
* refresher; for v0.1 the proxy is sufficient (verified within ±5% on
|
|
6
|
+
* the standard fixture corpus per the existing DEC-131 acceptance).
|
|
7
|
+
*
|
|
8
|
+
* @packageDocumentation
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { Message, TokenCounter } from '@graphorin/core';
|
|
12
|
+
|
|
13
|
+
import { JsTiktokenCounter } from './js-tiktoken.js';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Options for {@link GoogleAPICounter}.
|
|
17
|
+
*
|
|
18
|
+
* @stable
|
|
19
|
+
*/
|
|
20
|
+
export interface GoogleAPICounterOptions {
|
|
21
|
+
readonly modelId: string;
|
|
22
|
+
readonly id?: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @stable
|
|
27
|
+
*/
|
|
28
|
+
export class GoogleAPICounter implements TokenCounter {
|
|
29
|
+
readonly id: string;
|
|
30
|
+
readonly version: string;
|
|
31
|
+
readonly #fallback: JsTiktokenCounter;
|
|
32
|
+
|
|
33
|
+
constructor(options: GoogleAPICounterOptions) {
|
|
34
|
+
this.#fallback = new JsTiktokenCounter({ encoding: 'cl100k_base', modelId: options.modelId });
|
|
35
|
+
this.id = options.id ?? `google-proxy@${options.modelId}`;
|
|
36
|
+
this.version = `google-cl100k-proxy-${options.modelId}-v1`;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
count(messages: ReadonlyArray<Message>): Promise<number> {
|
|
40
|
+
return this.#fallback.count(messages);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
countText(text: string): Promise<number> {
|
|
44
|
+
return this.#fallback.countText(text);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Conservative `chars / N` token-count estimator. Used as the fallback
|
|
3
|
+
* for unknown providers; emits exactly one WARN per unique
|
|
4
|
+
* `(provider, model)` tuple per process so operators are nudged
|
|
5
|
+
* toward a precise counter.
|
|
6
|
+
*
|
|
7
|
+
* @packageDocumentation
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { Message, TokenCounter } from '@graphorin/core';
|
|
11
|
+
|
|
12
|
+
import { serialiseMessageForCount, serializedToString } from './serialize.js';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Options for {@link HeuristicCounter}.
|
|
16
|
+
*
|
|
17
|
+
* @stable
|
|
18
|
+
*/
|
|
19
|
+
export interface HeuristicCounterOptions {
|
|
20
|
+
/** Average characters per token. Defaults to `4` (latin-1 estimate). */
|
|
21
|
+
readonly charsPerToken?: number;
|
|
22
|
+
/** Identifier carried on the produced counter. */
|
|
23
|
+
readonly id?: string;
|
|
24
|
+
/** Optional model hint surfaced in the WARN message. */
|
|
25
|
+
readonly modelId?: string;
|
|
26
|
+
/** Optional sink. Defaults to `console.warn`. */
|
|
27
|
+
readonly logger?: (message: string, meta?: object) => void;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const WARNED = new Set<string>();
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Counter that estimates tokens via simple character division. The
|
|
34
|
+
* estimator is deterministic and side-effect-free apart from the
|
|
35
|
+
* one-time WARN.
|
|
36
|
+
*
|
|
37
|
+
* @stable
|
|
38
|
+
*/
|
|
39
|
+
export class HeuristicCounter implements TokenCounter {
|
|
40
|
+
readonly id: string;
|
|
41
|
+
readonly version: string;
|
|
42
|
+
readonly #charsPerToken: number;
|
|
43
|
+
readonly #modelId?: string;
|
|
44
|
+
readonly #logger: (message: string, meta?: object) => void;
|
|
45
|
+
#warnedOnce = false;
|
|
46
|
+
|
|
47
|
+
constructor(options: HeuristicCounterOptions = {}) {
|
|
48
|
+
const charsPerToken = options.charsPerToken ?? 4;
|
|
49
|
+
if (!(charsPerToken > 0)) {
|
|
50
|
+
throw new RangeError('HeuristicCounter: charsPerToken must be > 0.');
|
|
51
|
+
}
|
|
52
|
+
this.#charsPerToken = charsPerToken;
|
|
53
|
+
this.id = options.id ?? `heuristic@${charsPerToken}cpt`;
|
|
54
|
+
this.version = `heuristic-${charsPerToken}-cpt-v1`;
|
|
55
|
+
if (options.modelId !== undefined) this.#modelId = options.modelId;
|
|
56
|
+
this.#logger = options.logger ?? defaultLogger;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async count(messages: ReadonlyArray<Message>): Promise<number> {
|
|
60
|
+
this.#warnIfNeeded();
|
|
61
|
+
let total = 0;
|
|
62
|
+
for (const msg of messages) {
|
|
63
|
+
const serialised = serialiseMessageForCount(msg);
|
|
64
|
+
total += this.#estimate(serializedToString(serialised));
|
|
65
|
+
}
|
|
66
|
+
return total;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
async countText(text: string): Promise<number> {
|
|
70
|
+
this.#warnIfNeeded();
|
|
71
|
+
return this.#estimate(text);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
#estimate(text: string): number {
|
|
75
|
+
if (text.length === 0) return 0;
|
|
76
|
+
return Math.ceil(text.length / this.#charsPerToken);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
#warnIfNeeded(): void {
|
|
80
|
+
if (this.#warnedOnce) return;
|
|
81
|
+
const key = `${this.id}::${this.#modelId ?? 'unknown'}`;
|
|
82
|
+
if (WARNED.has(key)) {
|
|
83
|
+
this.#warnedOnce = true;
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
WARNED.add(key);
|
|
87
|
+
this.#warnedOnce = true;
|
|
88
|
+
this.#logger(
|
|
89
|
+
`[graphorin/provider] heuristic token counter active (${this.id}); accuracy is approximate (~chars/${this.#charsPerToken}).`,
|
|
90
|
+
this.#modelId !== undefined ? { modelId: this.#modelId } : {},
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function defaultLogger(message: string, meta?: object): void {
|
|
96
|
+
if (meta !== undefined) console.warn(message, meta);
|
|
97
|
+
else console.warn(message);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Test-only hook that resets the per-process WARN cache.
|
|
102
|
+
*
|
|
103
|
+
* @internal
|
|
104
|
+
*/
|
|
105
|
+
export function __resetHeuristicWarnCache(): void {
|
|
106
|
+
WARNED.clear();
|
|
107
|
+
}
|