@bitbaum/ai-kit 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -0
- package/dist/capability/classify.d.ts +52 -0
- package/dist/capability/classify.js +96 -0
- package/dist/capability/decide.d.ts +75 -0
- package/dist/capability/decide.js +143 -0
- package/dist/capability/index.d.ts +33 -0
- package/dist/capability/index.js +2 -0
- package/dist/capability/types.d.ts +81 -0
- package/dist/capability/types.js +18 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/policy.d.ts +127 -0
- package/dist/policy.js +122 -0
- package/package.json +5 -2
- package/src/capability/classify.ts +149 -0
- package/src/capability/decide.ts +183 -0
- package/src/capability/index.ts +52 -0
- package/src/capability/types.ts +87 -0
- package/src/index.ts +15 -0
- package/src/policy.ts +206 -0
package/dist/policy.d.ts
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Who may spend what, and what to offer when they may not.
|
|
3
|
+
*
|
|
4
|
+
* Pure by the same rule as `fair-share`: no database, no clock, no provider.
|
|
5
|
+
* The caller owns "what has this user done"; this file owns "what are they
|
|
6
|
+
* entitled to, and what should they be shown next".
|
|
7
|
+
*
|
|
8
|
+
* ── WHY THIS IS ONE OBJECT AND NOT TWO ───────────────────────────────────────
|
|
9
|
+
*
|
|
10
|
+
* Across this fleet the same feature was built twice, half each. One app
|
|
11
|
+
* enforces a daily budget with no interface at all — the user is refused and
|
|
12
|
+
* never learns why, or what their share was. Another shows a quota meter with
|
|
13
|
+
* its own separate notion of tier, which the enforcement path does not read.
|
|
14
|
+
* Both are correct in isolation and they cannot agree, because there was no
|
|
15
|
+
* single statement of the policy for them to agree ABOUT.
|
|
16
|
+
*
|
|
17
|
+
* So the policy is data, and the same object answers both questions: the gate
|
|
18
|
+
* asks `decide()` whether to allow, and the settings page asks `decide()` what
|
|
19
|
+
* to draw. A number on screen that the gate does not use is decoration; a gate
|
|
20
|
+
* whose reasoning cannot be rendered is a wall.
|
|
21
|
+
*
|
|
22
|
+
* ── WHY FREE CAPACITY IS A LADDER AND NOT A LIMIT ────────────────────────────
|
|
23
|
+
*
|
|
24
|
+
* Shared free capacity is a fixed cost carried on behalf of strangers, and its
|
|
25
|
+
* only job is to be good enough, once, that someone wants more of it. The
|
|
26
|
+
* moment it runs out is therefore the only moment anybody changes anything —
|
|
27
|
+
* which makes the refusal the most important screen in the product, not an
|
|
28
|
+
* error state.
|
|
29
|
+
*
|
|
30
|
+
* Hence `Wall`: a refusal always carries the ways out, in order, and the
|
|
31
|
+
* default order puts the exits that cost the user NOTHING ahead of the one that
|
|
32
|
+
* costs money. Asking someone to make a free account elsewhere converts far
|
|
33
|
+
* better than asking them to pay, and it removes them from the shared pool just
|
|
34
|
+
* as completely.
|
|
35
|
+
*
|
|
36
|
+
* And there is always a floor. `wait` is a real rung with a real time on it,
|
|
37
|
+
* because "come back at 01:00" is a path and "rate limit exceeded" is a dead
|
|
38
|
+
* end that reads as broken.
|
|
39
|
+
*/
|
|
40
|
+
/** Where the capacity for a turn comes from. */
|
|
41
|
+
export type PoolId =
|
|
42
|
+
/** Your keys, your cost, shared by everyone. The sample. */
|
|
43
|
+
"platform"
|
|
44
|
+
/** The user's own key. Free to you, uncapped for them. */
|
|
45
|
+
| "user"
|
|
46
|
+
/** A model on the user's machine. Free to everyone. */
|
|
47
|
+
| "local";
|
|
48
|
+
/** The ways off the shared pool, and the floor beneath them. */
|
|
49
|
+
export type RungId = "byok" | "local" | "earn" | "paid" | "wait";
|
|
50
|
+
export interface TierPolicy {
|
|
51
|
+
/** Turns per UTC day. `null` means uncapped — the user is not your cost. */
|
|
52
|
+
turnsPerDay: number | null;
|
|
53
|
+
pool: PoolId;
|
|
54
|
+
/** Present when spending is metered against a balance rather than a count. */
|
|
55
|
+
meter?: "credits";
|
|
56
|
+
}
|
|
57
|
+
export interface AiPolicy {
|
|
58
|
+
/** Tier id → what it may do. App-defined; only the shape is fixed here. */
|
|
59
|
+
tiers: Record<string, TierPolicy>;
|
|
60
|
+
/**
|
|
61
|
+
* What to offer at a wall, in order.
|
|
62
|
+
*
|
|
63
|
+
* Defaults to free exits first. Overriding it to lead with `paid` is a
|
|
64
|
+
* product decision this package will not make for you, but it is worth
|
|
65
|
+
* knowing you are then asking for money from someone who has not yet
|
|
66
|
+
* finished evaluating the thing.
|
|
67
|
+
*/
|
|
68
|
+
ladder?: RungId[];
|
|
69
|
+
}
|
|
70
|
+
/** Sensible order: everything that costs the user nothing, then money, then the floor. */
|
|
71
|
+
export declare const DEFAULT_LADDER: RungId[];
|
|
72
|
+
/** What the caller knows about this user right now. */
|
|
73
|
+
export interface UserState {
|
|
74
|
+
/** Which tier they are on — a key of `policy.tiers`. */
|
|
75
|
+
tier: string;
|
|
76
|
+
/** Turns already spent this UTC day. */
|
|
77
|
+
spentToday: number;
|
|
78
|
+
/** Credit balance, when the tier meters credits. */
|
|
79
|
+
credits?: number;
|
|
80
|
+
/** Rungs this user could actually take. A rung they cannot reach is not offered. */
|
|
81
|
+
available?: Partial<Record<RungId, boolean>>;
|
|
82
|
+
}
|
|
83
|
+
/** One way out of a wall, ready to render. */
|
|
84
|
+
export interface WallOption {
|
|
85
|
+
rung: RungId;
|
|
86
|
+
/** Whether this user can actually take it. */
|
|
87
|
+
available: boolean;
|
|
88
|
+
}
|
|
89
|
+
/** Everything needed to draw a refusal that is not a dead end. */
|
|
90
|
+
export interface Wall {
|
|
91
|
+
reason: "day-spent" | "no-credits" | "unknown-tier";
|
|
92
|
+
/** Ordered, best first. Never empty — `wait` is always last. */
|
|
93
|
+
options: WallOption[];
|
|
94
|
+
/** Epoch ms when the allowance refills, when the caller supplied a day boundary. */
|
|
95
|
+
resetAt: number | null;
|
|
96
|
+
}
|
|
97
|
+
export interface Decision {
|
|
98
|
+
allowed: boolean;
|
|
99
|
+
/** The resolved tier policy, or null when the tier is not in the policy. */
|
|
100
|
+
policy: TierPolicy | null;
|
|
101
|
+
pool: PoolId | null;
|
|
102
|
+
/**
|
|
103
|
+
* Turns left today. `null` means uncapped, which is NOT the same as zero and
|
|
104
|
+
* must not render as a gauge — there is nothing to draw a level against.
|
|
105
|
+
*/
|
|
106
|
+
remaining: number | null;
|
|
107
|
+
/** Present only when `allowed` is false. */
|
|
108
|
+
wall?: Wall;
|
|
109
|
+
}
|
|
110
|
+
/** Next UTC midnight — when a per-day allowance refills. */
|
|
111
|
+
export declare function nextUtcReset(now?: number): number;
|
|
112
|
+
/**
|
|
113
|
+
* May this user spend a turn, and if not, what should they be offered?
|
|
114
|
+
*
|
|
115
|
+
* The same call answers the gate and the settings page. A caller that only
|
|
116
|
+
* needs the number reads `remaining`; one drawing a refusal reads `wall`.
|
|
117
|
+
*/
|
|
118
|
+
export declare function decide(policy: AiPolicy, state: UserState, now?: number): Decision;
|
|
119
|
+
/**
|
|
120
|
+
* Should the remaining count be put in front of the user yet?
|
|
121
|
+
*
|
|
122
|
+
* A permanent gauge at 96% trains people to ignore the one at 4%. The indicator
|
|
123
|
+
* earns attention near the edge and stays quiet before it — so this returns
|
|
124
|
+
* false for an uncapped tier (nothing to say) and for a user who has barely
|
|
125
|
+
* started.
|
|
126
|
+
*/
|
|
127
|
+
export declare function shouldSurface(decision: Decision, threshold?: number): boolean;
|
package/dist/policy.js
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Who may spend what, and what to offer when they may not.
|
|
3
|
+
*
|
|
4
|
+
* Pure by the same rule as `fair-share`: no database, no clock, no provider.
|
|
5
|
+
* The caller owns "what has this user done"; this file owns "what are they
|
|
6
|
+
* entitled to, and what should they be shown next".
|
|
7
|
+
*
|
|
8
|
+
* ── WHY THIS IS ONE OBJECT AND NOT TWO ───────────────────────────────────────
|
|
9
|
+
*
|
|
10
|
+
* Across this fleet the same feature was built twice, half each. One app
|
|
11
|
+
* enforces a daily budget with no interface at all — the user is refused and
|
|
12
|
+
* never learns why, or what their share was. Another shows a quota meter with
|
|
13
|
+
* its own separate notion of tier, which the enforcement path does not read.
|
|
14
|
+
* Both are correct in isolation and they cannot agree, because there was no
|
|
15
|
+
* single statement of the policy for them to agree ABOUT.
|
|
16
|
+
*
|
|
17
|
+
* So the policy is data, and the same object answers both questions: the gate
|
|
18
|
+
* asks `decide()` whether to allow, and the settings page asks `decide()` what
|
|
19
|
+
* to draw. A number on screen that the gate does not use is decoration; a gate
|
|
20
|
+
* whose reasoning cannot be rendered is a wall.
|
|
21
|
+
*
|
|
22
|
+
* ── WHY FREE CAPACITY IS A LADDER AND NOT A LIMIT ────────────────────────────
|
|
23
|
+
*
|
|
24
|
+
* Shared free capacity is a fixed cost carried on behalf of strangers, and its
|
|
25
|
+
* only job is to be good enough, once, that someone wants more of it. The
|
|
26
|
+
* moment it runs out is therefore the only moment anybody changes anything —
|
|
27
|
+
* which makes the refusal the most important screen in the product, not an
|
|
28
|
+
* error state.
|
|
29
|
+
*
|
|
30
|
+
* Hence `Wall`: a refusal always carries the ways out, in order, and the
|
|
31
|
+
* default order puts the exits that cost the user NOTHING ahead of the one that
|
|
32
|
+
* costs money. Asking someone to make a free account elsewhere converts far
|
|
33
|
+
* better than asking them to pay, and it removes them from the shared pool just
|
|
34
|
+
* as completely.
|
|
35
|
+
*
|
|
36
|
+
* And there is always a floor. `wait` is a real rung with a real time on it,
|
|
37
|
+
* because "come back at 01:00" is a path and "rate limit exceeded" is a dead
|
|
38
|
+
* end that reads as broken.
|
|
39
|
+
*/
|
|
40
|
+
/** Sensible order: everything that costs the user nothing, then money, then the floor. */
|
|
41
|
+
export const DEFAULT_LADDER = ["byok", "local", "earn", "paid", "wait"];
|
|
42
|
+
/** Next UTC midnight — when a per-day allowance refills. */
|
|
43
|
+
export function nextUtcReset(now = Date.now()) {
|
|
44
|
+
const d = new Date(now);
|
|
45
|
+
return Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate() + 1);
|
|
46
|
+
}
|
|
47
|
+
function buildWall(reason, policy, state, now) {
|
|
48
|
+
const order = policy.ladder ?? DEFAULT_LADDER;
|
|
49
|
+
const options = order
|
|
50
|
+
// `wait` is appended below so it can never be configured away.
|
|
51
|
+
.filter((rung) => rung !== "wait")
|
|
52
|
+
.map((rung) => ({ rung, available: state.available?.[rung] ?? true }));
|
|
53
|
+
// The floor. A refusal without a way forward reads as a broken product, and
|
|
54
|
+
// the honest way forward is always "this refills, here is when".
|
|
55
|
+
options.push({ rung: "wait", available: true });
|
|
56
|
+
return { reason, options, resetAt: nextUtcReset(now) };
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* May this user spend a turn, and if not, what should they be offered?
|
|
60
|
+
*
|
|
61
|
+
* The same call answers the gate and the settings page. A caller that only
|
|
62
|
+
* needs the number reads `remaining`; one drawing a refusal reads `wall`.
|
|
63
|
+
*/
|
|
64
|
+
export function decide(policy, state, now = Date.now()) {
|
|
65
|
+
const tier = policy.tiers[state.tier];
|
|
66
|
+
// An unknown tier is not a licence to spend. It is a configuration error, and
|
|
67
|
+
// failing closed here is the difference between a typo costing nothing and a
|
|
68
|
+
// typo handing out the shared pool.
|
|
69
|
+
if (!tier) {
|
|
70
|
+
return {
|
|
71
|
+
allowed: false,
|
|
72
|
+
policy: null,
|
|
73
|
+
pool: null,
|
|
74
|
+
remaining: 0,
|
|
75
|
+
wall: buildWall("unknown-tier", policy, state, now),
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
if (tier.meter === "credits") {
|
|
79
|
+
const credits = state.credits ?? 0;
|
|
80
|
+
return credits > 0
|
|
81
|
+
? { allowed: true, policy: tier, pool: tier.pool, remaining: null }
|
|
82
|
+
: {
|
|
83
|
+
allowed: false,
|
|
84
|
+
policy: tier,
|
|
85
|
+
pool: tier.pool,
|
|
86
|
+
remaining: 0,
|
|
87
|
+
wall: buildWall("no-credits", policy, state, now),
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
// Uncapped: the user is not your cost, so there is no number to police.
|
|
91
|
+
if (tier.turnsPerDay === null) {
|
|
92
|
+
return { allowed: true, policy: tier, pool: tier.pool, remaining: null };
|
|
93
|
+
}
|
|
94
|
+
const remaining = Math.max(0, tier.turnsPerDay - state.spentToday);
|
|
95
|
+
return remaining > 0
|
|
96
|
+
? { allowed: true, policy: tier, pool: tier.pool, remaining }
|
|
97
|
+
: {
|
|
98
|
+
allowed: false,
|
|
99
|
+
policy: tier,
|
|
100
|
+
pool: tier.pool,
|
|
101
|
+
remaining: 0,
|
|
102
|
+
wall: buildWall("day-spent", policy, state, now),
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Should the remaining count be put in front of the user yet?
|
|
107
|
+
*
|
|
108
|
+
* A permanent gauge at 96% trains people to ignore the one at 4%. The indicator
|
|
109
|
+
* earns attention near the edge and stays quiet before it — so this returns
|
|
110
|
+
* false for an uncapped tier (nothing to say) and for a user who has barely
|
|
111
|
+
* started.
|
|
112
|
+
*/
|
|
113
|
+
export function shouldSurface(decision, threshold = 0.34) {
|
|
114
|
+
if (decision.remaining === null)
|
|
115
|
+
return false;
|
|
116
|
+
if (!decision.allowed)
|
|
117
|
+
return true;
|
|
118
|
+
const cap = decision.policy?.turnsPerDay;
|
|
119
|
+
if (!cap)
|
|
120
|
+
return false;
|
|
121
|
+
return decision.remaining / cap <= threshold;
|
|
122
|
+
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bitbaum/ai-kit",
|
|
3
|
-
"version": "1.
|
|
4
|
-
"description": "One install for the AI layer of an app: which model to call, what to do when the vendor retires it, how to walk the fallback chain and know when none of it worked, how to read the three kinds of 429, a fair daily budget across users, headless AI form filling \u2014 and now the model registry (one SSOT for every callable id, with the paid/free boundary as a field) and the grounding harness (facts, contract, deterministic fabrication check).",
|
|
3
|
+
"version": "1.3.0",
|
|
5
4
|
"license": "MIT",
|
|
6
5
|
"author": "Mao Nakamoto",
|
|
7
6
|
"homepage": "https://github.com/bitbaum/ai-kit#readme",
|
|
@@ -67,6 +66,10 @@
|
|
|
67
66
|
"types": "./dist/web/index.d.ts",
|
|
68
67
|
"default": "./dist/web/index.js"
|
|
69
68
|
},
|
|
69
|
+
"./capability": {
|
|
70
|
+
"types": "./dist/capability/index.d.ts",
|
|
71
|
+
"default": "./dist/capability/index.js"
|
|
72
|
+
},
|
|
70
73
|
"./grounding": {
|
|
71
74
|
"types": "./dist/grounding/index.d.ts",
|
|
72
75
|
"require": "./dist-cjs/grounding/index.js",
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reading a real response for what it proves about capability.
|
|
3
|
+
*
|
|
4
|
+
* The whole design rests on one asymmetry:
|
|
5
|
+
*
|
|
6
|
+
* A POSITIVE is cheap. One response carrying `tool_calls` proves, beyond
|
|
7
|
+
* argument, that this model on this provider with this key can call tools.
|
|
8
|
+
* Write it down immediately.
|
|
9
|
+
*
|
|
10
|
+
* A NEGATIVE is expensive and sticky. If a 400 gets recorded as "this model
|
|
11
|
+
* has no tools", that model is crippled until the record expires — and the
|
|
12
|
+
* user sees a capable model behaving like a toy with nothing explaining why.
|
|
13
|
+
* So a negative requires the vendor to SAY it is about tools. A 400 for a
|
|
14
|
+
* context-length overflow, a malformed parameter, a content filter or a
|
|
15
|
+
* billing problem proves nothing about tools and must be recorded as
|
|
16
|
+
* nothing at all.
|
|
17
|
+
*
|
|
18
|
+
* That asymmetry is why `record: false` exists. Most failures are
|
|
19
|
+
* uninformative, and the correct response to an uninformative failure is to
|
|
20
|
+
* learn nothing, not to guess.
|
|
21
|
+
*/
|
|
22
|
+
import type { Classification } from "./types.js";
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Phrases that mean "this model does not do tools", conservatively.
|
|
26
|
+
*
|
|
27
|
+
* Every entry names tools or functions explicitly. Deliberately absent:
|
|
28
|
+
* "invalid request", "bad parameter", "unsupported" on its own — each of those
|
|
29
|
+
* appears in vendor 400s for a dozen unrelated reasons, and a match on one of
|
|
30
|
+
* them would silently disable a working model. When in doubt the answer is to
|
|
31
|
+
* record nothing; an unobserved model gets asked again on the next message,
|
|
32
|
+
* whereas a wrongly-negative one does not.
|
|
33
|
+
*/
|
|
34
|
+
const TOOLS_UNSUPPORTED_PATTERNS: RegExp[] = [
|
|
35
|
+
/tool[\s_-]?(use|call|calls|calling)\s+(is\s+)?(not|un)[\s_-]?support/i,
|
|
36
|
+
/does\s+not\s+support\s+tool/i,
|
|
37
|
+
/doesn'?t\s+support\s+tool/i,
|
|
38
|
+
/no\s+support\s+for\s+tool/i,
|
|
39
|
+
/function[\s_-]?call(ing)?\s+(is\s+)?(not|un)[\s_-]?support/i,
|
|
40
|
+
/does\s+not\s+support\s+function/i,
|
|
41
|
+
/doesn'?t\s+support\s+function/i,
|
|
42
|
+
/model\s+.{0,60}?\s+does\s+not\s+support\s+(the\s+)?(`?tools`?|`?functions`?)/i,
|
|
43
|
+
/unsupported\s+parameter:?\s*'?"?tools?"?'?/i,
|
|
44
|
+
/unknown\s+(field|parameter):?\s*'?"?tools?"?'?/i,
|
|
45
|
+
/`?tools`?\s+is\s+not\s+(a\s+)?(valid|supported|allowed)/i,
|
|
46
|
+
];
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Does this error body explicitly say the model cannot do tools?
|
|
50
|
+
*
|
|
51
|
+
* Conservative on purpose — see the header. A false positive here is a model
|
|
52
|
+
* permanently downgraded for a reason nobody can see; a false negative just
|
|
53
|
+
* means we ask again next time, which costs one request.
|
|
54
|
+
*/
|
|
55
|
+
export function saysToolsUnsupported(body: string): boolean {
|
|
56
|
+
if (!body) return false;
|
|
57
|
+
return TOOLS_UNSUPPORTED_PATTERNS.some((re) => re.test(body));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Shape of an OpenAI-compatible chat completion, as far as we care. */
|
|
61
|
+
type ChatBody = {
|
|
62
|
+
choices?: Array<{
|
|
63
|
+
finish_reason?: unknown;
|
|
64
|
+
message?: { content?: unknown; tool_calls?: unknown };
|
|
65
|
+
}>;
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
export type ToolAttempt = {
|
|
69
|
+
/** HTTP status. 0 or undefined for a transport failure. */
|
|
70
|
+
status?: number;
|
|
71
|
+
/** Parsed JSON body, when there was one. */
|
|
72
|
+
parsed?: unknown;
|
|
73
|
+
/** Raw body text. Used only for error classification. */
|
|
74
|
+
bodyText?: string;
|
|
75
|
+
/**
|
|
76
|
+
* Did the caller's own text-protocol parser find a usable tool call in the
|
|
77
|
+
* assistant's prose? Only the app knows its envelope, so it answers this.
|
|
78
|
+
* Absent means "not checked", which is not the same as "no".
|
|
79
|
+
*/
|
|
80
|
+
textProtocolFound?: boolean;
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* What does this attempt prove?
|
|
85
|
+
*
|
|
86
|
+
* Call it after EVERY request that carried tool definitions. Real traffic then
|
|
87
|
+
* classifies every model a user brings, on its first message, at no extra cost
|
|
88
|
+
* — which is the property that makes this scale to models nobody has heard of.
|
|
89
|
+
*/
|
|
90
|
+
export function classifyToolAttempt(attempt: ToolAttempt): Classification {
|
|
91
|
+
const { status, parsed, bodyText = "", textProtocolFound } = attempt;
|
|
92
|
+
|
|
93
|
+
// ── transport failures prove nothing ────────────────────────────────────
|
|
94
|
+
if (!status) {
|
|
95
|
+
return { verdict: "unobserved", record: false, evidence: "no response" };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// ── the vendor refused, and we must be careful about why ────────────────
|
|
99
|
+
if (status >= 400) {
|
|
100
|
+
if (saysToolsUnsupported(bodyText)) {
|
|
101
|
+
return {
|
|
102
|
+
verdict: "none",
|
|
103
|
+
record: true,
|
|
104
|
+
evidence: `${status}: the vendor says this model does not support tools`,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
// Everything else — 429, 401, 500, a 400 about context length or a bad
|
|
108
|
+
// parameter — says nothing about tools. Learning nothing is correct.
|
|
109
|
+
return {
|
|
110
|
+
verdict: "unobserved",
|
|
111
|
+
record: false,
|
|
112
|
+
evidence: `${status}: not a statement about tool support`,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// ── a success: did it actually call a tool? ─────────────────────────────
|
|
117
|
+
const body = (parsed ?? {}) as ChatBody;
|
|
118
|
+
const choice = body.choices?.[0];
|
|
119
|
+
const toolCalls = choice?.message?.tool_calls;
|
|
120
|
+
|
|
121
|
+
if (Array.isArray(toolCalls) && toolCalls.length > 0) {
|
|
122
|
+
return { verdict: "native", record: true, evidence: "returned tool_calls" };
|
|
123
|
+
}
|
|
124
|
+
if (choice?.finish_reason === "tool_calls") {
|
|
125
|
+
return { verdict: "native", record: true, evidence: "finish_reason was tool_calls" };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// The app's own envelope parser found a call in the prose. That is the text
|
|
129
|
+
// protocol, and it is a real capability — five of nine free models probed in
|
|
130
|
+
// this fleet answer only this way.
|
|
131
|
+
if (textProtocolFound === true) {
|
|
132
|
+
return {
|
|
133
|
+
verdict: "text",
|
|
134
|
+
record: true,
|
|
135
|
+
evidence: "no tool_calls, but a tool call was parsed from the text",
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// A successful answer with no tool call is the ambiguous case, and the
|
|
140
|
+
// ambiguity is real: the model may be incapable, or it may simply have
|
|
141
|
+
// decided no tool was needed — which is the correct behaviour for most
|
|
142
|
+
// messages. Treating this as evidence of incapacity would mark almost every
|
|
143
|
+
// model `none` within a few turns of ordinary chat.
|
|
144
|
+
return {
|
|
145
|
+
verdict: "unobserved",
|
|
146
|
+
record: false,
|
|
147
|
+
evidence: "answered without calling a tool, which is not evidence either way",
|
|
148
|
+
};
|
|
149
|
+
}
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What to send this time, and what to tell the user we can do.
|
|
3
|
+
*
|
|
4
|
+
* Two different questions, deliberately separated:
|
|
5
|
+
*
|
|
6
|
+
* `planToolAttempt` decides what to PUT ON THE WIRE. It is optimistic about
|
|
7
|
+
* an unobserved model, because the only way to learn is to ask, and the cost
|
|
8
|
+
* of asking is one request that may ignore the tools.
|
|
9
|
+
*
|
|
10
|
+
* `claimableVerdict` decides what to SAY. It is pessimistic about an
|
|
11
|
+
* unobserved model, because promising a capability we have never seen is how
|
|
12
|
+
* an assistant comes to announce an action it cannot perform.
|
|
13
|
+
*
|
|
14
|
+
* Those two pulling in opposite directions is the whole point. A single
|
|
15
|
+
* "supportsTools" boolean cannot express it, and every app that has tried has
|
|
16
|
+
* either refused to learn or lied to its users.
|
|
17
|
+
*/
|
|
18
|
+
import { createHash } from "node:crypto";
|
|
19
|
+
import type { CapabilityKind, CapabilityRecord, ToolVerdict } from "./types.js";
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* How long an observation stands before it must be re-earned.
|
|
23
|
+
*
|
|
24
|
+
* Models change under their own names — a vendor updates the weights behind an
|
|
25
|
+
* alias, an org enables a feature, a local user swaps a quantization. A record
|
|
26
|
+
* with no expiry is a hardcoded list again, just one we wrote ourselves.
|
|
27
|
+
*
|
|
28
|
+
* Negatives expire sooner than positives: a model that gained tool support and
|
|
29
|
+
* is still marked `none` is invisibly crippled, while a model that lost it
|
|
30
|
+
* announces itself loudly on the next call.
|
|
31
|
+
*/
|
|
32
|
+
export const DEFAULT_TTL_MS = {
|
|
33
|
+
native: 30 * 24 * 60 * 60 * 1000,
|
|
34
|
+
text: 30 * 24 * 60 * 60 * 1000,
|
|
35
|
+
none: 7 * 24 * 60 * 60 * 1000,
|
|
36
|
+
unobserved: 0,
|
|
37
|
+
} as const;
|
|
38
|
+
|
|
39
|
+
export function isStale(
|
|
40
|
+
record: Pick<CapabilityRecord, "verdict" | "observedAt">,
|
|
41
|
+
now: Date = new Date(),
|
|
42
|
+
ttl: Partial<Record<ToolVerdict, number>> = {},
|
|
43
|
+
): boolean {
|
|
44
|
+
const limit = ttl[record.verdict] ?? DEFAULT_TTL_MS[record.verdict];
|
|
45
|
+
if (!limit) return true;
|
|
46
|
+
const age = now.getTime() - new Date(record.observedAt).getTime();
|
|
47
|
+
return !Number.isFinite(age) || age > limit;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** The verdict a record still supports, or `unobserved` once it has expired. */
|
|
51
|
+
export function currentVerdict(
|
|
52
|
+
record: CapabilityRecord | null | undefined,
|
|
53
|
+
now: Date = new Date(),
|
|
54
|
+
ttl: Partial<Record<ToolVerdict, number>> = {},
|
|
55
|
+
): ToolVerdict {
|
|
56
|
+
if (!record) return "unobserved";
|
|
57
|
+
return isStale(record, now, ttl) ? "unobserved" : record.verdict;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export type ToolPlan = {
|
|
61
|
+
/** Put tool definitions on the request? */
|
|
62
|
+
sendTools: boolean;
|
|
63
|
+
/** Parse the prose for a tool envelope as well? */
|
|
64
|
+
expectTextProtocol: boolean;
|
|
65
|
+
/** True when this request is also the thing that will teach us. */
|
|
66
|
+
isLearning: boolean;
|
|
67
|
+
reason: string;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* What to send. Optimistic about the unknown, because asking is how we learn
|
|
72
|
+
* and a declared prior is only a guess about where to start.
|
|
73
|
+
*/
|
|
74
|
+
export function planToolAttempt(input: {
|
|
75
|
+
observed: ToolVerdict;
|
|
76
|
+
/** What a registry or the vendor's docs claim. A prior, never a fact. */
|
|
77
|
+
declared?: ToolVerdict;
|
|
78
|
+
}): ToolPlan {
|
|
79
|
+
const { observed, declared } = input;
|
|
80
|
+
|
|
81
|
+
if (observed === "native") {
|
|
82
|
+
return {
|
|
83
|
+
sendTools: true,
|
|
84
|
+
expectTextProtocol: false,
|
|
85
|
+
isLearning: false,
|
|
86
|
+
reason: "observed to return tool_calls",
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
if (observed === "text") {
|
|
90
|
+
return {
|
|
91
|
+
sendTools: false,
|
|
92
|
+
expectTextProtocol: true,
|
|
93
|
+
isLearning: false,
|
|
94
|
+
reason: "observed to answer tools only in prose",
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
if (observed === "none") {
|
|
98
|
+
return {
|
|
99
|
+
sendTools: false,
|
|
100
|
+
expectTextProtocol: false,
|
|
101
|
+
isLearning: false,
|
|
102
|
+
reason: "the vendor said this model does not support tools",
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Unobserved. Ask — and accept EITHER answer, because a model that ignores
|
|
107
|
+
// the definitions and writes the envelope in prose is capable, just not
|
|
108
|
+
// natively, and a native-only client silently loses most of a free chain.
|
|
109
|
+
return {
|
|
110
|
+
sendTools: declared !== "none",
|
|
111
|
+
expectTextProtocol: true,
|
|
112
|
+
isLearning: true,
|
|
113
|
+
reason:
|
|
114
|
+
declared === "none"
|
|
115
|
+
? "never observed, and the registry says no — asking in prose only"
|
|
116
|
+
: "never observed — this request is also the probe",
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* What we may TELL the user, and what the prompt may claim.
|
|
122
|
+
*
|
|
123
|
+
* Pessimistic about the unknown. `unobserved` returns `none` here on purpose:
|
|
124
|
+
* until a model has demonstrated a capability, an assistant that announces it
|
|
125
|
+
* is writing a cheque the model may not honour, and the user discovers that as
|
|
126
|
+
* a broken promise rather than as a missing feature.
|
|
127
|
+
*/
|
|
128
|
+
export function claimableVerdict(observed: ToolVerdict): Exclude<ToolVerdict, "unobserved"> {
|
|
129
|
+
return observed === "unobserved" ? "none" : observed;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* A stable, non-reversible handle for the credential an observation was made
|
|
134
|
+
* through. Capability differs per key, so observations must not leak across
|
|
135
|
+
* keys — and the key itself must never be stored to achieve that.
|
|
136
|
+
*/
|
|
137
|
+
export function scopeKey(secret: string | undefined | null): string {
|
|
138
|
+
if (!secret) return "anonymous";
|
|
139
|
+
return createHash("sha256").update(secret).digest("hex").slice(0, 16);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Build a record from a classification. Keeps `observedAt` in one place. */
|
|
143
|
+
export function makeRecord(input: {
|
|
144
|
+
provider: string;
|
|
145
|
+
model: string;
|
|
146
|
+
scope: string;
|
|
147
|
+
capability: CapabilityKind;
|
|
148
|
+
verdict: ToolVerdict;
|
|
149
|
+
via: CapabilityRecord["via"];
|
|
150
|
+
evidence?: string;
|
|
151
|
+
now?: Date;
|
|
152
|
+
}): CapabilityRecord {
|
|
153
|
+
return {
|
|
154
|
+
provider: input.provider,
|
|
155
|
+
model: input.model,
|
|
156
|
+
scope: input.scope,
|
|
157
|
+
capability: input.capability,
|
|
158
|
+
verdict: input.verdict,
|
|
159
|
+
via: input.via,
|
|
160
|
+
observedAt: (input.now ?? new Date()).toISOString(),
|
|
161
|
+
...(input.evidence ? { evidence: input.evidence } : {}),
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Should a new observation overwrite the stored one?
|
|
167
|
+
*
|
|
168
|
+
* Strength beats age, and `live` beats `declared`, so a real call always
|
|
169
|
+
* overrules a registry guess. Between two observations of equal provenance the
|
|
170
|
+
* newer wins — including a `none` replacing a `native`, because a model really
|
|
171
|
+
* can lose a capability and refusing to believe that is how a chain keeps
|
|
172
|
+
* calling something that no longer works.
|
|
173
|
+
*/
|
|
174
|
+
export function shouldReplace(
|
|
175
|
+
existing: CapabilityRecord | null | undefined,
|
|
176
|
+
incoming: CapabilityRecord,
|
|
177
|
+
): boolean {
|
|
178
|
+
if (!existing) return true;
|
|
179
|
+
const rank = { declared: 0, probe: 1, live: 2 } as const;
|
|
180
|
+
if (rank[incoming.via] > rank[existing.via]) return true;
|
|
181
|
+
if (rank[incoming.via] < rank[existing.via]) return false;
|
|
182
|
+
return new Date(incoming.observedAt).getTime() >= new Date(existing.observedAt).getTime();
|
|
183
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ai-kit/capability — what a model can do, observed rather than declared.
|
|
3
|
+
*
|
|
4
|
+
* Replaces the list every app writes and every app gets wrong:
|
|
5
|
+
*
|
|
6
|
+
* const TOOL_CAPABLE_PROVIDERS = ['groq', 'openrouter'];
|
|
7
|
+
*
|
|
8
|
+
* That line is wrong the moment a user brings a model nobody on the team has
|
|
9
|
+
* heard of, which is every day. It is also wrong in the other direction: it
|
|
10
|
+
* cannot express that five of nine free models answer tools only in prose, so
|
|
11
|
+
* a native-only client silently loses most of its chain while believing it is
|
|
12
|
+
* fine.
|
|
13
|
+
*
|
|
14
|
+
* The replacement is not a better list. It is three rules:
|
|
15
|
+
*
|
|
16
|
+
* 1. The first real call IS the probe. Send tools, read what comes back,
|
|
17
|
+
* write down what it proved. Every model a user brings classifies itself
|
|
18
|
+
* on its first message, at no extra cost and with no release from us.
|
|
19
|
+
* 2. A positive is cheap and a negative is expensive. One `tool_calls`
|
|
20
|
+
* proves capability. A 400 proves nothing unless the vendor SAYS it is
|
|
21
|
+
* about tools — otherwise a context-length overflow permanently cripples
|
|
22
|
+
* a capable model and nothing explains why.
|
|
23
|
+
* 3. "Never asked" is its own answer. Optimistic on the wire, because asking
|
|
24
|
+
* is how we learn; pessimistic in the prompt and the UI, because
|
|
25
|
+
* announcing an unproven capability is a promise the model may not keep.
|
|
26
|
+
*
|
|
27
|
+
* Storage stays with the app — a table, a KV, a file. This package owns the
|
|
28
|
+
* shape and the rules, which is the part everyone gets wrong; it does not own
|
|
29
|
+
* where rows live, which is the part where apps legitimately differ.
|
|
30
|
+
*/
|
|
31
|
+
export type {
|
|
32
|
+
ToolVerdict,
|
|
33
|
+
CapabilityKind,
|
|
34
|
+
Provenance,
|
|
35
|
+
CapabilityRecord,
|
|
36
|
+
CapabilityStore,
|
|
37
|
+
Classification,
|
|
38
|
+
} from "./types.js";
|
|
39
|
+
|
|
40
|
+
export { classifyToolAttempt, saysToolsUnsupported, type ToolAttempt } from "./classify.js";
|
|
41
|
+
|
|
42
|
+
export {
|
|
43
|
+
planToolAttempt,
|
|
44
|
+
claimableVerdict,
|
|
45
|
+
currentVerdict,
|
|
46
|
+
isStale,
|
|
47
|
+
makeRecord,
|
|
48
|
+
scopeKey,
|
|
49
|
+
shouldReplace,
|
|
50
|
+
DEFAULT_TTL_MS,
|
|
51
|
+
type ToolPlan,
|
|
52
|
+
} from "./decide.js";
|