modelrig 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +115 -8
- package/dist/adapters/anthropic.d.ts +46 -0
- package/dist/adapters/anthropic.d.ts.map +1 -0
- package/dist/adapters/anthropic.js +202 -0
- package/dist/adapters/deepseek.d.ts +41 -0
- package/dist/adapters/deepseek.d.ts.map +1 -0
- package/dist/adapters/deepseek.js +163 -0
- package/dist/adapters/gemini.d.ts +44 -0
- package/dist/adapters/gemini.d.ts.map +1 -0
- package/dist/adapters/gemini.js +222 -0
- package/dist/adapters/grok.d.ts +44 -0
- package/dist/adapters/grok.d.ts.map +1 -0
- package/dist/adapters/grok.js +168 -0
- package/dist/adapters/openai-compat.d.ts +74 -0
- package/dist/adapters/openai-compat.d.ts.map +1 -0
- package/dist/adapters/openai-compat.js +187 -0
- package/dist/adapters/openai.d.ts +49 -0
- package/dist/adapters/openai.d.ts.map +1 -0
- package/dist/adapters/openai.js +222 -0
- package/dist/bakeoff/engine.d.ts +52 -0
- package/dist/bakeoff/engine.d.ts.map +1 -0
- package/dist/bakeoff/engine.js +163 -0
- package/dist/bakeoff/replay.d.ts +52 -0
- package/dist/bakeoff/replay.d.ts.map +1 -0
- package/dist/bakeoff/replay.js +94 -0
- package/dist/bakeoff/samples.d.ts +62 -0
- package/dist/bakeoff/samples.d.ts.map +1 -0
- package/dist/bakeoff/samples.js +128 -0
- package/dist/bakeoff/stats.d.ts +13 -0
- package/dist/bakeoff/stats.d.ts.map +1 -0
- package/dist/bakeoff/stats.js +48 -0
- package/dist/bakeoff/types.d.ts +37 -0
- package/dist/bakeoff/types.d.ts.map +1 -0
- package/dist/bakeoff/types.js +10 -0
- package/dist/cli.d.ts +27 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +572 -0
- package/dist/config.d.ts +104 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +148 -0
- package/dist/feedback/coverage.d.ts +32 -0
- package/dist/feedback/coverage.d.ts.map +1 -0
- package/dist/feedback/coverage.js +95 -0
- package/dist/feedback/feedback.d.ts +35 -0
- package/dist/feedback/feedback.d.ts.map +1 -0
- package/dist/feedback/feedback.js +39 -0
- package/dist/index.d.ts +155 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +448 -0
- package/dist/init/finder.d.ts +79 -0
- package/dist/init/finder.d.ts.map +1 -0
- package/dist/init/finder.js +297 -0
- package/dist/init/run.d.ts +99 -0
- package/dist/init/run.d.ts.map +1 -0
- package/dist/init/run.js +281 -0
- package/dist/init/signatures.d.ts +39 -0
- package/dist/init/signatures.d.ts.map +1 -0
- package/dist/init/signatures.js +116 -0
- package/dist/matrix/aggregate.d.ts +89 -0
- package/dist/matrix/aggregate.d.ts.map +1 -0
- package/dist/matrix/aggregate.js +186 -0
- package/dist/query/grammar.d.ts +35 -0
- package/dist/query/grammar.d.ts.map +1 -0
- package/dist/query/grammar.js +279 -0
- package/dist/query/run.d.ts +8 -0
- package/dist/query/run.d.ts.map +1 -0
- package/dist/query/run.js +21 -0
- package/dist/registry/build.d.ts +126 -0
- package/dist/registry/build.d.ts.map +1 -0
- package/dist/registry/build.js +192 -0
- package/dist/registry/call-notes.d.ts +11 -0
- package/dist/registry/call-notes.d.ts.map +1 -0
- package/dist/registry/call-notes.js +72 -0
- package/dist/registry/claims.d.ts +63 -0
- package/dist/registry/claims.d.ts.map +1 -0
- package/dist/registry/claims.js +107 -0
- package/dist/registry/facts.d.ts +32 -0
- package/dist/registry/facts.d.ts.map +1 -0
- package/dist/registry/facts.js +120 -0
- package/dist/registry/fingerprint.d.ts +20 -0
- package/dist/registry/fingerprint.d.ts.map +1 -0
- package/dist/registry/fingerprint.js +150 -0
- package/dist/registry/parity.d.ts +26 -0
- package/dist/registry/parity.d.ts.map +1 -0
- package/dist/registry/parity.js +41 -0
- package/dist/registry/pricing.d.ts +19 -0
- package/dist/registry/pricing.d.ts.map +1 -0
- package/dist/registry/pricing.js +69 -0
- package/dist/registry/rows.d.ts +16 -0
- package/dist/registry/rows.d.ts.map +1 -0
- package/dist/registry/rows.js +57 -0
- package/dist/registry/seed.d.ts +43 -0
- package/dist/registry/seed.d.ts.map +1 -0
- package/dist/registry/seed.js +97 -0
- package/dist/registry/select.d.ts +68 -0
- package/dist/registry/select.d.ts.map +1 -0
- package/dist/registry/select.js +141 -0
- package/dist/registry/staleness.d.ts +18 -0
- package/dist/registry/staleness.d.ts.map +1 -0
- package/dist/registry/staleness.js +39 -0
- package/dist/routes/draft.d.ts +83 -0
- package/dist/routes/draft.d.ts.map +1 -0
- package/dist/routes/draft.js +135 -0
- package/dist/routes/loader.d.ts +26 -0
- package/dist/routes/loader.d.ts.map +1 -0
- package/dist/routes/loader.js +436 -0
- package/dist/routes/manifest.d.ts +24 -0
- package/dist/routes/manifest.d.ts.map +1 -0
- package/dist/routes/manifest.js +118 -0
- package/dist/routes/serveability.d.ts +52 -0
- package/dist/routes/serveability.d.ts.map +1 -0
- package/dist/routes/serveability.js +73 -0
- package/dist/routes/template.d.ts +18 -0
- package/dist/routes/template.d.ts.map +1 -0
- package/dist/routes/template.js +94 -0
- package/dist/run/cost.d.ts +26 -0
- package/dist/run/cost.d.ts.map +1 -0
- package/dist/run/cost.js +41 -0
- package/dist/run/envelope.d.ts +21 -0
- package/dist/run/envelope.d.ts.map +1 -0
- package/dist/run/envelope.js +45 -0
- package/dist/run/grounding.d.ts +56 -0
- package/dist/run/grounding.d.ts.map +1 -0
- package/dist/run/grounding.js +89 -0
- package/dist/run/repair.d.ts +36 -0
- package/dist/run/repair.d.ts.map +1 -0
- package/dist/run/repair.js +61 -0
- package/dist/run/resolve.d.ts +26 -0
- package/dist/run/resolve.d.ts.map +1 -0
- package/dist/run/resolve.js +81 -0
- package/dist/run/retry.d.ts +19 -0
- package/dist/run/retry.d.ts.map +1 -0
- package/dist/run/retry.js +50 -0
- package/dist/run/run.d.ts +41 -0
- package/dist/run/run.d.ts.map +1 -0
- package/dist/run/run.js +0 -0
- package/dist/run/validate.d.ts +15 -0
- package/dist/run/validate.d.ts.map +1 -0
- package/dist/run/validate.js +56 -0
- package/dist/savings/emit.d.ts +25 -0
- package/dist/savings/emit.d.ts.map +1 -0
- package/dist/savings/emit.js +75 -0
- package/dist/savings/odometer.d.ts +32 -0
- package/dist/savings/odometer.d.ts.map +1 -0
- package/dist/savings/odometer.js +70 -0
- package/dist/savings/swap-execute.d.ts +44 -0
- package/dist/savings/swap-execute.d.ts.map +1 -0
- package/dist/savings/swap-execute.js +87 -0
- package/dist/savings/swaps-client.d.ts +41 -0
- package/dist/savings/swaps-client.d.ts.map +1 -0
- package/dist/savings/swaps-client.js +81 -0
- package/dist/telemetry/capture.d.ts +28 -0
- package/dist/telemetry/capture.d.ts.map +1 -0
- package/dist/telemetry/capture.js +74 -0
- package/dist/telemetry/exporter.d.ts +62 -0
- package/dist/telemetry/exporter.d.ts.map +1 -0
- package/dist/telemetry/exporter.js +233 -0
- package/dist/telemetry/ingest-protocol.d.ts +161 -0
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -0
- package/dist/telemetry/ingest-protocol.js +163 -0
- package/dist/telemetry/sinks.d.ts +54 -0
- package/dist/telemetry/sinks.d.ts.map +1 -0
- package/dist/telemetry/sinks.js +89 -0
- package/dist/telemetry/store.d.ts +38 -0
- package/dist/telemetry/store.d.ts.map +1 -0
- package/dist/telemetry/store.js +332 -0
- package/dist/telemetry/supabase-exporter.d.ts +64 -0
- package/dist/telemetry/supabase-exporter.d.ts.map +1 -0
- package/dist/telemetry/supabase-exporter.js +211 -0
- package/dist/tenancy/api-key-format.d.ts +67 -0
- package/dist/tenancy/api-key-format.d.ts.map +1 -0
- package/dist/tenancy/api-key-format.js +91 -0
- package/dist/tenancy/constants.d.ts +23 -0
- package/dist/tenancy/constants.d.ts.map +1 -0
- package/dist/tenancy/constants.js +25 -0
- package/dist/tenancy/scrub.d.ts +47 -0
- package/dist/tenancy/scrub.d.ts.map +1 -0
- package/dist/tenancy/scrub.js +85 -0
- package/dist/types/failures-teaching.d.ts +56 -0
- package/dist/types/failures-teaching.d.ts.map +1 -0
- package/dist/types/failures-teaching.js +177 -0
- package/dist/types/failures.d.ts +27 -0
- package/dist/types/failures.d.ts.map +1 -0
- package/dist/types/failures.js +41 -0
- package/dist/types/provider.d.ts +79 -0
- package/dist/types/provider.d.ts.map +1 -0
- package/dist/types/provider.js +20 -0
- package/dist/types/route.d.ts +127 -0
- package/dist/types/route.d.ts.map +1 -0
- package/dist/types/route.js +6 -0
- package/dist/types/telemetry.d.ts +89 -0
- package/dist/types/telemetry.d.ts.map +1 -0
- package/dist/types/telemetry.js +6 -0
- package/dist/watch/budget.d.ts +19 -0
- package/dist/watch/budget.d.ts.map +1 -0
- package/dist/watch/budget.js +35 -0
- package/dist/watch/evaluate.d.ts +66 -0
- package/dist/watch/evaluate.d.ts.map +1 -0
- package/dist/watch/evaluate.js +94 -0
- package/dist/watch/proposals.d.ts +73 -0
- package/dist/watch/proposals.d.ts.map +1 -0
- package/dist/watch/proposals.js +169 -0
- package/dist/watch/run-record.d.ts +64 -0
- package/dist/watch/run-record.d.ts.map +1 -0
- package/dist/watch/run-record.js +86 -0
- package/dist/watch/run-reported.d.ts +20 -0
- package/dist/watch/run-reported.d.ts.map +1 -0
- package/dist/watch/run-reported.js +53 -0
- package/dist/watch/run.d.ts +40 -0
- package/dist/watch/run.d.ts.map +1 -0
- package/dist/watch/run.js +44 -0
- package/dist/watch/state-store.d.ts +48 -0
- package/dist/watch/state-store.d.ts.map +1 -0
- package/dist/watch/state-store.js +86 -0
- package/dist/watch/state.d.ts +21 -0
- package/dist/watch/state.d.ts.map +1 -0
- package/dist/watch/state.js +93 -0
- package/package.json +50 -14
- package/index.js +0 -3
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Schema fingerprinting — spec §2 WS6, network-effects instrumentation.
|
|
4
|
+
* Pure function: JSON schema → bucketed structural features + task category
|
|
5
|
+
* + a stable cell key. Computed at route load; stored on every telemetry row.
|
|
6
|
+
* PRIVACY: the fingerprint must be irreversible — features and category only,
|
|
7
|
+
* never field names, titles, descriptions, or enum values.
|
|
8
|
+
*/
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.fingerprintSchema = fingerprintSchema;
|
|
11
|
+
const node_crypto_1 = require("node:crypto");
|
|
12
|
+
const MAX_WALK_DEPTH = 12;
|
|
13
|
+
/** minLength at or above this marks a prose field (a deliberate length floor,
|
|
14
|
+
* not an id/enum constraint) — the "analysis" signal. */
|
|
15
|
+
const PROSE_FLOOR_MIN_LENGTH = 200;
|
|
16
|
+
const LOW_ENUM_RATIO = 0.2;
|
|
17
|
+
function isRecord(v) {
|
|
18
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
19
|
+
}
|
|
20
|
+
const SCALAR_TYPES = new Set(["string", "number", "integer", "boolean", "null"]);
|
|
21
|
+
function isScalarNode(node) {
|
|
22
|
+
return typeof node.type === "string" && SCALAR_TYPES.has(node.type);
|
|
23
|
+
}
|
|
24
|
+
function walk(node, depth, stats) {
|
|
25
|
+
if (!isRecord(node) || depth > MAX_WALK_DEPTH)
|
|
26
|
+
return;
|
|
27
|
+
if (depth > stats.maxDepth)
|
|
28
|
+
stats.maxDepth = depth;
|
|
29
|
+
if (isScalarNode(node)) {
|
|
30
|
+
stats.leafCount += 1;
|
|
31
|
+
if (Array.isArray(node.enum) || node.const !== undefined)
|
|
32
|
+
stats.enumLeafCount += 1;
|
|
33
|
+
if (node.type === "string" &&
|
|
34
|
+
typeof node.minLength === "number" &&
|
|
35
|
+
node.minLength >= PROSE_FLOOR_MIN_LENGTH) {
|
|
36
|
+
stats.proseFloorCount += 1;
|
|
37
|
+
}
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
const properties = node.properties;
|
|
41
|
+
if (isRecord(properties)) {
|
|
42
|
+
for (const child of Object.values(properties)) {
|
|
43
|
+
stats.fieldCount += 1;
|
|
44
|
+
walk(child, depth + 1, stats);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
const items = node.items;
|
|
48
|
+
if (items !== undefined) {
|
|
49
|
+
const itemNodes = Array.isArray(items) ? items : [items];
|
|
50
|
+
for (const item of itemNodes) {
|
|
51
|
+
if (isRecord(item) && isScalarNode(item)) {
|
|
52
|
+
stats.hasFlatArray = true;
|
|
53
|
+
walk(item, depth + 1, stats);
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
stats.hasNestedArray = true;
|
|
57
|
+
walk(item, depth + 1, stats);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (isRecord(node.additionalProperties))
|
|
62
|
+
walk(node.additionalProperties, depth + 1, stats);
|
|
63
|
+
for (const combinator of ["anyOf", "oneOf", "allOf"]) {
|
|
64
|
+
const branches = node[combinator];
|
|
65
|
+
if (Array.isArray(branches)) {
|
|
66
|
+
// Combinator branches describe the SAME position — no depth increase.
|
|
67
|
+
for (const branch of branches)
|
|
68
|
+
walk(branch, depth, stats);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
function bucketDepth(depth) {
|
|
73
|
+
if (depth <= 2)
|
|
74
|
+
return "1-2";
|
|
75
|
+
if (depth <= 4)
|
|
76
|
+
return "3-4";
|
|
77
|
+
return "5+";
|
|
78
|
+
}
|
|
79
|
+
function bucketFieldCount(count) {
|
|
80
|
+
if (count <= 5)
|
|
81
|
+
return "1-5";
|
|
82
|
+
if (count <= 20)
|
|
83
|
+
return "6-20";
|
|
84
|
+
if (count <= 60)
|
|
85
|
+
return "21-60";
|
|
86
|
+
return "61+";
|
|
87
|
+
}
|
|
88
|
+
function bucketEnumDensity(ratio) {
|
|
89
|
+
if (ratio === 0)
|
|
90
|
+
return "none";
|
|
91
|
+
if (ratio < LOW_ENUM_RATIO)
|
|
92
|
+
return "low";
|
|
93
|
+
return "high";
|
|
94
|
+
}
|
|
95
|
+
function categorize(stats, enumDensity) {
|
|
96
|
+
// Order matters (v0 heuristic, spec WS6): a deliberate prose floor is the
|
|
97
|
+
// strongest signal (analysis) → enum dominance (classification) → shallow
|
|
98
|
+
// typed scalars (extraction) → everything else (generation).
|
|
99
|
+
if (stats.proseFloorCount > 0)
|
|
100
|
+
return "analysis";
|
|
101
|
+
if (enumDensity === "high")
|
|
102
|
+
return "classification";
|
|
103
|
+
if (stats.maxDepth <= 2 && !stats.hasNestedArray && stats.leafCount > 0)
|
|
104
|
+
return "extraction";
|
|
105
|
+
return "generation";
|
|
106
|
+
}
|
|
107
|
+
function fingerprintSchema(schema) {
|
|
108
|
+
if (schema === null)
|
|
109
|
+
return null;
|
|
110
|
+
const stats = {
|
|
111
|
+
maxDepth: 0,
|
|
112
|
+
fieldCount: 0,
|
|
113
|
+
leafCount: 0,
|
|
114
|
+
enumLeafCount: 0,
|
|
115
|
+
proseFloorCount: 0,
|
|
116
|
+
hasFlatArray: false,
|
|
117
|
+
hasNestedArray: false,
|
|
118
|
+
};
|
|
119
|
+
walk(schema, 1, stats);
|
|
120
|
+
const enumRatio = stats.leafCount === 0 ? 0 : stats.enumLeafCount / stats.leafCount;
|
|
121
|
+
const enumDensityBucket = bucketEnumDensity(enumRatio);
|
|
122
|
+
const arrayComplexityBucket = stats.hasNestedArray
|
|
123
|
+
? "nested"
|
|
124
|
+
: stats.hasFlatArray
|
|
125
|
+
? "flat"
|
|
126
|
+
: "none";
|
|
127
|
+
const category = categorize(stats, enumDensityBucket);
|
|
128
|
+
const depthBucket = bucketDepth(stats.maxDepth);
|
|
129
|
+
const fieldCountBucket = bucketFieldCount(stats.fieldCount);
|
|
130
|
+
const cellKey = (0, node_crypto_1.createHash)("sha256")
|
|
131
|
+
.update(JSON.stringify([
|
|
132
|
+
depthBucket,
|
|
133
|
+
fieldCountBucket,
|
|
134
|
+
enumDensityBucket,
|
|
135
|
+
stats.proseFloorCount,
|
|
136
|
+
arrayComplexityBucket,
|
|
137
|
+
category,
|
|
138
|
+
]))
|
|
139
|
+
.digest("hex")
|
|
140
|
+
.slice(0, 16);
|
|
141
|
+
return {
|
|
142
|
+
depthBucket,
|
|
143
|
+
fieldCountBucket,
|
|
144
|
+
enumDensityBucket,
|
|
145
|
+
lengthFloorCount: stats.proseFloorCount,
|
|
146
|
+
arrayComplexityBucket,
|
|
147
|
+
category,
|
|
148
|
+
cellKey,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* parity-50 — addendum A1 / provider-coverage-plan §3. The honest coverage
|
|
3
|
+
* KPI: how much of OpenRouter's top-50-by-usage list is PROBED (not merely
|
|
4
|
+
* listed) in our registry. A model counts as probed when ANY of its mapped
|
|
5
|
+
* registry keys carries probed schema samples — same weights on any host.
|
|
6
|
+
* Gaps are named on the leaderboard ("not yet probed"), never invisible.
|
|
7
|
+
*/
|
|
8
|
+
import type { Registry } from "./build";
|
|
9
|
+
export interface ParityListEntry {
|
|
10
|
+
readonly name: string;
|
|
11
|
+
readonly registry_keys: readonly string[];
|
|
12
|
+
}
|
|
13
|
+
export interface ParityGap {
|
|
14
|
+
readonly name: string;
|
|
15
|
+
readonly status: "not yet probed" | "not in registry";
|
|
16
|
+
}
|
|
17
|
+
export interface ParityReport {
|
|
18
|
+
readonly total: number;
|
|
19
|
+
readonly probed: number;
|
|
20
|
+
readonly listedUnprobed: number;
|
|
21
|
+
readonly missing: number;
|
|
22
|
+
readonly probedPct: number;
|
|
23
|
+
readonly gaps: readonly ParityGap[];
|
|
24
|
+
}
|
|
25
|
+
export declare function computeParity(list: readonly ParityListEntry[], registry: Registry): ParityReport;
|
|
26
|
+
//# sourceMappingURL=parity.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parity.d.ts","sourceRoot":"","sources":["../../src/registry/parity.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAExC,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;CAC3C;AAED,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,gBAAgB,GAAG,iBAAiB,CAAC;CACvD;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,SAAS,SAAS,EAAE,CAAC;CACrC;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,SAAS,eAAe,EAAE,EAAE,QAAQ,EAAE,QAAQ,GAAG,YAAY,CA4BhG"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* parity-50 — addendum A1 / provider-coverage-plan §3. The honest coverage
|
|
4
|
+
* KPI: how much of OpenRouter's top-50-by-usage list is PROBED (not merely
|
|
5
|
+
* listed) in our registry. A model counts as probed when ANY of its mapped
|
|
6
|
+
* registry keys carries probed schema samples — same weights on any host.
|
|
7
|
+
* Gaps are named on the leaderboard ("not yet probed"), never invisible.
|
|
8
|
+
*/
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.computeParity = computeParity;
|
|
11
|
+
function computeParity(list, registry) {
|
|
12
|
+
const byKey = new Map(registry.models.map((m) => [m.model_key, m]));
|
|
13
|
+
let probed = 0;
|
|
14
|
+
let listedUnprobed = 0;
|
|
15
|
+
let missing = 0;
|
|
16
|
+
const gaps = [];
|
|
17
|
+
for (const entry of list) {
|
|
18
|
+
const matches = entry.registry_keys
|
|
19
|
+
.map((k) => byKey.get(k))
|
|
20
|
+
.filter((m) => m !== undefined);
|
|
21
|
+
if (matches.some((m) => (m.probed?.schema?.samples ?? 0) > 0)) {
|
|
22
|
+
probed += 1;
|
|
23
|
+
}
|
|
24
|
+
else if (matches.length > 0) {
|
|
25
|
+
listedUnprobed += 1;
|
|
26
|
+
gaps.push({ name: entry.name, status: "not yet probed" });
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
missing += 1;
|
|
30
|
+
gaps.push({ name: entry.name, status: "not in registry" });
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return {
|
|
34
|
+
total: list.length,
|
|
35
|
+
probed,
|
|
36
|
+
listedUnprobed,
|
|
37
|
+
missing,
|
|
38
|
+
probedPct: list.length === 0 ? 0 : probed / list.length,
|
|
39
|
+
gaps,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Thin pricing accessor over the vendored LiteLLM pricing snapshot
|
|
3
|
+
* (data/pricing-map.json, extracted at the pinned SHA — see
|
|
4
|
+
* upstream/litellm.pin.json). The full three-layer registry is Phase 2;
|
|
5
|
+
* Phase 1 needs only cost-per-token lookups for envelopes, prefer:[cost]
|
|
6
|
+
* ordering, and telemetry cost estimates.
|
|
7
|
+
*
|
|
8
|
+
* Data source: model_prices_and_context_window.json,
|
|
9
|
+
* Copyright (c) 2023 Berri AI, MIT License (https://github.com/BerriAI/litellm).
|
|
10
|
+
*/
|
|
11
|
+
import type { ProviderId } from "../types/provider";
|
|
12
|
+
export interface ModelPricing {
|
|
13
|
+
readonly inputUsdPerTok: number;
|
|
14
|
+
readonly outputUsdPerTok: number;
|
|
15
|
+
readonly cachedInputUsdPerTok: number | null;
|
|
16
|
+
}
|
|
17
|
+
export declare function getPricing(provider: ProviderId, model: string): ModelPricing | null;
|
|
18
|
+
export declare function estimateCostUsd(pricing: ModelPricing, tokensIn: number, tokensOut: number, tokensCached: number): number;
|
|
19
|
+
//# sourceMappingURL=pricing.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pricing.d.ts","sourceRoot":"","sources":["../../src/registry/pricing.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAEpD,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9C;AAmCD,wBAAgB,UAAU,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CAqBnF;AAED,wBAAgB,eAAe,CAC7B,OAAO,EAAE,YAAY,EACrB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GACnB,MAAM,CAWR"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Thin pricing accessor over the vendored LiteLLM pricing snapshot
|
|
4
|
+
* (data/pricing-map.json, extracted at the pinned SHA — see
|
|
5
|
+
* upstream/litellm.pin.json). The full three-layer registry is Phase 2;
|
|
6
|
+
* Phase 1 needs only cost-per-token lookups for envelopes, prefer:[cost]
|
|
7
|
+
* ordering, and telemetry cost estimates.
|
|
8
|
+
*
|
|
9
|
+
* Data source: model_prices_and_context_window.json,
|
|
10
|
+
* Copyright (c) 2023 Berri AI, MIT License (https://github.com/BerriAI/litellm).
|
|
11
|
+
*/
|
|
12
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
|
+
exports.getPricing = getPricing;
|
|
14
|
+
exports.estimateCostUsd = estimateCostUsd;
|
|
15
|
+
const node_fs_1 = require("node:fs");
|
|
16
|
+
const node_path_1 = require("node:path");
|
|
17
|
+
/** LiteLLM key prefixes per provider, most-specific first. Verified against
|
|
18
|
+
* the snapshot (see data/PROVENANCE.md): Gemini API entries live under
|
|
19
|
+
* "gemini/" (bare ids resolve to Vertex pricing — different product), OpenAI
|
|
20
|
+
* under bare ids, DeepSeek under both. */
|
|
21
|
+
const KEY_PREFIXES = {
|
|
22
|
+
gemini: ["gemini/"],
|
|
23
|
+
openai: ["", "openai/"],
|
|
24
|
+
deepseek: ["deepseek/", ""],
|
|
25
|
+
anthropic: ["anthropic/", ""],
|
|
26
|
+
grok: ["xai/"],
|
|
27
|
+
// A1 hosts: DeepInfra models keep their org path ("Qwen/QwQ-32B");
|
|
28
|
+
// Fireworks pricing keys carry the accounts/… wire prefix.
|
|
29
|
+
deepinfra: ["deepinfra/"],
|
|
30
|
+
fireworks: ["fireworks_ai/accounts/fireworks/models/", "fireworks_ai/"],
|
|
31
|
+
};
|
|
32
|
+
let cachedMap = null;
|
|
33
|
+
function loadMap() {
|
|
34
|
+
if (cachedMap === null) {
|
|
35
|
+
const raw = (0, node_fs_1.readFileSync)((0, node_path_1.join)(__dirname, "..", "..", "data", "pricing-map.json"), "utf8");
|
|
36
|
+
cachedMap = JSON.parse(raw);
|
|
37
|
+
}
|
|
38
|
+
return cachedMap;
|
|
39
|
+
}
|
|
40
|
+
function getPricing(provider, model) {
|
|
41
|
+
if (model === "")
|
|
42
|
+
return null;
|
|
43
|
+
const map = loadMap();
|
|
44
|
+
for (const prefix of KEY_PREFIXES[provider]) {
|
|
45
|
+
const entry = map[`${prefix}${model}`];
|
|
46
|
+
if (entry !== undefined &&
|
|
47
|
+
typeof entry.input_cost_per_token === "number" &&
|
|
48
|
+
typeof entry.output_cost_per_token === "number") {
|
|
49
|
+
return {
|
|
50
|
+
inputUsdPerTok: entry.input_cost_per_token,
|
|
51
|
+
outputUsdPerTok: entry.output_cost_per_token,
|
|
52
|
+
cachedInputUsdPerTok: typeof entry.cache_read_input_token_cost === "number"
|
|
53
|
+
? entry.cache_read_input_token_cost
|
|
54
|
+
: null,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
function estimateCostUsd(pricing, tokensIn, tokensOut, tokensCached) {
|
|
61
|
+
// Providers report tokensCached as a subset of tokensIn; bill the cached
|
|
62
|
+
// part at the cached rate (input rate when none) and never negatively.
|
|
63
|
+
const cached = Math.min(Math.max(tokensCached, 0), Math.max(tokensIn, tokensCached));
|
|
64
|
+
const uncachedIn = Math.max(tokensIn - cached, 0);
|
|
65
|
+
const cachedRate = pricing.cachedInputUsdPerTok ?? pricing.inputUsdPerTok;
|
|
66
|
+
return (uncachedIn * pricing.inputUsdPerTok +
|
|
67
|
+
cached * cachedRate +
|
|
68
|
+
Math.max(tokensOut, 0) * pricing.outputUsdPerTok);
|
|
69
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Registry → selection rows. One projection, used by every surface that
|
|
3
|
+
* asks a capability question from the FILES (the CLI, `modelrig init`, the
|
|
4
|
+
* docs build). The console projects from its Supabase mirror and the oracle
|
|
5
|
+
* from its own loader, but all three land on the same `ModelSelectRow`
|
|
6
|
+
* shape so the three cannot answer differently.
|
|
7
|
+
*/
|
|
8
|
+
import type { ModelSelectRow } from "./select";
|
|
9
|
+
/**
|
|
10
|
+
* Load selection rows from registry.json (+ leaderboard.json for effective
|
|
11
|
+
* cost). Returns [] rather than throwing when the registry is absent or
|
|
12
|
+
* unreadable — a repo without a built registry should still get drafts,
|
|
13
|
+
* carrying the incumbent alone.
|
|
14
|
+
*/
|
|
15
|
+
export declare function loadRegistryRows(registryPath?: string): ModelSelectRow[];
|
|
16
|
+
//# sourceMappingURL=rows.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rows.d.ts","sourceRoot":"","sources":["../../src/registry/rows.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAyC/C;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,cAAc,EAAE,CA2BxE"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Registry → selection rows. One projection, used by every surface that
|
|
4
|
+
* asks a capability question from the FILES (the CLI, `modelrig init`, the
|
|
5
|
+
* docs build). The console projects from its Supabase mirror and the oracle
|
|
6
|
+
* from its own loader, but all three land on the same `ModelSelectRow`
|
|
7
|
+
* shape so the three cannot answer differently.
|
|
8
|
+
*/
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.loadRegistryRows = loadRegistryRows;
|
|
11
|
+
const node_fs_1 = require("node:fs");
|
|
12
|
+
const node_path_1 = require("node:path");
|
|
13
|
+
function packagedRegistryDir() {
|
|
14
|
+
// dist/ and src/ both sit one level under the package root.
|
|
15
|
+
return (0, node_path_1.join)(__dirname, "..", "..", "registry");
|
|
16
|
+
}
|
|
17
|
+
function readJson(path) {
|
|
18
|
+
if (!(0, node_fs_1.existsSync)(path))
|
|
19
|
+
return null;
|
|
20
|
+
try {
|
|
21
|
+
return JSON.parse((0, node_fs_1.readFileSync)(path, "utf8"));
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Load selection rows from registry.json (+ leaderboard.json for effective
|
|
29
|
+
* cost). Returns [] rather than throwing when the registry is absent or
|
|
30
|
+
* unreadable — a repo without a built registry should still get drafts,
|
|
31
|
+
* carrying the incumbent alone.
|
|
32
|
+
*/
|
|
33
|
+
function loadRegistryRows(registryPath) {
|
|
34
|
+
const registryFile = registryPath ?? (0, node_path_1.join)(packagedRegistryDir(), "registry.json");
|
|
35
|
+
const registry = readJson(registryFile);
|
|
36
|
+
if (registry === null)
|
|
37
|
+
return [];
|
|
38
|
+
const dir = registryFile.slice(0, Math.max(registryFile.lastIndexOf("/"), 0));
|
|
39
|
+
const leaderboard = readJson((0, node_path_1.join)(dir, "leaderboard.json")) ?? [];
|
|
40
|
+
const effective = new Map(leaderboard.map((r) => [r.model_key, r.effective_usd_per_1k_conformant ?? null]));
|
|
41
|
+
return (registry.models ?? []).map((entry) => {
|
|
42
|
+
const [provider] = entry.model_key.split("/");
|
|
43
|
+
return {
|
|
44
|
+
key: entry.model_key,
|
|
45
|
+
provider: provider ?? entry.model_key,
|
|
46
|
+
inputUsdPerMtok: entry.declared?.pricing?.input_usd_per_mtok ?? null,
|
|
47
|
+
outputUsdPerMtok: entry.declared?.pricing?.output_usd_per_mtok ?? null,
|
|
48
|
+
conformRate: entry.probed?.schema?.conform_rate ?? null,
|
|
49
|
+
nativeRungRate: entry.probed?.schema?.native_rung_rate ?? null,
|
|
50
|
+
groundedRate: entry.probed?.grounding?.grounded_rate ?? null,
|
|
51
|
+
cacheHitRate: entry.probed?.caching?.cache_hit_rate ?? null,
|
|
52
|
+
effectiveUsdPer1k: effective.get(entry.model_key) ?? null,
|
|
53
|
+
discrepancies: entry.discrepancies?.length ?? 0,
|
|
54
|
+
probedAsOf: entry.probed?.as_of ?? null,
|
|
55
|
+
};
|
|
56
|
+
});
|
|
57
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Candidate seeding — the "what should serve this route?" answer, from the
|
|
3
|
+
* registry rather than from taste. Shared by `modelrig init` (WS2) and the
|
|
4
|
+
* oracle's `create_route` (WS1), so a drafted route looks the same however
|
|
5
|
+
* it was drafted.
|
|
6
|
+
*
|
|
7
|
+
* The rule: the INCUMBENT leads. The model you already call goes first, so
|
|
8
|
+
* the drafted route reproduces today's behavior on its first run; the
|
|
9
|
+
* cheapest probed models that satisfy the inferred capability needs follow
|
|
10
|
+
* as challengers a bake-off can put to the test. We never silently promote
|
|
11
|
+
* a cheaper model into first position — that decision belongs to evidence
|
|
12
|
+
* and a human.
|
|
13
|
+
*/
|
|
14
|
+
import type { CapabilityPredicate, ModelSelectRow } from "./select";
|
|
15
|
+
import type { DraftCandidate } from "../routes/draft";
|
|
16
|
+
export interface SeedCandidatesInput {
|
|
17
|
+
readonly capabilities: readonly CapabilityPredicate[];
|
|
18
|
+
/** "provider/model" the caller uses today, if known. */
|
|
19
|
+
readonly incumbent: string | null;
|
|
20
|
+
readonly maxInputUsdPerMtok: number | null;
|
|
21
|
+
/** Caller-supplied candidates short-circuit seeding entirely. */
|
|
22
|
+
readonly explicit: ReadonlyArray<{
|
|
23
|
+
readonly provider: string;
|
|
24
|
+
readonly model: string;
|
|
25
|
+
}> | null;
|
|
26
|
+
/** How many challengers to add behind the incumbent. */
|
|
27
|
+
readonly challengerCount?: number;
|
|
28
|
+
}
|
|
29
|
+
export interface SeededCandidates {
|
|
30
|
+
readonly candidates: readonly DraftCandidate[];
|
|
31
|
+
readonly incumbent: {
|
|
32
|
+
readonly provider: string;
|
|
33
|
+
readonly model: string;
|
|
34
|
+
} | null;
|
|
35
|
+
/** Human-readable account of why this list looks like it does. */
|
|
36
|
+
readonly reasoning: readonly string[];
|
|
37
|
+
}
|
|
38
|
+
export declare function splitModelKey(modelKey: string): {
|
|
39
|
+
provider: string;
|
|
40
|
+
model: string;
|
|
41
|
+
} | null;
|
|
42
|
+
export declare function seedCandidates(rows: readonly ModelSelectRow[], input: SeedCandidatesInput): SeededCandidates;
|
|
43
|
+
//# sourceMappingURL=seed.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"seed.d.ts","sourceRoot":"","sources":["../../src/registry/seed.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,KAAK,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtD,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,YAAY,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACtD,wDAAwD;IACxD,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,iEAAiE;IACjE,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;QAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,IAAI,CAAC;IAC/F,wDAAwD;IACxD,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;CACnC;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,CAAC;IAC/C,QAAQ,CAAC,SAAS,EAAE;QAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACjF,kEAAkE;IAClE,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;CACvC;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAI1F;AAED,wBAAgB,cAAc,CAC5B,IAAI,EAAE,SAAS,cAAc,EAAE,EAC/B,KAAK,EAAE,mBAAmB,GACzB,gBAAgB,CAqFlB"}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Candidate seeding — the "what should serve this route?" answer, from the
|
|
4
|
+
* registry rather than from taste. Shared by `modelrig init` (WS2) and the
|
|
5
|
+
* oracle's `create_route` (WS1), so a drafted route looks the same however
|
|
6
|
+
* it was drafted.
|
|
7
|
+
*
|
|
8
|
+
* The rule: the INCUMBENT leads. The model you already call goes first, so
|
|
9
|
+
* the drafted route reproduces today's behavior on its first run; the
|
|
10
|
+
* cheapest probed models that satisfy the inferred capability needs follow
|
|
11
|
+
* as challengers a bake-off can put to the test. We never silently promote
|
|
12
|
+
* a cheaper model into first position — that decision belongs to evidence
|
|
13
|
+
* and a human.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.splitModelKey = splitModelKey;
|
|
17
|
+
exports.seedCandidates = seedCandidates;
|
|
18
|
+
const select_1 = require("./select");
|
|
19
|
+
function splitModelKey(modelKey) {
|
|
20
|
+
const slash = modelKey.indexOf("/");
|
|
21
|
+
if (slash <= 0 || slash === modelKey.length - 1)
|
|
22
|
+
return null;
|
|
23
|
+
return { provider: modelKey.slice(0, slash), model: modelKey.slice(slash + 1) };
|
|
24
|
+
}
|
|
25
|
+
function seedCandidates(rows, input) {
|
|
26
|
+
if (input.explicit !== null && input.explicit.length > 0) {
|
|
27
|
+
return {
|
|
28
|
+
candidates: input.explicit.map((c) => ({ ...c })),
|
|
29
|
+
incumbent: input.explicit[0] ?? null,
|
|
30
|
+
reasoning: ["Candidates were given explicitly — the registry was not consulted."],
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
const reasoning = [];
|
|
34
|
+
const candidates = [];
|
|
35
|
+
let incumbent = null;
|
|
36
|
+
if (input.incumbent !== null && input.incumbent !== "") {
|
|
37
|
+
const parsed = splitModelKey(input.incumbent);
|
|
38
|
+
if (parsed === null) {
|
|
39
|
+
reasoning.push(`Could not read "${input.incumbent}" as an incumbent — model keys are "provider/model". ` +
|
|
40
|
+
"Drafted without one, so review the ordering before you run it.");
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
incumbent = parsed;
|
|
44
|
+
const known = rows.some((r) => r.key === input.incumbent);
|
|
45
|
+
candidates.push({
|
|
46
|
+
...parsed,
|
|
47
|
+
note: known
|
|
48
|
+
? "incumbent — your current model, listed first so behavior is unchanged on the first run"
|
|
49
|
+
: "incumbent — your current model (not in our probed registry, so unmeasured here)",
|
|
50
|
+
});
|
|
51
|
+
reasoning.push(`Incumbent ${input.incumbent} leads: the drafted route reproduces what you do today.`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
reasoning.push("No incumbent was identified, so the cheapest probed model that meets the requirements " +
|
|
56
|
+
"leads. If you already use a specific model, put it first before running this.");
|
|
57
|
+
}
|
|
58
|
+
const challengerCount = input.challengerCount ?? 2;
|
|
59
|
+
const wanted = challengerCount + (incumbent === null ? 0 : 1);
|
|
60
|
+
const hits = (0, select_1.selectModels)(rows, {
|
|
61
|
+
caps: input.capabilities,
|
|
62
|
+
maxInputUsdPerMtok: input.maxInputUsdPerMtok,
|
|
63
|
+
limit: challengerCount + 1,
|
|
64
|
+
});
|
|
65
|
+
for (const row of hits) {
|
|
66
|
+
if (candidates.length >= wanted)
|
|
67
|
+
break;
|
|
68
|
+
if (incumbent !== null && row.key === `${incumbent.provider}/${incumbent.model}`)
|
|
69
|
+
continue;
|
|
70
|
+
const parsed = splitModelKey(row.key);
|
|
71
|
+
if (parsed === null)
|
|
72
|
+
continue;
|
|
73
|
+
const effective = row.effectiveUsdPer1k;
|
|
74
|
+
candidates.push({
|
|
75
|
+
...parsed,
|
|
76
|
+
note: effective === null
|
|
77
|
+
? "probed, but no effective-cost figure yet"
|
|
78
|
+
: `probed: $${effective.toFixed(4)} per 1k conformant outputs`,
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
if (input.capabilities.length > 0) {
|
|
82
|
+
reasoning.push(`Challengers had to satisfy every requirement asked for (${input.capabilities.join(", ")}) ` +
|
|
83
|
+
"in probing — declared support alone was not enough.");
|
|
84
|
+
}
|
|
85
|
+
if (input.maxInputUsdPerMtok !== null) {
|
|
86
|
+
reasoning.push(`Challengers were capped at $${input.maxInputUsdPerMtok}/M input tokens.`);
|
|
87
|
+
}
|
|
88
|
+
if (candidates.length === 0) {
|
|
89
|
+
reasoning.push("No probed model satisfied the requirements, so this draft has no candidates and will not " +
|
|
90
|
+
"load — name a model explicitly, or loosen the requirements.");
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
reasoning.push("Challengers are ranked by effective cost of conformance, not list price. Nothing switches " +
|
|
94
|
+
"to them without a bake-off and an explicit approval.");
|
|
95
|
+
}
|
|
96
|
+
return { candidates, incumbent, reasoning };
|
|
97
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Model selection — the capability-first question, answered once.
|
|
3
|
+
*
|
|
4
|
+
* "Which models can serve strict JSON under $1/M?" is asked in three places:
|
|
5
|
+
* the console's query box, the MCP oracle's `query_registry` tool, and the
|
|
6
|
+
* docs. Phase 5 plan §10 names the risk by name — oracle answers drifting
|
|
7
|
+
* from console answers — so the grammar and the predicates live HERE and
|
|
8
|
+
* both surfaces call in. If this file changes, every surface changes.
|
|
9
|
+
*
|
|
10
|
+
* Dependency-free on purpose: the console imports this source directly (its
|
|
11
|
+
* tsconfig path alias) and must not drag SQLite or the adapters into a
|
|
12
|
+
* browser bundle.
|
|
13
|
+
*
|
|
14
|
+
* Bars are the registry's own, not new judgments: 0.9 conformance (the
|
|
15
|
+
* discrepancy detector's bar) and 0.5 grounding (facts.ts's GROUNDED_BAR).
|
|
16
|
+
*/
|
|
17
|
+
/** The capability predicates a question can carry. Deliberately the probed
|
|
18
|
+
* dimensions only — we answer from measurement, not from provider claims. */
|
|
19
|
+
export type CapabilityPredicate = "structured-native" | "structured-coached" | "grounding" | "caching";
|
|
20
|
+
export declare const ALL_CAPABILITY_PREDICATES: readonly CapabilityPredicate[];
|
|
21
|
+
/** Plain-language gloss per predicate — the explainability layer travels
|
|
22
|
+
* WITH the answer (console-explainability.md §1), it is not a separate doc. */
|
|
23
|
+
export declare const PREDICATE_GLOSS: Readonly<Record<CapabilityPredicate, string>>;
|
|
24
|
+
/** The row shape selection needs. A projection, not the registry entry —
|
|
25
|
+
* console (Supabase mirror) and oracle (registry.json) both flatten into it. */
|
|
26
|
+
export interface ModelSelectRow {
|
|
27
|
+
readonly key: string;
|
|
28
|
+
readonly provider: string;
|
|
29
|
+
readonly inputUsdPerMtok: number | null;
|
|
30
|
+
readonly outputUsdPerMtok: number | null;
|
|
31
|
+
readonly conformRate: number | null;
|
|
32
|
+
readonly nativeRungRate: number | null;
|
|
33
|
+
readonly groundedRate: number | null;
|
|
34
|
+
readonly cacheHitRate: number | null;
|
|
35
|
+
readonly effectiveUsdPer1k: number | null;
|
|
36
|
+
readonly discrepancies: number;
|
|
37
|
+
readonly probedAsOf: string | null;
|
|
38
|
+
}
|
|
39
|
+
export declare function hasCapability(row: ModelSelectRow, cap: CapabilityPredicate): boolean;
|
|
40
|
+
export interface ParsedQuery {
|
|
41
|
+
readonly caps: readonly CapabilityPredicate[];
|
|
42
|
+
readonly maxInputUsdPerMtok: number | null;
|
|
43
|
+
/** Leftover words — substring-matched against the model key. */
|
|
44
|
+
readonly text: readonly string[];
|
|
45
|
+
}
|
|
46
|
+
/** The tiny grammar behind the query box and the oracle's free-text input:
|
|
47
|
+
* capability words + an "under $N/M" price cap; everything else is a
|
|
48
|
+
* substring match on the model key. Small on purpose — a question we cannot
|
|
49
|
+
* parse should fall through to text, not guess. */
|
|
50
|
+
export declare function parseCapabilityQuery(query: string): ParsedQuery;
|
|
51
|
+
export interface SelectCriteria {
|
|
52
|
+
/** Capability predicates ANDed together — every one must hold. */
|
|
53
|
+
readonly caps?: readonly CapabilityPredicate[];
|
|
54
|
+
readonly maxInputUsdPerMtok?: number | null;
|
|
55
|
+
readonly provider?: string | null;
|
|
56
|
+
readonly discrepanciesOnly?: boolean;
|
|
57
|
+
/** Substring terms ANDed against the model key. */
|
|
58
|
+
readonly text?: readonly string[];
|
|
59
|
+
/** Sort by effective cost of conformance ascending (default true) — the
|
|
60
|
+
* only ranking that answers "cheapest that actually works". */
|
|
61
|
+
readonly sortByEffectiveCost?: boolean;
|
|
62
|
+
readonly limit?: number;
|
|
63
|
+
}
|
|
64
|
+
export declare function selectModels(rows: readonly ModelSelectRow[], criteria?: SelectCriteria): ModelSelectRow[];
|
|
65
|
+
/** Why a question found nothing — the difference between "no such model" and
|
|
66
|
+
* "we have not probed one yet" is the whole point of the demand meter. */
|
|
67
|
+
export declare function explainEmptySelection(criteria: SelectCriteria): string;
|
|
68
|
+
//# sourceMappingURL=select.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"select.d.ts","sourceRoot":"","sources":["../../src/registry/select.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH;6EAC6E;AAC7E,MAAM,MAAM,mBAAmB,GAC3B,mBAAmB,GACnB,oBAAoB,GACpB,WAAW,GACX,SAAS,CAAC;AAEd,eAAO,MAAM,yBAAyB,EAAE,SAAS,mBAAmB,EAKnE,CAAC;AAEF;+EAC+E;AAC/E,eAAO,MAAM,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAQzE,CAAC;AAEF;gFACgF;AAChF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAKD,wBAAgB,aAAa,CAAC,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,mBAAmB,GAAG,OAAO,CAWpF;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC9C,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,gEAAgE;IAChE,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;CAClC;AA0BD;;;mDAGmD;AACnD,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,CAkB/D;AAED,MAAM,WAAW,cAAc;IAC7B,kEAAkE;IAClE,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC/C,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC;IACrC,mDAAmD;IACnD,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC;mEAC+D;IAC/D,QAAQ,CAAC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IACvC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,SAAS,cAAc,EAAE,EAC/B,QAAQ,GAAE,cAAmB,GAC5B,cAAc,EAAE,CA6BlB;AAED;0EAC0E;AAC1E,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,cAAc,GAAG,MAAM,CAatE"}
|