modelrig 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +238 -0
- package/README.md +65 -3
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +36 -7
- package/dist/adapters/attempt-meta-map.d.ts +63 -0
- package/dist/adapters/attempt-meta-map.d.ts.map +1 -0
- package/dist/adapters/attempt-meta-map.js +154 -0
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +25 -2
- package/dist/adapters/gemini.d.ts +39 -0
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +82 -4
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +25 -2
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +25 -2
- package/dist/adapters/openai-stream.d.ts +2 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -1
- package/dist/adapters/openai-stream.js +23 -3
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +36 -6
- package/dist/artifacts/artifact.d.ts +3 -2
- package/dist/artifacts/artifact.d.ts.map +1 -1
- package/dist/artifacts/artifact.js +35 -9
- package/dist/artifacts/classification.d.ts +27 -0
- package/dist/artifacts/classification.d.ts.map +1 -0
- package/dist/artifacts/classification.js +38 -0
- package/dist/artifacts/index.d.ts +3 -1
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/index.js +10 -1
- package/dist/artifacts/run-context.d.ts +33 -2
- package/dist/artifacts/run-context.d.ts.map +1 -1
- package/dist/artifacts/run-context.js +95 -1
- package/dist/artifacts/scrub.d.ts +26 -0
- package/dist/artifacts/scrub.d.ts.map +1 -1
- package/dist/artifacts/scrub.js +53 -0
- package/dist/artifacts/step.d.ts +9 -0
- package/dist/artifacts/step.d.ts.map +1 -1
- package/dist/artifacts/step.js +9 -0
- package/dist/bakeoff/engine.d.ts +46 -4
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +269 -19
- package/dist/bakeoff/eval-cases.d.ts +5 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -1
- package/dist/bakeoff/eval-cases.js +3 -0
- package/dist/bakeoff/judge.d.ts +39 -0
- package/dist/bakeoff/judge.d.ts.map +1 -0
- package/dist/bakeoff/judge.js +102 -0
- package/dist/bakeoff/quality.d.ts +44 -0
- package/dist/bakeoff/quality.d.ts.map +1 -0
- package/dist/bakeoff/quality.js +126 -0
- package/dist/bakeoff/reference.d.ts +34 -0
- package/dist/bakeoff/reference.d.ts.map +1 -0
- package/dist/bakeoff/reference.js +68 -0
- package/dist/bakeoff/replay.d.ts +25 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +65 -2
- package/dist/bakeoff/report.d.ts +8 -1
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +83 -12
- package/dist/bakeoff/samples.d.ts +35 -2
- package/dist/bakeoff/samples.d.ts.map +1 -1
- package/dist/bakeoff/samples.js +67 -6
- package/dist/bakeoff/stats.d.ts +17 -1
- package/dist/bakeoff/stats.d.ts.map +1 -1
- package/dist/bakeoff/stats.js +67 -0
- package/dist/bakeoff/types.d.ts +76 -5
- package/dist/bakeoff/types.d.ts.map +1 -1
- package/dist/bakeoff/types.js +4 -3
- package/dist/cli-query.d.ts +14 -0
- package/dist/cli-query.d.ts.map +1 -0
- package/dist/cli-query.js +59 -0
- package/dist/cli.d.ts +2 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +47 -24
- package/dist/config.d.ts +6 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +14 -0
- package/dist/detect/detector.d.ts +82 -0
- package/dist/detect/detector.d.ts.map +1 -0
- package/dist/detect/detector.js +23 -0
- package/dist/index.d.ts +34 -23
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +49 -12
- package/dist/query/analytics.d.ts +84 -0
- package/dist/query/analytics.d.ts.map +1 -0
- package/dist/query/analytics.js +280 -0
- package/dist/query/columns.d.ts +59 -0
- package/dist/query/columns.d.ts.map +1 -0
- package/dist/query/columns.js +89 -0
- package/dist/query/grammar.d.ts +74 -7
- package/dist/query/grammar.d.ts.map +1 -1
- package/dist/query/grammar.js +179 -71
- package/dist/query/saved.d.ts +26 -0
- package/dist/query/saved.d.ts.map +1 -0
- package/dist/query/saved.js +51 -0
- package/dist/registry/select.d.ts +59 -0
- package/dist/registry/select.d.ts.map +1 -1
- package/dist/registry/select.js +66 -0
- package/dist/routes/loader-warnings.d.ts +28 -0
- package/dist/routes/loader-warnings.d.ts.map +1 -0
- package/dist/routes/loader-warnings.js +56 -0
- package/dist/routes/loader.d.ts +1 -11
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +54 -31
- package/dist/routes/objective-loader.d.ts +35 -0
- package/dist/routes/objective-loader.d.ts.map +1 -0
- package/dist/routes/objective-loader.js +286 -0
- package/dist/run/accept.d.ts +43 -0
- package/dist/run/accept.d.ts.map +1 -0
- package/dist/run/accept.js +65 -0
- package/dist/run/cost.d.ts +13 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +27 -0
- package/dist/run/dispatch.d.ts +24 -1
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +19 -0
- package/dist/run/ladder-key.d.ts +4 -0
- package/dist/run/ladder-key.d.ts.map +1 -1
- package/dist/run/ladder-key.js +3 -0
- package/dist/run/quality-gate.d.ts +39 -0
- package/dist/run/quality-gate.d.ts.map +1 -0
- package/dist/run/quality-gate.js +70 -0
- package/dist/run/raw-defaults.d.ts +52 -0
- package/dist/run/raw-defaults.d.ts.map +1 -0
- package/dist/run/raw-defaults.js +123 -0
- package/dist/run/raw.d.ts +12 -0
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +107 -8
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +22 -1
- package/dist/run/resolve.d.ts +17 -1
- package/dist/run/resolve.d.ts.map +1 -1
- package/dist/run/resolve.js +20 -4
- package/dist/run/routing-docs.d.ts.map +1 -1
- package/dist/run/routing-docs.js +12 -5
- package/dist/run/run.d.ts +7 -1
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +54 -69
- package/dist/run/variant.d.ts +43 -0
- package/dist/run/variant.d.ts.map +1 -0
- package/dist/run/variant.js +86 -0
- package/dist/savings/emit.d.ts +33 -0
- package/dist/savings/emit.d.ts.map +1 -1
- package/dist/savings/emit.js +75 -3
- package/dist/settings/index.d.ts +1 -1
- package/dist/settings/index.d.ts.map +1 -1
- package/dist/settings/index.js +2 -1
- package/dist/settings/task-settings.d.ts +10 -0
- package/dist/settings/task-settings.d.ts.map +1 -1
- package/dist/settings/task-settings.js +17 -0
- package/dist/telemetry/attempt-meta.d.ts +178 -0
- package/dist/telemetry/attempt-meta.d.ts.map +1 -0
- package/dist/telemetry/attempt-meta.js +319 -0
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +6 -0
- package/dist/telemetry/ingest-protocol.d.ts +11 -0
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +3 -0
- package/dist/telemetry/otel-map.d.ts +10 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -1
- package/dist/telemetry/otel-map.js +64 -0
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +15 -0
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +25 -7
- package/dist/types/failures-teaching.d.ts.map +1 -1
- package/dist/types/failures-teaching.js +20 -0
- package/dist/types/failures.d.ts +1 -1
- package/dist/types/failures.d.ts.map +1 -1
- package/dist/types/failures.js +1 -0
- package/dist/types/provider.d.ts +9 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/quality.d.ts +41 -0
- package/dist/types/quality.d.ts.map +1 -0
- package/dist/types/quality.js +12 -0
- package/dist/types/route.d.ts +120 -0
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +13 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts.map +1 -1
- package/dist/validate-cli.js +10 -0
- package/dist/watch/evaluate.d.ts +27 -1
- package/dist/watch/evaluate.d.ts.map +1 -1
- package/dist/watch/evaluate.js +13 -0
- package/dist/watch/proposals.d.ts +28 -0
- package/dist/watch/proposals.d.ts.map +1 -1
- package/dist/watch/proposals.js +51 -0
- package/package.json +1 -1
package/dist/query/grammar.js
CHANGED
|
@@ -1,25 +1,57 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
* The telemetry query grammar — Phase 4 WS5 ("your exhaust is your data")
|
|
3
|
+
* The telemetry query grammar — Phase 4 WS5 ("your exhaust is your data"),
|
|
4
|
+
* extended for WI-1b (the attempt-envelope analytics surface).
|
|
5
|
+
*
|
|
4
6
|
* A hand-rolled recursive-descent parser over a DELIBERATELY tiny SQL-ish
|
|
5
7
|
* language:
|
|
6
8
|
*
|
|
7
9
|
* SELECT item (, item)* FROM inferences
|
|
8
10
|
* [WHERE cond (AND cond)*] [GROUP BY col (, col)*] [LIMIT n]
|
|
9
11
|
* item := col | count(*) | sum(m)|avg(m)|min(m)|max(m)
|
|
10
|
-
* col := route|provider|model|ts|failure_class|served_variant
|
|
12
|
+
* col := route|provider|model|ts|failure_class|served_variant
|
|
13
|
+
* | subject|feature|user|session|cost_center (reserved dims)
|
|
14
|
+
* | meta.<field> (attempt envelope)
|
|
15
|
+
* | tag.<name>
|
|
11
16
|
* m := cost|tokens_in|tokens_out|tokens_cached|latency_ms|ttfb_ms
|
|
17
|
+
* | meta.http_status|meta.reasoning_tokens|meta.response_chars
|
|
18
|
+
* | meta.cost.in|meta.cost.out|meta.cost.cache_read|meta.cost.cache_write|meta.cost.other
|
|
12
19
|
* cond := col op literal op := = != > < >= <=
|
|
13
20
|
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
21
|
+
* ── TWO EXECUTION TARGETS, ONE PARSER (S1: extend, don't twin) ────────────
|
|
22
|
+
* `parseQuery` produces a {@link QueryAst}. `compileQuery` renders that AST to a
|
|
23
|
+
* single read-only SQLite SELECT (the local `/query` surface, `runTelemetryQuery`).
|
|
24
|
+
* The server analytics surface (`/v1/analytics/query`) evaluates the SAME AST
|
|
25
|
+
* over PostgREST-read rows in JS (query/analytics.ts) — so the grammar, its
|
|
26
|
+
* SQLite semantics, and its multi-tenant semantics can never disagree.
|
|
27
|
+
*
|
|
28
|
+
* Security model = the allowlist (columns.ts): unknown tokens throw, string
|
|
29
|
+
* literals only ever travel as bind parameters, the SQLite output is a single
|
|
30
|
+
* read-only SELECT. There is no raw-SQL path — by construction, not by filtering.
|
|
31
|
+
* The metadata-safe invariant is structural: every selectable column is an
|
|
32
|
+
* enum/number/boolean/hash/identifier/timestamp (columns.ts), never content.
|
|
19
33
|
*/
|
|
20
34
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
21
|
-
exports.QueryGrammarError = void 0;
|
|
35
|
+
exports.QueryGrammarError = exports.RESERVED_DIMENSION_NAMES = exports.METRIC_COLUMNS = exports.DIMENSION_COLUMNS = exports.sinceIsoFor = exports.savedQueryById = exports.SAVED_QUERIES = exports.selectableColumnsAreMetaSafe = exports.ANALYTICS_SCAN_CAP = exports.planAnalyticsRead = exports.evaluateAnalyticsQuery = exports.describeAnalyticsSchema = void 0;
|
|
36
|
+
exports.parseQuery = parseQuery;
|
|
22
37
|
exports.compileQuery = compileQuery;
|
|
38
|
+
const columns_1 = require("./columns");
|
|
39
|
+
// Re-surface the analytics + saved-query symbols through this module so the
|
|
40
|
+
// package barrel (index.ts) can name them without a new export line (LOC cap).
|
|
41
|
+
var analytics_1 = require("./analytics");
|
|
42
|
+
Object.defineProperty(exports, "describeAnalyticsSchema", { enumerable: true, get: function () { return analytics_1.describeAnalyticsSchema; } });
|
|
43
|
+
Object.defineProperty(exports, "evaluateAnalyticsQuery", { enumerable: true, get: function () { return analytics_1.evaluateAnalyticsQuery; } });
|
|
44
|
+
Object.defineProperty(exports, "planAnalyticsRead", { enumerable: true, get: function () { return analytics_1.planAnalyticsRead; } });
|
|
45
|
+
Object.defineProperty(exports, "ANALYTICS_SCAN_CAP", { enumerable: true, get: function () { return analytics_1.ANALYTICS_SCAN_CAP; } });
|
|
46
|
+
Object.defineProperty(exports, "selectableColumnsAreMetaSafe", { enumerable: true, get: function () { return analytics_1.selectableColumnsAreMetaSafe; } });
|
|
47
|
+
var saved_1 = require("./saved");
|
|
48
|
+
Object.defineProperty(exports, "SAVED_QUERIES", { enumerable: true, get: function () { return saved_1.SAVED_QUERIES; } });
|
|
49
|
+
Object.defineProperty(exports, "savedQueryById", { enumerable: true, get: function () { return saved_1.savedQueryById; } });
|
|
50
|
+
Object.defineProperty(exports, "sinceIsoFor", { enumerable: true, get: function () { return saved_1.sinceIsoFor; } });
|
|
51
|
+
var columns_2 = require("./columns");
|
|
52
|
+
Object.defineProperty(exports, "DIMENSION_COLUMNS", { enumerable: true, get: function () { return columns_2.DIMENSION_COLUMNS; } });
|
|
53
|
+
Object.defineProperty(exports, "METRIC_COLUMNS", { enumerable: true, get: function () { return columns_2.METRIC_COLUMNS; } });
|
|
54
|
+
Object.defineProperty(exports, "RESERVED_DIMENSION_NAMES", { enumerable: true, get: function () { return columns_2.RESERVED_DIMENSION_NAMES; } });
|
|
23
55
|
class QueryGrammarError extends Error {
|
|
24
56
|
constructor(message) {
|
|
25
57
|
super(message);
|
|
@@ -27,22 +59,6 @@ class QueryGrammarError extends Error {
|
|
|
27
59
|
}
|
|
28
60
|
}
|
|
29
61
|
exports.QueryGrammarError = QueryGrammarError;
|
|
30
|
-
const PLAIN_COLUMNS = {
|
|
31
|
-
route: "route",
|
|
32
|
-
provider: "provider",
|
|
33
|
-
model: "model",
|
|
34
|
-
ts: "ts",
|
|
35
|
-
failure_class: "failure_class",
|
|
36
|
-
served_variant: "served_variant",
|
|
37
|
-
};
|
|
38
|
-
const METRICS = {
|
|
39
|
-
cost: "cost_estimate_usd",
|
|
40
|
-
tokens_in: "tokens_in",
|
|
41
|
-
tokens_out: "tokens_out",
|
|
42
|
-
tokens_cached: "tokens_cached",
|
|
43
|
-
latency_ms: "latency_ms",
|
|
44
|
-
ttfb_ms: "ttfb_ms",
|
|
45
|
-
};
|
|
46
62
|
const FUNCTIONS = new Set(["sum", "avg", "min", "max", "count"]);
|
|
47
63
|
const OPERATORS = new Set(["=", "!=", ">", "<", ">=", "<="]);
|
|
48
64
|
const TAG_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
@@ -121,23 +137,69 @@ function tokenize(input) {
|
|
|
121
137
|
}
|
|
122
138
|
return tokens;
|
|
123
139
|
}
|
|
124
|
-
/**
|
|
125
|
-
*
|
|
126
|
-
|
|
127
|
-
|
|
140
|
+
/** The alias a column reads out under — `tag.client`/`meta.finish_reason` lose
|
|
141
|
+
* their prefix so result keys read naturally (`client`, `finish_reason`). */
|
|
142
|
+
function aliasFor(word) {
|
|
143
|
+
if (word.startsWith("tag."))
|
|
144
|
+
return word.slice(4);
|
|
145
|
+
if (word.startsWith("meta."))
|
|
146
|
+
return word.slice(5);
|
|
147
|
+
return word;
|
|
148
|
+
}
|
|
149
|
+
/** Resolve a grammar word to a DIMENSION reference (group/filter), or throw. A
|
|
150
|
+
* `tag.<name>` compiles to a validated tags key; every other name must be a
|
|
151
|
+
* registered dimension (columns.ts). */
|
|
152
|
+
function resolveDimension(word) {
|
|
128
153
|
if (word.startsWith("tag.")) {
|
|
129
154
|
const tag = word.slice(4);
|
|
130
155
|
if (!TAG_NAME.test(tag))
|
|
131
156
|
throw new QueryGrammarError(`bad tag name: ${tag}`);
|
|
132
|
-
return
|
|
157
|
+
return { name: word, source: "tag", key: tag, classification: "identifier", numeric: false, alias: tag };
|
|
133
158
|
}
|
|
134
|
-
const
|
|
135
|
-
if (
|
|
136
|
-
throw new QueryGrammarError(`unknown column "${word}" — allowed: ${
|
|
159
|
+
const def = columns_1.DIMENSIONS_BY_NAME.get(word);
|
|
160
|
+
if (def === undefined) {
|
|
161
|
+
throw new QueryGrammarError(`unknown column "${word}" — allowed: ${[...columns_1.DIMENSIONS_BY_NAME.keys()].join(", ")}, tag.<name>`);
|
|
137
162
|
}
|
|
138
|
-
return
|
|
163
|
+
return refOf(def);
|
|
139
164
|
}
|
|
140
|
-
|
|
165
|
+
/** Resolve a grammar word to a METRIC reference (aggregate target), or throw. */
|
|
166
|
+
function resolveMetric(word) {
|
|
167
|
+
const def = columns_1.METRICS_BY_NAME.get(word);
|
|
168
|
+
if (def === undefined) {
|
|
169
|
+
throw new QueryGrammarError(`unknown metric "${word}" — allowed: ${[...columns_1.METRICS_BY_NAME.keys()].join(", ")}`);
|
|
170
|
+
}
|
|
171
|
+
return refOf(def);
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Resolve the argument of an aggregate. `sum`/`avg` require a NUMERIC metric.
|
|
175
|
+
* `min`/`max` also accept a dimension (e.g. `min(ts)` for first-seen) — MIN/MAX
|
|
176
|
+
* over an ordered non-numeric column is well-defined (SQL + the JS fold agree).
|
|
177
|
+
*/
|
|
178
|
+
function resolveAggArg(fn, word) {
|
|
179
|
+
if (fn === "sum" || fn === "avg")
|
|
180
|
+
return resolveMetric(word);
|
|
181
|
+
const metric = columns_1.METRICS_BY_NAME.get(word);
|
|
182
|
+
if (metric !== undefined)
|
|
183
|
+
return refOf(metric);
|
|
184
|
+
return resolveDimension(word);
|
|
185
|
+
}
|
|
186
|
+
function refOf(def) {
|
|
187
|
+
return {
|
|
188
|
+
name: def.name,
|
|
189
|
+
source: def.source,
|
|
190
|
+
key: def.key,
|
|
191
|
+
classification: def.classification,
|
|
192
|
+
numeric: def.numeric,
|
|
193
|
+
alias: aliasFor(def.name),
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Parse a query string into a {@link QueryAst}. Pure and total: every path
|
|
198
|
+
* either returns a validated AST or throws {@link QueryGrammarError}. This is
|
|
199
|
+
* the single parser — both the SQLite compiler (below) and the PostgREST
|
|
200
|
+
* evaluator (analytics.ts) consume its output.
|
|
201
|
+
*/
|
|
202
|
+
function parseQuery(input) {
|
|
141
203
|
const tokens = tokenize(input);
|
|
142
204
|
let pos = 0;
|
|
143
205
|
const peek = () => tokens[pos];
|
|
@@ -161,7 +223,7 @@ function compileQuery(input, scope) {
|
|
|
161
223
|
}
|
|
162
224
|
}
|
|
163
225
|
expectWord("select");
|
|
164
|
-
const
|
|
226
|
+
const select = [];
|
|
165
227
|
for (;;) {
|
|
166
228
|
const token = next();
|
|
167
229
|
if (token.kind !== "word")
|
|
@@ -175,22 +237,19 @@ function compileQuery(input, scope) {
|
|
|
175
237
|
if (arg.kind !== "punct" || arg.text !== "*") {
|
|
176
238
|
throw new QueryGrammarError("count takes exactly count(*)");
|
|
177
239
|
}
|
|
178
|
-
|
|
240
|
+
select.push({ kind: "count", alias: "count" });
|
|
179
241
|
}
|
|
180
242
|
else {
|
|
181
243
|
if (arg.kind !== "word")
|
|
182
244
|
throw new QueryGrammarError(`bad metric in ${lower}()`);
|
|
183
|
-
const
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
}
|
|
187
|
-
selectSql.push(`${lower.toUpperCase()}(${metric}) AS "${lower}_${arg.text.toLowerCase()}"`);
|
|
245
|
+
const fn = lower;
|
|
246
|
+
const metric = resolveAggArg(fn, arg.text.toLowerCase());
|
|
247
|
+
const aliasBase = metric.alias.replace(/\./g, "_");
|
|
248
|
+
select.push({ kind: "aggregate", fn, metric, alias: `${fn}_${aliasBase}` });
|
|
188
249
|
}
|
|
189
250
|
}
|
|
190
251
|
else {
|
|
191
|
-
|
|
192
|
-
const alias = token.text.startsWith("tag.") ? token.text.slice(4) : token.text;
|
|
193
|
-
selectSql.push(`${columnSql(token.text)} AS "${alias}"`);
|
|
252
|
+
select.push({ kind: "column", column: resolveDimension(token.text) });
|
|
194
253
|
}
|
|
195
254
|
if (peek()?.kind === "punct" && peek()?.text === ",") {
|
|
196
255
|
pos += 1;
|
|
@@ -203,32 +262,35 @@ function compileQuery(input, scope) {
|
|
|
203
262
|
if (table.kind !== "word" || table.text.toLowerCase() !== "inferences") {
|
|
204
263
|
throw new QueryGrammarError(`only "inferences" is queryable, got "${table.text}"`);
|
|
205
264
|
}
|
|
206
|
-
const
|
|
207
|
-
const params = [];
|
|
265
|
+
const where = [];
|
|
208
266
|
if (peek()?.kind === "word" && peek()?.text.toLowerCase() === "where") {
|
|
209
267
|
pos += 1;
|
|
210
268
|
for (;;) {
|
|
211
|
-
const
|
|
212
|
-
if (
|
|
269
|
+
const columnTok = next();
|
|
270
|
+
if (columnTok.kind !== "word")
|
|
213
271
|
throw new QueryGrammarError(`bad condition column`);
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
272
|
+
// A condition may target a metric (cost > 0.01) or a dimension (route = 'x').
|
|
273
|
+
const column = columns_1.METRICS_BY_NAME.has(columnTok.text.toLowerCase())
|
|
274
|
+
? resolveMetric(columnTok.text.toLowerCase())
|
|
275
|
+
: resolveDimension(columnTok.text);
|
|
217
276
|
const op = next();
|
|
218
277
|
if (op.kind !== "punct" || !OPERATORS.has(op.text)) {
|
|
219
278
|
throw new QueryGrammarError(`bad operator "${op.text}" — allowed: ${[...OPERATORS].join(" ")}`);
|
|
220
279
|
}
|
|
221
280
|
const value = next();
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
else if (value.kind === "number")
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
else {
|
|
281
|
+
let literal;
|
|
282
|
+
if (value.kind === "string")
|
|
283
|
+
literal = value.text;
|
|
284
|
+
else if (value.kind === "number")
|
|
285
|
+
literal = Number(value.text);
|
|
286
|
+
else
|
|
229
287
|
throw new QueryGrammarError(`condition value must be a 'string' or number, got "${value.text}"`);
|
|
288
|
+
// A `ts` bound is pushed down to PostgREST by the hosted read: validate it
|
|
289
|
+
// here so a bad literal is a 422 teaching error, never an upstream 400.
|
|
290
|
+
if (column.source === "plain" && column.key === "ts" && typeof literal === "string" && Number.isNaN(Date.parse(literal))) {
|
|
291
|
+
throw new QueryGrammarError(`ts bound must be an ISO-8601 timestamp (got '${literal}')`);
|
|
230
292
|
}
|
|
231
|
-
|
|
293
|
+
where.push({ column, op: op.text, value: literal });
|
|
232
294
|
if (peek()?.kind === "word" && peek()?.text.toLowerCase() === "and") {
|
|
233
295
|
pos += 1;
|
|
234
296
|
continue;
|
|
@@ -236,22 +298,15 @@ function compileQuery(input, scope) {
|
|
|
236
298
|
break;
|
|
237
299
|
}
|
|
238
300
|
}
|
|
239
|
-
|
|
240
|
-
if (scope.kind === "tag") {
|
|
241
|
-
if (!TAG_NAME.test(scope.tag))
|
|
242
|
-
throw new QueryGrammarError(`bad scope tag: ${scope.tag}`);
|
|
243
|
-
whereSql.push(`json_extract(tags_json, '$.${scope.tag}') = ?`);
|
|
244
|
-
params.push(scope.value);
|
|
245
|
-
}
|
|
246
|
-
const groupSql = [];
|
|
301
|
+
const groupBy = [];
|
|
247
302
|
if (peek()?.kind === "word" && peek()?.text.toLowerCase() === "group") {
|
|
248
303
|
pos += 1;
|
|
249
304
|
expectWord("by");
|
|
250
305
|
for (;;) {
|
|
251
|
-
const
|
|
252
|
-
if (
|
|
306
|
+
const columnTok = next();
|
|
307
|
+
if (columnTok.kind !== "word")
|
|
253
308
|
throw new QueryGrammarError("bad GROUP BY column");
|
|
254
|
-
|
|
309
|
+
groupBy.push(resolveDimension(columnTok.text));
|
|
255
310
|
if (peek()?.kind === "punct" && peek()?.text === ",") {
|
|
256
311
|
pos += 1;
|
|
257
312
|
continue;
|
|
@@ -271,9 +326,62 @@ function compileQuery(input, scope) {
|
|
|
271
326
|
if (pos !== tokens.length) {
|
|
272
327
|
throw new QueryGrammarError(`unexpected trailing input: "${tokens[pos].text}"`);
|
|
273
328
|
}
|
|
329
|
+
return { select, where, groupBy, limit };
|
|
330
|
+
}
|
|
331
|
+
// ── SQLite rendering (the local `/query` surface) ────────────────────────────
|
|
332
|
+
/** A column reference → SQLite expression. `meta.<field>` and reserved dims read
|
|
333
|
+
* out of `meta_json` / `tags_json` via json_extract with a VALIDATED path. */
|
|
334
|
+
function sqliteColumnSql(ref) {
|
|
335
|
+
if (ref.source === "tag")
|
|
336
|
+
return `json_extract(tags_json, '$.${ref.key}')`;
|
|
337
|
+
if (ref.source === "meta")
|
|
338
|
+
return `json_extract(meta_json, '$.${ref.key}')`;
|
|
339
|
+
return ref.key;
|
|
340
|
+
}
|
|
341
|
+
/** An aggregate's SQLite argument. A `meta.*` value under a NUMERIC aggregate
|
|
342
|
+
* (`sum`/`avg`) is cast to REAL so the fold is numeric on the json_extract text;
|
|
343
|
+
* `min`/`max` over an ordered column (ts, a snapshot id) keep the raw expression
|
|
344
|
+
* so the comparison stays lexicographic — matching the JS fold. */
|
|
345
|
+
function sqliteAggArgSql(fn, ref) {
|
|
346
|
+
if ((fn === "sum" || fn === "avg") && ref.source === "meta") {
|
|
347
|
+
return `CAST(json_extract(meta_json, '$.${ref.key}') AS REAL)`;
|
|
348
|
+
}
|
|
349
|
+
return sqliteColumnSql(ref);
|
|
350
|
+
}
|
|
351
|
+
/**
|
|
352
|
+
* Compile a query string to a single read-only SQLite SELECT (the local
|
|
353
|
+
* telemetry store). Row scoping is ANDed into WHERE unconditionally — the caller
|
|
354
|
+
* cannot opt out of it.
|
|
355
|
+
*/
|
|
356
|
+
function compileQuery(input, scope) {
|
|
357
|
+
const ast = parseQuery(input);
|
|
358
|
+
const params = [];
|
|
359
|
+
const selectSql = ast.select.map((item) => {
|
|
360
|
+
if (item.kind === "count")
|
|
361
|
+
return `COUNT(*) AS "count"`;
|
|
362
|
+
if (item.kind === "aggregate") {
|
|
363
|
+
return `${item.fn.toUpperCase()}(${sqliteAggArgSql(item.fn, item.metric)}) AS "${item.alias}"`;
|
|
364
|
+
}
|
|
365
|
+
return `${sqliteColumnSql(item.column)} AS "${item.column.alias}"`;
|
|
366
|
+
});
|
|
367
|
+
const whereSql = ast.where.map((cond) => {
|
|
368
|
+
params.push(cond.value);
|
|
369
|
+
return `${sqliteColumnSql(cond.column)} ${cond.op} ?`;
|
|
370
|
+
});
|
|
371
|
+
// Live rows only — bake-off replay rows (`tags.replay`) never move a route's
|
|
372
|
+
// numbers; mirrored in the JS fold (evaluateAnalyticsQuery).
|
|
373
|
+
whereSql.push("json_extract(tags_json, '$.replay') IS NULL");
|
|
374
|
+
// Row scoping — ANDed in unconditionally; the caller cannot remove it.
|
|
375
|
+
if (scope.kind === "tag") {
|
|
376
|
+
if (!TAG_NAME.test(scope.tag))
|
|
377
|
+
throw new QueryGrammarError(`bad scope tag: ${scope.tag}`);
|
|
378
|
+
whereSql.push(`json_extract(tags_json, '$.${scope.tag}') = ?`);
|
|
379
|
+
params.push(scope.value);
|
|
380
|
+
}
|
|
381
|
+
const groupSql = ast.groupBy.map((ref) => sqliteColumnSql(ref));
|
|
274
382
|
const sql = `SELECT ${selectSql.join(", ")} FROM inferences` +
|
|
275
383
|
(whereSql.length > 0 ? ` WHERE ${whereSql.join(" AND ")}` : "") +
|
|
276
384
|
(groupSql.length > 0 ? ` GROUP BY ${groupSql.join(", ")}` : "") +
|
|
277
|
-
` LIMIT ${limit}`;
|
|
385
|
+
` LIMIT ${ast.limit}`;
|
|
278
386
|
return { sql, params };
|
|
279
387
|
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The three saved analytics questions — WI-1b (spec §5). ONE shared module the
|
|
3
|
+
* server, console, and CLI import, so the console chip, the CLI `--saved <id>`,
|
|
4
|
+
* and any docs answer the SAME grammar. Each question is a trailing-window
|
|
5
|
+
* query; the ts bound is filled at call time (`grammar(sinceIso)`) so the module
|
|
6
|
+
* stays pure and testable — the caller computes `sinceIso = now − windowDays`.
|
|
7
|
+
*
|
|
8
|
+
* The questions return COUNTS and components; a RATE (truncation %, refusal %)
|
|
9
|
+
* is derived by the surface from those counts — the grammar measures, it never
|
|
10
|
+
* editorializes (F5: measured-only).
|
|
11
|
+
*/
|
|
12
|
+
export interface SavedQuery {
|
|
13
|
+
readonly id: string;
|
|
14
|
+
readonly title: string;
|
|
15
|
+
readonly description: string;
|
|
16
|
+
/** The trailing window the question asks over, in days. */
|
|
17
|
+
readonly windowDays: number;
|
|
18
|
+
/** The grammar string with the `ts` lower bound filled in. */
|
|
19
|
+
grammar(sinceIso: string): string;
|
|
20
|
+
}
|
|
21
|
+
export declare const SAVED_QUERIES: readonly SavedQuery[];
|
|
22
|
+
/** Look up a saved question by id, or null. */
|
|
23
|
+
export declare function savedQueryById(id: string): SavedQuery | null;
|
|
24
|
+
/** The ISO lower bound for a trailing window of `windowDays` ending at `now`. */
|
|
25
|
+
export declare function sinceIsoFor(windowDays: number, now: Date): string;
|
|
26
|
+
//# sourceMappingURL=saved.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"saved.d.ts","sourceRoot":"","sources":["../../src/query/saved.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,2DAA2D;IAC3D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC;AAED,eAAO,MAAM,aAAa,EAAE,SAAS,UAAU,EA8B9C,CAAC;AAEF,+CAA+C;AAC/C,wBAAgB,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CAE5D;AAED,iFAAiF;AACjF,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,GAAG,MAAM,CAEjE"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* The three saved analytics questions — WI-1b (spec §5). ONE shared module the
|
|
4
|
+
* server, console, and CLI import, so the console chip, the CLI `--saved <id>`,
|
|
5
|
+
* and any docs answer the SAME grammar. Each question is a trailing-window
|
|
6
|
+
* query; the ts bound is filled at call time (`grammar(sinceIso)`) so the module
|
|
7
|
+
* stays pure and testable — the caller computes `sinceIso = now − windowDays`.
|
|
8
|
+
*
|
|
9
|
+
* The questions return COUNTS and components; a RATE (truncation %, refusal %)
|
|
10
|
+
* is derived by the surface from those counts — the grammar measures, it never
|
|
11
|
+
* editorializes (F5: measured-only).
|
|
12
|
+
*/
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
exports.SAVED_QUERIES = void 0;
|
|
15
|
+
exports.savedQueryById = savedQueryById;
|
|
16
|
+
exports.sinceIsoFor = sinceIsoFor;
|
|
17
|
+
exports.SAVED_QUERIES = [
|
|
18
|
+
{
|
|
19
|
+
id: "cost-per-subject",
|
|
20
|
+
title: "Cost per subject this week",
|
|
21
|
+
description: "Total estimated spend grouped by the reserved `subject` dimension over the last 7 days.",
|
|
22
|
+
windowDays: 7,
|
|
23
|
+
grammar: (since) => `SELECT subject, sum(cost), count(*) FROM inferences WHERE ts >= '${since}' GROUP BY subject`,
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
id: "truncation-refusal",
|
|
27
|
+
title: "Truncation & refusal rate by task × model",
|
|
28
|
+
description: "Attempt counts grouped by route (task), model, and finish reason over the last 7 days — the surface " +
|
|
29
|
+
"derives truncation (finish_reason=length) and refusal rates from the counts.",
|
|
30
|
+
windowDays: 7,
|
|
31
|
+
grammar: (since) => `SELECT route, model, meta.finish_reason, count(*) FROM inferences ` +
|
|
32
|
+
`WHERE ts >= '${since}' GROUP BY route, model, meta.finish_reason`,
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
id: "snapshots-seen",
|
|
36
|
+
title: "Model snapshots seen per route (28 d)",
|
|
37
|
+
description: "Distinct provider-reported model snapshots per route over the last 28 days, with first-seen (min ts) and " +
|
|
38
|
+
"last-seen (max ts) and the attempt count — a drift signal, not proof (spec §7).",
|
|
39
|
+
windowDays: 28,
|
|
40
|
+
grammar: (since) => `SELECT route, meta.model_snapshot, min(ts), max(ts), count(*) FROM inferences ` +
|
|
41
|
+
`WHERE ts >= '${since}' GROUP BY route, meta.model_snapshot`,
|
|
42
|
+
},
|
|
43
|
+
];
|
|
44
|
+
/** Look up a saved question by id, or null. */
|
|
45
|
+
function savedQueryById(id) {
|
|
46
|
+
return exports.SAVED_QUERIES.find((q) => q.id === id) ?? null;
|
|
47
|
+
}
|
|
48
|
+
/** The ISO lower bound for a trailing window of `windowDays` ending at `now`. */
|
|
49
|
+
function sinceIsoFor(windowDays, now) {
|
|
50
|
+
return new Date(now.getTime() - windowDays * 24 * 60 * 60 * 1000).toISOString();
|
|
51
|
+
}
|
|
@@ -14,6 +14,8 @@
|
|
|
14
14
|
* Bars are the registry's own, not new judgments: 0.9 conformance (the
|
|
15
15
|
* discrepancy detector's bar) and 0.5 grounding (facts.ts's GROUNDED_BAR).
|
|
16
16
|
*/
|
|
17
|
+
import type { CapabilityFlag } from "../types/provider";
|
|
18
|
+
import type { Requirement } from "../types/route";
|
|
17
19
|
/** The capability predicates a question can carry. Deliberately the probed
|
|
18
20
|
* dimensions only — we answer from measurement, not from provider claims. */
|
|
19
21
|
export type CapabilityPredicate = "structured-native" | "structured-coached" | "grounding" | "caching";
|
|
@@ -62,6 +64,63 @@ export interface SelectCriteria {
|
|
|
62
64
|
readonly limit?: number;
|
|
63
65
|
}
|
|
64
66
|
export declare function selectModels(rows: readonly ModelSelectRow[], criteria?: SelectCriteria): ModelSelectRow[];
|
|
67
|
+
/**
|
|
68
|
+
* Declare-first challenger eligibility (routing-console spec §4.4, R3) — the ONE
|
|
69
|
+
* place that answers "can this model be a trial challenger for route X?", called
|
|
70
|
+
* by the console picker (`GET …/routes/:route/challengers`) AND the oracle
|
|
71
|
+
* `propose_candidates`. Kept here beside `selectModels` so eligibility and the
|
|
72
|
+
* registry grammar never drift between surfaces.
|
|
73
|
+
*
|
|
74
|
+
* It decomposes a route's `require` + json rung into registry CAPABILITY FLAGS
|
|
75
|
+
* (the same mechanism the live resolver's `satisfiesRequirements` uses — an
|
|
76
|
+
* eligible challenger is one that could actually SERVE) rather than probed
|
|
77
|
+
* rates: probes GATE CLAIMS, not serving, so an UNPROBED model that carries the
|
|
78
|
+
* flags is eligible (the picker badges it `unprobed`; the caller sees there's no
|
|
79
|
+
* measured evidence yet). Types are imported type-only so select.ts stays
|
|
80
|
+
* dependency-free (browser-safe).
|
|
81
|
+
*/
|
|
82
|
+
/** A flag OR-group: the challenger must carry AT LEAST ONE flag in the group.
|
|
83
|
+
* `json_mode` routes accept native strict OR syntactic json forcing, so that is
|
|
84
|
+
* a two-flag group; everything else is a singleton. */
|
|
85
|
+
export type RequirementFlagGroup = readonly CapabilityFlag[];
|
|
86
|
+
export interface RouteRequirement {
|
|
87
|
+
/** Flag OR-groups, ANDed together. Empty ⇒ no capability requirement. */
|
|
88
|
+
readonly flagGroups: readonly RequirementFlagGroup[];
|
|
89
|
+
/** The route requires `grounded`: satisfied by a `grounded_native` model OR —
|
|
90
|
+
* when the deployment has a search provider configured — by ANY model (the run
|
|
91
|
+
* loop orchestrates search-and-inject). Carried separately so the picker can
|
|
92
|
+
* apply the "or any model when search is configured" relaxation. */
|
|
93
|
+
readonly grounded: boolean;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Map a route's `require` + json rung to the capability-flag requirement a
|
|
97
|
+
* challenger must satisfy (spec §4.4). Mirrors `run/resolve.ts`
|
|
98
|
+
* `satisfiesRequirements` flag-for-flag (an invariant test pins the agreement):
|
|
99
|
+
* schema_conformant + json:native → structured_native
|
|
100
|
+
* schema_conformant + json:json_mode → structured_native | json_mode
|
|
101
|
+
* grounded → grounded_native (or search-configured)
|
|
102
|
+
* trace_visible → trace_visible
|
|
103
|
+
* zero_retention → zero_retention
|
|
104
|
+
*/
|
|
105
|
+
export declare function requirementPredicates(route: {
|
|
106
|
+
readonly require: readonly Requirement[];
|
|
107
|
+
readonly policy: {
|
|
108
|
+
readonly json: "native" | "json_mode";
|
|
109
|
+
};
|
|
110
|
+
}): RouteRequirement;
|
|
111
|
+
export interface ChallengerEligibility {
|
|
112
|
+
readonly eligible: boolean;
|
|
113
|
+
/** Present only when NOT eligible — the missing capability, in plain words. */
|
|
114
|
+
readonly why?: string;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Evaluate one candidate model's eligibility against a route's requirement
|
|
118
|
+
* (spec §4.4). `flags` is the registry capability set for that model (the
|
|
119
|
+
* server's `RegistryFacts.flagsFor`, the SDK's `CapabilityLookup`).
|
|
120
|
+
* `searchAvailable` lets `grounded` be satisfied by any model when a search
|
|
121
|
+
* provider is configured. Returns `{eligible:true}` or `{eligible:false, why}`.
|
|
122
|
+
*/
|
|
123
|
+
export declare function evaluateChallengerEligibility(req: RouteRequirement, flags: ReadonlySet<CapabilityFlag>, searchAvailable?: boolean): ChallengerEligibility;
|
|
65
124
|
/** Why a question found nothing — the difference between "no such model" and
|
|
66
125
|
* "we have not probed one yet" is the whole point of the demand meter. */
|
|
67
126
|
export declare function explainEmptySelection(criteria: SelectCriteria): string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"select.d.ts","sourceRoot":"","sources":["../../src/registry/select.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH;6EAC6E;AAC7E,MAAM,MAAM,mBAAmB,GAC3B,mBAAmB,GACnB,oBAAoB,GACpB,WAAW,GACX,SAAS,CAAC;AAEd,eAAO,MAAM,yBAAyB,EAAE,SAAS,mBAAmB,EAKnE,CAAC;AAEF;+EAC+E;AAC/E,eAAO,MAAM,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAQzE,CAAC;AAEF;gFACgF;AAChF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAKD,wBAAgB,aAAa,CAAC,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,mBAAmB,GAAG,OAAO,CAWpF;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC9C,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,gEAAgE;IAChE,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;CAClC;AA0BD;;;mDAGmD;AACnD,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,CAkB/D;AAED,MAAM,WAAW,cAAc;IAC7B,kEAAkE;IAClE,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC/C,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC;IACrC,mDAAmD;IACnD,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC;mEAC+D;IAC/D,QAAQ,CAAC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IACvC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,SAAS,cAAc,EAAE,EAC/B,QAAQ,GAAE,cAAmB,GAC5B,cAAc,EAAE,CA6BlB;AAED;0EAC0E;AAC1E,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,cAAc,GAAG,MAAM,CAatE"}
|
|
1
|
+
{"version":3,"file":"select.d.ts","sourceRoot":"","sources":["../../src/registry/select.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAElD;6EAC6E;AAC7E,MAAM,MAAM,mBAAmB,GAC3B,mBAAmB,GACnB,oBAAoB,GACpB,WAAW,GACX,SAAS,CAAC;AAEd,eAAO,MAAM,yBAAyB,EAAE,SAAS,mBAAmB,EAKnE,CAAC;AAEF;+EAC+E;AAC/E,eAAO,MAAM,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAQzE,CAAC;AAEF;gFACgF;AAChF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAKD,wBAAgB,aAAa,CAAC,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,mBAAmB,GAAG,OAAO,CAWpF;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC9C,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,gEAAgE;IAChE,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;CAClC;AA0BD;;;mDAGmD;AACnD,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,CAkB/D;AAED,MAAM,WAAW,cAAc;IAC7B,kEAAkE;IAClE,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC/C,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC;IACrC,mDAAmD;IACnD,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC;mEAC+D;IAC/D,QAAQ,CAAC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IACvC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,SAAS,cAAc,EAAE,EAC/B,QAAQ,GAAE,cAAmB,GAC5B,cAAc,EAAE,CA6BlB;AAED;;;;;;;;;;;;;;GAcG;AAEH;;uDAEuD;AACvD,MAAM,MAAM,oBAAoB,GAAG,SAAS,cAAc,EAAE,CAAC;AAE7D,MAAM,WAAW,gBAAgB;IAC/B,yEAAyE;IACzE,QAAQ,CAAC,UAAU,EAAE,SAAS,oBAAoB,EAAE,CAAC;IACrD;;;wEAGoE;IACpE,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE;IAC3C,QAAQ,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,CAAC;IACzC,QAAQ,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,WAAW,CAAA;KAAE,CAAC;CAC5D,GAAG,gBAAgB,CAsBnB;AAeD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,+EAA+E;IAC/E,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;GAMG;AACH,wBAAgB,6BAA6B,CAC3C,GAAG,EAAE,gBAAgB,EACrB,KAAK,EAAE,WAAW,CAAC,cAAc,CAAC,EAClC,eAAe,UAAQ,GACtB,qBAAqB,CAYvB;AAED;0EAC0E;AAC1E,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,cAAc,GAAG,MAAM,CAatE"}
|
package/dist/registry/select.js
CHANGED
|
@@ -20,6 +20,8 @@ exports.PREDICATE_GLOSS = exports.ALL_CAPABILITY_PREDICATES = void 0;
|
|
|
20
20
|
exports.hasCapability = hasCapability;
|
|
21
21
|
exports.parseCapabilityQuery = parseCapabilityQuery;
|
|
22
22
|
exports.selectModels = selectModels;
|
|
23
|
+
exports.requirementPredicates = requirementPredicates;
|
|
24
|
+
exports.evaluateChallengerEligibility = evaluateChallengerEligibility;
|
|
23
25
|
exports.explainEmptySelection = explainEmptySelection;
|
|
24
26
|
exports.ALL_CAPABILITY_PREDICATES = [
|
|
25
27
|
"structured-native",
|
|
@@ -124,6 +126,70 @@ function selectModels(rows, criteria = {}) {
|
|
|
124
126
|
out = out.slice(0, criteria.limit);
|
|
125
127
|
return out;
|
|
126
128
|
}
|
|
129
|
+
/**
|
|
130
|
+
* Map a route's `require` + json rung to the capability-flag requirement a
|
|
131
|
+
* challenger must satisfy (spec §4.4). Mirrors `run/resolve.ts`
|
|
132
|
+
* `satisfiesRequirements` flag-for-flag (an invariant test pins the agreement):
|
|
133
|
+
* schema_conformant + json:native → structured_native
|
|
134
|
+
* schema_conformant + json:json_mode → structured_native | json_mode
|
|
135
|
+
* grounded → grounded_native (or search-configured)
|
|
136
|
+
* trace_visible → trace_visible
|
|
137
|
+
* zero_retention → zero_retention
|
|
138
|
+
*/
|
|
139
|
+
function requirementPredicates(route) {
|
|
140
|
+
const flagGroups = [];
|
|
141
|
+
let grounded = false;
|
|
142
|
+
for (const requirement of route.require) {
|
|
143
|
+
switch (requirement) {
|
|
144
|
+
case "schema_conformant":
|
|
145
|
+
flagGroups.push(route.policy.json === "native" ? ["structured_native"] : ["structured_native", "json_mode"]);
|
|
146
|
+
break;
|
|
147
|
+
case "grounded":
|
|
148
|
+
grounded = true;
|
|
149
|
+
break;
|
|
150
|
+
case "trace_visible":
|
|
151
|
+
flagGroups.push(["trace_visible"]);
|
|
152
|
+
break;
|
|
153
|
+
case "zero_retention":
|
|
154
|
+
flagGroups.push(["zero_retention"]);
|
|
155
|
+
break;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return { flagGroups, grounded };
|
|
159
|
+
}
|
|
160
|
+
/** Human label for a flag requirement, used in the picker's `why` line. */
|
|
161
|
+
const FLAG_LABEL = {
|
|
162
|
+
structured_native: "native strict JSON-schema enforcement",
|
|
163
|
+
json_mode: "JSON-mode forcing",
|
|
164
|
+
grounded_native: "native web grounding",
|
|
165
|
+
context_cache_explicit: "explicit context caching",
|
|
166
|
+
prompt_cache_key: "key-based prompt caching",
|
|
167
|
+
prefix_cache_implicit: "implicit prefix caching",
|
|
168
|
+
tier_flex: "a discounted flex tier",
|
|
169
|
+
trace_visible: "a visible reasoning trace",
|
|
170
|
+
zero_retention: "a zero-retention endpoint",
|
|
171
|
+
};
|
|
172
|
+
/**
|
|
173
|
+
* Evaluate one candidate model's eligibility against a route's requirement
|
|
174
|
+
* (spec §4.4). `flags` is the registry capability set for that model (the
|
|
175
|
+
* server's `RegistryFacts.flagsFor`, the SDK's `CapabilityLookup`).
|
|
176
|
+
* `searchAvailable` lets `grounded` be satisfied by any model when a search
|
|
177
|
+
* provider is configured. Returns `{eligible:true}` or `{eligible:false, why}`.
|
|
178
|
+
*/
|
|
179
|
+
function evaluateChallengerEligibility(req, flags, searchAvailable = false) {
|
|
180
|
+
const missing = [];
|
|
181
|
+
for (const group of req.flagGroups) {
|
|
182
|
+
if (!group.some((f) => flags.has(f))) {
|
|
183
|
+
missing.push(group.map((f) => FLAG_LABEL[f]).join(" or "));
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
if (req.grounded && !flags.has("grounded_native") && !searchAvailable) {
|
|
187
|
+
missing.push("native web grounding (or a configured search provider)");
|
|
188
|
+
}
|
|
189
|
+
if (missing.length === 0)
|
|
190
|
+
return { eligible: true };
|
|
191
|
+
return { eligible: false, why: `needs ${missing.join("; and ")}` };
|
|
192
|
+
}
|
|
127
193
|
/** Why a question found nothing — the difference between "no such model" and
|
|
128
194
|
* "we have not probed one yet" is the whole point of the demand meter. */
|
|
129
195
|
function explainEmptySelection(criteria) {
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Non-fatal loader advisories — extracted from loader.ts (kept it under the
|
|
3
|
+
* 700-LOC cap). Each is a pure function returning a warning line or null; the
|
|
4
|
+
* loader console.warns them after a bundle loads. Unit-tested directly.
|
|
5
|
+
*/
|
|
6
|
+
import type { CandidateSpec, RouteVariant } from "../types/route";
|
|
7
|
+
/**
|
|
8
|
+
* CE-2 A5 co-item: warn (non-fatal) when a route's output schema has an ARRAY
|
|
9
|
+
* root. OpenAI strict structured output (`response_format: json_schema`, strict)
|
|
10
|
+
* requires a top-level OBJECT — an array-root schema cannot use strict
|
|
11
|
+
* enforcement on OpenAI candidates and silently falls back to non-strict there
|
|
12
|
+
* (the walkthrough §10 provider-matrix gap). The route still LOADS and serves as
|
|
13
|
+
* declared; the fix is to wrap the array in an object property so strict
|
|
14
|
+
* enforcement holds across every provider. Returns the warning line, or null.
|
|
15
|
+
*/
|
|
16
|
+
export declare function arrayRootSchemaWarning(route: string, outputSchema: object | null): string | null;
|
|
17
|
+
/**
|
|
18
|
+
* Declare-first dead-declaration warning (routing-console spec §4.4): a `trial`
|
|
19
|
+
* candidate that no variant references can never be reached — not live (trial
|
|
20
|
+
* candidates are dropped on every live path) and not in a bake-off (a bake-off
|
|
21
|
+
* replays a NAMED variant, and no variant narrows to it). Warn so the author
|
|
22
|
+
* either adds the `trial-<provider>-<model>` variant the declare flow emits or
|
|
23
|
+
* removes the dangling candidate. Non-fatal (the route still loads and serves
|
|
24
|
+
* its live candidates as declared). Pure — unit-tested directly. Returns the
|
|
25
|
+
* warning line, or null when every trial candidate is referenced.
|
|
26
|
+
*/
|
|
27
|
+
export declare function unreferencedTrialCandidatesWarning(route: string, candidates: readonly CandidateSpec[], variants: readonly RouteVariant[]): string | null;
|
|
28
|
+
//# sourceMappingURL=loader-warnings.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loader-warnings.d.ts","sourceRoot":"","sources":["../../src/routes/loader-warnings.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAElE;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAWhG;AAED;;;;;;;;;GASG;AACH,wBAAgB,kCAAkC,CAChD,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,SAAS,aAAa,EAAE,EACpC,QAAQ,EAAE,SAAS,YAAY,EAAE,GAChC,MAAM,GAAG,IAAI,CAef"}
|