modelrig 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +125 -0
- package/README.md +54 -4
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +36 -7
- package/dist/adapters/attempt-meta-map.d.ts +63 -0
- package/dist/adapters/attempt-meta-map.d.ts.map +1 -0
- package/dist/adapters/attempt-meta-map.js +154 -0
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +25 -2
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +29 -2
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +25 -2
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +25 -2
- package/dist/adapters/openai-stream.d.ts +2 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -1
- package/dist/adapters/openai-stream.js +23 -3
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +36 -6
- package/dist/artifacts/artifact.d.ts.map +1 -1
- package/dist/artifacts/artifact.js +24 -5
- package/dist/artifacts/classification.d.ts +27 -0
- package/dist/artifacts/classification.d.ts.map +1 -0
- package/dist/artifacts/classification.js +38 -0
- package/dist/artifacts/index.d.ts +3 -1
- package/dist/artifacts/index.d.ts.map +1 -1
- package/dist/artifacts/index.js +10 -1
- package/dist/artifacts/run-context.d.ts +25 -3
- package/dist/artifacts/run-context.d.ts.map +1 -1
- package/dist/artifacts/run-context.js +61 -0
- package/dist/artifacts/scrub.d.ts +26 -0
- package/dist/artifacts/scrub.d.ts.map +1 -1
- package/dist/artifacts/scrub.js +53 -0
- package/dist/artifacts/step.d.ts +9 -0
- package/dist/artifacts/step.d.ts.map +1 -1
- package/dist/artifacts/step.js +9 -0
- package/dist/bakeoff/engine.d.ts +46 -4
- package/dist/bakeoff/engine.d.ts.map +1 -1
- package/dist/bakeoff/engine.js +269 -19
- package/dist/bakeoff/eval-cases.d.ts +5 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -1
- package/dist/bakeoff/eval-cases.js +3 -0
- package/dist/bakeoff/judge.d.ts +39 -0
- package/dist/bakeoff/judge.d.ts.map +1 -0
- package/dist/bakeoff/judge.js +102 -0
- package/dist/bakeoff/quality.d.ts +44 -0
- package/dist/bakeoff/quality.d.ts.map +1 -0
- package/dist/bakeoff/quality.js +126 -0
- package/dist/bakeoff/reference.d.ts +34 -0
- package/dist/bakeoff/reference.d.ts.map +1 -0
- package/dist/bakeoff/reference.js +68 -0
- package/dist/bakeoff/replay.d.ts +25 -0
- package/dist/bakeoff/replay.d.ts.map +1 -1
- package/dist/bakeoff/replay.js +48 -1
- package/dist/bakeoff/report.d.ts +8 -1
- package/dist/bakeoff/report.d.ts.map +1 -1
- package/dist/bakeoff/report.js +83 -12
- package/dist/bakeoff/samples.d.ts +35 -2
- package/dist/bakeoff/samples.d.ts.map +1 -1
- package/dist/bakeoff/samples.js +67 -6
- package/dist/bakeoff/stats.d.ts +17 -1
- package/dist/bakeoff/stats.d.ts.map +1 -1
- package/dist/bakeoff/stats.js +67 -0
- package/dist/bakeoff/types.d.ts +76 -5
- package/dist/bakeoff/types.d.ts.map +1 -1
- package/dist/bakeoff/types.js +4 -3
- package/dist/cli-query.d.ts +14 -0
- package/dist/cli-query.d.ts.map +1 -0
- package/dist/cli-query.js +59 -0
- package/dist/cli.d.ts +2 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +26 -23
- package/dist/config.d.ts +6 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +14 -0
- package/dist/detect/detector.d.ts +82 -0
- package/dist/detect/detector.d.ts.map +1 -0
- package/dist/detect/detector.js +23 -0
- package/dist/index.d.ts +19 -14
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +38 -11
- package/dist/query/analytics.d.ts +84 -0
- package/dist/query/analytics.d.ts.map +1 -0
- package/dist/query/analytics.js +280 -0
- package/dist/query/columns.d.ts +59 -0
- package/dist/query/columns.d.ts.map +1 -0
- package/dist/query/columns.js +89 -0
- package/dist/query/grammar.d.ts +74 -7
- package/dist/query/grammar.d.ts.map +1 -1
- package/dist/query/grammar.js +179 -71
- package/dist/query/saved.d.ts +26 -0
- package/dist/query/saved.d.ts.map +1 -0
- package/dist/query/saved.js +51 -0
- package/dist/routes/loader.js +15 -10
- package/dist/routes/objective-loader.d.ts +35 -0
- package/dist/routes/objective-loader.d.ts.map +1 -0
- package/dist/routes/objective-loader.js +286 -0
- package/dist/run/accept.d.ts +43 -0
- package/dist/run/accept.d.ts.map +1 -0
- package/dist/run/accept.js +65 -0
- package/dist/run/cost.d.ts +13 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +27 -0
- package/dist/run/dispatch.d.ts +24 -1
- package/dist/run/dispatch.d.ts.map +1 -1
- package/dist/run/dispatch.js +19 -0
- package/dist/run/ladder-key.d.ts +4 -0
- package/dist/run/ladder-key.d.ts.map +1 -1
- package/dist/run/ladder-key.js +3 -0
- package/dist/run/quality-gate.d.ts +39 -0
- package/dist/run/quality-gate.d.ts.map +1 -0
- package/dist/run/quality-gate.js +70 -0
- package/dist/run/raw.d.ts.map +1 -1
- package/dist/run/raw.js +22 -3
- package/dist/run/record-attempt.d.ts.map +1 -1
- package/dist/run/record-attempt.js +22 -1
- package/dist/run/routing-docs.d.ts.map +1 -1
- package/dist/run/routing-docs.js +11 -4
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +46 -39
- package/dist/settings/index.d.ts +1 -1
- package/dist/settings/index.d.ts.map +1 -1
- package/dist/settings/index.js +2 -1
- package/dist/settings/task-settings.d.ts +10 -0
- package/dist/settings/task-settings.d.ts.map +1 -1
- package/dist/settings/task-settings.js +17 -0
- package/dist/telemetry/attempt-meta.d.ts +178 -0
- package/dist/telemetry/attempt-meta.d.ts.map +1 -0
- package/dist/telemetry/attempt-meta.js +319 -0
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +6 -0
- package/dist/telemetry/ingest-protocol.d.ts +11 -0
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +3 -0
- package/dist/telemetry/otel-map.d.ts +10 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -1
- package/dist/telemetry/otel-map.js +64 -0
- package/dist/telemetry/store-migrations.d.ts.map +1 -1
- package/dist/telemetry/store-migrations.js +15 -0
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +25 -7
- package/dist/types/failures-teaching.d.ts.map +1 -1
- package/dist/types/failures-teaching.js +20 -0
- package/dist/types/failures.d.ts +1 -1
- package/dist/types/failures.d.ts.map +1 -1
- package/dist/types/failures.js +1 -0
- package/dist/types/provider.d.ts +9 -0
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/quality.d.ts +41 -0
- package/dist/types/quality.d.ts.map +1 -0
- package/dist/types/quality.js +12 -0
- package/dist/types/route.d.ts +89 -0
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +13 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts.map +1 -1
- package/dist/validate-cli.js +10 -0
- package/package.json +1 -1
package/dist/validate-cli.js
CHANGED
|
@@ -16,6 +16,7 @@ const config_1 = require("./config");
|
|
|
16
16
|
const rawonly_1 = require("./run/rawonly");
|
|
17
17
|
const loader_1 = require("./routes/loader");
|
|
18
18
|
const manifest_1 = require("./routes/manifest");
|
|
19
|
+
const attempt_meta_1 = require("./telemetry/attempt-meta");
|
|
19
20
|
function runValidateCli() {
|
|
20
21
|
const config = (0, config_1.loadConfigFromEnv)();
|
|
21
22
|
const routesDir = (0, rawonly_1.requireRoutesDir)(config, "modelrig validate");
|
|
@@ -26,6 +27,15 @@ function runValidateCli() {
|
|
|
26
27
|
try {
|
|
27
28
|
const manifest = (0, manifest_1.loadManifest)(rigDir);
|
|
28
29
|
console.log(manifest === null ? `— rig.yaml (none at ${rigDir})` : "OK rig.yaml");
|
|
30
|
+
// WI-1 (spec §4): nudge a declared dimension that is a near-miss of a
|
|
31
|
+
// reserved dimension toward the well-known key, so cost/coverage group-bys
|
|
32
|
+
// line up across rigs. A WARN, never a failure — the customer's key is valid.
|
|
33
|
+
if (manifest !== null) {
|
|
34
|
+
for (const nm of (0, attempt_meta_1.reservedTagNearMisses)(manifest.dimensions.map((d) => d.key))) {
|
|
35
|
+
console.warn(`WARN rig.yaml dimension "${nm.key}" looks like the reserved dimension "${nm.suggestion}" — ` +
|
|
36
|
+
`rename it to "${nm.suggestion}" to use ModelRig's built-in cost/coverage group-bys.`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
29
39
|
}
|
|
30
40
|
catch (err) {
|
|
31
41
|
failures += 1;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "modelrig",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "ModelRig — capability-aware AI gateway: routes-as-code, provider adapters, typed failures, budget envelopes, and local-first telemetry.",
|
|
6
6
|
"license": "Apache-2.0",
|