modelrig 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/CHANGELOG.md +125 -0
  2. package/README.md +54 -4
  3. package/dist/adapters/anthropic.d.ts.map +1 -1
  4. package/dist/adapters/anthropic.js +36 -7
  5. package/dist/adapters/attempt-meta-map.d.ts +63 -0
  6. package/dist/adapters/attempt-meta-map.d.ts.map +1 -0
  7. package/dist/adapters/attempt-meta-map.js +154 -0
  8. package/dist/adapters/deepseek.d.ts.map +1 -1
  9. package/dist/adapters/deepseek.js +25 -2
  10. package/dist/adapters/gemini.d.ts.map +1 -1
  11. package/dist/adapters/gemini.js +29 -2
  12. package/dist/adapters/grok.d.ts.map +1 -1
  13. package/dist/adapters/grok.js +25 -2
  14. package/dist/adapters/openai-compat.d.ts.map +1 -1
  15. package/dist/adapters/openai-compat.js +25 -2
  16. package/dist/adapters/openai-stream.d.ts +2 -0
  17. package/dist/adapters/openai-stream.d.ts.map +1 -1
  18. package/dist/adapters/openai-stream.js +23 -3
  19. package/dist/adapters/openai.d.ts.map +1 -1
  20. package/dist/adapters/openai.js +36 -6
  21. package/dist/artifacts/artifact.d.ts.map +1 -1
  22. package/dist/artifacts/artifact.js +24 -5
  23. package/dist/artifacts/classification.d.ts +27 -0
  24. package/dist/artifacts/classification.d.ts.map +1 -0
  25. package/dist/artifacts/classification.js +38 -0
  26. package/dist/artifacts/index.d.ts +3 -1
  27. package/dist/artifacts/index.d.ts.map +1 -1
  28. package/dist/artifacts/index.js +10 -1
  29. package/dist/artifacts/run-context.d.ts +25 -3
  30. package/dist/artifacts/run-context.d.ts.map +1 -1
  31. package/dist/artifacts/run-context.js +61 -0
  32. package/dist/artifacts/scrub.d.ts +26 -0
  33. package/dist/artifacts/scrub.d.ts.map +1 -1
  34. package/dist/artifacts/scrub.js +53 -0
  35. package/dist/artifacts/step.d.ts +9 -0
  36. package/dist/artifacts/step.d.ts.map +1 -1
  37. package/dist/artifacts/step.js +9 -0
  38. package/dist/bakeoff/engine.d.ts +46 -4
  39. package/dist/bakeoff/engine.d.ts.map +1 -1
  40. package/dist/bakeoff/engine.js +269 -19
  41. package/dist/bakeoff/eval-cases.d.ts +5 -0
  42. package/dist/bakeoff/eval-cases.d.ts.map +1 -1
  43. package/dist/bakeoff/eval-cases.js +3 -0
  44. package/dist/bakeoff/judge.d.ts +39 -0
  45. package/dist/bakeoff/judge.d.ts.map +1 -0
  46. package/dist/bakeoff/judge.js +102 -0
  47. package/dist/bakeoff/quality.d.ts +44 -0
  48. package/dist/bakeoff/quality.d.ts.map +1 -0
  49. package/dist/bakeoff/quality.js +126 -0
  50. package/dist/bakeoff/reference.d.ts +34 -0
  51. package/dist/bakeoff/reference.d.ts.map +1 -0
  52. package/dist/bakeoff/reference.js +68 -0
  53. package/dist/bakeoff/replay.d.ts +25 -0
  54. package/dist/bakeoff/replay.d.ts.map +1 -1
  55. package/dist/bakeoff/replay.js +48 -1
  56. package/dist/bakeoff/report.d.ts +8 -1
  57. package/dist/bakeoff/report.d.ts.map +1 -1
  58. package/dist/bakeoff/report.js +83 -12
  59. package/dist/bakeoff/samples.d.ts +35 -2
  60. package/dist/bakeoff/samples.d.ts.map +1 -1
  61. package/dist/bakeoff/samples.js +67 -6
  62. package/dist/bakeoff/stats.d.ts +17 -1
  63. package/dist/bakeoff/stats.d.ts.map +1 -1
  64. package/dist/bakeoff/stats.js +67 -0
  65. package/dist/bakeoff/types.d.ts +76 -5
  66. package/dist/bakeoff/types.d.ts.map +1 -1
  67. package/dist/bakeoff/types.js +4 -3
  68. package/dist/cli-query.d.ts +14 -0
  69. package/dist/cli-query.d.ts.map +1 -0
  70. package/dist/cli-query.js +59 -0
  71. package/dist/cli.d.ts +2 -1
  72. package/dist/cli.d.ts.map +1 -1
  73. package/dist/cli.js +26 -23
  74. package/dist/config.d.ts +6 -0
  75. package/dist/config.d.ts.map +1 -1
  76. package/dist/config.js +14 -0
  77. package/dist/detect/detector.d.ts +82 -0
  78. package/dist/detect/detector.d.ts.map +1 -0
  79. package/dist/detect/detector.js +23 -0
  80. package/dist/index.d.ts +19 -14
  81. package/dist/index.d.ts.map +1 -1
  82. package/dist/index.js +38 -11
  83. package/dist/query/analytics.d.ts +84 -0
  84. package/dist/query/analytics.d.ts.map +1 -0
  85. package/dist/query/analytics.js +280 -0
  86. package/dist/query/columns.d.ts +59 -0
  87. package/dist/query/columns.d.ts.map +1 -0
  88. package/dist/query/columns.js +89 -0
  89. package/dist/query/grammar.d.ts +74 -7
  90. package/dist/query/grammar.d.ts.map +1 -1
  91. package/dist/query/grammar.js +179 -71
  92. package/dist/query/saved.d.ts +26 -0
  93. package/dist/query/saved.d.ts.map +1 -0
  94. package/dist/query/saved.js +51 -0
  95. package/dist/routes/loader.js +15 -10
  96. package/dist/routes/objective-loader.d.ts +35 -0
  97. package/dist/routes/objective-loader.d.ts.map +1 -0
  98. package/dist/routes/objective-loader.js +286 -0
  99. package/dist/run/accept.d.ts +43 -0
  100. package/dist/run/accept.d.ts.map +1 -0
  101. package/dist/run/accept.js +65 -0
  102. package/dist/run/cost.d.ts +13 -0
  103. package/dist/run/cost.d.ts.map +1 -1
  104. package/dist/run/cost.js +27 -0
  105. package/dist/run/dispatch.d.ts +24 -1
  106. package/dist/run/dispatch.d.ts.map +1 -1
  107. package/dist/run/dispatch.js +19 -0
  108. package/dist/run/ladder-key.d.ts +4 -0
  109. package/dist/run/ladder-key.d.ts.map +1 -1
  110. package/dist/run/ladder-key.js +3 -0
  111. package/dist/run/quality-gate.d.ts +39 -0
  112. package/dist/run/quality-gate.d.ts.map +1 -0
  113. package/dist/run/quality-gate.js +70 -0
  114. package/dist/run/raw.d.ts.map +1 -1
  115. package/dist/run/raw.js +22 -3
  116. package/dist/run/record-attempt.d.ts.map +1 -1
  117. package/dist/run/record-attempt.js +22 -1
  118. package/dist/run/routing-docs.d.ts.map +1 -1
  119. package/dist/run/routing-docs.js +11 -4
  120. package/dist/run/run.d.ts.map +1 -1
  121. package/dist/run/run.js +46 -39
  122. package/dist/settings/index.d.ts +1 -1
  123. package/dist/settings/index.d.ts.map +1 -1
  124. package/dist/settings/index.js +2 -1
  125. package/dist/settings/task-settings.d.ts +10 -0
  126. package/dist/settings/task-settings.d.ts.map +1 -1
  127. package/dist/settings/task-settings.js +17 -0
  128. package/dist/telemetry/attempt-meta.d.ts +178 -0
  129. package/dist/telemetry/attempt-meta.d.ts.map +1 -0
  130. package/dist/telemetry/attempt-meta.js +319 -0
  131. package/dist/telemetry/exporter.d.ts.map +1 -1
  132. package/dist/telemetry/exporter.js +6 -0
  133. package/dist/telemetry/ingest-protocol.d.ts +11 -0
  134. package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
  135. package/dist/telemetry/ingest-protocol.js +3 -0
  136. package/dist/telemetry/otel-map.d.ts +10 -0
  137. package/dist/telemetry/otel-map.d.ts.map +1 -1
  138. package/dist/telemetry/otel-map.js +64 -0
  139. package/dist/telemetry/store-migrations.d.ts.map +1 -1
  140. package/dist/telemetry/store-migrations.js +15 -0
  141. package/dist/telemetry/store.d.ts.map +1 -1
  142. package/dist/telemetry/store.js +25 -7
  143. package/dist/types/failures-teaching.d.ts.map +1 -1
  144. package/dist/types/failures-teaching.js +20 -0
  145. package/dist/types/failures.d.ts +1 -1
  146. package/dist/types/failures.d.ts.map +1 -1
  147. package/dist/types/failures.js +1 -0
  148. package/dist/types/provider.d.ts +9 -0
  149. package/dist/types/provider.d.ts.map +1 -1
  150. package/dist/types/quality.d.ts +41 -0
  151. package/dist/types/quality.d.ts.map +1 -0
  152. package/dist/types/quality.js +12 -0
  153. package/dist/types/route.d.ts +89 -0
  154. package/dist/types/route.d.ts.map +1 -1
  155. package/dist/types/telemetry.d.ts +13 -0
  156. package/dist/types/telemetry.d.ts.map +1 -1
  157. package/dist/validate-cli.d.ts.map +1 -1
  158. package/dist/validate-cli.js +10 -0
  159. package/package.json +1 -1
@@ -16,6 +16,7 @@ const config_1 = require("./config");
16
16
  const rawonly_1 = require("./run/rawonly");
17
17
  const loader_1 = require("./routes/loader");
18
18
  const manifest_1 = require("./routes/manifest");
19
+ const attempt_meta_1 = require("./telemetry/attempt-meta");
19
20
  function runValidateCli() {
20
21
  const config = (0, config_1.loadConfigFromEnv)();
21
22
  const routesDir = (0, rawonly_1.requireRoutesDir)(config, "modelrig validate");
@@ -26,6 +27,15 @@ function runValidateCli() {
26
27
  try {
27
28
  const manifest = (0, manifest_1.loadManifest)(rigDir);
28
29
  console.log(manifest === null ? `— rig.yaml (none at ${rigDir})` : "OK rig.yaml");
30
+ // WI-1 (spec §4): nudge a declared dimension that is a near-miss of a
31
+ // reserved dimension toward the well-known key, so cost/coverage group-bys
32
+ // line up across rigs. A WARN, never a failure — the customer's key is valid.
33
+ if (manifest !== null) {
34
+ for (const nm of (0, attempt_meta_1.reservedTagNearMisses)(manifest.dimensions.map((d) => d.key))) {
35
+ console.warn(`WARN rig.yaml dimension "${nm.key}" looks like the reserved dimension "${nm.suggestion}" — ` +
36
+ `rename it to "${nm.suggestion}" to use ModelRig's built-in cost/coverage group-bys.`);
37
+ }
38
+ }
29
39
  }
30
40
  catch (err) {
31
41
  failures += 1;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "modelrig",
3
- "version": "0.5.0",
3
+ "version": "0.7.0",
4
4
  "private": false,
5
5
  "description": "ModelRig — capability-aware AI gateway: routes-as-code, provider adapters, typed failures, budget envelopes, and local-first telemetry.",
6
6
  "license": "Apache-2.0",