modelrig 0.7.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/CHANGELOG.md +118 -0
  2. package/README.md +86 -0
  3. package/dist/bakeoff/disagreement.d.ts +93 -0
  4. package/dist/bakeoff/disagreement.d.ts.map +1 -0
  5. package/dist/bakeoff/disagreement.js +140 -0
  6. package/dist/bakeoff/engine.d.ts.map +1 -1
  7. package/dist/bakeoff/engine.js +58 -2
  8. package/dist/bakeoff/pairwise.d.ts +102 -0
  9. package/dist/bakeoff/pairwise.d.ts.map +1 -0
  10. package/dist/bakeoff/pairwise.js +214 -0
  11. package/dist/bakeoff/replay.d.ts +6 -0
  12. package/dist/bakeoff/replay.d.ts.map +1 -1
  13. package/dist/bakeoff/replay.js +6 -2
  14. package/dist/bakeoff/report.d.ts +7 -0
  15. package/dist/bakeoff/report.d.ts.map +1 -1
  16. package/dist/bakeoff/report.js +46 -0
  17. package/dist/bakeoff/types.d.ts +24 -0
  18. package/dist/bakeoff/types.d.ts.map +1 -1
  19. package/dist/cli.d.ts.map +1 -1
  20. package/dist/cli.js +9 -9
  21. package/dist/mask/apply.d.ts +80 -0
  22. package/dist/mask/apply.d.ts.map +1 -0
  23. package/dist/mask/apply.js +228 -0
  24. package/dist/mask/detect.d.ts +38 -0
  25. package/dist/mask/detect.d.ts.map +1 -0
  26. package/dist/mask/detect.js +167 -0
  27. package/dist/mask/index.d.ts +21 -0
  28. package/dist/mask/index.d.ts.map +1 -0
  29. package/dist/mask/index.js +54 -0
  30. package/dist/mask/mask-policy-cli.d.ts +11 -0
  31. package/dist/mask/mask-policy-cli.d.ts.map +1 -0
  32. package/dist/mask/mask-policy-cli.js +33 -0
  33. package/dist/mask/policy.d.ts +38 -0
  34. package/dist/mask/policy.d.ts.map +1 -0
  35. package/dist/mask/policy.js +187 -0
  36. package/dist/mask/proof.d.ts +54 -0
  37. package/dist/mask/proof.d.ts.map +1 -0
  38. package/dist/mask/proof.js +120 -0
  39. package/dist/mask/receipt.d.ts +30 -0
  40. package/dist/mask/receipt.d.ts.map +1 -0
  41. package/dist/mask/receipt.js +49 -0
  42. package/dist/mask/seam.d.ts +53 -0
  43. package/dist/mask/seam.d.ts.map +1 -0
  44. package/dist/mask/seam.js +120 -0
  45. package/dist/mask/types.d.ts +81 -0
  46. package/dist/mask/types.d.ts.map +1 -0
  47. package/dist/mask/types.js +10 -0
  48. package/dist/mask/vault.d.ts +50 -0
  49. package/dist/mask/vault.d.ts.map +1 -0
  50. package/dist/mask/vault.js +81 -0
  51. package/dist/receipts/types.d.ts +65 -0
  52. package/dist/receipts/types.d.ts.map +1 -0
  53. package/dist/receipts/types.js +134 -0
  54. package/dist/routes/loader-warnings.d.ts +8 -0
  55. package/dist/routes/loader-warnings.d.ts.map +1 -1
  56. package/dist/routes/loader-warnings.js +23 -0
  57. package/dist/routes/loader.d.ts +1 -1
  58. package/dist/routes/loader.d.ts.map +1 -1
  59. package/dist/routes/loader.js +24 -25
  60. package/dist/routes/mask-loader.d.ts +18 -0
  61. package/dist/routes/mask-loader.d.ts.map +1 -0
  62. package/dist/routes/mask-loader.js +39 -0
  63. package/dist/routes/objective-loader.d.ts.map +1 -1
  64. package/dist/routes/objective-loader.js +51 -17
  65. package/dist/routes/screen-loader.d.ts +23 -0
  66. package/dist/routes/screen-loader.d.ts.map +1 -0
  67. package/dist/routes/screen-loader.js +79 -0
  68. package/dist/routes/template.d.ts +10 -0
  69. package/dist/routes/template.d.ts.map +1 -1
  70. package/dist/routes/template.js +10 -3
  71. package/dist/run/accept.d.ts +4 -0
  72. package/dist/run/accept.d.ts.map +1 -1
  73. package/dist/run/accept.js +2 -0
  74. package/dist/run/dispatch.d.ts +35 -0
  75. package/dist/run/dispatch.d.ts.map +1 -1
  76. package/dist/run/dispatch.js +33 -0
  77. package/dist/run/grounding.d.ts +47 -2
  78. package/dist/run/grounding.d.ts.map +1 -1
  79. package/dist/run/grounding.js +113 -6
  80. package/dist/run/raw.d.ts.map +1 -1
  81. package/dist/run/raw.js +6 -1
  82. package/dist/run/record-attempt.d.ts.map +1 -1
  83. package/dist/run/record-attempt.js +12 -1
  84. package/dist/run/run.d.ts.map +1 -1
  85. package/dist/run/run.js +19 -16
  86. package/dist/screen/provenance.d.ts +59 -0
  87. package/dist/screen/provenance.d.ts.map +1 -0
  88. package/dist/screen/provenance.js +104 -0
  89. package/dist/screen/types.d.ts +71 -0
  90. package/dist/screen/types.d.ts.map +1 -0
  91. package/dist/screen/types.js +9 -0
  92. package/dist/status-cli.d.ts +11 -0
  93. package/dist/status-cli.d.ts.map +1 -0
  94. package/dist/status-cli.js +21 -0
  95. package/dist/status-runs.d.ts +38 -0
  96. package/dist/status-runs.d.ts.map +1 -0
  97. package/dist/status-runs.js +80 -0
  98. package/dist/status.d.ts +7 -2
  99. package/dist/status.d.ts.map +1 -1
  100. package/dist/status.js +11 -2
  101. package/dist/telemetry/ingest-protocol.d.ts +8 -0
  102. package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
  103. package/dist/telemetry/store-migrations.d.ts.map +1 -1
  104. package/dist/telemetry/store-migrations.js +11 -0
  105. package/dist/telemetry/store.d.ts.map +1 -1
  106. package/dist/telemetry/store.js +12 -3
  107. package/dist/types/provider.d.ts +11 -0
  108. package/dist/types/provider.d.ts.map +1 -1
  109. package/dist/types/provider.js +9 -0
  110. package/dist/types/route.d.ts +55 -0
  111. package/dist/types/route.d.ts.map +1 -1
  112. package/dist/types/telemetry.d.ts +7 -0
  113. package/dist/types/telemetry.d.ts.map +1 -1
  114. package/dist/validate-cli.d.ts.map +1 -1
  115. package/dist/validate-cli.js +13 -0
  116. package/dist/verify/types.d.ts +6 -0
  117. package/dist/verify/types.d.ts.map +1 -1
  118. package/docs/agent-skill/SKILL.md +131 -0
  119. package/docs/how-it-fits.md +34 -0
  120. package/docs/instrumentation.md +221 -0
  121. package/docs/llms.txt +276 -0
  122. package/docs/migration-playbook.md +512 -0
  123. package/docs/quickstart.md +339 -0
  124. package/package.json +18 -2
  125. package/receipts.d.ts +12 -0
@@ -1 +1 @@
1
- {"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../src/types/telemetry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAE1E;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,WAAW,GACX,kBAAkB,CAAC;AAEvB,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;;wCAEoC;IACpC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,sDAAsD;IACtD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;;;kFAM8E;IAC9E,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;wFACoF;IACpF,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;uDACmD;IACnD,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;wEACoE;IACpE,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;;;oDAGgD;IAChD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;oFACgF;IAChF,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C;;6DAEyD;IACzD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC;oDACgD;IAChD,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;uDACmD;IACnD,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;;;;;;+EAM2E;IAC3E,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC;mEAC+D;IAC/D,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC;;;sDAGkD;IAClD,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;;;;;uDAKmD;IACnD,QAAQ,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC;CACvC;AAED;;6EAE6E;AAC7E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,IAAI,GAAG,MAAM,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;OAcG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B;;gDAE4C;IAC5C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED;;;;cAIc;AACd,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;IAClE;;;6EAGyE;IACzE,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;CAClC;AAED;;;gDAGgD;AAChD,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,cAAc,GAAG,UAAU,CAAC;IACjD,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED;;mFAEmF;AACnF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;;;kEAG8D;IAC9D,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,sDAAsD;AACtD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,cAAc,CAAC;CAC5C;AAED,4DAA4D;AAC5D,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,gFAAgF;IAChF,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;mDAC+C;IAC/C,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,aAAa,EAAE,eAAe,GAAG,aAAa,GAAG,OAAO,CAAC;IAClE,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED,6EAA6E;AAC7E,MAAM,MAAM,cAAc,GAAG,kBAAkB,GAAG,iBAAiB,GAAG,QAAQ,CAAC;AAE/E;;+EAE+E;AAC/E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,GAAG,EAAE,YAAY,GAAG,IAAI,CAAC;IAChC,cAAc,CAAC,GAAG,EAAE,WAAW,GAAG,IAAI,CAAC;IACvC,UAAU,CAAC,GAAG,EAAE,MAAM,OAAO,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAAC;IACvE,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAAC;IAC7C,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,WAAW,CAAC;IAC9C,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,WAAW,CAAC;IAC1D,KAAK,IAAI,IAAI,CAAC;CACf"}
1
+ {"version":3,"file":"telemetry.d.ts","sourceRoot":"","sources":["../../src/types/telemetry.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC1E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,WAAW,GACX,kBAAkB,CAAC;AAEvB,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;;wCAEoC;IACpC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,sDAAsD;IACtD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;;;kFAM8E;IAC9E,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;wFACoF;IACpF,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;uDACmD;IACnD,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;wEACoE;IACpE,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;;;oDAGgD;IAChD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;oFACgF;IAChF,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C;;6DAEyD;IACzD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC;oDACgD;IAChD,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;uDACmD;IACnD,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;;;;;;+EAM2E;IAC3E,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC;mEAC+D;IAC/D,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC;;;sDAGkD;IAClD,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC;;;;;uDAKmD;IACnD,QAAQ,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IACrC;;;;uEAImE;IACnE,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;CACpC;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC;CACvC;AAED;;6EAE6E;AAC7E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,IAAI,GAAG,MAAM,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;;;;;;;;;;;;;;OAcG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B;;gDAE4C;IAC5C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED;;;;cAIc;AACd,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;IAClE;;;6EAGyE;IACzE,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;CAClC;AAED;;;gDAGgD;AAChD,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,cAAc,GAAG,UAAU,CAAC;IACjD,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED;;mFAEmF;AACnF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7C,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC;;;kEAG8D;IAC9D,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,sDAAsD;AACtD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,cAAc,CAAC;CAC5C;AAED,4DAA4D;AAC5D,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,gFAAgF;IAChF,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;mDAC+C;IAC/C,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,aAAa,EAAE,eAAe,GAAG,aAAa,GAAG,OAAO,CAAC;IAClE,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED,6EAA6E;AAC7E,MAAM,MAAM,cAAc,GAAG,kBAAkB,GAAG,iBAAiB,GAAG,QAAQ,CAAC;AAE/E;;+EAE+E;AAC/E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,GAAG,EAAE,YAAY,GAAG,IAAI,CAAC;IAChC,cAAc,CAAC,GAAG,EAAE,WAAW,GAAG,IAAI,CAAC;IACvC,UAAU,CAAC,GAAG,EAAE,MAAM,OAAO,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAAC;IACvE,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAAC;IAC7C,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,WAAW,CAAC;IAC9C,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,WAAW,CAAC;IAC1D,KAAK,IAAI,IAAI,CAAC;CACf"}
@@ -1 +1 @@
1
- {"version":3,"file":"validate-cli.d.ts","sourceRoot":"","sources":["../src/validate-cli.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AASH,wBAAgB,cAAc,IAAI,MAAM,CAyDvC"}
1
+ {"version":3,"file":"validate-cli.d.ts","sourceRoot":"","sources":["../src/validate-cli.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAUH,wBAAgB,cAAc,IAAI,MAAM,CAwEvC"}
@@ -17,6 +17,7 @@ const rawonly_1 = require("./run/rawonly");
17
17
  const loader_1 = require("./routes/loader");
18
18
  const manifest_1 = require("./routes/manifest");
19
19
  const attempt_meta_1 = require("./telemetry/attempt-meta");
20
+ const status_runs_1 = require("./status-runs");
20
21
  function runValidateCli() {
21
22
  const config = (0, config_1.loadConfigFromEnv)();
22
23
  const routesDir = (0, rawonly_1.requireRoutesDir)(config, "modelrig validate");
@@ -66,6 +67,18 @@ function runValidateCli() {
66
67
  console.error(`FAIL ${r.route ?? r.file}: ${r.problems.join("; ")}`);
67
68
  }
68
69
  }
70
+ // GS-1 teaching hint: a rig that is USED (has recorded inferences) but never
71
+ // wrapped an execution in run.start / run.scope records no runs, so /runs stays
72
+ // empty and /projects shows "(no run)". A WARN, never a failure — the config is
73
+ // valid; runs are the standard path (get-started v5). Best-effort + silent when
74
+ // there is no local buffer to read yet.
75
+ const adoption = (0, status_runs_1.readRunAdoption)(config.telemetryDbPath);
76
+ if (adoption !== null && adoption.inferences > 0 && adoption.runs === 0) {
77
+ console.warn(`WARN this rig has recorded ${adoption.inferences} call${adoption.inferences === 1 ? "" : "s"} ` +
78
+ "but has never started a run — wrap each pipeline execution in run.start()/run.scope() " +
79
+ "(one step() per model-call family, artifact.save for work products) so it appears on /runs. " +
80
+ "Runs are the standard path: https://modelrig.dev/instrumentation.html");
81
+ }
69
82
  if (failures === 0) {
70
83
  console.log(`\n${results.length} route(s) valid — no provider key required for this check.`);
71
84
  return 0;
@@ -75,6 +75,12 @@ export interface SampleReport extends SampleDiff {
75
75
  export interface EquivalenceReport {
76
76
  readonly schemaVersion: 1;
77
77
  readonly kind: "shadow-equivalence";
78
+ /** What change this equivalence run proves, when it is not a plain model swap
79
+ * (the default, absent). `screen-assembly` marks a SC-0 B3 run proving an
80
+ * assembly change — grounding `legacy`→`spotlight`, or an `x-trust: untrusted`
81
+ * variable flip — is output-equivalent to the prior assembly on the pinned
82
+ * model (spec §5), so the flip ships through the bake-off like a swap. Additive. */
83
+ readonly subject?: "swap" | "mask-policy" | "screen-assembly";
78
84
  readonly callSiteId: string;
79
85
  readonly route: string;
80
86
  readonly routeVersion: number;
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/verify/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpD;;6BAE6B;AAC7B,MAAM,WAAW,aAAa;IAC5B;8EAC0E;IAC1E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;gFACgF;AAChF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,gBAAgB,GAAG,cAAc,CAAC;IACjE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;oEACoE;AACpE,MAAM,WAAW,oBAAoB;IACnC,0EAA0E;IAC1E,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;mEAC+D;IAC/D,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,SAAS,aAAa,EAAE,CAAC;IAC3C,4EAA4E;IAC5E,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC9C;;;sDAGkD;IAClD,QAAQ,CAAC,qBAAqB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACpD;AAED,4DAA4D;AAC5D,MAAM,MAAM,aAAa,GACrB,YAAY,GACZ,kCAAkC,GAClC,WAAW,CAAC;AAEhB;kDACkD;AAClD,MAAM,MAAM,kBAAkB,GAAG,aAAa,GAAG,SAAS,CAAC;AAE3D,6DAA6D;AAC7D,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;IAChC;;wEAEoE;IACpE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAa,SAAQ,UAAU;IAC9C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;gDAC4C;IAC5C,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;4EACwE;IACxE,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,oBAAoB,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;+CAC2C;IAC3C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE;QACf,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;QAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,QAAQ,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,CAAC;IAC1C,wEAAwE;IACxE,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C,gEAAgE;IAChE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;sEACkE;IAClE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/verify/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpD;;6BAE6B;AAC7B,MAAM,WAAW,aAAa;IAC5B;8EAC0E;IAC1E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACxC,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;gFACgF;AAChF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,gBAAgB,GAAG,cAAc,CAAC;IACjE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;oEACoE;AACpE,MAAM,WAAW,oBAAoB;IACnC,0EAA0E;IAC1E,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;mEAC+D;IAC/D,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,SAAS,aAAa,EAAE,CAAC;IAC3C,4EAA4E;IAC5E,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC9C;;;sDAGkD;IAClD,QAAQ,CAAC,qBAAqB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACpD;AAED,4DAA4D;AAC5D,MAAM,MAAM,aAAa,GACrB,YAAY,GACZ,kCAAkC,GAClC,WAAW,CAAC;AAEhB;kDACkD;AAClD,MAAM,MAAM,kBAAkB,GAAG,aAAa,GAAG,SAAS,CAAC;AAE3D,6DAA6D;AAC7D,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;IAChC;;wEAEoE;IACpE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAa,SAAQ,UAAU;IAC9C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;gDAC4C;IAC5C,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;4EACwE;IACxE,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,oBAAoB,CAAC;IACpC;;;;wFAIoF;IACpF,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,aAAa,GAAG,iBAAiB,CAAC;IAC9D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;+CAC2C;IAC3C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE;QACf,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;QAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,QAAQ,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,CAAC;IAC1C,wEAAwE;IACxE,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C,gEAAgE;IAChE,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;sEACkE;IAClE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B"}
@@ -0,0 +1,131 @@
1
+ ---
2
+ name: modelrig
3
+ description: Use when working with LLM/model selection, cost, or reliability in a codebase — choosing a model, migrating direct provider SDK calls to routes, explaining what a model call actually costs, or debugging schema/grounding/caching failures. Answers from PROBED measurements rather than provider claims.
4
+ ---
5
+
6
+ # ModelRig
7
+
8
+ **A pointer skill.** The canonical playbook lives at
9
+ <https://modelrig.dev/llms.txt> and is regenerated whenever the registry
10
+ is rebuilt. Fetch it rather than trusting anything cached here — model
11
+ capabilities change, and a stale answer about a model is worse than no answer.
12
+
13
+ One copy, every ecosystem: this file is the same in Claude Code, Cursor,
14
+ Codex, and anywhere else skills load. It deliberately holds almost no facts.
15
+
16
+ ## Start here
17
+
18
+ 1. **Fetch <https://modelrig.dev/llms.txt>.** It carries the current probed
19
+ facts, the model list with `as_of` stamps, and the tool advertisements.
20
+ 2. **Prefer the MCP tools over parsing text.** If the ModelRig oracle is
21
+ registered, call it — the answers are typed, dated, and scoped:
22
+
23
+ ```
24
+ query_registry which models satisfy a capability + price bound
25
+ get_leaderboard standings by effective cost of conformance
26
+ explain_pricing list price vs what conformance actually costs
27
+ get_call_notes per-model quirks that make requests fail
28
+ ```
29
+
30
+ Not registered? Add it — it needs no account for those four:
31
+
32
+ ```json
33
+ { "mcpServers": { "modelrig": { "command": "npx", "args": ["modelrig-oracle"] } } }
34
+ ```
35
+
36
+ 3. **Fall back to the raw registry** when neither is available:
37
+ <https://raw.githubusercontent.com/modelrig/modelrig/main/registry/registry.json>
38
+
39
+ ## Rules that matter more than speed
40
+
41
+ - **Never state a model's capabilities from memory.** Look them up. Every
42
+ fact in the registry carries an `as_of` date and a sample count; quote both.
43
+ - **Absence is not a limit.** A model missing from an answer is one nobody has
44
+ probed — say that, rather than implying it can't do the thing.
45
+ - **Probed beats declared.** Where a provider's claim and our measurement
46
+ disagree, the entry's `discrepancies` field says so. Surface the
47
+ disagreement; it is usually the useful part.
48
+ - **Compare on effective cost of conformance,** not list price. A model at half
49
+ the price that conforms two-thirds as often is not cheaper.
50
+
51
+ ## Just want to see your runs first? (observe, no migration)
52
+
53
+ Before migrating anything, you can just **watch** an existing pipeline. ModelRig
54
+ records each run as a run → step → artifact chain you click through in the
55
+ console — no routes moved, no model calls rerouted, and with the flag off the
56
+ pipeline behaves byte-for-byte as before. Scaffold it:
57
+ `npx modelrig observe [path] --scope <one pipeline>` writes a self-contained
58
+ `record-step` seam and an `OBSERVE.md` whose wiring is emitted as diffs you
59
+ apply. The instrumentation is a code-level SDK seam (no zero-code observe), but
60
+ you no longer hand-write it — see <https://modelrig.dev/instrumentation.html>.
61
+ The namespace is **on by default once a control plane is configured** (a
62
+ `rig_sk_` API key, or the self-host Supabase pair); a pure local-only rig stays
63
+ off — force it on there with `MODELRIG_ARTIFACTS=1`, and `MODELRIG_ARTIFACTS=0`
64
+ opts out anywhere. Observing the runs in the console needs a free
65
+ account + key from <https://app.modelrig.ai> — the one step a human has to do
66
+ (console login to mint the key), so plan the handoff rather than stalling on it.
67
+
68
+ ## Migrating a repo — the T0–T2 playbook
69
+
70
+ When you import this skill to migrate a codebase, **fetch the canonical
71
+ playbook** and follow it: <https://modelrig.dev/migration-playbook.html>. It is
72
+ a four-tier autonomy ladder — T0 gateway-pointing and T1 instrumentation are
73
+ behaviour-preserving (apply after a plan review); T2 route extraction replaces a
74
+ direct provider call with `rig.run()` **always same-model**, and is
75
+ behaviour-affecting, so the agent *proposes* it, proves it with
76
+ `modelrig verify-swap` (shadow-equivalence), embeds the report in the PR, does
77
+ **one call site per PR**, and a **human merges**. Model optimization (T3) is
78
+ never the skill's job.
79
+
80
+ **Runs are the standard path, not an appendix.** Every pipeline execution should
81
+ be a **run**: wrap it in `rig.artifacts.run.start` / `run.scope`, name one step
82
+ per model-call family (pass `stepKey`), `artifact.save` the work products, and tag
83
+ `subject` / `feature`. Inside a run context the SDK auto-stamps `tags.run_id` for
84
+ you, so the run fills `/runs` **and** `/projects`; acceptance is the run appearing
85
+ on `/runs` with its steps. (Inside a run context the SDK also auto-stamps
86
+ `tags.step`; a value you pass always wins, so name the step with `stepKey` or
87
+ `tags.step` when you want a specific label.) This is Tier 1 and it is **not
88
+ optional for a raw-lane (Lane-B) seam** — a pipeline that reaches the model
89
+ through `rig.runRaw` without a `run.start` around it records inferences but no
90
+ run. The full API is the reference at
91
+ <https://modelrig.dev/instrumentation.html>.
92
+
93
+ The machinery underneath: `npx modelrig init [path] [--scope <glob>]` scans for
94
+ provider call sites and drafts a route bundle for each, plus a `MIGRATION.md`
95
+ mapping call site → route → the replacement `rig.run` line. It writes only under
96
+ `modelrig/routes/drafts/` and **never edits source** — applying the diff is the
97
+ human's decision, one call site at a time. The replaced call is **deleted** (not
98
+ commented out) with a one-line breadcrumb; rollback is `git revert`.
99
+
100
+ ## Capabilities beyond migration (the full surface)
101
+
102
+ Shipped features a newbie migration won't stumble onto — one line each, with the
103
+ reference doc. The canonical detail is always <https://modelrig.dev/llms.txt>.
104
+
105
+ - **Runs & projects** — the run → step → artifact graph at `/runs`, grouped by tag
106
+ at `/projects`: <https://modelrig.dev/instrumentation.html>.
107
+ - **Tenants & statements** — a `subject` tag or project-scoped key becomes a
108
+ per-customer tenant with a derived statement: <https://modelrig.dev/tenants.html>.
109
+ - **Published receipts** — a signed, login-free receipt of a bake-off or route at
110
+ `/r/<id>`, verifiable at `/v1/verify/<id>`, content never leaves:
111
+ <https://modelrig.dev/receipts.html>.
112
+ - **Analytics query API** — self-describing, read-only, metadata-only SELECT over
113
+ your own telemetry (`GET /v1/analytics/meta`, `POST /v1/analytics/query`):
114
+ <https://modelrig.dev/optimization-loop.html>.
115
+ - **Provenance & trust** — provenance-first prompt assembly plus per-variable
116
+ `x-trust`: <https://modelrig.dev/screening.html>.
117
+ - **Bake-offs — pairwise, disagreement & regret** — replay your own traffic, judge
118
+ pairwise with disagreement shown, propose the lowest-regret arm under a quality
119
+ gate: <https://modelrig.dev/bakeoffs.html> and <https://modelrig.dev/run-verdicts.html>.
120
+
121
+ Content custody stays early access (metadata and hashes); no capability here is a
122
+ claim beyond what ships.
123
+
124
+ ## What ModelRig is, in three sentences
125
+
126
+ A route is a task your code calls by name; the route's YAML declares which
127
+ models are allowed to serve it, the schema the output must match, and the
128
+ retry and budget policy. Your code calls the task, not the model, so swapping
129
+ models is a config change with evidence behind it rather than a code change
130
+ with hope behind it. Nothing switches what serves a route without a bake-off
131
+ on your own traffic and an explicit human approval.
@@ -0,0 +1,34 @@
1
+ # How it fits
2
+
3
+ One content source (this file) renders in the console `/setup` page and on modelrig.dev — edit here, surfaces follow.
4
+
5
+ ModelRig sits between your code and the model providers. Your repo declares **routes** — small YAML files naming a prompt, an output schema, and the models allowed to serve the task. Your app calls `rig.run("task.name")`; ModelRig renders the prompt, picks a model from the route's candidates using verified registry facts, enforces the schema, retries typed failures within per-class budgets, and hard-stops spending at your envelope caps. How the candidate ladder actually runs — ordering, validate → repair → fall-through, and what is not built — is on [Routing & reliability](routing-reliability.md).
6
+
7
+ Every call writes one telemetry row to a local SQLite file first — the run never blocks on the cloud. An async exporter mirrors rows to the console you are reading. Provider keys are the one thing that deliberately never leaves your machine; the exporter has no code path to them.
8
+
9
+ Routing is half of it. ModelRig also records the **execution graph** of your workstream — the run, its steps, and the artifacts each step produced — versioned, hashed, and lineaged, into an inspectable store you can open: when someone asks why the pipeline did that, you pull up the run and show them. This is the **standard path**, not an add-on: every pipeline execution is a run — wrap it in `run.start` / `run.scope`, name a step per model-call family (name it with `stepKey`), save the work products, and inside that context the SDK auto-stamps `run_id` for you, so the run appears on `/runs` and `/projects` with no hand-written id (the SDK auto-stamps the `step` key inside a run context too; the `stepKey` you pass names the step, and always wins). The namespace is **on by default once a control plane is configured** (a ModelRig API key, or the self-host Supabase pair); a pure local-only rig stays off, and `MODELRIG_ARTIFACTS=0` opts out anywhere. Artifact *metadata and lineage* ride the same exporter; artifact *content* moves only with per-route consent, off by default. A failing artifact can be promoted to a regression case and replayed as a bake-off — see → prove → save is the loop, and the store is where seeing happens.
10
+
11
+ What is retained beyond that is your choice, per route, and there are two lanes.
12
+
13
+ **Pure router** (capture off). Nothing retained. Real, supported, selectable, and the honest answer for a ZDR-strict or data-residency-bound workload.
14
+
15
+ **Optimization on** (recommended). Your traffic is what makes the routing smarter and the bill smaller. **What is kept:** per-inference telemetry rows, and bounded review samples for the bake-offs you create. **Where:** your organization, row-scoped by the database, not by our application code. **How you leave:** see it all in the console, delete it, export it, take your routes and go — they were always YAML in your git. **What you get back:** community priors on your very first bake-off and RigIndex-grade evidence on your own routes — at the same flat 2% either way; the corpus earns its place by being useful, not by being a discount.
16
+
17
+ Captures power **bake-offs**: replay your own recent traffic through route variants — a cheaper model, a different prompt — and get the effective cost of conformance for each, with confidence intervals. A winning variant is a proposal for you to apply in your YAML; nothing swaps models automatically.
18
+
19
+ The **registry** feeds routing from the side: declared claims, probed measurements, and observed traffic per model, with disagreements shown. A probed fact beats a declared claim — a model that declares schema support but failed our probes is not eligible to serve a schema route natively.
20
+
21
+ The loop, end to end: routes in your repo → `rig.run` in your app → local telemetry (+ captures on the routes where you turn Optimization on) → synced console → bake-offs prove cheaper variants → you edit the YAML. The console shows the truth; actions live in your code.
22
+
23
+ ## More capabilities
24
+
25
+ The five-step get-started is the front door; these are the shipped surfaces it does not stop to teach. One line each, with the reference doc:
26
+
27
+ - **Runs & projects** — the run → step → artifact graph on `/runs`, grouped by tag on `/projects`. The standard path above; full API in [instrumentation](instrumentation.md).
28
+ - **Tenants & statements** — a `subject` tag or a project-scoped key becomes a per-customer tenant with a derived statement (attempted, verified, billable, true cost), on `/tenants`. See [Tenants & statements](tenants.md).
29
+ - **Published receipts** — publish a signed, login-free receipt of a bake-off or route at `/r/<id>` — counts, scores and hashes only, verifiable at `/v1/verify/<id>`; names aliased, content never leaves. See [Published receipts](receipts.md).
30
+ - **Analytics query API** — a self-describing, read-only query surface over your own telemetry (`GET /v1/analytics/meta`, `POST /v1/analytics/query`): an allowlisted SELECT grammar that returns metadata only, never content. See [Optimization loop](optimization-loop.md).
31
+ - **Provenance & trust** — provenance-first prompt assembly keeps third-party content out of the most-trusted slot; annotate a variable `x-trust: untrusted` to spotlight it. See [Provenance & trust](screening.md).
32
+ - **Bake-offs — pairwise, disagreement & regret** — replay your own traffic, judge candidates pairwise with disagreement shown, and propose the lowest-**regret** arm under a quality gate (near-optimal, not merely cheapest). See [Bake-offs & replay](bakeoffs.md) and [Run verdicts](run-verdicts.md).
33
+
34
+ New claims are held to the same bar as the five steps: content custody stays early access (metadata and hashes), and pricing stays the launch-terms sentence.
@@ -0,0 +1,221 @@
1
+ # Artifact instrumentation (runs, steps, artifacts)
2
+
3
+ ModelRig can record the **execution graph** of any pipeline your code runs — the
4
+ run, its steps, and the artifacts each step produced (the prompt, the raw model
5
+ text, the parsed output, the evidence it grounded on) — with lineage,
6
+ evaluations, and an integrity hash for each. The console then renders that run as
7
+ a semantic chain you click through: *what your pipeline actually did*.
8
+
9
+ It is **on by default when a control plane is configured** (an api-mode
10
+ `rig_sk_` key, or the direct-mode Supabase pair) — metadata + hashes only. A
11
+ **pure local-only rig** (no sink to ship to) stays off. Either way it is
12
+ additive; set `MODELRIG_ARTIFACTS=0` to opt out, and with the namespace off it
13
+ is inert: `run.start` is a no-op, `artifact.save` returns null, no rows are
14
+ written, and your pipeline behaves byte-for-byte as before. `modelrig status`
15
+ prints the current posture (on/off + why).
16
+
17
+ > **Metadata + hashes only, this release.** The serialized value you save is
18
+ > hashed (sha256) and the bytes are **discarded** — nothing but the metadata row
19
+ > is persisted. Content custody (the blob path) ships in a later release; the
20
+ > hash proves integrity today. A `zeroRetention` run refuses every artifact
21
+ > fail-closed at the SDK gate.
22
+
23
+ ## The namespace toggle
24
+
25
+ ```bash
26
+ # On by default once a control plane is configured — nothing to set.
27
+ export MODELRIG_ARTIFACTS=0 # opt OUT (or force ON with =1 on a
28
+ # local-only rig)
29
+ export MODELRIG_ARTIFACT_STEPS=step_a,step_b # optional: instrument a subset
30
+ # (unset/empty = every instrumented step)
31
+ ```
32
+
33
+ Rows land in the local telemetry buffer and export to the control plane through
34
+ the standard sink (`MODELRIG_API_KEY` + `MODELRIG_INGEST_URL`, or the direct-mode
35
+ Supabase pair). With no sink configured they stay local.
36
+
37
+ An agent can do everything here except mint that `MODELRIG_API_KEY` — creating
38
+ it requires a human console login at <https://app.modelrig.ai>, so an
39
+ agent-driven setup should plan that one handoff rather than stall on it.
40
+
41
+ ## Instrument a pipeline
42
+
43
+ The namespace hangs off `rig.artifacts`. Two moving parts: a **run context**
44
+ around the whole pipeline, and a **per-step save** of the work products.
45
+
46
+ ```ts
47
+ import { createRig, loadConfigFromEnv } from "modelrig";
48
+
49
+ const rig = createRig(loadConfigFromEnv());
50
+
51
+ // 1. Open a run context around the pipeline. episodeKey bridges feedback;
52
+ // it is a grouping key, not a unique id — a natural choice is your own
53
+ // per-run attempt id.
54
+ const run = rig.artifacts.run.start({
55
+ pipeline: "my-pipeline",
56
+ episodeKey: runAttemptId,
57
+ environment: process.env.NODE_ENV,
58
+ });
59
+ try {
60
+ for (const step of steps) {
61
+ const prompt = assemble(step);
62
+ const result = await callModel(prompt); // your own model call
63
+
64
+ // 2. Save the step's artifacts. stepKey groups them in the console's
65
+ // chain view; task carries the semantic task name.
66
+ const p = rig.artifacts.artifact.save(prompt, { name: `${step}.prompt`, type: "prompt", stepKey: step });
67
+ const raw = rig.artifacts.artifact.save(result.rawText, { name: `${step}.raw`, type: "raw_response", stepKey: step });
68
+ const parsed = rig.artifacts.artifact.save(result.json, {
69
+ name: `${step}.output`, type: "step_output", task: step, stepKey: step, schema: step.schema,
70
+ });
71
+
72
+ // 3. Lineage: the parsed output derives from the prompt + raw; this step's
73
+ // prompt consumed the previous step's output.
74
+ if (p && parsed) rig.artifacts.artifact.link(parsed, p, "derived_from");
75
+ if (raw && parsed) rig.artifacts.artifact.link(parsed, raw, "derived_from");
76
+
77
+ // 4. Evaluations (optional, deterministic): flag conformance, degraded
78
+ // provenance, anything you can decide without an LLM judge.
79
+ if (parsed) rig.artifacts.artifact.evaluate(parsed, {
80
+ evaluator: "schema-check", kind: "deterministic", passed: conforms(result.json),
81
+ });
82
+ }
83
+ run.end("succeeded");
84
+ } catch (err) {
85
+ run.end("failed");
86
+ throw err;
87
+ }
88
+ ```
89
+
90
+ For **concurrent** runs in one process, use `rig.artifacts.run.scope(opts, fn)`
91
+ instead of `start`/`end` — each scope keeps its own ambient context so
92
+ overlapping runs never cross-attribute. `run.start` uses `enterWith`, which is
93
+ right for one run at a time on a request/async path.
94
+
95
+ ## Steps vs. artifacts
96
+
97
+ - An **artifact** is a work product — a durable thing your pipeline made. It
98
+ always saves (metadata + hash), whether or not the model call went through the
99
+ rig.
100
+ - A **step** is an execution row. `rig.run()` auto-creates a **ground-truth**
101
+ step (its inference is real, so cost and latency are measured) when it runs
102
+ inside a run context. A stage whose model call does *not* go through the rig
103
+ produces artifacts with `step_key` set and **no step row** — which is correct
104
+ and honest. Never route a stage through the rig just to mint a step row; the
105
+ console renders "cost n/a" for un-routed steps rather than inventing a number.
106
+
107
+ ## Tags, dimensions, and the graph
108
+
109
+ ModelRig has **two** grouping mechanisms, and they compose:
110
+
111
+ - **Tags** are a flat filter/rollup. Every `rig.run` / `rig.runRaw` takes a `tags`
112
+ object; those key/values ride the telemetry row and drive `/costs` and the route
113
+ dashboard (group cost by `project`, by `environment`, by whatever you tag).
114
+ - **The graph** is the per-run tree at `/runs/[id]` — the ground-truth steps a run
115
+ recorded, in order, with their inferences, costs, and the artifacts attached to
116
+ each step.
117
+
118
+ **Declare your dimensions** in `modelrig/rig.yaml` so the cost views know which
119
+ tags matter:
120
+
121
+ ```yaml
122
+ name: my-rig
123
+ dimensions:
124
+ - { key: project, label: Project, required: true }
125
+ - { key: environment, label: Environment, required: false }
126
+ ```
127
+
128
+ A `required` dimension that is missing from a run's `tags` is **warned once per
129
+ process** (`required dimension "project" missing from run tags — costs … will be
130
+ under-attributed on /costs`) — on the routed lane and the raw lane alike. It is a
131
+ hygiene warning, never an error: the run proceeds.
132
+
133
+ **Conventional tag keys** (documented, not enforced): `subject` (what a run is
134
+ *about* — a ticker, a client alias), `feature`, `project` (flat grouping),
135
+ `step` (names the step this call belongs to), and `run_id` (bridges a call to a
136
+ run's `episodeKey`). **Inside a `run.start` / `run.scope` context the SDK stamps
137
+ both `run_id` and `step` for you** — `run_id` = the run's
138
+ `episodeKey`, or its id when you gave none; `step` = the current step key (your
139
+ explicit `stepKey`, else `tags.step`, else the auto `route#n` / `provider/model#n`
140
+ key) — on every attempt row of both lanes, unless you passed your own value (yours
141
+ always wins). Outside a run context nothing is stamped, so the row is
142
+ byte-identical. This is why runs are the standard path: a `run.start` alone fills
143
+ both `/runs` (the graph) and `/projects` (the tag tree), with no hand-written id.
144
+ ModelRig owns `lane`, `zdr_enforced`, `rung`, and — on the hosted gateway —
145
+ `key_ownership`; you cannot override those.
146
+
147
+ **How they compose.** Inside a `run.start` context, both `rig.run` and
148
+ `rig.runRaw` record a **ground-truth step** and carry the run's `run_id` tag, so
149
+ the same calls appear on `/runs/[id]` (the graph) **and** under that run on
150
+ `/projects` (the tag tree). Name it with `stepKey` (either lane)
151
+ — explicit `stepKey` wins, else `tags.step`, else an auto key (`route#n` /
152
+ `provider/model#n`). An `artifact.save` **without** a `stepKey` lands in the
153
+ **ambient step** (the most recent step in the context), so a raw call and the
154
+ artifacts you save around it end up in one group on `/runs/[id]`:
155
+
156
+ ```ts
157
+ const run = rig.artifacts.run.start({ pipeline: "report" });
158
+ // A raw (BYOK) call names its step; the save with no stepKey joins it.
159
+ const out = await rig.runRaw({ provider, model, apiKey, systemPrompt, userPrompt,
160
+ tags: { project: "acme" }, stepKey: "earnings_5a" });
161
+ rig.artifacts.artifact.save(out.output, { name: "earnings.output", type: "step_output" });
162
+ run.end("succeeded");
163
+ ```
164
+
165
+ > **Raw lane and the tree.** A `rig.runRaw` / `rig.runRawStream` call inside a run
166
+ > context records a ground-truth step **since modelrig 0.5.0**, on success and on
167
+ > failure. Before 0.5.0 the raw lane never read the run context, so a pipeline
168
+ > that wrapped raw calls in `run.start` got an **empty tree**. (Raw steps are
169
+ > metadata only — no artifact *content* is written on the raw lane.)
170
+
171
+ ## Never throws into your pipeline
172
+
173
+ Every entry point is fail-open: a telemetry-buffer write that fails is logged and
174
+ dropped, never thrown into your code. Instrumentation that runs *after* an
175
+ expensive model call must never be the thing that kills the run.
176
+
177
+ ## See it
178
+
179
+ The console's **Runs** tab lists every instrumented run; `/runs/[id]` is the
180
+ semantic chain (artifacts grouped by step, with hashes, costs, and evaluation
181
+ badges) and `/artifacts/[id]` shows one artifact's lineage, versions, and
182
+ evaluations. Enabling the flag also lights up the `/setup` artifacts line.
183
+
184
+ ## A reusable per-step seam
185
+
186
+ Most pipelines want one small helper that saves the three work products of a
187
+ step and wires their lineage, so the call sites stay a single line.
188
+ **`modelrig observe [path] --scope <one pipeline>` generates this seam for you**
189
+ (a `record-step` file) plus an `OBSERVE.md` whose per-call-site wiring is emitted
190
+ as diffs to apply — it never edits your source. The seam is equally yours to
191
+ write by hand; here it is, self-contained:
192
+
193
+ ```ts
194
+ import { createRig, loadConfigFromEnv, type Rig } from "modelrig";
195
+
196
+ const rig: Rig = createRig(loadConfigFromEnv());
197
+
198
+ /** Save prompt + raw + parsed for one step, link lineage, flag conformance. */
199
+ function recordStep(
200
+ stepKey: string,
201
+ work: { prompt: string; rawText: string; json: unknown; conforms: boolean },
202
+ ): void {
203
+ const p = rig.artifacts.artifact.save(work.prompt, { name: `${stepKey}.prompt`, type: "prompt", stepKey });
204
+ const raw = rig.artifacts.artifact.save(work.rawText, { name: `${stepKey}.raw`, type: "raw_response", stepKey });
205
+ const parsed = rig.artifacts.artifact.save(work.json, { name: `${stepKey}.output`, type: "step_output", task: stepKey, stepKey });
206
+ if (p && parsed) rig.artifacts.artifact.link(parsed, p, "derived_from");
207
+ if (raw && parsed) rig.artifacts.artifact.link(parsed, raw, "derived_from");
208
+ if (parsed) rig.artifacts.artifact.evaluate(parsed, { evaluator: "schema-check", kind: "deterministic", passed: work.conforms });
209
+ }
210
+ ```
211
+
212
+ Call `recordStep(step, { … })` once per stage inside the `run.start` / `run.end`
213
+ context above, and every entry point stays fail-open — a save that fails is
214
+ logged and dropped, never thrown into your pipeline.
215
+
216
+ > **In-repo example (reconstructed via onboarding-test run-2).** ModelRig's own
217
+ > customer-zero pipeline (the InferWealth v3 report generator) was stripped back
218
+ > to a clean slate to be re-onboarded through this exact doc. Its live seam is
219
+ > reconstructed as part of onboarding-test run-2; until then, the snippet above
220
+ > is the reference. Do not resurrect a deleted seam from git history — writing a
221
+ > fresh one is the point.