@sanity/ailf 7.0.0 → 7.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/config/rubrics.ts +12 -13
  2. package/dist/_vendor/ailf-core/examples/index.d.ts +3 -3
  3. package/dist/_vendor/ailf-core/examples/index.js +3 -3
  4. package/dist/_vendor/ailf-core/ports/context.d.ts +45 -3
  5. package/dist/_vendor/ailf-core/ports/index.d.ts +1 -1
  6. package/dist/_vendor/ailf-core/schemas/branded-string.d.ts +9 -1
  7. package/dist/_vendor/ailf-core/schemas/branded-string.js +16 -6
  8. package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +2 -0
  9. package/dist/_vendor/ailf-core/schemas/pipeline-request.js +7 -0
  10. package/dist/_vendor/ailf-core/schemas/report.d.ts +12 -0
  11. package/dist/_vendor/ailf-core/schemas/report.js +2 -0
  12. package/dist/_vendor/ailf-core/schemas/team.d.ts +22 -0
  13. package/dist/_vendor/ailf-core/schemas/team.js +63 -0
  14. package/dist/_vendor/ailf-core/types/grader-judgment.d.ts +51 -0
  15. package/dist/_vendor/ailf-core/types/index.d.ts +8 -1
  16. package/dist/_vendor/ailf-core/types/pipeline-request.d.ts +17 -0
  17. package/dist/_vendor/ailf-core/types/team.d.ts +65 -0
  18. package/dist/_vendor/ailf-core/types/team.js +1 -0
  19. package/dist/_vendor/ailf-shared/eval-modes.d.ts +2 -0
  20. package/dist/_vendor/ailf-shared/eval-modes.js +5 -0
  21. package/dist/_vendor/ailf-shared/event-types.d.ts +15 -0
  22. package/dist/_vendor/ailf-shared/event-types.js +23 -0
  23. package/dist/_vendor/ailf-shared/generated/help-content.js +2 -2
  24. package/dist/_vendor/ailf-shared/index.d.ts +4 -2
  25. package/dist/_vendor/ailf-shared/index.js +4 -2
  26. package/dist/_vendor/ailf-shared/member-roles.d.ts +16 -0
  27. package/dist/_vendor/ailf-shared/member-roles.js +16 -0
  28. package/dist/_vendor/ailf-shared/owner-teams.d.ts +19 -0
  29. package/dist/_vendor/ailf-shared/owner-teams.js +7 -0
  30. package/dist/_vendor/ailf-shared/run-context.d.ts +8 -1
  31. package/dist/adapters/grader-outputs/promptfoo-grader-output.d.ts +65 -1
  32. package/dist/adapters/grader-outputs/promptfoo-grader-output.js +35 -0
  33. package/dist/adapters/task-sources/changed-docs-filter.d.ts +12 -0
  34. package/dist/adapters/task-sources/changed-docs-filter.js +30 -0
  35. package/dist/adapters/task-sources/content-lake-task-source.js +2 -1
  36. package/dist/adapters/task-sources/repo-task-source.js +2 -1
  37. package/dist/commands/pipeline-action.d.ts +4 -3
  38. package/dist/commands/pipeline-action.js +7 -5
  39. package/dist/commands/run.js +2 -2
  40. package/dist/config/rubrics.ts +12 -13
  41. package/dist/job-store.d.ts +18 -0
  42. package/dist/job-store.js +34 -0
  43. package/dist/orchestration/build-app-context.js +8 -1
  44. package/dist/orchestration/pipeline-orchestrator.js +46 -1
  45. package/dist/orchestration/steps/compare-step.d.ts +7 -0
  46. package/dist/orchestration/steps/compare-step.js +59 -23
  47. package/dist/orchestration/steps/fetch-docs-step.js +3 -0
  48. package/dist/orchestration/steps/finalize-run-step.js +2 -0
  49. package/dist/orchestration/steps/generate-configs-step.d.ts +32 -1
  50. package/dist/orchestration/steps/generate-configs-step.js +47 -13
  51. package/dist/orchestration/steps/grader-consistency-step.js +11 -0
  52. package/dist/orchestration/steps/publish-report-step.d.ts +12 -1
  53. package/dist/orchestration/steps/publish-report-step.js +19 -3
  54. package/dist/pipeline/cache-hit-restore.d.ts +30 -5
  55. package/dist/pipeline/cache-hit-restore.js +36 -6
  56. package/dist/pipeline/calculate-scores.js +57 -21
  57. package/dist/pipeline/compiler/mode-handlers/literacy/assertions.d.ts +7 -2
  58. package/dist/pipeline/compiler/mode-handlers/literacy/assertions.js +13 -4
  59. package/dist/pipeline/compiler/mode-handlers/literacy/compiler.js +1 -1
  60. package/dist/pipeline/compiler/provider-assembler.d.ts +15 -1
  61. package/dist/pipeline/compiler/provider-assembler.js +16 -3
  62. package/dist/pipeline/failure-modes.d.ts +20 -10
  63. package/dist/pipeline/failure-modes.js +84 -15
  64. package/dist/pipeline/map-request-to-config.js +2 -0
  65. package/dist/pipeline/normalize-mode.d.ts +1 -1
  66. package/dist/pipeline/normalize-mode.js +2 -0
  67. package/dist/pipeline/run-context.d.ts +16 -1
  68. package/dist/pipeline/run-context.js +12 -1
  69. package/dist/pipeline/validate.d.ts +8 -4
  70. package/dist/pipeline/validate.js +8 -18
  71. package/dist/report-store.d.ts +14 -1
  72. package/dist/report-store.js +32 -0
  73. package/dist/sanity/client.js +2 -2
  74. package/package.json +3 -3
@@ -270,6 +270,38 @@ export class ReportStore {
270
270
  return null;
271
271
  }
272
272
  }
273
+ /**
274
+ * Load a previously-published report's score summary as a baseline
275
+ * for comparison. Returns a discriminated result so the caller can
276
+ * distinguish a genuine 404 (skip compare with a clear reason) from
277
+ * a transport failure (fail the step — the user pinned a baseline
278
+ * and deserves to know it didn't actually compare).
279
+ *
280
+ * The report's `summary` field is a `ReportSummary` — a superset of
281
+ * `ComparableSummary` — so the projection below carries everything
282
+ * the `compare()` primitive needs (`overall`, `perModel`, `scores`)
283
+ * without re-hydrating the slim prose/array fields.
284
+ */
285
+ async loadBaselineFromReport(reportId) {
286
+ try {
287
+ const doc = await this.client.fetch(`*[_type == $type && reportId == $id][0]{ summary }`, { id: reportId, type: REPORT_TYPE });
288
+ const summary = doc?.summary;
289
+ if (!summary)
290
+ return { kind: "not_found" };
291
+ return {
292
+ kind: "ok",
293
+ baseline: {
294
+ overall: summary.overall,
295
+ perModel: summary.perModel,
296
+ scores: summary.scores,
297
+ },
298
+ };
299
+ }
300
+ catch (error) {
301
+ const message = error instanceof Error ? error.message : String(error);
302
+ return { kind: "error", message };
303
+ }
304
+ }
273
305
  /**
274
306
  * Write a report to the Sanity Content Lake.
275
307
  *
@@ -108,8 +108,8 @@ export function getSanityClient(overrides, source) {
108
108
  * fall back to `SANITY_DATASET` so existing CI workflows that pin a
109
109
  * test/staging dataset (e.g. Tier 2 with `SANITY_DATASET=ailf-test`)
110
110
  * continue to work without a new env var. The hard-coded fallback is
111
- * the editorial dataset name during the D0043 cutover window the flip
112
- * to `ailf-prod-private` happens after the migration script runs.
111
+ * `AILF_DATASET_DEFAULT` (`ailf-prod-private`, D0043) only reached for
112
+ * ad-hoc runs with no env at all.
113
113
  *
114
114
  * Token resolution prefers the AILF-scoped token, falling back to
115
115
  * the shared `SANITY_API_TOKEN`.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sanity/ailf",
3
- "version": "7.0.0",
3
+ "version": "7.1.0",
4
4
  "private": false,
5
5
  "publishConfig": {
6
6
  "access": "public"
@@ -56,8 +56,8 @@
56
56
  "tsx": "^4.19.2",
57
57
  "typescript": "^5.7.3",
58
58
  "vitest": "^4.1.5",
59
- "@sanity/ailf-shared": "0.1.0",
60
- "@sanity/ailf-core": "0.1.0"
59
+ "@sanity/ailf-core": "0.1.0",
60
+ "@sanity/ailf-shared": "0.1.0"
61
61
  },
62
62
  "scripts": {
63
63
  "build": "tsc && tsc -p tsconfig.scripts.json && tsx scripts/bundle-workspace-deps.ts",