@sanity/ailf 7.0.0 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/rubrics.ts +12 -13
- package/dist/_vendor/ailf-core/examples/index.d.ts +3 -3
- package/dist/_vendor/ailf-core/examples/index.js +3 -3
- package/dist/_vendor/ailf-core/ports/context.d.ts +45 -3
- package/dist/_vendor/ailf-core/ports/index.d.ts +1 -1
- package/dist/_vendor/ailf-core/schemas/branded-string.d.ts +9 -1
- package/dist/_vendor/ailf-core/schemas/branded-string.js +16 -6
- package/dist/_vendor/ailf-core/schemas/pipeline-request.d.ts +2 -0
- package/dist/_vendor/ailf-core/schemas/pipeline-request.js +7 -0
- package/dist/_vendor/ailf-core/schemas/report.d.ts +12 -0
- package/dist/_vendor/ailf-core/schemas/report.js +2 -0
- package/dist/_vendor/ailf-core/schemas/team.d.ts +22 -0
- package/dist/_vendor/ailf-core/schemas/team.js +63 -0
- package/dist/_vendor/ailf-core/types/grader-judgment.d.ts +51 -0
- package/dist/_vendor/ailf-core/types/index.d.ts +8 -1
- package/dist/_vendor/ailf-core/types/pipeline-request.d.ts +17 -0
- package/dist/_vendor/ailf-core/types/team.d.ts +65 -0
- package/dist/_vendor/ailf-core/types/team.js +1 -0
- package/dist/_vendor/ailf-shared/eval-modes.d.ts +2 -0
- package/dist/_vendor/ailf-shared/eval-modes.js +5 -0
- package/dist/_vendor/ailf-shared/event-types.d.ts +15 -0
- package/dist/_vendor/ailf-shared/event-types.js +23 -0
- package/dist/_vendor/ailf-shared/generated/help-content.js +2 -2
- package/dist/_vendor/ailf-shared/index.d.ts +4 -2
- package/dist/_vendor/ailf-shared/index.js +4 -2
- package/dist/_vendor/ailf-shared/member-roles.d.ts +16 -0
- package/dist/_vendor/ailf-shared/member-roles.js +16 -0
- package/dist/_vendor/ailf-shared/owner-teams.d.ts +19 -0
- package/dist/_vendor/ailf-shared/owner-teams.js +7 -0
- package/dist/_vendor/ailf-shared/run-context.d.ts +8 -1
- package/dist/adapters/grader-outputs/promptfoo-grader-output.d.ts +65 -1
- package/dist/adapters/grader-outputs/promptfoo-grader-output.js +35 -0
- package/dist/adapters/task-sources/changed-docs-filter.d.ts +12 -0
- package/dist/adapters/task-sources/changed-docs-filter.js +30 -0
- package/dist/adapters/task-sources/content-lake-task-source.js +2 -1
- package/dist/adapters/task-sources/repo-task-source.js +2 -1
- package/dist/commands/pipeline-action.d.ts +4 -3
- package/dist/commands/pipeline-action.js +7 -5
- package/dist/commands/run.js +2 -2
- package/dist/config/rubrics.ts +12 -13
- package/dist/job-store.d.ts +18 -0
- package/dist/job-store.js +34 -0
- package/dist/orchestration/build-app-context.js +8 -1
- package/dist/orchestration/pipeline-orchestrator.js +46 -1
- package/dist/orchestration/steps/compare-step.d.ts +7 -0
- package/dist/orchestration/steps/compare-step.js +59 -23
- package/dist/orchestration/steps/fetch-docs-step.js +3 -0
- package/dist/orchestration/steps/finalize-run-step.js +2 -0
- package/dist/orchestration/steps/generate-configs-step.d.ts +32 -1
- package/dist/orchestration/steps/generate-configs-step.js +47 -13
- package/dist/orchestration/steps/grader-consistency-step.js +11 -0
- package/dist/orchestration/steps/publish-report-step.d.ts +12 -1
- package/dist/orchestration/steps/publish-report-step.js +19 -3
- package/dist/pipeline/cache-hit-restore.d.ts +30 -5
- package/dist/pipeline/cache-hit-restore.js +36 -6
- package/dist/pipeline/calculate-scores.js +57 -21
- package/dist/pipeline/compiler/mode-handlers/literacy/assertions.d.ts +7 -2
- package/dist/pipeline/compiler/mode-handlers/literacy/assertions.js +13 -4
- package/dist/pipeline/compiler/mode-handlers/literacy/compiler.js +1 -1
- package/dist/pipeline/compiler/provider-assembler.d.ts +15 -1
- package/dist/pipeline/compiler/provider-assembler.js +16 -3
- package/dist/pipeline/failure-modes.d.ts +20 -10
- package/dist/pipeline/failure-modes.js +84 -15
- package/dist/pipeline/map-request-to-config.js +2 -0
- package/dist/pipeline/normalize-mode.d.ts +1 -1
- package/dist/pipeline/normalize-mode.js +2 -0
- package/dist/pipeline/run-context.d.ts +16 -1
- package/dist/pipeline/run-context.js +12 -1
- package/dist/pipeline/validate.d.ts +8 -4
- package/dist/pipeline/validate.js +8 -18
- package/dist/report-store.d.ts +14 -1
- package/dist/report-store.js +32 -0
- package/dist/sanity/client.js +2 -2
- package/package.json +3 -3
package/dist/report-store.js
CHANGED
|
@@ -270,6 +270,38 @@ export class ReportStore {
|
|
|
270
270
|
return null;
|
|
271
271
|
}
|
|
272
272
|
}
|
|
273
|
+
/**
|
|
274
|
+
* Load a previously-published report's score summary as a baseline
|
|
275
|
+
* for comparison. Returns a discriminated result so the caller can
|
|
276
|
+
* distinguish a genuine 404 (skip compare with a clear reason) from
|
|
277
|
+
* a transport failure (fail the step — the user pinned a baseline
|
|
278
|
+
* and deserves to know it didn't actually compare).
|
|
279
|
+
*
|
|
280
|
+
* The report's `summary` field is a `ReportSummary` — a superset of
|
|
281
|
+
* `ComparableSummary` — so the projection below carries everything
|
|
282
|
+
* the `compare()` primitive needs (`overall`, `perModel`, `scores`)
|
|
283
|
+
* without re-hydrating the slim prose/array fields.
|
|
284
|
+
*/
|
|
285
|
+
async loadBaselineFromReport(reportId) {
|
|
286
|
+
try {
|
|
287
|
+
const doc = await this.client.fetch(`*[_type == $type && reportId == $id][0]{ summary }`, { id: reportId, type: REPORT_TYPE });
|
|
288
|
+
const summary = doc?.summary;
|
|
289
|
+
if (!summary)
|
|
290
|
+
return { kind: "not_found" };
|
|
291
|
+
return {
|
|
292
|
+
kind: "ok",
|
|
293
|
+
baseline: {
|
|
294
|
+
overall: summary.overall,
|
|
295
|
+
perModel: summary.perModel,
|
|
296
|
+
scores: summary.scores,
|
|
297
|
+
},
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
catch (error) {
|
|
301
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
302
|
+
return { kind: "error", message };
|
|
303
|
+
}
|
|
304
|
+
}
|
|
273
305
|
/**
|
|
274
306
|
* Write a report to the Sanity Content Lake.
|
|
275
307
|
*
|
package/dist/sanity/client.js
CHANGED
|
@@ -108,8 +108,8 @@ export function getSanityClient(overrides, source) {
|
|
|
108
108
|
* fall back to `SANITY_DATASET` so existing CI workflows that pin a
|
|
109
109
|
* test/staging dataset (e.g. Tier 2 with `SANITY_DATASET=ailf-test`)
|
|
110
110
|
* continue to work without a new env var. The hard-coded fallback is
|
|
111
|
-
*
|
|
112
|
-
*
|
|
111
|
+
* `AILF_DATASET_DEFAULT` (`ailf-prod-private`, D0043) — only reached for
|
|
112
|
+
* ad-hoc runs with no env at all.
|
|
113
113
|
*
|
|
114
114
|
* Token resolution prefers the AILF-scoped token, falling back to
|
|
115
115
|
* the shared `SANITY_API_TOKEN`.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sanity/ailf",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.1.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
@@ -56,8 +56,8 @@
|
|
|
56
56
|
"tsx": "^4.19.2",
|
|
57
57
|
"typescript": "^5.7.3",
|
|
58
58
|
"vitest": "^4.1.5",
|
|
59
|
-
"@sanity/ailf-
|
|
60
|
-
"@sanity/ailf-
|
|
59
|
+
"@sanity/ailf-core": "0.1.0",
|
|
60
|
+
"@sanity/ailf-shared": "0.1.0"
|
|
61
61
|
},
|
|
62
62
|
"scripts": {
|
|
63
63
|
"build": "tsc && tsc -p tsconfig.scripts.json && tsx scripts/bundle-workspace-deps.ts",
|