@sanity/ailf 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js
CHANGED
|
@@ -147,6 +147,8 @@ import { createAgentReportCommand } from "./commands/agent-report.js";
|
|
|
147
147
|
program.addCommand(createAgentReportCommand().helpGroup(CommandGroup.AnalysisReports));
|
|
148
148
|
import { createWeeklyDigestCommand } from "./commands/weekly-digest.js";
|
|
149
149
|
program.addCommand(createWeeklyDigestCommand().helpGroup(CommandGroup.AnalysisReports));
|
|
150
|
+
import { createCheckStalenessCommand } from "./commands/check-staleness.js";
|
|
151
|
+
program.addCommand(createCheckStalenessCommand().helpGroup(CommandGroup.AnalysisReports));
|
|
150
152
|
// ── Grader Reliability ────────────────────────────────────────────────
|
|
151
153
|
import { createGraderCommand } from "./commands/grader/index.js";
|
|
152
154
|
program.addCommand(createGraderCommand().helpGroup(CommandGroup.GraderReliability));
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* check-staleness command — verifies recent evaluation reports exist.
|
|
3
|
+
*
|
|
4
|
+
* Exits 0 when the most recent report in the Sanity Content Lake is within
|
|
5
|
+
* the max-age window, 1 otherwise (including "no reports at all"). Emits a
|
|
6
|
+
* single JSON line on stdout summarizing the decision so CI can pipe it
|
|
7
|
+
* directly into an alert payload.
|
|
8
|
+
*
|
|
9
|
+
* Used by the scheduled staleness workflow to detect silent pipeline
|
|
10
|
+
* failures — cases where scheduled evaluations stop producing reports but
|
|
11
|
+
* no workflow run fails loudly enough to be noticed.
|
|
12
|
+
*/
|
|
13
|
+
import { Command } from "commander";
|
|
14
|
+
export declare function createCheckStalenessCommand(): Command;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* check-staleness command — verifies recent evaluation reports exist.
|
|
3
|
+
*
|
|
4
|
+
* Exits 0 when the most recent report in the Sanity Content Lake is within
|
|
5
|
+
* the max-age window, 1 otherwise (including "no reports at all"). Emits a
|
|
6
|
+
* single JSON line on stdout summarizing the decision so CI can pipe it
|
|
7
|
+
* directly into an alert payload.
|
|
8
|
+
*
|
|
9
|
+
* Used by the scheduled staleness workflow to detect silent pipeline
|
|
10
|
+
* failures — cases where scheduled evaluations stop producing reports but
|
|
11
|
+
* no workflow run fails loudly enough to be noticed.
|
|
12
|
+
*/
|
|
13
|
+
import { Command } from "commander";
|
|
14
|
+
export function createCheckStalenessCommand() {
|
|
15
|
+
return new Command("check-staleness")
|
|
16
|
+
.description("Exit 1 if no evaluation report has been produced within the max-age window")
|
|
17
|
+
.option("--max-age <days>", "Max age in days before reports are considered stale", (v) => Number.parseInt(v, 10), 3)
|
|
18
|
+
.action(async (opts) => {
|
|
19
|
+
const { getSanityClient } = await import("../sanity/client.js");
|
|
20
|
+
// Resolve report-store credentials with the same precedence as
|
|
21
|
+
// weekly-digest.ts and composition-root.ts — AILF_REPORT_* wins over
|
|
22
|
+
// the evaluated-source SANITY_* defaults so the staleness probe tracks
|
|
23
|
+
// the actual report dataset even when it diverges from the eval source.
|
|
24
|
+
const client = getSanityClient({
|
|
25
|
+
dataset: process.env.AILF_REPORT_DATASET,
|
|
26
|
+
projectId: process.env.AILF_REPORT_PROJECT_ID,
|
|
27
|
+
token: process.env.AILF_REPORT_SANITY_API_TOKEN ??
|
|
28
|
+
process.env.SANITY_API_TOKEN,
|
|
29
|
+
});
|
|
30
|
+
const maxAgeDays = opts.maxAge;
|
|
31
|
+
// Bound the GROQ sort with a `completedAt > $floor` filter. Beyond
|
|
32
|
+
// ~10,000 reports the unbounded `order(completedAt desc)[0]` scan
|
|
33
|
+
// becomes a noticeable cost; a floor proportional to the max-age
|
|
34
|
+
// window keeps the scan cheap regardless of corpus size. The factor
|
|
35
|
+
// of 10× max-age gives plenty of headroom — if the last report
|
|
36
|
+
// predates the floor, the absence of any result still yields the
|
|
37
|
+
// correct "stale" verdict.
|
|
38
|
+
const floorDays = Math.max(maxAgeDays * 10, 30);
|
|
39
|
+
const floor = new Date(Date.now() - floorDays * 24 * 60 * 60 * 1000).toISOString();
|
|
40
|
+
const QUERY = `*[_type == "ailf.report" && completedAt > $floor] | order(completedAt desc)[0]{
|
|
41
|
+
"reportId": reportId,
|
|
42
|
+
"completedAt": completedAt,
|
|
43
|
+
"tag": tag
|
|
44
|
+
}`;
|
|
45
|
+
const latest = await client.fetch(QUERY, { floor });
|
|
46
|
+
// Use `process.exitCode` + `return` rather than `process.exit()` so
|
|
47
|
+
// stdout flushes cleanly when the caller captures via `$(...)` — a
|
|
48
|
+
// hard exit can drop buffered output on piped captures. Matches the
|
|
49
|
+
// pattern used by agent-report.ts, capture-list.ts, etc.
|
|
50
|
+
if (!latest || !latest.completedAt) {
|
|
51
|
+
console.log(JSON.stringify({
|
|
52
|
+
floorDays,
|
|
53
|
+
maxAgeDays,
|
|
54
|
+
reason: "no-reports",
|
|
55
|
+
stale: true,
|
|
56
|
+
}));
|
|
57
|
+
process.exitCode = 1;
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
const ageMs = Date.now() - new Date(latest.completedAt).getTime();
|
|
61
|
+
const ageDays = Number((ageMs / (24 * 60 * 60 * 1000)).toFixed(2));
|
|
62
|
+
const stale = ageDays > maxAgeDays;
|
|
63
|
+
console.log(JSON.stringify({
|
|
64
|
+
ageDays,
|
|
65
|
+
floorDays,
|
|
66
|
+
latestCompletedAt: latest.completedAt,
|
|
67
|
+
latestReportId: latest.reportId,
|
|
68
|
+
latestTag: latest.tag,
|
|
69
|
+
maxAgeDays,
|
|
70
|
+
stale,
|
|
71
|
+
}));
|
|
72
|
+
process.exitCode = stale ? 1 : 0;
|
|
73
|
+
});
|
|
74
|
+
}
|