@sanity/ailf 2.0.2 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/LICENSE +21 -0
  2. package/dist/cli.js +0 -0
  3. package/package.json +24 -24
  4. package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.d.ts +0 -10
  5. package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.js +0 -185
  6. package/dist/_vendor/ailf-core/artifact-capture/__tests__/noop-collector.test.d.ts +0 -6
  7. package/dist/_vendor/ailf-core/artifact-capture/__tests__/noop-collector.test.js +0 -42
  8. package/dist/_vendor/ailf-tasks/cli.d.ts +0 -8
  9. package/dist/_vendor/ailf-tasks/cli.js +0 -61
  10. package/dist/_vendor/ailf-tasks/index.d.ts +0 -13
  11. package/dist/_vendor/ailf-tasks/index.js +0 -16
  12. package/dist/_vendor/ailf-tasks/parser.d.ts +0 -27
  13. package/dist/_vendor/ailf-tasks/parser.js +0 -73
  14. package/dist/_vendor/ailf-tasks/schemas.d.ts +0 -198
  15. package/dist/_vendor/ailf-tasks/schemas.js +0 -180
  16. package/dist/_vendor/ailf-tasks/validation.d.ts +0 -47
  17. package/dist/_vendor/ailf-tasks/validation.js +0 -162
  18. package/dist/adapters/task-sources/yaml-task-source.d.ts +0 -18
  19. package/dist/adapters/task-sources/yaml-task-source.js +0 -139
  20. package/dist/agent-observer/test-imports.d.ts +0 -7
  21. package/dist/agent-observer/test-imports.js +0 -185
  22. package/dist/commands/update-quality-scores.d.ts +0 -5
  23. package/dist/commands/update-quality-scores.js +0 -20
  24. package/dist/lib/agent-behavior-report.d.ts +0 -8
  25. package/dist/lib/agent-behavior-report.js +0 -185
  26. package/dist/lib/baseline.d.ts +0 -19
  27. package/dist/lib/baseline.js +0 -153
  28. package/dist/lib/calculate-scores.d.ts +0 -23
  29. package/dist/lib/calculate-scores.js +0 -42
  30. package/dist/lib/compare.d.ts +0 -18
  31. package/dist/lib/compare.js +0 -170
  32. package/dist/lib/coverage-audit.d.ts +0 -4
  33. package/dist/lib/coverage-audit.js +0 -42
  34. package/dist/lib/discovery-report.d.ts +0 -13
  35. package/dist/lib/discovery-report.js +0 -57
  36. package/dist/lib/fetch-docs.d.ts +0 -30
  37. package/dist/lib/fetch-docs.js +0 -171
  38. package/dist/lib/generate-configs.d.ts +0 -25
  39. package/dist/lib/generate-configs.js +0 -42
  40. package/dist/lib/grader-api.d.ts +0 -21
  41. package/dist/lib/grader-api.js +0 -34
  42. package/dist/lib/grader-compare.d.ts +0 -19
  43. package/dist/lib/grader-compare.js +0 -91
  44. package/dist/lib/grader-consistency.d.ts +0 -27
  45. package/dist/lib/grader-consistency.js +0 -79
  46. package/dist/lib/grader-sensitivity.d.ts +0 -19
  47. package/dist/lib/grader-sensitivity.js +0 -75
  48. package/dist/lib/grader-validate.d.ts +0 -19
  49. package/dist/lib/grader-validate.js +0 -78
  50. package/dist/lib/measure-retrieval.d.ts +0 -14
  51. package/dist/lib/measure-retrieval.js +0 -71
  52. package/dist/lib/pr-comment.d.ts +0 -16
  53. package/dist/lib/pr-comment.js +0 -28
  54. package/dist/lib/readiness-report.d.ts +0 -13
  55. package/dist/lib/readiness-report.js +0 -108
  56. package/dist/lib/webhook-server.d.ts +0 -11
  57. package/dist/lib/webhook-server.js +0 -24
  58. package/dist/lib/weekly-digest.d.ts +0 -24
  59. package/dist/lib/weekly-digest.js +0 -148
  60. package/dist/orchestration/env-bridge.d.ts +0 -21
  61. package/dist/orchestration/env-bridge.js +0 -66
  62. package/dist/orchestration/steps/fetch-docs-shell.d.ts +0 -17
  63. package/dist/orchestration/steps/fetch-docs-shell.js +0 -30
  64. package/dist/pipeline/compiler/__tests__/task-bridge.test.d.ts +0 -9
  65. package/dist/pipeline/compiler/__tests__/task-bridge.test.js +0 -339
  66. package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.d.ts +0 -70
  67. package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.js +0 -485
  68. package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.d.ts +0 -76
  69. package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.js +0 -245
  70. package/dist/pipeline/compiler/mode-handlers/literacy-handler.d.ts +0 -89
  71. package/dist/pipeline/compiler/mode-handlers/literacy-handler.js +0 -379
  72. package/dist/pipeline/compiler/mode-handlers/mcp-assertions.d.ts +0 -50
  73. package/dist/pipeline/compiler/mode-handlers/mcp-assertions.js +0 -334
  74. package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.d.ts +0 -69
  75. package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.js +0 -307
  76. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.d.ts +0 -65
  77. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.js +0 -368
  78. package/dist/pipeline/compiler/task-bridge.d.ts +0 -41
  79. package/dist/pipeline/compiler/task-bridge.js +0 -92
  80. package/dist/pipeline/expand-tasks.d.ts +0 -232
  81. package/dist/pipeline/expand-tasks.js +0 -467
  82. package/dist/pipeline/generate-configs.d.ts +0 -92
  83. package/dist/pipeline/generate-configs.js +0 -445
  84. package/dist/pipeline/steps/calculate-scores-step.d.ts +0 -11
  85. package/dist/pipeline/steps/calculate-scores-step.js +0 -89
  86. package/dist/pipeline/steps/compare-step.d.ts +0 -18
  87. package/dist/pipeline/steps/compare-step.js +0 -90
  88. package/dist/pipeline/steps/eval-step.d.ts +0 -53
  89. package/dist/pipeline/steps/eval-step.js +0 -347
  90. package/dist/pipeline/steps/fetch-docs-step.d.ts +0 -11
  91. package/dist/pipeline/steps/fetch-docs-step.js +0 -84
  92. package/dist/pipeline/steps/generate-configs-step.d.ts +0 -11
  93. package/dist/pipeline/steps/generate-configs-step.js +0 -98
  94. package/dist/pipeline/steps/grader-consistency-step.d.ts +0 -21
  95. package/dist/pipeline/steps/grader-consistency-step.js +0 -74
  96. package/dist/pipeline/steps/publish-report-step.d.ts +0 -57
  97. package/dist/pipeline/steps/publish-report-step.js +0 -243
  98. package/dist/pipeline/steps/report-step.d.ts +0 -13
  99. package/dist/pipeline/steps/report-step.js +0 -56
  100. package/dist/pipeline/steps/update-scores-step.d.ts +0 -11
  101. package/dist/pipeline/steps/update-scores-step.js +0 -42
  102. package/dist/scripts/agent-behavior-report.d.ts +0 -19
  103. package/dist/scripts/agent-behavior-report.js +0 -315
  104. package/dist/scripts/baseline.d.ts +0 -43
  105. package/dist/scripts/baseline.js +0 -267
  106. package/dist/scripts/calculate-scores.d.ts +0 -166
  107. package/dist/scripts/calculate-scores.js +0 -1296
  108. package/dist/scripts/compare.d.ts +0 -22
  109. package/dist/scripts/compare.js +0 -334
  110. package/dist/scripts/coverage-audit.d.ts +0 -44
  111. package/dist/scripts/coverage-audit.js +0 -209
  112. package/dist/scripts/debug-eval.d.ts +0 -19
  113. package/dist/scripts/debug-eval.js +0 -73
  114. package/dist/scripts/discovery-report.d.ts +0 -58
  115. package/dist/scripts/discovery-report.js +0 -250
  116. package/dist/scripts/fetch-docs.d.ts +0 -35
  117. package/dist/scripts/fetch-docs.js +0 -472
  118. package/dist/scripts/generate-configs.d.ts +0 -66
  119. package/dist/scripts/generate-configs.js +0 -459
  120. package/dist/scripts/grader-api.d.ts +0 -27
  121. package/dist/scripts/grader-api.js +0 -206
  122. package/dist/scripts/grader-compare.d.ts +0 -22
  123. package/dist/scripts/grader-compare.js +0 -368
  124. package/dist/scripts/grader-consistency.d.ts +0 -20
  125. package/dist/scripts/grader-consistency.js +0 -313
  126. package/dist/scripts/grader-sensitivity.d.ts +0 -22
  127. package/dist/scripts/grader-sensitivity.js +0 -354
  128. package/dist/scripts/grader-validate.d.ts +0 -19
  129. package/dist/scripts/grader-validate.js +0 -267
  130. package/dist/scripts/measure-retrieval.d.ts +0 -10
  131. package/dist/scripts/measure-retrieval.js +0 -145
  132. package/dist/scripts/migrate-tasks-to-content-lake.d.ts +0 -24
  133. package/dist/scripts/migrate-tasks-to-content-lake.js +0 -328
  134. package/dist/scripts/pipeline.d.ts +0 -76
  135. package/dist/scripts/pipeline.js +0 -1031
  136. package/dist/scripts/pr-comment.d.ts +0 -10
  137. package/dist/scripts/pr-comment.js +0 -510
  138. package/dist/scripts/readiness-report.d.ts +0 -88
  139. package/dist/scripts/readiness-report.js +0 -342
  140. package/dist/scripts/update-quality-scores.d.ts +0 -15
  141. package/dist/scripts/update-quality-scores.js +0 -184
  142. package/dist/scripts/validate-task-sources.d.ts +0 -21
  143. package/dist/scripts/validate-task-sources.js +0 -210
  144. package/dist/scripts/validate.d.ts +0 -13
  145. package/dist/scripts/validate.js +0 -79
  146. package/dist/scripts/webhook-server.d.ts +0 -26
  147. package/dist/scripts/webhook-server.js +0 -147
  148. package/dist/scripts/weekly-digest.d.ts +0 -24
  149. package/dist/scripts/weekly-digest.js +0 -144
  150. package/dist/sinks/format-slack.d.ts +0 -64
  151. package/dist/sinks/format-slack.js +0 -306
  152. package/dist/sinks/slack-sink.d.ts +0 -27
  153. package/dist/sinks/slack-sink.js +0 -78
  154. package/dist/sinks/webhook-sink.d.ts +0 -19
  155. package/dist/sinks/webhook-sink.js +0 -50
  156. package/tasks/.expanded.agentic.yaml +0 -280
  157. package/tasks/.expanded.yaml +0 -565
@@ -1,78 +0,0 @@
1
- /**
2
- * lib/grader-validate.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/grader-validate-runner.ts.
5
- * The pure analysis functions live in pipeline/grader-validation.ts.
6
- * This shim preserves backward compatibility for direct CLI invocation.
7
- *
8
- * TODO: Update all importers to use pipeline/ modules, then delete this file.
9
- *
10
- * @deprecated Import from ../pipeline/grader-validate-runner.js instead.
11
- */
12
- // oxlint-disable-next-line import/no-unassigned-import -- side-effect: loads .env into process.env
13
- import "dotenv/config";
14
- import { dirname, resolve } from "path";
15
- import { fileURLToPath } from "url";
16
- // Re-export from pipeline modules
17
- export { formatValidationReport, runGraderValidate, } from "../pipeline/grader-validate-runner.js";
18
- export { classifyCorrelation, validateGrader, } from "../pipeline/grader-validation.js";
19
- import { runGraderValidate } from "../pipeline/grader-validate-runner.js";
20
- const __dirname = dirname(fileURLToPath(import.meta.url));
21
- const ROOT = resolve(__dirname, "..", "..");
22
- // ---------------------------------------------------------------------------
23
- // CLI argument parsing
24
- // ---------------------------------------------------------------------------
25
- function parseCliArgs() {
26
- const args = process.argv.slice(2);
27
- function getOption(name) {
28
- const idx = args.indexOf(`--${name}`);
29
- return idx !== -1 && idx + 1 < args.length ? args[idx + 1] : undefined;
30
- }
31
- function getFlag(name) {
32
- return args.includes(`--${name}`);
33
- }
34
- const showHelp = getFlag("help") || getFlag("h");
35
- if (showHelp) {
36
- console.log(`
37
- Usage: pnpm grader-validate [options]
38
-
39
- Validate grader accuracy against human reference grades.
40
-
41
- Options:
42
- --grader <model> Grader model to validate (default: from config/models.yaml)
43
- --threshold <n> MAE threshold for pass/fail (default: 10)
44
- --help, -h Show this help
45
- `);
46
- process.exit(0);
47
- }
48
- const thresholdStr = getOption("threshold");
49
- return {
50
- graderOverride: getOption("grader"),
51
- maeThreshold: thresholdStr ? parseFloat(thresholdStr) : 10,
52
- };
53
- }
54
- /**
55
- * Legacy main() entry point.
56
- * @deprecated Use runGraderValidate() from pipeline/grader-validate-runner.js instead.
57
- */
58
- export async function main() {
59
- const { graderOverride, maeThreshold } = parseCliArgs();
60
- const result = await runGraderValidate({
61
- graderModel: graderOverride,
62
- maeThreshold,
63
- rootDir: ROOT,
64
- });
65
- // Exit with error code if threshold not met
66
- if (!result.passesThreshold) {
67
- console.error(`\n ❌ VALIDATION FAILED: MAE ${result.overallMae} exceeds threshold ${maeThreshold}`);
68
- process.exit(1);
69
- }
70
- }
71
- // Only run when invoked directly
72
- if (process.argv[1]?.endsWith("grader-validate.ts") ||
73
- process.argv[1]?.endsWith("grader-validate.js")) {
74
- main().catch((err) => {
75
- console.error("❌ Fatal error:", err);
76
- process.exit(1);
77
- });
78
- }
@@ -1,14 +0,0 @@
1
- /**
2
- * lib/measure-retrieval.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/measure-retrieval.ts.
5
- * This shim preserves backward compatibility for direct CLI invocation.
6
- *
7
- * TODO: Update all importers to use pipeline/measure-retrieval.ts, then delete this file.
8
- *
9
- * @deprecated Import from ../pipeline/measure-retrieval.js instead.
10
- */
11
- import "dotenv/config";
12
- export { calculateNDCG, calculateRecall, formatRetrievalTable, measureRetrieval, type MeasureRetrievalOptions, type RetrievalResult, type RetrievalSummary, type RetrieverFn, } from "../pipeline/measure-retrieval.js";
13
- /** @deprecated Use measureRetrieval() from pipeline/measure-retrieval.js instead. */
14
- export declare function main(): Promise<void>;
@@ -1,71 +0,0 @@
1
- /**
2
- * lib/measure-retrieval.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/measure-retrieval.ts.
5
- * This shim preserves backward compatibility for direct CLI invocation.
6
- *
7
- * TODO: Update all importers to use pipeline/measure-retrieval.ts, then delete this file.
8
- *
9
- * @deprecated Import from ../pipeline/measure-retrieval.js instead.
10
- */
11
- // oxlint-disable-next-line import/no-unassigned-import -- side-effect: loads .env into process.env
12
- import "dotenv/config";
13
- import { writeFileSync, mkdirSync } from "fs";
14
- import { join, dirname } from "path";
15
- import { getSanityClient } from "../sanity/client.js";
16
- import { formatRetrievalTable, measureRetrieval, } from "../pipeline/measure-retrieval.js";
17
- // Re-export pipeline types and functions
18
- export { calculateNDCG, calculateRecall, formatRetrievalTable, measureRetrieval, } from "../pipeline/measure-retrieval.js";
19
- // ---------------------------------------------------------------------------
20
- // Sanity text search retriever (side-effecting — uses Sanity client)
21
- // ---------------------------------------------------------------------------
22
- async function retrieveDocsForQuery(query, k = 10) {
23
- const client = getSanityClient();
24
- const results = await client.fetch(`
25
- *[_type == "article" && !(_id in path("drafts.**"))]
26
- | score(
27
- boost(title match $query, 3),
28
- boost(pt::text(content) match $query, 1)
29
- )
30
- | order(_score desc)
31
- [0...$k] {
32
- "slug": slug.current,
33
- _score
34
- }
35
- `, { k, query });
36
- return results.map((r) => r.slug);
37
- }
38
- // ---------------------------------------------------------------------------
39
- // Legacy main() entry point
40
- // ---------------------------------------------------------------------------
41
- /** @deprecated Use measureRetrieval() from pipeline/measure-retrieval.js instead. */
42
- export async function main() {
43
- console.log("=== Sanity AI Literacy — Retrieval Quality Measurement ===\n");
44
- const ROOT = join(dirname(new URL(import.meta.url).pathname), "..", "..");
45
- const summary = await measureRetrieval({
46
- onProgress: (area, taskId, result) => {
47
- console.log(` ${taskId}:`);
48
- console.log(` Recall@5: ${(result.recall_at_5 * 100).toFixed(1)}%`);
49
- console.log(` Recall@10: ${(result.recall_at_10 * 100).toFixed(1)}%`);
50
- console.log(` NDCG@10: ${(result.ndcg_at_10 * 100).toFixed(1)}%`);
51
- },
52
- retriever: retrieveDocsForQuery,
53
- rootDir: ROOT,
54
- });
55
- // Print summary
56
- console.log();
57
- console.log(formatRetrievalTable(summary));
58
- // Persist results
59
- const outDir = join(ROOT, "results", "latest");
60
- mkdirSync(outDir, { recursive: true });
61
- writeFileSync(join(outDir, "retrieval-results.json"), JSON.stringify(summary, null, 2));
62
- console.log("\nResults written to results/latest/retrieval-results.json");
63
- }
64
- // Only run when invoked directly (not when imported)
65
- if (process.argv[1]?.endsWith("measure-retrieval.ts") ||
66
- process.argv[1]?.endsWith("measure-retrieval.js")) {
67
- main().catch((err) => {
68
- console.error("Fatal error:", err);
69
- process.exit(1);
70
- });
71
- }
@@ -1,16 +0,0 @@
1
- /**
2
- * lib/pr-comment.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/pr-comment.ts.
5
- *
6
- * @deprecated Import from ../pipeline/pr-comment.js instead.
7
- */
8
- export { generatePrComment, type PrCommentOptions, } from "../pipeline/pr-comment.js";
9
- import type { PrCommentOptions } from "../pipeline/pr-comment.js";
10
- /**
11
- * Legacy main() entry point.
12
- * @deprecated Use generatePrComment() from pipeline/pr-comment.ts instead.
13
- */
14
- export declare function main(options?: Omit<PrCommentOptions, "rootDir"> & {
15
- rootDir?: string;
16
- }): void;
@@ -1,28 +0,0 @@
1
- /**
2
- * lib/pr-comment.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/pr-comment.ts.
5
- *
6
- * @deprecated Import from ../pipeline/pr-comment.js instead.
7
- */
8
- import { dirname, resolve } from "path";
9
- import { fileURLToPath } from "url";
10
- export { generatePrComment, } from "../pipeline/pr-comment.js";
11
- import { generatePrComment } from "../pipeline/pr-comment.js";
12
- const __dirname = dirname(fileURLToPath(import.meta.url));
13
- const ROOT = resolve(__dirname, "..", "..");
14
- /**
15
- * Legacy main() entry point.
16
- * @deprecated Use generatePrComment() from pipeline/pr-comment.ts instead.
17
- */
18
- export function main(options) {
19
- generatePrComment({
20
- rootDir: options?.rootDir ?? ROOT,
21
- outputPath: options?.outputPath,
22
- promptfooUrl: options?.promptfooUrl,
23
- });
24
- }
25
- if (process.argv[1]?.endsWith("pr-comment.ts") ||
26
- process.argv[1]?.endsWith("pr-comment.js")) {
27
- main();
28
- }
@@ -1,13 +0,0 @@
1
- /**
2
- * lib/readiness-report.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/readiness-report.ts.
5
- *
6
- * @deprecated Import from ../pipeline/readiness-report.js instead.
7
- */
8
- export { formatReadinessMarkdown, generateReadinessReport, type DimensionCheck, type HistoryEntry, type ReadinessReport, } from "../pipeline/readiness-report.js";
9
- /**
10
- * Legacy main() entry point.
11
- * @deprecated Use generateReadinessReport() + formatReadinessMarkdown() directly.
12
- */
13
- export declare function main(): void;
@@ -1,108 +0,0 @@
1
- /**
2
- * lib/readiness-report.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/readiness-report.ts.
5
- *
6
- * @deprecated Import from ../pipeline/readiness-report.js instead.
7
- */
8
- import { existsSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
9
- import { dirname, join, resolve } from "node:path";
10
- import { fileURLToPath } from "node:url";
11
- import { load } from "js-yaml";
12
- import { ThresholdConfigSchema, } from "../pipeline/schemas.js";
13
- export { formatReadinessMarkdown, generateReadinessReport, } from "../pipeline/readiness-report.js";
14
- import { generateReadinessReport, formatReadinessMarkdown, } from "../pipeline/readiness-report.js";
15
- const __dirname = dirname(fileURLToPath(import.meta.url));
16
- const ROOT = resolve(__dirname, "..", "..");
17
- const SCORE_SUMMARY_PATH = join(ROOT, "results", "latest", "score-summary.json");
18
- const GAP_ANALYSIS_PATH = join(ROOT, "results", "latest", "gap-analysis.json");
19
- const THRESHOLDS_PATH = join(ROOT, "config", "thresholds.yaml");
20
- const BASELINES_DIR = join(ROOT, "results", "baselines");
21
- /**
22
- * Legacy main() entry point.
23
- * @deprecated Use generateReadinessReport() + formatReadinessMarkdown() directly.
24
- */
25
- export function main() {
26
- const args = process.argv.slice(2);
27
- let area;
28
- let history = false;
29
- let output;
30
- for (let i = 0; i < args.length; i++) {
31
- const arg = args[i];
32
- if (arg === "--area" && i + 1 < args.length) {
33
- area = args[++i];
34
- }
35
- else if (arg === "--history") {
36
- history = true;
37
- }
38
- else if (arg === "--output" && i + 1 < args.length) {
39
- output = args[++i];
40
- }
41
- }
42
- if (!area) {
43
- console.error("Usage: readiness-report --area <area> [--history] [--output <file>]");
44
- process.exit(1);
45
- }
46
- // Load data
47
- if (!existsSync(SCORE_SUMMARY_PATH)) {
48
- throw new Error(`Score summary not found at ${SCORE_SUMMARY_PATH}. Run \`pnpm pipeline\` first.`);
49
- }
50
- const scoreSummary = JSON.parse(readFileSync(SCORE_SUMMARY_PATH, "utf-8"));
51
- if (!existsSync(THRESHOLDS_PATH)) {
52
- throw new Error(`Threshold config not found at ${THRESHOLDS_PATH}.`);
53
- }
54
- const rawThresholds = readFileSync(THRESHOLDS_PATH, "utf-8");
55
- const thresholdConfig = ThresholdConfigSchema.parse(load(rawThresholds));
56
- let gapAnalysis;
57
- if (existsSync(GAP_ANALYSIS_PATH)) {
58
- gapAnalysis = JSON.parse(readFileSync(GAP_ANALYSIS_PATH, "utf-8"));
59
- }
60
- const historyEntries = [];
61
- if (history && existsSync(BASELINES_DIR)) {
62
- const files = readdirSync(BASELINES_DIR)
63
- .filter((f) => f.endsWith(".json"))
64
- .sort();
65
- for (const file of files) {
66
- try {
67
- const raw = readFileSync(join(BASELINES_DIR, file), "utf-8");
68
- const data = JSON.parse(raw);
69
- const areaScore = data.scores?.find((s) => s.feature === area);
70
- if (!areaScore)
71
- continue;
72
- const nameWithoutExt = file.replace(/\.json$/, "");
73
- const parts = nameWithoutExt.split("_");
74
- const tag = parts.length > 4 ? parts.slice(4).join("_") : undefined;
75
- historyEntries.push({
76
- score: areaScore.totalScore,
77
- tag,
78
- timestamp: data.timestamp ?? nameWithoutExt,
79
- });
80
- }
81
- catch {
82
- // Skip malformed baseline files
83
- }
84
- }
85
- }
86
- const report = generateReadinessReport({
87
- area,
88
- gapAnalysis,
89
- history: historyEntries,
90
- scoreSummary,
91
- thresholdConfig,
92
- });
93
- const markdown = formatReadinessMarkdown(report);
94
- if (output) {
95
- writeFileSync(output, markdown, "utf-8");
96
- console.error(`✅ Readiness report written to ${output}`);
97
- }
98
- else {
99
- console.log(markdown);
100
- }
101
- if (!report.pass) {
102
- process.exit(1);
103
- }
104
- }
105
- if (process.argv[1]?.endsWith("readiness-report.ts") ||
106
- process.argv[1]?.endsWith("readiness-report.js")) {
107
- main();
108
- }
@@ -1,11 +0,0 @@
1
- /**
2
- * lib/webhook-server.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/webhook-server.ts.
5
- * This shim preserves backward compatibility for direct CLI invocation.
6
- *
7
- * TODO: Update all importers to use pipeline/webhook-server.ts, then delete this file.
8
- *
9
- * @deprecated Import from ../pipeline/webhook-server.js instead.
10
- */
11
- export { startWebhookServer, type WebhookServerOptions, } from "../pipeline/webhook-server.js";
@@ -1,24 +0,0 @@
1
- /**
2
- * lib/webhook-server.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/webhook-server.ts.
5
- * This shim preserves backward compatibility for direct CLI invocation.
6
- *
7
- * TODO: Update all importers to use pipeline/webhook-server.ts, then delete this file.
8
- *
9
- * @deprecated Import from ../pipeline/webhook-server.js instead.
10
- */
11
- import { dirname, resolve } from "path";
12
- import { fileURLToPath } from "url";
13
- import { startWebhookServer } from "../pipeline/webhook-server.js";
14
- export { startWebhookServer, } from "../pipeline/webhook-server.js";
15
- const __dirname = dirname(fileURLToPath(import.meta.url));
16
- const ROOT = resolve(__dirname, "..", "..");
17
- // When imported directly (e.g., `tsx src/lib/webhook-server.ts`), start the server
18
- startWebhookServer({
19
- dailyBudget: parseInt(process.env.WEBHOOK_DAILY_BUDGET ?? "20", 10),
20
- debounceMs: parseInt(process.env.WEBHOOK_DEBOUNCE_MS ?? "10000", 10),
21
- githubToken: process.env.GITHUB_TOKEN ?? "",
22
- port: parseInt(process.env.WEBHOOK_PORT ?? "3333", 10),
23
- rootDir: ROOT,
24
- });
@@ -1,24 +0,0 @@
1
- /**
2
- * weekly-digest.ts
3
- *
4
- * CLI script to generate and deliver a weekly evaluation digest.
5
- *
6
- * Queries the Sanity Content Lake for all reports within the configured
7
- * lookback window, computes trend analysis, and delivers the digest
8
- * via configured channels (Slack, stdout, or both).
9
- *
10
- * Usage:
11
- * pnpm weekly-digest # send to configured Slack webhook
12
- * pnpm weekly-digest --dry-run # print to stdout only
13
- * pnpm weekly-digest --lookback 14 # 14-day lookback window
14
- * pnpm weekly-digest --json # output raw JSON
15
- *
16
- * Environment variables:
17
- * SLACK_WEBHOOK_URL — Slack incoming webhook URL
18
- * SANITY_API_TOKEN — Sanity read token
19
- * AILF_TRIGGER_TYPE — set to "scheduled" by the cron workflow
20
- * AILF_SCHEDULE — the schedule name (e.g., "weekly-digest")
21
- *
22
- * @see docs/design-docs/report-store/implementation.md — Phase 5
23
- */
24
- export declare function main(): Promise<void>;
@@ -1,148 +0,0 @@
1
- /**
2
- * weekly-digest.ts
3
- *
4
- * CLI script to generate and deliver a weekly evaluation digest.
5
- *
6
- * Queries the Sanity Content Lake for all reports within the configured
7
- * lookback window, computes trend analysis, and delivers the digest
8
- * via configured channels (Slack, stdout, or both).
9
- *
10
- * Usage:
11
- * pnpm weekly-digest # send to configured Slack webhook
12
- * pnpm weekly-digest --dry-run # print to stdout only
13
- * pnpm weekly-digest --lookback 14 # 14-day lookback window
14
- * pnpm weekly-digest --json # output raw JSON
15
- *
16
- * Environment variables:
17
- * SLACK_WEBHOOK_URL — Slack incoming webhook URL
18
- * SANITY_API_TOKEN — Sanity read token
19
- * AILF_TRIGGER_TYPE — set to "scheduled" by the cron workflow
20
- * AILF_SCHEDULE — the schedule name (e.g., "weekly-digest")
21
- *
22
- * @see docs/design-docs/report-store/implementation.md — Phase 5
23
- */
24
- import { config as dotenvConfig } from "dotenv";
25
- import { existsSync } from "fs";
26
- import { dirname, resolve } from "path";
27
- import { fileURLToPath } from "url";
28
- import { generateDigest } from "../schedules/digest.js";
29
- import { getDigestConfig } from "../schedules/loader.js";
30
- import { formatWeeklyDigest } from "../sinks/slack/format.js";
31
- // Load root .env (same override behavior as pipeline.ts)
32
- const __dirname = dirname(fileURLToPath(import.meta.url));
33
- const envPath = resolve(__dirname, "..", "..", "..", "..", ".env");
34
- if (existsSync(envPath)) {
35
- dotenvConfig({ override: true, path: envPath });
36
- }
37
- // ---------------------------------------------------------------------------
38
- // CLI argument parsing
39
- // ---------------------------------------------------------------------------
40
- const args = process.argv.slice(2);
41
- function getOption(name) {
42
- const idx = args.indexOf(`--${name}`);
43
- return idx >= 0 && idx + 1 < args.length ? args[idx + 1] : undefined;
44
- }
45
- function hasFlag(name) {
46
- return args.includes(`--${name}`);
47
- }
48
- const DRY_RUN = hasFlag("dry-run");
49
- const JSON_OUTPUT = hasFlag("json");
50
- const lookbackOverride = getOption("lookback");
51
- // ---------------------------------------------------------------------------
52
- // Main
53
- // ---------------------------------------------------------------------------
54
- export async function main() {
55
- console.log();
56
- console.log("=== AI Literacy Weekly Digest ===");
57
- console.log();
58
- // Load digest config
59
- const digestConfig = getDigestConfig();
60
- const lookbackDays = lookbackOverride
61
- ? parseInt(lookbackOverride, 10)
62
- : (digestConfig?.lookbackDays ?? 7);
63
- console.log(` Lookback: ${lookbackDays} days`);
64
- console.log(` Mode: ${DRY_RUN ? "dry run (stdout only)" : "live"}`);
65
- console.log();
66
- // Generate digest — uses AILF_REPORT_* env vars for report store access,
67
- // independent of SANITY_DATASET/SANITY_PROJECT_ID (which control doc evaluation)
68
- const digest = await generateDigest({
69
- dataset: process.env.AILF_REPORT_DATASET,
70
- lookbackDays,
71
- projectId: process.env.AILF_REPORT_PROJECT_ID,
72
- token: process.env.AILF_REPORT_SANITY_API_TOKEN ?? process.env.SANITY_API_TOKEN,
73
- });
74
- if (!digest) {
75
- console.log(" No reports found in the lookback window. Nothing to send.");
76
- process.exit(0);
77
- }
78
- // Output
79
- console.log(` Reports found: ${digest.reportCount}`);
80
- console.log(` Overall: ${Math.round(digest.overallLatest)} (${digest.overallTrend})`);
81
- console.log(` Improved: ${digest.improved.join(", ") || "none"}`);
82
- console.log(` Regressed: ${digest.regressed.join(", ") || "none"}`);
83
- console.log(` Stable: ${digest.stable.join(", ") || "none"}`);
84
- console.log();
85
- if (JSON_OUTPUT) {
86
- console.log(JSON.stringify(digest, null, 2));
87
- return;
88
- }
89
- // Format for Slack
90
- const message = formatWeeklyDigest(digest);
91
- if (DRY_RUN) {
92
- console.log(" --- Slack Message Preview ---");
93
- console.log(` Text: ${message.text}`);
94
- console.log();
95
- for (const block of message.blocks) {
96
- if (block.text) {
97
- console.log(` [${block.type}] ${block.text.text}`);
98
- }
99
- if (block.fields) {
100
- for (const field of block.fields) {
101
- console.log(` [field] ${field.text}`);
102
- }
103
- }
104
- if (block.elements) {
105
- for (const el of block.elements) {
106
- console.log(` [element] ${el.text}`);
107
- }
108
- }
109
- }
110
- console.log();
111
- return;
112
- }
113
- // Deliver via Slack
114
- const webhookUrl = digestConfig?.slackWebhookUrl ?? process.env.SLACK_WEBHOOK_URL;
115
- if (!webhookUrl) {
116
- console.warn(" ⚠️ No Slack webhook URL configured. Set SLACK_WEBHOOK_URL or configure in schedules.yaml");
117
- console.log(" Printing digest to stdout instead:");
118
- console.log();
119
- console.log(` ${message.text}`);
120
- return;
121
- }
122
- console.log(" Sending to Slack...");
123
- try {
124
- const response = await fetch(webhookUrl, {
125
- body: JSON.stringify(message),
126
- headers: { "Content-Type": "application/json" },
127
- method: "POST",
128
- });
129
- if (response.ok) {
130
- console.log(" ✅ Digest sent successfully");
131
- }
132
- else {
133
- const text = await response.text();
134
- console.warn(` ⚠️ Slack delivery failed: ${response.status} ${text}`);
135
- }
136
- }
137
- catch (error) {
138
- console.warn(` ⚠️ Slack delivery error: ${error instanceof Error ? error.message : String(error)}`);
139
- }
140
- }
141
- // Only run when invoked directly (not when imported)
142
- if (process.argv[1]?.endsWith("weekly-digest.ts") ||
143
- process.argv[1]?.endsWith("weekly-digest.js")) {
144
- main().catch((error) => {
145
- console.error("Fatal error:", error);
146
- process.exit(1);
147
- });
148
- }
@@ -1,21 +0,0 @@
1
- /**
2
- * Environment variable bridge — writes ResolvedConfig values to process.env
3
- * so that lib/*.ts modules (which still read process.env) work correctly.
4
- *
5
- * This replaces the former global applyEnvironment() with an explicit
6
- * per-step bridge. Each orchestration step calls this before invoking
7
- * its lib/*.ts main() function.
8
- *
9
- * Phase 9 will eliminate this file entirely by giving lib/*.ts main()
10
- * functions typed option parameters.
11
- *
12
- * @see docs/exec-plans/active/ports-and-adapters/phase-8-delete-legacy-step-layer.md
13
- */
14
- import type { ResolvedConfig } from "../_vendor/ailf-core/index.d.ts";
15
- /**
16
- * Bridge ResolvedConfig values to process.env.
17
- *
18
- * Idempotent — safe to call multiple times. Only sets env vars for
19
- * config values that are defined (never deletes or resets).
20
- */
21
- export declare function bridgeConfigToEnv(config: ResolvedConfig): void;
@@ -1,66 +0,0 @@
1
- /**
2
- * Environment variable bridge — writes ResolvedConfig values to process.env
3
- * so that lib/*.ts modules (which still read process.env) work correctly.
4
- *
5
- * This replaces the former global applyEnvironment() with an explicit
6
- * per-step bridge. Each orchestration step calls this before invoking
7
- * its lib/*.ts main() function.
8
- *
9
- * Phase 9 will eliminate this file entirely by giving lib/*.ts main()
10
- * functions typed option parameters.
11
- *
12
- * @see docs/exec-plans/active/ports-and-adapters/phase-8-delete-legacy-step-layer.md
13
- */
14
- /**
15
- * Bridge ResolvedConfig values to process.env.
16
- *
17
- * Idempotent — safe to call multiple times. Only sets env vars for
18
- * config values that are defined (never deletes or resets).
19
- */
20
- export function bridgeConfigToEnv(config) {
21
- // Mode
22
- process.env.EVAL_MODE = config.mode;
23
- // Search mode
24
- if (config.searchMode !== "open") {
25
- process.env.EVAL_SEARCH_MODE = config.searchMode;
26
- }
27
- // Source
28
- if (config.source) {
29
- process.env.DOC_SOURCE = config.source;
30
- }
31
- // URL-derived overrides
32
- if (config.urls?.[0]) {
33
- process.env.DOC_BASE_URL = config.urls[0];
34
- }
35
- // Sanity overrides
36
- if (config.datasetOverride) {
37
- process.env.SANITY_DATASET = config.datasetOverride;
38
- }
39
- if (config.projectIdOverride) {
40
- process.env.SANITY_PROJECT_ID = config.projectIdOverride;
41
- }
42
- if (config.perspectiveOverride) {
43
- process.env.SANITY_PERSPECTIVE = config.perspectiveOverride;
44
- }
45
- if (config.studioOriginOverride) {
46
- process.env.SANITY_STUDIO_ORIGIN = config.studioOriginOverride;
47
- }
48
- if (config.sanityDocumentArgs?.length) {
49
- process.env.SANITY_DOCUMENT_IDS = config.sanityDocumentArgs.join(",");
50
- }
51
- // Custom headers
52
- if (config.headers) {
53
- process.env.DOC_HEADERS = JSON.stringify(config.headers);
54
- }
55
- // Allowed origins
56
- if (config.allowedOrigins?.length) {
57
- process.env.DOC_ALLOWED_ORIGINS = config.allowedOrigins.join(",");
58
- }
59
- // Scoping filters
60
- if (config.areas) {
61
- process.env.EVAL_FILTER_AREAS = config.areas.join(",");
62
- }
63
- if (config.tasks) {
64
- process.env.EVAL_FILTER_TASKS = config.tasks.join(",");
65
- }
66
- }
@@ -1,17 +0,0 @@
1
- /**
2
- * Shell delegation for the fetch-docs step.
3
- *
4
- * Isolates the execSync call so it can be replaced when the pipeline
5
- * fully migrates to the DocFetcher port.
6
- */
7
- export interface ShellResult {
8
- ok: boolean;
9
- error?: string;
10
- }
11
- /**
12
- * Run `pnpm fetch-docs` via shell.
13
- *
14
- * Returns a result object instead of throwing so the step can
15
- * handle the failure uniformly.
16
- */
17
- export declare function runFetchDocsShell(rootDir: string, source?: string): ShellResult;