@sanity/ailf 2.0.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/LICENSE +21 -0
  2. package/dist/cli.js +0 -0
  3. package/dist/orchestration/steps/run-eval-step.js +1 -1
  4. package/dist/pipeline/checks.d.ts +8 -3
  5. package/dist/pipeline/checks.js +23 -3
  6. package/package.json +25 -25
  7. package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.d.ts +0 -10
  8. package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.js +0 -185
  9. package/dist/_vendor/ailf-core/artifact-capture/__tests__/noop-collector.test.d.ts +0 -6
  10. package/dist/_vendor/ailf-core/artifact-capture/__tests__/noop-collector.test.js +0 -42
  11. package/dist/_vendor/ailf-tasks/cli.d.ts +0 -8
  12. package/dist/_vendor/ailf-tasks/cli.js +0 -61
  13. package/dist/_vendor/ailf-tasks/index.d.ts +0 -13
  14. package/dist/_vendor/ailf-tasks/index.js +0 -16
  15. package/dist/_vendor/ailf-tasks/parser.d.ts +0 -27
  16. package/dist/_vendor/ailf-tasks/parser.js +0 -73
  17. package/dist/_vendor/ailf-tasks/schemas.d.ts +0 -198
  18. package/dist/_vendor/ailf-tasks/schemas.js +0 -180
  19. package/dist/_vendor/ailf-tasks/validation.d.ts +0 -47
  20. package/dist/_vendor/ailf-tasks/validation.js +0 -162
  21. package/dist/adapters/task-sources/yaml-task-source.d.ts +0 -18
  22. package/dist/adapters/task-sources/yaml-task-source.js +0 -139
  23. package/dist/agent-observer/test-imports.d.ts +0 -7
  24. package/dist/agent-observer/test-imports.js +0 -185
  25. package/dist/commands/update-quality-scores.d.ts +0 -5
  26. package/dist/commands/update-quality-scores.js +0 -20
  27. package/dist/lib/agent-behavior-report.d.ts +0 -8
  28. package/dist/lib/agent-behavior-report.js +0 -185
  29. package/dist/lib/baseline.d.ts +0 -19
  30. package/dist/lib/baseline.js +0 -153
  31. package/dist/lib/calculate-scores.d.ts +0 -23
  32. package/dist/lib/calculate-scores.js +0 -42
  33. package/dist/lib/compare.d.ts +0 -18
  34. package/dist/lib/compare.js +0 -170
  35. package/dist/lib/coverage-audit.d.ts +0 -4
  36. package/dist/lib/coverage-audit.js +0 -42
  37. package/dist/lib/discovery-report.d.ts +0 -13
  38. package/dist/lib/discovery-report.js +0 -57
  39. package/dist/lib/fetch-docs.d.ts +0 -30
  40. package/dist/lib/fetch-docs.js +0 -171
  41. package/dist/lib/generate-configs.d.ts +0 -25
  42. package/dist/lib/generate-configs.js +0 -42
  43. package/dist/lib/grader-api.d.ts +0 -21
  44. package/dist/lib/grader-api.js +0 -34
  45. package/dist/lib/grader-compare.d.ts +0 -19
  46. package/dist/lib/grader-compare.js +0 -91
  47. package/dist/lib/grader-consistency.d.ts +0 -27
  48. package/dist/lib/grader-consistency.js +0 -79
  49. package/dist/lib/grader-sensitivity.d.ts +0 -19
  50. package/dist/lib/grader-sensitivity.js +0 -75
  51. package/dist/lib/grader-validate.d.ts +0 -19
  52. package/dist/lib/grader-validate.js +0 -78
  53. package/dist/lib/measure-retrieval.d.ts +0 -14
  54. package/dist/lib/measure-retrieval.js +0 -71
  55. package/dist/lib/pr-comment.d.ts +0 -16
  56. package/dist/lib/pr-comment.js +0 -28
  57. package/dist/lib/readiness-report.d.ts +0 -13
  58. package/dist/lib/readiness-report.js +0 -108
  59. package/dist/lib/webhook-server.d.ts +0 -11
  60. package/dist/lib/webhook-server.js +0 -24
  61. package/dist/lib/weekly-digest.d.ts +0 -24
  62. package/dist/lib/weekly-digest.js +0 -148
  63. package/dist/orchestration/env-bridge.d.ts +0 -21
  64. package/dist/orchestration/env-bridge.js +0 -66
  65. package/dist/orchestration/steps/fetch-docs-shell.d.ts +0 -17
  66. package/dist/orchestration/steps/fetch-docs-shell.js +0 -30
  67. package/dist/pipeline/compiler/__tests__/task-bridge.test.d.ts +0 -9
  68. package/dist/pipeline/compiler/__tests__/task-bridge.test.js +0 -339
  69. package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.d.ts +0 -70
  70. package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.js +0 -485
  71. package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.d.ts +0 -76
  72. package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.js +0 -245
  73. package/dist/pipeline/compiler/mode-handlers/literacy-handler.d.ts +0 -89
  74. package/dist/pipeline/compiler/mode-handlers/literacy-handler.js +0 -379
  75. package/dist/pipeline/compiler/mode-handlers/mcp-assertions.d.ts +0 -50
  76. package/dist/pipeline/compiler/mode-handlers/mcp-assertions.js +0 -334
  77. package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.d.ts +0 -69
  78. package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.js +0 -307
  79. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.d.ts +0 -65
  80. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.js +0 -368
  81. package/dist/pipeline/compiler/task-bridge.d.ts +0 -41
  82. package/dist/pipeline/compiler/task-bridge.js +0 -92
  83. package/dist/pipeline/expand-tasks.d.ts +0 -232
  84. package/dist/pipeline/expand-tasks.js +0 -467
  85. package/dist/pipeline/generate-configs.d.ts +0 -92
  86. package/dist/pipeline/generate-configs.js +0 -445
  87. package/dist/pipeline/steps/calculate-scores-step.d.ts +0 -11
  88. package/dist/pipeline/steps/calculate-scores-step.js +0 -89
  89. package/dist/pipeline/steps/compare-step.d.ts +0 -18
  90. package/dist/pipeline/steps/compare-step.js +0 -90
  91. package/dist/pipeline/steps/eval-step.d.ts +0 -53
  92. package/dist/pipeline/steps/eval-step.js +0 -347
  93. package/dist/pipeline/steps/fetch-docs-step.d.ts +0 -11
  94. package/dist/pipeline/steps/fetch-docs-step.js +0 -84
  95. package/dist/pipeline/steps/generate-configs-step.d.ts +0 -11
  96. package/dist/pipeline/steps/generate-configs-step.js +0 -98
  97. package/dist/pipeline/steps/grader-consistency-step.d.ts +0 -21
  98. package/dist/pipeline/steps/grader-consistency-step.js +0 -74
  99. package/dist/pipeline/steps/publish-report-step.d.ts +0 -57
  100. package/dist/pipeline/steps/publish-report-step.js +0 -243
  101. package/dist/pipeline/steps/report-step.d.ts +0 -13
  102. package/dist/pipeline/steps/report-step.js +0 -56
  103. package/dist/pipeline/steps/update-scores-step.d.ts +0 -11
  104. package/dist/pipeline/steps/update-scores-step.js +0 -42
  105. package/dist/scripts/agent-behavior-report.d.ts +0 -19
  106. package/dist/scripts/agent-behavior-report.js +0 -315
  107. package/dist/scripts/baseline.d.ts +0 -43
  108. package/dist/scripts/baseline.js +0 -267
  109. package/dist/scripts/calculate-scores.d.ts +0 -166
  110. package/dist/scripts/calculate-scores.js +0 -1296
  111. package/dist/scripts/compare.d.ts +0 -22
  112. package/dist/scripts/compare.js +0 -334
  113. package/dist/scripts/coverage-audit.d.ts +0 -44
  114. package/dist/scripts/coverage-audit.js +0 -209
  115. package/dist/scripts/debug-eval.d.ts +0 -19
  116. package/dist/scripts/debug-eval.js +0 -73
  117. package/dist/scripts/discovery-report.d.ts +0 -58
  118. package/dist/scripts/discovery-report.js +0 -250
  119. package/dist/scripts/fetch-docs.d.ts +0 -35
  120. package/dist/scripts/fetch-docs.js +0 -472
  121. package/dist/scripts/generate-configs.d.ts +0 -66
  122. package/dist/scripts/generate-configs.js +0 -459
  123. package/dist/scripts/grader-api.d.ts +0 -27
  124. package/dist/scripts/grader-api.js +0 -206
  125. package/dist/scripts/grader-compare.d.ts +0 -22
  126. package/dist/scripts/grader-compare.js +0 -368
  127. package/dist/scripts/grader-consistency.d.ts +0 -20
  128. package/dist/scripts/grader-consistency.js +0 -313
  129. package/dist/scripts/grader-sensitivity.d.ts +0 -22
  130. package/dist/scripts/grader-sensitivity.js +0 -354
  131. package/dist/scripts/grader-validate.d.ts +0 -19
  132. package/dist/scripts/grader-validate.js +0 -267
  133. package/dist/scripts/measure-retrieval.d.ts +0 -10
  134. package/dist/scripts/measure-retrieval.js +0 -145
  135. package/dist/scripts/migrate-tasks-to-content-lake.d.ts +0 -24
  136. package/dist/scripts/migrate-tasks-to-content-lake.js +0 -328
  137. package/dist/scripts/pipeline.d.ts +0 -76
  138. package/dist/scripts/pipeline.js +0 -1031
  139. package/dist/scripts/pr-comment.d.ts +0 -10
  140. package/dist/scripts/pr-comment.js +0 -510
  141. package/dist/scripts/readiness-report.d.ts +0 -88
  142. package/dist/scripts/readiness-report.js +0 -342
  143. package/dist/scripts/update-quality-scores.d.ts +0 -15
  144. package/dist/scripts/update-quality-scores.js +0 -184
  145. package/dist/scripts/validate-task-sources.d.ts +0 -21
  146. package/dist/scripts/validate-task-sources.js +0 -210
  147. package/dist/scripts/validate.d.ts +0 -13
  148. package/dist/scripts/validate.js +0 -79
  149. package/dist/scripts/webhook-server.d.ts +0 -26
  150. package/dist/scripts/webhook-server.js +0 -147
  151. package/dist/scripts/weekly-digest.d.ts +0 -24
  152. package/dist/scripts/weekly-digest.js +0 -144
  153. package/dist/sinks/format-slack.d.ts +0 -64
  154. package/dist/sinks/format-slack.js +0 -306
  155. package/dist/sinks/slack-sink.d.ts +0 -27
  156. package/dist/sinks/slack-sink.js +0 -78
  157. package/dist/sinks/webhook-sink.d.ts +0 -19
  158. package/dist/sinks/webhook-sink.js +0 -50
  159. package/tasks/.expanded.agentic.yaml +0 -280
  160. package/tasks/.expanded.yaml +0 -565
@@ -1,78 +0,0 @@
1
- /**
2
- * lib/grader-validate.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/grader-validate-runner.ts.
5
- * The pure analysis functions live in pipeline/grader-validation.ts.
6
- * This shim preserves backward compatibility for direct CLI invocation.
7
- *
8
- * TODO: Update all importers to use pipeline/ modules, then delete this file.
9
- *
10
- * @deprecated Import from ../pipeline/grader-validate-runner.js instead.
11
- */
12
- // oxlint-disable-next-line import/no-unassigned-import -- side-effect: loads .env into process.env
13
- import "dotenv/config";
14
- import { dirname, resolve } from "path";
15
- import { fileURLToPath } from "url";
16
- // Re-export from pipeline modules
17
- export { formatValidationReport, runGraderValidate, } from "../pipeline/grader-validate-runner.js";
18
- export { classifyCorrelation, validateGrader, } from "../pipeline/grader-validation.js";
19
- import { runGraderValidate } from "../pipeline/grader-validate-runner.js";
20
- const __dirname = dirname(fileURLToPath(import.meta.url));
21
- const ROOT = resolve(__dirname, "..", "..");
22
- // ---------------------------------------------------------------------------
23
- // CLI argument parsing
24
- // ---------------------------------------------------------------------------
25
- function parseCliArgs() {
26
- const args = process.argv.slice(2);
27
- function getOption(name) {
28
- const idx = args.indexOf(`--${name}`);
29
- return idx !== -1 && idx + 1 < args.length ? args[idx + 1] : undefined;
30
- }
31
- function getFlag(name) {
32
- return args.includes(`--${name}`);
33
- }
34
- const showHelp = getFlag("help") || getFlag("h");
35
- if (showHelp) {
36
- console.log(`
37
- Usage: pnpm grader-validate [options]
38
-
39
- Validate grader accuracy against human reference grades.
40
-
41
- Options:
42
- --grader <model> Grader model to validate (default: from config/models.yaml)
43
- --threshold <n> MAE threshold for pass/fail (default: 10)
44
- --help, -h Show this help
45
- `);
46
- process.exit(0);
47
- }
48
- const thresholdStr = getOption("threshold");
49
- return {
50
- graderOverride: getOption("grader"),
51
- maeThreshold: thresholdStr ? parseFloat(thresholdStr) : 10,
52
- };
53
- }
54
- /**
55
- * Legacy main() entry point.
56
- * @deprecated Use runGraderValidate() from pipeline/grader-validate-runner.js instead.
57
- */
58
- export async function main() {
59
- const { graderOverride, maeThreshold } = parseCliArgs();
60
- const result = await runGraderValidate({
61
- graderModel: graderOverride,
62
- maeThreshold,
63
- rootDir: ROOT,
64
- });
65
- // Exit with error code if threshold not met
66
- if (!result.passesThreshold) {
67
- console.error(`\n ❌ VALIDATION FAILED: MAE ${result.overallMae} exceeds threshold ${maeThreshold}`);
68
- process.exit(1);
69
- }
70
- }
71
- // Only run when invoked directly
72
- if (process.argv[1]?.endsWith("grader-validate.ts") ||
73
- process.argv[1]?.endsWith("grader-validate.js")) {
74
- main().catch((err) => {
75
- console.error("❌ Fatal error:", err);
76
- process.exit(1);
77
- });
78
- }
@@ -1,14 +0,0 @@
1
- /**
2
- * lib/measure-retrieval.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/measure-retrieval.ts.
5
- * This shim preserves backward compatibility for direct CLI invocation.
6
- *
7
- * TODO: Update all importers to use pipeline/measure-retrieval.ts, then delete this file.
8
- *
9
- * @deprecated Import from ../pipeline/measure-retrieval.js instead.
10
- */
11
- import "dotenv/config";
12
- export { calculateNDCG, calculateRecall, formatRetrievalTable, measureRetrieval, type MeasureRetrievalOptions, type RetrievalResult, type RetrievalSummary, type RetrieverFn, } from "../pipeline/measure-retrieval.js";
13
- /** @deprecated Use measureRetrieval() from pipeline/measure-retrieval.js instead. */
14
- export declare function main(): Promise<void>;
@@ -1,71 +0,0 @@
1
- /**
2
- * lib/measure-retrieval.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/measure-retrieval.ts.
5
- * This shim preserves backward compatibility for direct CLI invocation.
6
- *
7
- * TODO: Update all importers to use pipeline/measure-retrieval.ts, then delete this file.
8
- *
9
- * @deprecated Import from ../pipeline/measure-retrieval.js instead.
10
- */
11
- // oxlint-disable-next-line import/no-unassigned-import -- side-effect: loads .env into process.env
12
- import "dotenv/config";
13
- import { writeFileSync, mkdirSync } from "fs";
14
- import { join, dirname } from "path";
15
- import { getSanityClient } from "../sanity/client.js";
16
- import { formatRetrievalTable, measureRetrieval, } from "../pipeline/measure-retrieval.js";
17
- // Re-export pipeline types and functions
18
- export { calculateNDCG, calculateRecall, formatRetrievalTable, measureRetrieval, } from "../pipeline/measure-retrieval.js";
19
- // ---------------------------------------------------------------------------
20
- // Sanity text search retriever (side-effecting — uses Sanity client)
21
- // ---------------------------------------------------------------------------
22
- async function retrieveDocsForQuery(query, k = 10) {
23
- const client = getSanityClient();
24
- const results = await client.fetch(`
25
- *[_type == "article" && !(_id in path("drafts.**"))]
26
- | score(
27
- boost(title match $query, 3),
28
- boost(pt::text(content) match $query, 1)
29
- )
30
- | order(_score desc)
31
- [0...$k] {
32
- "slug": slug.current,
33
- _score
34
- }
35
- `, { k, query });
36
- return results.map((r) => r.slug);
37
- }
38
- // ---------------------------------------------------------------------------
39
- // Legacy main() entry point
40
- // ---------------------------------------------------------------------------
41
- /** @deprecated Use measureRetrieval() from pipeline/measure-retrieval.js instead. */
42
- export async function main() {
43
- console.log("=== Sanity AI Literacy — Retrieval Quality Measurement ===\n");
44
- const ROOT = join(dirname(new URL(import.meta.url).pathname), "..", "..");
45
- const summary = await measureRetrieval({
46
- onProgress: (area, taskId, result) => {
47
- console.log(` ${taskId}:`);
48
- console.log(` Recall@5: ${(result.recall_at_5 * 100).toFixed(1)}%`);
49
- console.log(` Recall@10: ${(result.recall_at_10 * 100).toFixed(1)}%`);
50
- console.log(` NDCG@10: ${(result.ndcg_at_10 * 100).toFixed(1)}%`);
51
- },
52
- retriever: retrieveDocsForQuery,
53
- rootDir: ROOT,
54
- });
55
- // Print summary
56
- console.log();
57
- console.log(formatRetrievalTable(summary));
58
- // Persist results
59
- const outDir = join(ROOT, "results", "latest");
60
- mkdirSync(outDir, { recursive: true });
61
- writeFileSync(join(outDir, "retrieval-results.json"), JSON.stringify(summary, null, 2));
62
- console.log("\nResults written to results/latest/retrieval-results.json");
63
- }
64
- // Only run when invoked directly (not when imported)
65
- if (process.argv[1]?.endsWith("measure-retrieval.ts") ||
66
- process.argv[1]?.endsWith("measure-retrieval.js")) {
67
- main().catch((err) => {
68
- console.error("Fatal error:", err);
69
- process.exit(1);
70
- });
71
- }
@@ -1,16 +0,0 @@
1
- /**
2
- * lib/pr-comment.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/pr-comment.ts.
5
- *
6
- * @deprecated Import from ../pipeline/pr-comment.js instead.
7
- */
8
- export { generatePrComment, type PrCommentOptions, } from "../pipeline/pr-comment.js";
9
- import type { PrCommentOptions } from "../pipeline/pr-comment.js";
10
- /**
11
- * Legacy main() entry point.
12
- * @deprecated Use generatePrComment() from pipeline/pr-comment.ts instead.
13
- */
14
- export declare function main(options?: Omit<PrCommentOptions, "rootDir"> & {
15
- rootDir?: string;
16
- }): void;
@@ -1,28 +0,0 @@
1
- /**
2
- * lib/pr-comment.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/pr-comment.ts.
5
- *
6
- * @deprecated Import from ../pipeline/pr-comment.js instead.
7
- */
8
- import { dirname, resolve } from "path";
9
- import { fileURLToPath } from "url";
10
- export { generatePrComment, } from "../pipeline/pr-comment.js";
11
- import { generatePrComment } from "../pipeline/pr-comment.js";
12
- const __dirname = dirname(fileURLToPath(import.meta.url));
13
- const ROOT = resolve(__dirname, "..", "..");
14
- /**
15
- * Legacy main() entry point.
16
- * @deprecated Use generatePrComment() from pipeline/pr-comment.ts instead.
17
- */
18
- export function main(options) {
19
- generatePrComment({
20
- rootDir: options?.rootDir ?? ROOT,
21
- outputPath: options?.outputPath,
22
- promptfooUrl: options?.promptfooUrl,
23
- });
24
- }
25
- if (process.argv[1]?.endsWith("pr-comment.ts") ||
26
- process.argv[1]?.endsWith("pr-comment.js")) {
27
- main();
28
- }
@@ -1,13 +0,0 @@
1
- /**
2
- * lib/readiness-report.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/readiness-report.ts.
5
- *
6
- * @deprecated Import from ../pipeline/readiness-report.js instead.
7
- */
8
- export { formatReadinessMarkdown, generateReadinessReport, type DimensionCheck, type HistoryEntry, type ReadinessReport, } from "../pipeline/readiness-report.js";
9
- /**
10
- * Legacy main() entry point.
11
- * @deprecated Use generateReadinessReport() + formatReadinessMarkdown() directly.
12
- */
13
- export declare function main(): void;
@@ -1,108 +0,0 @@
1
- /**
2
- * lib/readiness-report.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/readiness-report.ts.
5
- *
6
- * @deprecated Import from ../pipeline/readiness-report.js instead.
7
- */
8
- import { existsSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
9
- import { dirname, join, resolve } from "node:path";
10
- import { fileURLToPath } from "node:url";
11
- import { load } from "js-yaml";
12
- import { ThresholdConfigSchema, } from "../pipeline/schemas.js";
13
- export { formatReadinessMarkdown, generateReadinessReport, } from "../pipeline/readiness-report.js";
14
- import { generateReadinessReport, formatReadinessMarkdown, } from "../pipeline/readiness-report.js";
15
- const __dirname = dirname(fileURLToPath(import.meta.url));
16
- const ROOT = resolve(__dirname, "..", "..");
17
- const SCORE_SUMMARY_PATH = join(ROOT, "results", "latest", "score-summary.json");
18
- const GAP_ANALYSIS_PATH = join(ROOT, "results", "latest", "gap-analysis.json");
19
- const THRESHOLDS_PATH = join(ROOT, "config", "thresholds.yaml");
20
- const BASELINES_DIR = join(ROOT, "results", "baselines");
21
- /**
22
- * Legacy main() entry point.
23
- * @deprecated Use generateReadinessReport() + formatReadinessMarkdown() directly.
24
- */
25
- export function main() {
26
- const args = process.argv.slice(2);
27
- let area;
28
- let history = false;
29
- let output;
30
- for (let i = 0; i < args.length; i++) {
31
- const arg = args[i];
32
- if (arg === "--area" && i + 1 < args.length) {
33
- area = args[++i];
34
- }
35
- else if (arg === "--history") {
36
- history = true;
37
- }
38
- else if (arg === "--output" && i + 1 < args.length) {
39
- output = args[++i];
40
- }
41
- }
42
- if (!area) {
43
- console.error("Usage: readiness-report --area <area> [--history] [--output <file>]");
44
- process.exit(1);
45
- }
46
- // Load data
47
- if (!existsSync(SCORE_SUMMARY_PATH)) {
48
- throw new Error(`Score summary not found at ${SCORE_SUMMARY_PATH}. Run \`pnpm pipeline\` first.`);
49
- }
50
- const scoreSummary = JSON.parse(readFileSync(SCORE_SUMMARY_PATH, "utf-8"));
51
- if (!existsSync(THRESHOLDS_PATH)) {
52
- throw new Error(`Threshold config not found at ${THRESHOLDS_PATH}.`);
53
- }
54
- const rawThresholds = readFileSync(THRESHOLDS_PATH, "utf-8");
55
- const thresholdConfig = ThresholdConfigSchema.parse(load(rawThresholds));
56
- let gapAnalysis;
57
- if (existsSync(GAP_ANALYSIS_PATH)) {
58
- gapAnalysis = JSON.parse(readFileSync(GAP_ANALYSIS_PATH, "utf-8"));
59
- }
60
- const historyEntries = [];
61
- if (history && existsSync(BASELINES_DIR)) {
62
- const files = readdirSync(BASELINES_DIR)
63
- .filter((f) => f.endsWith(".json"))
64
- .sort();
65
- for (const file of files) {
66
- try {
67
- const raw = readFileSync(join(BASELINES_DIR, file), "utf-8");
68
- const data = JSON.parse(raw);
69
- const areaScore = data.scores?.find((s) => s.feature === area);
70
- if (!areaScore)
71
- continue;
72
- const nameWithoutExt = file.replace(/\.json$/, "");
73
- const parts = nameWithoutExt.split("_");
74
- const tag = parts.length > 4 ? parts.slice(4).join("_") : undefined;
75
- historyEntries.push({
76
- score: areaScore.totalScore,
77
- tag,
78
- timestamp: data.timestamp ?? nameWithoutExt,
79
- });
80
- }
81
- catch {
82
- // Skip malformed baseline files
83
- }
84
- }
85
- }
86
- const report = generateReadinessReport({
87
- area,
88
- gapAnalysis,
89
- history: historyEntries,
90
- scoreSummary,
91
- thresholdConfig,
92
- });
93
- const markdown = formatReadinessMarkdown(report);
94
- if (output) {
95
- writeFileSync(output, markdown, "utf-8");
96
- console.error(`✅ Readiness report written to ${output}`);
97
- }
98
- else {
99
- console.log(markdown);
100
- }
101
- if (!report.pass) {
102
- process.exit(1);
103
- }
104
- }
105
- if (process.argv[1]?.endsWith("readiness-report.ts") ||
106
- process.argv[1]?.endsWith("readiness-report.js")) {
107
- main();
108
- }
@@ -1,11 +0,0 @@
1
- /**
2
- * lib/webhook-server.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/webhook-server.ts.
5
- * This shim preserves backward compatibility for direct CLI invocation.
6
- *
7
- * TODO: Update all importers to use pipeline/webhook-server.ts, then delete this file.
8
- *
9
- * @deprecated Import from ../pipeline/webhook-server.js instead.
10
- */
11
- export { startWebhookServer, type WebhookServerOptions, } from "../pipeline/webhook-server.js";
@@ -1,24 +0,0 @@
1
- /**
2
- * lib/webhook-server.ts — DEPRECATED re-export shim.
3
- *
4
- * The real implementation has moved to pipeline/webhook-server.ts.
5
- * This shim preserves backward compatibility for direct CLI invocation.
6
- *
7
- * TODO: Update all importers to use pipeline/webhook-server.ts, then delete this file.
8
- *
9
- * @deprecated Import from ../pipeline/webhook-server.js instead.
10
- */
11
- import { dirname, resolve } from "path";
12
- import { fileURLToPath } from "url";
13
- import { startWebhookServer } from "../pipeline/webhook-server.js";
14
- export { startWebhookServer, } from "../pipeline/webhook-server.js";
15
- const __dirname = dirname(fileURLToPath(import.meta.url));
16
- const ROOT = resolve(__dirname, "..", "..");
17
- // When imported directly (e.g., `tsx src/lib/webhook-server.ts`), start the server
18
- startWebhookServer({
19
- dailyBudget: parseInt(process.env.WEBHOOK_DAILY_BUDGET ?? "20", 10),
20
- debounceMs: parseInt(process.env.WEBHOOK_DEBOUNCE_MS ?? "10000", 10),
21
- githubToken: process.env.GITHUB_TOKEN ?? "",
22
- port: parseInt(process.env.WEBHOOK_PORT ?? "3333", 10),
23
- rootDir: ROOT,
24
- });
@@ -1,24 +0,0 @@
1
- /**
2
- * weekly-digest.ts
3
- *
4
- * CLI script to generate and deliver a weekly evaluation digest.
5
- *
6
- * Queries the Sanity Content Lake for all reports within the configured
7
- * lookback window, computes trend analysis, and delivers the digest
8
- * via configured channels (Slack, stdout, or both).
9
- *
10
- * Usage:
11
- * pnpm weekly-digest # send to configured Slack webhook
12
- * pnpm weekly-digest --dry-run # print to stdout only
13
- * pnpm weekly-digest --lookback 14 # 14-day lookback window
14
- * pnpm weekly-digest --json # output raw JSON
15
- *
16
- * Environment variables:
17
- * SLACK_WEBHOOK_URL — Slack incoming webhook URL
18
- * SANITY_API_TOKEN — Sanity read token
19
- * AILF_TRIGGER_TYPE — set to "scheduled" by the cron workflow
20
- * AILF_SCHEDULE — the schedule name (e.g., "weekly-digest")
21
- *
22
- * @see docs/design-docs/report-store/implementation.md — Phase 5
23
- */
24
- export declare function main(): Promise<void>;
@@ -1,148 +0,0 @@
1
- /**
2
- * weekly-digest.ts
3
- *
4
- * CLI script to generate and deliver a weekly evaluation digest.
5
- *
6
- * Queries the Sanity Content Lake for all reports within the configured
7
- * lookback window, computes trend analysis, and delivers the digest
8
- * via configured channels (Slack, stdout, or both).
9
- *
10
- * Usage:
11
- * pnpm weekly-digest # send to configured Slack webhook
12
- * pnpm weekly-digest --dry-run # print to stdout only
13
- * pnpm weekly-digest --lookback 14 # 14-day lookback window
14
- * pnpm weekly-digest --json # output raw JSON
15
- *
16
- * Environment variables:
17
- * SLACK_WEBHOOK_URL — Slack incoming webhook URL
18
- * SANITY_API_TOKEN — Sanity read token
19
- * AILF_TRIGGER_TYPE — set to "scheduled" by the cron workflow
20
- * AILF_SCHEDULE — the schedule name (e.g., "weekly-digest")
21
- *
22
- * @see docs/design-docs/report-store/implementation.md — Phase 5
23
- */
24
- import { config as dotenvConfig } from "dotenv";
25
- import { existsSync } from "fs";
26
- import { dirname, resolve } from "path";
27
- import { fileURLToPath } from "url";
28
- import { generateDigest } from "../schedules/digest.js";
29
- import { getDigestConfig } from "../schedules/loader.js";
30
- import { formatWeeklyDigest } from "../sinks/slack/format.js";
31
- // Load root .env (same override behavior as pipeline.ts)
32
- const __dirname = dirname(fileURLToPath(import.meta.url));
33
- const envPath = resolve(__dirname, "..", "..", "..", "..", ".env");
34
- if (existsSync(envPath)) {
35
- dotenvConfig({ override: true, path: envPath });
36
- }
37
- // ---------------------------------------------------------------------------
38
- // CLI argument parsing
39
- // ---------------------------------------------------------------------------
40
- const args = process.argv.slice(2);
41
- function getOption(name) {
42
- const idx = args.indexOf(`--${name}`);
43
- return idx >= 0 && idx + 1 < args.length ? args[idx + 1] : undefined;
44
- }
45
- function hasFlag(name) {
46
- return args.includes(`--${name}`);
47
- }
48
- const DRY_RUN = hasFlag("dry-run");
49
- const JSON_OUTPUT = hasFlag("json");
50
- const lookbackOverride = getOption("lookback");
51
- // ---------------------------------------------------------------------------
52
- // Main
53
- // ---------------------------------------------------------------------------
54
- export async function main() {
55
- console.log();
56
- console.log("=== AI Literacy Weekly Digest ===");
57
- console.log();
58
- // Load digest config
59
- const digestConfig = getDigestConfig();
60
- const lookbackDays = lookbackOverride
61
- ? parseInt(lookbackOverride, 10)
62
- : (digestConfig?.lookbackDays ?? 7);
63
- console.log(` Lookback: ${lookbackDays} days`);
64
- console.log(` Mode: ${DRY_RUN ? "dry run (stdout only)" : "live"}`);
65
- console.log();
66
- // Generate digest — uses AILF_REPORT_* env vars for report store access,
67
- // independent of SANITY_DATASET/SANITY_PROJECT_ID (which control doc evaluation)
68
- const digest = await generateDigest({
69
- dataset: process.env.AILF_REPORT_DATASET,
70
- lookbackDays,
71
- projectId: process.env.AILF_REPORT_PROJECT_ID,
72
- token: process.env.AILF_REPORT_SANITY_API_TOKEN ?? process.env.SANITY_API_TOKEN,
73
- });
74
- if (!digest) {
75
- console.log(" No reports found in the lookback window. Nothing to send.");
76
- process.exit(0);
77
- }
78
- // Output
79
- console.log(` Reports found: ${digest.reportCount}`);
80
- console.log(` Overall: ${Math.round(digest.overallLatest)} (${digest.overallTrend})`);
81
- console.log(` Improved: ${digest.improved.join(", ") || "none"}`);
82
- console.log(` Regressed: ${digest.regressed.join(", ") || "none"}`);
83
- console.log(` Stable: ${digest.stable.join(", ") || "none"}`);
84
- console.log();
85
- if (JSON_OUTPUT) {
86
- console.log(JSON.stringify(digest, null, 2));
87
- return;
88
- }
89
- // Format for Slack
90
- const message = formatWeeklyDigest(digest);
91
- if (DRY_RUN) {
92
- console.log(" --- Slack Message Preview ---");
93
- console.log(` Text: ${message.text}`);
94
- console.log();
95
- for (const block of message.blocks) {
96
- if (block.text) {
97
- console.log(` [${block.type}] ${block.text.text}`);
98
- }
99
- if (block.fields) {
100
- for (const field of block.fields) {
101
- console.log(` [field] ${field.text}`);
102
- }
103
- }
104
- if (block.elements) {
105
- for (const el of block.elements) {
106
- console.log(` [element] ${el.text}`);
107
- }
108
- }
109
- }
110
- console.log();
111
- return;
112
- }
113
- // Deliver via Slack
114
- const webhookUrl = digestConfig?.slackWebhookUrl ?? process.env.SLACK_WEBHOOK_URL;
115
- if (!webhookUrl) {
116
- console.warn(" ⚠️ No Slack webhook URL configured. Set SLACK_WEBHOOK_URL or configure in schedules.yaml");
117
- console.log(" Printing digest to stdout instead:");
118
- console.log();
119
- console.log(` ${message.text}`);
120
- return;
121
- }
122
- console.log(" Sending to Slack...");
123
- try {
124
- const response = await fetch(webhookUrl, {
125
- body: JSON.stringify(message),
126
- headers: { "Content-Type": "application/json" },
127
- method: "POST",
128
- });
129
- if (response.ok) {
130
- console.log(" ✅ Digest sent successfully");
131
- }
132
- else {
133
- const text = await response.text();
134
- console.warn(` ⚠️ Slack delivery failed: ${response.status} ${text}`);
135
- }
136
- }
137
- catch (error) {
138
- console.warn(` ⚠️ Slack delivery error: ${error instanceof Error ? error.message : String(error)}`);
139
- }
140
- }
141
- // Only run when invoked directly (not when imported)
142
- if (process.argv[1]?.endsWith("weekly-digest.ts") ||
143
- process.argv[1]?.endsWith("weekly-digest.js")) {
144
- main().catch((error) => {
145
- console.error("Fatal error:", error);
146
- process.exit(1);
147
- });
148
- }
@@ -1,21 +0,0 @@
1
- /**
2
- * Environment variable bridge — writes ResolvedConfig values to process.env
3
- * so that lib/*.ts modules (which still read process.env) work correctly.
4
- *
5
- * This replaces the former global applyEnvironment() with an explicit
6
- * per-step bridge. Each orchestration step calls this before invoking
7
- * its lib/*.ts main() function.
8
- *
9
- * Phase 9 will eliminate this file entirely by giving lib/*.ts main()
10
- * functions typed option parameters.
11
- *
12
- * @see docs/exec-plans/active/ports-and-adapters/phase-8-delete-legacy-step-layer.md
13
- */
14
- import type { ResolvedConfig } from "../_vendor/ailf-core/index.d.ts";
15
- /**
16
- * Bridge ResolvedConfig values to process.env.
17
- *
18
- * Idempotent — safe to call multiple times. Only sets env vars for
19
- * config values that are defined (never deletes or resets).
20
- */
21
- export declare function bridgeConfigToEnv(config: ResolvedConfig): void;
@@ -1,66 +0,0 @@
1
- /**
2
- * Environment variable bridge — writes ResolvedConfig values to process.env
3
- * so that lib/*.ts modules (which still read process.env) work correctly.
4
- *
5
- * This replaces the former global applyEnvironment() with an explicit
6
- * per-step bridge. Each orchestration step calls this before invoking
7
- * its lib/*.ts main() function.
8
- *
9
- * Phase 9 will eliminate this file entirely by giving lib/*.ts main()
10
- * functions typed option parameters.
11
- *
12
- * @see docs/exec-plans/active/ports-and-adapters/phase-8-delete-legacy-step-layer.md
13
- */
14
- /**
15
- * Bridge ResolvedConfig values to process.env.
16
- *
17
- * Idempotent — safe to call multiple times. Only sets env vars for
18
- * config values that are defined (never deletes or resets).
19
- */
20
- export function bridgeConfigToEnv(config) {
21
- // Mode
22
- process.env.EVAL_MODE = config.mode;
23
- // Search mode
24
- if (config.searchMode !== "open") {
25
- process.env.EVAL_SEARCH_MODE = config.searchMode;
26
- }
27
- // Source
28
- if (config.source) {
29
- process.env.DOC_SOURCE = config.source;
30
- }
31
- // URL-derived overrides
32
- if (config.urls?.[0]) {
33
- process.env.DOC_BASE_URL = config.urls[0];
34
- }
35
- // Sanity overrides
36
- if (config.datasetOverride) {
37
- process.env.SANITY_DATASET = config.datasetOverride;
38
- }
39
- if (config.projectIdOverride) {
40
- process.env.SANITY_PROJECT_ID = config.projectIdOverride;
41
- }
42
- if (config.perspectiveOverride) {
43
- process.env.SANITY_PERSPECTIVE = config.perspectiveOverride;
44
- }
45
- if (config.studioOriginOverride) {
46
- process.env.SANITY_STUDIO_ORIGIN = config.studioOriginOverride;
47
- }
48
- if (config.sanityDocumentArgs?.length) {
49
- process.env.SANITY_DOCUMENT_IDS = config.sanityDocumentArgs.join(",");
50
- }
51
- // Custom headers
52
- if (config.headers) {
53
- process.env.DOC_HEADERS = JSON.stringify(config.headers);
54
- }
55
- // Allowed origins
56
- if (config.allowedOrigins?.length) {
57
- process.env.DOC_ALLOWED_ORIGINS = config.allowedOrigins.join(",");
58
- }
59
- // Scoping filters
60
- if (config.areas) {
61
- process.env.EVAL_FILTER_AREAS = config.areas.join(",");
62
- }
63
- if (config.tasks) {
64
- process.env.EVAL_FILTER_TASKS = config.tasks.join(",");
65
- }
66
- }
@@ -1,17 +0,0 @@
1
- /**
2
- * Shell delegation for the fetch-docs step.
3
- *
4
- * Isolates the execSync call so it can be replaced when the pipeline
5
- * fully migrates to the DocFetcher port.
6
- */
7
- export interface ShellResult {
8
- ok: boolean;
9
- error?: string;
10
- }
11
- /**
12
- * Run `pnpm fetch-docs` via shell.
13
- *
14
- * Returns a result object instead of throwing so the step can
15
- * handle the failure uniformly.
16
- */
17
- export declare function runFetchDocsShell(rootDir: string, source?: string): ShellResult;