@sanity/ailf 2.0.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/LICENSE +21 -0
  2. package/dist/cli.js +0 -0
  3. package/dist/orchestration/steps/run-eval-step.js +1 -1
  4. package/dist/pipeline/checks.d.ts +8 -3
  5. package/dist/pipeline/checks.js +23 -3
  6. package/package.json +25 -25
  7. package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.d.ts +0 -10
  8. package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.js +0 -185
  9. package/dist/_vendor/ailf-core/artifact-capture/__tests__/noop-collector.test.d.ts +0 -6
  10. package/dist/_vendor/ailf-core/artifact-capture/__tests__/noop-collector.test.js +0 -42
  11. package/dist/_vendor/ailf-tasks/cli.d.ts +0 -8
  12. package/dist/_vendor/ailf-tasks/cli.js +0 -61
  13. package/dist/_vendor/ailf-tasks/index.d.ts +0 -13
  14. package/dist/_vendor/ailf-tasks/index.js +0 -16
  15. package/dist/_vendor/ailf-tasks/parser.d.ts +0 -27
  16. package/dist/_vendor/ailf-tasks/parser.js +0 -73
  17. package/dist/_vendor/ailf-tasks/schemas.d.ts +0 -198
  18. package/dist/_vendor/ailf-tasks/schemas.js +0 -180
  19. package/dist/_vendor/ailf-tasks/validation.d.ts +0 -47
  20. package/dist/_vendor/ailf-tasks/validation.js +0 -162
  21. package/dist/adapters/task-sources/yaml-task-source.d.ts +0 -18
  22. package/dist/adapters/task-sources/yaml-task-source.js +0 -139
  23. package/dist/agent-observer/test-imports.d.ts +0 -7
  24. package/dist/agent-observer/test-imports.js +0 -185
  25. package/dist/commands/update-quality-scores.d.ts +0 -5
  26. package/dist/commands/update-quality-scores.js +0 -20
  27. package/dist/lib/agent-behavior-report.d.ts +0 -8
  28. package/dist/lib/agent-behavior-report.js +0 -185
  29. package/dist/lib/baseline.d.ts +0 -19
  30. package/dist/lib/baseline.js +0 -153
  31. package/dist/lib/calculate-scores.d.ts +0 -23
  32. package/dist/lib/calculate-scores.js +0 -42
  33. package/dist/lib/compare.d.ts +0 -18
  34. package/dist/lib/compare.js +0 -170
  35. package/dist/lib/coverage-audit.d.ts +0 -4
  36. package/dist/lib/coverage-audit.js +0 -42
  37. package/dist/lib/discovery-report.d.ts +0 -13
  38. package/dist/lib/discovery-report.js +0 -57
  39. package/dist/lib/fetch-docs.d.ts +0 -30
  40. package/dist/lib/fetch-docs.js +0 -171
  41. package/dist/lib/generate-configs.d.ts +0 -25
  42. package/dist/lib/generate-configs.js +0 -42
  43. package/dist/lib/grader-api.d.ts +0 -21
  44. package/dist/lib/grader-api.js +0 -34
  45. package/dist/lib/grader-compare.d.ts +0 -19
  46. package/dist/lib/grader-compare.js +0 -91
  47. package/dist/lib/grader-consistency.d.ts +0 -27
  48. package/dist/lib/grader-consistency.js +0 -79
  49. package/dist/lib/grader-sensitivity.d.ts +0 -19
  50. package/dist/lib/grader-sensitivity.js +0 -75
  51. package/dist/lib/grader-validate.d.ts +0 -19
  52. package/dist/lib/grader-validate.js +0 -78
  53. package/dist/lib/measure-retrieval.d.ts +0 -14
  54. package/dist/lib/measure-retrieval.js +0 -71
  55. package/dist/lib/pr-comment.d.ts +0 -16
  56. package/dist/lib/pr-comment.js +0 -28
  57. package/dist/lib/readiness-report.d.ts +0 -13
  58. package/dist/lib/readiness-report.js +0 -108
  59. package/dist/lib/webhook-server.d.ts +0 -11
  60. package/dist/lib/webhook-server.js +0 -24
  61. package/dist/lib/weekly-digest.d.ts +0 -24
  62. package/dist/lib/weekly-digest.js +0 -148
  63. package/dist/orchestration/env-bridge.d.ts +0 -21
  64. package/dist/orchestration/env-bridge.js +0 -66
  65. package/dist/orchestration/steps/fetch-docs-shell.d.ts +0 -17
  66. package/dist/orchestration/steps/fetch-docs-shell.js +0 -30
  67. package/dist/pipeline/compiler/__tests__/task-bridge.test.d.ts +0 -9
  68. package/dist/pipeline/compiler/__tests__/task-bridge.test.js +0 -339
  69. package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.d.ts +0 -70
  70. package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.js +0 -485
  71. package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.d.ts +0 -76
  72. package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.js +0 -245
  73. package/dist/pipeline/compiler/mode-handlers/literacy-handler.d.ts +0 -89
  74. package/dist/pipeline/compiler/mode-handlers/literacy-handler.js +0 -379
  75. package/dist/pipeline/compiler/mode-handlers/mcp-assertions.d.ts +0 -50
  76. package/dist/pipeline/compiler/mode-handlers/mcp-assertions.js +0 -334
  77. package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.d.ts +0 -69
  78. package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.js +0 -307
  79. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.d.ts +0 -65
  80. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.js +0 -368
  81. package/dist/pipeline/compiler/task-bridge.d.ts +0 -41
  82. package/dist/pipeline/compiler/task-bridge.js +0 -92
  83. package/dist/pipeline/expand-tasks.d.ts +0 -232
  84. package/dist/pipeline/expand-tasks.js +0 -467
  85. package/dist/pipeline/generate-configs.d.ts +0 -92
  86. package/dist/pipeline/generate-configs.js +0 -445
  87. package/dist/pipeline/steps/calculate-scores-step.d.ts +0 -11
  88. package/dist/pipeline/steps/calculate-scores-step.js +0 -89
  89. package/dist/pipeline/steps/compare-step.d.ts +0 -18
  90. package/dist/pipeline/steps/compare-step.js +0 -90
  91. package/dist/pipeline/steps/eval-step.d.ts +0 -53
  92. package/dist/pipeline/steps/eval-step.js +0 -347
  93. package/dist/pipeline/steps/fetch-docs-step.d.ts +0 -11
  94. package/dist/pipeline/steps/fetch-docs-step.js +0 -84
  95. package/dist/pipeline/steps/generate-configs-step.d.ts +0 -11
  96. package/dist/pipeline/steps/generate-configs-step.js +0 -98
  97. package/dist/pipeline/steps/grader-consistency-step.d.ts +0 -21
  98. package/dist/pipeline/steps/grader-consistency-step.js +0 -74
  99. package/dist/pipeline/steps/publish-report-step.d.ts +0 -57
  100. package/dist/pipeline/steps/publish-report-step.js +0 -243
  101. package/dist/pipeline/steps/report-step.d.ts +0 -13
  102. package/dist/pipeline/steps/report-step.js +0 -56
  103. package/dist/pipeline/steps/update-scores-step.d.ts +0 -11
  104. package/dist/pipeline/steps/update-scores-step.js +0 -42
  105. package/dist/scripts/agent-behavior-report.d.ts +0 -19
  106. package/dist/scripts/agent-behavior-report.js +0 -315
  107. package/dist/scripts/baseline.d.ts +0 -43
  108. package/dist/scripts/baseline.js +0 -267
  109. package/dist/scripts/calculate-scores.d.ts +0 -166
  110. package/dist/scripts/calculate-scores.js +0 -1296
  111. package/dist/scripts/compare.d.ts +0 -22
  112. package/dist/scripts/compare.js +0 -334
  113. package/dist/scripts/coverage-audit.d.ts +0 -44
  114. package/dist/scripts/coverage-audit.js +0 -209
  115. package/dist/scripts/debug-eval.d.ts +0 -19
  116. package/dist/scripts/debug-eval.js +0 -73
  117. package/dist/scripts/discovery-report.d.ts +0 -58
  118. package/dist/scripts/discovery-report.js +0 -250
  119. package/dist/scripts/fetch-docs.d.ts +0 -35
  120. package/dist/scripts/fetch-docs.js +0 -472
  121. package/dist/scripts/generate-configs.d.ts +0 -66
  122. package/dist/scripts/generate-configs.js +0 -459
  123. package/dist/scripts/grader-api.d.ts +0 -27
  124. package/dist/scripts/grader-api.js +0 -206
  125. package/dist/scripts/grader-compare.d.ts +0 -22
  126. package/dist/scripts/grader-compare.js +0 -368
  127. package/dist/scripts/grader-consistency.d.ts +0 -20
  128. package/dist/scripts/grader-consistency.js +0 -313
  129. package/dist/scripts/grader-sensitivity.d.ts +0 -22
  130. package/dist/scripts/grader-sensitivity.js +0 -354
  131. package/dist/scripts/grader-validate.d.ts +0 -19
  132. package/dist/scripts/grader-validate.js +0 -267
  133. package/dist/scripts/measure-retrieval.d.ts +0 -10
  134. package/dist/scripts/measure-retrieval.js +0 -145
  135. package/dist/scripts/migrate-tasks-to-content-lake.d.ts +0 -24
  136. package/dist/scripts/migrate-tasks-to-content-lake.js +0 -328
  137. package/dist/scripts/pipeline.d.ts +0 -76
  138. package/dist/scripts/pipeline.js +0 -1031
  139. package/dist/scripts/pr-comment.d.ts +0 -10
  140. package/dist/scripts/pr-comment.js +0 -510
  141. package/dist/scripts/readiness-report.d.ts +0 -88
  142. package/dist/scripts/readiness-report.js +0 -342
  143. package/dist/scripts/update-quality-scores.d.ts +0 -15
  144. package/dist/scripts/update-quality-scores.js +0 -184
  145. package/dist/scripts/validate-task-sources.d.ts +0 -21
  146. package/dist/scripts/validate-task-sources.js +0 -210
  147. package/dist/scripts/validate.d.ts +0 -13
  148. package/dist/scripts/validate.js +0 -79
  149. package/dist/scripts/webhook-server.d.ts +0 -26
  150. package/dist/scripts/webhook-server.js +0 -147
  151. package/dist/scripts/weekly-digest.d.ts +0 -24
  152. package/dist/scripts/weekly-digest.js +0 -144
  153. package/dist/sinks/format-slack.d.ts +0 -64
  154. package/dist/sinks/format-slack.js +0 -306
  155. package/dist/sinks/slack-sink.d.ts +0 -27
  156. package/dist/sinks/slack-sink.js +0 -78
  157. package/dist/sinks/webhook-sink.d.ts +0 -19
  158. package/dist/sinks/webhook-sink.js +0 -50
  159. package/tasks/.expanded.agentic.yaml +0 -280
  160. package/tasks/.expanded.yaml +0 -565
@@ -1,210 +0,0 @@
1
- #!/usr/bin/env tsx
2
- /**
3
- * Validation script: Compare YamlTaskSource vs ContentLakeTaskSource
4
- *
5
- * Loads tasks from both sources and compares them field-by-field to verify
6
- * that the Content Lake migration produced identical LiteracyTaskDefinition[] output.
7
- *
8
- * This is Phase 3b of the tasks-as-content exec plan — parallel validation
9
- * before deleting YAML files.
10
- *
11
- * Usage:
12
- * cd packages/eval
13
- * npx tsx src/scripts/validate-task-sources.ts
14
- *
15
- * Prerequisites:
16
- * - Migration script has been run (ailf.task documents exist in CL)
17
- * - SANITY_API_TOKEN configured for Content Lake reads
18
- *
19
- * @see docs/archive/exec-plans/tasks-as-content/phase-3-migration.md
20
- */
21
- import { config as dotenvConfig } from "dotenv";
22
- import { existsSync } from "fs";
23
- import { dirname, resolve } from "path";
24
- import { fileURLToPath } from "url";
25
- import { isSlugRef } from "../_vendor/ailf-core/index.js";
26
- import { ContentLakeTaskSource } from "../adapters/task-sources/content-lake-task-source.js";
27
- import { YamlTaskSource } from "../adapters/task-sources/yaml-task-source.js";
28
- import { getSanityClient } from "../sanity/client.js";
29
- const __dirname = dirname(fileURLToPath(import.meta.url));
30
- const ROOT = resolve(__dirname, "..", "..");
31
- // Load .env from repository root (same as CLI entry point)
32
- const envPath = resolve(ROOT, "..", "..", ".env");
33
- if (existsSync(envPath)) {
34
- dotenvConfig({ override: true, path: envPath });
35
- }
36
- /**
37
- * Compare two LiteracyTaskDefinition arrays field-by-field.
38
- * Returns a list of differences.
39
- */
40
- function compareTasks(yamlTasks, clTasks) {
41
- const yamlMap = new Map(yamlTasks.map((t) => [t.id, t]));
42
- const clMap = new Map(clTasks.map((t) => [t.id, t]));
43
- const missingInCl = yamlTasks.filter((t) => !clMap.has(t.id)).map((t) => t.id);
44
- const missingInYaml = clTasks
45
- .filter((t) => !yamlMap.has(t.id))
46
- .map((t) => t.id);
47
- const diffs = [];
48
- for (const [id, yamlTask] of yamlMap) {
49
- const clTask = clMap.get(id);
50
- if (!clTask)
51
- continue;
52
- // Compare each field
53
- compareField(diffs, id, "title", yamlTask.title, clTask.title);
54
- compareField(diffs, id, "area", yamlTask.area, clTask.area);
55
- compareField(diffs, id, "docCoverage", yamlTask.docCoverage, clTask.docCoverage);
56
- // prompt.text — normalize whitespace for comparison (YAML multiline vs CL text)
57
- const yamlPrompt = (yamlTask.prompt?.text ?? "").trim();
58
- const clPrompt = (clTask.prompt?.text ?? "").trim();
59
- if (yamlPrompt !== clPrompt) {
60
- diffs.push({
61
- clValue: truncate(clPrompt, 100),
62
- field: "prompt.text",
63
- taskId: id,
64
- yamlValue: truncate(yamlPrompt, 100),
65
- });
66
- }
67
- // context.docs — compare slug sets (slug refs only for comparison)
68
- const yamlSlugs = (yamlTask.context?.docs ?? [])
69
- .filter(isSlugRef)
70
- .map((d) => d.slug)
71
- .sort();
72
- const clSlugs = (clTask.context?.docs ?? [])
73
- .filter(isSlugRef)
74
- .map((d) => d.slug)
75
- .sort();
76
- if (JSON.stringify(yamlSlugs) !== JSON.stringify(clSlugs)) {
77
- diffs.push({
78
- clValue: clSlugs,
79
- field: "canonicalDocs.slugs",
80
- taskId: id,
81
- yamlValue: yamlSlugs,
82
- });
83
- }
84
- // assertions — compare type + template arrays
85
- const yamlAssertTypes = (yamlTask.assertions ?? [])
86
- .map((a) => {
87
- const base = a.type;
88
- if ("template" in a)
89
- return `${base}:${a.template}`;
90
- return base;
91
- })
92
- .sort();
93
- const clAssertTypes = (clTask.assertions ?? [])
94
- .map((a) => {
95
- const base = a.type;
96
- if ("template" in a)
97
- return `${base}:${a.template}`;
98
- return base;
99
- })
100
- .sort();
101
- if (JSON.stringify(yamlAssertTypes) !== JSON.stringify(clAssertTypes)) {
102
- diffs.push({
103
- clValue: clAssertTypes,
104
- field: "assertions.types",
105
- taskId: id,
106
- yamlValue: yamlAssertTypes,
107
- });
108
- }
109
- // referenceSolution — CL always returns "" since solution content is a
110
- // separate document. YAML has a path. This is an intentional difference.
111
- // We only flag it if YAML also has no solution and CL somehow does.
112
- // (In practice, we expect all YAML tasks to have paths and CL to have "")
113
- }
114
- return { diffs, missingInCl, missingInYaml };
115
- }
116
- function compareField(diffs, taskId, field, yamlValue, clValue) {
117
- if (JSON.stringify(yamlValue) !== JSON.stringify(clValue)) {
118
- diffs.push({ clValue, field, taskId, yamlValue });
119
- }
120
- }
121
- function truncate(s, maxLen) {
122
- return s.length <= maxLen ? s : s.slice(0, maxLen) + "...";
123
- }
124
- // ---------------------------------------------------------------------------
125
- // Main validation
126
- // ---------------------------------------------------------------------------
127
- async function validate() {
128
- console.log("\n🔍 Task Source Validation");
129
- console.log("=".repeat(50));
130
- // Load from YAML
131
- console.log("\n1️⃣ Loading from YAML (tasks/*.yaml)...");
132
- const yamlSource = new YamlTaskSource(ROOT);
133
- const yamlTasks = (await yamlSource.loadTasks()).filter((t) => t.mode === "literacy");
134
- console.log(` Loaded ${yamlTasks.length} tasks`);
135
- // Load from Content Lake (use write token since task docs may require it)
136
- console.log("\n2️⃣ Loading from Content Lake (ailf.task)...");
137
- const client = getSanityClient({
138
- token: process.env.AILF_REPORT_SANITY_API_TOKEN ??
139
- process.env.SANITY_API_TOKEN ??
140
- undefined,
141
- });
142
- const clSource = new ContentLakeTaskSource(client);
143
- const clTasks = (await clSource.loadTasks()).filter((t) => t.mode === "literacy");
144
- console.log(` Loaded ${clTasks.length} tasks`);
145
- // Compare
146
- console.log("\n3️⃣ Comparing...");
147
- const { diffs, missingInCl, missingInYaml } = compareTasks(yamlTasks, clTasks);
148
- // Report
149
- console.log("\n" + "=".repeat(50));
150
- console.log("📊 Validation Results");
151
- console.log(` YAML tasks: ${yamlTasks.length}`);
152
- console.log(` CL tasks: ${clTasks.length}`);
153
- console.log(` Missing in CL: ${missingInCl.length}`);
154
- console.log(` Missing in YAML: ${missingInYaml.length}`);
155
- console.log(` Field diffs: ${diffs.length}`);
156
- if (missingInCl.length > 0) {
157
- console.log("\n ❌ Tasks in YAML but missing from Content Lake:");
158
- for (const id of missingInCl) {
159
- console.log(` - ${id}`);
160
- }
161
- }
162
- if (missingInYaml.length > 0) {
163
- console.log("\n ℹ️ Tasks in Content Lake but not in YAML (expected for new CL tasks):");
164
- for (const id of missingInYaml) {
165
- console.log(` - ${id}`);
166
- }
167
- }
168
- if (diffs.length > 0) {
169
- console.log("\n ⚠️ Field differences:");
170
- for (const d of diffs) {
171
- console.log(` ${d.taskId}.${d.field}:`);
172
- console.log(` YAML: ${JSON.stringify(d.yamlValue)}`);
173
- console.log(` CL: ${JSON.stringify(d.clValue)}`);
174
- }
175
- }
176
- // Known intentional differences:
177
- // - referenceSolution: CL returns "" (content is a separate document)
178
- // - canonicalDocs.slugs: CL may have fewer slugs if some didn't resolve
179
- // during migration (broken article references). This is actually an
180
- // improvement — broken refs are visible in Studio instead of silently
181
- // failing at fetch time.
182
- const INTENTIONAL_FIELDS = new Set([
183
- "referenceSolution",
184
- "canonicalDocs.slugs",
185
- ]);
186
- const intentional = diffs.filter((d) => INTENTIONAL_FIELDS.has(d.field));
187
- const unexpected = diffs.filter((d) => !INTENTIONAL_FIELDS.has(d.field));
188
- if (unexpected.length === 0 && missingInCl.length === 0) {
189
- console.log("\n ✅ All tasks match (ignoring known intentional differences)");
190
- if (intentional.length > 0) {
191
- console.log(` (${intentional.length} intentional diff(s):`);
192
- for (const d of intentional) {
193
- console.log(` - ${d.taskId}.${d.field}`);
194
- }
195
- console.log(" )");
196
- }
197
- }
198
- else {
199
- console.log("\n ❌ Validation FAILED — see differences above");
200
- process.exit(1);
201
- }
202
- console.log("");
203
- }
204
- // ---------------------------------------------------------------------------
205
- // Run
206
- // ---------------------------------------------------------------------------
207
- validate().catch((err) => {
208
- console.error("Validation failed:", err);
209
- process.exit(1);
210
- });
@@ -1,13 +0,0 @@
1
- /**
2
- * validate.ts
3
- *
4
- * CLI script that validates pipeline configuration.
5
- * Checks that all YAML files are consistent, all task-to-mapping
6
- * cross-references are valid, and reference solutions exist.
7
- *
8
- * Usage:
9
- * pnpm validate # validate everything
10
- * pnpm validate --strict # treat warnings as errors
11
- * pnpm validate --contexts # also check that context files exist
12
- */
13
- export {};
@@ -1,79 +0,0 @@
1
- /**
2
- * validate.ts
3
- *
4
- * CLI script that validates pipeline configuration.
5
- * Checks that all YAML files are consistent, all task-to-mapping
6
- * cross-references are valid, and reference solutions exist.
7
- *
8
- * Usage:
9
- * pnpm validate # validate everything
10
- * pnpm validate --strict # treat warnings as errors
11
- * pnpm validate --contexts # also check that context files exist
12
- */
13
- import { dirname, resolve } from "path";
14
- import { fileURLToPath } from "url";
15
- import { checkContextsExist, checkEnvironment } from "../pipeline/checks.js";
16
- import { validateConfiguration } from "../pipeline/validate.js";
17
- const __dirname = dirname(fileURLToPath(import.meta.url));
18
- const ROOT = resolve(__dirname, "..", "..");
19
- // ---------------------------------------------------------------------------
20
- // CLI argument parsing
21
- // ---------------------------------------------------------------------------
22
- const args = process.argv.slice(2);
23
- const strict = args.includes("--strict");
24
- const checkCtx = args.includes("--contexts");
25
- // ---------------------------------------------------------------------------
26
- // Run validation
27
- // ---------------------------------------------------------------------------
28
- console.log("=== ai-literacy-framework — Configuration Validator ===\n");
29
- const result = validateConfiguration(ROOT);
30
- // Optionally check contexts
31
- if (checkCtx) {
32
- // Dynamically import to get feature areas
33
- const { ALL_FEATURE_AREAS } = await import("../sanity/queries.js");
34
- const contextIssues = checkContextsExist(ROOT, ALL_FEATURE_AREAS);
35
- result.issues.push(...contextIssues);
36
- result.valid =
37
- result.valid && contextIssues.every((i) => i.severity !== "error");
38
- }
39
- // Check environment
40
- const envIssues = checkEnvironment(ROOT);
41
- result.issues.push(...envIssues);
42
- // ---------------------------------------------------------------------------
43
- // Report results
44
- // ---------------------------------------------------------------------------
45
- const errors = result.issues.filter((i) => i.severity === "error");
46
- const warnings = result.issues.filter((i) => i.severity === "warning");
47
- if (errors.length > 0) {
48
- console.log(`❌ ${errors.length} error(s):\n`);
49
- for (const issue of errors) {
50
- console.log(` ERROR [${issue.source}] ${issue.message}`);
51
- if (issue.path)
52
- console.log(` at ${issue.path}`);
53
- }
54
- console.log();
55
- }
56
- if (warnings.length > 0) {
57
- console.log(`⚠️ ${warnings.length} warning(s):\n`);
58
- for (const issue of warnings) {
59
- console.log(` WARN [${issue.source}] ${issue.message}`);
60
- if (issue.path)
61
- console.log(` at ${issue.path}`);
62
- }
63
- console.log();
64
- }
65
- if (errors.length === 0 && warnings.length === 0) {
66
- console.log("✅ All checks passed — configuration is valid.\n");
67
- }
68
- if (errors.length === 0 && warnings.length > 0) {
69
- console.log(`✅ Configuration is valid (${warnings.length} warning(s)).\n`);
70
- }
71
- // In strict mode, warnings are treated as errors
72
- const exitCode = strict
73
- ? result.issues.length > 0
74
- ? 1
75
- : 0
76
- : errors.length > 0
77
- ? 1
78
- : 0;
79
- process.exit(exitCode);
@@ -1,26 +0,0 @@
1
- /**
2
- * webhook-server.ts
3
- *
4
- * Local development server for testing the webhook handler.
5
- *
6
- * Starts an HTTP server that receives Sanity webhook payloads, processes
7
- * them through the WebhookHandler, and logs results. Useful for local
8
- * development and testing the full event-driven trigger flow.
9
- *
10
- * Usage:
11
- * pnpm webhook-server # start on port 3333
12
- * WEBHOOK_PORT=8080 pnpm webhook-server # custom port
13
- *
14
- * Test with curl:
15
- * curl -X POST http://localhost:3333/webhook \
16
- * -H "Content-Type: application/json" \
17
- * -d '{"operation":"update","result":{"_id":"abc","_type":"article","slug":{"current":"groq-introduction"}}}'
18
- *
19
- * Endpoints:
20
- * POST /webhook — handle a Sanity webhook payload
21
- * GET /health — handler diagnostics (budget, pending, tracked slugs)
22
- * GET /mappings — list all tracked document slugs and their areas
23
- *
24
- * @see docs/design-docs/report-store/visibility-workflows.md
25
- */
26
- export {};
@@ -1,147 +0,0 @@
1
- /**
2
- * webhook-server.ts
3
- *
4
- * Local development server for testing the webhook handler.
5
- *
6
- * Starts an HTTP server that receives Sanity webhook payloads, processes
7
- * them through the WebhookHandler, and logs results. Useful for local
8
- * development and testing the full event-driven trigger flow.
9
- *
10
- * Usage:
11
- * pnpm webhook-server # start on port 3333
12
- * WEBHOOK_PORT=8080 pnpm webhook-server # custom port
13
- *
14
- * Test with curl:
15
- * curl -X POST http://localhost:3333/webhook \
16
- * -H "Content-Type: application/json" \
17
- * -d '{"operation":"update","result":{"_id":"abc","_type":"article","slug":{"current":"groq-introduction"}}}'
18
- *
19
- * Endpoints:
20
- * POST /webhook — handle a Sanity webhook payload
21
- * GET /health — handler diagnostics (budget, pending, tracked slugs)
22
- * GET /mappings — list all tracked document slugs and their areas
23
- *
24
- * @see docs/design-docs/report-store/visibility-workflows.md
25
- */
26
- import { createServer } from "http";
27
- import { dirname, resolve } from "path";
28
- import { fileURLToPath } from "url";
29
- import { allTrackedSlugs, buildReverseMapping, } from "../pipeline/reverse-mapping.js";
30
- import { WebhookHandler } from "../webhook/handler.js";
31
- const __dirname = dirname(fileURLToPath(import.meta.url));
32
- const ROOT = resolve(__dirname, "..", "..");
33
- // ---------------------------------------------------------------------------
34
- // Configuration
35
- // ---------------------------------------------------------------------------
36
- const PORT = parseInt(process.env.WEBHOOK_PORT ?? "3333", 10);
37
- const GITHUB_TOKEN = process.env.GITHUB_TOKEN ?? "";
38
- const DRY_RUN = !GITHUB_TOKEN;
39
- // ---------------------------------------------------------------------------
40
- // Handler
41
- // ---------------------------------------------------------------------------
42
- const handler = new WebhookHandler({
43
- dailyBudget: parseInt(process.env.WEBHOOK_DAILY_BUDGET ?? "20", 10),
44
- debounceMs: parseInt(process.env.WEBHOOK_DEBOUNCE_MS ?? "10000", 10), // 10s for local dev
45
- githubToken: GITHUB_TOKEN,
46
- rootDir: ROOT,
47
- });
48
- // ---------------------------------------------------------------------------
49
- // HTTP Server
50
- // ---------------------------------------------------------------------------
51
- const server = createServer((req, res) => {
52
- void handleRequest(req, res);
53
- });
54
- async function handleRequest(req, res) {
55
- const url = new URL(req.url ?? "/", `http://localhost:${PORT}`);
56
- // CORS headers for local dev
57
- res.setHeader("Access-Control-Allow-Origin", "*");
58
- res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
59
- res.setHeader("Access-Control-Allow-Headers", "Content-Type");
60
- if (req.method === "OPTIONS") {
61
- res.writeHead(200);
62
- res.end();
63
- return;
64
- }
65
- // Health check
66
- if (url.pathname === "/health" && req.method === "GET") {
67
- const diagnostics = handler.diagnostics();
68
- res.writeHead(200, { "Content-Type": "application/json" });
69
- res.end(JSON.stringify({ dryRun: DRY_RUN, ...diagnostics }, null, 2));
70
- return;
71
- }
72
- // Mappings
73
- if (url.pathname === "/mappings" && req.method === "GET") {
74
- const reverseMapping = buildReverseMapping(ROOT);
75
- const slugs = allTrackedSlugs(reverseMapping);
76
- const mappings = Object.fromEntries(slugs.map((slug) => [slug, reverseMapping.get(slug)]));
77
- res.writeHead(200, { "Content-Type": "application/json" });
78
- res.end(JSON.stringify({ mappings, slugCount: slugs.length }, null, 2));
79
- return;
80
- }
81
- // Webhook handler
82
- if (url.pathname === "/webhook" && req.method === "POST") {
83
- const body = await readBody(req);
84
- let payload;
85
- try {
86
- payload = JSON.parse(body);
87
- }
88
- catch {
89
- res.writeHead(400, { "Content-Type": "application/json" });
90
- res.end(JSON.stringify({ error: "Invalid JSON" }));
91
- return;
92
- }
93
- if (DRY_RUN) {
94
- console.log("\n 🔶 DRY RUN — would dispatch (no GITHUB_TOKEN set)");
95
- }
96
- const result = handler.handle(payload);
97
- console.log(` → ${result.status}:`, JSON.stringify(result));
98
- res.writeHead(200, { "Content-Type": "application/json" });
99
- res.end(JSON.stringify(result, null, 2));
100
- return;
101
- }
102
- // 404
103
- res.writeHead(404, { "Content-Type": "application/json" });
104
- res.end(JSON.stringify({
105
- endpoints: ["POST /webhook", "GET /health", "GET /mappings"],
106
- error: "Not found",
107
- }));
108
- }
109
- // ---------------------------------------------------------------------------
110
- // Start
111
- // ---------------------------------------------------------------------------
112
- server.listen(PORT, () => {
113
- const reverseMapping = buildReverseMapping(ROOT);
114
- const slugCount = allTrackedSlugs(reverseMapping).length;
115
- console.log();
116
- console.log("=== AILF Webhook Server ===");
117
- console.log();
118
- console.log(` Port: ${PORT}`);
119
- console.log(` Mode: ${DRY_RUN ? "DRY RUN (set GITHUB_TOKEN to dispatch)" : "LIVE"}`);
120
- console.log(` Tracked slugs: ${slugCount}`);
121
- console.log(` Debounce: ${handler.diagnostics().pendingSlugs}`);
122
- console.log();
123
- console.log(" Endpoints:");
124
- console.log(` POST http://localhost:${PORT}/webhook — handle webhook`);
125
- console.log(` GET http://localhost:${PORT}/health — diagnostics`);
126
- console.log(` GET http://localhost:${PORT}/mappings — slug → area map`);
127
- console.log();
128
- });
129
- // Graceful shutdown
130
- process.on("SIGINT", () => {
131
- console.log("\n Shutting down — flushing debounce window...");
132
- void handler.shutdown().then(() => {
133
- server.close();
134
- process.exit(0);
135
- });
136
- });
137
- // ---------------------------------------------------------------------------
138
- // Helpers
139
- // ---------------------------------------------------------------------------
140
- function readBody(req) {
141
- return new Promise((resolve, reject) => {
142
- const chunks = [];
143
- req.on("data", (chunk) => chunks.push(chunk));
144
- req.on("end", () => resolve(Buffer.concat(chunks).toString()));
145
- req.on("error", reject);
146
- });
147
- }
@@ -1,24 +0,0 @@
1
- /**
2
- * weekly-digest.ts
3
- *
4
- * CLI script to generate and deliver a weekly evaluation digest.
5
- *
6
- * Queries the Sanity Content Lake for all reports within the configured
7
- * lookback window, computes trend analysis, and delivers the digest
8
- * via configured channels (Slack, stdout, or both).
9
- *
10
- * Usage:
11
- * pnpm weekly-digest # send to configured Slack webhook
12
- * pnpm weekly-digest --dry-run # print to stdout only
13
- * pnpm weekly-digest --lookback 14 # 14-day lookback window
14
- * pnpm weekly-digest --json # output raw JSON
15
- *
16
- * Environment variables:
17
- * SLACK_WEBHOOK_URL — Slack incoming webhook URL
18
- * SANITY_API_TOKEN — Sanity read token
19
- * AILF_TRIGGER_TYPE — set to "scheduled" by the cron workflow
20
- * AILF_SCHEDULE — the schedule name (e.g., "weekly-digest")
21
- *
22
- * @see docs/design-docs/report-store/implementation.md — Phase 5
23
- */
24
- export {};
@@ -1,144 +0,0 @@
1
- /**
2
- * weekly-digest.ts
3
- *
4
- * CLI script to generate and deliver a weekly evaluation digest.
5
- *
6
- * Queries the Sanity Content Lake for all reports within the configured
7
- * lookback window, computes trend analysis, and delivers the digest
8
- * via configured channels (Slack, stdout, or both).
9
- *
10
- * Usage:
11
- * pnpm weekly-digest # send to configured Slack webhook
12
- * pnpm weekly-digest --dry-run # print to stdout only
13
- * pnpm weekly-digest --lookback 14 # 14-day lookback window
14
- * pnpm weekly-digest --json # output raw JSON
15
- *
16
- * Environment variables:
17
- * SLACK_WEBHOOK_URL — Slack incoming webhook URL
18
- * SANITY_API_TOKEN — Sanity read token
19
- * AILF_TRIGGER_TYPE — set to "scheduled" by the cron workflow
20
- * AILF_SCHEDULE — the schedule name (e.g., "weekly-digest")
21
- *
22
- * @see docs/design-docs/report-store/implementation.md — Phase 5
23
- */
24
- import { config as dotenvConfig } from "dotenv";
25
- import { existsSync } from "fs";
26
- import { dirname, resolve } from "path";
27
- import { fileURLToPath } from "url";
28
- import { generateDigest } from "../schedules/digest.js";
29
- import { getDigestConfig } from "../schedules/loader.js";
30
- import { formatWeeklyDigest } from "../sinks/slack/format.js";
31
- // Load root .env (same override behavior as pipeline.ts)
32
- const __dirname = dirname(fileURLToPath(import.meta.url));
33
- const envPath = resolve(__dirname, "..", "..", "..", "..", ".env");
34
- if (existsSync(envPath)) {
35
- dotenvConfig({ override: true, path: envPath });
36
- }
37
- // ---------------------------------------------------------------------------
38
- // CLI argument parsing
39
- // ---------------------------------------------------------------------------
40
- const args = process.argv.slice(2);
41
- function getOption(name) {
42
- const idx = args.indexOf(`--${name}`);
43
- return idx >= 0 && idx + 1 < args.length ? args[idx + 1] : undefined;
44
- }
45
- function hasFlag(name) {
46
- return args.includes(`--${name}`);
47
- }
48
- const DRY_RUN = hasFlag("dry-run");
49
- const JSON_OUTPUT = hasFlag("json");
50
- const lookbackOverride = getOption("lookback");
51
- // ---------------------------------------------------------------------------
52
- // Main
53
- // ---------------------------------------------------------------------------
54
- async function main() {
55
- console.log();
56
- console.log("=== AI Literacy Weekly Digest ===");
57
- console.log();
58
- // Load digest config
59
- const digestConfig = getDigestConfig();
60
- const lookbackDays = lookbackOverride
61
- ? parseInt(lookbackOverride, 10)
62
- : (digestConfig?.lookbackDays ?? 7);
63
- console.log(` Lookback: ${lookbackDays} days`);
64
- console.log(` Mode: ${DRY_RUN ? "dry run (stdout only)" : "live"}`);
65
- console.log();
66
- // Generate digest — uses AILF_REPORT_* env vars for report store access,
67
- // independent of SANITY_DATASET/SANITY_PROJECT_ID (which control doc evaluation)
68
- const digest = await generateDigest({
69
- dataset: process.env.AILF_REPORT_DATASET,
70
- lookbackDays,
71
- projectId: process.env.AILF_REPORT_PROJECT_ID,
72
- token: process.env.AILF_REPORT_SANITY_API_TOKEN ?? process.env.SANITY_API_TOKEN,
73
- });
74
- if (!digest) {
75
- console.log(" No reports found in the lookback window. Nothing to send.");
76
- process.exit(0);
77
- }
78
- // Output
79
- console.log(` Reports found: ${digest.reportCount}`);
80
- console.log(` Overall: ${Math.round(digest.overallLatest)} (${digest.overallTrend})`);
81
- console.log(` Improved: ${digest.improved.join(", ") || "none"}`);
82
- console.log(` Regressed: ${digest.regressed.join(", ") || "none"}`);
83
- console.log(` Stable: ${digest.stable.join(", ") || "none"}`);
84
- console.log();
85
- if (JSON_OUTPUT) {
86
- console.log(JSON.stringify(digest, null, 2));
87
- return;
88
- }
89
- // Format for Slack
90
- const message = formatWeeklyDigest(digest);
91
- if (DRY_RUN) {
92
- console.log(" --- Slack Message Preview ---");
93
- console.log(` Text: ${message.text}`);
94
- console.log();
95
- for (const block of message.blocks) {
96
- if (block.text) {
97
- console.log(` [${block.type}] ${block.text.text}`);
98
- }
99
- if (block.fields) {
100
- for (const field of block.fields) {
101
- console.log(` [field] ${field.text}`);
102
- }
103
- }
104
- if (block.elements) {
105
- for (const el of block.elements) {
106
- console.log(` [element] ${el.text}`);
107
- }
108
- }
109
- }
110
- console.log();
111
- return;
112
- }
113
- // Deliver via Slack
114
- const webhookUrl = digestConfig?.slackWebhookUrl ?? process.env.SLACK_WEBHOOK_URL;
115
- if (!webhookUrl) {
116
- console.warn(" ⚠️ No Slack webhook URL configured. Set SLACK_WEBHOOK_URL or configure in schedules.yaml");
117
- console.log(" Printing digest to stdout instead:");
118
- console.log();
119
- console.log(` ${message.text}`);
120
- return;
121
- }
122
- console.log(" Sending to Slack...");
123
- try {
124
- const response = await fetch(webhookUrl, {
125
- body: JSON.stringify(message),
126
- headers: { "Content-Type": "application/json" },
127
- method: "POST",
128
- });
129
- if (response.ok) {
130
- console.log(" ✅ Digest sent successfully");
131
- }
132
- else {
133
- const text = await response.text();
134
- console.warn(` ⚠️ Slack delivery failed: ${response.status} ${text}`);
135
- }
136
- }
137
- catch (error) {
138
- console.warn(` ⚠️ Slack delivery error: ${error instanceof Error ? error.message : String(error)}`);
139
- }
140
- }
141
- main().catch((error) => {
142
- console.error("Fatal error:", error);
143
- process.exit(1);
144
- });