@sanity/ailf 2.0.2 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. package/LICENSE +21 -0
  2. package/dist/_vendor/ailf-core/examples/index.d.ts +50 -1
  3. package/dist/_vendor/ailf-core/examples/index.js +66 -1
  4. package/dist/agent-harness/assertions-runtime.d.ts +49 -0
  5. package/dist/agent-harness/assertions-runtime.js +138 -0
  6. package/dist/agent-harness/provider.d.ts +58 -0
  7. package/dist/agent-harness/provider.js +104 -0
  8. package/dist/cli.js +0 -0
  9. package/dist/commands/init.js +3 -0
  10. package/dist/orchestration/steps/generate-configs-step.d.ts +7 -0
  11. package/dist/orchestration/steps/generate-configs-step.js +35 -2
  12. package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.js +39 -25
  13. package/dist/pipeline/compiler/compiler-to-yaml.js +78 -7
  14. package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.d.ts +9 -0
  15. package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.js +28 -85
  16. package/dist/pipeline/compiler/mode-handlers/agent-harness/compiler.js +22 -15
  17. package/dist/pipeline/compiler/mode-handlers/agent-harness/sandbox.d.ts +8 -1
  18. package/dist/pipeline/compiler/mode-handlers/agent-harness/sandbox.js +42 -12
  19. package/package.json +25 -24
  20. package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.d.ts +0 -10
  21. package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.js +0 -185
  22. package/dist/_vendor/ailf-core/artifact-capture/__tests__/noop-collector.test.d.ts +0 -6
  23. package/dist/_vendor/ailf-core/artifact-capture/__tests__/noop-collector.test.js +0 -42
  24. package/dist/_vendor/ailf-tasks/cli.d.ts +0 -8
  25. package/dist/_vendor/ailf-tasks/cli.js +0 -61
  26. package/dist/_vendor/ailf-tasks/index.d.ts +0 -13
  27. package/dist/_vendor/ailf-tasks/index.js +0 -16
  28. package/dist/_vendor/ailf-tasks/parser.d.ts +0 -27
  29. package/dist/_vendor/ailf-tasks/parser.js +0 -73
  30. package/dist/_vendor/ailf-tasks/schemas.d.ts +0 -198
  31. package/dist/_vendor/ailf-tasks/schemas.js +0 -180
  32. package/dist/_vendor/ailf-tasks/validation.d.ts +0 -47
  33. package/dist/_vendor/ailf-tasks/validation.js +0 -162
  34. package/dist/adapters/task-sources/yaml-task-source.d.ts +0 -18
  35. package/dist/adapters/task-sources/yaml-task-source.js +0 -139
  36. package/dist/agent-observer/test-imports.d.ts +0 -7
  37. package/dist/agent-observer/test-imports.js +0 -185
  38. package/dist/commands/update-quality-scores.d.ts +0 -5
  39. package/dist/commands/update-quality-scores.js +0 -20
  40. package/dist/lib/agent-behavior-report.d.ts +0 -8
  41. package/dist/lib/agent-behavior-report.js +0 -185
  42. package/dist/lib/baseline.d.ts +0 -19
  43. package/dist/lib/baseline.js +0 -153
  44. package/dist/lib/calculate-scores.d.ts +0 -23
  45. package/dist/lib/calculate-scores.js +0 -42
  46. package/dist/lib/compare.d.ts +0 -18
  47. package/dist/lib/compare.js +0 -170
  48. package/dist/lib/coverage-audit.d.ts +0 -4
  49. package/dist/lib/coverage-audit.js +0 -42
  50. package/dist/lib/discovery-report.d.ts +0 -13
  51. package/dist/lib/discovery-report.js +0 -57
  52. package/dist/lib/fetch-docs.d.ts +0 -30
  53. package/dist/lib/fetch-docs.js +0 -171
  54. package/dist/lib/generate-configs.d.ts +0 -25
  55. package/dist/lib/generate-configs.js +0 -42
  56. package/dist/lib/grader-api.d.ts +0 -21
  57. package/dist/lib/grader-api.js +0 -34
  58. package/dist/lib/grader-compare.d.ts +0 -19
  59. package/dist/lib/grader-compare.js +0 -91
  60. package/dist/lib/grader-consistency.d.ts +0 -27
  61. package/dist/lib/grader-consistency.js +0 -79
  62. package/dist/lib/grader-sensitivity.d.ts +0 -19
  63. package/dist/lib/grader-sensitivity.js +0 -75
  64. package/dist/lib/grader-validate.d.ts +0 -19
  65. package/dist/lib/grader-validate.js +0 -78
  66. package/dist/lib/measure-retrieval.d.ts +0 -14
  67. package/dist/lib/measure-retrieval.js +0 -71
  68. package/dist/lib/pr-comment.d.ts +0 -16
  69. package/dist/lib/pr-comment.js +0 -28
  70. package/dist/lib/readiness-report.d.ts +0 -13
  71. package/dist/lib/readiness-report.js +0 -108
  72. package/dist/lib/webhook-server.d.ts +0 -11
  73. package/dist/lib/webhook-server.js +0 -24
  74. package/dist/lib/weekly-digest.d.ts +0 -24
  75. package/dist/lib/weekly-digest.js +0 -148
  76. package/dist/orchestration/env-bridge.d.ts +0 -21
  77. package/dist/orchestration/env-bridge.js +0 -66
  78. package/dist/orchestration/steps/fetch-docs-shell.d.ts +0 -17
  79. package/dist/orchestration/steps/fetch-docs-shell.js +0 -30
  80. package/dist/pipeline/compiler/__tests__/task-bridge.test.d.ts +0 -9
  81. package/dist/pipeline/compiler/__tests__/task-bridge.test.js +0 -339
  82. package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.d.ts +0 -70
  83. package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.js +0 -485
  84. package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.d.ts +0 -76
  85. package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.js +0 -245
  86. package/dist/pipeline/compiler/mode-handlers/literacy-handler.d.ts +0 -89
  87. package/dist/pipeline/compiler/mode-handlers/literacy-handler.js +0 -379
  88. package/dist/pipeline/compiler/mode-handlers/mcp-assertions.d.ts +0 -50
  89. package/dist/pipeline/compiler/mode-handlers/mcp-assertions.js +0 -334
  90. package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.d.ts +0 -69
  91. package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.js +0 -307
  92. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.d.ts +0 -65
  93. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.js +0 -368
  94. package/dist/pipeline/compiler/task-bridge.d.ts +0 -41
  95. package/dist/pipeline/compiler/task-bridge.js +0 -92
  96. package/dist/pipeline/expand-tasks.d.ts +0 -232
  97. package/dist/pipeline/expand-tasks.js +0 -467
  98. package/dist/pipeline/generate-configs.d.ts +0 -92
  99. package/dist/pipeline/generate-configs.js +0 -445
  100. package/dist/pipeline/steps/calculate-scores-step.d.ts +0 -11
  101. package/dist/pipeline/steps/calculate-scores-step.js +0 -89
  102. package/dist/pipeline/steps/compare-step.d.ts +0 -18
  103. package/dist/pipeline/steps/compare-step.js +0 -90
  104. package/dist/pipeline/steps/eval-step.d.ts +0 -53
  105. package/dist/pipeline/steps/eval-step.js +0 -347
  106. package/dist/pipeline/steps/fetch-docs-step.d.ts +0 -11
  107. package/dist/pipeline/steps/fetch-docs-step.js +0 -84
  108. package/dist/pipeline/steps/generate-configs-step.d.ts +0 -11
  109. package/dist/pipeline/steps/generate-configs-step.js +0 -98
  110. package/dist/pipeline/steps/grader-consistency-step.d.ts +0 -21
  111. package/dist/pipeline/steps/grader-consistency-step.js +0 -74
  112. package/dist/pipeline/steps/publish-report-step.d.ts +0 -57
  113. package/dist/pipeline/steps/publish-report-step.js +0 -243
  114. package/dist/pipeline/steps/report-step.d.ts +0 -13
  115. package/dist/pipeline/steps/report-step.js +0 -56
  116. package/dist/pipeline/steps/update-scores-step.d.ts +0 -11
  117. package/dist/pipeline/steps/update-scores-step.js +0 -42
  118. package/dist/scripts/agent-behavior-report.d.ts +0 -19
  119. package/dist/scripts/agent-behavior-report.js +0 -315
  120. package/dist/scripts/baseline.d.ts +0 -43
  121. package/dist/scripts/baseline.js +0 -267
  122. package/dist/scripts/calculate-scores.d.ts +0 -166
  123. package/dist/scripts/calculate-scores.js +0 -1296
  124. package/dist/scripts/compare.d.ts +0 -22
  125. package/dist/scripts/compare.js +0 -334
  126. package/dist/scripts/coverage-audit.d.ts +0 -44
  127. package/dist/scripts/coverage-audit.js +0 -209
  128. package/dist/scripts/debug-eval.d.ts +0 -19
  129. package/dist/scripts/debug-eval.js +0 -73
  130. package/dist/scripts/discovery-report.d.ts +0 -58
  131. package/dist/scripts/discovery-report.js +0 -250
  132. package/dist/scripts/fetch-docs.d.ts +0 -35
  133. package/dist/scripts/fetch-docs.js +0 -472
  134. package/dist/scripts/generate-configs.d.ts +0 -66
  135. package/dist/scripts/generate-configs.js +0 -459
  136. package/dist/scripts/grader-api.d.ts +0 -27
  137. package/dist/scripts/grader-api.js +0 -206
  138. package/dist/scripts/grader-compare.d.ts +0 -22
  139. package/dist/scripts/grader-compare.js +0 -368
  140. package/dist/scripts/grader-consistency.d.ts +0 -20
  141. package/dist/scripts/grader-consistency.js +0 -313
  142. package/dist/scripts/grader-sensitivity.d.ts +0 -22
  143. package/dist/scripts/grader-sensitivity.js +0 -354
  144. package/dist/scripts/grader-validate.d.ts +0 -19
  145. package/dist/scripts/grader-validate.js +0 -267
  146. package/dist/scripts/measure-retrieval.d.ts +0 -10
  147. package/dist/scripts/measure-retrieval.js +0 -145
  148. package/dist/scripts/migrate-tasks-to-content-lake.d.ts +0 -24
  149. package/dist/scripts/migrate-tasks-to-content-lake.js +0 -328
  150. package/dist/scripts/pipeline.d.ts +0 -76
  151. package/dist/scripts/pipeline.js +0 -1031
  152. package/dist/scripts/pr-comment.d.ts +0 -10
  153. package/dist/scripts/pr-comment.js +0 -510
  154. package/dist/scripts/readiness-report.d.ts +0 -88
  155. package/dist/scripts/readiness-report.js +0 -342
  156. package/dist/scripts/update-quality-scores.d.ts +0 -15
  157. package/dist/scripts/update-quality-scores.js +0 -184
  158. package/dist/scripts/validate-task-sources.d.ts +0 -21
  159. package/dist/scripts/validate-task-sources.js +0 -210
  160. package/dist/scripts/validate.d.ts +0 -13
  161. package/dist/scripts/validate.js +0 -79
  162. package/dist/scripts/webhook-server.d.ts +0 -26
  163. package/dist/scripts/webhook-server.js +0 -147
  164. package/dist/scripts/weekly-digest.d.ts +0 -24
  165. package/dist/scripts/weekly-digest.js +0 -144
  166. package/dist/sinks/format-slack.d.ts +0 -64
  167. package/dist/sinks/format-slack.js +0 -306
  168. package/dist/sinks/slack-sink.d.ts +0 -27
  169. package/dist/sinks/slack-sink.js +0 -78
  170. package/dist/sinks/webhook-sink.d.ts +0 -19
  171. package/dist/sinks/webhook-sink.js +0 -50
  172. package/tasks/.expanded.agentic.yaml +0 -280
  173. package/tasks/.expanded.yaml +0 -565
@@ -1,210 +0,0 @@
1
- #!/usr/bin/env tsx
2
- /**
3
- * Validation script: Compare YamlTaskSource vs ContentLakeTaskSource
4
- *
5
- * Loads tasks from both sources and compares them field-by-field to verify
6
- * that the Content Lake migration produced identical LiteracyTaskDefinition[] output.
7
- *
8
- * This is Phase 3b of the tasks-as-content exec plan — parallel validation
9
- * before deleting YAML files.
10
- *
11
- * Usage:
12
- * cd packages/eval
13
- * npx tsx src/scripts/validate-task-sources.ts
14
- *
15
- * Prerequisites:
16
- * - Migration script has been run (ailf.task documents exist in CL)
17
- * - SANITY_API_TOKEN configured for Content Lake reads
18
- *
19
- * @see docs/archive/exec-plans/tasks-as-content/phase-3-migration.md
20
- */
21
- import { config as dotenvConfig } from "dotenv";
22
- import { existsSync } from "fs";
23
- import { dirname, resolve } from "path";
24
- import { fileURLToPath } from "url";
25
- import { isSlugRef } from "../_vendor/ailf-core/index.js";
26
- import { ContentLakeTaskSource } from "../adapters/task-sources/content-lake-task-source.js";
27
- import { YamlTaskSource } from "../adapters/task-sources/yaml-task-source.js";
28
- import { getSanityClient } from "../sanity/client.js";
29
- const __dirname = dirname(fileURLToPath(import.meta.url));
30
- const ROOT = resolve(__dirname, "..", "..");
31
- // Load .env from repository root (same as CLI entry point)
32
- const envPath = resolve(ROOT, "..", "..", ".env");
33
- if (existsSync(envPath)) {
34
- dotenvConfig({ override: true, path: envPath });
35
- }
36
- /**
37
- * Compare two LiteracyTaskDefinition arrays field-by-field.
38
- * Returns a list of differences.
39
- */
40
- function compareTasks(yamlTasks, clTasks) {
41
- const yamlMap = new Map(yamlTasks.map((t) => [t.id, t]));
42
- const clMap = new Map(clTasks.map((t) => [t.id, t]));
43
- const missingInCl = yamlTasks.filter((t) => !clMap.has(t.id)).map((t) => t.id);
44
- const missingInYaml = clTasks
45
- .filter((t) => !yamlMap.has(t.id))
46
- .map((t) => t.id);
47
- const diffs = [];
48
- for (const [id, yamlTask] of yamlMap) {
49
- const clTask = clMap.get(id);
50
- if (!clTask)
51
- continue;
52
- // Compare each field
53
- compareField(diffs, id, "title", yamlTask.title, clTask.title);
54
- compareField(diffs, id, "area", yamlTask.area, clTask.area);
55
- compareField(diffs, id, "docCoverage", yamlTask.docCoverage, clTask.docCoverage);
56
- // prompt.text — normalize whitespace for comparison (YAML multiline vs CL text)
57
- const yamlPrompt = (yamlTask.prompt?.text ?? "").trim();
58
- const clPrompt = (clTask.prompt?.text ?? "").trim();
59
- if (yamlPrompt !== clPrompt) {
60
- diffs.push({
61
- clValue: truncate(clPrompt, 100),
62
- field: "prompt.text",
63
- taskId: id,
64
- yamlValue: truncate(yamlPrompt, 100),
65
- });
66
- }
67
- // context.docs — compare slug sets (slug refs only for comparison)
68
- const yamlSlugs = (yamlTask.context?.docs ?? [])
69
- .filter(isSlugRef)
70
- .map((d) => d.slug)
71
- .sort();
72
- const clSlugs = (clTask.context?.docs ?? [])
73
- .filter(isSlugRef)
74
- .map((d) => d.slug)
75
- .sort();
76
- if (JSON.stringify(yamlSlugs) !== JSON.stringify(clSlugs)) {
77
- diffs.push({
78
- clValue: clSlugs,
79
- field: "canonicalDocs.slugs",
80
- taskId: id,
81
- yamlValue: yamlSlugs,
82
- });
83
- }
84
- // assertions — compare type + template arrays
85
- const yamlAssertTypes = (yamlTask.assertions ?? [])
86
- .map((a) => {
87
- const base = a.type;
88
- if ("template" in a)
89
- return `${base}:${a.template}`;
90
- return base;
91
- })
92
- .sort();
93
- const clAssertTypes = (clTask.assertions ?? [])
94
- .map((a) => {
95
- const base = a.type;
96
- if ("template" in a)
97
- return `${base}:${a.template}`;
98
- return base;
99
- })
100
- .sort();
101
- if (JSON.stringify(yamlAssertTypes) !== JSON.stringify(clAssertTypes)) {
102
- diffs.push({
103
- clValue: clAssertTypes,
104
- field: "assertions.types",
105
- taskId: id,
106
- yamlValue: yamlAssertTypes,
107
- });
108
- }
109
- // referenceSolution — CL always returns "" since solution content is a
110
- // separate document. YAML has a path. This is an intentional difference.
111
- // We only flag it if YAML also has no solution and CL somehow does.
112
- // (In practice, we expect all YAML tasks to have paths and CL to have "")
113
- }
114
- return { diffs, missingInCl, missingInYaml };
115
- }
116
- function compareField(diffs, taskId, field, yamlValue, clValue) {
117
- if (JSON.stringify(yamlValue) !== JSON.stringify(clValue)) {
118
- diffs.push({ clValue, field, taskId, yamlValue });
119
- }
120
- }
121
- function truncate(s, maxLen) {
122
- return s.length <= maxLen ? s : s.slice(0, maxLen) + "...";
123
- }
124
- // ---------------------------------------------------------------------------
125
- // Main validation
126
- // ---------------------------------------------------------------------------
127
- async function validate() {
128
- console.log("\n🔍 Task Source Validation");
129
- console.log("=".repeat(50));
130
- // Load from YAML
131
- console.log("\n1️⃣ Loading from YAML (tasks/*.yaml)...");
132
- const yamlSource = new YamlTaskSource(ROOT);
133
- const yamlTasks = (await yamlSource.loadTasks()).filter((t) => t.mode === "literacy");
134
- console.log(` Loaded ${yamlTasks.length} tasks`);
135
- // Load from Content Lake (use write token since task docs may require it)
136
- console.log("\n2️⃣ Loading from Content Lake (ailf.task)...");
137
- const client = getSanityClient({
138
- token: process.env.AILF_REPORT_SANITY_API_TOKEN ??
139
- process.env.SANITY_API_TOKEN ??
140
- undefined,
141
- });
142
- const clSource = new ContentLakeTaskSource(client);
143
- const clTasks = (await clSource.loadTasks()).filter((t) => t.mode === "literacy");
144
- console.log(` Loaded ${clTasks.length} tasks`);
145
- // Compare
146
- console.log("\n3️⃣ Comparing...");
147
- const { diffs, missingInCl, missingInYaml } = compareTasks(yamlTasks, clTasks);
148
- // Report
149
- console.log("\n" + "=".repeat(50));
150
- console.log("📊 Validation Results");
151
- console.log(` YAML tasks: ${yamlTasks.length}`);
152
- console.log(` CL tasks: ${clTasks.length}`);
153
- console.log(` Missing in CL: ${missingInCl.length}`);
154
- console.log(` Missing in YAML: ${missingInYaml.length}`);
155
- console.log(` Field diffs: ${diffs.length}`);
156
- if (missingInCl.length > 0) {
157
- console.log("\n ❌ Tasks in YAML but missing from Content Lake:");
158
- for (const id of missingInCl) {
159
- console.log(` - ${id}`);
160
- }
161
- }
162
- if (missingInYaml.length > 0) {
163
- console.log("\n ℹ️ Tasks in Content Lake but not in YAML (expected for new CL tasks):");
164
- for (const id of missingInYaml) {
165
- console.log(` - ${id}`);
166
- }
167
- }
168
- if (diffs.length > 0) {
169
- console.log("\n ⚠️ Field differences:");
170
- for (const d of diffs) {
171
- console.log(` ${d.taskId}.${d.field}:`);
172
- console.log(` YAML: ${JSON.stringify(d.yamlValue)}`);
173
- console.log(` CL: ${JSON.stringify(d.clValue)}`);
174
- }
175
- }
176
- // Known intentional differences:
177
- // - referenceSolution: CL returns "" (content is a separate document)
178
- // - canonicalDocs.slugs: CL may have fewer slugs if some didn't resolve
179
- // during migration (broken article references). This is actually an
180
- // improvement — broken refs are visible in Studio instead of silently
181
- // failing at fetch time.
182
- const INTENTIONAL_FIELDS = new Set([
183
- "referenceSolution",
184
- "canonicalDocs.slugs",
185
- ]);
186
- const intentional = diffs.filter((d) => INTENTIONAL_FIELDS.has(d.field));
187
- const unexpected = diffs.filter((d) => !INTENTIONAL_FIELDS.has(d.field));
188
- if (unexpected.length === 0 && missingInCl.length === 0) {
189
- console.log("\n ✅ All tasks match (ignoring known intentional differences)");
190
- if (intentional.length > 0) {
191
- console.log(` (${intentional.length} intentional diff(s):`);
192
- for (const d of intentional) {
193
- console.log(` - ${d.taskId}.${d.field}`);
194
- }
195
- console.log(" )");
196
- }
197
- }
198
- else {
199
- console.log("\n ❌ Validation FAILED — see differences above");
200
- process.exit(1);
201
- }
202
- console.log("");
203
- }
204
- // ---------------------------------------------------------------------------
205
- // Run
206
- // ---------------------------------------------------------------------------
207
- validate().catch((err) => {
208
- console.error("Validation failed:", err);
209
- process.exit(1);
210
- });
@@ -1,13 +0,0 @@
1
- /**
2
- * validate.ts
3
- *
4
- * CLI script that validates pipeline configuration.
5
- * Checks that all YAML files are consistent, all task-to-mapping
6
- * cross-references are valid, and reference solutions exist.
7
- *
8
- * Usage:
9
- * pnpm validate # validate everything
10
- * pnpm validate --strict # treat warnings as errors
11
- * pnpm validate --contexts # also check that context files exist
12
- */
13
- export {};
@@ -1,79 +0,0 @@
1
- /**
2
- * validate.ts
3
- *
4
- * CLI script that validates pipeline configuration.
5
- * Checks that all YAML files are consistent, all task-to-mapping
6
- * cross-references are valid, and reference solutions exist.
7
- *
8
- * Usage:
9
- * pnpm validate # validate everything
10
- * pnpm validate --strict # treat warnings as errors
11
- * pnpm validate --contexts # also check that context files exist
12
- */
13
- import { dirname, resolve } from "path";
14
- import { fileURLToPath } from "url";
15
- import { checkContextsExist, checkEnvironment } from "../pipeline/checks.js";
16
- import { validateConfiguration } from "../pipeline/validate.js";
17
- const __dirname = dirname(fileURLToPath(import.meta.url));
18
- const ROOT = resolve(__dirname, "..", "..");
19
- // ---------------------------------------------------------------------------
20
- // CLI argument parsing
21
- // ---------------------------------------------------------------------------
22
- const args = process.argv.slice(2);
23
- const strict = args.includes("--strict");
24
- const checkCtx = args.includes("--contexts");
25
- // ---------------------------------------------------------------------------
26
- // Run validation
27
- // ---------------------------------------------------------------------------
28
- console.log("=== ai-literacy-framework — Configuration Validator ===\n");
29
- const result = validateConfiguration(ROOT);
30
- // Optionally check contexts
31
- if (checkCtx) {
32
- // Dynamically import to get feature areas
33
- const { ALL_FEATURE_AREAS } = await import("../sanity/queries.js");
34
- const contextIssues = checkContextsExist(ROOT, ALL_FEATURE_AREAS);
35
- result.issues.push(...contextIssues);
36
- result.valid =
37
- result.valid && contextIssues.every((i) => i.severity !== "error");
38
- }
39
- // Check environment
40
- const envIssues = checkEnvironment(ROOT);
41
- result.issues.push(...envIssues);
42
- // ---------------------------------------------------------------------------
43
- // Report results
44
- // ---------------------------------------------------------------------------
45
- const errors = result.issues.filter((i) => i.severity === "error");
46
- const warnings = result.issues.filter((i) => i.severity === "warning");
47
- if (errors.length > 0) {
48
- console.log(`❌ ${errors.length} error(s):\n`);
49
- for (const issue of errors) {
50
- console.log(` ERROR [${issue.source}] ${issue.message}`);
51
- if (issue.path)
52
- console.log(` at ${issue.path}`);
53
- }
54
- console.log();
55
- }
56
- if (warnings.length > 0) {
57
- console.log(`⚠️ ${warnings.length} warning(s):\n`);
58
- for (const issue of warnings) {
59
- console.log(` WARN [${issue.source}] ${issue.message}`);
60
- if (issue.path)
61
- console.log(` at ${issue.path}`);
62
- }
63
- console.log();
64
- }
65
- if (errors.length === 0 && warnings.length === 0) {
66
- console.log("✅ All checks passed — configuration is valid.\n");
67
- }
68
- if (errors.length === 0 && warnings.length > 0) {
69
- console.log(`✅ Configuration is valid (${warnings.length} warning(s)).\n`);
70
- }
71
- // In strict mode, warnings are treated as errors
72
- const exitCode = strict
73
- ? result.issues.length > 0
74
- ? 1
75
- : 0
76
- : errors.length > 0
77
- ? 1
78
- : 0;
79
- process.exit(exitCode);
@@ -1,26 +0,0 @@
1
- /**
2
- * webhook-server.ts
3
- *
4
- * Local development server for testing the webhook handler.
5
- *
6
- * Starts an HTTP server that receives Sanity webhook payloads, processes
7
- * them through the WebhookHandler, and logs results. Useful for local
8
- * development and testing the full event-driven trigger flow.
9
- *
10
- * Usage:
11
- * pnpm webhook-server # start on port 3333
12
- * WEBHOOK_PORT=8080 pnpm webhook-server # custom port
13
- *
14
- * Test with curl:
15
- * curl -X POST http://localhost:3333/webhook \
16
- * -H "Content-Type: application/json" \
17
- * -d '{"operation":"update","result":{"_id":"abc","_type":"article","slug":{"current":"groq-introduction"}}}'
18
- *
19
- * Endpoints:
20
- * POST /webhook — handle a Sanity webhook payload
21
- * GET /health — handler diagnostics (budget, pending, tracked slugs)
22
- * GET /mappings — list all tracked document slugs and their areas
23
- *
24
- * @see docs/design-docs/report-store/visibility-workflows.md
25
- */
26
- export {};
@@ -1,147 +0,0 @@
1
- /**
2
- * webhook-server.ts
3
- *
4
- * Local development server for testing the webhook handler.
5
- *
6
- * Starts an HTTP server that receives Sanity webhook payloads, processes
7
- * them through the WebhookHandler, and logs results. Useful for local
8
- * development and testing the full event-driven trigger flow.
9
- *
10
- * Usage:
11
- * pnpm webhook-server # start on port 3333
12
- * WEBHOOK_PORT=8080 pnpm webhook-server # custom port
13
- *
14
- * Test with curl:
15
- * curl -X POST http://localhost:3333/webhook \
16
- * -H "Content-Type: application/json" \
17
- * -d '{"operation":"update","result":{"_id":"abc","_type":"article","slug":{"current":"groq-introduction"}}}'
18
- *
19
- * Endpoints:
20
- * POST /webhook — handle a Sanity webhook payload
21
- * GET /health — handler diagnostics (budget, pending, tracked slugs)
22
- * GET /mappings — list all tracked document slugs and their areas
23
- *
24
- * @see docs/design-docs/report-store/visibility-workflows.md
25
- */
26
- import { createServer } from "http";
27
- import { dirname, resolve } from "path";
28
- import { fileURLToPath } from "url";
29
- import { allTrackedSlugs, buildReverseMapping, } from "../pipeline/reverse-mapping.js";
30
- import { WebhookHandler } from "../webhook/handler.js";
31
- const __dirname = dirname(fileURLToPath(import.meta.url));
32
- const ROOT = resolve(__dirname, "..", "..");
33
- // ---------------------------------------------------------------------------
34
- // Configuration
35
- // ---------------------------------------------------------------------------
36
- const PORT = parseInt(process.env.WEBHOOK_PORT ?? "3333", 10);
37
- const GITHUB_TOKEN = process.env.GITHUB_TOKEN ?? "";
38
- const DRY_RUN = !GITHUB_TOKEN;
39
- // ---------------------------------------------------------------------------
40
- // Handler
41
- // ---------------------------------------------------------------------------
42
- const handler = new WebhookHandler({
43
- dailyBudget: parseInt(process.env.WEBHOOK_DAILY_BUDGET ?? "20", 10),
44
- debounceMs: parseInt(process.env.WEBHOOK_DEBOUNCE_MS ?? "10000", 10), // 10s for local dev
45
- githubToken: GITHUB_TOKEN,
46
- rootDir: ROOT,
47
- });
48
- // ---------------------------------------------------------------------------
49
- // HTTP Server
50
- // ---------------------------------------------------------------------------
51
- const server = createServer((req, res) => {
52
- void handleRequest(req, res);
53
- });
54
- async function handleRequest(req, res) {
55
- const url = new URL(req.url ?? "/", `http://localhost:${PORT}`);
56
- // CORS headers for local dev
57
- res.setHeader("Access-Control-Allow-Origin", "*");
58
- res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
59
- res.setHeader("Access-Control-Allow-Headers", "Content-Type");
60
- if (req.method === "OPTIONS") {
61
- res.writeHead(200);
62
- res.end();
63
- return;
64
- }
65
- // Health check
66
- if (url.pathname === "/health" && req.method === "GET") {
67
- const diagnostics = handler.diagnostics();
68
- res.writeHead(200, { "Content-Type": "application/json" });
69
- res.end(JSON.stringify({ dryRun: DRY_RUN, ...diagnostics }, null, 2));
70
- return;
71
- }
72
- // Mappings
73
- if (url.pathname === "/mappings" && req.method === "GET") {
74
- const reverseMapping = buildReverseMapping(ROOT);
75
- const slugs = allTrackedSlugs(reverseMapping);
76
- const mappings = Object.fromEntries(slugs.map((slug) => [slug, reverseMapping.get(slug)]));
77
- res.writeHead(200, { "Content-Type": "application/json" });
78
- res.end(JSON.stringify({ mappings, slugCount: slugs.length }, null, 2));
79
- return;
80
- }
81
- // Webhook handler
82
- if (url.pathname === "/webhook" && req.method === "POST") {
83
- const body = await readBody(req);
84
- let payload;
85
- try {
86
- payload = JSON.parse(body);
87
- }
88
- catch {
89
- res.writeHead(400, { "Content-Type": "application/json" });
90
- res.end(JSON.stringify({ error: "Invalid JSON" }));
91
- return;
92
- }
93
- if (DRY_RUN) {
94
- console.log("\n 🔶 DRY RUN — would dispatch (no GITHUB_TOKEN set)");
95
- }
96
- const result = handler.handle(payload);
97
- console.log(` → ${result.status}:`, JSON.stringify(result));
98
- res.writeHead(200, { "Content-Type": "application/json" });
99
- res.end(JSON.stringify(result, null, 2));
100
- return;
101
- }
102
- // 404
103
- res.writeHead(404, { "Content-Type": "application/json" });
104
- res.end(JSON.stringify({
105
- endpoints: ["POST /webhook", "GET /health", "GET /mappings"],
106
- error: "Not found",
107
- }));
108
- }
109
- // ---------------------------------------------------------------------------
110
- // Start
111
- // ---------------------------------------------------------------------------
112
- server.listen(PORT, () => {
113
- const reverseMapping = buildReverseMapping(ROOT);
114
- const slugCount = allTrackedSlugs(reverseMapping).length;
115
- console.log();
116
- console.log("=== AILF Webhook Server ===");
117
- console.log();
118
- console.log(` Port: ${PORT}`);
119
- console.log(` Mode: ${DRY_RUN ? "DRY RUN (set GITHUB_TOKEN to dispatch)" : "LIVE"}`);
120
- console.log(` Tracked slugs: ${slugCount}`);
121
- console.log(` Debounce: ${handler.diagnostics().pendingSlugs}`);
122
- console.log();
123
- console.log(" Endpoints:");
124
- console.log(` POST http://localhost:${PORT}/webhook — handle webhook`);
125
- console.log(` GET http://localhost:${PORT}/health — diagnostics`);
126
- console.log(` GET http://localhost:${PORT}/mappings — slug → area map`);
127
- console.log();
128
- });
129
- // Graceful shutdown
130
- process.on("SIGINT", () => {
131
- console.log("\n Shutting down — flushing debounce window...");
132
- void handler.shutdown().then(() => {
133
- server.close();
134
- process.exit(0);
135
- });
136
- });
137
- // ---------------------------------------------------------------------------
138
- // Helpers
139
- // ---------------------------------------------------------------------------
140
- function readBody(req) {
141
- return new Promise((resolve, reject) => {
142
- const chunks = [];
143
- req.on("data", (chunk) => chunks.push(chunk));
144
- req.on("end", () => resolve(Buffer.concat(chunks).toString()));
145
- req.on("error", reject);
146
- });
147
- }
@@ -1,24 +0,0 @@
1
- /**
2
- * weekly-digest.ts
3
- *
4
- * CLI script to generate and deliver a weekly evaluation digest.
5
- *
6
- * Queries the Sanity Content Lake for all reports within the configured
7
- * lookback window, computes trend analysis, and delivers the digest
8
- * via configured channels (Slack, stdout, or both).
9
- *
10
- * Usage:
11
- * pnpm weekly-digest # send to configured Slack webhook
12
- * pnpm weekly-digest --dry-run # print to stdout only
13
- * pnpm weekly-digest --lookback 14 # 14-day lookback window
14
- * pnpm weekly-digest --json # output raw JSON
15
- *
16
- * Environment variables:
17
- * SLACK_WEBHOOK_URL — Slack incoming webhook URL
18
- * SANITY_API_TOKEN — Sanity read token
19
- * AILF_TRIGGER_TYPE — set to "scheduled" by the cron workflow
20
- * AILF_SCHEDULE — the schedule name (e.g., "weekly-digest")
21
- *
22
- * @see docs/design-docs/report-store/implementation.md — Phase 5
23
- */
24
- export {};
@@ -1,144 +0,0 @@
1
- /**
2
- * weekly-digest.ts
3
- *
4
- * CLI script to generate and deliver a weekly evaluation digest.
5
- *
6
- * Queries the Sanity Content Lake for all reports within the configured
7
- * lookback window, computes trend analysis, and delivers the digest
8
- * via configured channels (Slack, stdout, or both).
9
- *
10
- * Usage:
11
- * pnpm weekly-digest # send to configured Slack webhook
12
- * pnpm weekly-digest --dry-run # print to stdout only
13
- * pnpm weekly-digest --lookback 14 # 14-day lookback window
14
- * pnpm weekly-digest --json # output raw JSON
15
- *
16
- * Environment variables:
17
- * SLACK_WEBHOOK_URL — Slack incoming webhook URL
18
- * SANITY_API_TOKEN — Sanity read token
19
- * AILF_TRIGGER_TYPE — set to "scheduled" by the cron workflow
20
- * AILF_SCHEDULE — the schedule name (e.g., "weekly-digest")
21
- *
22
- * @see docs/design-docs/report-store/implementation.md — Phase 5
23
- */
24
- import { config as dotenvConfig } from "dotenv";
25
- import { existsSync } from "fs";
26
- import { dirname, resolve } from "path";
27
- import { fileURLToPath } from "url";
28
- import { generateDigest } from "../schedules/digest.js";
29
- import { getDigestConfig } from "../schedules/loader.js";
30
- import { formatWeeklyDigest } from "../sinks/slack/format.js";
31
- // Load root .env (same override behavior as pipeline.ts)
32
- const __dirname = dirname(fileURLToPath(import.meta.url));
33
- const envPath = resolve(__dirname, "..", "..", "..", "..", ".env");
34
- if (existsSync(envPath)) {
35
- dotenvConfig({ override: true, path: envPath });
36
- }
37
- // ---------------------------------------------------------------------------
38
- // CLI argument parsing
39
- // ---------------------------------------------------------------------------
40
- const args = process.argv.slice(2);
41
- function getOption(name) {
42
- const idx = args.indexOf(`--${name}`);
43
- return idx >= 0 && idx + 1 < args.length ? args[idx + 1] : undefined;
44
- }
45
- function hasFlag(name) {
46
- return args.includes(`--${name}`);
47
- }
48
- const DRY_RUN = hasFlag("dry-run");
49
- const JSON_OUTPUT = hasFlag("json");
50
- const lookbackOverride = getOption("lookback");
51
- // ---------------------------------------------------------------------------
52
- // Main
53
- // ---------------------------------------------------------------------------
54
- async function main() {
55
- console.log();
56
- console.log("=== AI Literacy Weekly Digest ===");
57
- console.log();
58
- // Load digest config
59
- const digestConfig = getDigestConfig();
60
- const lookbackDays = lookbackOverride
61
- ? parseInt(lookbackOverride, 10)
62
- : (digestConfig?.lookbackDays ?? 7);
63
- console.log(` Lookback: ${lookbackDays} days`);
64
- console.log(` Mode: ${DRY_RUN ? "dry run (stdout only)" : "live"}`);
65
- console.log();
66
- // Generate digest — uses AILF_REPORT_* env vars for report store access,
67
- // independent of SANITY_DATASET/SANITY_PROJECT_ID (which control doc evaluation)
68
- const digest = await generateDigest({
69
- dataset: process.env.AILF_REPORT_DATASET,
70
- lookbackDays,
71
- projectId: process.env.AILF_REPORT_PROJECT_ID,
72
- token: process.env.AILF_REPORT_SANITY_API_TOKEN ?? process.env.SANITY_API_TOKEN,
73
- });
74
- if (!digest) {
75
- console.log(" No reports found in the lookback window. Nothing to send.");
76
- process.exit(0);
77
- }
78
- // Output
79
- console.log(` Reports found: ${digest.reportCount}`);
80
- console.log(` Overall: ${Math.round(digest.overallLatest)} (${digest.overallTrend})`);
81
- console.log(` Improved: ${digest.improved.join(", ") || "none"}`);
82
- console.log(` Regressed: ${digest.regressed.join(", ") || "none"}`);
83
- console.log(` Stable: ${digest.stable.join(", ") || "none"}`);
84
- console.log();
85
- if (JSON_OUTPUT) {
86
- console.log(JSON.stringify(digest, null, 2));
87
- return;
88
- }
89
- // Format for Slack
90
- const message = formatWeeklyDigest(digest);
91
- if (DRY_RUN) {
92
- console.log(" --- Slack Message Preview ---");
93
- console.log(` Text: ${message.text}`);
94
- console.log();
95
- for (const block of message.blocks) {
96
- if (block.text) {
97
- console.log(` [${block.type}] ${block.text.text}`);
98
- }
99
- if (block.fields) {
100
- for (const field of block.fields) {
101
- console.log(` [field] ${field.text}`);
102
- }
103
- }
104
- if (block.elements) {
105
- for (const el of block.elements) {
106
- console.log(` [element] ${el.text}`);
107
- }
108
- }
109
- }
110
- console.log();
111
- return;
112
- }
113
- // Deliver via Slack
114
- const webhookUrl = digestConfig?.slackWebhookUrl ?? process.env.SLACK_WEBHOOK_URL;
115
- if (!webhookUrl) {
116
- console.warn(" ⚠️ No Slack webhook URL configured. Set SLACK_WEBHOOK_URL or configure in schedules.yaml");
117
- console.log(" Printing digest to stdout instead:");
118
- console.log();
119
- console.log(` ${message.text}`);
120
- return;
121
- }
122
- console.log(" Sending to Slack...");
123
- try {
124
- const response = await fetch(webhookUrl, {
125
- body: JSON.stringify(message),
126
- headers: { "Content-Type": "application/json" },
127
- method: "POST",
128
- });
129
- if (response.ok) {
130
- console.log(" ✅ Digest sent successfully");
131
- }
132
- else {
133
- const text = await response.text();
134
- console.warn(` ⚠️ Slack delivery failed: ${response.status} ${text}`);
135
- }
136
- }
137
- catch (error) {
138
- console.warn(` ⚠️ Slack delivery error: ${error instanceof Error ? error.message : String(error)}`);
139
- }
140
- }
141
- main().catch((error) => {
142
- console.error("Fatal error:", error);
143
- process.exit(1);
144
- });