@sanity/ailf 2.0.2 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/LICENSE +21 -0
  2. package/dist/cli.js +0 -0
  3. package/package.json +24 -24
  4. package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.d.ts +0 -10
  5. package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.js +0 -185
  6. package/dist/_vendor/ailf-core/artifact-capture/__tests__/noop-collector.test.d.ts +0 -6
  7. package/dist/_vendor/ailf-core/artifact-capture/__tests__/noop-collector.test.js +0 -42
  8. package/dist/_vendor/ailf-tasks/cli.d.ts +0 -8
  9. package/dist/_vendor/ailf-tasks/cli.js +0 -61
  10. package/dist/_vendor/ailf-tasks/index.d.ts +0 -13
  11. package/dist/_vendor/ailf-tasks/index.js +0 -16
  12. package/dist/_vendor/ailf-tasks/parser.d.ts +0 -27
  13. package/dist/_vendor/ailf-tasks/parser.js +0 -73
  14. package/dist/_vendor/ailf-tasks/schemas.d.ts +0 -198
  15. package/dist/_vendor/ailf-tasks/schemas.js +0 -180
  16. package/dist/_vendor/ailf-tasks/validation.d.ts +0 -47
  17. package/dist/_vendor/ailf-tasks/validation.js +0 -162
  18. package/dist/adapters/task-sources/yaml-task-source.d.ts +0 -18
  19. package/dist/adapters/task-sources/yaml-task-source.js +0 -139
  20. package/dist/agent-observer/test-imports.d.ts +0 -7
  21. package/dist/agent-observer/test-imports.js +0 -185
  22. package/dist/commands/update-quality-scores.d.ts +0 -5
  23. package/dist/commands/update-quality-scores.js +0 -20
  24. package/dist/lib/agent-behavior-report.d.ts +0 -8
  25. package/dist/lib/agent-behavior-report.js +0 -185
  26. package/dist/lib/baseline.d.ts +0 -19
  27. package/dist/lib/baseline.js +0 -153
  28. package/dist/lib/calculate-scores.d.ts +0 -23
  29. package/dist/lib/calculate-scores.js +0 -42
  30. package/dist/lib/compare.d.ts +0 -18
  31. package/dist/lib/compare.js +0 -170
  32. package/dist/lib/coverage-audit.d.ts +0 -4
  33. package/dist/lib/coverage-audit.js +0 -42
  34. package/dist/lib/discovery-report.d.ts +0 -13
  35. package/dist/lib/discovery-report.js +0 -57
  36. package/dist/lib/fetch-docs.d.ts +0 -30
  37. package/dist/lib/fetch-docs.js +0 -171
  38. package/dist/lib/generate-configs.d.ts +0 -25
  39. package/dist/lib/generate-configs.js +0 -42
  40. package/dist/lib/grader-api.d.ts +0 -21
  41. package/dist/lib/grader-api.js +0 -34
  42. package/dist/lib/grader-compare.d.ts +0 -19
  43. package/dist/lib/grader-compare.js +0 -91
  44. package/dist/lib/grader-consistency.d.ts +0 -27
  45. package/dist/lib/grader-consistency.js +0 -79
  46. package/dist/lib/grader-sensitivity.d.ts +0 -19
  47. package/dist/lib/grader-sensitivity.js +0 -75
  48. package/dist/lib/grader-validate.d.ts +0 -19
  49. package/dist/lib/grader-validate.js +0 -78
  50. package/dist/lib/measure-retrieval.d.ts +0 -14
  51. package/dist/lib/measure-retrieval.js +0 -71
  52. package/dist/lib/pr-comment.d.ts +0 -16
  53. package/dist/lib/pr-comment.js +0 -28
  54. package/dist/lib/readiness-report.d.ts +0 -13
  55. package/dist/lib/readiness-report.js +0 -108
  56. package/dist/lib/webhook-server.d.ts +0 -11
  57. package/dist/lib/webhook-server.js +0 -24
  58. package/dist/lib/weekly-digest.d.ts +0 -24
  59. package/dist/lib/weekly-digest.js +0 -148
  60. package/dist/orchestration/env-bridge.d.ts +0 -21
  61. package/dist/orchestration/env-bridge.js +0 -66
  62. package/dist/orchestration/steps/fetch-docs-shell.d.ts +0 -17
  63. package/dist/orchestration/steps/fetch-docs-shell.js +0 -30
  64. package/dist/pipeline/compiler/__tests__/task-bridge.test.d.ts +0 -9
  65. package/dist/pipeline/compiler/__tests__/task-bridge.test.js +0 -339
  66. package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.d.ts +0 -70
  67. package/dist/pipeline/compiler/mode-handlers/agent-harness-handler.js +0 -485
  68. package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.d.ts +0 -76
  69. package/dist/pipeline/compiler/mode-handlers/knowledge-probe-handler.js +0 -245
  70. package/dist/pipeline/compiler/mode-handlers/literacy-handler.d.ts +0 -89
  71. package/dist/pipeline/compiler/mode-handlers/literacy-handler.js +0 -379
  72. package/dist/pipeline/compiler/mode-handlers/mcp-assertions.d.ts +0 -50
  73. package/dist/pipeline/compiler/mode-handlers/mcp-assertions.js +0 -334
  74. package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.d.ts +0 -69
  75. package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.js +0 -307
  76. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.d.ts +0 -65
  77. package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.js +0 -368
  78. package/dist/pipeline/compiler/task-bridge.d.ts +0 -41
  79. package/dist/pipeline/compiler/task-bridge.js +0 -92
  80. package/dist/pipeline/expand-tasks.d.ts +0 -232
  81. package/dist/pipeline/expand-tasks.js +0 -467
  82. package/dist/pipeline/generate-configs.d.ts +0 -92
  83. package/dist/pipeline/generate-configs.js +0 -445
  84. package/dist/pipeline/steps/calculate-scores-step.d.ts +0 -11
  85. package/dist/pipeline/steps/calculate-scores-step.js +0 -89
  86. package/dist/pipeline/steps/compare-step.d.ts +0 -18
  87. package/dist/pipeline/steps/compare-step.js +0 -90
  88. package/dist/pipeline/steps/eval-step.d.ts +0 -53
  89. package/dist/pipeline/steps/eval-step.js +0 -347
  90. package/dist/pipeline/steps/fetch-docs-step.d.ts +0 -11
  91. package/dist/pipeline/steps/fetch-docs-step.js +0 -84
  92. package/dist/pipeline/steps/generate-configs-step.d.ts +0 -11
  93. package/dist/pipeline/steps/generate-configs-step.js +0 -98
  94. package/dist/pipeline/steps/grader-consistency-step.d.ts +0 -21
  95. package/dist/pipeline/steps/grader-consistency-step.js +0 -74
  96. package/dist/pipeline/steps/publish-report-step.d.ts +0 -57
  97. package/dist/pipeline/steps/publish-report-step.js +0 -243
  98. package/dist/pipeline/steps/report-step.d.ts +0 -13
  99. package/dist/pipeline/steps/report-step.js +0 -56
  100. package/dist/pipeline/steps/update-scores-step.d.ts +0 -11
  101. package/dist/pipeline/steps/update-scores-step.js +0 -42
  102. package/dist/scripts/agent-behavior-report.d.ts +0 -19
  103. package/dist/scripts/agent-behavior-report.js +0 -315
  104. package/dist/scripts/baseline.d.ts +0 -43
  105. package/dist/scripts/baseline.js +0 -267
  106. package/dist/scripts/calculate-scores.d.ts +0 -166
  107. package/dist/scripts/calculate-scores.js +0 -1296
  108. package/dist/scripts/compare.d.ts +0 -22
  109. package/dist/scripts/compare.js +0 -334
  110. package/dist/scripts/coverage-audit.d.ts +0 -44
  111. package/dist/scripts/coverage-audit.js +0 -209
  112. package/dist/scripts/debug-eval.d.ts +0 -19
  113. package/dist/scripts/debug-eval.js +0 -73
  114. package/dist/scripts/discovery-report.d.ts +0 -58
  115. package/dist/scripts/discovery-report.js +0 -250
  116. package/dist/scripts/fetch-docs.d.ts +0 -35
  117. package/dist/scripts/fetch-docs.js +0 -472
  118. package/dist/scripts/generate-configs.d.ts +0 -66
  119. package/dist/scripts/generate-configs.js +0 -459
  120. package/dist/scripts/grader-api.d.ts +0 -27
  121. package/dist/scripts/grader-api.js +0 -206
  122. package/dist/scripts/grader-compare.d.ts +0 -22
  123. package/dist/scripts/grader-compare.js +0 -368
  124. package/dist/scripts/grader-consistency.d.ts +0 -20
  125. package/dist/scripts/grader-consistency.js +0 -313
  126. package/dist/scripts/grader-sensitivity.d.ts +0 -22
  127. package/dist/scripts/grader-sensitivity.js +0 -354
  128. package/dist/scripts/grader-validate.d.ts +0 -19
  129. package/dist/scripts/grader-validate.js +0 -267
  130. package/dist/scripts/measure-retrieval.d.ts +0 -10
  131. package/dist/scripts/measure-retrieval.js +0 -145
  132. package/dist/scripts/migrate-tasks-to-content-lake.d.ts +0 -24
  133. package/dist/scripts/migrate-tasks-to-content-lake.js +0 -328
  134. package/dist/scripts/pipeline.d.ts +0 -76
  135. package/dist/scripts/pipeline.js +0 -1031
  136. package/dist/scripts/pr-comment.d.ts +0 -10
  137. package/dist/scripts/pr-comment.js +0 -510
  138. package/dist/scripts/readiness-report.d.ts +0 -88
  139. package/dist/scripts/readiness-report.js +0 -342
  140. package/dist/scripts/update-quality-scores.d.ts +0 -15
  141. package/dist/scripts/update-quality-scores.js +0 -184
  142. package/dist/scripts/validate-task-sources.d.ts +0 -21
  143. package/dist/scripts/validate-task-sources.js +0 -210
  144. package/dist/scripts/validate.d.ts +0 -13
  145. package/dist/scripts/validate.js +0 -79
  146. package/dist/scripts/webhook-server.d.ts +0 -26
  147. package/dist/scripts/webhook-server.js +0 -147
  148. package/dist/scripts/weekly-digest.d.ts +0 -24
  149. package/dist/scripts/weekly-digest.js +0 -144
  150. package/dist/sinks/format-slack.d.ts +0 -64
  151. package/dist/sinks/format-slack.js +0 -306
  152. package/dist/sinks/slack-sink.d.ts +0 -27
  153. package/dist/sinks/slack-sink.js +0 -78
  154. package/dist/sinks/webhook-sink.d.ts +0 -19
  155. package/dist/sinks/webhook-sink.js +0 -50
  156. package/tasks/.expanded.agentic.yaml +0 -280
  157. package/tasks/.expanded.yaml +0 -565
@@ -1,210 +0,0 @@
1
- #!/usr/bin/env tsx
2
- /**
3
- * Validation script: Compare YamlTaskSource vs ContentLakeTaskSource
4
- *
5
- * Loads tasks from both sources and compares them field-by-field to verify
6
- * that the Content Lake migration produced identical LiteracyTaskDefinition[] output.
7
- *
8
- * This is Phase 3b of the tasks-as-content exec plan — parallel validation
9
- * before deleting YAML files.
10
- *
11
- * Usage:
12
- * cd packages/eval
13
- * npx tsx src/scripts/validate-task-sources.ts
14
- *
15
- * Prerequisites:
16
- * - Migration script has been run (ailf.task documents exist in CL)
17
- * - SANITY_API_TOKEN configured for Content Lake reads
18
- *
19
- * @see docs/archive/exec-plans/tasks-as-content/phase-3-migration.md
20
- */
21
- import { config as dotenvConfig } from "dotenv";
22
- import { existsSync } from "fs";
23
- import { dirname, resolve } from "path";
24
- import { fileURLToPath } from "url";
25
- import { isSlugRef } from "../_vendor/ailf-core/index.js";
26
- import { ContentLakeTaskSource } from "../adapters/task-sources/content-lake-task-source.js";
27
- import { YamlTaskSource } from "../adapters/task-sources/yaml-task-source.js";
28
- import { getSanityClient } from "../sanity/client.js";
29
- const __dirname = dirname(fileURLToPath(import.meta.url));
30
- const ROOT = resolve(__dirname, "..", "..");
31
- // Load .env from repository root (same as CLI entry point)
32
- const envPath = resolve(ROOT, "..", "..", ".env");
33
- if (existsSync(envPath)) {
34
- dotenvConfig({ override: true, path: envPath });
35
- }
36
- /**
37
- * Compare two LiteracyTaskDefinition arrays field-by-field.
38
- * Returns a list of differences.
39
- */
40
- function compareTasks(yamlTasks, clTasks) {
41
- const yamlMap = new Map(yamlTasks.map((t) => [t.id, t]));
42
- const clMap = new Map(clTasks.map((t) => [t.id, t]));
43
- const missingInCl = yamlTasks.filter((t) => !clMap.has(t.id)).map((t) => t.id);
44
- const missingInYaml = clTasks
45
- .filter((t) => !yamlMap.has(t.id))
46
- .map((t) => t.id);
47
- const diffs = [];
48
- for (const [id, yamlTask] of yamlMap) {
49
- const clTask = clMap.get(id);
50
- if (!clTask)
51
- continue;
52
- // Compare each field
53
- compareField(diffs, id, "title", yamlTask.title, clTask.title);
54
- compareField(diffs, id, "area", yamlTask.area, clTask.area);
55
- compareField(diffs, id, "docCoverage", yamlTask.docCoverage, clTask.docCoverage);
56
- // prompt.text — normalize whitespace for comparison (YAML multiline vs CL text)
57
- const yamlPrompt = (yamlTask.prompt?.text ?? "").trim();
58
- const clPrompt = (clTask.prompt?.text ?? "").trim();
59
- if (yamlPrompt !== clPrompt) {
60
- diffs.push({
61
- clValue: truncate(clPrompt, 100),
62
- field: "prompt.text",
63
- taskId: id,
64
- yamlValue: truncate(yamlPrompt, 100),
65
- });
66
- }
67
- // context.docs — compare slug sets (slug refs only for comparison)
68
- const yamlSlugs = (yamlTask.context?.docs ?? [])
69
- .filter(isSlugRef)
70
- .map((d) => d.slug)
71
- .sort();
72
- const clSlugs = (clTask.context?.docs ?? [])
73
- .filter(isSlugRef)
74
- .map((d) => d.slug)
75
- .sort();
76
- if (JSON.stringify(yamlSlugs) !== JSON.stringify(clSlugs)) {
77
- diffs.push({
78
- clValue: clSlugs,
79
- field: "canonicalDocs.slugs",
80
- taskId: id,
81
- yamlValue: yamlSlugs,
82
- });
83
- }
84
- // assertions — compare type + template arrays
85
- const yamlAssertTypes = (yamlTask.assertions ?? [])
86
- .map((a) => {
87
- const base = a.type;
88
- if ("template" in a)
89
- return `${base}:${a.template}`;
90
- return base;
91
- })
92
- .sort();
93
- const clAssertTypes = (clTask.assertions ?? [])
94
- .map((a) => {
95
- const base = a.type;
96
- if ("template" in a)
97
- return `${base}:${a.template}`;
98
- return base;
99
- })
100
- .sort();
101
- if (JSON.stringify(yamlAssertTypes) !== JSON.stringify(clAssertTypes)) {
102
- diffs.push({
103
- clValue: clAssertTypes,
104
- field: "assertions.types",
105
- taskId: id,
106
- yamlValue: yamlAssertTypes,
107
- });
108
- }
109
- // referenceSolution — CL always returns "" since solution content is a
110
- // separate document. YAML has a path. This is an intentional difference.
111
- // We only flag it if YAML also has no solution and CL somehow does.
112
- // (In practice, we expect all YAML tasks to have paths and CL to have "")
113
- }
114
- return { diffs, missingInCl, missingInYaml };
115
- }
116
- function compareField(diffs, taskId, field, yamlValue, clValue) {
117
- if (JSON.stringify(yamlValue) !== JSON.stringify(clValue)) {
118
- diffs.push({ clValue, field, taskId, yamlValue });
119
- }
120
- }
121
- function truncate(s, maxLen) {
122
- return s.length <= maxLen ? s : s.slice(0, maxLen) + "...";
123
- }
124
- // ---------------------------------------------------------------------------
125
- // Main validation
126
- // ---------------------------------------------------------------------------
127
- async function validate() {
128
- console.log("\n🔍 Task Source Validation");
129
- console.log("=".repeat(50));
130
- // Load from YAML
131
- console.log("\n1️⃣ Loading from YAML (tasks/*.yaml)...");
132
- const yamlSource = new YamlTaskSource(ROOT);
133
- const yamlTasks = (await yamlSource.loadTasks()).filter((t) => t.mode === "literacy");
134
- console.log(` Loaded ${yamlTasks.length} tasks`);
135
- // Load from Content Lake (use write token since task docs may require it)
136
- console.log("\n2️⃣ Loading from Content Lake (ailf.task)...");
137
- const client = getSanityClient({
138
- token: process.env.AILF_REPORT_SANITY_API_TOKEN ??
139
- process.env.SANITY_API_TOKEN ??
140
- undefined,
141
- });
142
- const clSource = new ContentLakeTaskSource(client);
143
- const clTasks = (await clSource.loadTasks()).filter((t) => t.mode === "literacy");
144
- console.log(` Loaded ${clTasks.length} tasks`);
145
- // Compare
146
- console.log("\n3️⃣ Comparing...");
147
- const { diffs, missingInCl, missingInYaml } = compareTasks(yamlTasks, clTasks);
148
- // Report
149
- console.log("\n" + "=".repeat(50));
150
- console.log("📊 Validation Results");
151
- console.log(` YAML tasks: ${yamlTasks.length}`);
152
- console.log(` CL tasks: ${clTasks.length}`);
153
- console.log(` Missing in CL: ${missingInCl.length}`);
154
- console.log(` Missing in YAML: ${missingInYaml.length}`);
155
- console.log(` Field diffs: ${diffs.length}`);
156
- if (missingInCl.length > 0) {
157
- console.log("\n ❌ Tasks in YAML but missing from Content Lake:");
158
- for (const id of missingInCl) {
159
- console.log(` - ${id}`);
160
- }
161
- }
162
- if (missingInYaml.length > 0) {
163
- console.log("\n ℹ️ Tasks in Content Lake but not in YAML (expected for new CL tasks):");
164
- for (const id of missingInYaml) {
165
- console.log(` - ${id}`);
166
- }
167
- }
168
- if (diffs.length > 0) {
169
- console.log("\n ⚠️ Field differences:");
170
- for (const d of diffs) {
171
- console.log(` ${d.taskId}.${d.field}:`);
172
- console.log(` YAML: ${JSON.stringify(d.yamlValue)}`);
173
- console.log(` CL: ${JSON.stringify(d.clValue)}`);
174
- }
175
- }
176
- // Known intentional differences:
177
- // - referenceSolution: CL returns "" (content is a separate document)
178
- // - canonicalDocs.slugs: CL may have fewer slugs if some didn't resolve
179
- // during migration (broken article references). This is actually an
180
- // improvement — broken refs are visible in Studio instead of silently
181
- // failing at fetch time.
182
- const INTENTIONAL_FIELDS = new Set([
183
- "referenceSolution",
184
- "canonicalDocs.slugs",
185
- ]);
186
- const intentional = diffs.filter((d) => INTENTIONAL_FIELDS.has(d.field));
187
- const unexpected = diffs.filter((d) => !INTENTIONAL_FIELDS.has(d.field));
188
- if (unexpected.length === 0 && missingInCl.length === 0) {
189
- console.log("\n ✅ All tasks match (ignoring known intentional differences)");
190
- if (intentional.length > 0) {
191
- console.log(` (${intentional.length} intentional diff(s):`);
192
- for (const d of intentional) {
193
- console.log(` - ${d.taskId}.${d.field}`);
194
- }
195
- console.log(" )");
196
- }
197
- }
198
- else {
199
- console.log("\n ❌ Validation FAILED — see differences above");
200
- process.exit(1);
201
- }
202
- console.log("");
203
- }
204
- // ---------------------------------------------------------------------------
205
- // Run
206
- // ---------------------------------------------------------------------------
207
- validate().catch((err) => {
208
- console.error("Validation failed:", err);
209
- process.exit(1);
210
- });
@@ -1,13 +0,0 @@
1
- /**
2
- * validate.ts
3
- *
4
- * CLI script that validates pipeline configuration.
5
- * Checks that all YAML files are consistent, all task-to-mapping
6
- * cross-references are valid, and reference solutions exist.
7
- *
8
- * Usage:
9
- * pnpm validate # validate everything
10
- * pnpm validate --strict # treat warnings as errors
11
- * pnpm validate --contexts # also check that context files exist
12
- */
13
- export {};
@@ -1,79 +0,0 @@
1
- /**
2
- * validate.ts
3
- *
4
- * CLI script that validates pipeline configuration.
5
- * Checks that all YAML files are consistent, all task-to-mapping
6
- * cross-references are valid, and reference solutions exist.
7
- *
8
- * Usage:
9
- * pnpm validate # validate everything
10
- * pnpm validate --strict # treat warnings as errors
11
- * pnpm validate --contexts # also check that context files exist
12
- */
13
- import { dirname, resolve } from "path";
14
- import { fileURLToPath } from "url";
15
- import { checkContextsExist, checkEnvironment } from "../pipeline/checks.js";
16
- import { validateConfiguration } from "../pipeline/validate.js";
17
- const __dirname = dirname(fileURLToPath(import.meta.url));
18
- const ROOT = resolve(__dirname, "..", "..");
19
- // ---------------------------------------------------------------------------
20
- // CLI argument parsing
21
- // ---------------------------------------------------------------------------
22
- const args = process.argv.slice(2);
23
- const strict = args.includes("--strict");
24
- const checkCtx = args.includes("--contexts");
25
- // ---------------------------------------------------------------------------
26
- // Run validation
27
- // ---------------------------------------------------------------------------
28
- console.log("=== ai-literacy-framework — Configuration Validator ===\n");
29
- const result = validateConfiguration(ROOT);
30
- // Optionally check contexts
31
- if (checkCtx) {
32
- // Dynamically import to get feature areas
33
- const { ALL_FEATURE_AREAS } = await import("../sanity/queries.js");
34
- const contextIssues = checkContextsExist(ROOT, ALL_FEATURE_AREAS);
35
- result.issues.push(...contextIssues);
36
- result.valid =
37
- result.valid && contextIssues.every((i) => i.severity !== "error");
38
- }
39
- // Check environment
40
- const envIssues = checkEnvironment(ROOT);
41
- result.issues.push(...envIssues);
42
- // ---------------------------------------------------------------------------
43
- // Report results
44
- // ---------------------------------------------------------------------------
45
- const errors = result.issues.filter((i) => i.severity === "error");
46
- const warnings = result.issues.filter((i) => i.severity === "warning");
47
- if (errors.length > 0) {
48
- console.log(`❌ ${errors.length} error(s):\n`);
49
- for (const issue of errors) {
50
- console.log(` ERROR [${issue.source}] ${issue.message}`);
51
- if (issue.path)
52
- console.log(` at ${issue.path}`);
53
- }
54
- console.log();
55
- }
56
- if (warnings.length > 0) {
57
- console.log(`⚠️ ${warnings.length} warning(s):\n`);
58
- for (const issue of warnings) {
59
- console.log(` WARN [${issue.source}] ${issue.message}`);
60
- if (issue.path)
61
- console.log(` at ${issue.path}`);
62
- }
63
- console.log();
64
- }
65
- if (errors.length === 0 && warnings.length === 0) {
66
- console.log("✅ All checks passed — configuration is valid.\n");
67
- }
68
- if (errors.length === 0 && warnings.length > 0) {
69
- console.log(`✅ Configuration is valid (${warnings.length} warning(s)).\n`);
70
- }
71
- // In strict mode, warnings are treated as errors
72
- const exitCode = strict
73
- ? result.issues.length > 0
74
- ? 1
75
- : 0
76
- : errors.length > 0
77
- ? 1
78
- : 0;
79
- process.exit(exitCode);
@@ -1,26 +0,0 @@
1
- /**
2
- * webhook-server.ts
3
- *
4
- * Local development server for testing the webhook handler.
5
- *
6
- * Starts an HTTP server that receives Sanity webhook payloads, processes
7
- * them through the WebhookHandler, and logs results. Useful for local
8
- * development and testing the full event-driven trigger flow.
9
- *
10
- * Usage:
11
- * pnpm webhook-server # start on port 3333
12
- * WEBHOOK_PORT=8080 pnpm webhook-server # custom port
13
- *
14
- * Test with curl:
15
- * curl -X POST http://localhost:3333/webhook \
16
- * -H "Content-Type: application/json" \
17
- * -d '{"operation":"update","result":{"_id":"abc","_type":"article","slug":{"current":"groq-introduction"}}}'
18
- *
19
- * Endpoints:
20
- * POST /webhook — handle a Sanity webhook payload
21
- * GET /health — handler diagnostics (budget, pending, tracked slugs)
22
- * GET /mappings — list all tracked document slugs and their areas
23
- *
24
- * @see docs/design-docs/report-store/visibility-workflows.md
25
- */
26
- export {};
@@ -1,147 +0,0 @@
1
- /**
2
- * webhook-server.ts
3
- *
4
- * Local development server for testing the webhook handler.
5
- *
6
- * Starts an HTTP server that receives Sanity webhook payloads, processes
7
- * them through the WebhookHandler, and logs results. Useful for local
8
- * development and testing the full event-driven trigger flow.
9
- *
10
- * Usage:
11
- * pnpm webhook-server # start on port 3333
12
- * WEBHOOK_PORT=8080 pnpm webhook-server # custom port
13
- *
14
- * Test with curl:
15
- * curl -X POST http://localhost:3333/webhook \
16
- * -H "Content-Type: application/json" \
17
- * -d '{"operation":"update","result":{"_id":"abc","_type":"article","slug":{"current":"groq-introduction"}}}'
18
- *
19
- * Endpoints:
20
- * POST /webhook — handle a Sanity webhook payload
21
- * GET /health — handler diagnostics (budget, pending, tracked slugs)
22
- * GET /mappings — list all tracked document slugs and their areas
23
- *
24
- * @see docs/design-docs/report-store/visibility-workflows.md
25
- */
26
- import { createServer } from "http";
27
- import { dirname, resolve } from "path";
28
- import { fileURLToPath } from "url";
29
- import { allTrackedSlugs, buildReverseMapping, } from "../pipeline/reverse-mapping.js";
30
- import { WebhookHandler } from "../webhook/handler.js";
31
- const __dirname = dirname(fileURLToPath(import.meta.url));
32
- const ROOT = resolve(__dirname, "..", "..");
33
- // ---------------------------------------------------------------------------
34
- // Configuration
35
- // ---------------------------------------------------------------------------
36
- const PORT = parseInt(process.env.WEBHOOK_PORT ?? "3333", 10);
37
- const GITHUB_TOKEN = process.env.GITHUB_TOKEN ?? "";
38
- const DRY_RUN = !GITHUB_TOKEN;
39
- // ---------------------------------------------------------------------------
40
- // Handler
41
- // ---------------------------------------------------------------------------
42
- const handler = new WebhookHandler({
43
- dailyBudget: parseInt(process.env.WEBHOOK_DAILY_BUDGET ?? "20", 10),
44
- debounceMs: parseInt(process.env.WEBHOOK_DEBOUNCE_MS ?? "10000", 10), // 10s for local dev
45
- githubToken: GITHUB_TOKEN,
46
- rootDir: ROOT,
47
- });
48
- // ---------------------------------------------------------------------------
49
- // HTTP Server
50
- // ---------------------------------------------------------------------------
51
- const server = createServer((req, res) => {
52
- void handleRequest(req, res);
53
- });
54
- async function handleRequest(req, res) {
55
- const url = new URL(req.url ?? "/", `http://localhost:${PORT}`);
56
- // CORS headers for local dev
57
- res.setHeader("Access-Control-Allow-Origin", "*");
58
- res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
59
- res.setHeader("Access-Control-Allow-Headers", "Content-Type");
60
- if (req.method === "OPTIONS") {
61
- res.writeHead(200);
62
- res.end();
63
- return;
64
- }
65
- // Health check
66
- if (url.pathname === "/health" && req.method === "GET") {
67
- const diagnostics = handler.diagnostics();
68
- res.writeHead(200, { "Content-Type": "application/json" });
69
- res.end(JSON.stringify({ dryRun: DRY_RUN, ...diagnostics }, null, 2));
70
- return;
71
- }
72
- // Mappings
73
- if (url.pathname === "/mappings" && req.method === "GET") {
74
- const reverseMapping = buildReverseMapping(ROOT);
75
- const slugs = allTrackedSlugs(reverseMapping);
76
- const mappings = Object.fromEntries(slugs.map((slug) => [slug, reverseMapping.get(slug)]));
77
- res.writeHead(200, { "Content-Type": "application/json" });
78
- res.end(JSON.stringify({ mappings, slugCount: slugs.length }, null, 2));
79
- return;
80
- }
81
- // Webhook handler
82
- if (url.pathname === "/webhook" && req.method === "POST") {
83
- const body = await readBody(req);
84
- let payload;
85
- try {
86
- payload = JSON.parse(body);
87
- }
88
- catch {
89
- res.writeHead(400, { "Content-Type": "application/json" });
90
- res.end(JSON.stringify({ error: "Invalid JSON" }));
91
- return;
92
- }
93
- if (DRY_RUN) {
94
- console.log("\n 🔶 DRY RUN — would dispatch (no GITHUB_TOKEN set)");
95
- }
96
- const result = handler.handle(payload);
97
- console.log(` → ${result.status}:`, JSON.stringify(result));
98
- res.writeHead(200, { "Content-Type": "application/json" });
99
- res.end(JSON.stringify(result, null, 2));
100
- return;
101
- }
102
- // 404
103
- res.writeHead(404, { "Content-Type": "application/json" });
104
- res.end(JSON.stringify({
105
- endpoints: ["POST /webhook", "GET /health", "GET /mappings"],
106
- error: "Not found",
107
- }));
108
- }
109
- // ---------------------------------------------------------------------------
110
- // Start
111
- // ---------------------------------------------------------------------------
112
- server.listen(PORT, () => {
113
- const reverseMapping = buildReverseMapping(ROOT);
114
- const slugCount = allTrackedSlugs(reverseMapping).length;
115
- console.log();
116
- console.log("=== AILF Webhook Server ===");
117
- console.log();
118
- console.log(` Port: ${PORT}`);
119
- console.log(` Mode: ${DRY_RUN ? "DRY RUN (set GITHUB_TOKEN to dispatch)" : "LIVE"}`);
120
- console.log(` Tracked slugs: ${slugCount}`);
121
- console.log(` Debounce: ${handler.diagnostics().pendingSlugs}`);
122
- console.log();
123
- console.log(" Endpoints:");
124
- console.log(` POST http://localhost:${PORT}/webhook — handle webhook`);
125
- console.log(` GET http://localhost:${PORT}/health — diagnostics`);
126
- console.log(` GET http://localhost:${PORT}/mappings — slug → area map`);
127
- console.log();
128
- });
129
- // Graceful shutdown
130
- process.on("SIGINT", () => {
131
- console.log("\n Shutting down — flushing debounce window...");
132
- void handler.shutdown().then(() => {
133
- server.close();
134
- process.exit(0);
135
- });
136
- });
137
- // ---------------------------------------------------------------------------
138
- // Helpers
139
- // ---------------------------------------------------------------------------
140
- function readBody(req) {
141
- return new Promise((resolve, reject) => {
142
- const chunks = [];
143
- req.on("data", (chunk) => chunks.push(chunk));
144
- req.on("end", () => resolve(Buffer.concat(chunks).toString()));
145
- req.on("error", reject);
146
- });
147
- }
@@ -1,24 +0,0 @@
1
- /**
2
- * weekly-digest.ts
3
- *
4
- * CLI script to generate and deliver a weekly evaluation digest.
5
- *
6
- * Queries the Sanity Content Lake for all reports within the configured
7
- * lookback window, computes trend analysis, and delivers the digest
8
- * via configured channels (Slack, stdout, or both).
9
- *
10
- * Usage:
11
- * pnpm weekly-digest # send to configured Slack webhook
12
- * pnpm weekly-digest --dry-run # print to stdout only
13
- * pnpm weekly-digest --lookback 14 # 14-day lookback window
14
- * pnpm weekly-digest --json # output raw JSON
15
- *
16
- * Environment variables:
17
- * SLACK_WEBHOOK_URL — Slack incoming webhook URL
18
- * SANITY_API_TOKEN — Sanity read token
19
- * AILF_TRIGGER_TYPE — set to "scheduled" by the cron workflow
20
- * AILF_SCHEDULE — the schedule name (e.g., "weekly-digest")
21
- *
22
- * @see docs/design-docs/report-store/implementation.md — Phase 5
23
- */
24
- export {};
@@ -1,144 +0,0 @@
1
- /**
2
- * weekly-digest.ts
3
- *
4
- * CLI script to generate and deliver a weekly evaluation digest.
5
- *
6
- * Queries the Sanity Content Lake for all reports within the configured
7
- * lookback window, computes trend analysis, and delivers the digest
8
- * via configured channels (Slack, stdout, or both).
9
- *
10
- * Usage:
11
- * pnpm weekly-digest # send to configured Slack webhook
12
- * pnpm weekly-digest --dry-run # print to stdout only
13
- * pnpm weekly-digest --lookback 14 # 14-day lookback window
14
- * pnpm weekly-digest --json # output raw JSON
15
- *
16
- * Environment variables:
17
- * SLACK_WEBHOOK_URL — Slack incoming webhook URL
18
- * SANITY_API_TOKEN — Sanity read token
19
- * AILF_TRIGGER_TYPE — set to "scheduled" by the cron workflow
20
- * AILF_SCHEDULE — the schedule name (e.g., "weekly-digest")
21
- *
22
- * @see docs/design-docs/report-store/implementation.md — Phase 5
23
- */
24
- import { config as dotenvConfig } from "dotenv";
25
- import { existsSync } from "fs";
26
- import { dirname, resolve } from "path";
27
- import { fileURLToPath } from "url";
28
- import { generateDigest } from "../schedules/digest.js";
29
- import { getDigestConfig } from "../schedules/loader.js";
30
- import { formatWeeklyDigest } from "../sinks/slack/format.js";
31
- // Load root .env (same override behavior as pipeline.ts)
32
- const __dirname = dirname(fileURLToPath(import.meta.url));
33
- const envPath = resolve(__dirname, "..", "..", "..", "..", ".env");
34
- if (existsSync(envPath)) {
35
- dotenvConfig({ override: true, path: envPath });
36
- }
37
- // ---------------------------------------------------------------------------
38
- // CLI argument parsing
39
- // ---------------------------------------------------------------------------
40
- const args = process.argv.slice(2);
41
- function getOption(name) {
42
- const idx = args.indexOf(`--${name}`);
43
- return idx >= 0 && idx + 1 < args.length ? args[idx + 1] : undefined;
44
- }
45
- function hasFlag(name) {
46
- return args.includes(`--${name}`);
47
- }
48
- const DRY_RUN = hasFlag("dry-run");
49
- const JSON_OUTPUT = hasFlag("json");
50
- const lookbackOverride = getOption("lookback");
51
- // ---------------------------------------------------------------------------
52
- // Main
53
- // ---------------------------------------------------------------------------
54
- async function main() {
55
- console.log();
56
- console.log("=== AI Literacy Weekly Digest ===");
57
- console.log();
58
- // Load digest config
59
- const digestConfig = getDigestConfig();
60
- const lookbackDays = lookbackOverride
61
- ? parseInt(lookbackOverride, 10)
62
- : (digestConfig?.lookbackDays ?? 7);
63
- console.log(` Lookback: ${lookbackDays} days`);
64
- console.log(` Mode: ${DRY_RUN ? "dry run (stdout only)" : "live"}`);
65
- console.log();
66
- // Generate digest — uses AILF_REPORT_* env vars for report store access,
67
- // independent of SANITY_DATASET/SANITY_PROJECT_ID (which control doc evaluation)
68
- const digest = await generateDigest({
69
- dataset: process.env.AILF_REPORT_DATASET,
70
- lookbackDays,
71
- projectId: process.env.AILF_REPORT_PROJECT_ID,
72
- token: process.env.AILF_REPORT_SANITY_API_TOKEN ?? process.env.SANITY_API_TOKEN,
73
- });
74
- if (!digest) {
75
- console.log(" No reports found in the lookback window. Nothing to send.");
76
- process.exit(0);
77
- }
78
- // Output
79
- console.log(` Reports found: ${digest.reportCount}`);
80
- console.log(` Overall: ${Math.round(digest.overallLatest)} (${digest.overallTrend})`);
81
- console.log(` Improved: ${digest.improved.join(", ") || "none"}`);
82
- console.log(` Regressed: ${digest.regressed.join(", ") || "none"}`);
83
- console.log(` Stable: ${digest.stable.join(", ") || "none"}`);
84
- console.log();
85
- if (JSON_OUTPUT) {
86
- console.log(JSON.stringify(digest, null, 2));
87
- return;
88
- }
89
- // Format for Slack
90
- const message = formatWeeklyDigest(digest);
91
- if (DRY_RUN) {
92
- console.log(" --- Slack Message Preview ---");
93
- console.log(` Text: ${message.text}`);
94
- console.log();
95
- for (const block of message.blocks) {
96
- if (block.text) {
97
- console.log(` [${block.type}] ${block.text.text}`);
98
- }
99
- if (block.fields) {
100
- for (const field of block.fields) {
101
- console.log(` [field] ${field.text}`);
102
- }
103
- }
104
- if (block.elements) {
105
- for (const el of block.elements) {
106
- console.log(` [element] ${el.text}`);
107
- }
108
- }
109
- }
110
- console.log();
111
- return;
112
- }
113
- // Deliver via Slack
114
- const webhookUrl = digestConfig?.slackWebhookUrl ?? process.env.SLACK_WEBHOOK_URL;
115
- if (!webhookUrl) {
116
- console.warn(" ⚠️ No Slack webhook URL configured. Set SLACK_WEBHOOK_URL or configure in schedules.yaml");
117
- console.log(" Printing digest to stdout instead:");
118
- console.log();
119
- console.log(` ${message.text}`);
120
- return;
121
- }
122
- console.log(" Sending to Slack...");
123
- try {
124
- const response = await fetch(webhookUrl, {
125
- body: JSON.stringify(message),
126
- headers: { "Content-Type": "application/json" },
127
- method: "POST",
128
- });
129
- if (response.ok) {
130
- console.log(" ✅ Digest sent successfully");
131
- }
132
- else {
133
- const text = await response.text();
134
- console.warn(` ⚠️ Slack delivery failed: ${response.status} ${text}`);
135
- }
136
- }
137
- catch (error) {
138
- console.warn(` ⚠️ Slack delivery error: ${error instanceof Error ? error.message : String(error)}`);
139
- }
140
- }
141
- main().catch((error) => {
142
- console.error("Fatal error:", error);
143
- process.exit(1);
144
- });