@sanity/ailf 3.8.0 → 3.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/canary-tasks.ts +64 -0
- package/config/test-budgets.ts +24 -0
- package/dist/_vendor/ailf-core/config-helpers.d.ts +19 -0
- package/dist/_vendor/ailf-core/config-helpers.js +27 -0
- package/dist/_vendor/ailf-core/index.d.ts +1 -1
- package/dist/_vendor/ailf-core/index.js +1 -1
- package/dist/_vendor/ailf-core/schemas/canary-tasks.d.ts +52 -0
- package/dist/_vendor/ailf-core/schemas/canary-tasks.js +46 -0
- package/dist/_vendor/ailf-core/schemas/index.d.ts +2 -0
- package/dist/_vendor/ailf-core/schemas/index.js +2 -0
- package/dist/_vendor/ailf-core/schemas/test-budgets.d.ts +19 -0
- package/dist/_vendor/ailf-core/schemas/test-budgets.js +34 -0
- package/dist/_vendor/ailf-shared/canary-drift.d.ts +84 -0
- package/dist/_vendor/ailf-shared/canary-drift.js +86 -0
- package/dist/_vendor/ailf-shared/index.d.ts +1 -0
- package/dist/_vendor/ailf-shared/index.js +1 -0
- package/dist/adapters/config-sources/file-config-adapter.js +4 -5
- package/dist/adapters/task-sources/repo-schemas.d.ts +3 -3
- package/dist/cli-program.d.ts +39 -0
- package/dist/cli-program.js +137 -0
- package/dist/cli.d.ts +6 -0
- package/dist/cli.js +12 -122
- package/dist/config/canary-tasks.ts +64 -0
- package/dist/config/test-budgets.ts +24 -0
- package/dist/pipeline/calculate-scores.d.ts +17 -2
- package/dist/pipeline/calculate-scores.js +99 -0
- package/dist/pipeline/compiler/mode-handlers/knowledge-probe/assertions.d.ts +5 -0
- package/dist/pipeline/compiler/mode-handlers/knowledge-probe/assertions.js +25 -2
- package/dist/pipeline/compiler/mode-handlers/knowledge-probe/index.js +5 -1
- package/dist/pipeline/compiler/mode-handlers/knowledge-probe/types.d.ts +4 -0
- package/dist/pipeline/compiler/promptfoo-compiler.js +23 -0
- package/dist/pipeline/mirror-repo-tasks.d.ts +1 -1
- package/dist/tasks/knowledge-probe/groq-projections.task.ts +29 -11
- package/package.json +6 -3
- package/tasks/knowledge-probe/groq-projections.task.ts +29 -11
- package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.d.ts +0 -10
- package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.js +0 -366
- package/dist/pipeline/compiler/__tests__/assertion-mapper.test.d.ts +0 -9
- package/dist/pipeline/compiler/__tests__/assertion-mapper.test.js +0 -145
- package/dist/pipeline/compiler/__tests__/knowledge-probe-handler.test.d.ts +0 -10
- package/dist/pipeline/compiler/__tests__/knowledge-probe-handler.test.js +0 -314
- package/dist/pipeline/compiler/__tests__/literacy-handler.test.d.ts +0 -10
- package/dist/pipeline/compiler/__tests__/literacy-handler.test.js +0 -486
- package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.d.ts +0 -10
- package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.js +0 -425
- package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.d.ts +0 -9
- package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.js +0 -332
- package/dist/pipeline/compiler/__tests__/sandbox-and-fixtures.test.d.ts +0 -12
- package/dist/pipeline/compiler/__tests__/sandbox-and-fixtures.test.js +0 -210
- package/dist/pipeline/compiler/__tests__/scoring-and-presets.test.d.ts +0 -7
- package/dist/pipeline/compiler/__tests__/scoring-and-presets.test.js +0 -404
- package/dist/pipeline/compiler/__tests__/scoring-bridge.test.d.ts +0 -10
- package/dist/pipeline/compiler/__tests__/scoring-bridge.test.js +0 -184
- package/dist/pipeline/compiler/__tests__/task-graph-builder.test.d.ts +0 -8
- package/dist/pipeline/compiler/__tests__/task-graph-builder.test.js +0 -301
- package/dist/pipeline/compiler/__tests__/telemetry.test.d.ts +0 -9
- package/dist/pipeline/compiler/__tests__/telemetry.test.js +0 -503
- package/dist/pipeline/compiler/__tests__/tool-loop-openai.test.d.ts +0 -10
- package/dist/pipeline/compiler/__tests__/tool-loop-openai.test.js +0 -509
|
@@ -1,301 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* task-graph-builder.test.ts — Unit tests for TaskGraphBuilder.
|
|
3
|
-
*
|
|
4
|
-
* Tests DAG construction, cycle detection, filtering, and priority assignment.
|
|
5
|
-
*
|
|
6
|
-
* Run: npx tsx --test src/pipeline/compiler/__tests__/task-graph-builder.test.ts
|
|
7
|
-
*/
|
|
8
|
-
import assert from "node:assert/strict";
|
|
9
|
-
import { describe, it } from "node:test";
|
|
10
|
-
import { buildTaskGraph, detectCycle } from "../task-graph-builder.js";
|
|
11
|
-
// ---------------------------------------------------------------------------
|
|
12
|
-
// Helpers
|
|
13
|
-
// ---------------------------------------------------------------------------
|
|
14
|
-
/**
|
|
15
|
-
* Build a LiteracyTaskDefinition for testing. The task-graph-builder now
|
|
16
|
-
* accepts GeneralizedTaskDefinition[] and LiteracyTaskDefinition is a
|
|
17
|
-
* valid variant of that union.
|
|
18
|
-
*/
|
|
19
|
-
function makeTask(overrides) {
|
|
20
|
-
return {
|
|
21
|
-
mode: "literacy",
|
|
22
|
-
title: `Task ${overrides.id}`,
|
|
23
|
-
area: overrides.area ?? "groq",
|
|
24
|
-
prompt: overrides.prompt ?? { text: `Do ${overrides.id}` },
|
|
25
|
-
...overrides,
|
|
26
|
-
};
|
|
27
|
-
}
|
|
28
|
-
// ---------------------------------------------------------------------------
|
|
29
|
-
// buildTaskGraph — basic construction
|
|
30
|
-
// ---------------------------------------------------------------------------
|
|
31
|
-
describe("buildTaskGraph", () => {
|
|
32
|
-
it("builds a graph from a list of tasks", () => {
|
|
33
|
-
const result = buildTaskGraph({
|
|
34
|
-
tasks: [makeTask({ id: "task-a" }), makeTask({ id: "task-b" })],
|
|
35
|
-
});
|
|
36
|
-
assert.ok(result.graph);
|
|
37
|
-
assert.equal(result.graph.nodes.size, 2);
|
|
38
|
-
assert.ok(result.graph.nodes.has("task-a"));
|
|
39
|
-
assert.ok(result.graph.nodes.has("task-b"));
|
|
40
|
-
assert.equal(result.warnings.length, 0);
|
|
41
|
-
});
|
|
42
|
-
it("returns null graph when no tasks match filter", () => {
|
|
43
|
-
const result = buildTaskGraph({
|
|
44
|
-
tasks: [makeTask({ id: "task-a", area: "groq" })],
|
|
45
|
-
filter: { areas: ["studio"] },
|
|
46
|
-
});
|
|
47
|
-
assert.equal(result.graph, null);
|
|
48
|
-
assert.equal(result.filteredOut.length, 1);
|
|
49
|
-
assert.ok(result.filteredOut.includes("task-a"));
|
|
50
|
-
});
|
|
51
|
-
it("filters by area", () => {
|
|
52
|
-
const result = buildTaskGraph({
|
|
53
|
-
tasks: [
|
|
54
|
-
makeTask({ id: "groq-1", area: "groq" }),
|
|
55
|
-
makeTask({ id: "studio-1", area: "studio" }),
|
|
56
|
-
],
|
|
57
|
-
filter: { areas: ["groq"] },
|
|
58
|
-
});
|
|
59
|
-
assert.ok(result.graph);
|
|
60
|
-
assert.equal(result.graph.nodes.size, 1);
|
|
61
|
-
assert.ok(result.graph.nodes.has("groq-1"));
|
|
62
|
-
});
|
|
63
|
-
it("filters by task IDs", () => {
|
|
64
|
-
const result = buildTaskGraph({
|
|
65
|
-
tasks: [makeTask({ id: "task-a" }), makeTask({ id: "task-b" })],
|
|
66
|
-
filter: { taskIds: ["task-a"] },
|
|
67
|
-
});
|
|
68
|
-
assert.ok(result.graph);
|
|
69
|
-
assert.equal(result.graph.nodes.size, 1);
|
|
70
|
-
});
|
|
71
|
-
it("filters by tags", () => {
|
|
72
|
-
const result = buildTaskGraph({
|
|
73
|
-
tasks: [
|
|
74
|
-
makeTask({ id: "task-a", tags: ["groq", "basics"] }),
|
|
75
|
-
makeTask({ id: "task-b", tags: ["studio"] }),
|
|
76
|
-
],
|
|
77
|
-
filter: { tags: ["groq"] },
|
|
78
|
-
});
|
|
79
|
-
assert.ok(result.graph);
|
|
80
|
-
assert.equal(result.graph.nodes.size, 1);
|
|
81
|
-
assert.ok(result.graph.nodes.has("task-a"));
|
|
82
|
-
});
|
|
83
|
-
it("filters out archived tasks", () => {
|
|
84
|
-
const result = buildTaskGraph({
|
|
85
|
-
tasks: [
|
|
86
|
-
makeTask({ id: "active" }),
|
|
87
|
-
makeTask({ id: "archived", status: "archived" }),
|
|
88
|
-
],
|
|
89
|
-
});
|
|
90
|
-
assert.ok(result.graph);
|
|
91
|
-
assert.equal(result.graph.nodes.size, 1);
|
|
92
|
-
assert.ok(result.graph.nodes.has("active"));
|
|
93
|
-
});
|
|
94
|
-
it("filters out paused tasks unless targeted by ID", () => {
|
|
95
|
-
const result = buildTaskGraph({
|
|
96
|
-
tasks: [makeTask({ id: "paused-task", status: "paused" })],
|
|
97
|
-
filter: { taskIds: ["paused-task"] },
|
|
98
|
-
});
|
|
99
|
-
assert.ok(result.graph);
|
|
100
|
-
assert.equal(result.graph.nodes.size, 1);
|
|
101
|
-
});
|
|
102
|
-
it("warns about duplicate task IDs", () => {
|
|
103
|
-
const result = buildTaskGraph({
|
|
104
|
-
tasks: [makeTask({ id: "dup" }), makeTask({ id: "dup", area: "studio" })],
|
|
105
|
-
});
|
|
106
|
-
assert.ok(result.graph);
|
|
107
|
-
assert.equal(result.graph.nodes.size, 1);
|
|
108
|
-
assert.ok(result.warnings.some((w) => w.includes("Duplicate")));
|
|
109
|
-
});
|
|
110
|
-
it("resolves task variables into envelope", () => {
|
|
111
|
-
const result = buildTaskGraph({
|
|
112
|
-
tasks: [makeTask({ id: "task-a", prompt: { text: "Do the thing" } })],
|
|
113
|
-
});
|
|
114
|
-
assert.ok(result.graph);
|
|
115
|
-
const node = result.graph.nodes.get("task-a");
|
|
116
|
-
assert.equal(node.resolvedVariables.values.task, "Do the thing");
|
|
117
|
-
});
|
|
118
|
-
it("includes prompt.vars in the variable envelope", () => {
|
|
119
|
-
const result = buildTaskGraph({
|
|
120
|
-
tasks: [
|
|
121
|
-
makeTask({
|
|
122
|
-
id: "task-a",
|
|
123
|
-
prompt: { text: "Do task-a", vars: { docs: "file://ctx.md" } },
|
|
124
|
-
}),
|
|
125
|
-
],
|
|
126
|
-
});
|
|
127
|
-
assert.ok(result.graph);
|
|
128
|
-
const node = result.graph.nodes.get("task-a");
|
|
129
|
-
assert.equal(node.resolvedVariables.values.docs, "file://ctx.md");
|
|
130
|
-
});
|
|
131
|
-
});
|
|
132
|
-
// ---------------------------------------------------------------------------
|
|
133
|
-
// Dependency edges and ordering
|
|
134
|
-
// ---------------------------------------------------------------------------
|
|
135
|
-
describe("buildTaskGraph — dependencies", () => {
|
|
136
|
-
it("resolves explicit dependsOn edges", () => {
|
|
137
|
-
const result = buildTaskGraph({
|
|
138
|
-
tasks: [
|
|
139
|
-
makeTask({ id: "base" }),
|
|
140
|
-
makeTask({
|
|
141
|
-
id: "derived",
|
|
142
|
-
prompt: { text: "Do derived", vars: { dependsOn: ["base"] } },
|
|
143
|
-
}),
|
|
144
|
-
],
|
|
145
|
-
});
|
|
146
|
-
assert.ok(result.graph);
|
|
147
|
-
assert.equal(result.graph.edges.length, 1);
|
|
148
|
-
assert.equal(result.graph.edges[0].from, "base");
|
|
149
|
-
assert.equal(result.graph.edges[0].to, "derived");
|
|
150
|
-
});
|
|
151
|
-
it("assigns lower priority to upstream nodes", () => {
|
|
152
|
-
const result = buildTaskGraph({
|
|
153
|
-
tasks: [
|
|
154
|
-
makeTask({ id: "step-1" }),
|
|
155
|
-
makeTask({
|
|
156
|
-
id: "step-2",
|
|
157
|
-
prompt: { text: "Do step-2", vars: { dependsOn: ["step-1"] } },
|
|
158
|
-
}),
|
|
159
|
-
makeTask({
|
|
160
|
-
id: "step-3",
|
|
161
|
-
prompt: { text: "Do step-3", vars: { dependsOn: ["step-2"] } },
|
|
162
|
-
}),
|
|
163
|
-
],
|
|
164
|
-
});
|
|
165
|
-
assert.ok(result.graph);
|
|
166
|
-
const p1 = result.graph.nodes.get("step-1").priority;
|
|
167
|
-
const p2 = result.graph.nodes.get("step-2").priority;
|
|
168
|
-
const p3 = result.graph.nodes.get("step-3").priority;
|
|
169
|
-
assert.ok(p1 < p2, `step-1 priority (${p1}) should be < step-2 (${p2})`);
|
|
170
|
-
assert.ok(p2 < p3, `step-2 priority (${p2}) should be < step-3 (${p3})`);
|
|
171
|
-
});
|
|
172
|
-
it("warns when dependency target is not in graph", () => {
|
|
173
|
-
const result = buildTaskGraph({
|
|
174
|
-
tasks: [
|
|
175
|
-
makeTask({
|
|
176
|
-
id: "task-a",
|
|
177
|
-
prompt: { text: "Do task-a", vars: { dependsOn: ["missing"] } },
|
|
178
|
-
}),
|
|
179
|
-
],
|
|
180
|
-
});
|
|
181
|
-
assert.ok(result.graph);
|
|
182
|
-
assert.ok(result.warnings.some((w) => w.includes("missing")));
|
|
183
|
-
});
|
|
184
|
-
});
|
|
185
|
-
// ---------------------------------------------------------------------------
|
|
186
|
-
// Cycle detection
|
|
187
|
-
// ---------------------------------------------------------------------------
|
|
188
|
-
describe("detectCycle", () => {
|
|
189
|
-
it("returns null for acyclic graph", () => {
|
|
190
|
-
const nodes = new Map([
|
|
191
|
-
[
|
|
192
|
-
"a",
|
|
193
|
-
{
|
|
194
|
-
taskId: "a",
|
|
195
|
-
dependsOn: [],
|
|
196
|
-
priority: 0,
|
|
197
|
-
resolvedPrompt: "",
|
|
198
|
-
resolvedVariables: { values: {}, provenance: {}, declarations: [] },
|
|
199
|
-
},
|
|
200
|
-
],
|
|
201
|
-
[
|
|
202
|
-
"b",
|
|
203
|
-
{
|
|
204
|
-
taskId: "b",
|
|
205
|
-
dependsOn: ["a"],
|
|
206
|
-
priority: 0,
|
|
207
|
-
resolvedPrompt: "",
|
|
208
|
-
resolvedVariables: { values: {}, provenance: {}, declarations: [] },
|
|
209
|
-
},
|
|
210
|
-
],
|
|
211
|
-
]);
|
|
212
|
-
const edges = [{ from: "a", to: "b", type: "ordering" }];
|
|
213
|
-
assert.equal(detectCycle(nodes, edges), null);
|
|
214
|
-
});
|
|
215
|
-
it("detects simple two-node cycle", () => {
|
|
216
|
-
const nodes = new Map([
|
|
217
|
-
[
|
|
218
|
-
"a",
|
|
219
|
-
{
|
|
220
|
-
taskId: "a",
|
|
221
|
-
dependsOn: ["b"],
|
|
222
|
-
priority: 0,
|
|
223
|
-
resolvedPrompt: "",
|
|
224
|
-
resolvedVariables: { values: {}, provenance: {}, declarations: [] },
|
|
225
|
-
},
|
|
226
|
-
],
|
|
227
|
-
[
|
|
228
|
-
"b",
|
|
229
|
-
{
|
|
230
|
-
taskId: "b",
|
|
231
|
-
dependsOn: ["a"],
|
|
232
|
-
priority: 0,
|
|
233
|
-
resolvedPrompt: "",
|
|
234
|
-
resolvedVariables: { values: {}, provenance: {}, declarations: [] },
|
|
235
|
-
},
|
|
236
|
-
],
|
|
237
|
-
]);
|
|
238
|
-
const edges = [
|
|
239
|
-
{ from: "a", to: "b", type: "ordering" },
|
|
240
|
-
{ from: "b", to: "a", type: "ordering" },
|
|
241
|
-
];
|
|
242
|
-
const cycle = detectCycle(nodes, edges);
|
|
243
|
-
assert.ok(cycle, "Should detect a cycle");
|
|
244
|
-
assert.ok(cycle.length >= 2, "Cycle should have at least 2 nodes");
|
|
245
|
-
});
|
|
246
|
-
it("detects three-node cycle", () => {
|
|
247
|
-
const nodes = new Map([
|
|
248
|
-
[
|
|
249
|
-
"a",
|
|
250
|
-
{
|
|
251
|
-
taskId: "a",
|
|
252
|
-
dependsOn: [],
|
|
253
|
-
priority: 0,
|
|
254
|
-
resolvedPrompt: "",
|
|
255
|
-
resolvedVariables: { values: {}, provenance: {}, declarations: [] },
|
|
256
|
-
},
|
|
257
|
-
],
|
|
258
|
-
[
|
|
259
|
-
"b",
|
|
260
|
-
{
|
|
261
|
-
taskId: "b",
|
|
262
|
-
dependsOn: [],
|
|
263
|
-
priority: 0,
|
|
264
|
-
resolvedPrompt: "",
|
|
265
|
-
resolvedVariables: { values: {}, provenance: {}, declarations: [] },
|
|
266
|
-
},
|
|
267
|
-
],
|
|
268
|
-
[
|
|
269
|
-
"c",
|
|
270
|
-
{
|
|
271
|
-
taskId: "c",
|
|
272
|
-
dependsOn: [],
|
|
273
|
-
priority: 0,
|
|
274
|
-
resolvedPrompt: "",
|
|
275
|
-
resolvedVariables: { values: {}, provenance: {}, declarations: [] },
|
|
276
|
-
},
|
|
277
|
-
],
|
|
278
|
-
]);
|
|
279
|
-
const edges = [
|
|
280
|
-
{ from: "a", to: "b", type: "ordering" },
|
|
281
|
-
{ from: "b", to: "c", type: "ordering" },
|
|
282
|
-
{ from: "c", to: "a", type: "ordering" },
|
|
283
|
-
];
|
|
284
|
-
const cycle = detectCycle(nodes, edges);
|
|
285
|
-
assert.ok(cycle, "Should detect a cycle");
|
|
286
|
-
});
|
|
287
|
-
it("throws when building a graph with cycles", () => {
|
|
288
|
-
assert.throws(() => buildTaskGraph({
|
|
289
|
-
tasks: [
|
|
290
|
-
makeTask({
|
|
291
|
-
id: "a",
|
|
292
|
-
prompt: { text: "Do a", vars: { dependsOn: ["b"] } },
|
|
293
|
-
}),
|
|
294
|
-
makeTask({
|
|
295
|
-
id: "b",
|
|
296
|
-
prompt: { text: "Do b", vars: { dependsOn: ["a"] } },
|
|
297
|
-
}),
|
|
298
|
-
],
|
|
299
|
-
}), /cycle/i);
|
|
300
|
-
});
|
|
301
|
-
});
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* telemetry.test.ts — Tests for the observability & telemetry subsystem.
|
|
3
|
-
*
|
|
4
|
-
* Covers tool call classification, trace collection, cost tracking,
|
|
5
|
-
* redaction pipeline, trace storage, and per-turn trace merging.
|
|
6
|
-
*
|
|
7
|
-
* Run: npx tsx --test src/pipeline/compiler/__tests__/telemetry.test.ts
|
|
8
|
-
*/
|
|
9
|
-
export {};
|