@akshatmittal/invoker 0.2.0 → 0.3.0

package/README.md

@@ -104,10 +104,10 @@ import { defineTask } from "@akshatmittal/invoker";
 
 export const evaluateModels = defineTask({
   name: "evaluate-models",
-  matrix: {
+  matrix: async () => ({
     model: ["gpt-5", "gpt-5-mini"],
     dataset: ["support", "sales"],
-  },
+  }),
   setup: async ({ cases }) => loadFixtures(cases),
   run: async ({ matrix, setup, vitest }) => {
     vitest.expect(setup.has(matrix.dataset)).toBe(true);
@@ -139,7 +139,8 @@ defineWorkflow({
 });
 ```
 
-Matrix literals determine the exact `matrix` type, `setup` determines the exact
+The Matrix function runs during collection and its returned literal determines the exact
+`matrix` type. `setup` determines the exact
 shared setup type, and the exact JSON return type is retained on the Task.
 Axis names must be non-empty, enumerable strings that are not array indexes.
 Omitting `matrix` creates one Case with `{}`. Setup runs once per Task, Cases

package/dist/index.d.mts

@@ -34,7 +34,7 @@ type TeardownContext<M extends Matrix, Setup> = SetupContext<M> & {
 declare const taskDefinitionBrand: unique symbol;
 interface TaskDefinition<Name extends string = string, M extends Matrix = Matrix, Setup = unknown, Output extends JsonValue = JsonValue> {
   readonly name: Name;
-  readonly matrix: M;
+  readonly matrix: () => Promise<M>;
   readonly [taskDefinitionBrand]: true;
   readonly setup?: (context: SetupContext<M>) => Awaitable<Setup>;
   readonly run: (context: TaskContext<CaseCoordinates<M>, Setup>) => Awaitable<Output>;
@@ -42,14 +42,14 @@ interface TaskDefinition<Name extends string = string, M extends Matrix = Matrix
 }
 type TaskWithSetup<Name extends string, M extends Matrix, Setup, Output extends JsonValue> = {
   readonly name: Name;
-  readonly matrix?: M;
+  readonly matrix?: () => Promise<M>;
   readonly setup: (context: SetupContext<M>) => Awaitable<Setup>;
   readonly run: (context: TaskContext<CaseCoordinates<M>, Setup>) => Awaitable<Output>;
   readonly teardown?: (context: TeardownContext<M, Setup>) => Awaitable<void>;
 };
 type TaskWithoutSetup<Name extends string, M extends Matrix, Output extends JsonValue> = {
   readonly name: Name;
-  readonly matrix?: M;
+  readonly matrix?: () => Promise<M>;
   readonly setup?: never;
   readonly run: (context: TaskContext<CaseCoordinates<M>, undefined>) => Awaitable<Output>;
   readonly teardown?: never;
@@ -58,7 +58,7 @@ declare function defineTask<const Name extends string, const M extends Matrix =
 declare function defineTask<const Name extends string, const M extends Matrix = Record<never, never>, const Output extends JsonValue = JsonValue>(definition: TaskWithoutSetup<Name, M, Output>): TaskDefinition<Name, M, undefined, Output>;
 type AnyTaskDefinition = {
   readonly name: string;
-  readonly matrix: Matrix;
+  readonly matrix: () => Promise<Matrix>;
   readonly [taskDefinitionBrand]: true;
 };
 //#endregion

package/dist/index.mjs

@@ -5,7 +5,7 @@ const taskDefinitionBrand = Symbol("invoker.task");
 function defineTask(definition) {
 	return {
 		...definition,
-		matrix: definition.matrix ?? {},
+		matrix: definition.matrix ?? (async () => ({})),
 		[taskDefinitionBrand]: true
 	};
 }
@@ -71,7 +71,6 @@ function isJsonObject(value) {
 //#region src/matrix.ts
 const axisSchema = z.string();
 function expandMatrix(matrix, owner) {
-	if (matrix === void 0) return [{}];
 	assertPlainObject(matrix, owner, ".matrix");
 	let cases = [{}];
 	for (const candidate of Reflect.ownKeys(matrix)) {
@@ -105,9 +104,9 @@ function caseName(matrix, index) {
 //#endregion
 //#region src/workflow.ts
 function defineWorkflow(definition) {
-	const workflow = prepareWorkflow(definition);
-	describe(workflow.name, { concurrent: false }, () => {
-		for (const prepared of workflow.tasks) describe(prepared.task.name, { concurrent: false }, () => {
+	describe(definition.name, { concurrent: false }, async () => {
+		const tasks = await prepareWorkflow(definition);
+		for (const prepared of tasks) describe(prepared.task.name, { concurrent: false }, () => {
 			let setup;
 			const setupTask = prepared.task.setup;
 			if (setupTask) beforeAll(async () => {
@@ -135,7 +134,7 @@ function defineWorkflow(definition) {
 		});
 	});
 }
-function prepareWorkflow(definition) {
+async function prepareWorkflow(definition) {
 	assertPlainObject(definition, "Workflow", "");
 	assertOnlyKeys(definition, [
 		"name",
@@ -148,44 +147,45 @@ function prepareWorkflow(definition) {
 	if (metadata !== void 0) assertPlainObject(metadata, `Workflow ${JSON.stringify(name)}`, ".metadata");
 	if (!Array.isArray(taskDefinitions) || taskDefinitions.length === 0) fail("Workflow", ".tasks", "expected a non-empty Task tuple");
 	const names = /* @__PURE__ */ new Set();
-	return {
-		name,
-		tasks: taskDefinitions.map((value, index) => {
-			const owner = `Workflow ${JSON.stringify(name)} Task ${index + 1}`;
-			assertPlainObject(value, owner, "");
-			if (value[taskDefinitionBrand] !== true) fail(owner, "", "expected a Task created by defineTask");
-			const task = value;
-			assertOnlyKeys(task, [
-				"name",
-				"matrix",
-				"setup",
-				"run",
-				"teardown"
-			], owner);
-			const taskSnapshot = { ...task };
-			assertName(taskSnapshot.name, owner, ".name");
-			if (names.has(taskSnapshot.name)) fail(owner, ".name", `duplicate Task name ${JSON.stringify(taskSnapshot.name)}`);
-			names.add(taskSnapshot.name);
-			if (!z.function().safeParse(taskSnapshot.run).success) fail(owner, ".run", "expected a function");
-			if (taskSnapshot.setup !== void 0 && !z.function().safeParse(taskSnapshot.setup).success) fail(owner, ".setup", "expected a function");
-			if (taskSnapshot.teardown !== void 0 && !z.function().safeParse(taskSnapshot.teardown).success) fail(owner, ".teardown", "expected a function");
-			if (taskSnapshot.teardown && !taskSnapshot.setup) fail(owner, ".teardown", "requires setup");
-			const cases = expandMatrix(taskSnapshot.matrix, `Task ${JSON.stringify(taskSnapshot.name)}`);
-			return {
-				task: taskSnapshot,
-				cases,
-				names: cases.map(caseName),
-				metadata: cases.map((matrix) => metadata === void 0 ? {
-					schema: 1,
-					matrix
-				} : {
-					schema: 1,
-					matrix,
-					metadata
-				})
-			};
-		})
-	};
+	const tasks = taskDefinitions.map((value, index) => {
+		const owner = `Workflow ${JSON.stringify(name)} Task ${index + 1}`;
+		assertPlainObject(value, owner, "");
+		if (value[taskDefinitionBrand] !== true) fail(owner, "", "expected a Task created by defineTask");
+		const task = value;
+		assertOnlyKeys(task, [
+			"name",
+			"matrix",
+			"setup",
+			"run",
+			"teardown"
+		], owner);
+		const taskSnapshot = { ...task };
+		assertName(taskSnapshot.name, owner, ".name");
+		if (names.has(taskSnapshot.name)) fail(owner, ".name", `duplicate Task name ${JSON.stringify(taskSnapshot.name)}`);
+		names.add(taskSnapshot.name);
+		if (!z.function().safeParse(taskSnapshot.run).success) fail(owner, ".run", "expected a function");
+		if (!z.function().safeParse(taskSnapshot.matrix).success) fail(owner, ".matrix", "expected a function");
+		if (taskSnapshot.setup !== void 0 && !z.function().safeParse(taskSnapshot.setup).success) fail(owner, ".setup", "expected a function");
+		if (taskSnapshot.teardown !== void 0 && !z.function().safeParse(taskSnapshot.teardown).success) fail(owner, ".teardown", "expected a function");
+		if (taskSnapshot.teardown && !taskSnapshot.setup) fail(owner, ".teardown", "requires setup");
+		return taskSnapshot;
+	});
+	return await Promise.all(tasks.map(async (task) => {
+		const cases = expandMatrix(await task.matrix(), `Task ${JSON.stringify(task.name)}`);
+		return {
+			task,
+			cases,
+			names: cases.map(caseName),
+			metadata: cases.map((matrix) => metadata === void 0 ? {
+				schema: 1,
+				matrix
+			} : {
+				schema: 1,
+				matrix,
+				metadata
+			})
+		};
+	}));
 }
 //#endregion
 export { defineTask, defineWorkflow };

package/dist/slack.mjs

@@ -48,7 +48,8 @@ function collectWorkflowReports(modules) {
 				failed: 0,
 				skipped: 0,
 				incomplete: 0,
-				failures: []
+				failures: [],
+				retries: []
 			};
 			workflow.tasks.set(taskSuite, task);
 		}
@@ -56,7 +57,17 @@ function collectWorkflowReports(modules) {
 		const result = testCase.result();
 		const diagnostic = testCase.diagnostic();
 		task[result.state === "pending" ? "incomplete" : result.state] += 1;
-		if ((diagnostic?.retryCount ?? 0) > 0) task.retried += 1;
+		const retryCount = diagnostic?.retryCount ?? 0;
+		if (retryCount > 0) {
+			task.retried += 1;
+			if (result.state === "passed") task.retries.push({
+				task: task.name,
+				caseName: testCase.name,
+				matrix: invoker.matrix,
+				count: retryCount,
+				messages: (result.errors ?? []).map(errorMessage)
+			});
+		}
 		if (diagnostic) {
 			task.startedAt = Math.min(task.startedAt ?? diagnostic.startTime, diagnostic.startTime);
 			task.endedAt = Math.max(task.endedAt ?? 0, diagnostic.startTime + diagnostic.duration);
@@ -70,28 +81,31 @@ function collectWorkflowReports(modules) {
 	}
 	return [...workflows.values()].map((workflow) => {
 		const failures = [];
+		const retries = [];
 		addErrors(failures, workflow.suite.errors());
 		addErrors(failures, workflow.module.errors());
 		const collectedTasks = [...workflow.tasks.values()];
 		for (const task of collectedTasks) {
 			failures.push(...task.failures);
+			retries.push(...task.retries);
 			addErrors(failures, task.suite.errors(), task.name);
 		}
 		const { startedAt, endedAt } = timeSpan(collectedTasks);
 		return {
 			name: workflow.name,
 			metadata: workflow.metadata,
-			tasks: collectedTasks.map(({ suite: _suite, failures: _failures, startedAt, endedAt, ...task }) => ({
+			tasks: collectedTasks.map(({ suite: _suite, failures: _failures, retries: _retries, startedAt, endedAt, ...task }) => ({
 				...task,
 				duration: startedAt === void 0 || endedAt === void 0 ? 0 : endedAt - startedAt
 			})),
 			failures: deduplicateFailures(failures),
+			retries,
 			startedAt,
 			endedAt
 		};
 	});
 }
-function summaryMessages(reports, runUrl) {
+function summaryMessage(reports, runUrl) {
 	const timestamp = Math.floor(Date.now() / 1e3);
 	const { startedAt, endedAt } = timeSpan(reports);
 	const footer = [
@@ -99,16 +113,16 @@ function summaryMessages(reports, runUrl) {
 		`<!date^${timestamp}^{date_short_pretty} at {time}|${(/* @__PURE__ */ new Date(timestamp * 1e3)).toISOString()}>`,
 		...runUrl ? [`<${escapeSlackControl(runUrl)}|View run>`] : []
 	].join(" • ");
-	return reports.flatMap(workflowAttachments).map((attachment, index) => ({
-		text: index === 0 ? "Invoker Report" : "Invoker Report (continued)",
-		attachments: [attachment, ...index === 0 ? [{ blocks: [{
+	return {
+		text: "Invoker Report",
+		attachments: [...reports.flatMap(workflowAttachments), { blocks: [{
 			type: "context",
 			elements: [{
 				type: "mrkdwn",
 				text: footer
 			}]
-		}] }] : []]
-	}));
+		}] }]
+	};
 }
 function workflowAttachments(report) {
 	const tables = taskTables(report.tasks);
@@ -194,6 +208,50 @@ function failureMessages(report) {
 		});
 	});
 }
+function retryMessages(report) {
+	return [...Map.groupBy(report.retries, (retry) => retry.task)].flatMap(([task, retries]) => {
+		const title = escapeSlack(truncate(task, NAME_CHARACTER_LIMIT));
+		const workflow = `*Workflow:* ${escapeSlack(truncate(report.name, NAME_CHARACTER_LIMIT))}`;
+		const metadata = metadataText(report.metadata);
+		const context = metadata ? `${workflow}  •  ${metadata}` : workflow;
+		return chunk(retries.map((retry) => {
+			const matrix = `\nMatrix: ${escapeSlack(JSON.stringify(retry.matrix))}`;
+			const messages = retry.messages.map((message) => `\n• ${escapeSlack(message)}`).join("");
+			return `*${escapeSlack(retry.caseName)}*\nRetries: ${retry.count}${matrix}${messages}`;
+		}), SECTION_CHARACTER_LIMIT).map((details, index, messages) => {
+			const part = messages.length > 1 ? ` (${index + 1}/${messages.length})` : "";
+			return {
+				text: `${truncate(report.name, NAME_CHARACTER_LIMIT)} › ${truncate(task, NAME_CHARACTER_LIMIT)} — ${retries.length} retried${part}`,
+				attachments: [{
+					color: "warning",
+					blocks: [
+						{
+							type: "section",
+							text: {
+								type: "mrkdwn",
+								text: `🟡 *${title} — ${retries.length} retried${part}*`
+							}
+						},
+						{
+							type: "context",
+							elements: [{
+								type: "mrkdwn",
+								text: context
+							}]
+						},
+						{
+							type: "section",
+							text: {
+								type: "mrkdwn",
+								text: details
+							}
+						}
+					]
+				}]
+			};
+		});
+	});
+}
 function unhandledErrorMessages(errors) {
 	const messages = [...new Set(errors.map(errorMessage))];
 	return chunk(messages.map((message) => `• ${escapeSlack(message)}`), SECTION_CHARACTER_LIMIT).map((details, index, chunks) => {
@@ -382,8 +440,7 @@ function slackReporter(options) {
 	return { async onTestRunEnd(modules, unhandledErrors) {
 		const reports = collectWorkflowReports(modules);
 		if (reports.length === 0) return;
-		const [parentMessage, ...continuations] = summaryMessages(reports, options.runUrl);
-		if (!parentMessage) return;
+		const parentMessage = summaryMessage(reports, options.runUrl);
 		let parentTimestamp;
 		try {
 			const parentArguments = {
@@ -398,8 +455,8 @@ function slackReporter(options) {
 			return;
 		}
 		const replies = [
-			...continuations,
 			...reports.flatMap(failureMessages),
+			...reports.flatMap(retryMessages),
 			...unhandledErrorMessages(unhandledErrors)
 		];
 		if (replies.length > 0 && !parentTimestamp) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@akshatmittal/invoker",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Typed matrix-driven regression workflows for Vitest.",
   "keywords": [
     "matrix",