@akshatmittal/invoker 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Akshat Mittal
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,128 @@
+# @akshatmittal/invoker
+
+A strictly typed TypeScript DSL for matrix-driven regression workflows. Invoker
+expands Tasks into Vitest tests, runs each Task's Cases concurrently, and stores
+validated JSON Output in Vitest metadata for reporters and later analysis.
+
+## Install
+
+```sh
+pnpm add -D @akshatmittal/invoker vitest
+```
+
+Invoker supports Node 24 and Vitest 4.1.10 or newer within Vitest 4.
+
+## Define a Workflow
+
+```ts
+// regressions/model/tasks/evaluate-models.ts
+import { defineTask } from "@akshatmittal/invoker";
+
+export const evaluateModels = defineTask({
+  name: "evaluate-models",
+  matrix: {
+    model: ["gpt-5", "gpt-5-mini"],
+    dataset: ["support", "sales"],
+  },
+  setup: async ({ cases }) => loadFixtures(cases),
+  run: async ({ matrix, setup, vitest }) => {
+    vitest.expect(setup.has(matrix.dataset)).toBe(true);
+
+    return {
+      model: matrix.model,
+      dataset: matrix.dataset,
+      score: await evaluate(matrix, setup),
+    };
+  },
+  teardown: async ({ setup }) => {
+    await setup.close();
+  },
+});
+```
+
+```ts
+// regressions/model/index.test.ts
+import { defineWorkflow } from "@akshatmittal/invoker";
+import { evaluateModels } from "./tasks/evaluate-models.js";
+
+defineWorkflow({
+  name: "model-regressions",
+  metadata: {
+    commit: process.env.GITHUB_SHA ?? "local",
+    baseline: "2026-08-01",
+  },
+  tasks: [evaluateModels],
+});
+```
+
+Matrix literals determine the exact `matrix` type, `setup` determines the exact
+shared setup type, and the exact JSON return type is retained on the Task.
+Omitting `matrix` creates one Case with `{}`. Setup runs once per Task, Cases
+within that Task run concurrently, and teardown runs once after successful
+setup. Tasks run sequentially in their Workflow.
+
+## Configure Vitest
+
+Invoker uses Vitest's built-in reporters. The JSON reporter includes each
+Case's data at `assertionResults[].meta.invoker`:
+
+```ts
+// vitest.config.ts
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    maxConcurrency: 5,
+    reporters: ["default", "json", ...(process.env.GITHUB_ACTIONS === "true" ? ["github-actions" as const] : [])],
+    outputFile: {
+      json: "./artifacts/invoker-results.json",
+    },
+  },
+});
+```
+
+Run every Workflow or filter to one Task with ordinary Vitest commands:
+
+```sh
+pnpm vitest run
+pnpm vitest run regressions/model/index.test.ts -t evaluate-models
+```
+
+The metadata envelope is stable and JSON-compatible:
+
+```json
+{
+  "schema": 1,
+  "matrix": { "model": "gpt-5", "dataset": "support" },
+  "metadata": { "commit": "abc123" },
+  "output": { "model": "gpt-5", "dataset": "support", "score": 0.92 }
+}
+```
+
+`output` is present only after a successful, JSON-valid Task return. Vitest's
+report remains authoritative for status, failures, timing, hierarchy, and
+retries.
+
+## GitHub Actions artifacts
+
+Create the output directory before Vitest and upload the report even when the
+run fails:
+
+```yaml
+- run: mkdir -p artifacts && pnpm vitest run
+- if: always()
+  uses: actions/upload-artifact@v4
+  with:
+    name: invoker-results
+    path: artifacts/invoker-results.json
+```
+
+The artifact provides per-run retention and can be downloaded later for custom
+queries or reports. Invoker does not upload, index, or persist results itself.
+
+## v1 scope
+
+Invoker does not provide a CLI, directory discovery, custom runner, custom
+reporter, configuration helper, Task-level parallelism, matrix include/exclude,
+or hosted result storage. Use Vitest configuration and your CI runner for those
+concerns.

package/dist/index.d.mts

@@ -0,0 +1,79 @@
+import { TestContext } from "vitest";
+//#region src/types.d.ts
+type JsonPrimitive = string | number | boolean | null;
+type JsonValue = JsonPrimitive | readonly JsonValue[] | {
+  readonly [key: string]: JsonValue;
+};
+type JsonObject = {
+  readonly [key: string]: JsonValue;
+};
+type Matrix = {
+  readonly [axis: string]: readonly JsonValue[];
+};
+type CaseCoordinates<M extends Matrix> = { readonly [K in keyof M]: M[K][number]; };
+interface InvokerMeta<Coordinates extends JsonObject, Output extends JsonValue, Metadata extends JsonObject = JsonObject> {
+  schema: 1;
+  matrix: Coordinates;
+  metadata?: Metadata;
+  output?: Output;
+}
+interface TaskContext<Coordinates, Setup> {
+  readonly matrix: Coordinates;
+  readonly setup: Setup;
+  readonly vitest: TestContext;
+}
+type Awaitable<Value> = Value | PromiseLike<Value>;
+type SetupContext<M extends Matrix> = {
+  readonly cases: readonly CaseCoordinates<M>[];
+};
+type TeardownContext<M extends Matrix, Setup> = SetupContext<M> & {
+  readonly setup: Setup;
+};
+//#endregion
+//#region src/task.d.ts
+declare const taskDefinitionBrand: unique symbol;
+interface TaskDefinition<Name extends string = string, M extends Matrix = Matrix, Setup = unknown, Output extends JsonValue = JsonValue> {
+  readonly name: Name;
+  readonly matrix: M;
+  readonly [taskDefinitionBrand]: {
+    readonly setup: Setup;
+    readonly output: Output;
+  };
+  readonly setup?: (context: SetupContext<M>) => Awaitable<Setup>;
+  readonly run: (context: TaskContext<CaseCoordinates<M>, Setup>) => Awaitable<Output>;
+  readonly teardown?: (context: TeardownContext<M, Setup>) => Awaitable<void>;
+}
+type TaskWithSetup<Name extends string, M extends Matrix, Setup, Output extends JsonValue> = {
+  readonly name: Name;
+  readonly matrix?: M;
+  readonly setup: (context: SetupContext<M>) => Awaitable<Setup>;
+  readonly run: (context: TaskContext<CaseCoordinates<M>, Setup>) => Awaitable<Output>;
+  readonly teardown?: (context: TeardownContext<M, Setup>) => Awaitable<void>;
+};
+type TaskWithoutSetup<Name extends string, M extends Matrix, Output extends JsonValue> = {
+  readonly name: Name;
+  readonly matrix?: M;
+  readonly setup?: never;
+  readonly run: (context: TaskContext<CaseCoordinates<M>, undefined>) => Awaitable<Output>;
+  readonly teardown?: never;
+};
+declare function defineTask<const Name extends string, const M extends Matrix = Record<never, never>, Setup = unknown, const Output extends JsonValue = JsonValue>(definition: TaskWithSetup<Name, M, Setup, Output>): TaskDefinition<Name, M, Setup, Output>;
+declare function defineTask<const Name extends string, const M extends Matrix = Record<never, never>, const Output extends JsonValue = JsonValue>(definition: TaskWithoutSetup<Name, M, Output>): TaskDefinition<Name, M, undefined, Output>;
+type AnyTaskDefinition = {
+  readonly name: string;
+  readonly matrix: Matrix;
+  readonly [taskDefinitionBrand]: {
+    readonly setup: unknown;
+    readonly output: JsonValue;
+  };
+};
+//#endregion
+//#region src/workflow.d.ts
+type WorkflowDefinition<Tasks extends readonly [AnyTaskDefinition, ...AnyTaskDefinition[]], Metadata extends JsonObject> = {
+  readonly name: string;
+  readonly metadata?: Metadata;
+  readonly tasks: Tasks;
+};
+declare function defineWorkflow<const Tasks extends readonly [AnyTaskDefinition, ...AnyTaskDefinition[]], const Metadata extends JsonObject = JsonObject>(definition: WorkflowDefinition<Tasks, Metadata>): void;
+//#endregion
+export { type CaseCoordinates, type InvokerMeta, type JsonObject, type JsonPrimitive, type JsonValue, type Matrix, type TaskContext, type TaskDefinition, defineTask, defineWorkflow };

package/dist/index.mjs

@@ -0,0 +1,174 @@
+import { beforeAll, describe, test } from "vitest";
+//#region src/task.ts
+const taskDefinitionBrand = Symbol("invoker.task");
+function defineTask(definition) {
+	return {
+		...definition,
+		matrix: definition.matrix ?? {},
+		[taskDefinitionBrand]: true
+	};
+}
+//#endregion
+//#region src/json.ts
+function assertJson(value, owner, path, ancestors = /* @__PURE__ */ new Set()) {
+	if (value === null || typeof value === "string" || typeof value === "boolean") return;
+	if (typeof value === "number") {
+		if (!Number.isFinite(value)) fail(owner, path, "expected a finite number");
+		return;
+	}
+	if (typeof value !== "object") fail(owner, path, `expected JSON, received ${typeof value}`);
+	if (ancestors.has(value)) fail(owner, path, "cyclic values are not JSON");
+	ancestors.add(value);
+	if (Array.isArray(value)) for (let index = 0; index < value.length; index += 1) {
+		if (!(index in value)) fail(owner, `${path}[${index}]`, "sparse arrays are not JSON");
+		assertJson(value[index], owner, `${path}[${index}]`, ancestors);
+	}
+	else {
+		assertPlainObject(value, owner, path);
+		if (Object.getOwnPropertySymbols(value).length > 0) fail(owner, path, "JSON objects cannot have symbol keys");
+		for (const [key, child] of Object.entries(value)) assertJson(child, owner, `${path}.${key}`, ancestors);
+	}
+	ancestors.delete(value);
+}
+function assertPlainObject(value, owner, path) {
+	if (value === null || typeof value !== "object" || Array.isArray(value)) fail(owner, path, "expected a plain object");
+	const prototype = Object.getPrototypeOf(value);
+	if (prototype !== Object.prototype && prototype !== null) fail(owner, path, "expected a plain object");
+}
+function assertName(value, owner, path) {
+	if (typeof value !== "string" || value.trim() === "") fail(owner, path, "expected a non-empty string");
+}
+function assertOnlyKeys(value, allowed, owner) {
+	for (const key of Object.keys(value)) if (!allowed.includes(key)) fail(owner, `.${key}`, "unknown property");
+}
+function canonicalJson(value) {
+	if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
+	if (value !== null && typeof value === "object") {
+		const object = value;
+		return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(object[key])}`).join(",")}}`;
+	}
+	return JSON.stringify(value);
+}
+function fail(owner, path, message) {
+	throw new TypeError(`${owner}${path}: ${message}`);
+}
+//#endregion
+//#region src/matrix.ts
+function expandMatrix(matrix, owner) {
+	if (matrix === void 0) return [{}];
+	assertPlainObject(matrix, owner, ".matrix");
+	let cases = [{}];
+	for (const [axis, values] of Object.entries(matrix)) {
+		if (axis.trim() === "") fail(owner, ".matrix", "axis names must not be empty");
+		if (!Array.isArray(values)) fail(owner, `.matrix.${axis}`, "expected an array");
+		if (values.length === 0) fail(owner, `.matrix.${axis}`, "expected at least one value");
+		values.forEach((value, index) => {
+			assertJson(value, owner, `.matrix.${axis}[${index}]`);
+		});
+		cases = cases.flatMap((coordinates) => values.map((value) => ({
+			...coordinates,
+			[axis]: value
+		})));
+	}
+	const coordinates = /* @__PURE__ */ new Set();
+	for (const value of cases) {
+		const key = canonicalJson(value);
+		if (coordinates.has(key)) fail(owner, ".matrix", `duplicate coordinate ${JSON.stringify(value)}`);
+		coordinates.add(key);
+	}
+	return cases;
+}
+function caseName(matrix, index) {
+	const axes = Object.entries(matrix).map(([axis, value]) => `${axis}=${JSON.stringify(value)}`).join(", ");
+	return axes === "" ? `[${index + 1}]` : `[${index + 1}] ${axes}`;
+}
+//#endregion
+//#region src/workflow.ts
+function defineWorkflow(definition) {
+	const workflow = prepareWorkflow(definition);
+	describe(workflow.name, { concurrent: false }, () => {
+		for (const prepared of workflow.tasks) describe(prepared.task.name, { concurrent: false }, () => {
+			let setup;
+			const setupTask = prepared.task.setup;
+			if (setupTask) beforeAll(async () => {
+				setup = await setupTask({ cases: prepared.cases });
+				const teardownTask = prepared.task.teardown;
+				if (teardownTask) return () => teardownTask({
+					cases: prepared.cases,
+					setup
+				});
+			});
+			for (const [index, matrix] of prepared.cases.entries()) {
+				const name = prepared.names[index];
+				const invoker = prepared.metadata[index];
+				test.concurrent(name, { meta: { invoker } }, async (vitest) => {
+					const meta = vitest.task.meta;
+					delete meta.invoker.output;
+					const output = await prepared.task.run({
+						matrix,
+						setup,
+						vitest
+					});
+					assertJson(output, `Task ${JSON.stringify(prepared.task.name)}`, ".output");
+					meta.invoker.output = output;
+				});
+			}
+		});
+	});
+}
+function prepareWorkflow(definition) {
+	assertPlainObject(definition, "Workflow", "");
+	assertOnlyKeys(definition, [
+		"name",
+		"metadata",
+		"tasks"
+	], "Workflow");
+	assertName(definition.name, "Workflow", ".name");
+	const metadata = definition.metadata;
+	if (metadata !== void 0) {
+		assertJson(metadata, `Workflow ${JSON.stringify(definition.name)}`, ".metadata");
+		assertPlainObject(metadata, `Workflow ${JSON.stringify(definition.name)}`, ".metadata");
+	}
+	if (!Array.isArray(definition.tasks) || definition.tasks.length === 0) fail("Workflow", ".tasks", "expected a non-empty Task tuple");
+	const names = /* @__PURE__ */ new Set();
+	const tasks = definition.tasks.map((value, index) => {
+		const owner = `Workflow ${JSON.stringify(definition.name)} Task ${index + 1}`;
+		assertPlainObject(value, owner, "");
+		const task = value;
+		if (task[taskDefinitionBrand] !== true) fail(owner, "", "expected a Task created by defineTask");
+		assertOnlyKeys(task, [
+			"name",
+			"matrix",
+			"setup",
+			"run",
+			"teardown"
+		], owner);
+		assertName(task.name, owner, ".name");
+		if (names.has(task.name)) fail(owner, ".name", `duplicate Task name ${JSON.stringify(task.name)}`);
+		names.add(task.name);
+		if (typeof task.run !== "function") fail(owner, ".run", "expected a function");
+		if (task.setup !== void 0 && typeof task.setup !== "function") fail(owner, ".setup", "expected a function");
+		if (task.teardown !== void 0 && typeof task.teardown !== "function") fail(owner, ".teardown", "expected a function");
+		if (task.teardown && !task.setup) fail(owner, ".teardown", "requires setup");
+		const cases = expandMatrix(task.matrix, `Task ${JSON.stringify(task.name)}`);
+		return {
+			task,
+			cases,
+			names: cases.map(caseName),
+			metadata: cases.map((matrix) => metadata === void 0 ? {
+				schema: 1,
+				matrix
+			} : {
+				schema: 1,
+				matrix,
+				metadata
+			})
+		};
+	});
+	return {
+		name: definition.name,
+		tasks
+	};
+}
+//#endregion
+export { defineTask, defineWorkflow };

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@akshatmittal/invoker",
+  "version": "0.1.0",
+  "description": "Typed matrix-driven regression workflows for Vitest.",
+  "keywords": [
+    "matrix",
+    "regression",
+    "vitest",
+    "workflow"
+  ],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/akshatmittal/invoker.git"
+  },
+  "files": [
+    "dist/**",
+    "README.md",
+    "LICENSE"
+  ],
+  "type": "module",
+  "sideEffects": false,
+  "main": "./dist/index.mjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "devDependencies": {
+    "@types/node": "^24.13.3",
+    "tsdown": "^0.22.14",
+    "typescript": "^7.0.2",
+    "vitest": "^4.1.10",
+    "@workspace/tsconfig": "0.0.0"
+  },
+  "peerDependencies": {
+    "vitest": "^4.1.10"
+  },
+  "engines": {
+    "node": "^24.18.1"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "dev": "tsdown --watch",
+    "typecheck": "tsc --noEmit"
+  }
+}