agent-skill-evals 0.1.0 → 0.1.1

package/dist/test-generator/index.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.mts","names":[],"sources":["../../src/test-pack.ts"],"mappings":";;;cAmBa,kBAAA,EAAkB,CAAA,CAAA,SAAA;;;;;;;;;cA+BlB,iBAAA,EAAiB,CAAA,CAAA,qBAAA,EAAA,CAAA,CAAA,SAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAmCjB,mBAAA,EAAmB,CAAA,CAAA,SAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAiBnB,mBAAA,EAAmB,CAAA,CAAA,SAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAwBpB,aAAA,GAAgB,CAAA,CAAE,KAAA,QAAa,mBAAA;AAAA,KAC/B,aAAA,GAAgB,CAAA,CAAE,KAAA,QAAa,mBAAA;AAAA,KAC/B,WAAA,GAAc,CAAA,CAAE,KAAA,QAAa,iBAAA;AAAA,UAExB,iBAAA;EACf,WAAA;EACA,IAAA,EAAM,MAAA;EACN,MAAA,EAAQ,KAAA,CAAM,MAAA;EACd,QAAA,GAAW,MAAA;AAAA;AAAA,UAGI,uBAAA;EACf,IAAA;EACA,aAAA;AAAA;AAAA,iBAYc,qBAAA,CAAsB,KAAA,YAAiB,aAAA;AAAA,iBAUjC,YAAA,CAAa,IAAA,WAAe,OAAA,CAAQ,aAAA;AAAA,iBAgC1C,gBAAA,CACd,IAAA,EAAM,aAAA,EACN,OAAA;EAAW,aAAA;EAAuB,WAAA;AAAA,IACjC,iBAAA;AAAA,iBA0DmB,iBAAA,CACpB,MAAA,EAAQ,uBAAA,GACP,OAAA,CAAQ,iBAAA"}

package/dist/test-generator/index.mjs

@@ -0,0 +1,2 @@
+import { a as loadTestPack, c as toPromptfooTests, i as MockServiceSchema, n as CleanTestPackSchema, o as parseTestPackDocument, r as ConversationSchema, s as testPackGenerator, t as CleanTestCaseSchema } from "../test-pack-Cu1WOssl.mjs";
+export { CleanTestCaseSchema, CleanTestPackSchema, ConversationSchema, MockServiceSchema, testPackGenerator as default, testPackGenerator, loadTestPack, parseTestPackDocument, toPromptfooTests };

package/dist/test-pack-Cu1WOssl.mjs

@@ -0,0 +1,215 @@
+import { t as RUNTIME_CHECKS } from "./catalog-9Wh-AfRv.mjs";
+import { z } from "zod";
+import { dirname } from "node:path";
+import { readFile, stat } from "node:fs/promises";
+import "node:fs";
+import { parse } from "yaml";
+//#region src/internal-services.ts
+async function pathExists(path) {
+	try {
+		await stat(path);
+		return true;
+	} catch {
+		return false;
+	}
+}
+/**
+* YAML parse with merge keys enabled (<<: *anchor), matching Promptfoo's
+* js-yaml loader so test packs can hoist shared vars into anchored blocks.
+*/
+function parseYaml(input) {
+	return parse(input, { merge: true });
+}
+//#endregion
+//#region src/test-pack.ts
+const NonEmptyString = z.string().min(1);
+const RuntimeShorthandSchemas = RUNTIME_CHECKS.map(({ type, argsSchema }) => z.object({ [type]: argsSchema }).strict());
+const CheckEntrySchema = z.union(RuntimeShorthandSchemas);
+const CheckListSchema = z.array(CheckEntrySchema);
+const RequiredCheckListSchema = z.array(CheckEntrySchema).min(1);
+const ConversationSchema = z.object({
+	max_turns: z.number().int().min(1).max(20).optional(),
+	scripted_user: z.array(NonEmptyString).min(1).optional(),
+	simulated_user: z.object({
+		goal: NonEmptyString,
+		persona: NonEmptyString.optional(),
+		allow_mocks: z.boolean().default(false)
+	}).optional()
+}).superRefine((conversation, ctx) => {
+	if (conversation.scripted_user && conversation.simulated_user) ctx.addIssue({
+		code: "custom",
+		message: "set either scripted_user or simulated_user, not both"
+	});
+	if (!conversation.scripted_user && !conversation.simulated_user) ctx.addIssue({
+		code: "custom",
+		message: "conversation requires scripted_user or simulated_user"
+	});
+});
+const MockBaseSchema = z.object({ name: NonEmptyString });
+const MockServiceSchema = z.discriminatedUnion("kind", [
+	MockBaseSchema.extend({
+		kind: z.literal("mcp"),
+		transport: z.enum(["stdio", "http"]).default("stdio"),
+		command: NonEmptyString.optional(),
+		args: z.array(z.string()).default([]),
+		url: NonEmptyString.optional(),
+		env: z.record(z.string(), z.string()).optional(),
+		provides_skill_evidence: z.boolean().default(false)
+	}),
+	MockBaseSchema.extend({
+		kind: z.literal("http"),
+		command: NonEmptyString,
+		args: z.array(z.string()).default([]),
+		env: z.record(z.string(), z.string()).optional(),
+		ready: z.object({
+			path: NonEmptyString.default("/health"),
+			timeout_ms: z.number().positive().optional()
+		}).default({ path: "/health" }),
+		expose_as: NonEmptyString
+	}),
+	MockBaseSchema.extend({
+		kind: z.literal("command"),
+		executable: NonEmptyString
+	})
+]);
+const BudgetSchema = z.object({
+	max_total_tokens: z.number().int().nonnegative().optional(),
+	max_prompt_tokens: z.number().int().nonnegative().optional(),
+	max_completion_tokens: z.number().int().nonnegative().optional(),
+	max_cached_tokens: z.number().int().nonnegative().optional()
+}).refine((budget) => Object.values(budget).some((value) => value !== void 0), { message: "budget requires at least one token limit" });
+const PromptfooAssertionSchema = z.record(z.string(), z.unknown());
+const CleanTestCaseSchema = z.object({
+	description: z.string().optional(),
+	mode: z.enum(["behavior", "routing"]).default("behavior"),
+	prompt: NonEmptyString,
+	fixture: NonEmptyString.optional(),
+	setup: z.array(NonEmptyString).default([]),
+	supporting_skills: z.array(NonEmptyString).optional(),
+	distractor_skills: z.array(NonEmptyString).optional(),
+	preconditions: CheckListSchema.default([]),
+	expect: RequiredCheckListSchema,
+	conversation: ConversationSchema.optional(),
+	budget: BudgetSchema.optional(),
+	promptfoo: z.object({ assert: z.array(PromptfooAssertionSchema).default([]) }).optional(),
+	environment: z.object({ mocks: z.array(MockServiceSchema).default([]) }).optional(),
+	metadata: z.record(z.string(), z.unknown()).optional()
+});
+const CleanTestPackSchema = z.object({
+	skill: NonEmptyString,
+	supporting_skills: z.array(NonEmptyString).default([]),
+	distractor_skills: z.array(NonEmptyString).default([]),
+	builtin_distractor: z.boolean().default(true),
+	environment: z.object({ mocks: z.array(MockServiceSchema).default([]) }).optional(),
+	tests: z.array(CleanTestCaseSchema).min(1)
+}).superRefine((pack, ctx) => {
+	pack.tests.forEach((test, index) => {
+		if (test.mode !== "routing") return;
+		const distractors = [...pack.distractor_skills, ...test.distractor_skills ?? []];
+		if (!pack.builtin_distractor && distractors.length === 0) ctx.addIssue({
+			code: "custom",
+			path: [
+				"tests",
+				index,
+				"distractor_skills"
+			],
+			message: "routing tests require at least one distractor skill"
+		});
+	});
+});
+function migrationError(input) {
+	if (Array.isArray(input)) return /* @__PURE__ */ new Error("Legacy Promptfoo-shaped test packs are no longer supported. Use the clean Test Pack format with top-level `skill` and `tests` fields; move each case's prompt, fixture, and expectations out of `vars`.");
+}
+function parseTestPackDocument(input) {
+	const legacy = migrationError(input);
+	if (legacy) throw legacy;
+	const parsed = CleanTestPackSchema.safeParse(input);
+	if (!parsed.success) throw new Error(`Invalid Agent Skill Evals Test Pack:\n${z.prettifyError(parsed.error)}`);
+	return parsed.data;
+}
+async function loadTestPack(path) {
+	return parseTestPackDocument(parseYaml(await readFile(path, "utf8")));
+}
+function budgetConfig(budget) {
+	if (!budget) return {};
+	return {
+		...budget.max_total_tokens !== void 0 ? { maxTotalTokens: budget.max_total_tokens } : {},
+		...budget.max_prompt_tokens !== void 0 ? { maxPromptTokens: budget.max_prompt_tokens } : {},
+		...budget.max_completion_tokens !== void 0 ? { maxCompletionTokens: budget.max_completion_tokens } : {},
+		...budget.max_cached_tokens !== void 0 ? { maxCachedTokens: budget.max_cached_tokens } : {}
+	};
+}
+function conversationVars(conversation) {
+	if (!conversation) return void 0;
+	return {
+		...conversation.max_turns !== void 0 ? { maxTurns: conversation.max_turns } : {},
+		...conversation.scripted_user ? { userTurns: conversation.scripted_user } : {},
+		...conversation.simulated_user ? {
+			user: {
+				goal: conversation.simulated_user.goal,
+				...conversation.simulated_user.persona ? { persona: conversation.simulated_user.persona } : {}
+			},
+			simulatedUserAllowMocks: conversation.simulated_user.allow_mocks
+		} : {}
+	};
+}
+function toPromptfooTests(pack, options) {
+	return pack.tests.map((test) => {
+		const supportingSkills = test.supporting_skills ?? pack.supporting_skills;
+		const distractorSkills = [...pack.distractor_skills, ...test.distractor_skills ?? []];
+		const environmentMocks = [...pack.environment?.mocks ?? [], ...test.environment?.mocks ?? []];
+		const assertions = [{
+			type: "javascript",
+			metric: "skill.test",
+			value: options.assertionPath,
+			config: { metric: "skill.test" }
+		}];
+		if (test.budget) assertions.push({
+			type: "javascript",
+			metric: "skill.budget",
+			value: options.assertionPath,
+			config: {
+				metric: "skill.budget",
+				agentSkillEvals: budgetConfig(test.budget)
+			}
+		});
+		assertions.push(...test.promptfoo?.assert ?? []);
+		return {
+			...test.description ? { description: test.description } : {},
+			vars: {
+				prompt: test.prompt,
+				...options.testPackDir ? { testPackDir: options.testPackDir } : {},
+				skillPath: pack.skill,
+				mode: test.mode,
+				supportingSkills,
+				distractorSkills,
+				builtinDistractor: test.mode === "routing" && pack.builtin_distractor,
+				...test.fixture ? { fixture: test.fixture } : {},
+				setup: test.setup,
+				preconditions: test.preconditions,
+				expect: test.expect,
+				...test.conversation ? { conversation: conversationVars(test.conversation) } : {},
+				...environmentMocks.length > 0 ? { environment: { mocks: environmentMocks } } : {}
+			},
+			assert: assertions,
+			metadata: {
+				...test.metadata ?? {},
+				agentSkillEvals: {
+					mode: test.mode,
+					skill: pack.skill
+				}
+			}
+		};
+	});
+}
+async function testPackGenerator(config) {
+	if (!config?.path) throw new Error("Agent Skill Evals test generator requires config.path");
+	return toPromptfooTests(await loadTestPack(config.path), {
+		assertionPath: config.assertionPath ?? "file://./agent-skill-evals/assertions.js",
+		testPackDir: dirname(config.path)
+	});
+}
+//#endregion
+export { loadTestPack as a, toPromptfooTests as c, MockServiceSchema as i, parseYaml as l, CleanTestPackSchema as n, parseTestPackDocument as o, ConversationSchema as r, testPackGenerator as s, CleanTestCaseSchema as t, pathExists as u };
+
+//# sourceMappingURL=test-pack-Cu1WOssl.mjs.map

package/dist/test-pack-Cu1WOssl.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-pack-Cu1WOssl.mjs","names":["parseYamlRaw"],"sources":["../src/internal-services.ts","../src/test-pack.ts"],"sourcesContent":["import { constants } from \"node:fs\";\nimport { access, stat } from \"node:fs/promises\";\nimport { parse as parseYamlRaw } from \"yaml\";\n\nexport async function pathExists(path: string): Promise<boolean> {\n  try {\n    await stat(path);\n    return true;\n  } catch {\n    return false;\n  }\n}\n\nexport async function pathExecutable(path: string): Promise<boolean> {\n  try {\n    await access(path, constants.X_OK);\n    return true;\n  } catch {\n    return false;\n  }\n}\n\n/**\n * YAML parse with merge keys enabled (<<: *anchor), matching Promptfoo's\n * js-yaml loader so test packs can hoist shared vars into anchored blocks.\n */\nexport function parseYaml(input: string): unknown {\n  return parseYamlRaw(input, { merge: true });\n}\n","import { readFile } from \"node:fs/promises\";\nimport { dirname } from \"node:path\";\nimport { z } from \"zod\";\nimport { parseYaml } from \"./internal-services.js\";\nimport { RUNTIME_CHECKS } from \"./runtime-checks/catalog.js\";\n\nconst NonEmptyString = z.string().min(1);\nconst RuntimeShorthandSchemas = RUNTIME_CHECKS.map(({ type, argsSchema }) =>\n  z.object({ [type]: argsSchema }).strict(),\n);\nconst RuntimeShorthandSchema = z.union(RuntimeShorthandSchemas as [\n  (typeof RuntimeShorthandSchemas)[number],\n  (typeof RuntimeShorthandSchemas)[number],\n  ...(typeof RuntimeShorthandSchemas)[number][],\n]);\nconst CheckEntrySchema = RuntimeShorthandSchema;\nconst CheckListSchema = z.array(CheckEntrySchema);\nconst RequiredCheckListSchema = z.array(CheckEntrySchema).min(1);\n\nexport const ConversationSchema = z\n  .object({\n    max_turns: z.number().int().min(1).max(20).optional(),\n    scripted_user: z.array(NonEmptyString).min(1).optional(),\n    simulated_user: z\n      .object({\n        goal: NonEmptyString,\n        persona: NonEmptyString.optional(),\n        allow_mocks: z.boolean().default(false),\n      })\n      .optional(),\n  })\n  .superRefine((conversation, ctx) => {\n    if (conversation.scripted_user && conversation.simulated_user) {\n      ctx.addIssue({\n        code: \"custom\",\n        message: \"set either scripted_user or simulated_user, not both\",\n      });\n    }\n    if (!conversation.scripted_user && !conversation.simulated_user) {\n      ctx.addIssue({\n        code: \"custom\",\n        message: \"conversation requires scripted_user or simulated_user\",\n      });\n    }\n  });\n\nconst MockBaseSchema = z.object({\n  name: NonEmptyString,\n});\n\nexport const MockServiceSchema = z.discriminatedUnion(\"kind\", [\n  MockBaseSchema.extend({\n    kind: z.literal(\"mcp\"),\n    transport: z.enum([\"stdio\", \"http\"]).default(\"stdio\"),\n    command: NonEmptyString.optional(),\n    args: z.array(z.string()).default([]),\n    url: NonEmptyString.optional(),\n    env: z.record(z.string(), z.string()).optional(),\n    provides_skill_evidence: z.boolean().default(false),\n  }),\n  MockBaseSchema.extend({\n    kind: z.literal(\"http\"),\n    command: NonEmptyString,\n    args: z.array(z.string()).default([]),\n    env: z.record(z.string(), z.string()).optional(),\n    ready: z.object({ path: NonEmptyString.default(\"/health\"), timeout_ms: z.number().positive().optional() }).default({ path: \"/health\" }),\n    expose_as: NonEmptyString,\n  }),\n  MockBaseSchema.extend({\n    kind: z.literal(\"command\"),\n    executable: NonEmptyString,\n  }),\n]);\n\nconst BudgetSchema = z.object({\n  max_total_tokens: z.number().int().nonnegative().optional(),\n  max_prompt_tokens: z.number().int().nonnegative().optional(),\n  max_completion_tokens: z.number().int().nonnegative().optional(),\n  max_cached_tokens: z.number().int().nonnegative().optional(),\n}).refine((budget) => Object.values(budget).some((value) => value !== undefined), {\n  message: \"budget requires at least one token limit\",\n});\n\nconst PromptfooAssertionSchema = z.record(z.string(), z.unknown());\n\nexport const CleanTestCaseSchema = z.object({\n  description: z.string().optional(),\n  mode: z.enum([\"behavior\", \"routing\"]).default(\"behavior\"),\n  prompt: NonEmptyString,\n  fixture: NonEmptyString.optional(),\n  setup: z.array(NonEmptyString).default([]),\n  supporting_skills: z.array(NonEmptyString).optional(),\n  distractor_skills: z.array(NonEmptyString).optional(),\n  preconditions: CheckListSchema.default([]),\n  expect: RequiredCheckListSchema,\n  conversation: ConversationSchema.optional(),\n  budget: BudgetSchema.optional(),\n  promptfoo: z.object({ assert: z.array(PromptfooAssertionSchema).default([]) }).optional(),\n  environment: z.object({ mocks: z.array(MockServiceSchema).default([]) }).optional(),\n  metadata: z.record(z.string(), z.unknown()).optional(),\n});\n\nexport const CleanTestPackSchema = z.object({\n  skill: NonEmptyString,\n  supporting_skills: z.array(NonEmptyString).default([]),\n  distractor_skills: z.array(NonEmptyString).default([]),\n  builtin_distractor: z.boolean().default(true),\n  environment: z.object({ mocks: z.array(MockServiceSchema).default([]) }).optional(),\n  tests: z.array(CleanTestCaseSchema).min(1),\n}).superRefine((pack, ctx) => {\n  pack.tests.forEach((test, index) => {\n    if (test.mode !== \"routing\") return;\n    const distractors = [\n      ...pack.distractor_skills,\n      ...(test.distractor_skills ?? []),\n    ];\n    if (!pack.builtin_distractor && distractors.length === 0) {\n      ctx.addIssue({\n        code: \"custom\",\n        path: [\"tests\", index, \"distractor_skills\"],\n        message: \"routing tests require at least one distractor skill\",\n      });\n    }\n  });\n});\n\nexport type CleanTestPack = z.infer<typeof CleanTestPackSchema>;\nexport type CleanTestCase = z.infer<typeof CleanTestCaseSchema>;\nexport type MockService = z.infer<typeof MockServiceSchema>;\n\nexport interface PromptfooTestCase {\n  description?: string;\n  vars: Record<string, unknown>;\n  assert: Array<Record<string, unknown>>;\n  metadata?: Record<string, unknown>;\n}\n\nexport interface TestPackGeneratorConfig {\n  path: string;\n  assertionPath?: string;\n}\n\nfunction migrationError(input: unknown): Error | undefined {\n  if (Array.isArray(input)) {\n    return new Error(\n      \"Legacy Promptfoo-shaped test packs are no longer supported. Use the clean Test Pack format with top-level `skill` and `tests` fields; move each case's prompt, fixture, and expectations out of `vars`.\",\n    );\n  }\n  return undefined;\n}\n\nexport function parseTestPackDocument(input: unknown): CleanTestPack {\n  const legacy = migrationError(input);\n  if (legacy) throw legacy;\n  const parsed = CleanTestPackSchema.safeParse(input);\n  if (!parsed.success) {\n    throw new Error(`Invalid Agent Skill Evals Test Pack:\\n${z.prettifyError(parsed.error)}`);\n  }\n  return parsed.data;\n}\n\nexport async function loadTestPack(path: string): Promise<CleanTestPack> {\n  const source = await readFile(path, \"utf8\");\n  return parseTestPackDocument(parseYaml(source));\n}\n\nfunction budgetConfig(budget: CleanTestCase[\"budget\"]): Record<string, number> {\n  if (!budget) return {};\n  return {\n    ...(budget.max_total_tokens !== undefined ? { maxTotalTokens: budget.max_total_tokens } : {}),\n    ...(budget.max_prompt_tokens !== undefined ? { maxPromptTokens: budget.max_prompt_tokens } : {}),\n    ...(budget.max_completion_tokens !== undefined ? { maxCompletionTokens: budget.max_completion_tokens } : {}),\n    ...(budget.max_cached_tokens !== undefined ? { maxCachedTokens: budget.max_cached_tokens } : {}),\n  };\n}\n\nfunction conversationVars(conversation: CleanTestCase[\"conversation\"]): Record<string, unknown> | undefined {\n  if (!conversation) return undefined;\n  return {\n    ...(conversation.max_turns !== undefined ? { maxTurns: conversation.max_turns } : {}),\n    ...(conversation.scripted_user ? { userTurns: conversation.scripted_user } : {}),\n    ...(conversation.simulated_user\n      ? {\n          user: {\n            goal: conversation.simulated_user.goal,\n            ...(conversation.simulated_user.persona ? { persona: conversation.simulated_user.persona } : {}),\n          },\n          simulatedUserAllowMocks: conversation.simulated_user.allow_mocks,\n        }\n      : {}),\n  };\n}\n\nexport function toPromptfooTests(\n  pack: CleanTestPack,\n  options: { assertionPath: string; testPackDir?: string },\n): PromptfooTestCase[] {\n  return pack.tests.map((test) => {\n    const supportingSkills = test.supporting_skills ?? pack.supporting_skills;\n    const distractorSkills = [\n      ...pack.distractor_skills,\n      ...(test.distractor_skills ?? []),\n    ];\n    const environmentMocks = [\n      ...(pack.environment?.mocks ?? []),\n      ...(test.environment?.mocks ?? []),\n    ];\n    const assertions: Array<Record<string, unknown>> = [\n      {\n        type: \"javascript\",\n        metric: \"skill.test\",\n        value: options.assertionPath,\n        config: { metric: \"skill.test\" },\n      },\n    ];\n    if (test.budget) {\n      assertions.push({\n        type: \"javascript\",\n        metric: \"skill.budget\",\n        value: options.assertionPath,\n        config: { metric: \"skill.budget\", agentSkillEvals: budgetConfig(test.budget) },\n      });\n    }\n    assertions.push(...(test.promptfoo?.assert ?? []));\n\n    return {\n      ...(test.description ? { description: test.description } : {}),\n      vars: {\n        prompt: test.prompt,\n        ...(options.testPackDir ? { testPackDir: options.testPackDir } : {}),\n        skillPath: pack.skill,\n        mode: test.mode,\n        supportingSkills,\n        distractorSkills,\n        builtinDistractor: test.mode === \"routing\" && pack.builtin_distractor,\n        ...(test.fixture ? { fixture: test.fixture } : {}),\n        setup: test.setup,\n        preconditions: test.preconditions,\n        expect: test.expect,\n        ...(test.conversation ? { conversation: conversationVars(test.conversation) } : {}),\n        ...(environmentMocks.length > 0 ? { environment: { mocks: environmentMocks } } : {}),\n      },\n      assert: assertions,\n      metadata: {\n        ...(test.metadata ?? {}),\n        agentSkillEvals: {\n          mode: test.mode,\n          skill: pack.skill,\n        },\n      },\n    };\n  });\n}\n\nexport async function testPackGenerator(\n  config: TestPackGeneratorConfig,\n): Promise<PromptfooTestCase[]> {\n  if (!config?.path) {\n    throw new Error(\"Agent Skill Evals test generator requires config.path\");\n  }\n  const pack = await loadTestPack(config.path);\n  return toPromptfooTests(pack, {\n    assertionPath: config.assertionPath ?? \"file://./agent-skill-evals/assertions.js\",\n    testPackDir: dirname(config.path),\n  });\n}\n\nexport default testPackGenerator;\n"],"mappings":";;;;;;;AAIA,eAAsB,WAAW,MAAgC;AAC/D,KAAI;AACF,QAAM,KAAK,KAAK;AAChB,SAAO;SACD;AACN,SAAO;;;;;;;AAiBX,SAAgB,UAAU,OAAwB;AAChD,QAAOA,MAAa,OAAO,EAAE,OAAO,MAAM,CAAC;;;;ACrB7C,MAAM,iBAAiB,EAAE,QAAQ,CAAC,IAAI,EAAE;AACxC,MAAM,0BAA0B,eAAe,KAAK,EAAE,MAAM,iBAC1D,EAAE,OAAO,GAAG,OAAO,YAAY,CAAC,CAAC,QAAQ,CAC1C;AAMD,MAAM,mBALyB,EAAE,MAAM,wBAKQ;AAC/C,MAAM,kBAAkB,EAAE,MAAM,iBAAiB;AACjD,MAAM,0BAA0B,EAAE,MAAM,iBAAiB,CAAC,IAAI,EAAE;AAEhE,MAAa,qBAAqB,EAC/B,OAAO;CACN,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,GAAG,CAAC,UAAU;CACrD,eAAe,EAAE,MAAM,eAAe,CAAC,IAAI,EAAE,CAAC,UAAU;CACxD,gBAAgB,EACb,OAAO;EACN,MAAM;EACN,SAAS,eAAe,UAAU;EAClC,aAAa,EAAE,SAAS,CAAC,QAAQ,MAAM;EACxC,CAAC,CACD,UAAU;CACd,CAAC,CACD,aAAa,cAAc,QAAQ;AAClC,KAAI,aAAa,iBAAiB,aAAa,eAC7C,KAAI,SAAS;EACX,MAAM;EACN,SAAS;EACV,CAAC;AAEJ,KAAI,CAAC,aAAa,iBAAiB,CAAC,aAAa,eAC/C,KAAI,SAAS;EACX,MAAM;EACN,SAAS;EACV,CAAC;EAEJ;AAEJ,MAAM,iBAAiB,EAAE,OAAO,EAC9B,MAAM,gBACP,CAAC;AAEF,MAAa,oBAAoB,EAAE,mBAAmB,QAAQ;CAC5D,eAAe,OAAO;EACpB,MAAM,EAAE,QAAQ,MAAM;EACtB,WAAW,EAAE,KAAK,CAAC,SAAS,OAAO,CAAC,CAAC,QAAQ,QAAQ;EACrD,SAAS,eAAe,UAAU;EAClC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,QAAQ,EAAE,CAAC;EACrC,KAAK,eAAe,UAAU;EAC9B,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,QAAQ,CAAC,CAAC,UAAU;EAChD,yBAAyB,EAAE,SAAS,CAAC,QAAQ,MAAM;EACpD,CAAC;CACF,eAAe,OAAO;EACpB,MAAM,EAAE,QAAQ,OAAO;EACvB,SAAS;EACT,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,QAAQ,EAAE,CAAC;EACrC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,QAAQ,CAAC,CAAC,UAAU;EAChD,OAAO,EAAE,OAAO;GAAE,MAAM,eAAe,QAAQ,UAAU;GAAE,YAAY,EAAE,QAAQ,CAAC,UAAU,CAAC,UAAU;GAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,WAAW,CAAC;EACvI,WAAW;EACZ,CAAC;CACF,eAAe,OAAO;EACpB,MAAM,EAAE,QAAQ,UAAU;EAC1B,YAAY;EACb,CAAC;CACH,CAAC;AAEF,MAAM,eAAe,EAAE,OAAO;CAC5B,kBAAkB,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,UAAU;CAC3D,mBAAmB,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,UAAU;CAC5D,uBAAuB,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,UAAU;CAChE,mBAAmB,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,UAAU;CAC7D,CAAC,CAAC,QAAQ,WAAW,OAAO,OAAO,OAAO,CAAC,MAAM,UAAU,UAAU,KAAA,EAAU,EAAE,EAChF,SAAS,4CACV,CAAC;AAEF,MAAM,2BAA2B,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,SAAS,CAAC;AAElE,MAAa,sBAAsB,EAAE,OAAO;CAC1C,aAAa,EAAE,QAAQ,CAAC,UAAU;CAClC,MAAM,EAAE,KAAK,CAAC,YAAY,UAAU,CAAC,CAAC,QAAQ,WAAW;CACzD,QAAQ;CACR,SAAS,eAAe,UAAU;CAClC,OAAO,EAAE,MAAM,eAAe,CAAC,QAAQ,EAAE,CAAC;CAC1C,mBAAmB,EAAE,MAAM,eAAe,CAAC,UAAU;CACrD,mBAAmB,EAAE,MAAM,eAAe,CAAC,UAAU;CACrD,eAAe,gBAAgB,QAAQ,EAAE,CAAC;CAC1C,QAAQ;CACR,cAAc,mBAAmB,UAAU;CAC3C,QAAQ,aAAa,UAAU;CAC/B,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,UAAU;CACzF,aAAa,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,UAAU;CACnF,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC,UAAU;CACvD,CAAC;AAEF,MAAa,sBAAsB,EAAE,OAAO;CAC1C,OAAO;CACP,mBAAmB,EAAE,MAAM,eAAe,CAAC,QAAQ,EAAE,CAAC;CACtD,mBAAmB,EAAE,MAAM,eAAe,CAAC,QAAQ,EAAE,CAAC;CACtD,oBAAoB,EAAE,SAAS,CAAC,QAAQ,KAAK;CAC7C,aAAa,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,UAAU;CACnF,OAAO,EAAE,MAAM,oBAAoB,CAAC,IAAI,EAAE;CAC3C,CAAC,CAAC,aAAa,MAAM,QAAQ;AAC5B,MAAK,MAAM,SAAS,MAAM,UAAU;AAClC,MAAI,KAAK,SAAS,UAAW;EAC7B,MAAM,cAAc,CAClB,GAAG,KAAK,mBACR,GAAI,KAAK,qBAAqB,EAAE,CACjC;AACD,MAAI,CAAC,KAAK,sBAAsB,YAAY,WAAW,EACrD,KAAI,SAAS;GACX,MAAM;GACN,MAAM;IAAC;IAAS;IAAO;IAAoB;GAC3C,SAAS;GACV,CAAC;GAEJ;EACF;AAkBF,SAAS,eAAe,OAAmC;AACzD,KAAI,MAAM,QAAQ,MAAM,CACtB,wBAAO,IAAI,MACT,0MACD;;AAKL,SAAgB,sBAAsB,OAA+B;CACnE,MAAM,SAAS,eAAe,MAAM;AACpC,KAAI,OAAQ,OAAM;CAClB,MAAM,SAAS,oBAAoB,UAAU,MAAM;AACnD,KAAI,CAAC,OAAO,QACV,OAAM,IAAI,MAAM,yCAAyC,EAAE,cAAc,OAAO,MAAM,GAAG;AAE3F,QAAO,OAAO;;AAGhB,eAAsB,aAAa,MAAsC;AAEvE,QAAO,sBAAsB,UAAU,MADlB,SAAS,MAAM,OAAO,CACG,CAAC;;AAGjD,SAAS,aAAa,QAAyD;AAC7E,KAAI,CAAC,OAAQ,QAAO,EAAE;AACtB,QAAO;EACL,GAAI,OAAO,qBAAqB,KAAA,IAAY,EAAE,gBAAgB,OAAO,kBAAkB,GAAG,EAAE;EAC5F,GAAI,OAAO,sBAAsB,KAAA,IAAY,EAAE,iBAAiB,OAAO,mBAAmB,GAAG,EAAE;EAC/F,GAAI,OAAO,0BAA0B,KAAA,IAAY,EAAE,qBAAqB,OAAO,uBAAuB,GAAG,EAAE;EAC3G,GAAI,OAAO,sBAAsB,KAAA,IAAY,EAAE,iBAAiB,OAAO,mBAAmB,GAAG,EAAE;EAChG;;AAGH,SAAS,iBAAiB,cAAkF;AAC1G,KAAI,CAAC,aAAc,QAAO,KAAA;AAC1B,QAAO;EACL,GAAI,aAAa,cAAc,KAAA,IAAY,EAAE,UAAU,aAAa,WAAW,GAAG,EAAE;EACpF,GAAI,aAAa,gBAAgB,EAAE,WAAW,aAAa,eAAe,GAAG,EAAE;EAC/E,GAAI,aAAa,iBACb;GACE,MAAM;IACJ,MAAM,aAAa,eAAe;IAClC,GAAI,aAAa,eAAe,UAAU,EAAE,SAAS,aAAa,eAAe,SAAS,GAAG,EAAE;IAChG;GACD,yBAAyB,aAAa,eAAe;GACtD,GACD,EAAE;EACP;;AAGH,SAAgB,iBACd,MACA,SACqB;AACrB,QAAO,KAAK,MAAM,KAAK,SAAS;EAC9B,MAAM,mBAAmB,KAAK,qBAAqB,KAAK;EACxD,MAAM,mBAAmB,CACvB,GAAG,KAAK,mBACR,GAAI,KAAK,qBAAqB,EAAE,CACjC;EACD,MAAM,mBAAmB,CACvB,GAAI,KAAK,aAAa,SAAS,EAAE,EACjC,GAAI,KAAK,aAAa,SAAS,EAAE,CAClC;EACD,MAAM,aAA6C,CACjD;GACE,MAAM;GACN,QAAQ;GACR,OAAO,QAAQ;GACf,QAAQ,EAAE,QAAQ,cAAc;GACjC,CACF;AACD,MAAI,KAAK,OACP,YAAW,KAAK;GACd,MAAM;GACN,QAAQ;GACR,OAAO,QAAQ;GACf,QAAQ;IAAE,QAAQ;IAAgB,iBAAiB,aAAa,KAAK,OAAO;IAAE;GAC/E,CAAC;AAEJ,aAAW,KAAK,GAAI,KAAK,WAAW,UAAU,EAAE,CAAE;AAElD,SAAO;GACL,GAAI,KAAK,cAAc,EAAE,aAAa,KAAK,aAAa,GAAG,EAAE;GAC7D,MAAM;IACJ,QAAQ,KAAK;IACb,GAAI,QAAQ,cAAc,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;IACnE,WAAW,KAAK;IAChB,MAAM,KAAK;IACX;IACA;IACA,mBAAmB,KAAK,SAAS,aAAa,KAAK;IACnD,GAAI,KAAK,UAAU,EAAE,SAAS,KAAK,SAAS,GAAG,EAAE;IACjD,OAAO,KAAK;IACZ,eAAe,KAAK;IACpB,QAAQ,KAAK;IACb,GAAI,KAAK,eAAe,EAAE,cAAc,iBAAiB,KAAK,aAAa,EAAE,GAAG,EAAE;IAClF,GAAI,iBAAiB,SAAS,IAAI,EAAE,aAAa,EAAE,OAAO,kBAAkB,EAAE,GAAG,EAAE;IACpF;GACD,QAAQ;GACR,UAAU;IACR,GAAI,KAAK,YAAY,EAAE;IACvB,iBAAiB;KACf,MAAM,KAAK;KACX,OAAO,KAAK;KACb;IACF;GACF;GACD;;AAGJ,eAAsB,kBACpB,QAC8B;AAC9B,KAAI,CAAC,QAAQ,KACX,OAAM,IAAI,MAAM,wDAAwD;AAG1E,QAAO,iBAAiB,MADL,aAAa,OAAO,KAAK,EACd;EAC5B,eAAe,OAAO,iBAAiB;EACvC,aAAa,QAAQ,OAAO,KAAK;EAClC,CAAC"}

package/package.json

@@ -1,32 +1,38 @@
 {
   "name": "agent-skill-evals",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "type": "module",
   "exports": {
     "./agent": {
       "types": "./dist/agent/index.d.mts",
       "import": "./dist/agent/index.mjs"
     },
-    "./skill-checks": {
-      "types": "./dist/skill-checks/index.d.mts",
-      "import": "./dist/skill-checks/index.mjs"
-    },
     "./assertions": {
       "types": "./dist/assertions/index.d.mts",
       "import": "./dist/assertions/index.mjs"
+    },
+    "./test-generator": {
+      "types": "./dist/test-generator/index.d.mts",
+      "import": "./dist/test-generator/index.mjs"
     }
   },
+  "bin": {
+    "agent-skill-evals": "./dist/cli/init.mjs"
+  },
   "scripts": {
     "build": "tsdown",
+    "schema:generate": "tsx scripts/write-test-pack-schema.ts",
     "typecheck": "tsc --noEmit -p tsconfig.json"
   },
   "dependencies": {
-    "@effect/platform": "^0.96.1",
-    "@effect/platform-node": "^0.106.0",
-    "effect": "^3.21.2",
-    "yaml": "^2.6.0"
+    "yaml": "^2.6.0",
+    "zod": "^4.4.3"
+  },
+  "peerDependencies": {
+    "promptfoo": "^0.121.18"
   },
   "devDependencies": {
+    "tsx": "^4.22.0",
     "typescript": "^5.6.2"
   },
   "description": "Promptfoo-native core package for evaluating reusable agent skills.",
@@ -48,6 +54,7 @@
   ],
   "files": [
     "dist",
+    "schema",
     "README.md"
   ],
   "publishConfig": {