agent-skill-evals 0.1.0 → 0.1.1

package/dist/assertions/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","names":["configObject","numberSetting","availableMetrics","metricFrom"],"sources":["../../src/assertions/_shared.ts","../../src/assertions/skill-test.ts","../../src/assertions/skill-budget.ts","../../src/skill-checks/assertions-static/_shared.ts","../../src/skill-checks/assertions-static/settings.ts","../../src/skill-checks/assertions-static/context-economy.ts","../../src/skill-checks/assertions-static/executable-helper.ts","../../src/skill-checks/assertions-static/instruction-calibration.ts","../../src/skill-checks/assertions-static/negative-coverage.ts","../../src/skill-checks/assertions-static/routing-metadata.ts","../../src/skill-checks/assertions-static/scenario-validity.ts","../../src/skill-checks/assertions-static/promptfoo.ts","../../src/assertions/promptfoo.ts"],"sourcesContent":["import { join } from \"node:path\";\nimport * as Either from \"effect/Either\";\nimport * as Effect from \"effect/Effect\";\nimport type { AssertionEntry } from \"../assertion-entries.js\";\nimport type {\n  AssertionMode,\n  EvidenceHandle,\n  AgentSkillEvalsAssertionResult,\n  WorldHandle,\n} from \"../internal-types.js\";\nimport { decodeEvidenceSnapshotEither } from \"../evidence-schema.js\";\nimport {\n  EvidenceCollector,\n  evidenceFromSnapshot,\n  type AgentSkillEvalsProviderMetadata,\n} from \"../agent/index.js\";\nimport { makeWorldHandle } from \"../agent/world.js\";\nimport {\n  getRuntimeCheck,\n  RuntimeCheckCatalog,\n  RuntimeCheckCatalogLive,\n} from \"../runtime-checks/catalog.js\";\nimport {\n  Environment,\n  FileSystem,\n  NodeServicesLive,\n} from \"../internal-services.js\";\n\nexport interface PromptfooAssertContext {\n  vars?: Record<string, unknown>;\n  providerResponse?: {\n    metadata?: unknown;\n    tokenUsage?: {\n      total?: number;\n      prompt?: number;\n      completion?: number;\n      cached?: number;\n    };\n  };\n  test?: { vars?: Record<string, unknown> };\n  assertion?: { metric?: string };\n  assert?: { metric?: string };\n  config?: { metric?: string; agentSkillEvals?: unknown };\n  metric?: string;\n}\n\nexport interface GradingResult {\n  pass: boolean;\n  score: number;\n  reason: string;\n  componentResults?: Array<{ pass: boolean; score: number; reason: string }>;\n  metadata?: Record<string, unknown>;\n}\n\nexport async function loadMetadata(\n  context: PromptfooAssertContext,\n): Promise<AgentSkillEvalsProviderMetadata | null> {\n  return Effect.runPromise(loadMetadataEffect(context).pipe(Effect.provide(NodeServicesLive)));\n}\n\nexport function loadMetadataEffect(\n  context: PromptfooAssertContext,\n): Effect.Effect<AgentSkillEvalsProviderMetadata | null, never, FileSystem | Environment> {\n  return Effect.gen(function* () {\n  const direct = context.providerResponse?.metadata;\n  if (direct && typeof direct === \"object\" && \"worldPath\" in direct) {\n    return direct as AgentSkillEvalsProviderMetadata;\n  }\n  // Fallback: AGENT_SKILL_EVALS_RUN_DIR env points at the most recent run.\n  const environment = yield* Environment;\n  const env = yield* environment.env;\n  const runDir = env.AGENT_SKILL_EVALS_RUN_DIR;\n  if (runDir) {\n    const fs = yield* FileSystem;\n    const parsed = yield* fs.readText(join(runDir, \"agent-skill-evals-meta.json\")).pipe(\n      Effect.map((buf) => {\n        try {\n          return JSON.parse(buf) as AgentSkillEvalsProviderMetadata;\n        } catch {\n          return null;\n        }\n      }),\n      Effect.catchAll(() => Effect.succeed(null)),\n    );\n    if (parsed) {\n      return parsed;\n    }\n  }\n  return null;\n  });\n}\n\nexport async function loadEvidence(\n  meta: AgentSkillEvalsProviderMetadata,\n): Promise<EvidenceCollector> {\n  return Effect.runPromise(loadEvidenceEffect(meta).pipe(Effect.provide(NodeServicesLive)));\n}\n\nexport function loadEvidenceEffect(\n  meta: AgentSkillEvalsProviderMetadata,\n): Effect.Effect<EvidenceCollector, Error, FileSystem> {\n  return Effect.gen(function* () {\n  const fs = yield* FileSystem;\n  const buf = yield* fs.readText(meta.evidencePath).pipe(\n    Effect.mapError((err) => new Error(`evidence: failed to read ${meta.evidencePath}: ${err instanceof Error ? err.message : String(err)}`)),\n  );\n  const parsed = yield* Effect.try({\n    try: () => JSON.parse(buf) as unknown,\n    catch: (err) =>\n      new Error(`evidence: invalid JSON in ${meta.evidencePath}: ${err instanceof Error ? err.message : String(err)}`),\n  });\n  const decoded = decodeEvidenceSnapshotEither(parsed);\n  if (Either.isLeft(decoded)) {\n    return yield* Effect.fail(\n      new Error(`evidence: invalid agent-skill-evals.evidence.v1 payload: ${decoded.left.message}`),\n    );\n  }\n  const snapshot = decoded.right;\n  return EvidenceCollector.fromSnapshot(snapshot);\n  });\n}\n\nexport function loadWorld(\n  meta: AgentSkillEvalsProviderMetadata,\n  evidenceCollector: EvidenceCollector,\n): WorldHandle {\n  return makeWorldHandle(meta.worldPath, (event) => evidenceCollector.addCommand(event));\n}\n\nexport async function runEntries(\n  entries: AssertionEntry[],\n  world: WorldHandle,\n  evidenceCollector: EvidenceCollector,\n  mode: AssertionMode,\n): Promise<AgentSkillEvalsAssertionResult[]> {\n  return Effect.runPromise(\n    runEntriesEffect(entries, world, evidenceCollector, mode).pipe(\n      Effect.provide(RuntimeCheckCatalogLive),\n    ),\n  );\n}\n\nexport function runEntriesEffect(\n  entries: AssertionEntry[],\n  world: WorldHandle,\n  evidenceCollector: EvidenceCollector,\n  mode: AssertionMode,\n): Effect.Effect<AgentSkillEvalsAssertionResult[], never, RuntimeCheckCatalog> {\n  return Effect.gen(function* () {\n  const results: AgentSkillEvalsAssertionResult[] = [];\n  for (const entry of entries) {\n    const plugin = yield* getRuntimeCheck(entry.type);\n    if (!plugin) {\n      results.push({\n        pass: false,\n        score: 0,\n        reason: `unknown effect type: ${entry.type}`,\n      });\n      continue;\n    }\n    const r = yield* plugin.verify({\n      assertion: entry.args,\n      world,\n      evidence: evidenceFromSnapshot(evidenceCollector.toSnapshot()),\n      mode,\n    });\n    results.push(r);\n  }\n  return results;\n  });\n}\n\nexport function aggregate(\n  results: AgentSkillEvalsAssertionResult[],\n  emptyReason: string,\n  options: { emptyPass?: boolean } = {},\n): GradingResult {\n  if (results.length === 0) {\n    const pass = options.emptyPass ?? true;\n    return { pass, score: pass ? 1 : 0, reason: emptyReason };\n  }\n  const allPass = results.every((r) => r.pass);\n  const failed = results.filter((r) => !r.pass).map((r) => r.reason);\n  return {\n    pass: allPass,\n    score: allPass ? 1 : 0,\n    reason: allPass\n      ? `${results.length} check(s) passed`\n      : failed.join(\"; \"),\n    componentResults: results.map((r) => ({\n      pass: r.pass,\n      score: r.score,\n      reason: r.reason,\n    })),\n  };\n}\n","import * as Either from \"effect/Either\";\nimport * as Effect from \"effect/Effect\";\nimport {\n  aggregate,\n  loadEvidenceEffect,\n  loadMetadataEffect,\n  loadWorld,\n  runEntriesEffect,\n  type GradingResult,\n  type PromptfooAssertContext,\n} from \"./_shared.js\";\nimport { parseRuntimeTestFields } from \"../assertion-entries.js\";\nimport { writeEvidenceToEffect } from \"../agent/evidence.js\";\nimport {\n  Environment,\n  FileSystem,\n  NodeServicesLive,\n} from \"../internal-services.js\";\nimport {\n  RuntimeCheckCatalog,\n  RuntimeCheckCatalogLive,\n} from \"../runtime-checks/catalog.js\";\n\nexport default async function skillTest(\n  _output: string,\n  context: PromptfooAssertContext,\n): Promise<GradingResult> {\n  return Effect.runPromise(\n    skillTestEffect(_output, context).pipe(\n      Effect.provide(RuntimeCheckCatalogLive),\n      Effect.provide(NodeServicesLive),\n    ),\n  );\n}\n\nfunction skillTestEffect(\n  _output: string,\n  context: PromptfooAssertContext,\n): Effect.Effect<GradingResult, never, FileSystem | Environment | RuntimeCheckCatalog> {\n  return Effect.gen(function* () {\n    const meta = yield* loadMetadataEffect(context);\n    if (!meta) {\n      return {\n        pass: false,\n        score: 0,\n        reason: \"skill.test: provider metadata missing\",\n      };\n    }\n\n    const preconditionResults = meta.preconditionResults ?? [];\n    if (!meta.preconditionsPassed) {\n      return aggregate(preconditionResults, \"skill.test: preconditions failed\");\n    }\n\n    const vars = (context.vars ?? context.test?.vars ?? {}) as Record<string, unknown>;\n    const loadedEvidence = yield* Effect.either(loadEvidenceEffect(meta));\n    if (Either.isLeft(loadedEvidence)) {\n      const err = loadedEvidence.left;\n      return { pass: false, score: 0, reason: err instanceof Error ? err.message : String(err) };\n    }\n    const evidenceCollector = loadedEvidence.right;\n    const world = loadWorld(meta, evidenceCollector);\n\n    const parsed = parseRuntimeTestFields(vars);\n    const parseResults = parsed.errors\n      .filter((error) => error.field !== \"preconditions\")\n      .map((error) => ({\n        pass: false,\n        score: 0,\n        reason: `runtime test field ${error.index === undefined ? error.field : `${error.field}[${error.index}]`}: ${error.reason}`,\n      }));\n    const shouldResults = yield* runEntriesEffect(parsed.should, world, evidenceCollector, \"should\");\n    const shouldNotResults = yield* runEntriesEffect(parsed.should_not, world, evidenceCollector, \"should_not\");\n    yield* writeEvidenceToEffect(evidenceCollector, meta.runDir).pipe(Effect.orDie);\n    const results = [...preconditionResults, ...parseResults, ...shouldResults, ...shouldNotResults];\n    return aggregate(results, \"skill.test: no Runtime Test Fields checks declared\", {\n      emptyPass: false,\n    });\n  });\n}\n","import type { GradingResult, PromptfooAssertContext } from \"./_shared.js\";\n\ninterface TokenUsage {\n  total?: number;\n  prompt?: number;\n  completion?: number;\n  cached?: number;\n}\n\ninterface BudgetSettings {\n  maxTotalTokens?: number;\n  maxPromptTokens?: number;\n  maxCompletionTokens?: number;\n  maxCachedTokens?: number;\n}\n\nconst budgetFields = [\n  [\"total\", \"maxTotalTokens\"],\n  [\"prompt\", \"maxPromptTokens\"],\n  [\"completion\", \"maxCompletionTokens\"],\n  [\"cached\", \"maxCachedTokens\"],\n] as const;\n\nfunction configObject(value: unknown): Record<string, unknown> {\n  return value && typeof value === \"object\" && !Array.isArray(value)\n    ? value as Record<string, unknown>\n    : {};\n}\n\nfunction numberSetting(config: Record<string, unknown>, key: keyof BudgetSettings): number | undefined {\n  const value = config[key];\n  return typeof value === \"number\" && Number.isFinite(value) ? value : undefined;\n}\n\nfunction budgetSettings(context: PromptfooAssertContext): BudgetSettings {\n  const config = configObject(context.config?.agentSkillEvals);\n  return {\n    maxTotalTokens: numberSetting(config, \"maxTotalTokens\"),\n    maxPromptTokens: numberSetting(config, \"maxPromptTokens\"),\n    maxCompletionTokens: numberSetting(config, \"maxCompletionTokens\"),\n    maxCachedTokens: numberSetting(config, \"maxCachedTokens\"),\n  };\n}\n\nfunction tokenUsage(context: PromptfooAssertContext): TokenUsage | undefined {\n  const usage = context.providerResponse?.tokenUsage;\n  return usage && typeof usage === \"object\" && !Array.isArray(usage)\n    ? usage as TokenUsage\n    : undefined;\n}\n\nexport default async function skillBudget(\n  _output: string,\n  context: PromptfooAssertContext,\n): Promise<GradingResult> {\n  const usage = tokenUsage(context);\n  if (!usage) {\n    return {\n      pass: false,\n      score: 0,\n      reason: \"skill.budget: provider tokenUsage missing\",\n    };\n  }\n\n  const settings = budgetSettings(context);\n  const configured = budgetFields.filter(([, limitKey]) => settings[limitKey] !== undefined);\n  if (configured.length === 0) {\n    return {\n      pass: false,\n      score: 0,\n      reason: \"skill.budget: configure at least one token limit\",\n    };\n  }\n\n  const components = configured.map(([usageKey, limitKey]) => {\n    const actual = usage[usageKey];\n    const limit = settings[limitKey] ?? 0;\n    if (typeof actual !== \"number\" || !Number.isFinite(actual)) {\n      return {\n        pass: false,\n        score: 0,\n        reason: `${usageKey} tokens missing`,\n      };\n    }\n    return {\n      pass: actual <= limit,\n      score: actual <= limit ? 1 : 0,\n      reason: `${usageKey} tokens ${actual} <= ${limit}`,\n    };\n  });\n  const failed = components.filter((component) => !component.pass);\n  return {\n    pass: failed.length === 0,\n    score: failed.length === 0 ? 1 : 0,\n    reason: failed.length === 0\n      ? `skill.budget: ${components.length} budget(s) passed`\n      : failed.map((component) => component.reason).join(\"; \"),\n    componentResults: components,\n  };\n}\n","import type { StaticProviderMetadata } from \"../index.js\";\n\nexport interface PromptfooAssertContext {\n  vars?: Record<string, unknown>;\n  providerResponse?: { metadata?: unknown };\n  test?: { vars?: Record<string, unknown> };\n  assertion?: { metric?: string };\n  assert?: { metric?: string };\n  config?: { metric?: string; agentSkillEvals?: unknown };\n  metric?: string;\n}\n\nexport interface GradingResult {\n  pass: boolean;\n  score: number;\n  reason: string;\n  componentResults?: Array<{ pass: boolean; score: number; reason: string }>;\n}\n\nexport function getStaticMeta(\n  context: PromptfooAssertContext,\n): StaticProviderMetadata | null {\n  const m = context.providerResponse?.metadata;\n  if (m && typeof m === \"object\" && (\"skill\" in m || \"tests\" in m)) {\n    return m as StaticProviderMetadata;\n  }\n  return null;\n}\n\nexport function pass(reason: string, components?: GradingResult[\"componentResults\"]): GradingResult {\n  return { pass: true, score: 1, reason, componentResults: components };\n}\n\nexport function fail(reason: string, components?: GradingResult[\"componentResults\"]): GradingResult {\n  return { pass: false, score: 0, reason, componentResults: components };\n}\n","import * as Schema from \"effect/Schema\";\nimport type { PromptfooAssertContext } from \"./_shared.js\";\n\nexport interface SkillCheckSettings {\n  maxSkillLines: number;\n  destructiveEffects: readonly string[];\n  requireTokenBudget: boolean;\n  riskyEffects: readonly string[];\n}\n\nconst DEFAULT_SETTINGS: SkillCheckSettings = {\n  maxSkillLines: 200,\n  destructiveEffects: [\"file.changes_outside_scope\", \"tool.called\"],\n  requireTokenBudget: false,\n  riskyEffects: [\"file.changes_outside_scope\", \"tool.called\"],\n};\n\nconst StringArraySchema = Schema.Array(Schema.String);\n\nfunction configObject(value: unknown): Record<string, unknown> {\n  return value && typeof value === \"object\" && !Array.isArray(value)\n    ? value as Record<string, unknown>\n    : {};\n}\n\nfunction numberSetting(\n  config: Record<string, unknown>,\n  key: string,\n  fallback: number,\n): number {\n  return Schema.is(Schema.Number)(config[key]) ? config[key] : fallback;\n}\n\nfunction stringArraySetting(\n  config: Record<string, unknown>,\n  key: string,\n  fallback: readonly string[],\n): readonly string[] {\n  return Schema.is(StringArraySchema)(config[key]) ? config[key] : fallback;\n}\n\nfunction booleanSetting(\n  config: Record<string, unknown>,\n  key: string,\n  fallback: boolean,\n): boolean {\n  return typeof config[key] === \"boolean\" ? config[key] : fallback;\n}\n\nexport function skillCheckSettings(context: PromptfooAssertContext): SkillCheckSettings {\n  const config = configObject(context.config?.agentSkillEvals);\n  return {\n    maxSkillLines: numberSetting(config, \"maxSkillLines\", DEFAULT_SETTINGS.maxSkillLines),\n    destructiveEffects: stringArraySetting(\n      config,\n      \"destructiveEffects\",\n      DEFAULT_SETTINGS.destructiveEffects,\n    ),\n    requireTokenBudget: booleanSetting(\n      config,\n      \"requireTokenBudget\",\n      DEFAULT_SETTINGS.requireTokenBudget,\n    ),\n    riskyEffects: stringArraySetting(\n      config,\n      \"riskyEffects\",\n      DEFAULT_SETTINGS.riskyEffects,\n    ),\n  };\n}\n","import { fail, getStaticMeta, pass, type GradingResult, type PromptfooAssertContext } from \"./_shared.js\";\nimport { skillCheckSettings } from \"./settings.js\";\n\n/**\n * SPEC §7.2 — context economy. Hard: referenced files must exist (already\n * surfaced via missingFiles). Warnings: SKILL.md size, reference depth,\n * missing TOC in long reference files.\n *\n * Implemented as warnings-only: this assertion always passes, but emits a\n * `score < 1` reason describing how SKILL.md compares to thresholds.\n */\nexport default async function contextEconomy(\n  _output: string,\n  context: PromptfooAssertContext,\n): Promise<GradingResult> {\n  const meta = getStaticMeta(context);\n  if (!meta) return fail(\"context-economy: provider metadata missing\");\n  const skill = meta.skill;\n  if (!skill) return fail(\"context-economy: skill not parsed\");\n\n  const { maxSkillLines: maxLines } = skillCheckSettings(context);\n\n  const components: GradingResult[\"componentResults\"] = [];\n\n  components.push({\n    pass: skill.totalLines <= maxLines,\n    score: skill.totalLines <= maxLines ? 1 : 0.5,\n    reason: `SKILL.md ${skill.totalLines} line(s) (limit ${maxLines})`,\n  });\n\n  components.push({\n    pass: skill.missingReferences.length === 0,\n    score: skill.missingReferences.length === 0 ? 1 : 0,\n    reason:\n      skill.missingReferences.length === 0\n        ? `${skill.references.length} reference(s) all resolved`\n        : `missing references: ${skill.missingReferences.slice(0, 3).join(\", \")}`,\n  });\n\n  // Hard fail only on missing references; soft on size.\n  const failed = components.filter((c) => !c.pass && c.reason.startsWith(\"missing references\"));\n  if (failed.length === 0) {\n    const oversize = components.find((c) => c.score < 1);\n    return oversize\n      ? { pass: true, score: 0.5, reason: oversize.reason, componentResults: components }\n      : pass(`context-economy: ok`, components);\n  }\n  return fail(failed.map((f) => `✗ ${f.reason}`).join(\"; \"), components);\n}\n","import { fail, getStaticMeta, pass, type GradingResult, type PromptfooAssertContext } from \"./_shared.js\";\n\n/**\n * SPEC §7.4 — verifier scripts referenced by tests must exist and be\n * runnable. Generated tests must not reference missing fixtures.\n */\nexport default async function executableHelper(\n  _output: string,\n  context: PromptfooAssertContext,\n): Promise<GradingResult> {\n  const meta = getStaticMeta(context);\n  if (!meta) return fail(\"executable-helper: provider metadata missing\");\n  const tests = meta.tests;\n  if (!tests) return fail(\"executable-helper: tests not parsed\");\n\n  const components: GradingResult[\"componentResults\"] = [];\n\n  components.push({\n    pass: tests.missingVerifierScripts.length === 0,\n    score: tests.missingVerifierScripts.length === 0 ? 1 : 0,\n    reason: tests.missingVerifierScripts.length === 0\n      ? `${tests.verifierScripts.length} verifier script(s) all present`\n      : `missing verifier scripts: ${tests.missingVerifierScripts.slice(0, 3).join(\", \")}`,\n  });\n\n  components.push({\n    pass: tests.nonExecutableVerifierScripts.length === 0,\n    score: tests.nonExecutableVerifierScripts.length === 0 ? 1 : 0,\n    reason: tests.nonExecutableVerifierScripts.length === 0\n      ? `${tests.verifierScripts.length} verifier script(s) executable`\n      : `non-executable verifier scripts: ${tests.nonExecutableVerifierScripts.slice(0, 3).join(\", \")}`,\n  });\n\n  components.push({\n    pass: tests.missingFixturePaths.length === 0,\n    score: tests.missingFixturePaths.length === 0 ? 1 : 0,\n    reason: tests.missingFixturePaths.length === 0\n      ? `${tests.fixturePaths.length} fixture(s) all present`\n      : `missing fixtures: ${tests.missingFixturePaths.slice(0, 3).join(\", \")}`,\n  });\n\n  const failed = components.filter((c) => !c.pass);\n  if (failed.length === 0) return pass(\"executable-helper: ok\", components);\n  return fail(failed.map((f) => `✗ ${f.reason}`).join(\"; \"), components);\n}\n","import { fail, getStaticMeta, pass, type GradingResult, type PromptfooAssertContext } from \"./_shared.js\";\nimport { skillCheckSettings } from \"./settings.js\";\n\n/**\n * SPEC §7.3 — for skills whose test pack uses destructive effects, the\n * SKILL.md must contain confirmation/clarification language and at least\n * one negative test must declare forbidden effects.\n *\n * `assert.config.agentSkillEvals.destructiveEffects` overrides the default list.\n */\nconst CONFIRMATION_RE =\n  /\\b(confirm|ask first|do not.*without|before.*push|require.*approval|do not.*destructive)\\b/i;\nconst PLAN_BEFORE_ACT_RE =\n  /\\b(plan first|plan before|read.*before.*write|validate.*before|dry.run)\\b/i;\n\nexport default async function instructionCalibration(\n  _output: string,\n  context: PromptfooAssertContext,\n): Promise<GradingResult> {\n  const meta = getStaticMeta(context);\n  if (!meta) return fail(\"skill.instructions: provider metadata missing\");\n  const skill = meta.skill;\n  const tests = meta.tests;\n  if (!skill) return fail(\"skill.instructions: skill not parsed\");\n\n  const destructive = new Set<string>(skillCheckSettings(context).destructiveEffects);\n\n  const usesDestructive = tests\n    ? tests.tests.some((t) => t.effectTypes.some((e) => destructive.has(e)))\n    : false;\n\n  if (!usesDestructive) {\n    return pass(\"skill.instructions: no destructive effects in test pack\");\n  }\n\n  const components: GradingResult[\"componentResults\"] = [];\n  const hasConfirm = CONFIRMATION_RE.test(skill.body) || PLAN_BEFORE_ACT_RE.test(skill.body);\n  components.push({\n    pass: hasConfirm,\n    score: hasConfirm ? 1 : 0,\n    reason: hasConfirm\n      ? \"SKILL.md describes confirmation / plan-before-act\"\n      : \"SKILL.md uses destructive effects but lacks confirmation / plan-before-act language\",\n  });\n\n  const declaresForbidden = tests\n    ? tests.tests.some((t) => Array.isArray(t.vars.should_not) && t.vars.should_not.length > 0)\n    : false;\n  components.push({\n    pass: declaresForbidden,\n    score: declaresForbidden ? 1 : 0,\n    reason: declaresForbidden\n      ? \"test pack declares forbidden effects (should_not)\"\n      : \"no should_not declared in any test, despite destructive effects\",\n  });\n\n  const failed = components.filter((c) => !c.pass);\n  if (failed.length === 0) return pass(\"skill.instructions: ok\", components);\n  return fail(failed.map((f) => `✗ ${f.reason}`).join(\"; \"), components);\n}\n","import { fail, getStaticMeta, pass, type GradingResult, type PromptfooAssertContext } from \"./_shared.js\";\nimport { skillCheckSettings } from \"./settings.js\";\n\n/**\n * SPEC §7.6 — risky skills (those whose test pack uses any risky effect)\n * must include at least one negative test.\n *\n * `assert.config.agentSkillEvals.riskyEffects` overrides the default risky-effect list.\n */\nexport default async function negativeCoverage(\n  _output: string,\n  context: PromptfooAssertContext,\n): Promise<GradingResult> {\n  const meta = getStaticMeta(context);\n  if (!meta) return fail(\"skill.tests: provider metadata missing\");\n  const tests = meta.tests;\n  if (!tests) return fail(\"skill.tests: tests not parsed\");\n\n  const risky = new Set<string>(skillCheckSettings(context).riskyEffects);\n\n  const usesRisky = tests.tests.some((t) => t.effectTypes.some((e) => risky.has(e)));\n  if (!usesRisky) {\n    return pass(\"skill.tests: no risky effects, negative test not required\");\n  }\n\n  const negatives = tests.tests.filter((t) => t.isNegative);\n  if (negatives.length === 0) {\n    return fail(\n      `skill.tests: skill uses risky effects (${[...risky].join(\", \")}) but no negative test (kind: negative) is declared`,\n    );\n  }\n  return pass(`skill.tests: ${negatives.length} negative test(s)`);\n}\n","import { fail, getStaticMeta, pass, type GradingResult, type PromptfooAssertContext } from \"./_shared.js\";\n\nconst GENERIC_PHRASES = [\n  /\\bhelp(s|ing)?\\b/i,\n  /\\bgithub workflows?\\b/i,\n  /\\bvarious\\b/i,\n  /\\bany kind of\\b/i,\n];\n\nconst WHEN_TO_USE_RE = /\\b(use when|use this|when (?:the )?(?:user|you))\\b/i;\nconst WHEN_NOT_RE = /\\bdo not use|do not invoke|don'?t use|not for|avoid using\\b/i;\n\n/**\n * SPEC §7.1 — routing metadata hard checks.\n */\nexport default async function routingMetadata(\n  _output: string,\n  context: PromptfooAssertContext,\n): Promise<GradingResult> {\n  const meta = getStaticMeta(context);\n  if (!meta) return fail(\"routing-metadata: provider metadata missing\");\n  const skill = meta.skill;\n  if (!skill) return fail(\"routing-metadata: skill not parsed (vars.skillPath missing?)\");\n\n  const fm = skill.frontmatter;\n  const components: GradingResult[\"componentResults\"] = [];\n\n  const name = typeof fm.name === \"string\" ? fm.name.trim() : \"\";\n  components.push({\n    pass: name.length > 0,\n    score: name.length > 0 ? 1 : 0,\n    reason: name ? `name: ${name}` : \"missing `name` frontmatter\",\n  });\n\n  const desc = typeof fm.description === \"string\" ? fm.description.trim() : \"\";\n  components.push({\n    pass: desc.length > 0,\n    score: desc.length > 0 ? 1 : 0,\n    reason: desc ? `description present (${desc.length} chars)` : \"missing `description` frontmatter\",\n  });\n\n  const sayWhen = WHEN_TO_USE_RE.test(desc);\n  components.push({\n    pass: sayWhen,\n    score: sayWhen ? 1 : 0,\n    reason: sayWhen ? \"description says when to use\" : \"description does not say when to use (e.g. 'Use when …')\",\n  });\n\n  const sayWhenNot = WHEN_NOT_RE.test(desc);\n  components.push({\n    pass: sayWhenNot,\n    score: sayWhenNot ? 1 : 0,\n    reason: sayWhenNot ? \"description says when not to use\" : \"description does not say when not to use (e.g. 'Do not use for …')\",\n  });\n\n  const generic = GENERIC_PHRASES.some((re) => re.test(desc)) && desc.length < 80;\n  components.push({\n    pass: !generic,\n    score: generic ? 0 : 1,\n    reason: generic ? \"description is too generic\" : \"description is specific enough\",\n  });\n\n  const failed = components.filter((c) => !c.pass);\n  if (failed.length === 0) {\n    return pass(`routing-metadata: ${components.length}/${components.length} checks ok`, components);\n  }\n  return fail(\n    failed.map((f) => `✗ ${f.reason}`).join(\"; \"),\n    components,\n  );\n}\n","import { fail, getStaticMeta, pass, type GradingResult, type PromptfooAssertContext } from \"./_shared.js\";\nimport { skillCheckSettings } from \"./settings.js\";\n\n/**\n * SPEC §7.6 — every runtime test must have prompt + fixture (or fixtureless)\n * + at least one of should/should_not. Unsupported effect types fail\n * static validation.\n */\nexport default async function scenarioValidity(\n  _output: string,\n  context: PromptfooAssertContext,\n): Promise<GradingResult> {\n  const meta = getStaticMeta(context);\n  if (!meta) return fail(\"skill.tests: provider metadata missing\");\n  const tests = meta.tests;\n  if (!tests) return fail(\"skill.tests: tests not parsed (vars.testsGlob missing?)\");\n  const settings = skillCheckSettings(context);\n\n  const components: GradingResult[\"componentResults\"] = [];\n\n  if (tests.parseErrors.length > 0) {\n    components.push({\n      pass: false,\n      score: 0,\n      reason: `parse errors: ${tests.parseErrors.map((e) => `${e.filePath}: ${e.error}`).join(\"; \")}`,\n    });\n  }\n\n  if (tests.matchedFiles.length === 0) {\n    components.push({\n      pass: false,\n      score: 0,\n      reason: \"testsGlob matched no test files\",\n    });\n  }\n\n  for (const t of tests.tests) {\n    const issues: string[] = [];\n    for (const error of t.entryErrors) {\n      const at = error.index === undefined ? error.field : `${error.field}[${error.index}]`;\n      issues.push(`${at}: ${error.reason}`);\n    }\n    if (typeof t.vars.prompt !== \"string\" || t.vars.prompt.length === 0) {\n      issues.push(\"missing vars.prompt\");\n    }\n    if (!t.hasFixture) {\n      issues.push(\"missing vars.fixture (or vars.fixtureless: true)\");\n    }\n    const hasCheck =\n      (Array.isArray(t.vars.should) && t.vars.should.length > 0) ||\n      (Array.isArray(t.vars.should_not) && t.vars.should_not.length > 0);\n    if (!hasCheck) issues.push(\"no should / should_not\");\n    if (settings.requireTokenBudget && !t.hasTokenBudget) {\n      issues.push(\"missing skill.budget assertion\");\n    }\n    components.push({\n      pass: issues.length === 0,\n      score: issues.length === 0 ? 1 : 0,\n      reason: `${t.description ?? \"(no description)\"}: ${issues.length === 0 ? \"ok\" : issues.join(\", \")}`,\n    });\n  }\n\n  if (meta.unresolvedEffectTypes.length > 0) {\n    components.push({\n      pass: false,\n      score: 0,\n      reason: `unsupported effect types: ${meta.unresolvedEffectTypes.join(\", \")}`,\n    });\n  }\n\n  if (meta.missingFiles.length > 0) {\n    components.push({\n      pass: false,\n      score: 0,\n      reason: `missing referenced files: ${meta.missingFiles.slice(0, 5).join(\", \")}`,\n    });\n  }\n\n  const failed = components.filter((c) => !c.pass);\n  if (failed.length === 0) {\n    return pass(`skill.tests: ${tests.tests.length} test(s) ok`, components);\n  }\n  return fail(\n    failed.map((f) => `✗ ${f.reason}`).join(\"; \"),\n    components,\n  );\n}\n","import * as Effect from \"effect/Effect\";\nimport contextEconomy from \"./context-economy.js\";\nimport executableHelper from \"./executable-helper.js\";\nimport instructionCalibration from \"./instruction-calibration.js\";\nimport negativeCoverage from \"./negative-coverage.js\";\nimport routingMetadata from \"./routing-metadata.js\";\nimport scenarioValidity from \"./scenario-validity.js\";\nimport type { GradingResult, PromptfooAssertContext } from \"./_shared.js\";\n\ntype AssertionFn = (\n  output: string,\n  context: PromptfooAssertContext,\n) => Promise<GradingResult>;\n\nconst metricChecks: Record<string, readonly AssertionFn[]> = {\n  \"skill.activation\": [routingMetadata],\n  \"skill.budgets\": [scenarioValidity],\n  \"skill.context\": [contextEconomy],\n  \"skill.instructions\": [instructionCalibration],\n  \"skill.tests\": [scenarioValidity, negativeCoverage],\n  \"skill.verifiers\": [executableHelper],\n};\n\nmetricChecks[\"skill.checks\"] = [\n  routingMetadata,\n  contextEconomy,\n  instructionCalibration,\n  scenarioValidity,\n  negativeCoverage,\n  executableHelper,\n];\n\nconst availableMetrics = Object.keys(metricChecks).sort().join(\", \");\n\nfunction metricFrom(context: PromptfooAssertContext): string | undefined {\n  const candidates = [\n    context.assertion?.metric,\n    context.assert?.metric,\n    context.config?.metric,\n    context.metric,\n  ];\n  return candidates.find((metric): metric is string => typeof metric === \"string\");\n}\n\nasync function runChecks(\n  output: string,\n  context: PromptfooAssertContext,\n  checks: readonly AssertionFn[],\n): Promise<GradingResult> {\n  return Effect.runPromise(runChecksEffect(output, context, checks));\n}\n\nfunction runChecksEffect(\n  output: string,\n  context: PromptfooAssertContext,\n  checks: readonly AssertionFn[],\n): Effect.Effect<GradingResult> {\n  return Effect.gen(function* () {\n  const results = yield* Effect.forEach(\n    checks,\n    (check) => Effect.promise(() => check(output, context)),\n    { concurrency: \"unbounded\" },\n  );\n  const failed = results.filter((result) => !result.pass);\n  const soft = results.filter((result) => result.pass && result.score < 1);\n  return {\n    pass: failed.length === 0,\n    score: failed.length === 0\n      ? Math.min(...results.map((result) => result.score))\n      : 0,\n    reason: failed.length === 0\n      ? soft.length > 0\n        ? soft.map((result) => result.reason).join(\"; \")\n        : `skill checks: ${results.length} check(s) passed`\n      : failed.map((result) => result.reason).join(\"; \"),\n    componentResults: results,\n  };\n  });\n}\n\nexport async function agentSkillEvalsStaticAssertions(\n  output: string,\n  context: PromptfooAssertContext,\n): Promise<GradingResult> {\n  return Effect.runPromise(agentSkillEvalsStaticAssertionsEffect(output, context));\n}\n\nfunction agentSkillEvalsStaticAssertionsEffect(\n  output: string,\n  context: PromptfooAssertContext,\n): Effect.Effect<GradingResult> {\n  return Effect.gen(function* () {\n  const metric = metricFrom(context);\n  const checks = metric ? metricChecks[metric] : undefined;\n  if (!checks) {\n    return {\n      pass: false,\n      score: 0,\n      reason: `agent-skill-evals skill checks: unknown metric \"${metric ?? \"missing\"}\". Available metrics: ${availableMetrics}`,\n    };\n  }\n  return yield* runChecksEffect(output, context, checks);\n  });\n}\n\nexport default agentSkillEvalsStaticAssertions;\n","import skillTest from \"./skill-test.js\";\nimport skillBudget from \"./skill-budget.js\";\nimport { agentSkillEvalsStaticAssertions } from \"../skill-checks/assertions-static/promptfoo.js\";\nimport type { GradingResult, PromptfooAssertContext } from \"./_shared.js\";\n\ntype AssertionFn = (\n  output: string,\n  context: PromptfooAssertContext,\n) => Promise<GradingResult>;\n\nconst routes: Record<string, AssertionFn> = {\n  \"skill.budget\": skillBudget,\n  \"skill.test\": skillTest,\n};\nconst staticMetrics = [\n  \"skill.checks\",\n  \"skill.activation\",\n  \"skill.budgets\",\n  \"skill.context\",\n  \"skill.instructions\",\n  \"skill.tests\",\n  \"skill.verifiers\",\n];\nconst availableMetrics = [...Object.keys(routes), ...staticMetrics].sort().join(\", \");\n\nexport { skillTest };\n\nfunction metricFrom(context: PromptfooAssertContext): string | undefined {\n  const candidates = [\n    context.assertion?.metric,\n    context.assert?.metric,\n    context.config?.metric,\n    context.metric,\n  ];\n  return candidates.find((metric): metric is string => typeof metric === \"string\");\n}\n\nexport default async function agentSkillEvalsAssertions(\n  output: string,\n  context: PromptfooAssertContext,\n): Promise<GradingResult> {\n  const metric = metricFrom(context);\n  const assertion = metric ? routes[metric] : undefined;\n  if (assertion) {\n    return assertion(output, context);\n  }\n  if (metric && staticMetrics.includes(metric)) {\n    return agentSkillEvalsStaticAssertions(output, context);\n  }\n  return {\n    pass: false,\n    score: 0,\n    reason: `agent-skill-evals assertions: unknown metric \"${metric ?? \"missing\"}\". Available metrics: ${availableMetrics}`,\n  };\n}\n"],"mappings":";;;;;;;AA4DA,SAAgB,mBACd,SACwF;AACxF,QAAO,OAAO,IAAI,aAAa;EAC/B,MAAM,SAAS,QAAQ,kBAAkB;AACzC,MAAI,UAAU,OAAO,WAAW,YAAY,eAAe,OACzD,QAAO;EAKT,MAAM,UAAS,QADI,OADQ,aACI,KACZ;AACnB,MAAI,QAAQ;GAEV,MAAM,SAAS,QAAO,OADJ,YACO,SAAS,KAAK,QAAQ,8BAA8B,CAAC,CAAC,KAC7E,OAAO,KAAK,QAAQ;AAClB,QAAI;AACF,YAAO,KAAK,MAAM,IAAI;YAChB;AACN,YAAO;;KAET,EACF,OAAO,eAAe,OAAO,QAAQ,KAAK,CAAC,CAC5C;AACD,OAAI,OACF,QAAO;;AAGX,SAAO;GACL;;AASJ,SAAgB,mBACd,MACqD;AACrD,QAAO,OAAO,IAAI,aAAa;EAE/B,MAAM,MAAM,QAAO,OADD,YACI,SAAS,KAAK,aAAa,CAAC,KAChD,OAAO,UAAU,wBAAQ,IAAI,MAAM,4BAA4B,KAAK,aAAa,IAAI,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GAAG,CAAC,CAC1I;EAMD,MAAM,UAAU,6BAA6B,OALvB,OAAO,IAAI;GAC/B,WAAW,KAAK,MAAM,IAAI;GAC1B,QAAQ,wBACN,IAAI,MAAM,6BAA6B,KAAK,aAAa,IAAI,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GAAG;GACnH,CAAC,CACkD;AACpD,MAAI,OAAO,OAAO,QAAQ,CACxB,QAAO,OAAO,OAAO,qBACnB,IAAI,MAAM,4DAA4D,QAAQ,KAAK,UAAU,CAC9F;EAEH,MAAM,WAAW,QAAQ;AACzB,SAAO,kBAAkB,aAAa,SAAS;GAC7C;;AAGJ,SAAgB,UACd,MACA,mBACa;AACb,QAAO,gBAAgB,KAAK,YAAY,UAAU,kBAAkB,WAAW,MAAM,CAAC;;AAgBxF,SAAgB,iBACd,SACA,OACA,mBACA,MAC6E;AAC7E,QAAO,OAAO,IAAI,aAAa;EAC/B,MAAM,UAA4C,EAAE;AACpD,OAAK,MAAM,SAAS,SAAS;GAC3B,MAAM,SAAS,OAAO,gBAAgB,MAAM,KAAK;AACjD,OAAI,CAAC,QAAQ;AACX,YAAQ,KAAK;KACX,MAAM;KACN,OAAO;KACP,QAAQ,wBAAwB,MAAM;KACvC,CAAC;AACF;;GAEF,MAAM,IAAI,OAAO,OAAO,OAAO;IAC7B,WAAW,MAAM;IACjB;IACA,UAAU,qBAAqB,kBAAkB,YAAY,CAAC;IAC9D;IACD,CAAC;AACF,WAAQ,KAAK,EAAE;;AAEjB,SAAO;GACL;;AAGJ,SAAgB,UACd,SACA,aACA,UAAmC,EAAE,EACtB;AACf,KAAI,QAAQ,WAAW,GAAG;EACxB,MAAM,OAAO,QAAQ,aAAa;AAClC,SAAO;GAAE;GAAM,OAAO,OAAO,IAAI;GAAG,QAAQ;GAAa;;CAE3D,MAAM,UAAU,QAAQ,OAAO,MAAM,EAAE,KAAK;CAC5C,MAAM,SAAS,QAAQ,QAAQ,MAAM,CAAC,EAAE,KAAK,CAAC,KAAK,MAAM,EAAE,OAAO;AAClE,QAAO;EACL,MAAM;EACN,OAAO,UAAU,IAAI;EACrB,QAAQ,UACJ,GAAG,QAAQ,OAAO,oBAClB,OAAO,KAAK,KAAK;EACrB,kBAAkB,QAAQ,KAAK,OAAO;GACpC,MAAM,EAAE;GACR,OAAO,EAAE;GACT,QAAQ,EAAE;GACX,EAAE;EACJ;;;;AC3KH,eAA8B,UAC5B,SACA,SACwB;AACxB,QAAO,OAAO,WACZ,gBAAgB,SAAS,QAAQ,CAAC,KAChC,OAAO,QAAQ,wBAAwB,EACvC,OAAO,QAAQ,iBAAiB,CACjC,CACF;;AAGH,SAAS,gBACP,SACA,SACqF;AACrF,QAAO,OAAO,IAAI,aAAa;EAC7B,MAAM,OAAO,OAAO,mBAAmB,QAAQ;AAC/C,MAAI,CAAC,KACH,QAAO;GACL,MAAM;GACN,OAAO;GACP,QAAQ;GACT;EAGH,MAAM,sBAAsB,KAAK,uBAAuB,EAAE;AAC1D,MAAI,CAAC,KAAK,oBACR,QAAO,UAAU,qBAAqB,mCAAmC;EAG3E,MAAM,OAAQ,QAAQ,QAAQ,QAAQ,MAAM,QAAQ,EAAE;EACtD,MAAM,iBAAiB,OAAO,OAAO,OAAO,mBAAmB,KAAK,CAAC;AACrE,MAAI,OAAO,OAAO,eAAe,EAAE;GACjC,MAAM,MAAM,eAAe;AAC3B,UAAO;IAAE,MAAM;IAAO,OAAO;IAAG,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;IAAE;;EAE5F,MAAM,oBAAoB,eAAe;EACzC,MAAM,QAAQ,UAAU,MAAM,kBAAkB;EAEhD,MAAM,SAAS,uBAAuB,KAAK;EAC3C,MAAM,eAAe,OAAO,OACzB,QAAQ,UAAU,MAAM,UAAU,gBAAgB,CAClD,KAAK,WAAW;GACf,MAAM;GACN,OAAO;GACP,QAAQ,sBAAsB,MAAM,UAAU,KAAA,IAAY,MAAM,QAAQ,GAAG,MAAM,MAAM,GAAG,MAAM,MAAM,GAAG,IAAI,MAAM;GACpH,EAAE;EACL,MAAM,gBAAgB,OAAO,iBAAiB,OAAO,QAAQ,OAAO,mBAAmB,SAAS;EAChG,MAAM,mBAAmB,OAAO,iBAAiB,OAAO,YAAY,OAAO,mBAAmB,aAAa;AAC3G,SAAO,sBAAsB,mBAAmB,KAAK,OAAO,CAAC,KAAK,OAAO,MAAM;AAE/E,SAAO,UAAU;GADA,GAAG;GAAqB,GAAG;GAAc,GAAG;GAAe,GAAG;GACvD,EAAE,sDAAsD,EAC9E,WAAW,OACZ,CAAC;GACF;;;;AC9DJ,MAAM,eAAe;CACnB,CAAC,SAAS,iBAAiB;CAC3B,CAAC,UAAU,kBAAkB;CAC7B,CAAC,cAAc,sBAAsB;CACrC,CAAC,UAAU,kBAAkB;CAC9B;AAED,SAASA,eAAa,OAAyC;AAC7D,QAAO,SAAS,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,MAAM,GAC9D,QACA,EAAE;;AAGR,SAASC,gBAAc,QAAiC,KAA+C;CACrG,MAAM,QAAQ,OAAO;AACrB,QAAO,OAAO,UAAU,YAAY,OAAO,SAAS,MAAM,GAAG,QAAQ,KAAA;;AAGvE,SAAS,eAAe,SAAiD;CACvE,MAAM,SAASD,eAAa,QAAQ,QAAQ,gBAAgB;AAC5D,QAAO;EACL,gBAAgBC,gBAAc,QAAQ,iBAAiB;EACvD,iBAAiBA,gBAAc,QAAQ,kBAAkB;EACzD,qBAAqBA,gBAAc,QAAQ,sBAAsB;EACjE,iBAAiBA,gBAAc,QAAQ,kBAAkB;EAC1D;;AAGH,SAAS,WAAW,SAAyD;CAC3E,MAAM,QAAQ,QAAQ,kBAAkB;AACxC,QAAO,SAAS,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,MAAM,GAC9D,QACA,KAAA;;AAGN,eAA8B,YAC5B,SACA,SACwB;CACxB,MAAM,QAAQ,WAAW,QAAQ;AACjC,KAAI,CAAC,MACH,QAAO;EACL,MAAM;EACN,OAAO;EACP,QAAQ;EACT;CAGH,MAAM,WAAW,eAAe,QAAQ;CACxC,MAAM,aAAa,aAAa,QAAQ,GAAG,cAAc,SAAS,cAAc,KAAA,EAAU;AAC1F,KAAI,WAAW,WAAW,EACxB,QAAO;EACL,MAAM;EACN,OAAO;EACP,QAAQ;EACT;CAGH,MAAM,aAAa,WAAW,KAAK,CAAC,UAAU,cAAc;EAC1D,MAAM,SAAS,MAAM;EACrB,MAAM,QAAQ,SAAS,aAAa;AACpC,MAAI,OAAO,WAAW,YAAY,CAAC,OAAO,SAAS,OAAO,CACxD,QAAO;GACL,MAAM;GACN,OAAO;GACP,QAAQ,GAAG,SAAS;GACrB;AAEH,SAAO;GACL,MAAM,UAAU;GAChB,OAAO,UAAU,QAAQ,IAAI;GAC7B,QAAQ,GAAG,SAAS,UAAU,OAAO,MAAM;GAC5C;GACD;CACF,MAAM,SAAS,WAAW,QAAQ,cAAc,CAAC,UAAU,KAAK;AAChE,QAAO;EACL,MAAM,OAAO,WAAW;EACxB,OAAO,OAAO,WAAW,IAAI,IAAI;EACjC,QAAQ,OAAO,WAAW,IACtB,iBAAiB,WAAW,OAAO,qBACnC,OAAO,KAAK,cAAc,UAAU,OAAO,CAAC,KAAK,KAAK;EAC1D,kBAAkB;EACnB;;;;AC/EH,SAAgB,cACd,SAC+B;CAC/B,MAAM,IAAI,QAAQ,kBAAkB;AACpC,KAAI,KAAK,OAAO,MAAM,aAAa,WAAW,KAAK,WAAW,GAC5D,QAAO;AAET,QAAO;;AAGT,SAAgB,KAAK,QAAgB,YAA+D;AAClG,QAAO;EAAE,MAAM;EAAM,OAAO;EAAG;EAAQ,kBAAkB;EAAY;;AAGvE,SAAgB,KAAK,QAAgB,YAA+D;AAClG,QAAO;EAAE,MAAM;EAAO,OAAO;EAAG;EAAQ,kBAAkB;EAAY;;;;ACxBxE,MAAM,mBAAuC;CAC3C,eAAe;CACf,oBAAoB,CAAC,8BAA8B,cAAc;CACjE,oBAAoB;CACpB,cAAc,CAAC,8BAA8B,cAAc;CAC5D;AAED,MAAM,oBAAoB,OAAO,MAAM,OAAO,OAAO;AAErD,SAAS,aAAa,OAAyC;AAC7D,QAAO,SAAS,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,MAAM,GAC9D,QACA,EAAE;;AAGR,SAAS,cACP,QACA,KACA,UACQ;AACR,QAAO,OAAO,GAAG,OAAO,OAAO,CAAC,OAAO,KAAK,GAAG,OAAO,OAAO;;AAG/D,SAAS,mBACP,QACA,KACA,UACmB;AACnB,QAAO,OAAO,GAAG,kBAAkB,CAAC,OAAO,KAAK,GAAG,OAAO,OAAO;;AAGnE,SAAS,eACP,QACA,KACA,UACS;AACT,QAAO,OAAO,OAAO,SAAS,YAAY,OAAO,OAAO;;AAG1D,SAAgB,mBAAmB,SAAqD;CACtF,MAAM,SAAS,aAAa,QAAQ,QAAQ,gBAAgB;AAC5D,QAAO;EACL,eAAe,cAAc,QAAQ,iBAAiB,iBAAiB,cAAc;EACrF,oBAAoB,mBAClB,QACA,sBACA,iBAAiB,mBAClB;EACD,oBAAoB,eAClB,QACA,sBACA,iBAAiB,mBAClB;EACD,cAAc,mBACZ,QACA,gBACA,iBAAiB,aAClB;EACF;;;;;;;;;;;;ACzDH,eAA8B,eAC5B,SACA,SACwB;CACxB,MAAM,OAAO,cAAc,QAAQ;AACnC,KAAI,CAAC,KAAM,QAAO,KAAK,6CAA6C;CACpE,MAAM,QAAQ,KAAK;AACnB,KAAI,CAAC,MAAO,QAAO,KAAK,oCAAoC;CAE5D,MAAM,EAAE,eAAe,aAAa,mBAAmB,QAAQ;CAE/D,MAAM,aAAgD,EAAE;AAExD,YAAW,KAAK;EACd,MAAM,MAAM,cAAc;EAC1B,OAAO,MAAM,cAAc,WAAW,IAAI;EAC1C,QAAQ,YAAY,MAAM,WAAW,kBAAkB,SAAS;EACjE,CAAC;AAEF,YAAW,KAAK;EACd,MAAM,MAAM,kBAAkB,WAAW;EACzC,OAAO,MAAM,kBAAkB,WAAW,IAAI,IAAI;EAClD,QACE,MAAM,kBAAkB,WAAW,IAC/B,GAAG,MAAM,WAAW,OAAO,8BAC3B,uBAAuB,MAAM,kBAAkB,MAAM,GAAG,EAAE,CAAC,KAAK,KAAK;EAC5E,CAAC;CAGF,MAAM,SAAS,WAAW,QAAQ,MAAM,CAAC,EAAE,QAAQ,EAAE,OAAO,WAAW,qBAAqB,CAAC;AAC7F,KAAI,OAAO,WAAW,GAAG;EACvB,MAAM,WAAW,WAAW,MAAM,MAAM,EAAE,QAAQ,EAAE;AACpD,SAAO,WACH;GAAE,MAAM;GAAM,OAAO;GAAK,QAAQ,SAAS;GAAQ,kBAAkB;GAAY,GACjF,KAAK,uBAAuB,WAAW;;AAE7C,QAAO,KAAK,OAAO,KAAK,MAAM,KAAK,EAAE,SAAS,CAAC,KAAK,KAAK,EAAE,WAAW;;;;;;;;ACzCxE,eAA8B,iBAC5B,SACA,SACwB;CACxB,MAAM,OAAO,cAAc,QAAQ;AACnC,KAAI,CAAC,KAAM,QAAO,KAAK,+CAA+C;CACtE,MAAM,QAAQ,KAAK;AACnB,KAAI,CAAC,MAAO,QAAO,KAAK,sCAAsC;CAE9D,MAAM,aAAgD,EAAE;AAExD,YAAW,KAAK;EACd,MAAM,MAAM,uBAAuB,WAAW;EAC9C,OAAO,MAAM,uBAAuB,WAAW,IAAI,IAAI;EACvD,QAAQ,MAAM,uBAAuB,WAAW,IAC5C,GAAG,MAAM,gBAAgB,OAAO,mCAChC,6BAA6B,MAAM,uBAAuB,MAAM,GAAG,EAAE,CAAC,KAAK,KAAK;EACrF,CAAC;AAEF,YAAW,KAAK;EACd,MAAM,MAAM,6BAA6B,WAAW;EACpD,OAAO,MAAM,6BAA6B,WAAW,IAAI,IAAI;EAC7D,QAAQ,MAAM,6BAA6B,WAAW,IAClD,GAAG,MAAM,gBAAgB,OAAO,kCAChC,oCAAoC,MAAM,6BAA6B,MAAM,GAAG,EAAE,CAAC,KAAK,KAAK;EAClG,CAAC;AAEF,YAAW,KAAK;EACd,MAAM,MAAM,oBAAoB,WAAW;EAC3C,OAAO,MAAM,oBAAoB,WAAW,IAAI,IAAI;EACpD,QAAQ,MAAM,oBAAoB,WAAW,IACzC,GAAG,MAAM,aAAa,OAAO,2BAC7B,qBAAqB,MAAM,oBAAoB,MAAM,GAAG,EAAE,CAAC,KAAK,KAAK;EAC1E,CAAC;CAEF,MAAM,SAAS,WAAW,QAAQ,MAAM,CAAC,EAAE,KAAK;AAChD,KAAI,OAAO,WAAW,EAAG,QAAO,KAAK,yBAAyB,WAAW;AACzE,QAAO,KAAK,OAAO,KAAK,MAAM,KAAK,EAAE,SAAS,CAAC,KAAK,KAAK,EAAE,WAAW;;;;;;;;;;;ACjCxE,MAAM,kBACJ;AACF,MAAM,qBACJ;AAEF,eAA8B,uBAC5B,SACA,SACwB;CACxB,MAAM,OAAO,cAAc,QAAQ;AACnC,KAAI,CAAC,KAAM,QAAO,KAAK,gDAAgD;CACvE,MAAM,QAAQ,KAAK;CACnB,MAAM,QAAQ,KAAK;AACnB,KAAI,CAAC,MAAO,QAAO,KAAK,uCAAuC;CAE/D,MAAM,cAAc,IAAI,IAAY,mBAAmB,QAAQ,CAAC,mBAAmB;AAMnF,KAAI,EAJoB,QACpB,MAAM,MAAM,MAAM,MAAM,EAAE,YAAY,MAAM,MAAM,YAAY,IAAI,EAAE,CAAC,CAAC,GACtE,OAGF,QAAO,KAAK,0DAA0D;CAGxE,MAAM,aAAgD,EAAE;CACxD,MAAM,aAAa,gBAAgB,KAAK,MAAM,KAAK,IAAI,mBAAmB,KAAK,MAAM,KAAK;AAC1F,YAAW,KAAK;EACd,MAAM;EACN,OAAO,aAAa,IAAI;EACxB,QAAQ,aACJ,sDACA;EACL,CAAC;CAEF,MAAM,oBAAoB,QACtB,MAAM,MAAM,MAAM,MAAM,MAAM,QAAQ,EAAE,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,SAAS,EAAE,GACzF;AACJ,YAAW,KAAK;EACd,MAAM;EACN,OAAO,oBAAoB,IAAI;EAC/B,QAAQ,oBACJ,sDACA;EACL,CAAC;CAEF,MAAM,SAAS,WAAW,QAAQ,MAAM,CAAC,EAAE,KAAK;AAChD,KAAI,OAAO,WAAW,EAAG,QAAO,KAAK,0BAA0B,WAAW;AAC1E,QAAO,KAAK,OAAO,KAAK,MAAM,KAAK,EAAE,SAAS,CAAC,KAAK,KAAK,EAAE,WAAW;;;;;;;;;;ACjDxE,eAA8B,iBAC5B,SACA,SACwB;CACxB,MAAM,OAAO,cAAc,QAAQ;AACnC,KAAI,CAAC,KAAM,QAAO,KAAK,yCAAyC;CAChE,MAAM,QAAQ,KAAK;AACnB,KAAI,CAAC,MAAO,QAAO,KAAK,gCAAgC;CAExD,MAAM,QAAQ,IAAI,IAAY,mBAAmB,QAAQ,CAAC,aAAa;AAGvE,KAAI,CADc,MAAM,MAAM,MAAM,MAAM,EAAE,YAAY,MAAM,MAAM,MAAM,IAAI,EAAE,CAAC,CACnE,CACZ,QAAO,KAAK,4DAA4D;CAG1E,MAAM,YAAY,MAAM,MAAM,QAAQ,MAAM,EAAE,WAAW;AACzD,KAAI,UAAU,WAAW,EACvB,QAAO,KACL,0CAA0C,CAAC,GAAG,MAAM,CAAC,KAAK,KAAK,CAAC,qDACjE;AAEH,QAAO,KAAK,gBAAgB,UAAU,OAAO,mBAAmB;;;;AC7BlE,MAAM,kBAAkB;CACtB;CACA;CACA;CACA;CACD;AAED,MAAM,iBAAiB;AACvB,MAAM,cAAc;;;;AAKpB,eAA8B,gBAC5B,SACA,SACwB;CACxB,MAAM,OAAO,cAAc,QAAQ;AACnC,KAAI,CAAC,KAAM,QAAO,KAAK,8CAA8C;CACrE,MAAM,QAAQ,KAAK;AACnB,KAAI,CAAC,MAAO,QAAO,KAAK,+DAA+D;CAEvF,MAAM,KAAK,MAAM;CACjB,MAAM,aAAgD,EAAE;CAExD,MAAM,OAAO,OAAO,GAAG,SAAS,WAAW,GAAG,KAAK,MAAM,GAAG;AAC5D,YAAW,KAAK;EACd,MAAM,KAAK,SAAS;EACpB,OAAO,KAAK,SAAS,IAAI,IAAI;EAC7B,QAAQ,OAAO,SAAS,SAAS;EAClC,CAAC;CAEF,MAAM,OAAO,OAAO,GAAG,gBAAgB,WAAW,GAAG,YAAY,MAAM,GAAG;AAC1E,YAAW,KAAK;EACd,MAAM,KAAK,SAAS;EACpB,OAAO,KAAK,SAAS,IAAI,IAAI;EAC7B,QAAQ,OAAO,wBAAwB,KAAK,OAAO,WAAW;EAC/D,CAAC;CAEF,MAAM,UAAU,eAAe,KAAK,KAAK;AACzC,YAAW,KAAK;EACd,MAAM;EACN,OAAO,UAAU,IAAI;EACrB,QAAQ,UAAU,iCAAiC;EACpD,CAAC;CAEF,MAAM,aAAa,YAAY,KAAK,KAAK;AACzC,YAAW,KAAK;EACd,MAAM;EACN,OAAO,aAAa,IAAI;EACxB,QAAQ,aAAa,qCAAqC;EAC3D,CAAC;CAEF,MAAM,UAAU,gBAAgB,MAAM,OAAO,GAAG,KAAK,KAAK,CAAC,IAAI,KAAK,SAAS;AAC7E,YAAW,KAAK;EACd,MAAM,CAAC;EACP,OAAO,UAAU,IAAI;EACrB,QAAQ,UAAU,+BAA+B;EAClD,CAAC;CAEF,MAAM,SAAS,WAAW,QAAQ,MAAM,CAAC,EAAE,KAAK;AAChD,KAAI,OAAO,WAAW,EACpB,QAAO,KAAK,qBAAqB,WAAW,OAAO,GAAG,WAAW,OAAO,aAAa,WAAW;AAElG,QAAO,KACL,OAAO,KAAK,MAAM,KAAK,EAAE,SAAS,CAAC,KAAK,KAAK,EAC7C,WACD;;;;;;;;;AC7DH,eAA8B,iBAC5B,SACA,SACwB;CACxB,MAAM,OAAO,cAAc,QAAQ;AACnC,KAAI,CAAC,KAAM,QAAO,KAAK,yCAAyC;CAChE,MAAM,QAAQ,KAAK;AACnB,KAAI,CAAC,MAAO,QAAO,KAAK,0DAA0D;CAClF,MAAM,WAAW,mBAAmB,QAAQ;CAE5C,MAAM,aAAgD,EAAE;AAExD,KAAI,MAAM,YAAY,SAAS,EAC7B,YAAW,KAAK;EACd,MAAM;EACN,OAAO;EACP,QAAQ,iBAAiB,MAAM,YAAY,KAAK,MAAM,GAAG,EAAE,SAAS,IAAI,EAAE,QAAQ,CAAC,KAAK,KAAK;EAC9F,CAAC;AAGJ,KAAI,MAAM,aAAa,WAAW,EAChC,YAAW,KAAK;EACd,MAAM;EACN,OAAO;EACP,QAAQ;EACT,CAAC;AAGJ,MAAK,MAAM,KAAK,MAAM,OAAO;EAC3B,MAAM,SAAmB,EAAE;AAC3B,OAAK,MAAM,SAAS,EAAE,aAAa;GACjC,MAAM,KAAK,MAAM,UAAU,KAAA,IAAY,MAAM,QAAQ,GAAG,MAAM,MAAM,GAAG,MAAM,MAAM;AACnF,UAAO,KAAK,GAAG,GAAG,IAAI,MAAM,SAAS;;AAEvC,MAAI,OAAO,EAAE,KAAK,WAAW,YAAY,EAAE,KAAK,OAAO,WAAW,EAChE,QAAO,KAAK,sBAAsB;AAEpC,MAAI,CAAC,EAAE,WACL,QAAO,KAAK,mDAAmD;AAKjE,MAAI,EAFD,MAAM,QAAQ,EAAE,KAAK,OAAO,IAAI,EAAE,KAAK,OAAO,SAAS,KACvD,MAAM,QAAQ,EAAE,KAAK,WAAW,IAAI,EAAE,KAAK,WAAW,SAAS,GACnD,QAAO,KAAK,yBAAyB;AACpD,MAAI,SAAS,sBAAsB,CAAC,EAAE,eACpC,QAAO,KAAK,iCAAiC;AAE/C,aAAW,KAAK;GACd,MAAM,OAAO,WAAW;GACxB,OAAO,OAAO,WAAW,IAAI,IAAI;GACjC,QAAQ,GAAG,EAAE,eAAe,mBAAmB,IAAI,OAAO,WAAW,IAAI,OAAO,OAAO,KAAK,KAAK;GAClG,CAAC;;AAGJ,KAAI,KAAK,sBAAsB,SAAS,EACtC,YAAW,KAAK;EACd,MAAM;EACN,OAAO;EACP,QAAQ,6BAA6B,KAAK,sBAAsB,KAAK,KAAK;EAC3E,CAAC;AAGJ,KAAI,KAAK,aAAa,SAAS,EAC7B,YAAW,KAAK;EACd,MAAM;EACN,OAAO;EACP,QAAQ,6BAA6B,KAAK,aAAa,MAAM,GAAG,EAAE,CAAC,KAAK,KAAK;EAC9E,CAAC;CAGJ,MAAM,SAAS,WAAW,QAAQ,MAAM,CAAC,EAAE,KAAK;AAChD,KAAI,OAAO,WAAW,EACpB,QAAO,KAAK,gBAAgB,MAAM,MAAM,OAAO,cAAc,WAAW;AAE1E,QAAO,KACL,OAAO,KAAK,MAAM,KAAK,EAAE,SAAS,CAAC,KAAK,KAAK,EAC7C,WACD;;;;ACvEH,MAAM,eAAuD;CAC3D,oBAAoB,CAAC,gBAAgB;CACrC,iBAAiB,CAAC,iBAAiB;CACnC,iBAAiB,CAAC,eAAe;CACjC,sBAAsB,CAAC,uBAAuB;CAC9C,eAAe,CAAC,kBAAkB,iBAAiB;CACnD,mBAAmB,CAAC,iBAAiB;CACtC;AAED,aAAa,kBAAkB;CAC7B;CACA;CACA;CACA;CACA;CACA;CACD;AAED,MAAMC,qBAAmB,OAAO,KAAK,aAAa,CAAC,MAAM,CAAC,KAAK,KAAK;AAEpE,SAASC,aAAW,SAAqD;AAOvE,QAAO;EALL,QAAQ,WAAW;EACnB,QAAQ,QAAQ;EAChB,QAAQ,QAAQ;EAChB,QAAQ;EAEO,CAAC,MAAM,WAA6B,OAAO,WAAW,SAAS;;AAWlF,SAAS,gBACP,QACA,SACA,QAC8B;AAC9B,QAAO,OAAO,IAAI,aAAa;EAC/B,MAAM,UAAU,OAAO,OAAO,QAC5B,SACC,UAAU,OAAO,cAAc,MAAM,QAAQ,QAAQ,CAAC,EACvD,EAAE,aAAa,aAAa,CAC7B;EACD,MAAM,SAAS,QAAQ,QAAQ,WAAW,CAAC,OAAO,KAAK;EACvD,MAAM,OAAO,QAAQ,QAAQ,WAAW,OAAO,QAAQ,OAAO,QAAQ,EAAE;AACxE,SAAO;GACL,MAAM,OAAO,WAAW;GACxB,OAAO,OAAO,WAAW,IACrB,KAAK,IAAI,GAAG,QAAQ,KAAK,WAAW,OAAO,MAAM,CAAC,GAClD;GACJ,QAAQ,OAAO,WAAW,IACtB,KAAK,SAAS,IACZ,KAAK,KAAK,WAAW,OAAO,OAAO,CAAC,KAAK,KAAK,GAC9C,iBAAiB,QAAQ,OAAO,oBAClC,OAAO,KAAK,WAAW,OAAO,OAAO,CAAC,KAAK,KAAK;GACpD,kBAAkB;GACnB;GACC;;AAGJ,eAAsB,gCACpB,QACA,SACwB;AACxB,QAAO,OAAO,WAAW,sCAAsC,QAAQ,QAAQ,CAAC;;AAGlF,SAAS,sCACP,QACA,SAC8B;AAC9B,QAAO,OAAO,IAAI,aAAa;EAC/B,MAAM,SAASA,aAAW,QAAQ;EAClC,MAAM,SAAS,SAAS,aAAa,UAAU,KAAA;AAC/C,MAAI,CAAC,OACH,QAAO;GACL,MAAM;GACN,OAAO;GACP,QAAQ,mDAAmD,UAAU,UAAU,wBAAwBD;GACxG;AAEH,SAAO,OAAO,gBAAgB,QAAQ,SAAS,OAAO;GACpD;;;;AC5FJ,MAAM,SAAsC;CAC1C,gBAAgB;CAChB,cAAc;CACf;AACD,MAAM,gBAAgB;CACpB;CACA;CACA;CACA;CACA;CACA;CACA;CACD;AACD,MAAM,mBAAmB,CAAC,GAAG,OAAO,KAAK,OAAO,EAAE,GAAG,cAAc,CAAC,MAAM,CAAC,KAAK,KAAK;AAIrF,SAAS,WAAW,SAAqD;AAOvE,QAAO;EALL,QAAQ,WAAW;EACnB,QAAQ,QAAQ;EAChB,QAAQ,QAAQ;EAChB,QAAQ;EAEO,CAAC,MAAM,WAA6B,OAAO,WAAW,SAAS;;AAGlF,eAA8B,0BAC5B,QACA,SACwB;CACxB,MAAM,SAAS,WAAW,QAAQ;CAClC,MAAM,YAAY,SAAS,OAAO,UAAU,KAAA;AAC5C,KAAI,UACF,QAAO,UAAU,QAAQ,QAAQ;AAEnC,KAAI,UAAU,cAAc,SAAS,OAAO,CAC1C,QAAO,gCAAgC,QAAQ,QAAQ;AAEzD,QAAO;EACL,MAAM;EACN,OAAO;EACP,QAAQ,iDAAiD,UAAU,UAAU,wBAAwB;EACtG"}
+{"version":3,"file":"index.mjs","names":[],"sources":["../../src/assertions/_shared.ts","../../src/assertions/skill-test.ts","../../src/assertions/skill-budget.ts","../../src/assertions/promptfoo.ts"],"sourcesContent":["import { join } from \"node:path\";\nimport { readFile } from \"node:fs/promises\";\nimport type { AssertionEntry } from \"../assertion-entries.js\";\nimport type {\n  AssertionMode,\n  AgentSkillEvalsAssertionResult,\n  WorldHandle,\n} from \"../internal-types.js\";\nimport { decodeEvidenceSnapshot } from \"../evidence-schema.js\";\nimport {\n  EvidenceCollector,\n  evidenceFromSnapshot,\n  type AgentSkillEvalsProviderMetadata,\n} from \"../agent/index.js\";\nimport { makeWorldHandle } from \"../agent/world.js\";\nimport { RUNTIME_CHECKS_BY_TYPE } from \"../runtime-checks/catalog.js\";\n\nexport interface PromptfooAssertContext {\n  vars?: Record<string, unknown>;\n  providerResponse?: {\n    metadata?: unknown;\n    tokenUsage?: {\n      total?: number;\n      prompt?: number;\n      completion?: number;\n      cached?: number;\n    };\n  };\n  test?: {\n    vars?: Record<string, unknown>;\n    assert?: Array<{ type?: string; metric?: string; value?: unknown; config?: unknown }>;\n  };\n  assertion?: { metric?: string };\n  assert?: { metric?: string };\n  config?: { metric?: string; agentSkillEvals?: unknown };\n  metric?: string;\n}\n\nexport interface GradingResult {\n  pass: boolean;\n  score: number;\n  reason: string;\n  componentResults?: Array<{ pass: boolean; score: number; reason: string }>;\n  metadata?: Record<string, unknown>;\n}\n\nexport async function loadMetadata(\n  context: PromptfooAssertContext,\n): Promise<AgentSkillEvalsProviderMetadata | null> {\n  const direct = context.providerResponse?.metadata;\n  if (direct && typeof direct === \"object\" && \"worldPath\" in direct) {\n    return direct as AgentSkillEvalsProviderMetadata;\n  }\n  // Fallback: AGENT_SKILL_EVALS_RUN_DIR env points at the most recent run.\n  const runDir = process.env.AGENT_SKILL_EVALS_RUN_DIR;\n  if (runDir) {\n    try {\n      const buf = await readFile(join(runDir, \"agent-skill-evals-meta.json\"), \"utf8\");\n      return JSON.parse(buf) as AgentSkillEvalsProviderMetadata;\n    } catch {\n      return null;\n    }\n  }\n  return null;\n}\n\nexport async function loadEvidence(\n  meta: AgentSkillEvalsProviderMetadata,\n): Promise<EvidenceCollector> {\n  let buf: string;\n  try {\n    buf = await readFile(meta.evidencePath, \"utf8\");\n  } catch (err) {\n    throw new Error(\n      `evidence: failed to read ${meta.evidencePath}: ${err instanceof Error ? err.message : String(err)}`,\n    );\n  }\n  let parsed: unknown;\n  try {\n    parsed = JSON.parse(buf) as unknown;\n  } catch (err) {\n    throw new Error(\n      `evidence: invalid JSON in ${meta.evidencePath}: ${err instanceof Error ? err.message : String(err)}`,\n    );\n  }\n  const decoded = decodeEvidenceSnapshot(parsed);\n  if (!decoded.ok) {\n    // decodeEvidenceSnapshot already produces a user-facing message.\n    throw new Error(`evidence: ${decoded.error.message}`);\n  }\n  return EvidenceCollector.fromSnapshot(decoded.value);\n}\n\nexport function loadWorld(\n  meta: AgentSkillEvalsProviderMetadata,\n  evidenceCollector: EvidenceCollector,\n): WorldHandle {\n  const world = makeWorldHandle(meta.worldPath, (event) => evidenceCollector.addCommand(event));\n  if (!meta.mockEnv) return world;\n  return {\n    ...world,\n    exec(command, args, opts = {}) {\n      return world.exec(command, args, { ...opts, env: { ...meta.mockEnv, ...(opts.env ?? {}) } });\n    },\n  };\n}\n\nexport async function runEntries(\n  entries: AssertionEntry[],\n  world: WorldHandle,\n  evidenceCollector: EvidenceCollector,\n  mode: AssertionMode,\n): Promise<AgentSkillEvalsAssertionResult[]> {\n  const results: AgentSkillEvalsAssertionResult[] = [];\n  for (const entry of entries) {\n    const plugin = RUNTIME_CHECKS_BY_TYPE.get(entry.type);\n    if (!plugin) {\n      results.push({\n        pass: false,\n        score: 0,\n        reason: `unknown effect type: ${entry.type}`,\n      });\n      continue;\n    }\n    const r = await plugin.verify({\n      assertion: entry.args,\n      world,\n      evidence: evidenceFromSnapshot(evidenceCollector.toSnapshot()),\n      mode,\n    });\n    results.push(r);\n  }\n  return results;\n}\n\nexport function aggregate(\n  results: AgentSkillEvalsAssertionResult[],\n  emptyReason: string,\n  options: { emptyPass?: boolean } = {},\n): GradingResult {\n  if (results.length === 0) {\n    const pass = options.emptyPass ?? true;\n    return { pass, score: pass ? 1 : 0, reason: emptyReason };\n  }\n  const allPass = results.every((r) => r.pass);\n  const failed = results.filter((r) => !r.pass).map((r) => r.reason);\n  return {\n    pass: allPass,\n    score: allPass ? 1 : 0,\n    reason: allPass\n      ? `${results.length} check(s) passed`\n      : failed.join(\"; \"),\n    componentResults: results.map((r) => ({\n      pass: r.pass,\n      score: r.score,\n      reason: r.reason,\n    })),\n  };\n}\n","import {\n  aggregate,\n  loadEvidence,\n  loadMetadata,\n  loadWorld,\n  runEntries,\n  type GradingResult,\n  type PromptfooAssertContext,\n} from \"./_shared.js\";\nimport { parseRuntimeTestFields } from \"../assertion-entries.js\";\nimport { writeEvidenceTo } from \"../agent/evidence.js\";\nimport { assertMockServicesHealthy, stopMockServices } from \"../agent/mock-services.js\";\n\nfunction withEvidencePointer(\n  result: GradingResult,\n  meta: { evidencePath: string; worldPath: string },\n): GradingResult {\n  if (result.pass) return result;\n  return {\n    ...result,\n    reason: `${result.reason}\\nEvidence: ${meta.evidencePath}\\nWorld: ${meta.worldPath}`,\n  };\n}\n\nexport default async function skillTest(\n  _output: string,\n  context: PromptfooAssertContext,\n): Promise<GradingResult> {\n  const meta = await loadMetadata(context);\n  if (!meta) {\n    return {\n      pass: false,\n      score: 0,\n      reason: \"skill.test: provider metadata missing\",\n    };\n  }\n\n  const preconditionResults = meta.preconditionResults ?? [];\n  if (!meta.preconditionsPassed) {\n    const result = withEvidencePointer(\n      aggregate(preconditionResults, \"skill.test: preconditions failed\"),\n      meta,\n    );\n    await stopMockServices(meta.runDir);\n    return result;\n  }\n\n  const vars = (context.vars ?? context.test?.vars ?? {}) as Record<string, unknown>;\n  let evidenceCollector;\n  try {\n    evidenceCollector = await loadEvidence(meta);\n  } catch (err) {\n    await stopMockServices(meta.runDir);\n    return { pass: false, score: 0, reason: err instanceof Error ? err.message : String(err) };\n  }\n  try {\n    assertMockServicesHealthy(meta.runDir);\n  } catch (err) {\n    await stopMockServices(meta.runDir);\n    return withEvidencePointer({ pass: false, score: 0, reason: `Mock Service failure: ${err instanceof Error ? err.message : String(err)}` }, meta);\n  }\n  const world = loadWorld(meta, evidenceCollector);\n  const warningResults = evidenceCollector.toSnapshot().warnings?.map((warning) => ({\n    pass: false,\n    score: 0,\n    reason: `evidence warning: ${warning}`,\n  })) ?? [];\n\n  const parsed = parseRuntimeTestFields(vars);\n  const parseResults = parsed.errors\n    .filter((error) => error.field !== \"preconditions\")\n    .map((error) => ({\n      pass: false,\n      score: 0,\n      reason: `runtime test field ${error.index === undefined ? error.field : `${error.field}[${error.index}]`}: ${error.reason}`,\n    }));\n  const expectationResults = await runEntries(parsed.expect, world, evidenceCollector, \"should\");\n  // Persisting verifier evidence back to disk is best effort: a write\n  // failure must not discard the check results already computed.\n  let writeError: string | undefined;\n  try {\n    await writeEvidenceTo(evidenceCollector, meta.runDir);\n  } catch (err) {\n    writeError = err instanceof Error ? err.message : String(err);\n  }\n  const results = [\n    ...preconditionResults,\n    ...warningResults,\n    ...parseResults,\n    ...expectationResults,\n  ];\n  const graded = withEvidencePointer(\n    aggregate(results, \"skill.test: no Runtime Test Fields checks declared\", {\n      emptyPass: false,\n    }),\n    meta,\n  );\n  if (writeError !== undefined) {\n    await stopMockServices(meta.runDir);\n    return { ...graded, reason: `${graded.reason}\\n(note: failed to update evidence.json: ${writeError})` };\n  }\n  await stopMockServices(meta.runDir);\n  return graded;\n}\n","import type { GradingResult, PromptfooAssertContext } from \"./_shared.js\";\n\ninterface TokenUsage {\n  total?: number;\n  prompt?: number;\n  completion?: number;\n  cached?: number;\n}\n\ninterface BudgetSettings {\n  maxTotalTokens?: number;\n  maxPromptTokens?: number;\n  maxCompletionTokens?: number;\n  maxCachedTokens?: number;\n}\n\nconst budgetFields = [\n  [\"total\", \"maxTotalTokens\"],\n  [\"prompt\", \"maxPromptTokens\"],\n  [\"completion\", \"maxCompletionTokens\"],\n  [\"cached\", \"maxCachedTokens\"],\n] as const;\n\nfunction configObject(value: unknown): Record<string, unknown> {\n  return value && typeof value === \"object\" && !Array.isArray(value)\n    ? value as Record<string, unknown>\n    : {};\n}\n\nfunction numberSetting(config: Record<string, unknown>, key: keyof BudgetSettings): number | undefined {\n  const value = config[key];\n  return typeof value === \"number\" && Number.isFinite(value) ? value : undefined;\n}\n\nfunction budgetSettings(context: PromptfooAssertContext): BudgetSettings {\n  const config = configObject(context.config?.agentSkillEvals);\n  return {\n    maxTotalTokens: numberSetting(config, \"maxTotalTokens\"),\n    maxPromptTokens: numberSetting(config, \"maxPromptTokens\"),\n    maxCompletionTokens: numberSetting(config, \"maxCompletionTokens\"),\n    maxCachedTokens: numberSetting(config, \"maxCachedTokens\"),\n  };\n}\n\nfunction tokenUsage(context: PromptfooAssertContext): TokenUsage | undefined {\n  const usage = context.providerResponse?.tokenUsage;\n  return usage && typeof usage === \"object\" && !Array.isArray(usage)\n    ? usage as TokenUsage\n    : undefined;\n}\n\nexport default async function skillBudget(\n  _output: string,\n  context: PromptfooAssertContext,\n): Promise<GradingResult> {\n  const usage = tokenUsage(context);\n  if (!usage) {\n    return {\n      pass: false,\n      score: 0,\n      reason: \"skill.budget: provider tokenUsage missing\",\n    };\n  }\n\n  const settings = budgetSettings(context);\n  const configured = budgetFields.filter(([, limitKey]) => settings[limitKey] !== undefined);\n  if (configured.length === 0) {\n    return {\n      pass: false,\n      score: 0,\n      reason: \"skill.budget: configure at least one token limit\",\n    };\n  }\n\n  const components = configured.map(([usageKey, limitKey]) => {\n    const actual = usage[usageKey];\n    const limit = settings[limitKey] ?? 0;\n    if (typeof actual !== \"number\" || !Number.isFinite(actual)) {\n      return {\n        pass: false,\n        score: 0,\n        reason: `${usageKey} tokens missing`,\n      };\n    }\n    return {\n      pass: actual <= limit,\n      score: actual <= limit ? 1 : 0,\n      reason: `${usageKey} tokens ${actual} <= ${limit}`,\n    };\n  });\n  const failed = components.filter((component) => !component.pass);\n  return {\n    pass: failed.length === 0,\n    score: failed.length === 0 ? 1 : 0,\n    reason: failed.length === 0\n      ? `skill.budget: ${components.length} budget(s) passed`\n      : failed.map((component) => component.reason).join(\"; \"),\n    componentResults: components,\n  };\n}\n","import skillTest from \"./skill-test.js\";\nimport skillBudget from \"./skill-budget.js\";\nimport type { GradingResult, PromptfooAssertContext } from \"./_shared.js\";\n\ntype AssertionFn = (\n  output: string,\n  context: PromptfooAssertContext,\n) => Promise<GradingResult>;\n\nconst routes: Record<string, AssertionFn> = {\n  \"skill.budget\": skillBudget,\n  \"skill.test\": skillTest,\n};\nconst knownMetrics = Object.keys(routes);\nconst availableMetrics = [...knownMetrics].sort().join(\", \");\n\nexport { skillTest };\n\nfunction sameConfig(a: unknown, b: unknown): boolean {\n  if (a === undefined && b === undefined) return true;\n  try {\n    return JSON.stringify(a) === JSON.stringify(b);\n  } catch {\n    return false;\n  }\n}\n\n/**\n * Promptfoo hands a javascript assertion its `config` but not its `metric`,\n * so a lone `metric:` declaration used to require a duplicate `config.metric`.\n * Recover the metric from the test's assert list instead: the entry whose\n * config matches the one Promptfoo passed is the assertion being graded.\n */\nfunction metricFromTestAsserts(context: PromptfooAssertContext): string | undefined {\n  const asserts = context.test?.assert;\n  if (!Array.isArray(asserts)) return undefined;\n  const candidates = asserts.filter(\n    (entry): entry is { metric: string; config?: unknown } =>\n      typeof entry === \"object\" &&\n      entry !== null &&\n      typeof (entry as { metric?: unknown }).metric === \"string\" &&\n      knownMetrics.includes((entry as { metric: string }).metric),\n  );\n  if (candidates.length === 1) return candidates[0]!.metric;\n  const matched = candidates.filter((entry) => sameConfig(entry.config, context.config));\n  if (matched.length === 1) return matched[0]!.metric;\n  return undefined;\n}\n\nfunction metricFrom(context: PromptfooAssertContext): string | undefined {\n  return [\n    context.assertion?.metric,\n    context.assert?.metric,\n    context.config?.metric,\n    context.metric,\n  ].find((metric): metric is string => typeof metric === \"string\") ?? metricFromTestAsserts(context);\n}\n\nexport default async function agentSkillEvalsAssertions(\n  output: string,\n  context: PromptfooAssertContext,\n): Promise<GradingResult> {\n  const metric = metricFrom(context);\n  const assertion = metric ? routes[metric] : undefined;\n  if (assertion) {\n    return assertion(output, context);\n  }\n  return {\n    pass: false,\n    score: 0,\n    reason:\n      `agent-skill-evals assertions: unknown metric \"${metric ?? \"missing\"}\". ` +\n      `Available metrics: ${availableMetrics}. ` +\n      \"Declare the metric on the assertion (metric: skill.test). If one test case uses \" +\n      \"several agent-skill-evals assertions with identical config blocks, disambiguate \" +\n      \"with config.metric.\",\n  };\n}\n"],"mappings":";;;;;AA8CA,eAAsB,aACpB,SACiD;CACjD,MAAM,SAAS,QAAQ,kBAAkB;AACzC,KAAI,UAAU,OAAO,WAAW,YAAY,eAAe,OACzD,QAAO;CAGT,MAAM,SAAS,QAAQ,IAAI;AAC3B,KAAI,OACF,KAAI;EACF,MAAM,MAAM,MAAM,SAAS,KAAK,QAAQ,8BAA8B,EAAE,OAAO;AAC/E,SAAO,KAAK,MAAM,IAAI;SAChB;AACN,SAAO;;AAGX,QAAO;;AAGT,eAAsB,aACpB,MAC4B;CAC5B,IAAI;AACJ,KAAI;AACF,QAAM,MAAM,SAAS,KAAK,cAAc,OAAO;UACxC,KAAK;AACZ,QAAM,IAAI,MACR,4BAA4B,KAAK,aAAa,IAAI,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GACnG;;CAEH,IAAI;AACJ,KAAI;AACF,WAAS,KAAK,MAAM,IAAI;UACjB,KAAK;AACZ,QAAM,IAAI,MACR,6BAA6B,KAAK,aAAa,IAAI,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI,GACpG;;CAEH,MAAM,UAAU,uBAAuB,OAAO;AAC9C,KAAI,CAAC,QAAQ,GAEX,OAAM,IAAI,MAAM,aAAa,QAAQ,MAAM,UAAU;AAEvD,QAAO,kBAAkB,aAAa,QAAQ,MAAM;;AAGtD,SAAgB,UACd,MACA,mBACa;CACb,MAAM,QAAQ,gBAAgB,KAAK,YAAY,UAAU,kBAAkB,WAAW,MAAM,CAAC;AAC7F,KAAI,CAAC,KAAK,QAAS,QAAO;AAC1B,QAAO;EACL,GAAG;EACH,KAAK,SAAS,MAAM,OAAO,EAAE,EAAE;AAC7B,UAAO,MAAM,KAAK,SAAS,MAAM;IAAE,GAAG;IAAM,KAAK;KAAE,GAAG,KAAK;KAAS,GAAI,KAAK,OAAO,EAAE;KAAG;IAAE,CAAC;;EAE/F;;AAGH,eAAsB,WACpB,SACA,OACA,mBACA,MAC2C;CAC3C,MAAM,UAA4C,EAAE;AACpD,MAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,SAAS,uBAAuB,IAAI,MAAM,KAAK;AACrD,MAAI,CAAC,QAAQ;AACX,WAAQ,KAAK;IACX,MAAM;IACN,OAAO;IACP,QAAQ,wBAAwB,MAAM;IACvC,CAAC;AACF;;EAEF,MAAM,IAAI,MAAM,OAAO,OAAO;GAC5B,WAAW,MAAM;GACjB;GACA,UAAU,qBAAqB,kBAAkB,YAAY,CAAC;GAC9D;GACD,CAAC;AACF,UAAQ,KAAK,EAAE;;AAEjB,QAAO;;AAGT,SAAgB,UACd,SACA,aACA,UAAmC,EAAE,EACtB;AACf,KAAI,QAAQ,WAAW,GAAG;EACxB,MAAM,OAAO,QAAQ,aAAa;AAClC,SAAO;GAAE;GAAM,OAAO,OAAO,IAAI;GAAG,QAAQ;GAAa;;CAE3D,MAAM,UAAU,QAAQ,OAAO,MAAM,EAAE,KAAK;CAC5C,MAAM,SAAS,QAAQ,QAAQ,MAAM,CAAC,EAAE,KAAK,CAAC,KAAK,MAAM,EAAE,OAAO;AAClE,QAAO;EACL,MAAM;EACN,OAAO,UAAU,IAAI;EACrB,QAAQ,UACJ,GAAG,QAAQ,OAAO,oBAClB,OAAO,KAAK,KAAK;EACrB,kBAAkB,QAAQ,KAAK,OAAO;GACpC,MAAM,EAAE;GACR,OAAO,EAAE;GACT,QAAQ,EAAE;GACX,EAAE;EACJ;;;;AChJH,SAAS,oBACP,QACA,MACe;AACf,KAAI,OAAO,KAAM,QAAO;AACxB,QAAO;EACL,GAAG;EACH,QAAQ,GAAG,OAAO,OAAO,cAAc,KAAK,aAAa,WAAW,KAAK;EAC1E;;AAGH,eAA8B,UAC5B,SACA,SACwB;CACxB,MAAM,OAAO,MAAM,aAAa,QAAQ;AACxC,KAAI,CAAC,KACH,QAAO;EACL,MAAM;EACN,OAAO;EACP,QAAQ;EACT;CAGH,MAAM,sBAAsB,KAAK,uBAAuB,EAAE;AAC1D,KAAI,CAAC,KAAK,qBAAqB;EAC7B,MAAM,SAAS,oBACb,UAAU,qBAAqB,mCAAmC,EAClE,KACD;AACD,QAAM,iBAAiB,KAAK,OAAO;AACnC,SAAO;;CAGT,MAAM,OAAQ,QAAQ,QAAQ,QAAQ,MAAM,QAAQ,EAAE;CACtD,IAAI;AACJ,KAAI;AACF,sBAAoB,MAAM,aAAa,KAAK;UACrC,KAAK;AACZ,QAAM,iBAAiB,KAAK,OAAO;AACnC,SAAO;GAAE,MAAM;GAAO,OAAO;GAAG,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GAAE;;AAE5F,KAAI;AACF,4BAA0B,KAAK,OAAO;UAC/B,KAAK;AACZ,QAAM,iBAAiB,KAAK,OAAO;AACnC,SAAO,oBAAoB;GAAE,MAAM;GAAO,OAAO;GAAG,QAAQ,yBAAyB,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GAAI,EAAE,KAAK;;CAElJ,MAAM,QAAQ,UAAU,MAAM,kBAAkB;CAChD,MAAM,iBAAiB,kBAAkB,YAAY,CAAC,UAAU,KAAK,aAAa;EAChF,MAAM;EACN,OAAO;EACP,QAAQ,qBAAqB;EAC9B,EAAE,IAAI,EAAE;CAET,MAAM,SAAS,uBAAuB,KAAK;CAC3C,MAAM,eAAe,OAAO,OACzB,QAAQ,UAAU,MAAM,UAAU,gBAAgB,CAClD,KAAK,WAAW;EACf,MAAM;EACN,OAAO;EACP,QAAQ,sBAAsB,MAAM,UAAU,KAAA,IAAY,MAAM,QAAQ,GAAG,MAAM,MAAM,GAAG,MAAM,MAAM,GAAG,IAAI,MAAM;EACpH,EAAE;CACL,MAAM,qBAAqB,MAAM,WAAW,OAAO,QAAQ,OAAO,mBAAmB,SAAS;CAG9F,IAAI;AACJ,KAAI;AACF,QAAM,gBAAgB,mBAAmB,KAAK,OAAO;UAC9C,KAAK;AACZ,eAAa,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;;CAQ/D,MAAM,SAAS,oBACb,UAAU;EANV,GAAG;EACH,GAAG;EACH,GAAG;EACH,GAAG;EAGc,EAAE,sDAAsD,EACvE,WAAW,OACZ,CAAC,EACF,KACD;AACD,KAAI,eAAe,KAAA,GAAW;AAC5B,QAAM,iBAAiB,KAAK,OAAO;AACnC,SAAO;GAAE,GAAG;GAAQ,QAAQ,GAAG,OAAO,OAAO,2CAA2C,WAAW;GAAI;;AAEzG,OAAM,iBAAiB,KAAK,OAAO;AACnC,QAAO;;;;ACtFT,MAAM,eAAe;CACnB,CAAC,SAAS,iBAAiB;CAC3B,CAAC,UAAU,kBAAkB;CAC7B,CAAC,cAAc,sBAAsB;CACrC,CAAC,UAAU,kBAAkB;CAC9B;AAED,SAAS,aAAa,OAAyC;AAC7D,QAAO,SAAS,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,MAAM,GAC9D,QACA,EAAE;;AAGR,SAAS,cAAc,QAAiC,KAA+C;CACrG,MAAM,QAAQ,OAAO;AACrB,QAAO,OAAO,UAAU,YAAY,OAAO,SAAS,MAAM,GAAG,QAAQ,KAAA;;AAGvE,SAAS,eAAe,SAAiD;CACvE,MAAM,SAAS,aAAa,QAAQ,QAAQ,gBAAgB;AAC5D,QAAO;EACL,gBAAgB,cAAc,QAAQ,iBAAiB;EACvD,iBAAiB,cAAc,QAAQ,kBAAkB;EACzD,qBAAqB,cAAc,QAAQ,sBAAsB;EACjE,iBAAiB,cAAc,QAAQ,kBAAkB;EAC1D;;AAGH,SAAS,WAAW,SAAyD;CAC3E,MAAM,QAAQ,QAAQ,kBAAkB;AACxC,QAAO,SAAS,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,MAAM,GAC9D,QACA,KAAA;;AAGN,eAA8B,YAC5B,SACA,SACwB;CACxB,MAAM,QAAQ,WAAW,QAAQ;AACjC,KAAI,CAAC,MACH,QAAO;EACL,MAAM;EACN,OAAO;EACP,QAAQ;EACT;CAGH,MAAM,WAAW,eAAe,QAAQ;CACxC,MAAM,aAAa,aAAa,QAAQ,GAAG,cAAc,SAAS,cAAc,KAAA,EAAU;AAC1F,KAAI,WAAW,WAAW,EACxB,QAAO;EACL,MAAM;EACN,OAAO;EACP,QAAQ;EACT;CAGH,MAAM,aAAa,WAAW,KAAK,CAAC,UAAU,cAAc;EAC1D,MAAM,SAAS,MAAM;EACrB,MAAM,QAAQ,SAAS,aAAa;AACpC,MAAI,OAAO,WAAW,YAAY,CAAC,OAAO,SAAS,OAAO,CACxD,QAAO;GACL,MAAM;GACN,OAAO;GACP,QAAQ,GAAG,SAAS;GACrB;AAEH,SAAO;GACL,MAAM,UAAU;GAChB,OAAO,UAAU,QAAQ,IAAI;GAC7B,QAAQ,GAAG,SAAS,UAAU,OAAO,MAAM;GAC5C;GACD;CACF,MAAM,SAAS,WAAW,QAAQ,cAAc,CAAC,UAAU,KAAK;AAChE,QAAO;EACL,MAAM,OAAO,WAAW;EACxB,OAAO,OAAO,WAAW,IAAI,IAAI;EACjC,QAAQ,OAAO,WAAW,IACtB,iBAAiB,WAAW,OAAO,qBACnC,OAAO,KAAK,cAAc,UAAU,OAAO,CAAC,KAAK,KAAK;EAC1D,kBAAkB;EACnB;;;;ACzFH,MAAM,SAAsC;CAC1C,gBAAgB;CAChB,cAAc;CACf;AACD,MAAM,eAAe,OAAO,KAAK,OAAO;AACxC,MAAM,mBAAmB,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,KAAK;AAI5D,SAAS,WAAW,GAAY,GAAqB;AACnD,KAAI,MAAM,KAAA,KAAa,MAAM,KAAA,EAAW,QAAO;AAC/C,KAAI;AACF,SAAO,KAAK,UAAU,EAAE,KAAK,KAAK,UAAU,EAAE;SACxC;AACN,SAAO;;;;;;;;;AAUX,SAAS,sBAAsB,SAAqD;CAClF,MAAM,UAAU,QAAQ,MAAM;AAC9B,KAAI,CAAC,MAAM,QAAQ,QAAQ,CAAE,QAAO,KAAA;CACpC,MAAM,aAAa,QAAQ,QACxB,UACC,OAAO,UAAU,YACjB,UAAU,QACV,OAAQ,MAA+B,WAAW,YAClD,aAAa,SAAU,MAA6B,OAAO,CAC9D;AACD,KAAI,WAAW,WAAW,EAAG,QAAO,WAAW,GAAI;CACnD,MAAM,UAAU,WAAW,QAAQ,UAAU,WAAW,MAAM,QAAQ,QAAQ,OAAO,CAAC;AACtF,KAAI,QAAQ,WAAW,EAAG,QAAO,QAAQ,GAAI;;AAI/C,SAAS,WAAW,SAAqD;AACvE,QAAO;EACL,QAAQ,WAAW;EACnB,QAAQ,QAAQ;EAChB,QAAQ,QAAQ;EAChB,QAAQ;EACT,CAAC,MAAM,WAA6B,OAAO,WAAW,SAAS,IAAI,sBAAsB,QAAQ;;AAGpG,eAA8B,0BAC5B,QACA,SACwB;CACxB,MAAM,SAAS,WAAW,QAAQ;CAClC,MAAM,YAAY,SAAS,OAAO,UAAU,KAAA;AAC5C,KAAI,UACF,QAAO,UAAU,QAAQ,QAAQ;AAEnC,QAAO;EACL,MAAM;EACN,OAAO;EACP,QACE,iDAAiD,UAAU,UAAU,wBAC/C,iBAAiB;EAI1C"}

package/dist/catalog-9Wh-AfRv.mjs

@@ -0,0 +1,409 @@
+import { z } from "zod";
+//#region src/runtime-checks/schemas.ts
+function result(pass, reason, evidence) {
+	return {
+		pass,
+		score: pass ? 1 : 0,
+		reason,
+		evidence
+	};
+}
+function validationFailure(reason) {
+	return result(false, reason);
+}
+/**
+* Map a "matched" boolean to a pass result based on mode. Used by checks
+* that don't self-encode polarity (file.exists, tool.called, ...).
+*
+* - should: pass = matched
+* - should_not: pass = !matched
+* - precondition: pass = matched (precondition asserts a current state)
+*/
+function applyMode(matched, mode, reasonMatched, reasonUnmatched) {
+	switch (mode) {
+		case "should":
+		case "precondition": return result(matched, matched ? reasonMatched : reasonUnmatched);
+		case "should_not": return result(!matched, matched ? reasonMatched : reasonUnmatched);
+	}
+}
+const NonEmptyString = z.string().trim().min(1);
+const OptionalString = z.string().optional();
+const OptionalNonEmptyString = NonEmptyString.optional();
+const PathArgsSchema = z.object({ path: NonEmptyString });
+const FileContainsArgsSchema = z.object({
+	path: NonEmptyString,
+	text: z.string()
+});
+const FileChangesWithinArgsSchema = z.object({ paths: z.array(NonEmptyString).min(1) });
+const VerifierArgsSchema = z.object({
+	run: NonEmptyString,
+	args: z.array(z.string()).optional(),
+	timeoutMs: z.number().optional()
+});
+const ToolCalledArgsSchema = z.object({
+	tool: NonEmptyString,
+	provider: OptionalString,
+	server: OptionalString,
+	args_match: z.unknown().optional()
+});
+const JsonValueSchema = z.union([
+	z.null(),
+	z.boolean(),
+	z.number(),
+	z.string(),
+	z.array(z.unknown()),
+	z.record(z.string(), z.unknown())
+]);
+const ToolNotCalledSelectorSchema = z.object({
+	tool: OptionalNonEmptyString,
+	provider: OptionalNonEmptyString,
+	server: OptionalNonEmptyString,
+	args_match: JsonValueSchema.optional()
+});
+const ToolNotCalledArgsSchema = z.intersection(ToolNotCalledSelectorSchema, z.union([
+	z.object({ tool: NonEmptyString }),
+	z.object({ provider: NonEmptyString }),
+	z.object({ server: NonEmptyString }),
+	z.object({ args_match: JsonValueSchema })
+]));
+const SkillSelectionArgsSchema = z.object({
+	skills: z.array(NonEmptyString).min(1),
+	delivery: z.enum(["native", "mcp"]).optional(),
+	provider: OptionalString,
+	server: OptionalString,
+	source: OptionalString
+});
+const ToolCountArgsSchema = z.object({
+	tool: OptionalString,
+	provider: OptionalString,
+	server: OptionalString,
+	args_match: z.unknown().optional(),
+	turn: z.number().optional(),
+	before_turn: z.number().optional(),
+	after_turn: z.number().optional(),
+	min: z.number().optional(),
+	max: z.number().optional()
+});
+const TurnCountArgsSchema = z.object({
+	min: z.number().optional(),
+	max: z.number().optional()
+});
+const ToolSequenceArgsSchema = z.object({ order: z.array(NonEmptyString).min(2) });
+const OutputContainsArgsSchema = z.object({ text: z.string() });
+const OutputMatchesArgsSchema = z.object({
+	pattern: NonEmptyString,
+	flags: z.string().optional()
+});
+/**
+* Decode assertion args with a zod schema. Returns the parsed value, or a
+* failed assertion result carrying `invalidReason` when parsing fails.
+*/
+function decodeCheckArgs(schema, assertion, invalidReason) {
+	const decoded = schema.safeParse(assertion ?? {});
+	return decoded.success ? decoded.data : validationFailure(invalidReason);
+}
+function isValidationFailure(value) {
+	return !!value && typeof value === "object" && "pass" in value && value.pass === false && typeof value.reason === "string";
+}
+function decodeToolNotCalledArgs(assertion) {
+	return decodeCheckArgs(ToolNotCalledArgsSchema, assertion, "tool.not_called: assertion must include at least one selector");
+}
+//#endregion
+//#region src/runtime-checks/verifier-checks.ts
+const verifierSucceeds = {
+	type: "verifier.succeeds",
+	async verify(ctx) {
+		const a = decodeCheckArgs(VerifierArgsSchema, ctx.assertion, "verifier.succeeds: assertion.run must be a non-empty string");
+		if (isValidationFailure(a)) return a;
+		const r = await ctx.world.exec(a.run, a.args ?? [], { timeoutMs: a.timeoutMs ?? 6e4 });
+		return applyMode(r.exitCode === 0, ctx.mode, `verifier.succeeds: ${a.run} exited 0`, `verifier.succeeds: ${a.run} exited ${r.exitCode}: ${r.stderr.slice(0, 200)}`);
+	}
+};
+const verifierFails = {
+	type: "verifier.fails",
+	async verify(ctx) {
+		const a = decodeCheckArgs(VerifierArgsSchema, ctx.assertion, "verifier.fails: assertion.run must be a non-empty string");
+		if (isValidationFailure(a)) return a;
+		const r = await ctx.world.exec(a.run, a.args ?? [], { timeoutMs: a.timeoutMs ?? 6e4 });
+		return applyMode(r.exitCode !== 0, ctx.mode, `verifier.fails: ${a.run} exited ${r.exitCode} (failed as expected)`, `verifier.fails: ${a.run} unexpectedly exited 0`);
+	}
+};
+//#endregion
+//#region src/runtime-checks/file-code-checks.ts
+const fileExists = {
+	type: "file.exists",
+	async verify(ctx) {
+		const args = decodeCheckArgs(PathArgsSchema, ctx.assertion, "file.exists: path must be a non-empty string");
+		if (isValidationFailure(args)) return args;
+		return applyMode(await ctx.world.readFile(args.path) !== null, ctx.mode, `file.exists: ${args.path} present`, `file.exists: ${args.path} not found`);
+	}
+};
+const fileCreated = {
+	type: "file.created",
+	verify(ctx) {
+		const args = decodeCheckArgs(PathArgsSchema, ctx.assertion, "file.created: path must be a non-empty string");
+		if (isValidationFailure(args)) return args;
+		return applyMode(ctx.evidence.filesWritten().some((event) => event.path === args.path && event.op === "create"), ctx.mode, `file.created: ${args.path} created`, `file.created: ${args.path} was not created`);
+	}
+};
+const fileUnchanged = {
+	type: "file.unchanged",
+	verify(ctx) {
+		const args = decodeCheckArgs(PathArgsSchema, ctx.assertion, "file.unchanged: path must be a non-empty string");
+		if (isValidationFailure(args)) return args;
+		return applyMode(!ctx.evidence.filesWritten().some((event) => event.path === args.path), ctx.mode, `file.unchanged: ${args.path} unchanged`, `file.unchanged: ${args.path} changed`);
+	}
+};
+const fileContains = {
+	type: "file.contains",
+	async verify(ctx) {
+		const args = decodeCheckArgs(FileContainsArgsSchema, ctx.assertion, "file.contains: path and text are required");
+		if (isValidationFailure(args)) return args;
+		const content = await ctx.world.readFile(args.path);
+		return applyMode(content !== null && content.includes(args.text), ctx.mode, `file.contains: ${args.path} contains ${JSON.stringify(args.text.slice(0, 40))}`, content === null ? `file.contains: ${args.path} not found` : `file.contains: ${args.path} missing ${JSON.stringify(args.text.slice(0, 40))}`);
+	}
+};
+/** A path declaration is either one exact file or a directory ending in /. */
+function pathAllowed(path, allowed) {
+	return path === allowed || allowed.endsWith("/") && path.startsWith(allowed);
+}
+const fileChangesWithin = {
+	type: "file.changes_within",
+	verify(ctx) {
+		const args = decodeCheckArgs(FileChangesWithinArgsSchema, ctx.assertion, "file.changes_within: paths must contain at least one path");
+		if (isValidationFailure(args)) return args;
+		const outside = ctx.evidence.filesWritten().filter((event) => !args.paths.some((allowed) => pathAllowed(event.path, allowed)));
+		return applyMode(outside.length === 0, ctx.mode, `file.changes_within: all changes stayed within ${args.paths.join(", ")}`, `file.changes_within: changed outside allowed paths: ${outside.slice(0, 3).map((event) => event.path).join(", ")}`);
+	}
+};
+//#endregion
+//#region src/runtime-checks/skill-loaded.ts
+function matchesLoad(event, args) {
+	return (!args.delivery || event.delivery === args.delivery) && (!args.provider || event.provider === args.provider) && (!args.server || event.server === args.server) && (!args.source || event.source === args.source);
+}
+function loadedSkills(events, args) {
+	return new Set(events.filter((event) => matchesLoad(event, args)).map((event) => event.skill));
+}
+const skillLoaded = {
+	type: "skill.loaded",
+	verify({ assertion, evidence, mode }) {
+		const args = decodeCheckArgs(SkillSelectionArgsSchema, assertion, "skill.loaded: skills must contain at least one name");
+		if (isValidationFailure(args)) return args;
+		const loaded = loadedSkills(evidence.skillsLoaded(), args);
+		const missing = args.skills.filter((skill) => !loaded.has(skill));
+		return applyMode(missing.length === 0, mode, `skill.loaded: loaded ${args.skills.join(", ")}`, `skill.loaded: missing ${missing.join(", ")}`);
+	}
+};
+const skillNotLoaded = {
+	type: "skill.not_loaded",
+	verify({ assertion, evidence, mode }) {
+		const args = decodeCheckArgs(SkillSelectionArgsSchema, assertion, "skill.not_loaded: skills must contain at least one name");
+		if (isValidationFailure(args)) return args;
+		const loaded = loadedSkills(evidence.skillsLoaded(), args);
+		const forbidden = args.skills.filter((skill) => loaded.has(skill));
+		return applyMode(forbidden.length === 0, mode, `skill.not_loaded: absent ${args.skills.join(", ")}`, `skill.not_loaded: unexpectedly loaded ${forbidden.join(", ")}`);
+	}
+};
+//#endregion
+//#region src/runtime-checks/_match.ts
+function matchesSubset(actual, expected) {
+	if (expected === actual) return true;
+	if (Array.isArray(expected)) return Array.isArray(actual) && expected.length === actual.length && expected.every((item, i) => matchesSubset(actual[i], item));
+	if (expected && typeof expected === "object") {
+		if (!actual || typeof actual !== "object" || Array.isArray(actual)) return false;
+		const actualRecord = actual;
+		return Object.entries(expected).every(([key, value]) => matchesSubset(actualRecord[key], value));
+	}
+	return false;
+}
+//#endregion
+//#region src/runtime-checks/tool-turn-checks.ts
+function matchesRecordedCall(call, args) {
+	return (!args.tool || call.tool === args.tool) && (!args.provider || call.provider === args.provider) && (!args.server || call.server === args.server) && (args.args_match === void 0 || matchesSubset(call.args, args.args_match));
+}
+//#endregion
+//#region src/runtime-checks/catalog.ts
+const RUNTIME_CHECKS = [
+	{
+		type: "verifier.succeeds",
+		plugin: verifierSucceeds,
+		argsSchema: VerifierArgsSchema
+	},
+	{
+		type: "verifier.fails",
+		plugin: verifierFails,
+		argsSchema: VerifierArgsSchema
+	},
+	{
+		type: "file.exists",
+		plugin: fileExists,
+		argsSchema: PathArgsSchema
+	},
+	{
+		type: "file.created",
+		plugin: fileCreated,
+		argsSchema: PathArgsSchema
+	},
+	{
+		type: "file.contains",
+		plugin: fileContains,
+		argsSchema: FileContainsArgsSchema
+	},
+	{
+		type: "file.unchanged",
+		plugin: fileUnchanged,
+		argsSchema: PathArgsSchema
+	},
+	{
+		type: "file.changes_within",
+		plugin: fileChangesWithin,
+		argsSchema: FileChangesWithinArgsSchema
+	},
+	{
+		type: "tool.called",
+		plugin: {
+			type: "tool.called",
+			verify({ assertion, evidence, mode }) {
+				const a = decodeCheckArgs(ToolCalledArgsSchema, assertion, "tool.called: assertion.tool must be a non-empty string");
+				if (isValidationFailure(a)) return a;
+				const calls = evidence.toolCalls();
+				if (!calls.length) return {
+					pass: false,
+					score: 0,
+					reason: "tool.called: no built-in tool evidence found"
+				};
+				return applyMode(calls.some((c) => matchesRecordedCall(c, a)), mode, "tool.called: matched built-in tool call", "tool.called: matching call not found");
+			}
+		},
+		argsSchema: ToolCalledArgsSchema
+	},
+	{
+		type: "tool.not_called",
+		plugin: {
+			type: "tool.not_called",
+			verify({ assertion, evidence }) {
+				const a = decodeToolNotCalledArgs(assertion);
+				if (isValidationFailure(a)) return a;
+				return evidence.toolCalls().some((c) => matchesRecordedCall(c, a)) ? {
+					pass: false,
+					score: 0,
+					reason: "tool.not_called: forbidden built-in tool call observed"
+				} : {
+					pass: true,
+					score: 1,
+					reason: "tool.not_called: no matching built-in tool calls observed"
+				};
+			}
+		},
+		argsSchema: ToolNotCalledArgsSchema
+	},
+	{
+		type: "tool.count",
+		plugin: {
+			type: "tool.count",
+			verify({ assertion, evidence, mode }) {
+				const a = decodeCheckArgs(ToolCountArgsSchema, assertion, "tool.count: invalid args (tool/provider/server/args_match/turn/before_turn/after_turn/min/max)");
+				if (isValidationFailure(a)) return a;
+				if (a.min === void 0 && a.max === void 0) return validationFailure("tool.count: set min and/or max");
+				const count = evidence.toolCalls().filter((call) => {
+					const turn = call.turn ?? 1;
+					if (a.turn !== void 0 && turn !== a.turn) return false;
+					if (a.before_turn !== void 0 && turn >= a.before_turn) return false;
+					if (a.after_turn !== void 0 && turn <= a.after_turn) return false;
+					return matchesRecordedCall(call, a);
+				}).length;
+				const within = (a.min === void 0 || count >= a.min) && (a.max === void 0 || count <= a.max);
+				const bounds = [a.min !== void 0 ? `min ${a.min}` : void 0, a.max !== void 0 ? `max ${a.max}` : void 0].filter(Boolean).join(", ");
+				return applyMode(within, mode, `tool.count: ${count} matching call(s) within bounds (${bounds})`, `tool.count: ${count} matching call(s) outside bounds (${bounds})`);
+			}
+		},
+		argsSchema: ToolCountArgsSchema
+	},
+	{
+		type: "tool.sequence",
+		plugin: {
+			type: "tool.sequence",
+			verify({ assertion, evidence, mode }) {
+				const a = decodeCheckArgs(ToolSequenceArgsSchema, assertion, "tool.sequence: assertion.order must list at least two tool names");
+				if (isValidationFailure(a)) return a;
+				const names = evidence.toolCalls().map((call) => call.tool);
+				let next = 0;
+				for (const name of names) {
+					if (name === a.order[next]) next += 1;
+					if (next === a.order.length) break;
+				}
+				const matched = next === a.order.length;
+				const missing = matched ? "" : ` (stuck at "${a.order[next]}")`;
+				return applyMode(matched, mode, `tool.sequence: observed ${a.order.join(" -> ")} in order`, `tool.sequence: did not observe ${a.order.join(" -> ")} in order${missing}`);
+			}
+		},
+		argsSchema: ToolSequenceArgsSchema
+	},
+	{
+		type: "turn.count",
+		plugin: {
+			type: "turn.count",
+			verify({ assertion, evidence, mode }) {
+				const a = decodeCheckArgs(TurnCountArgsSchema, assertion, "turn.count: invalid args (min/max must be numbers)");
+				if (isValidationFailure(a)) return a;
+				if (a.min === void 0 && a.max === void 0) return validationFailure("turn.count: set min and/or max");
+				const agentTurns = (evidence.turns?.() ?? []).filter((turn) => turn.role === "agent").length;
+				const count = agentTurns > 0 ? agentTurns : 1;
+				const within = (a.min === void 0 || count >= a.min) && (a.max === void 0 || count <= a.max);
+				const bounds = [a.min !== void 0 ? `min ${a.min}` : void 0, a.max !== void 0 ? `max ${a.max}` : void 0].filter(Boolean).join(", ");
+				return applyMode(within, mode, `turn.count: ${count} agent turn(s) within bounds (${bounds})`, `turn.count: ${count} agent turn(s) outside bounds (${bounds})`);
+			}
+		},
+		argsSchema: TurnCountArgsSchema
+	},
+	{
+		type: "skill.loaded",
+		plugin: skillLoaded,
+		argsSchema: SkillSelectionArgsSchema
+	},
+	{
+		type: "skill.not_loaded",
+		plugin: skillNotLoaded,
+		argsSchema: SkillSelectionArgsSchema
+	},
+	{
+		type: "output.contains",
+		plugin: {
+			type: "output.contains",
+			verify(ctx) {
+				const args = decodeCheckArgs(OutputContainsArgsSchema, ctx.assertion, "output.contains: text is required");
+				if (isValidationFailure(args)) return args;
+				return applyMode(ctx.evidence.output().includes(args.text), ctx.mode, `output.contains: found ${JSON.stringify(args.text)}`, `output.contains: missing ${JSON.stringify(args.text)}`);
+			}
+		},
+		argsSchema: OutputContainsArgsSchema
+	},
+	{
+		type: "output.matches",
+		plugin: {
+			type: "output.matches",
+			verify(ctx) {
+				const args = decodeCheckArgs(OutputMatchesArgsSchema, ctx.assertion, "output.matches: pattern is required");
+				if (isValidationFailure(args)) return args;
+				let regex;
+				try {
+					regex = new RegExp(args.pattern, args.flags);
+				} catch (error) {
+					return {
+						pass: false,
+						score: 0,
+						reason: `output.matches: invalid regex: ${error instanceof Error ? error.message : String(error)}`
+					};
+				}
+				return applyMode(regex.test(ctx.evidence.output()), ctx.mode, `output.matches: /${args.pattern}/${args.flags ?? ""} matched`, `output.matches: /${args.pattern}/${args.flags ?? ""} did not match`);
+			}
+		},
+		argsSchema: OutputMatchesArgsSchema
+	}
+];
+const RUNTIME_CHECK_TYPES = RUNTIME_CHECKS.map((check) => check.type);
+const RUNTIME_CHECKS_BY_TYPE = new Map(RUNTIME_CHECKS.map(({ type, plugin }) => [type, plugin]));
+//#endregion
+export { RUNTIME_CHECKS_BY_TYPE as n, RUNTIME_CHECK_TYPES as r, RUNTIME_CHECKS as t };
+
+//# sourceMappingURL=catalog-9Wh-AfRv.mjs.map