@klhapp/skillmux 1.1.0 → 1.3.0

package/src/calibrate.ts

@@ -3,7 +3,7 @@ import { join } from "node:path";
 import { Database } from "bun:sqlite";
 import { z } from "zod";
 import { decideResolveOutcome } from "./decision";
-import type { RankedCandidate } from "./types";
+import type { AuditRow, RankedCandidate } from "./types";
 
 export { generateDataset, type GenerateDatasetOptions } from "./dataset-generator";
 
@@ -15,11 +15,21 @@ export { generateDataset, type GenerateDatasetOptions } from "./dataset-generato
 export type DecisionSplit = "tune" | "test";
 export type DecisionOutcome = "matched" | "ambiguous" | "no_match";
 
+export interface DecisionCaseProvenance {
+  version: 1;
+  source: "authored" | "audit_import";
+  review_status: "human_labelled" | "unreviewed";
+  query_storage: "raw" | "redacted";
+  audit_id?: number;
+  labelled_at?: string;
+}
+
 export interface DecisionCase {
   query: string;
   split: DecisionSplit;
   expected_outcome: DecisionOutcome;
   relevant_skill_ids: string[];
+  provenance?: DecisionCaseProvenance;
 }
 
 // ---------------------------------------------------------------------------
@@ -31,6 +41,14 @@ const rawCaseSchema = z.object({
   split: z.enum(["tune", "test"]),
   expected_outcome: z.enum(["matched", "ambiguous", "no_match"]),
   relevant_skill_ids: z.array(z.string()),
+  provenance: z.object({
+    version: z.literal(1),
+    source: z.enum(["authored", "audit_import"]),
+    review_status: z.enum(["human_labelled", "unreviewed"]),
+    query_storage: z.enum(["raw", "redacted"]),
+    audit_id: z.number().int().positive().optional(),
+    labelled_at: z.string().datetime().optional(),
+  }).strict().optional(),
 }).strict();
 
 type RawCase = z.infer<typeof rawCaseSchema>;
@@ -45,6 +63,25 @@ function validateCase(raw: RawCase, idx: number): DecisionCase {
   }
 
   const { expected_outcome, relevant_skill_ids } = raw;
+  const provenance: DecisionCaseProvenance = raw.provenance ?? {
+    version: 1,
+    source: "authored",
+    review_status: "human_labelled",
+    query_storage: "raw",
+  };
+
+  if (provenance.source === "audit_import") {
+    if (provenance.audit_id === undefined) {
+      throw new Error(
+        `Validation error at case ${idx}: imported field "provenance.audit_id" is required`,
+      );
+    }
+    if (provenance.review_status !== "human_labelled" || !provenance.labelled_at) {
+      throw new Error(
+        `Validation error at case ${idx}: imported audit case is unreviewed; human label and "provenance.labelled_at" are required for certification`,
+      );
+    }
+  }
 
   if (expected_outcome === "matched") {
     if (relevant_skill_ids.length !== 1) {
@@ -67,7 +104,85 @@ function validateCase(raw: RawCase, idx: number): DecisionCase {
     }
   }
 
-  return raw as DecisionCase;
+  return { ...raw, provenance };
+}
+
+export interface AuditFeedbackLabel {
+  split: DecisionSplit;
+  expected_outcome: DecisionOutcome;
+  relevant_skill_ids: string[];
+  labelled_at: string;
+}
+
+export type AuditQueryPrivacy =
+  | { include_raw_query: true }
+  | { include_raw_query: false; redacted_query: string };
+
+/** Import an audit outcome only after a separate human label is supplied. */
+export function importLabelledAuditCase(
+  audit: AuditRow,
+  label: AuditFeedbackLabel,
+  privacy: AuditQueryPrivacy,
+): DecisionCase {
+  const query = privacy.include_raw_query ? audit.query : privacy.redacted_query.trim();
+  if (!query) {
+    throw new Error("A non-empty redacted_query is required when raw audit queries are excluded");
+  }
+  const parsed = rawCaseSchema.parse({
+    query,
+    split: label.split,
+    expected_outcome: label.expected_outcome,
+    relevant_skill_ids: label.relevant_skill_ids,
+    provenance: {
+      version: 1,
+      source: "audit_import",
+      review_status: "human_labelled",
+      query_storage: privacy.include_raw_query ? "raw" : "redacted",
+      audit_id: audit.id,
+      labelled_at: label.labelled_at,
+    },
+  });
+  return validateCase(parsed, audit.id);
+}
+
+export interface DatasetProvenanceSummary {
+  version: 1;
+  human_labelled_case_count: number;
+  imported_labelled_case_count: number;
+  imported_unreviewed_case_count: number;
+  raw_query_case_count: number;
+  redacted_query_case_count: number;
+}
+
+export function summarizeDatasetProvenance(
+  cases: DecisionCase[],
+): DatasetProvenanceSummary {
+  const provenance = (item: DecisionCase): DecisionCaseProvenance =>
+    item.provenance ?? {
+      version: 1,
+      source: "authored",
+      review_status: "human_labelled",
+      query_storage: "raw",
+    };
+  return {
+    version: 1,
+    human_labelled_case_count:
+      cases.filter((item) => provenance(item).review_status === "human_labelled").length,
+    imported_labelled_case_count:
+      cases.filter((item) =>
+        provenance(item).source === "audit_import" &&
+        provenance(item).review_status === "human_labelled"
+      ).length,
+    imported_unreviewed_case_count:
+      cases.filter((item) =>
+        provenance(item).source === "audit_import" &&
+        provenance(item).review_status === "unreviewed"
+      ).length,
+    raw_query_case_count:
+      cases.filter((item) => provenance(item).query_storage === "raw").length,
+    redacted_query_case_count:
+      cases.filter((item) => provenance(item).query_storage === "redacted").length,
+  };
 }
 
 // ---------------------------------------------------------------------------
@@ -113,8 +228,12 @@ function validateDatasetCompleteness(cases: DecisionCase[]): void {
  *     no_match → 0)
  *   - Dataset completeness (both splits, all outcome types in each split)
  */
-export function loadDecisionCases(raw: unknown[]): DecisionCase[] {
+export function loadDecisionCases(
+  raw: unknown[],
+  validSkillIds?: Iterable<string>,
+): DecisionCase[] {
   const parsed: DecisionCase[] = [];
+  const validIds = validSkillIds ? new Set(validSkillIds) : undefined;
 
   for (let i = 0; i < raw.length; i++) {
     const item = raw[i];
@@ -128,7 +247,17 @@ export function loadDecisionCases(raw: unknown[]): DecisionCase[] {
       );
     }
 
-    parsed.push(validateCase(result.data, i));
+    const parsedCase = validateCase(result.data, i);
+    if (validIds) {
+      for (const skillId of parsedCase.relevant_skill_ids) {
+        if (!validIds.has(skillId)) {
+          throw new Error(
+            `Validation error at case ${i}: field "relevant_skill_ids" references unknown vault skill "${skillId}"`,
+          );
+        }
+      }
+    }
+    parsed.push(parsedCase);
   }
 
   validateDatasetCompleteness(parsed);
@@ -139,9 +268,12 @@ export function loadDecisionCases(raw: unknown[]): DecisionCase[] {
  * Read a JSON file from disk and validate it as a decision-policy dataset.
  * Throws if the file cannot be read or the contents fail validation.
  */
-export function loadDecisionCasesFromFile(path: string): DecisionCase[] {
+export function loadDecisionCasesFromFile(
+  path: string,
+  validSkillIds?: Iterable<string>,
+): DecisionCase[] {
   const raw = JSON.parse(readFileSync(path, "utf8")) as unknown[];
-  return loadDecisionCases(raw);
+  return loadDecisionCases(raw, validSkillIds);
 }
 
 // ---------------------------------------------------------------------------
@@ -734,6 +866,7 @@ export interface CalibrationRunRecord {
   embedding_fingerprint: string;
   corpus_fingerprint: string;
   dataset_hash: string;
+  dataset_provenance?: DatasetProvenanceSummary;
   candidate_limit: number;
   attempt_count?: number;
   min_auto_match_precision: number;
@@ -756,6 +889,8 @@ export interface CalibrationRunSummary {
   embedding_fingerprint: string;
   corpus_fingerprint: string;
   dataset_hash: string;
+  human_labelled_case_count: number;
+  imported_labelled_case_count: number;
   candidate_limit: number;
   attempt_count: number;
   min_auto_match_precision: number;
@@ -806,6 +941,15 @@ export function openCalibrateDb(stateDir: string): Database {
   if (!columns.some((column) => column.name === "failed_reason")) {
     db.run("ALTER TABLE calibration_runs ADD COLUMN failed_reason TEXT");
   }
+  if (!columns.some((column) => column.name === "dataset_provenance")) {
+    db.run("ALTER TABLE calibration_runs ADD COLUMN dataset_provenance TEXT NOT NULL DEFAULT '{}'");
+  }
+  if (!columns.some((column) => column.name === "human_labelled_case_count")) {
+    db.run("ALTER TABLE calibration_runs ADD COLUMN human_labelled_case_count INTEGER NOT NULL DEFAULT 0");
+  }
+  if (!columns.some((column) => column.name === "imported_labelled_case_count")) {
+    db.run("ALTER TABLE calibration_runs ADD COLUMN imported_labelled_case_count INTEGER NOT NULL DEFAULT 0");
+  }
   return db;
 }
 
@@ -822,8 +966,9 @@ export function insertCalibrationRun(db: Database, run: CalibrationRunRecord): v
       candidate_limit,
       attempt_count, min_auto_match_precision, min_auto_match_count,
       min_delivered_shortlist_recall_at_k, min_shortlist_recall_at_5, failed_reason,
-      selected_thresholds, tune_metrics, test_metrics, observations
-    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      selected_thresholds, tune_metrics, test_metrics, observations,
+      dataset_provenance, human_labelled_case_count, imported_labelled_case_count
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
     [
       run.run_id,
       run.created_at,
@@ -843,6 +988,9 @@ export function insertCalibrationRun(db: Database, run: CalibrationRunRecord): v
       run.tune_metrics != null ? JSON.stringify(run.tune_metrics) : null,
       run.test_metrics != null ? JSON.stringify(run.test_metrics) : null,
       JSON.stringify(run.observations),
+      JSON.stringify(run.dataset_provenance ?? {}),
+      run.dataset_provenance?.human_labelled_case_count ?? 0,
+      run.dataset_provenance?.imported_labelled_case_count ?? 0,
     ],
   );
 }
@@ -866,6 +1014,9 @@ interface RawCalibrationRow {
   tune_metrics: string | null;
   test_metrics: string | null;
   observations: string;
+  dataset_provenance: string;
+  human_labelled_case_count: number;
+  imported_labelled_case_count: number;
 }
 
 function parseMetrics(json: string): CalibrationMetrics {
@@ -900,6 +1051,10 @@ function rowToRecord(row: RawCalibrationRow): CalibrationRunRecord {
     embedding_fingerprint: row.embedding_fingerprint,
     corpus_fingerprint: row.corpus_fingerprint,
     dataset_hash: row.dataset_hash,
+    dataset_provenance:
+      Object.keys(JSON.parse(row.dataset_provenance) as object).length > 0
+        ? JSON.parse(row.dataset_provenance) as DatasetProvenanceSummary
+        : undefined,
     candidate_limit: row.candidate_limit,
     attempt_count: row.attempt_count,
     min_auto_match_precision: row.min_auto_match_precision,
@@ -939,7 +1094,8 @@ export function listCalibrationRuns(db: Database): CalibrationRunSummary[] {
         reranker_fingerprint, embedding_fingerprint, corpus_fingerprint, dataset_hash,
         candidate_limit,
         attempt_count, min_auto_match_precision, min_auto_match_count,
-        min_delivered_shortlist_recall_at_k, min_shortlist_recall_at_5, failed_reason
+        min_delivered_shortlist_recall_at_k, min_shortlist_recall_at_5, failed_reason,
+        human_labelled_case_count, imported_labelled_case_count
        FROM calibration_runs ORDER BY created_at DESC`,
     )
     .all() as CalibrationRunSummary[];

package/src/cli.ts

@@ -132,6 +132,31 @@ const KNOWN_COMMANDS = [
   "local-vault",
 ];
 
+const DOCKER_HOST_MANAGEMENT_GUIDANCE =
+  "This command manages local Skillmux or agent directories and is not supported inside the Docker image. Install the Skillmux CLI on the host using the Bun package or standalone Linux executable.";
+
+function isDockerHostManagementCommand(command: string, subCommand: string): boolean {
+  if (
+    [
+      "init",
+      "sync",
+      "install",
+      "project",
+      "target",
+      "core",
+      "local-vault",
+      "models",
+      "context",
+      "calibrate",
+      "eval",
+    ].includes(command)
+  ) {
+    return true;
+  }
+
+  return command === "config" && ["init", "set"].includes(subCommand);
+}
+
 async function main() {
   const rawArgv = Bun.argv.slice(2);
 
@@ -141,6 +166,8 @@ async function main() {
   let flagContext: string | undefined;
   let flagServer: string | undefined;
   let isDryRun = false;
+  const subCommand = rawArgv[1] ?? "";
+  const commandArgs = rawArgv.slice(2);
 
   const command = rawArgv[0];
   if (!command || command === "--help" || command === "-h") {
@@ -161,6 +188,18 @@ async function main() {
 
   let resolvedTarget: ResolvedTarget = { type: "local", name: "local" };
 
+  if (
+    process.env.RUNNING_IN_DOCKER === "true" &&
+    isDockerHostManagementCommand(command, subCommand)
+  ) {
+    handleError(new Error(DOCKER_HOST_MANAGEMENT_GUIDANCE), {
+      target: resolvedTarget,
+      isJson,
+      isVerbose,
+    });
+    return;
+  }
+
   // Only resolve target if command is target-aware or context/config/calibrate
   const isLocalConfigInit = command === "config" && rawArgv[1] === "init";
   if (
@@ -181,8 +220,6 @@ async function main() {
   }
 
   const adapter = createTargetAdapter(resolvedTarget, { allowInsecure });
-  const subCommand = rawArgv[1] ?? "";
-  const commandArgs = rawArgv.slice(2);
 
   try {
     switch (command) {
@@ -446,7 +483,7 @@ async function handleCalibrateCommand(
       minAutoMatchCount,
     });
     emitSuccess({ isJson: ctx.isJson, target: ctx.target }, res, () => {
-      renderTargetBanner(ctx.target);
+      renderCalibrationTarget(ctx.target);
       console.log(`Calibration run complete.`);
       if (res.result) console.log(JSON.stringify(res.result, null, 2));
     });
@@ -456,7 +493,7 @@ async function handleCalibrateCommand(
   if (sub === "list") {
     const res = await adapter.calibrateList();
     emitSuccess({ isJson: ctx.isJson, target: ctx.target }, res, () => {
-      renderTargetBanner(ctx.target);
+      renderCalibrationTarget(ctx.target);
       renderTable(
         [
           { key: "run_id", header: "RUN_ID" },
@@ -474,7 +511,7 @@ async function handleCalibrateCommand(
     if (!runId) throw new Error("usage: skillmux calibrate show <run_id>");
     const res = await adapter.calibrateShow(runId);
     emitSuccess({ isJson: ctx.isJson, target: ctx.target }, res, () => {
-      renderTargetBanner(ctx.target);
+      renderCalibrationTarget(ctx.target);
       console.log(JSON.stringify(res, null, 2));
     });
     return;
@@ -485,7 +522,7 @@ async function handleCalibrateCommand(
     if (!runId) throw new Error("usage: skillmux calibrate apply <run_id>");
     const res = await adapter.calibrateApply(runId);
     emitSuccess({ isJson: ctx.isJson, target: ctx.target }, res, () => {
-      renderTargetBanner(ctx.target);
+      renderCalibrationTarget(ctx.target);
       console.log(`Applied calibration run "${runId}"`);
     });
     return;
@@ -501,6 +538,14 @@ async function handleCalibrateCommand(
   );
 }
 
+function renderCalibrationTarget(target: ResolvedTarget): void {
+  if (target.type === "local") {
+    console.log("Target: local");
+  } else {
+    console.log(`Target: remote (${target.name} -> ${target.server})`);
+  }
+}
+
 async function handleCompletionsCommand(shell: string) {
   if (shell !== "bash" && shell !== "zsh" && shell !== "fish") {
     throw new Error("usage: skillmux completions <bash|zsh|fish>");

package/src/config-watcher.ts

@@ -1,4 +1,4 @@
-import { watch } from "node:fs";
+import { mkdirSync, watch } from "node:fs";
 import { dirname } from "node:path";
 import { loadConfig } from "./config";
 import type { Config } from "./types";
@@ -133,6 +133,10 @@ export class ConfigWatcher {
     const dir = dirname(tomlPath);
     const filename = tomlPath.split(/[/\\]/).pop()!;
 
+    // A config file is optional. Ensure its parent exists so zero-config
+    // startup is safe and later config writes are still observed.
+    mkdirSync(dir, { recursive: true });
+
     this.watcher = watch(dir, { recursive: false }, (_event, changedName) => {
       if (this.stopped) return;
       // Fire for: the config file itself, or any .tmp variant of it (handles

package/src/dataset-generator.ts

@@ -13,14 +13,45 @@ export interface GenerateDatasetOptions {
   queriesPerSplit?: number;
 }
 
-const GENERIC_NO_MATCH_QUERIES = [
-  "what is the weather in Paris today",
-  "recipe for baking sourdough bread at home",
-  "what is the distance from Earth to Mars",
-  "explain quantum entanglement simply",
-  "how do I solve a quadratic equation",
-  "who won the 1998 World Cup",
-];
+const STOP_WORDS = new Set([
+  "a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "in", "is",
+  "it", "of", "on", "or", "the", "this", "to", "use", "with",
+]);
+
+function words(value: string): string[] {
+  return value
+    .toLowerCase()
+    .match(/[a-z0-9]+/g)
+    ?.filter((word) => word.length > 2 && !STOP_WORDS.has(word)) ?? [];
+}
+
+function anchors(skill: VaultSkill): string[] {
+  const preferred = [...skill.aliases.flatMap(words), ...words(skill.title)];
+  const fallback = words(skill.description);
+  return [...new Set([...preferred, ...fallback])].slice(0, 2);
+}
+
+function matchedQuery(skill: VaultSkill, variant: number): string {
+  const [first = "specialized", second = "workflow"] = anchors(skill);
+  const templates = [
+    `I need practical guidance completing an unfamiliar ${first} ${second} task safely`,
+    `Which available workflow can handle my unusual ${first} ${second} problem end to end`,
+    `Please guide me through a difficult unfamiliar ${first} ${second} operation safely`,
+  ];
+  return templates[variant % templates.length]!;
+}
+
+function ambiguousQuery(first: VaultSkill, second: VaultSkill): string {
+  const [firstAnchor = "first"] = anchors(first);
+  const [secondAnchor = "second"] = anchors(second);
+  return `Help with a workflow spanning both ${firstAnchor} and ${secondAnchor} responsibilities`;
+}
+
+function nearMissQuery(first: VaultSkill, second: VaultSkill): string {
+  const [firstAnchor = "one"] = anchors(first);
+  const [secondAnchor = "another"] = anchors(second);
+  return `Explain the theory comparing ${firstAnchor} and ${secondAnchor} without performing either workflow`;
+}
 
 /**
  * Automatically generate a synthetic decision-policy calibration dataset
@@ -30,110 +61,58 @@ export function generateDataset(
   skills: VaultSkill[],
   options: GenerateDatasetOptions = {},
 ): RawDecisionCase[] {
-  const cases: RawDecisionCase[] = [];
+  if (skills.length < 4) {
+    throw new Error(
+      "Dataset generation requires at least 4 vault skills so tune and test can each contain matched and ambiguous cases without skill leakage",
+    );
+  }
 
-  // --- 1. Matched Cases ---
-  for (const skill of skills) {
-    // Primary query from title + description
-    cases.push({
-      query: `how do I ${skill.title.toLowerCase()}: ${skill.description.toLowerCase()}`,
-      split: "tune",
-      expected_outcome: "matched",
-      relevant_skill_ids: [skill.skill_id],
-    });
+  const cases: RawDecisionCase[] = [];
+  const sorted = [...skills].sort((a, b) => a.skill_id.localeCompare(b.skill_id));
+  const splitAt = Math.ceil(sorted.length / 2);
+  const bySplit: Record<DecisionSplit, VaultSkill[]> = {
+    tune: sorted.slice(0, splitAt),
+    test: sorted.slice(splitAt),
+  };
+  const targetPerSplit = Math.max(3, options.queriesPerSplit ?? 10);
 
-    // Secondary query from aliases
-    if (skill.aliases.length > 0) {
-      cases.push({
-        query: `help me with ${skill.aliases[0]}`,
-        split: "test",
-        expected_outcome: "matched",
-        relevant_skill_ids: [skill.skill_id],
-      });
-    } else {
+  for (const split of ["tune", "test"] as const) {
+    const splitSkills = bySplit[split];
+    for (let i = 0; i < splitSkills.length; i++) {
+      const skill = splitSkills[i]!;
       cases.push({
-        query: `execute task related to ${skill.title}`,
-        split: "test",
+        query: matchedQuery(skill, i),
+        split,
         expected_outcome: "matched",
         relevant_skill_ids: [skill.skill_id],
       });
     }
-  }
 
-  // --- 2. Ambiguous Cases ---
-  if (skills.length >= 2) {
-    // Pair skills for ambiguous multi-match
-    for (let i = 0; i < skills.length - 1; i += 2) {
-      const s1 = skills[i]!;
-      const s2 = skills[i + 1]!;
-      const split: DecisionSplit = i % 4 === 0 ? "tune" : "test";
-      cases.push({
-        query: `automated task using ${s1.title} and ${s2.title}`,
-        split,
-        expected_outcome: "ambiguous",
-        relevant_skill_ids: [s1.skill_id, s2.skill_id],
-      });
-    }
-  } else {
-    // Fallback ambiguous cases if fewer than 2 skills
-    cases.push({
-      query: "automate browser workflow testing",
-      split: "tune",
-      expected_outcome: "ambiguous",
-      relevant_skill_ids: ["mock-e2e", "mock-browser"],
-    });
-    cases.push({
-      query: "extract and fetch clean web text",
-      split: "test",
-      expected_outcome: "ambiguous",
-      relevant_skill_ids: ["mock-fetch", "mock-extract"],
-    });
-  }
-
-  // Ensure both tune and test have ambiguous cases
-  if (!cases.some((c) => c.split === "tune" && c.expected_outcome === "ambiguous")) {
-    const sIds = skills.length >= 2 ? [skills[0]!.skill_id, skills[1]!.skill_id] : ["mock-a", "mock-b"];
+    const first = splitSkills[0]!;
+    const second = splitSkills[1]!;
     cases.push({
-      query: "integrated workflow multi skill query",
-      split: "tune",
+      query: ambiguousQuery(first, second),
+      split,
       expected_outcome: "ambiguous",
-      relevant_skill_ids: sIds,
+      relevant_skill_ids: [first.skill_id, second.skill_id],
     });
-  }
-  if (!cases.some((c) => c.split === "test" && c.expected_outcome === "ambiguous")) {
-    const sIds = skills.length >= 2 ? [skills[0]!.skill_id, skills[1]!.skill_id] : ["mock-a", "mock-b"];
     cases.push({
-      query: "combined operations multi skill query",
-      split: "test",
-      expected_outcome: "ambiguous",
-      relevant_skill_ids: sIds,
-    });
-  }
-
-  // --- 3. No Match Cases ---
-  GENERIC_NO_MATCH_QUERIES.forEach((q, idx) => {
-    cases.push({
-      query: q,
-      split: idx % 2 === 0 ? "tune" : "test",
+      query: nearMissQuery(first, second),
+      split,
       expected_outcome: "no_match",
       relevant_skill_ids: [],
     });
-  });
 
-  // Ensure both splits have at least 1 matched case if skills were empty
-  if (skills.length === 0) {
-    cases.push({
-      query: "run mock container action",
-      split: "tune",
-      expected_outcome: "matched",
-      relevant_skill_ids: ["mock-container"],
-    });
-    cases.push({
-      query: "search mock API docs",
-      split: "test",
-      expected_outcome: "matched",
-      relevant_skill_ids: ["mock-docs"],
-    });
+    for (let i = cases.filter((item) => item.split === split).length; i < targetPerSplit; i++) {
+      const left = splitSkills[i % splitSkills.length]!;
+      const right = splitSkills[(i + 1) % splitSkills.length]!;
+      cases.push({
+        query: i % 2 === 0 ? matchedQuery(left, i) : nearMissQuery(left, right),
+        split,
+        expected_outcome: i % 2 === 0 ? "matched" : "no_match",
+        relevant_skill_ids: i % 2 === 0 ? [left.skill_id] : [],
+      });
+    }
   }
 
   return cases;