@unbrained/pm-cli 2026.6.5 → 2026.6.7

package/packages/pm-lifecycle-hooks/README.md

@@ -11,6 +11,27 @@ Transition-aware packages can inspect `afterCommand`'s optional `affected`
 entries for item mutations. Each entry includes the item id, operation,
 previous/current status, changed fields, and compact front matter snapshots.
 
+Copy this pattern when a package needs command-level notifications without
+parsing command results or reading item files:
+
+```ts
+api.hooks.afterCommand((event) => {
+  const transitions = event.affected?.filter(
+    (item) => item.previous_status !== item.status && item.status !== undefined,
+  );
+
+  if (!transitions?.length) {
+    return;
+  }
+
+  // Notify, enqueue, or cache based on item.id, item.previous_status,
+  // item.status, item.changed_fields, and item.current.
+});
+```
+
+Use `afterCommand` for command-level context such as notifications. Use
+`onWrite` when a package needs file-level sync or audit behavior.
+
 ## Capabilities
 
 - `hooks`

package/packages/pm-search-advanced/README.md

@@ -16,6 +16,8 @@ First-party package that restores optional advanced search surfaces in bare-core
 - Adds `pm reindex` with:
   - `--mode keyword|semantic|hybrid`
   - `--progress`
+  - `--eval`
+  - `--eval-fixtures <path>`
 
 ## Install
 
@@ -30,6 +32,42 @@ pm search-advanced "vector cache" --mode hybrid --limit 5 --json
 pm search-advanced --hybrid "vector cache" --limit 5 --json
 pm search-advanced "calendar package" --mode keyword --fields id,title,score --compact --json
 pm reindex --mode hybrid --progress --json
+pm reindex --mode keyword --eval --eval-fixtures tests/search-eval/golden-queries.json --json
 ```
 
 Without `--mode`, `--semantic`, or `--hybrid`, `search-advanced` stays keyword-first for fast agent reads.
+
+## Reindex eval harness
+
+`pm reindex --eval` runs a golden-query relevance harness after the reindex pass and appends a deterministic `eval` object to JSON output:
+
+- per-fixture pass/fail verdicts
+- `ndcg_at_5` score for each fixture
+- aggregate `average_ndcg_at_5`, pass/fail counts, and overall status
+
+The default fixture file is `tests/search-eval/golden-queries.json`. Override with `--eval-fixtures <path>`.
+
+### Fixture schema
+
+Fixtures can be either a top-level array or an object with a `fixtures` array:
+
+```json
+{
+  "fixtures": [
+    {
+      "name": "search-eval-task",
+      "query": "pm-22x2 search relevance evaluation harness",
+      "mode": "keyword",
+      "expected_top_ids": ["pm-22x2", "pm-fhsg"],
+      "min_ndcg_at_5": 0.8
+    }
+  ]
+}
+```
+
+Rules:
+
+- `query` is required and must be non-empty.
+- `mode` is optional (`keyword`, `semantic`, `hybrid`), default `keyword`.
+- `expected_top_ids` is required and defines the ideal ranking for nDCG@5.
+- `min_ndcg_at_5` is optional (`0..1`), default `0.7`.

package/packages/pm-search-advanced/extensions/search-advanced/index.js

@@ -200,6 +200,23 @@ const reindexFlags = [
     value_type: "boolean",
     description: "Emit non-interactive progress lines to stderr.",
   },
+  {
+    long: "--eval",
+    value_type: "boolean",
+    description: "Run golden-query relevance eval and append nDCG@5 summary output.",
+  },
+  {
+    long: "--eval-fixtures",
+    value_name: "path",
+    value_type: "string",
+    description: "Path to reindex eval fixtures JSON (default: tests/search-eval/golden-queries.json).",
+  },
+  {
+    long: "--eval_fixtures",
+    value_name: "path",
+    value_type: "string",
+    description: "Alias for --eval-fixtures.",
+  },
 ];
 
 function searchAdvancedCommand() {

package/packages/pm-search-advanced/extensions/search-advanced/index.ts

@@ -222,6 +222,23 @@ const reindexFlags = [
     value_type: "boolean",
     description: "Emit non-interactive progress lines to stderr.",
   },
+  {
+    long: "--eval",
+    value_type: "boolean",
+    description: "Run golden-query relevance eval and append nDCG@5 summary output.",
+  },
+  {
+    long: "--eval-fixtures",
+    value_name: "path",
+    value_type: "string",
+    description: "Path to reindex eval fixtures JSON (default: tests/search-eval/golden-queries.json).",
+  },
+  {
+    long: "--eval_fixtures",
+    value_name: "path",
+    value_type: "string",
+    description: "Alias for --eval-fixtures.",
+  },
 ] as const;
 
 function searchAdvancedCommand(): CommandDefinition {

package/packages/pm-search-advanced/extensions/search-advanced/runtime.js

@@ -1,7 +1,13 @@
+import { readFile } from "node:fs/promises";
 import path from "node:path";
 import { pathToFileURL } from "node:url";
 
 const PM_PACKAGE_ROOT_ENV = "PM_CLI_PACKAGE_ROOT";
+const DEFAULT_EVAL_FIXTURES_PATH = path.join("tests", "search-eval", "golden-queries.json");
+const DEFAULT_EVAL_MIN_NDCG_AT_5 = 0.7;
+const EVAL_RANK_CUTOFF = 5;
+const VALID_EVAL_SEARCH_MODES = new Set(["keyword", "semantic", "hybrid"]);
+
 const sdk = await loadSearchSdkModule();
 const {
   EXIT_CODE,
@@ -136,10 +142,219 @@ function normalizeReindexOptions(rawOptions) {
   };
 }
 
+function normalizeEvalFixturePath(rawPath) {
+  const candidate = typeof rawPath === "string" ? rawPath : DEFAULT_EVAL_FIXTURES_PATH;
+  return path.resolve(process.cwd(), candidate);
+}
+
+function normalizeReindexRuntimeOptions(rawOptions) {
+  const reindex = normalizeReindexOptions(rawOptions);
+  const evalEnabled = readBooleanOption(rawOptions, "eval") === true;
+  const evalFixturePath = readStringOption(rawOptions, "evalFixtures", ["eval_fixtures"]);
+  if (!evalEnabled) {
+    if (typeof evalFixturePath === "string" && evalFixturePath.trim().length > 0) {
+      throw new PmCliError("`--eval-fixtures` requires `--eval`.", EXIT_CODE.USAGE);
+    }
+    return { reindex };
+  }
+  return {
+    reindex,
+    eval: {
+      fixturesPath: normalizeEvalFixturePath(evalFixturePath),
+    },
+  };
+}
+
+function toScoreAtRank(relevance, rankIndex) {
+  if (relevance <= 0) {
+    return 0;
+  }
+  return (2 ** relevance - 1) / Math.log2(rankIndex + 2);
+}
+
+function roundMetric(value) {
+  return Math.round(value * 10_000) / 10_000;
+}
+
+function toCompactExpectedIds(value, fixtureLabel, fixturesPath) {
+  if (!Array.isArray(value) || value.length === 0) {
+    throw new PmCliError(
+      `Reindex eval fixture ${fixtureLabel} in ${fixturesPath} must provide a non-empty expected_top_ids array.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  const expectedIds = value
+    .filter((entry) => typeof entry === "string")
+    .map((entry) => entry.trim())
+    .filter((entry) => entry.length > 0);
+  if (expectedIds.length === 0) {
+    throw new PmCliError(
+      `Reindex eval fixture ${fixtureLabel} in ${fixturesPath} must provide at least one non-empty expected_top_ids value.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  return [...new Set(expectedIds)];
+}
+
+function normalizeFixtureMode(rawMode, fixtureLabel, fixturesPath) {
+  if (rawMode === undefined) {
+    return "keyword";
+  }
+  const normalized = typeof rawMode === "string" ? rawMode.trim().toLowerCase() : "";
+  if (VALID_EVAL_SEARCH_MODES.has(normalized)) {
+    return normalized;
+  }
+  throw new PmCliError(
+    `Reindex eval fixture ${fixtureLabel} in ${fixturesPath} has unsupported mode "${String(rawMode)}". Expected keyword|semantic|hybrid.`,
+    EXIT_CODE.USAGE,
+  );
+}
+
+function normalizeFixtureThreshold(rawValue, fixtureLabel, fixturesPath) {
+  if (rawValue === undefined) {
+    return DEFAULT_EVAL_MIN_NDCG_AT_5;
+  }
+  const parsed = typeof rawValue === "number" ? rawValue : Number.NaN;
+  if (!Number.isFinite(parsed) || parsed < 0 || parsed > 1) {
+    throw new PmCliError(
+      `Reindex eval fixture ${fixtureLabel} in ${fixturesPath} has invalid min_ndcg_at_5; expected a number between 0 and 1.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  return parsed;
+}
+
+function normalizeFixtureEntry(rawFixture, index, fixturesPath) {
+  if (!rawFixture || typeof rawFixture !== "object") {
+    throw new PmCliError(
+      `Reindex eval fixture at index ${index + 1} in ${fixturesPath} must be an object.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  const rawQuery = typeof rawFixture.query === "string" ? rawFixture.query.trim() : "";
+  if (rawQuery.length === 0) {
+    throw new PmCliError(
+      `Reindex eval fixture at index ${index + 1} in ${fixturesPath} must provide a non-empty query.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  const fallbackName = `fixture-${index + 1}`;
+  const fixtureName =
+    typeof rawFixture.name === "string" && rawFixture.name.trim().length > 0 ? rawFixture.name.trim() : fallbackName;
+  return {
+    name: fixtureName,
+    query: rawQuery,
+    mode: normalizeFixtureMode(rawFixture.mode, fixtureName, fixturesPath),
+    expected_top_ids: toCompactExpectedIds(rawFixture.expected_top_ids, fixtureName, fixturesPath),
+    min_ndcg_at_5: normalizeFixtureThreshold(rawFixture.min_ndcg_at_5, fixtureName, fixturesPath),
+  };
+}
+
+function normalizeFixtureCollection(parsed, fixturesPath) {
+  const parsedObject = parsed && typeof parsed === "object" ? parsed : null;
+  const candidateFixtures = Array.isArray(parsed)
+    ? parsed
+    : parsedObject && Array.isArray(parsedObject.fixtures)
+      ? parsedObject.fixtures
+      : null;
+  if (!candidateFixtures) {
+    throw new PmCliError(
+      `Reindex eval fixtures at ${fixturesPath} must be an array or an object with a fixtures array.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  if (candidateFixtures.length === 0) {
+    throw new PmCliError(`Reindex eval fixtures at ${fixturesPath} must contain at least one fixture.`, EXIT_CODE.USAGE);
+  }
+  return candidateFixtures.map((entry, index) => normalizeFixtureEntry(entry, index, fixturesPath));
+}
+
+async function loadEvalFixtures(fixturesPath) {
+  let rawText = "";
+  try {
+    rawText = await readFile(fixturesPath, "utf8");
+  } catch {
+    throw new PmCliError(`Unable to read reindex eval fixtures at ${fixturesPath}.`, EXIT_CODE.USAGE);
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(rawText);
+  } catch {
+    throw new PmCliError(`Reindex eval fixtures at ${fixturesPath} must be valid JSON.`, EXIT_CODE.USAGE);
+  }
+  return normalizeFixtureCollection(parsed, fixturesPath);
+}
+
+function computeNdcgAt5(actualIds, expectedIds) {
+  const expectedTop = expectedIds.slice(0, EVAL_RANK_CUTOFF);
+  const relevanceById = new Map();
+  for (let index = 0; index < expectedTop.length; index += 1) {
+    relevanceById.set(expectedTop[index], expectedTop.length - index);
+  }
+  const dcg = actualIds.slice(0, EVAL_RANK_CUTOFF).reduce((sum, id, index) => {
+    const relevance = relevanceById.get(id) ?? 0;
+    return sum + toScoreAtRank(relevance, index);
+  }, 0);
+  const idealDcg = expectedTop.reduce((sum, _id, index) => sum + toScoreAtRank(expectedTop.length - index, index), 0);
+  return dcg / idealDcg;
+}
+
+async function runFixtureEvaluation(fixture, global) {
+  const searchResult = await runSearch(
+    fixture.query,
+    {
+      mode: fixture.mode,
+      limit: String(EVAL_RANK_CUTOFF),
+      fields: "id,score",
+    },
+    global,
+  );
+  const actualTopIds = searchResult.items.map((item) => item.id).slice(0, EVAL_RANK_CUTOFF);
+  const ndcgAt5 = roundMetric(computeNdcgAt5(actualTopIds, fixture.expected_top_ids));
+  const minNdcgAt5 = roundMetric(fixture.min_ndcg_at_5);
+  return {
+    fixture: fixture.name,
+    query: fixture.query,
+    requested_mode: fixture.mode,
+    resolved_mode: searchResult.mode,
+    expected_top_ids: fixture.expected_top_ids,
+    actual_top_ids: actualTopIds,
+    ndcg_at_5: ndcgAt5,
+    min_ndcg_at_5: minNdcgAt5,
+    passed: ndcgAt5 >= minNdcgAt5,
+    warnings: searchResult.warnings ?? [],
+  };
+}
+
+async function runReindexEvaluation(fixturesPath, global) {
+  const fixtures = await loadEvalFixtures(fixturesPath);
+  const results = await Promise.all(fixtures.map((fixture) => runFixtureEvaluation(fixture, global)));
+  const passCount = results.filter((result) => result.passed).length;
+  const averageNdcg = roundMetric(results.reduce((sum, result) => sum + result.ndcg_at_5, 0) / results.length);
+  return {
+    enabled: true,
+    fixtures_path: fixturesPath,
+    k: EVAL_RANK_CUTOFF,
+    fixture_count: results.length,
+    pass_count: passCount,
+    fail_count: results.length - passCount,
+    average_ndcg_at_5: averageNdcg,
+    passed: passCount === results.length,
+    results,
+    evaluated_at: new Date().toISOString(),
+  };
+}
+
 export async function runAdvancedSearchPackage(args, rawOptions, global) {
   return runSearch(resolveSearchQuery(args), normalizeAdvancedSearchOptions(rawOptions, args), global);
 }
 
 export async function runAdvancedReindexPackage(rawOptions, global) {
-  return runReindex(normalizeReindexOptions(rawOptions), global);
+  const options = normalizeReindexRuntimeOptions(rawOptions);
+  const reindexResult = await runReindex(options.reindex, global);
+  if (!options.eval) {
+    return reindexResult;
+  }
+  const evalSummary = await runReindexEvaluation(options.eval.fixturesPath, global);
+  return { ...reindexResult, eval: evalSummary };
 }

package/packages/pm-search-advanced/extensions/search-advanced/runtime.ts

@@ -1,3 +1,4 @@
+import { readFile } from "node:fs/promises";
 import path from "node:path";
 import { pathToFileURL } from "node:url";
 import type {
@@ -9,6 +10,12 @@ import type {
 } from "../../../../src/sdk/runtime.js";
 
 const PM_PACKAGE_ROOT_ENV = "PM_CLI_PACKAGE_ROOT";
+const DEFAULT_EVAL_FIXTURES_PATH = path.join("tests", "search-eval", "golden-queries.json");
+const DEFAULT_EVAL_MIN_NDCG_AT_5 = 0.7;
+const EVAL_RANK_CUTOFF = 5;
+const VALID_EVAL_SEARCH_MODES = new Set(["keyword", "semantic", "hybrid"] as const);
+
+type EvalSearchMode = "keyword" | "semantic" | "hybrid";
 
 interface SearchRuntimeSdkModule {
   EXIT_CODE: {
@@ -31,6 +38,59 @@ const {
   readBooleanOption,
 } = sdk;
 
+interface SearchEvalFixtureInput {
+  query?: unknown;
+  mode?: unknown;
+  expected_top_ids?: unknown;
+  min_ndcg_at_5?: unknown;
+  name?: unknown;
+}
+
+interface SearchEvalFixture {
+  name: string;
+  query: string;
+  mode: EvalSearchMode;
+  expected_top_ids: string[];
+  min_ndcg_at_5: number;
+}
+
+interface SearchEvalResult {
+  fixture: string;
+  query: string;
+  requested_mode: EvalSearchMode;
+  resolved_mode: SearchResult["mode"];
+  expected_top_ids: string[];
+  actual_top_ids: string[];
+  ndcg_at_5: number;
+  min_ndcg_at_5: number;
+  passed: boolean;
+  warnings: string[];
+}
+
+interface ReindexEvalSummary {
+  enabled: true;
+  fixtures_path: string;
+  k: number;
+  fixture_count: number;
+  pass_count: number;
+  fail_count: number;
+  average_ndcg_at_5: number;
+  passed: boolean;
+  results: SearchEvalResult[];
+  evaluated_at: string;
+}
+
+interface ReindexResultWithEval extends ReindexResult {
+  eval?: ReindexEvalSummary;
+}
+
+interface ReindexRuntimeOptions {
+  reindex: ReindexOptions;
+  eval?: {
+    fixturesPath: string;
+  };
+}
+
 async function loadSearchSdkModule(): Promise<SearchRuntimeSdkModule> {
   const envRoot = process.env[PM_PACKAGE_ROOT_ENV];
   if (typeof envRoot !== "string" || envRoot.trim().length === 0) {
@@ -155,6 +215,209 @@ function normalizeReindexOptions(rawOptions: Record<string, unknown>): ReindexOp
   };
 }
 
+function normalizeEvalFixturePath(rawPath: string | undefined): string {
+  const candidate = typeof rawPath === "string" ? rawPath : DEFAULT_EVAL_FIXTURES_PATH;
+  return path.resolve(process.cwd(), candidate);
+}
+
+function normalizeReindexRuntimeOptions(rawOptions: Record<string, unknown>): ReindexRuntimeOptions {
+  const reindex = normalizeReindexOptions(rawOptions);
+  const evalEnabled = readBooleanOption(rawOptions, "eval") === true;
+  const evalFixturePath = readStringOption(rawOptions, "evalFixtures", ["eval_fixtures"]);
+  if (!evalEnabled) {
+    if (typeof evalFixturePath === "string" && evalFixturePath.trim().length > 0) {
+      throw new PmCliError("`--eval-fixtures` requires `--eval`.", EXIT_CODE.USAGE);
+    }
+    return { reindex };
+  }
+  return {
+    reindex,
+    eval: {
+      fixturesPath: normalizeEvalFixturePath(evalFixturePath),
+    },
+  };
+}
+
+function toScoreAtRank(relevance: number, rankIndex: number): number {
+  if (relevance <= 0) {
+    return 0;
+  }
+  return (2 ** relevance - 1) / Math.log2(rankIndex + 2);
+}
+
+function roundMetric(value: number): number {
+  return Math.round(value * 10_000) / 10_000;
+}
+
+function toCompactExpectedIds(value: unknown, fixtureLabel: string, fixturesPath: string): string[] {
+  if (!Array.isArray(value) || value.length === 0) {
+    throw new PmCliError(
+      `Reindex eval fixture ${fixtureLabel} in ${fixturesPath} must provide a non-empty expected_top_ids array.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  const expectedIds = value
+    .filter((entry): entry is string => typeof entry === "string")
+    .map((entry) => entry.trim())
+    .filter((entry) => entry.length > 0);
+  if (expectedIds.length === 0) {
+    throw new PmCliError(
+      `Reindex eval fixture ${fixtureLabel} in ${fixturesPath} must provide at least one non-empty expected_top_ids value.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  return [...new Set(expectedIds)];
+}
+
+function normalizeFixtureMode(rawMode: unknown, fixtureLabel: string, fixturesPath: string): EvalSearchMode {
+  if (rawMode === undefined) {
+    return "keyword";
+  }
+  const normalized = typeof rawMode === "string" ? rawMode.trim().toLowerCase() : "";
+  if (VALID_EVAL_SEARCH_MODES.has(normalized as EvalSearchMode)) {
+    return normalized as EvalSearchMode;
+  }
+  throw new PmCliError(
+    `Reindex eval fixture ${fixtureLabel} in ${fixturesPath} has unsupported mode "${String(rawMode)}". Expected keyword|semantic|hybrid.`,
+    EXIT_CODE.USAGE,
+  );
+}
+
+function normalizeFixtureThreshold(rawValue: unknown, fixtureLabel: string, fixturesPath: string): number {
+  if (rawValue === undefined) {
+    return DEFAULT_EVAL_MIN_NDCG_AT_5;
+  }
+  const parsed = typeof rawValue === "number" ? rawValue : Number.NaN;
+  if (!Number.isFinite(parsed) || parsed < 0 || parsed > 1) {
+    throw new PmCliError(
+      `Reindex eval fixture ${fixtureLabel} in ${fixturesPath} has invalid min_ndcg_at_5; expected a number between 0 and 1.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  return parsed;
+}
+
+function normalizeFixtureEntry(rawFixture: unknown, index: number, fixturesPath: string): SearchEvalFixture {
+  if (!rawFixture || typeof rawFixture !== "object") {
+    throw new PmCliError(
+      `Reindex eval fixture at index ${index + 1} in ${fixturesPath} must be an object.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  const fixture = rawFixture as SearchEvalFixtureInput;
+  const rawQuery = typeof fixture.query === "string" ? fixture.query.trim() : "";
+  if (rawQuery.length === 0) {
+    throw new PmCliError(
+      `Reindex eval fixture at index ${index + 1} in ${fixturesPath} must provide a non-empty query.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  const fallbackName = `fixture-${index + 1}`;
+  const fixtureName = typeof fixture.name === "string" && fixture.name.trim().length > 0 ? fixture.name.trim() : fallbackName;
+  return {
+    name: fixtureName,
+    query: rawQuery,
+    mode: normalizeFixtureMode(fixture.mode, fixtureName, fixturesPath),
+    expected_top_ids: toCompactExpectedIds(fixture.expected_top_ids, fixtureName, fixturesPath),
+    min_ndcg_at_5: normalizeFixtureThreshold(fixture.min_ndcg_at_5, fixtureName, fixturesPath),
+  };
+}
+
+function normalizeFixtureCollection(parsed: unknown, fixturesPath: string): SearchEvalFixture[] {
+  const parsedObject = parsed as { fixtures?: unknown } | null;
+  const candidateFixtures = Array.isArray(parsed)
+    ? parsed
+    : parsedObject && Array.isArray(parsedObject.fixtures)
+      ? parsedObject.fixtures
+      : null;
+  if (!candidateFixtures) {
+    throw new PmCliError(
+      `Reindex eval fixtures at ${fixturesPath} must be an array or an object with a fixtures array.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  if (candidateFixtures.length === 0) {
+    throw new PmCliError(`Reindex eval fixtures at ${fixturesPath} must contain at least one fixture.`, EXIT_CODE.USAGE);
+  }
+  return candidateFixtures.map((entry, index) => normalizeFixtureEntry(entry, index, fixturesPath));
+}
+
+async function loadEvalFixtures(fixturesPath: string): Promise<SearchEvalFixture[]> {
+  let rawText = "";
+  try {
+    rawText = await readFile(fixturesPath, "utf8");
+  } catch {
+    throw new PmCliError(`Unable to read reindex eval fixtures at ${fixturesPath}.`, EXIT_CODE.USAGE);
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(rawText);
+  } catch {
+    throw new PmCliError(`Reindex eval fixtures at ${fixturesPath} must be valid JSON.`, EXIT_CODE.USAGE);
+  }
+  return normalizeFixtureCollection(parsed, fixturesPath);
+}
+
+function computeNdcgAt5(actualIds: readonly string[], expectedIds: readonly string[]): number {
+  const expectedTop = expectedIds.slice(0, EVAL_RANK_CUTOFF);
+  const relevanceById = new Map<string, number>();
+  for (let index = 0; index < expectedTop.length; index += 1) {
+    relevanceById.set(expectedTop[index], expectedTop.length - index);
+  }
+  const dcg = actualIds.slice(0, EVAL_RANK_CUTOFF).reduce((sum, id, index) => {
+    const relevance = relevanceById.get(id) ?? 0;
+    return sum + toScoreAtRank(relevance, index);
+  }, 0);
+  const idealDcg = expectedTop.reduce((sum, _id, index) => sum + toScoreAtRank(expectedTop.length - index, index), 0);
+  return dcg / idealDcg;
+}
+
+async function runFixtureEvaluation(fixture: SearchEvalFixture, global: GlobalOptions): Promise<SearchEvalResult> {
+  const searchResult = await runSearch(
+    fixture.query,
+    {
+      mode: fixture.mode,
+      limit: String(EVAL_RANK_CUTOFF),
+      fields: "id,score",
+    },
+    global,
+  );
+  const actualTopIds = searchResult.items.map((item) => (item as { id: string }).id).slice(0, EVAL_RANK_CUTOFF);
+  const ndcgAt5 = roundMetric(computeNdcgAt5(actualTopIds, fixture.expected_top_ids));
+  const minNdcgAt5 = roundMetric(fixture.min_ndcg_at_5);
+  return {
+    fixture: fixture.name,
+    query: fixture.query,
+    requested_mode: fixture.mode,
+    resolved_mode: searchResult.mode,
+    expected_top_ids: fixture.expected_top_ids,
+    actual_top_ids: actualTopIds,
+    ndcg_at_5: ndcgAt5,
+    min_ndcg_at_5: minNdcgAt5,
+    passed: ndcgAt5 >= minNdcgAt5,
+    warnings: searchResult.warnings ?? [],
+  };
+}
+
+async function runReindexEvaluation(fixturesPath: string, global: GlobalOptions): Promise<ReindexEvalSummary> {
+  const fixtures = await loadEvalFixtures(fixturesPath);
+  const results: SearchEvalResult[] = await Promise.all(fixtures.map((fixture) => runFixtureEvaluation(fixture, global)));
+  const passCount = results.filter((result) => result.passed).length;
+  const averageNdcg = roundMetric(results.reduce((sum, result) => sum + result.ndcg_at_5, 0) / results.length);
+  return {
+    enabled: true,
+    fixtures_path: fixturesPath,
+    k: EVAL_RANK_CUTOFF,
+    fixture_count: results.length,
+    pass_count: passCount,
+    fail_count: results.length - passCount,
+    average_ndcg_at_5: averageNdcg,
+    passed: passCount === results.length,
+    results,
+    evaluated_at: new Date().toISOString(),
+  };
+}
+
 export async function runAdvancedSearchPackage(
   args: string[],
   rawOptions: Record<string, unknown>,
@@ -166,6 +429,12 @@ export async function runAdvancedSearchPackage(
 export async function runAdvancedReindexPackage(
   rawOptions: Record<string, unknown>,
   global: GlobalOptions,
-): Promise<ReindexResult> {
-  return runReindex(normalizeReindexOptions(rawOptions), global);
+): Promise<ReindexResultWithEval> {
+  const options = normalizeReindexRuntimeOptions(rawOptions);
+  const reindexResult = await runReindex(options.reindex, global);
+  if (!options.eval) {
+    return reindexResult;
+  }
+  const evalSummary = await runReindexEvaluation(options.eval.fixturesPath, global);
+  return { ...reindexResult, eval: evalSummary };
 }

package/plugins/pm-claude/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "pm-claude",
-  "description": "Native pm CLI integration for Claude Code — 18 MCP tools, 5 workflow skills, 14 slash commands, 3 subagents (coordinator, triage, verification), hybrid TUI task tracking (pm as persistent store + Claude Code task panel as live view), session context injection, and full git-based project management without leaving Claude Code.",
+  "description": "Native pm CLI integration for Claude Code — 21 MCP tools, 5 workflow skills, 14 slash commands, 3 subagents (coordinator, triage, verification), hybrid TUI task tracking (pm as persistent store + Claude Code task panel as live view), session context injection, and full git-based project management without leaving Claude Code.",
   "version": "1.4.0",
   "author": {
     "name": "unbrained",