@unbrained/pm-cli 2026.6.6 → 2026.6.8

package/packages/pm-governance-audit/extensions/governance-audit/index.js

@@ -1,3 +1,5 @@
+import { appendFileSync, mkdirSync } from "node:fs";
+import path from "node:path";
 import {
   runCommentsAuditPackage,
   runDedupeAuditPackage,
@@ -9,9 +11,12 @@ export const manifest = {
   version: "0.1.0",
   entry: "./index.js",
   priority: 0,
-  capabilities: ["commands", "schema"],
+  capabilities: ["commands", "schema", "hooks"],
 };
 
+const HOOK_LOG_ENV = "PM_GOVERNANCE_AUDIT_HOOK_LOG";
+const createdHookLogDirs = new Set();
+
 const dedupeAuditFlags = [
   { long: "--mode", value_name: "value", value_type: "string", description: "Audit mode: title_exact|title_fuzzy|parent_scope." },
   { long: "--status", value_name: "value", value_type: "string", description: "Filter by status." },
@@ -105,10 +110,43 @@ function normalizeCommand() {
   };
 }
 
+function appendHookAuditRecord(kind, context) {
+  const logPath = process.env[HOOK_LOG_ENV]?.trim();
+  if (!logPath) {
+    return;
+  }
+  try {
+    const absoluteLogPath = path.resolve(logPath);
+    const logDir = path.dirname(absoluteLogPath);
+    if (!createdHookLogDirs.has(logDir)) {
+      mkdirSync(logDir, { recursive: true });
+      createdHookLogDirs.add(logDir);
+    }
+    const writeContext = kind === "on_write" ? context : undefined;
+    appendFileSync(
+      absoluteLogPath,
+      `${JSON.stringify({
+        kind,
+        path: context.path,
+        scope: context.scope,
+        op: writeContext?.op,
+        item_id: writeContext?.item_id,
+        item_type: writeContext?.item_type,
+        changed_fields: writeContext?.changed_fields,
+      })}\n`,
+      "utf8",
+    );
+  } catch {
+    // Best-effort sidecar logging must not interrupt core read/write flows.
+  }
+}
+
 export function activate(api) {
   api.registerCommand(dedupeAuditCommand());
   api.registerCommand(commentsAuditCommand());
   api.registerCommand(normalizeCommand());
+  api.hooks.onRead((context) => appendHookAuditRecord("on_read", context));
+  api.hooks.onWrite((context) => appendHookAuditRecord("on_write", context));
 }
 
 export default {

package/packages/pm-governance-audit/extensions/governance-audit/index.ts

@@ -1,4 +1,6 @@
-import type { CommandDefinition, ExtensionApi } from "../../../../src/sdk/index.js";
+import { appendFileSync, mkdirSync } from "node:fs";
+import path from "node:path";
+import type { CommandDefinition, ExtensionApi, OnReadHookContext, OnWriteHookContext } from "../../../../src/sdk/index.js";
 import {
   runCommentsAuditPackage,
   runDedupeAuditPackage,
@@ -10,9 +12,12 @@ export const manifest = {
   version: "0.1.0",
   entry: "./index.js",
   priority: 0,
-  capabilities: ["commands", "schema"],
+  capabilities: ["commands", "schema", "hooks"],
 };
 
+const HOOK_LOG_ENV = "PM_GOVERNANCE_AUDIT_HOOK_LOG";
+const createdHookLogDirs = new Set<string>();
+
 const dedupeAuditFlags = [
   { long: "--mode", value_name: "value", value_type: "string", description: "Audit mode: title_exact|title_fuzzy|parent_scope." },
   { long: "--status", value_name: "value", value_type: "string", description: "Filter by status." },
@@ -106,10 +111,43 @@ function normalizeCommand(): CommandDefinition {
   };
 }
 
+function appendHookAuditRecord(kind: "on_read" | "on_write", context: OnReadHookContext | OnWriteHookContext): void {
+  const logPath = process.env[HOOK_LOG_ENV]?.trim();
+  if (!logPath) {
+    return;
+  }
+  try {
+    const absoluteLogPath = path.resolve(logPath);
+    const logDir = path.dirname(absoluteLogPath);
+    if (!createdHookLogDirs.has(logDir)) {
+      mkdirSync(logDir, { recursive: true });
+      createdHookLogDirs.add(logDir);
+    }
+    const writeContext = kind === "on_write" ? (context as OnWriteHookContext) : undefined;
+    appendFileSync(
+      absoluteLogPath,
+      `${JSON.stringify({
+        kind,
+        path: context.path,
+        scope: context.scope,
+        op: writeContext?.op,
+        item_id: writeContext?.item_id,
+        item_type: writeContext?.item_type,
+        changed_fields: writeContext?.changed_fields,
+      })}\n`,
+      "utf8",
+    );
+  } catch {
+    // Best-effort sidecar logging must not interrupt core read/write flows.
+  }
+}
+
 export function activate(api: ExtensionApi): void {
   api.registerCommand(dedupeAuditCommand());
   api.registerCommand(commentsAuditCommand());
   api.registerCommand(normalizeCommand());
+  api.hooks.onRead((context) => appendHookAuditRecord("on_read", context));
+  api.hooks.onWrite((context) => appendHookAuditRecord("on_write", context));
 }
 
 export default {

package/packages/pm-governance-audit/extensions/governance-audit/manifest.json

@@ -3,7 +3,7 @@
   "version": "0.1.0",
   "entry": "./index.js",
   "priority": 0,
-  "capabilities": ["commands", "schema"],
+  "capabilities": ["commands", "schema", "hooks"],
   "manifest_version": 1,
   "pm_min_version": "2026.5.0",
   "activation": {

package/packages/pm-governance-audit/package.json

@@ -18,7 +18,8 @@
     "project-management",
     "governance",
     "audit",
-    "normalize"
+    "normalize",
+    "hooks"
   ],
   "pm": {
     "aliases": [
@@ -34,7 +35,8 @@
       "tags": [
         "governance",
         "audit",
-        "normalize"
+        "normalize",
+        "hooks"
       ],
       "links": {
         "docs": "https://github.com/unbraind/pm-cli/tree/main/packages/pm-governance-audit#readme",

package/packages/pm-lifecycle-hooks/README.md

@@ -11,6 +11,27 @@ Transition-aware packages can inspect `afterCommand`'s optional `affected`
 entries for item mutations. Each entry includes the item id, operation,
 previous/current status, changed fields, and compact front matter snapshots.
 
+Copy this pattern when a package needs command-level notifications without
+parsing command results or reading item files:
+
+```ts
+api.hooks.afterCommand((event) => {
+  const transitions = event.affected?.filter(
+    (item) => item.previous_status !== item.status && item.status !== undefined,
+  );
+
+  if (!transitions?.length) {
+    return;
+  }
+
+  // Notify, enqueue, or cache based on item.id, item.previous_status,
+  // item.status, item.changed_fields, and item.current.
+});
+```
+
+Use `afterCommand` for command-level context such as notifications. Use
+`onWrite` when a package needs file-level sync or audit behavior.
+
 ## Capabilities
 
 - `hooks`

package/packages/pm-search-advanced/README.md

@@ -16,6 +16,8 @@ First-party package that restores optional advanced search surfaces in bare-core
 - Adds `pm reindex` with:
   - `--mode keyword|semantic|hybrid`
   - `--progress`
+  - `--eval`
+  - `--eval-fixtures <path>`
 
 ## Install
 
@@ -30,6 +32,42 @@ pm search-advanced "vector cache" --mode hybrid --limit 5 --json
 pm search-advanced --hybrid "vector cache" --limit 5 --json
 pm search-advanced "calendar package" --mode keyword --fields id,title,score --compact --json
 pm reindex --mode hybrid --progress --json
+pm reindex --mode keyword --eval --eval-fixtures tests/search-eval/golden-queries.json --json
 ```
 
 Without `--mode`, `--semantic`, or `--hybrid`, `search-advanced` stays keyword-first for fast agent reads.
+
+## Reindex eval harness
+
+`pm reindex --eval` runs a golden-query relevance harness after the reindex pass and appends a deterministic `eval` object to JSON output:
+
+- per-fixture pass/fail verdicts
+- `ndcg_at_5` score for each fixture
+- aggregate `average_ndcg_at_5`, pass/fail counts, and overall status
+
+The default fixture file is `tests/search-eval/golden-queries.json`. Override with `--eval-fixtures <path>`.
+
+### Fixture schema
+
+Fixtures can be either a top-level array or an object with a `fixtures` array:
+
+```json
+{
+  "fixtures": [
+    {
+      "name": "search-eval-task",
+      "query": "pm-22x2 search relevance evaluation harness",
+      "mode": "keyword",
+      "expected_top_ids": ["pm-22x2", "pm-fhsg"],
+      "min_ndcg_at_5": 0.8
+    }
+  ]
+}
+```
+
+Rules:
+
+- `query` is required and must be non-empty.
+- `mode` is optional (`keyword`, `semantic`, `hybrid`), default `keyword`.
+- `expected_top_ids` is required and defines the ideal ranking for nDCG@5.
+- `min_ndcg_at_5` is optional (`0..1`), default `0.7`.

package/packages/pm-search-advanced/extensions/search-advanced/index.js

@@ -200,6 +200,23 @@ const reindexFlags = [
     value_type: "boolean",
     description: "Emit non-interactive progress lines to stderr.",
   },
+  {
+    long: "--eval",
+    value_type: "boolean",
+    description: "Run golden-query relevance eval and append nDCG@5 summary output.",
+  },
+  {
+    long: "--eval-fixtures",
+    value_name: "path",
+    value_type: "string",
+    description: "Path to reindex eval fixtures JSON (default: tests/search-eval/golden-queries.json).",
+  },
+  {
+    long: "--eval_fixtures",
+    value_name: "path",
+    value_type: "string",
+    description: "Alias for --eval-fixtures.",
+  },
 ];
 
 function searchAdvancedCommand() {

package/packages/pm-search-advanced/extensions/search-advanced/index.ts

@@ -222,6 +222,23 @@ const reindexFlags = [
     value_type: "boolean",
     description: "Emit non-interactive progress lines to stderr.",
   },
+  {
+    long: "--eval",
+    value_type: "boolean",
+    description: "Run golden-query relevance eval and append nDCG@5 summary output.",
+  },
+  {
+    long: "--eval-fixtures",
+    value_name: "path",
+    value_type: "string",
+    description: "Path to reindex eval fixtures JSON (default: tests/search-eval/golden-queries.json).",
+  },
+  {
+    long: "--eval_fixtures",
+    value_name: "path",
+    value_type: "string",
+    description: "Alias for --eval-fixtures.",
+  },
 ] as const;
 
 function searchAdvancedCommand(): CommandDefinition {

package/packages/pm-search-advanced/extensions/search-advanced/runtime.js

@@ -1,7 +1,13 @@
+import { readFile } from "node:fs/promises";
 import path from "node:path";
 import { pathToFileURL } from "node:url";
 
 const PM_PACKAGE_ROOT_ENV = "PM_CLI_PACKAGE_ROOT";
+const DEFAULT_EVAL_FIXTURES_PATH = path.join("tests", "search-eval", "golden-queries.json");
+const DEFAULT_EVAL_MIN_NDCG_AT_5 = 0.7;
+const EVAL_RANK_CUTOFF = 5;
+const VALID_EVAL_SEARCH_MODES = new Set(["keyword", "semantic", "hybrid"]);
+
 const sdk = await loadSearchSdkModule();
 const {
   EXIT_CODE,
@@ -136,10 +142,219 @@ function normalizeReindexOptions(rawOptions) {
   };
 }
 
+function normalizeEvalFixturePath(rawPath) {
+  const candidate = typeof rawPath === "string" ? rawPath : DEFAULT_EVAL_FIXTURES_PATH;
+  return path.resolve(process.cwd(), candidate);
+}
+
+function normalizeReindexRuntimeOptions(rawOptions) {
+  const reindex = normalizeReindexOptions(rawOptions);
+  const evalEnabled = readBooleanOption(rawOptions, "eval") === true;
+  const evalFixturePath = readStringOption(rawOptions, "evalFixtures", ["eval_fixtures"]);
+  if (!evalEnabled) {
+    if (typeof evalFixturePath === "string" && evalFixturePath.trim().length > 0) {
+      throw new PmCliError("`--eval-fixtures` requires `--eval`.", EXIT_CODE.USAGE);
+    }
+    return { reindex };
+  }
+  return {
+    reindex,
+    eval: {
+      fixturesPath: normalizeEvalFixturePath(evalFixturePath),
+    },
+  };
+}
+
+function toScoreAtRank(relevance, rankIndex) {
+  if (relevance <= 0) {
+    return 0;
+  }
+  return (2 ** relevance - 1) / Math.log2(rankIndex + 2);
+}
+
+function roundMetric(value) {
+  return Math.round(value * 10_000) / 10_000;
+}
+
+function toCompactExpectedIds(value, fixtureLabel, fixturesPath) {
+  if (!Array.isArray(value) || value.length === 0) {
+    throw new PmCliError(
+      `Reindex eval fixture ${fixtureLabel} in ${fixturesPath} must provide a non-empty expected_top_ids array.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  const expectedIds = value
+    .filter((entry) => typeof entry === "string")
+    .map((entry) => entry.trim())
+    .filter((entry) => entry.length > 0);
+  if (expectedIds.length === 0) {
+    throw new PmCliError(
+      `Reindex eval fixture ${fixtureLabel} in ${fixturesPath} must provide at least one non-empty expected_top_ids value.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  return [...new Set(expectedIds)];
+}
+
+function normalizeFixtureMode(rawMode, fixtureLabel, fixturesPath) {
+  if (rawMode === undefined) {
+    return "keyword";
+  }
+  const normalized = typeof rawMode === "string" ? rawMode.trim().toLowerCase() : "";
+  if (VALID_EVAL_SEARCH_MODES.has(normalized)) {
+    return normalized;
+  }
+  throw new PmCliError(
+    `Reindex eval fixture ${fixtureLabel} in ${fixturesPath} has unsupported mode "${String(rawMode)}". Expected keyword|semantic|hybrid.`,
+    EXIT_CODE.USAGE,
+  );
+}
+
+function normalizeFixtureThreshold(rawValue, fixtureLabel, fixturesPath) {
+  if (rawValue === undefined) {
+    return DEFAULT_EVAL_MIN_NDCG_AT_5;
+  }
+  const parsed = typeof rawValue === "number" ? rawValue : Number.NaN;
+  if (!Number.isFinite(parsed) || parsed < 0 || parsed > 1) {
+    throw new PmCliError(
+      `Reindex eval fixture ${fixtureLabel} in ${fixturesPath} has invalid min_ndcg_at_5; expected a number between 0 and 1.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  return parsed;
+}
+
+function normalizeFixtureEntry(rawFixture, index, fixturesPath) {
+  if (!rawFixture || typeof rawFixture !== "object") {
+    throw new PmCliError(
+      `Reindex eval fixture at index ${index + 1} in ${fixturesPath} must be an object.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  const rawQuery = typeof rawFixture.query === "string" ? rawFixture.query.trim() : "";
+  if (rawQuery.length === 0) {
+    throw new PmCliError(
+      `Reindex eval fixture at index ${index + 1} in ${fixturesPath} must provide a non-empty query.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  const fallbackName = `fixture-${index + 1}`;
+  const fixtureName =
+    typeof rawFixture.name === "string" && rawFixture.name.trim().length > 0 ? rawFixture.name.trim() : fallbackName;
+  return {
+    name: fixtureName,
+    query: rawQuery,
+    mode: normalizeFixtureMode(rawFixture.mode, fixtureName, fixturesPath),
+    expected_top_ids: toCompactExpectedIds(rawFixture.expected_top_ids, fixtureName, fixturesPath),
+    min_ndcg_at_5: normalizeFixtureThreshold(rawFixture.min_ndcg_at_5, fixtureName, fixturesPath),
+  };
+}
+
+function normalizeFixtureCollection(parsed, fixturesPath) {
+  const parsedObject = parsed && typeof parsed === "object" ? parsed : null;
+  const candidateFixtures = Array.isArray(parsed)
+    ? parsed
+    : parsedObject && Array.isArray(parsedObject.fixtures)
+      ? parsedObject.fixtures
+      : null;
+  if (!candidateFixtures) {
+    throw new PmCliError(
+      `Reindex eval fixtures at ${fixturesPath} must be an array or an object with a fixtures array.`,
+      EXIT_CODE.USAGE,
+    );
+  }
+  if (candidateFixtures.length === 0) {
+    throw new PmCliError(`Reindex eval fixtures at ${fixturesPath} must contain at least one fixture.`, EXIT_CODE.USAGE);
+  }
+  return candidateFixtures.map((entry, index) => normalizeFixtureEntry(entry, index, fixturesPath));
+}
+
+async function loadEvalFixtures(fixturesPath) {
+  let rawText = "";
+  try {
+    rawText = await readFile(fixturesPath, "utf8");
+  } catch {
+    throw new PmCliError(`Unable to read reindex eval fixtures at ${fixturesPath}.`, EXIT_CODE.USAGE);
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(rawText);
+  } catch {
+    throw new PmCliError(`Reindex eval fixtures at ${fixturesPath} must be valid JSON.`, EXIT_CODE.USAGE);
+  }
+  return normalizeFixtureCollection(parsed, fixturesPath);
+}
+
+function computeNdcgAt5(actualIds, expectedIds) {
+  const expectedTop = expectedIds.slice(0, EVAL_RANK_CUTOFF);
+  const relevanceById = new Map();
+  for (let index = 0; index < expectedTop.length; index += 1) {
+    relevanceById.set(expectedTop[index], expectedTop.length - index);
+  }
+  const dcg = actualIds.slice(0, EVAL_RANK_CUTOFF).reduce((sum, id, index) => {
+    const relevance = relevanceById.get(id) ?? 0;
+    return sum + toScoreAtRank(relevance, index);
+  }, 0);
+  const idealDcg = expectedTop.reduce((sum, _id, index) => sum + toScoreAtRank(expectedTop.length - index, index), 0);
+  return dcg / idealDcg;
+}
+
+async function runFixtureEvaluation(fixture, global) {
+  const searchResult = await runSearch(
+    fixture.query,
+    {
+      mode: fixture.mode,
+      limit: String(EVAL_RANK_CUTOFF),
+      fields: "id,score",
+    },
+    global,
+  );
+  const actualTopIds = searchResult.items.map((item) => item.id).slice(0, EVAL_RANK_CUTOFF);
+  const ndcgAt5 = roundMetric(computeNdcgAt5(actualTopIds, fixture.expected_top_ids));
+  const minNdcgAt5 = roundMetric(fixture.min_ndcg_at_5);
+  return {
+    fixture: fixture.name,
+    query: fixture.query,
+    requested_mode: fixture.mode,
+    resolved_mode: searchResult.mode,
+    expected_top_ids: fixture.expected_top_ids,
+    actual_top_ids: actualTopIds,
+    ndcg_at_5: ndcgAt5,
+    min_ndcg_at_5: minNdcgAt5,
+    passed: ndcgAt5 >= minNdcgAt5,
+    warnings: searchResult.warnings ?? [],
+  };
+}
+
+async function runReindexEvaluation(fixturesPath, global) {
+  const fixtures = await loadEvalFixtures(fixturesPath);
+  const results = await Promise.all(fixtures.map((fixture) => runFixtureEvaluation(fixture, global)));
+  const passCount = results.filter((result) => result.passed).length;
+  const averageNdcg = roundMetric(results.reduce((sum, result) => sum + result.ndcg_at_5, 0) / results.length);
+  return {
+    enabled: true,
+    fixtures_path: fixturesPath,
+    k: EVAL_RANK_CUTOFF,
+    fixture_count: results.length,
+    pass_count: passCount,
+    fail_count: results.length - passCount,
+    average_ndcg_at_5: averageNdcg,
+    passed: passCount === results.length,
+    results,
+    evaluated_at: new Date().toISOString(),
+  };
+}
+
 export async function runAdvancedSearchPackage(args, rawOptions, global) {
   return runSearch(resolveSearchQuery(args), normalizeAdvancedSearchOptions(rawOptions, args), global);
 }
 
 export async function runAdvancedReindexPackage(rawOptions, global) {
-  return runReindex(normalizeReindexOptions(rawOptions), global);
+  const options = normalizeReindexRuntimeOptions(rawOptions);
+  const reindexResult = await runReindex(options.reindex, global);
+  if (!options.eval) {
+    return reindexResult;
+  }
+  const evalSummary = await runReindexEvaluation(options.eval.fixturesPath, global);
+  return { ...reindexResult, eval: evalSummary };
 }