@dzhechkov/harness-core 0.8.34 → 0.8.35

package/src/skills.ts

@@ -4,8 +4,8 @@
  * @packageDocumentation
  */
 
-import { existsSync, readdirSync, readFileSync } from 'node:fs';
-import { join, relative } from 'node:path';
+import { existsSync, lstatSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs';
+import { isAbsolute, join, relative, sep } from 'node:path';
 
 import { parse as parseYaml } from 'yaml';
 
@@ -55,15 +55,259 @@ function readAssetContent(path: string): { encoding: 'utf-8' | 'base64'; content
   return { encoding: 'utf-8', content: decoded };
 }
 
-/** Recursively list every file under `dir`. */
-function walkFiles(dir: string): string[] {
-  const out: string[] = [];
-  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+/**
+ * One filesystem entry skipped during skill asset discovery, named so nothing
+ * vanishes silently (feature `skills-walk-symlinks-and-junk`, FR-1/FR-2). `path`
+ * is the entry's own path (not its symlink target); `reason` is a short,
+ * stable, human-readable tag: `'junk file (<pattern>)'` (e.g. `'junk file
+ * (*.pyc)'` — fix-round 1 HIGH-1(b): the pattern that matched, not just the
+ * verdict), `'junk directory (<name>)'`, `'broken symlink'`, `'symlink escapes
+ * the skill directory'` (fix-round 1 AM-8), `'symlink cycle — directory
+ * already visited'`, or `'unreadable directory (<errno message>)'` (fix-round
+ * 1 MEDIUM-3).
+ */
+export interface SkippedEntry {
+  readonly path: string;
+  readonly reason: string;
+}
+
+/** The result of {@link walkFiles}: the real assets found, plus everything skipped. */
+export interface SkillWalkResult {
+  readonly files: readonly string[];
+  readonly skipped: readonly SkippedEntry[];
+}
+
+/**
+ * Directory names that never carry legitimate skill assets — build/cache artifacts a
+ * skill author does not intend to ship. **This list IS the published contract for "what
+ * counts as junk"** (fix-round 1 HIGH-1(a), REFUTED-by-contract — see
+ * `packages/@dzhechkov/harness-core/README.md`, "What counts as junk"): a skill cannot
+ * ship a directory bearing one of these exact names as an asset, on purpose or by
+ * accident — that is the deliberate, documented trade the design makes, not an
+ * oversight to be widened into content-sniffing heuristics. NOT a whitelist of allowed
+ * directories: any OTHER name (including a dotdir the author added on purpose) is
+ * walked as usual — filtering a user's own files is not this list's job (FR-2,
+ * requirements AC-1).
+ */
+export const SKILL_JUNK_DIRS: ReadonlySet<string> = new Set([
+  '__pycache__', // Python bytecode cache
+  'node_modules', // an accidentally-vendored dependency tree
+  '.git', // VCS metadata
+  '__MACOSX', // macOS zip-archive resource-fork sidecar directory (fix-round 1 MEDIUM-2)
+  '.pytest_cache', // pytest's cache directory (fix-round 1 MEDIUM-2)
+  '.mypy_cache', // mypy's cache directory (fix-round 1 MEDIUM-2)
+]);
+
+/** Exact junk filenames skipped during skill asset discovery (FR-2). */
+export const SKILL_JUNK_FILES: ReadonlySet<string> = new Set([
+  '.DS_Store', // macOS Finder folder metadata
+  'Thumbs.db', // Windows Explorer thumbnail cache
+]);
+
+/** Junk filename SUFFIXES skipped during skill asset discovery (FR-2). */
+const SKILL_JUNK_FILE_SUFFIXES: readonly string[] = [
+  '.pyc', // Python bytecode
+  '.pyo', // Python optimized bytecode
+  '.swp', // Vim swap file
+  '.swo', // Vim swap file, second form left after a crash recovery (fix-round 1 MEDIUM-2)
+  // A trailing `~` (editor backup) is deliberately NOT on this list (Codex r2 HIGH, lead
+  // decision): it is the one pattern a legitimate asset name can plausibly end with
+  // (`notes~`), and FR-2's list is conservative by contract — a false positive here would
+  // silently drop a real asset, which AC-1 forbids. Backup files ending in `~` ship as assets.
+];
+
+/** Junk filename PREFIXES skipped during skill asset discovery (fix-round 1 MEDIUM-2). */
+const SKILL_JUNK_FILE_PREFIXES: readonly string[] = [
+  '.#', // Emacs lock file (e.g. `.#notes.txt`), left behind by an unclean editor exit
+];
+
+/** True when `name` matches a documented junk-file pattern (FR-2). */
+export function isSkillJunkFile(name: string): boolean {
+  if (SKILL_JUNK_FILES.has(name)) return true;
+  if (SKILL_JUNK_FILE_SUFFIXES.some((suffix) => name.endsWith(suffix))) return true;
+  return SKILL_JUNK_FILE_PREFIXES.some((prefix) => name.startsWith(prefix));
+}
+
+/**
+ * The ONE decision point `walkFiles` calls to ask "is this entry junk?" — a directory
+ * check and a file check both funnel through here so a single mutation can prove (or
+ * disprove) that junk filtering, as a whole, is wired in (registry entry
+ * `walk-filters-junk`). Splitting this into two never-both-mutated call sites would let
+ * a mutation of just one half pass unnoticed while the other half still filtered.
+ */
+function isJunkEntry(name: string, kind: 'file' | 'directory'): boolean {
+  return kind === 'directory' ? SKILL_JUNK_DIRS.has(name) : isSkillJunkFile(name);
+}
+
+/**
+ * Which junk PATTERN `name` matched — used to build a `reason` that NAMES the pattern,
+ * not just the verdict (fix-round 1 HIGH-1(b): `'junk file (*.pyc)'`, never a bare
+ * `'junk file'`). Kept separate from {@link isJunkEntry} (the one go/no-go point the
+ * `walk-filters-junk` mutation targets) so a mutation of the decision does not also
+ * have to fake this label to stay silent.
+ */
+function junkPatternLabel(name: string, kind: 'file' | 'directory'): string {
+  if (kind === 'directory') return name;
+  if (SKILL_JUNK_FILES.has(name)) return name;
+  for (const suffix of SKILL_JUNK_FILE_SUFFIXES) {
+    if (name.endsWith(suffix)) return `*${suffix}`;
+  }
+  for (const prefix of SKILL_JUNK_FILE_PREFIXES) {
+    if (name.startsWith(prefix)) return `${prefix}*`;
+  }
+  return name; // unreachable when isJunkEntry(name, kind) is true; kept total, not partial.
+}
+
+/**
+ * True when `targetRealPath` lies within `rootRealDir` — refuses a symlink whose
+ * resolved target escapes the skill directory (lead item AM-8: `assets/secret ->
+ * /etc/hostname`, or `-> ../../..`, must never be followed and bundled as an asset).
+ * Uses `relative()` plus a leading-`..`/absolute check rather than a lexical
+ * `startsWith`, the SAME idiom as `containedUnderRoot` in
+ * `packages/@dzhechkov/harness-cli/src/cli.ts` (read-only reference for technique) —
+ * a lexical string-prefix check is fooled by a sibling directory that happens to share
+ * the root as a text prefix (root `/a/b` vs. target `/a/bc`).
+ */
+function isContained(rootRealDir: string, targetRealPath: string): boolean {
+  if (targetRealPath === rootRealDir) return true;
+  const rel = relative(rootRealDir, targetRealPath);
+  // Codex r2 MEDIUM (lead fix): only a `..` COMPONENT escapes — a file legitimately named
+  // `..asset` yields rel === '..asset', which is inside the root.
+  return rel !== '' && rel !== '..' && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
+}
+
+/**
+ * Recursively list every real, non-junk file under `dir`, resolving symlinks to
+ * their targets and guarding against symlink cycles.
+ *
+ * A `Dirent` from `readdirSync` answers `false` to BOTH `isDirectory()` and
+ * `isFile()` for a symlink entry — trusting those two checks alone silently drops
+ * every symlinked asset (MEASURED: 2 of 4 fixture assets vanished, exit 0). This
+ * resolves each symlink with `statSync` (which follows the link) before deciding
+ * whether it names a file or a directory.
+ *
+ * Cycle guard: each directory's `realpath` is recorded in `seenRealDirs` while
+ * it is being walked and REMOVED again when its walk returns — the set is the
+ * chain of ANCESTORS on the current recursion path, not every directory ever
+ * visited. A directory (reached directly or through a symlink) whose real path is
+ * already on that chain ends the walk there instead of recursing — this is what
+ * stops `a -> ..` from hanging (AC-2; registry entry `walk-guards-cycles`). Two
+ * non-cyclic aliases of the same directory (`alias1 -> shared`, `alias2 -> shared`)
+ * are BOTH walked under their own logical paths (Codex r2 HIGH, lead fix): an
+ * alias is not a cycle, and the earlier visited-set semantics silently dropped the
+ * second one as if it were.
+ *
+ * Containment guard (lead item AM-8): every symlink's resolved target is checked
+ * against `rootRealDir` — the real path of the directory the OUTERMOST call was
+ * given (the skill directory itself, for every caller in this file) — before it is
+ * followed. A symlink whose target resolves outside that root is skipped, named,
+ * never bundled as an asset; `rootRealDir` is threaded through every recursive call
+ * so a nested symlinked directory is still checked against the ORIGINAL skill root,
+ * not against whichever subdirectory happens to be walking it.
+ *
+ * `readdirSync` failure (fix-round 1 MEDIUM-3) — e.g. an unreadable directory whose
+ * own `realpath` still resolved — produces a named `skipped` entry, same as an
+ * unresolvable `realpathSync`; it never throws out of this function or out of
+ * {@link loadSkillFromDir}.
+ */
+export function walkFiles(
+  dir: string,
+  seenRealDirs: Set<string> = new Set<string>(),
+  rootRealDir?: string,
+): SkillWalkResult {
+  const files: string[] = [];
+  const skipped: SkippedEntry[] = [];
+
+  let realDir: string;
+  try {
+    realDir = realpathSync(dir);
+  } catch (error) {
+    skipped.push({ path: dir, reason: `unreadable directory (${error instanceof Error ? error.message : String(error)})` });
+    return { files, skipped };
+  }
+  if (seenRealDirs.has(realDir)) {
+    skipped.push({ path: dir, reason: 'symlink cycle — directory already visited' });
+    return { files, skipped };
+  }
+  seenRealDirs.add(realDir);
+  const root = rootRealDir ?? realDir;
+
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch (error) {
+    skipped.push({ path: dir, reason: `unreadable directory (${error instanceof Error ? error.message : String(error)})` });
+    seenRealDirs.delete(realDir);
+    return { files, skipped };
+  }
+
+  for (const entry of entries) {
     const full = join(dir, entry.name);
-    if (entry.isDirectory()) out.push(...walkFiles(full));
-    else if (entry.isFile()) out.push(full);
+
+    if (entry.isSymbolicLink()) {
+      let target;
+      try {
+        target = statSync(full); // follows the link
+      } catch {
+        skipped.push({ path: full, reason: 'broken symlink' });
+        continue;
+      }
+      let resolvedReal: string;
+      try {
+        resolvedReal = realpathSync(full);
+      } catch {
+        // statSync just followed this same link successfully, so this is very unlikely
+        // (a race with something deleting the target); treat it the same as broken.
+        skipped.push({ path: full, reason: 'broken symlink' });
+        continue;
+      }
+      if (!isContained(root, resolvedReal)) {
+        skipped.push({ path: full, reason: 'symlink escapes the skill directory' });
+        continue;
+      }
+      if (target.isDirectory()) {
+        if (isJunkEntry(entry.name, 'directory')) {
+          skipped.push({ path: full, reason: `junk directory (${junkPatternLabel(entry.name, 'directory')})` });
+          continue;
+        }
+        const nested = walkFiles(full, seenRealDirs, root);
+        files.push(...nested.files);
+        skipped.push(...nested.skipped);
+      } else if (target.isFile()) {
+        if (isJunkEntry(entry.name, 'file')) {
+          skipped.push({ path: full, reason: `junk file (${junkPatternLabel(entry.name, 'file')})` });
+          continue;
+        }
+        files.push(full);
+      } else {
+        skipped.push({ path: full, reason: 'symlink target is neither a file nor a directory' });
+      }
+      continue;
+    }
+
+    if (entry.isDirectory()) {
+      if (isJunkEntry(entry.name, 'directory')) {
+        skipped.push({ path: full, reason: `junk directory (${junkPatternLabel(entry.name, 'directory')})` });
+        continue;
+      }
+      const nested = walkFiles(full, seenRealDirs, root);
+      files.push(...nested.files);
+      skipped.push(...nested.skipped);
+    } else if (entry.isFile()) {
+      if (isJunkEntry(entry.name, 'file')) {
+        skipped.push({ path: full, reason: `junk file (${junkPatternLabel(entry.name, 'file')})` });
+        continue;
+      }
+      files.push(full);
+    } else {
+      skipped.push({ path: full, reason: 'not a regular file, directory, or symlink' });
+    }
   }
-  return out;
+
+  // Ancestor-chain semantics (see the cycle-guard doc above): this directory's walk is
+  // over, so a sibling alias of it must be allowed to walk it again under its own path.
+  seenRealDirs.delete(realDir);
+  return { files, skipped };
 }
 
 /** Return the ids of every `<skillsDir>/<id>/SKILL.md`, sorted. */
@@ -88,6 +332,9 @@ export function discoverSkillIds(skillsDir: string): string[] {
  */
 export function listSkills(skillsDir: string): SkillSummary[] {
   return discoverSkillIds(skillsDir).map((id) => {
+    // Codex r3 (lead fix): the SKILL.md containment check guards EVERY public reader, not
+    // only loadSkillFromDir/getSkillInfo — an escaping SKILL.md symlink is refused here too.
+    assertSkillMdContained(join(skillsDir, id), join(skillsDir, id, 'SKILL.md'), id);
     const document = parseSkillDocument(readFileSync(join(skillsDir, id, 'SKILL.md'), 'utf-8'));
     const frontmatter = ClaudeSkillFrontmatterSchema.parse(parseYaml(document.frontmatterYaml));
     return { id, description: frontmatter.description };
@@ -146,6 +393,9 @@ export function describeSkillLoadFailure(
   const path = join(skillsDir, id, 'SKILL.md');
   let firstLine = '';
   try {
+    // Codex r3 (lead fix): never read an ESCAPING SKILL.md even for a diagnostic snippet —
+    // the first line of a file outside the skill directory is not ours to print.
+    assertSkillMdContained(join(skillsDir, id), path, id);
     const raw = readFileSync(path, 'utf-8');
     const line = (raw.split('\n', 1)[0] ?? '').replace(/\r$/, '').trim();
     firstLine =
@@ -176,6 +426,8 @@ export function listSkillsDetailed(skillsDir: string): SkillListing {
   const failures: SkillLoadFailure[] = [];
   for (const id of discoverSkillIds(skillsDir)) {
     try {
+      // Codex r3 (lead fix): refused BEFORE the read; the throw lands in `failures` below.
+      assertSkillMdContained(join(skillsDir, id), join(skillsDir, id, 'SKILL.md'), id);
       const document = parseSkillDocument(readFileSync(join(skillsDir, id, 'SKILL.md'), 'utf-8'));
       const frontmatter = ClaudeSkillFrontmatterSchema.parse(parseYaml(document.frontmatterYaml));
       skills.push({ id, description: frontmatter.description });
@@ -258,15 +510,36 @@ export function formatSkillApplyFailures(failures: readonly SkillApplyFailure[])
 }
 
 /** Get detailed info about a single skill without loading all assets. */
+/**
+ * Codex r2 CRITICAL (lead fix, AM-8 completed): the walk's containment guard runs AFTER
+ * `SKILL.md` has already been read, so a `SKILL.md` that is itself a symlink escaping
+ * the skill directory (`SKILL.md -> /etc/motd`, `-> ../../outside.md`) was followed and
+ * installed regardless. `SKILL.md` is mandatory, so an escaping one cannot be "skipped" —
+ * the whole skill is REFUSED with a named reason, never loaded from outside its directory.
+ * An in-tree `SKILL.md` symlink (a real file elsewhere inside the same skill directory)
+ * still loads. Returns nothing; throws on escape.
+ */
+function assertSkillMdContained(skillDir: string, skillMdPath: string, id: string): void {
+  if (!lstatSync(skillMdPath).isSymbolicLink()) return;
+  const rootReal = realpathSync(skillDir);
+  const targetReal = realpathSync(skillMdPath);
+  if (!isContained(rootReal, targetReal)) {
+    throw new Error(
+      `skill ${JSON.stringify(id)}: SKILL.md is a symlink escaping the skill directory (-> ${targetReal}) — refused`,
+    );
+  }
+}
+
 export function getSkillInfo(skillsDir: string, id: string): SkillInfo | undefined {
   const skillDir = join(skillsDir, id);
   const skillMdPath = join(skillDir, 'SKILL.md');
   if (!existsSync(skillMdPath)) return undefined;
+  assertSkillMdContained(skillDir, skillMdPath, id);
   const document = parseSkillDocument(readFileSync(skillMdPath, 'utf-8'));
   const fm = parseYaml(document.frontmatterYaml) as Record<string, unknown>;
   const parsed = ClaudeSkillFrontmatterSchema.parse(fm);
   const assetPaths = walkFiles(skillDir)
-    .filter((p) => p !== skillMdPath)
+    .files.filter((p) => p !== skillMdPath)
     .map((p) => relative(skillDir, p).split('\\').join('/'))
     .sort();
   return {
@@ -281,21 +554,35 @@ export function getSkillInfo(skillsDir: string, id: string): SkillInfo | undefin
   };
 }
 
+/**
+ * A {@link CanonicalSkill} plus, optionally, the {@link SkippedEntry} list
+ * {@link loadSkillFromDir} collected while walking the skill's directory. The
+ * field is additive and optional — a `CanonicalSkill`-typed caller (every
+ * adapter, every existing consumer) sees exactly the shape it always saw;
+ * only a caller that reads `.skipped` learns about junk/broken-symlink skips.
+ */
+export type LoadedSkill = CanonicalSkill & { readonly skipped?: readonly SkippedEntry[] };
+
 /**
  * Load one `<skillsDir>/<id>/` directory into a {@link CanonicalSkill}: its
- * `SKILL.md` document plus every other file as a bundled asset.
+ * `SKILL.md` document plus every other file as a bundled asset. Junk entries
+ * and broken/cyclic symlinks encountered along the way are named in
+ * `.skipped` (feature `skills-walk-symlinks-and-junk`, FR-1/FR-2) — omitted
+ * entirely when nothing was skipped, never a silent drop.
  *
  * @throws if the skill directory has no `SKILL.md`.
  */
-export function loadSkillFromDir(skillsDir: string, id: string): CanonicalSkill {
+export function loadSkillFromDir(skillsDir: string, id: string): LoadedSkill {
   const skillDir = join(skillsDir, id);
   const skillMdPath = join(skillDir, 'SKILL.md');
   if (!existsSync(skillMdPath)) {
     throw new Error(`skill not found: ${JSON.stringify(id)} (looked in ${skillsDir})`);
   }
+  assertSkillMdContained(skillDir, skillMdPath, id);
   const document = parseSkillDocument(readFileSync(skillMdPath, 'utf-8'));
   const frontmatter = ClaudeSkillFrontmatterSchema.parse(parseYaml(document.frontmatterYaml));
-  const assets: SkillAsset[] = walkFiles(skillDir)
+  const walk = walkFiles(skillDir);
+  const assets: SkillAsset[] = walk.files
     .filter((path) => path !== skillMdPath)
     .map((path) => {
       const { encoding, content } = readAssetContent(path);
@@ -306,5 +593,7 @@ export function loadSkillFromDir(skillsDir: string, id: string): CanonicalSkill
       };
     })
     .sort((a, b) => a.path.localeCompare(b.path));
-  return { id, frontmatter, document, assets };
+  return walk.skipped.length > 0
+    ? { id, frontmatter, document, assets, skipped: walk.skipped }
+    : { id, frontmatter, document, assets };
 }

package/src/vector-tier.ts

@@ -69,6 +69,7 @@ import {
   reindexAgentdbRows,
   readAgentdbRowsByTaskType,
   DZ_OWNED_TASK_TYPES,
+  readStoreGeneration,
 } from './agentdb-index.js';
 // smart-backlog (ADR-001/005 lifecycle): `dz vector reindex` must re-embed dz-backlog rows too, or
 // they rot in a stale embedding space after a model bump. One-directional import — backlog.ts imports
@@ -857,12 +858,17 @@ interface EngineCacheEntry {
   readonly stat: AgentdbDbStat;
 }
 
-/** The three independent invalidation signals AM-6 asks for, plus the recency stamp
+/** The four independent invalidation signals: the three AM-6 asks for (mtime/size/inode) PLUS the
+ * store's own write-generation counter (`store-generation-counter`, FR-2) — a same-size same-tick
+ * temp+rename replace can still leave mtime/size/inode all coincidentally unchanged on a coarse
+ * filesystem, but `indexPatternsToAgentdb` bumps the generation on every real write, so it is the
+ * one signal that can never coincidentally match a stale cache entry. Plus the recency stamp
  * {@link touchEngineCacheEntry} needs for the bounded-size eviction below. */
 interface AgentdbDbStat {
   readonly mtimeMs: number;
   readonly size: number;
   readonly ino: number;
+  readonly generation: number;
   lastUsedAt: number;
 }
 
@@ -897,21 +903,25 @@ function engineCacheKey(projectRoot: string): string {
 
 /** stat facts of `<root>/.dz/agentdb.db`, or `-1`/`-1`/`-1` when absent — a distinct, stable cache
  * key for "no store yet" so a project that later gains a store is never confused with one that
- * never had (statSync's own floor is mtime 0). Never throws. */
-function agentdbDbStat(projectRoot: string): { mtimeMs: number; size: number; ino: number } {
+ * never had (statSync's own floor is mtime 0). `generation` is read regardless of whether the stat
+ * itself succeeded (`readStoreGeneration` already degrades a missing/corrupt counter file to `0`,
+ * FR-2's compatibility floor for a pre-existing store). Never throws. */
+function agentdbDbStat(projectRoot: string): { mtimeMs: number; size: number; ino: number; generation: number } {
+  const generation = readStoreGeneration(projectRoot);
   try {
     const st = statSync(join(projectRoot, '.dz', 'agentdb.db'));
-    return { mtimeMs: st.mtimeMs, size: st.size, ino: st.ino };
+    return { mtimeMs: st.mtimeMs, size: st.size, ino: st.ino, generation };
   } catch {
-    return { mtimeMs: -1, size: -1, ino: -1 };
+    return { mtimeMs: -1, size: -1, ino: -1, generation };
   }
 }
 
-/** True when NONE of the three independent signals changed — the only case where a cached engine
- * may still be trusted (AM-6). Any one of them differing (a same-tick replace still bumps size or
- * gets a fresh inode from a temp+rename write) forces a re-resolve. */
-function agentdbDbStatUnchanged(a: AgentdbDbStat, b: { mtimeMs: number; size: number; ino: number }): boolean {
-  return a.mtimeMs === b.mtimeMs && a.size === b.size && a.ino === b.ino;
+/** True when NONE of the four independent signals changed — the only case where a cached engine
+ * may still be trusted (AM-6, and `store-generation-counter` FR-2). Any one of them differing (a
+ * same-tick replace still bumps size or gets a fresh inode from a temp+rename write, and every real
+ * write bumps the generation regardless) forces a re-resolve. */
+function agentdbDbStatUnchanged(a: AgentdbDbStat, b: { mtimeMs: number; size: number; ino: number; generation: number }): boolean {
+  return a.mtimeMs === b.mtimeMs && a.size === b.size && a.ino === b.ino && a.generation === b.generation;
 }
 
 /** Evict the least-recently-used entry once the cache is at capacity — called only on a genuine