@ngockhoale/ukit 3.0.9 → 3.0.11

package/src/core/metadata.js

@@ -1,5 +1,5 @@
 import path from 'node:path';
-import { ensureDir, writeJson, readJsonIfExists } from './fileOps.js';
+import { ensureDir, writeJson, readJsonIfExists, withFileLock } from './fileOps.js';
 
 function normalizeTrackedRelativePath(relativePath) {
   if (typeof relativePath !== 'string') {
@@ -33,100 +33,130 @@ export async function writeInstallMetadata({
   projectRoot,
   managedRelativePaths = null,
   retainedManagedRelativePaths = [],
+  lockStaleMs,
+  lockMaxWaitMs,
 }) {
-  // Accumulate file tracking across re-installs so files written in earlier runs
-  // remain tracked (they are 'skip'/'unchanged' on re-install and don't appear in
-  // the current writes, but we still need to remove them on uninstall).
-  const previousData = await readJsonIfExists(installMetaPath);
-  const previousFiles = previousData?.tool === 'ukit' && Array.isArray(previousData?.files)
-    ? previousData.files
-    : [];
-  const managedPathSet = Array.isArray(managedRelativePaths)
-    ? new Set(managedRelativePaths.map((value) => normalizeTrackedRelativePath(value)).filter(Boolean))
-    : null;
-  const retainedManagedPathSet = new Set(
-    retainedManagedRelativePaths.map((value) => normalizeTrackedRelativePath(value)).filter(Boolean),
-  );
-
-  // Convert writes to tracked entries stored as relative paths for portability.
-  const newEntries = projectRoot
-    ? writes.map((w) => ({
-        p: normalizeTrackedRelativePath(path.relative(projectRoot, w.targetPath)),
-        t: w.type, // 'link', 'command', 'agent', 'hook', 'config', 'skill', etc.
-      })).filter((entry) => {
-        if (!entry.p) {
-          return false;
-        }
-        if (!managedPathSet) {
-          return true;
-        }
-        return managedPathSet.has(entry.p) || retainedManagedPathSet.has(entry.p);
-      })
-    : [];
-
-  // Merge: keep entries from previous runs for paths not re-written this run,
-  // then append new entries (updating type info for any that were re-written).
-  const newPaths = new Set(newEntries.map((e) => e.p));
-  const retained = previousFiles.filter((e) => {
-    const p = normalizeTrackedRelativePath(typeof e === 'string' ? e : e?.p);
-    if (!p) {
-      return false;
-    }
-    if (managedPathSet && !managedPathSet.has(p) && !retainedManagedPathSet.has(p)) {
-      return false;
-    }
-    return !newPaths.has(p);
-  });
-  const allFiles = [...retained, ...newEntries];
-
-  const metadata = {
-    tool: 'ukit',
-    version: packageVersion,
-    installedAt: new Date().toISOString(),
-    manifest: {
-      name: manifest.name,
-      version: manifest.version,
-    },
-    stack: stackContext,
-    providers: providerContext,
-    summary,
-    files: allFiles,
-  };
-
-  await ensureDir(path.dirname(installMetaPath));
-  await writeJson(installMetaPath, metadata);
+  const writeResult = await withFileLock(installMetaPath, async () => {
+    // Accumulate file tracking across re-installs so files written in earlier
+    // runs remain tracked (they are 'skip'/'unchanged' on re-install and don't
+    // appear in the current writes, but we still need to remove them on
+    // uninstall). The read→merge→write runs under withFileLock (C79-12):
+    // concurrent installs used to read the same pre-state and last-writer-wins
+    // the merge, silently losing tracked-file entries.
+    const previousData = await readJsonIfExists(installMetaPath);
+    const previousFiles = previousData?.tool === 'ukit' && Array.isArray(previousData?.files)
+      ? previousData.files
+      : [];
+    const managedPathSet = Array.isArray(managedRelativePaths)
+      ? new Set(managedRelativePaths.map((value) => normalizeTrackedRelativePath(value)).filter(Boolean))
+      : null;
+    const retainedManagedPathSet = new Set(
+      retainedManagedRelativePaths.map((value) => normalizeTrackedRelativePath(value)).filter(Boolean),
+    );
+
+    // Convert writes to tracked entries stored as relative paths for portability.
+    const newEntries = projectRoot
+      ? writes.map((w) => ({
+          p: normalizeTrackedRelativePath(path.relative(projectRoot, w.targetPath)),
+          t: w.type, // 'link', 'command', 'agent', 'hook', 'config', 'skill', etc.
+        })).filter((entry) => {
+          if (!entry.p) {
+            return false;
+          }
+          if (!managedPathSet) {
+            return true;
+          }
+          return managedPathSet.has(entry.p) || retainedManagedPathSet.has(entry.p);
+        })
+      : [];
+
+    // Merge: keep entries from previous runs for paths not re-written this run,
+    // then append new entries (updating type info for any that were re-written).
+    const newPaths = new Set(newEntries.map((e) => e.p));
+    const retained = previousFiles.filter((e) => {
+      const p = normalizeTrackedRelativePath(typeof e === 'string' ? e : e?.p);
+      if (!p) {
+        return false;
+      }
+      if (managedPathSet && !managedPathSet.has(p) && !retainedManagedPathSet.has(p)) {
+        return false;
+      }
+      return !newPaths.has(p);
+    });
+    const allFiles = [...retained, ...newEntries];
+
+    const metadata = {
+      tool: 'ukit',
+      version: packageVersion,
+      installedAt: new Date().toISOString(),
+      manifest: {
+        name: manifest.name,
+        version: manifest.version,
+      },
+      stack: stackContext,
+      providers: providerContext,
+      summary,
+      files: allFiles,
+    };
+
+    await ensureDir(path.dirname(installMetaPath));
+    await writeJson(installMetaPath, metadata);
+    return 'written';
+  }, { staleMs: lockStaleMs, maxWaitMs: lockMaxWaitMs });
+
+  if (writeResult === undefined) {
+    // FR-012 / SPEC §14: lock-wait expiry degrades explicitly — warn and skip
+    // the bookkeeping write rather than aborting the install or writing
+    // unlocked (the drop is also journaled by withFileLock).
+    console.warn(
+      `[UKit] Skipping install metadata write — install.json lock stayed held past the wait budget (${installMetaPath}). Tracked files for this run may be incomplete.`,
+    );
+  }
 }
 
 export async function removeTrackedPathsFromMetadata({
   installMetaPath,
   projectRoot,
   removeRelativePaths = [],
+  lockStaleMs,
+  lockMaxWaitMs,
 }) {
   if (!removeRelativePaths.length) {
     return;
   }
 
-  const metadata = await readJsonIfExists(installMetaPath);
-  if (!metadata || metadata.tool !== 'ukit' || !Array.isArray(metadata.files)) {
-    return;
-  }
-
-  const targetPaths = [...new Set(removeRelativePaths
-    .map((relPath) => (typeof relPath === 'string' ? relPath.trim().replace(/\\/g, '/') : ''))
-    .filter(Boolean))];
+  const removeResult = await withFileLock(installMetaPath, async () => {
+    const metadata = await readJsonIfExists(installMetaPath);
+    if (!metadata || metadata.tool !== 'ukit' || !Array.isArray(metadata.files)) {
+      return 'noop';
+    }
 
-  const filteredFiles = metadata.files.filter((entry) => {
-    const p = typeof entry === 'string' ? entry : entry?.p;
-    if (!p) {
-      return true;
+    const targetPaths = [...new Set(removeRelativePaths
+      .map((relPath) => (typeof relPath === 'string' ? relPath.trim().replace(/\\/g, '/') : ''))
+      .filter(Boolean))];
+
+    const filteredFiles = metadata.files.filter((entry) => {
+      const p = typeof entry === 'string' ? entry : entry?.p;
+      if (!p) {
+        return true;
+      }
+      const normalized = p.trim().replace(/\\/g, '/');
+      return !targetPaths.some((targetPath) => isSameOrDescendantPath(normalized, targetPath));
+    });
+
+    if (filteredFiles.length === metadata.files.length) {
+      return 'noop';
     }
-    const normalized = p.trim().replace(/\\/g, '/');
-    return !targetPaths.some((targetPath) => isSameOrDescendantPath(normalized, targetPath));
-  });
 
-  if (filteredFiles.length === metadata.files.length) {
-    return;
-  }
+    await writeJson(installMetaPath, { ...metadata, files: filteredFiles });
+    return 'written';
+  }, { staleMs: lockStaleMs, maxWaitMs: lockMaxWaitMs });
 
-  await writeJson(installMetaPath, { ...metadata, files: filteredFiles });
+  if (removeResult === undefined) {
+    // Same explicit-degrade contract as writeInstallMetadata: warn, skip the
+    // write, never mutate install.json unlocked.
+    console.warn(
+      `[UKit] Skipping install metadata update — install.json lock stayed held past the wait budget (${installMetaPath}). Removed paths may still be listed as tracked.`,
+    );
+  }
 }

package/src/core/token/index.js

@@ -1,5 +1,5 @@
 import crypto from 'node:crypto';
-import { readJsonIfExists, writeJson } from '../fileOps.js';
+import { readJsonIfExists, writeJson, withFileLock } from '../fileOps.js';
 import { buildRuntimePaths } from '../runtimePaths.js';
 
 export const DEFAULT_PROMPT_CACHE_MAX_ENTRIES = 20;
@@ -265,15 +265,39 @@ export async function readPromptCacheEntry(
     lastHitAt: Date.now(),
     hitCount: (entry.hitCount ?? 0) + 1,
   };
-  const entries = [
-    touchedEntry,
-    ...cache.entries.slice(0, entryIndex),
-    ...cache.entries.slice(entryIndex + 1),
-  ].slice(0, maxEntries);
   try {
-    const nextDocument = normalizePromptCacheDocument({ entries }, { maxEntries });
-    await writePromptCacheDocument(projectRoot, nextDocument);
-    return nextDocument.entries[0] ?? touchedEntry;
+    const runtimePaths = buildRuntimePaths(projectRoot);
+    // SPEC §5 FR-019: the touch's read-merge-write runs under the cache-file
+    // lock — concurrent hook processes used to merge over the same snapshot
+    // and drop hitCounts/entries. The lock RE-READS the document so the merge
+    // applies to the freshest state, and an entry a concurrent delete already
+    // dropped is never resurrected.
+    const locked = await withFileLock(runtimePaths.promptCachePath, async () => {
+      const lockedCache = await readPromptCacheDocument(projectRoot, { maxEntries });
+      const lockedIndex = lockedCache.entries.findIndex(
+        (lockedEntry) => lockedEntry.requestKey === requestKey,
+      );
+      if (lockedIndex < 0) {
+        return null;
+      }
+      const mergedEntry = {
+        ...lockedCache.entries[lockedIndex],
+        updatedAt: Date.now(),
+        lastHitAt: Date.now(),
+        hitCount: (lockedCache.entries[lockedIndex].hitCount ?? 0) + 1,
+      };
+      const entries = [
+        mergedEntry,
+        ...lockedCache.entries.slice(0, lockedIndex),
+        ...lockedCache.entries.slice(lockedIndex + 1),
+      ].slice(0, maxEntries);
+      const nextDocument = normalizePromptCacheDocument({ entries }, { maxEntries });
+      await writePromptCacheDocument(projectRoot, nextDocument);
+      return nextDocument.entries[0] ?? mergedEntry;
+    });
+    // Lock-busy (fail-closed skip, journaled) or entry-gone-under-lock →
+    // degrade to the in-memory touched entry, same as the failure path below.
+    return locked ?? touchedEntry;
   } catch {
     // TASK-006 (SPEC §5 FR-009): the touch-write is a side effect of a READ —
     // a full/unwritable cache (EACCES, ENOSPC, ...) must never reject the
@@ -295,14 +319,23 @@ export async function writePromptCacheEntry(
     return readPromptCacheDocument(projectRoot, { maxEntries });
   }
 
-  const cache = await readPromptCacheDocument(projectRoot, { maxEntries });
-  const entries = [
-    normalizedEntry,
-    ...cache.entries.filter((existingEntry) => existingEntry.requestKey !== normalizedEntry.requestKey),
-  ].slice(0, maxEntries);
-  const nextDocument = normalizePromptCacheDocument({ entries }, { maxEntries });
-  await writePromptCacheDocument(projectRoot, nextDocument);
-  return nextDocument;
+  const runtimePaths = buildRuntimePaths(projectRoot);
+  // SPEC §5 FR-019: the merge must re-read under the lock — merging over a
+  // pre-lock snapshot is the lost-update race this lock exists to close, and
+  // no code path may write the cache unlocked.
+  const lockedDocument = await withFileLock(runtimePaths.promptCachePath, async () => {
+    const cache = await readPromptCacheDocument(projectRoot, { maxEntries });
+    const entries = [
+      normalizedEntry,
+      ...cache.entries.filter((existingEntry) => existingEntry.requestKey !== normalizedEntry.requestKey),
+    ].slice(0, maxEntries);
+    const nextDocument = normalizePromptCacheDocument({ entries }, { maxEntries });
+    await writePromptCacheDocument(projectRoot, nextDocument);
+    return nextDocument;
+  });
+  // Lock-busy → the write was skipped (journaled); return the freshest doc so
+  // callers keep a consistent shape without an unlocked write.
+  return lockedDocument ?? readPromptCacheDocument(projectRoot, { maxEntries });
 }
 
 /**
@@ -317,18 +350,26 @@ export async function deletePromptCacheEntries(projectRoot, { selectedIds } = {}
   try {
     const ids = new Set(Array.isArray(selectedIds) ? selectedIds : []);
     if (ids.size === 0) return { removed: 0 };
-    const cache = await readPromptCacheDocument(projectRoot);
-    if (cache.entries.length === 0) return { removed: 0 };
-    const kept = cache.entries.filter((entry) => {
-      const entryIds = Array.isArray(entry?.metadata?.selectedIds)
-        ? entry.metadata.selectedIds
-        : [];
-      return !entryIds.some((id) => ids.has(id));
+    const runtimePaths = buildRuntimePaths(projectRoot);
+    // SPEC §5 FR-019: the filter+rewrite is a read-merge-write and runs under
+    // the cache-file lock — a purge racing a cache writer must not resurrect
+    // (or lose) entries. Lock-busy → the purge is skipped (journaled) and the
+    // function reports its usual no-op shape.
+    const locked = await withFileLock(runtimePaths.promptCachePath, async () => {
+      const cache = await readPromptCacheDocument(projectRoot);
+      if (cache.entries.length === 0) return { removed: 0 };
+      const kept = cache.entries.filter((entry) => {
+        const entryIds = Array.isArray(entry?.metadata?.selectedIds)
+          ? entry.metadata.selectedIds
+          : [];
+        return !entryIds.some((id) => ids.has(id));
+      });
+      const removed = cache.entries.length - kept.length;
+      if (removed === 0) return { removed: 0 };
+      await writePromptCacheDocument(projectRoot, { entries: kept });
+      return { removed };
     });
-    const removed = cache.entries.length - kept.length;
-    if (removed === 0) return { removed: 0 };
-    await writePromptCacheDocument(projectRoot, { entries: kept });
-    return { removed };
+    return locked ?? { removed: 0 };
   } catch {
     return { removed: 0 };
   }

package/src/index/buildIndex.js

@@ -697,6 +697,13 @@ async function resolveChangedHintUpdates(rootDir, changedPaths, previousRecordsB
     if (!isSafeContainedRelative(rootDir, relativePath)) {
       return null;
     }
+    // Hints under directories discovery never indexes are no-ops, not trust
+    // failures: `node_modules/`, `.cache/` and friends bypassed this lane and
+    // entered the index as phantom rows that kept the index perpetually stale.
+    // Same predicate as collectGitTrackedFiles.
+    if (relativePath.split('/').some((segment) => EXCLUDED_DIR_NAMES.has(segment))) {
+      continue;
+    }
 
     const absolutePath = path.join(rootDir, relativePath);
     let stat = null;

package/src/index/impactContext.js

@@ -31,6 +31,17 @@ async function readArtifact(rootDir, artifactName) {
   }
 }
 
+// Returns the artifact's item rows as plain objects, dropping poisoned rows
+// (null, undefined, primitives). Hand-corrupted or truncated `.cache/index/*.json`
+// files can carry `items: [null]` or `items: "corrupted"` — every consumer that
+// dereferences `item.filePath` must see only object rows.
+function safeItems(artifact) {
+  if (!artifact || typeof artifact !== 'object' || !Array.isArray(artifact.items)) {
+    return [];
+  }
+  return artifact.items.filter((entry) => entry && typeof entry === 'object');
+}
+
 function normalizePath(filePath) {
   return String(filePath ?? '').trim().replace(/\\/g, '/').replace(/^\.\//, '');
 }
@@ -93,9 +104,8 @@ export async function resolveImpactContext({
     readArtifact(absoluteRoot, INDEX_ARTIFACTS.files),
     loadRelatedTestArtifacts({ rootDir: absoluteRoot }),
   ]);
-
-  const callItems = callsArtifact.items ?? [];
-  const importItems = importsArtifact.items ?? [];
+  const callItems = safeItems(callsArtifact);
+  const importItems = safeItems(importsArtifact);
   const risk = classifyImpactRisk(normalizedChangedFiles);
   const mirrorCounterparts = findMirrorCounterparts(normalizedChangedFiles).slice(0, budget.maxMirrors);
 
@@ -145,8 +155,7 @@ export async function resolveImpactContext({
   // Non-relative specifiers (aliases like `@/x`, `~/x`, tsconfig paths) resolve
   // through the same machinery the index builder uses; relative specifiers keep
   // the candidate-list match so a changed file absent from files.json (e.g. a
-  // brand-new file) still counts.
-  const indexedFileSet = new Set((filesArtifact.items ?? []).map((item) => item.filePath));
+  const indexedFileSet = new Set(safeItems(filesArtifact).map((item) => item.filePath));
   const importAliasContext = importsNeedAliasContext(importItems)
     ? await loadImportAliasContext({ rootDir: absoluteRoot }).catch(() => null)
     : null;

package/src/render/instructionRenderer.js

@@ -19,19 +19,48 @@ import { renderTemplateString } from './renderTemplate.js';
 
 export const PREAMBLE_KEY = '__preamble__';
 
-const SECTION_HEADING_RE = /^## (.+)$/gm;
+const FENCE_OPEN_RE = /^ {0,3}(`{3,}|~{3,})/;
+const FENCE_CLOSE_RES = {
+  '`': /^ {0,3}(`{3,})\s*$/,
+  '~': /^ {0,3}(~{3,})\s*$/,
+};
 const LEFTOVER_TOKEN_RE = /\{\{\s*([a-zA-Z0-9_.-]+)\s*\}\}/;
 
 /**
  * Split markdown into a Map<heading, body> at `## ` boundaries.
- * `### ` and deeper stay inside the enclosing body. Leading non-## text lands
- * under the reserved `__preamble__` key (absent when the file starts with `## `).
- * Throws on a duplicate `## ` heading, naming it.
+ * The reserved key PREAMBLE_KEY holds the leading non-## block verbatim.
+ * Bodies are raw slices (leading+trailing whitespace byte-meaningful).
+ *
+ * FR-017 (SPEC §5): heading detection is fence-aware — a `## ` line inside a
+ * fenced code block is content, not a boundary. Fences follow CommonMark: the
+ * opener is ``` or ~~~ indented ≤3 spaces; the closer uses the same character
+ * at the same or greater length with nothing after it. Duplicate `## `
+ * headings warn and keep the FIRST boundary (the duplicate line and its text
+ * stay inside the first section's verbatim body — no content is dropped).
  */
 export function parseSections(markdown) {
+  // Collect `## ` heading boundaries by line scan, tracking fence state.
+  // Byte offsets are maintained manually so bodies stay raw slices.
+  const boundaries = []; // {index, matchLength, heading}
+  let fence = null; // {char, length} of the open fence
+  let offset = 0;
+  for (const line of markdown.split('\n')) {
+    if (fence) {
+      const close = line.match(FENCE_CLOSE_RES[fence.char]);
+      if (close && close[1].length >= fence.length) fence = null;
+    } else {
+      const open = line.match(FENCE_OPEN_RE);
+      if (open) {
+        fence = { char: open[1][0], length: open[1].length };
+      } else if (line.startsWith('## ') && line.length > 3) {
+        boundaries.push({ index: offset, matchLength: line.length, heading: line.slice(3).trimEnd() });
+      }
+    }
+    offset += line.length + 1;
+  }
+
   const sections = new Map();
-  const matches = [...markdown.matchAll(SECTION_HEADING_RE)];
-  const first = matches[0];
+  const first = boundaries[0];
 
   const preamble = first ? markdown.slice(0, first.index) : markdown;
   if (preamble.trim().length > 0) {
@@ -40,18 +69,30 @@ export function parseSections(markdown) {
     sections.set(PREAMBLE_KEY, preamble);
   }
 
-  for (let i = 0; i < matches.length; i++) {
-    const m = matches[i];
-    const heading = m[1].trimEnd();
-    if (sections.has(heading)) {
-      throw new Error(`duplicate ## heading in source: ${heading}`);
+  // Seed with the reserved key so a literal `## __preamble__` heading after a
+  // preamble degrades to keep-first instead of overwriting it.
+  const seen = new Set(sections.has(PREAMBLE_KEY) ? [PREAMBLE_KEY] : []);
+  const heads = [];
+  for (const b of boundaries) {
+    if (seen.has(b.heading)) {
+      console.warn(
+        `instructionRenderer: duplicate ## heading '${b.heading}' — keeping first occurrence`,
+      );
+      continue;
     }
-    const bodyStart = m.index + m[0].length;
-    const bodyEnd = i + 1 < matches.length ? matches[i + 1].index : markdown.length;
+    seen.add(b.heading);
+    heads.push(b);
+  }
+
+  for (let i = 0; i < heads.length; i++) {
+    const h = heads[i];
+    const bodyStart = h.index + h.matchLength;
+    const bodyEnd = i + 1 < heads.length ? heads[i + 1].index : markdown.length;
     // Raw slice up to (not including) the next `## ` heading — leading AND
     // trailing whitespace are byte-meaningful (marker line vs blank line
-    // under the heading; multi-blank separators between sections).
-    sections.set(heading, markdown.slice(bodyStart, bodyEnd));
+    // under the heading; multi-blank separators between sections). A skipped
+    // duplicate boundary leaves its line inside this slice.
+    sections.set(h.heading, markdown.slice(bodyStart, bodyEnd));
   }
 
   return sections;
@@ -111,8 +152,10 @@ export function renderOutput(outputSpec, sources, vars = {}) {
     out = renderTemplateString(out, vars);
     const leftover = out.match(LEFTOVER_TOKEN_RE);
     if (leftover) {
-      throw new Error(
-        `output ${outputSpec.target}: unresolved template variable '{{${leftover[1]}}}'`,
+      // FR-017(d): warn + emit the token verbatim instead of bricking
+      // install/render on an imperfect user overlay.
+      console.warn(
+        `instructionRenderer: output '${outputSpec.target}' has unresolved template variable '{{${leftover[1]}}}' — emitted verbatim`,
       );
     }
   }
@@ -124,8 +167,10 @@ export function renderOutput(outputSpec, sources, vars = {}) {
  * Source names are looked up as `sources.get(ref.from)`; overlay sources are
  * named `overlay:<file-stem>` by convention in renderInstructionsFromDisk.
  * `varsByResolve` supplies variables to outputs with resolve_vars: true.
- * FR-006 strictness: a `## ` section in any source referenced by zero outputs
- * throws, naming the heading.
+ * FR-006 strictness: a `## ` section referenced by zero outputs throws for
+ * shipped sources, preserving the layout contract. FR-017(c): the same
+ * condition in a USER overlay (`overlay:*` source name) downgrades to
+ * console.warn + keep — an imperfect repo.md must not brick install/render.
  */
 export function renderAll(layout, sourceTexts, varsByResolve = {}) {
   if (!layout || !Array.isArray(layout.outputs) || layout.outputs.length === 0) {
@@ -156,9 +201,16 @@ export function renderAll(layout, sourceTexts, varsByResolve = {}) {
     }
   }
   for (const [name, source] of sources) {
+    const isOverlay = name.startsWith('overlay:');
     for (const heading of source.keys()) {
       if (heading === PREAMBLE_KEY) continue;
       if (!referenced.get(name) || !referenced.get(name).has(heading)) {
+        if (isOverlay) {
+          console.warn(
+            `instructionRenderer: overlay source '${name}' has ## section '${heading}' referenced by zero outputs — kept`,
+          );
+          continue;
+        }
         throw new Error(`source '${name}': ## section '${heading}' referenced by zero outputs`);
       }
     }