@pruddiman/hem 0.0.1-beta-5671db0

package/dist/decision-queue.js

@@ -0,0 +1,265 @@
+/**
+ * Decision queue for Hem's parallel organization pass.
+ *
+ * Solves the race condition where the arbiter issues a MERGE and a DELETE
+ * for the same source file simultaneously.  The MERGE must complete (worker
+ * reads source → writes merged target) before the DELETE removes the source.
+ *
+ * ## How it works
+ *
+ * 1. **MERGE src INTO dst** — relayed immediately.  The queue records that
+ *    `src` has a pending read (the merge worker needs to read it).
+ *    A filesystem watcher on `dst` waits for the merged file to appear.
+ *
+ * 2. **DELETE filepath** — if `filepath` has a pending MERGE read, the
+ *    DELETE is queued.  Otherwise it's relayed immediately.
+ *
+ * 3. **UPDATE-LINKS / free-form** — always relayed immediately (graceful
+ *    degradation for unparsed decisions).
+ *
+ * ## Release triggers
+ *
+ * - **Filesystem watcher** (`fs.watch`) detects when the merge target file
+ *   is written, releasing queued DELETEs for the corresponding source.
+ * - **Worker completion** — when a worker's `promptAndWait` resolves, any
+ *   decisions still blocked on that worker's actions are released.
+ * - **Timeout** — queued decisions are released after a configurable
+ *   deadline (default 60 s) to prevent permanent blocking.
+ *
+ * The watcher uses `node:fs` callback-based `watch()` (not the async
+ * iterator from `node:fs/promises`) so we can `.close()` it cleanly.
+ */
+import { watch } from "node:fs";
+import { resolve } from "node:path";
+// ── Constants ───────────────────────────────────────────────────────────
+/** Default timeout (ms) after which a queued decision is released regardless. */
+export const QUEUE_TIMEOUT_MS = 60_000;
+/** Regex patterns for structured DECISION parsing. */
+const RE_MERGE = /^DECISION:\s*@(org-worker-\d+)\s+MERGE\s+(\S+)\s+INTO\s+(\S+)/i;
+const RE_DELETE = /^DECISION:\s*@(org-worker-\d+)\s+DELETE\s+(\S+)/i;
+const RE_UPDATE_LINKS = /^DECISION:\s*@(org-worker-\d+)\s+UPDATE-LINKS\s+(\S+)/i;
+const RE_FREEFORM = /^DECISION:\s*@(org-worker-\d+)/i;
+/**
+ * Parse a DECISION message into a structured action.
+ * Returns `undefined` if the message is not a DECISION.
+ *
+ * NOTE: `@all-workers` decisions intentionally return `undefined` here
+ * because the regex only matches `@org-worker-\d+`. This is correct —
+ * `@all-workers` decisions are broadcast to all workers by the SSE relay
+ * routing logic and do not need file-based sequencing via DecisionQueue.
+ */
+export function parseDecision(message) {
+    let m;
+    m = message.match(RE_MERGE);
+    if (m)
+        return { kind: "merge", worker: m[1], src: m[2], dst: m[3] };
+    m = message.match(RE_DELETE);
+    if (m)
+        return { kind: "delete", worker: m[1], filePath: m[2] };
+    m = message.match(RE_UPDATE_LINKS);
+    if (m)
+        return { kind: "update-links", worker: m[1], filePath: m[2] };
+    m = message.match(RE_FREEFORM);
+    if (m)
+        return { kind: "freeform", worker: m[1] };
+    return undefined;
+}
+// ── DecisionQueue ───────────────────────────────────────────────────────
+/**
+ * Filesystem-watched decision queue.
+ *
+ * Instantiated once per `runParallel()` invocation.  Call `start()` before
+ * the SSE relay loop and `stop()` in the `finally` block.
+ */
+export class DecisionQueue {
+    /** Files that have a pending MERGE read (source → merge-target). */
+    pendingReads = new Map();
+    /** Queued DELETE decisions blocked on a pending MERGE. */
+    queue = [];
+    /** The filesystem watcher (recursive), or null if not started. */
+    watcher = null;
+    /** Absolute path to the documentation destination directory. */
+    destPath;
+    /** Timeout in ms for queued decisions. */
+    timeoutMs;
+    /** Relay callback provided by the caller. */
+    relay;
+    /** Optional verbose logger. */
+    verbose;
+    constructor(opts) {
+        this.destPath = resolve(opts.destPath);
+        this.relay = opts.relay;
+        this.timeoutMs = opts.timeoutMs ?? QUEUE_TIMEOUT_MS;
+        this.verbose = opts.verbose;
+    }
+    // ── Lifecycle ───────────────────────────────────────────────────────
+    /**
+     * Start the filesystem watcher on the destination directory.
+     * Must be called before processing any decisions.
+     */
+    start() {
+        if (this.watcher)
+            return; // already running
+        try {
+            this.watcher = watch(this.destPath, { recursive: true }, (_eventType, filename) => {
+                if (!filename)
+                    return;
+                // Normalize: fs.watch may return OS-native separators
+                const relPath = filename.replace(/\\/g, "/");
+                this.onFileChange(relPath);
+            });
+            // Suppress watcher errors (e.g., ENOENT if dir is deleted)
+            this.watcher.on("error", () => { });
+            if (this.verbose) {
+                this.verbose(`[decision-queue] Watcher started on ${this.destPath}`);
+            }
+        }
+        catch {
+            // If fs.watch fails (e.g., dir doesn't exist yet), proceed without
+            // it — timeouts and worker completion will still release queued items.
+            if (this.verbose) {
+                this.verbose(`[decision-queue] Could not start watcher — using timeout fallback`);
+            }
+        }
+    }
+    /**
+     * Stop the filesystem watcher and release all queued decisions.
+     * Call in the `finally` block of `runParallel()`.
+     */
+    stop() {
+        if (this.watcher) {
+            this.watcher.close();
+            this.watcher = null;
+        }
+        // Release all remaining queued decisions
+        this.releaseAll("stop");
+    }
+    // ── Public API ──────────────────────────────────────────────────────
+    /**
+     * Process a parsed DECISION and either relay it immediately or queue it.
+     *
+     * @param action    - Parsed decision action.
+     * @param target    - The worker session to relay to.
+     * @param relayText - Full relay text string.
+     */
+    async handleDecision(action, target, relayText) {
+        switch (action.kind) {
+            case "merge":
+                // Record that the source file has a pending read
+                this.pendingReads.set(action.src, action.dst);
+                if (this.verbose) {
+                    this.verbose(`[decision-queue] MERGE ${action.src} INTO ${action.dst} — pending read recorded`);
+                }
+                // Relay the MERGE immediately
+                await this.relay(target, relayText);
+                break;
+            case "delete":
+                if (this.pendingReads.has(action.filePath)) {
+                    // Block this DELETE until the MERGE completes
+                    const blockedBy = this.pendingReads.get(action.filePath);
+                    this.enqueue(target, relayText, action.filePath, blockedBy);
+                }
+                else {
+                    // No pending MERGE — relay immediately
+                    await this.relay(target, relayText);
+                }
+                break;
+            case "update-links":
+            case "freeform":
+                // Always relay immediately
+                await this.relay(target, relayText);
+                break;
+        }
+    }
+    /**
+     * Called when a worker completes.  Releases any queued decisions that
+     * were blocked on actions by that worker (identified by label match
+     * in the target).
+     */
+    releaseForWorker(workerLabel) {
+        const toRelease = this.queue.filter((q) => {
+            // Release decisions targeting this worker OR blocked by files
+            // this worker was responsible for merging
+            return q.target.label === workerLabel;
+        });
+        for (const item of toRelease) {
+            this.release(item, "worker-complete");
+        }
+    }
+    /**
+     * Returns the number of currently queued (blocked) decisions.
+     */
+    get pendingCount() {
+        return this.queue.length;
+    }
+    /**
+     * Returns a snapshot of pending reads (source → merge-target).
+     * Useful for testing.
+     */
+    get pendingReadSnapshot() {
+        return new Map(this.pendingReads);
+    }
+    // ── Internal ────────────────────────────────────────────────────────
+    /** Enqueue a blocked DELETE decision. */
+    enqueue(target, relayText, filePath, blockedBy) {
+        const timer = setTimeout(() => {
+            this.release(item, "timeout");
+        }, this.timeoutMs);
+        const item = {
+            target,
+            relayText,
+            action: "DELETE",
+            filePath,
+            blockedBy,
+            queuedAt: Date.now(),
+            timer,
+        };
+        this.queue.push(item);
+        if (this.verbose) {
+            this.verbose(`[decision-queue] DELETE ${filePath} queued — waiting for MERGE target ${blockedBy}`);
+        }
+    }
+    /**
+     * Release a single queued decision — relay it and remove from queue.
+     */
+    release(item, reason) {
+        const idx = this.queue.indexOf(item);
+        if (idx === -1)
+            return; // already released
+        this.queue.splice(idx, 1);
+        clearTimeout(item.timer);
+        // Remove the pending read since the MERGE is considered done
+        this.pendingReads.delete(item.filePath);
+        if (this.verbose) {
+            this.verbose(`[decision-queue] Releasing DELETE ${item.filePath} (${reason})`);
+        }
+        // Fire-and-forget — don't block the watcher/timer callback
+        void this.relay(item.target, item.relayText).catch((err) => {
+            if (this.verbose) {
+                this.verbose(`[decision-queue] Relay failed for DELETE ${item.filePath}: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        });
+    }
+    /** Release all queued decisions (used during stop/cleanup). */
+    releaseAll(reason) {
+        // Copy to avoid mutation during iteration
+        const items = [...this.queue];
+        for (const item of items) {
+            this.release(item, reason);
+        }
+    }
+    /**
+     * Filesystem watcher callback.  When a file in the destination directory
+     * changes, check if it matches a merge target and release blocked DELETEs.
+     */
+    onFileChange(relPath) {
+        // Check if any queued DELETE is blocked by this file being written
+        const toRelease = this.queue.filter((q) => q.blockedBy === relPath);
+        if (toRelease.length > 0 && this.verbose) {
+            this.verbose(`[decision-queue] File change detected: ${relPath} — releasing ${toRelease.length} queued decision(s)`);
+        }
+        for (const item of toRelease) {
+            this.release(item, "fs-watch");
+        }
+    }
+}

package/dist/diff-scope.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Diff-based scoping for incremental documentation generation.
+ *
+ * When a previous Hem run's commit SHA is available in `changelog.md`,
+ * this module filters the discovered source files down to only those
+ * that changed since the last run. This enables incremental generation
+ * where only modified/added source files are re-documented.
+ *
+ * Reference: specs/004-changelog-diff-scope.
+ */
+import type { FileInfo } from "./types.js";
+/**
+ * Filters discovered source files to only those that changed since the
+ * last Hem run.
+ *
+ * Matches are performed by comparing `FileInfo.path` (relative to source
+ * root) against the paths returned by `git diff --name-only`. The
+ * comparison normalizes forward slashes for cross-platform consistency.
+ *
+ * @param allFiles      - All discovered source files from the discovery phase.
+ * @param changedPaths  - Relative paths of changed files from `computeChangedFiles()`.
+ * @returns The subset of `allFiles` whose paths appear in `changedPaths`.
+ */
+export declare function scopeToChangedFiles(allFiles: FileInfo[], changedPaths: string[]): FileInfo[];

package/dist/diff-scope.js

@@ -0,0 +1,28 @@
+/**
+ * Diff-based scoping for incremental documentation generation.
+ *
+ * When a previous Hem run's commit SHA is available in `changelog.md`,
+ * this module filters the discovered source files down to only those
+ * that changed since the last run. This enables incremental generation
+ * where only modified/added source files are re-documented.
+ *
+ * Reference: specs/004-changelog-diff-scope.
+ */
+// ── Scope to Changed Files ────────────────────────────────────────────
+/**
+ * Filters discovered source files to only those that changed since the
+ * last Hem run.
+ *
+ * Matches are performed by comparing `FileInfo.path` (relative to source
+ * root) against the paths returned by `git diff --name-only`. The
+ * comparison normalizes forward slashes for cross-platform consistency.
+ *
+ * @param allFiles      - All discovered source files from the discovery phase.
+ * @param changedPaths  - Relative paths of changed files from `computeChangedFiles()`.
+ * @returns The subset of `allFiles` whose paths appear in `changedPaths`.
+ */
+export function scopeToChangedFiles(allFiles, changedPaths) {
+    // Build a set for O(1) lookup, normalizing path separators
+    const changedSet = new Set(changedPaths.map((p) => p.replace(/\\/g, "/")));
+    return allFiles.filter((f) => changedSet.has(f.path.replace(/\\/g, "/")));
+}

package/dist/discovery.d.ts

@@ -0,0 +1,54 @@
+/**
+ * File discovery module for Hem.
+ *
+ * Scans a source directory for files matching a glob pattern, classifies
+ * each file as text or binary, and returns a `FileInfo[]` array.
+ *
+ * Also provides `detectProjectName()` for inferring the project name
+ * from manifest files (package.json, Cargo.toml, etc.) in ancestor
+ * directories of the source path.
+ *
+ * Reference: FR-002 (file scanning), FR-014 (binary detection), FR-015 (destination exclusion).
+ */
+import type { FileInfo } from "./types.js";
+/**
+ * Discovers source files within `sourcePath` that match the given glob
+ * pattern. Each file is classified as binary or text and returned as a
+ * `FileInfo` object.
+ *
+ * Key behaviours:
+ *   - `sourcePath` and `destinationPath` are resolved to absolute paths.
+ *   - Files within `destinationPath` are excluded when the destination
+ *     overlaps with the source (FR-015).
+ *   - Binary files are included in the returned array with
+ *     `isBinary: true` so callers can count/warn about them.
+ *
+ * @param sourcePath - Root directory to scan.
+ * @param globPattern - Glob pattern relative to `sourcePath`.
+ * @param destinationPath - Destination directory for generated docs
+ *   (excluded from results when overlapping with source).
+ * @returns Array of `FileInfo` objects for every matched file.
+ */
+export declare function discoverFiles(sourcePath: string, globPattern: string, destinationPath: string): Promise<FileInfo[]>;
+/**
+ * Detect the project name by walking up from the source directory
+ * and inspecting manifest files.
+ *
+ * Detection priority (first match wins):
+ *   1. `package.json` → `name` field (scoped names stripped)
+ *   2. `Cargo.toml` → `[package] name`
+ *   3. `pyproject.toml` → `[project] name` or `[tool.poetry] name`
+ *   4. `go.mod` → last segment of module path
+ *
+ * The walk starts at the parent of `sourcePath` (since `sourcePath`
+ * itself is typically `src/` or similar) and continues up to the
+ * git root or a maximum of {@link MAX_WALK_DEPTH} levels.
+ *
+ * If no manifest is found, falls back to the basename of the
+ * directory containing the first manifest-like file, the parent
+ * directory, or ultimately `basename(sourcePath)`.
+ *
+ * @param sourcePath - The absolute source directory path.
+ * @returns The detected project name.
+ */
+export declare function detectProjectName(sourcePath: string): Promise<string>;