@smithers-orchestrator/engine 0.20.3 → 0.21.0

package/src/hot/overlay.js

@@ -13,6 +13,17 @@ const DEFAULT_EXCLUDE = [
     ".smithers",
     ".DS_Store",
 ];
+
+function hotOverlayError(cause, operation, details) {
+    return toSmithersError(cause, operation, {
+        code: "HOT_OVERLAY_FAILED",
+        details,
+    });
+}
+
+function hotOverlayErrorMapper(operation, details) {
+    return (cause) => hotOverlayError(cause, operation, details);
+}
 /**
  * Build a generation overlay by hardlinking (or copying) the hot root
  * tree into a new generation directory.
@@ -31,9 +42,10 @@ export function buildOverlayEffect(hotRoot, outDir, generation, opts) {
     return Effect.gen(function* () {
         yield* Effect.tryPromise({
             try: () => mkdir(genDir, { recursive: true }),
-            catch: (cause) => toSmithersError(cause, "create hot overlay generation dir", {
-                code: "HOT_OVERLAY_FAILED",
-                details: { hotRoot, outDir, generation },
+            catch: hotOverlayErrorMapper("create hot overlay generation dir", {
+                hotRoot,
+                outDir,
+                generation,
             }),
         });
         yield* mirrorTreeEffect(hotRoot, genDir, exclude);
@@ -63,9 +75,9 @@ function mirrorTreeEffect(src, dest, exclude) {
     return Effect.gen(function* () {
         const entries = yield* Effect.tryPromise({
             try: () => readdir(src, { withFileTypes: true }),
-            catch: (cause) => toSmithersError(cause, "read hot overlay source dir", {
-                code: "HOT_OVERLAY_FAILED",
-                details: { src, dest },
+            catch: hotOverlayErrorMapper("read hot overlay source dir", {
+                src,
+                dest,
             }),
         });
         for (const entry of entries) {
@@ -78,9 +90,9 @@ function mirrorTreeEffect(src, dest, exclude) {
             if (entry.isDirectory()) {
                 yield* Effect.tryPromise({
                     try: () => mkdir(destPath, { recursive: true }),
-                    catch: (cause) => toSmithersError(cause, "create mirrored hot overlay dir", {
-                        code: "HOT_OVERLAY_FAILED",
-                        details: { srcPath, destPath },
+                    catch: hotOverlayErrorMapper("create mirrored hot overlay dir", {
+                        srcPath,
+                        destPath,
                     }),
                 });
                 yield* mirrorTreeEffect(srcPath, destPath, exclude);
@@ -88,24 +100,24 @@ function mirrorTreeEffect(src, dest, exclude) {
             else if (entry.isFile()) {
                 const linked = yield* Effect.either(Effect.tryPromise({
                     try: () => link(srcPath, destPath),
-                    catch: (cause) => toSmithersError(cause, "hardlink overlay file", {
-                        code: "HOT_OVERLAY_FAILED",
-                        details: { srcPath, destPath },
+                    catch: hotOverlayErrorMapper("hardlink overlay file", {
+                        srcPath,
+                        destPath,
                     }),
                 }));
                 if (linked._tag === "Left") {
                     yield* Effect.tryPromise({
                         try: () => mkdir(dirname(destPath), { recursive: true }),
-                        catch: (cause) => toSmithersError(cause, "create overlay file parent dir", {
-                            code: "HOT_OVERLAY_FAILED",
-                            details: { srcPath, destPath },
+                        catch: hotOverlayErrorMapper("create overlay file parent dir", {
+                            srcPath,
+                            destPath,
                         }),
                     });
                     yield* Effect.tryPromise({
                         try: () => copyFile(srcPath, destPath),
-                        catch: (cause) => toSmithersError(cause, "copy overlay file", {
-                            code: "HOT_OVERLAY_FAILED",
-                            details: { srcPath, destPath },
+                        catch: hotOverlayErrorMapper("copy overlay file", {
+                            srcPath,
+                            destPath,
                         }),
                     });
                 }
@@ -126,9 +138,9 @@ export function cleanupGenerationsEffect(outDir, keepLast) {
             return;
         const entries = yield* Effect.tryPromise({
             try: () => readdir(outDir, { withFileTypes: true }),
-            catch: (cause) => toSmithersError(cause, "read hot overlay generations", {
-                code: "HOT_OVERLAY_FAILED",
-                details: { outDir, keepLast },
+            catch: hotOverlayErrorMapper("read hot overlay generations", {
+                outDir,
+                keepLast,
             }),
         });
         const genDirs = entries
@@ -143,9 +155,9 @@ export function cleanupGenerationsEffect(outDir, keepLast) {
         for (const dir of toRemove) {
             yield* Effect.either(Effect.tryPromise({
                 try: () => rm(join(outDir, dir.name), { recursive: true, force: true }),
-                catch: (cause) => toSmithersError(cause, "remove stale hot overlay generation", {
-                    code: "HOT_OVERLAY_FAILED",
-                    details: { outDir, generationDir: dir.name },
+                catch: hotOverlayErrorMapper("remove stale hot overlay generation", {
+                    outDir,
+                    generationDir: dir.name,
                 }),
             }));
         }
@@ -175,3 +187,9 @@ export function resolveOverlayEntry(entryPath, hotRoot, genDir) {
     const rel = relative(hotRoot, entryPath);
     return resolve(genDir, rel);
 }
+
+export const __overlayInternals = {
+    hotOverlayError,
+    hotOverlayErrorMapper,
+    mirrorTreeEffect,
+};

package/src/hot/watch.js

@@ -1,6 +1,6 @@
 import { watch } from "node:fs";
-import { readdir } from "node:fs/promises";
-import { resolve } from "node:path";
+import { readdir, stat } from "node:fs/promises";
+import { basename, resolve } from "node:path";
 import { Effect } from "effect";
 import { toSmithersError } from "@smithers-orchestrator/errors/toSmithersError";
 import { logDebug, logInfo } from "@smithers-orchestrator/observability/logging";
@@ -12,13 +12,27 @@ const DEFAULT_IGNORE = [
     ".jj",
     ".smithers",
 ];
+const MIN_POLL_MS = 1000;
+const MAX_POLL_MS = 10_000;
+const MAX_POLL_FILES = 5000;
+class HotWatchScanLimitError extends Error {
+    constructor() {
+        super(`Hot watch polling skipped after ${MAX_POLL_FILES} files.`);
+        this.name = "HotWatchScanLimitError";
+    }
+}
 export class WatchTree {
     watchers = [];
     rootDir;
     ignore;
     debounceMs;
     changedFiles = new Set();
+    fileSignatures = new Map();
     debounceTimer = null;
+    pollTimer = null;
+    polling = false;
+    pollingDisabled = false;
+    currentPollIntervalMs = MIN_POLL_MS;
     waitResolve = null;
     closed = false;
     /**
@@ -53,6 +67,8 @@ export class WatchTree {
         this.closed = true;
         if (this.debounceTimer)
             clearTimeout(this.debounceTimer);
+        if (this.pollTimer)
+            clearTimeout(this.pollTimer);
         for (const w of this.watchers) {
             try {
                 w.close();
@@ -71,7 +87,31 @@ export class WatchTree {
     }
     startEffect() {
         return Effect.tryPromise({
-            try: () => this.watchDir(this.rootDir),
+            try: async () => {
+                try {
+                    this.fileSignatures = await this.scanFileSignatures(this.rootDir);
+                }
+                catch (error) {
+                    this.pollingDisabled = true;
+                    this.fileSignatures = new Map();
+                    if (error instanceof HotWatchScanLimitError) {
+                        logInfo("hot watch polling disabled by file count guard", {
+                            rootDir: this.rootDir,
+                            maxFiles: MAX_POLL_FILES,
+                        }, "hot:watch");
+                    }
+                    else {
+                        logInfo("hot watch polling disabled after initial scan failed", {
+                            rootDir: this.rootDir,
+                            errorName: error instanceof Error ? error.name : typeof error,
+                        }, "hot:watch");
+                    }
+                }
+                await this.watchDir(this.rootDir);
+                if (!this.pollingDisabled) {
+                    this.startPolling();
+                }
+            },
             catch: (cause) => toSmithersError(cause, "start hot watch tree"),
         }).pipe(Effect.annotateLogs({
             rootDir: this.rootDir,
@@ -105,6 +145,122 @@ export class WatchTree {
     shouldIgnore(name) {
         return this.ignore.includes(name) || name.startsWith(".");
     }
+    pollIntervalMs() {
+        return Math.min(MAX_POLL_MS, Math.max(MIN_POLL_MS, this.debounceMs * 4));
+    }
+    resetPollBackoff() {
+        this.currentPollIntervalMs = this.pollIntervalMs();
+    }
+    advancePollBackoff(changed) {
+        if (changed) {
+            this.resetPollBackoff();
+            return;
+        }
+        this.currentPollIntervalMs = Math.min(MAX_POLL_MS, Math.max(this.pollIntervalMs(), this.currentPollIntervalMs * 2));
+    }
+    scheduleNextPoll() {
+        if (this.pollTimer || this.closed || this.pollingDisabled)
+            return;
+        this.pollTimer = setTimeout(() => {
+            this.pollTimer = null;
+            void this.pollOnce().finally(() => {
+                this.scheduleNextPoll();
+            });
+        }, this.currentPollIntervalMs);
+    }
+    startPolling() {
+        if (this.pollTimer || this.closed || this.pollingDisabled)
+            return;
+        this.resetPollBackoff();
+        this.scheduleNextPoll();
+    }
+    async pollOnce() {
+        if (this.closed || this.polling || this.pollingDisabled)
+            return false;
+        this.polling = true;
+        try {
+            const next = await this.scanFileSignatures(this.rootDir);
+            const changed = this.recordScanChanges(next);
+            this.advancePollBackoff(changed);
+            return changed;
+        }
+        catch (error) {
+            if (error instanceof HotWatchScanLimitError) {
+                this.pollingDisabled = true;
+                logInfo("hot watch polling disabled by file count guard", {
+                    rootDir: this.rootDir,
+                    maxFiles: MAX_POLL_FILES,
+                }, "hot:watch");
+            }
+            else {
+                this.advancePollBackoff(false);
+            }
+            // Ignore transient filesystem races; the next interval will retry.
+            return false;
+        }
+        finally {
+            this.polling = false;
+        }
+    }
+    /**
+   * @param {string} dir
+   * @returns {Promise<Map<string, string>>}
+   */
+    async scanFileSignatures(dir) {
+        const files = new Map();
+        await this.scanDir(dir, files);
+        return files;
+    }
+    /**
+   * @param {string} dir
+   * @param {Map<string, string>} files
+   * @returns {Promise<void>}
+   */
+    async scanDir(dir, files) {
+        if (this.closed)
+            return;
+        const baseName = basename(dir);
+        if (baseName && this.shouldIgnore(baseName) && dir !== this.rootDir)
+            return;
+        const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
+        for (const entry of entries) {
+            if (this.shouldIgnore(entry.name))
+                continue;
+            if (files.size >= MAX_POLL_FILES) {
+                throw new HotWatchScanLimitError();
+            }
+            const fullPath = resolve(dir, entry.name);
+            if (entry.isDirectory()) {
+                await this.scanDir(fullPath, files);
+            }
+            else if (entry.isFile()) {
+                const info = await stat(fullPath).catch(() => null);
+                if (info?.isFile()) {
+                    files.set(fullPath, `${info.mtimeMs}:${info.size}`);
+                }
+            }
+        }
+    }
+    /**
+   * @param {Map<string, string>} next
+   */
+    recordScanChanges(next) {
+        let changed = false;
+        for (const [filePath, signature] of next) {
+            if (this.fileSignatures.get(filePath) !== signature) {
+                this.onFileChange(filePath);
+                changed = true;
+            }
+        }
+        for (const filePath of this.fileSignatures.keys()) {
+            if (!next.has(filePath)) {
+                this.onFileChange(filePath);
+                changed = true;
+            }
+        }
+        this.fileSignatures = next;
+        return changed;
+    }
     /**
    * @param {string} dir
    * @returns {Promise<void>}
@@ -112,7 +268,7 @@ export class WatchTree {
     async watchDir(dir) {
         if (this.closed)
             return;
-        const baseName = dir.split("/").pop() ?? "";
+        const baseName = basename(dir);
         if (baseName && this.shouldIgnore(baseName) && dir !== this.rootDir)
             return;
         try {
@@ -130,6 +286,8 @@ export class WatchTree {
                     fullPath,
                 }, "hot:watch");
                 this.onFileChange(fullPath);
+                this.resetPollBackoff();
+                void this.pollOnce();
             });
             this.watchers.push(watcher);
             // Recursively watch subdirectories

package/src/human-requests.js

@@ -118,3 +118,6 @@ export function validateHumanRequestValue(request, value) {
     }
     return { ok: true };
 }
+export const __humanRequestInternals = {
+    formatValidationIssues,
+};

package/src/workflow-hash.js

@@ -0,0 +1,157 @@
+import { existsSync, statSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { createHash } from "node:crypto";
+import { SmithersError } from "@smithers-orchestrator/errors/SmithersError";
+
+const STATIC_IMPORT_RE = /\b(?:import|export)\s+(?:[^"'`]*?\s+from\s*)?["']([^"']+)["']/g;
+const DYNAMIC_IMPORT_RE = /\bimport\s*\(\s*["']([^"']+)["']\s*\)/g;
+const WORKFLOW_IMPORT_EXTENSIONS = [
+    "",
+    ".ts",
+    ".tsx",
+    ".mts",
+    ".cts",
+    ".js",
+    ".jsx",
+    ".mjs",
+    ".cjs",
+];
+
+/**
+ * @param {string} input
+ * @returns {string}
+ */
+export function sha256Hex(input) {
+    return createHash("sha256").update(input).digest("hex");
+}
+
+/**
+ * @param {string | null | undefined} sourcePath
+ */
+export function getWorkflowImportScanLoader(sourcePath) {
+    const lower = sourcePath?.toLowerCase() ?? "";
+    if (lower.endsWith(".tsx"))
+        return "tsx";
+    if (lower.endsWith(".jsx"))
+        return "jsx";
+    if (lower.endsWith(".ts") ||
+        lower.endsWith(".mts") ||
+        lower.endsWith(".cts")) {
+        return "ts";
+    }
+    return "js";
+}
+
+/**
+ * @param {string | null} workflowPath
+ * @returns {Promise<string | null>}
+ */
+export async function readWorkflowEntryHash(workflowPath) {
+    if (!workflowPath)
+        return null;
+    try {
+        const raw = await readFile(workflowPath, "utf8");
+        return sha256Hex(raw);
+    }
+    catch {
+        return null;
+    }
+}
+
+/**
+ * @param {string} source
+ * @param {string | null} [sourcePath]
+ * @returns {string[]}
+ */
+export function extractWorkflowImportSpecifiers(source, sourcePath) {
+    if (typeof Bun !== "undefined" && typeof Bun.Transpiler === "function") {
+        try {
+            const scanned = new Bun.Transpiler({
+                loader: getWorkflowImportScanLoader(sourcePath),
+            }).scanImports(source);
+            const specifiers = new Set();
+            for (const entry of scanned) {
+                const specifier = entry?.path?.trim();
+                if (specifier?.startsWith(".")) {
+                    specifiers.add(specifier);
+                }
+            }
+            return [...specifiers];
+        }
+        catch {
+            // Fall back to regex scanning if Bun's parser cannot handle the source.
+        }
+    }
+    const specifiers = new Set();
+    for (const pattern of [STATIC_IMPORT_RE, DYNAMIC_IMPORT_RE]) {
+        pattern.lastIndex = 0;
+        let match;
+        while ((match = pattern.exec(source)) !== null) {
+            const specifier = match[1]?.trim();
+            if (!specifier?.startsWith("."))
+                continue;
+            specifiers.add(specifier);
+        }
+    }
+    return [...specifiers];
+}
+
+/**
+ * @param {string} baseFile
+ * @param {string} specifier
+ * @returns {string | null}
+ */
+export function resolveWorkflowImport(baseFile, specifier) {
+    const basePath = resolve(dirname(baseFile), specifier);
+    const candidates = [
+        ...WORKFLOW_IMPORT_EXTENSIONS.map((ext) => `${basePath}${ext}`),
+        ...WORKFLOW_IMPORT_EXTENSIONS
+            .filter((ext) => ext.length > 0)
+            .map((ext) => resolve(basePath, `index${ext}`)),
+    ];
+    for (const candidate of candidates) {
+        if (existsSync(candidate) && statSync(candidate).isFile()) {
+            return resolve(candidate);
+        }
+    }
+    return null;
+}
+
+/**
+ * @param {string} workflowPath
+ * @returns {Promise<string[]>}
+ */
+async function collectWorkflowModuleHashEntries(workflowPath, visited = new Set()) {
+    const resolvedPath = resolve(workflowPath);
+    if (visited.has(resolvedPath)) {
+        return [];
+    }
+    visited.add(resolvedPath);
+    const source = await readFile(resolvedPath, "utf8");
+    const entries = [`${resolvedPath}:${sha256Hex(source)}`];
+    for (const specifier of extractWorkflowImportSpecifiers(source, resolvedPath)) {
+        const importedPath = resolveWorkflowImport(resolvedPath, specifier);
+        if (!importedPath) {
+            throw new SmithersError("WORKFLOW_HASH_RESOLUTION_FAILED", `Unable to resolve workflow import "${specifier}" from ${resolvedPath}.`, { workflowPath: resolvedPath, specifier });
+        }
+        entries.push(...(await collectWorkflowModuleHashEntries(importedPath, visited)));
+    }
+    return entries;
+}
+
+/**
+ * @param {string | null} workflowPath
+ * @returns {Promise<string | null>}
+ */
+export async function readWorkflowGraphHash(workflowPath) {
+    if (!workflowPath)
+        return null;
+    try {
+        const entries = await collectWorkflowModuleHashEntries(workflowPath);
+        return sha256Hex(entries.sort().join("|"));
+    }
+    catch {
+        return null;
+    }
+}