@omnitron-dev/omnitron 0.3.3 → 0.4.6

package/dist/orchestrator/orchestrator.service.js

@@ -1,3 +1,4 @@
+import { configuredStartEntry, effectiveAppName, ensureNamespacedEntry, registeredAppName } from './app-name.js';
 import { Errors } from '@omnitron-dev/titan/errors';
 import { OWNER_CAPABILITIES } from '../daemon/execution-capabilities.js';
 import { parsePoolScaleRequest, assertPoolScaleTarget } from '../shared/pool-scale.js';
@@ -34,6 +35,7 @@ import { stackExecutionKey, executionScopeKey, executionScopeOwns } from '../sha
  *     → Two Netron instances: PM management plane + app HTTP data plane
  */
 import { ReadinessMonitor } from '../health/readiness.js';
+import { FileWatcher } from './file-watcher.js';
 import fs from 'node:fs';
 import { readDeclaredConfig, withDeclaredConfig } from '../project/declared-config.js';
 import path from 'node:path';
@@ -116,46 +118,7 @@ function mapStrategy(strategy) {
             return SupervisionStrategy.ONE_FOR_ONE;
     }
 }
-/**
- * Strip the `${project}/${stack}/` prefix from a handle key (or
- * leave a bare name untouched). Used to identify "the same physical
- * app under any registration form" so duplicate registrations from
- * stack mode + bare-name RPC paths can be deduplicated.
- *
- * Examples:
- *   effectiveAppName('omni/dev/payments') === 'payments'
- *   effectiveAppName('payments')           === 'payments'
- *   effectiveAppName('a/b/c/payments')     === 'payments'  (last segment)
- */
-export function effectiveAppName(key) {
-    const idx = key.lastIndexOf('/');
-    return idx >= 0 ? key.slice(idx + 1) : key;
-}
-/**
- * Auto-promote a bare `entry.name` to canonical `${project}/${stack}/${name}`
- * form when the env carries OMNITRON_PROJECT + OMNITRON_STACK. The stack-mode
- * boot path (ProjectService.startStack) already prefixes the name explicitly;
- * this helper closes every other entry point that could otherwise register a
- * bare-name handle which then becomes invisible to the stack-prefix-based
- * UI lookup (ProjectService.toStackInfo filters on `${project}/${stack}/`).
- *
- * Idempotent: if the entry already looks namespaced (or env lacks the keys),
- * the input is returned unchanged. An entry qualified under a DIFFERENT
- * project/stack is preserved too — that's operator intent (cross-project
- * administrative run), not a bug.
- */
-export function ensureNamespacedEntry(entry) {
-    const project = entry.env?.['OMNITRON_PROJECT'];
-    const stack = entry.env?.['OMNITRON_STACK'];
-    if (!project || !stack)
-        return entry;
-    const canonicalPrefix = `${project}/${stack}/`;
-    if (entry.name.startsWith(canonicalPrefix))
-        return entry;
-    if (entry.name.includes('/'))
-        return entry;
-    return { ...entry, name: `${canonicalPrefix}${entry.name}` };
-}
+export { effectiveAppName, ensureNamespacedEntry } from './app-name.js';
 const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /**
  * Startup budget for a child process that declares none of its own.
@@ -248,6 +211,7 @@ export class OrchestratorService extends EventEmitter {
     devMode = false;
     /** esbuild bundling pipeline — pre-bundles TS entry points for child processes */
     buildService = null;
+    fileWatcher = null;
     watchDisabled = false;
     watchSelection = null;
     bootstrapSources = new Map();
@@ -264,11 +228,42 @@ export class OrchestratorService extends EventEmitter {
             if (!enabled || !this.handles.get(name) || !this.allowsWatch(this.handles.get(name).entry))
                 state.watchSerial = 0;
         }
+        this.syncClassicWatches();
         await Promise.all([...this.bootstrapSources.keys()].map((name) => this.syncBootstrapWatch(name)));
     }
-    getBuildWatchedApps() {
-        return [...this.bootstrapSources].filter(([name]) => this.buildService?.isWatching(name))
-            .map(([name, source]) => ({ name, directory: path.dirname(source.path) }));
+    getWatchedApps() {
+        const registrations = [...(this.fileWatcher?.getWatchedApps() ?? []),
+            ...[...this.bootstrapSources].filter(([name]) => this.buildService?.isWatching(name))
+                .map(([name, source]) => ({ name, directory: path.dirname(source.path) }))];
+        return registrations.flatMap(watch => {
+            const handle = this.handles.get(watch.name);
+            return handle ? [{ ...watch, generation: handle.generation, definitionRevision: handle.definitionRevision }] : [];
+        });
+    }
+    /** Native subscriptions follow the same owned handles and policy as esbuild. */
+    syncClassicWatches() {
+        if (!this.devMode || this.watchDisabled || this.stoppingApps || !this.config) {
+            this.fileWatcher?.stop();
+            this.fileWatcher = null;
+            return;
+        }
+        const apps = [...this.handles.values()].filter(handle => {
+            if (handle.entry.bootstrap || !this.allowsWatch(handle.entry) || this.executionBusy(handle.name))
+                return false;
+            if (handle.status === 'online' && !this.appStopIntents.get(handle.name)?.stopped)
+                return true;
+            // An internal restart keeps its existing source subscription and pending
+            // edits. Re-arming fs.watch on every child generation loses macOS events.
+            return !['crashed', 'errored'].includes(handle.status) &&
+                Boolean(this.restartCoalescer.get(handle.name)?.inFlight) && this.fileWatcher?.isWatching(handle.entry);
+        }).map(handle => handle.entry);
+        const config = { ...this.config, apps };
+        if (this.fileWatcher)
+            this.fileWatcher.applyConfig(config);
+        else {
+            this.fileWatcher = new FileWatcher(this.logger, this, config, this.cwd);
+            this.fileWatcher.start();
+        }
     }
     async syncBootstrapWatch(name) {
         const source = this.bootstrapSources.get(name);
@@ -292,12 +287,42 @@ export class OrchestratorService extends EventEmitter {
         }, this.devMode);
     }
     /** Automatic triggers remain revocable while queued; manual restart is independent. */
-    async restartAfterChange(name) {
+    async restartAfterChange(name, expected) {
         const canonical = this.resolveAppName(name) ?? name;
+        if (expected && canonical !== name)
+            throw Errors.badRequest(`Source refresh requires the observed canonical application name '${canonical}'`);
         const handle = this.handles.get(canonical);
-        if (!handle || !this.allowsWatch(handle.entry))
-            return handle;
-        return this.restartAppCoalesced(canonical, true, 'watch');
+        if (expected && (!handle || handle.generation !== expected.generation || handle.definitionRevision !== expected.definitionRevision))
+            throw Errors.conflict(`Application '${canonical}' source binding changed; observe its watch status before retrying`);
+        if (!handle || !this.allowsWatch(handle.entry)) {
+            if (expected)
+                throw Errors.conflict(`Source refresh of '${canonical}' is excluded by the current watch policy`);
+            return undefined;
+        }
+        if (expected && !this.getWatchedApps().some(app => app.name === canonical &&
+            app.generation === expected.generation && app.definitionRevision === expected.definitionRevision))
+            throw Errors.conflict(`Source watcher for '${canonical}' is unavailable; SDK refresh was not started`);
+        const entry = handle.entry;
+        const definitionRevision = handle.definitionRevision;
+        const generation = handle.generation;
+        const result = await this.restartAppCoalesced(canonical, true, 'watch');
+        if (handle.definitionRevision !== definitionRevision || result.entry !== entry ||
+            result.generation === generation || result.status !== 'online') {
+            if (expected)
+                throw Errors.conflict(`Source refresh of '${canonical}' was cancelled or its source binding changed`);
+            return undefined;
+        }
+        const source = this.getWatchedApps().find(app => app.name === canonical &&
+            app.generation === result.generation && app.definitionRevision === result.definitionRevision);
+        if (expected && !source)
+            throw Errors.conflict(`Source watcher for '${canonical}' is unavailable after restart; SDK readiness is not confirmed`);
+        const process = this.getApp(canonical);
+        if (!source || !process || process.status !== 'online' || process.generation !== source.generation) {
+            if (expected)
+                throw Errors.conflict(`Source refresh of '${canonical}' has no matching online observation`);
+            return undefined;
+        }
+        return { process, source };
     }
     /** Cached build results per app name */
     buildResults = new Map();
@@ -449,6 +474,7 @@ export class OrchestratorService extends EventEmitter {
      */
     setConfig(config) {
         this.config = config;
+        this.syncClassicWatches();
         // Metrics polling is armed here rather than in `startAll` alone.
         //
         // It used to run only from `startAll`, which is the `omnitron start`
@@ -815,6 +841,7 @@ export class OrchestratorService extends EventEmitter {
                         throw new Error(`Cannot replace '${key}' with '${entry.name}': the previous application did not stop`, { cause: err });
                     }
                     this.handles.delete(key);
+                    this.syncClassicWatches();
                     if (key !== entry.name)
                         this.metricsBridge?.evictApp(key);
                     continue;
@@ -832,6 +859,7 @@ export class OrchestratorService extends EventEmitter {
             // processes still run: an errored app's other children and its pools.
             await this.tearDownApp(h);
             this.handles.delete(key);
+            this.syncClassicWatches();
             // Drop any metrics that were collected under the stale alias so it
             // doesn't haunt future snapshots as a ghost offline app.
             if (key !== entry.name)
@@ -868,6 +896,7 @@ export class OrchestratorService extends EventEmitter {
             handle.restartWindow = existing.restartWindow;
         }
         this.handles.set(entry.name, handle);
+        this.syncClassicWatches();
         // Arm metrics sampling here — the one place an app enters the registry.
         //
         // It used to be armed only from `startAll` (the `omnitron start` path).
@@ -940,6 +969,9 @@ export class OrchestratorService extends EventEmitter {
                 this.scheduleRecovery(entry, handle, cfg);
             throw failure;
         }
+        finally {
+            this.syncClassicWatches();
+        }
     }
     /**
      * Everything a handle still runs, stopped: its supervisor's children (or its
@@ -1030,8 +1062,12 @@ export class OrchestratorService extends EventEmitter {
             return active.promise;
         this.assertExecutionIdle(scope);
         // Reserve before invoking work, including its synchronous checks and first write.
-        const promise = Promise.resolve().then(work).finally(() => this.executionOperations.delete(key));
+        const promise = Promise.resolve().then(work).finally(() => {
+            this.executionOperations.delete(key);
+            this.syncClassicWatches();
+        });
         this.executionOperations.set(key, { id: operationId, fingerprint, promise });
+        this.syncClassicWatches();
         return promise;
     }
     /**
@@ -1106,6 +1142,7 @@ export class OrchestratorService extends EventEmitter {
                 }
             }
             this.persistState();
+            this.syncClassicWatches();
             for (const name of retired) {
                 this.emit('app:retired', name);
                 this.metricsBridge?.evictApp(name);
@@ -1389,7 +1426,12 @@ export class OrchestratorService extends EventEmitter {
         }
     }
     async stopApp(name, force = false, timeout = 10_000) {
+        return this.stopAppQueued(name, force, timeout);
+    }
+    async stopAppQueued(name, force = false, timeout = 10_000, preserveWatch = false) {
         const canonical = this.resolveAppName(name) ?? name;
+        if (!preserveWatch)
+            this.fileWatcher?.removeApp(canonical);
         // Outside the queue on purpose: the operator's intent lands the
         // moment the command arrives, not whenever the queue drains.
         // Unknown names need no retained intent. Start admission registers its
@@ -1426,6 +1468,7 @@ export class OrchestratorService extends EventEmitter {
             this.clearCoolDown(name);
         const operation = Promise.resolve().then(() => this.stopAllOnce(force));
         this.stoppingApps = operation;
+        this.syncClassicWatches();
         const settled = () => { if (this.stoppingApps === operation)
             this.stoppingApps = null; };
         void operation.then(settled, settled);
@@ -1520,11 +1563,13 @@ export class OrchestratorService extends EventEmitter {
             throw new Error('An installed stack start is active; its application entry cannot be changed');
         }
         handle.redefine(entry);
-        if (!this.allowsWatch(entry)) {
-            const state = this.restartCoalescer.get(handle.name);
-            if (state)
-                state.watchSerial = 0;
-        }
+        this.fileWatcher?.revokePending(handle.name);
+        // A pending source event belongs to the previous definition, even when
+        // the replacement also enables watching or restores the original path.
+        const state = this.restartCoalescer.get(handle.name);
+        if (state)
+            state.watchSerial = 0;
+        this.syncClassicWatches();
         await this.syncBootstrapWatch(handle.name);
         return true;
     }
@@ -1565,6 +1610,10 @@ export class OrchestratorService extends EventEmitter {
         const canonical = this.resolveAppName(name) ?? name;
         const handle = this.handles.get(canonical);
         if (!handle) {
+            // A queued source event cannot resurrect a registration removed while
+            // awaiting a cooldown or another lifecycle operation.
+            if (!fromExternal || cause === 'watch')
+                throw new Error(`Application '${canonical}' was removed before restart`);
             // A PROJECT-QUALIFIED name with no live handle is not this daemon's to
             // start, and matching it against the daemon's own config starts the
             // WRONG APP.
@@ -1584,35 +1633,14 @@ export class OrchestratorService extends EventEmitter {
             // this exact shape and names the command that works; this is the same
             // refusal on the same reasoning, at the point where "no live handle" is
             // already known.
-            const qualified = name.includes('/');
-            const ownProject = this.config?.project;
-            if (qualified && !ownProject) {
-                const [project, stack] = name.split('/');
-                throw new Error(`'${name}' belongs to a registered project, which this daemon restarts through its ` +
-                    `stack. Use \`omnitron stack start ${project} ${stack}\`, or restart the app by its ` +
-                    `bare name if you mean the daemon's own config.`);
-            }
-            // P1-F — fall through to startApp when there's no live handle but
-            // the ecosystem config declares the app. Operators expect
-            // `omnitron restart foo` to work even when `foo` was never started
-            // (or was stopped, or crashed and was reaped). Previously this
-            // threw "Unknown app", forcing the operator to first run start
-            // explicitly.
-            const effective = effectiveAppName(name);
-            const entry = this.config?.apps.find((a) => a.name === name || a.name === effective || effectiveAppName(a.name) === effective);
+            const entry = this.config && configuredStartEntry(name, this.config);
             if (entry) {
-                this.logger.info({ app: name }, 'restartApp: no live handle — falling through to startApp');
-                // Restart under the name the OPERATOR asked for. The match above
-                // accepts a bare entry for a qualified request, and handing it to
-                // `startApp` unchanged re-registers the app bare —
-                // `ensureNamespacedEntry` leaves an entry with no OMNITRON_PROJECT /
-                // OMNITRON_STACK env alone by design. The app then silently leaves its
-                // stack, because `ProjectService.toStackInfo` filters handles on the
-                // `${project}/${stack}/` prefix.
-                const requested = name.includes('/') && !entry.name.includes('/')
-                    ? { ...entry, name }
-                    : entry;
-                return this.startApp(requested);
+                this.logger.info({ app: entry.name }, 'Restarting declared application with no live handle');
+                return this.startApp(entry);
+            }
+            if (name.includes('/')) {
+                const [project, stack] = name.split('/');
+                throw new Error("Unknown app '" + name + "'. To start the registered stack use: omnitron stack start " + project + ' ' + stack);
             }
             throw new Error(`Unknown app: ${name}`);
         }
@@ -1621,12 +1649,17 @@ export class OrchestratorService extends EventEmitter {
         const state = this.getCoalescerState(name);
         if (fromExternal) {
             state.triggerSerial += 1;
-            if (cause === 'watch')
+            if (cause === 'watch') {
                 state.watchSerial = state.triggerSerial;
+                state.watchBinding = { handle, entry: handle.entry, definitionRevision: handle.definitionRevision };
+            }
             else
                 state.manualSerial = state.triggerSerial;
         }
-        if (!this.allowsWatch(handle.entry) || ['errored', 'crashed', 'stopped', 'stopping'].includes(handle.status)) {
+        if (!this.allowsWatch(handle.entry) || ['errored', 'crashed'].includes(handle.status) ||
+            (!state.inFlight && ['stopped', 'stopping'].includes(handle.status)) ||
+            !state.watchBinding || state.watchBinding.entry !== handle.entry ||
+            state.watchBinding.handle.definitionRevision !== state.watchBinding.definitionRevision) {
             state.watchSerial = 0;
         }
         const observedSerial = Math.max(state.manualSerial, state.watchSerial);
@@ -1728,7 +1761,7 @@ export class OrchestratorService extends EventEmitter {
         if (this.devMode && entry.bootstrap) {
             clearCacheFor(path.resolve(entry.cwd ?? this.cwd, entry.bootstrap));
         }
-        const stopped = this.stopApp(name);
+        const stopped = this.stopAppQueued(name, false, 10_000, true);
         const intent = this.appStopIntents.get(name);
         const generation = this.stopGeneration;
         await stopped;
@@ -1743,7 +1776,7 @@ export class OrchestratorService extends EventEmitter {
     getCoalescerState(name) {
         let state = this.restartCoalescer.get(name);
         if (!state) {
-            state = { inFlight: null, triggerSerial: 0, firedSerial: 0, manualSerial: 0, watchSerial: 0, cooldownUntil: 0 };
+            state = { inFlight: null, triggerSerial: 0, firedSerial: 0, manualSerial: 0, watchSerial: 0, watchBinding: null, cooldownUntil: 0 };
             this.restartCoalescer.set(name, state);
         }
         return state;
@@ -1778,29 +1811,9 @@ export class OrchestratorService extends EventEmitter {
     resolveAppName(rawName) {
         if (this.handles.has(rawName))
             return rawName;
-        // A qualified name is an ADDRESS, and the only handle that satisfies it
-        // is the one with that address. Falling through to the short-name match
-        // answered a question about one deployment with another's data:
-        //
-        //     omnitron logs sample/deployed/main   →  sample/dev/main
-        //
-        // — the node's app asked for, this laptop's app printed, with nothing
-        // said. `startApp` already refuses this for the same reason ("a
-        // PROJECT-QUALIFIED name with no live handle is not this daemon's to
-        // start"); the resolver every other caller goes through did not.
         if (rawName.includes('/'))
             return undefined;
-        const effective = effectiveAppName(rawName);
-        const matches = [];
-        for (const key of this.handles.keys()) {
-            if (effectiveAppName(key) === effective)
-                matches.push(key);
-        }
-        if (matches.length === 0)
-            return undefined;
-        if (matches.length === 1)
-            return matches[0];
-        throw new Error(`App name '${rawName}' is ambiguous — multiple matches: ${matches.join(', ')}. Use the full canonical name.`);
+        return registeredAppName(rawName, this.handles.keys());
     }
     /**
      * Reload app with zero-downtime strategy:
@@ -2213,6 +2226,7 @@ export class OrchestratorService extends EventEmitter {
                 this.scheduleRecovery(entry, handle, config);
             }
             this.persistState();
+            this.syncClassicWatches();
         });
         return launch;
     }
@@ -2237,7 +2251,7 @@ export class OrchestratorService extends EventEmitter {
         let topology;
         let definition;
         try {
-            const captured = await readBootstrapMetadata(bootstrapAbsPath, this.devMode && !entry.releaseRoot, 60_000);
+            const captured = await readBootstrapMetadata(bootstrapAbsPath, this.devMode && !entry.releaseRoot, 60_000, entry.releaseRoot);
             definition = captured.definition;
             handle.bootstrapMetadata = captured.metadata;
             topology = definition.processes;
@@ -2330,6 +2344,7 @@ export class OrchestratorService extends EventEmitter {
         return launch;
     }
     async discardSourceBuild(name) {
+        this.fileWatcher?.removeApp(name);
         const state = this.restartCoalescer.get(name);
         if (state)
             state.watchSerial = 0;
@@ -2389,7 +2404,8 @@ export class OrchestratorService extends EventEmitter {
         // application generation. Resolve before allocating topology resources.
         const runtime = await this.bootstrapRuntime(bootstrapAbsPath);
         const policy = buildRestartPolicy(entry, config);
-        const spawnEnv = withClosureLibraryPath(entry.env, [entry.releaseRoot, entry.cwd, bootstrapAbsPath]);
+        const spawnEnv = withClosureLibraryPath(entry.env, entry.releaseRoot
+            ? { verifiedRoot: entry.releaseRoot } : [entry.cwd, bootstrapAbsPath]);
         const needsRouting = topology.some(proc => proc.topology?.expose === true || proc.topology?.access?.length);
         handle.deferStartup();
         if (needsRouting) {
@@ -2631,7 +2647,8 @@ export class OrchestratorService extends EventEmitter {
     async launchSingleProcess(entry, handle, config, bootstrapAbsPath, buildResult) {
         const instances = entry.instances ?? 1;
         const policy = buildRestartPolicy(entry, config);
-        const spawnEnv = withClosureLibraryPath(entry.env, [entry.releaseRoot, entry.cwd, bootstrapAbsPath]);
+        const spawnEnv = withClosureLibraryPath(entry.env, entry.releaseRoot
+            ? { verifiedRoot: entry.releaseRoot } : [entry.cwd, bootstrapAbsPath]);
         // For single-process apps, use the first (only) process entry's bundled module
         const firstProcessName = buildResult?.modulePaths.keys().next().value;
         const bundledModulePath = firstProcessName
@@ -2751,6 +2768,7 @@ export class OrchestratorService extends EventEmitter {
         handle.clearLastExit();
         this.startPostOnlineCooldown(name);
         this.persistState();
+        this.syncClassicWatches();
         this.emit('app:online', name);
     }
     /**