@harperfast/harper 5.3.0-beta.3 → 5.3.0-beta.4

package/bin/cliOperations.ts

@@ -978,27 +978,37 @@ async function cliOperations(req: any, skipResponseLog = false) {
 		// when response headers arrive (streamResponse: true), which happens before the full
 		// upload completes — calling endUpload() here would snap the bar prematurely.
 
+		const { statusCode } = response;
+		const failedStatus = statusCode < 200 || (statusCode >= 300 && statusCode !== 304);
+		const eventStream = useSse && response.headers['content-type']?.startsWith('text/event-stream');
 		let responseData;
-		if (useSse && response.headers['content-type']?.startsWith('text/event-stream')) {
+		if (eventStream && !failedStatus) {
 			// Consume SSE: render phase events live, capture the final result from the `done`
 			// event (or the error message from the `error` event). The HTTP status stays 200
 			// until end-of-stream; failures are signaled in-band.
 			let finalResult;
 			let sseError;
+			let terminated = false;
+			// Older servers write a result as unnamed frames, with no `done` event.
+			let lastUnnamedData: string | undefined;
 			for await (const message of parseSSE(response)) {
 				renderer?.renderEvent(message);
 				if (message.event === 'done') {
+					terminated = true;
 					try {
 						finalResult = JSON.parse(message.data)?.result;
 					} catch {
 						finalResult = message.data;
 					}
 				} else if (message.event === 'error') {
+					terminated = true;
 					try {
 						sseError = JSON.parse(message.data);
 					} catch {
 						sseError = { message: message.data };
 					}
+				} else if (message.event === 'message') {
+					lastUnnamedData = message.data;
 				}
 			}
 			if (sseError) {
@@ -1006,6 +1016,13 @@ async function cliOperations(req: any, skipResponseLog = false) {
 				console.error(`error: ${errMsg}`);
 				process.exit(1);
 			}
+			if (finalResult === undefined && !terminated && lastUnnamedData !== undefined) {
+				try {
+					finalResult = JSON.parse(lastUnnamedData);
+				} catch {
+					finalResult = { message: lastUnnamedData };
+				}
+			}
 			responseData = finalResult ?? { message: 'Deploy completed (no result payload).' };
 		} else {
 			// When useSse is true, httpRequest returns a raw IncomingMessage (streamResponse mode),
@@ -1018,6 +1035,12 @@ async function cliOperations(req: any, skipResponseLog = false) {
 			} else {
 				bodyText = response.body;
 			}
+			// A request refused before its stream started can still arrive labeled as an event stream,
+			// with the error written as an unnamed frame (older servers). Unwrap it; a body with no
+			// frames at all (a proxy's error page) stays as it came.
+			if (eventStream) {
+				for await (const message of parseSSE(Readable.from([bodyText]))) bodyText = message.data;
+			}
 			try {
 				responseData = JSON.parse(bodyText);
 			} catch {
@@ -1035,8 +1058,7 @@ async function cliOperations(req: any, skipResponseLog = false) {
 			responseLog = YAML.stringify(responseData).trim();
 		}
 
-		const { statusCode } = response;
-		if (statusCode < 200 || (statusCode >= 300 && statusCode !== 304)) {
+		if (failedStatus) {
 			const errorPrefix = responseLog.startsWith('error:') ? '' : 'error: ';
 			console.error(`${errorPrefix}${responseLog}`);
 			process.exit(1);

package/bin/run.ts

@@ -26,6 +26,8 @@ import { startHTTPThreads } from '../server/threads/socketRouter.ts';
 import { beginProcessShutdown } from '../server/threads/manageThreads.js';
 import * as hdbInfoController from '../dataLayer/hdbInfoController.ts';
 import { isReadOnlyMode } from '../resources/databases.ts';
+import { declareTokenUseTable } from '../security/authn/oidc/tokenUseTable.ts';
+import { ensureCertificateVerificationTables } from '../security/certificateVerification/verificationTables.ts';
 import { getThisNodeName, getThisNodeHostname } from '../server/nodeName.ts';
 import * as hdbTerms from '../utility/hdbTerms.ts';
 import { getHdbPid, isProcessRunning } from '../utility/processManagement/processManagement.js';
@@ -184,6 +186,22 @@ async function initialize(calledByInstall = false, calledByMain = false) {
 		process.exit(1);
 	}
 
+	// Every boot: data already at a 5.3.0 pre-release never runs the 5.3.0 directive (compareVersions sorts
+	// those versions above 5.3.0). Before worker threads start, so they load the table with its expiration.
+	if (!isReadOnlyMode()) {
+		try {
+			declareTokenUseTable();
+		} catch (error) {
+			hdbLogger.error('Starting without the declared system.hdb_oidc_token_use; the next start retries', error);
+		}
+		// A node that never verifies a client certificate still holds the rows its peers replicate.
+		try {
+			await ensureCertificateVerificationTables();
+		} catch (error) {
+			hdbLogger.error('Starting without the declared certificate verification tables; the next start retries', error);
+		}
+	}
+
 	// A built-in component only activates when its config key is present. Fresh installs get these
 	// keys from defaultConfig.yaml, but an in-place upgrade carries the old config forward without a
 	// key added in a newer release (harper-pro#585). Ensure them on every boot — idempotent and

package/components/Application.ts

@@ -55,6 +55,10 @@ import {
 	writeFile,
 } from 'node:fs/promises';
 import { spawn, type ChildProcess } from 'node:child_process';
+import {
+	confirmWindowsProcessTreeGone,
+	type WindowsProcessTreeIdentity,
+} from '../server/threads/windowsProcessTree.ts';
 import { tmpdir } from 'node:os';
 import { randomUUID } from 'node:crypto';
 import { chmodSync, createReadStream, existsSync, lstatSync, renameSync } from 'node:fs';
@@ -610,6 +614,19 @@ const RECOVERY_LOCK_WAIT = {
 const COMPONENT_RECOVERY_LOCK_PURPOSE = 'component-recovery';
 const MAX_GIT_EXTRACTION_COMMANDS = 4;
 const MAX_INSTALL_COMMANDS = 2;
+
+/**
+ * The longest one preparation may legitimately run: the longest extraction path runs clone, tag listing,
+ * checkout, and npm pack, and a custom package manager configured to warn can then fall back to npm, yielding
+ * two install commands. What a preparation lock waits on, and what an origin allows a peer's replicated deploy.
+ */
+export function componentPreparationBudgetMs(installTimeoutMs: number = DEFAULT_COMMAND_TIMEOUT_MS): number {
+	return (
+		MAX_GIT_EXTRACTION_COMMANDS * DEFAULT_COMMAND_TIMEOUT_MS +
+		MAX_INSTALL_COMMANDS * installTimeoutMs +
+		COMPONENT_PREPARATION_WAIT_MARGIN_MS
+	);
+}
 const PRODUCTION_DEPENDENCY_FIELDS = ['dependencies', 'optionalDependencies', 'peerDependencies'] as const;
 const INSTALL_LIFECYCLE_SCRIPTS = new Set([
 	'preinstall',
@@ -4474,13 +4491,8 @@ export async function prepareApplication(application: Application, options: Prep
 				}
 			},
 			{
-				// The longest extraction path runs clone, tag listing, checkout, and npm pack. A custom
-				// package manager configured to warn can then fall back to npm, yielding two install
-				// commands. Bound orphaned same-process worker locks without rejecting behind a valid holder.
-				timeoutMs:
-					MAX_GIT_EXTRACTION_COMMANDS * DEFAULT_COMMAND_TIMEOUT_MS +
-					MAX_INSTALL_COMMANDS * commandTimeoutMs +
-					COMPONENT_PREPARATION_WAIT_MARGIN_MS,
+				// Bound orphaned same-process worker locks without rejecting behind a valid holder.
+				timeoutMs: componentPreparationBudgetMs(commandTimeoutMs),
 				onWait: (owner) =>
 					application.logger.info(
 						`Waiting for in-progress preparation of ${application.name}` +
@@ -5209,6 +5221,10 @@ function spawnWithEnv(
 		}
 		const spawnLogger = logger.loggerWithTag(`${applicationName}:spawn:${command}`);
 
+		// The child is created inside spawn(), so the interval around the call bounds its creation
+		// time exactly — what the Windows tree scan needs to tell its children from an orphan of the
+		// PID's previous owner.
+		const spawnStartedAt = Date.now();
 		const childProcess = spawn(command, args, {
 			shell: true,
 			cwd,
@@ -5218,13 +5234,29 @@ function spawnWithEnv(
 			// and install-script descendant before the component preparation lock is released.
 			detached: process.platform !== 'win32',
 		});
+		const rootKnownAt = Date.now();
 		const trackedProcessId = childProcess.pid;
-		if (trackedProcessId) registerProcessGroup(trackedProcessId);
+		// Read live by the Windows tree scan: a root that exits mid-wait must bound its children
+		// from that moment, since its PID is reusable from then on.
+		const treeIdentity: WindowsProcessTreeIdentity = {
+			rootPid: trackedProcessId ?? 0,
+			rootKnownAt,
+			// A backward wall-clock step between the two Date.now() calls above (e.g. an NTP
+			// correction) would otherwise make this negative — the same case the registry path
+			// already guards (manageThreads.js's addProcessGroup discards spawnStartedAt entirely
+			// when it reads later than spawnedAt). Falling back to the same allowance here keeps both
+			// paths consistent instead of silently narrowing the window that admits the root's own
+			// earliest children.
+			rootStartedWithinMs: spawnStartedAt <= rootKnownAt ? rootKnownAt - spawnStartedAt : undefined,
+		};
+		const processGroupRegistration = trackedProcessId
+			? registerProcessGroup(trackedProcessId, rootKnownAt, spawnStartedAt)
+			: undefined;
 		let processGroupIsTracked = Boolean(trackedProcessId);
 		const untrackProcessGroup = () => {
 			if (!processGroupIsTracked || !trackedProcessId) return;
 			processGroupIsTracked = false;
-			unregisterProcessGroup(trackedProcessId);
+			unregisterProcessGroup(trackedProcessId, processGroupRegistration);
 		};
 
 		let didTimeout = false;
@@ -5235,7 +5267,7 @@ function spawnWithEnv(
 		});
 		const timeout = setTimeout(() => {
 			didTimeout = true;
-			void terminateProcessTree(childProcess, closePromise).then(
+			void terminateProcessTree(childProcess, closePromise, treeIdentity).then(
 				() => {
 					// Only untrack once terminateProcessTree has confirmed the group is actually gone.
 					// The direct child can emit 'close' mid-grace-period (e.g. right after SIGTERM)
@@ -5302,6 +5334,7 @@ function spawnWithEnv(
 		});
 
 		childProcess.on('exit', (code, signal) => {
+			treeIdentity.rootExitedAt = Date.now();
 			spawnLogger.debug?.(`Direct child exited with code ${code}, signal ${signal}; awaiting stdio close`);
 		});
 
@@ -5327,7 +5360,7 @@ function spawnWithEnv(
 			// when a timeout is already driving its own terminateProcessTree call for this process.
 			if (!didTimeout && trackedProcessId) {
 				try {
-					await terminateProcessTree(childProcess, closePromise);
+					await terminateProcessTree(childProcess, closePromise, treeIdentity);
 				} catch (error) {
 					untrackProcessGroup();
 					flushOutput();
@@ -5397,71 +5430,6 @@ export async function waitForConfirmedTermination(
 	while (await isAlive()) await delay(pollMs);
 }
 
-export async function waitForWindowsTreeTermination(
-	attemptTermination: () => boolean | Promise<boolean>,
-	treeIsAlive: () => boolean | null | Promise<boolean | null>,
-	pollMs: number = PROCESS_TERMINATION_POLL_MS
-): Promise<void> {
-	for (;;) {
-		// A successful taskkill exit only proves the request was accepted, not that the whole tree
-		// has actually exited — Windows termination is asynchronous, and taskkill can report overall
-		// success even when a descendant is not yet (or never) reaped. Only an explicit `false` from
-		// treeIsAlive, independently confirming no member of the tree remains, is safe to return on;
-		// `true` or `null` (unknown) must keep the loop retrying.
-		await attemptTermination();
-		if ((await treeIsAlive()) === false) return;
-		await delay(pollMs);
-	}
-}
-
-async function windowsProcessTreeIsAlive(rootPid: number): Promise<boolean | null> {
-	// Query the process table rather than probing only the parent PID: descendants retain their
-	// ParentProcessId after the parent exits, which is exactly the taskkill "process not found" race.
-	// Exit code 1 must mean "queried the process table and positively found nothing" — never
-	// "the query itself failed" (e.g. Get-CimInstance denied or WMI unavailable), which would
-	// otherwise read identically to a confirmed-gone tree and release the lock while a descendant
-	// may still be alive. ErrorActionPreference=Stop plus the wrapping try/catch turns a query
-	// failure into its own exit code (2), which the caller below already treats as unknown.
-	const script =
-		"$ErrorActionPreference = 'Stop'; try { " +
-		`$rootPid = ${rootPid}; ` +
-		'$all = @(Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId); ' +
-		'$frontier = @($rootPid); $seen = @{}; $found = $false; ' +
-		'while ($frontier.Count -gt 0) { ' +
-		'$next = @(); foreach ($parentPid in $frontier) { if ($seen[$parentPid]) { continue }; ' +
-		'$seen[$parentPid] = $true; foreach ($p in $all) { ' +
-		'if ($p.ProcessId -eq $parentPid) { $found = $true }; ' +
-		'if ($p.ParentProcessId -eq $parentPid) { $found = $true; $next += [int]$p.ProcessId } } }; ' +
-		'$frontier = $next }; ' +
-		'if ($found) { exit 0 } else { exit 1 } ' +
-		'} catch { exit 2 }';
-	return new Promise<boolean | null>((resolve) => {
-		const query = spawn('powershell.exe', ['-NoLogo', '-NoProfile', '-NonInteractive', '-Command', script], {
-			stdio: 'ignore',
-			windowsHide: true,
-		});
-		query.once('close', (code) => resolve(code === 0 ? true : code === 1 ? false : null));
-		query.once('error', () => resolve(null));
-	});
-}
-
-async function terminateWindowsProcessTree(childProcess: ChildProcess): Promise<void> {
-	if (!childProcess.pid) return;
-	const rootPid = childProcess.pid;
-	await waitForWindowsTreeTermination(
-		() =>
-			new Promise<boolean>((resolve) => {
-				const taskkill = spawn('taskkill', ['/pid', String(rootPid), '/T', '/F'], {
-					stdio: 'ignore',
-					windowsHide: true,
-				});
-				taskkill.once('close', (code) => resolve(code === 0));
-				taskkill.once('error', () => resolve(false));
-			}),
-		() => windowsProcessTreeIsAlive(rootPid)
-	);
-}
-
 async function waitForProcessClose(childProcess: ChildProcess, closePromise: Promise<void>): Promise<void> {
 	let timer: ReturnType<typeof setTimeout>;
 	const closeTimeout = new Promise<false>((resolve) => {
@@ -5478,13 +5446,19 @@ async function waitForProcessClose(childProcess: ChildProcess, closePromise: Pro
 	}
 }
 
-export async function terminateProcessTree(childProcess: ChildProcess, closePromise: Promise<void>): Promise<void> {
+export async function terminateProcessTree(
+	childProcess: ChildProcess,
+	closePromise: Promise<void>,
+	treeIdentity: WindowsProcessTreeIdentity
+): Promise<void> {
 	if (!childProcess.pid) {
 		await waitForProcessClose(childProcess, closePromise);
 		return;
 	}
 	if (process.platform === 'win32') {
-		await terminateWindowsProcessTree(childProcess);
+		await confirmWindowsProcessTreeGone(treeIdentity, {
+			label: `pid ${childProcess.pid} (${childProcess.spawnargs.join(' ')})`,
+		});
 		await waitForProcessClose(childProcess, closePromise);
 		return;
 	}

package/components/DESIGN.md

@@ -49,12 +49,45 @@ The ordering is the design. Three things used to be wrong, the first two in a wa
   load-error PROBE, not a safety guarantee: it executes the component's own top-level code with
   incomplete side-effect isolation. It also remains a no-op on the main thread, and the operations API
   deploys on the main thread — so operator deploys are still unvalidated, exactly as before. Fixing that
-  is separate work; this only fixed the order.
+  is separate work; this only fixed the order. Nor does it run on a worker under the default lockdown —
+  see the next section.
 - **Root config was written before the build and never rolled back**, so a build or validation that failed
   still left config naming the release, and `installApplications()` installed it at the next restart. The
   entry is now the transaction's third effect, published after the commit — see "Root config is an effect
   of the activation" below.
 
+### The load validation runs only where it can reproduce a boot load
+
+`validateComponentLoads` (`components/operations.js`) loads the candidate in the thread running the deploy,
+so its verdict is only worth what that thread has in common with a freshly started worker. Two kinds of
+thread have too little, and it runs in neither:
+
+- **The main thread**, where the operations API deploys: application code does not run there.
+- **A worker under `applications.lockdown: freeze-after-load`**, the default, which is where a replicated
+  peer runs the deploy. The lockdown freezes a worker's intrinsics once its boot load finishes, and a freeze
+  is per realm and cannot be undone. A dependency that extends an intrinsic as it loads (reflect-metadata
+  defines `Reflect.decorate`) then throws in the deploying worker, yet loads in every worker a restart
+  starts. 5.2 hid this by swapping before it validated, so a failed first deploy still landed and
+  `restart: true` loaded it. Once validation moved before the swap, every peer refused every retry of a
+  working component while the origin, which does not validate, ran it (#2881).
+  `laterLoadsMeetFrozenIntrinsics()` (`security/jsLoader.ts`) is the test. It holds in any thread that
+  runs the boot load, so a deploy racing that load is excluded as well.
+
+Workers still validate under the other modes
+(`integrationTests/deploy/worker-deploy-frozen-intrinsics.test.ts`). `freeze` and `ses` freeze before the
+boot load, so a later load meets exactly the intrinsics it met. `none` freezes nothing, but an intrinsic the
+live version of a component changed, such as a property it made non-configurable, still differs from what a
+fresh worker's load of the candidate meets. That narrower mismatch predates this and is left alone.
+
+Under the default, no node load-validates a deploy. A candidate that throws at load goes live on every node
+alike, and each restarted worker fails that component closed and serves the rest. That was always the
+origin's behavior, and it replaces a cluster split between versions. A node on an earlier 5.3 build still
+validates in its frozen workers and keeps refusing a candidate whose dependency extends an intrinsic at load,
+so such a deploy converges only once every node is upgraded; retry it then. The `load` progress phase no
+longer fires on a worker either; the operations API, on the main thread, never emitted it. Validating there
+again needs a load in a fresh realm: #2315 step 2 plans it as a canary rollout, whose first replacement
+worker boots the candidate before it takes traffic.
+
 ### Staging a build now and activating it later
 
 `deploy_component { activate: false }` stops after certification, leaving a dormant artifact; a later
@@ -94,7 +127,8 @@ did not need it:
   them:** nothing stops a dormant artifact being edited while it waits, so the link rule and the load
   validation are both re-run at activation rather than trusted from the marker. Content tampering is still
   not detected — that needs a manifest the marker is bound to, and the load validation that would catch a
-  broken entry point is a no-op on the main thread until #2315 step 2.
+  broken entry point is a no-op on the main thread, and on a worker under the default lockdown, until
+  #2315 step 2.
 
 Certification moved out of `activateCandidateApplication` and up into `prepareApplication` for the same
 reason: `markCandidateComplete` fsyncs the whole candidate tree, and a delayed activation must not re-walk
@@ -196,7 +230,8 @@ and the journal removal: the journal is what would carry the activation forward
 
 Two limits are deliberate and tracked separately: activation is two renames, so the live _pathname_ is
 briefly absent (in-memory resources are unaffected, but a component that opens its own files during a
-request can still see a gap); and validation does not run on the main-thread deploy path.
+request can still see a gap); and validation does not run on the main-thread deploy path, nor on a worker
+under the default lockdown.
 
 ### Root config is an effect of the activation
 
@@ -346,6 +381,19 @@ starting `handleApplication`; if a deploy begins during the load, the plugin tim
 unpaused load time. This prevents a long install from looking like a hung plugin while its entry handlers
 are deliberately paused against the intermediate tree.
 
+A deploy's own validation load is exempt: its Scopes (`isTransientValidation`, set at construction from
+`collectScopes`) never read the deploy state, subscribe to `deploy:start`/`deploy:end`, pause their
+watchers, or suspend their plugin timeout. The load runs inside the deploy's lifecycle bracket, between
+build and swap, so any wait for a deploy to end can be a wait on itself. It was, for a component deployed
+as `harper`: the validation load passes no `appName`, so its Scopes take the loader's default name
+`harper`, matched the deploy in flight, and waited for its end while the deploy waited for them. A
+replicated deploy of that component hung on every peer, since only worker threads validate. The same
+cycle formed when a `harper` deploy started on a worker while another component's validation was running
+there, because that deploy's own validation queues behind the running one (`validationChain`). The
+candidate tree is complete before validation starts and nothing else writes to it, so there is nothing to
+wait for. `unitTests/components/componentLoader.test.js` ("deploy validation loads never wait on a
+deploy") holds both routes and the plain timeout.
+
 ### The component load lock is keyed by plugin type, so a plugin's promise is everyone's clock
 
 `sequentiallyHandleApplication` (`components/componentLoader.ts`) holds a cross-thread lock keyed by the
@@ -424,6 +472,31 @@ inputs retain their existing symlink behavior.
 - **Retry is only safe before any byte has reached the consumer.** Once a chunk is handed downstream, re-opening `Blob.stream()` from byte 0 would duplicate it (there's no cheap way to resume from an arbitrary offset across a fresh stream without also plumbing `Blob.slice()`, which was out of scope for this fix). So the helper retries only while the current attempt has yielded nothing yet; a stall after partial content fails immediately, same as before this existed.
 - **Backpressure and cancellation are two different problems, and both are easy to get wrong with a hand-rolled `ReadableStream`.** An early version wrapped the retry loop in `new ReadableStream({ async start(controller) { for await (...) controller.enqueue(chunk) } })` — this eagerly drains `streamFactory()` regardless of `controller.desiredSize`, defeating the whole point of not buffering a multi-GB payload in memory. Switching to an `async function*` consumed via `Readable.from()` restores real backpressure (the generator only resumes when the consumer wants more, matching how the un-wrapped `Blob.stream()` behaved pre-fix). But `Readable.from()`'s `return()`-on-destroy cancellation only takes effect at the generator's _next_ `yield` — while the loop is stuck retrying (no `yield` reached yet), destroying the `Readable` does nothing until the loop naturally exits, verified empirically (a generator that never yields kept retrying long after `.destroy()`). The fix: thread the constructed `Readable` back into the generator via a mutable cell (`readable` doesn't exist until after `Readable.from()` returns, so it can't be closed over directly) and check `.destroyed` explicitly at each loop iteration and after each backoff sleep.
 
+## An origin waits for a peer's deploy answer only as long as the peer may take
+
+The origin of a replicated `deploy_component` hands `server.replication.replicateOperation` a `timeoutMs`
+from `peerDeployAnswerTimeoutMs(req)` (`components/operations.js`). It is the sum of what the peer is
+allowed for that request: its payload wait (`deployment_timeout`, counted twice when credential references
+must also replicate in), two full preparation budgets (`componentPreparationBudgetMs`: every extraction
+command and both install commands at their allowances), a margin for validation and the swap, and the
+restart ceiling when the peer restarts before answering. It is clamped to the longest delay a timer holds.
+One budget is the peer's own preparation. The other is the preparation lock's wait: a peer already preparing
+the same component for another deploy holds this one at the lock for a budget before the lock re-checks the
+holder.
+
+It must never undercut a healthy peer. A shorter bound turns a slow success into a reported failure, and a
+two-command install (a custom package manager falling back to npm) is exactly such a success. So the
+default is hours, and its job is only that the origin eventually settles, rather than holding the
+operation, the deployment row and its own restart for as long as a wedged peer stays wedged. Two things are
+not budgeted. The lock keeps waiting while its holder is alive, so queueing behind a preparation that
+outlasts the lock's wait, or behind several, can run past the deadline. So can plugin `timeout`s a component
+configures beyond the validation margin, which live in the payload the origin does not parse. Covering the
+first would take a deadline that follows the peer's progress rather than a sum of its allowances. The
+deadline is not cancellation: a peer past it may still finish, so the failure the replicator records says
+the outcome there is unknown.
+It stays a `failed` peer result, because `getFailedPeers()` counts only that status, and a new one would
+read as success.
+
 ## A dangling symlink silently truncates the deploy tarball (`components/packageComponent.ts`)
 
 Packaging uses `tar-fs.pack(dir, { dereference: true })` by default (`skip_symlinks` off).

package/components/Scope.ts

@@ -81,6 +81,8 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 	// Set by the loader on deploy pre-flight validation loads (collectScopes):
 	// the scope exists to validate a component, not to run it. Plugins with
 	// process-global side effects should validate fully but skip activation.
+	// Such a scope never follows a deploy: it loads a finished candidate from inside
+	// its own deploy's lifecycle, so waiting or pausing for a deploy waits on itself.
 	isTransientValidation?: boolean;
 
 	/**
@@ -103,18 +105,20 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 		applicationScope: ApplicationScope,
 		origin: string = appName,
 		isRootConfig?: boolean,
-		mount?: ScopeMount
+		mount?: ScopeMount,
+		isTransientValidation?: boolean
 	) {
 		super();
 
 		this.mount = mount;
+		this.isTransientValidation = isTransientValidation;
 		this.#appName = appName;
 		this.#pluginName = pluginName;
 		this.#origin = typeof origin === 'string' ? origin : appName;
 		this.#directory = directory;
 		this.#configFilePath = configFilePath;
 		this.#logger = loggerWithTag(this.#appName);
-		this.#deployInFlight = deployLifecycle.loadsAwaitDeploy(this.#appName);
+		this.#deployInFlight = !isTransientValidation && deployLifecycle.loadsAwaitDeploy(this.#appName);
 
 		this.databaseEvents = databaseEventsEmitter;
 		this.applicationScope = applicationScope;
@@ -170,8 +174,10 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 		this.#deployEndHandler = (name) => {
 			if (name === this.#appName) this.#onDeployEnd(name);
 		};
-		deployLifecycle.on('deploy:start', this.#deployStartHandler);
-		deployLifecycle.on('deploy:end', this.#deployEndHandler);
+		if (!isTransientValidation) {
+			deployLifecycle.on('deploy:start', this.#deployStartHandler);
+			deployLifecycle.on('deploy:end', this.#deployEndHandler);
+		}
 	}
 
 	get logger(): Logger {

package/components/anthropic/index.ts

@@ -89,6 +89,15 @@ export interface AnthropicBackendConfig {
  * `tools: true` — Anthropic has first-class tool-use support via
  * `tool_use` / `tool_result` content blocks.
  */
+const CAPABILITIES: ModelCapabilities = Object.freeze({
+	embed: false,
+	generate: true,
+	stream: true,
+	tools: true,
+	adapters: false,
+	structuredOutput: false,
+});
+
 export class AnthropicBackend implements ModelBackend {
 	readonly name = 'anthropic';
 	readonly #baseUrl: string;
@@ -106,7 +115,7 @@ export class AnthropicBackend implements ModelBackend {
 	}
 
 	capabilities(): ModelCapabilities {
-		return { embed: false, generate: true, stream: true, tools: true, adapters: false };
+		return CAPABILITIES;
 	}
 
 	async generate(input: GenerateInput, opts: BackendOpts<GenerateOpts>): Promise<ModelCallResult<GenerateResult>> {

package/components/bedrock/index.ts

@@ -145,6 +145,15 @@ export function _injectSdkForTests(sdk: SdkLike): void {
  * support tools (or that haven't been wired into the per-family dispatcher
  * yet) raise structured errors.
  */
+const CAPABILITIES: ModelCapabilities = Object.freeze({
+	embed: true,
+	generate: true,
+	stream: true,
+	tools: true,
+	adapters: false,
+	structuredOutput: false,
+});
+
 export class BedrockBackend implements ModelBackend {
 	readonly name = 'bedrock';
 	readonly #region?: string;
@@ -159,7 +168,7 @@ export class BedrockBackend implements ModelBackend {
 	}
 
 	capabilities(): ModelCapabilities {
-		return { embed: true, generate: true, stream: true, tools: true, adapters: false };
+		return CAPABILITIES;
 	}
 
 	async embed(input: string | string[], opts: BackendOpts<EmbedOpts>): Promise<ModelCallResult<Float32Array[]>> {

package/components/componentLoader.ts

@@ -49,6 +49,7 @@ import * as auth from '../security/auth.ts';
 import * as mqtt from '../server/mqtt.ts';
 import { getConfigFilePath, getConfigObj, getConfigPath } from '../config/configUtils.ts';
 import { bootstrapModels, startModelsConfigHotReload } from '../resources/models/bootstrap.ts';
+import { declareDecisionTablesAtBoot } from '../resources/models/decisionStore.ts';
 import { ErrorResource } from '../resources/ErrorResource.ts';
 import { Scope } from './Scope.ts';
 import { ApplicationScope } from './ApplicationScope.ts';
@@ -690,13 +691,18 @@ function sequentiallyHandleApplication(scope: Scope, plugin: PluginModule) {
 			whenResolved(sequentiallyHandleApplication(scope, plugin));
 		};
 		const store = Status.primaryStore;
-		const lockAcquired = store.tryLock(scope.pluginName, callback);
+		// Keyed per (application, plugin), not the plugin name alone: applications load concurrently,
+		// and a plugin-wide lock lets one application's hung handleApplication starve every other
+		// application's load of that plugin into the timeout below (#3184). NUL separator: appName
+		// can contain dots and slashes, so a printable separator could alias two distinct pairs.
+		const lockId = `${scope.appName}\0${scope.pluginName}`;
+		const lockAcquired = store.tryLock(lockId, callback);
 
 		if (!lockAcquired) {
 			return new Promise((resolve, reject) => {
 				whenResolved = resolve;
 				timer = setTimeout(() => {
-					reject(new Error(`Timeout waiting for lock on ${scope.pluginName}`));
+					reject(new Error(`Timeout waiting for lock on ${scope.pluginName} on behalf of ${scope.appName}`));
 				}, timeout + 5_000); // extra time for lock acquisition
 			});
 		}
@@ -712,13 +718,14 @@ function sequentiallyHandleApplication(scope: Scope, plugin: PluginModule) {
 				timeout
 			);
 		} finally {
-			Status.primaryStore.unlock(scope.pluginName);
+			Status.primaryStore.unlock(lockId);
 		}
 	});
 }
 
 function withDeployAwareTimeout<T>(operation: Promise<T>, scope: Scope, timeout: number): Promise<T> {
 	return new Promise((resolve, reject) => {
+		const followsDeploys = !scope.isTransientValidation;
 		const absoluteTimeout = timeout + 6 * 60 * 60 * 1000;
 		let remaining = timeout;
 		let activeSince = 0;
@@ -739,7 +746,7 @@ function withDeployAwareTimeout<T>(operation: Promise<T>, scope: Scope, timeout:
 		absoluteTimer = setTimeout(() => rejectTimeout(absoluteTimeout), absoluteTimeout);
 		absoluteTimer.unref?.();
 		const arm = () => {
-			if (timer || deployLifecycle.loadsAwaitDeploy(scope.appName)) return;
+			if (timer || (followsDeploys && deployLifecycle.loadsAwaitDeploy(scope.appName))) return;
 			if (remaining <= 0) return rejectTimeout();
 			activeSince = Date.now();
 			timer = setTimeout(rejectTimeout, remaining);
@@ -754,8 +761,10 @@ function withDeployAwareTimeout<T>(operation: Promise<T>, scope: Scope, timeout:
 			if (componentName === scope.appName) arm();
 		}
 
-		deployLifecycle.on('deploy:start', handleDeployStart);
-		deployLifecycle.on('deploy:end', handleDeployEnd);
+		if (followsDeploys) {
+			deployLifecycle.on('deploy:start', handleDeployStart);
+			deployLifecycle.on('deploy:end', handleDeployEnd);
+		}
 		operation.then(
 			(value) => {
 				cleanup();
@@ -885,6 +894,7 @@ export async function loadComponent(
 		if (isRoot) {
 			await bootstrapModels(config);
 			startModelsConfigHotReload();
+			declareDecisionTablesAtBoot();
 		}
 
 		// The `env:` block declares the component's environment expectations (string literal →
@@ -1073,19 +1083,18 @@ export async function loadComponent(
 						isRoot,
 						// A root-declared plugin reads its own `host`/`urlPath` straight from this
 						// config, so only an inherited application mount applies here.
-						mount
+						mount,
+						// Marked so plugins with process-global side effects (e.g. the scheduler
+						// registering jobs into its engine) can validate without activating —
+						// validation scopes may reuse a live component's identity, so activating
+						// from one can displace the real component's registrations.
+						Boolean(options.collectScopes)
 					);
 
 					if (options.collectScopes) {
 						// A transient/validation load owns these scopes and closes them itself once the
 						// load is validated (see operations.js deploy pre-flight). Skip the worker-shutdown
-						// auto-close so their deploy-lifecycle listeners — and this SHUTDOWN handler — don't
-						// accumulate across deploys (#1462).
-						// Mark it so plugins with process-global side effects (e.g. the scheduler
-						// registering jobs into its engine) can validate without activating —
-						// validation scopes may reuse a live component's identity, so activating
-						// from one can displace the real component's registrations.
-						scope.isTransientValidation = true;
+						// auto-close so this SHUTDOWN handler doesn't accumulate across deploys (#1462).
 						options.collectScopes.add(scope);
 					} else {
 						// Track the close so the worker's shutdown path waits for it (and thus for any async

package/components/mcp/tools/operations.ts

@@ -360,7 +360,9 @@ export function makeOperationToolHandler(operationName: string) {
 		} catch (err) {
 			const e = err as { message?: string; http_resp_msg?: string; statusCode?: number };
 			const message = e?.http_resp_msg ?? e?.message ?? `operation '${operationName}' failed`;
-			harperLogger.trace(`MCP operations/${operationName} threw: ${(err as Error).stack ?? message}`);
+			harperLogger.trace(
+				`MCP operations/${operationName} threw: ${(err as Error)?.stack ?? (err as Error)?.message ?? message}`
+			);
 			return {
 				isError: true,
 				content: [