@harperfast/harper 5.3.0-alpha.1 → 5.3.0-beta.2

package/components/Scope.ts

@@ -20,6 +20,7 @@ import {
 } from './componentSecrets.ts';
 import type { SecretsView } from './componentSecrets.ts';
 import { deployLifecycle } from './deployLifecycle.ts';
+import { thisThreadOwnsApplication } from '../server/threads/isolatedApplications.ts';
 
 export class MissingDefaultFilesOptionError extends Error {
 	constructor() {
@@ -68,6 +69,7 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 	#deployEndHandler: (name: string) => void;
 	#deployInFlight: boolean = false;
 	#restartRequestedDuringDeploy: boolean = false;
+	#optionsReady: boolean = false;
 	applicationScope?: ApplicationScope;
 
 	options: OptionsWatcher;
@@ -241,7 +243,10 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 
 	ensureTable<TableResourceType = unknown>(options: any): TableResourceType {
 		options.origin = this.#origin;
-		return scopedTableFactory(this.applicationScope?.branches)<TableResourceType>(options);
+		return scopedTableFactory(
+			this.applicationScope?.branches,
+			thisThreadOwnsApplication(this.applicationScope?.name)
+		)<TableResourceType>(options);
 	}
 
 	#handleOptionsWatcherReady(): void {
@@ -251,7 +256,21 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 		// We could make the user call `await scope.ready()` in their `handleApplication` function, but that could lead to the same issue and it'd
 		// be harder for the user to understand why.
 
+		// A second `ready` means the scope had no config of its own and now does — a config file
+		// that was unreadable when this worker booted, or one recreated after deletion. Re-emitting
+		// reaches nobody: componentLoader is long past its `await scope.ready`, so the component is
+		// running on the defaults until something restarts it. Same recovery as the `remove`
+		// listener, and the same convention — a plugin with its own `ready` handler owns it.
+		const started = this.#optionsReady;
+		this.#optionsReady = true;
+		const restartNeeded = started && this.listenerCount('ready') === 0;
+
 		this.emit('ready');
+
+		if (restartNeeded) {
+			this.#logger.debug?.('Options arrived after the scope started, requesting restart');
+			this.requestRestart();
+		}
 	}
 
 	#handleError(error: unknown): void {

package/components/componentLoader.ts

@@ -32,8 +32,13 @@ import { restartWorkers, getWorkerIndex } from '../server/threads/manageThreads.
 import { resetRestartNeeded, subscribeToRestartRequests } from './requestRestart.ts';
 import { trackScopeClose } from './scopeShutdown.ts';
 import { deployLifecycle } from './deployLifecycle.ts';
-import { assertBranchedDatabases } from './Application.ts';
+import { assertBranchedDatabases, assertIsolationConfig } from './Application.ts';
 import { prepareBranches } from '../resources/branchDatabase.ts';
+import {
+	isIsolatedApplication,
+	shouldLoadApplicationHere,
+	thisThreadOwnsApplication,
+} from '../server/threads/isolatedApplications.ts';
 import { toScopeMount, nestScopeMount, type ScopeMount } from './scopeMount.ts';
 import { scopedImport } from '../security/jsLoader.ts';
 import { server } from '../server/Server.ts';
@@ -161,6 +166,32 @@ function rootConfigBranchedDatabases(appName: string): string[] | true | undefin
 	return (getConfigObj()?.[appName] as any)?.branchedDatabases;
 }
 
+/**
+ * Whether this thread is the one that loads `appName` (harper#642, tier 2). An isolated application
+ * is loaded by its dedicated worker and by nothing else; every other thread loads only the
+ * applications that are not isolated. Decided before any of the application's modules are imported.
+ * With no worker threads at all there is nowhere for an isolated application to run, so it fails
+ * closed here rather than silently sharing the only thread.
+ */
+function placedOnThisThread(appName: string): boolean {
+	if (Object.hasOwn(TRUSTED_RESOURCE_PLUGINS, appName)) return true;
+	try {
+		assertIsolationConfig(appName, (getConfigObj()?.[appName] as any)?.isolated);
+	} catch (error) {
+		componentLifecycle.failed(appName, error, `Component '${appName}' failed to load`);
+		return false;
+	}
+	if (isIsolatedApplication(appName) && isMainThread && getWorkerIndex() === 0) {
+		componentLifecycle.failed(
+			appName,
+			new Error(`Application '${appName}' is isolated, which needs a worker thread of its own, but threads.count is 0`),
+			`Component '${appName}' failed to load`
+		);
+		return false;
+	}
+	return shouldLoadApplicationHere(appName);
+}
+
 /**
  * Resolves the mount for `appName`, or reports the failure and returns `undefined` if the
  * configured `host`/`urlPath` is invalid. The caller must skip loading the application in that
@@ -268,6 +299,7 @@ export async function loadComponentDirectories(
 						}
 						return;
 					}
+					if (!placedOnThisThread(appName)) return;
 					const mountResult = tryRootConfigMount(appName);
 					if (!mountResult.ok) return;
 					const loadedModules = new Set<any>();
@@ -316,6 +348,7 @@ export async function loadComponentDirectories(
 				);
 				continue;
 			}
+			if (!placedOnThisThread(appName)) continue;
 			const appFolder = join(CF_ROUTES_DIR, appName);
 			const mountResult = tryRootConfigMount(appName);
 			if (!mountResult.ok) continue;
@@ -347,7 +380,7 @@ export async function loadComponentDirectories(
 	if (hdbAppFolder) {
 		if (getWorkerIndex() === 0) harperLogger.info?.('Loading application from ' + hdbAppFolder);
 		const mountResult = tryRootConfigMount(basename(hdbAppFolder));
-		if (mountResult.ok) {
+		if (mountResult.ok && placedOnThisThread(basename(hdbAppFolder))) {
 			cfsLoaded.push(
 				serializeComponentLoad(hdbAppFolder, () =>
 					loadComponent(hdbAppFolder, cycleResources, hdbAppFolder, {
@@ -849,6 +882,9 @@ export async function loadComponent(
 			if (!componentConfig) continue;
 
 			// Initialize loading status for all components (applications and extensions)
+			// A root-config application (`package:`) is placed like a directory one: an isolated application
+			// loads only in its dedicated worker, and that worker loads no other application.
+			if (isRoot && componentConfig.package && !placedOnThisThread(componentName)) continue;
 			componentLifecycle.loading(componentStatusName);
 
 			const subApplicationScope = isRoot
@@ -940,7 +976,10 @@ export async function loadComponent(
 				// our own trusted modules can be directly retrieved from our map, otherwise use the (configurable) secure module loader
 				const ensureTable = (options: any) => {
 					options.origin = origin;
-					return scopedTableFactory(applicationScope.branches)(options);
+					return scopedTableFactory(
+						applicationScope.branches,
+						thisThreadOwnsApplication(applicationScope.name)
+					)(options);
 				};
 				// call the main start hook
 				const network =
@@ -1160,7 +1199,7 @@ export async function loadComponent(
 						// — not abort the loop and discard a pending follow-up, like the save that fixes it.
 						try {
 							await loadComponentDirectories();
-							await restartWorkers();
+							await restartWorkers(undefined, undefined, true, null, '*');
 						} catch (error) {
 							harperLogger.error('Error during component reload', error);
 						}

package/components/deploymentOperations.ts

@@ -13,7 +13,10 @@ import type { ProgressEmitter } from '../server/serverHelpers/progressEmitter.ts
 const { HTTP_STATUS_CODES } = hdbErrors;
 
 const DEPLOYMENT_TABLE = terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME;
-const TERMINAL_STATUSES = new Set(['success', 'failed', 'rolled_back']);
+// `staged` belongs here: the build is complete on disk and nothing further happens to this deployment, so
+// an SSE tail must finish and `delete_deployment_payload` must not refuse a tarball nobody is installing
+// from any more. Activating the artifact is a separate deployment with a row of its own.
+const TERMINAL_STATUSES = new Set(['success', 'failed', 'rolled_back', 'staged']);
 
 interface ListRequest {
 	project?: string;

package/components/deploymentRecorder.ts

@@ -55,7 +55,11 @@ type DeploymentStatus =
 	| 'restarting'
 	| 'success'
 	| 'failed'
-	| 'rolled_back';
+	| 'rolled_back'
+	// Terminal: the build is complete and certified on disk but was not activated. Terminal rather than
+	// in-flight because nothing further happens to THIS deployment — activating its artifact is a separate
+	// operator action with a row of its own.
+	| 'staged';
 
 interface CreateOptions {
 	project?: string;
@@ -63,6 +67,8 @@ interface CreateOptions {
 	user?: string;
 	restart_mode?: 'immediate' | 'rolling' | null;
 	rollback_of?: string | null;
+	/** The staged deployment whose artifact this deployment activated. */
+	activated_from?: string | null;
 	// Deploy credentials in reference form (`{ registry, secret, scope? }` / `{ host, secret,
 	// username? }`) — never a literal token. Kept so a rollback can re-resolve the credential from
 	// hdb_secret without the operator re-supplying it. Null when the deploy used no credentials or a
@@ -141,6 +147,7 @@ export class DeploymentRecorder {
 			completed_at: null,
 			user: options.user ?? null,
 			rollback_of: options.rollback_of ?? null,
+			activated_from: options.activated_from ?? null,
 			credentials: options.credentials ?? null,
 			error: null,
 		};
@@ -465,7 +472,7 @@ export class DeploymentRecorder {
 		this.sealed = true;
 	}
 
-	async finish(status: 'success' | 'failed' | 'rolled_back', error?: unknown): Promise<void> {
+	async finish(status: 'success' | 'failed' | 'rolled_back' | 'staged', error?: unknown): Promise<void> {
 		if (this.finished) return;
 		// Send a terminal sentinel through the emitter (if any) BEFORE we unsubscribe and
 		// remove it from the registry, so any SSE tail subscribers can resolve their wait

package/components/mcp/DESIGN.md

@@ -0,0 +1,109 @@
+# components/mcp/ — Design notes
+
+The MCP Streamable-HTTP surface.
+
+**Read this when:** adding or changing an MCP method, tool, prompt or profile.
+
+Index of every design note: [DESIGN.md](../../DESIGN.md).
+
+---
+
+## MCP protocol surface (`components/mcp/`)
+
+The MCP Streamable-HTTP transport (spec `2025-06-18`) is served at `/mcp` under **two profiles**: an
+_operations_ profile (mounted on the Fastify operations server) and an _application_ profile (mounted on
+the Harper application HTTP server). Both share `transport.ts` (the JSON-RPC dispatcher) but differ in what
+they expose — operations surfaces management operations as tools; application surfaces exported
+Resources/tables. Profile gating runs throughout (`completeResourceArgument`, prompt visibility, tool
+`visibleTo`), so when adding a method, decide which profile(s) it belongs to rather than assuming both.
+
+A handful of design points are non-obvious and easy to break:
+
+- **Per-call POST SSE streaming has a close-before-subscribe race.** A `tools/call` that opts into streaming
+  (`Accept: text/event-stream`) gets an `IterableEventQueue` whose frames the adapter consumes via the event
+  API (`on('data')` / `once('close')`), **not** `for await` — the async iterator does not terminate on
+  `'close'`. The streaming tool handler is therefore dispatched inside a `setImmediate` (a _detached_,
+  deferred task) so the adapter's consumer attaches **before** any frame is produced. Without the defer, a
+  fast handler emits its final frame + `close` synchronously; the queue buffers `'data'` but not `'close'`,
+  and the stream hangs. Any handler on this path must check `signal.aborted` first (cancellation can land
+  before the deferred task runs).
+
+- **Server→client requests are correlated across workers.** `serverRequests.ts` lets a streaming `tools/call`
+  call _back_ into the client (`sampling/createMessage`, `elicitation/create`, `roots/list`) and await the
+  reply. The request frame rides **the call's POST SSE stream**; the client's response is a _fresh POST_ that
+  can land on **any worker**. The pending-promise registry is per-worker, so a response with no local match
+  is fanned out over ITC (`MCP_CLIENT_RESPONSE`) and the worker holding the promise resolves it (mirrors
+  `components/status/crossThread.ts`). Request ids are `srv-${randomUUID()}` — **not** a per-worker counter,
+  which would collide on `(sessionId, id)` across workers and misroute responses. Methods are capability-gated
+  (`METHOD_CAPABILITY`) against the client capabilities captured at `initialize`; the registry is bounded
+  (timeout + high-water-mark) so a non-responding client can't leak promises.
+
+- **Application tools must be rebuilt after JS resources register, not just on schema changes.** The
+  application-profile tool scan (`registerApplicationTools`) runs at MCP component boot and on schema-change
+  ITC events — both of which fire while the `@table` classes register, **before** the `jsResource` plugin
+  registers the component's exported `class X extends tables.X` subclass. That subclass is the object the
+  registry ends up holding (REST routes to it) and the only place author opt-ins (`static mcpTools`/
+  `mcpPrompts`) live, so a scan that ran earlier sees only the base table class and misses them (#1448). The
+  fix: `jsResource` fires `signalResourcesRegistered()` (a deliberately **local-only**, non-ITC signal in
+  `utility/signalling.ts`, backed by `resourceHandler` in `server/itc/serverHandlers.js` — each worker
+  registers its own JS resources, so the rebuild belongs in that worker) after registration; `listChanged`
+  subscribes and re-runs the scan. Consequence: the verb tools (`create_*` etc.) now bind to the subclass and
+  honor its `post`/`patch` overrides, matching REST — previously they bound to the base table class and
+  silently bypassed those overrides. Advertised CRUD output schemas are still table-derived, so an overridden
+  write verb whose return diverges from `{ id }`/`{ ok }`/`{ deleted }` advertises a subset shape (the in-use
+  SDK tolerates supersets; tightening per-override envelopes is sibling-issue work — see the `derive.ts`
+  envelope note).
+
+- **Resource subscriptions are row-backed via the audit log.** `resources/subscribe` resolves the URI to a
+  Resource and drives `Table.subscribe` off the audit-store `'committed'` path (same machinery as the
+  "Audit-store `'committed'` notification batching" section above). The targeting is the subtle part:
+  `getMatch` returns the matched Resource plus the remaining path on `relativeURL`, and `subscribeToResource`
+  sets **both** `request.id` (the record key, or `undefined`) **and** `request.isCollection` from it. A record
+  URI (`…/WorkItem/42`) watches that record; a collection URI (`…/WorkItem`, what `resources/list` advertises)
+  watches the whole table. `new RequestTarget(path)` parses an id out of the path on its own, so _both_ fields
+  must be overridden — otherwise a collection URI silently watches a phantom record named after the resource
+  and receives nothing. `harper://*` pseudo-resources are **list-changed-only** (not row-backed). Subscriptions
+  use `omitCurrent` (notify on change, not a retained snapshot — the notification just says "re-read this").
+
+- **Subscribe requires a live GET stream; teardown is asymmetric.** `resources/subscribe` rejects (`-32602`)
+  if no GET SSE stream has registered the session — the audit-log iterator has nowhere to deliver, and there'd
+  be no `RegisteredSession` close hook to stop it. The GET `'close'` handler drops **subscriptions only**
+  (`dropSessionSubscriptions`), _not_ pending server requests: those ride the per-call POST stream, so a normal
+  GET reconnect must not reject an in-flight `ctx.serverRequest`. A `DELETE` (explicit session teardown) drops
+  **both**, because it may arrive with no open GET stream.
+
+- **SSE resumability (`Last-Event-ID`).** Every GET-channel frame goes through `pushSessionFrame`, which
+  assigns a monotonic event id and appends to a bounded per-session `replayBuffer`. On reconnect with a
+  `Last-Event-ID` header, `replaySince` re-sends only the frames after that id. The event-id sequence **and**
+  the buffer carry across a supersede (a fresh GET replacing the old one for the same session id), so ids stay
+  monotonic and no frame is lost across a reconnect.
+
+- **Test seams avoid loading thread/audit machinery in unit tests.** `_setSubscribeImplForTest`
+  (`resources.ts`) and `_setItcForTest` (`serverRequests.ts`) inject fakes so the unit suite needn't spin up
+  the audit log or ITC. Consequence: the subscribe **targeting** logic (`id`/`isCollection` derivation) is
+  _bypassed_ by the seam and is therefore covered at the **integration** level (`sse-listchanged.test.ts` N3
+  record / N4 collection), not in unit tests.
+
+Two related traps: the create/schema-update path's exclusive `update-attributes` lock is a
+synchronous bounded wait (`acquireUpdateAttributesLock` in `Table.ts`: brief hot spin, then
+`Atomics.wait` backoff, retryable `ServerError` after the 10s `UPDATE_ATTRIBUTES_LOCK_TIMEOUT` — harper#2251; it used to be
+an unbounded `while (!tryLock()) {}` spin that pinned a worker core forever if the holder never
+released). Release is structural — `table()` releases in a single `finally` and `dropTable` uses
+`withUpdateAttributesLock` — so a throw inside the locked window cannot leak the lock (regression
+suite: `unitTests/resources/updateAttributesLock.test.js`). Because the acquire can now throw,
+`table()` takes the RocksDB lock _before_ it mutates the live `Table` (attributes, class metadata,
+index handles): losing the race then leaves this worker's in-memory schema exactly as it found it,
+and moving any mutation above that acquire reintroduces schema drift the catalog never saw. LMDB
+keeps the lazy acquire — its `exclusiveLock()` is an environment-wide write transaction that cannot
+time out, so taking it eagerly would stall every write to the database on an unchanged reload. A
+successful acquire that waited past `UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT` (1s) warns once, since
+contention is otherwise invisible until it becomes a timeout. The locked
+sections MUST stay synchronous: the wait blocks the event loop, so an awaited operation inside
+one would stall a concurrent acquirer to its deadline. And dropping then recreating a
+same-named table within one process requires @harperfast/rocksdb-js >= the column-family
+eviction fix (1.4.3 / rocksdb-js#<main PR>): older bindings keep the dropped column family's
+by-name registry entry alive whenever other worker threads hold handles, so the recreate
+silently reuses a dangling handle and every write fails with "Invalid column family specified
+in write batch", poisoning the whole database env until restart. The regression suite for all
+of this is `unitTests/resources/dropTableGhost.test.js` (it fails by design on pre-fix
+bindings).

package/components/mcp/audit.ts

@@ -6,7 +6,7 @@
  * (session id, profile, tool name).
  *
  * Argument summarization runs through a redaction step that drops anything
- * that looks like a credential (key/secret/password). Operators who need
+ * that looks like a credential (secret/password/token). Operators who need
  * stricter PII handling configure `mcp.audit.argumentRedactor` to a custom
  * function via a future component-author hook (v1.1).
  */
@@ -25,17 +25,17 @@ export interface AuditEntry {
 }
 
 const REDACTION_PATTERN = /(secret|password|token|api[-_]?key|credentials?|auth)/i;
-// Exact field names that carry secret material without a credential-looking name: `value`/`values`
-// (set_secret plaintext, set_env_value .env secrets) and `envelope` (set_secret ciphertext). These
-// mirror the fields processLocalTransaction strips from the REST operations log — the MCP audit
-// path must not become a bypass. Exact-match so e.g. `search_value` stays auditable.
-const REDACTION_EXACT_FIELDS = /^(value|values|envelope)$/i;
-// ...but only for the operations that actually put secrets in those generically-named fields.
-// `value`/`values` are common, non-secret params on many other operations (record data, config
-// values), so blanket-redacting them would gut the audit trail for unrelated tools. The global
-// REDACTION_PATTERN still applies to every tool; this just narrows the exact-field masking to the
-// secret-bearing ops. NOTE: extend this set if a new op ever carries a secret in a plain field.
-const EXACT_FIELD_TOOLS = new Set(['set_secret', 'set_env_value']);
+// Both secret tools carry all three fields, not just the ones each declares: MCP forwards arguments
+// as-is and audits them after the handler, so a field the tool rejects is still logged. Mirrors what
+// the REST operations log strips unconditionally (UNLOGGABLE_OPERATION_FIELDS).
+const SECRET_VALUE_FIELDS: ReadonlySet<string> = new Set(['value', 'values', 'envelope']);
+// Per tool, not global: `value`/`values` are ordinary params on many other operations.
+const REDACTION_FIELDS_BY_TOOL = new Map<string, ReadonlySet<string>>([
+	['set_secret', SECRET_VALUE_FIELDS],
+	['set_env_value', SECRET_VALUE_FIELDS],
+	['add_ssh_key', new Set(['key'])],
+	['update_ssh_key', new Set(['key'])],
+]);
 const REDACTION_PLACEHOLDER = '[redacted]';
 const MAX_REDACTION_DEPTH = 10;
 
@@ -46,23 +46,27 @@ const MAX_REDACTION_DEPTH = 10;
  * a credential buried below the depth limit cannot leak. Returns a shallow
  * clone — the caller's payload is never mutated.
  */
-export function redactArgs(value: unknown, depth = 0, redactExactFields = false): unknown {
+export function redactArgs(value: unknown, depth = 0, redactionFields?: ReadonlySet<string>): unknown {
 	if (value === null || typeof value !== 'object') return value;
 	if (depth > MAX_REDACTION_DEPTH) return REDACTION_PLACEHOLDER;
 	if (Array.isArray(value)) {
-		return value.map((v) => redactArgs(v, depth + 1, redactExactFields));
+		return value.map((v) => redactArgs(v, depth + 1, redactionFields));
 	}
 	const out: Record<string, unknown> = {};
 	for (const [k, v] of Object.entries(value)) {
-		if (REDACTION_PATTERN.test(k) || (redactExactFields && REDACTION_EXACT_FIELDS.test(k))) {
+		if (REDACTION_PATTERN.test(k) || redactionFields?.has(k.toLowerCase())) {
 			out[k] = REDACTION_PLACEHOLDER;
 		} else {
-			out[k] = redactArgs(v, depth + 1, redactExactFields);
+			out[k] = redactArgs(v, depth + 1, redactionFields);
 		}
 	}
 	return out;
 }
 
+export function redactArgsForTool(value: unknown, tool: string): unknown {
+	return redactArgs(value, 0, REDACTION_FIELDS_BY_TOOL.get(tool));
+}
+
 /** Mask a session id for logging — first 8 chars, suffix elided. */
 export function maskSessionId(id: string): string {
 	if (typeof id !== 'string' || id.length <= 8) return id;
@@ -79,7 +83,7 @@ export function emitAuditEntry(entry: AuditEntry): void {
 		const masked = {
 			...entry,
 			sessionId: maskSessionId(entry.sessionId),
-			args: redactArgs(entry.args, 0, EXACT_FIELD_TOOLS.has(entry.tool)),
+			args: redactArgsForTool(entry.args, entry.tool),
 		};
 		harperLogger.info({ category: 'mcp.audit', ...masked });
 	} catch (err) {