@harperfast/harper 5.3.0-beta.4 → 5.3.0

package/components/DESIGN.md

@@ -106,19 +106,25 @@ did not need it:
   activation cannot re-derive. An _optional_ record could not distinguish a payload build from a package
   build whose record was lost, so a missing, malformed or wrong-version one is refused rather than defaulted.
 - **Claiming an id is exclusive.** `buildCandidateApplication` used to tolerate an existing deployment
-  directory because a fresh UUID could not collide; a public id can be repeated by an operator or by a
-  redelivered replication, so a claim now rejects another component's directory and any directory carrying
-  `.complete`, and rebuilds only over an uncertified partial of its _own_ component. Ownership is published
-  as part of the claim — the `.component` sidecar is written right after the exclusive `mkdir`, not at
-  certification — because `buildCandidateApplication` can spend minutes resolving and packing before any
-  tree exists to infer an owner from. For that whole window the directory answered to nobody, and an empty
-  `readdir` is indistinguishable from an abandoned claim, so a second component could delete a build that
-  was still running. **Emptiness is not a verdict:** an unattributed directory is refused, never reclaimed,
-  which is the same reading recovery already gives it. The id this request names is also pinned
-  through the preparation preamble, so retention cannot evict the artifact the request is about to use —
-  which it otherwise would, immediately, at `deployment_stagingRetention_maxCount: 0`. The contract is
-  bounded: an id names one artifact _while that artifact exists_. Activation consumes it (the swap is a
-  rename) and retention can prune it, after which the id is free again.
+  directory because a fresh UUID could not collide; a public id can be repeated by an operator, so a claim
+  rejects another component's directory and any directory carrying `.complete`, and rebuilds only over an
+  uncertified partial of its _own_ component — in a fresh claim, never inside the partial tree. Ownership is
+  published as part of the claim, not at certification, because `buildCandidateApplication` can spend minutes
+  resolving and packing before any tree exists to infer an owner from. The claim is built at
+  `.deploy-staging/.claiming-<uuid>-<digest>`, named there, and renamed onto the id, so the directory only
+  ever appears at its id already attributed. It used to be an exclusive `mkdir` followed by the sidecar write,
+  and a death between the two left an unattributed directory — indistinguishable from a claim in flight, so
+  refused forever. Now no live claim is ever visible unattributed: an EMPTY directory at an id is that
+  wreckage from an older build and is taken over (`rmdir`, which removes nothing else); a non-empty
+  unattributed one is still refused. A claim that died before its rename names no id, is skipped by every scan
+  of the staging root (all of them skip dot-prefixed entries), and is removed under the component's lock by its
+  next deploy, or by `drop_component`; a digest of the component in its name, rather than the name itself, makes
+  that attributable without a read and cannot push a long name past a filename limit.
+  On Windows a rename onto any existing directory fails with the `EPERM` a transient holder raises, so the
+  destination is classified before renaming rather than after a spent retry budget. The id this request names
+  is also pinned through the preparation preamble, so retention cannot evict the artifact the request is about
+  to use — which it otherwise would, immediately, at `deployment_stagingRetention_maxCount: 0`. What an id
+  names after its activation is the next section's subject.
 - **Staging owns its bytes.** A `file:<directory>` source is refused, and so is any symlink in the built
   tree resolving outside it (bar the `node_modules/harper`/`harperdb` links the loader owns and repairs).
   Certification fsyncs the tree but follows no links, and the post-swap relocation repair leaves external
@@ -221,8 +227,9 @@ never depends on it. Roll-forward requires the journal, the candidate and `.comp
 observable, which means a lost directory update degrades to a roll back rather than to a wrong decision.
 
 Retiring the rollback record only marks the displaced tree disposable; both the activation path and
-recovery then sweep it, or the components root would grow by a whole component version per deploy. The
-retire is **correctness, not hygiene** — that marker is what stops the legacy pass treating the record as
+recovery then put it back into its deployment's record, or sweep it when nothing can keep it (see "A deployment
+id keeps naming its release"), since otherwise the components root would grow by a whole component version per
+deploy. The retire is **correctness, not hygiene** — that marker is what stops the legacy pass treating the record as
 authoritative once the journal is gone — so a failure to retire propagates and the component fails closed
 with its journal intact. Only the sweep itself is best-effort, because it costs disk rather than a wrong
 decision. For the same reason, a swap whose rename cannot be confirmed on storage skips both the retire
@@ -278,9 +285,10 @@ publication-lock timeout is rethrown as a 503 `ServerError` rather than the prep
 class, which recovery reads as "a live deploy holds this component's lock" — a deferral, not a verdict.
 
 The throw also means `deploy_component` never reaches replication: the release is live on this node only, and
-the error says so. A fresh deploy converges; a retry of a `deployment_id` activation does not, because its
-preamble settles the kept journal and the artifact is gone — the unconvergeable-retry gap #2315 step 5 owns.
-So a condition that would fail every publish is refused BEFORE anything moves: `assertRootConfigEffectPublishable`
+the error says so. A fresh deploy converges. So does a retry of a `deployment_id` activation: its preamble settles
+the kept journal, publishing the entry, and then answers that the id is already live here, so the operation goes
+on to replicate. Peers of an immediate deploy hold no artifact, so its id answers 404 there. Even so, a condition
+that would fail every publish is refused BEFORE anything moves: `assertRootConfigEffectPublishable`
 runs ahead of the journal and refuses an effect that would have to change a document that does not parse, or
 whose directory this process cannot write. Only a static condition is caught; a publish can still fail.
 
@@ -347,9 +355,9 @@ and validated, activated by nobody. Recovery used to remove every owned journal-
 dormant builds and bounds them per component to `deployment_stagingRetention_maxCount` (default 5, 0 keeps
 none), newest by `.complete` mtime, ties broken by deployment id so concurrent passes pick the same victims.
 Everything else journal-less — a partial tree, a directory whose tree already moved live, a stale
-`.unsettled` — is still residue and still removed. Nothing here produces a dormant build yet beyond the crash
-window between `.complete` and the journal; #2315 step 6 (deploy from an existing aside) is the producer this
-bound exists for.
+`.unsettled` — is still residue and still removed. Two things produce them: a stage (`activate: false`), and every
+activation that displaces a release it can put back under its own id (next section). A deployment's record — its
+directory without a tree — is not a build and is not counted.
 
 Removal is decided **only under the owner's preparation lock**: activation writes `.complete` moments
 before its journal while holding that lock, so an unlocked read of "complete, no journal" is a candidate, not
@@ -370,6 +378,86 @@ hygiene: it never fails a component closed and never replaces a deploy's own err
 best-effort under filesystem failure and is not a storage quota — journaled, unsettled and unowned
 directories are preserved by design and can still fill a volume.
 
+### A deployment id keeps naming its release after the swap
+
+On a node, a deployment id names its release for as long as the node holds that release: while it is dormant,
+`deployment_id` activates it; while it is live, activating it answers that it already is, without a swap; when a
+later activation displaces it, it becomes dormant again under the same id, within
+`deployment_stagingRetention_maxCount`. A node holding neither answers 404 (#2315 step 5). Four pieces:
+
+- **Provenance rides the tree.** `buildCandidateApplication` writes `.harper-deployment.json` — `{ v, component,
+deploymentId }` — at the top of every candidate it extracts, after the install and before `.complete`, replacing
+  any marker the payload carried. It moves with the tree through the commit rename, so the live tree always names
+  the build that made it live, and anything else that replaces the directory replaces the marker too: it is
+  absent, never stale. A `file:` link gets none, since its target is not the deploy's to write, and a live link
+  has no provenance whatever its target holds. The tree is the component's own to write, which is why the marker
+  carries only an id: nothing published on activation comes from it. A component rewriting its own marker can
+  mislabel only its own releases. `package_component` and `get_components` leave the top-level marker out; a
+  deeper file of that name is the component's.
+- **The id's record outlives the swap.** A build whose caller declared what it publishes (`describeArtifact` — every
+  `deploy_component`) writes `.artifact.json` before `.complete`, as a stage does, and a committed activation
+  keeps `.deploy-staging/<id>` minus its tree and journal: `.component`, `.complete`, `.artifact.json`. That
+  record is where the release goes back to, so its directory entry is flushed before the swap, as a stage's is. A
+  boot install, `add_component` and a link build declare nothing and keep none, as before. Keeping the descriptor
+  here rather than snapshotting the entry at displacement is
+  deliberate: for a boot install the entry in force is the new build's, so after an out-of-band edit the displaced
+  tree would be described with the wrong package. Re-activating an id publishes what that deployment declared,
+  which is the contract a staged artifact already had; `set_configuration` edits made while it was live are not
+  carried back, just as a package redeploy replaces the entry whole. One side effect, stated: an immediate deploy
+  that crashes before its first rename is now DESCRIBED, so recovery returns its candidate to dormant rather than
+  discarding it.
+- **The displaced tree goes back into its record.** `retainDisplacedRelease` runs where a committed activation
+  disposes of the tree it displaced — after the retire, in `activateCandidateApplication`'s post-commit block, in
+  `sweepAsideRecords` on a roll forward, and in the legacy pass for a retired tree still in the aside — and
+  renames it into `<record>/<component>` when its marker names this component's record. **Only an explicit
+  retention decision deletes a tree**, so its verdict is three-way: _kept_; _ineligible_ (no marker, no usable
+  record, or `maxCount: 0`, when the record goes too) — swept as before; or _failed_ (a read error other than
+  ENOENT, or a failed rename) — left retired in the aside with its record for the legacy pass to retry at the next
+  preparation, since on Windows the rename and the delete both fail while something holds a handle in the tree.
+  A failure after the rename (syncing its parents) propagates, as the aside syncs do. At the activation it keeps
+  the journal, so recovery repeats the aside barrier before letting it go — not the kept tree's own entry, whose
+  loss costs only that kept copy. In recovery it fails the component closed with its journal, the retire's own
+  contract. The kept tree's
+  `.complete` mtime is refreshed (retried briefly) so retention orders it by when it stopped being live; left at
+  its build time, stages nobody activated would outrank it. **Only the activation's own site prunes** after a
+  keep, pinning the kept id: settlement runs inside a request's preamble, whose pin it cannot see, and a prune
+  there evicted the very artifact that request was activating. The pass a settlement runs inside counts what it
+  kept and pins it beside its own pin, so a refresh that failed cannot let that pass evict the release it just put
+  back. A kept release is not re-certified: it is kept as
+  it was when displaced, including what it wrote into its own directory while live, and activation re-runs the
+  link rule and the load validation. An absolute link `repairRelocatedDependencyLinks` wrote after its own swap (a
+  Windows `file:`/workspace junction) names the live path, so such a release's re-activation is refused 409.
+- **Activating the live id answers without a swap.** After the preamble, `prepareApplication` in `activate` mode
+  compares the live marker with the id: equal means the release is already serving here — a retry whose earlier
+  attempt swapped on this node and failed later or elsewhere — so it sets `alreadyActive`, requests a restart
+  (nothing can tell whether the running generation ever loaded it, the reasoning a late startup preparation
+  follows), and returns; `deploy_component` emits the `prepare` phase end itself, since nothing ran
+  `validateCandidate`, and replicates. A restart it performs replaces every worker, dedicated ones included: which
+  ones loaded the previous release is unknowable, since the preamble may just have published an isolation change.
+  Per node, with no cross-node inference: a node that swapped answers success, one still holding the artifact
+  activates it, one with neither answers 404. Only `activate`: a deploy or stage claims an id the origin minted
+  fresh, and harper-pro never resends a replicated operation (`sendOperationToNode` is one socket per call), so a
+  replicated deploy of an id this node already holds — only a hand-crafted `_deploymentId` produces one — is refused
+  by the claim. A record whose release is neither live nor kept answers 404, where a missing tree used to answer 409.
+
+**A record is stale only when nothing refers to it.** "Not live" is not enough: a crash after the commit and
+before the re-home leaves the displaced release's record and its tree in `.deploy-aside`, and a boot scan can
+reach the record before the journal that will re-home into it — the record would be deleted and its only tree
+swept. So a record is residue only when neither the live marker nor the marker of any tree in the owner's aside,
+retired or not, names it. Boot recovery checks that unlocked and leaves a referenced record alone without the
+lock, as it does a dormant build (#2531: a per-directory lock on every pass makes healthy components lose the
+250 ms probe to sibling threads), then re-derives it under the owner's lock before removing a stale one.
+`drop_component` removes the dropped component's records and abandoned claims along with its dormant builds.
+
+Compatibility: a tree made live before this change has no marker, so the first deploy after the upgrade sweeps
+it. An older build reads a record as residue and removes it, and a kept release as an ordinary dormant build it
+can activate; the marker is an ignored file to it. The journal format is unchanged. A peer on an older build
+answers 404 to a consumed id, as before. `deployment_stagingRetention_maxCount` (default 5) now bounds kept
+releases with staged builds, and each is a whole installed tree, so at the default a component holds up to five
+extra copies of itself; `0` keeps no previous release, and the rest of this section still applies. Until #2315
+step 7, a package component reinstalls from its source on the first start after each package deploy; that
+reinstall keeps no record, but the certified tree it displaces is kept, which is the one worth keeping.
+
 ## Component preparation is serialized across worker threads
 
 `prepareApplication()` performs one transaction per component: build the replacement, validate it, then swap it in (see "A deploy builds off to the side" below). Deploy operations can execute on worker threads as well as main, so a module-local promise queue is insufficient—each worker has its own module registry. `withComponentPreparationLock()` (`components/componentPreparationLock.ts`) instead acquires an atomic filesystem lock keyed by the absolute component path. The deprecated `install_node_modules` operation uses the same lock, so it cannot run npm concurrently with a deploy.

package/components/deploymentProvenance.ts

@@ -0,0 +1,28 @@
+import { basename } from 'node:path';
+
+/** Rides the rename that makes a tree live, so it is never stale. Only an id: the tree is the component's to write. */
+export const DEPLOYMENT_PROVENANCE_FILE = '.harper-deployment.json';
+
+const DEPLOYMENT_PROVENANCE_VERSION = 1;
+
+export function formatDeploymentProvenance(component: string, deploymentId: string): string {
+	return JSON.stringify({ v: DEPLOYMENT_PROVENANCE_VERSION, component, deploymentId });
+}
+
+export function parseDeploymentProvenance(raw: string, component: string): string | undefined {
+	let parsed: any;
+	try {
+		parsed = JSON.parse(raw);
+	} catch {
+		return undefined;
+	}
+	if (parsed?.v !== DEPLOYMENT_PROVENANCE_VERSION || parsed.component !== component) return undefined;
+	const deploymentId = parsed.deploymentId;
+	// A single path segment that cannot name a claim in progress or any other dot-prefixed staging entry.
+	return typeof deploymentId === 'string' &&
+		deploymentId.length > 0 &&
+		deploymentId === basename(deploymentId) &&
+		!deploymentId.startsWith('.')
+		? deploymentId
+		: undefined;
+}

package/components/operations.js

@@ -39,6 +39,7 @@ const {
 	retireComponentDirectory,
 } = require('./Application.ts');
 const { COMPONENT_PREPARATION_LOCK_DIR, withComponentPreparationLock } = require('./componentPreparationLock.ts');
+const { DEPLOYMENT_PROVENANCE_FILE } = require('./deploymentProvenance.ts');
 const {
 	applyRootConfigEffect,
 	assertRootConfigEffectPublishable,
@@ -888,6 +889,8 @@ async function deployComponent(req) {
 				await validateComponentLoads(candidateDirPath, emit);
 			},
 		});
+		// Nothing ran `validateCandidate`, which ends this phase.
+		if (application.alreadyActive) emit('phase', { phase: 'prepare', status: 'done' });
 		// The build is certified on disk from here on, so every later failure — a peer result, or a rejection
 		// thrown by the replication layer itself — still leaves an artifact this id can activate.
 		if (mode === 'stage') stagedOnOrigin = true;
@@ -938,8 +941,9 @@ async function deployComponent(req) {
 		}
 		// A still-isolated application restarts only its own dedicated worker. Everything else restarts the
 		// pool: a shared application, and either direction of an isolation flip, where the reconcile in
-		// restartWorkers starts or stops the moving application's own worker.
-		const restartScope = wasIsolated && nowIsolated ? application.name : undefined;
+		// restartWorkers starts or stops the moving application's own worker. An already-live retry cannot know
+		// which workers loaded the previous release, so it restarts them all.
+		const restartScope = application.alreadyActive ? '*' : wasIsolated && nowIsolated ? application.name : undefined;
 		if (mode === 'stage') {
 			// No restart and no restart-required flag: nothing about the running component changed. The
 			// marker is what tells the origin which peers understood the request — see the confirmation
@@ -967,7 +971,8 @@ async function deployComponent(req) {
 				operation: 'restart_service',
 				service: 'http',
 				scope: manageThreads.encodeRestartScope(restartScope),
-				scopeFallback: restartScope === undefined ? undefined : manageThreads.encodeRestartScope(undefined),
+				scopeFallback:
+					restartScope === undefined || restartScope === '*' ? undefined : manageThreads.encodeRestartScope(undefined),
 				replicated: true,
 			});
 			emit('phase', { phase: 'restart', status: 'done' });
@@ -1204,6 +1209,7 @@ async function getComponents() {
 	// Recursive function that will traverse the components dir and build json
 	// directory tree as it goes.
 	const rootConfig = configUtils.getConfiguration();
+	const componentsRoot = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
 	const walkDir = async (dir, result) => {
 		try {
 			const list = await fs.readdir(dir, { withFileTypes: true });
@@ -1211,12 +1217,13 @@ async function getComponents() {
 				const itemName = item.name;
 				// Deny-list, not a dot-prefix skip: component CONTENTS legitimately include dot-files
 				// (`.aiignore`, `.env.example`) that callers expect to see, so only Harper's own
-				// bookkeeping directories are excluded by name.
+				// bookkeeping is excluded by name: its directories, and each component's provenance marker.
 				if (
 					itemName === 'node_modules' ||
 					itemName === ASIDE_STAGING_DIR ||
 					itemName === DEPLOY_STAGING_DIR ||
-					itemName === COMPONENT_PREPARATION_LOCK_DIR
+					itemName === COMPONENT_PREPARATION_LOCK_DIR ||
+					(itemName === DEPLOYMENT_PROVENANCE_FILE && path.dirname(dir) === componentsRoot)
 				)
 					continue;
 				const itemPath = path.join(dir, itemName);
@@ -1248,8 +1255,8 @@ async function getComponents() {
 		}
 	};
 
-	const results = await walkDir(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), {
-		name: configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT).split(path.sep).slice(-1).pop(),
+	const results = await walkDir(componentsRoot, {
+		name: componentsRoot.split(path.sep).slice(-1).pop(),
 		entries: [],
 	});
 	const { getUnsatisfiedEnv } = require('./componentSecrets.ts');

package/components/packageComponent.ts

@@ -3,6 +3,7 @@ import { stat, readdir } from 'node:fs/promises';
 import { Readable, pipeline } from 'node:stream';
 import tar from 'tar-fs';
 import { createGzip } from 'node:zlib';
+import { DEPLOYMENT_PROVENANCE_FILE } from './deploymentProvenance.ts';
 
 interface PackageOptions {
 	skip_node_modules?: boolean;
@@ -42,9 +43,12 @@ const INSTALL_OWNED_PATH = join('node_modules', 'harper');
  * excluded the whole tree. Shared by the stream packer and the directory walk so they cannot diverge.
  */
 function isExcluded(directory: string, fullPath: string, options: PackageOptions): boolean {
+	const name = basename(fullPath);
 	// Unconditional, including under `skip_symlinks`: packed literally, the link ships an absolute path to
 	// the packaging host's install root, which arrives dangling.
-	if (basename(fullPath) === 'harper' && relative(directory, fullPath) === INSTALL_OWNED_PATH) return true;
+	if (name === 'harper' && relative(directory, fullPath) === INSTALL_OWNED_PATH) return true;
+	// A package made from a live tree would otherwise carry that release's deployment id into the next one.
+	if (name === DEPLOYMENT_PROVENANCE_FILE && relative(directory, fullPath) === DEPLOYMENT_PROVENANCE_FILE) return true;
 	if (!options.skip_node_modules) return false;
 	const rel = relative(directory, fullPath);
 	return rel.split(sep).includes('node_modules') || rel.includes(WEBPACK_CACHE_SEGMENT);

package/config-root.schema.json

@@ -25,7 +25,7 @@
 				"compressionThreshold": {
 					"type": "integer",
 					"minimum": 0,
-					"description": "Responses larger than this threshold (bytes) will be compressed for clients that accept compression. Default: 1200"
+					"description": "Responses larger than this threshold (bytes) will be compressed for clients that accept brotli. Default: 0 (compression disabled)"
 				},
 				"cors": { "type": "boolean", "description": "Enable Cross Origin Resource Sharing. Default: true" },
 				"corsAccessList": {

package/dataLayer/DESIGN.md

@@ -89,13 +89,41 @@ Three non-obvious mechanics keep that safe:
   it and broadcasting a reload would surface the earlier attempt's partial/corrupt directory as
   healthy. Only a _fresh_ marker on a _previously healthy_ database that failed before destruction is
   safe to clear.
-- **The ITC close broadcast is best-effort, so closure is verified before the purge.** The SCHEMA
-  broadcast (`signalSchemaChange`) resolves after remote handlers complete but times out at 30s
-  "best-effort", swallows errors, and never reaches job-worker threads at all (their ports are
-  excluded from broadcasts to avoid re-entrant deadlocks). A destructive purge cannot trust it:
-  `restoreBackup` polls rocksdb-js `registryStatus()` (process-global across worker threads) until
-  the database path has no open instance, and aborts with a 409 — _cleaning up the marker, since
-  nothing was destroyed_ — if handles remain.
+- **The ITC close broadcast is normally best-effort, so closure is verified before the purge.** A
+  SCHEMA broadcast (`signalSchemaChange`) usually resolves after remote handlers complete but times
+  out at 30s "best-effort" and swallows errors. The restore `close` phase is stricter: it waits until
+  every eligible recipient acknowledges or its port closes, and aborts the restore with a retryable
+  409 if that does not happen within 30s, because proceeding past an unconfirmed blob-save barrier
+  would re-open the race the barrier exists to close. A destructive purge still
+  verifies closure independently: `restoreBackup` polls rocksdb-js `registryStatus()` (process-global
+  across worker threads) until the database path has no open instance, and aborts with a 409 —
+  _cleaning up the marker, since nothing was destroyed_ — if handles remain.
+- **The close acknowledgement fences blob saves, deferred reclamation and orphan cleanup, not just
+  database handles.** A store
+  handle can close while a `saveBlob` file pipeline it started is still pending, because blob roots
+  live outside RocksDB and streamed saves settle independently of the record write; a queued
+  reclamation is worse still, being a timer that never consults the database at all. The restore
+  signal therefore has explicit `close` and `reload` phases: every worker refuses new saves for that
+  database, stops draining reclamations for it, stops any `cleanup_orphan_blobs` walk in progress
+  (its "nothing references this path" verdict was reached against the generation being replaced),
+  awaits the saves and unlinks already dispatched -- including a save-failure cleanup's -- then
+  closes the store and acknowledges. Restore signals are the one schema broadcast that includes job
+  workers; ordinary gossip still excludes them to avoid re-entrant broadcast deadlocks.
+
+  The fence records the token of the restore that currently owns it, not a flag and not a set. Two
+  restores of the same database can overlap — a restore releases its lock before awaiting its `reload`
+  broadcast — so a database-wide boolean lets the first restore's late reload lift the second one's
+  fence, admitting saves during the second one's post-close check. A later close takes ownership
+  without the database ever unfencing in between, and a release whose token is not the current owner
+  does nothing. A _set_ of tokens fails the opposite way: a restore that dies after destruction never
+  reloads, so its token would keep the database fenced for the life of the process, defeating the
+  rerun the failure tells the operator to perform.
+
+  Releasing the fence discards that database's queued reclamations rather than resuming them: they
+  condemn file paths belonging to the generation the restore just replaced, so draining them would
+  unlink the bytes the restore wrote at those same paths. A restore that aborted before destroying
+  anything says so, and its queue is kept and rewoken instead.
+
 - **Online restore is impossible for a database a component holds open — and that failure is
   correct.** rocksdb-js's registry is process-global but records only a per-path refCount, with no
   attribution to a thread or component; Harper keeps no component→database ownership map. So when a

package/dataLayer/blobBackup.ts

@@ -101,6 +101,25 @@ const BACKUP_MARKER_REASONS: CaptureMarkerReasons = {
 	pending: 'blob was not yet complete when this backup was taken',
 };
 
+export async function* walkBlobFiles(root: string): AsyncGenerator<string> {
+	const stack: string[] = [root];
+	while (stack.length > 0) {
+		const dir = stack.pop() as string;
+		let entries;
+		try {
+			entries = await readdir(dir, { withFileTypes: true });
+		} catch (error: any) {
+			if (error.code === 'ENOENT') continue;
+			throw error;
+		}
+		for (const entry of entries) {
+			const path = join(dir, entry.name);
+			if (entry.isDirectory()) stack.push(path);
+			else if (entry.isFile()) yield path;
+		}
+	}
+}
+
 /** Put one blob file into the destination, returning how the entry was captured. */
 async function captureBlobFile(
 	srcPath: string,
@@ -151,34 +170,16 @@ export async function copyTree(
 	onProgress?: () => void
 ): Promise<{ substituted: number; captured: number; copied: number }> {
 	const counts = { substituted: 0, captured: 0, copied: 0 };
-	if (!existsSync(srcRoot)) return counts;
-	const stack: string[] = [srcRoot];
-	while (stack.length > 0) {
-		const dir = stack.pop() as string;
-		let entries;
-		try {
-			entries = await readdir(dir, { withFileTypes: true });
-		} catch (error: any) {
-			if (error.code === 'ENOENT') continue; // directory removed mid-walk
-			throw error;
-		}
-		for (const entry of entries) {
-			const srcPath = join(dir, entry.name);
-			if (entry.isDirectory()) {
-				stack.push(srcPath);
-			} else if (entry.isFile()) {
-				const destPath = join(destRoot, relative(srcRoot, srcPath));
-				if (classify) {
-					const disposition = await captureBlobFile(srcPath, destPath, reasons, counts);
-					if (disposition === 'pending' || disposition === 'gone') counts.substituted++;
-					else if (disposition === 'capture') counts.captured++;
-				} else {
-					await linkOrCopy(srcPath, destPath, counts);
-				}
-				onProgress?.();
-			}
-			// symlinks/other node types in a blob root are not expected and are intentionally skipped
+	for await (const srcPath of walkBlobFiles(srcRoot)) {
+		const destPath = join(destRoot, relative(srcRoot, srcPath));
+		if (classify) {
+			const disposition = await captureBlobFile(srcPath, destPath, reasons, counts);
+			if (disposition === 'pending' || disposition === 'gone') counts.substituted++;
+			else if (disposition === 'capture') counts.captured++;
+		} else {
+			await linkOrCopy(srcPath, destPath, counts);
 		}
+		onProgress?.();
 	}
 	return counts;
 }
@@ -335,6 +336,44 @@ export async function assertBlobSnapshotRestorable(
 	}
 }
 
+/**
+ * Refuse an engine-only restore unless the operator has accepted the mixed result.
+ *
+ * Restoring engine files without the matching blobs produces a mixed generation: records rolled back
+ * to the backup point, addressing whichever blobs the roots happen to hold. There are two ways that
+ * goes wrong, and the destination being empty rules out only the first:
+ *
+ * 1. Blobs are present now, so a restored record can resolve onto a blob that no longer belongs to it,
+ *    or onto nothing at all if the blob it referenced was deleted since the backup point.
+ * 2. The roots are empty, so nothing collides today -- but blob ids are a per-database counter that
+ *    `getNextFileId` (`resources/blob.ts`) re-seeds by scanning the roots. Empty roots re-seed at 1,
+ *    which is exactly the id space the restored records already reference, so the *next* blob written
+ *    is placed at a path a restored record points at. A brand-new target database is not exempt: its
+ *    roots are empty for the same reason.
+ *
+ * Purging the roots instead is not the answer -- that strips blobs the restored records still
+ * reference -- so the operator has to choose, and the choice is recorded. Case 2 would stop being a
+ * hazard if the blob-id high-water mark were persisted with the engine and restored alongside it, so
+ * the counter could not fall behind the references it has already handed out.
+ */
+export function assertEngineOnlyRestoreAllowed(
+	databaseName: string,
+	{ backupHasBlobs, allowEngineOnly }: { backupHasBlobs: boolean; allowEngineOnly: boolean }
+): void {
+	// Strictly true, not merely truthy: a manifest is on-disk data, and a corrupt `blobs` value must not
+	// be read as "this backup has blobs" and waved past the guard.
+	if (backupHasBlobs === true || allowEngineOnly === true) return;
+	// Decided from the manifest alone. Walking the destination to tailor the message would turn a
+	// deterministic 400 into a traversal that can be slow or fail outright (EACCES on an unreadable
+	// root), for wording -- and both hazards are worth stating anyway, since the operator who passes
+	// the opt-in is accepting whichever one applies.
+	throw new ClientError(
+		`Cannot restore an engine-only backup over database '${databaseName}': the backup captured no blobs, so the restored records would address whichever blobs are on disk now. ` +
+			`If the database still has blob files, a record can resolve to a blob that no longer belongs to it; if it has none, blob ids restart at 1 and the next blob written lands on a path a restored record already references — so a new target database is not an escape. ` +
+			`Restore a backup that includes blobs, or pass 'allow_engine_only' to accept the mixed result.`
+	);
+}
+
 /**
  * Restore a backup's blob snapshot back into the database's blob roots. Each root is purged and
  * rewritten from `blobs/<backupId>/<rootIndex>/` so the restored blob set matches the backup exactly
@@ -342,9 +381,11 @@ export async function assertBlobSnapshotRestorable(
  *
  * A backup created with blobs excluded (or an older backup that predates blob snapshots) has no
  * snapshot directory: in that case the live blob roots are left untouched and a warning is logged,
- * since purging them would strip blobs the restored records may still reference. Roots are restored
- * by index into the *same* configured root; an incompatible root count is rejected up front (see
- * `assertBlobSnapshotRestorable`) rather than collapsed, so blobs are never mis-addressed.
+ * since purging them would strip blobs the restored records may still reference. A restore only
+ * reaches that state when the operator accepted it with `allow_engine_only`
+ * (`assertEngineOnlyRestoreAllowed` refuses otherwise, whatever the destination roots hold). Roots are restored by index into the *same* configured root;
+ * an incompatible root count is rejected up front (see `assertBlobSnapshotRestorable`) rather than
+ * collapsed, so blobs are never mis-addressed.
  */
 export async function restoreBlobSnapshot(
 	backupDir: string,

package/dataLayer/harperBridge/ResourceBridge.ts

@@ -56,6 +56,7 @@ export type SearchByConditionsRequest = Query &
 		database?: string;
 		table: string;
 		get_attributes: Select;
+		hdb_user?: Context['user'];
 		reverse?: boolean;
 		operator?: Operator;
 	};
@@ -85,6 +86,12 @@ export class ResourceBridge extends BridgeMethods {
 					attribute: c.attribute ?? c.search_attribute,
 					comparator: c.comparator ?? c.search_type,
 					value: c.value !== undefined ? c.value : c.search_value, // null is valid value
+					...(c.fields === undefined ? null : { fields: c.fields }),
+					...(c.includeHighlights === undefined ? null : { includeHighlights: c.includeHighlights }),
+					...(c.maxIndexLagMilliseconds === undefined ? null : { maxIndexLagMilliseconds: c.maxIndexLagMilliseconds }),
+					...(c.waitForIndexMilliseconds === undefined
+						? null
+						: { waitForIndexMilliseconds: c.waitForIndexMilliseconds }),
 				};
 			}
 		}
@@ -107,6 +114,7 @@ export class ResourceBridge extends BridgeMethods {
 				allowFullScan: true, // operations API can do full scans by default, but REST is more cautious about what it allows
 			} as any,
 			{
+				user: searchObject.hdb_user,
 				onlyIfCached: searchObject.onlyIfCached,
 				noCacheStore: searchObject.noCacheStore,
 				noCache: searchObject.noCache,
@@ -727,8 +735,14 @@ export class ResourceBridge extends BridgeMethods {
 function getSelect({ get_attributes }: any, table: any) {
 	if (get_attributes) {
 		if (get_attributes[0] === '*') {
-			if (table.schemaDefined) return;
-			else get_attributes = table.attributes.map((attribute) => attribute.name);
+			const metadata = get_attributes.slice(1).filter((name) => name === '$score' || name === '$highlights');
+			if (table.schemaDefined && metadata.length === 0) return;
+			get_attributes = [
+				...table.attributes
+					.filter((attribute) => !attribute.computed && !attribute.relationship)
+					.map((attribute) => attribute.name),
+				...metadata,
+			];
 		}
 		get_attributes.forceNulls = true;
 		return get_attributes;