@harperfast/harper 5.2.5 → 5.2.7

package/resources/Table.ts

@@ -36,6 +36,7 @@ import {
 	DatabaseTransaction,
 	ImmediateTransaction,
 	priorStagedWrite,
+	isJoinableScope,
 	isReleasedTransaction,
 	TRANSACTION_STATE,
 	writeKeyId,
@@ -71,7 +72,15 @@ import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.
 import { HAS_BLOBS, auditRetention, removeAuditEntry } from './auditStore.ts';
 import { buildEmbedBefore, createDefaultEmbedder, type EmbedAttribute, type Embedder } from './models/embedHook.ts';
 import { autoCast, autoCastBooleanStrict } from '../utility/common_utils.ts';
-import { recordUpdater, removeEntry, PENDING_LOCAL_TIME, RecordObject, type Entry, entryMap } from './RecordEncoder.ts';
+import {
+	recordUpdater,
+	removeEntry,
+	PENDING_LOCAL_TIME,
+	RecordObject,
+	type Entry,
+	entryMap,
+	storedFieldsOnly,
+} from './RecordEncoder.ts';
 import { recordAction, recordActionBinary } from './analytics/write.ts';
 import { rebuildUpdateBefore } from './crdt.ts';
 import { appendHeader } from '../server/serverHelpers/Headers.ts';
@@ -452,6 +461,18 @@ export function makeTable(options) {
 	// it, so such a table takes the guarded serialization path rather than the raw fast path.
 	let hasSurfacedComputed = false;
 	let runningRecordExpiration: boolean;
+	const reportedResolverCollisions = new Set<string>();
+	// Reached from record materialization, so it can never be the reason a record fails to load: the
+	// name is marked before the log call, and a throwing log sink is swallowed.
+	function reportResolverCollision(name: string) {
+		if (reportedResolverCollisions.has(name)) return;
+		reportedResolverCollisions.add(name);
+		try {
+			logger.warn?.(
+				`Table "${tableName}" has a stored value under "${name}", which is a computed attribute; the stored value is being discarded and the computed value used instead`
+			);
+		} catch {}
+	}
 	const isRocksDB = primaryStore instanceof RocksDatabase;
 	type BigInt64ArrayAndMaxSafeId = BigInt64Array & { maxSafeId: number };
 	let idIncrementer: BigInt64ArrayAndMaxSafeId;
@@ -1873,7 +1894,28 @@ export function makeTable(options) {
 		}
 		#saveOperation(operation: any) {
 			const transaction = txnForContext(this.getContext());
-			if (transaction.save) return transaction.save(operation) || operation.promise || operation.result;
+			const holder = operation.stagedIn;
+			// never-drop-on-conflict lives on the transaction and would not travel with the write, so an
+			// apply or a replay keeps it (harper-pro#348)
+			const holderOwnsPolicy = holder?.sourceApply || holder?.isReplay;
+			// stagesWriteOnSave: LMDBTransaction's addWrite never runs the write (its commit applies
+			// `writes`), so handing it one is a dead end
+			if (
+				holder &&
+				holder !== transaction &&
+				!holderOwnsPolicy &&
+				transaction.stagesWriteOnSave &&
+				isJoinableScope(transaction)
+			) {
+				holder.detachWrite(operation);
+				// The basis chain belongs to the holder: derived from a write this scope cannot commit, the
+				// merge and index diff would be relative to a record that may never land.
+				operation.priorWrite = undefined;
+				operation.deferSave = false;
+				return when(transaction.addWrite(operation), () => operation.promise ?? operation.result);
+			}
+			const owner = holder ?? transaction;
+			if (owner.save) return owner.save(operation) || operation.promise || operation.result;
 		}
 
 		addTo(property: any, value: any) {
@@ -2314,11 +2356,11 @@ export function makeTable(options) {
 				nodeName: (context as any)?.nodeName,
 				fullUpdate,
 				deferSave: true,
-				validate: (txnTime) => {
+				validate: (txnTime, committedBy = transaction) => {
 					if (!recordUpdate) recordUpdate = this.#changes;
 					if (fullUpdate || (recordUpdate && hasChanges(this.#changes === recordUpdate ? this : recordUpdate))) {
 						if (!(context as any)?.source) {
-							transaction.checkOverloaded();
+							committedBy.checkOverloaded();
 							// A record must be a plain object. Reject primitive, string/number, bare-binary,
 							// and bare-array roots — e.g. a raw Buffer from an application/octet-stream PUT, a
 							// JSON string/number body, or a top-level JSON array. Such roots carry no primary
@@ -2356,7 +2398,7 @@ export function makeTable(options) {
 							// by replayLogs). Records were valid when originally written; post-crash schema
 							// evolution (e.g. newly required fields) must not prevent replaying them
 							// (harper#1316, facet b).
-							if (!transaction.isReplay) this.validate(recordUpdate, !fullUpdate);
+							if (!committedBy.isReplay) this.validate(recordUpdate, !fullUpdate);
 							if (updatedTimeProperty) {
 								recordUpdate[updatedTimeProperty.name] =
 									updatedTimeProperty.type === 'Date'
@@ -2391,7 +2433,7 @@ export function makeTable(options) {
 							// TODO: else freeze after we have applied the changes
 						}
 					} else {
-						(transaction as any).removeWrite?.(write);
+						(committedBy as any).removeWrite?.(write);
 						return false;
 					}
 				},
@@ -2434,6 +2476,7 @@ export function makeTable(options) {
 					let incrementalUpdateToApply: boolean;
 
 					this.#savingOperation = null;
+					write.stagedIn = undefined; // nothing may pin this write's transaction past its commit
 					let omitLocalRecord = false;
 					// we use optimistic locking to only commit if the existing record state still holds true.
 					// this is superior to using an async transaction since it doesn't require JS execution
@@ -4967,6 +5010,9 @@ export function makeTable(options) {
 			for (const attribute of this.attributes) {
 				if (attribute.isPrimaryKey) primaryKeyAttribute = attribute;
 				attribute.resolve = null; // reset this
+				// Also the setter, or a reload that turns a @relationship into a @computed keeps the
+				// relationship's setter and writes a foreign key from a computed assignment.
+				attribute.set = null;
 				const relationship = attribute.relationship;
 				const computed = attribute.computed;
 				// Register the default embedder unless an author override is set. Sits outside
@@ -5128,14 +5174,19 @@ export function makeTable(options) {
 			enumerableAttributeNames = [];
 			enumerableRelationDefs.clear();
 			hasSurfacedComputed = false;
+			const resolvedAttributeNames: string[] = [];
 			for (const attribute of attributes) {
 				const name = attribute.name;
 				if (attribute.resolve) {
+					resolvedAttributeNames.push(name);
 					Object.defineProperty(primaryStore.encoder.structPrototype, name, {
 						get() {
 							return attribute.resolve(this, contextStorage.getStore()); // it is only possible to get the context from ALS, we don't have a direct reference to the current context
 						},
 						set(related) {
+							// A read-only resolver must never be the reason a record fails to materialize — the
+							// same reason the one-to-many branch above installs a no-op.
+							if (!attribute.set) return reportResolverCollision(name);
 							return attribute.set(this, related);
 						},
 						configurable: true,
@@ -5161,6 +5212,12 @@ export function makeTable(options) {
 			if (enumerableAttributeNames.length > 0)
 				installEnumerableToJSON(primaryStore.encoder.structPrototype, this, hasSurfacedComputed);
 			else if (primaryStore.encoder.structPrototype.toJSON) delete primaryStore.encoder.structPrototype.toJSON;
+			// Undefined rather than an empty set for a table with no resolved attributes: that is the
+			// check which keeps the projection off an unaffected store's write path.
+			primaryStore.encoder.resolvedAttributeNames =
+				resolvedAttributeNames.length > 0 ? new Set(resolvedAttributeNames) : undefined;
+			primaryStore.encoder.resolvedAttributeNamesList = resolvedAttributeNames;
+			primaryStore.encoder.surfacedToJSON = primaryStore.encoder.structPrototype.toJSON;
 		}
 		// #section: computed-history
 		static setComputedAttribute(attribute_name, resolver) {
@@ -5645,10 +5702,33 @@ export function makeTable(options) {
 				// See if this is a transaction for our database and if so, use it
 				if (transaction.db?.path === primaryStore.path) return transaction;
 				// try the next one:
-				const nextTxn = transaction.next;
+				let nextTxn = transaction.next;
+				// A self-committing link is CLOSED once it has committed, and a further write through it
+				// commits on a native handle nothing awaits (#2323). Spent — closed, handle detached, none of
+				// its OWN writes left (hasPendingWrites walks successors, which is not this question) — it
+				// holds nothing, so drop it. A run of them can be spent, hence the loop. A timeout-poisoned
+				// link is kept: reusing it is what makes the rest of the operation fail atomically (#1411).
+				while (
+					nextTxn?.saveCommits &&
+					nextTxn.open !== TRANSACTION_STATE.OPEN &&
+					!nextTxn.timedOut &&
+					!nextTxn.transaction &&
+					!nextTxn.writes.some((write) => write)
+				) {
+					transaction.next = nextTxn.next;
+					nextTxn = transaction.next;
+				}
 				if (!nextTxn) {
 					// no next one, then add our database
-					transaction.next = isRocksDB ? new DatabaseTransaction() : new LMDBTransaction();
+					// A staging link under a self-committing head is committed only if the head's own database
+					// is written again and cascades the chain, so a handler writing this one last loses it (#2292).
+					transaction.next = transaction.saveCommits
+						? ((isRocksDB
+								? new ImmediateTransaction(primaryStore as any)
+								: new ImmediateLMDBTransaction(primaryStore as any)) as any)
+						: isRocksDB
+							? new DatabaseTransaction()
+							: new LMDBTransaction();
 					// The chain root, so a link that only ever receives a blind write is supervised by the
 					// long-transaction monitor as part of its logical transaction rather than as its own
 					// timeout root (issue #2231).
@@ -5670,8 +5750,11 @@ export function makeTable(options) {
 					// A second database joined after a mid-scope commit belongs to the same snapshot-free
 					// generation as the head, or its reads would re-pin what the commit just unpinned.
 					transaction.next.snapshotFree = transaction.snapshotFree;
-					if (transaction.open === TRANSACTION_STATE.CLOSED) {
+					if (transaction.open === TRANSACTION_STATE.CLOSED && !transaction.next.saveCommits) {
 						// if the current transaction is already closed, we need to retain that state on new databases we work with
+						// Never onto a self-committing link: CLOSED is what routes its first write through the
+						// commit re-entry that drops the native commit promise (#2323), and it commits per write
+						// regardless of this state.
 						transaction.next.open = TRANSACTION_STATE.CLOSED;
 					}
 					transaction = transaction.next;
@@ -5948,6 +6031,31 @@ export function makeTable(options) {
 						// before stamping the primary key and created/updated times below (records are immutable —
 						// 5.2 record caching relies on it — so we must not write through the frozen object).
 						if (isFrozenRecordObject(updatedRecord)) updatedRecord = { ...updatedRecord };
+						// A writable resolver (a relationship) keeps its meaning for source payloads: a source
+						// returning the related object instead of the foreign key gets the key derived, the way
+						// the promotion setter used to. Derivation runs against a probe first — a malformed
+						// value (a scalar, an object without the related primary key) derives undefined, and
+						// applying that would durably wipe the foreign key. The projection then drops the
+						// resolver-owned key itself, so the cache-fill response cannot be the one read
+						// reporting the source's value.
+						const resolvedNames = primaryStore.encoder.resolvedAttributeNamesList;
+						if (resolvedNames) {
+							for (const name of resolvedNames) {
+								if (Object.hasOwn(updatedRecord, name)) {
+									const resolvedAttribute = findAttribute(attributes, name);
+									if (resolvedAttribute?.set && updatedRecord[name] != null) {
+										const probe = {};
+										resolvedAttribute.set(probe, updatedRecord[name]);
+										for (const key in probe) {
+											const derived = probe[key];
+											const usable = Array.isArray(derived) ? derived.every((one) => one != null) : derived != null;
+											if (usable) updatedRecord[key] = derived;
+										}
+									}
+								}
+							}
+						}
+						updatedRecord = storedFieldsOnly(primaryStore.encoder, updatedRecord);
 						if (primaryKey && updatedRecord[primaryKey] !== id) updatedRecord[primaryKey] = id;
 					}
 					resolved = true;

package/resources/graphql.ts

@@ -351,4 +351,4 @@ async function processGraphQLSchema(gqlContent, urlPath, filePath, resources) {
 }
 
 // useful for testing
-export const loadGQLSchema = (content) => processGraphQLSchema(content, null, null, new Resources());
+export const loadGQLSchema = (content) => processGraphQLSchema(content, null, '<inline-schema>', new Resources());

package/resources/tracked.ts

@@ -45,6 +45,10 @@ export function assignTrackedAccessors(Target, typeDef, useFullPropertyProxy = f
 					return attribute.resolve(this, this.getContext?.());
 				},
 				set(related) {
+					// Unlike the struct-prototype accessor (which tolerates a stored collision so a record can
+					// still load), this is a user assignment to a resolved attribute, and a read-only resolver
+					// (a scalar @computed has no attribute.set) has nowhere to put it (harper#2359).
+					if (!attribute.set) throw new ClientError(`${name} is a computed attribute and can not be assigned`);
 					return attribute.set(this, related);
 				},
 				configurable: true,

package/resources/transaction.ts

@@ -2,9 +2,9 @@ import type { Context } from './ResourceInterface.ts';
 import { _assignPackageExport } from '../globals.js';
 import {
 	DatabaseTransaction,
+	isJoinableScope,
 	isReleasedTransaction,
 	type Transaction,
-	TRANSACTION_STATE,
 } from './DatabaseTransaction.ts';
 import { AsyncLocalStorage } from 'async_hooks';
 
@@ -41,7 +41,7 @@ export function transaction<T>(
 	if (typeof callback !== 'function') {
 		throw new TypeError('Callback function must be provided to transaction');
 	}
-	if (context?.transaction?.open === TRANSACTION_STATE.OPEN && typeof callback === 'function') {
+	if (isJoinableScope(context?.transaction) && typeof callback === 'function') {
 		return callback(context.transaction); // nothing to be done, already in open transaction
 	}
 

package/server/DESIGN.md

@@ -68,6 +68,12 @@ A request entering `http.ts` does **not** go through Fastify. The two `handleApp
 | `threads/itc.js`           | Inter-thread comms primitives.                           |
 | `transactionLogCooling.ts` | Main-thread timer that cools transaction-log mmaps.      |
 
+Process-wide shutdown begins by calling `beginProcessShutdown()` in `threads/manageThreads.js`.
+Once set, this terminal state prevents every worker replacement path and makes new `startWorker()`
+calls fail with `ERR_HARPER_PROCESS_SHUTTING_DOWN`; scoped worker-type restarts do not set it.
+`shutdownWorkersNow()` remains an immediate teardown: its worker shutdown messages are best-effort,
+and it force-terminates the remaining worker set rather than waiting for application drain hooks.
+
 > Workers receive `workerData.noServerStart = true` — never start the server inside a worker.
 >
 > `threadServer.listenOnDomainSocket()` skips a listener only when its path exceeds the platform's

package/server/threads/manageThreads.js

@@ -46,6 +46,7 @@ const chokidar = require('chokidar');
 const isBun = typeof globalThis.Bun !== 'undefined';
 const MB = 1024 * 1024;
 const workers = []; // these are our child workers that we are managing
+let processShuttingDown = false;
 const connectedPorts = []; // these are all known connected worker ports (siblings, children, parents)
 const MAX_UNEXPECTED_RESTARTS = 50;
 // Threads get 10s to die before they're forced. In dev (`harper dev`) we widen this: a reload's old
@@ -150,6 +151,7 @@ module.exports = {
 	setTerminateTimeout,
 	extendShutdownDeadline,
 	restoreShutdownDeadline,
+	beginProcessShutdown,
 	registerWorkerDataProvider,
 	onThreadExit,
 	registerProcessGroup,
@@ -324,6 +326,11 @@ listenersByType.set(THREAD_INFO, null);
 listenersByType.set(PROCESS_GROUP_TERMINATION_CONFIRMED, null);
 
 function startWorker(path, options = {}) {
+	if (processShuttingDown) {
+		const error = new Error('Cannot start a worker while the Harper process is shutting down');
+		error.code = 'ERR_HARPER_PROCESS_SHUTTING_DOWN';
+		throw error;
+	}
 	// Take a percentage of total memory to determine the max memory for each thread. The percentage is based
 	// on the thread count. Generally, it is unrealistic to efficiently use the majority of total memory for a single
 	// NodeJS worker since it would lead to massive swap space usage with other processes and there is significant
@@ -435,7 +442,7 @@ function startWorker(path, options = {}) {
 	});
 	worker.on('exit', (_code) => {
 		workers.splice(workers.indexOf(worker), 1);
-		if (!worker.wasShutdown && options.autoRestart !== false) {
+		if (!processShuttingDown && !worker.wasShutdown && options.autoRestart !== false) {
 			// if this wasn't an intentional shutdown, restart now (unless we have tried too many times)
 			if (worker.unexpectedRestarts < MAX_UNEXPECTED_RESTARTS) {
 				options.unexpectedRestarts = worker.unexpectedRestarts + 1;
@@ -469,6 +476,7 @@ async function restartWorkers(
 	startReplacementThreads = true
 ) {
 	if (isMainThread) {
+		if (processShuttingDown && startReplacementThreads) return;
 		try {
 			// we do this because it is possible for a component to chdir to itself, get re-deployed and then the cwd
 			// inode link is invalid and it can cause a lot of problems. But process.cwd() still returns the path, for
@@ -506,6 +514,8 @@ async function restartWorkers(
 		// listenOnPorts() treat a dedicated listener's EADDRINUSE as an external conflict.
 		const canPreStartReplacement = process.platform !== 'win32' && process.platform !== 'darwin' && !isBun;
 		for (let worker of workers.slice(0)) {
+			// Terminal shutdown: stop replacing workers mid-loop — the guard for every replacement start below.
+			if (processShuttingDown && startReplacementThreads) break;
 			if ((name && worker.name !== name) || worker.wasShutdown) continue; // filter by type, if specified
 			const overlapping = OVERLAPPING_RESTART_TYPES.indexOf(worker.name) > -1;
 			if (overlapping && startReplacementThreads && canPreStartReplacement) {
@@ -590,7 +600,7 @@ async function restartWorkers(
 			// Overlapping types we couldn't pre-start (Windows/Bun): start the replacement now that the old
 			// worker is releasing its port. server.close() stops accepting immediately, so the port frees up
 			// well before the replacement finishes booting and binds.
-			if (overlapping && startReplacementThreads && !canPreStartReplacement) worker.startCopy();
+			if (overlapping && startReplacementThreads && !canPreStartReplacement && !processShuttingDown) worker.startCopy();
 			let whenDone = new Promise((resolve) => {
 				// in case the exit inside the thread doesn't timeout, force it from the outside
 				const armTerminate = (delay) =>
@@ -627,7 +637,7 @@ async function restartWorkers(
 					const index = waitingToFinish.indexOf(whenDone);
 					if (index > -1) waitingToFinish.splice(index, 1);
 					// non-overlapping types have no advance replacement, so start it once the old one is gone
-					if (!overlapping && startReplacementThreads) worker.startCopy();
+					if (!overlapping && startReplacementThreads && !processShuttingDown) worker.startCopy();
 					resolve();
 				});
 			});
@@ -650,7 +660,11 @@ async function restartWorkers(
 function shutdownWorkers(name) {
 	return restartWorkers(name, Infinity, false);
 }
+function beginProcessShutdown() {
+	processShuttingDown = true;
+}
 async function shutdownWorkersNow(name) {
+	if (name == null) beginProcessShutdown();
 	shutdownWorkers(name); // set the state of all the workers to shut down. this should finish the important stuff synchronously
 	if (isBun) {
 		// worker.terminate() triggers a NAPI segfault in Bun; ask workers to self-exit instead