@harperfast/harper 5.2.6 → 5.2.8

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@harperfast/harper",
 	"description": "Harper is an open-source Node.js performance platform that unifies database, cache, application, and messaging layers into one in-memory process.",
-	"version": "5.2.6",
+	"version": "5.2.8",
 	"license": "Apache-2.0",
 	"homepage": "https://harper.fast",
 	"bugs": {
@@ -179,7 +179,7 @@
 		"@fastify/cors": "^11.2.0",
 		"@fastify/static": "^9.1.3",
 		"@harperfast/extended-iterable": "1.0.3",
-		"@harperfast/rocksdb-js": "2.7.1",
+		"@harperfast/rocksdb-js": "2.8.0",
 		"@harperfast/skills": "^1.10.8",
 		"@turf/area": "6.5.0",
 		"@turf/boolean-contains": "6.5.0",
@@ -223,7 +223,7 @@
 		"minimist": "1.2.8",
 		"moment": "2.30.1",
 		"mqtt-packet": "~9.0.1",
-		"msgpackr": "2.0.5",
+		"msgpackr": "2.0.6",
 		"needle": "3.5.0",
 		"node-forge": "^1.3.1",
 		"node-stream-zip": "1.16.0",
@@ -244,7 +244,7 @@
 		"ses": "^1.15.0",
 		"stream-chain": "2.2.5",
 		"stream-json": "1.9.1",
-		"structon": "1.0.8",
+		"structon": "1.1.0",
 		"systeminformation": "^5.31.4",
 		"tar-fs": "^3.1.2",
 		"tar-stream": "^3.1.8",

package/resources/DESIGN.md

@@ -166,6 +166,11 @@ Tests: `../unitTests/resources/defineResource.test.js`, `../unitTests/resources/
 
 ## Conventions
 
+- A record's `toJSON` is a response projection, not durable state. `recordUpdater` (and the cache-fill
+  and audit paths feeding it) projects records to stored fields via `storedFieldsOnly`, so no durable
+  form carries a resolver-owned (`@computed`/`@relationship`) name; materialization skips such names
+  when promoting a decoded record (`assignStoredFields`), which is what keeps rows written by affected
+  releases readable. See harper#2359 and `integrationTests/resources/cachedComputedAttribute.test.ts`.
 - **Never** remove `transactional()` from a static method on `Resource` — it owns transaction context lifetime.
 - New `Resource` subclasses should override **instance** methods (`get`, `put`, ...) for behavior; static methods are the protocol entry points and stay generic.
 - **Overriding a static entry point takes over its whole contract.** Assigning `static post`/`put`/`patch` on a subclass shadows the `transactional()`-wrapped static, so none of that wrapper runs — including the `when(data, ...)` that resolves `data` and the `allowCreate`/`allowUpdate` gate. That is by design (it is how `login.ts` implements a deliberately pre-authentication endpoint), and it means an override owns two obligations the wrapper would otherwise have met:

package/resources/PrimaryRocksDatabase.ts

@@ -3,7 +3,7 @@ import { RocksDatabase, type RocksDatabaseOptions, constants, type Store } from
 const FRESH_VERSION_FLAG = constants.FRESH_VERSION_FLAG;
 import { WeakLRUCache } from 'weak-lru-cache';
 import { when } from '../utility/when.ts';
-import { entryMap, METADATA, type Entry } from './RecordEncoder.ts';
+import { assignStoredFields, entryMap, METADATA, type Entry } from './RecordEncoder.ts';
 
 /**
  * RocksDatabase subclass that owns all primary-store behaviour for Harper tables:
@@ -60,7 +60,7 @@ export class PrimaryRocksDatabase extends RocksDatabase {
 			if (entry.value.constructor === Object && this.#enc.structPrototype) {
 				const originalValue = entry.value;
 				entry.value = new this.#enc.structPrototype.constructor();
-				Object.assign(entry.value, originalValue);
+				assignStoredFields(entry.value, originalValue, this.#enc);
 			}
 			if (typeof entry.value === 'object' && entry.value !== null) {
 				entryMap.set(entry.value, entry);
@@ -164,7 +164,7 @@ export class PrimaryRocksDatabase extends RocksDatabase {
 			if (entry.value?.constructor === Object && enc.structPrototype) {
 				const originalValue = entry.value;
 				entry.value = new enc.structPrototype.constructor();
-				for (const key in originalValue) entry.value[key] = originalValue[key];
+				assignStoredFields(entry.value, originalValue, enc);
 			}
 			return entry;
 		});

package/resources/RecordEncoder.ts

@@ -6,7 +6,7 @@
  */
 
 import { Encoder } from 'msgpackr';
-import { createStructon } from 'structon';
+import { createStructon, SOURCE_SYMBOL } from 'structon';
 import {
 	HAS_PREVIOUS_RESIDENCY_ID,
 	HAS_CURRENT_RESIDENCY_ID,
@@ -123,12 +123,86 @@ let timestampNextEncoding = 0,
 	additionalAuditRefsNextEncoding: Array<{ version: number; nodeId: number }> | undefined;
 // tracking metadata with a singleton works better than trying to alter response of getEntry/get and coordinating that across caching layers
 export let lastMetadata: Entry | null = null;
+/**
+ * A resolver owns its attribute's name, so no durable form of a record may carry a value under it.
+ * A value gets there either through the response `toJSON` the table installs on the record prototype
+ * (msgpackr consults an instance's toJSON when it encodes one) or as an own key from a source or
+ * peer payload.
+ */
+export function storedFieldsOnly(encoder: any, record: any) {
+	if (record === null || typeof record !== 'object') return record;
+	// Optional: this also runs for a store whose encode hook was grafted onto a foreign encoder
+	// (copyDb's migration target), which carries no table schema.
+	const resolved = encoder?.resolvedAttributeNames;
+	if (resolved === undefined) return record;
+	const resolvedList = encoder.resolvedAttributeNamesList ?? Array.from(resolved);
+	// A typed-struct instance keeps its stored fields as accessors on a structon-generated child
+	// prototype carrying structon's own toJSON, so it never reaches the response projection — and
+	// copying its own keys would encode an empty record.
+	let project =
+		encoder.surfacedToJSON !== undefined &&
+		record.toJSON === encoder.surfacedToJSON &&
+		record[SOURCE_SYMBOL] === undefined;
+	if (!project && typeof record.toJSON === 'function' && record[SOURCE_SYMBOL] === undefined) {
+		// A class instance with its own serialization: the encoder will consult that toJSON, so the
+		// projection has to run on its OUTPUT, or the serialized form smuggles resolver-owned keys past
+		// a check that only saw the instance. A non-object (or self) result is left for the encoder,
+		// which invokes toJSON again — serializers are assumed pure.
+		const json = record.toJSON();
+		if (json !== record && json !== null && typeof json === 'object') record = json;
+	}
+	if (!project) {
+		for (let i = 0; i < resolvedList.length; i++) {
+			if (Object.hasOwn(record, resolvedList[i])) {
+				project = true;
+				break;
+			}
+		}
+	}
+	if (!project) return record;
+	const stored = {};
+	for (const key of Object.keys(record)) {
+		if (resolved.has(key)) continue;
+		// '__proto__' as an own key must stay a data property: plain assignment would run the inherited
+		// setter, dropping the field from the durable write and swapping the temporary's prototype.
+		if (key === '__proto__')
+			Object.defineProperty(stored, key, { value: record[key], enumerable: true, writable: true, configurable: true });
+		else stored[key] = record[key];
+	}
+	return stored;
+}
+
+/**
+ * A resolver-owned name is skipped rather than assigned during record-instance promotion: through a
+ * read-only resolver the assignment throws, and through a writable one (a many-to-one
+ * `@relationship`) it would derive a foreign key from the stale stored value.
+ */
+export function assignStoredFields(target: any, source: any, encoder: any) {
+	const resolved = encoder?.resolvedAttributeNames;
+	if (resolved === undefined) return Object.assign(target, source);
+	// Own keys, matching the Object.assign fallback above and storedFieldsOnly: a decoded plain
+	// object has no inherited enumerables of its own, so walking the chain could only ever pick up
+	// somebody else's (a polluted Object.prototype included).
+	for (const key of Object.keys(source)) {
+		if (resolved.has(key)) continue;
+		if (key === '__proto__')
+			Object.defineProperty(target, key, { value: source[key], enumerable: true, writable: true, configurable: true });
+		else target[key] = source[key];
+	}
+	return target;
+}
+
 export class RecordEncoder extends StructonEncoder {
 	rootStore: any;
 	declare saveStructures: any;
 	declare getStructures: any;
 	declare _writeStruct: any;
 	structureUpdate?: any;
+	// Set by TableResource.updatedAttributes. Undefined for a store with no resolved attributes, which
+	// is what keeps the projection off that store's writes entirely.
+	resolvedAttributeNames?: Set<string>;
+	resolvedAttributeNamesList?: string[];
+	surfacedToJSON?: () => any;
 	isRocksDB: boolean;
 	name: string;
 	useVersions: boolean;
@@ -539,7 +613,7 @@ export function handleLocalTimeForGets(store, rootStore) {
 					// if an object was deserialized as a plain object, give it the right prototype for computed properties to be accessible
 					const originalValue = entry.value;
 					entry.value = new store.encoder.structPrototype.constructor();
-					Object.assign(entry.value, originalValue);
+					assignStoredFields(entry.value, originalValue, store.encoder);
 				}
 				entryMap.set(entry.value, entry); // allow the record to access the entry
 			}
@@ -591,7 +665,7 @@ export function handleLocalTimeForGets(store, rootStore) {
 					// if an object was deserialized as a plain object, give it the right prototype for computed properties to be accessible
 					const originalValue = entry.value;
 					entry.value = new store.encoder.structPrototype.constructor();
-					for (const key in originalValue) entry.value[key] = originalValue[key];
+					assignStoredFields(entry.value, originalValue, store.encoder);
 				}
 			}
 			return entry;
@@ -708,6 +782,10 @@ export function recordUpdater(store, tableId, auditStore) {
 		// harper#1309: reset so a record===undefined call (delete/no-op) cannot carry stale bytes
 		// into the audit encodedRecord or write-size analytics of this call.
 		lastValueEncoding = undefined;
+		// Every durable form of the record — the stored value, the audit entry's encodedRecord and so the
+		// transaction log and replication payload — comes from this record, so projecting it here covers
+		// them all; nested values keep their own toJSON serialization.
+		record = storedFieldsOnly(store.encoder, record);
 		const isRocksDB = store instanceof RocksDatabase;
 		// determine if and how we apply the local timestamp
 		if (isRocksDB) {
@@ -809,6 +887,10 @@ export function recordUpdater(store, tableId, auditStore) {
 			if (audit) {
 				const username = typeof options?.user === 'string' ? options.user : options?.user?.username;
 				if (auditRecord) {
+					// The audit encodedRecord is durable record state too (it is the transaction log and the
+					// replication payload), so it takes the same projection — except for message/publish
+					// entries, whose auditRecord IS the published payload and must reach subscribers verbatim.
+					if (type !== 'message' && type !== 'publish') auditRecord = storedFieldsOnly(store.encoder, auditRecord);
 					encodeBlobsWithFilePath(() => store.encoder.encode(auditRecord), id, store.rootStore);
 					if (blobsWereEncoded) {
 						extendedType |= HAS_BLOBS;

package/resources/RocksTransactionLogStore.ts

@@ -117,7 +117,14 @@ export class RocksTransactionLogStore extends EventEmitter {
 			}
 			entryBinary = createAuditEntry(auditRecord, position);
 		}
-		if (this.listenerCount('aftercommit')) (options.transaction.logEntries ??= []).push(auditRecord);
+		if (this.listenerCount('aftercommit')) {
+			if (!(auditRecord instanceof Uint8Array)) {
+				const txnTimestamp = options.transaction.getTimestamp?.();
+				if (txnTimestamp != null) auditRecord.localTime = txnTimestamp;
+				auditRecord.recordVersion = auditRecord.version;
+			}
+			(options.transaction.logEntries ??= []).push(auditRecord);
+		}
 		// One commit hook per transaction handles both deferred side effects — the `aftercommit` emit and the
 		// per-(log, table) structureVersion watermark advances — so they apply only after a durable commit (never
 		// on an aborted/discarded transaction). This store is the sole setter of transaction.onCommit.
@@ -414,7 +421,10 @@ export class RocksTransactionLogStore extends EventEmitter {
 					position += 8;
 				}
 				const auditRecord = readAuditEntry(data, position, undefined);
+				// version stays the log key (replication resume relies on version === log key);
+				// the record's own version survives as recordVersion
 				auditRecord.version = timestamp;
+				auditRecord.localTime = timestamp;
 				auditRecord.endTxn = endTxn;
 				auditRecord.previousResidencyId = previousResidencyId;
 				auditRecord.previousVersion = previousVersion;

package/resources/Table.ts

@@ -72,7 +72,15 @@ import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.
 import { HAS_BLOBS, auditRetention, removeAuditEntry } from './auditStore.ts';
 import { buildEmbedBefore, createDefaultEmbedder, type EmbedAttribute, type Embedder } from './models/embedHook.ts';
 import { autoCast, autoCastBooleanStrict } from '../utility/common_utils.ts';
-import { recordUpdater, removeEntry, PENDING_LOCAL_TIME, RecordObject, type Entry, entryMap } from './RecordEncoder.ts';
+import {
+	recordUpdater,
+	removeEntry,
+	PENDING_LOCAL_TIME,
+	RecordObject,
+	type Entry,
+	entryMap,
+	storedFieldsOnly,
+} from './RecordEncoder.ts';
 import { recordAction, recordActionBinary } from './analytics/write.ts';
 import { rebuildUpdateBefore } from './crdt.ts';
 import { appendHeader } from '../server/serverHelpers/Headers.ts';
@@ -453,6 +461,18 @@ export function makeTable(options) {
 	// it, so such a table takes the guarded serialization path rather than the raw fast path.
 	let hasSurfacedComputed = false;
 	let runningRecordExpiration: boolean;
+	const reportedResolverCollisions = new Set<string>();
+	// Reached from record materialization, so it can never be the reason a record fails to load: the
+	// name is marked before the log call, and a throwing log sink is swallowed.
+	function reportResolverCollision(name: string) {
+		if (reportedResolverCollisions.has(name)) return;
+		reportedResolverCollisions.add(name);
+		try {
+			logger.warn?.(
+				`Table "${tableName}" has a stored value under "${name}", which is a computed attribute; the stored value is being discarded and the computed value used instead`
+			);
+		} catch {}
+	}
 	const isRocksDB = primaryStore instanceof RocksDatabase;
 	type BigInt64ArrayAndMaxSafeId = BigInt64Array & { maxSafeId: number };
 	let idIncrementer: BigInt64ArrayAndMaxSafeId;
@@ -4113,7 +4133,8 @@ export function makeTable(options) {
 							// been written, so are fresh in memory.
 							const entry: Entry = primaryStore.getEntry(id);
 							if (entry) {
-								if (entry.version !== auditRecord.version) return; // out of order event, with old update, don't send anything
+								// staleness is a record-version comparison; auditRecord.version is the log key on RocksDB
+								if (entry.version !== (auditRecord.recordVersion ?? auditRecord.version)) return; // out of order event, with old update, don't send anything
 								value = entry.value;
 								type = entry.metadataFlags & INVALIDATED ? 'invalidate' : value ? 'put' : 'delete';
 							} else {
@@ -4990,6 +5011,9 @@ export function makeTable(options) {
 			for (const attribute of this.attributes) {
 				if (attribute.isPrimaryKey) primaryKeyAttribute = attribute;
 				attribute.resolve = null; // reset this
+				// Also the setter, or a reload that turns a @relationship into a @computed keeps the
+				// relationship's setter and writes a foreign key from a computed assignment.
+				attribute.set = null;
 				const relationship = attribute.relationship;
 				const computed = attribute.computed;
 				// Register the default embedder unless an author override is set. Sits outside
@@ -5151,14 +5175,19 @@ export function makeTable(options) {
 			enumerableAttributeNames = [];
 			enumerableRelationDefs.clear();
 			hasSurfacedComputed = false;
+			const resolvedAttributeNames: string[] = [];
 			for (const attribute of attributes) {
 				const name = attribute.name;
 				if (attribute.resolve) {
+					resolvedAttributeNames.push(name);
 					Object.defineProperty(primaryStore.encoder.structPrototype, name, {
 						get() {
 							return attribute.resolve(this, contextStorage.getStore()); // it is only possible to get the context from ALS, we don't have a direct reference to the current context
 						},
 						set(related) {
+							// A read-only resolver must never be the reason a record fails to materialize — the
+							// same reason the one-to-many branch above installs a no-op.
+							if (!attribute.set) return reportResolverCollision(name);
 							return attribute.set(this, related);
 						},
 						configurable: true,
@@ -5184,6 +5213,12 @@ export function makeTable(options) {
 			if (enumerableAttributeNames.length > 0)
 				installEnumerableToJSON(primaryStore.encoder.structPrototype, this, hasSurfacedComputed);
 			else if (primaryStore.encoder.structPrototype.toJSON) delete primaryStore.encoder.structPrototype.toJSON;
+			// Undefined rather than an empty set for a table with no resolved attributes: that is the
+			// check which keeps the projection off an unaffected store's write path.
+			primaryStore.encoder.resolvedAttributeNames =
+				resolvedAttributeNames.length > 0 ? new Set(resolvedAttributeNames) : undefined;
+			primaryStore.encoder.resolvedAttributeNamesList = resolvedAttributeNames;
+			primaryStore.encoder.surfacedToJSON = primaryStore.encoder.structPrototype.toJSON;
 		}
 		// #section: computed-history
 		static setComputedAttribute(attribute_name, resolver) {
@@ -5997,6 +6032,31 @@ export function makeTable(options) {
 						// before stamping the primary key and created/updated times below (records are immutable —
 						// 5.2 record caching relies on it — so we must not write through the frozen object).
 						if (isFrozenRecordObject(updatedRecord)) updatedRecord = { ...updatedRecord };
+						// A writable resolver (a relationship) keeps its meaning for source payloads: a source
+						// returning the related object instead of the foreign key gets the key derived, the way
+						// the promotion setter used to. Derivation runs against a probe first — a malformed
+						// value (a scalar, an object without the related primary key) derives undefined, and
+						// applying that would durably wipe the foreign key. The projection then drops the
+						// resolver-owned key itself, so the cache-fill response cannot be the one read
+						// reporting the source's value.
+						const resolvedNames = primaryStore.encoder.resolvedAttributeNamesList;
+						if (resolvedNames) {
+							for (const name of resolvedNames) {
+								if (Object.hasOwn(updatedRecord, name)) {
+									const resolvedAttribute = findAttribute(attributes, name);
+									if (resolvedAttribute?.set && updatedRecord[name] != null) {
+										const probe = {};
+										resolvedAttribute.set(probe, updatedRecord[name]);
+										for (const key in probe) {
+											const derived = probe[key];
+											const usable = Array.isArray(derived) ? derived.every((one) => one != null) : derived != null;
+											if (usable) updatedRecord[key] = derived;
+										}
+									}
+								}
+							}
+						}
+						updatedRecord = storedFieldsOnly(primaryStore.encoder, updatedRecord);
 						if (primaryKey && updatedRecord[primaryKey] !== id) updatedRecord[primaryKey] = id;
 					}
 					resolved = true;

package/resources/auditStore.ts

@@ -34,7 +34,8 @@ initSync();
 
 export type AuditRecord = {
 	version: number;
-	localTime: number; // only to be used by LMDB (from the key)
+	recordVersion?: number; // the record's own version; on RocksDB reads `version` becomes the log key, so record identity uses this
+	localTime: number; // log position: LMDB audit-store key; RocksDB transaction-log timestamp
 	type: string;
 	encodedRecord?: Buffer;
 	extendedType?: number;
@@ -620,6 +621,7 @@ export function readAuditEntry(buffer: Uint8Array, start = 0, end = undefined):
 				return buffer.subarray(recordIdStart, recordIdEnd);
 			},
 			version,
+			recordVersion: version,
 			previousVersion,
 			get user() {
 				try {

package/resources/graphql.ts

@@ -351,4 +351,4 @@ async function processGraphQLSchema(gqlContent, urlPath, filePath, resources) {
 }
 
 // useful for testing
-export const loadGQLSchema = (content) => processGraphQLSchema(content, null, null, new Resources());
+export const loadGQLSchema = (content) => processGraphQLSchema(content, null, '<inline-schema>', new Resources());

package/resources/tracked.ts

@@ -45,6 +45,10 @@ export function assignTrackedAccessors(Target, typeDef, useFullPropertyProxy = f
 					return attribute.resolve(this, this.getContext?.());
 				},
 				set(related) {
+					// Unlike the struct-prototype accessor (which tolerates a stored collision so a record can
+					// still load), this is a user assignment to a resolved attribute, and a read-only resolver
+					// (a scalar @computed has no attribute.set) has nowhere to put it (harper#2359).
+					if (!attribute.set) throw new ClientError(`${name} is a computed attribute and can not be assigned`);
 					return attribute.set(this, related);
 				},
 				configurable: true,

package/resources/transactionBroadcast.ts

@@ -180,6 +180,9 @@ function notifyFromTransactionData(subscriptions, auditLogIterable?, allowYield
 			const auditRecord = result.value;
 			const timestamp: number = auditRecord.localTime ?? auditRecord.version;
 			subscriptions.lastTxnTime = timestamp;
+			// the transaction extent: RocksDB entries committed together share the log key (record
+			// versions may differ); LMDB's localTime is a per-entry audit key, so version delimits there
+			const txnKey = auditStore.reusableIterable ? timestamp : auditRecord.version;
 			if (ACTIONS_OF_INTEREST.includes(auditRecord.type)) {
 				const tableSubscriptions = subscriptions[auditRecord.tableId];
 				if (tableSubscriptions) {
@@ -204,7 +207,7 @@ function notifyFromTransactionData(subscriptions, auditLogIterable?, allowYield
 								}
 								try {
 									let beginTxn;
-									if (subscription.supportsTransactions && subscription.txnInProgress !== auditRecord.version) {
+									if (subscription.supportsTransactions && subscription.txnInProgress !== txnKey) {
 										// if the subscriber supports transactions, we mark this as the beginning of a new transaction
 										// tracking the subscription so that we can delimit the transaction on next transaction
 										// (with a beginTxn flag, which may be on an endTxn event)
@@ -214,10 +217,7 @@ function notifyFromTransactionData(subscriptions, auditLogIterable?, allowYield
 											if (!subscribersWithTxns) subscribersWithTxns = [subscription];
 											else subscribersWithTxns.push(subscription);
 										}
-										// the version defines the extent of a transaction, all audit records with the same version
-										// are part of the same transaction, and when the version changes, we know it is a new
-										// transaction
-										subscription.txnInProgress = auditRecord.version;
+										subscription.txnInProgress = txnKey;
 									}
 									subscription.listener(recordId, auditRecord, timestamp, beginTxn);
 								} catch (error) {