@harperfast/harper 5.2.0-beta.4 → 5.2.1

package/resources/analytics/write.ts

@@ -397,26 +397,28 @@ function storeDBSizeMetrics(analyticsTable: Table, databases: Databases) {
 	}
 }
 
-function storeVolumeMetrics(analyticsTable: Table, databases: Databases) {
-	for (const [db, tables] of Object.entries(databases)) {
-		try {
-			const [firstTable] = Object.values(tables);
-			const storageStats = firstTable?.getStorageStats();
-			if (!storageStats) {
-				continue;
+async function storeVolumeMetrics(analyticsTable: Table, databases: Databases) {
+	await Promise.all(
+		Object.entries(databases).map(async ([db, tables]) => {
+			try {
+				const [firstTable] = Object.values(tables);
+				const storageStats = await firstTable?.getStorageStats();
+				if (!storageStats) {
+					return;
+				}
+				const metric = {
+					metric: METRIC.STORAGE_VOLUME,
+					database: db,
+					...storageStats,
+				};
+				storeMetric(analyticsTable, metric);
+				log.trace?.(`db ${db} storage volume metrics: ${JSON.stringify(metric)}`);
+			} catch (error) {
+				// a table or db was deleted, could get an error here
+				log.warn?.(`Error getting DB volume metrics`, error);
 			}
-			const metric = {
-				metric: METRIC.STORAGE_VOLUME,
-				database: db,
-				...storageStats,
-			};
-			storeMetric(analyticsTable, metric);
-			log.trace?.(`db ${db} storage volume metrics: ${JSON.stringify(metric)}`);
-		} catch (error) {
-			// a table or db was deleted, could get an error here
-			log.warn?.(`Error getting DB volume metrics`, error);
-		}
-	}
+		})
+	);
 }
 
 // RocksDB stat names are kebab-case with a "rocksdb." prefix (e.g. "rocksdb.block.cache.hit");
@@ -1047,7 +1049,7 @@ async function aggregation(fromPeriod, toPeriod = 60000) {
 	storeDBSizeMetrics(analyticsTable, databases);
 
 	// database storage volume metrics
-	storeVolumeMetrics(analyticsTable, databases);
+	await storeVolumeMetrics(analyticsTable, databases);
 
 	// rocksdb engine stats (only for RocksDB-backed databases)
 	const rocksDBPeriod = lastRocksDBStatsTime ? now - lastRocksDBStatsTime : undefined;

package/resources/blob.ts

@@ -55,7 +55,6 @@ type StorageInfo = {
 	recordId?: number;
 	contentBuffer?: any;
 	source?: Readable;
-	storageBuffer?: Buffer;
 	compress?: boolean;
 	flush?: boolean;
 	start?: number;
@@ -829,7 +828,7 @@ class FileBackedBlob extends (Blob as unknown as { new (): Blob }) implements Bl
 			storageInfoForBlob.set(slicedBlob, slicedStorageInfo);
 			if (this.size != undefined)
 				slicedBlob.size = (end == undefined ? this.size : Math.min(end, this.size)) - (start ?? 0);
-		} else if (sourceStorageInfo?.contentBuffer && !sourceStorageInfo.storageBuffer) {
+		} else if (sourceStorageInfo?.contentBuffer) {
 			const slicedStorageInfo = {
 				...sourceStorageInfo,
 				contentBuffer: sourceStorageInfo.contentBuffer.subarray(start, end),
@@ -1186,16 +1185,26 @@ export function getRootBlobPathsForDB(store: RootDatabase) {
 			logger.warn?.('No database name specified, can not determine blob storage path');
 			return [];
 		}
-		const blobPaths: string[] = envGet(CONFIG_PARAMS.STORAGE_BLOBPATHS);
-		if (blobPaths) {
-			paths = blobPaths.map((path) => join(path, (store as any).databaseName));
-		} else {
-			paths = [join(getHdbBasePath(), 'blobs', (store as any).databaseName)];
-		}
+		paths = getBlobPathsForDatabaseName((store as any).databaseName);
 		databasePaths.set(store, paths);
 	}
 	return paths;
 }
+
+/**
+ * Resolve a database's blob root directories from configuration by name, without needing a live
+ * store handle. Mirrors `getRootBlobPathsForDB`'s path derivation so backup/restore code (which runs
+ * while the database may be closed) computes the same roots: one per configured `storage.blobPaths`
+ * entry, or a single `<hdb_root>/blobs/<database>` when none are configured. A database can therefore
+ * have more than one blob root.
+ */
+export function getBlobPathsForDatabaseName(databaseName: string): string[] {
+	const blobPaths: string[] = envGet(CONFIG_PARAMS.STORAGE_BLOBPATHS);
+	if (blobPaths) {
+		return blobPaths.map((path) => join(path, databaseName));
+	}
+	return [join(getHdbBasePath(), 'blobs', databaseName)];
+}
 export async function deleteRootBlobPathsForDB(store: RootDatabase): Promise<void> {
 	const paths = getRootBlobPathsForDB(store);
 	if (paths) {
@@ -1330,8 +1339,11 @@ function getNextStorageIndex(blobStoragePaths: string[], fileId: number) {
 	}
 	if (((blobStoragePaths as any).lastUpdated ?? 0) + 60000 < now) {
 		(blobStoragePaths as any).lastUpdated = now;
-		// create a new frequency table based on the available space
-		createFrequencyTableForStoragePaths(blobStoragePaths);
+		// create a new frequency table based on the available space; fire-and-forget, so a
+		// transient stat/mkdir failure must not become an unhandled rejection
+		createFrequencyTableForStoragePaths(blobStoragePaths).catch((error) => {
+			logger.warn?.('Error creating storage path frequency table', error);
+		});
 	}
 	const nextIndex = (blobStoragePaths as any).frequencyTable[fileId % FREQUENCY_TABLE_SIZE];
 	return nextIndex;
@@ -1346,6 +1358,10 @@ async function createFrequencyTableForStoragePaths(blobStoragePaths: string[]) {
 	if (!statfs) return; // statfs is not available on all older node versions
 	const availableSpaces = await Promise.all(
 		blobStoragePaths.map(async (path) => {
+			// This compares MULTIPLE distinct paths against each other, so it always needs the
+			// real per-path number: quota-status.json (see getStorageSpaceStats, #1976) is a
+			// single instance-wide figure that can't tell two disks apart, and would collapse
+			// every path in a multi-volume STORAGE_BLOBPATHS config to the same "available" value.
 			let stats: StatsFs;
 			try {
 				stats = await statfs(path);
@@ -1723,10 +1739,14 @@ addExtension({
 				throw new Error('No store specified, cannot load blob from storage');
 			}
 		} else {
+			// contentBuffer is a *copy* (copyingUnpacker uses copyBuffers), so it stays valid for the
+			// lifetime of the blob. Do not retain the raw ext-body `buffer` here: it is a view into the
+			// store's read buffer, which is recycled by later reads. Keeping it and re-emitting it on
+			// re-encode (e.g. a read-modify-write / REST PATCH) serialized whatever foreign bytes had
+			// since overwritten that buffer, corrupting the record (harper#2103).
 			storageInfoForBlob.set(blob, {
 				storageIndex: 0,
 				fileId: null,
-				storageBuffer: buffer as any,
 				contentBuffer: blobInfo[1] as any,
 			});
 			blob.size = blobInfo[1]?.length;
@@ -1747,9 +1767,6 @@ addExtension({
 		if (blob.type) options.type = blob.type;
 		if (blob.size !== undefined) options.size = blob.size;
 		if (storageInfo) {
-			if (storageInfo.storageBuffer) {
-				return storageInfo.storageBuffer;
-			}
 			if (
 				storageInfo.contentBuffer &&
 				(storageInfo.contentBuffer?.length < FILE_STORAGE_THRESHOLD || blob.saveInRecord)