@harperfast/harper-pro 5.0.22 → 5.0.24

package/core/DESIGN.md

@@ -38,6 +38,8 @@ When `migrateOnStart` opens a source LMDB primary store to read records out for
 
 Harper's normal `databases.ts` path already does this (search for `dbiInit.compression = primaryKeyAttribute.compression`); the migration path in `bin/copyDb.ts` has to match.
 
+The same source-dbi open has a second non-obvious requirement: assign `sourceDbi.encoder.rootStore = sourceRootStore` for the primary store. The primary dbi decodes through a `RecordEncoder`, and decoding a record that holds a file-backed blob reference resolves that reference against `rootStore` (it locates the blob file). With `rootStore` unset, the `Blob` msgpackr extension throws `No store specified, cannot load blob from storage`; `RecordEncoder.decode` swallows the error and yields `null`, and `copyDbToRocks` then skips the `null` value — so every record with a file-backed blob is silently dropped from the migration. The runtime path gets `rootStore` from `handleLocalTimeForGets`; the migration path opens the source dbi raw and must set it explicitly (issue #857).
+
 ## Schema migration and `runIndexing` internals (`databases.ts`)
 
 When `table()` is called with an attribute newly marked `indexed: true` (or with any change that requires re-building the secondary index), `runIndexing` is launched asynchronously and `Table.indexingOperation` is set to its promise. While running:

package/core/bin/copyDb.ts

@@ -376,7 +376,7 @@ export async function migrateOnStart() {
 	}
 }
 
-async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath: string) {
+export async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath: string) {
 	console.log(`Migrating database ${sourceDatabase} to RocksDB at ${targetPath}`);
 	const sourceDbisDb = sourceRootStore.dbisDb;
 
@@ -413,6 +413,11 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath
 			const dbiInit = new OpenDBIObject(!isPrimary, isPrimary);
 			dbiInit.compression = attribute.compression;
 			const sourceDbi = sourceRootStore.openDB(key, dbiInit);
+			// The primary dbi uses a RecordEncoder, whose decode resolves file-backed blob references
+			// against `rootStore`. Without it, decoding any record that holds a blob throws "No store
+			// specified, cannot load blob from storage", the error is swallowed (record decodes to null),
+			// and the record is silently dropped from the migration (HarperFast/harper#857).
+			if (isPrimary && sourceDbi.encoder) sourceDbi.encoder.rootStore = sourceRootStore;
 
 			let targetDbi;
 			if (!isPrimary) {
@@ -425,9 +430,23 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath
 				existingEncoder.isRocksDB = true;
 				existingEncoder.rootStore = targetRootStore;
 				const tempEncoder = new RecordEncoder({ name: key }) as any;
+				// msgpackr's pack closure captures `packr = this` at construction, so during
+				// re-encoding the structure callbacks resolve to tempEncoder's getStructures/
+				// saveStructures (invoked with this === tempEncoder), not existingEncoder's.
+				// tempEncoder must therefore carry the RocksDB wiring too, or getStructures hits
+				// the non-RocksDB branch where the captured super is undefined and throws.
+				tempEncoder.name = key;
+				tempEncoder.isRocksDB = true;
+				tempEncoder.rootStore = targetRootStore;
 				existingEncoder.encode = tempEncoder.encode;
-				existingEncoder.saveStructures = tempEncoder.saveStructures;
 				existingEncoder.getStructures = tempEncoder.getStructures;
+				// The shared structures dictionary is copied verbatim from the source by
+				// copyStructures() below, so re-encoding never needs to persist new structures.
+				// A no-op saveStructures avoids opening a targetRootStore.transactionSync() in the
+				// middle of each record's encode, which otherwise discards the targetDbi record writes.
+				const noopSaveStructures = () => true;
+				existingEncoder.saveStructures = noopSaveStructures;
+				tempEncoder.saveStructures = noopSaveStructures;
 			}
 
 			copyStructures(sourceDbi, key);

package/core/package-lock.json

@@ -1,12 +1,12 @@
 {
 	"name": "harper",
-	"version": "5.0.22",
+	"version": "5.0.24",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "harper",
-			"version": "5.0.22",
+			"version": "5.0.24",
 			"license": "Apache-2.0",
 			"dependencies": {
 				"@aws-sdk/client-s3": "^3.1012.0",
@@ -18,7 +18,7 @@
 				"@fastify/cors": "^11.2.0",
 				"@fastify/static": "^9.1.3",
 				"@harperfast/extended-iterable": "^1.0.1",
-				"@harperfast/rocksdb-js": "^1.2.0",
+				"@harperfast/rocksdb-js": "^1.4.0",
 				"@turf/area": "6.5.0",
 				"@turf/boolean-contains": "6.5.0",
 				"@turf/boolean-disjoint": "6.5.0",
@@ -53,7 +53,7 @@
 				"json-bigint-fixes": "1.1.0",
 				"jsonata": "1.8.7",
 				"jsonwebtoken": "9.0.3",
-				"lmdb": "3.5.4",
+				"lmdb": "3.5.5",
 				"lodash": "^4.17.23",
 				"mathjs": "11.12.0",
 				"micromatch": "^4.0.8",
@@ -2218,9 +2218,9 @@
 			"license": "Apache-2.0"
 		},
 		"node_modules/@harperfast/rocksdb-js": {
-			"version": "1.2.0",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js/-/rocksdb-js-1.2.0.tgz",
-			"integrity": "sha512-s/JOC720wmJCTG9rBKYJMY73yfZU1gpWxAeFTJXolFSYvc6uQzPwsAfxYxayziZrjO5LcmHg6dlNR5JiZo6ZqA==",
+			"version": "1.4.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js/-/rocksdb-js-1.4.0.tgz",
+			"integrity": "sha512-Qa62jWYG5GO+OI3gEHJI3WyhzgA6OdFLthbmB7TRJJMa2Zu8s3psKuAnM0t2v963ohRFF4UbyRw24zt5PX3k3A==",
 			"license": "Apache-2.0",
 			"dependencies": {
 				"@harperfast/extended-iterable": "1.0.3",
@@ -2234,20 +2234,20 @@
 				"node": ">=18"
 			},
 			"optionalDependencies": {
-				"@harperfast/rocksdb-js-darwin-arm64": "1.2.0",
-				"@harperfast/rocksdb-js-darwin-x64": "1.2.0",
-				"@harperfast/rocksdb-js-linux-arm64-glibc": "1.2.0",
-				"@harperfast/rocksdb-js-linux-arm64-musl": "1.2.0",
-				"@harperfast/rocksdb-js-linux-x64-glibc": "1.2.0",
-				"@harperfast/rocksdb-js-linux-x64-musl": "1.2.0",
-				"@harperfast/rocksdb-js-win32-arm64": "1.2.0",
-				"@harperfast/rocksdb-js-win32-x64": "1.2.0"
+				"@harperfast/rocksdb-js-darwin-arm64": "1.4.0",
+				"@harperfast/rocksdb-js-darwin-x64": "1.4.0",
+				"@harperfast/rocksdb-js-linux-arm64-glibc": "1.4.0",
+				"@harperfast/rocksdb-js-linux-arm64-musl": "1.4.0",
+				"@harperfast/rocksdb-js-linux-x64-glibc": "1.4.0",
+				"@harperfast/rocksdb-js-linux-x64-musl": "1.4.0",
+				"@harperfast/rocksdb-js-win32-arm64": "1.4.0",
+				"@harperfast/rocksdb-js-win32-x64": "1.4.0"
 			}
 		},
 		"node_modules/@harperfast/rocksdb-js-darwin-arm64": {
-			"version": "1.2.0",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-arm64/-/rocksdb-js-darwin-arm64-1.2.0.tgz",
-			"integrity": "sha512-+rCncLUYuBWFzVcfP5ZrtX0ZXPw1eVtnUoJMV2cvOIl6sOqkKkPVd7o0kQOoc9GWgqEn0+QB/AWcFSDUmPOuoQ==",
+			"version": "1.4.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-arm64/-/rocksdb-js-darwin-arm64-1.4.0.tgz",
+			"integrity": "sha512-78nT13T3P0Am0UfppPkMgUcTqS5Xx2ftXGl9IsmjWfrDawM1HOJt00n0GtPLoqOqrs80QVGbH9qgBZ22Bpty6Q==",
 			"cpu": [
 				"arm64"
 			],
@@ -2261,9 +2261,9 @@
 			}
 		},
 		"node_modules/@harperfast/rocksdb-js-darwin-x64": {
-			"version": "1.2.0",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-x64/-/rocksdb-js-darwin-x64-1.2.0.tgz",
-			"integrity": "sha512-kHvbdzuVP4oZjEBmI2+4bcQFYh2vkz3gCP+p6uHlZwD8cBIohIX19fw66U6tVxEU5Cpt9Q/D+rfhbYnbwqPKbA==",
+			"version": "1.4.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-x64/-/rocksdb-js-darwin-x64-1.4.0.tgz",
+			"integrity": "sha512-Ck+m4bfaiGikq1I0IAHS8AELnRL+GeKB99hR8ASR7DRAvQqYGYbPtUQQUKIQk9/8S8LviKUfto/cla1nRAcslg==",
 			"cpu": [
 				"x64"
 			],
@@ -2277,12 +2277,15 @@
 			}
 		},
 		"node_modules/@harperfast/rocksdb-js-linux-arm64-glibc": {
-			"version": "1.2.0",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-glibc/-/rocksdb-js-linux-arm64-glibc-1.2.0.tgz",
-			"integrity": "sha512-q7mRuJTfrZ3iKxxeOx9flxDuuj3jdoOj8PvtnGzMyUR4BnBa7K/ly2KNcdT3zmOQA+sJM0XN/Uq2iusTWxhATw==",
+			"version": "1.4.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-glibc/-/rocksdb-js-linux-arm64-glibc-1.4.0.tgz",
+			"integrity": "sha512-1yJkGMYR5Zh45j6joCu0cgQ+LPlsU0hjiPfv0f0cmXdjPiccXu7TemADjf3fnLz6p4p2VLP46s1fwJL7W+4lFw==",
 			"cpu": [
 				"arm64"
 			],
+			"libc": [
+				"glibc"
+			],
 			"license": "Apache-2.0",
 			"optional": true,
 			"os": [
@@ -2293,12 +2296,15 @@
 			}
 		},
 		"node_modules/@harperfast/rocksdb-js-linux-arm64-musl": {
-			"version": "1.2.0",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-musl/-/rocksdb-js-linux-arm64-musl-1.2.0.tgz",
-			"integrity": "sha512-f5Myta9ke0DBF901ehckMflm1DIOTNaKWMfzoJVuQIHwW8QT1+ZFUBz2dLKnoveIKWuE8r4TVW29YDW/oo6m2g==",
+			"version": "1.4.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-musl/-/rocksdb-js-linux-arm64-musl-1.4.0.tgz",
+			"integrity": "sha512-wIUqCl95Fs7WL0Ecm2j6Lu9TAFMbeDageMn2MfzqTtjx6fcQRL9dPH9o8HfIOfQ6Q0T0HTPpRwBTAtryYLmcFA==",
 			"cpu": [
 				"arm64"
 			],
+			"libc": [
+				"musl"
+			],
 			"license": "Apache-2.0",
 			"optional": true,
 			"os": [
@@ -2309,12 +2315,15 @@
 			}
 		},
 		"node_modules/@harperfast/rocksdb-js-linux-x64-glibc": {
-			"version": "1.2.0",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-glibc/-/rocksdb-js-linux-x64-glibc-1.2.0.tgz",
-			"integrity": "sha512-gMVjvANBh15mGwNntHFL1jPsfn1stW+IV8CT+QYbdR6dZ6XboCdnxFalOiyeRsY18ouNURKx3UzxnAelAMSqhA==",
+			"version": "1.4.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-glibc/-/rocksdb-js-linux-x64-glibc-1.4.0.tgz",
+			"integrity": "sha512-J8anAA31Dv6+h+Uu/mDlszQOoEZiTOGqrLVbWQ7/W5cET2pKZ9IC6NJjvG3rjkkFT82G+spqJ+aESY2yuEJv5Q==",
 			"cpu": [
 				"x64"
 			],
+			"libc": [
+				"glibc"
+			],
 			"license": "Apache-2.0",
 			"optional": true,
 			"os": [
@@ -2325,12 +2334,15 @@
 			}
 		},
 		"node_modules/@harperfast/rocksdb-js-linux-x64-musl": {
-			"version": "1.2.0",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-musl/-/rocksdb-js-linux-x64-musl-1.2.0.tgz",
-			"integrity": "sha512-u0LN3maKDemGDGfA77aYr2HSpLl9ilOhQOFcLSjAtYrZlquxvNRGQR60aE0VIC/quui6Xdu7zHOVY9aLZpqW3g==",
+			"version": "1.4.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-musl/-/rocksdb-js-linux-x64-musl-1.4.0.tgz",
+			"integrity": "sha512-bHJvxKlx/UqeG++aLpEcm6DDzNeGvDWvBZ+LKstVmF4mNZnR/QZ8TX9dYEGHFHMB+/PVrotRzUaMA5jgjfaMFQ==",
 			"cpu": [
 				"x64"
 			],
+			"libc": [
+				"musl"
+			],
 			"license": "Apache-2.0",
 			"optional": true,
 			"os": [
@@ -2341,9 +2353,9 @@
 			}
 		},
 		"node_modules/@harperfast/rocksdb-js-win32-arm64": {
-			"version": "1.2.0",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-arm64/-/rocksdb-js-win32-arm64-1.2.0.tgz",
-			"integrity": "sha512-ZNF5er1udx3Ntl9dE1vtspT8jUokA503Evb1Z/qGDqVeewC2QAZvAI3R4T82qILQtirUjtbUhWOlJEJyv9K9XQ==",
+			"version": "1.4.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-arm64/-/rocksdb-js-win32-arm64-1.4.0.tgz",
+			"integrity": "sha512-r6F+ZQWZmppu4mkuZwlyYDaXWXZ/AQldP2qdIRhICxXNfxcvQUxW0+CKmfnor3X3GNk46ThIaP7tM08fu7/Zag==",
 			"cpu": [
 				"arm64"
 			],
@@ -2357,9 +2369,9 @@
 			}
 		},
 		"node_modules/@harperfast/rocksdb-js-win32-x64": {
-			"version": "1.2.0",
-			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-x64/-/rocksdb-js-win32-x64-1.2.0.tgz",
-			"integrity": "sha512-B6up+3PzrGMpSK1vx8cf5bMnJH5z/Wb1WK3OaQKXmS8xGUhAPXzE46Jjb74VOj8tNaTF3sL3wZqPuwjEi2oqFQ==",
+			"version": "1.4.0",
+			"resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-x64/-/rocksdb-js-win32-x64-1.4.0.tgz",
+			"integrity": "sha512-msYlg3D2sm33sw7Uyiuxw04Jt8mrzcKOhjHFZbGAM9bqlUE0lar3txcO03oHwcKYVyoIEsnYgyddQFHdRLAU+A==",
 			"cpu": [
 				"x64"
 			],
@@ -2607,9 +2619,9 @@
 			}
 		},
 		"node_modules/@lmdb/lmdb-darwin-arm64": {
-			"version": "3.5.4",
-			"resolved": "https://registry.npmjs.org/@lmdb/lmdb-darwin-arm64/-/lmdb-darwin-arm64-3.5.4.tgz",
-			"integrity": "sha512-Kk4Kz3iyu1QiLsLZBS9Af1eSKUC8VR2T+/jyE2iAyuGw2VwK08pp5iTbZnXn6sWu0LogO/RFktMxOjiDA2sS3w==",
+			"version": "3.5.5",
+			"resolved": "https://registry.npmjs.org/@lmdb/lmdb-darwin-arm64/-/lmdb-darwin-arm64-3.5.5.tgz",
+			"integrity": "sha512-hcSIUgSblRtwEuPzV/88cTbEPxaWfxVSrTaMIVUreyAA02ysE3S9tsiQdmT7KNmUaXlwgRGs+x+GaVDCuk7Prg==",
 			"cpu": [
 				"arm64"
 			],
@@ -2620,9 +2632,9 @@
 			]
 		},
 		"node_modules/@lmdb/lmdb-darwin-x64": {
-			"version": "3.5.4",
-			"resolved": "https://registry.npmjs.org/@lmdb/lmdb-darwin-x64/-/lmdb-darwin-x64-3.5.4.tgz",
-			"integrity": "sha512-BEe5Rp3trn26oxoXOVL5HVDoiYmjUDwr8NRPkBOdUdCSBEorKI+7JrZLRKAdxO+G6cGQLgseXk0gR7qIQa7aGw==",
+			"version": "3.5.5",
+			"resolved": "https://registry.npmjs.org/@lmdb/lmdb-darwin-x64/-/lmdb-darwin-x64-3.5.5.tgz",
+			"integrity": "sha512-cLlp9ypk3W9Hr4fuTYwg1kL2vPfz887v9NXiHQE1raSKtJ8/hnghekqUu62wUQdQPgThLV8Om+Ran08wWuM/3w==",
 			"cpu": [
 				"x64"
 			],
@@ -2633,9 +2645,9 @@
 			]
 		},
 		"node_modules/@lmdb/lmdb-linux-arm": {
-			"version": "3.5.4",
-			"resolved": "https://registry.npmjs.org/@lmdb/lmdb-linux-arm/-/lmdb-linux-arm-3.5.4.tgz",
-			"integrity": "sha512-SGbFR7816uBcTHc2ZY4S6WyOkl9bICnzqTQd2Mv4V/j24cfds88xx2nC6cm/y8zGQL7Ds31YF/5NGxjgcdM5Hw==",
+			"version": "3.5.5",
+			"resolved": "https://registry.npmjs.org/@lmdb/lmdb-linux-arm/-/lmdb-linux-arm-3.5.5.tgz",
+			"integrity": "sha512-eDeK9vZww0dIXlRCZa7pvbEXSUpUNIJKUs13FxDzazL8CRLDLggbeirm5m0hZvJcRgZzTYGESE8UUEBqHcdZ9Q==",
 			"cpu": [
 				"arm"
 			],
@@ -2646,9 +2658,9 @@
 			]
 		},
 		"node_modules/@lmdb/lmdb-linux-arm64": {
-			"version": "3.5.4",
-			"resolved": "https://registry.npmjs.org/@lmdb/lmdb-linux-arm64/-/lmdb-linux-arm64-3.5.4.tgz",
-			"integrity": "sha512-cUXEengO8o60v1SWerJTH4/RH4U3+9jC0/4njp2Z9NdmvaGzhKsbRM2wpXuRYrN8tytsoJCg0SvWEWwHAwLbCA==",
+			"version": "3.5.5",
+			"resolved": "https://registry.npmjs.org/@lmdb/lmdb-linux-arm64/-/lmdb-linux-arm64-3.5.5.tgz",
+			"integrity": "sha512-tn8KSYdvX5sietpuGyJJSXocvQQtWlenzquap/Mzr+wWx5jrjK5I3nEAIMFxvTtuMMrpPTGHGNuE/Hanh3o0EQ==",
 			"cpu": [
 				"arm64"
 			],
@@ -2659,9 +2671,9 @@
 			]
 		},
 		"node_modules/@lmdb/lmdb-linux-x64": {
-			"version": "3.5.4",
-			"resolved": "https://registry.npmjs.org/@lmdb/lmdb-linux-x64/-/lmdb-linux-x64-3.5.4.tgz",
-			"integrity": "sha512-Gxq8jpgOWXwd0PUR+c9R2Ik1/uBnGd5GMIIzRRDqABCkvmjtC3KWcyhesV9jSPCz759isl0NlbsstZ2oyvk8lA==",
+			"version": "3.5.5",
+			"resolved": "https://registry.npmjs.org/@lmdb/lmdb-linux-x64/-/lmdb-linux-x64-3.5.5.tgz",
+			"integrity": "sha512-o6VRcJjZ6Zc67MnymLEa3AtbYSreFzm6Um6LMHh4lkynlE5Qk/XUHXU2beTZBdB3QuZBiMoXO2gjPSgP0tYUKg==",
 			"cpu": [
 				"x64"
 			],
@@ -2672,9 +2684,9 @@
 			]
 		},
 		"node_modules/@lmdb/lmdb-win32-arm64": {
-			"version": "3.5.4",
-			"resolved": "https://registry.npmjs.org/@lmdb/lmdb-win32-arm64/-/lmdb-win32-arm64-3.5.4.tgz",
-			"integrity": "sha512-pKv1DJ1bPZAaHkdFsSz5IDfUG8x9vntgquXF9/Dm2xuupcIe/EkLzylpoBxppFVK5vzbV561Dq26jNY2fIMA7g==",
+			"version": "3.5.5",
+			"resolved": "https://registry.npmjs.org/@lmdb/lmdb-win32-arm64/-/lmdb-win32-arm64-3.5.5.tgz",
+			"integrity": "sha512-k0k/CwywhymTAJT7Uj5PdehH31Vem2ov/Jp0rn5McuJEHgzj7vbE0cmnByhjrQv+VeHLALI8wUZNicqwocPB4Q==",
 			"cpu": [
 				"arm64"
 			],
@@ -2685,9 +2697,9 @@
 			]
 		},
 		"node_modules/@lmdb/lmdb-win32-x64": {
-			"version": "3.5.4",
-			"resolved": "https://registry.npmjs.org/@lmdb/lmdb-win32-x64/-/lmdb-win32-x64-3.5.4.tgz",
-			"integrity": "sha512-JF1BmLCm9kGEVZgYmJq43zeQVdHVgAJnTi/NURWEsy6L1ZrrlSmdltS+D17QN4LODwf+1LMXAA9auIZVXtWwzw==",
+			"version": "3.5.5",
+			"resolved": "https://registry.npmjs.org/@lmdb/lmdb-win32-x64/-/lmdb-win32-x64-3.5.5.tgz",
+			"integrity": "sha512-dkV3r04ro8MfhBaPj4ZsARy0LIP3sK+rm2LAuVe2QLCepBIpSzRx8p8RLWfRfkKV5YNyJ3c0NUJxGNMqtTB6Fg==",
 			"cpu": [
 				"x64"
 			],
@@ -7535,9 +7547,9 @@
 			"license": "MIT"
 		},
 		"node_modules/lmdb": {
-			"version": "3.5.4",
-			"resolved": "https://registry.npmjs.org/lmdb/-/lmdb-3.5.4.tgz",
-			"integrity": "sha512-9FKQA6G1MMtqNxfxvSBNXD/axeG2QRjYbNh0/ykRL5xYcRbCm2vXq7B9bhc7nSuKdHzr8/BHIwfPuYYH1UsXXw==",
+			"version": "3.5.5",
+			"resolved": "https://registry.npmjs.org/lmdb/-/lmdb-3.5.5.tgz",
+			"integrity": "sha512-e2q1RtHdJoDaQQNMJ55wJnD5BqpvzoJIEL3NBxfZXNBjYxY0QjCh3RSL/sqmi+y9l97cERdO4WwGFfYYVft1fA==",
 			"hasInstallScript": true,
 			"license": "MIT",
 			"dependencies": {
@@ -7552,13 +7564,13 @@
 				"download-lmdb-prebuilds": "bin/download-prebuilds.js"
 			},
 			"optionalDependencies": {
-				"@lmdb/lmdb-darwin-arm64": "3.5.4",
-				"@lmdb/lmdb-darwin-x64": "3.5.4",
-				"@lmdb/lmdb-linux-arm": "3.5.4",
-				"@lmdb/lmdb-linux-arm64": "3.5.4",
-				"@lmdb/lmdb-linux-x64": "3.5.4",
-				"@lmdb/lmdb-win32-arm64": "3.5.4",
-				"@lmdb/lmdb-win32-x64": "3.5.4"
+				"@lmdb/lmdb-darwin-arm64": "3.5.5",
+				"@lmdb/lmdb-darwin-x64": "3.5.5",
+				"@lmdb/lmdb-linux-arm": "3.5.5",
+				"@lmdb/lmdb-linux-arm64": "3.5.5",
+				"@lmdb/lmdb-linux-x64": "3.5.5",
+				"@lmdb/lmdb-win32-arm64": "3.5.5",
+				"@lmdb/lmdb-win32-x64": "3.5.5"
 			}
 		},
 		"node_modules/lmdb/node_modules/node-addon-api": {

package/core/resources/RecordEncoder.ts

@@ -17,7 +17,14 @@ import {
 } from './auditStore.ts';
 import * as harperLogger from '../utility/logging/harper_logger.js';
 import './blob.ts';
-import { blobsWereEncoded, decodeFromDatabase, deleteBlobsInObject, encodeBlobsWithFilePath } from './blob.ts';
+import {
+	blobsWereEncoded,
+	decodeFromDatabase,
+	deleteBlobsInObject,
+	encodeBlobsWithFilePath,
+	findBlobsInObject,
+	getFileId,
+} from './blob.ts';
 import { getThisNodeId } from './nodeIdMapping.ts';
 import { recordAction } from './analytics/write.ts';
 import { RocksDatabase } from '@harperfast/rocksdb-js';
@@ -606,8 +613,19 @@ export function recordUpdater(store, tableId, auditStore) {
 			// we use resolveRecord outside of transaction, so must explicitly make it conditional
 			if (resolveRecord) putOptions.ifVersion = ifVersion = existingEntry?.version ?? null;
 			if (existingEntry && existingEntry.value && type !== 'message' && existingEntry.metadataFlags & HAS_BLOBS) {
-				// delete the old blobs
-				deleteBlobsInObject(existingEntry.value);
+				// Delete the prior row's blob files — except any the new record still references.
+				// Without the retention check, updating an unrelated attribute on a row that
+				// carries a file-backed blob unlinks the blob ~deletionDelay ms later, leaving
+				// the new (otherwise valid) row pointing at a missing file. See HarperFast/harper#641
+				// (deployment tracking) for the production repro.
+				let retainedFileIds: Set<string> | undefined;
+				if (record) {
+					findBlobsInObject(record, (blob) => {
+						const fileId = getFileId(blob);
+						if (fileId) (retainedFileIds ??= new Set()).add(fileId);
+					});
+				}
+				deleteBlobsInObject(existingEntry.value, retainedFileIds);
 			}
 			let result: Promise<void>;
 			if (record !== undefined) {

package/core/resources/Table.ts

@@ -1441,6 +1441,7 @@ export function makeTable(options) {
 			const lmdbTransaction = txnForContext({ transaction: new DatabaseTransaction() });
 			let transaction = lmdbTransaction.getReadTxn();
 			let options = { transaction };
+			let committed = false;
 			try {
 				if (hasSourceGet || audit) {
 					if (!existingRecord) return;
@@ -1460,19 +1461,30 @@ export function makeTable(options) {
 					primaryStore.ifVersion?.(id, existingVersion, () => {
 						updateIndices(id, existingRecord, null);
 					});
-					return removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), existingVersion);
+					removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), existingVersion);
 				} else {
 					updateIndices(id, existingRecord, null, options);
-					return removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), options);
+					removeEntry(primaryStore, entry ?? primaryStore.getEntry(id), options);
 				}
-			} finally {
+				committed = true;
 				if (primaryStore.ifVersion) {
 					// LMDB: committing the wrapper calls doneReadTxn(), removing it from trackedTxns
 					return lmdbTransaction.commit();
 				}
 				// RocksDB: eviction writes went directly into the raw transaction via options;
-				// commit it directly, as DatabaseTransaction.commit() would abort it (no tracked writes)
-				return transaction?.commit();
+				// commit it directly, as DatabaseTransaction.commit() would abort it (no tracked writes).
+				// Wrap in Promise.resolve so callers can rely on a thenable return regardless of engine.
+				return Promise.resolve((transaction as any).commit());
+			} finally {
+				if (!committed) {
+					// Skip path or thrown error: abort instead of committing so we don't apply
+					// partial work and the txn handle is released.
+					if (primaryStore.ifVersion) {
+						(lmdbTransaction as any).abort?.();
+					} else {
+						(transaction as any)?.abort?.();
+					}
+				}
 			}
 		}
 		/**
@@ -4459,7 +4471,7 @@ export function makeTable(options) {
 										const { key, value: record, version, expiresAt, metadataFlags } = entry;
 										// if there is no auditing cleanup and we are tracking deletion, need to do cleanup of
 										// these deletion entries (LMDB audit cleanup has its own scheduled job for this)
-										let resolution: Promise<void>;
+										let resolution: Promise<void> | undefined;
 										if (record === null && removeDeletedRecords && version + auditRetention < Date.now()) {
 											// make sure it is still deleted when we do the removal
 											resolution = removeEntry(primaryStore, entry, version);

package/core/resources/blob.ts

@@ -970,12 +970,26 @@ export function decodeFromDatabase<T>(callback: () => T, rootStore: LMDBStore) {
 }
 
 /**
- * Delete blobs in an object, recursively searching for blobs
+ * Delete blobs in an object, recursively searching for blobs. When `retainedFileIds`
+ * is supplied, blobs whose fileId is in that set are skipped — used by the update path
+ * to avoid deleting a blob the new record still references.
+ *
+ * Background: `RecordEncoder` calls this with the *prior* row's value on every update.
+ * If a caller updates an unrelated attribute on a row that still carries a file-backed
+ * blob, the unfiltered delete unlinks the blob file ~`deletionDelay` ms later, even
+ * though the new row still references the same fileId. The retainedFileIds set
+ * prevents that data loss.
+ *
  * @param object
+ * @param retainedFileIds optional set of fileIds the caller wants to keep on disk
  */
-export function deleteBlobsInObject(object) {
-	findBlobsInObject(object, (object) => {
-		deleteBlob(object);
+export function deleteBlobsInObject(object: any, retainedFileIds?: Set<string>): void {
+	findBlobsInObject(object, (blob) => {
+		if (retainedFileIds?.size) {
+			const fileId = getFileId(blob);
+			if (fileId && retainedFileIds.has(fileId)) return;
+		}
+		deleteBlob(blob);
 	});
 }
 

package/core/resources/databases.ts

@@ -1317,6 +1317,24 @@ async function runIndexing(Table, attributes, indicesToRemove) {
 		}
 	} catch (error) {
 		logger.error('Error in indexing', error);
+		// Persist indexingFailed so the next restart re-triggers the rebuild from an
+		// explicitly failed state rather than silently looping. Without this,
+		// indexingPID (written before runIndexing was called) stays in the descriptor
+		// but indexingFailed is never set, leaving isIndexing stuck with no recovery
+		// signal. Mirrors the hadIndexingErrors path. harper#843
+		try {
+			const puts: Promise<unknown>[] = [];
+			for (const attribute of attributes) {
+				attribute.indexingFailed = true;
+				puts.push(Table.dbisDB.put(attribute.key, attribute));
+				attribute.dbi.isIndexing = true;
+				const activeDbi = Table.indices[attribute.name];
+				if (activeDbi) activeDbi.isIndexing = true;
+			}
+			await Promise.all(puts);
+		} catch (persistError) {
+			logger.warn('Failed to persist indexing failure state', persistError);
+		}
 	}
 }
 

package/core/utility/logging/logRotator.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const { promises: fsProm, createReadStream, createWriteStream } = require('fs');
+const { promises: fsProm, createReadStream, createWriteStream, mkdirSync } = require('fs');
 const { createGzip } = require('zlib');
 const { promisify } = require('util');
 const { pipeline } = require('stream');
@@ -17,8 +17,6 @@ const { onStorageReclamation } = require('../../server/storageReclamation.ts');
 const LOG_AUDIT_INTERVAL = 60000;
 const INT_SIZE_UNDEFINED_MSG =
 	"'interval' and 'maxSize' are both undefined, to enable logging rotation at least one of these values must be defined in harperdb-config.yaml";
-const PATH_UNDEFINED_MSG =
-	"'logging.rotation.path' is undefined, to enable logging rotation set this value in harperdb-config.yaml";
 
 let lastRotationTime;
 let setIntervalId;
@@ -47,8 +45,9 @@ function logRotator({ logger, maxSize, interval, retention, enabled, path: rotat
 	}
 
 	if (!rotatedLogDir) {
-		throw new Error(PATH_UNDEFINED_MSG);
+		rotatedLogDir = path.join(path.dirname(logger.path), 'rotated');
 	}
+	mkdirSync(rotatedLogDir, { recursive: true });
 
 	// Convert maxSize param to bytes.
 	let maxBytes;

package/dist/core/bin/copyDb.js

@@ -39,6 +39,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.compactOnStart = compactOnStart;
 exports.copyDb = copyDb;
 exports.migrateOnStart = migrateOnStart;
+exports.copyDbToRocks = copyDbToRocks;
 const databases_ts_1 = require("../resources/databases.js");
 const lmdb_1 = require("lmdb");
 const path_1 = require("path");
@@ -423,6 +424,12 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase, targetPath) {
             const dbiInit = new OpenDBIObject_js_1.OpenDBIObject(!isPrimary, isPrimary);
             dbiInit.compression = attribute.compression;
             const sourceDbi = sourceRootStore.openDB(key, dbiInit);
+            // The primary dbi uses a RecordEncoder, whose decode resolves file-backed blob references
+            // against `rootStore`. Without it, decoding any record that holds a blob throws "No store
+            // specified, cannot load blob from storage", the error is swallowed (record decodes to null),
+            // and the record is silently dropped from the migration (HarperFast/harper#857).
+            if (isPrimary && sourceDbi.encoder)
+                sourceDbi.encoder.rootStore = sourceRootStore;
             let targetDbi;
             if (!isPrimary) {
                 targetDbi = openRocksDb(targetPath, { dupSort: true, name: key });
@@ -435,9 +442,23 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase, targetPath) {
                 existingEncoder.isRocksDB = true;
                 existingEncoder.rootStore = targetRootStore;
                 const tempEncoder = new RecordEncoder_ts_1.RecordEncoder({ name: key });
+                // msgpackr's pack closure captures `packr = this` at construction, so during
+                // re-encoding the structure callbacks resolve to tempEncoder's getStructures/
+                // saveStructures (invoked with this === tempEncoder), not existingEncoder's.
+                // tempEncoder must therefore carry the RocksDB wiring too, or getStructures hits
+                // the non-RocksDB branch where the captured super is undefined and throws.
+                tempEncoder.name = key;
+                tempEncoder.isRocksDB = true;
+                tempEncoder.rootStore = targetRootStore;
                 existingEncoder.encode = tempEncoder.encode;
-                existingEncoder.saveStructures = tempEncoder.saveStructures;
                 existingEncoder.getStructures = tempEncoder.getStructures;
+                // The shared structures dictionary is copied verbatim from the source by
+                // copyStructures() below, so re-encoding never needs to persist new structures.
+                // A no-op saveStructures avoids opening a targetRootStore.transactionSync() in the
+                // middle of each record's encode, which otherwise discards the targetDbi record writes.
+                const noopSaveStructures = () => true;
+                existingEncoder.saveStructures = noopSaveStructures;
+                tempEncoder.saveStructures = noopSaveStructures;
             }
             copyStructures(sourceDbi, key);
             console.log('migrating', key, 'from', sourceDatabase, 'to RocksDB');