@harperfast/rocksdb-js 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +122 -9
- package/dist/index.cjs +441 -30
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1480 -1368
- package/dist/index.d.mts +1480 -1368
- package/dist/index.mjs +442 -33
- package/dist/index.mjs.map +1 -1
- package/docs/backups.md +238 -0
- package/package.json +20 -20
package/dist/index.cjs
CHANGED
|
@@ -114,6 +114,21 @@ const coolTransactionLogs = binding.coolTransactionLogs;
|
|
|
114
114
|
* the underlying mapping rather than leaving it retained.
|
|
115
115
|
*/
|
|
116
116
|
const transactionLogMapCount = binding.transactionLogMapCount;
|
|
117
|
+
/**
|
|
118
|
+
* Creates a native file lock using the specified file path (`flock` on POSIX,
|
|
119
|
+
* `LockFileEx` on Windows). Returns an opaque non-zero token to pass to
|
|
120
|
+
* `fileLockRelease`, or `0` if another holder — in any process, container, or
|
|
121
|
+
* worker thread — currently has it. Throws if `file` is missing or on a hard
|
|
122
|
+
* error. The OS handle is owned entirely in native code (no fd crosses into
|
|
123
|
+
* JS), and the kernel releases the lock when the handle closes, including on
|
|
124
|
+
* process death.
|
|
125
|
+
*/
|
|
126
|
+
const tryFileLock = binding.tryFileLock;
|
|
127
|
+
/**
|
|
128
|
+
* Releases a file lock acquired via `tryFileLock`. A no-op for
|
|
129
|
+
* token `0` or an unknown token.
|
|
130
|
+
*/
|
|
131
|
+
const fileLockRelease = binding.fileLockRelease;
|
|
117
132
|
const nativeBackupRestore = binding.backupRestore;
|
|
118
133
|
const nativeBackupList = binding.backupList;
|
|
119
134
|
const nativeBackupDelete = binding.backupDelete;
|
|
@@ -124,6 +139,126 @@ const version = binding.version;
|
|
|
124
139
|
|
|
125
140
|
//#endregion
|
|
126
141
|
//#region src/backup.ts
|
|
142
|
+
/** Subdirectory (under a backup directory) holding per-backup transaction log snapshots. */
|
|
143
|
+
const TRANSACTION_LOGS_DIRNAME = "transaction_logs";
|
|
144
|
+
/** Non-blocking existence check (`fs.existsSync` would block the event loop). */
|
|
145
|
+
async function exists(path) {
|
|
146
|
+
try {
|
|
147
|
+
await (0, node_fs_promises.access)(path);
|
|
148
|
+
return true;
|
|
149
|
+
} catch {
|
|
150
|
+
return false;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Runs `fn` while holding a native file lock for `backupDir`, releasing it when
|
|
155
|
+
* `fn` settles. Used by the writable-engine operations `backups.delete` and
|
|
156
|
+
* `backups.purge`; backup creation (`Store.backup`) takes the same lock on the
|
|
157
|
+
* same file natively inside `Database::Backup` (see `runCreateBackup` in
|
|
158
|
+
* `src/binding/database/backup.cpp`), where the backup directory is also
|
|
159
|
+
* created. Throws immediately (without running `fn`) if another writer holds
|
|
160
|
+
* the directory lock.
|
|
161
|
+
*
|
|
162
|
+
* Read-only operations (`list`, `verify`, and a restore's source read) use
|
|
163
|
+
* `BackupEngineReadOnly` and are not locked: concurrent readers are safe.
|
|
164
|
+
* Running a reader concurrently with a `delete`/`purge` on the same directory is
|
|
165
|
+
* a caller-managed hazard, matching RocksDB's one-writable-engine-per-dir model.
|
|
166
|
+
*
|
|
167
|
+
* RocksDB only serializes work within a single `BackupEngine` instance and has
|
|
168
|
+
* no lock on the backup directory itself. Two writable engines — in any
|
|
169
|
+
* processes — creating/deleting/purging backups in the same directory
|
|
170
|
+
* concurrently race on the per-backup staging directory and both fail,
|
|
171
|
+
* potentially leaving the directory with no usable backup. An in-memory lock
|
|
172
|
+
* cannot prevent this across processes, so the lock lives on disk: a kernel
|
|
173
|
+
* advisory lock (`flock` on POSIX, `LockFileEx` on Windows) held on a
|
|
174
|
+
* `.backup.lock` file at the directory root.
|
|
175
|
+
*
|
|
176
|
+
* The lock is taken entirely in native code (`tryFileLock`), which opens,
|
|
177
|
+
* locks, and later closes the file's OS handle without ever exposing a
|
|
178
|
+
* descriptor to JS — the addon statically links its own C runtime, so a
|
|
179
|
+
* Node/libuv fd is not usable across that boundary. The kernel owning the lock
|
|
180
|
+
* buys two properties a pidfile cannot:
|
|
181
|
+
*
|
|
182
|
+
* - No staleness heuristic. The lock is released when the holder's handle closes
|
|
183
|
+
* — normal release, crash, `kill -9`, container exit — so there is nothing to
|
|
184
|
+
* reclaim and no liveness check.
|
|
185
|
+
* - True cross-context exclusion. The lock conflicts across processes, containers
|
|
186
|
+
* sharing a volume (same kernel), and `worker_threads`.
|
|
187
|
+
*
|
|
188
|
+
* The lock file is never deleted, only locked and unlocked; an unlocked, empty
|
|
189
|
+
* `.backup.lock` at the directory root is the steady state.
|
|
190
|
+
*/
|
|
191
|
+
async function withBackupDirLock(backupDir, fn) {
|
|
192
|
+
const token = tryFileLock((0, node_path.join)(backupDir, ".backup.lock"));
|
|
193
|
+
if (token === 0) throw new Error(`Backup directory is locked: ${backupDir}`);
|
|
194
|
+
try {
|
|
195
|
+
return await fn();
|
|
196
|
+
} finally {
|
|
197
|
+
fileLockRelease(token);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Removes `<backupDir>/transaction_logs/<id>` subtrees whose backup id no longer
|
|
202
|
+
* corresponds to a surviving backup. Called after a `purge` (which holds the
|
|
203
|
+
* backup-dir lock); a no-op when no transaction logs were backed up. Self-healing:
|
|
204
|
+
* it drops every log subtree without a live backup. Note the surviving set comes
|
|
205
|
+
* from `nativeBackupList`, which omits corrupt backups — a still-present but
|
|
206
|
+
* corrupt backup's logs may be pruned (acceptable: it cannot be restored anyway).
|
|
207
|
+
* This also sweeps `.staging-*` leftovers from a crashed backup process (a
|
|
208
|
+
* staging name never matches a live backup id); backup creation sweeps them
|
|
209
|
+
* natively as well, under the backup-dir lock.
|
|
210
|
+
*/
|
|
211
|
+
async function pruneOrphanedTransactionLogs(backupDir) {
|
|
212
|
+
const logsRoot = (0, node_path.join)(backupDir, TRANSACTION_LOGS_DIRNAME);
|
|
213
|
+
let ids;
|
|
214
|
+
try {
|
|
215
|
+
ids = await (0, node_fs_promises.readdir)(logsRoot);
|
|
216
|
+
} catch (err) {
|
|
217
|
+
if (err.code === "ENOENT") return;
|
|
218
|
+
throw err;
|
|
219
|
+
}
|
|
220
|
+
const list = await new Promise((resolve, reject) => nativeBackupList(resolve, reject, backupDir));
|
|
221
|
+
const surviving = new Set(list.map((info) => String(info.backupId)));
|
|
222
|
+
await Promise.all(ids.filter((id) => !surviving.has(id)).map((id) => (0, node_fs_promises.rm)((0, node_path.join)(logsRoot, id), {
|
|
223
|
+
recursive: true,
|
|
224
|
+
force: true
|
|
225
|
+
})));
|
|
226
|
+
}
|
|
227
|
+
/**
|
|
228
|
+
* Restores the transaction log snapshot for the restored backup into
|
|
229
|
+
* `<dbDir>/transaction_logs/`, wiping the destination first so that restoring an
|
|
230
|
+
* older backup over a newer one cannot leave stale (newer) log files behind.
|
|
231
|
+
*
|
|
232
|
+
* Only acts when THIS backup actually captured logs: if the restored id has no
|
|
233
|
+
* snapshot, the destination is left untouched. Wiping unconditionally would
|
|
234
|
+
* destroy on-disk logs that a mixed-mode (`transactionLogs:false`) backup never
|
|
235
|
+
* managed.
|
|
236
|
+
*
|
|
237
|
+
* Offline only: the files are picked up when the database is next opened. mtimes
|
|
238
|
+
* are preserved because the store derives file age (rotation/retention) from
|
|
239
|
+
* mtime — a fresh mtime would break retention.
|
|
240
|
+
*/
|
|
241
|
+
async function restoreTransactionLogs(backupDir, dbDir, backupId) {
|
|
242
|
+
const logsRoot = (0, node_path.join)(backupDir, TRANSACTION_LOGS_DIRNAME);
|
|
243
|
+
if (!await exists(logsRoot)) return;
|
|
244
|
+
let id = backupId;
|
|
245
|
+
if (id === void 0) {
|
|
246
|
+
const list = await new Promise((resolve, reject) => nativeBackupList(resolve, reject, backupDir));
|
|
247
|
+
if (list.length === 0) return;
|
|
248
|
+
id = Math.max(...list.map((info) => info.backupId));
|
|
249
|
+
}
|
|
250
|
+
const logsSrc = (0, node_path.join)(logsRoot, String(id));
|
|
251
|
+
if (!await exists(logsSrc)) return;
|
|
252
|
+
const logsDest = (0, node_path.join)(dbDir, TRANSACTION_LOGS_DIRNAME);
|
|
253
|
+
await (0, node_fs_promises.rm)(logsDest, {
|
|
254
|
+
recursive: true,
|
|
255
|
+
force: true
|
|
256
|
+
});
|
|
257
|
+
await (0, node_fs_promises.cp)(logsSrc, logsDest, {
|
|
258
|
+
recursive: true,
|
|
259
|
+
preserveTimestamps: true
|
|
260
|
+
});
|
|
261
|
+
}
|
|
127
262
|
/**
|
|
128
263
|
* Backup management operations that act on a backup directory and do not
|
|
129
264
|
* require an open database. To create a backup, use the `db.backup()` instance
|
|
@@ -149,11 +284,12 @@ const backups = {
|
|
|
149
284
|
const walDir = options?.walDir ?? dbDir;
|
|
150
285
|
await (0, node_fs_promises.mkdir)(dbDir, { recursive: true });
|
|
151
286
|
if (walDir !== dbDir) await (0, node_fs_promises.mkdir)(walDir, { recursive: true });
|
|
152
|
-
|
|
287
|
+
await new Promise((resolve, reject) => nativeBackupRestore(resolve, reject, backupDir, dbDir, walDir, {
|
|
153
288
|
backupId: options?.backupId,
|
|
154
289
|
keepLogFiles: options?.keepLogFiles,
|
|
155
290
|
mode: options?.mode
|
|
156
291
|
}));
|
|
292
|
+
await restoreTransactionLogs(backupDir, dbDir, options?.backupId);
|
|
157
293
|
},
|
|
158
294
|
/**
|
|
159
295
|
* Lists the non-corrupt backups in a backup directory, ordered by id.
|
|
@@ -165,14 +301,27 @@ const backups = {
|
|
|
165
301
|
* Deletes a specific backup. Shared files are reference-counted and only
|
|
166
302
|
* removed once no remaining backup references them.
|
|
167
303
|
*/
|
|
168
|
-
delete(backupDir, backupId) {
|
|
169
|
-
return
|
|
304
|
+
async delete(backupDir, backupId) {
|
|
305
|
+
return withBackupDirLock(backupDir, async () => {
|
|
306
|
+
await new Promise((resolve, reject) => nativeBackupDelete(resolve, reject, backupDir, backupId));
|
|
307
|
+
try {
|
|
308
|
+
await (0, node_fs_promises.rm)((0, node_path.join)(backupDir, TRANSACTION_LOGS_DIRNAME, String(backupId)), {
|
|
309
|
+
recursive: true,
|
|
310
|
+
force: true
|
|
311
|
+
});
|
|
312
|
+
} catch {}
|
|
313
|
+
});
|
|
170
314
|
},
|
|
171
315
|
/**
|
|
172
316
|
* Deletes all but the newest `keepCount` backups.
|
|
173
317
|
*/
|
|
174
|
-
purge(backupDir, keepCount) {
|
|
175
|
-
return
|
|
318
|
+
async purge(backupDir, keepCount) {
|
|
319
|
+
return withBackupDirLock(backupDir, async () => {
|
|
320
|
+
await new Promise((resolve, reject) => nativeBackupPurge(resolve, reject, backupDir, keepCount));
|
|
321
|
+
try {
|
|
322
|
+
await pruneOrphanedTransactionLogs(backupDir);
|
|
323
|
+
} catch {}
|
|
324
|
+
});
|
|
176
325
|
},
|
|
177
326
|
/**
|
|
178
327
|
* Verifies a backup's file sizes, and optionally their checksums (which
|
|
@@ -543,6 +692,258 @@ var DBI = class DBI {
|
|
|
543
692
|
}
|
|
544
693
|
};
|
|
545
694
|
|
|
695
|
+
//#endregion
|
|
696
|
+
//#region src/tar.ts
|
|
697
|
+
/**
|
|
698
|
+
* A minimal, dependency-free streaming USTAR (POSIX tar) encoder.
|
|
699
|
+
*
|
|
700
|
+
* Built for streaming whole-database backups: the set of live RocksDB files
|
|
701
|
+
* (SSTs, blobs, MANIFEST, CURRENT, OPTIONS, WAL) is serialized into a single
|
|
702
|
+
* tar byte stream that can be piped to any destination and unpacked with
|
|
703
|
+
* standard tools (`tar -xf`). The encoder never buffers a whole file — each
|
|
704
|
+
* entry's payload is streamed through in chunks — so peak memory stays flat
|
|
705
|
+
* regardless of database size.
|
|
706
|
+
*
|
|
707
|
+
* Backpressure is the caller's lever: every method `await`s the {@link TarSink},
|
|
708
|
+
* so wiring the sink to a stream writer's `write()` naturally paces production
|
|
709
|
+
* to the consumer.
|
|
710
|
+
*
|
|
711
|
+
* Only what a RocksDB backup needs is implemented: regular files. Paths longer
|
|
712
|
+
* than USTAR's 100-byte `name` field are split across the `name` and `prefix`
|
|
713
|
+
* fields (~255 bytes total), which covers `transaction_logs/<store>/<file>`
|
|
714
|
+
* entries; GNU/pax long-name extensions are not implemented, so a path that
|
|
715
|
+
* still doesn't fit throws rather than being silently truncated. File sizes
|
|
716
|
+
* larger than the octal `size` field can hold (~8 GiB) fall back to the GNU
|
|
717
|
+
* base-256 encoding.
|
|
718
|
+
*/
|
|
719
|
+
/** Size of a tar block. Every header and the archive length are multiples of this. */
|
|
720
|
+
const BLOCK_SIZE = 512;
|
|
721
|
+
/** Maximum bytes for the USTAR `name` field. */
|
|
722
|
+
const MAX_NAME_LENGTH = 100;
|
|
723
|
+
/** Maximum bytes for the USTAR `prefix` field (holds the leading directories). */
|
|
724
|
+
const MAX_PREFIX_LENGTH = 155;
|
|
725
|
+
const textEncoder = new TextEncoder();
|
|
726
|
+
const textDecoder = new TextDecoder();
|
|
727
|
+
/**
|
|
728
|
+
* Writes a UTF-8 string into a fixed-width header field, left-justified and
|
|
729
|
+
* NUL-padded (the block is already zero-filled, so nothing else is needed).
|
|
730
|
+
* Throws if the encoded value does not fit — tar fields are silently lossy
|
|
731
|
+
* otherwise, which corrupts the archive.
|
|
732
|
+
*/
|
|
733
|
+
function writeField(block, value, offset, length) {
|
|
734
|
+
const bytes = textEncoder.encode(value);
|
|
735
|
+
if (bytes.length > length) throw new RangeError(`tar field value ${JSON.stringify(value)} exceeds ${length} bytes`);
|
|
736
|
+
block.set(bytes, offset);
|
|
737
|
+
}
|
|
738
|
+
/**
|
|
739
|
+
* Writes a non-negative integer as a NUL-terminated, zero-padded octal string
|
|
740
|
+
* into a `length`-byte field (so `length - 1` octal digits). Values too large
|
|
741
|
+
* for octal fall back to the GNU base-256 extension, where the high bit of the
|
|
742
|
+
* first byte is set and the magnitude is stored big-endian in the remainder.
|
|
743
|
+
*/
|
|
744
|
+
function writeOctal(block, value, offset, length) {
|
|
745
|
+
const digits = length - 1;
|
|
746
|
+
if (value > 8 ** digits - 1) {
|
|
747
|
+
let v = value;
|
|
748
|
+
for (let i = offset + length - 1; i > offset; i--) {
|
|
749
|
+
block[i] = v & 255;
|
|
750
|
+
v = Math.floor(v / 256);
|
|
751
|
+
}
|
|
752
|
+
block[offset] = 128;
|
|
753
|
+
return;
|
|
754
|
+
}
|
|
755
|
+
const str = value.toString(8).padStart(digits, "0");
|
|
756
|
+
for (let i = 0; i < digits; i++) block[offset + i] = str.charCodeAt(i);
|
|
757
|
+
}
|
|
758
|
+
/**
|
|
759
|
+
* Computes and writes the USTAR header checksum: the unsigned sum of all 512
|
|
760
|
+
* header bytes with the checksum field itself taken as ASCII spaces. Stored as
|
|
761
|
+
* six octal digits, a NUL, then a space (the conventional encoding).
|
|
762
|
+
*/
|
|
763
|
+
function writeChecksum(block) {
|
|
764
|
+
for (let i = 148; i < 156; i++) block[i] = 32;
|
|
765
|
+
let sum = 0;
|
|
766
|
+
for (let i = 0; i < BLOCK_SIZE; i++) sum += block[i];
|
|
767
|
+
const str = sum.toString(8).padStart(6, "0");
|
|
768
|
+
for (let i = 0; i < 6; i++) block[148 + i] = str.charCodeAt(i);
|
|
769
|
+
block[154] = 0;
|
|
770
|
+
block[155] = 32;
|
|
771
|
+
}
|
|
772
|
+
/**
|
|
773
|
+
* Splits a path into USTAR `name` (<=100 bytes) and `prefix` (<=155 bytes)
|
|
774
|
+
* fields at a `/` boundary; a reader rejoins them as `prefix + "/" + name`.
|
|
775
|
+
* Returns an empty prefix when the whole path fits in `name`. Throws when the
|
|
776
|
+
* path cannot be represented — a trailing component over 100 bytes, or leading
|
|
777
|
+
* directories that don't fit in 155. RocksDB and transaction-log filenames never
|
|
778
|
+
* hit this; only a pathological log-store name would.
|
|
779
|
+
*/
|
|
780
|
+
function splitTarName(path) {
|
|
781
|
+
const encoded = textEncoder.encode(path);
|
|
782
|
+
if (encoded.length <= MAX_NAME_LENGTH) return {
|
|
783
|
+
name: path,
|
|
784
|
+
prefix: ""
|
|
785
|
+
};
|
|
786
|
+
let best = -1;
|
|
787
|
+
for (let i = 0; i < encoded.length; i++) {
|
|
788
|
+
if (encoded[i] !== 47) continue;
|
|
789
|
+
const prefixLength = i;
|
|
790
|
+
const nameLength = encoded.length - i - 1;
|
|
791
|
+
if (nameLength > 0 && nameLength <= MAX_NAME_LENGTH && prefixLength <= MAX_PREFIX_LENGTH) best = i;
|
|
792
|
+
}
|
|
793
|
+
if (best === -1) throw new RangeError(`tar entry name ${JSON.stringify(path)} does not fit the USTAR name (<=${MAX_NAME_LENGTH}) + prefix (<=${MAX_PREFIX_LENGTH}) fields`);
|
|
794
|
+
return {
|
|
795
|
+
name: textDecoder.decode(encoded.subarray(best + 1)),
|
|
796
|
+
prefix: textDecoder.decode(encoded.subarray(0, best))
|
|
797
|
+
};
|
|
798
|
+
}
|
|
799
|
+
/**
|
|
800
|
+
* Builds a single 512-byte USTAR header block for a regular file. Long paths are
|
|
801
|
+
* split across the `name` and `prefix` fields (see {@link splitTarName}).
|
|
802
|
+
*/
|
|
803
|
+
function buildHeader(name, size, mode, mtime) {
|
|
804
|
+
const block = new Uint8Array(BLOCK_SIZE);
|
|
805
|
+
const split = splitTarName(name);
|
|
806
|
+
writeField(block, split.name, 0, MAX_NAME_LENGTH);
|
|
807
|
+
writeOctal(block, mode & 4095, 100, 8);
|
|
808
|
+
writeOctal(block, 0, 108, 8);
|
|
809
|
+
writeOctal(block, 0, 116, 8);
|
|
810
|
+
writeOctal(block, size, 124, 12);
|
|
811
|
+
writeOctal(block, Math.floor(mtime), 136, 12);
|
|
812
|
+
block[156] = 48;
|
|
813
|
+
writeField(block, "ustar", 257, 6);
|
|
814
|
+
block[263] = 48;
|
|
815
|
+
block[264] = 48;
|
|
816
|
+
if (split.prefix.length > 0) writeField(block, split.prefix, 345, MAX_PREFIX_LENGTH);
|
|
817
|
+
writeChecksum(block);
|
|
818
|
+
return block;
|
|
819
|
+
}
|
|
820
|
+
/**
|
|
821
|
+
* Streaming USTAR encoder. Usage is a strict sequence per file —
|
|
822
|
+
* {@link TarEncoder.addFile} once, then {@link TarEncoder.writeData} until the
|
|
823
|
+
* declared size is reached — followed by a single {@link TarEncoder.finalize}:
|
|
824
|
+
*
|
|
825
|
+
* ```ts
|
|
826
|
+
* const tar = new TarEncoder((chunk) => writer.write(chunk));
|
|
827
|
+
* await tar.addFile('CURRENT', current.length);
|
|
828
|
+
* await tar.writeData(current);
|
|
829
|
+
* await tar.addFile('000123.sst', sstSize);
|
|
830
|
+
* for await (const chunk of sstChunks) await tar.writeData(chunk);
|
|
831
|
+
* await tar.finalize();
|
|
832
|
+
* ```
|
|
833
|
+
*
|
|
834
|
+
* The size passed to `addFile` is authoritative: tar records it in the header
|
|
835
|
+
* before any payload, so it must be known up front (it is — RocksDB's
|
|
836
|
+
* `GetLiveFilesStorageInfo` returns each file's exact size). Writing more or
|
|
837
|
+
* fewer bytes than declared throws.
|
|
838
|
+
*/
|
|
839
|
+
var TarEncoder = class {
|
|
840
|
+
#sink;
|
|
841
|
+
/** Payload bytes still expected for the current entry. */
|
|
842
|
+
#remaining = 0;
|
|
843
|
+
/** Zero-padding bytes owed after the current entry's payload reaches a block boundary. */
|
|
844
|
+
#pad = 0;
|
|
845
|
+
#finalized = false;
|
|
846
|
+
constructor(sink) {
|
|
847
|
+
this.#sink = sink;
|
|
848
|
+
}
|
|
849
|
+
/**
|
|
850
|
+
* Emits the header for a new file entry. Must be preceded by completing any
|
|
851
|
+
* prior entry's payload. After this, exactly `size` bytes must be supplied
|
|
852
|
+
* via {@link TarEncoder.writeData} before the next `addFile` or `finalize`.
|
|
853
|
+
*/
|
|
854
|
+
async addFile(name, size, options) {
|
|
855
|
+
if (this.#finalized) throw new Error("TarEncoder already finalized");
|
|
856
|
+
if (this.#remaining !== 0) throw new Error(`previous tar entry incomplete: ${this.#remaining} byte(s) unwritten`);
|
|
857
|
+
if (!Number.isInteger(size) || size < 0) throw new RangeError("tar entry size must be a non-negative integer");
|
|
858
|
+
const header = buildHeader(name, size, options?.mode ?? 420, options?.mtime ?? 0);
|
|
859
|
+
this.#remaining = size;
|
|
860
|
+
this.#pad = (BLOCK_SIZE - size % BLOCK_SIZE) % BLOCK_SIZE;
|
|
861
|
+
await this.#sink(header);
|
|
862
|
+
}
|
|
863
|
+
/**
|
|
864
|
+
* Streams a chunk of the current entry's payload. Chunks may be any size; the
|
|
865
|
+
* total across all `writeData` calls for an entry must equal the size passed
|
|
866
|
+
* to {@link TarEncoder.addFile}. Block padding is emitted automatically once
|
|
867
|
+
* the final byte of an entry is written.
|
|
868
|
+
*/
|
|
869
|
+
async writeData(chunk) {
|
|
870
|
+
if (this.#finalized) throw new Error("TarEncoder already finalized");
|
|
871
|
+
if (chunk.length === 0) return;
|
|
872
|
+
if (chunk.length > this.#remaining) throw new RangeError(`tar data overruns entry size by ${chunk.length - this.#remaining} byte(s)`);
|
|
873
|
+
this.#remaining -= chunk.length;
|
|
874
|
+
await this.#sink(chunk);
|
|
875
|
+
if (this.#remaining === 0 && this.#pad > 0) {
|
|
876
|
+
const pad = this.#pad;
|
|
877
|
+
this.#pad = 0;
|
|
878
|
+
await this.#sink(new Uint8Array(pad));
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
/**
|
|
882
|
+
* Convenience for an entry whose contents are already fully in memory (e.g.
|
|
883
|
+
* the small `CURRENT`/`OPTIONS` files RocksDB returns inline).
|
|
884
|
+
*/
|
|
885
|
+
async addFileData(name, data, options) {
|
|
886
|
+
await this.addFile(name, data.length, options);
|
|
887
|
+
await this.writeData(data);
|
|
888
|
+
}
|
|
889
|
+
/**
|
|
890
|
+
* Writes the two zero-filled blocks that mark end-of-archive. The encoder is
|
|
891
|
+
* unusable afterward. Throws if the current entry is not fully written, so a
|
|
892
|
+
* truncated stream cannot be silently finalized into a valid-looking archive.
|
|
893
|
+
*/
|
|
894
|
+
async finalize() {
|
|
895
|
+
if (this.#finalized) throw new Error("TarEncoder already finalized");
|
|
896
|
+
if (this.#remaining !== 0) throw new Error(`cannot finalize: current tar entry has ${this.#remaining} byte(s) unwritten`);
|
|
897
|
+
this.#finalized = true;
|
|
898
|
+
await this.#sink(new Uint8Array(BLOCK_SIZE * 2));
|
|
899
|
+
}
|
|
900
|
+
};
|
|
901
|
+
|
|
902
|
+
//#endregion
|
|
903
|
+
//#region src/backup-stream.ts
|
|
904
|
+
/** Native event discriminator: a new file header (vs. a payload chunk). */
|
|
905
|
+
const EVENT_FILE = 0;
|
|
906
|
+
/**
|
|
907
|
+
* Streams a consistent snapshot of the database to a `WritableStream` as a tar
|
|
908
|
+
* archive, with no intermediate copy on disk. The set of live RocksDB files is
|
|
909
|
+
* enumerated and read natively; each file header and payload chunk is framed
|
|
910
|
+
* into the {@link TarEncoder} here in JS and written to `stream`.
|
|
911
|
+
*
|
|
912
|
+
* Backpressure flows end to end: the encoder awaits each `writer.write()`, and
|
|
913
|
+
* the native producer awaits each `emit()` before reading the next chunk, so a
|
|
914
|
+
* slow consumer paces the entire pipeline rather than buffering in memory.
|
|
915
|
+
*
|
|
916
|
+
* The resulting archive can be unpacked with any tar tool (`tar -xf`) into a
|
|
917
|
+
* directory that opens directly as a RocksDB database.
|
|
918
|
+
*/
|
|
919
|
+
async function backupToStream(db, stream, options) {
|
|
920
|
+
let tarDestination;
|
|
921
|
+
let piped;
|
|
922
|
+
if (options?.gzip) {
|
|
923
|
+
const gzip = new CompressionStream("gzip");
|
|
924
|
+
piped = gzip.readable.pipeTo(stream);
|
|
925
|
+
tarDestination = gzip.writable;
|
|
926
|
+
} else tarDestination = stream;
|
|
927
|
+
const writer = tarDestination.getWriter();
|
|
928
|
+
const tar = new TarEncoder((chunk) => writer.write(chunk));
|
|
929
|
+
const emit = async (kind, data, size, mtime) => {
|
|
930
|
+
if (kind === EVENT_FILE) await tar.addFile(data, size, { mtime });
|
|
931
|
+
else await tar.writeData(data);
|
|
932
|
+
};
|
|
933
|
+
try {
|
|
934
|
+
await new Promise((resolve, reject) => {
|
|
935
|
+
db.backupStream(resolve, reject, emit, options);
|
|
936
|
+
});
|
|
937
|
+
await tar.finalize();
|
|
938
|
+
await writer.close();
|
|
939
|
+
if (piped) await piped;
|
|
940
|
+
} catch (err) {
|
|
941
|
+
await writer.abort(err).catch(() => {});
|
|
942
|
+
if (piped) await piped.catch(() => {});
|
|
943
|
+
throw err;
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
|
|
546
947
|
//#endregion
|
|
547
948
|
//#region src/dbi-iterator.ts
|
|
548
949
|
const { ITERATOR_RESULT_DONE, ITERATOR_RESULT_FAST } = constants;
|
|
@@ -925,19 +1326,9 @@ var Store = class {
|
|
|
925
1326
|
}
|
|
926
1327
|
return new Promise((resolve, reject) => this.db.compact(resolve, reject, startBuffer, endBuffer));
|
|
927
1328
|
}
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
* are created as needed. See `backups` for restore and management.
|
|
932
|
-
*
|
|
933
|
-
* @example
|
|
934
|
-
* ```typescript
|
|
935
|
-
* const id = await db.backup('/path/to/backups');
|
|
936
|
-
* ```
|
|
937
|
-
*/
|
|
938
|
-
async backup(backupDir, options) {
|
|
939
|
-
await (0, node_fs_promises.mkdir)(backupDir, { recursive: true });
|
|
940
|
-
return new Promise((resolve, reject) => this.db.backup(resolve, reject, backupDir, options));
|
|
1329
|
+
async backup(target, options) {
|
|
1330
|
+
if (typeof target !== "string" && typeof target?.getWriter === "function") return backupToStream(this.db, target, options);
|
|
1331
|
+
return new Promise((resolve, reject) => this.db.backup(resolve, reject, target, options));
|
|
941
1332
|
}
|
|
942
1333
|
/**
|
|
943
1334
|
* Compacts the entire key range of the database synchronously.
|
|
@@ -1506,25 +1897,43 @@ var RocksDatabase = class RocksDatabase extends DBI {
|
|
|
1506
1897
|
compact(options) {
|
|
1507
1898
|
return this.store.compact(options);
|
|
1508
1899
|
}
|
|
1900
|
+
backup(target, options) {
|
|
1901
|
+
if (typeof target === "string") return this.store.backup(target, options);
|
|
1902
|
+
return this.store.backup(target, options);
|
|
1903
|
+
}
|
|
1509
1904
|
/**
|
|
1510
|
-
* Creates a
|
|
1511
|
-
*
|
|
1512
|
-
* the new backup id. Use the `backups` namespace to restore, list, delete,
|
|
1513
|
-
* purge, or verify backups.
|
|
1905
|
+
* Creates a hardlinked, point-in-time, fully independent copy of the entire
|
|
1906
|
+
* database (all column families) at `targetPath` and resolves once written.
|
|
1514
1907
|
*
|
|
1515
|
-
*
|
|
1516
|
-
*
|
|
1908
|
+
* Unlike a backup, a checkpoint is a normal, writable sibling database: open
|
|
1909
|
+
* it with {@link RocksDatabase.open} and it diverges independently from the
|
|
1910
|
+
* source. SST and blob files are hardlinked when `targetPath` is on the same
|
|
1911
|
+
* filesystem as the database and copied otherwise (other files such as the
|
|
1912
|
+
* MANIFEST are always copied), so the operation is near-instant on the same
|
|
1913
|
+
* filesystem. The memtable is flushed so the checkpoint includes the latest
|
|
1914
|
+
* writes even when the WAL is disabled.
|
|
1915
|
+
*
|
|
1916
|
+
* Parent directories are created as needed. `targetPath` itself must not
|
|
1917
|
+
* already exist (RocksDB creates the checkpoint directory) and rejects with
|
|
1918
|
+
* `Create checkpoint failed: target path exists` if it does. The caller is
|
|
1919
|
+
* responsible for eventual cleanup of the directory.
|
|
1517
1920
|
*
|
|
1518
1921
|
* @example
|
|
1519
1922
|
* ```typescript
|
|
1520
|
-
* import { backups } from '@harperfast/rocksdb-js';
|
|
1521
|
-
*
|
|
1522
1923
|
* const db = RocksDatabase.open('/path/to/database');
|
|
1523
|
-
*
|
|
1924
|
+
* await db.createCheckpoint('/path/to/checkpoint');
|
|
1925
|
+
* const branch = RocksDatabase.open('/path/to/checkpoint');
|
|
1524
1926
|
* ```
|
|
1525
1927
|
*/
|
|
1526
|
-
|
|
1527
|
-
return
|
|
1928
|
+
createCheckpoint(targetPath) {
|
|
1929
|
+
return new Promise((resolve, reject) => {
|
|
1930
|
+
if ((0, node_fs.existsSync)(targetPath)) {
|
|
1931
|
+
reject(/* @__PURE__ */ new Error("Create checkpoint failed: target path exists"));
|
|
1932
|
+
return;
|
|
1933
|
+
}
|
|
1934
|
+
(0, node_fs.mkdirSync)((0, node_path.dirname)(targetPath), { recursive: true });
|
|
1935
|
+
this.store.db.createCheckpoint(resolve, reject, targetPath);
|
|
1936
|
+
});
|
|
1528
1937
|
}
|
|
1529
1938
|
/**
|
|
1530
1939
|
* Compacts the entire key range of the database synchronously.
|
|
@@ -2375,7 +2784,7 @@ function loadLastPosition(transactionLog, readUncommitted) {
|
|
|
2375
2784
|
//#region src/index.ts
|
|
2376
2785
|
const versions = {
|
|
2377
2786
|
rocksdb: version,
|
|
2378
|
-
"rocksdb-js": "2.
|
|
2787
|
+
"rocksdb-js": "2.4.0"
|
|
2379
2788
|
};
|
|
2380
2789
|
|
|
2381
2790
|
//#endregion
|
|
@@ -2389,9 +2798,11 @@ exports.backups = backups;
|
|
|
2389
2798
|
exports.constants = constants;
|
|
2390
2799
|
exports.coolTransactionLogs = coolTransactionLogs;
|
|
2391
2800
|
exports.currentThreadId = currentThreadId;
|
|
2801
|
+
exports.fileLockRelease = fileLockRelease;
|
|
2392
2802
|
exports.parseTransactionLog = parseTransactionLog;
|
|
2393
2803
|
exports.registryStatus = registryStatus;
|
|
2394
2804
|
exports.shutdown = shutdown;
|
|
2395
2805
|
exports.stats = stats;
|
|
2806
|
+
exports.tryFileLock = tryFileLock;
|
|
2396
2807
|
exports.versions = versions;
|
|
2397
2808
|
//# sourceMappingURL=index.cjs.map
|