event-storage 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -0
- package/package.json +3 -5
- package/src/Consumer.js +9 -7
- package/src/EventStore.js +165 -83
- package/src/EventStream.js +49 -11
- package/src/Index/ReadableIndex.js +7 -5
- package/src/Index/WritableIndex.js +4 -6
- package/src/IndexMatcher.js +2 -2
- package/src/JoinEventStream.js +30 -46
- package/src/Partition/ReadablePartition.js +153 -85
- package/src/Partition/WritablePartition.js +37 -28
- package/src/Storage/ReadOnlyStorage.js +3 -3
- package/src/Storage/ReadableStorage.js +73 -102
- package/src/Storage/WritableStorage.js +43 -25
- package/src/Watcher.js +1 -1
- package/src/utils/jsonUtil.js +87 -0
- package/src/utils/metadataUtil.js +247 -0
- package/src/{util.js → utils/util.js} +52 -17
- package/src/metadataUtil.js +0 -126
- /package/src/{fsUtil.js → utils/fsUtil.js} +0 -0
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import fs from 'fs';
|
|
2
2
|
import path from 'path';
|
|
3
3
|
import events from 'events';
|
|
4
|
-
import { assert, alignTo, hash, binarySearch } from '../util.js';
|
|
4
|
+
import { assert, alignTo, hash, binarySearch } from '../utils/util.js';
|
|
5
|
+
|
|
6
|
+
|
|
5
7
|
|
|
6
8
|
const DEFAULT_READ_BUFFER_SIZE = 64 * 1024;
|
|
7
9
|
const DOCUMENT_HEADER_SIZE = 16;
|
|
@@ -114,10 +116,10 @@ class ReadablePartition extends events.EventEmitter {
|
|
|
114
116
|
fs.readSync(this.fd, headerBuffer, 0, 8 + 4, 0);
|
|
115
117
|
const headerMagic = headerBuffer.toString('utf8', 0, 8);
|
|
116
118
|
|
|
117
|
-
assert(headerMagic.
|
|
119
|
+
assert(headerMagic.substring(0, 6) === HEADER_MAGIC.substring(0, 6), `Invalid file header in partition ${this.name}.`);
|
|
118
120
|
|
|
119
121
|
this.header = headerMagic;
|
|
120
|
-
assert(headerMagic === HEADER_MAGIC, `Invalid file version. The partition ${this.name} was created with a different library version (${headerMagic.
|
|
122
|
+
assert(headerMagic === HEADER_MAGIC, `Invalid file version. The partition ${this.name} was created with a different library version (${headerMagic.substring(6)}).`);
|
|
121
123
|
|
|
122
124
|
const metadataSize = headerBuffer.readUInt32BE(8);
|
|
123
125
|
assert(metadataSize > 2 && metadataSize <= 4096, 'Invalid metadata size.');
|
|
@@ -200,51 +202,35 @@ class ReadablePartition extends events.EventEmitter {
|
|
|
200
202
|
const dataSize = buffer.readUInt32BE(offset + 0);
|
|
201
203
|
assert(dataSize > 0 && dataSize <= 64 * 1024 * 1024, `Error reading document size from ${position}, got ${dataSize}.`);
|
|
202
204
|
|
|
203
|
-
|
|
204
|
-
throw new InvalidDataSizeError(`Invalid document size ${dataSize} at position ${position}, expected ${size}.`);
|
|
205
|
-
}
|
|
205
|
+
assert(!size || dataSize === size, `Invalid document size ${dataSize} at position ${position}, expected ${size}.`, InvalidDataSizeError);
|
|
206
206
|
|
|
207
207
|
const sequenceNumber = buffer.readUInt32BE(offset + 4);
|
|
208
208
|
const time64 = buffer.readDoubleBE(offset + 8);
|
|
209
209
|
return ({ dataSize, sequenceNumber, time64 });
|
|
210
210
|
}
|
|
211
211
|
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
*
|
|
215
|
-
* @protected
|
|
216
|
-
* @param {number} position The position in the file to prepare the read buffer for reading from.
|
|
217
|
-
* @returns {{ buffer: Buffer|null, cursor: number, length: number }} A reader object with properties `buffer`, `cursor` and `length`.
|
|
218
|
-
*/
|
|
219
|
-
prepareReadBuffer(position) {
|
|
220
|
-
if (position + DOCUMENT_HEADER_SIZE >= this.size) {
|
|
221
|
-
return ({ buffer: null, cursor: 0, length: 0 });
|
|
222
|
-
}
|
|
223
|
-
let bufferCursor = position - this.readBufferPos;
|
|
224
|
-
if (this.readBufferPos < 0 || bufferCursor < 0 || bufferCursor + DOCUMENT_HEADER_SIZE + DOCUMENT_ALIGNMENT > this.readBufferLength) {
|
|
225
|
-
this.fillBuffer(position);
|
|
226
|
-
bufferCursor = 0;
|
|
227
|
-
}
|
|
228
|
-
return ({ buffer: this.readBuffer, cursor: bufferCursor, length: this.readBufferLength });
|
|
212
|
+
resolveIterationPosition(position) {
|
|
213
|
+
return position < 0 ? this.size + position + 1 : position;
|
|
229
214
|
}
|
|
230
215
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
* @returns {{ buffer: Buffer|null, cursor: number, length: number }} A reader object with properties `buffer`, `cursor` and `length`.
|
|
237
|
-
*/
|
|
238
|
-
prepareReadBufferBackwards(position) {
|
|
239
|
-
if (position < 0) {
|
|
240
|
-
return ({ buffer: null, cursor: 0, length: 0 });
|
|
216
|
+
selectReader(position, size, backwardsHint) {
|
|
217
|
+
if (size > 0 && backwardsHint) {
|
|
218
|
+
const bufferOffset = DOCUMENT_HEADER_SIZE + size;
|
|
219
|
+
const reader = this.prepareReadBufferBackwards(position + bufferOffset, bufferOffset);
|
|
220
|
+
return { reader, bufferOffset };
|
|
241
221
|
}
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
222
|
+
return { reader: this.prepareReadBuffer(position), bufferOffset: 0 };
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
assignHeaderOutput(headerOut, header) {
|
|
226
|
+
if (headerOut === null) {
|
|
227
|
+
return;
|
|
246
228
|
}
|
|
247
|
-
|
|
229
|
+
headerOut.dataSize = header.dataSize;
|
|
230
|
+
headerOut.sequenceNumber = header.sequenceNumber;
|
|
231
|
+
// Denormalize time64 relative to this partition's epoch so callers can compare
|
|
232
|
+
// timestamps across partitions without needing to know the epoch value.
|
|
233
|
+
headerOut.time64 = this.metadata.epoch + header.time64;
|
|
248
234
|
}
|
|
249
235
|
|
|
250
236
|
/**
|
|
@@ -255,39 +241,37 @@ class ReadablePartition extends events.EventEmitter {
|
|
|
255
241
|
* @param {number} [size] The expected byte size of the document at the given position.
|
|
256
242
|
* @param {object|null} [headerOut] Optional object to populate with the document header fields
|
|
257
243
|
* (`dataSize`, `sequenceNumber`, `time64`). Pass an existing object to avoid extra allocation.
|
|
258
|
-
* @
|
|
244
|
+
* @param {boolean} [backwardsHint] If set to true, will optimize buffering for backwards reads.
|
|
245
|
+
* @returns {Buffer|boolean} The data stored at the given position or false if no data could be read.
|
|
259
246
|
* @throws {Error} if the storage entry at the given position is corrupted.
|
|
260
247
|
* @throws {InvalidDataSizeError} if the document size at the given position does not match the provided size.
|
|
261
248
|
* @throws {CorruptFileError} if the document at the given position can not be read completely.
|
|
262
249
|
*/
|
|
263
|
-
readFrom(position, size = 0, headerOut = null) {
|
|
250
|
+
readFrom(position, size = 0, headerOut = null, backwardsHint = false) {
|
|
264
251
|
assert(this.fd, 'Partition is not opened.');
|
|
265
252
|
assert((position % DOCUMENT_ALIGNMENT) === 0, `Invalid read position ${position}. Needs to be a multiple of ${DOCUMENT_ALIGNMENT}.`);
|
|
266
253
|
|
|
267
|
-
const reader = this.
|
|
268
|
-
if (reader.length <
|
|
254
|
+
const { reader, bufferOffset } = this.selectReader(position, size, backwardsHint);
|
|
255
|
+
if (reader.length < DOCUMENT_HEADER_SIZE) {
|
|
269
256
|
return false;
|
|
270
257
|
}
|
|
271
258
|
|
|
259
|
+
// prepareReadBufferBackwards positions the cursor at position + bufferOffset (the end of
|
|
260
|
+
// the document data), so the previous document in file order lands inside the buffer on the next
|
|
261
|
+
// backwards read. Adjust the cursor back to the document header start before reading.
|
|
262
|
+
reader.cursor -= bufferOffset;
|
|
272
263
|
let dataPosition = reader.cursor + DOCUMENT_HEADER_SIZE;
|
|
273
|
-
const
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
headerOut.sequenceNumber = sequenceNumber;
|
|
277
|
-
headerOut.time64 = time64;
|
|
278
|
-
}
|
|
264
|
+
const header = this.readDocumentHeader(reader.buffer, reader.cursor, position, size);
|
|
265
|
+
const dataSize = header.dataSize;
|
|
266
|
+
this.assignHeaderOutput(headerOut, header);
|
|
279
267
|
|
|
280
|
-
// TODO: This should only be checked on opening
|
|
281
268
|
const writeSize = this.documentWriteSize(dataSize);
|
|
282
|
-
|
|
283
|
-
throw new CorruptFileError(`Invalid document at position ${position}. This may be caused by an unfinished write.`);
|
|
284
|
-
}
|
|
269
|
+
assert(position + writeSize <= this.size, `Invalid document at position ${position}. This may be caused by an unfinished write.`, CorruptFileError);
|
|
285
270
|
|
|
286
271
|
if (dataSize + DOCUMENT_HEADER_SIZE > reader.buffer.byteLength) {
|
|
287
|
-
//console.log('sync read for large document size', dataLength, 'at position', position);
|
|
288
272
|
const tempReadBuffer = Buffer.allocUnsafe(dataSize);
|
|
289
273
|
fs.readSync(this.fd, tempReadBuffer, 0, dataSize, this.headerSize + position + DOCUMENT_HEADER_SIZE);
|
|
290
|
-
return tempReadBuffer
|
|
274
|
+
return tempReadBuffer;
|
|
291
275
|
}
|
|
292
276
|
|
|
293
277
|
if (reader.cursor > 0 && dataPosition + dataSize > reader.length) {
|
|
@@ -295,7 +279,48 @@ class ReadablePartition extends events.EventEmitter {
|
|
|
295
279
|
dataPosition = DOCUMENT_HEADER_SIZE;
|
|
296
280
|
}
|
|
297
281
|
|
|
298
|
-
|
|
282
|
+
// reader.buffer is a shared buffer filled by fillBuffer; callers must consume the returned
|
|
283
|
+
// view before the next readFrom call (which may fill the same buffer region).
|
|
284
|
+
return reader.buffer.subarray(dataPosition, dataPosition + dataSize);
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* Prepare the read buffer for reading from the specified position.
|
|
289
|
+
*
|
|
290
|
+
* @protected
|
|
291
|
+
* @param {number} position The position in the file to prepare the read buffer for reading from.
|
|
292
|
+
* @returns {{ buffer: Buffer|null, cursor: number, length: number }} A reader object with properties `buffer`, `cursor` and `length`.
|
|
293
|
+
*/
|
|
294
|
+
prepareReadBuffer(position) {
|
|
295
|
+
if (position + DOCUMENT_HEADER_SIZE >= this.size) {
|
|
296
|
+
return ({ buffer: null, cursor: 0, length: 0 });
|
|
297
|
+
}
|
|
298
|
+
let bufferCursor = position - this.readBufferPos;
|
|
299
|
+
if (this.readBufferPos < 0 || bufferCursor < 0 || bufferCursor + DOCUMENT_HEADER_SIZE + DOCUMENT_ALIGNMENT > this.readBufferLength) {
|
|
300
|
+
this.fillBuffer(position);
|
|
301
|
+
bufferCursor = 0;
|
|
302
|
+
}
|
|
303
|
+
return ({ buffer: this.readBuffer, cursor: bufferCursor, length: this.readBufferLength });
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* Prepare the read buffer for reading *before* the specified position. Don't try to read *after* the returned cursor.
|
|
308
|
+
*
|
|
309
|
+
* @protected
|
|
310
|
+
* @param {number} position The position in the file to prepare the read buffer for reading before.
|
|
311
|
+
* @param {number} [size] The amount of bytes that need to be buffered before position. By default, only guarantees that the document footer can be read.
|
|
312
|
+
* @returns {{ buffer: Buffer|null, cursor: number, length: number }} A reader object with properties `buffer`, `cursor` and `length`.
|
|
313
|
+
*/
|
|
314
|
+
prepareReadBufferBackwards(position, size = 0) {
|
|
315
|
+
if (position < 0) {
|
|
316
|
+
return ({ buffer: null, cursor: 0, length: 0 });
|
|
317
|
+
}
|
|
318
|
+
let bufferCursor = position - this.readBufferPos;
|
|
319
|
+
if (this.readBufferPos < 0 || (this.readBufferPos > 0 && bufferCursor < size + DOCUMENT_FOOTER_SIZE) || bufferCursor > this.readBufferLength) {
|
|
320
|
+
this.fillBuffer(Math.max(position - this.readBuffer.byteLength, 0));
|
|
321
|
+
bufferCursor = position - this.readBufferPos;
|
|
322
|
+
}
|
|
323
|
+
return ({ buffer: this.readBuffer, cursor: bufferCursor, length: this.readBufferLength });
|
|
299
324
|
}
|
|
300
325
|
|
|
301
326
|
/**
|
|
@@ -307,7 +332,7 @@ class ReadablePartition extends events.EventEmitter {
|
|
|
307
332
|
*/
|
|
308
333
|
findDocumentPositionBefore(position) {
|
|
309
334
|
assert(this.fd, 'Partition is not opened.');
|
|
310
|
-
position -= (position % DOCUMENT_ALIGNMENT);
|
|
335
|
+
if (position > 0) position -= (position % DOCUMENT_ALIGNMENT);
|
|
311
336
|
if (position <= 0) {
|
|
312
337
|
return false;
|
|
313
338
|
}
|
|
@@ -373,49 +398,49 @@ class ReadablePartition extends events.EventEmitter {
|
|
|
373
398
|
}
|
|
374
399
|
|
|
375
400
|
/**
|
|
376
|
-
* Find the
|
|
401
|
+
* Find a document around the target sequence number using one of two binary-search modes.
|
|
377
402
|
* Uses readLast() to short-circuit when the partition contains no such document.
|
|
378
403
|
* Uses a binary search over file positions via readDocumentBefore() to locate the
|
|
379
|
-
* document
|
|
380
|
-
* confirmed "< sequenceNumber" doc) and the upper bound (minimum position of any
|
|
381
|
-
* probed doc with sequenceNumber >= target). The upper bound, when available, is
|
|
382
|
-
* the exact target document, so no further linear scan is needed.
|
|
404
|
+
* document and tracks nearest candidates on both sides of the target.
|
|
383
405
|
*
|
|
384
406
|
* @api
|
|
385
407
|
* @param {number} sequenceNumber The 0-based sequence number to search for.
|
|
386
|
-
* @
|
|
387
|
-
*
|
|
408
|
+
* @param {boolean} [min=true] When true, returns the first document with sequenceNumber >= target.
|
|
409
|
+
* When false, returns the last document with sequenceNumber <= target.
|
|
410
|
+
* @returns {{ data: Buffer, header: object, position: number }|null}
|
|
411
|
+
* The matched document with its header and position, or null if no such document exists.
|
|
388
412
|
*/
|
|
389
|
-
findDocument(sequenceNumber) {
|
|
413
|
+
findDocument(sequenceNumber, min = true) {
|
|
390
414
|
const last = this.readLast();
|
|
391
|
-
if (!last
|
|
415
|
+
if (!last) {
|
|
392
416
|
return null;
|
|
393
417
|
}
|
|
394
418
|
|
|
395
|
-
|
|
396
|
-
|
|
419
|
+
if (min && last.header.sequenceNumber < sequenceNumber) {
|
|
420
|
+
return null;
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
const [low, high] = binarySearch(
|
|
397
424
|
sequenceNumber,
|
|
398
425
|
this.size,
|
|
399
426
|
(pos) => {
|
|
400
427
|
const doc = this.readDocumentBefore(pos);
|
|
401
|
-
|
|
402
|
-
if (doc.header.sequenceNumber < sequenceNumber) {
|
|
403
|
-
startPosition = Math.max(startPosition, doc.position + this.documentWriteSize(doc.header.dataSize));
|
|
404
|
-
} else {
|
|
405
|
-
startPosition = Math.min(startPosition, doc.position);
|
|
406
|
-
}
|
|
407
|
-
return doc.header.sequenceNumber;
|
|
428
|
+
return doc ? doc.header.sequenceNumber : Number.MIN_SAFE_INTEGER;
|
|
408
429
|
}
|
|
409
430
|
);
|
|
410
431
|
|
|
411
|
-
const
|
|
412
|
-
|
|
432
|
+
const position = this.findDocumentPositionBefore(min ? low : high);
|
|
433
|
+
if (position === false || position < 0) {
|
|
434
|
+
return null;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
const header = {};
|
|
438
|
+
const data = this.readFrom(position, 0, header);
|
|
413
439
|
/* istanbul ignore if */
|
|
414
440
|
if (data === false) {
|
|
415
441
|
return null;
|
|
416
442
|
}
|
|
417
|
-
|
|
418
|
-
return { headerOut, data };
|
|
443
|
+
return { data, header, position };
|
|
419
444
|
}
|
|
420
445
|
|
|
421
446
|
/**
|
|
@@ -424,10 +449,10 @@ class ReadablePartition extends events.EventEmitter {
|
|
|
424
449
|
* @param {object|null} [headerOut] Optional object to populate with document header fields
|
|
425
450
|
* (`dataSize`, `sequenceNumber`, `time64`, `position`) on each yield. Pass an existing object
|
|
426
451
|
* to avoid extra allocation. The object is mutated in place before each yield.
|
|
427
|
-
* @returns {Generator<
|
|
452
|
+
* @returns {Generator<Buffer>} A generator that returns all documents in this partition.
|
|
428
453
|
*/
|
|
429
454
|
*readAll(after = 0, headerOut = null) {
|
|
430
|
-
let position =
|
|
455
|
+
let position = this.resolveIterationPosition(after);
|
|
431
456
|
const internalHeader = headerOut !== null ? headerOut : {};
|
|
432
457
|
let data;
|
|
433
458
|
while ((data = this.readFrom(position, 0, internalHeader)) !== false) {
|
|
@@ -442,16 +467,59 @@ class ReadablePartition extends events.EventEmitter {
|
|
|
442
467
|
/**
|
|
443
468
|
* @api
|
|
444
469
|
* @param {number} [before] The document position to start reading backward from.
|
|
445
|
-
* @
|
|
470
|
+
* @param {object|null} [headerOut] Optional object to populate with document header fields
|
|
471
|
+
* (`dataSize`, `sequenceNumber`, `time64`, `position`) on each yield.
|
|
472
|
+
* @returns {Generator<Buffer>} A generator that returns all documents in this partition in reverse order.
|
|
446
473
|
*/
|
|
447
|
-
*readAllBackwards(before = -1) {
|
|
448
|
-
let position =
|
|
474
|
+
*readAllBackwards(before = -1, headerOut = null) {
|
|
475
|
+
let position = this.resolveIterationPosition(before);
|
|
476
|
+
const internalHeader = headerOut !== null ? headerOut : {};
|
|
449
477
|
while ((position = this.findDocumentPositionBefore(position)) !== false) {
|
|
450
|
-
const data = this.readFrom(position);
|
|
478
|
+
const data = this.readFrom(position, 0, internalHeader, true);
|
|
479
|
+
if (headerOut !== null) {
|
|
480
|
+
headerOut.position = position;
|
|
481
|
+
}
|
|
451
482
|
yield data;
|
|
452
483
|
}
|
|
453
484
|
}
|
|
485
|
+
|
|
486
|
+
/**
|
|
487
|
+
* Read documents with sequenceNumber in the inclusive range [from, until].
|
|
488
|
+
* If from > until, documents are yielded in reverse order.
|
|
489
|
+
*
|
|
490
|
+
* @api
|
|
491
|
+
* @param {number} [from=0]
|
|
492
|
+
* @param {number} [until=Number.MAX_SAFE_INTEGER]
|
|
493
|
+
* @returns {Generator<{ data: Buffer, header: object, entry: object }>}
|
|
494
|
+
*/
|
|
495
|
+
*readRange(from = 0, until = Number.MAX_SAFE_INTEGER) {
|
|
496
|
+
const forwards = from <= until;
|
|
497
|
+
const lo = Math.min(from, until);
|
|
498
|
+
const hi = Math.max(from, until);
|
|
499
|
+
const found = this.findDocument(forwards ? lo : hi, forwards);
|
|
500
|
+
if (!found || found.header.sequenceNumber < lo || found.header.sequenceNumber > hi) {
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
const entry = { partition: this.id };
|
|
505
|
+
const header = {};
|
|
506
|
+
const iterator = forwards
|
|
507
|
+
? this.readAll(found.position, header)
|
|
508
|
+
: this.readAllBackwards(found.position + this.documentWriteSize(found.header.dataSize), header);
|
|
509
|
+
|
|
510
|
+
for (const data of iterator) {
|
|
511
|
+
if (header.sequenceNumber < lo || header.sequenceNumber > hi) {
|
|
512
|
+
return;
|
|
513
|
+
}
|
|
514
|
+
entry.number = header.sequenceNumber;
|
|
515
|
+
entry.position = header.position;
|
|
516
|
+
entry.size = header.dataSize;
|
|
517
|
+
yield { data, header, entry };
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
|
|
454
522
|
}
|
|
455
523
|
|
|
456
524
|
export default ReadablePartition;
|
|
457
|
-
export { CorruptFileError, InvalidDataSizeError, HEADER_MAGIC, DOCUMENT_SEPARATOR, DOCUMENT_ALIGNMENT, DOCUMENT_HEADER_SIZE, DOCUMENT_FOOTER_SIZE };
|
|
525
|
+
export { CorruptFileError, InvalidDataSizeError, HEADER_MAGIC, DOCUMENT_SEPARATOR, DOCUMENT_ALIGNMENT, DOCUMENT_HEADER_SIZE, DOCUMENT_FOOTER_SIZE };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import fs from 'fs';
|
|
2
2
|
import ReadablePartition, { CorruptFileError, HEADER_MAGIC, DOCUMENT_ALIGNMENT, DOCUMENT_SEPARATOR, DOCUMENT_HEADER_SIZE, DOCUMENT_FOOTER_SIZE } from './ReadablePartition.js';
|
|
3
|
-
import { assert, alignTo } from '../util.js';
|
|
4
|
-
import { buildMetadataHeader } from '../metadataUtil.js';
|
|
5
|
-
import { ensureDirectory } from '../fsUtil.js';
|
|
3
|
+
import { assert, alignTo } from '../utils/util.js';
|
|
4
|
+
import { buildMetadataHeader } from '../utils/metadataUtil.js';
|
|
5
|
+
import { ensureDirectory } from '../utils/fsUtil.js';
|
|
6
6
|
import Clock from '../Clock.js';
|
|
7
7
|
|
|
8
8
|
const DEFAULT_WRITE_BUFFER_SIZE = 16 * 1024;
|
|
@@ -177,22 +177,34 @@ class WritablePartition extends ReadablePartition {
|
|
|
177
177
|
* @returns {number} The size of the document header
|
|
178
178
|
*/
|
|
179
179
|
writeDocumentHeader(buffer, offset, dataSize, sequenceNumber = null, time64 = null) {
|
|
180
|
-
|
|
181
|
-
sequenceNumber = 0;
|
|
182
|
-
}
|
|
183
|
-
if (time64 === null) {
|
|
184
|
-
time64 = this.clock.time();
|
|
185
|
-
}
|
|
180
|
+
({ sequenceNumber, time64 } = this.normalizeWriteMetadata(sequenceNumber, time64));
|
|
186
181
|
/* istanbul ignore if */
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
}
|
|
182
|
+
assert(time64 >= 0, 'Time may not be negative!');
|
|
183
|
+
|
|
190
184
|
buffer.writeUInt32BE(dataSize, offset);
|
|
191
185
|
buffer.writeUInt32BE(sequenceNumber, offset + 4);
|
|
192
186
|
buffer.writeDoubleBE(time64, offset + 8);
|
|
193
187
|
return DOCUMENT_HEADER_SIZE;
|
|
194
188
|
}
|
|
195
189
|
|
|
190
|
+
normalizeWriteMetadata(sequenceNumber, time64) {
|
|
191
|
+
return {
|
|
192
|
+
sequenceNumber: sequenceNumber === null ? 0 : sequenceNumber,
|
|
193
|
+
time64: time64 === null ? this.clock.time() : time64
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
normalizeWriteArguments(sequenceNumber, callback) {
|
|
198
|
+
if (typeof sequenceNumber === 'function') {
|
|
199
|
+
return { sequenceNumber: null, callback: sequenceNumber };
|
|
200
|
+
}
|
|
201
|
+
return { sequenceNumber, callback };
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
shouldWriteUnbuffered(dataSize) {
|
|
205
|
+
return dataSize + DOCUMENT_HEADER_SIZE >= this.writeBuffer.byteLength * 4 / 5;
|
|
206
|
+
}
|
|
207
|
+
|
|
196
208
|
/**
|
|
197
209
|
* Write the given data to the partition without buffering.
|
|
198
210
|
* @private
|
|
@@ -210,7 +222,7 @@ class WritablePartition extends ReadablePartition {
|
|
|
210
222
|
bytesWritten += fs.writeSync(this.fd, this.writeMetaBuffer, 0, DOCUMENT_HEADER_SIZE);
|
|
211
223
|
bytesWritten += fs.writeSync(this.fd, data);
|
|
212
224
|
const padSize = alignTo(dataSize + DOCUMENT_FOOTER_SIZE, DOCUMENT_ALIGNMENT);
|
|
213
|
-
bytesWritten += fs.writeSync(this.fd, DOCUMENT_PAD.
|
|
225
|
+
bytesWritten += fs.writeSync(this.fd, DOCUMENT_PAD.substring(0, padSize));
|
|
214
226
|
this.writeMetaBuffer.writeUInt32BE(dataSize, 0);
|
|
215
227
|
bytesWritten += fs.writeSync(this.fd, this.writeMetaBuffer, 0, 4);
|
|
216
228
|
bytesWritten += fs.writeSync(this.fd, DOCUMENT_SEPARATOR);
|
|
@@ -237,7 +249,7 @@ class WritablePartition extends ReadablePartition {
|
|
|
237
249
|
bytesWritten += this.writeDocumentHeader(this.writeBuffer, this.writeBufferCursor, dataSize, sequenceNumber);
|
|
238
250
|
bytesWritten += this.writeBuffer.write(data, this.writeBufferCursor + bytesWritten, 'utf8');
|
|
239
251
|
const padSize = alignTo(dataSize + DOCUMENT_FOOTER_SIZE, DOCUMENT_ALIGNMENT);
|
|
240
|
-
bytesWritten += this.writeBuffer.write(DOCUMENT_PAD.
|
|
252
|
+
bytesWritten += this.writeBuffer.write(DOCUMENT_PAD.substring(0, padSize), this.writeBufferCursor + bytesWritten, 'utf8');
|
|
241
253
|
this.writeBuffer.writeUInt32BE(dataSize, this.writeBufferCursor + bytesWritten);
|
|
242
254
|
bytesWritten += 4;
|
|
243
255
|
bytesWritten += this.writeBuffer.write(DOCUMENT_SEPARATOR, this.writeBufferCursor + bytesWritten, 'utf8');
|
|
@@ -259,15 +271,12 @@ class WritablePartition extends ReadablePartition {
|
|
|
259
271
|
*/
|
|
260
272
|
write(data, sequenceNumber, callback) {
|
|
261
273
|
assert(this.fd, 'Partition is not opened.');
|
|
262
|
-
|
|
263
|
-
callback = sequenceNumber;
|
|
264
|
-
sequenceNumber = null;
|
|
265
|
-
}
|
|
274
|
+
({ sequenceNumber, callback } = this.normalizeWriteArguments(sequenceNumber, callback));
|
|
266
275
|
const dataSize = Buffer.byteLength(data, 'utf8');
|
|
267
276
|
assert(dataSize <= 64 * 1024 * 1024, 'Document is too large! Maximum is 64 MB');
|
|
268
277
|
|
|
269
278
|
const dataPosition = this.size;
|
|
270
|
-
if (
|
|
279
|
+
if (this.shouldWriteUnbuffered(dataSize)) {
|
|
271
280
|
this.size += this.writeUnbuffered(data, dataSize, sequenceNumber, callback);
|
|
272
281
|
} else {
|
|
273
282
|
this.size += this.writeBuffered(data, dataSize, sequenceNumber, callback);
|
|
@@ -300,15 +309,16 @@ class WritablePartition extends ReadablePartition {
|
|
|
300
309
|
*
|
|
301
310
|
* @protected
|
|
302
311
|
* @param {number} position The position in the file to prepare the read buffer for reading before.
|
|
312
|
+
* @param {number} [size] The amount of bytes that need to be buffered before position. By default, only guarantees that the document footer can be read.
|
|
303
313
|
* @returns {object} A reader object with properties `buffer`, `cursor` and `length`.
|
|
304
314
|
*/
|
|
305
|
-
prepareReadBufferBackwards(position) {
|
|
315
|
+
prepareReadBufferBackwards(position, size = 0) {
|
|
306
316
|
const bufferPos = this.size - this.writeBufferCursor;
|
|
307
317
|
// Handle the case when data that is still in write buffer is supposed to be read backwards
|
|
308
318
|
if (this.dirtyReads && this.writeBufferCursor > 0 && position > bufferPos) {
|
|
309
319
|
return { buffer: this.writeBuffer, cursor: position - bufferPos, length: this.writeBufferCursor };
|
|
310
320
|
}
|
|
311
|
-
return super.prepareReadBufferBackwards(position);
|
|
321
|
+
return super.prepareReadBufferBackwards(position, size);
|
|
312
322
|
}
|
|
313
323
|
|
|
314
324
|
/**
|
|
@@ -317,16 +327,17 @@ class WritablePartition extends ReadablePartition {
|
|
|
317
327
|
*
|
|
318
328
|
* @api
|
|
319
329
|
* @param {number} [before] The document position to start reading backward from.
|
|
320
|
-
* @
|
|
330
|
+
* @param {object|null} [headerOut] Optional object to populate with document header fields on each yield.
|
|
331
|
+
* @returns {Generator<Buffer>} A generator that returns all documents in this partition in reverse order.
|
|
321
332
|
*/
|
|
322
|
-
*readAllBackwards(before = -1) {
|
|
333
|
+
*readAllBackwards(before = -1, headerOut = null) {
|
|
323
334
|
if (!this.dirtyReads && this.writeBufferCursor > 0) {
|
|
324
335
|
const flushedSize = this.size - this.writeBufferCursor;
|
|
325
336
|
const clampedBefore = before < 0 ? flushedSize : Math.min(before, flushedSize);
|
|
326
|
-
yield* super.readAllBackwards(clampedBefore);
|
|
337
|
+
yield* super.readAllBackwards(clampedBefore, headerOut);
|
|
327
338
|
return;
|
|
328
339
|
}
|
|
329
|
-
yield* super.readAllBackwards(before);
|
|
340
|
+
yield* super.readAllBackwards(before, headerOut);
|
|
330
341
|
}
|
|
331
342
|
|
|
332
343
|
/**
|
|
@@ -388,9 +399,7 @@ class WritablePartition extends ReadablePartition {
|
|
|
388
399
|
try {
|
|
389
400
|
this.readFrom(after);
|
|
390
401
|
} catch (e) {
|
|
391
|
-
|
|
392
|
-
throw new Error('Can only truncate on valid document boundaries.');
|
|
393
|
-
}
|
|
402
|
+
assert(e instanceof CorruptFileError, 'Can only truncate on valid document boundaries.');
|
|
394
403
|
}
|
|
395
404
|
|
|
396
405
|
fs.truncateSync(this.fileName, this.headerSize + after);
|
|
@@ -23,7 +23,7 @@ class ReadOnlyStorage extends ReadableStorage {
|
|
|
23
23
|
* @returns {boolean}
|
|
24
24
|
*/
|
|
25
25
|
storageFilesFilter(filename) {
|
|
26
|
-
return filename.
|
|
26
|
+
return !filename.endsWith('.branch') && filename.substring(0, this.storageFile.length) === this.storageFile;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
/**
|
|
@@ -46,8 +46,8 @@ class ReadOnlyStorage extends ReadableStorage {
|
|
|
46
46
|
* @param {string} filename
|
|
47
47
|
*/
|
|
48
48
|
onStorageFileChanged(filename) {
|
|
49
|
-
if (filename.
|
|
50
|
-
const indexName = filename.
|
|
49
|
+
if (filename.endsWith('.index')) {
|
|
50
|
+
const indexName = filename.substring(this.storageFile.length + 1, filename.length - 6);
|
|
51
51
|
// New indexes are not automatically opened in the reader
|
|
52
52
|
this.emit('index-created', indexName);
|
|
53
53
|
return;
|