event-storage 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/index.js +1 -1
- package/package.json +3 -4
- package/src/Consumer.js +9 -6
- package/src/EventStore.js +336 -80
- package/src/EventStream.js +73 -11
- package/src/Index/ReadableIndex.js +7 -5
- package/src/Index/WritableIndex.js +4 -4
- package/src/IndexMatcher.js +205 -0
- package/src/JoinEventStream.js +44 -46
- package/src/Partition/ReadablePartition.js +153 -85
- package/src/Partition/WritablePartition.js +37 -26
- package/src/PartitionPool.js +149 -0
- package/src/Storage/ReadOnlyStorage.js +5 -5
- package/src/Storage/ReadableStorage.js +203 -140
- package/src/Storage/WritableStorage.js +81 -45
- package/src/Watcher.js +1 -1
- package/src/utils/fsUtil.js +123 -0
- package/src/utils/jsonUtil.js +87 -0
- package/src/utils/metadataUtil.js +247 -0
- package/src/utils/util.js +202 -0
- package/src/metadataUtil.js +0 -79
- package/src/util.js +0 -218
package/src/util.js
DELETED
|
@@ -1,218 +0,0 @@
|
|
|
1
|
-
import fs from 'fs';
|
|
2
|
-
import { mkdirpSync } from 'mkdirp';
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* Assert that actual and expected match or throw an Error with the given message appended by information about expected and actual value.
|
|
6
|
-
*
|
|
7
|
-
* @param {*} actual
|
|
8
|
-
* @param {*} expected
|
|
9
|
-
* @param {string} message
|
|
10
|
-
*/
|
|
11
|
-
function assertEqual(actual, expected, message) {
|
|
12
|
-
if (actual !== expected) {
|
|
13
|
-
throw new Error(message + (message ? ' ' : '') + `Expected "${expected}" but got "${actual}".`);
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
/**
|
|
18
|
-
* Assert that the condition holds and if not, throw an error with the given message.
|
|
19
|
-
*
|
|
20
|
-
* @param {boolean} condition
|
|
21
|
-
* @param {string} message
|
|
22
|
-
* @param {typeof Error} ErrorType
|
|
23
|
-
*/
|
|
24
|
-
function assert(condition, message, ErrorType = Error) {
|
|
25
|
-
if (!condition) {
|
|
26
|
-
throw new ErrorType(message);
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Return the amount required to align value to the given alignment.
|
|
32
|
-
* It calculates the difference of the alignment and the modulo of value by alignment.
|
|
33
|
-
* @param {number} value
|
|
34
|
-
* @param {number} alignment
|
|
35
|
-
* @returns {number}
|
|
36
|
-
*/
|
|
37
|
-
function alignTo(value, alignment) {
|
|
38
|
-
return (alignment - (value % alignment)) % alignment;
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
/**
|
|
42
|
-
* Method for hashing a string (e.g. a partition name) to a 32-bit unsigned integer.
|
|
43
|
-
*
|
|
44
|
-
* @param {string} str
|
|
45
|
-
* @returns {number}
|
|
46
|
-
*/
|
|
47
|
-
function hash(str) {
|
|
48
|
-
/* istanbul ignore if */
|
|
49
|
-
if (str.length === 0) {
|
|
50
|
-
return 0;
|
|
51
|
-
}
|
|
52
|
-
let hash = 5381,
|
|
53
|
-
i = str.length;
|
|
54
|
-
|
|
55
|
-
while(i) {
|
|
56
|
-
hash = ((hash << 5) + hash) ^ str.charCodeAt(--i); // jshint ignore:line
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
/* JavaScript does bitwise operations (like XOR, above) on 32-bit signed
|
|
60
|
-
* integers. Since we want the results to be always positive, convert the
|
|
61
|
-
* signed int to an unsigned by doing an unsigned bitshift. */
|
|
62
|
-
return hash >>> 0; // jshint ignore:line
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
/**
|
|
66
|
-
* Build a buffer containing the file magic header and a JSON stringified metadata block, padded to be a multiple of 16 bytes long.
|
|
67
|
-
*
|
|
68
|
-
* @param {string} magic
|
|
69
|
-
* @param {object} metadata
|
|
70
|
-
* @returns {Buffer} A buffer containing the header data
|
|
71
|
-
*/
|
|
72
|
-
function buildMetadataHeader(magic, metadata) {
|
|
73
|
-
assertEqual(magic.length, 8, 'The header magic bytes length is wrong.');
|
|
74
|
-
let metadataString = JSON.stringify(metadata);
|
|
75
|
-
let metadataSize = Buffer.byteLength(metadataString, 'utf8');
|
|
76
|
-
// 8 byte MAGIC, 4 byte metadata size, 1 byte line break
|
|
77
|
-
const pad = (16 - ((8 + 4 + metadataSize + 1) % 16)) % 16;
|
|
78
|
-
metadataString += ' '.repeat(pad) + "\n";
|
|
79
|
-
metadataSize += pad + 1;
|
|
80
|
-
const metadataBuffer = Buffer.allocUnsafe(8 + 4 + metadataSize);
|
|
81
|
-
metadataBuffer.write(magic, 0, 8, 'utf8');
|
|
82
|
-
metadataBuffer.writeUInt32BE(metadataSize, 8);
|
|
83
|
-
metadataBuffer.write(metadataString, 8 + 4, metadataSize, 'utf8');
|
|
84
|
-
return metadataBuffer;
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
/**
|
|
88
|
-
* Do a binary search for number in the range 1-length with values retrieved via a provided getter.
|
|
89
|
-
*
|
|
90
|
-
* @param {number} number The value to search for
|
|
91
|
-
* @param {number} length The upper position to search up to
|
|
92
|
-
* @param {function(number)} get The getter function to retrieve the values at the specific position
|
|
93
|
-
* @returns {Array<number>} An array of the low and high position that match the searched number
|
|
94
|
-
*/
|
|
95
|
-
function binarySearch(number, length, get) {
|
|
96
|
-
let low = 1;
|
|
97
|
-
let high = length;
|
|
98
|
-
|
|
99
|
-
if (get(low) > number) {
|
|
100
|
-
return [low, 0];
|
|
101
|
-
}
|
|
102
|
-
if (get(high) < number) {
|
|
103
|
-
return [0, high];
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
while (low <= high) {
|
|
107
|
-
const mid = low + ((high - low) >> 1);
|
|
108
|
-
const value = get(mid);
|
|
109
|
-
if (value === number) {
|
|
110
|
-
return [mid, mid];
|
|
111
|
-
}
|
|
112
|
-
if (value < number) {
|
|
113
|
-
low = mid + 1;
|
|
114
|
-
} else {
|
|
115
|
-
high = mid - 1;
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
return [low, high];
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
/**
|
|
122
|
-
* @param {number} index The 1-based index position to wrap around if < 0 and check against the bounds.
|
|
123
|
-
* @param {number} length The length of the index and upper bound.
|
|
124
|
-
* @returns {number} The wrapped index position or -1 if index out of bounds.
|
|
125
|
-
*/
|
|
126
|
-
function wrapAndCheck(index, length) {
|
|
127
|
-
if (typeof index !== 'number') {
|
|
128
|
-
return -1;
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
if (index < 0) {
|
|
132
|
-
index += length + 1;
|
|
133
|
-
}
|
|
134
|
-
if (index < 1 || index > length) {
|
|
135
|
-
return -1;
|
|
136
|
-
}
|
|
137
|
-
return index;
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/**
|
|
141
|
-
* Ensure that the given directory exists.
|
|
142
|
-
* @param {string} dirName
|
|
143
|
-
* @return {boolean} true if the directory existed already
|
|
144
|
-
*/
|
|
145
|
-
function ensureDirectory(dirName) {
|
|
146
|
-
if (!fs.existsSync(dirName)) {
|
|
147
|
-
try {
|
|
148
|
-
mkdirpSync(dirName);
|
|
149
|
-
} catch (e) {
|
|
150
|
-
}
|
|
151
|
-
return false;
|
|
152
|
-
}
|
|
153
|
-
return true;
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
/**
|
|
157
|
-
* Perform a k-way merge over multiple streams, invoking a callback for each item in ascending key order.
|
|
158
|
-
* Each stream object is mutated in place by the `advance` function.
|
|
159
|
-
*
|
|
160
|
-
* @param {object[]} streams Array of stream state objects; entries are removed when exhausted.
|
|
161
|
-
* @param {function(object): number} getKey Returns the current sort key for a stream state.
|
|
162
|
-
* @param {function(object): boolean} advance Advances the stream to its next item.
|
|
163
|
-
* Returns true if the stream has more items within range, false if exhausted.
|
|
164
|
-
* @param {function(object): void} visit Called for each stream state in merged order.
|
|
165
|
-
*/
|
|
166
|
-
function kWayMerge(streams, getKey, advance, visit) {
|
|
167
|
-
while (streams.length > 0) {
|
|
168
|
-
let minIdx = 0;
|
|
169
|
-
for (let i = 1; i < streams.length; i++) {
|
|
170
|
-
if (getKey(streams[i]) < getKey(streams[minIdx])) {
|
|
171
|
-
minIdx = i;
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
visit(streams[minIdx]);
|
|
175
|
-
if (!advance(streams[minIdx])) {
|
|
176
|
-
streams.splice(minIdx, 1);
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
/**
|
|
182
|
-
* Scan a directory for files whose names match a regex pattern, calling a callback for each match.
|
|
183
|
-
* The `onEach` callback receives the first capturing group of the match (`match[1]`), or the full
|
|
184
|
-
* match (`match[0]`) when no capturing group is defined in the pattern.
|
|
185
|
-
*
|
|
186
|
-
* @param {string} directory The directory to scan.
|
|
187
|
-
* @param {RegExp} regexPattern The pattern to match file names against.
|
|
188
|
-
* @param {function(string)} onEach Called with the first capturing group (or full match) for each matching file name.
|
|
189
|
-
* @param {function(Error?)} onDone Called when the scan is complete, or with an error if one occurred.
|
|
190
|
-
*/
|
|
191
|
-
function scanForFiles(directory, regexPattern, onEach, onDone) {
|
|
192
|
-
fs.readdir(directory, (err, files) => {
|
|
193
|
-
if (err) {
|
|
194
|
-
return onDone(err);
|
|
195
|
-
}
|
|
196
|
-
let match;
|
|
197
|
-
for (let file of files) {
|
|
198
|
-
if ((match = file.match(regexPattern)) !== null) {
|
|
199
|
-
onEach(match[1] !== undefined ? match[1] : match[0]);
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
onDone(null);
|
|
203
|
-
});
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
export {
|
|
208
|
-
assert,
|
|
209
|
-
assertEqual,
|
|
210
|
-
hash,
|
|
211
|
-
wrapAndCheck,
|
|
212
|
-
binarySearch,
|
|
213
|
-
buildMetadataHeader,
|
|
214
|
-
alignTo,
|
|
215
|
-
ensureDirectory,
|
|
216
|
-
scanForFiles,
|
|
217
|
-
kWayMerge
|
|
218
|
-
};
|