eventsourcingdb-merkle 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/lib/backupReader.js +186 -32
- package/dist/lib/eventText.js +20 -10
- package/dist/lib/isObject.js +3 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -212,7 +212,12 @@ In both versions, the data is hashed as it appears in the backup or in the event
|
|
|
212
212
|
|
|
213
213
|
The Merkle tree is constructed by treating event hashes, as they are stored, prefix included, as leaf nodes. Internal nodes are created by hashing the concatenation of their child hashes. If the number of nodes at any level is odd, the last node is duplicated to create a complete binary tree. This process continues until a single root hash remains.
|
|
214
214
|
|
|
215
|
-
Backup files are expected to be in the NDJSON
|
|
215
|
+
Backup files are expected to be in one of the two NDJSON formats that EventSourcingDB writes, and the tool reads every backup that a restore of EventSourcingDB takes. It checks the structure of a backup, the hashes, and the chain, but neither the values of the events nor the schemas, which a restore checks with the rules of the server:
|
|
216
|
+
|
|
217
|
+
- A backup of the v1 API, which EventSourcingDB writes on `/api/v1/backup`, holds a JSON object with a `type` and a `payload` field on each line. A line of the type `event` holds a CloudEvents-compliant event and its hash. A line of the type `schema` holds the schema of an event type, which the tool skips, since neither the hashes nor the chain cover it.
|
|
218
|
+
- A backup of the v2 API, which EventSourcingDB writes on `/api/v2/management/create-backup` from release 1.3 on, holds a JSON object with a `type` and a `data` field on each line. It begins with a line of the type `backup`, which names the version of its format, and ends with a line of the type `end`. An incremental backup follows on the last event of the backup before it. To verify one, append it to the full backup and the increments before it, in their order, as a restore takes them, for example with `cat full.ndjson increment-1.ndjson increment-2.ndjson`: across them, the chain is the ordinary chain of the events.
|
|
219
|
+
|
|
220
|
+
A backup that holds a line of the type `error` was cut short, and the tool refuses it, as it refuses a line of any type that its format does not know, a line that is not valid JSON, and a backup of the v2 API whose structure a restore would refuse, naming the line.
|
|
216
221
|
|
|
217
222
|
## Running Quality Assurance
|
|
218
223
|
|
package/dist/lib/backupReader.js
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
import { createReadStream } from 'node:fs';
|
|
2
2
|
import { access } from 'node:fs/promises';
|
|
3
3
|
import { createInterface } from 'node:readline';
|
|
4
|
-
import {
|
|
4
|
+
import { isObject } from './isObject.js';
|
|
5
|
+
import { getCompactValue, getRawValue } from './rawJson.js';
|
|
6
|
+
// The version of the format in which the v2 API writes a backup. The backups
|
|
7
|
+
// of the v1 API carry no version of their own.
|
|
8
|
+
const backupFormatVersion = 2;
|
|
9
|
+
// An event ID is an integer, which EventSourcingDB reads with an optional sign.
|
|
10
|
+
const eventIdPattern = /^[+-]?\d+$/u;
|
|
5
11
|
export const readBackupFile = async (filePath) => {
|
|
6
12
|
// Check if file exists and is readable
|
|
7
13
|
try {
|
|
@@ -16,62 +22,210 @@ export const readBackupFile = async (filePath) => {
|
|
|
16
22
|
input: fileStream,
|
|
17
23
|
crlfDelay: Number.POSITIVE_INFINITY,
|
|
18
24
|
});
|
|
25
|
+
let formatReader;
|
|
19
26
|
let lineNumber = 0;
|
|
20
|
-
for await (const
|
|
27
|
+
for await (const text of rl) {
|
|
21
28
|
lineNumber++;
|
|
22
|
-
if (
|
|
29
|
+
if (text.trim() === '') {
|
|
23
30
|
continue;
|
|
24
31
|
}
|
|
25
|
-
const
|
|
32
|
+
const line = parseLine(text, lineNumber);
|
|
33
|
+
// A server sends heartbeats while it reads a large store, and they carry
|
|
34
|
+
// nothing to verify.
|
|
35
|
+
if (line.type === 'heartbeat') {
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
// A backup that the v1 API wrote begins with a line of its own rather than
|
|
39
|
+
// with a backup line, and holds payloads rather than data, which is how
|
|
40
|
+
// the restore of EventSourcingDB tells the two formats apart as well.
|
|
41
|
+
formatReader ??=
|
|
42
|
+
line.type !== 'backup' && line.payload !== undefined ? createV1Reader() : createV2Reader();
|
|
43
|
+
const entry = formatReader.readLine(line);
|
|
26
44
|
if (entry !== undefined) {
|
|
27
45
|
entries.push(entry);
|
|
28
46
|
}
|
|
29
47
|
}
|
|
48
|
+
formatReader?.finish();
|
|
30
49
|
return entries;
|
|
31
50
|
};
|
|
32
|
-
|
|
33
|
-
// for a line that holds a schema, which neither the hashes nor the chain cover.
|
|
34
|
-
// Every other line ends the reading: an error line means that the backup was
|
|
35
|
-
// cut short, and a line of a type the format does not know means that the file
|
|
36
|
-
// is no backup the tool can read. Skipping either would let the events before
|
|
37
|
-
// it pass as a complete, valid chain.
|
|
38
|
-
const readLine = (line, lineNumber) => {
|
|
51
|
+
const parseLine = (text, lineNumber) => {
|
|
39
52
|
let parsed;
|
|
40
53
|
try {
|
|
41
|
-
parsed = JSON.parse(
|
|
54
|
+
parsed = JSON.parse(text);
|
|
42
55
|
}
|
|
43
56
|
catch (error) {
|
|
44
57
|
throw new Error(`line ${lineNumber} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
|
|
45
58
|
}
|
|
46
|
-
const { type, payload } = (
|
|
59
|
+
const { type, payload, data } = isObject(parsed) ? parsed : {};
|
|
47
60
|
if (typeof type !== 'string') {
|
|
48
61
|
throw new Error(`line ${lineNumber} has no type`);
|
|
49
62
|
}
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
63
|
+
return { text, number: lineNumber, type, payload, data };
|
|
64
|
+
};
|
|
65
|
+
// A backup of the v1 API holds a line for every schema, which neither the hashes
|
|
66
|
+
// nor the chain cover, and a line for every event. Every other line ends the
|
|
67
|
+
// reading: an error line means that the backup was cut short, and a line of a
|
|
68
|
+
// type the format does not know means that the file is no backup the tool can
|
|
69
|
+
// read. Skipping either would let the events before it pass as a complete,
|
|
70
|
+
// valid chain.
|
|
71
|
+
const createV1Reader = () => {
|
|
72
|
+
return {
|
|
73
|
+
readLine: line => {
|
|
74
|
+
switch (line.type) {
|
|
75
|
+
case 'schema':
|
|
76
|
+
return undefined;
|
|
77
|
+
case 'error':
|
|
78
|
+
throw new Error(`the backup is incomplete: line ${line.number} reports an error: ${typeof line.payload === 'string' ? line.payload : JSON.stringify(line.payload)}`);
|
|
79
|
+
case 'event':
|
|
80
|
+
return readV1EventLine(line);
|
|
81
|
+
default:
|
|
82
|
+
throw new Error(`line ${line.number} has an unknown type: ${line.type}`);
|
|
83
|
+
}
|
|
84
|
+
},
|
|
85
|
+
finish: () => {
|
|
86
|
+
// The format has no line that ends it, so a backup that was cut short
|
|
87
|
+
// without an error line cannot be told from a complete one.
|
|
88
|
+
},
|
|
89
|
+
};
|
|
60
90
|
};
|
|
61
|
-
const
|
|
62
|
-
const { event, hash } = (
|
|
63
|
-
if (
|
|
64
|
-
throw new Error(`line ${
|
|
91
|
+
const readV1EventLine = (line) => {
|
|
92
|
+
const { event, hash } = isObject(line.payload) ? line.payload : {};
|
|
93
|
+
if (!isObject(event)) {
|
|
94
|
+
throw new Error(`line ${line.number} has an event line without an event`);
|
|
65
95
|
}
|
|
66
96
|
if (typeof hash !== 'string') {
|
|
67
|
-
throw new Error(`line ${
|
|
97
|
+
throw new Error(`line ${line.number} has an event line without a hash`);
|
|
68
98
|
}
|
|
69
|
-
|
|
99
|
+
return {
|
|
70
100
|
type: 'event',
|
|
71
|
-
payload: {
|
|
101
|
+
payload: {
|
|
102
|
+
event: event,
|
|
103
|
+
hash,
|
|
104
|
+
originalDataString: getCompactValue(line.text, ['payload', 'event', 'data']),
|
|
105
|
+
},
|
|
106
|
+
};
|
|
107
|
+
};
|
|
108
|
+
// A backup of the v2 API is a sequence of parts in one file: a full backup,
|
|
109
|
+
// followed by any number of increments, each beginning with a backup line and
|
|
110
|
+
// ending with an end line, as the restore of EventSourcingDB takes them. An
|
|
111
|
+
// increment follows on the event that the part before it ended at, so across
|
|
112
|
+
// the parts the chain is the ordinary chain of the events. The reader checks the
|
|
113
|
+
// structure as the restore does, since a backup that the restore refuses holds
|
|
114
|
+
// no chain that could be restored, but leaves the values of the events and the
|
|
115
|
+
// schemas to the restore, which checks them with the rules of the server.
|
|
116
|
+
const createV2Reader = () => {
|
|
117
|
+
let partIndex = -1;
|
|
118
|
+
let isPartOpen = false;
|
|
119
|
+
let hasPartEvents = false;
|
|
120
|
+
let endedAt;
|
|
121
|
+
const readBackupLine = (line) => {
|
|
122
|
+
if (isPartOpen) {
|
|
123
|
+
throw new Error(`line ${line.number} begins a part before the part before it ended`);
|
|
124
|
+
}
|
|
125
|
+
const { lastBackedUpEvent } = isObject(line.data) ? line.data : {};
|
|
126
|
+
partIndex++;
|
|
127
|
+
// The restore reads the version as the text of a number, so it takes one
|
|
128
|
+
// in quotes as well, and counts null as a version that is left out.
|
|
129
|
+
const rawFormatVersion = getRawValue(line.text, ['data', 'formatVersion']);
|
|
130
|
+
const formatVersion = rawFormatVersion === undefined || rawFormatVersion === 'null'
|
|
131
|
+
? undefined
|
|
132
|
+
: rawFormatVersion.startsWith('"')
|
|
133
|
+
? JSON.parse(rawFormatVersion)
|
|
134
|
+
: rawFormatVersion;
|
|
135
|
+
if (formatVersion !== String(backupFormatVersion)) {
|
|
136
|
+
throw new Error(`line ${line.number} has an unsupported backup format version: ${formatVersion ?? 'none'}`);
|
|
137
|
+
}
|
|
138
|
+
// The first part is a full backup, and every later one follows on the event
|
|
139
|
+
// the part before it ended at, or on its anchor, where it held no event.
|
|
140
|
+
if (partIndex === 0 && lastBackedUpEvent !== null && lastBackedUpEvent !== undefined) {
|
|
141
|
+
throw new Error(`line ${line.number} begins an increment, but a backup has to begin with a full backup`);
|
|
142
|
+
}
|
|
143
|
+
if (partIndex > 0 && !isSameEvent(lastBackedUpEvent, endedAt)) {
|
|
144
|
+
throw new Error(`line ${line.number} begins an increment that does not follow on the part before it`);
|
|
145
|
+
}
|
|
146
|
+
isPartOpen = true;
|
|
147
|
+
hasPartEvents = false;
|
|
148
|
+
return undefined;
|
|
149
|
+
};
|
|
150
|
+
const readEventLine = (line) => {
|
|
151
|
+
if (!isObject(line.data)) {
|
|
152
|
+
throw new Error(`line ${line.number} has an event line without an event`);
|
|
153
|
+
}
|
|
154
|
+
const { hash } = line.data;
|
|
155
|
+
if (typeof hash !== 'string') {
|
|
156
|
+
throw new Error(`line ${line.number} has an event line without a hash`);
|
|
157
|
+
}
|
|
158
|
+
const event = line.data;
|
|
159
|
+
hasPartEvents = true;
|
|
160
|
+
endedAt = { id: event.id, hash };
|
|
161
|
+
return {
|
|
162
|
+
type: 'event',
|
|
163
|
+
payload: {
|
|
164
|
+
event,
|
|
165
|
+
hash,
|
|
166
|
+
originalDataString: getCompactValue(line.text, ['data', 'data']),
|
|
167
|
+
},
|
|
168
|
+
};
|
|
169
|
+
};
|
|
170
|
+
return {
|
|
171
|
+
readLine: line => {
|
|
172
|
+
if (!isPartOpen && line.type !== 'backup') {
|
|
173
|
+
throw new Error(`line ${line.number} has to begin a part with a backup line, but has the type ${line.type}`);
|
|
174
|
+
}
|
|
175
|
+
switch (line.type) {
|
|
176
|
+
case 'backup':
|
|
177
|
+
return readBackupLine(line);
|
|
178
|
+
case 'eventSchema':
|
|
179
|
+
if (hasPartEvents) {
|
|
180
|
+
throw new Error(`line ${line.number} holds a schema after the events of its part`);
|
|
181
|
+
}
|
|
182
|
+
return undefined;
|
|
183
|
+
case 'event':
|
|
184
|
+
return readEventLine(line);
|
|
185
|
+
case 'lastBackedUpEvent':
|
|
186
|
+
// The event a part ended at follows from its events or its anchor
|
|
187
|
+
// already, so the line that names it adds nothing to check.
|
|
188
|
+
return undefined;
|
|
189
|
+
case 'error': {
|
|
190
|
+
const { description } = isObject(line.data) ? line.data : {};
|
|
191
|
+
throw new Error(`the backup is incomplete: line ${line.number} reports an error: ${typeof description === 'string' ? description : JSON.stringify(line.data)}`);
|
|
192
|
+
}
|
|
193
|
+
case 'end':
|
|
194
|
+
isPartOpen = false;
|
|
195
|
+
return undefined;
|
|
196
|
+
default:
|
|
197
|
+
throw new Error(`line ${line.number} has an unknown type: ${line.type}`);
|
|
198
|
+
}
|
|
199
|
+
},
|
|
200
|
+
finish: () => {
|
|
201
|
+
if (isPartOpen) {
|
|
202
|
+
throw new Error(`the backup is incomplete: part ${partIndex} lacks its end line`);
|
|
203
|
+
}
|
|
204
|
+
},
|
|
72
205
|
};
|
|
73
|
-
|
|
74
|
-
|
|
206
|
+
};
|
|
207
|
+
// isSameEvent tells whether the anchor that an increment names, as its backup
|
|
208
|
+
// line holds it, is the event that the part before it ended at, by its ID as a
|
|
209
|
+
// number, since creating a backup takes the ID in any notation that
|
|
210
|
+
// EventSourcingDB reads, and by its hash. An anchor of null names no event, and
|
|
211
|
+
// follows on a part that ended at none, such as a full backup of an empty store.
|
|
212
|
+
// Any other anchor that does not name an event by its ID and hash follows on
|
|
213
|
+
// nothing.
|
|
214
|
+
const isSameEvent = (anchor, endedAt) => {
|
|
215
|
+
if (anchor === null || anchor === undefined) {
|
|
216
|
+
return endedAt === undefined;
|
|
217
|
+
}
|
|
218
|
+
if (endedAt === undefined || !isObject(anchor)) {
|
|
219
|
+
return false;
|
|
220
|
+
}
|
|
221
|
+
const { id, hash } = anchor;
|
|
222
|
+
if (typeof id !== 'string' || typeof hash !== 'string') {
|
|
223
|
+
return false;
|
|
224
|
+
}
|
|
225
|
+
if (!eventIdPattern.test(id) || !eventIdPattern.test(endedAt.id)) {
|
|
226
|
+
return false;
|
|
227
|
+
}
|
|
228
|
+
return BigInt(id) === BigInt(endedAt.id) && hash === endedAt.hash;
|
|
75
229
|
};
|
|
76
230
|
export const findEventById = (entries, eventId) => {
|
|
77
231
|
return entries.find(entry => entry.payload.event.id === eventId);
|
package/dist/lib/eventText.js
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
|
+
import { isObject } from './isObject.js';
|
|
1
2
|
import { getCompactValue } from './rawJson.js';
|
|
2
|
-
const isObject = (value) => {
|
|
3
|
-
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
4
|
-
};
|
|
5
3
|
const isEvent = (value) => {
|
|
6
4
|
return isObject(value) && typeof value.specversion === 'string';
|
|
7
5
|
};
|
|
8
6
|
// readEventText reads an event from text in every form one can copy it from:
|
|
9
|
-
// a line of a backup, which holds the event and its hash side by
|
|
10
|
-
// of read-events, which holds the event with its
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
7
|
+
// a line of a backup of the v1 API, which holds the event and its hash side by
|
|
8
|
+
// side, a line of read-events of the v1 API, which holds the event with its
|
|
9
|
+
// hash, a line of a backup or of read-events of the v2 API, which holds the
|
|
10
|
+
// event with its hash as data, or the event alone, with its hash or without.
|
|
11
|
+
// The data is taken from the text rather than from what JSON.parse makes of
|
|
12
|
+
// it, since the hash covers it as EventSourcingDB wrote it. Text in any other
|
|
13
|
+
// form gives undefined, a line whose event or hash is missing included, since
|
|
14
|
+
// a line carries no spec version of its own and so is never taken for an event
|
|
15
|
+
// alone.
|
|
16
16
|
export const readEventText = (text) => {
|
|
17
17
|
const parsed = JSON.parse(text);
|
|
18
18
|
if (isObject(parsed) && parsed.type === 'event' && isObject(parsed.payload)) {
|
|
@@ -32,6 +32,16 @@ export const readEventText = (text) => {
|
|
|
32
32
|
};
|
|
33
33
|
}
|
|
34
34
|
}
|
|
35
|
+
if (isObject(parsed) && parsed.type === 'event' && isEvent(parsed.data)) {
|
|
36
|
+
const { data } = parsed;
|
|
37
|
+
if (typeof data.hash === 'string') {
|
|
38
|
+
return {
|
|
39
|
+
event: data,
|
|
40
|
+
storedHash: data.hash,
|
|
41
|
+
originalDataString: getCompactValue(text, ['data', 'data']),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
}
|
|
35
45
|
if (isEvent(parsed)) {
|
|
36
46
|
return {
|
|
37
47
|
event: parsed,
|