@git-stunts/git-warp 10.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +16 -0
- package/README.md +480 -0
- package/SECURITY.md +30 -0
- package/bin/git-warp +24 -0
- package/bin/warp-graph.js +1574 -0
- package/index.d.ts +2366 -0
- package/index.js +180 -0
- package/package.json +129 -0
- package/scripts/install-git-warp.sh +258 -0
- package/scripts/uninstall-git-warp.sh +139 -0
- package/src/domain/WarpGraph.js +3157 -0
- package/src/domain/crdt/Dot.js +160 -0
- package/src/domain/crdt/LWW.js +154 -0
- package/src/domain/crdt/ORSet.js +371 -0
- package/src/domain/crdt/VersionVector.js +222 -0
- package/src/domain/entities/GraphNode.js +60 -0
- package/src/domain/errors/EmptyMessageError.js +47 -0
- package/src/domain/errors/ForkError.js +30 -0
- package/src/domain/errors/IndexError.js +23 -0
- package/src/domain/errors/OperationAbortedError.js +22 -0
- package/src/domain/errors/QueryError.js +39 -0
- package/src/domain/errors/SchemaUnsupportedError.js +17 -0
- package/src/domain/errors/ShardCorruptionError.js +56 -0
- package/src/domain/errors/ShardLoadError.js +57 -0
- package/src/domain/errors/ShardValidationError.js +61 -0
- package/src/domain/errors/StorageError.js +57 -0
- package/src/domain/errors/SyncError.js +30 -0
- package/src/domain/errors/TraversalError.js +23 -0
- package/src/domain/errors/WarpError.js +31 -0
- package/src/domain/errors/WormholeError.js +28 -0
- package/src/domain/errors/WriterError.js +39 -0
- package/src/domain/errors/index.js +21 -0
- package/src/domain/services/AnchorMessageCodec.js +99 -0
- package/src/domain/services/BitmapIndexBuilder.js +225 -0
- package/src/domain/services/BitmapIndexReader.js +435 -0
- package/src/domain/services/BoundaryTransitionRecord.js +463 -0
- package/src/domain/services/CheckpointMessageCodec.js +147 -0
- package/src/domain/services/CheckpointSerializerV5.js +281 -0
- package/src/domain/services/CheckpointService.js +384 -0
- package/src/domain/services/CommitDagTraversalService.js +156 -0
- package/src/domain/services/DagPathFinding.js +712 -0
- package/src/domain/services/DagTopology.js +239 -0
- package/src/domain/services/DagTraversal.js +245 -0
- package/src/domain/services/Frontier.js +108 -0
- package/src/domain/services/GCMetrics.js +101 -0
- package/src/domain/services/GCPolicy.js +122 -0
- package/src/domain/services/GitLogParser.js +205 -0
- package/src/domain/services/HealthCheckService.js +246 -0
- package/src/domain/services/HookInstaller.js +326 -0
- package/src/domain/services/HttpSyncServer.js +262 -0
- package/src/domain/services/IndexRebuildService.js +426 -0
- package/src/domain/services/IndexStalenessChecker.js +103 -0
- package/src/domain/services/JoinReducer.js +582 -0
- package/src/domain/services/KeyCodec.js +113 -0
- package/src/domain/services/LegacyAnchorDetector.js +67 -0
- package/src/domain/services/LogicalTraversal.js +351 -0
- package/src/domain/services/MessageCodecInternal.js +132 -0
- package/src/domain/services/MessageSchemaDetector.js +145 -0
- package/src/domain/services/MigrationService.js +55 -0
- package/src/domain/services/ObserverView.js +265 -0
- package/src/domain/services/PatchBuilderV2.js +669 -0
- package/src/domain/services/PatchMessageCodec.js +140 -0
- package/src/domain/services/ProvenanceIndex.js +337 -0
- package/src/domain/services/ProvenancePayload.js +242 -0
- package/src/domain/services/QueryBuilder.js +835 -0
- package/src/domain/services/StateDiff.js +300 -0
- package/src/domain/services/StateSerializerV5.js +156 -0
- package/src/domain/services/StreamingBitmapIndexBuilder.js +709 -0
- package/src/domain/services/SyncProtocol.js +593 -0
- package/src/domain/services/TemporalQuery.js +201 -0
- package/src/domain/services/TranslationCost.js +221 -0
- package/src/domain/services/TraversalService.js +8 -0
- package/src/domain/services/WarpMessageCodec.js +29 -0
- package/src/domain/services/WarpStateIndexBuilder.js +127 -0
- package/src/domain/services/WormholeService.js +353 -0
- package/src/domain/types/TickReceipt.js +285 -0
- package/src/domain/types/WarpTypes.js +209 -0
- package/src/domain/types/WarpTypesV2.js +200 -0
- package/src/domain/utils/CachedValue.js +140 -0
- package/src/domain/utils/EventId.js +89 -0
- package/src/domain/utils/LRUCache.js +112 -0
- package/src/domain/utils/MinHeap.js +114 -0
- package/src/domain/utils/RefLayout.js +280 -0
- package/src/domain/utils/WriterId.js +205 -0
- package/src/domain/utils/cancellation.js +33 -0
- package/src/domain/utils/canonicalStringify.js +42 -0
- package/src/domain/utils/defaultClock.js +20 -0
- package/src/domain/utils/defaultCodec.js +51 -0
- package/src/domain/utils/nullLogger.js +21 -0
- package/src/domain/utils/roaring.js +181 -0
- package/src/domain/utils/shardVersion.js +9 -0
- package/src/domain/warp/PatchSession.js +217 -0
- package/src/domain/warp/Writer.js +181 -0
- package/src/hooks/post-merge.sh +60 -0
- package/src/infrastructure/adapters/BunHttpAdapter.js +225 -0
- package/src/infrastructure/adapters/ClockAdapter.js +57 -0
- package/src/infrastructure/adapters/ConsoleLogger.js +150 -0
- package/src/infrastructure/adapters/DenoHttpAdapter.js +230 -0
- package/src/infrastructure/adapters/GitGraphAdapter.js +787 -0
- package/src/infrastructure/adapters/GlobalClockAdapter.js +5 -0
- package/src/infrastructure/adapters/NoOpLogger.js +62 -0
- package/src/infrastructure/adapters/NodeCryptoAdapter.js +32 -0
- package/src/infrastructure/adapters/NodeHttpAdapter.js +98 -0
- package/src/infrastructure/adapters/PerformanceClockAdapter.js +5 -0
- package/src/infrastructure/adapters/WebCryptoAdapter.js +121 -0
- package/src/infrastructure/codecs/CborCodec.js +384 -0
- package/src/ports/BlobPort.js +30 -0
- package/src/ports/ClockPort.js +25 -0
- package/src/ports/CodecPort.js +25 -0
- package/src/ports/CommitPort.js +114 -0
- package/src/ports/ConfigPort.js +31 -0
- package/src/ports/CryptoPort.js +38 -0
- package/src/ports/GraphPersistencePort.js +57 -0
- package/src/ports/HttpServerPort.js +25 -0
- package/src/ports/IndexStoragePort.js +39 -0
- package/src/ports/LoggerPort.js +68 -0
- package/src/ports/RefPort.js +51 -0
- package/src/ports/TreePort.js +51 -0
- package/src/visualization/index.js +26 -0
- package/src/visualization/layouts/converters.js +75 -0
- package/src/visualization/layouts/elkAdapter.js +86 -0
- package/src/visualization/layouts/elkLayout.js +95 -0
- package/src/visualization/layouts/index.js +29 -0
- package/src/visualization/renderers/ascii/box.js +16 -0
- package/src/visualization/renderers/ascii/check.js +271 -0
- package/src/visualization/renderers/ascii/colors.js +13 -0
- package/src/visualization/renderers/ascii/formatters.js +73 -0
- package/src/visualization/renderers/ascii/graph.js +344 -0
- package/src/visualization/renderers/ascii/history.js +335 -0
- package/src/visualization/renderers/ascii/index.js +14 -0
- package/src/visualization/renderers/ascii/info.js +245 -0
- package/src/visualization/renderers/ascii/materialize.js +255 -0
- package/src/visualization/renderers/ascii/path.js +240 -0
- package/src/visualization/renderers/ascii/progress.js +32 -0
- package/src/visualization/renderers/ascii/symbols.js +33 -0
- package/src/visualization/renderers/ascii/table.js +19 -0
- package/src/visualization/renderers/browser/index.js +1 -0
- package/src/visualization/renderers/svg/index.js +159 -0
- package/src/visualization/utils/ansi.js +14 -0
- package/src/visualization/utils/time.js +40 -0
- package/src/visualization/utils/truncate.js +40 -0
- package/src/visualization/utils/unicode.js +52 -0
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Checkpoint Serialization for WARP V5
|
|
3
|
+
*
|
|
4
|
+
* Provides full V5 state serialization including ORSet internals (entries + tombstones).
|
|
5
|
+
* This is the AUTHORITATIVE checkpoint format for V5 state.
|
|
6
|
+
*
|
|
7
|
+
* Key differences from StateSerializerV5:
|
|
8
|
+
* - StateSerializerV5 serializes the VISIBLE PROJECTION (for hashing)
|
|
9
|
+
* - CheckpointSerializerV5 serializes the FULL INTERNAL STATE (for resume)
|
|
10
|
+
*
|
|
11
|
+
* @module CheckpointSerializerV5
|
|
12
|
+
* @see WARP Spec Section 10 (Checkpoints)
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import defaultCodec from '../utils/defaultCodec.js';
|
|
16
|
+
import { orsetSerialize, orsetDeserialize } from '../crdt/ORSet.js';
|
|
17
|
+
import { vvSerialize, vvDeserialize } from '../crdt/VersionVector.js';
|
|
18
|
+
import { decodeDot } from '../crdt/Dot.js';
|
|
19
|
+
import { createEmptyStateV5 } from './JoinReducer.js';
|
|
20
|
+
|
|
21
|
+
// ============================================================================
|
|
22
|
+
// Full State Serialization (for Checkpoints)
|
|
23
|
+
// ============================================================================
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Serializes full V5 state including ORSet internals (entries + tombstones).
|
|
27
|
+
* This is the AUTHORITATIVE checkpoint format.
|
|
28
|
+
*
|
|
29
|
+
* Structure:
|
|
30
|
+
* {
|
|
31
|
+
* nodeAlive: { entries: [[element, [dots...]], ...], tombstones: [dots...] },
|
|
32
|
+
* edgeAlive: { entries: [[element, [dots...]], ...], tombstones: [dots...] },
|
|
33
|
+
* prop: [[propKey, {eventId: {...}, value: ...}], ...],
|
|
34
|
+
* observedFrontier: { writerId: counter, ... }
|
|
35
|
+
* }
|
|
36
|
+
*
|
|
37
|
+
* @param {import('./JoinReducer.js').WarpStateV5} state
|
|
38
|
+
* @param {Object} [options]
|
|
39
|
+
* @param {import('../../ports/CodecPort.js').default} [options.codec] - Codec for serialization
|
|
40
|
+
* @returns {Buffer} CBOR-encoded full state
|
|
41
|
+
*/
|
|
42
|
+
export function serializeFullStateV5(state, { codec } = {}) {
|
|
43
|
+
const c = codec || defaultCodec;
|
|
44
|
+
// Serialize ORSets using existing serialization
|
|
45
|
+
const nodeAliveObj = orsetSerialize(state.nodeAlive);
|
|
46
|
+
const edgeAliveObj = orsetSerialize(state.edgeAlive);
|
|
47
|
+
|
|
48
|
+
// Serialize props as sorted array of [key, register] pairs
|
|
49
|
+
const propArray = [];
|
|
50
|
+
for (const [key, register] of state.prop) {
|
|
51
|
+
propArray.push([key, serializeLWWRegister(register)]);
|
|
52
|
+
}
|
|
53
|
+
// Sort by key for determinism
|
|
54
|
+
propArray.sort((a, b) => {
|
|
55
|
+
const keyA = /** @type {string} */ (a[0]);
|
|
56
|
+
const keyB = /** @type {string} */ (b[0]);
|
|
57
|
+
return keyA < keyB ? -1 : keyA > keyB ? 1 : 0;
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
// Serialize observedFrontier
|
|
61
|
+
const observedFrontierObj = vvSerialize(state.observedFrontier);
|
|
62
|
+
|
|
63
|
+
// Serialize edgeBirthEvent as sorted array of [edgeKey, eventId] pairs
|
|
64
|
+
const edgeBirthArray = [];
|
|
65
|
+
if (state.edgeBirthEvent) {
|
|
66
|
+
for (const [key, eventId] of state.edgeBirthEvent) {
|
|
67
|
+
edgeBirthArray.push([key, { lamport: eventId.lamport, writerId: eventId.writerId, patchSha: eventId.patchSha, opIndex: eventId.opIndex }]);
|
|
68
|
+
}
|
|
69
|
+
edgeBirthArray.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const obj = {
|
|
73
|
+
version: 'full-v5',
|
|
74
|
+
nodeAlive: nodeAliveObj,
|
|
75
|
+
edgeAlive: edgeAliveObj,
|
|
76
|
+
prop: propArray,
|
|
77
|
+
observedFrontier: observedFrontierObj,
|
|
78
|
+
edgeBirthEvent: edgeBirthArray,
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
return c.encode(obj);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Deserializes full V5 state. Used for resume.
|
|
86
|
+
*
|
|
87
|
+
* @param {Buffer} buffer - CBOR-encoded full state
|
|
88
|
+
* @param {Object} [options]
|
|
89
|
+
* @param {import('../../ports/CodecPort.js').default} [options.codec] - Codec for deserialization
|
|
90
|
+
* @returns {import('./JoinReducer.js').WarpStateV5}
|
|
91
|
+
*/
|
|
92
|
+
// eslint-disable-next-line complexity
|
|
93
|
+
export function deserializeFullStateV5(buffer, { codec: codecOpt } = {}) {
|
|
94
|
+
const codec = codecOpt || defaultCodec;
|
|
95
|
+
// Handle null/undefined buffer before attempting decode
|
|
96
|
+
if (buffer === null || buffer === undefined) {
|
|
97
|
+
return createEmptyStateV5();
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const obj = codec.decode(buffer);
|
|
101
|
+
|
|
102
|
+
// Handle null/undefined decoded result: return empty state
|
|
103
|
+
if (obj === null || obj === undefined) {
|
|
104
|
+
return createEmptyStateV5();
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// Handle version mismatch: throw with diagnostic info
|
|
108
|
+
// Accept both 'full-v5' and missing version (for backward compatibility with pre-versioned data)
|
|
109
|
+
if (obj.version !== undefined && obj.version !== 'full-v5') {
|
|
110
|
+
throw new Error(
|
|
111
|
+
`Unsupported full state version: expected 'full-v5', got '${obj.version}'`
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return {
|
|
116
|
+
nodeAlive: orsetDeserialize(obj.nodeAlive || {}),
|
|
117
|
+
edgeAlive: orsetDeserialize(obj.edgeAlive || {}),
|
|
118
|
+
prop: deserializeProps(obj.prop),
|
|
119
|
+
observedFrontier: vvDeserialize(obj.observedFrontier || {}),
|
|
120
|
+
edgeBirthEvent: deserializeEdgeBirthEvent(obj),
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// ============================================================================
|
|
125
|
+
// AppliedVV Computation and Serialization
|
|
126
|
+
// ============================================================================
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Computes appliedVV by scanning all dots in state.
|
|
130
|
+
* Scans state.nodeAlive.entries and state.edgeAlive.entries for all dots.
|
|
131
|
+
* Returns Map<writerId, maxCounter>.
|
|
132
|
+
*
|
|
133
|
+
* CRITICAL: This scans ALL dots, including those that may be tombstoned.
|
|
134
|
+
* The appliedVV represents what operations have been applied, not what is visible.
|
|
135
|
+
*
|
|
136
|
+
* @param {import('./JoinReducer.js').WarpStateV5} state
|
|
137
|
+
* @returns {Map<string, number>} Map<writerId, maxCounter>
|
|
138
|
+
*/
|
|
139
|
+
export function computeAppliedVV(state) {
|
|
140
|
+
const vv = new Map();
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Helper to scan all dots from an ORSet and update vv with max counters.
|
|
144
|
+
* @param {import('../crdt/ORSet.js').ORSet} orset
|
|
145
|
+
*/
|
|
146
|
+
function scanORSet(orset) {
|
|
147
|
+
for (const dots of orset.entries.values()) {
|
|
148
|
+
for (const encodedDot of dots) {
|
|
149
|
+
const dot = decodeDot(encodedDot);
|
|
150
|
+
const current = vv.get(dot.writerId) || 0;
|
|
151
|
+
if (dot.counter > current) {
|
|
152
|
+
vv.set(dot.writerId, dot.counter);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// Scan nodeAlive entries
|
|
159
|
+
scanORSet(state.nodeAlive);
|
|
160
|
+
|
|
161
|
+
// Scan edgeAlive entries
|
|
162
|
+
scanORSet(state.edgeAlive);
|
|
163
|
+
|
|
164
|
+
return vv;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Serializes appliedVV to CBOR format.
|
|
169
|
+
*
|
|
170
|
+
* @param {Map<string, number>} vv - Version vector (Map<writerId, counter>)
|
|
171
|
+
* @param {Object} [options]
|
|
172
|
+
* @param {import('../../ports/CodecPort.js').default} [options.codec] - Codec for serialization
|
|
173
|
+
* @returns {Buffer} CBOR-encoded version vector
|
|
174
|
+
*/
|
|
175
|
+
export function serializeAppliedVV(vv, { codec } = {}) {
|
|
176
|
+
const c = codec || defaultCodec;
|
|
177
|
+
const obj = vvSerialize(vv);
|
|
178
|
+
return c.encode(obj);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Deserializes appliedVV from CBOR format.
|
|
183
|
+
*
|
|
184
|
+
* @param {Buffer} buffer - CBOR-encoded version vector
|
|
185
|
+
* @param {Object} [options]
|
|
186
|
+
* @param {import('../../ports/CodecPort.js').default} [options.codec] - Codec for deserialization
|
|
187
|
+
* @returns {Map<string, number>} Version vector
|
|
188
|
+
*/
|
|
189
|
+
export function deserializeAppliedVV(buffer, { codec } = {}) {
|
|
190
|
+
const c = codec || defaultCodec;
|
|
191
|
+
const obj = c.decode(buffer);
|
|
192
|
+
return vvDeserialize(obj);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// ============================================================================
|
|
196
|
+
// Helper Functions
|
|
197
|
+
// ============================================================================
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Deserializes the props array from checkpoint format.
|
|
201
|
+
* @param {Array} propArray - Array of [key, registerObj] pairs
|
|
202
|
+
* @returns {Map<string, import('../crdt/LWW.js').LWWRegister>}
|
|
203
|
+
*/
|
|
204
|
+
function deserializeProps(propArray) {
|
|
205
|
+
const prop = new Map();
|
|
206
|
+
if (propArray && Array.isArray(propArray)) {
|
|
207
|
+
for (const [key, registerObj] of propArray) {
|
|
208
|
+
prop.set(key, deserializeLWWRegister(registerObj));
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return prop;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Deserializes edge birth event data, supporting both legacy and current formats.
|
|
216
|
+
* @param {Object} obj - The decoded checkpoint object
|
|
217
|
+
* @returns {Map<string, Object>}
|
|
218
|
+
*/
|
|
219
|
+
function deserializeEdgeBirthEvent(obj) {
|
|
220
|
+
const edgeBirthEvent = new Map();
|
|
221
|
+
const birthData = obj.edgeBirthEvent || obj.edgeBirthLamport;
|
|
222
|
+
if (birthData && Array.isArray(birthData)) {
|
|
223
|
+
for (const [key, val] of birthData) {
|
|
224
|
+
if (typeof val === 'number') {
|
|
225
|
+
// Legacy format: bare lamport number → synthesize minimal EventId.
|
|
226
|
+
// Empty writerId and placeholder patchSha are sentinels indicating
|
|
227
|
+
// this EventId was reconstructed from pre-v5 data, not a real writer.
|
|
228
|
+
edgeBirthEvent.set(key, { lamport: val, writerId: '', patchSha: '0000', opIndex: 0 });
|
|
229
|
+
} else {
|
|
230
|
+
// Shallow copy to avoid sharing a reference with the decoded CBOR object
|
|
231
|
+
edgeBirthEvent.set(key, { lamport: val.lamport, writerId: val.writerId, patchSha: val.patchSha, opIndex: val.opIndex });
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
return edgeBirthEvent;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Serializes an LWW register for CBOR encoding.
|
|
240
|
+
* EventId is serialized as a plain object with sorted keys.
|
|
241
|
+
*
|
|
242
|
+
* @param {import('../crdt/LWW.js').LWWRegister} register
|
|
243
|
+
* @returns {Object}
|
|
244
|
+
*/
|
|
245
|
+
function serializeLWWRegister(register) {
|
|
246
|
+
if (!register) {
|
|
247
|
+
return null;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
return {
|
|
251
|
+
eventId: {
|
|
252
|
+
lamport: register.eventId.lamport,
|
|
253
|
+
opIndex: register.eventId.opIndex,
|
|
254
|
+
patchSha: register.eventId.patchSha,
|
|
255
|
+
writerId: register.eventId.writerId,
|
|
256
|
+
},
|
|
257
|
+
value: register.value,
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Deserializes an LWW register from CBOR.
|
|
263
|
+
*
|
|
264
|
+
* @param {Object} obj
|
|
265
|
+
* @returns {import('../crdt/LWW.js').LWWRegister}
|
|
266
|
+
*/
|
|
267
|
+
function deserializeLWWRegister(obj) {
|
|
268
|
+
if (!obj) {
|
|
269
|
+
return null;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
return {
|
|
273
|
+
eventId: {
|
|
274
|
+
lamport: obj.eventId.lamport,
|
|
275
|
+
writerId: obj.eventId.writerId,
|
|
276
|
+
patchSha: obj.eventId.patchSha,
|
|
277
|
+
opIndex: obj.eventId.opIndex,
|
|
278
|
+
},
|
|
279
|
+
value: obj.value,
|
|
280
|
+
};
|
|
281
|
+
}
|
|
@@ -0,0 +1,384 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Checkpoint Service for WARP multi-writer graph database.
|
|
3
|
+
*
|
|
4
|
+
* Provides functionality for creating and loading schema:2 and schema:3
|
|
5
|
+
* checkpoints, as well as incremental state materialization from checkpoints.
|
|
6
|
+
*
|
|
7
|
+
* This service supports schema:2 and schema:3 (V5) checkpoints. Schema:1 (V4)
|
|
8
|
+
* checkpoints must be migrated before use.
|
|
9
|
+
*
|
|
10
|
+
* @module CheckpointService
|
|
11
|
+
* @see WARP Spec Section 10
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { serializeStateV5, computeStateHashV5 } from './StateSerializerV5.js';
|
|
15
|
+
import {
|
|
16
|
+
serializeFullStateV5,
|
|
17
|
+
deserializeFullStateV5,
|
|
18
|
+
computeAppliedVV,
|
|
19
|
+
serializeAppliedVV,
|
|
20
|
+
deserializeAppliedVV,
|
|
21
|
+
} from './CheckpointSerializerV5.js';
|
|
22
|
+
import { serializeFrontier, deserializeFrontier } from './Frontier.js';
|
|
23
|
+
import { encodeCheckpointMessage, decodeCheckpointMessage } from './WarpMessageCodec.js';
|
|
24
|
+
import { createORSet, orsetAdd, orsetCompact } from '../crdt/ORSet.js';
|
|
25
|
+
import { createDot } from '../crdt/Dot.js';
|
|
26
|
+
import { createVersionVector } from '../crdt/VersionVector.js';
|
|
27
|
+
import { cloneStateV5, reduceV5 } from './JoinReducer.js';
|
|
28
|
+
import { encodeEdgeKey, encodePropKey } from './KeyCodec.js';
|
|
29
|
+
import { ProvenanceIndex } from './ProvenanceIndex.js';
|
|
30
|
+
|
|
31
|
+
// ============================================================================
|
|
32
|
+
// Checkpoint Creation (WARP spec Section 10)
|
|
33
|
+
// ============================================================================
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Creates a schema:2 checkpoint commit containing serialized V5 state and frontier.
|
|
37
|
+
*
|
|
38
|
+
* Tree structure:
|
|
39
|
+
* ```
|
|
40
|
+
* <checkpoint_commit_tree>/
|
|
41
|
+
* ├── state.cbor # AUTHORITATIVE: Full V5 state (ORSets + props)
|
|
42
|
+
* ├── visible.cbor # CACHE ONLY: Visible projection for fast queries
|
|
43
|
+
* ├── frontier.cbor # Writer frontiers
|
|
44
|
+
* ├── appliedVV.cbor # Version vector of dots in state
|
|
45
|
+
* └── provenanceIndex.cbor # Optional: node-to-patchSha index (HG/IO/2)
|
|
46
|
+
* ```
|
|
47
|
+
*
|
|
48
|
+
* @param {Object} options - Checkpoint creation options
|
|
49
|
+
* @param {import('../../ports/GraphPersistencePort.js').default} options.persistence - Git persistence adapter
|
|
50
|
+
* @param {string} options.graphName - Name of the graph
|
|
51
|
+
* @param {import('./JoinReducer.js').WarpStateV5} options.state - The V5 state to checkpoint
|
|
52
|
+
* @param {import('./Frontier.js').Frontier} options.frontier - Writer frontier map
|
|
53
|
+
* @param {string[]} [options.parents=[]] - Parent commit SHAs (typically prior checkpoint or patch commits)
|
|
54
|
+
* @param {boolean} [options.compact=true] - Whether to compact tombstoned dots before saving
|
|
55
|
+
* @param {import('./ProvenanceIndex.js').ProvenanceIndex} [options.provenanceIndex] - Optional provenance index to persist
|
|
56
|
+
* @param {import('../../ports/CodecPort.js').default} [options.codec] - Codec for CBOR serialization
|
|
57
|
+
* @param {import('../../ports/CryptoPort.js').default} [options.crypto] - CryptoPort for state hash computation
|
|
58
|
+
* @returns {Promise<string>} The checkpoint commit SHA
|
|
59
|
+
*/
|
|
60
|
+
export async function create({ persistence, graphName, state, frontier, parents = [], compact = true, provenanceIndex, codec, crypto }) {
|
|
61
|
+
return await createV5({ persistence, graphName, state, frontier, parents, compact, provenanceIndex, codec, crypto });
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Creates a V5 checkpoint commit with full ORSet state.
|
|
66
|
+
*
|
|
67
|
+
* V5 Checkpoint Tree Structure:
|
|
68
|
+
* ```
|
|
69
|
+
* <checkpoint_tree>/
|
|
70
|
+
* ├── state.cbor # AUTHORITATIVE: Full V5 state (ORSets + props)
|
|
71
|
+
* ├── visible.cbor # CACHE ONLY: Visible projection for fast queries
|
|
72
|
+
* ├── frontier.cbor # Writer frontiers
|
|
73
|
+
* ├── appliedVV.cbor # Version vector of dots in state
|
|
74
|
+
* └── provenanceIndex.cbor # Optional: node-to-patchSha index (HG/IO/2)
|
|
75
|
+
* ```
|
|
76
|
+
*
|
|
77
|
+
* @param {Object} options - Checkpoint creation options
|
|
78
|
+
* @param {import('../../ports/GraphPersistencePort.js').default} options.persistence - Git persistence adapter
|
|
79
|
+
* @param {string} options.graphName - Name of the graph
|
|
80
|
+
* @param {import('./JoinReducer.js').WarpStateV5} options.state - The V5 state to checkpoint
|
|
81
|
+
* @param {import('./Frontier.js').Frontier} options.frontier - Writer frontier map
|
|
82
|
+
* @param {string[]} [options.parents=[]] - Parent commit SHAs
|
|
83
|
+
* @param {boolean} [options.compact=true] - Whether to compact tombstoned dots before saving
|
|
84
|
+
* @param {import('./ProvenanceIndex.js').ProvenanceIndex} [options.provenanceIndex] - Optional provenance index to persist
|
|
85
|
+
* @param {import('../../ports/CodecPort.js').default} [options.codec] - Codec for CBOR serialization
|
|
86
|
+
* @param {import('../../ports/CryptoPort.js').default} [options.crypto] - CryptoPort for state hash computation
|
|
87
|
+
* @returns {Promise<string>} The checkpoint commit SHA
|
|
88
|
+
*/
|
|
89
|
+
export async function createV5({
|
|
90
|
+
persistence,
|
|
91
|
+
graphName,
|
|
92
|
+
state,
|
|
93
|
+
frontier,
|
|
94
|
+
parents = [],
|
|
95
|
+
compact = true,
|
|
96
|
+
provenanceIndex,
|
|
97
|
+
codec,
|
|
98
|
+
crypto,
|
|
99
|
+
}) {
|
|
100
|
+
// 1. Compute appliedVV from actual state dots
|
|
101
|
+
const appliedVV = computeAppliedVV(state);
|
|
102
|
+
|
|
103
|
+
// 2. Optionally compact (only tombstoned dots <= appliedVV)
|
|
104
|
+
let checkpointState = state;
|
|
105
|
+
if (compact) {
|
|
106
|
+
checkpointState = cloneStateV5(state);
|
|
107
|
+
orsetCompact(checkpointState.nodeAlive, appliedVV);
|
|
108
|
+
orsetCompact(checkpointState.edgeAlive, appliedVV);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// 3. Serialize full state (AUTHORITATIVE)
|
|
112
|
+
const stateBuffer = serializeFullStateV5(checkpointState, { codec });
|
|
113
|
+
|
|
114
|
+
// 4. Serialize visible projection (CACHE)
|
|
115
|
+
const visibleBuffer = serializeStateV5(checkpointState, { codec });
|
|
116
|
+
const stateHash = await computeStateHashV5(checkpointState, { codec, crypto });
|
|
117
|
+
|
|
118
|
+
// 5. Serialize frontier and appliedVV
|
|
119
|
+
const frontierBuffer = serializeFrontier(frontier, { codec });
|
|
120
|
+
const appliedVVBuffer = serializeAppliedVV(appliedVV, { codec });
|
|
121
|
+
|
|
122
|
+
// 6. Write blobs to git
|
|
123
|
+
const stateBlobOid = await persistence.writeBlob(stateBuffer);
|
|
124
|
+
const visibleBlobOid = await persistence.writeBlob(visibleBuffer);
|
|
125
|
+
const frontierBlobOid = await persistence.writeBlob(frontierBuffer);
|
|
126
|
+
const appliedVVBlobOid = await persistence.writeBlob(appliedVVBuffer);
|
|
127
|
+
|
|
128
|
+
// 6b. Optionally serialize and write provenance index
|
|
129
|
+
let provenanceIndexBlobOid = null;
|
|
130
|
+
if (provenanceIndex) {
|
|
131
|
+
const provenanceIndexBuffer = provenanceIndex.serialize({ codec });
|
|
132
|
+
provenanceIndexBlobOid = await persistence.writeBlob(provenanceIndexBuffer);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// 7. Create tree with sorted entries
|
|
136
|
+
const treeEntries = [
|
|
137
|
+
`100644 blob ${appliedVVBlobOid}\tappliedVV.cbor`,
|
|
138
|
+
`100644 blob ${frontierBlobOid}\tfrontier.cbor`,
|
|
139
|
+
`100644 blob ${stateBlobOid}\tstate.cbor`,
|
|
140
|
+
`100644 blob ${visibleBlobOid}\tvisible.cbor`,
|
|
141
|
+
];
|
|
142
|
+
|
|
143
|
+
// Add provenance index if present
|
|
144
|
+
if (provenanceIndexBlobOid) {
|
|
145
|
+
treeEntries.push(`100644 blob ${provenanceIndexBlobOid}\tprovenanceIndex.cbor`);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Sort entries by filename for deterministic tree (git requires sorted entries by path)
|
|
149
|
+
treeEntries.sort((a, b) => {
|
|
150
|
+
const filenameA = a.split('\t')[1];
|
|
151
|
+
const filenameB = b.split('\t')[1];
|
|
152
|
+
return filenameA.localeCompare(filenameB);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
const treeOid = await persistence.writeTree(treeEntries);
|
|
156
|
+
|
|
157
|
+
// 8. Create checkpoint commit message with v5 trailer
|
|
158
|
+
const message = encodeCheckpointMessage({
|
|
159
|
+
graph: graphName,
|
|
160
|
+
stateHash,
|
|
161
|
+
frontierOid: frontierBlobOid,
|
|
162
|
+
indexOid: treeOid,
|
|
163
|
+
schema: 2,
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
// 9. Create the checkpoint commit
|
|
167
|
+
const checkpointSha = await persistence.commitNodeWithTree({
|
|
168
|
+
treeOid,
|
|
169
|
+
parents,
|
|
170
|
+
message,
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
return checkpointSha;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// ============================================================================
|
|
177
|
+
// Checkpoint Loading
|
|
178
|
+
// ============================================================================
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Loads a schema:2 checkpoint from a commit SHA.
|
|
182
|
+
*
|
|
183
|
+
* Reads the checkpoint commit, extracts the tree entries,
|
|
184
|
+
* and deserializes the V5 state and frontier.
|
|
185
|
+
*
|
|
186
|
+
* Loads state.cbor as AUTHORITATIVE full ORSet state
|
|
187
|
+
* (NEVER uses visible.cbor for resume - it's cache only)
|
|
188
|
+
*
|
|
189
|
+
* Schema:1 checkpoints are not supported and will throw an error.
|
|
190
|
+
* Use MigrationService to upgrade schema:1 checkpoints first.
|
|
191
|
+
*
|
|
192
|
+
* @param {import('../../ports/GraphPersistencePort.js').default} persistence - Git persistence adapter
|
|
193
|
+
* @param {string} checkpointSha - The checkpoint commit SHA to load
|
|
194
|
+
* @param {Object} [options] - Load options
|
|
195
|
+
* @param {import('../../ports/CodecPort.js').default} [options.codec] - Codec for CBOR deserialization
|
|
196
|
+
* @returns {Promise<{state: import('./JoinReducer.js').WarpStateV5, frontier: import('./Frontier.js').Frontier, stateHash: string, schema: number, appliedVV?: Map<string, number>, provenanceIndex?: import('./ProvenanceIndex.js').ProvenanceIndex}>} The loaded checkpoint data
|
|
197
|
+
* @throws {Error} If checkpoint is schema:1 (migration required)
|
|
198
|
+
*/
|
|
199
|
+
export async function loadCheckpoint(persistence, checkpointSha, { codec } = {}) {
|
|
200
|
+
// 1. Read commit message and decode
|
|
201
|
+
const message = await persistence.showNode(checkpointSha);
|
|
202
|
+
const decoded = decodeCheckpointMessage(message);
|
|
203
|
+
|
|
204
|
+
// 2. Reject schema:1 checkpoints - migration required
|
|
205
|
+
if (decoded.schema !== 2 && decoded.schema !== 3) {
|
|
206
|
+
throw new Error(
|
|
207
|
+
`Checkpoint ${checkpointSha} is schema:${decoded.schema}. ` +
|
|
208
|
+
`Only schema:2 and schema:3 checkpoints are supported. Please migrate using MigrationService.`
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// 3. Read tree entries via the indexOid from the message (points to the tree)
|
|
213
|
+
const treeOids = await persistence.readTreeOids(decoded.indexOid);
|
|
214
|
+
|
|
215
|
+
// 4. Read frontier.cbor blob
|
|
216
|
+
const frontierOid = treeOids['frontier.cbor'];
|
|
217
|
+
if (!frontierOid) {
|
|
218
|
+
throw new Error(`Checkpoint ${checkpointSha} missing frontier.cbor in tree`);
|
|
219
|
+
}
|
|
220
|
+
const frontierBuffer = await persistence.readBlob(frontierOid);
|
|
221
|
+
const frontier = deserializeFrontier(frontierBuffer, { codec });
|
|
222
|
+
|
|
223
|
+
// 5. Read state.cbor blob and deserialize as V5 full state
|
|
224
|
+
const stateOid = treeOids['state.cbor'];
|
|
225
|
+
if (!stateOid) {
|
|
226
|
+
throw new Error(`Checkpoint ${checkpointSha} missing state.cbor in tree`);
|
|
227
|
+
}
|
|
228
|
+
const stateBuffer = await persistence.readBlob(stateOid);
|
|
229
|
+
|
|
230
|
+
// V5: Load AUTHORITATIVE full state from state.cbor (NEVER use visible.cbor for resume)
|
|
231
|
+
const state = deserializeFullStateV5(stateBuffer, { codec });
|
|
232
|
+
|
|
233
|
+
// Load appliedVV if present
|
|
234
|
+
let appliedVV = null;
|
|
235
|
+
const appliedVVOid = treeOids['appliedVV.cbor'];
|
|
236
|
+
if (appliedVVOid) {
|
|
237
|
+
const appliedVVBuffer = await persistence.readBlob(appliedVVOid);
|
|
238
|
+
appliedVV = deserializeAppliedVV(appliedVVBuffer, { codec });
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// Load provenanceIndex if present (HG/IO/2)
|
|
242
|
+
let provenanceIndex = null;
|
|
243
|
+
const provenanceIndexOid = treeOids['provenanceIndex.cbor'];
|
|
244
|
+
if (provenanceIndexOid) {
|
|
245
|
+
const provenanceIndexBuffer = await persistence.readBlob(provenanceIndexOid);
|
|
246
|
+
provenanceIndex = ProvenanceIndex.deserialize(provenanceIndexBuffer, { codec });
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
return {
|
|
250
|
+
state,
|
|
251
|
+
frontier,
|
|
252
|
+
stateHash: decoded.stateHash,
|
|
253
|
+
schema: decoded.schema,
|
|
254
|
+
appliedVV,
|
|
255
|
+
provenanceIndex,
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// ============================================================================
|
|
260
|
+
// Incremental Materialization
|
|
261
|
+
// ============================================================================
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* Materializes V5 state incrementally from a schema:2 checkpoint.
|
|
265
|
+
*
|
|
266
|
+
* Loads the checkpoint state and frontier, then applies all patches
|
|
267
|
+
* since the checkpoint frontier to reach the target frontier.
|
|
268
|
+
*
|
|
269
|
+
* Only supports schema:2 checkpoints. Schema:1 checkpoints will cause
|
|
270
|
+
* loadCheckpoint to throw an error.
|
|
271
|
+
*
|
|
272
|
+
* @param {Object} options - Materialization options
|
|
273
|
+
* @param {import('../../ports/GraphPersistencePort.js').default} options.persistence - Git persistence adapter
|
|
274
|
+
* @param {string} options.graphName - Name of the graph
|
|
275
|
+
* @param {string} options.checkpointSha - The schema:2 checkpoint commit SHA to start from
|
|
276
|
+
* @param {import('./Frontier.js').Frontier} options.targetFrontier - The target frontier to materialize to
|
|
277
|
+
* @param {Function} options.patchLoader - Async function to load patches: (writerId, fromSha, toSha) => Array<{patch, sha}>
|
|
278
|
+
* @param {import('../../ports/CodecPort.js').default} [options.codec] - Codec for CBOR deserialization
|
|
279
|
+
* @returns {Promise<import('./JoinReducer.js').WarpStateV5>} The materialized V5 state at targetFrontier
|
|
280
|
+
* @throws {Error} If checkpoint is schema:1 (migration required)
|
|
281
|
+
* @throws {Error} If checkpoint is missing required blobs (state.cbor, frontier.cbor)
|
|
282
|
+
*/
|
|
283
|
+
export async function materializeIncremental({
|
|
284
|
+
persistence,
|
|
285
|
+
graphName: _graphName,
|
|
286
|
+
checkpointSha,
|
|
287
|
+
targetFrontier,
|
|
288
|
+
patchLoader,
|
|
289
|
+
codec,
|
|
290
|
+
}) {
|
|
291
|
+
// 1. Load checkpoint state and frontier (schema:2 returns full V5 state)
|
|
292
|
+
const checkpoint = await loadCheckpoint(persistence, checkpointSha, { codec });
|
|
293
|
+
const checkpointFrontier = checkpoint.frontier;
|
|
294
|
+
|
|
295
|
+
// 2. Use checkpoint state directly (schema:2 stores full V5 state)
|
|
296
|
+
const initialState = checkpoint.state;
|
|
297
|
+
|
|
298
|
+
// 3. Collect patches since checkpoint frontier for each writer
|
|
299
|
+
const allPatches = [];
|
|
300
|
+
|
|
301
|
+
for (const [writerId, targetSha] of targetFrontier) {
|
|
302
|
+
const cpSha = checkpointFrontier.get(writerId);
|
|
303
|
+
|
|
304
|
+
// If writer wasn't in checkpoint frontier, load all their patches up to targetSha
|
|
305
|
+
// If writer was in checkpoint, load patches from checkpoint SHA to target SHA
|
|
306
|
+
const patches = await patchLoader(writerId, cpSha || null, targetSha);
|
|
307
|
+
allPatches.push(...patches);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
// 4. If no new patches, return the checkpoint state as-is
|
|
311
|
+
if (allPatches.length === 0) {
|
|
312
|
+
return initialState;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// 5. Apply new patches using V5 reducer with checkpoint state as initial
|
|
316
|
+
const finalState = reduceV5(allPatches, initialState);
|
|
317
|
+
|
|
318
|
+
return finalState;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* Reconstructs WarpStateV5 (ORSet-based) from a checkpoint's visible projection.
|
|
323
|
+
*
|
|
324
|
+
* Creates ORSet-based state with synthetic dots for all visible elements.
|
|
325
|
+
* This is used when loading a v5 checkpoint for incremental materialization.
|
|
326
|
+
*
|
|
327
|
+
* @param {Object} visibleProjection - The checkpoint's visible projection
|
|
328
|
+
* @param {string[]} visibleProjection.nodes - Visible node IDs
|
|
329
|
+
* @param {Array<{from: string, to: string, label: string}>} visibleProjection.edges - Visible edges
|
|
330
|
+
* @param {Array<{node: string, key: string, value: *}>} visibleProjection.props - Visible properties
|
|
331
|
+
* @returns {import('./JoinReducer.js').WarpStateV5} Reconstructed WarpStateV5
|
|
332
|
+
* @public
|
|
333
|
+
*/
|
|
334
|
+
export function reconstructStateV5FromCheckpoint(visibleProjection) {
|
|
335
|
+
const { nodes, edges, props } = visibleProjection;
|
|
336
|
+
|
|
337
|
+
// Create a synthetic dot for checkpoint entries
|
|
338
|
+
// Uses a special writerId that won't conflict with real writers
|
|
339
|
+
// Counter starts at 1 (0 is invalid for dots)
|
|
340
|
+
const syntheticDot = createDot('__checkpoint__', 1);
|
|
341
|
+
|
|
342
|
+
// Create a synthetic eventId for LWW props
|
|
343
|
+
const syntheticEventId = {
|
|
344
|
+
lamport: 0,
|
|
345
|
+
writerId: '__checkpoint__',
|
|
346
|
+
patchSha: '0000000000000000000000000000000000000000',
|
|
347
|
+
opIndex: 0,
|
|
348
|
+
};
|
|
349
|
+
|
|
350
|
+
const nodeAlive = createORSet();
|
|
351
|
+
const edgeAlive = createORSet();
|
|
352
|
+
const prop = new Map();
|
|
353
|
+
const observedFrontier = createVersionVector();
|
|
354
|
+
|
|
355
|
+
// Reconstruct nodes as ORSet entries
|
|
356
|
+
for (const nodeId of nodes) {
|
|
357
|
+
orsetAdd(nodeAlive, nodeId, syntheticDot);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// Reconstruct edges as ORSet entries
|
|
361
|
+
for (const edge of edges) {
|
|
362
|
+
const edgeKey = encodeEdgeKey(edge.from, edge.to, edge.label);
|
|
363
|
+
orsetAdd(edgeAlive, edgeKey, syntheticDot);
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// Reconstruct props with LWW registers (same as v4)
|
|
367
|
+
for (const p of props) {
|
|
368
|
+
const propKey = encodePropKey(p.node, p.key);
|
|
369
|
+
prop.set(propKey, {
|
|
370
|
+
eventId: syntheticEventId,
|
|
371
|
+
value: p.value,
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
// Reconstruct edgeBirthEvent: synthetic birth at lamport 0
|
|
376
|
+
// so checkpoint-loaded props pass the visibility filter
|
|
377
|
+
const edgeBirthEvent = new Map();
|
|
378
|
+
for (const edge of edges) {
|
|
379
|
+
const edgeKey = encodeEdgeKey(edge.from, edge.to, edge.label);
|
|
380
|
+
edgeBirthEvent.set(edgeKey, { lamport: 0, writerId: '', patchSha: '0000', opIndex: 0 });
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
return { nodeAlive, edgeAlive, prop, observedFrontier, edgeBirthEvent };
|
|
384
|
+
}
|