@fizzyflow/wdoublesync 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -0
- package/WDoubleSync.js +450 -0
- package/index.js +27 -0
- package/package.json +34 -0
- package/test/fixture.js +84 -0
- package/test/helpers.js +66 -0
- package/test/wdoublesync.test.js +472 -0
- package/test/wdoublesync_deletion.test.js +244 -0
- package/test/wdoublesync_with_walrus.test.js +371 -0
- package/vitest.config.js +13 -0
package/README.md
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# WDoubleSync
|
|
2
|
+
|
|
3
|
+
Folder-tree delta sync on the Sui blockchain.
|
|
4
|
+
|
|
5
|
+
WDoubleSync combines the power of two. Two libraries walk into a blockchain...
|
|
6
|
+
|
|
7
|
+
- **EndlessVector** is a scalable append-only `vector<vector<u8>>` on Sui that grows beyond object size limits by automatically splitting data into history segments and offloading large items as Walrus blobs. It has built-in Seal encryption support — all stored data can be AES encrypted with Seal-wrapped keys, so only the vector owner can decrypt. This is advised for any production data.
|
|
8
|
+
- **DoubleSync** is a content-defined chunking engine that splits files at content-determined boundaries so that small edits only affect nearby chunks, deduplicates identical chunks by SHA-256, fingerprints folder trees as Merkle trees for fast skip of unchanged subtrees, and produces compact incremental diff patches that carry only the changed operations and chunks.
|
|
9
|
+
|
|
10
|
+
Together: DoubleSync builds snapshot and diff patch documents from a folder tree, and EndlessVector stores them as an ordered chain of items on Sui, giving you versioned, deduplicated, incrementally-updated folder sync on the blockchain. Both libraries work and are tested in Node.js and in the browser, and have abstract filesystem interfaces that can be backed by anything — real disk, in-memory, or any custom state layer.
|
|
11
|
+
|
|
12
|
+
**TL;DR:** git-like filesystem/folder versioning on Sui(+Walrus+Seal).
|
|
13
|
+
|
|
14
|
+
Both libraries are fully standalone and can be used independently of each other and without WDoubleSync. EndlessVector works as general-purpose scalable on-chain storage for any `vector<u8>` data, and DoubleSync works as a pure CDC diff engine for any folder sync scenario — network, disk, database, cloud, any abstraction and whatever transport you want.
|
|
15
|
+
|
|
16
|
+
## Overview
|
|
17
|
+
|
|
18
|
+
WDoubleSync stores a chain of patch documents inside an EndlessVector on Sui:
|
|
19
|
+
|
|
20
|
+
```
|
|
21
|
+
EndlessVector (on-chain)
|
|
22
|
+
item[0] full DoubleSyncPatch (initial state)
|
|
23
|
+
item[1] DoubleSyncDiffPatch (v1 -> v2, only changed chunks)
|
|
24
|
+
item[2] DoubleSyncDiffPatch (v2 -> v3)
|
|
25
|
+
...
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
The first push is always a self-contained full patch. Every subsequent push produces an incremental diff patch that carries only the operations and chunks the receiver doesn't already have. Patch type is auto-detected by magic header — no metadata needed.
|
|
29
|
+
|
|
30
|
+
EndlessVector handles all on-chain size constraints automatically (history/archive tiers, Walrus blob offloading for large payloads).
|
|
31
|
+
|
|
32
|
+
## Features
|
|
33
|
+
|
|
34
|
+
- **Incremental sync**: Content-defined chunking means only changed bytes cross the wire and land on-chain
|
|
35
|
+
- **Version history**: Every push is a new version; restore any past version by replaying the chain
|
|
36
|
+
- **Auto-detection**: Patch type (full / diff / compressed) detected from magic bytes
|
|
37
|
+
- **Compression**: Optional gzip envelope to reduce on-chain storage cost
|
|
38
|
+
- **Stateless resume**: A new sender instance rebuilds its CDC store and session state by replaying existing chain items — no local state required between sessions
|
|
39
|
+
- **Transparent blob routing**: Large patches are automatically stored as Walrus blobs by EndlessVector
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pnpm add wdoublesync
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Usage
|
|
48
|
+
|
|
49
|
+
### Sender: push folder state to chain
|
|
50
|
+
|
|
51
|
+
```js
|
|
52
|
+
import EndlessVector from 'endless_vector';
|
|
53
|
+
import { DoubleSync, DoubleSyncMemoryFolder } from 'doublesync';
|
|
54
|
+
import WDoubleSync from 'wdoublesync';
|
|
55
|
+
|
|
56
|
+
const ev = new EndlessVector({
|
|
57
|
+
suiClient,
|
|
58
|
+
id: '0x...',
|
|
59
|
+
packageId,
|
|
60
|
+
signAndExecuteTransaction,
|
|
61
|
+
walrusClient, // optional, for large patches
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
const sync = new DoubleSync({ avgSize: 8192 });
|
|
65
|
+
const w = new WDoubleSync({ endlessVector: ev, sync });
|
|
66
|
+
await w.initialize();
|
|
67
|
+
|
|
68
|
+
// Build your folder tree
|
|
69
|
+
const root = new DoubleSyncMemoryFolder('project');
|
|
70
|
+
const src = await root.addFolder('src');
|
|
71
|
+
await src.addFile('index.js', new TextEncoder().encode('console.log("hello")'));
|
|
72
|
+
await root.addFile('README.md', new TextEncoder().encode('# My Project'));
|
|
73
|
+
|
|
74
|
+
// Push to chain (first call = full patch, subsequent = diff)
|
|
75
|
+
await w.push(root);
|
|
76
|
+
|
|
77
|
+
// Edit and push again — only the diff goes on-chain
|
|
78
|
+
await src.addFile('utils.js', new TextEncoder().encode('export function add(a, b) { return a + b; }'));
|
|
79
|
+
await w.push(root);
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
### Receiver: restore folder state from chain
|
|
83
|
+
|
|
84
|
+
```js
|
|
85
|
+
const ev = new EndlessVector({ suiClient, id: '0x...' });
|
|
86
|
+
const w = new WDoubleSync({ endlessVector: ev });
|
|
87
|
+
|
|
88
|
+
// Restore latest version
|
|
89
|
+
const folder = await w.restore();
|
|
90
|
+
|
|
91
|
+
// Or restore a specific version
|
|
92
|
+
const v1 = await w.restore(1);
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### Compression
|
|
96
|
+
|
|
97
|
+
```js
|
|
98
|
+
const w = new WDoubleSync({ endlessVector: ev, sync, compress: 'gzip' });
|
|
99
|
+
await w.initialize();
|
|
100
|
+
await w.push(root); // patches are gzip-wrapped before pushing
|
|
101
|
+
|
|
102
|
+
// Restore auto-detects compression — no flag needed on the reader side
|
|
103
|
+
const w2 = new WDoubleSync({ endlessVector: ev });
|
|
104
|
+
const folder = await w2.restore();
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
### Resume from existing chain
|
|
108
|
+
|
|
109
|
+
A new WDoubleSync instance on a populated EndlessVector rebuilds its internal state (CDC store + last snapshot) by replaying all existing items. The next push produces a diff patch — not a full patch.
|
|
110
|
+
|
|
111
|
+
```js
|
|
112
|
+
// Some time later, new process...
|
|
113
|
+
const w = new WDoubleSync({ endlessVector: ev, sync });
|
|
114
|
+
await w.initialize(); // replays chain to rebuild state
|
|
115
|
+
await w.push(updatedRoot); // produces a diff patch
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
## API
|
|
119
|
+
|
|
120
|
+
### `new WDoubleSync({ endlessVector, sync?, compress? })`
|
|
121
|
+
|
|
122
|
+
| Param | Type | Default | Description |
|
|
123
|
+
|---|---|---|---|
|
|
124
|
+
| `endlessVector` | `EndlessVector` | required | EndlessVector instance (read-only or read+write) |
|
|
125
|
+
| `sync` | `DoubleSync` | `new DoubleSync()` | DoubleSync instance with CDC parameters |
|
|
126
|
+
| `compress` | `'gzip'` \| `false` | `false` | Wrap patches in gzip before pushing |
|
|
127
|
+
|
|
128
|
+
### `initialize(): Promise<void>`
|
|
129
|
+
|
|
130
|
+
Load EndlessVector state and rebuild internal CDC store + session by replaying existing chain items. Safe to call multiple times.
|
|
131
|
+
|
|
132
|
+
### `push(root, params?): Promise<{ version: number }>`
|
|
133
|
+
|
|
134
|
+
Build the next patch from `root` and push it to the EndlessVector. Auto-initializes if needed.
|
|
135
|
+
|
|
136
|
+
### `restore(version?): Promise<DoubleSyncMemoryFolder>`
|
|
137
|
+
|
|
138
|
+
Reconstruct the folder tree by replaying patches from the chain. Defaults to latest version.
|
|
139
|
+
|
|
140
|
+
### `length(): Promise<number>`
|
|
141
|
+
|
|
142
|
+
Number of patch versions stored on chain.
|
|
143
|
+
|
|
144
|
+
### `getPatch(index): Promise<Uint8Array>`
|
|
145
|
+
|
|
146
|
+
Raw patch bytes at the given chain index.
|
|
147
|
+
|
|
148
|
+
### `reInitialize()`
|
|
149
|
+
|
|
150
|
+
Force full state reset. Next `initialize()` replays the chain from scratch.
|
|
151
|
+
|
|
152
|
+
## Testing
|
|
153
|
+
|
|
154
|
+
Tests run against a real local Sui blockchain node using the same infrastructure as EndlessVector's own test suite:
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
pnpm test
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Requires the `seal_walrus_localnet` setup at `../seal_walrus_localnet/` and the `endless_vector` Move package at `../endless_vector/move/`.
|
|
161
|
+
|
|
162
|
+
## Dependencies
|
|
163
|
+
|
|
164
|
+
- [doublesync](../doublesync/) — CDC chunking, snapshot/patch encoding, folder-tree diff
|
|
165
|
+
- [endless_vector](https://github.com/fizzyFlow/endless_vector) — on-chain append-only vector with auto history/archive/Walrus management
|
package/WDoubleSync.js
ADDED
|
@@ -0,0 +1,450 @@
|
|
|
1
|
+
import DoubleSync from '../doublesync/DoubleSync.js';
|
|
2
|
+
import CDCStore from '../doublesync/CDCStore.js';
|
|
3
|
+
import DoubleSyncSnapshot from '../doublesync/DoubleSyncSnapshot.js';
|
|
4
|
+
import DoubleSyncPatch from '../doublesync/DoubleSyncPatch.js';
|
|
5
|
+
import DoubleSyncDiffPatch from '../doublesync/DoubleSyncDiffPatch.js';
|
|
6
|
+
import DoubleSyncCompressed from '../doublesync/DoubleSyncCompressed.js';
|
|
7
|
+
import DoubleSyncFormat from '../doublesync/DoubleSyncFormat.js';
|
|
8
|
+
import { DoubleSyncMemoryFolder } from '../doublesync/DoubleSyncFolder.js';
|
|
9
|
+
|
|
10
|
+
const SEGMENT_MAGIC = 0x57445347; // WDSG
|
|
11
|
+
const SEGMENT_VERSION = 1;
|
|
12
|
+
const SEGMENT_HEADER_SIZE = 24;
|
|
13
|
+
const DEFAULT_MAX_PATCH_ITEM_BYTES = 8 * 1024 * 1024;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Combines EndlessVector (on-chain append-only storage on Sui) with DoubleSync
|
|
17
|
+
* (CDC-based folder-tree delta sync) to store snapshots and patches on-chain and
|
|
18
|
+
* reconstruct folder state from the chain.
|
|
19
|
+
*
|
|
20
|
+
* Each EndlessVector item holds one patch document — either a full `DoubleSyncPatch`
|
|
21
|
+
* (item 0) or an incremental `DoubleSyncDiffPatch` (items 1..N). The document type
|
|
22
|
+
* is auto-detected via the magic header (`DoubleSyncFormat.detect`).
|
|
23
|
+
*
|
|
24
|
+
* Sender workflow:
|
|
25
|
+
* const w = new WDoubleSync({ endlessVector, sync });
|
|
26
|
+
* await w.initialize();
|
|
27
|
+
* await w.push(senderRoot); // first → full patch
|
|
28
|
+
* await w.push(senderRoot); // next → diff patch
|
|
29
|
+
*
|
|
30
|
+
* Receiver workflow:
|
|
31
|
+
* const w = new WDoubleSync({ endlessVector });
|
|
32
|
+
* const folder = await w.restore(); // latest version
|
|
33
|
+
* const older = await w.restore(2); // specific version
|
|
34
|
+
*
|
|
35
|
+
* @example
|
|
36
|
+
* import EndlessVector from 'endless_vector';
|
|
37
|
+
* import DoubleSync from 'doublesync';
|
|
38
|
+
* import WDoubleSync from './WDoubleSync.js';
|
|
39
|
+
*
|
|
40
|
+
* const ev = new EndlessVector({ suiClient, id, packageId, signAndExecuteTransaction });
|
|
41
|
+
* const sync = new DoubleSync({ avgSize: 8192 });
|
|
42
|
+
* const w = new WDoubleSync({ endlessVector: ev, sync });
|
|
43
|
+
* await w.initialize();
|
|
44
|
+
* await w.push(senderRoot);
|
|
45
|
+
*/
|
|
46
|
+
export default class WDoubleSync {
|
|
47
|
+
/**
|
|
48
|
+
* @param {Object} params
|
|
49
|
+
* @param {import('endless_vector').default} params.endlessVector - EndlessVector instance (read or read+write)
|
|
50
|
+
* @param {DoubleSync} [params.sync] - DoubleSync instance; defaults to `new DoubleSync()` if omitted
|
|
51
|
+
* @param {'gzip'|false} [params.compress=false] - wrap patches in a compression envelope before pushing
|
|
52
|
+
* @param {number} [params.maxPatchItemBytes=8388608] - split larger patch documents across multiple vector items
|
|
53
|
+
*/
|
|
54
|
+
constructor(params = {}) {
|
|
55
|
+
if (!params.endlessVector) throw new Error('WDoubleSync: endlessVector is required');
|
|
56
|
+
|
|
57
|
+
/** @type {import('endless_vector').default} */
|
|
58
|
+
this._ev = params.endlessVector;
|
|
59
|
+
/** @type {DoubleSync} */
|
|
60
|
+
this._sync = params.sync || new DoubleSync();
|
|
61
|
+
/** @type {?'gzip'} */
|
|
62
|
+
this._compress = params.compress || null;
|
|
63
|
+
/** @type {number} */
|
|
64
|
+
this._maxPatchItemBytes = params.maxPatchItemBytes || DEFAULT_MAX_PATCH_ITEM_BYTES;
|
|
65
|
+
|
|
66
|
+
/** @type {CDCStore} */
|
|
67
|
+
this._senderStore = new CDCStore({ copyBytes: false });
|
|
68
|
+
/** @type {CDCStore} */
|
|
69
|
+
this._receiverMirror = new CDCStore();
|
|
70
|
+
|
|
71
|
+
/** @type {?Uint8Array} - snapshot bytes from the last patch produced or replayed */
|
|
72
|
+
this._lastSnapshot = null;
|
|
73
|
+
/** @type {number} - how many items from chain have been replayed into _senderStore / _lastSnapshot */
|
|
74
|
+
this._replayedCount = 0;
|
|
75
|
+
|
|
76
|
+
/** @type {boolean} */
|
|
77
|
+
this._isInitialized = false;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Number of patch versions stored on chain.
|
|
82
|
+
* @returns {Promise<number>}
|
|
83
|
+
*/
|
|
84
|
+
async length() {
|
|
85
|
+
await this._ev.initialize();
|
|
86
|
+
return this._ev.length;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Initialize: load EndlessVector state and rebuild CDCStore + session state
|
|
91
|
+
* by replaying every existing item from the chain.
|
|
92
|
+
*
|
|
93
|
+
* Safe to call multiple times — skips items already replayed.
|
|
94
|
+
* @returns {Promise<void>}
|
|
95
|
+
*/
|
|
96
|
+
async initialize() {
|
|
97
|
+
await this._ev.initialize();
|
|
98
|
+
|
|
99
|
+
const total = this._ev.length;
|
|
100
|
+
|
|
101
|
+
// Replay items we haven't seen yet
|
|
102
|
+
if (this._replayedCount < total) {
|
|
103
|
+
await this._replayRange(this._replayedCount, total);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
this._isInitialized = true;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Re-initialize: force reload from chain. Useful after external pushes.
|
|
111
|
+
*/
|
|
112
|
+
reInitialize() {
|
|
113
|
+
this._ev.reInitialize();
|
|
114
|
+
this._senderStore = new CDCStore({ copyBytes: false });
|
|
115
|
+
this._receiverMirror = new CDCStore();
|
|
116
|
+
this._lastSnapshot = null;
|
|
117
|
+
this._replayedCount = 0;
|
|
118
|
+
this._isInitialized = false;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Build the next patch from `root` and push it to the EndlessVector.
|
|
123
|
+
*
|
|
124
|
+
* First push produces a full `DoubleSyncPatch`; every subsequent push
|
|
125
|
+
* produces a `DoubleSyncDiffPatch` against the previous snapshot.
|
|
126
|
+
*
|
|
127
|
+
* @param {import('doublesync').DoubleSyncFolder} root - sender's current folder tree
|
|
128
|
+
* @param {Object} [params]
|
|
129
|
+
* @param {number} [params.timeout] - tx confirmation timeout in ms
|
|
130
|
+
* @param {number} [params.pollIntervalMs] - tx poll interval in ms
|
|
131
|
+
* @returns {Promise<{version: number}>}
|
|
132
|
+
*/
|
|
133
|
+
async push(root, params = {}) {
|
|
134
|
+
if (!this._isInitialized) await this.initialize();
|
|
135
|
+
|
|
136
|
+
let patchBytes;
|
|
137
|
+
let newSnapshot = null;
|
|
138
|
+
let newChunks = [];
|
|
139
|
+
|
|
140
|
+
if (!this._lastSnapshot) {
|
|
141
|
+
patchBytes = await this._sync.buildPatch({
|
|
142
|
+
root,
|
|
143
|
+
senderStore: this._senderStore,
|
|
144
|
+
receiverStore: this._receiverMirror,
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
const parsed = new DoubleSyncPatch(patchBytes);
|
|
148
|
+
newSnapshot = parsed.snapshot.slice();
|
|
149
|
+
|
|
150
|
+
for (const { hash, bytes } of parsed.chunks()) {
|
|
151
|
+
newChunks.push({ hash, bytes });
|
|
152
|
+
}
|
|
153
|
+
} else {
|
|
154
|
+
const result = await this._sync.buildDiffPatch({
|
|
155
|
+
root,
|
|
156
|
+
senderStore: this._senderStore,
|
|
157
|
+
receiverStore: this._receiverMirror,
|
|
158
|
+
prevSnapshot: this._lastSnapshot,
|
|
159
|
+
});
|
|
160
|
+
patchBytes = result.patch;
|
|
161
|
+
newSnapshot = result.newSnapshot;
|
|
162
|
+
|
|
163
|
+
const parsedDiff = new DoubleSyncDiffPatch(patchBytes);
|
|
164
|
+
for (const { hash, bytes } of parsedDiff.chunks()) {
|
|
165
|
+
newChunks.push({ hash, bytes });
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
if (this._compress) {
|
|
170
|
+
patchBytes = await DoubleSyncCompressed.encode(patchBytes, this._compress);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
this._senderStore = this._cloneReceiverMirror();
|
|
174
|
+
const pushedItems = await this._pushPatchDocument(patchBytes, params);
|
|
175
|
+
|
|
176
|
+
this._lastSnapshot = newSnapshot;
|
|
177
|
+
for (const { hash, bytes } of newChunks) {
|
|
178
|
+
this._receiverMirror.putWithHash(hash, bytes);
|
|
179
|
+
this._senderStore.putWithHash(hash, bytes);
|
|
180
|
+
}
|
|
181
|
+
this._replayedCount += pushedItems;
|
|
182
|
+
|
|
183
|
+
return { version: this._replayedCount };
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Read raw patch bytes at the given chain index.
|
|
188
|
+
*
|
|
189
|
+
* @param {number} index
|
|
190
|
+
* @returns {Promise<Uint8Array>}
|
|
191
|
+
*/
|
|
192
|
+
async getPatch(index) {
|
|
193
|
+
await this._ev.initialize();
|
|
194
|
+
return await this._ev.at(index);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Reconstruct the folder tree by replaying patches 0..version from the chain.
|
|
199
|
+
* Returns a `DoubleSyncMemoryFolder` containing the reconstructed tree.
|
|
200
|
+
*
|
|
201
|
+
* @param {number} [version] - replay up to this version (exclusive); defaults to latest
|
|
202
|
+
* @returns {Promise<DoubleSyncMemoryFolder>}
|
|
203
|
+
*/
|
|
204
|
+
async restore(version) {
|
|
205
|
+
await this._ev.initialize();
|
|
206
|
+
|
|
207
|
+
const total = this._ev.length;
|
|
208
|
+
const end = (version !== undefined) ? Math.min(version, total) : total;
|
|
209
|
+
|
|
210
|
+
if (end === 0) return new DoubleSyncMemoryFolder('root');
|
|
211
|
+
|
|
212
|
+
const store = new CDCStore({ copyBytes: false });
|
|
213
|
+
const dest = new DoubleSyncMemoryFolder('root');
|
|
214
|
+
let lastSnapshot = null;
|
|
215
|
+
|
|
216
|
+
for (let i = 0; i < end;) {
|
|
217
|
+
const item = await this._readPatchDocument(i, end);
|
|
218
|
+
let patchBytes = item.patchBytes;
|
|
219
|
+
i = item.nextIndex;
|
|
220
|
+
|
|
221
|
+
if (DoubleSyncFormat.isCompressed(patchBytes)) {
|
|
222
|
+
patchBytes = await DoubleSyncCompressed.decode(patchBytes);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const kind = DoubleSyncFormat.detect(patchBytes);
|
|
226
|
+
|
|
227
|
+
if (kind === 'patch') {
|
|
228
|
+
await this._sync.applyPatch({ patch: patchBytes, store, dest });
|
|
229
|
+
const parsed = new DoubleSyncPatch(patchBytes);
|
|
230
|
+
lastSnapshot = parsed.snapshot.slice();
|
|
231
|
+
} else if (kind === 'diff-patch') {
|
|
232
|
+
if (!lastSnapshot) {
|
|
233
|
+
throw new Error(`WDoubleSync.restore: diff-patch at index ${i} but no prior snapshot`);
|
|
234
|
+
}
|
|
235
|
+
const newSnapshot = await this._sync.applyDiffPatch({
|
|
236
|
+
patch: patchBytes,
|
|
237
|
+
store,
|
|
238
|
+
dest,
|
|
239
|
+
prevSnapshot: lastSnapshot,
|
|
240
|
+
});
|
|
241
|
+
lastSnapshot = newSnapshot;
|
|
242
|
+
} else {
|
|
243
|
+
throw new Error(`WDoubleSync.restore: unknown patch type at index ${i}`);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
return dest;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* Get the Merkle tree hash for a specific version.
|
|
252
|
+
* Reads backwards from the target to find the nearest full snapshot,
|
|
253
|
+
* then replays only the diff-patches forward.
|
|
254
|
+
* Best case (latest is a full snapshot): single read, no replay.
|
|
255
|
+
*
|
|
256
|
+
* @param {number} [version] - version to query; defaults to latest
|
|
257
|
+
* @returns {Promise<Uint8Array>} 32-byte tree hash
|
|
258
|
+
*/
|
|
259
|
+
async getTreeHash(version) {
|
|
260
|
+
await this._ev.initialize();
|
|
261
|
+
|
|
262
|
+
const total = this._ev.length;
|
|
263
|
+
const end = (version !== undefined) ? Math.min(version, total) : total;
|
|
264
|
+
if (end === 0) return new Uint8Array(32);
|
|
265
|
+
|
|
266
|
+
const diffPatches = [];
|
|
267
|
+
|
|
268
|
+
for (let i = end - 1; i >= 0; i--) {
|
|
269
|
+
const item = await this._readPatchDocument(i, total);
|
|
270
|
+
let patchBytes = item.patchBytes;
|
|
271
|
+
|
|
272
|
+
if (DoubleSyncFormat.isCompressed(patchBytes)) {
|
|
273
|
+
patchBytes = await DoubleSyncCompressed.decode(patchBytes);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
const kind = DoubleSyncFormat.detect(patchBytes);
|
|
277
|
+
|
|
278
|
+
if (kind === 'patch') {
|
|
279
|
+
const parsed = new DoubleSyncPatch(patchBytes);
|
|
280
|
+
if (diffPatches.length === 0) {
|
|
281
|
+
return new DoubleSyncSnapshot(parsed.snapshot).treeHash;
|
|
282
|
+
}
|
|
283
|
+
let currentSnapshot = parsed.snapshot;
|
|
284
|
+
const store = new CDCStore({ copyBytes: false });
|
|
285
|
+
for (const { hash, bytes } of parsed.chunks()) {
|
|
286
|
+
store.putWithHash(hash, bytes);
|
|
287
|
+
}
|
|
288
|
+
const dest = new DoubleSyncMemoryFolder('_hash');
|
|
289
|
+
for (const diffBytes of diffPatches) {
|
|
290
|
+
currentSnapshot = await this._sync.applyDiffPatch({
|
|
291
|
+
patch: diffBytes, store, dest, prevSnapshot: currentSnapshot,
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
return new DoubleSyncSnapshot(currentSnapshot).treeHash;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
diffPatches.unshift(patchBytes);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
return new Uint8Array(32);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/**
|
|
304
|
+
* Replay items from the chain into _senderStore and _lastSnapshot so the sender
|
|
305
|
+
* session state is up to date for the next push().
|
|
306
|
+
*
|
|
307
|
+
* @param {number} from - inclusive start index
|
|
308
|
+
* @param {number} to - exclusive end index
|
|
309
|
+
*/
|
|
310
|
+
async _replayRange(from, to) {
|
|
311
|
+
// We need a temporary dest to drive applyPatch / applyDiffPatch, but we only
|
|
312
|
+
// care about the CDCStore and snapshot state — not the folder contents. A
|
|
313
|
+
// throwaway memory folder is cheap.
|
|
314
|
+
const dest = new DoubleSyncMemoryFolder('_replay');
|
|
315
|
+
|
|
316
|
+
for (let i = from; i < to;) {
|
|
317
|
+
const item = await this._readPatchDocument(i, to);
|
|
318
|
+
let patchBytes = item.patchBytes;
|
|
319
|
+
i = item.nextIndex;
|
|
320
|
+
|
|
321
|
+
if (DoubleSyncFormat.isCompressed(patchBytes)) {
|
|
322
|
+
patchBytes = await DoubleSyncCompressed.decode(patchBytes);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
const kind = DoubleSyncFormat.detect(patchBytes);
|
|
326
|
+
|
|
327
|
+
if (kind === 'patch') {
|
|
328
|
+
await this._sync.applyPatch({ patch: patchBytes, store: this._senderStore, dest });
|
|
329
|
+
|
|
330
|
+
const parsed = new DoubleSyncPatch(patchBytes);
|
|
331
|
+
this._lastSnapshot = parsed.snapshot.slice();
|
|
332
|
+
|
|
333
|
+
// Receiver mirror: after a full patch, receiver has all embedded chunks
|
|
334
|
+
for (const { hash, bytes } of parsed.chunks()) {
|
|
335
|
+
this._receiverMirror.putWithHash(hash, bytes);
|
|
336
|
+
}
|
|
337
|
+
} else if (kind === 'diff-patch') {
|
|
338
|
+
if (!this._lastSnapshot) {
|
|
339
|
+
throw new Error(`WDoubleSync._replayRange: diff-patch at index ${i} but no prior snapshot`);
|
|
340
|
+
}
|
|
341
|
+
const newSnapshot = await this._sync.applyDiffPatch({
|
|
342
|
+
patch: patchBytes,
|
|
343
|
+
store: this._senderStore,
|
|
344
|
+
dest,
|
|
345
|
+
prevSnapshot: this._lastSnapshot,
|
|
346
|
+
});
|
|
347
|
+
this._lastSnapshot = newSnapshot;
|
|
348
|
+
|
|
349
|
+
const parsedDiff = new DoubleSyncDiffPatch(patchBytes);
|
|
350
|
+
for (const { hash, bytes } of parsedDiff.chunks()) {
|
|
351
|
+
this._receiverMirror.putWithHash(hash, bytes);
|
|
352
|
+
}
|
|
353
|
+
} else {
|
|
354
|
+
throw new Error(`WDoubleSync._replayRange: unknown patch type at index ${i}`);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
this._replayedCount = to;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
async _pushPatchDocument(patchBytes, params) {
|
|
362
|
+
if (!(patchBytes instanceof Uint8Array)) throw new Error('WDoubleSync._pushPatchDocument: patchBytes must be Uint8Array');
|
|
363
|
+
|
|
364
|
+
if (!this._maxPatchItemBytes || patchBytes.length <= this._maxPatchItemBytes) {
|
|
365
|
+
await this._ev.push(patchBytes, params);
|
|
366
|
+
return 1;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
const segmentPayloadBytes = Math.max(1, this._maxPatchItemBytes - SEGMENT_HEADER_SIZE);
|
|
370
|
+
const total = Math.ceil(patchBytes.length / segmentPayloadBytes);
|
|
371
|
+
for (let index = 0; index < total; index++) {
|
|
372
|
+
const start = index * segmentPayloadBytes;
|
|
373
|
+
const payload = patchBytes.subarray(start, Math.min(start + segmentPayloadBytes, patchBytes.length));
|
|
374
|
+
await this._ev.push(encodeSegment(payload, { index, total, originalLength: patchBytes.length }), params);
|
|
375
|
+
}
|
|
376
|
+
return total;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
async _readPatchDocument(index, end) {
|
|
380
|
+
const first = await this._ev.at(index);
|
|
381
|
+
const segment = parseSegment(first);
|
|
382
|
+
if (!segment) return { patchBytes: first, nextIndex: index + 1 };
|
|
383
|
+
|
|
384
|
+
if (segment.index !== 0) {
|
|
385
|
+
throw new Error(`WDoubleSync: segmented patch at index ${index} starts with segment ${segment.index}`);
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
const parts = new Array(segment.total);
|
|
389
|
+
parts[0] = segment.payload;
|
|
390
|
+
let nextIndex = index + 1;
|
|
391
|
+
for (let expected = 1; expected < segment.total; expected++, nextIndex++) {
|
|
392
|
+
if (nextIndex >= end) {
|
|
393
|
+
throw new Error(`WDoubleSync: incomplete segmented patch at index ${index}`);
|
|
394
|
+
}
|
|
395
|
+
const nextSegment = parseSegment(await this._ev.at(nextIndex));
|
|
396
|
+
if (!nextSegment || nextSegment.index !== expected || nextSegment.total !== segment.total || nextSegment.originalLength !== segment.originalLength) {
|
|
397
|
+
throw new Error(`WDoubleSync: invalid segmented patch segment at index ${nextIndex}`);
|
|
398
|
+
}
|
|
399
|
+
parts[expected] = nextSegment.payload;
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
const out = new Uint8Array(segment.originalLength);
|
|
403
|
+
let cur = 0;
|
|
404
|
+
for (const part of parts) {
|
|
405
|
+
out.set(part, cur);
|
|
406
|
+
cur += part.length;
|
|
407
|
+
}
|
|
408
|
+
if (cur !== out.length) throw new Error(`WDoubleSync: segmented patch size mismatch (${cur} != ${out.length})`);
|
|
409
|
+
return { patchBytes: out, nextIndex };
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
_cloneReceiverMirror() {
|
|
413
|
+
const store = new CDCStore({ copyBytes: false });
|
|
414
|
+
for (const hash of this._receiverMirror.hashes()) {
|
|
415
|
+
const bytes = this._receiverMirror.get(hash);
|
|
416
|
+
if (bytes) store.putWithHash(hash, bytes);
|
|
417
|
+
}
|
|
418
|
+
return store;
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
function encodeSegment(payload, { index, total, originalLength }) {
|
|
423
|
+
const out = new Uint8Array(SEGMENT_HEADER_SIZE + payload.length);
|
|
424
|
+
const dv = new DataView(out.buffer, out.byteOffset, out.byteLength);
|
|
425
|
+
dv.setUint32(0, SEGMENT_MAGIC, true);
|
|
426
|
+
out[4] = SEGMENT_VERSION;
|
|
427
|
+
dv.setUint32(8, index, true);
|
|
428
|
+
dv.setUint32(12, total, true);
|
|
429
|
+
dv.setBigUint64(16, BigInt(originalLength), true);
|
|
430
|
+
out.set(payload, SEGMENT_HEADER_SIZE);
|
|
431
|
+
return out;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
function parseSegment(bytes) {
|
|
435
|
+
if (!(bytes instanceof Uint8Array) || bytes.length < SEGMENT_HEADER_SIZE) return null;
|
|
436
|
+
const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
437
|
+
if (dv.getUint32(0, true) !== SEGMENT_MAGIC || bytes[4] !== SEGMENT_VERSION) return null;
|
|
438
|
+
const index = dv.getUint32(8, true);
|
|
439
|
+
const total = dv.getUint32(12, true);
|
|
440
|
+
const originalLength = Number(dv.getBigUint64(16, true));
|
|
441
|
+
if (!total || index >= total || !Number.isSafeInteger(originalLength)) {
|
|
442
|
+
throw new Error('WDoubleSync: invalid segmented patch header');
|
|
443
|
+
}
|
|
444
|
+
return {
|
|
445
|
+
index,
|
|
446
|
+
total,
|
|
447
|
+
originalLength,
|
|
448
|
+
payload: bytes.subarray(SEGMENT_HEADER_SIZE),
|
|
449
|
+
};
|
|
450
|
+
}
|
package/index.js
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import WDoubleSync from './WDoubleSync.js';
|
|
2
|
+
|
|
3
|
+
export {
|
|
4
|
+
WDoubleSync as default,
|
|
5
|
+
WDoubleSync,
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
export {
|
|
9
|
+
CDCSync,
|
|
10
|
+
CDCManifest,
|
|
11
|
+
CDCStore,
|
|
12
|
+
FastCDC,
|
|
13
|
+
GEAR,
|
|
14
|
+
DoubleSyncFile,
|
|
15
|
+
DoubleSyncMemoryFile,
|
|
16
|
+
DoubleSyncFolder,
|
|
17
|
+
DoubleSyncMemoryFolder,
|
|
18
|
+
DoubleSyncSnapshot,
|
|
19
|
+
DoubleSync,
|
|
20
|
+
DoubleSyncPatch,
|
|
21
|
+
DoubleSyncDiffPatch,
|
|
22
|
+
DoubleSyncCompressed,
|
|
23
|
+
DoubleSyncFormat,
|
|
24
|
+
MAGICS,
|
|
25
|
+
diffSnapshots,
|
|
26
|
+
DoubleSyncSession,
|
|
27
|
+
} from 'doublesync';
|
package/package.json
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@fizzyflow/wdoublesync",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "Combines EndlessVector (on-chain Sui storage) with DoubleSync (CDC delta sync) to store and replay folder-tree snapshots on-chain.",
|
|
6
|
+
"main": "./index.js",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": {
|
|
9
|
+
"import": "./index.js",
|
|
10
|
+
"default": "./index.js"
|
|
11
|
+
}
|
|
12
|
+
},
|
|
13
|
+
"publishConfig": {
|
|
14
|
+
"access": "public"
|
|
15
|
+
},
|
|
16
|
+
"scripts": {
|
|
17
|
+
"test": "vitest run ./test/*.test.js"
|
|
18
|
+
},
|
|
19
|
+
"keywords": [],
|
|
20
|
+
"author": "suidouble (https://github.com/suidouble)",
|
|
21
|
+
"license": "AGPL-3.0",
|
|
22
|
+
"packageManager": "pnpm@10.16.0",
|
|
23
|
+
"dependencies": {
|
|
24
|
+
"@fizzyflow/doublesync": "^0.1.0",
|
|
25
|
+
"@fizzyflow/endless-vector": "^0.0.10"
|
|
26
|
+
},
|
|
27
|
+
"devDependencies": {
|
|
28
|
+
"@mysten/seal": "^1.1.3",
|
|
29
|
+
"@mysten/sui": "^2.16.0",
|
|
30
|
+
"@noble/ciphers": "^2.2.0",
|
|
31
|
+
"suidouble": "^2.16.0",
|
|
32
|
+
"vitest": "^4.1.6"
|
|
33
|
+
}
|
|
34
|
+
}
|