@interocitor/core 0.0.0-beta.3 → 0.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +445 -91
- package/dist/adapters/cloudflare.d.ts +8 -9
- package/dist/adapters/cloudflare.d.ts.map +1 -1
- package/dist/adapters/cloudflare.js +38 -12
- package/dist/adapters/google-drive.d.ts +1 -1
- package/dist/adapters/google-drive.js +1 -2
- package/dist/adapters/memory.d.ts +4 -1
- package/dist/adapters/memory.d.ts.map +1 -1
- package/dist/adapters/memory.js +13 -2
- package/dist/adapters/webdav.d.ts +5 -0
- package/dist/adapters/webdav.d.ts.map +1 -1
- package/dist/adapters/webdav.js +18 -1
- package/dist/core/codec.d.ts +1 -1
- package/dist/core/codec.d.ts.map +1 -1
- package/dist/core/codec.js +39 -3
- package/dist/core/compaction.d.ts +8 -1
- package/dist/core/compaction.d.ts.map +1 -1
- package/dist/core/compaction.js +13 -5
- package/dist/core/crdt.d.ts +6 -3
- package/dist/core/crdt.d.ts.map +1 -1
- package/dist/core/crdt.js +38 -60
- package/dist/core/errors.d.ts +47 -0
- package/dist/core/errors.d.ts.map +1 -0
- package/dist/core/errors.js +61 -0
- package/dist/core/flush.d.ts +3 -3
- package/dist/core/flush.d.ts.map +1 -1
- package/dist/core/flush.js +64 -7
- package/dist/core/hlc.js +0 -1
- package/dist/core/ids.d.ts +49 -0
- package/dist/core/ids.d.ts.map +1 -0
- package/dist/core/ids.js +132 -0
- package/dist/core/internals.d.ts +10 -2
- package/dist/core/internals.d.ts.map +1 -1
- package/dist/core/internals.js +27 -9
- package/dist/core/manifest.d.ts +20 -5
- package/dist/core/manifest.d.ts.map +1 -1
- package/dist/core/manifest.js +65 -11
- package/dist/core/pull.d.ts +1 -1
- package/dist/core/pull.d.ts.map +1 -1
- package/dist/core/pull.js +21 -6
- package/dist/core/row-id.js +0 -1
- package/dist/core/schema-types.d.ts +22 -11
- package/dist/core/schema-types.d.ts.map +1 -1
- package/dist/core/schema-types.js +18 -9
- package/dist/core/schema-types.type-test.js +59 -5
- package/dist/core/sync-engine.d.ts +163 -12
- package/dist/core/sync-engine.d.ts.map +1 -1
- package/dist/core/sync-engine.js +1521 -219
- package/dist/core/table.d.ts +217 -17
- package/dist/core/table.d.ts.map +1 -1
- package/dist/core/table.js +376 -24
- package/dist/core/types.d.ts +382 -17
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js +0 -1
- package/dist/crypto/encryption.d.ts.map +1 -1
- package/dist/crypto/encryption.js +6 -1
- package/dist/crypto/keys.js +0 -1
- package/dist/handshake/channel.js +0 -1
- package/dist/handshake/index.d.ts +5 -2
- package/dist/handshake/index.d.ts.map +1 -1
- package/dist/handshake/index.js +19 -2
- package/dist/handshake/qr.js +0 -1
- package/dist/index.d.ts +9 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -6
- package/dist/storage/credential-store.d.ts +25 -2
- package/dist/storage/credential-store.d.ts.map +1 -1
- package/dist/storage/credential-store.js +55 -8
- package/dist/storage/local-store.d.ts +4 -1
- package/dist/storage/local-store.d.ts.map +1 -1
- package/dist/storage/local-store.js +37 -21
- package/package.json +3 -3
- package/dist/adapters/cloudflare.js.map +0 -1
- package/dist/adapters/google-drive.js.map +0 -1
- package/dist/adapters/memory.js.map +0 -1
- package/dist/adapters/webdav.js.map +0 -1
- package/dist/core/codec.js.map +0 -1
- package/dist/core/compaction.js.map +0 -1
- package/dist/core/crdt.js.map +0 -1
- package/dist/core/flush.js.map +0 -1
- package/dist/core/hlc.js.map +0 -1
- package/dist/core/internals.js.map +0 -1
- package/dist/core/manifest.js.map +0 -1
- package/dist/core/pull.js.map +0 -1
- package/dist/core/row-id.js.map +0 -1
- package/dist/core/schema-types.js.map +0 -1
- package/dist/core/schema-types.type-test.js.map +0 -1
- package/dist/core/sync-engine.js.map +0 -1
- package/dist/core/table.js.map +0 -1
- package/dist/core/types.js.map +0 -1
- package/dist/crypto/encryption.js.map +0 -1
- package/dist/crypto/keys.js.map +0 -1
- package/dist/handshake/channel.js.map +0 -1
- package/dist/handshake/index.js.map +0 -1
- package/dist/handshake/qr.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/storage/credential-store.js.map +0 -1
- package/dist/storage/local-store.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,151 +1,482 @@
|
|
|
1
1
|
<p align="center">
|
|
2
2
|
<a href="https://github.com/TheUiTeam/interocitor">
|
|
3
|
-
<img src="
|
|
3
|
+
<img src="../../docs/assets/hero.svg" alt="Interocitor" width="640" />
|
|
4
4
|
</a>
|
|
5
5
|
</p>
|
|
6
6
|
|
|
7
7
|
<p align="center">
|
|
8
|
-
<
|
|
8
|
+
<em>Encrypted local-first CRDT sync for browser apps.</em>
|
|
9
9
|
</p>
|
|
10
10
|
|
|
11
|
-
# interocitor
|
|
11
|
+
# @interocitor/core
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
End‑to‑end encrypted, local‑first sync over a remote folder you already
|
|
14
|
+
own (Google Drive, WebDAV, Cloudflare R2, your own server). The cloud is
|
|
15
|
+
a mailbox; merge happens on the device.
|
|
14
16
|
|
|
15
|
-
##
|
|
17
|
+
## What it is
|
|
18
|
+
|
|
19
|
+
- A sync engine, not a database. Local reads/writes go through an embedded
|
|
20
|
+
store (IndexedDB in the browser). The engine ships diffs, not queries.
|
|
21
|
+
- A CRDT over per‑column HLC values. Every device converges to the same
|
|
22
|
+
state without a central merge authority.
|
|
23
|
+
- An end‑to‑end encryption layer. The remote sees ciphertext blobs and
|
|
24
|
+
enough metadata to route them; nothing else.
|
|
25
|
+
- A pluggable transport. Any backend that can list/read/write/delete
|
|
26
|
+
files works. See `docs/adapter-contract.md`.
|
|
16
27
|
|
|
17
|
-
|
|
18
|
-
Your devices hold the key. The cloud is only a mailbox. It carries encrypted sync artifacts and cannot read your mail.
|
|
28
|
+
## Why
|
|
19
29
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
-
|
|
23
|
-
|
|
24
|
-
|
|
30
|
+
- **Offline‑first by construction.** Every read and write hits a local
|
|
31
|
+
store. Network is needed only to share state with other devices.
|
|
32
|
+
- **No server you have to operate.** The remote is dumb storage. You can
|
|
33
|
+
run the engine against a user's own Google Drive and ship zero
|
|
34
|
+
backend.
|
|
35
|
+
- **End‑to‑end encryption by default.** The default config encrypts
|
|
36
|
+
every change file and snapshot before it leaves the device.
|
|
37
|
+
- **Small, embeddable runtime.** No workers, no background services
|
|
38
|
+
required.
|
|
25
39
|
|
|
26
40
|
## Quick start
|
|
27
41
|
|
|
28
42
|
```ts
|
|
29
|
-
import {
|
|
30
|
-
import {
|
|
43
|
+
import { Interocitor } from '@interocitor/core';
|
|
44
|
+
import { GoogleDriveAdapter } from '@interocitor/core/adapters/google-drive';
|
|
31
45
|
|
|
32
|
-
const adapter = new
|
|
33
|
-
baseUrl: 'https://your-webdav-server.example.com',
|
|
34
|
-
auth: { username: 'user', password: 'pass' },
|
|
35
|
-
});
|
|
46
|
+
const adapter = new GoogleDriveAdapter({ clientId: 'YOUR_GOOGLE_CLIENT_ID' });
|
|
36
47
|
|
|
37
|
-
const
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
48
|
+
const db = new Interocitor(adapter, {
|
|
49
|
+
dbName: 'my-app', // local DB name; keep stable across reloads
|
|
50
|
+
appName: 'My App', // shown in biometric prompts / OS keychain
|
|
51
|
+
remotePath: '/MyApp', // mesh-scoped folder on the remote
|
|
52
|
+
encrypted: true, // E2E encryption (default)
|
|
42
53
|
});
|
|
43
54
|
|
|
44
|
-
await
|
|
45
|
-
await
|
|
55
|
+
await db.init();
|
|
56
|
+
await db.connect();
|
|
46
57
|
|
|
47
|
-
|
|
48
|
-
await
|
|
49
|
-
text: 'Ship privacy-first sync',
|
|
50
|
-
|
|
51
|
-
|
|
58
|
+
// Write — local-first, no network required
|
|
59
|
+
await db.table('todos').add(
|
|
60
|
+
{ text: 'Ship privacy-first sync', done: false },
|
|
61
|
+
{ prefix: 'todo' },
|
|
62
|
+
);
|
|
63
|
+
|
|
64
|
+
// Share with another device — see "New device / restore" below
|
|
65
|
+
console.log('Passphrase:', db.getPassphrase());
|
|
52
66
|
```
|
|
53
67
|
|
|
54
|
-
|
|
68
|
+
> The constructor accepts either `(adapter, config)` or `(config)` alone.
|
|
69
|
+
> Use the `(config)` form for local‑only mode (no remote). When you
|
|
70
|
+
> attach a remote later, call `setRemoteStorage(adapter)` and then
|
|
71
|
+
> `connect()`.
|
|
72
|
+
|
|
73
|
+
## Mental model
|
|
55
74
|
|
|
56
75
|
```mermaid
|
|
57
76
|
flowchart LR
|
|
58
77
|
A[Browser app] --> B[Interocitor engine]
|
|
59
78
|
B --> C[Local store]
|
|
60
|
-
B --> D[Encrypt
|
|
79
|
+
B --> D[Encrypt locally]
|
|
61
80
|
D --> E[Adapter]
|
|
62
|
-
E --> F[Remote mailbox
|
|
81
|
+
E --> F[Remote mailbox<br/>ciphertext only]
|
|
63
82
|
F --> E
|
|
64
|
-
E --> G[
|
|
65
|
-
G -->
|
|
66
|
-
H --> B
|
|
83
|
+
E --> G[Decrypt locally]
|
|
84
|
+
G --> B
|
|
67
85
|
```
|
|
68
86
|
|
|
69
|
-
|
|
87
|
+
The remote is a mailbox. The engine puts encrypted change files into it
|
|
88
|
+
and pulls down change files from peers. All merging happens on the
|
|
89
|
+
device. There is no server‑side compute.
|
|
70
90
|
|
|
71
|
-
|
|
72
|
-
await engine.init();
|
|
73
|
-
await engine.connect();
|
|
74
|
-
await engine.put(table, id, data);
|
|
75
|
-
await engine.get(table, id);
|
|
76
|
-
await engine.query(table);
|
|
77
|
-
await engine.sync();
|
|
78
|
-
|
|
79
|
-
await engine.secureWithBiometrics(); // optional, explicit keychain enrollment
|
|
80
|
-
await engine.restoreWithBiometrics(); // explicit recovery flow
|
|
81
|
-
```
|
|
91
|
+
Three artifacts live on the remote:
|
|
82
92
|
|
|
83
|
-
|
|
93
|
+
- **change files** — one per write batch, named `<HLC>-chg_<id>.json`;
|
|
94
|
+
- **snapshots** — periodic full state, written by compaction;
|
|
95
|
+
- **a manifest** — pointer to the current generation, plus mesh metadata.
|
|
84
96
|
|
|
85
|
-
|
|
97
|
+
See `docs/adapter-contract.md` for the full layout.
|
|
86
98
|
|
|
87
|
-
|
|
99
|
+
## Security model
|
|
88
100
|
|
|
89
|
-
|
|
90
|
-
|
|
101
|
+
Encryption is on by default (`encrypted: true`). The engine uses
|
|
102
|
+
**AES‑GCM 256** with a key derived from a base58 passphrase.
|
|
91
103
|
|
|
92
|
-
|
|
93
|
-
|
|
104
|
+
What it protects:
|
|
105
|
+
|
|
106
|
+
- Row contents (field names, field values, table names) inside change
|
|
107
|
+
files and snapshots.
|
|
108
|
+
- Integrity of each entry (AES‑GCM authentication tag).
|
|
109
|
+
|
|
110
|
+
What it does **not** protect:
|
|
111
|
+
|
|
112
|
+
- File names (they encode the HLC and a device id).
|
|
113
|
+
- Manifest contents (mesh id, schema version, epoch, encrypted flag,
|
|
114
|
+
serverId).
|
|
115
|
+
- Sizes, write timing, device count.
|
|
116
|
+
- The local store on the device (IndexedDB / SQLite stores plaintext).
|
|
117
|
+
|
|
118
|
+
What a malicious remote can still do:
|
|
94
119
|
|
|
95
|
-
|
|
120
|
+
- Drop, withhold, replay, or roll back files.
|
|
121
|
+
- Observe activity timing and device identities.
|
|
96
122
|
|
|
97
|
-
|
|
123
|
+
For the threat model, the metadata table, and full mitigations see
|
|
124
|
+
`docs/security-model.md`.
|
|
98
125
|
|
|
99
|
-
|
|
100
|
-
`sync()` is the only call that touches the remote mailbox.
|
|
126
|
+
## Offline guarantees
|
|
101
127
|
|
|
102
128
|
| Operation | Network? |
|
|
103
129
|
| --- | --- |
|
|
130
|
+
| `new Interocitor()` | No |
|
|
104
131
|
| `init()` | No |
|
|
105
|
-
| `
|
|
106
|
-
| `
|
|
107
|
-
| `
|
|
108
|
-
| `
|
|
132
|
+
| `table.add()` / `patch()` / `replace()` / `delete()` | No |
|
|
133
|
+
| `table.row()` / `table.query()` / `table.where()` | No |
|
|
134
|
+
| `connect()` | Yes (if adapter configured) |
|
|
135
|
+
| `flush()` / `pull()` / `compact()` | Yes |
|
|
136
|
+
|
|
137
|
+
Writes are persisted locally and queued in an outbox. They survive
|
|
138
|
+
reload, crash, and offline periods. On reconnect the engine drains the
|
|
139
|
+
outbox to the remote.
|
|
140
|
+
|
|
141
|
+
## Sync guarantees
|
|
142
|
+
|
|
143
|
+
- **Eventual convergence.** Two devices that have seen the same set of
|
|
144
|
+
change files (in any order) reach byte‑identical local state.
|
|
145
|
+
- **Per‑column merge.** Conflicts are resolved field‑by‑field, not at
|
|
146
|
+
the row level. Default strategy is `'remote-wins'` — see
|
|
147
|
+
"Conflict resolution".
|
|
148
|
+
- **Idempotent merge.** Replaying an already‑applied change is a no‑op.
|
|
149
|
+
Safe to re‑pull, safe to re‑process the same change file twice.
|
|
150
|
+
- **Per‑device HLC monotonicity.** A single device's HLCs strictly
|
|
151
|
+
increase. Cross‑device order is total but only as wall‑clocks allow.
|
|
152
|
+
- **Restore via snapshot.** A device that joins late (or rehydrates after
|
|
153
|
+
a long offline) loads the latest snapshot, then applies any change
|
|
154
|
+
files newer than the snapshot watermark.
|
|
155
|
+
|
|
156
|
+
What we do **not** guarantee:
|
|
157
|
+
|
|
158
|
+
- Cross‑device atomicity. A `db.batch()` is one atomic remote file, but
|
|
159
|
+
two unrelated batches from different devices are independent.
|
|
160
|
+
- Real‑time delivery. The default poll interval is 30 s; some adapters
|
|
161
|
+
(Cloudflare) layer push notifications on top.
|
|
162
|
+
- Recovery if the remote silently lies (drops writes, rolls back the
|
|
163
|
+
manifest). See `docs/security-model.md`.
|
|
164
|
+
- Recovery if the passphrase is lost — see "New device / restore".
|
|
165
|
+
|
|
166
|
+
## Setup lifecycle
|
|
109
167
|
|
|
110
|
-
|
|
168
|
+
```
|
|
169
|
+
new Interocitor(adapter?, config)
|
|
170
|
+
│
|
|
171
|
+
▼
|
|
172
|
+
configureMesh({...}) ← optional; pin meshId / passphrase upfront
|
|
173
|
+
│
|
|
174
|
+
▼
|
|
175
|
+
init() ← opens local store, restores credentials
|
|
176
|
+
│
|
|
177
|
+
▼
|
|
178
|
+
setRemoteStorage(adapter) ← only if no adapter passed to ctor
|
|
179
|
+
│
|
|
180
|
+
▼
|
|
181
|
+
connect() ← loads/creates manifest, pulls, flushes, polls
|
|
182
|
+
```
|
|
111
183
|
|
|
112
|
-
|
|
113
|
-
|
|
184
|
+
`connect()` will auto‑`init()` if needed. App code should still treat
|
|
185
|
+
init as explicit so credential restore happens before any writes.
|
|
114
186
|
|
|
115
|
-
|
|
116
|
-
|
|
187
|
+
The `encrypted` flag is **pinned at mesh bootstrap** and cannot change
|
|
188
|
+
between sessions for the same `dbName`. Reconnecting with a different
|
|
189
|
+
mode throws a typed `MeshEncryptionMismatchError`.
|
|
117
190
|
|
|
118
|
-
|
|
119
|
-
|
|
191
|
+
```ts
|
|
192
|
+
import { MeshEncryptionMismatchError } from '@interocitor/core';
|
|
193
|
+
|
|
194
|
+
try {
|
|
195
|
+
await db.connect();
|
|
196
|
+
} catch (err) {
|
|
197
|
+
if (err instanceof MeshEncryptionMismatchError) {
|
|
198
|
+
// err.expectedMode is the mode the remote was bootstrapped with
|
|
199
|
+
}
|
|
200
|
+
throw err;
|
|
201
|
+
}
|
|
202
|
+
```
|
|
120
203
|
|
|
121
|
-
|
|
122
|
-
For tests and adapter-contract validation.
|
|
204
|
+
## New device / restore
|
|
123
205
|
|
|
124
|
-
|
|
206
|
+
Joining a new device to an existing mesh requires three things:
|
|
125
207
|
|
|
126
|
-
|
|
208
|
+
1. The **`meshId`** of the existing mesh.
|
|
209
|
+
2. The **passphrase** that decrypts the mesh.
|
|
210
|
+
3. Access to the same **remote mailbox** (the same `remotePath` on a
|
|
211
|
+
storage backend the new device can reach).
|
|
127
212
|
|
|
128
|
-
|
|
213
|
+
The pairing flow ships these three over an ECDH relay handshake (see
|
|
214
|
+
`generateShareQR` / `handleScannedQR` in the handshake module). After
|
|
215
|
+
the handshake the new device:
|
|
129
216
|
|
|
130
|
-
|
|
217
|
+
```ts
|
|
218
|
+
const db = new Interocitor(adapter, {
|
|
219
|
+
dbName: 'my-app',
|
|
220
|
+
appName: 'My App',
|
|
221
|
+
remotePath: '/MyApp',
|
|
222
|
+
passphrase: 'base58-from-handshake',
|
|
223
|
+
encrypted: true,
|
|
224
|
+
});
|
|
131
225
|
|
|
132
|
-
|
|
133
|
-
|
|
226
|
+
await db.init();
|
|
227
|
+
await db.connect(); // pulls the manifest, rehydrates from snapshot, then catches up
|
|
228
|
+
```
|
|
134
229
|
|
|
135
|
-
|
|
230
|
+
On `connect()` the engine:
|
|
231
|
+
|
|
232
|
+
1. Reads `manifest.json` from the remote.
|
|
233
|
+
2. Compares stored `meshId` (from the credential store) against the live
|
|
234
|
+
one. Mismatch → `MeshCredentialMismatchError`.
|
|
235
|
+
3. If local epoch < remote epoch, calls `rehydrate()` to load the
|
|
236
|
+
latest snapshot.
|
|
237
|
+
4. Pulls any change files newer than the snapshot watermark.
|
|
238
|
+
5. Starts polling.
|
|
239
|
+
|
|
240
|
+
> **If the passphrase is lost and no other device holds it, the mesh is
|
|
241
|
+
> unreadable.** The engine has no recovery path — the data is end‑to‑end
|
|
242
|
+
> encrypted and the key is the passphrase. Treat the passphrase as the
|
|
243
|
+
> only thing that matters; back it up out of band (1Password, paper,
|
|
244
|
+
> another device's `WebAuthnCredentialStore`).
|
|
245
|
+
|
|
246
|
+
For the credential store details (records, anchors, biometric paths)
|
|
247
|
+
see `docs/credential-store.md`.
|
|
248
|
+
|
|
249
|
+
## Core API
|
|
136
250
|
|
|
137
251
|
```ts
|
|
138
|
-
|
|
139
|
-
|
|
252
|
+
const db = new Interocitor(adapter, {
|
|
253
|
+
dbName: 'my-app',
|
|
254
|
+
appName: 'My App',
|
|
255
|
+
remotePath: '/MyApp',
|
|
256
|
+
schema,
|
|
257
|
+
});
|
|
258
|
+
await db.init();
|
|
259
|
+
|
|
260
|
+
await db.table('tasks').add({ title: 'Ship it', done: false }, { prefix: 'task' });
|
|
261
|
+
await db.table('tasks').patch(taskId, { done: true });
|
|
262
|
+
await db.table('tasks').replace(taskId, fullTask);
|
|
263
|
+
await db.table('tasks').delete(taskId);
|
|
264
|
+
|
|
265
|
+
await db.table('tasks').row(taskId); // single row
|
|
266
|
+
await db.table('tasks').query(); // all rows
|
|
267
|
+
await db.table('tasks').where('done').equals(false).orderBy('title');
|
|
268
|
+
|
|
269
|
+
await db.connect(); // attach + sync
|
|
270
|
+
await db.flush(); // force outbox drain
|
|
271
|
+
await db.pull(); // force pull
|
|
272
|
+
await db.compact(); // see docs/compaction.md
|
|
273
|
+
await db.disconnect(); // tear down
|
|
274
|
+
|
|
275
|
+
// Credentials
|
|
276
|
+
db.getPassphrase();
|
|
277
|
+
db.setPassphrase(passphrase);
|
|
278
|
+
await db.secureWithBiometrics();
|
|
279
|
+
await db.restoreWithBiometrics();
|
|
280
|
+
await db.clearCredentials();
|
|
281
|
+
|
|
282
|
+
// Batched writes — one ChangeEntry per batch
|
|
283
|
+
await db.batch(async () => {
|
|
284
|
+
await db.table('todos').add({ title: 'a' });
|
|
285
|
+
await db.table('todos').patch(otherId, { done: true });
|
|
140
286
|
});
|
|
141
287
|
```
|
|
142
288
|
|
|
143
|
-
|
|
289
|
+
### Schema typing
|
|
144
290
|
|
|
145
|
-
|
|
291
|
+
```ts
|
|
292
|
+
import { types } from '@interocitor/core';
|
|
293
|
+
import type { DatabaseSchemaDefinition } from '@interocitor/core';
|
|
294
|
+
|
|
295
|
+
const schema = {
|
|
296
|
+
version: 1,
|
|
297
|
+
tables: {
|
|
298
|
+
todos: {
|
|
299
|
+
fields: {
|
|
300
|
+
text: types.string,
|
|
301
|
+
done: types.boolean,
|
|
302
|
+
createdAt: types.index(types.date),
|
|
303
|
+
note: types.string.optional,
|
|
304
|
+
items: types.typed<TodoItem[]>('json'),
|
|
305
|
+
},
|
|
306
|
+
},
|
|
307
|
+
},
|
|
308
|
+
} satisfies DatabaseSchemaDefinition;
|
|
309
|
+
|
|
310
|
+
// Inferred row type:
|
|
311
|
+
// { text: string; done: boolean; createdAt: Date; note?: string; items: TodoItem[] }
|
|
312
|
+
```
|
|
146
313
|
|
|
147
|
-
|
|
148
|
-
|
|
314
|
+
`.optional` makes the field optional in the inferred type. Indexed and
|
|
315
|
+
unique fields cannot be optional. `types.index(...)` marks a field for
|
|
316
|
+
efficient `where`/`orderBy`.
|
|
317
|
+
|
|
318
|
+
### Conflict resolution
|
|
319
|
+
|
|
320
|
+
Per‑column CRDT with HLC. Default merge strategy: **`'remote-wins'`**.
|
|
321
|
+
|
|
322
|
+
> "Remote‑wins" is per‑column, not per‑row. When two devices write the
|
|
323
|
+
> same column on the same row, the merge keeps the value with the higher
|
|
324
|
+
> HLC. Because HLCs are timestamp‑first, this is "later wall‑clock
|
|
325
|
+
> wins, ties broken by device id". Calling it "remote‑wins" is a
|
|
326
|
+
> historical accident of where the merge runs (during pull); both sides
|
|
327
|
+
> apply the same rule and reach the same answer. Override per database,
|
|
328
|
+
> table, or field if you need `'lww'`, `'local-wins'`, or a custom
|
|
329
|
+
> `MergeFunction`.
|
|
330
|
+
|
|
331
|
+
Available strategies:
|
|
332
|
+
|
|
333
|
+
- `'lww'` — last‑write‑wins by HLC. Functionally identical to
|
|
334
|
+
`'remote-wins'` for this CRDT but keeps semantics explicit.
|
|
335
|
+
- `'remote-wins'` (default).
|
|
336
|
+
- `'local-wins'` — keep local on tie; remote still wins on a strictly
|
|
337
|
+
greater HLC.
|
|
338
|
+
- `MergeFunction` — `(local, remote, ctx) => result` for custom logic.
|
|
339
|
+
|
|
340
|
+
### Deletion semantics
|
|
341
|
+
|
|
342
|
+
`delete()` writes a **tombstone**, not an unlink. Tombstones:
|
|
343
|
+
|
|
344
|
+
- carry an HLC like any other write;
|
|
345
|
+
- propagate through change files and snapshots like any other change;
|
|
346
|
+
- are needed so that a slower device that re‑syncs an older `add()` is
|
|
347
|
+
overridden by the newer `delete()` (no resurrection).
|
|
348
|
+
|
|
349
|
+
Compaction collapses tombstones into the snapshot. After compaction:
|
|
350
|
+
|
|
351
|
+
- The snapshot still records the row as deleted.
|
|
352
|
+
- The change file is pruned at HLC ≤ watermark.
|
|
353
|
+
- A device that was offline before the deletion catches up via
|
|
354
|
+
rehydrate (snapshot first, change files second). The tombstone in
|
|
355
|
+
the snapshot prevents resurrection.
|
|
356
|
+
|
|
357
|
+
A device that was offline **with a local pending re‑add** of the same
|
|
358
|
+
row id will still have its re‑add applied; whether it wins depends on
|
|
359
|
+
HLC ordering. Treat hard‑deleted IDs as burned — generate a fresh row
|
|
360
|
+
id when re‑creating.
|
|
361
|
+
|
|
362
|
+
There is no built‑in retention policy ("forget tombstones older than N
|
|
363
|
+
days"). Add one at the app layer if you need it; physically deleting a
|
|
364
|
+
tombstone risks resurrection.
|
|
365
|
+
|
|
366
|
+
### Local store
|
|
367
|
+
|
|
368
|
+
The local store is a pluggable `LocalStoreAdapter`. Browser default is
|
|
369
|
+
IndexedDB. The Swift package ships SQLite. Tests use an in‑memory
|
|
370
|
+
implementation. Reads, writes, queries, and the outbox all go through
|
|
371
|
+
this interface.
|
|
372
|
+
|
|
373
|
+
## Adapters
|
|
374
|
+
|
|
375
|
+
| Adapter | Use when | Notes |
|
|
376
|
+
| --- | --- | --- |
|
|
377
|
+
| `GoogleDriveAdapter` | You want zero infrastructure | OAuth in the browser; the user owns the data |
|
|
378
|
+
| `WebDAVAdapter` | Self‑hosted (Nextcloud, OwnCloud, custom WebDAV) | Easy to inspect remotely |
|
|
379
|
+
| `CloudflareAdapter` | You operate a worker; want push invalidations | Experimental |
|
|
380
|
+
| `MemoryAdapter` | Tests and demos | No persistence |
|
|
381
|
+
|
|
382
|
+
Implementing your own adapter: see `docs/adapter-contract.md` for
|
|
383
|
+
required semantics, consistency assumptions, and the contract test
|
|
384
|
+
suite.
|
|
385
|
+
|
|
386
|
+
## Remote mailbox layout
|
|
387
|
+
|
|
388
|
+
For `remotePath: '/MyApp'`:
|
|
389
|
+
|
|
390
|
+
```
|
|
391
|
+
/MyApp/
|
|
392
|
+
├── manifest.json # pointer { currentGeneration, file }
|
|
393
|
+
├── manifest-1.json # generation 1
|
|
394
|
+
├── manifest-2.json # ...
|
|
395
|
+
├── changes/
|
|
396
|
+
│ ├── head.json # { latestHlc } — pull fast path
|
|
397
|
+
│ └── <HLC>-chg_<id>.json # encrypted change entries
|
|
398
|
+
├── devices/
|
|
399
|
+
│ └── <deviceId>.json # device metadata
|
|
400
|
+
└── mainline/
|
|
401
|
+
└── snapshot-<epoch>-<serverId>.json # encrypted snapshot
|
|
402
|
+
```
|
|
403
|
+
|
|
404
|
+
`remotePath` is **mesh‑scoped**. One folder = one mesh = one logical
|
|
405
|
+
database. Two meshes sharing the same folder will fight over the
|
|
406
|
+
manifest and poison the remote.
|
|
407
|
+
|
|
408
|
+
## Maintenance / compaction
|
|
409
|
+
|
|
410
|
+
Compaction collapses the change log into a snapshot, bumps the manifest,
|
|
411
|
+
and prunes old change files. Sync works without it; the remote folder
|
|
412
|
+
just grows until *some* device compacts.
|
|
413
|
+
|
|
414
|
+
Two paths run automatically:
|
|
415
|
+
|
|
416
|
+
- **Immediate sampled** — after a flush of ≥ `compactAutoThreshold`
|
|
417
|
+
(default 50) ops, with probability ≈
|
|
418
|
+
`compactAutoSampleNumerator / compactAutoDeviceCount`.
|
|
419
|
+
- **Delayed two‑phase** — per‑write timer (10 ± 5 min) → check that
|
|
420
|
+
remote change files exceed `compactRemoteChangeThreshold` (default 2)
|
|
421
|
+
→ second timer (15 ± 5 min) → run.
|
|
422
|
+
|
|
423
|
+
Both paths are deduped by a single in‑flight guard. You can also call
|
|
424
|
+
`db.compact()` manually.
|
|
425
|
+
|
|
426
|
+
> **The auto defaults and the recommended manual policy are different
|
|
427
|
+
> things.** The manual policy ("idle > 1 min, churn > 20") is what to
|
|
428
|
+
> gate a "Sync now" button on. The auto defaults are what runs without
|
|
429
|
+
> any button. See `docs/compaction.md`.
|
|
430
|
+
|
|
431
|
+
> **Compaction is not race‑safe across devices.** The adapter contract
|
|
432
|
+
> has no CAS/ETag write, so two simultaneous compactors can both
|
|
433
|
+
> overwrite the manifest pointer. Mitigations: rely on probabilistic
|
|
434
|
+
> avoidance for small meshes, or run with `serverManaged: true` and a
|
|
435
|
+
> single authorized writer.
|
|
436
|
+
|
|
437
|
+
Full protocol, events, lock story, prune invariants, and tuning
|
|
438
|
+
checklist: **`docs/compaction.md`**.
|
|
439
|
+
|
|
440
|
+
## Schema migration
|
|
441
|
+
|
|
442
|
+
Schema versions are integers in `manifest.schema`. The engine tracks the
|
|
443
|
+
current version on every write. There is **no in‑place rewrite of the
|
|
444
|
+
remote history** — change files written under v1 stay v1 ciphertext.
|
|
445
|
+
|
|
446
|
+
Recommended migration pattern:
|
|
447
|
+
|
|
448
|
+
1. Bump `schema.version` in your code.
|
|
449
|
+
2. Implement a one‑shot `onInit` migration that reads v1 rows from the
|
|
450
|
+
local store and writes v2 rows back. Use `db.batch(...)` to keep it
|
|
451
|
+
atomic per row group.
|
|
452
|
+
3. Trigger compaction after the migration so the snapshot is written
|
|
453
|
+
under v2 and old v1 change files are pruned.
|
|
454
|
+
4. Devices that have not yet migrated will read v2 change files; your
|
|
455
|
+
schema definitions need to handle the transition (e.g. accept both
|
|
456
|
+
shapes during the rollout window).
|
|
457
|
+
|
|
458
|
+
For breaking changes that cannot be rolled out gradually, the
|
|
459
|
+
heavier path is to bootstrap a fresh mesh, replicate data over, and
|
|
460
|
+
retire the old mesh. The engine does not automate this.
|
|
461
|
+
|
|
462
|
+
## Events
|
|
463
|
+
|
|
464
|
+
```ts
|
|
465
|
+
db.on(event => {
|
|
466
|
+
switch (event.type) {
|
|
467
|
+
case 'sync:start': /* pull began */ break;
|
|
468
|
+
case 'sync:complete': /* event.entriesMerged */ break;
|
|
469
|
+
case 'change': /* event.table, event.rowId, event.row */ break;
|
|
470
|
+
case 'delete': /* event.table, event.rowId */ break;
|
|
471
|
+
case 'flush:start': /* event.entryCount */ break;
|
|
472
|
+
case 'flush:complete': break;
|
|
473
|
+
case 'flush:error': /* event.error */ break;
|
|
474
|
+
case 'remote:poisoned': /* unrecoverable; see security-model.md */ break;
|
|
475
|
+
case 'credentials:meshMismatch': /* stored meshId != live; offer clearCredentials() */ break;
|
|
476
|
+
case 'compact:warning': /* outbox is large */ break;
|
|
477
|
+
// compact:auto:start / complete / skip / error / delayed:* — see docs/compaction.md
|
|
478
|
+
}
|
|
479
|
+
});
|
|
149
480
|
```
|
|
150
481
|
|
|
151
482
|
## What this is not
|
|
@@ -156,22 +487,45 @@ yarn demo:todo
|
|
|
156
487
|
- not Replicache
|
|
157
488
|
- not a hosted backend
|
|
158
489
|
- not a query engine over the cloud
|
|
159
|
-
- not a server
|
|
490
|
+
- not a server‑trusted merge layer
|
|
491
|
+
|
|
492
|
+
## What can go wrong
|
|
493
|
+
|
|
494
|
+
A short field guide. Detailed mitigations in the linked docs.
|
|
495
|
+
|
|
496
|
+
| Symptom | Likely cause | Where to look |
|
|
497
|
+
| --- | --- | --- |
|
|
498
|
+
| `MeshCredentialMismatchError` on connect | Same `dbName`, new mesh; stale credential record | `engine.clearCredentials()` then reconnect; `docs/credential-store.md` |
|
|
499
|
+
| `MeshEncryptionMismatchError` on connect | App flipped `encrypted` between sessions | Pin `encrypted` per `dbName`, never change |
|
|
500
|
+
| `remote:poisoned` event | Decode failure on a manifest, change file, or snapshot | `docs/security-model.md` — usually wrong key, schema drift, or remote tampering |
|
|
501
|
+
| Writes never appear on peer | Peer never compacted, or peer's poll interval is long, or remote dropped writes | Check `flush:complete` events; check remote folder by hand |
|
|
502
|
+
| Local store has rows that are "old" after re‑pair | Engine kept local data when you re‑paired with a fresh mesh | Either delete local DB on re‑pair, or accept the merge |
|
|
503
|
+
| Lost passphrase | No recovery | Passphrase is the key. Back it up out of band |
|
|
504
|
+
| Long‑offline device "lost" recent edits | Rehydrate replaced local state with the snapshot | Local writes already in the outbox survive; in‑flight uncommitted UI state does not |
|
|
505
|
+
| Compaction never runs | `autoCompact: false`, or no remote, or `compactAutoThreshold` never reached | `docs/compaction.md` — subscribe to `compact:auto:skip` |
|
|
506
|
+
| Two compactors race | No CAS in the adapter; small probability in small meshes | Use `serverManaged: true` for large meshes |
|
|
160
507
|
|
|
161
508
|
## Tests
|
|
162
509
|
|
|
163
|
-
|
|
510
|
+
```bash
|
|
511
|
+
yarn workspace @interocitor/core test
|
|
512
|
+
```
|
|
513
|
+
|
|
514
|
+
Adapter contract tests (run for every adapter):
|
|
164
515
|
|
|
165
516
|
```bash
|
|
166
|
-
yarn workspace interocitor test
|
|
517
|
+
yarn workspace @interocitor/core test webdav.adapter.contract
|
|
167
518
|
```
|
|
168
519
|
|
|
169
520
|
## Package context
|
|
170
521
|
|
|
171
|
-
|
|
522
|
+
Part of the Interocitor monorepo. See:
|
|
523
|
+
|
|
172
524
|
- root `README.md` — monorepo overview
|
|
173
|
-
- `
|
|
174
|
-
- `
|
|
525
|
+
- `docs/security-model.md` — threat model
|
|
526
|
+
- `docs/adapter-contract.md` — adapter requirements
|
|
527
|
+
- `docs/compaction.md` — compaction protocol & tuning
|
|
528
|
+
- `docs/credential-store.md` — credential persistence
|
|
175
529
|
|
|
176
530
|
## License
|
|
177
531
|
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* The adapter derives:
|
|
11
11
|
* wss://<worker>/notify/<prefix> (WebSocket invalidations via InterocitorRelay DO)
|
|
12
12
|
*/
|
|
13
|
-
import type { StorageAdapter, FileEntry } from '../core/types.ts';
|
|
13
|
+
import type { StorageAdapter, FileEntry, RemoteInvalidationPayload, RemoteInvalidationHooks } from '../core/types.ts';
|
|
14
14
|
export interface CloudflareAdapterConfig {
|
|
15
15
|
/** Worker IO base URL that includes prefix, e.g. https://worker/io/team-a */
|
|
16
16
|
baseUrl: string;
|
|
@@ -40,8 +40,10 @@ export declare class CloudflareAdapter implements StorageAdapter {
|
|
|
40
40
|
readonly name = "cloudflare";
|
|
41
41
|
private readonly config;
|
|
42
42
|
private authenticated;
|
|
43
|
+
private ensuredFolders;
|
|
43
44
|
constructor(config: CloudflareAdapterConfig);
|
|
44
45
|
private headers;
|
|
46
|
+
private parseBaseUrl;
|
|
45
47
|
private get ioBaseUrl();
|
|
46
48
|
private get notifyUrl();
|
|
47
49
|
private ioUrl;
|
|
@@ -53,15 +55,12 @@ export declare class CloudflareAdapter implements StorageAdapter {
|
|
|
53
55
|
*/
|
|
54
56
|
getHandshakeConfig(): string;
|
|
55
57
|
isAuthenticated(): boolean;
|
|
56
|
-
subscribeToInvalidations(onInvalidate: (payload:
|
|
57
|
-
type: string;
|
|
58
|
-
path: string;
|
|
59
|
-
ts: number;
|
|
60
|
-
}) => void, hooks?: {
|
|
61
|
-
onReady?: () => void;
|
|
62
|
-
onError?: () => void;
|
|
63
|
-
}): () => void;
|
|
58
|
+
subscribeToInvalidations(onInvalidate: (payload: RemoteInvalidationPayload) => void, hooks?: RemoteInvalidationHooks): () => void;
|
|
64
59
|
ensureFolder(path: string): Promise<void>;
|
|
60
|
+
/** Drop the per-session ensureFolder cache. Call after mesh swap, poison,
|
|
61
|
+
* or any state where a previous "this folder exists" observation must
|
|
62
|
+
* not be trusted. */
|
|
63
|
+
resetFolderCache(): void;
|
|
65
64
|
listFiles(path: string): Promise<FileEntry[]>;
|
|
66
65
|
listFolders(path: string): Promise<string[]>;
|
|
67
66
|
readFile(path: string): Promise<Uint8Array>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cloudflare.d.ts","sourceRoot":"","sources":["../../src/adapters/cloudflare.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"cloudflare.d.ts","sourceRoot":"","sources":["../../src/adapters/cloudflare.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,yBAAyB,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAEtH,MAAM,WAAW,uBAAuB;IACtC,6EAA6E;IAC7E,OAAO,EAAE,MAAM,CAAC;IAChB,6EAA6E;IAC7E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAUD;;;;;;;;;;;;;GAaG;AACH,wFAAwF;AACxF,MAAM,WAAW,yBAAyB;IACxC,kEAAkE;IAClE,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,iBAAkB,YAAW,cAAc;IACtD,QAAQ,CAAC,IAAI,gBAAgB;IAE7B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;IACjD,OAAO,CAAC,aAAa,CAAS;IAM9B,OAAO,CAAC,cAAc,CAA0B;gBAEpC,MAAM,EAAE,uBAAuB;IAI3C,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,YAAY;IAWpB,OAAO,KAAK,SAAS,GAMpB;IAED,OAAO,KAAK,SAAS,GAWpB;IAED,OAAO,CAAC,KAAK;IAKb,OAAO,CAAC,OAAO;IAMT,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAkBnC;;;OAGG;IACH,kBAAkB,IAAI,MAAM;IAK5B,eAAe,IAAI,OAAO;IAI1B,wBAAwB,CACtB,YAAY,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,EAC1D,KAAK,CAAC,EAAE,uBAAuB,GAC9B,MAAM,IAAI;IA2DP,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAe/C;;0BAEsB;IACtB,gBAAgB,IAAI,IAAI;IAIlB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAqB7C,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAe5C,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAa3C,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAcjE,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAWvC,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;CAsB/D"}
|