@le-space/orbitdb-storage-bridge 0.16.1 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -26,7 +26,7 @@ Where "somewhere else" is, is a choice:
26
26
  | Backend | Needs | |
27
27
  | --- | --- | --- |
28
28
  | `aleph` | nothing | the only backend a browser can write to with no key; ingest only |
29
- | `aleph-pin` | a wallet | the signed STORE message that makes Aleph keep it |
29
+ | `aleph-pin` | a signing key: the paying account's, or one it authorised | the signed STORE message that makes Aleph keep it, paid in credits |
30
30
  | `pinata` | a scoped JWT | pin-by-CID as well as CAR upload |
31
31
  | `lighthouse` | an API key | pay once, stored in perpetuity |
32
32
  | `memory` | nothing | in-process, for tests and demos |
@@ -83,6 +83,8 @@ Galaxy A57 with the same key brought the database back and wrote to it. Source:
83
83
  | [docs/STORAGE-BACKENDS.md](docs/STORAGE-BACKENDS.md) | every backend evaluated — prices, limits, and what has been verified against a live account |
84
84
  | [docs/RECOVERY-ON-A-SECOND-DEVICE.md](docs/RECOVERY-ON-A-SECOND-DEVICE.md) | the passkey recovery procedure, step by step |
85
85
  | [docs/CAR-BACKUP.md](docs/CAR-BACKUP.md) | CAR-based timestamped backups |
86
+ | [docs/APP-BACKUP.md](docs/APP-BACKUP.md) | an application's backup in one sealed file: several databases, a keyring header, kept on Aleph from the browser, restored by merging |
87
+ | [docs/LIVE-BACKUP.md](docs/LIVE-BACKUP.md) | increments between two full backups: what was written since the last one, shipped as one small CAR, with nobody clicking and no peer online |
86
88
  | [SVELTE-COMPONENTS.md](SVELTE-COMPONENTS.md) | the browser components — older than this repository's copy of them; [docs/STORACHA-UI-HISTORY.md](docs/STORACHA-UI-HISTORY.md) records what each piece could do and where it went |
87
89
  | [docs/LOGGING.md](docs/LOGGING.md) | debug namespaces, in Node and in a browser |
88
90
  | [ROADMAP.md](ROADMAP.md) | what is planned |
@@ -0,0 +1,300 @@
1
+ /**
2
+ * @fileoverview An application's backup in one sealed file (#147).
3
+ *
4
+ * Two applications, Le-Space/belege and Le-Space/invoice, back up the same way:
5
+ * every OrbitDB database they keep, and blocks of their own (belege's receipt
6
+ * files), go into one CAR whose root is a manifest; the CAR is sealed with the
7
+ * application's key; one upload, one STORE. This module is that file,
8
+ * generalised from belege's `archive.js` (Le-Space/belege#77), so that the
9
+ * applications stop inventing their own.
10
+ *
11
+ * ## The file
12
+ *
13
+ * "OSBA" | version | header length (4 bytes, big-endian) | header | envelope
14
+ *
15
+ * - **The header** is the caller's, and is read without a key
16
+ * ({@link readAppBackupHeader}). It is room for a keyring: invoice puts its
17
+ * vault there, one sealed slot per passkey, so that any registered passkey
18
+ * finds the key to the rest. This module does not interpret it. Its SHA-256
19
+ * is in the manifest, so a header swapped after the backup was made is
20
+ * refused once the body is open.
21
+ * - **The envelope** is the package's own (`OSBE`, see ./backends/encryption.js)
22
+ * around the CAR, encrypted by the caller's `encrypt`. The package holds no
23
+ * keys, here as everywhere.
24
+ * - **The CAR's root** is a dag-cbor manifest: `kind` and `v` say what this is,
25
+ * `app` whose it is, and `metadata` is `bundleDatabases`'s, naming every
26
+ * database with its heads. `extra` is the application's own (belege lists its
27
+ * receipt files there).
28
+ *
29
+ * Everything is built in memory, as the rest of the package does; a backup is
30
+ * as large as the books it holds.
31
+ *
32
+ * @requires ./extract-blocks.js - bundleDatabases
33
+ * @requires ./backup-car.js - createCARFromBlocks
34
+ * @requires ./restore-cid.js - the verifying CAR reader, restoreFromBlocks
35
+ */
36
+
37
+ import * as dagCbor from "@ipld/dag-cbor";
38
+ import { CID } from "multiformats/cid";
39
+ import { sha256 } from "multiformats/hashes/sha2";
40
+ import { bundleDatabases } from "./extract-blocks.js";
41
+ import { createCARFromBlocks } from "./backup-car.js";
42
+ import { readBlocksFromCAR, restoreFromBlocks } from "./restore-cid.js";
43
+ import { wrapEnvelope, readEnvelope } from "./backends/encryption.js";
44
+
45
+ /** `OSBA` — orbitdb-storage-bridge, application backup. */
46
+ export const APP_BACKUP_MAGIC = new Uint8Array([0x4f, 0x53, 0x42, 0x41]);
47
+
48
+ /** Bumped when the file changes in a way readers must notice. */
49
+ export const APP_BACKUP_VERSION = 1;
50
+
51
+ /** What the manifest at the CAR's root calls itself. */
52
+ export const APP_BACKUP_KIND = "app-backup";
53
+
54
+ const PREAMBLE = APP_BACKUP_MAGIC.length + 1 + 4;
55
+
56
+ const hex = (bytes) => [...bytes].map((b) => b.toString(16).padStart(2, "0")).join("");
57
+
58
+ async function sha256Hex(bytes) {
59
+ return hex(new Uint8Array(await globalThis.crypto.subtle.digest("SHA-256", bytes)));
60
+ }
61
+
62
+ /**
63
+ * @typedef {object} AppBackupManifest
64
+ * @property {"app-backup"} kind
65
+ * @property {number} v - {@link APP_BACKUP_VERSION}
66
+ * @property {string} app - whose backup this is, e.g. `"invoice"`
67
+ * @property {string} createdAt - ISO
68
+ * @property {string} [appVersion]
69
+ * @property {any} metadata - `bundleDatabases`'s: `databases` with each one's
70
+ * `address`, `name`, `type`, `manifestCID`, `entryCount`, `heads`, and
71
+ * `collection` when the databases were given by name
72
+ * @property {string} [header] - hex SHA-256 of the file's header, when it has one
73
+ * @property {any} [extra] - the application's own, as it passed it
74
+ */
75
+
76
+ /**
77
+ * @typedef {{ stage: "database", index: number, total: number, name: string, entries: number }
78
+ * | { stage: "sealing", bytes: number }} AppBackupProgress
79
+ */
80
+
81
+ /**
82
+ * Build a backup file.
83
+ *
84
+ * @param {object} params
85
+ * @param {string} params.app - whose backup this is; {@link openAppBackup} can insist on it
86
+ * @param {Object[]|Object<string, Object>} params.databases - open OrbitDB
87
+ * databases. Given by name, the names go into the metadata as `collection`.
88
+ * @param {(plaintext: Uint8Array) => Promise<{ ciphertext: Uint8Array, iv: Uint8Array }>} params.encrypt -
89
+ * the caller's, with the caller's key
90
+ * @param {Map<string, { bytes: Uint8Array }>} [params.blocks] - blocks of the
91
+ * application's own, by CID string: they travel in the CAR and come back into
92
+ * the blockstore on restore
93
+ * @param {Uint8Array} [params.header] - read without a key, e.g. a keyring
94
+ * @param {string} [params.appVersion]
95
+ * @param {any} [params.extra] - anything dag-cbor can encode
96
+ * @param {() => Date} [params.now]
97
+ * @param {(progress: AppBackupProgress) => void} [params.onProgress]
98
+ * @returns {Promise<{ bytes: Uint8Array, manifest: AppBackupManifest, blocks: number, carBytes: number }>}
99
+ */
100
+ export async function buildAppBackup({
101
+ app,
102
+ databases,
103
+ encrypt,
104
+ blocks: own,
105
+ header,
106
+ appVersion,
107
+ extra,
108
+ now = () => new Date(),
109
+ onProgress,
110
+ }) {
111
+ if (typeof app !== "string" || !app) throw new Error("buildAppBackup needs the application's name as `app`");
112
+ if (typeof encrypt !== "function") throw new Error("buildAppBackup needs an `encrypt` function: the package holds no keys");
113
+ if (header !== undefined && !(header instanceof Uint8Array)) throw new Error("A backup's header is bytes");
114
+ if (own !== undefined && !(own instanceof Map)) throw new Error("An application's own blocks come as a Map, by CID string");
115
+
116
+ const at = now();
117
+ const names = Array.isArray(databases) ? null : Object.keys(databases ?? {});
118
+ const { blocks, metadata } = await bundleDatabases(databases, {
119
+ timestamp: at.getTime(),
120
+ onProgress: (p) =>
121
+ onProgress?.({
122
+ stage: "database",
123
+ index: p.index,
124
+ total: p.total,
125
+ name: names?.[p.index] ?? p.name,
126
+ entries: p.entries,
127
+ }),
128
+ });
129
+ if (names) metadata.databases.forEach((d, i) => (d.collection = names[i]));
130
+ for (const [name, block] of own ?? []) if (!blocks.has(name)) blocks.set(name, block);
131
+
132
+ /** @type {AppBackupManifest} */
133
+ const manifest = {
134
+ kind: APP_BACKUP_KIND,
135
+ v: APP_BACKUP_VERSION,
136
+ app,
137
+ createdAt: at.toISOString(),
138
+ ...(appVersion ? { appVersion } : {}),
139
+ metadata,
140
+ ...(header?.length ? { header: await sha256Hex(header) } : {}),
141
+ ...(extra !== undefined ? { extra } : {}),
142
+ };
143
+ const root = dagCbor.encode(manifest);
144
+ const rootCid = CID.createV1(dagCbor.code, await sha256.digest(root));
145
+ blocks.set(rootCid.toString(), { cid: rootCid, bytes: root });
146
+
147
+ const car = await createCARFromBlocks(blocks, rootCid.toString());
148
+ onProgress?.({ stage: "sealing", bytes: car.length });
149
+ const { ciphertext, iv } = await encrypt(car);
150
+ const envelope = wrapEnvelope(ciphertext, iv);
151
+
152
+ const head = header ?? new Uint8Array(0);
153
+ const bytes = new Uint8Array(PREAMBLE + head.length + envelope.length);
154
+ bytes.set(APP_BACKUP_MAGIC, 0);
155
+ bytes[APP_BACKUP_MAGIC.length] = APP_BACKUP_VERSION;
156
+ new DataView(bytes.buffer).setUint32(APP_BACKUP_MAGIC.length + 1, head.length);
157
+ bytes.set(head, PREAMBLE);
158
+ bytes.set(envelope, PREAMBLE + head.length);
159
+ return { bytes, manifest, blocks: blocks.size, carBytes: car.length };
160
+ }
161
+
162
+ /** Is this an application backup this module wrote, of any version? */
163
+ export function isAppBackup(bytes) {
164
+ return (
165
+ bytes instanceof Uint8Array &&
166
+ bytes.length >= PREAMBLE &&
167
+ APP_BACKUP_MAGIC.every((byte, index) => bytes[index] === byte)
168
+ );
169
+ }
170
+
171
+ /**
172
+ * The header, without a key: what a keyring needs before anything can be opened.
173
+ *
174
+ * @param {Uint8Array} bytes
175
+ * @returns {{ version: number, header: Uint8Array, body: Uint8Array }}
176
+ */
177
+ export function readAppBackupHeader(bytes) {
178
+ if (!isAppBackup(bytes)) throw new Error("This is not an application backup.");
179
+ const version = bytes[APP_BACKUP_MAGIC.length];
180
+ if (version !== APP_BACKUP_VERSION) {
181
+ throw new Error(
182
+ `This backup was written in format ${version}, and this build reads ${APP_BACKUP_VERSION}. ` +
183
+ "Upgrade @le-space/orbitdb-storage-bridge to open it.",
184
+ );
185
+ }
186
+ const length = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(APP_BACKUP_MAGIC.length + 1);
187
+ if (PREAMBLE + length > bytes.length) throw new Error("This backup is cut short.");
188
+ return {
189
+ version,
190
+ header: bytes.subarray(PREAMBLE, PREAMBLE + length),
191
+ body: bytes.subarray(PREAMBLE + length),
192
+ };
193
+ }
194
+
195
+ /**
196
+ * Open a backup file: the envelope decrypted, every block checked against its
197
+ * own CID, the manifest found among them, and the header checked against it.
198
+ *
199
+ * A wrong key fails in `decrypt` itself (AES-GCM refuses it), and its error
200
+ * comes through unchanged, for the caller to name.
201
+ *
202
+ * @param {Uint8Array} bytes
203
+ * @param {object} options
204
+ * @param {(ciphertext: Uint8Array, iv: Uint8Array) => Promise<Uint8Array>} options.decrypt
205
+ * @param {string} [options.app] - refuse a backup of another application
206
+ * @returns {Promise<{ header: Uint8Array, manifest: AppBackupManifest, blocks: Map<string, { cid: any, bytes: Uint8Array }> }>}
207
+ */
208
+ export async function openAppBackup(bytes, { decrypt, app } = {}) {
209
+ if (typeof decrypt !== "function") throw new Error("openAppBackup needs a `decrypt` function");
210
+ const { header, body } = readAppBackupHeader(bytes);
211
+ const { ciphertext, iv } = readEnvelope(body);
212
+ const car = await decrypt(ciphertext, iv);
213
+ const blocks = await readBlocksFromCAR(car, { verify: true });
214
+
215
+ let manifest;
216
+ for (const [name, { bytes: block }] of blocks) {
217
+ if (CID.parse(name).code !== dagCbor.code) continue;
218
+ let value;
219
+ try {
220
+ value = dagCbor.decode(block);
221
+ } catch {
222
+ continue;
223
+ }
224
+ if (value?.kind === APP_BACKUP_KIND && value?.v === APP_BACKUP_VERSION) {
225
+ manifest = value;
226
+ break;
227
+ }
228
+ }
229
+ if (!manifest) throw new Error("This backup holds no manifest.");
230
+ if (app && manifest.app !== app) throw new Error(`This is a backup of ${manifest.app}, not of ${app}.`);
231
+ // The header is outside the seal; the manifest inside it says which header belongs.
232
+ const expected = manifest.header ?? null;
233
+ const actual = header.length ? await sha256Hex(header) : null;
234
+ if (expected !== actual) throw new Error("This backup's header is not the one it was made with.");
235
+ return { header, manifest, blocks };
236
+ }
237
+
238
+ /**
239
+ * @typedef {{ stage: "database", index: number, total: number, name: string, joined: number }} AppRestoreProgress
240
+ */
241
+
242
+ /**
243
+ * Put an opened backup back into books that are open here.
244
+ *
245
+ * Every database the backup names must be one of these books, by address:
246
+ * restoring merges, and merging another's books into these is never wanted.
247
+ * The databases go back through {@link restoreFromBlocks}: every block into the
248
+ * blockstore (the application's own with them), each database reopened and its
249
+ * heads joined. Joining merges — what is here stays, what the backup holds is
250
+ * added, nothing is deleted. A head that cannot be joined fails the restore
251
+ * with the reason, where `restoreFromBlocks` alone only warns.
252
+ *
253
+ * The open databases are closed and reopened behind the caller's back; reload
254
+ * the application's handles afterwards.
255
+ *
256
+ * @param {object} params
257
+ * @param {any} params.orbitdb
258
+ * @param {{ manifest: AppBackupManifest, blocks: Map<string, any> }} params.opened - from {@link openAppBackup}
259
+ * @param {Record<string, string> | string[]} params.addresses - these books'
260
+ * database addresses, by collection or as a list
261
+ * @param {Record<string, any>} [params.open] - what `orbitdb.open` needs for them (`encryption`, `AccessController`, …)
262
+ * @param {(progress: AppRestoreProgress) => void} [params.onProgress]
263
+ * @returns {Promise<{ databases: { address: string, collection?: string, joined: number, entries: number | null }[] }>}
264
+ */
265
+ export async function restoreAppBackup({ orbitdb, opened, addresses, open, onProgress }) {
266
+ const metadata = opened?.manifest?.metadata;
267
+ if (!metadata?.databases?.length) throw new Error("This backup names no databases.");
268
+ const byAddress = Array.isArray(addresses)
269
+ ? Object.fromEntries(addresses.map((address) => [String(address), undefined]))
270
+ : Object.fromEntries(Object.entries(addresses ?? {}).map(([collection, address]) => [String(address), collection]));
271
+ for (const d of metadata.databases) {
272
+ if (!(String(d.address) in byAddress)) {
273
+ throw new Error("This backup holds databases these books do not have.");
274
+ }
275
+ }
276
+
277
+ const warnings = [];
278
+ const restored = await restoreFromBlocks(orbitdb, opened.blocks, metadata, {
279
+ open,
280
+ log: { info() {}, debug() {}, warn: (message) => warnings.push(String(message)) },
281
+ onProgress: (p) =>
282
+ onProgress?.({
283
+ stage: "database",
284
+ index: p.index,
285
+ total: p.total,
286
+ name: byAddress[p.address] ?? p.address,
287
+ joined: p.joined,
288
+ }),
289
+ });
290
+ const failed = warnings.find((w) => /could not join head/.test(w));
291
+ if (failed) throw new Error(`The backup could not be put back: ${failed.replace(/^\W+/, "")}`);
292
+ return {
293
+ databases: restored.databases.map((d) => ({
294
+ address: d.address,
295
+ ...(byAddress[d.address] ? { collection: byAddress[d.address] } : {}),
296
+ joined: d.joined,
297
+ entries: d.entries ?? null,
298
+ })),
299
+ };
300
+ }
@@ -9,11 +9,29 @@
9
9
  *
10
10
  * ## No tokens move
11
11
  *
12
- * The wallet **signs a string**. Aleph then checks that the signing address has
13
- * enough balance or credit to cover what it is being asked to keep. There is no
14
- * transaction, no gas and no transfer: the token is the evidence, not the
15
- * payment. Worth stating because "pay for storage with a wallet" reads as the
16
- * opposite.
12
+ * The wallet **signs a string**. Aleph then checks that the account the STORE
13
+ * is for has enough credit to cover what it is being asked to keep. There is no
14
+ * transaction, no gas and no transfer at that moment: credits are drawn from the
15
+ * account by the hour, afterwards. Worth stating because "pay for storage with a
16
+ * wallet" reads as the opposite.
17
+ *
18
+ * ## Paid in credits
19
+ *
20
+ * Measured against `api2.aleph.im` on 2026-10-03, with throwaway accounts and
21
+ * 2 MiB of random bytes:
22
+ *
23
+ * - A STORE whose content names no `payment` is booked as **`hold`**: ALEPH
24
+ * tokens locked on the account, a model Aleph has deprecated. It was
25
+ * processed at once for an account holding nothing, so "processed" says
26
+ * nothing about cover there.
27
+ * - With `payment: { type: "credit" }` — what Aleph's own CLI sends by default —
28
+ * Aleph wants credit for at least a day (`min_runtime_days: 1`) and rejects
29
+ * the STORE otherwise: 107.8 credits for 2 MiB, about 54 per MiB and day.
30
+ * - It is the account in `content.address` whose credit is checked and to
31
+ * which the cost is booked, not the sender: a delegate with no credit at all
32
+ * stored for a funded owner, and the cost appeared on the owner.
33
+ *
34
+ * So {@link buildStoreMessage} pays in credits unless told otherwise.
17
35
  *
18
36
  * ## Injected, not imported
19
37
  *
@@ -50,6 +68,9 @@ import { BackendError } from "./types.js";
50
68
  export const DEFAULT_ALEPH_API_HOST = "https://api2.aleph.im";
51
69
  export const DEFAULT_ALEPH_CHANNEL = "ALEPH-CLOUDSOLUTIONS";
52
70
 
71
+ /** How a STORE may be paid for; `null` leaves `payment` out, which Aleph books as `hold`. */
72
+ export const STORE_PAYMENTS = Object.freeze(["credit", "hold"]);
73
+
53
74
  /** Hex sha-256 of a string, via WebCrypto — present in browsers and in Node 18+. */
54
75
  async function sha256Hex(payload) {
55
76
  const digest = await globalThis.crypto.subtle.digest(
@@ -71,10 +92,27 @@ async function sha256Hex(payload) {
71
92
  * by default
72
93
  * @param {string} args.cid - what to keep
73
94
  * @param {string} [args.channel]
95
+ * @param {"credit" | "hold" | null} [args.payment] - `credit` by default; `null`
96
+ * sends no `payment`, as every version up to 0.16.1 did, which Aleph books as
97
+ * `hold`
74
98
  * @param {number} [args.now] - seconds; injected so a test is not a clock
75
99
  * @param {(payload: string) => Promise<string>} [args.hasher]
76
100
  */
77
- export async function buildStoreMessage({ sender, owner, cid, channel = DEFAULT_ALEPH_CHANNEL, now, hasher = sha256Hex }) {
101
+ export async function buildStoreMessage({
102
+ sender,
103
+ owner,
104
+ cid,
105
+ channel = DEFAULT_ALEPH_CHANNEL,
106
+ payment = "credit",
107
+ now,
108
+ hasher = sha256Hex,
109
+ }) {
110
+ if (payment !== null && !STORE_PAYMENTS.includes(payment)) {
111
+ throw new BackendError(
112
+ "INVALID_BACKEND",
113
+ `a STORE is paid by "credit" or "hold", or names no payment (null), not ${JSON.stringify(payment)}`,
114
+ );
115
+ }
78
116
  const time = now ?? Date.now() / 1000;
79
117
  const content = {
80
118
  // Whose STORE this is. Equal to the sender unless the owner's `security`
@@ -83,6 +121,9 @@ export async function buildStoreMessage({ sender, owner, cid, channel = DEFAULT_
83
121
  // "the thing to keep is an IPFS CID" — not the envelope's item_type
84
122
  item_type: "ipfs",
85
123
  item_hash: cid,
124
+ // How that account pays. Without it Aleph books the STORE as `hold`, and an
125
+ // account funded with credits covers nothing (see the file's header).
126
+ ...(payment ? { payment: { type: payment } } : {}),
86
127
  time,
87
128
  };
88
129
  const item_content = JSON.stringify(content);
@@ -114,15 +155,20 @@ export const signaturePayload = (message) =>
114
155
  * `personal_sign`, or anything shaped like it. The library never sees a key.
115
156
  * @param {string} [options.apiHost]
116
157
  * @param {string} [options.channel]
158
+ * @param {"credit" | "hold" | null} [options.payment] - see {@link buildStoreMessage}
117
159
  * @param {(payload: string) => Promise<string>} [options.hasher]
118
160
  * @param {typeof fetch} [options.fetch]
119
161
  * @param {() => number} [options.now] - seconds
120
162
  * @returns {(cid: string, meta?: object) => Promise<{ itemHash: string, status: string }>}
163
+ * `status` is what Aleph answered: `processed`, or `pending` until it has
164
+ * checked the signature, the authorization and the credit — follow a pending
165
+ * one with {@link waitForMessage}.
121
166
  */
122
167
  export function createAlephPin(options = {}) {
123
168
  const { sender, sign, owner } = options;
124
169
  const apiHost = options.apiHost || DEFAULT_ALEPH_API_HOST;
125
170
  const channel = options.channel || DEFAULT_ALEPH_CHANNEL;
171
+ const payment = options.payment === undefined ? "credit" : options.payment;
126
172
  const hasher = options.hasher || sha256Hex;
127
173
  const doFetch = options.fetch || globalThis.fetch;
128
174
  const now = options.now;
@@ -132,7 +178,7 @@ export function createAlephPin(options = {}) {
132
178
  if (typeof doFetch !== "function") throw new BackendError("INVALID_BACKEND", "createAlephPin needs fetch");
133
179
 
134
180
  return async function pin(cid) {
135
- const unsigned = await buildStoreMessage({ sender, owner, cid, channel, hasher, now: now?.() });
181
+ const unsigned = await buildStoreMessage({ sender, owner, cid, channel, payment, hasher, now: now?.() });
136
182
  const signature = await sign(sender, signaturePayload(unsigned));
137
183
  const message = {
138
184
  ...unsigned,
@@ -165,6 +211,126 @@ export function createAlephPin(options = {}) {
165
211
  };
166
212
  }
167
213
 
214
+ /** Statuses after which a message no longer changes by itself. */
215
+ const SETTLED = new Set(["processed", "rejected", "removing", "removed", "forgotten"]);
216
+
217
+ /**
218
+ * Follow a message until Aleph has decided about it.
219
+ *
220
+ * A STORE answered with 202 is `pending`: Aleph has taken it, and checks the
221
+ * signature, the authorization and the account's credit afterwards. A
222
+ * rejected one says why. For a credit STORE without enough credit,
223
+ * `details.errors[0]` names `account_credits`, `required_credits` and
224
+ * `min_runtime_days`, with `errorCode` 6 (measured 2026-10-03).
225
+ *
226
+ * Never throws for what Aleph answers: a message still pending when the time is
227
+ * up comes back as `{ status: "pending", timedOut: true }`, for the caller to
228
+ * report as exactly that.
229
+ *
230
+ * @param {string} itemHash
231
+ * @param {object} [options]
232
+ * @param {string} [options.apiHost]
233
+ * @param {typeof fetch} [options.fetch]
234
+ * @param {number} [options.timeout] - ms in all; default 60 000
235
+ * @param {number} [options.interval] - ms between asks; default 2 000
236
+ * @param {(ms: number) => Promise<void>} [options.sleep] - injected so a test is not a clock
237
+ * @returns {Promise<{ itemHash: string, status: string, errorCode?: number, details?: unknown, timedOut?: true }>}
238
+ */
239
+ export async function waitForMessage(itemHash, options = {}) {
240
+ const apiHost = options.apiHost || DEFAULT_ALEPH_API_HOST;
241
+ const doFetch = options.fetch || globalThis.fetch;
242
+ const timeout = options.timeout ?? 60_000;
243
+ const interval = options.interval ?? 2_000;
244
+ const sleep = options.sleep || ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
245
+ if (!itemHash) throw new BackendError("INVALID_BACKEND", "waitForMessage needs the message's item hash");
246
+ if (typeof doFetch !== "function") throw new BackendError("INVALID_BACKEND", "waitForMessage needs fetch");
247
+
248
+ // Counted in asks rather than read off a clock, so an injected `sleep` is enough for a test.
249
+ const asks = Math.max(1, Math.floor(timeout / interval) + 1);
250
+ let status = "pending";
251
+ for (let ask = 0; ask < asks; ask++) {
252
+ if (ask > 0) await sleep(interval);
253
+ let response;
254
+ try {
255
+ response = await doFetch(`${apiHost}/api/v0/messages/${itemHash}`);
256
+ } catch {
257
+ continue; // the network, not Aleph's answer: ask again
258
+ }
259
+ // 404: not known on this node yet. Anything else not ok: ask again.
260
+ if (!response.ok) continue;
261
+ const body = await response.json().catch(() => ({}));
262
+ if (typeof body?.status === "string") status = body.status;
263
+ if (SETTLED.has(status)) {
264
+ return {
265
+ itemHash,
266
+ status,
267
+ ...(body.error_code != null ? { errorCode: body.error_code } : {}),
268
+ ...(body.details != null ? { details: body.details } : {}),
269
+ };
270
+ }
271
+ }
272
+ return { itemHash, status, timedOut: true };
273
+ }
274
+
275
+ /**
276
+ * The STORE messages kept for an account, newest first: its own, and those a
277
+ * delegate sent for it.
278
+ *
279
+ * This is how an empty device finds a backup with nothing but the paying
280
+ * account's public address. Aleph's `addresses` filter matches the *sender*,
281
+ * so it misses a delegate's STORE; `owners` matches `content.address`, the
282
+ * account the STORE is for (measured 2026-10-03).
283
+ *
284
+ * @param {object} options
285
+ * @param {string} options.owner - the paying account, EIP-55 checksummed (Aleph
286
+ * keys accounts by that form)
287
+ * @param {string} [options.channel] - one channel; every channel when absent
288
+ * @param {string} [options.apiHost]
289
+ * @param {typeof fetch} [options.fetch]
290
+ * @param {number} [options.pagination] - per page; default 50
291
+ * @param {number} [options.page] - from 1
292
+ * @returns {Promise<{ stores: Array<{ cid: string, itemHash: string, sender: string, owner: string, time: number, channel?: string, payment?: string }>, total: number | null }>}
293
+ * `cid` is what the STORE keeps — for an upload through `createAlephBackend`,
294
+ * the handle's `id`
295
+ */
296
+ export async function listAlephStores(options = {}) {
297
+ const { owner, channel } = options;
298
+ const apiHost = options.apiHost || DEFAULT_ALEPH_API_HOST;
299
+ const doFetch = options.fetch || globalThis.fetch;
300
+ if (!owner) throw new BackendError("INVALID_BACKEND", "listAlephStores needs the paying account's address as `owner`");
301
+ if (typeof doFetch !== "function") throw new BackendError("INVALID_BACKEND", "listAlephStores needs fetch");
302
+
303
+ const url = new URL(`${apiHost}/api/v0/messages.json`);
304
+ url.searchParams.set("owners", owner);
305
+ url.searchParams.set("msgTypes", "STORE");
306
+ if (channel) url.searchParams.set("channels", channel);
307
+ url.searchParams.set("pagination", String(options.pagination ?? 50));
308
+ url.searchParams.set("page", String(options.page ?? 1));
309
+
310
+ const response = await doFetch(url.toString());
311
+ if (!response.ok) {
312
+ throw new BackendError("UNSUPPORTED", `Aleph did not list the STORE messages: ${response.status}`);
313
+ }
314
+ const body = await response.json().catch(() => ({}));
315
+ const messages = Array.isArray(body?.messages) ? body.messages : [];
316
+ const same = (a, b) => String(a).toLowerCase() === String(b).toLowerCase();
317
+ const stores = messages
318
+ .map((m) => ({
319
+ cid: String(m?.content?.item_hash ?? ""),
320
+ itemHash: String(m?.item_hash ?? ""),
321
+ sender: String(m?.sender ?? ""),
322
+ owner: String(m?.content?.address ?? ""),
323
+ time: Number(m?.content?.time ?? m?.time ?? 0),
324
+ ...(m?.channel ? { channel: m.channel } : {}),
325
+ ...(m?.content?.payment?.type ? { payment: m.content.payment.type } : {}),
326
+ }))
327
+ // Aleph answers what it was asked; keeping only this owner's costs nothing
328
+ // and keeps a misread filter from handing back someone else's files.
329
+ .filter((store) => store.cid && store.itemHash && same(store.owner, owner))
330
+ .sort((a, b) => b.time - a.time);
331
+ return { stores, total: Number.isFinite(body?.pagination_total) ? body.pagination_total : null };
332
+ }
333
+
168
334
  /** The reserved channel and aggregate key Aleph keeps permissions in. */
169
335
  export const SECURITY = "security";
170
336
 
@@ -227,8 +393,9 @@ export async function buildAuthorizationMessage({ owner, authorizations, now, ha
227
393
  * (`types: ["STORE"]`, one channel), and revoked on its own. The owner signs
228
394
  * these grants; the delegates then pass `owner` to {@link createAlephPin}.
229
395
  *
230
- * Which balance Aleph charges for a STORE sent by a delegate is not stated in
231
- * its documentation; measure it before relying on it.
396
+ * Aleph's documentation does not say whose credit pays for a STORE a delegate
397
+ * sends. Measured on 2026-10-03: the owner's, the account in `content.address`
398
+ * (see the file's header).
232
399
  *
233
400
  * Aleph keys accounts by their EIP-55 checksummed address; pass `owner` in that
234
401
  * form, or `read()` finds nothing.
@@ -104,8 +104,9 @@ export function createAlephBackend(options = {}) {
104
104
  // No key exists to leak.
105
105
  browserSafeAuth: true,
106
106
  delegation: false,
107
- // The IPFS host offers no listing, and the messages API would need the
108
- // wallet address this driver deliberately does not hold.
107
+ // The IPFS host offers no listing, and the messages API needs the paying
108
+ // account's address, which this driver deliberately does not hold:
109
+ // `listAlephStores` in ./aleph-pin.js takes it.
109
110
  listing: false,
110
111
  // Unpinning is a FORGET message, so it belongs with `pin` rather than
111
112
  // here. Claiming deletion the driver cannot perform would be worse than
@@ -167,8 +168,11 @@ export function createAlephBackend(options = {}) {
167
168
  size: bytes.length,
168
169
  ...(meta.name ? { name: meta.name } : {}),
169
170
  // Said out loud in the handle, because "stored" and "kept" are not the
170
- // same thing here and a caller should not have to read this file.
171
- retained: Boolean(pin),
171
+ // same thing here and a caller should not have to read this file. An
172
+ // upload is never kept by itself, `pin` or not: keeping is a STORE,
173
+ // which `pinCid` sends. (Up to 0.16.1 this said `Boolean(pin)`, a
174
+ // promise no call had kept.)
175
+ retained: false,
172
176
  };
173
177
  },
174
178
 
@@ -190,8 +194,21 @@ export function createAlephBackend(options = {}) {
190
194
  * predicted, and the reason `pinCid` here does not upload anything.
191
195
  */
192
196
  backend.pinCid = async (cid, meta = {}) => {
193
- await pin(cid, meta);
194
- return { id: cid, cid, backend: "aleph", retained: true, ...(meta.name ? { name: meta.name } : {}) };
197
+ const kept = await pin(cid, meta);
198
+ // A pin that reports Aleph's answer (createAlephPin does) is believed:
199
+ // retained only once the STORE is processed, and a pending one says so,
200
+ // with the item hash to follow it by (waitForMessage). A pin that
201
+ // reports nothing keeps the old answer.
202
+ const status = typeof kept?.status === "string" ? kept.status : undefined;
203
+ return {
204
+ id: cid,
205
+ cid,
206
+ backend: "aleph",
207
+ retained: status === undefined || status === "processed",
208
+ ...(status ? { status } : {}),
209
+ ...(kept?.itemHash ? { itemHash: kept.itemHash } : {}),
210
+ ...(meta.name ? { name: meta.name } : {}),
211
+ };
195
212
  };
196
213
  }
197
214
 
@@ -0,0 +1,734 @@
1
+ /**
2
+ * @fileoverview Incremental live backup: what was written since the last
3
+ * increment, shipped as one small CAR (#144).
4
+ *
5
+ * A full backup (`bundleDatabases`, `buildAppBackup`) is the baseline, and it
6
+ * stays one. This module adds what happens *between* two of them: a block
7
+ * should be durable somewhere else soon after it is written, without anyone
8
+ * clicking "back up" and without another peer being online.
9
+ *
10
+ * ## Write-behind, never write-through
11
+ *
12
+ * A local write never waits for the network. The watcher keeps, per database,
13
+ * the heads it last shipped; a flush walks from the current heads back over
14
+ * `next` until it meets an entry it has already shipped, and everything it
15
+ * passed on the way is the increment. Entries that arrived by replication from
16
+ * another device are reachable from the heads like any other, so they ship
17
+ * too — otherwise each device would back up only its own writes.
18
+ *
19
+ * The walk needs to know what was shipped, and the previous heads alone do not
20
+ * say it: a branch another writer forked from an old entry leads back past
21
+ * them. So the set of shipped hashes is kept in memory and, after a restart,
22
+ * rebuilt once from the persisted heads by walking the log. What has to be
23
+ * persisted is therefore tiny — a sequence number and a few hashes per
24
+ * database — and a crash between "written" and "shipped" loses nothing: the
25
+ * next flush finds the entry where it left it.
26
+ *
27
+ * ## One upload, one STORE
28
+ *
29
+ * An increment is one CAR whose root is a dag-cbor manifest
30
+ * (`{ kind: "increment", v: 1, sequence, base, previous, metadata }`), so a
31
+ * single blob is self-describing. Every increment carries each database's
32
+ * manifest and access controller (a few hundred bytes) and the identity of
33
+ * every writer it has entries from, so a chain of increments restores on its
34
+ * own, with no base, if it is complete from the beginning; and so
35
+ * `restoreFromBlocks` can verify each one the way it verifies a backup.
36
+ *
37
+ * The CAR goes through any `StorageBackend`'s `putBlob`. A backend that says
38
+ * `retained: false` — Aleph, where an upload is ingest and keeping is a
39
+ * STORE — is then asked to keep it with `pinCid`. Sealing is the caller's,
40
+ * through `withEncryption` around the backend: nothing here changes.
41
+ *
42
+ * ## When the backend is unreachable
43
+ *
44
+ * The flush fails, the state stays as it was, and the next flush ships
45
+ * everything that is still pending — the increment just grows. `status()`
46
+ * says how many updates are waiting, when the last increment went out and
47
+ * what the last failure was, so an application can show "last backed up …".
48
+ *
49
+ * @requires ./backup-car.js - createCARFromBlocks
50
+ * @requires ./restore-cid.js - readBlocksFromCAR, restoreFromBlocks
51
+ */
52
+
53
+ import * as dagCbor from "@ipld/dag-cbor";
54
+ import * as Block from "multiformats/block";
55
+ import { CID } from "multiformats/cid";
56
+ import { sha256 } from "multiformats/hashes/sha2";
57
+ import { base58btc } from "multiformats/bases/base58";
58
+ import { createCARFromBlocks } from "./backup-car.js";
59
+ import { readBlocksFromCAR, restoreFromBlocks } from "./restore-cid.js";
60
+
61
+ /** What the manifest at an increment's root calls itself. */
62
+ export const INCREMENT_KIND = "increment";
63
+
64
+ /** Bumped when the manifest changes in a way readers must notice. */
65
+ export const INCREMENT_VERSION = 1;
66
+
67
+ /** The persisted state's format. */
68
+ export const LIVE_BACKUP_STATE_VERSION = 1;
69
+
70
+ const DAG_CBOR = dagCbor.code;
71
+
72
+ /**
73
+ * @typedef {object} LiveBackupState
74
+ * @property {number} v - {@link LIVE_BACKUP_STATE_VERSION}
75
+ * @property {number} sequence - of the last increment shipped; 0 before the first
76
+ * @property {string|null} base - the full backup the increments build on, as the caller named it
77
+ * @property {string|null} previous - the backend's id of the last increment
78
+ * @property {Object<string, { heads: string[] }>} databases - by address: the heads last shipped
79
+ */
80
+
81
+ /**
82
+ * @typedef {object} IncrementManifest
83
+ * @property {"increment"} kind
84
+ * @property {number} v
85
+ * @property {number} sequence - 1 for the first increment after the base
86
+ * @property {string|null} base - the full backup this builds on, or null
87
+ * @property {string|null} previous - the id of the increment before this one
88
+ * @property {string} createdAt - ISO
89
+ * @property {object} metadata - the shape `restoreFromBlocks` reads: `databases`
90
+ * with each one's `address`, `name`, `type`, `manifestCID`, `heads`, the
91
+ * `since` heads it continues from and `entries`, how many it holds
92
+ */
93
+
94
+ const manifestCIDOf = (database) => database.address.split("/").pop();
95
+
96
+ /** Blocks keyed the way OrbitDB names them, deduplicated by that name. */
97
+ const addBlock = (blocks, name, bytes) => {
98
+ if (!bytes || blocks.has(name)) return;
99
+ blocks.set(name, { cid: CID.parse(name), bytes });
100
+ };
101
+
102
+ /**
103
+ * A database's manifest and access controller: what makes a CAR of entries a
104
+ * backup of *this* database rather than a bag of valid blocks.
105
+ */
106
+ async function skeletonBlocks(database, blocks) {
107
+ const manifestCID = manifestCIDOf(database);
108
+ const manifestBytes = await database.log.storage
109
+ .get(manifestCID)
110
+ .catch(() => null);
111
+ addBlock(blocks, manifestCID, manifestBytes);
112
+ if (!manifestBytes) return;
113
+ try {
114
+ const { value } = await Block.decode({
115
+ cid: CID.parse(manifestCID),
116
+ bytes: manifestBytes,
117
+ codec: dagCbor,
118
+ hasher: sha256,
119
+ });
120
+ const accessController = value?.accessController?.replace("/ipfs/", "");
121
+ if (accessController) {
122
+ addBlock(
123
+ blocks,
124
+ accessController,
125
+ await database.log.storage.get(accessController).catch(() => null),
126
+ );
127
+ }
128
+ } catch {
129
+ // A manifest that does not decode has no access controller to find.
130
+ }
131
+ }
132
+
133
+ /** The identity block an entry's signature refers to, from wherever this node has it. */
134
+ async function identityBytes(database, hash) {
135
+ const own = database.identity ?? database.log.identity;
136
+ if (own?.hash === hash && own.bytes) return own.bytes;
137
+ const identities = database.log.identities;
138
+ if (identities && typeof identities.getIdentity === "function") {
139
+ const identity = await identities.getIdentity(hash).catch(() => null);
140
+ if (identity?.bytes) return identity.bytes;
141
+ }
142
+ return database.log.storage.get(hash).catch(() => null);
143
+ }
144
+
145
+ /**
146
+ * Every hash reachable from these heads: what a restarted watcher counts as
147
+ * already shipped.
148
+ */
149
+ async function reachable(database, heads) {
150
+ const seen = new Set();
151
+ const stack = [...heads];
152
+ while (stack.length) {
153
+ const hash = stack.pop();
154
+ if (seen.has(hash)) continue;
155
+ seen.add(hash);
156
+ const entry = await database.log.get(hash).catch(() => null);
157
+ if (entry?.next) stack.push(...entry.next);
158
+ }
159
+ return seen;
160
+ }
161
+
162
+ /**
163
+ * Walk from the current heads down to what was shipped, and collect what is new.
164
+ *
165
+ * @returns {Promise<{ entries: object[], heads: string[] }>} the new entries,
166
+ * newest first, and the heads the walk started from
167
+ */
168
+ async function newEntries(database, shipped) {
169
+ const heads = await database.log.heads();
170
+ const entries = [];
171
+ const seen = new Set();
172
+ const stack = [...heads];
173
+ while (stack.length) {
174
+ const entry = stack.pop();
175
+ if (!entry || seen.has(entry.hash) || shipped.has(entry.hash)) continue;
176
+ seen.add(entry.hash);
177
+ entries.push(entry);
178
+ for (const hash of entry.next ?? []) {
179
+ if (!seen.has(hash) && !shipped.has(hash))
180
+ stack.push(await database.log.get(hash).catch(() => null));
181
+ }
182
+ }
183
+ return { entries, heads: heads.map((head) => head.hash) };
184
+ }
185
+
186
+ /**
187
+ * Collect one increment across several databases, without shipping it.
188
+ *
189
+ * Exposed for callers that ship by other means; {@link createLiveBackup} is
190
+ * the usual way in.
191
+ *
192
+ * @param {Map<string, { database: object, shipped: Set<string>, collection?: string }>} watched - by address
193
+ * @param {object} options
194
+ * @param {number} options.sequence - the increment's number
195
+ * @param {string|null} [options.base]
196
+ * @param {string|null} [options.previous]
197
+ * @param {Map<string, { bytes: Uint8Array }>} [options.blocks] - blocks of the application's own to carry along
198
+ * @param {() => Date} [options.now]
199
+ * @returns {Promise<null | { blocks: Map<string, { cid: CID, bytes: Uint8Array }>, manifest: IncrementManifest,
200
+ * root: string, shipped: Map<string, { heads: string[], hashes: string[] }>, entries: number }>}
201
+ * null when nothing is new
202
+ */
203
+ export async function collectIncrement(
204
+ watched,
205
+ {
206
+ sequence,
207
+ base = null,
208
+ previous = null,
209
+ blocks: own,
210
+ now = () => new Date(),
211
+ },
212
+ ) {
213
+ const blocks = new Map();
214
+ const described = [];
215
+ const shippedNow = new Map();
216
+ let total = 0;
217
+
218
+ for (const [address, { database, shipped, collection }] of watched) {
219
+ const { entries, heads } = await newEntries(database, shipped);
220
+ if (entries.length === 0) continue;
221
+ await skeletonBlocks(database, blocks);
222
+ const identities = new Set();
223
+ for (const entry of entries) {
224
+ addBlock(
225
+ blocks,
226
+ entry.hash,
227
+ await database.log.storage.get(entry.hash).catch(() => null),
228
+ );
229
+ if (entry.identity) identities.add(entry.identity);
230
+ }
231
+ for (const hash of identities)
232
+ addBlock(blocks, hash, await identityBytes(database, hash));
233
+ total += entries.length;
234
+ described.push({
235
+ address,
236
+ name: database.name,
237
+ type: database.type,
238
+ manifestCID: manifestCIDOf(database),
239
+ heads,
240
+ since: [...(shipped.heads ?? [])],
241
+ entries: entries.length,
242
+ ...(collection ? { collection } : {}),
243
+ });
244
+ shippedNow.set(address, {
245
+ heads,
246
+ hashes: entries.map((entry) => entry.hash),
247
+ });
248
+ }
249
+
250
+ for (const [name, block] of own ?? [])
251
+ if (!blocks.has(name)) blocks.set(name, block);
252
+ if (total === 0 && (own?.size ?? 0) === 0) return null;
253
+
254
+ const at = now();
255
+ /** @type {IncrementManifest} */
256
+ const manifest = {
257
+ kind: INCREMENT_KIND,
258
+ v: INCREMENT_VERSION,
259
+ sequence,
260
+ base,
261
+ previous,
262
+ createdAt: at.toISOString(),
263
+ metadata: {
264
+ version: "1.0",
265
+ timestamp: at.getTime(),
266
+ databaseCount: described.length,
267
+ totalBlocks: blocks.size,
268
+ totalEntries: total,
269
+ ...(described[0] ? { manifestCID: described[0].manifestCID } : {}),
270
+ databases: described,
271
+ },
272
+ };
273
+ const rootBytes = dagCbor.encode(manifest);
274
+ const rootCid = CID.createV1(DAG_CBOR, await sha256.digest(rootBytes));
275
+ blocks.set(rootCid.toString(), { cid: rootCid, bytes: rootBytes });
276
+ return {
277
+ blocks,
278
+ manifest,
279
+ root: rootCid.toString(),
280
+ shipped: shippedNow,
281
+ entries: total,
282
+ };
283
+ }
284
+
285
+ /**
286
+ * Read an increment back: every block checked against its CID, the manifest
287
+ * found among them.
288
+ *
289
+ * @param {Uint8Array} carBytes
290
+ * @returns {Promise<{ manifest: IncrementManifest, blocks: Map<string, { bytes: Uint8Array }> }>}
291
+ */
292
+ export async function readIncrement(carBytes) {
293
+ const blocks = await readBlocksFromCAR(carBytes, { verify: true });
294
+ for (const [name, { bytes }] of blocks) {
295
+ if (CID.parse(name).code !== DAG_CBOR) continue;
296
+ let value;
297
+ try {
298
+ value = dagCbor.decode(bytes);
299
+ } catch {
300
+ continue;
301
+ }
302
+ if (value?.kind === INCREMENT_KIND) {
303
+ if (value.v !== INCREMENT_VERSION) {
304
+ throw new Error(
305
+ `This increment was written in format ${value.v}, and this build reads ${INCREMENT_VERSION}. ` +
306
+ "Upgrade @le-space/orbitdb-storage-bridge to restore it.",
307
+ );
308
+ }
309
+ return { manifest: value, blocks };
310
+ }
311
+ }
312
+ throw new Error("This CAR holds no increment manifest.");
313
+ }
314
+
315
+ const metadataOf = (source) => source?.metadata ?? source?.manifest?.metadata;
316
+
317
+ /**
318
+ * Fold a base backup and its increments into one block Map and one metadata:
319
+ * every block of every source, and for each database the heads of the newest
320
+ * source that names it.
321
+ *
322
+ * Sources are `{ metadata | manifest, blocks }`: what `bundleDatabases`,
323
+ * `openAppBackup` and {@link readIncrement} return. Increments are ordered by
324
+ * their `sequence`; the base, if given, comes first.
325
+ *
326
+ * @param {object} params
327
+ * @param {{ metadata?: object, manifest?: object, blocks: Map }} [params.base]
328
+ * @param {{ manifest: IncrementManifest, blocks: Map }[]} params.increments
329
+ * @returns {{ blocks: Map, metadata: object, sequences: number[] }}
330
+ */
331
+ export function mergeIncrements({ base, increments = [] }) {
332
+ const ordered = [...increments].sort(
333
+ (a, b) => (a.manifest?.sequence ?? 0) - (b.manifest?.sequence ?? 0),
334
+ );
335
+ const sources = base ? [base, ...ordered] : ordered;
336
+ if (sources.length === 0)
337
+ throw new Error("mergeIncrements needs a base or at least one increment");
338
+
339
+ const blocks = new Map();
340
+ const databases = new Map();
341
+ let timestamp = 0;
342
+ for (const source of sources) {
343
+ for (const [name, block] of source.blocks ?? [])
344
+ if (!blocks.has(name)) blocks.set(name, block);
345
+ const metadata = metadataOf(source);
346
+ if (!metadata?.databases) continue;
347
+ timestamp = Math.max(timestamp, Number(metadata.timestamp) || 0);
348
+ for (const d of metadata.databases) {
349
+ const known = databases.get(String(d.address)) ?? {};
350
+ databases.set(String(d.address), {
351
+ ...known,
352
+ address: String(d.address),
353
+ ...(d.name ? { name: d.name } : {}),
354
+ ...(d.type ? { type: d.type } : {}),
355
+ ...(d.manifestCID ? { manifestCID: d.manifestCID } : {}),
356
+ ...(d.collection ? { collection: d.collection } : {}),
357
+ // Later sources are newer: their heads supersede.
358
+ ...(Array.isArray(d.heads) && d.heads.length ? { heads: d.heads } : {}),
359
+ ...(typeof d.entryCount === "number"
360
+ ? { entryCount: d.entryCount }
361
+ : {}),
362
+ });
363
+ }
364
+ }
365
+ const described = [...databases.values()];
366
+ return {
367
+ blocks,
368
+ metadata: {
369
+ version: "1.0",
370
+ timestamp,
371
+ databaseCount: described.length,
372
+ totalBlocks: blocks.size,
373
+ ...(described[0]?.manifestCID
374
+ ? { manifestCID: described[0].manifestCID }
375
+ : {}),
376
+ databases: described,
377
+ },
378
+ sequences: ordered.map((i) => i.manifest?.sequence ?? 0),
379
+ };
380
+ }
381
+
382
+ /**
383
+ * Put a base backup and its increments back into a node, through
384
+ * `restoreFromBlocks`: the union of their blocks, and each database joined at
385
+ * the heads of the newest increment that names it.
386
+ *
387
+ * @param {object} orbitdb
388
+ * @param {object} params
389
+ * @param {{ metadata?: object, manifest?: object, blocks: Map }} [params.base]
390
+ * @param {{ manifest: IncrementManifest, blocks: Map }[]} params.increments
391
+ * @param {object} [params.open] - options for `orbitdb.open` (`encryption`, `AccessController`, …)
392
+ * @param {object} [params.log]
393
+ * @param {Function} [params.onProgress]
394
+ * @returns {Promise<{ blocks: number, databases: object[], sequences: number[] }>}
395
+ */
396
+ export async function restoreFromIncrements(
397
+ orbitdb,
398
+ { base, increments = [], open, log, onProgress } = {},
399
+ ) {
400
+ const { blocks, metadata, sequences } = mergeIncrements({ base, increments });
401
+ const restored = await restoreFromBlocks(orbitdb, blocks, metadata, {
402
+ open,
403
+ log,
404
+ onProgress,
405
+ });
406
+ return { ...restored, sequences };
407
+ }
408
+
409
+ /**
410
+ * Join a base and its increments into a database that is open *now*, without
411
+ * closing it: the blocks go into the log's storage, and each head the newest
412
+ * source names is joined on the open instance.
413
+ *
414
+ * `restoreFromIncrements` reopens the database, as `restoreFromBlocks` does,
415
+ * which leaves an application holding a closed instance. An application whose
416
+ * books are open — a todo list on screen, belege's books — wants the entries
417
+ * to appear in what it already holds. Joining merges: what is here stays.
418
+ *
419
+ * @param {object} database - an open OrbitDB database
420
+ * @param {object} params
421
+ * @param {{ metadata?: object, manifest?: object, blocks: Map }} [params.base]
422
+ * @param {{ manifest: IncrementManifest, blocks: Map }[]} params.increments
423
+ * @returns {Promise<{ joined: number, heads: number, blocks: number }>}
424
+ */
425
+ export async function joinIncrements(database, { base, increments = [] } = {}) {
426
+ const { blocks, metadata } = mergeIncrements({ base, increments });
427
+ const address = String(database.address);
428
+ const described = metadata.databases.find(
429
+ (d) => String(d.address) === address,
430
+ );
431
+ if (!described)
432
+ throw new Error("These increments hold nothing for this database.");
433
+
434
+ for (const [name, { bytes }] of blocks) {
435
+ const cid = CID.parse(name);
436
+ // OrbitDB names its blocks in base58btc; a CAR in base32. The log reads
437
+ // its own names, and identities are looked up by theirs, so each block
438
+ // goes in under the name its reader will ask for.
439
+ const stored =
440
+ cid.code === DAG_CBOR ? cid.toV1().toString(base58btc) : name;
441
+ await database.log.storage.put(stored, bytes);
442
+ }
443
+
444
+ let joined = 0;
445
+ for (const hash of described.heads ?? []) {
446
+ const name = CID.parse(hash).toV1().toString(base58btc);
447
+ const entry = await database.log.get(name).catch(() => null);
448
+ if (entry && (await database.log.joinEntry(entry))) joined++;
449
+ }
450
+ return { joined, heads: (described.heads ?? []).length, blocks: blocks.size };
451
+ }
452
+
453
+ /** A state with nothing shipped yet. */
454
+ export const emptyLiveBackupState = () => ({
455
+ v: LIVE_BACKUP_STATE_VERSION,
456
+ sequence: 0,
457
+ base: null,
458
+ previous: null,
459
+ databases: {},
460
+ });
461
+
462
+ /**
463
+ * Keep the databases backed up, increment by increment.
464
+ *
465
+ * @param {object} params
466
+ * @param {object[]|Object<string, object>} [params.databases] - open OrbitDB
467
+ * databases, as a list or by name; more can be added with `watch()`
468
+ * @param {import("./backends/types.js").StorageBackend} params.backend - where
469
+ * increments go; wrap it with `withEncryption` to seal them
470
+ * @param {number} [params.every] - ms between flushes while something is
471
+ * pending; 0 or undefined for none (flush by `after`, or by hand)
472
+ * @param {number} [params.after] - flush once this many updates are pending
473
+ * @param {number} [params.settle=250] - ms to wait after an update before an
474
+ * `after`-triggered flush, so a burst of writes becomes one increment
475
+ * @param {{ load: () => Promise<LiveBackupState|null|undefined>, save: (state: LiveBackupState) => Promise<void> }} [params.state] -
476
+ * where the few bytes that survive a restart live; in memory when omitted
477
+ * @param {() => Promise<Map<string, { bytes: Uint8Array }>>|Map} [params.blocks] - blocks of the
478
+ * application's own to carry in the next increment (receipt files, …)
479
+ * @param {boolean} [params.keep=true] - ask a backend whose handle says
480
+ * `retained: false` to keep the increment (`pinCid`)
481
+ * @param {(sequence: number, at: Date) => string} [params.name] - the blob's name
482
+ * @param {(shipped: { sequence: number, handle: object, kept?: object, entries: number, blocks: number, bytes: number, at: Date }) => void} [params.onShipped]
483
+ * @param {(error: Error) => void} [params.onError]
484
+ * @param {() => Date} [params.now]
485
+ * @returns {{ watch: Function, unwatch: Function, start: Function, stop: Function,
486
+ * flush: () => Promise<object|null>, rebase: Function, status: () => object }}
487
+ */
488
+ export function createLiveBackup({
489
+ databases,
490
+ backend,
491
+ every = 0,
492
+ after = 0,
493
+ settle = 250,
494
+ state: store,
495
+ blocks: ownBlocks,
496
+ keep = true,
497
+ name = (sequence, at) =>
498
+ `increment-${String(sequence).padStart(6, "0")}-${at.toISOString().replace(/[:.]/g, "-")}.car`,
499
+ onShipped,
500
+ onError,
501
+ now = () => new Date(),
502
+ } = {}) {
503
+ if (!backend || typeof backend.putBlob !== "function") {
504
+ throw new Error("createLiveBackup needs a storage backend with putBlob()");
505
+ }
506
+
507
+ /** @type {Map<string, { database: object, shipped: Set<string>, collection?: string, off: Function, loaded: boolean }>} */
508
+ const watched = new Map();
509
+ /** @type {LiveBackupState} */
510
+ let state = emptyLiveBackupState();
511
+ let loaded = !store;
512
+ let running = false;
513
+ let pending = 0;
514
+ let inflight = null;
515
+ let again = false;
516
+ let timer = null;
517
+ let settling = null;
518
+ let lastShipped = null;
519
+ let lastError = null;
520
+ let failures = 0;
521
+
522
+ const schedule = () => {
523
+ if (!running || !every || timer) return;
524
+ timer = setTimeout(() => {
525
+ timer = null;
526
+ if (pending > 0) flush().catch(() => {});
527
+ schedule();
528
+ }, every);
529
+ timer.unref?.();
530
+ };
531
+
532
+ /** A flush soon, once the writes of this moment have settled. */
533
+ const scheduleFlush = () => {
534
+ if (!running || settling) return;
535
+ settling = setTimeout(() => {
536
+ settling = null;
537
+ flush().catch(() => {});
538
+ }, settle);
539
+ settling.unref?.();
540
+ };
541
+
542
+ const onUpdate = () => {
543
+ pending++;
544
+ if (after && pending >= after) scheduleFlush();
545
+ };
546
+
547
+ const watch = (database, collection) => {
548
+ const address = String(database.address);
549
+ if (watched.has(address)) return;
550
+ const handler = () => onUpdate();
551
+ database.events?.on?.("update", handler);
552
+ watched.set(address, {
553
+ database,
554
+ collection,
555
+ shipped: new Set(),
556
+ loaded: false,
557
+ off: () => database.events?.off?.("update", handler),
558
+ });
559
+ };
560
+
561
+ const unwatch = (database) => {
562
+ const address = String(database?.address ?? database);
563
+ watched.get(address)?.off();
564
+ watched.delete(address);
565
+ };
566
+
567
+ if (Array.isArray(databases)) databases.forEach((d) => watch(d));
568
+ else
569
+ for (const [collection, d] of Object.entries(databases ?? {}))
570
+ watch(d, collection);
571
+
572
+ const load = async () => {
573
+ if (!loaded) {
574
+ const saved = await store.load();
575
+ if (saved && saved.v === LIVE_BACKUP_STATE_VERSION)
576
+ state = { ...emptyLiveBackupState(), ...saved };
577
+ loaded = true;
578
+ }
579
+ for (const [address, w] of watched) {
580
+ if (w.loaded) continue;
581
+ const heads = state.databases[address]?.heads ?? [];
582
+ w.shipped = await reachable(w.database, heads);
583
+ w.shipped.heads = heads;
584
+ w.loaded = true;
585
+ }
586
+ };
587
+
588
+ const save = async () => {
589
+ // Plain JSON, so a copy is a copy and the caller may keep or serialise it.
590
+ if (store) await store.save(JSON.parse(JSON.stringify(state)));
591
+ };
592
+
593
+ const ship = async () => {
594
+ await load();
595
+ const own = typeof ownBlocks === "function" ? await ownBlocks() : ownBlocks;
596
+ const collected = await collectIncrement(watched, {
597
+ sequence: state.sequence + 1,
598
+ base: state.base,
599
+ previous: state.previous,
600
+ blocks: own,
601
+ now,
602
+ });
603
+ const countedPending = pending;
604
+ if (!collected) {
605
+ pending = 0;
606
+ return null;
607
+ }
608
+ const at = now();
609
+ const car = await createCARFromBlocks(collected.blocks, collected.root);
610
+ const handle = await backend.putBlob(car, {
611
+ name: name(collected.manifest.sequence, at),
612
+ contentType: "application/vnd.ipld.car",
613
+ });
614
+ let kept;
615
+ if (
616
+ keep &&
617
+ handle?.retained === false &&
618
+ typeof backend.pinCid === "function"
619
+ ) {
620
+ kept = await backend.pinCid(handle.id, { name: handle.name });
621
+ }
622
+
623
+ // Only now, with the bytes out of the house, does the state move.
624
+ state.sequence = collected.manifest.sequence;
625
+ state.previous = String(handle.id ?? handle.cid ?? "");
626
+ for (const [address, { heads, hashes }] of collected.shipped) {
627
+ const w = watched.get(address);
628
+ hashes.forEach((hash) => w.shipped.add(hash));
629
+ w.shipped.heads = heads;
630
+ state.databases[address] = { heads };
631
+ }
632
+ pending = Math.max(0, pending - countedPending);
633
+ await save();
634
+
635
+ const shipped = {
636
+ sequence: state.sequence,
637
+ handle,
638
+ ...(kept ? { kept } : {}),
639
+ entries: collected.entries,
640
+ blocks: collected.blocks.size,
641
+ bytes: car.length,
642
+ at,
643
+ };
644
+ lastShipped = shipped;
645
+ lastError = null;
646
+ onShipped?.(shipped);
647
+ return shipped;
648
+ };
649
+
650
+ /** Ship what is pending; one flush at a time, and a request during one runs after it. */
651
+ const flush = async () => {
652
+ if (inflight) {
653
+ again = true;
654
+ return inflight;
655
+ }
656
+ inflight = (async () => {
657
+ try {
658
+ return await ship();
659
+ } catch (error) {
660
+ failures++;
661
+ lastError = error;
662
+ onError?.(error);
663
+ throw error;
664
+ } finally {
665
+ inflight = null;
666
+ if (again) {
667
+ again = false;
668
+ flush().catch(() => {});
669
+ }
670
+ }
671
+ })();
672
+ return inflight;
673
+ };
674
+
675
+ /**
676
+ * Everything now in the databases counts as shipped.
677
+ *
678
+ * With `base`, a full backup was just made: the increments start again at 1
679
+ * from it. Without, the content arrived by other means — a restore joined
680
+ * into the open databases — and must not be sent out again, while the
681
+ * numbering carries on from where it was.
682
+ *
683
+ * @param {{ base?: string|null }} [params] - what the caller calls the full backup (its id, CID, …)
684
+ */
685
+ const rebase = async ({ base } = {}) => {
686
+ await load();
687
+ if (base !== undefined)
688
+ state = { ...emptyLiveBackupState(), base: base ?? null };
689
+ for (const [address, w] of watched) {
690
+ const heads = (await w.database.log.heads()).map((head) => head.hash);
691
+ w.shipped = await reachable(w.database, heads);
692
+ w.shipped.heads = heads;
693
+ state.databases[address] = { heads };
694
+ }
695
+ pending = 0;
696
+ await save();
697
+ };
698
+
699
+ const start = async () => {
700
+ if (running) return;
701
+ running = true;
702
+ await load();
703
+ schedule();
704
+ // What was written before the watcher came — after a restart, or before
705
+ // the archive was switched on — is as unshipped as anything, and nothing
706
+ // would announce it. So one flush soon, which ships nothing if there is
707
+ // nothing.
708
+ if (after || every) scheduleFlush();
709
+ };
710
+
711
+ const stop = async ({ flush: last = false } = {}) => {
712
+ running = false;
713
+ if (timer) clearTimeout(timer);
714
+ if (settling) clearTimeout(settling);
715
+ timer = settling = null;
716
+ if (last) await flush();
717
+ };
718
+
719
+ const status = () => ({
720
+ running,
721
+ sequence: state.sequence,
722
+ base: state.base,
723
+ previous: state.previous,
724
+ pending,
725
+ watched: watched.size,
726
+ lastShipped,
727
+ lastError,
728
+ failures,
729
+ });
730
+
731
+ return { watch, unwatch, start, stop, flush, rebase, status };
732
+ }
733
+
734
+ export default createLiveBackup;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@le-space/orbitdb-storage-bridge",
3
- "version": "0.16.1",
3
+ "version": "0.18.0",
4
4
  "description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
5
5
  "main": "lib/orbitdb-storacha-bridge.js",
6
6
  "svelte": "dist/components/",
@@ -16,6 +16,8 @@
16
16
  "./dehydrate": "./lib/dehydrate.js",
17
17
  "./restore-cid": "./lib/restore-cid.js",
18
18
  "./extract-blocks": "./lib/extract-blocks.js",
19
+ "./app-backup": "./lib/app-backup.js",
20
+ "./live-backup": "./lib/live-backup.js",
19
21
  "./gateway-fetch": "./lib/gateway-fetch.js",
20
22
  "./peer-fetch": "./lib/peer-fetch.js",
21
23
  "./backends/types": "./lib/backends/types.js",