@le-space/orbitdb-storage-bridge 0.19.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -94,6 +94,11 @@ async function sha256Hex(payload) {
94
94
  * by default
95
95
  * @param {string} args.cid - what to keep
96
96
  * @param {string} [args.channel]
97
+ * @param {string} [args.ref] - Aleph's free-text `ref` on the STORE: a label the
98
+ * messages API filters by (`refs=`), so an application can keep several
99
+ * chains on one account and channel and list one of them — the increments of
100
+ * one database, say — without downloading anything. Opaque to Aleph; what it
101
+ * discloses is the caller's choice (a derived fingerprint, not an address)
97
102
  * @param {"credit" | "hold" | null} [args.payment] - `credit` by default; `null`
98
103
  * sends no `payment`, as every version up to 0.16.1 did, which Aleph books as
99
104
  * `hold`
@@ -105,10 +110,17 @@ export async function buildStoreMessage({
105
110
  owner,
106
111
  cid,
107
112
  channel = DEFAULT_ALEPH_CHANNEL,
113
+ ref,
108
114
  payment = "credit",
109
115
  now,
110
116
  hasher = sha256Hex,
111
117
  }) {
118
+ if (ref != null && (typeof ref !== "string" || !ref)) {
119
+ throw new BackendError(
120
+ "INVALID_BACKEND",
121
+ "a STORE's ref is a non-empty string, or absent",
122
+ );
123
+ }
112
124
  if (payment !== null && !STORE_PAYMENTS.includes(payment)) {
113
125
  throw new BackendError(
114
126
  "INVALID_BACKEND",
@@ -123,6 +135,7 @@ export async function buildStoreMessage({
123
135
  // "the thing to keep is an IPFS CID" — not the envelope's item_type
124
136
  item_type: "ipfs",
125
137
  item_hash: cid,
138
+ ...(ref ? { ref } : {}),
126
139
  // How that account pays. Without it Aleph books the STORE as `hold`, and an
127
140
  // account funded with credits covers nothing (see the file's header).
128
141
  ...(payment ? { payment: { type: payment } } : {}),
@@ -157,11 +170,13 @@ export const signaturePayload = (message) =>
157
170
  * `personal_sign`, or anything shaped like it. The library never sees a key.
158
171
  * @param {string} [options.apiHost]
159
172
  * @param {string} [options.channel]
173
+ * @param {string} [options.ref] - the `ref` every STORE from this pin carries
174
+ * (see {@link buildStoreMessage}); a `meta.ref` on one call wins over it
160
175
  * @param {"credit" | "hold" | null} [options.payment] - see {@link buildStoreMessage}
161
176
  * @param {(payload: string) => Promise<string>} [options.hasher]
162
177
  * @param {typeof fetch} [options.fetch]
163
178
  * @param {() => number} [options.now] - seconds
164
- * @returns {(cid: string, meta?: object) => Promise<{ itemHash: string, status: string }>}
179
+ * @returns {(cid: string, meta?: { name?: string, ref?: string }) => Promise<{ itemHash: string, status: string }>}
165
180
  * `status` is what Aleph answered: `processed`, or `pending` until it has
166
181
  * checked the signature, the authorization and the credit — follow a pending
167
182
  * one with {@link waitForMessage}.
@@ -188,12 +203,13 @@ export function createAlephPin(options = {}) {
188
203
  if (typeof doFetch !== "function")
189
204
  throw new BackendError("INVALID_BACKEND", "createAlephPin needs fetch");
190
205
 
191
- return async function pin(cid) {
206
+ return async function pin(cid, meta = {}) {
192
207
  const unsigned = await buildStoreMessage({
193
208
  sender,
194
209
  owner,
195
210
  cid,
196
211
  channel,
212
+ ref: meta?.ref ?? options.ref,
197
213
  payment,
198
214
  hasher,
199
215
  now: now?.(),
@@ -317,6 +333,7 @@ export async function waitForMessage(itemHash, options = {}) {
317
333
  * @param {string} options.owner - the paying account, EIP-55 checksummed (Aleph
318
334
  * keys accounts by that form)
319
335
  * @param {string} [options.channel] - one channel; every channel when absent
336
+ * @param {string} [options.ref] - only STOREs carrying this `ref` (`refs=`); every STORE when absent
320
337
  * @param {string} [options.apiHost]
321
338
  * @param {typeof fetch} [options.fetch]
322
339
  * @param {number} [options.pagination] - per page; default 50
@@ -326,7 +343,7 @@ export async function waitForMessage(itemHash, options = {}) {
326
343
  * the handle's `id`
327
344
  */
328
345
  export async function listAlephStores(options = {}) {
329
- const { owner, channel } = options;
346
+ const { owner, channel, ref } = options;
330
347
  const apiHost = options.apiHost || DEFAULT_ALEPH_API_HOST;
331
348
  const doFetch = options.fetch || globalThis.fetch;
332
349
  if (!owner)
@@ -334,6 +351,11 @@ export async function listAlephStores(options = {}) {
334
351
  "INVALID_BACKEND",
335
352
  "listAlephStores needs the paying account's address as `owner`",
336
353
  );
354
+ if (ref != null && (typeof ref !== "string" || !ref))
355
+ throw new BackendError(
356
+ "INVALID_BACKEND",
357
+ "listAlephStores takes `ref` as a non-empty string, or not at all",
358
+ );
337
359
  if (typeof doFetch !== "function")
338
360
  throw new BackendError("INVALID_BACKEND", "listAlephStores needs fetch");
339
361
 
@@ -341,6 +363,7 @@ export async function listAlephStores(options = {}) {
341
363
  url.searchParams.set("owners", owner);
342
364
  url.searchParams.set("msgTypes", "STORE");
343
365
  if (channel) url.searchParams.set("channels", channel);
366
+ if (ref) url.searchParams.set("refs", ref);
344
367
  url.searchParams.set("pagination", String(options.pagination ?? 50));
345
368
  url.searchParams.set("page", String(options.page ?? 1));
346
369
 
@@ -362,11 +385,16 @@ export async function listAlephStores(options = {}) {
362
385
  owner: String(m?.content?.address ?? ""),
363
386
  time: Number(m?.content?.time ?? m?.time ?? 0),
364
387
  ...(m?.channel ? { channel: m.channel } : {}),
388
+ ...(typeof m?.content?.ref === "string" && m.content.ref
389
+ ? { ref: m.content.ref }
390
+ : {}),
365
391
  ...(m?.content?.payment?.type ? { payment: m.content.payment.type } : {}),
366
392
  }))
367
393
  // Aleph answers what it was asked; keeping only this owner's costs nothing
368
394
  // and keeps a misread filter from handing back someone else's files.
369
395
  .filter((store) => store.cid && store.itemHash && same(store.owner, owner))
396
+ // A host that ignored `refs=` must not hand back the other chains.
397
+ .filter((store) => !ref || store.ref === ref)
370
398
  .sort((a, b) => b.time - a.time);
371
399
  return {
372
400
  stores,
@@ -413,6 +441,19 @@ export async function buildForgetMessage({
413
441
  "a FORGET names at least one item hash",
414
442
  );
415
443
  }
444
+ // Aleph's limits, checked before anything is signed.
445
+ if (hashes.length > 1000) {
446
+ throw new BackendError(
447
+ "INVALID_BACKEND",
448
+ `a FORGET names at most 1000 item hashes, not ${hashes.length}`,
449
+ );
450
+ }
451
+ if (reason != null && String(reason).length > 1000) {
452
+ throw new BackendError(
453
+ "INVALID_BACKEND",
454
+ "a FORGET's reason is at most 1000 characters",
455
+ );
456
+ }
416
457
  const time = now ?? Date.now() / 1000;
417
458
  const content = {
418
459
  address: owner || sender,
@@ -519,6 +560,7 @@ export function createAlephForget(options = {}) {
519
560
  * @param {object} options
520
561
  * @param {string} options.owner - the account the STOREs are for
521
562
  * @param {string} options.channel - one channel; a FORGET across channels is not what anyone means
563
+ * @param {string} [options.ref] - only the STOREs carrying this `ref` (one chain of several on the account); all of the channel when absent
522
564
  * @param {ReturnType<typeof createAlephForget>} options.forget - from {@link createAlephForget}, signing for that account
523
565
  * @param {(store: { cid: string, itemHash: string, time: number }) => boolean} [options.keep]
524
566
  * @param {string} [options.reason]
@@ -527,12 +569,16 @@ export function createAlephForget(options = {}) {
527
569
  * @param {typeof fetch} [options.fetch]
528
570
  * @param {{ timeout?: number, interval?: number }} [options.settle] - for {@link waitForMessage}
529
571
  * @param {(progress: { done: number, total: number }) => void} [options.onProgress]
530
- * @returns {Promise<{ forgotten: string[], kept: string[], rejected: { itemHash: string, errorCode?: string }[] }>}
572
+ * @returns {Promise<{ forgotten: string[], kept: string[], rejected: { itemHash: string, errorCode?: string | number }[],
573
+ * pending: { itemHash: string, forget: string, status: string }[], failed: { itemHash: string, error: string }[] }>}
574
+ * `forgotten` only once Aleph said `processed`; `pending` when it had not decided in time (follow
575
+ * `forget`, the FORGET's item hash, with {@link waitForMessage}); `failed` when a batch could not be sent
531
576
  */
532
577
  export async function forgetAlephStores(options = {}) {
533
578
  const {
534
579
  owner,
535
580
  channel,
581
+ ref,
536
582
  forget,
537
583
  keep,
538
584
  reason,
@@ -561,53 +607,80 @@ export async function forgetAlephStores(options = {}) {
561
607
  /** @type {{ cid: string, itemHash: string, time: number }[]} */
562
608
  const all = [];
563
609
  for (let page = 1; ; page++) {
564
- const { stores } = await listAlephStores({
610
+ const { stores, total } = await listAlephStores({
565
611
  owner,
566
612
  channel,
613
+ ...(ref ? { ref } : {}),
567
614
  apiHost,
568
615
  fetch: doFetch,
569
616
  pagination: 100,
570
617
  page,
571
618
  });
572
619
  all.push(...stores);
573
- if (stores.length < 100) break;
620
+ // The listing filters after the page was cut, so a short page is not the
621
+ // end: the API's own total is.
622
+ if (stores.length === 0 || (total != null && all.length >= total)) break;
623
+ if (total == null && stores.length < 100) break;
574
624
  }
575
- const kept = all
576
- .filter((store) => keep?.(store))
577
- .map((store) => store.itemHash);
625
+ const kept = new Set(
626
+ all.filter((store) => keep?.(store)).map((store) => store.itemHash),
627
+ );
578
628
  const targets = all
579
- .filter((store) => !kept.includes(store.itemHash))
629
+ .filter((store) => !kept.has(store.itemHash))
580
630
  .map((store) => store.itemHash);
581
631
 
582
632
  const forgotten = [];
583
633
  const rejected = [];
634
+ const pending = [];
635
+ const failed = [];
584
636
  for (let i = 0; i < targets.length; i += batch) {
585
637
  const hashes = targets.slice(i, i + batch);
586
- const sent = await forget(hashes, reason);
587
- const final =
588
- sent.status === "pending"
589
- ? await waitForMessage(sent.itemHash, {
590
- apiHost,
591
- fetch: doFetch,
592
- ...settle,
593
- })
594
- : sent;
595
- if (final.status === "rejected") {
596
- rejected.push(
638
+ try {
639
+ const sent = await forget(hashes, reason);
640
+ const final =
641
+ sent.status === "pending"
642
+ ? await waitForMessage(sent.itemHash, {
643
+ apiHost,
644
+ fetch: doFetch,
645
+ ...settle,
646
+ })
647
+ : sent;
648
+ if (final.status === "rejected") {
649
+ rejected.push(
650
+ ...hashes.map((itemHash) => ({
651
+ itemHash,
652
+ ...(final.errorCode != null ? { errorCode: final.errorCode } : {}),
653
+ })),
654
+ );
655
+ } else if (final.status === "processed") {
656
+ forgotten.push(...hashes);
657
+ } else {
658
+ // Still pending when the time ran out, or a status in between: not
659
+ // forgotten, not refused — the caller follows the FORGET itself.
660
+ pending.push(
661
+ ...hashes.map((itemHash) => ({
662
+ itemHash,
663
+ forget: sent.itemHash,
664
+ status: final.status,
665
+ })),
666
+ );
667
+ }
668
+ } catch (error) {
669
+ // One batch that could not be sent does not undo the ones before it,
670
+ // and the caller learns which hashes are still there.
671
+ failed.push(
597
672
  ...hashes.map((itemHash) => ({
598
673
  itemHash,
599
- ...(final.errorCode ? { errorCode: final.errorCode } : {}),
674
+ error: error instanceof Error ? error.message : String(error),
600
675
  })),
601
676
  );
602
- } else {
603
- forgotten.push(...hashes);
604
677
  }
605
678
  onProgress?.({
606
679
  done: Math.min(i + batch, targets.length),
607
680
  total: targets.length,
608
681
  });
609
682
  }
610
- return { forgotten, kept, rejected };
683
+ return { forgotten, kept: [...kept], rejected, pending, failed };
611
684
  }
612
685
 
613
686
  /** The reserved channel and aggregate key Aleph keeps permissions in. */
@@ -39,7 +39,7 @@
39
39
  * @module backends/encryption
40
40
  */
41
41
 
42
- import { defineBackend, BackendError } from "./types.js";
42
+ import { defineBackend, BackendError, handleId } from "./types.js";
43
43
 
44
44
  /** `OSBE` — orbitdb-storage-bridge, encrypted. */
45
45
  // Not frozen: freezing a typed array throws, because its elements live in a
@@ -53,7 +53,8 @@ const HEADER_LENGTH = ENVELOPE_MAGIC.length + 2;
53
53
 
54
54
  /** Do these bytes start with an envelope this module wrote? */
55
55
  export function isEncrypted(bytes) {
56
- if (!(bytes instanceof Uint8Array) || bytes.length < HEADER_LENGTH) return false;
56
+ if (!(bytes instanceof Uint8Array) || bytes.length < HEADER_LENGTH)
57
+ return false;
57
58
  return ENVELOPE_MAGIC.every((byte, index) => bytes[index] === byte);
58
59
  }
59
60
 
@@ -66,10 +67,16 @@ export function isEncrypted(bytes) {
66
67
  */
67
68
  export function wrapEnvelope(ciphertext, iv) {
68
69
  if (!(ciphertext instanceof Uint8Array) || !(iv instanceof Uint8Array)) {
69
- throw new BackendError("INVALID_BACKEND", "encrypt must return Uint8Array ciphertext and iv");
70
+ throw new BackendError(
71
+ "INVALID_BACKEND",
72
+ "encrypt must return Uint8Array ciphertext and iv",
73
+ );
70
74
  }
71
75
  if (iv.length > 255) {
72
- throw new BackendError("INVALID_BACKEND", `An IV of ${iv.length} bytes does not fit the envelope`);
76
+ throw new BackendError(
77
+ "INVALID_BACKEND",
78
+ `An IV of ${iv.length} bytes does not fit the envelope`,
79
+ );
73
80
  }
74
81
  const out = new Uint8Array(HEADER_LENGTH + iv.length + ciphertext.length);
75
82
  out.set(ENVELOPE_MAGIC, 0);
@@ -88,7 +95,10 @@ export function wrapEnvelope(ciphertext, iv) {
88
95
  */
89
96
  export function readEnvelope(bytes) {
90
97
  if (!isEncrypted(bytes)) {
91
- throw new BackendError("INVALID_BACKEND", "These bytes are not an encrypted backup");
98
+ throw new BackendError(
99
+ "INVALID_BACKEND",
100
+ "These bytes are not an encrypted backup",
101
+ );
92
102
  }
93
103
  const version = bytes[ENVELOPE_MAGIC.length];
94
104
  if (version !== ENVELOPE_VERSION) {
@@ -111,15 +121,24 @@ export function readEnvelope(bytes) {
111
121
  * @param {object} options
112
122
  * @param {(plaintext: Uint8Array) => Promise<{ciphertext: Uint8Array, iv: Uint8Array}>} options.encrypt
113
123
  * @param {(ciphertext: Uint8Array, iv: Uint8Array) => Promise<Uint8Array>} options.decrypt
114
- * @param {boolean} [options.allowPlaintextReads=true] - restore a backup written
115
- * before backups were encrypted, or written with `dontEncrypt`. On by
116
- * default: a pointer does not say which kind it names, and refusing would
117
- * make yesterday's backups unreadable for no gain in secrecy.
124
+ * @param {boolean} [options.allowPlaintextReads=false] - also hand back a
125
+ * blob that is not an envelope: a backup written before backups were
126
+ * encrypted, or with `dontEncrypt`. Off by default since 0.19.0: a caller
127
+ * that brings a key expects ciphertext, and whoever answers the fetch
128
+ * (gateway, service, network) could otherwise hand it a plaintext backup
129
+ * of *another* database, which it would open and join. Turn it on only for
130
+ * the one restore that needs it, and the acceptance is logged.
118
131
  * @returns {import("./types.js").StorageBackend}
119
132
  */
120
- export function withEncryption(backend, { encrypt, decrypt, allowPlaintextReads = true } = {}) {
133
+ export function withEncryption(
134
+ backend,
135
+ { encrypt, decrypt, allowPlaintextReads = false } = {},
136
+ ) {
121
137
  if (!backend || typeof backend.putBlob !== "function") {
122
- throw new BackendError("INVALID_BACKEND", "withEncryption needs a backend to wrap");
138
+ throw new BackendError(
139
+ "INVALID_BACKEND",
140
+ "withEncryption needs a backend to wrap",
141
+ );
123
142
  }
124
143
  if (typeof encrypt !== "function" || typeof decrypt !== "function") {
125
144
  throw new BackendError(
@@ -149,10 +168,15 @@ export function withEncryption(backend, { encrypt, decrypt, allowPlaintextReads
149
168
  async getBlob(handle) {
150
169
  const bytes = await inner.getBlob(handle);
151
170
  if (!isEncrypted(bytes)) {
152
- if (allowPlaintextReads) return bytes;
171
+ if (allowPlaintextReads) {
172
+ console.warn(
173
+ `[orbitdb-storage-bridge] accepting an unencrypted blob (${handleId(handle)}) because allowPlaintextReads is on`,
174
+ );
175
+ return bytes;
176
+ }
153
177
  throw new BackendError(
154
178
  "INVALID_BACKEND",
155
- "This backup is not encrypted, and allowPlaintextReads is off",
179
+ "This backup is not encrypted. A sealed archive never holds plaintext; pass allowPlaintextReads: true only to read a backup written before encryption.",
156
180
  );
157
181
  }
158
182
  const { ciphertext, iv } = readEnvelope(bytes);
@@ -185,24 +209,38 @@ export function withEncryption(backend, { encrypt, decrypt, allowPlaintextReads
185
209
  * @param {(cid: string, options?: object) => Promise<Uint8Array>} fetchBytes
186
210
  * @param {object} options
187
211
  * @param {(ciphertext: Uint8Array, iv: Uint8Array) => Promise<Uint8Array>} options.decrypt
188
- * @param {boolean} [options.allowPlaintextReads=true]
212
+ * @param {boolean} [options.allowPlaintextReads=false] - see {@link withEncryption}
189
213
  * @returns {(cid: string, options?: object) => Promise<Uint8Array>}
190
214
  */
191
- export function decryptingFetch(fetchBytes, { decrypt, allowPlaintextReads = true } = {}) {
215
+ export function decryptingFetch(
216
+ fetchBytes,
217
+ { decrypt, allowPlaintextReads = false } = {},
218
+ ) {
192
219
  if (typeof fetchBytes !== "function") {
193
- throw new BackendError("INVALID_BACKEND", "decryptingFetch needs a fetcher to wrap");
220
+ throw new BackendError(
221
+ "INVALID_BACKEND",
222
+ "decryptingFetch needs a fetcher to wrap",
223
+ );
194
224
  }
195
225
  if (typeof decrypt !== "function") {
196
- throw new BackendError("INVALID_BACKEND", "decryptingFetch needs a decrypt function");
226
+ throw new BackendError(
227
+ "INVALID_BACKEND",
228
+ "decryptingFetch needs a decrypt function",
229
+ );
197
230
  }
198
231
 
199
232
  return async function fetchAndDecrypt(cid, options) {
200
233
  const bytes = await fetchBytes(cid, options);
201
234
  if (!isEncrypted(bytes)) {
202
- if (allowPlaintextReads) return bytes;
235
+ if (allowPlaintextReads) {
236
+ console.warn(
237
+ `[orbitdb-storage-bridge] accepting an unencrypted backup at ${cid} because allowPlaintextReads is on`,
238
+ );
239
+ return bytes;
240
+ }
203
241
  throw new BackendError(
204
242
  "INVALID_BACKEND",
205
- `The backup at ${cid} is not encrypted, and allowPlaintextReads is off`,
243
+ `The backup at ${cid} is not encrypted. A sealed backup never comes back as plaintext; pass allowPlaintextReads: true only to read a backup written before encryption.`,
206
244
  );
207
245
  }
208
246
  const { ciphertext, iv } = readEnvelope(bytes);
@@ -40,6 +40,10 @@
40
40
  * @property {string} backend - the `name` of the backend that issued it
41
41
  * @property {string} [cid] - set when the id is, or carries, a content identifier
42
42
  * @property {number} [size] - stored size in bytes, when the backend reports one
43
+ * @property {boolean} [retained] - whether the service keeps it: a backend whose upload is
44
+ * only ingest (Aleph) says false until `pinCid` has made it stick; absent means stored is kept
45
+ * @property {string} [status] - the service's own word for where a pin stands (Aleph: `processed`, `pending`, `rejected`)
46
+ * @property {string} [itemHash] - the message that keeps it (Aleph), to follow with `waitForMessage`
43
47
  */
44
48
 
45
49
  /**
package/lib/dehydrate.js CHANGED
@@ -83,7 +83,10 @@ export async function dehydrate({
83
83
 
84
84
  const { backupDatabaseCAR } = await import("./backup-car.js");
85
85
  const storeIn = encrypt
86
- ? (await import("./backends/encryption.js")).withEncryption(backend, { encrypt, decrypt })
86
+ ? (await import("./backends/encryption.js")).withEncryption(backend, {
87
+ encrypt,
88
+ decrypt,
89
+ })
87
90
  : backend;
88
91
 
89
92
  const result = await backupDatabaseCAR(orbitdb, address, {
@@ -129,7 +132,8 @@ export async function dehydrate({
129
132
  * @param {string[]} [params.endpoints]
130
133
  * @param {Object} [params.open] Options for `orbitdb.open` — a node without a
131
134
  * pubsub service needs `{ sync: false }`.
132
- * @param {Object} [params.restore] Extra options for `restoreFromCID`.
135
+ * @param {Object} [params.restore] Extra options for `restoreFromCID`; plus
136
+ * `allowPlaintextReads: true` to accept an unencrypted backup with a key in hand.
133
137
  * @returns {Promise<{db: Object, address: string, name: string, metadataCID: string, blocks: number, entries: number|null}>}
134
138
  */
135
139
  export async function hydrate({
@@ -145,21 +149,28 @@ export async function hydrate({
145
149
  const pointer = await resolvePointer({ privateKey, endpoints });
146
150
 
147
151
  const { restoreFromCID } = await import("./restore-cid.js");
148
- const { decryptingFetch, explainIfEncrypted } = await import("./backends/encryption.js");
152
+ const { decryptingFetch, explainIfEncrypted } =
153
+ await import("./backends/encryption.js");
149
154
  const { fetchFromGateways } = await import("./gateway-fetch.js");
150
155
 
151
156
  // A restore does not read through a backend — it fetches from a gateway or
152
157
  // from peers — so decryption belongs here, around the fetcher. Without a
153
158
  // key, an encrypted backup is still recognised, and says so.
154
159
  const fetchBytes = restore.fetchBytes ?? fetchFromGateways;
160
+ // Plaintext is refused with a key in hand (0.19.0); `restore.allowPlaintextReads`
161
+ // is the way to read a backup written before encryption, for that one restore.
155
162
  const reader = decrypt
156
- ? decryptingFetch(fetchBytes, { decrypt })
163
+ ? decryptingFetch(fetchBytes, {
164
+ decrypt,
165
+ allowPlaintextReads: restore.allowPlaintextReads === true,
166
+ })
157
167
  : explainIfEncrypted(fetchBytes);
168
+ const { allowPlaintextReads: _plain, ...restoreOptions } = restore;
158
169
 
159
170
  const result = await restoreFromCID(orbitdb, {
160
171
  metadataCID: pointer.cid,
161
172
  open,
162
- ...restore,
173
+ ...restoreOptions,
163
174
  fetchBytes: reader,
164
175
  });
165
176
 
@@ -30,6 +30,8 @@
30
30
  * rotted unnoticed. The replacement is not a longer list: it is fetching from
31
31
  * the providers that hold the blocks, over libp2p — see issue #112.
32
32
  */
33
+
34
+ import { CID } from "multiformats/cid";
33
35
  export const DEFAULT_GATEWAYS = ["https://ipfs.aleph.cloud/ipfs"];
34
36
 
35
37
  /**
@@ -74,11 +76,16 @@ export const MAX_BACKOFF_MS = 30_000;
74
76
  * because the header is not always honest.
75
77
  */
76
78
  const looksLikeAnErrorPage = (bytes, contentType = "") => {
77
- if (contentType.includes("text/html") || contentType.includes("xhtml")) return true;
79
+ if (contentType.includes("text/html") || contentType.includes("xhtml"))
80
+ return true;
78
81
  const start = new TextDecoder("utf-8", { fatal: false })
79
82
  .decode(bytes.subarray(0, Math.min(100, bytes.length)))
80
83
  .trim();
81
- return start.startsWith("<!DOCTYPE") || start.startsWith("<html") || start.startsWith("<?xml");
84
+ return (
85
+ start.startsWith("<!DOCTYPE") ||
86
+ start.startsWith("<html") ||
87
+ start.startsWith("<?xml")
88
+ );
82
89
  };
83
90
 
84
91
  const waitFor = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -93,9 +100,13 @@ const backoffFor = (response, attempt) => {
93
100
  const seconds = Number.parseInt(retryAfter, 10);
94
101
  if (Number.isFinite(seconds)) return seconds * 1000;
95
102
  const date = new Date(retryAfter);
96
- if (!Number.isNaN(date.getTime())) return Math.max(0, date.getTime() - Date.now());
103
+ if (!Number.isNaN(date.getTime()))
104
+ return Math.max(0, date.getTime() - Date.now());
97
105
  }
98
- const reset = Number.parseInt(response.headers?.get?.("X-RateLimit-Reset") ?? "", 10);
106
+ const reset = Number.parseInt(
107
+ response.headers?.get?.("X-RateLimit-Reset") ?? "",
108
+ 10,
109
+ );
99
110
  if (Number.isFinite(reset)) return Math.max(0, reset * 1000 - Date.now());
100
111
  return 2000 * (attempt + 1);
101
112
  };
@@ -115,7 +126,45 @@ const backoffFor = (response, attempt) => {
115
126
  * @param {typeof fetch} [options.fetchImpl] injectable for tests
116
127
  * @returns {Promise<Uint8Array>}
117
128
  */
118
- export async function fetchFromGateways(cid, { gateways = DEFAULT_GATEWAYS, timeout = DEFAULT_TIMEOUT_MS, signal = null, log = SILENT, accept = null, maxBackoff = MAX_BACKOFF_MS, sleep = waitFor, fetchImpl = fetch } = {}) {
129
+ /**
130
+ * True unless the CID is a raw sha-256 block whose digest the bytes miss.
131
+ * Anything this cannot check (another codec, another hash, an unparsable
132
+ * string) passes: the check is for the case it can prove, not a gate.
133
+ *
134
+ * @param {string} cid
135
+ * @param {Uint8Array} bytes
136
+ */
137
+ export async function matchesRawCid(cid, bytes) {
138
+ let parsed;
139
+ try {
140
+ parsed = CID.parse(String(cid));
141
+ } catch {
142
+ return true;
143
+ }
144
+ if (parsed.code !== 0x55 || parsed.multihash.code !== 0x12) return true;
145
+ const digest = new Uint8Array(
146
+ await globalThis.crypto.subtle.digest("SHA-256", bytes),
147
+ );
148
+ const expected = parsed.multihash.digest;
149
+ if (digest.length !== expected.length) return false;
150
+ for (let i = 0; i < digest.length; i++)
151
+ if (digest[i] !== expected[i]) return false;
152
+ return true;
153
+ }
154
+
155
+ export async function fetchFromGateways(
156
+ cid,
157
+ {
158
+ gateways = DEFAULT_GATEWAYS,
159
+ timeout = DEFAULT_TIMEOUT_MS,
160
+ signal = null,
161
+ log = SILENT,
162
+ accept = null,
163
+ maxBackoff = MAX_BACKOFF_MS,
164
+ sleep = waitFor,
165
+ fetchImpl = fetch,
166
+ } = {},
167
+ ) {
119
168
  let lastError = null;
120
169
 
121
170
  for (const gateway of gateways) {
@@ -139,10 +188,14 @@ export async function fetchFromGateways(cid, { gateways = DEFAULT_GATEWAYS, time
139
188
  if (response.status === 429 && attempt < MAX_ATTEMPTS_PER_GATEWAY - 1) {
140
189
  const wait = backoffFor(response, attempt);
141
190
  if (wait > maxBackoff) {
142
- log.warn(` ⚠️ ${gateway} wants ${Math.round(wait / 1000)}s — treating it as closed`);
191
+ log.warn(
192
+ ` ⚠️ ${gateway} wants ${Math.round(wait / 1000)}s — treating it as closed`,
193
+ );
143
194
  break;
144
195
  }
145
- log.warn(` ⚠️ ${gateway} rate-limited; waiting ${Math.round(wait / 1000)}s`);
196
+ log.warn(
197
+ ` ⚠️ ${gateway} rate-limited; waiting ${Math.round(wait / 1000)}s`,
198
+ );
146
199
  await sleep(wait);
147
200
  continue;
148
201
  }
@@ -152,10 +205,24 @@ export async function fetchFromGateways(cid, { gateways = DEFAULT_GATEWAYS, time
152
205
  }
153
206
 
154
207
  const bytes = new Uint8Array(await response.arrayBuffer());
155
- if (looksLikeAnErrorPage(bytes, response.headers?.get?.("content-type") ?? "")) {
208
+ if (
209
+ looksLikeAnErrorPage(
210
+ bytes,
211
+ response.headers?.get?.("content-type") ?? "",
212
+ )
213
+ ) {
156
214
  log.warn(` ⚠️ ${gateway} returned an error page with a 200`);
157
215
  break;
158
216
  }
217
+ // What came back is what was asked for — provable when the CID names
218
+ // the bytes themselves (a raw block, sha-256). A UnixFS CID names a
219
+ // DAG the gateway assembled, and the chunking that made it is the
220
+ // service's; those bytes are trusted as far as the envelope around
221
+ // them goes, which is why a sealed backup refuses plaintext.
222
+ if (!(await matchesRawCid(cid, bytes))) {
223
+ log.warn(` ⚠️ ${gateway} served bytes that do not hash to ${cid}`);
224
+ break;
225
+ }
159
226
  log.info(` ✅ ${bytes.length} bytes from ${gateway}`);
160
227
  return bytes;
161
228
  } catch (error) {
@@ -347,6 +347,16 @@ export function mergeIncrements({ base, increments = [] }) {
347
347
  timestamp = Math.max(timestamp, Number(metadata.timestamp) || 0);
348
348
  for (const d of metadata.databases) {
349
349
  const known = databases.get(String(d.address)) ?? {};
350
+ // Every source's heads, not only the newest's: a join is a merge, a
351
+ // stale head costs nothing, and a branch only the older source saw —
352
+ // two devices shipping in parallel, or a store that renumbered — is
353
+ // kept instead of dropped.
354
+ const heads = [
355
+ ...new Set([
356
+ ...(known.heads ?? []),
357
+ ...(Array.isArray(d.heads) ? d.heads : []),
358
+ ]),
359
+ ];
350
360
  databases.set(String(d.address), {
351
361
  ...known,
352
362
  address: String(d.address),
@@ -354,8 +364,7 @@ export function mergeIncrements({ base, increments = [] }) {
354
364
  ...(d.type ? { type: d.type } : {}),
355
365
  ...(d.manifestCID ? { manifestCID: d.manifestCID } : {}),
356
366
  ...(d.collection ? { collection: d.collection } : {}),
357
- // Later sources are newer: their heads supersede.
358
- ...(Array.isArray(d.heads) && d.heads.length ? { heads: d.heads } : {}),
367
+ ...(heads.length ? { heads } : {}),
359
368
  ...(typeof d.entryCount === "number"
360
369
  ? { entryCount: d.entryCount }
361
370
  : {}),
@@ -363,6 +372,17 @@ export function mergeIncrements({ base, increments = [] }) {
363
372
  }
364
373
  }
365
374
  const described = [...databases.values()];
375
+ const sequences = ordered.map((i) => i.manifest?.sequence ?? 0);
376
+ // A chain with a hole restores the heads of the newer part onto entries
377
+ // nobody has: the join then searches the network for each missing block,
378
+ // thirty seconds at a time. Named here, so a caller can refuse up front.
379
+ const gaps = [];
380
+ const duplicates = [];
381
+ for (let i = 1; i < sequences.length; i++) {
382
+ if (sequences[i] === sequences[i - 1]) duplicates.push(sequences[i]);
383
+ for (let missing = sequences[i - 1] + 1; missing < sequences[i]; missing++)
384
+ gaps.push(missing);
385
+ }
366
386
  return {
367
387
  blocks,
368
388
  metadata: {
@@ -375,10 +395,27 @@ export function mergeIncrements({ base, increments = [] }) {
375
395
  : {}),
376
396
  databases: described,
377
397
  },
378
- sequences: ordered.map((i) => i.manifest?.sequence ?? 0),
398
+ sequences,
399
+ gaps,
400
+ duplicates,
379
401
  };
380
402
  }
381
403
 
404
+ /** Refuse a chain with holes, unless the caller said it knows. */
405
+ function assertContiguous({ gaps, duplicates }, allowGaps) {
406
+ if (allowGaps) return;
407
+ if (gaps.length) {
408
+ throw new Error(
409
+ `The increments have a gap: sequence ${gaps.join(", ")} missing. Fetch them, or pass allowGaps: true to join what is there.`,
410
+ );
411
+ }
412
+ if (duplicates.length) {
413
+ throw new Error(
414
+ `Two increments carry the same sequence ${[...new Set(duplicates)].join(", ")}: the chain was written twice, or renumbered. Pass allowGaps: true to join them all.`,
415
+ );
416
+ }
417
+ }
418
+
382
419
  /**
383
420
  * Put a base backup and its increments back into a node, through
384
421
  * `restoreFromBlocks`: the union of their blocks, and each database joined at
@@ -391,19 +428,28 @@ export function mergeIncrements({ base, increments = [] }) {
391
428
  * @param {object} [params.open] - options for `orbitdb.open` (`encryption`, `AccessController`, …)
392
429
  * @param {object} [params.log]
393
430
  * @param {Function} [params.onProgress]
431
+ * @param {boolean} [params.allowGaps=false] - join a chain with missing or
432
+ * duplicate sequence numbers instead of refusing it
394
433
  * @returns {Promise<{ blocks: number, databases: object[], sequences: number[] }>}
395
434
  */
396
435
  export async function restoreFromIncrements(
397
436
  orbitdb,
398
- { base, increments = [], open, log, onProgress } = {},
437
+ { base, increments = [], open, log, onProgress, allowGaps = false } = {},
399
438
  ) {
400
- const { blocks, metadata, sequences } = mergeIncrements({ base, increments });
439
+ const merged = mergeIncrements({ base, increments });
440
+ assertContiguous(merged, allowGaps);
441
+ const { blocks, metadata, sequences } = merged;
401
442
  const restored = await restoreFromBlocks(orbitdb, blocks, metadata, {
402
443
  open,
403
444
  log,
404
445
  onProgress,
405
446
  });
406
- return { ...restored, sequences };
447
+ return {
448
+ ...restored,
449
+ sequences,
450
+ gaps: merged.gaps,
451
+ duplicates: merged.duplicates,
452
+ };
407
453
  }
408
454
 
409
455
  /**
@@ -420,10 +466,22 @@ export async function restoreFromIncrements(
420
466
  * @param {object} params
421
467
  * @param {{ metadata?: object, manifest?: object, blocks: Map }} [params.base]
422
468
  * @param {{ manifest: IncrementManifest, blocks: Map }[]} params.increments
423
- * @returns {Promise<{ joined: number, heads: number, blocks: number }>}
469
+ * @param {{ put: (cid: CID, bytes: Uint8Array) => Promise<unknown> }} [params.blockstore] -
470
+ * where the application's own (non-dag-cbor) blocks go, e.g. `orbitdb.ipfs.blockstore`;
471
+ * without it they are counted in `skippedRaw` and left out
472
+ * @param {boolean} [params.allowGaps=false]
473
+ * @returns {Promise<{ joined: number, heads: number, blocks: number, gaps: number[], duplicates: number[], skippedRaw?: number }>}
474
+ * `gaps`: sequence numbers missing from the chain; `duplicates`: sequence
475
+ * numbers carried by more than one increment (two devices numbering on
476
+ * their own) — both empty unless the join was told `allowGaps`
424
477
  */
425
- export async function joinIncrements(database, { base, increments = [] } = {}) {
426
- const { blocks, metadata } = mergeIncrements({ base, increments });
478
+ export async function joinIncrements(
479
+ database,
480
+ { base, increments = [], blockstore, allowGaps = false } = {},
481
+ ) {
482
+ const merged = mergeIncrements({ base, increments });
483
+ assertContiguous(merged, allowGaps);
484
+ const { blocks, metadata } = merged;
427
485
  const address = String(database.address);
428
486
  const described = metadata.databases.find(
429
487
  (d) => String(d.address) === address,
@@ -431,14 +489,21 @@ export async function joinIncrements(database, { base, increments = [] } = {}) {
431
489
  if (!described)
432
490
  throw new Error("These increments hold nothing for this database.");
433
491
 
492
+ let skippedRaw = 0;
434
493
  for (const [name, { bytes }] of blocks) {
435
494
  const cid = CID.parse(name);
495
+ if (cid.code !== DAG_CBOR) {
496
+ // The application's own blocks (receipt files): not the log's, and the
497
+ // log's storage refuses their base32 names. They go where Helia looks,
498
+ // when the caller says where that is.
499
+ if (blockstore) await blockstore.put(cid, bytes);
500
+ else skippedRaw++;
501
+ continue;
502
+ }
436
503
  // OrbitDB names its blocks in base58btc; a CAR in base32. The log reads
437
504
  // its own names, and identities are looked up by theirs, so each block
438
505
  // goes in under the name its reader will ask for.
439
- const stored =
440
- cid.code === DAG_CBOR ? cid.toV1().toString(base58btc) : name;
441
- await database.log.storage.put(stored, bytes);
506
+ await database.log.storage.put(cid.toV1().toString(base58btc), bytes);
442
507
  }
443
508
 
444
509
  let joined = 0;
@@ -447,7 +512,15 @@ export async function joinIncrements(database, { base, increments = [] } = {}) {
447
512
  const entry = await database.log.get(name).catch(() => null);
448
513
  if (entry && (await database.log.joinEntry(entry))) joined++;
449
514
  }
450
- return { joined, heads: (described.heads ?? []).length, blocks: blocks.size };
515
+ return {
516
+ joined,
517
+ heads: (described.heads ?? []).length,
518
+ blocks: blocks.size,
519
+ // What the chain looked like, for a caller that joined it with allowGaps.
520
+ gaps: merged.gaps,
521
+ duplicates: merged.duplicates,
522
+ ...(skippedRaw ? { skippedRaw } : {}),
523
+ };
451
524
  }
452
525
 
453
526
  /** A state with nothing shipped yet. */
@@ -481,8 +554,13 @@ export const emptyLiveBackupState = () => ({
481
554
  * @param {(sequence: number, at: Date) => string} [params.name] - the blob's name
482
555
  * @param {(shipped: { sequence: number, handle: object, kept?: object, entries: number, blocks: number, bytes: number, at: Date }) => void} [params.onShipped]
483
556
  * @param {(error: Error) => void} [params.onError]
557
+ * @param {(kept: object, handle: object) => Promise<object>} [params.confirm] -
558
+ * follow a backend's answer before the increment counts as shipped: Aleph
559
+ * answers `pending` first and may reject later (no credit, no grant). Given
560
+ * `(kept, handle)`, return the settled answer; `status: "rejected"` fails the
561
+ * flush and the state stays, so the next one ships it all again.
484
562
  * @param {() => Date} [params.now]
485
- * @returns {{ watch: Function, unwatch: Function, start: Function, stop: Function,
563
+ * @returns {{ watch: Function, unwatch: Function, start: Function, stop: Function, close: Function,
486
564
  * flush: () => Promise<object|null>, rebase: Function, status: () => object }}
487
565
  */
488
566
  export function createLiveBackup({
@@ -498,6 +576,7 @@ export function createLiveBackup({
498
576
  `increment-${String(sequence).padStart(6, "0")}-${at.toISOString().replace(/[:.]/g, "-")}.car`,
499
577
  onShipped,
500
578
  onError,
579
+ confirm: confirmKept,
501
580
  now = () => new Date(),
502
581
  } = {}) {
503
582
  if (!backend || typeof backend.putBlob !== "function") {
@@ -564,6 +643,13 @@ export function createLiveBackup({
564
643
  watched.delete(address);
565
644
  };
566
645
 
646
+ /** Stop, and let go of the databases: no listener keeps counting after this. */
647
+ const close = async ({ flush: last = false } = {}) => {
648
+ await stop({ flush: last });
649
+ for (const w of watched.values()) w.off();
650
+ watched.clear();
651
+ };
652
+
567
653
  if (Array.isArray(databases)) databases.forEach((d) => watch(d));
568
654
  else
569
655
  for (const [collection, d] of Object.entries(databases ?? {}))
@@ -572,8 +658,17 @@ export function createLiveBackup({
572
658
  const load = async () => {
573
659
  if (!loaded) {
574
660
  const saved = await store.load();
575
- if (saved && saved.v === LIVE_BACKUP_STATE_VERSION)
661
+ if (saved) {
662
+ // A state this build does not read must not be dropped: starting at 1
663
+ // again would ship everything twice under sequence numbers the chain
664
+ // already has.
665
+ if (saved.v !== LIVE_BACKUP_STATE_VERSION) {
666
+ throw new Error(
667
+ `The live backup's state is version ${saved.v}, and this build reads ${LIVE_BACKUP_STATE_VERSION}. Upgrade @le-space/orbitdb-storage-bridge, or clear the state and rebase.`,
668
+ );
669
+ }
576
670
  state = { ...emptyLiveBackupState(), ...saved };
671
+ }
577
672
  loaded = true;
578
673
  }
579
674
  for (const [address, w] of watched) {
@@ -592,6 +687,10 @@ export function createLiveBackup({
592
687
 
593
688
  const ship = async () => {
594
689
  await load();
690
+ // Counted before the walk: OrbitDB appends before it emits `update`, so
691
+ // every update counted here is in the log the walk is about to read, and
692
+ // one that fires during the walk stays pending for the next flush.
693
+ const countedPending = pending;
595
694
  const own = typeof ownBlocks === "function" ? await ownBlocks() : ownBlocks;
596
695
  const collected = await collectIncrement(watched, {
597
696
  sequence: state.sequence + 1,
@@ -600,9 +699,8 @@ export function createLiveBackup({
600
699
  blocks: own,
601
700
  now,
602
701
  });
603
- const countedPending = pending;
604
702
  if (!collected) {
605
- pending = 0;
703
+ pending = Math.max(0, pending - countedPending);
606
704
  return null;
607
705
  }
608
706
  const at = now();
@@ -618,13 +716,25 @@ export function createLiveBackup({
618
716
  typeof backend.pinCid === "function"
619
717
  ) {
620
718
  kept = await backend.pinCid(handle.id, { name: handle.name });
719
+ if (confirmKept) kept = await confirmKept(kept, handle);
720
+ if (kept?.status === "rejected") {
721
+ throw new Error(
722
+ `The store refused to keep increment ${collected.manifest.sequence}${kept.errorCode != null ? ` (${kept.errorCode})` : ""}: it was uploaded but is not kept.`,
723
+ );
724
+ }
621
725
  }
622
726
 
623
- // Only now, with the bytes out of the house, does the state move.
624
- state.sequence = collected.manifest.sequence;
625
- state.previous = String(handle.id ?? handle.cid ?? "");
727
+ // Only now, with the bytes out of the house, does the state move — and
728
+ // all of it at once, so a database unwatched during the upload cannot
729
+ // leave it half moved.
730
+ const moved = [];
626
731
  for (const [address, { heads, hashes }] of collected.shipped) {
627
732
  const w = watched.get(address);
733
+ if (w) moved.push({ address, w, heads, hashes });
734
+ }
735
+ state.sequence = collected.manifest.sequence;
736
+ state.previous = String(handle.id ?? handle.cid ?? "");
737
+ for (const { address, w, heads, hashes } of moved) {
628
738
  hashes.forEach((hash) => w.shipped.add(hash));
629
739
  w.shipped.heads = heads;
630
740
  state.databases[address] = { heads };
@@ -665,7 +775,7 @@ export function createLiveBackup({
665
775
  inflight = null;
666
776
  if (again) {
667
777
  again = false;
668
- flush().catch(() => {});
778
+ if (running) flush().catch(() => {});
669
779
  }
670
780
  }
671
781
  })();
@@ -683,6 +793,8 @@ export function createLiveBackup({
683
793
  * @param {{ base?: string|null }} [params] - what the caller calls the full backup (its id, CID, …)
684
794
  */
685
795
  const rebase = async ({ base } = {}) => {
796
+ // Not under a flush: ship() would write the pre-rebase state back.
797
+ if (inflight) await inflight.catch(() => {});
686
798
  await load();
687
799
  if (base !== undefined)
688
800
  state = { ...emptyLiveBackupState(), base: base ?? null };
@@ -713,7 +825,11 @@ export function createLiveBackup({
713
825
  if (timer) clearTimeout(timer);
714
826
  if (settling) clearTimeout(settling);
715
827
  timer = settling = null;
716
- if (last) await flush();
828
+ // A flush asked for during one runs after it; the caller gets its own.
829
+ if (last) {
830
+ if (inflight) await inflight.catch(() => {});
831
+ await flush();
832
+ }
717
833
  };
718
834
 
719
835
  const status = () => ({
@@ -728,7 +844,7 @@ export function createLiveBackup({
728
844
  failures,
729
845
  });
730
846
 
731
- return { watch, unwatch, start, stop, flush, rebase, status };
847
+ return { watch, unwatch, start, stop, close, flush, rebase, status };
732
848
  }
733
849
 
734
850
  export default createLiveBackup;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@le-space/orbitdb-storage-bridge",
3
- "version": "0.19.0",
3
+ "version": "0.21.0",
4
4
  "description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
5
5
  "main": "lib/orbitdb-storacha-bridge.js",
6
6
  "svelte": "dist/components/",