@le-space/orbitdb-storage-bridge 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2789 @@
1
+ /**
2
+ * OrbitDB Storacha Bridge - Main Library
3
+ *
4
+ * Provides complete OrbitDB database backup and restoration via Storacha/Filecoin
5
+ * with 100% hash preservation and identity recovery.
6
+ */
7
+
8
+ import * as Client from "@storacha/client";
9
+ import { StoreMemory } from "@storacha/client/stores/memory";
10
+ import { Signer } from "@storacha/client/principal/ed25519";
11
+ import * as Proof from "@storacha/client/proof";
12
+ import { BackendError } from "./backends/types.js";
13
+ import { CID } from "multiformats/cid";
14
+ import * as Block from "multiformats/block";
15
+ import * as dagCbor from "@ipld/dag-cbor";
16
+ import { sha256 } from "multiformats/hashes/sha2";
17
+ import { bases } from "multiformats/basics";
18
+ import { EventEmitter } from "events";
19
+ import { createHeliaOrbitDB, cleanupOrbitDBDirectories } from "./utils.js";
20
+ import {
21
+ generateBackupPrefix,
22
+ getBackupFilenames,
23
+ findLatestBackup,
24
+ } from "./backup-helpers.js";
25
+ import { logger, createLogger } from "./logger.js";
26
+ import { unixfs } from "@helia/unixfs";
27
+ import { readBlockBytes } from "./block-bytes.js";
28
+ // Moved out of this file so that backing up does not drag the Storacha client
29
+ // into a browser bundle; re-exported here, because this is where callers of
30
+ // every vintage look for it.
31
+ import { extractDatabaseBlocks } from "./extract-blocks.js";
32
+ import { resolveBackend } from "./backends/resolve.js";
33
+
34
+ export { extractDatabaseBlocks };
35
+
36
+ const HELIA_LOGGER = createLogger("helia");
37
+ const HELIA_COLOR = "\x1b[95m";
38
+ const COLOR_RESET = "\x1b[0m";
39
+
40
+ function logHeliaActivity(message) {
41
+ HELIA_LOGGER(`${HELIA_COLOR}${message}${COLOR_RESET}`);
42
+ }
43
+
44
+ /**
45
+ * Default configuration options
46
+ */
47
+ const DEFAULT_OPTIONS = {
48
+ // Core configuration
49
+ timeout: 30000, // Timeout in milliseconds
50
+ gateway: "https://dweb.link", // IPFS gateway URL (w3s.link only redirects here now)
51
+ verbose: false, // Enable verbose debug logging
52
+
53
+ // Network download options
54
+ useIPFSNetwork: true, // Enable network downloads by default
55
+ gatewayFallback: true, // Fallback to gateway if network fails
56
+
57
+ // Performance tuning (used in specific functions)
58
+ batchSize: 10, // Batch size for upload/download operations (removeLayerFiles)
59
+ maxConcurrency: 3, // Maximum concurrent batches (uploadBlocks)
60
+ uploadBatchSize: 10, // Upload batch size for uploadBlocks
61
+ uploadMaxConcurrency: 3, // Upload batch concurrency for uploadBlocks
62
+
63
+ // Credentials (required - can also be set via environment variables)
64
+ storachaKey: undefined, // Storacha private key (required)
65
+ storachaProof: undefined, // Storacha proof (required)
66
+
67
+ // Restore options
68
+ fallbackDatabaseName: undefined, // Custom name for fallback reconstruction
69
+ forceFallback: false, // Force fallback reconstruction mode
70
+ };
71
+
72
+ /**
73
+ * Convert Storacha CID format to OrbitDB CID format
74
+ *
75
+ * @param {string} storachaCID - Storacha CID (bafkre... format)
76
+ * @returns {string} - OrbitDB CID (zdpu... format)
77
+ */
78
+ export function convertStorachaCIDToOrbitDB(storachaCID) {
79
+ const storachaParsed = CID.parse(storachaCID);
80
+
81
+ // Create CIDv1 with dag-cbor codec using the same multihash
82
+ const orbitdbCID = CID.createV1(0x71, storachaParsed.multihash); // 0x71 = dag-cbor
83
+
84
+ // Return in base58btc format (zdpu prefix)
85
+ return orbitdbCID.toString(bases.base58btc);
86
+ }
87
+
88
+ /**
89
+ * Extract manifest CID from OrbitDB address
90
+ *
91
+ * @param {string} databaseAddress - OrbitDB address (/orbitdb/zdpu...)
92
+ * @returns {string} - Manifest CID
93
+ */
94
+ export function extractManifestCID(databaseAddress) {
95
+ return databaseAddress.split("/").pop();
96
+ }
97
+
98
+
99
+ /**
100
+ * Initialize Storacha client with credentials
101
+ *
102
+ * @param {string} storachaKey - Storacha private key
103
+ * @param {string} storachaProof - Storacha proof
104
+ * @param {Object} [serviceConf] - Optional service configuration for custom endpoints
105
+ * @param {string|URL} [receiptsEndpoint] - Optional receipts endpoint override
106
+ * @returns {Promise<Object>} - Initialized Storacha client
107
+ */
108
+ async function initializeStorachaClient(
109
+ storachaKey,
110
+ storachaProof,
111
+ serviceConf,
112
+ receiptsEndpoint,
113
+ ) {
114
+ const principal = Signer.parse(storachaKey);
115
+ const store = new StoreMemory();
116
+ const clientOptions = { principal, store };
117
+
118
+ if (serviceConf) {
119
+ clientOptions.serviceConf = serviceConf;
120
+ }
121
+ if (receiptsEndpoint) {
122
+ clientOptions.receiptsEndpoint = receiptsEndpoint;
123
+ }
124
+
125
+ const client = await Client.create(clientOptions);
126
+
127
+ const proof = await Proof.parse(storachaProof);
128
+ const space = await client.addSpace(proof);
129
+ await client.setCurrentSpace(space.did());
130
+
131
+ return client;
132
+ }
133
+
134
+ /**
135
+ * Initialize Storacha client with UCAN authentication
136
+ *
137
+ * @param {Object} options - UCAN options
138
+ * @param {Object} options.client - Pre-initialized w3up client
139
+ * @param {string} options.spaceDID - Target space DID
140
+ * @returns {Promise<Object>} - Initialized Storacha client
141
+ */
142
+ async function initializeStorachaClientWithUCAN(options) {
143
+ logger.info("🔐 Initializing Storacha client with UCAN authentication...");
144
+
145
+ if (!options.client) {
146
+ throw new Error("UCAN client is required");
147
+ }
148
+
149
+ // If spaceDID is provided, set it as current space
150
+ if (options.spaceDID) {
151
+ logger.info(` 🚀 Setting current space: ${options.spaceDID}`);
152
+ await options.client.setCurrentSpace(options.spaceDID);
153
+ }
154
+
155
+ logger.info("✅ UCAN Storacha client initialized");
156
+ logger.info(` 🤖 Agent: ${options.client.agent.did()}`);
157
+ logger.info(` 🚀 Current space: ${options.client.currentSpace()?.did()}`);
158
+
159
+ return options.client;
160
+ }
161
+
162
+
163
+ /**
164
+ * Upload blocks to Storacha with parallel batch processing and progress events
165
+ *
166
+ * @param {Map} blocks - Map of blocks to upload
167
+ * @param {import('./backends/types.js').StorageBackend} backend - storage backend
168
+ * @param {number} batchSize - Number of files to upload in parallel (default: 10)
169
+ * @param {number} maxConcurrency - Maximum concurrent batches (default: 3)
170
+ * @param {EventEmitter} eventEmitter - Optional event emitter for progress updates
171
+ * @returns {Promise<Object>} - Upload results and CID mappings
172
+ */
173
+ async function uploadBlocks(
174
+ blocks,
175
+ backend,
176
+ batchSize = 10,
177
+ maxConcurrency = 3,
178
+ eventEmitter = null,
179
+ ) {
180
+ logger.info(
181
+ `📤 Uploading ${blocks.size} blocks to ${backend.name} in batches of ${batchSize}...`,
182
+ );
183
+
184
+ const uploadResults = [];
185
+ const cidMappings = new Map();
186
+ const blocksArray = Array.from(blocks.entries());
187
+ const totalBlocks = blocks.size;
188
+ let completedBlocks = 0;
189
+
190
+ // Emit initial progress
191
+ if (eventEmitter) {
192
+ eventEmitter.emit("uploadProgress", {
193
+ type: "upload",
194
+ current: 0,
195
+ total: totalBlocks,
196
+ percentage: 0,
197
+ status: "starting",
198
+ });
199
+ }
200
+
201
+ // Helper function to upload a single block
202
+ const uploadSingleBlock = async ([hash, blockData]) => {
203
+ try {
204
+ logger.info(
205
+ ` 📤 Uploading block ${hash} (${blockData.bytes.length} bytes)...`,
206
+ );
207
+
208
+ const handle = await backend.putBlob(blockData.bytes, { name: hash });
209
+ const uploadedCID = handle.cid || handle.id;
210
+
211
+ logger.info(` ✅ Uploaded: ${hash} → ${uploadedCID}`);
212
+
213
+ // Update progress
214
+ completedBlocks++;
215
+ if (eventEmitter) {
216
+ eventEmitter.emit("uploadProgress", {
217
+ type: "upload",
218
+ current: completedBlocks,
219
+ total: totalBlocks,
220
+ percentage: Math.round((completedBlocks / totalBlocks) * 100),
221
+ status: "uploading",
222
+ currentBlock: {
223
+ hash,
224
+ uploadedCID,
225
+ size: blockData.bytes.length,
226
+ },
227
+ });
228
+ }
229
+
230
+ return {
231
+ originalHash: hash,
232
+ uploadedCID,
233
+ size: blockData.bytes.length,
234
+ };
235
+ } catch (error) {
236
+ // Enhanced error handling for rate limits and service errors
237
+ let errorMessage = error.message;
238
+ let rateLimitInfo = "";
239
+
240
+ // Check if it's a rate limit or service error with retry info
241
+ if (error.status === 503 || error.code === 503) {
242
+ const retryAfter =
243
+ error.headers?.get?.("Retry-After") || error["retry-after"];
244
+ const rateLimitReset =
245
+ error.headers?.get?.("X-RateLimit-Reset") ||
246
+ error["x-ratelimit-reset"];
247
+ const rateLimitRemaining =
248
+ error.headers?.get?.("X-RateLimit-Remaining") ||
249
+ error["x-ratelimit-remaining"];
250
+
251
+ rateLimitInfo += `\n 📊 Service Status: 503 Service Unavailable (rate limited or maintenance)`;
252
+
253
+ if (retryAfter) {
254
+ const retrySeconds = parseInt(retryAfter);
255
+ let retryTime = retryAfter;
256
+ if (!isNaN(retrySeconds)) {
257
+ const futureTime = new Date(Date.now() + retrySeconds * 1000);
258
+ retryTime = `${retrySeconds} seconds (available at ${futureTime.toLocaleTimeString()})`;
259
+ } else {
260
+ // Try to parse as HTTP date
261
+ const retryDate = new Date(retryAfter);
262
+ if (!isNaN(retryDate.getTime())) {
263
+ const waitMs = Math.max(0, retryDate.getTime() - Date.now());
264
+ const waitSecs = Math.ceil(waitMs / 1000);
265
+ retryTime = `${waitSecs} seconds (available at ${retryDate.toLocaleTimeString()})`;
266
+ }
267
+ }
268
+ rateLimitInfo += `\n ⏱️ Retry-After: ${retryTime}`;
269
+ }
270
+
271
+ if (rateLimitReset) {
272
+ const resetTime = parseInt(rateLimitReset);
273
+ if (!isNaN(resetTime)) {
274
+ const resetDate = new Date(resetTime * 1000);
275
+ const waitMs = Math.max(0, resetDate.getTime() - Date.now());
276
+ const waitSecs = Math.ceil(waitMs / 1000);
277
+ rateLimitInfo += `\n 🔄 Rate Limit Reset: ${resetDate.toLocaleTimeString()} (in ${waitSecs}s)`;
278
+ }
279
+ }
280
+
281
+ if (rateLimitRemaining) {
282
+ rateLimitInfo += `\n 📈 Requests Remaining: ${rateLimitRemaining}`;
283
+ }
284
+
285
+ if (!retryAfter && !rateLimitReset) {
286
+ rateLimitInfo += `\n 💡 No retry timing provided - service may be temporarily unavailable`;
287
+ }
288
+
289
+ errorMessage = `${errorMessage}${rateLimitInfo}`;
290
+ } else if (error.status || error.code) {
291
+ // Other HTTP errors
292
+ const statusCode = error.status || error.code;
293
+ errorMessage = `HTTP ${statusCode}: ${errorMessage}`;
294
+ }
295
+
296
+ logger.error(` ❌ Failed to upload block ${hash}: ${errorMessage}`);
297
+
298
+ // Update progress even for failed uploads
299
+ completedBlocks++;
300
+ if (eventEmitter) {
301
+ eventEmitter.emit("uploadProgress", {
302
+ type: "upload",
303
+ current: completedBlocks,
304
+ total: totalBlocks,
305
+ percentage: Math.round((completedBlocks / totalBlocks) * 100),
306
+ status: "uploading",
307
+ error: {
308
+ hash,
309
+ message: error.message,
310
+ },
311
+ });
312
+ }
313
+
314
+ return {
315
+ originalHash: hash,
316
+ error: error.message,
317
+ size: blockData.bytes.length,
318
+ };
319
+ }
320
+ };
321
+
322
+ // Process blocks in batches with controlled concurrency
323
+ for (let i = 0; i < blocksArray.length; i += batchSize * maxConcurrency) {
324
+ const megaBatch = blocksArray.slice(i, i + batchSize * maxConcurrency);
325
+ const batches = [];
326
+
327
+ // Split mega-batch into smaller batches
328
+ for (let j = 0; j < megaBatch.length; j += batchSize) {
329
+ const batch = megaBatch.slice(j, j + batchSize);
330
+ batches.push(batch);
331
+ }
332
+
333
+ logger.info(
334
+ ` 🔄 Processing ${batches.length} concurrent batches (${megaBatch.length} blocks)...`,
335
+ );
336
+
337
+ // Process all batches in this mega-batch concurrently
338
+ const batchPromises = batches.map(async (batch, batchIndex) => {
339
+ logger.info(
340
+ ` 📦 Batch ${batchIndex + 1}/${batches.length}: ${batch.length} blocks`,
341
+ );
342
+
343
+ // Upload all blocks in this batch in parallel
344
+ const batchResults = await Promise.allSettled(
345
+ batch.map(uploadSingleBlock),
346
+ );
347
+
348
+ return batchResults.map((result) =>
349
+ result.status === "fulfilled"
350
+ ? result.value
351
+ : {
352
+ originalHash: "unknown",
353
+ error: result.reason?.message || "Unknown error",
354
+ size: 0,
355
+ },
356
+ );
357
+ });
358
+
359
+ // Wait for all batches to complete
360
+ const batchResults = await Promise.all(batchPromises);
361
+
362
+ // Flatten and process results
363
+ for (const batchResult of batchResults) {
364
+ for (const result of batchResult) {
365
+ uploadResults.push(result);
366
+ if (result.uploadedCID) {
367
+ cidMappings.set(result.originalHash, result.uploadedCID);
368
+ }
369
+ }
370
+ }
371
+ }
372
+
373
+ const successful = uploadResults.filter((r) => r.uploadedCID);
374
+ const failed = uploadResults.filter((r) => r.error);
375
+
376
+ logger.info(` 📊 Upload summary:`);
377
+ logger.info(` Total blocks: ${blocks.size}`);
378
+ logger.info(` Successful: ${successful.length}`);
379
+ logger.info(` Failed: ${failed.length}`);
380
+ logger.info(` Batch size: ${batchSize}`);
381
+ logger.info(` Max concurrency: ${maxConcurrency}`);
382
+
383
+ // Emit completion
384
+ if (eventEmitter) {
385
+ eventEmitter.emit("uploadProgress", {
386
+ type: "upload",
387
+ current: totalBlocks,
388
+ total: totalBlocks,
389
+ percentage: 100,
390
+ status: "completed",
391
+ summary: {
392
+ successful: successful.length,
393
+ failed: failed.length,
394
+ },
395
+ });
396
+ }
397
+
398
+ return { uploadResults, successful, failed, cidMappings };
399
+ }
400
+
401
+ // executeW3Command has been removed as it was deprecated and used Node.js-specific child_process
402
+ // Use SDK equivalents like listStorachaSpaceFiles() instead
403
+
404
+ /**
405
+ * List all files in Storacha space using SDK (IMPROVED: Direct API access)
406
+ *
407
+ * @param {Object} options - Configuration options
408
+ * @param {string} [options.storachaKey] - Storacha private key (defaults to env)
409
+ * @param {string} [options.storachaProof] - Storacha proof (defaults to env)
410
+ * @param {number} [options.size] - Maximum number of items to retrieve (default: 1000000)
411
+ * @param {string} [options.cursor] - Pagination cursor
412
+ * @returns {Promise<Array>} - Space files with metadata
413
+ */
414
+ export async function listStorachaSpaceFiles(options = {}) {
415
+ const _config = { ...DEFAULT_OPTIONS, ...options };
416
+ logger.info("📋 Listing files in Storacha space using SDK...");
417
+
418
+ try {
419
+ // Initialize client - support both credential and UCAN authentication
420
+ const backend = await resolveBackend(options);
421
+
422
+ // Prepare list options
423
+ const listOptions = {};
424
+ if (options.size) {
425
+ listOptions.size = parseInt(String(options.size));
426
+ } else {
427
+ listOptions.size = 1000000; // Default to get ALL files
428
+ }
429
+ if (options.cursor) {
430
+ listOptions.cursor = options.cursor;
431
+ }
432
+ if (options.pre) {
433
+ listOptions.pre = options.pre;
434
+ }
435
+
436
+ // List through the backend contract, so this works for any driver
437
+ const entries = await backend.list(listOptions);
438
+
439
+ logger.info(` ✅ Found ${entries.length} uploads in space`);
440
+
441
+ // Convert to the format we expect, with enhanced metadata
442
+ const spaceFiles = entries.map((entry) => ({
443
+ root: entry.id,
444
+ uploaded: entry.insertedAt ? new Date(entry.insertedAt) : new Date(),
445
+ size: entry.size || "unknown",
446
+ shards: entry.raw?.shards?.length || 0,
447
+ insertedAt: entry.insertedAt,
448
+ updatedAt: entry.updatedAt,
449
+ }));
450
+
451
+ return spaceFiles;
452
+ } catch (error) {
453
+ logger.error(" ❌ SDK listing error:", error.message);
454
+ throw error;
455
+ }
456
+ }
457
+
458
+ /**
459
+ * List files in a specific Storacha layer using SDK
460
+ *
461
+ * @param {string} layer - Layer to list ('upload', 'blob')
462
+ * @param {Object} options - Configuration options
463
+ * @returns {Promise<Array>} - Layer files
464
+ */
465
+ export async function listLayerFiles(layer, options = {}) {
466
+ const _config = { ...DEFAULT_OPTIONS, ...options };
467
+
468
+ try {
469
+ // Initialize client - support both credential and UCAN authentication
470
+ const backend = await resolveBackend(options);
471
+ const client = backend.client;
472
+
473
+ // Prepare list options
474
+ const listOptions = {};
475
+ if (options.size) {
476
+ listOptions.size = parseInt(String(options.size));
477
+ } else {
478
+ listOptions.size = 1000000; // Default to get ALL files
479
+ }
480
+ if (options.cursor) {
481
+ listOptions.cursor = options.cursor;
482
+ }
483
+ if (options.pre) {
484
+ listOptions.pre = options.pre;
485
+ }
486
+
487
+ let result;
488
+ switch (layer) {
489
+ case "upload": {
490
+ result = await client.capability.upload.list(listOptions);
491
+ return result.results.map((upload) => upload.root.toString());
492
+ }
493
+
494
+ case "blob": {
495
+ result = await client.capability.blob.list(listOptions);
496
+ return result.results.map((blob) => {
497
+ // The blob structure is: { blob: { size: number, digest: Uint8Array }, cause: CID, insertedAt: string }
498
+ if (blob.blob && blob.blob.digest) {
499
+ // blob.blob.digest is already a Uint8Array, encode it to base64
500
+ const encodedDigest = Buffer.from(blob.blob.digest).toString(
501
+ "base64",
502
+ );
503
+ return encodedDigest;
504
+ } else {
505
+ logger.warn({ blob }, ` ⚠️ Unexpected blob structure`);
506
+ return blob.toString(); // Fallback
507
+ }
508
+ });
509
+ }
510
+
511
+ default:
512
+ throw new Error(`Unknown layer: ${layer}. Use 'upload' or 'blob'.`);
513
+ }
514
+ } catch (error) {
515
+ logger.warn(` ⚠️ Failed to list ${layer}: ${error.message}`);
516
+ return [];
517
+ }
518
+ }
519
+
520
+ /**
521
+ * Remove files from a specific layer in batches
522
+ *
523
+ * @param {string} layer - Layer to clear ('upload', 'blob')
524
+ * @param {Array} cids - Array of CIDs to remove
525
+ * @param {Object} options - Configuration options
526
+ * @returns {Promise<Object>} - Removal results
527
+ */
528
+ export async function removeLayerFiles(layer, cids, options = {}) {
529
+ if (cids.length === 0) {
530
+ logger.info(` ✓ ${layer}: No files to remove`);
531
+ return { removed: 0, failed: 0 };
532
+ }
533
+
534
+ const batchSize = options.batchSize || 10;
535
+ logger.info(
536
+ ` 🗑️ Removing ${cids.length} files from ${layer} layer using SDK (batch size: ${batchSize})...`,
537
+ );
538
+
539
+ try {
540
+ // Initialize client - support both credential and UCAN authentication
541
+ const backend = await resolveBackend(options);
542
+ const client = backend.client;
543
+
544
+ let removed = 0;
545
+ let failed = 0;
546
+
547
+ // Process CIDs in batches
548
+ for (let i = 0; i < cids.length; i += batchSize) {
549
+ const batch = cids.slice(i, i + batchSize);
550
+ const batchNum = Math.floor(i / batchSize) + 1;
551
+ const totalBatches = Math.ceil(cids.length / batchSize);
552
+
553
+ logger.info(
554
+ ` 📦 Processing batch ${batchNum}/${totalBatches} (${batch.length} files)...`,
555
+ );
556
+
557
+ // Process batch in parallel
558
+ const batchPromises = batch.map(async (cid) => {
559
+ try {
560
+ switch (layer) {
561
+ case "upload": {
562
+ await client.capability.upload.remove(CID.parse(cid));
563
+ break;
564
+ }
565
+ case "blob": {
566
+ // For blob, we need to parse the digest format
567
+ // cid should be a base64 encoded string, convert it back to Uint8Array
568
+ let digest;
569
+ if (typeof cid === "string") {
570
+ // Convert to Buffer first, then to Uint8Array (API expects Uint8Array, not Buffer)
571
+ const buffer = Buffer.from(cid, "base64");
572
+ digest = new Uint8Array(buffer);
573
+ } else {
574
+ logger.error(
575
+ { cidType: typeof cid, cid },
576
+ ` ❌ Invalid blob CID format`,
577
+ );
578
+ throw new Error(
579
+ `Invalid blob CID format: expected string, got ${typeof cid}`,
580
+ );
581
+ }
582
+
583
+ // Remove the blob using the correct API format: { bytes: Uint8Array }
584
+ await client.capability.blob.remove({ bytes: digest });
585
+ break;
586
+ }
587
+
588
+ default:
589
+ throw new Error(
590
+ `Unknown layer: ${layer}. Use 'upload' or 'blob'.`,
591
+ );
592
+ }
593
+
594
+ return { success: true, cid };
595
+ } catch (error) {
596
+ return { success: false, cid, error: error.message };
597
+ }
598
+ });
599
+
600
+ // Wait for all deletions in this batch to complete
601
+ const batchResults = await Promise.allSettled(batchPromises);
602
+
603
+ // Process results
604
+ for (const result of batchResults) {
605
+ if (result.status === "fulfilled") {
606
+ if (result.value.success) {
607
+ removed++;
608
+ logger.info(` ✓ Removed: ${result.value.cid}`);
609
+ } else {
610
+ failed++;
611
+ logger.info(
612
+ ` ❌ Failed to remove ${result.value.cid}: ${result.value.error}`,
613
+ );
614
+ }
615
+ } else {
616
+ failed++;
617
+ logger.info(` ❌ Batch operation failed: ${result.reason}`);
618
+ }
619
+ }
620
+
621
+ // Add a small delay between batches to avoid overwhelming the API
622
+ if (i + batchSize < cids.length) {
623
+ await new Promise((resolve) => setTimeout(resolve, 100));
624
+ }
625
+ }
626
+
627
+ logger.info(` 📊 ${layer}: ${removed} removed, ${failed} failed`);
628
+ return { removed, failed };
629
+ } catch (error) {
630
+ logger.error(` ❌ Error removing from ${layer}: ${error.message}`);
631
+ return { removed: 0, failed: cids.length };
632
+ }
633
+ }
634
+
635
+ /**
636
+ * Clear all files from Storacha space using SDK
637
+ *
638
+ * @param {Object} options - Configuration options
639
+ * @returns {Promise<Object>} - Clearing results
640
+ */
641
+ export async function clearStorachaSpace(options = {}) {
642
+ logger.info("🧹 Clearing Storacha space using SDK...");
643
+ logger.info("=".repeat(50));
644
+
645
+ // FIXED: Remove 'store' layer as it's no longer available in the API
646
+ const layers = ["upload", "blob"]; // Removed "store"
647
+ const summary = {
648
+ totalFiles: 0,
649
+ totalRemoved: 0,
650
+ totalFailed: 0,
651
+ byLayer: {},
652
+ };
653
+
654
+ for (const layer of layers) {
655
+ logger.info(`\n📋 Checking ${layer} layer...`);
656
+ const cids = await listLayerFiles(layer, options);
657
+ summary.totalFiles += cids.length;
658
+
659
+ if (cids.length > 0) {
660
+ const result = await removeLayerFiles(layer, cids, options);
661
+ summary.totalRemoved += result.removed;
662
+ summary.totalFailed += result.failed;
663
+ summary.byLayer[layer] = result;
664
+ } else {
665
+ summary.byLayer[layer] = { removed: 0, failed: 0 };
666
+ logger.info(` ✓ ${layer}: Already empty`);
667
+ }
668
+ }
669
+
670
+ logger.info("\n" + "=".repeat(50));
671
+ logger.info("🧹 SPACE CLEARING RESULTS (SDK)");
672
+ logger.info("=".repeat(50));
673
+ logger.info(`📊 Total files found: ${summary.totalFiles}`);
674
+ logger.info(`✅ Total files removed: ${summary.totalRemoved}`);
675
+ logger.info(`❌ Total failures: ${summary.totalFailed}`);
676
+
677
+ for (const [layer, stats] of Object.entries(summary.byLayer)) {
678
+ logger.info(
679
+ ` ${layer}: ${stats.removed} removed, ${stats.failed} failed`,
680
+ );
681
+ }
682
+
683
+ const success =
684
+ summary.totalFailed === 0 && summary.totalFiles === summary.totalRemoved;
685
+ logger.info(
686
+ `\n${success ? "✅" : "⚠️"} Space clearing: ${success ? "COMPLETE" : "PARTIAL"}`,
687
+ );
688
+
689
+ return {
690
+ success,
691
+ ...summary,
692
+ };
693
+ }
694
+
695
+ /**
696
+ * Download content from IPFS network using Helia with UnixFS
697
+ *
698
+ * Uses unixfs.cat() which handles DAG traversal automatically, making it suitable
699
+ * for both single blocks and chunked files (like large CAR files).
700
+ *
701
+ * @param {string} cid - CID to download
702
+ * @param {Object} helia - Helia IPFS instance
703
+ * @param {Object} options - Configuration options
704
+ * @param {Object} [options.unixfs] - Optional unixfs instance (will be created if not provided)
705
+ * @returns {Promise<Uint8Array>} - Complete content bytes
706
+ */
707
+ export async function downloadBlockFromIPFSNetwork(cid, helia, options = {}) {
708
+ const config = { ...DEFAULT_OPTIONS, ...options };
709
+
710
+ try {
711
+ if (typeof config.reconnectToInMemoryHelia === "function") {
712
+ try {
713
+ await config.reconnectToInMemoryHelia();
714
+ } catch (error) {
715
+ logger.warn(
716
+ ` ⚠️ In-memory Helia reconnect failed: ${error?.message ?? error}`,
717
+ );
718
+ }
719
+ }
720
+
721
+ // Check if helia instance is available and ready
722
+ if (!helia) {
723
+ throw new Error("Helia instance is not available");
724
+ }
725
+
726
+ const parsedCID = typeof cid === "string" ? CID.parse(cid) : cid;
727
+ logHeliaActivity(`📥 helia unixfs.cat ${parsedCID.toString()}`);
728
+
729
+ // Create unixfs instance from helia (or use provided one)
730
+ const fs = options.unixfs || unixfs(helia);
731
+
732
+ // Use unixfs.cat() to get complete content (handles DAGs automatically)
733
+ // This uses Bitswap protocol to fetch from IPFS network and traverses DAGs
734
+ const chunks = [];
735
+ let timeoutId;
736
+
737
+ // Create a timeout promise
738
+ const timeoutPromise = new Promise((_, reject) => {
739
+ timeoutId = setTimeout(
740
+ () => reject(new Error("Network download timeout")),
741
+ config.timeout,
742
+ );
743
+ });
744
+
745
+ // Race between cat() and timeout
746
+ try {
747
+ const catPromise = (async () => {
748
+ for await (const chunk of fs.cat(parsedCID)) {
749
+ chunks.push(chunk);
750
+ }
751
+ })();
752
+
753
+ await Promise.race([catPromise, timeoutPromise]);
754
+ } finally {
755
+ // Clear timeout if cat() completes first
756
+ if (timeoutId) {
757
+ clearTimeout(timeoutId);
758
+ }
759
+ }
760
+
761
+ // Concatenate all chunks into a single Uint8Array
762
+ const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
763
+ const bytes = new Uint8Array(totalLength);
764
+ let offset = 0;
765
+ for (const chunk of chunks) {
766
+ bytes.set(chunk, offset);
767
+ offset += chunk.length;
768
+ }
769
+
770
+ logger.info(
771
+ ` ✅ Downloaded ${bytes.length} bytes from IPFS network (CID: ${cid.substring(0, 12)}...)`,
772
+ );
773
+ logHeliaActivity(
774
+ `✅ helia unixfs.cat complete ${parsedCID.toString()} (${bytes.length} bytes)`,
775
+ );
776
+ return bytes;
777
+ } catch (error) {
778
+ logHeliaActivity(
779
+ `❌ helia unixfs.cat failed ${cid.toString?.() ?? cid}: ${error.message}`,
780
+ );
781
+ logger.warn(` ⚠️ Failed to download from IPFS network: ${error.message}`);
782
+ throw error;
783
+ }
784
+ }
785
+
786
+ /**
787
+ * Download a block by CID, peers first and HTTP gateways as the fallback.
788
+ *
789
+ * Nothing here has ever touched a Storacha client -- the old name
790
+ * (`downloadBlockFromStoracha`, still exported as an alias) described where the block
791
+ * had been put rather than where it is read from, which stopped being true the moment
792
+ * a second backend existed. Bitswap does not care who paid for the pin.
793
+ *
794
+ * @param {string} cid - CID of the block to fetch
795
+ * @param {Object} options - Configuration options
796
+ * @param {Object} [options.helia] - Optional Helia instance for network download
797
+ * @param {boolean} [options.useIPFSNetwork] - try peers before gateways
798
+ * @param {boolean} [options.gatewayFallback] - fall back to HTTP gateways
799
+ * @returns {Promise<Uint8Array>} - Block bytes
800
+ */
801
+ export async function downloadBlock(cid, options = {}) {
802
+ const config = { ...DEFAULT_OPTIONS, ...options };
803
+
804
+ // Try IPFS network first if enabled and Helia instance is available
805
+ if (config.useIPFSNetwork && config.helia) {
806
+ try {
807
+ const bytes = await downloadBlockFromIPFSNetwork(
808
+ cid,
809
+ config.helia,
810
+ config,
811
+ );
812
+ return bytes;
813
+ } catch (error) {
814
+ logger.info(
815
+ ` ⚠️ Network download failed, ${config.gatewayFallback ? "falling back to gateway" : "no fallback enabled"}: ${error.message}`,
816
+ );
817
+ if (!config.gatewayFallback) {
818
+ throw new Error(
819
+ `Could not download block ${cid} from IPFS network and gateway fallback is disabled`, { cause: error },
820
+ );
821
+ }
822
+ // Continue to gateway fallback
823
+ }
824
+ }
825
+
826
+ // Fallback to gateway download
827
+ // storacha.link and w3s.link answer 301 to dweb.link as of 2026-09-05, so going
828
+ // through them only buys a redirect. Ask the destination directly.
829
+ const gateways = [
830
+ `${config.gateway}/ipfs`,
831
+ "https://dweb.link/ipfs",
832
+ "https://ipfs.io/ipfs",
833
+ ];
834
+
835
+ for (const gateway of gateways) {
836
+ try {
837
+ const response = await fetch(`${gateway}/${cid}`, {
838
+ signal: AbortSignal.timeout(config.timeout),
839
+ });
840
+
841
+ if (response.ok) {
842
+ const contentType = response.headers?.get("content-type") || "";
843
+
844
+ // Reject HTML responses (error pages)
845
+ if (
846
+ contentType.includes("text/html") ||
847
+ contentType.includes("application/xhtml")
848
+ ) {
849
+ logger.debug(
850
+ ` ⚠️ Gateway ${gateway} returned HTML (likely error page), trying next gateway...`,
851
+ );
852
+ continue; // Try next gateway
853
+ }
854
+
855
+ const bytes = new Uint8Array(await response.arrayBuffer());
856
+
857
+ // Validate that we didn't get an HTML error page
858
+ // Check first few bytes for HTML markers
859
+ const textStart = new TextDecoder("utf-8", { fatal: false }).decode(
860
+ bytes.slice(0, Math.min(100, bytes.length)),
861
+ );
862
+ if (
863
+ textStart.trim().startsWith("<!DOCTYPE") ||
864
+ textStart.trim().startsWith("<html") ||
865
+ textStart.trim().startsWith("<?xml")
866
+ ) {
867
+ logger.debug(
868
+ ` ⚠️ Gateway ${gateway} returned HTML/XML (likely error page), trying next gateway...`,
869
+ );
870
+ continue; // Try next gateway
871
+ }
872
+
873
+ logger.info(` ✅ Downloaded ${bytes.length} bytes from ${gateway}`);
874
+ return bytes;
875
+ } else {
876
+ // Enhanced logging for 503 and other error statuses
877
+ let errorLog = ` ⚠️ Gateway ${gateway} returned status ${response.status}`;
878
+
879
+ if (response.status === 503) {
880
+ const retryAfter = response.headers?.get("Retry-After");
881
+ const rateLimitReset = response.headers?.get("X-RateLimit-Reset");
882
+ const rateLimitRemaining = response.headers?.get(
883
+ "X-RateLimit-Remaining",
884
+ );
885
+
886
+ errorLog += ` (Service Unavailable - likely rate limited)`;
887
+
888
+ if (retryAfter) {
889
+ const retrySeconds = parseInt(retryAfter);
890
+ if (!isNaN(retrySeconds)) {
891
+ const futureTime = new Date(Date.now() + retrySeconds * 1000);
892
+ errorLog += `\n ⏱️ Retry-After: ${retrySeconds}s (available at ${futureTime.toLocaleTimeString()})`;
893
+ } else {
894
+ const retryDate = new Date(retryAfter);
895
+ if (!isNaN(retryDate.getTime())) {
896
+ const waitSecs = Math.ceil(
897
+ Math.max(0, retryDate.getTime() - Date.now()) / 1000,
898
+ );
899
+ errorLog += `\n ⏱️ Retry-After: ${waitSecs}s (available at ${retryDate.toLocaleTimeString()})`;
900
+ } else {
901
+ errorLog += `\n ⏱️ Retry-After: ${retryAfter}`;
902
+ }
903
+ }
904
+ }
905
+
906
+ if (rateLimitReset) {
907
+ const resetTime = parseInt(rateLimitReset);
908
+ if (!isNaN(resetTime)) {
909
+ const resetDate = new Date(resetTime * 1000);
910
+ const waitSecs = Math.ceil(
911
+ Math.max(0, resetDate.getTime() - Date.now()) / 1000,
912
+ );
913
+ errorLog += `\n 🔄 Rate Limit Reset: ${resetDate.toLocaleTimeString()} (in ${waitSecs}s)`;
914
+ }
915
+ }
916
+
917
+ if (rateLimitRemaining !== null && rateLimitRemaining !== undefined) {
918
+ errorLog += `\n 📈 Requests Remaining: ${rateLimitRemaining}`;
919
+ }
920
+ }
921
+
922
+ errorLog += `, trying next gateway...`;
923
+ logger.debug(errorLog);
924
+ }
925
+ } catch (error) {
926
+ logger.debug(` ⚠️ Failed from ${gateway}: ${error.message}`);
927
+ }
928
+ }
929
+
930
+ throw new Error(`Could not download block ${cid} from any gateway`);
931
+ }
932
+
933
+ /**
934
+ * @deprecated Use {@link downloadBlock}. Kept because downstream code imports this name.
935
+ * @type {typeof downloadBlock}
936
+ */
937
+ export const downloadBlockFromStoracha = downloadBlock;
938
+
939
+ /**
940
+ * Advanced block analysis (classification and log head detection)
941
+ *
942
+ * @param {Object} blockstore - IPFS blockstore
943
+ * @param {Map} downloadedBlocks - Downloaded blocks map
944
+ * @returns {Promise<Object>} - Analysis results
945
+ */
946
+ export async function analyzeBlocks(blockstore, downloadedBlocks = null) {
947
+ logger.debug("Analyzing downloaded blocks...");
948
+
949
+ const analysis = {
950
+ manifestBlocks: [],
951
+ accessControllerBlocks: [],
952
+ logEntryBlocks: [],
953
+ identityBlocks: [],
954
+ unknownBlocks: [],
955
+ logStructure: new Map(),
956
+ potentialHeads: [],
957
+ logChain: new Map(),
958
+ };
959
+
960
+ const allCIDStrings = downloadedBlocks
961
+ ? Array.from(downloadedBlocks.keys())
962
+ : [];
963
+
964
+ for (const cidString of allCIDStrings) {
965
+ try {
966
+ const cid = CID.parse(cidString);
967
+
968
+ // Check if blockstore is available and accessible
969
+ if (!blockstore || typeof blockstore.get !== "function") {
970
+ throw new Error("Blockstore is not available or accessible");
971
+ }
972
+
973
+ let bytes;
974
+ try {
975
+ bytes = await readBlockBytes(blockstore, cid);
976
+ } catch (error) {
977
+ if (
978
+ error.message?.includes("not open") ||
979
+ error.message?.includes("closed") ||
980
+ error.message?.includes("Database is not open")
981
+ ) {
982
+ throw new Error(
983
+ `Blockstore operation failed - OrbitDB was closed during analysis: ${error.message}`, { cause: error },
984
+ );
985
+ }
986
+ throw error;
987
+ }
988
+
989
+ if (cid.code === 0x71) {
990
+ // dag-cbor codec
991
+ try {
992
+ const block = await Block.decode({
993
+ cid,
994
+ bytes,
995
+ codec: dagCbor,
996
+ hasher: sha256,
997
+ });
998
+
999
+ const content = block.value;
1000
+ logger.info("content.type", content);
1001
+ // Smart block classification
1002
+ //if (content.type && content.name && content.accessController) {
1003
+ if (content.accessController) {
1004
+ analysis.manifestBlocks.push({ cid: cidString, content });
1005
+ logger.info(` 📋 Manifest: ${cidString} (${content.name})`);
1006
+ } else if (content.sig && content.key && content.identity) {
1007
+ analysis.logEntryBlocks.push({ cid: cidString, content });
1008
+ analysis.logStructure.set(cidString, content);
1009
+ logger.info(` 📝 Log Entry: ${cidString}`);
1010
+
1011
+ // Build log chain for head detection
1012
+ if (content.next && Array.isArray(content.next)) {
1013
+ for (const nextHash of content.next) {
1014
+ analysis.logChain.set(nextHash, cidString);
1015
+ }
1016
+ }
1017
+ } else if (content.id && content.type) {
1018
+ analysis.identityBlocks.push({ cid: cidString, content });
1019
+ logger.info(` 👤 Identity: ${cidString}`);
1020
+ } else if (
1021
+ content.type === "orbitdb-access-controller" ||
1022
+ content.type === "ipfs"
1023
+ ) {
1024
+ analysis.accessControllerBlocks.push({ cid: cidString, content });
1025
+ logger.info(` 🔒 Access Controller: ${cidString}`);
1026
+ } else {
1027
+ analysis.unknownBlocks.push({ cid: cidString, content });
1028
+ logger.info(` ❓ Unknown: ${cidString}`);
1029
+ }
1030
+ } catch (decodeError) {
1031
+ analysis.unknownBlocks.push({
1032
+ cid: cidString,
1033
+ decodeError: decodeError.message,
1034
+ });
1035
+ logger.info(` ⚠️ Decode failed: ${cidString}`);
1036
+ }
1037
+ } else {
1038
+ analysis.unknownBlocks.push({ cid: cidString, reason: "not dag-cbor" });
1039
+ logger.info(` 🔧 Raw block: ${cidString}`);
1040
+ }
1041
+ } catch (error) {
1042
+ logger.warn(` ❌ Error analyzing block ${cidString}: ${error.message}`);
1043
+ }
1044
+ }
1045
+
1046
+ // Intelligent head detection
1047
+ logger.info("🎯 Determining log heads:");
1048
+ for (const [entryHash, _entryContent] of analysis.logStructure) {
1049
+ if (!analysis.logChain.has(entryHash)) {
1050
+ analysis.potentialHeads.push(entryHash);
1051
+ logger.info(` 🎯 HEAD: ${entryHash}`);
1052
+ }
1053
+ }
1054
+
1055
+ logger.info("📊 Analysis Summary:");
1056
+ logger.info(` 📋 Manifests: ${analysis.manifestBlocks.length}`);
1057
+ logger.info(` 📝 Log Entries: ${analysis.logEntryBlocks.length}`);
1058
+ logger.info(` 👤 Identities: ${analysis.identityBlocks.length}`);
1059
+ logger.info(
1060
+ ` 🔒 Access Controllers: ${analysis.accessControllerBlocks.length}`,
1061
+ );
1062
+ logger.info(` 🎯 Heads Discovered: ${analysis.potentialHeads.length}`);
1063
+
1064
+ return analysis;
1065
+ }
1066
+
1067
+ /**
1068
+ * Download blocks from Storacha and bridge CID formats for OrbitDB
1069
+ *
1070
+ * @param {Map} cidMappings - Mapping of original → uploaded CIDs
1071
+ * @param {Object} client - Storacha client (unused, kept for compatibility)
1072
+ * @param {Object} targetBlockstore - Target blockstore to store blocks
1073
+ * @param {Object} options - Configuration options
1074
+ * @returns {Promise<Object>} - Bridge results
1075
+ */
1076
+ async function downloadAndBridgeBlocks(
1077
+ cidMappings,
1078
+ client,
1079
+ targetBlockstore,
1080
+ options = {},
1081
+ ) {
1082
+ const config = { ...DEFAULT_OPTIONS, ...options };
1083
+ logger.info(
1084
+ `📥 Downloading and bridging ${cidMappings.size} blocks for OrbitDB...`,
1085
+ );
1086
+
1087
+ const bridgedBlocks = [];
1088
+
1089
+ for (const [originalCID, storachaCID] of cidMappings) {
1090
+ try {
1091
+ logger.info(` 📥 Downloading ${storachaCID}...`);
1092
+
1093
+ // Try network download first if enabled and Helia instance available
1094
+ let blockBytes = null;
1095
+ if (config.useIPFSNetwork && config.helia) {
1096
+ try {
1097
+ // Create unixfs if not provided
1098
+ if (!config.unixfs) {
1099
+ config.unixfs = unixfs(config.helia);
1100
+ }
1101
+ blockBytes = await downloadBlockFromIPFSNetwork(
1102
+ storachaCID,
1103
+ config.helia,
1104
+ { ...config, unixfs: config.unixfs },
1105
+ );
1106
+ logger.info(
1107
+ ` ✅ Downloaded ${blockBytes.length} bytes from IPFS network`,
1108
+ );
1109
+ } catch (error) {
1110
+ logger.info(
1111
+ ` ⚠️ Network download failed, ${config.gatewayFallback ? "falling back to gateway" : "no fallback"}: ${error.message}`,
1112
+ );
1113
+ if (!config.gatewayFallback) {
1114
+ throw new Error(
1115
+ `Failed to download block from IPFS network and gateway fallback is disabled: ${error.message}`, { cause: error },
1116
+ );
1117
+ }
1118
+ }
1119
+ }
1120
+
1121
+ // Fallback to gateway if network failed or disabled
1122
+ if (!blockBytes) {
1123
+ const response = await fetch(`${config.gateway}/ipfs/${storachaCID}`, {
1124
+ signal: AbortSignal.timeout(config.timeout),
1125
+ });
1126
+
1127
+ if (!response.ok) {
1128
+ throw new Error(`HTTP ${response.status}: ${response.statusText}`);
1129
+ }
1130
+
1131
+ blockBytes = new Uint8Array(await response.arrayBuffer());
1132
+ logger.info(
1133
+ ` ✅ Downloaded ${blockBytes.length} bytes from gateway ${config.gateway}`,
1134
+ );
1135
+ }
1136
+
1137
+ // Convert Storacha CID to OrbitDB format
1138
+ const bridgedCID = convertStorachaCIDToOrbitDB(storachaCID);
1139
+ logger.info(` 🌉 Bridged CID: ${storachaCID} → ${bridgedCID}`);
1140
+
1141
+ // Verify the bridged CID matches the original
1142
+ const match = bridgedCID === originalCID;
1143
+ if (match) {
1144
+ logger.info(` ✅ CID bridge successful: ${bridgedCID}`);
1145
+ } else {
1146
+ logger.warn(
1147
+ ` ⚠️ CID bridge mismatch: expected ${originalCID}, got ${bridgedCID}`,
1148
+ );
1149
+ }
1150
+
1151
+ // Store block in target blockstore under OrbitDB CID format
1152
+ const parsedBridgedCID = CID.parse(bridgedCID);
1153
+ await targetBlockstore.put(parsedBridgedCID, blockBytes);
1154
+ logger.info(` 💾 Stored in blockstore as: ${bridgedCID}`);
1155
+
1156
+ bridgedBlocks.push({
1157
+ originalCID,
1158
+ storachaCID,
1159
+ bridgedCID,
1160
+ size: blockBytes.length,
1161
+ match,
1162
+ });
1163
+ } catch (error) {
1164
+ logger.error(
1165
+ ` ❌ Failed to download/bridge ${storachaCID}: ${error.message}`,
1166
+ );
1167
+ bridgedBlocks.push({
1168
+ originalCID,
1169
+ storachaCID,
1170
+ error: error.message,
1171
+ });
1172
+ }
1173
+ }
1174
+
1175
+ const successful = bridgedBlocks.filter((b) => b.bridgedCID);
1176
+ const failed = bridgedBlocks.filter((b) => b.error);
1177
+ const matches = successful.filter((b) => b.match);
1178
+
1179
+ logger.info(` 📊 Bridge summary:`);
1180
+ logger.info(` Total blocks: ${cidMappings.size}`);
1181
+ logger.info(` Downloaded: ${successful.length}`);
1182
+ logger.info(` Failed: ${failed.length}`);
1183
+ logger.info(` CID matches: ${matches.length}`);
1184
+
1185
+ return { bridgedBlocks, successful, failed, matches };
1186
+ }
1187
+
1188
+ /**
1189
+ * Back up an OrbitDB database through whichever backend is configured.
1190
+ *
1191
+ * **A CAR by default** (#54, 0.5.2). One opaque blob rather than a stream of
1192
+ * small ones: hash preservation is then structural instead of a property we
1193
+ * hope a vendor keeps, and it clears the minimum piece sizes that individual
1194
+ * OrbitDB blocks do not.
1195
+ *
1196
+ * The two strategies return **different fields**, so read `result.method`
1197
+ * rather than guessing from the shape. A CAR backup yields `backupFiles` with
1198
+ * the pointer to restore from; a block backup yields `cidMappings`, which the
1199
+ * CAR path cannot produce because there are no per-block handles to map. If
1200
+ * you were calling this before 0.5.2 and using `cidMappings`, pass
1201
+ * `strategy: "blocks"` — and read `assertCanTakeBlocks` for when a backend
1202
+ * will refuse.
1203
+ *
1204
+ * @param {Object} orbitdb - OrbitDB instance
1205
+ * @param {string} databaseAddress - Database address or name
1206
+ * @param {Object} options - Backup options
1207
+ * @param {"car"|"blocks"} [options.strategy="car"] - How to hand the database
1208
+ * to the backend. `"blocks"` is refused by a backend that re-chunks or has a
1209
+ * minimum piece size, before anything is sent.
1210
+ * @param {EventEmitter} options.eventEmitter - Optional event emitter for progress updates
1211
+ * @param {boolean} [options.logEntriesOnly] - Salvage mode: only log entries,
1212
+ * for a database whose manifest cannot be read. Implies `strategy: "blocks"`,
1213
+ * because a CAR without its manifest is one `restoreFromCID` refuses — and
1214
+ * should, since it could be anybody's log.
1215
+ * @returns {Promise<Object>} - Backup result, carrying `method`
1216
+ */
1217
+ /**
1218
+ * Whether this backend can be handed one OrbitDB block at a time.
1219
+ *
1220
+ * Two reasons it usually cannot, and both are quiet failures rather than loud
1221
+ * ones, which is why they are checked before a single block is sent:
1222
+ *
1223
+ * - **A minimum piece size.** Filecoin Onchain Cloud rejects anything under
1224
+ * 127 bytes and an OrbitDB block is routinely smaller, so a per-block backup
1225
+ * dies partway through with some blocks stored and some not — the worst
1226
+ * possible outcome, because it looks like a backup.
1227
+ * - **Re-chunking.** A backend that computes its own CID for what we send hands
1228
+ * back a name for content we have never seen. Inside a CAR that cannot happen,
1229
+ * because the CAR is one opaque blob and the CIDs inside it stay ours.
1230
+ *
1231
+ * @param {import("./backends/types.js").StorageBackend} backend
1232
+ */
1233
+ function assertCanTakeBlocks(backend) {
1234
+ const caps = backend.capabilities ?? {};
1235
+ if (!caps.preservesInnerCids) {
1236
+ throw new BackendError(
1237
+ "UNSUPPORTED",
1238
+ `The ${backend.name} backend does not preserve the CIDs it is given, so it cannot be sent blocks one at a time. Use the default CAR strategy.`,
1239
+ );
1240
+ }
1241
+ if (caps.minBlobSize > 0) {
1242
+ throw new BackendError(
1243
+ "TOO_SMALL",
1244
+ `The ${backend.name} backend has a ${caps.minBlobSize}-byte minimum and OrbitDB blocks are routinely smaller, so a per-block backup would fail partway through. Use the default CAR strategy.`,
1245
+ );
1246
+ }
1247
+ }
1248
+
1249
+ export async function backupDatabase(orbitdb, databaseAddress, options = {}) {
1250
+ const config = { ...DEFAULT_OPTIONS, ...options };
1251
+ const eventEmitter = options.eventEmitter;
1252
+ const logEntriesOnly = options.logEntriesOnly || false;
1253
+
1254
+ // CAR is the default (#54, 0.5.2). Hash preservation is then structural
1255
+ // rather than a property we hope a vendor keeps, and one blob clears the
1256
+ // minimum piece sizes that a stream of small blocks does not.
1257
+ //
1258
+ // `logEntriesOnly` stays on the block path and is not a strategy: it exists
1259
+ // to salvage a database whose manifest cannot be read, and a CAR without its
1260
+ // manifest is one `restoreFromCID` refuses — correctly, since it could be
1261
+ // anybody's log.
1262
+ const strategy = options.strategy ?? (logEntriesOnly ? "blocks" : "car");
1263
+ if (strategy !== "car" && strategy !== "blocks") {
1264
+ throw new BackendError("UNSUPPORTED", `Unknown backup strategy "${strategy}"`);
1265
+ }
1266
+ if (strategy === "car") {
1267
+ // Imported here rather than at the top: `backup-car.js` imports this
1268
+ // module, and a static edge back would close the cycle.
1269
+ const { backupDatabaseCAR } = await import("./backup-car.js");
1270
+ return backupDatabaseCAR(orbitdb, databaseAddress, options);
1271
+ }
1272
+
1273
+ const backupMode = logEntriesOnly
1274
+ ? "Log Entries Only (Fallback Mode)"
1275
+ : "Full Backup";
1276
+
1277
+ logger.info("🚀 Starting OrbitDB Database Backup to Storacha");
1278
+ logger.info(`📍 Database: ${databaseAddress}`);
1279
+ logger.info(`🔧 Backup Mode: ${backupMode}`);
1280
+
1281
+ try {
1282
+ // Initialize Storacha client - support both credential and UCAN authentication
1283
+ const backend = await resolveBackend(config);
1284
+ // Before a single block is sent, not after some of them are.
1285
+ assertCanTakeBlocks(backend);
1286
+
1287
+ // Open the database
1288
+ const database = await orbitdb.open(databaseAddress, options.dbConfig);
1289
+
1290
+ // Extract blocks based on backup mode
1291
+ const { blocks, blockSources, manifestCID } = await extractDatabaseBlocks(
1292
+ database,
1293
+ {
1294
+ logEntriesOnly,
1295
+ },
1296
+ );
1297
+
1298
+ // Upload blocks to Storacha with progress tracking
1299
+ let successful, cidMappings;
1300
+ try {
1301
+ const result = await uploadBlocks(
1302
+ blocks,
1303
+ backend,
1304
+ config.uploadBatchSize,
1305
+ config.uploadMaxConcurrency,
1306
+ eventEmitter,
1307
+ );
1308
+ successful = result.successful;
1309
+ cidMappings = result.cidMappings;
1310
+
1311
+ if (successful.length === 0) {
1312
+ throw new Error("No blocks were successfully uploaded");
1313
+ }
1314
+ } catch (error) {
1315
+ if (
1316
+ error.message?.includes("Unauthorized") ||
1317
+ error.message?.includes("Capability")
1318
+ ) {
1319
+ if (eventEmitter) {
1320
+ eventEmitter.emit("uploadProgress", {
1321
+ type: "upload",
1322
+ current: 0,
1323
+ total: blocks.size,
1324
+ percentage: 0,
1325
+ status: "error",
1326
+ error: {
1327
+ type: "ucan",
1328
+ message:
1329
+ "UCAN authorization failed: Your capabilities are not sufficient for uploading. Please check your space permissions.",
1330
+ details: error.message,
1331
+ },
1332
+ });
1333
+ }
1334
+ throw new Error(
1335
+ "UCAN authorization failed: Your capabilities are not sufficient for uploading. Please check your space permissions.", { cause: error },
1336
+ );
1337
+ }
1338
+ throw error;
1339
+ }
1340
+
1341
+ // Get block summary
1342
+ const blockSummary = {};
1343
+ for (const [_hash, source] of blockSources) {
1344
+ blockSummary[source] = (blockSummary[source] || 0) + 1;
1345
+ }
1346
+
1347
+ logger.info("✅ Backup completed successfully!");
1348
+
1349
+ return {
1350
+ success: true,
1351
+ // Named, because the two strategies return different fields and a caller
1352
+ // that branches on shape rather than on this is guessing.
1353
+ method: "blocks",
1354
+ manifestCID,
1355
+ databaseAddress: database.address,
1356
+ databaseName: database.name,
1357
+ blocksTotal: blocks.size,
1358
+ blocksUploaded: successful.length,
1359
+ blockSummary,
1360
+ cidMappings: Object.fromEntries(cidMappings),
1361
+ };
1362
+ } catch (error) {
1363
+ logger.error("❌ Backup failed:", error.message);
1364
+ return {
1365
+ success: false,
1366
+ error: error.message,
1367
+ // A BackendError says the backend cannot serve this request, which is a
1368
+ // different thing from the backup having failed, and a caller that has to
1369
+ // read the message to tell them apart will get it wrong.
1370
+ ...(error instanceof BackendError ? { code: error.code } : {}),
1371
+ };
1372
+ }
1373
+ }
1374
+
1375
+ /**
1376
+ * Optimized restore from Storacha for fallback reconstruction (log entries only)
1377
+ *
1378
+ * @param {Object} orbitdb - Target OrbitDB instance
1379
+ * @param {Object} options - Restore options
1380
+ * @param {string} [options.storachaKey] - Storacha private key (defaults to env)
1381
+ * @param {string} [options.storachaProof] - Storacha proof (defaults to env)
1382
+ * @param {EventEmitter} [options.eventEmitter] - Optional event emitter for progress updates
1383
+ * @returns {Promise<Object>} - Restore result
1384
+ */
1385
+ export async function restoreLogEntriesOnly(orbitdb, options = {}) {
1386
+ const config = { ...DEFAULT_OPTIONS, ...options };
1387
+ const eventEmitter = options.eventEmitter;
1388
+
1389
+ logger.info("⚡ Starting Optimized Log-Entries-Only Restore from Storacha");
1390
+
1391
+ try {
1392
+ // Step 1: List ALL files in Storacha space
1393
+ logger.info("\n📋 Step 1: Discovering all files in Storacha space...");
1394
+ const spaceFiles = await listStorachaSpaceFiles(config);
1395
+
1396
+ if (spaceFiles.length === 0) {
1397
+ throw new Error("No files found in Storacha space");
1398
+ }
1399
+
1400
+ logger.info(` 🎉 SUCCESS! Found ${spaceFiles.length} files in space`);
1401
+
1402
+ // Step 2: Download ONLY log entry blocks (optimized)
1403
+ // Ensure helia instance is included in config for network downloads
1404
+ const downloadConfig = {
1405
+ ...config,
1406
+ helia: config.helia || orbitdb.ipfs,
1407
+ };
1408
+ const logEntryBlocks = await downloadLogEntriesOnly(
1409
+ spaceFiles,
1410
+ orbitdb,
1411
+ downloadConfig,
1412
+ eventEmitter,
1413
+ );
1414
+
1415
+ if (logEntryBlocks.size === 0) {
1416
+ throw new Error("No log entries found in Storacha space");
1417
+ }
1418
+
1419
+ logger.info(
1420
+ ` ⚡ OPTIMIZATION: Downloaded only ${logEntryBlocks.size} log entries instead of ${spaceFiles.length} total files`,
1421
+ );
1422
+
1423
+ // Step 3: Direct fallback reconstruction with joinEntry support
1424
+ logger.info(
1425
+ "\n🔧 Step 3: Reconstructing database from log entries with joinEntry...",
1426
+ );
1427
+ const fallbackResult = await reconstructWithoutManifest(
1428
+ orbitdb,
1429
+ logEntryBlocks,
1430
+ config,
1431
+ true, // Enable joinEntry mode for proper log traversal
1432
+ );
1433
+
1434
+ logger.info(
1435
+ "✅ Optimized Log-Entries-Only Restore completed successfully!",
1436
+ );
1437
+
1438
+ return {
1439
+ database: fallbackResult.database,
1440
+ metadata: fallbackResult.metadata,
1441
+ entriesCount: fallbackResult.entriesCount,
1442
+ entriesRecovered: fallbackResult.entriesCount,
1443
+ method: "optimized-log-entries-only",
1444
+ success: true,
1445
+ preservedHashes: false,
1446
+ preservedAddress: false,
1447
+ spaceFilesFound: spaceFiles.length,
1448
+ logEntriesDownloaded: logEntryBlocks.size,
1449
+ optimizationSavings: {
1450
+ totalFiles: spaceFiles.length,
1451
+ filesDownloaded: logEntryBlocks.size,
1452
+ filesSkipped: spaceFiles.length - logEntryBlocks.size,
1453
+ percentageSaved: Math.round(
1454
+ ((spaceFiles.length - logEntryBlocks.size) / spaceFiles.length) * 100,
1455
+ ),
1456
+ },
1457
+ };
1458
+ } catch (error) {
1459
+ logger.error(
1460
+ "❌ Optimized Log-Entries-Only Restore failed:",
1461
+ error.message,
1462
+ );
1463
+
1464
+ return {
1465
+ success: false,
1466
+ error: error.message,
1467
+ };
1468
+ }
1469
+ }
1470
+
1471
+ /**
1472
+ * Mapping-independent restore from Storacha (ENHANCED)
1473
+ *
1474
+ * @param {Object} orbitdb - Target OrbitDB instance
1475
+ * @param {Object} options - Restore options
1476
+ * @param {string} [options.storachaKey] - Storacha private key (defaults to env)
1477
+ * @param {string} [options.storachaProof] - Storacha proof (defaults to env)
1478
+ * @param {boolean} [options.forceFallback] - Force fallback reconstruction mode (default: false)
1479
+ * @param {string} [options.fallbackDatabaseName] - Custom name for fallback reconstruction
1480
+ * @param {EventEmitter} [options.eventEmitter] - Optional event emitter for progress updates
1481
+ * @returns {Promise<Object>} - Restore result
1482
+ */
1483
+ export async function restoreDatabaseFromSpace(orbitdb, options = {}) {
1484
+ const config = { ...DEFAULT_OPTIONS, ...options };
1485
+ const eventEmitter = options.eventEmitter;
1486
+ // Use orbitdb parameter directly instead of creating an alias
1487
+
1488
+ logger.info("🔄 Starting Mapping-Independent OrbitDB Restore from Storacha");
1489
+
1490
+ // Backups are CARs since 0.5.2, so look for one first. Both paths are the
1491
+ // same intent — scan the space, reconstruct without being told the mapping —
1492
+ // and differ only in what the space turns out to hold, which is exactly the
1493
+ // decision a caller cannot make for itself.
1494
+ //
1495
+ // The fallback is not politeness: every backup written before 0.5.2 is a set
1496
+ // of individual blocks, and dropping that path would strand them.
1497
+ const strategy = options.strategy ?? "auto";
1498
+ if (strategy === "auto" || strategy === "car") {
1499
+ // Dynamic, because backup-car.js imports this module.
1500
+ const { listAvailableBackups, restoreFromSpaceCAR } = await import("./backup-car.js");
1501
+ let carBackups = [];
1502
+ try {
1503
+ carBackups = (await listAvailableBackups(config)) ?? [];
1504
+ } catch (error) {
1505
+ if (strategy === "car") throw error;
1506
+ logger.info(` ⚠️ Could not look for CAR backups (${error.message}); scanning for blocks`);
1507
+ }
1508
+
1509
+ if (carBackups.length > 0) {
1510
+ const latest = carBackups[0];
1511
+ logger.info(` 📦 Restoring the CAR backup from ${latest.timestamp}`);
1512
+ // The listing already found it; passing both spares a second scan of the
1513
+ // whole space, which downloads every file in it to identify the metadata.
1514
+ return restoreFromSpaceCAR(orbitdb, {
1515
+ ...options,
1516
+ timestamp: latest.timestamp,
1517
+ metadataCID: latest.metadataCID,
1518
+ });
1519
+ }
1520
+
1521
+ if (strategy === "car") {
1522
+ throw new Error("No CAR backup found in this space");
1523
+ }
1524
+ logger.info(" ↩︎ No CAR backup in this space; scanning for individual blocks");
1525
+ }
1526
+
1527
+ try {
1528
+ // Step 1: List ALL files in Storacha space
1529
+ logger.info("\n📋 Step 1: Discovering all files in Storacha space...");
1530
+ const spaceFiles = await listStorachaSpaceFiles(config);
1531
+
1532
+ if (spaceFiles.length === 0) {
1533
+ throw new Error("No files found in Storacha space");
1534
+ }
1535
+
1536
+ logger.info(
1537
+ ` 🎉 SUCCESS! Found ${spaceFiles.length} files in space without requiring CID mappings`,
1538
+ );
1539
+
1540
+ // Step 2: Download ALL files from space with progress tracking
1541
+ // Ensure helia instance is included in config for network downloads
1542
+ const downloadConfig = {
1543
+ ...config,
1544
+ helia: config.helia || orbitdb.ipfs,
1545
+ };
1546
+ const downloadedBlocks = await downloadBlocksWithProgress(
1547
+ spaceFiles,
1548
+ orbitdb,
1549
+ downloadConfig,
1550
+ eventEmitter,
1551
+ );
1552
+
1553
+ // ... rest of existing code remains the same ...
1554
+
1555
+ // Step 3: Intelligent block analysis
1556
+ logger.info("\n🔍 Step 3: Analyzing block structure...");
1557
+ const analysis = await analyzeBlocks(
1558
+ orbitdb.ipfs.blockstore,
1559
+ downloadedBlocks,
1560
+ );
1561
+
1562
+ if (analysis.manifestBlocks.length === 0 || options.forceFallback) {
1563
+ logger.info(
1564
+ "⚠️ No manifest blocks found - attempting fallback reconstruction...",
1565
+ );
1566
+
1567
+ // Fallback: decode blocks and extract payloads to import as new entries
1568
+ const fallbackResult = await reconstructWithoutManifest(
1569
+ orbitdb,
1570
+ downloadedBlocks,
1571
+ config,
1572
+ );
1573
+
1574
+ return {
1575
+ database: fallbackResult.database,
1576
+ metadata: fallbackResult.metadata,
1577
+ entriesCount: fallbackResult.entriesCount,
1578
+ entriesRecovered: fallbackResult.entriesCount,
1579
+ method: "fallback-reconstruction",
1580
+ success: true,
1581
+ preservedHashes: false,
1582
+ preservedAddress: false,
1583
+ };
1584
+ }
1585
+
1586
+ // Step 4: Reconstruct database using discovered manifest
1587
+ logger.info("\n🔄 Step 4: Reconstructing database from analysis...");
1588
+
1589
+ // Find the correct manifest by matching log entries to database IDs
1590
+ const correctManifest = findCorrectManifest(analysis);
1591
+ if (!correctManifest) {
1592
+ throw new Error("Could not determine correct manifest from log entries");
1593
+ }
1594
+
1595
+ const databaseAddress = `/orbitdb/${correctManifest.cid}`;
1596
+
1597
+ logger.info(` 📥 Opening database at: ${databaseAddress}`);
1598
+ logger.info(
1599
+ ` 🎯 Selected manifest: ${correctManifest.cid} (matched from log entries)`,
1600
+ );
1601
+
1602
+ // Extract database type from manifest if available, otherwise infer from log entries
1603
+ let databaseType = inferDatabaseType(analysis.logEntryBlocks); // fallback
1604
+ if (correctManifest.content && correctManifest.content.type) {
1605
+ databaseType = correctManifest.content.type;
1606
+ logger.info(` 📋 Database type from manifest: ${databaseType}`);
1607
+ } else {
1608
+ logger.debug(`Inferred database type: ${databaseType}`);
1609
+ }
1610
+
1611
+ const reconstructedDB = await orbitdb.open(
1612
+ databaseAddress,
1613
+ config.dbConfig ? config.dbConfig : { type: databaseType },
1614
+ );
1615
+
1616
+ // OPTIMIZED: Only join HEAD entries - OrbitDB will traverse backward automatically
1617
+ logger.info(
1618
+ " 🔗 Step 4a: Joining HEAD entries to reconstruct complete log...",
1619
+ );
1620
+ logger.info(
1621
+ ` 🎯 Found ${analysis.potentialHeads.length} HEAD(s) in ${analysis.logEntryBlocks.length} total log entries`,
1622
+ );
1623
+ let joinedHeads = 0;
1624
+
1625
+ // Only process HEAD entries - OrbitDB automatically traverses backward from each head
1626
+ for (const headCID of analysis.potentialHeads) {
1627
+ try {
1628
+ // Find the corresponding log entry block for this head
1629
+ const logEntryBlock = analysis.logEntryBlocks.find(
1630
+ (block) => block.cid === headCID,
1631
+ );
1632
+
1633
+ if (!logEntryBlock) {
1634
+ logger.warn(
1635
+ ` ⚠️ HEAD ${headCID.slice(0, 12)}... not found in log entry blocks`,
1636
+ );
1637
+ continue;
1638
+ }
1639
+
1640
+ // Create an entry object that matches OrbitDB's expected format
1641
+ const entryData = {
1642
+ hash: logEntryBlock.cid,
1643
+ v: logEntryBlock.content.v,
1644
+ id: logEntryBlock.content.id,
1645
+ key: logEntryBlock.content.key,
1646
+ sig: logEntryBlock.content.sig,
1647
+ next: logEntryBlock.content.next,
1648
+ refs: logEntryBlock.content.refs,
1649
+ clock: logEntryBlock.content.clock,
1650
+ payload: logEntryBlock.content.payload,
1651
+ identity: logEntryBlock.content.identity,
1652
+ };
1653
+
1654
+ // Use joinEntry - this will automatically traverse and join all connected entries
1655
+ const updated = await reconstructedDB.log.joinEntry(entryData);
1656
+ if (updated) {
1657
+ joinedHeads++;
1658
+ logger.info(
1659
+ ` ✓ Joined HEAD ${joinedHeads}/${analysis.potentialHeads.length}: ${headCID.slice(0, 12)}...`,
1660
+ );
1661
+ logger.info(
1662
+ ` (OrbitDB will automatically traverse and load all connected entries)`,
1663
+ );
1664
+ } else {
1665
+ logger.info(` → HEAD already in log: ${headCID.slice(0, 12)}...`);
1666
+ }
1667
+ } catch (joinError) {
1668
+ logger.warn(
1669
+ ` ⚠️ Failed to join HEAD ${headCID}: ${joinError.message}`,
1670
+ );
1671
+ // Continue with other heads even if one fails
1672
+ }
1673
+ }
1674
+
1675
+ logger.info(
1676
+ ` 📊 Successfully joined ${joinedHeads}/${analysis.potentialHeads.length} HEAD entries`,
1677
+ );
1678
+
1679
+ // Wait for log to settle after joining entries
1680
+ logger.info(" ⏳ Waiting for log to settle after joining entries...");
1681
+ await new Promise((resolve) => setTimeout(resolve, config.timeout / 5));
1682
+
1683
+ const reconstructedEntries = await reconstructedDB.all();
1684
+ logger.info(" ⏳ Additional wait for entries to fully load...");
1685
+ await new Promise((resolve) => setTimeout(resolve, config.timeout / 10));
1686
+
1687
+ // Handle different database types properly
1688
+ let entriesArray;
1689
+ let entriesCount;
1690
+ if (reconstructedDB.type === "keyvalue") {
1691
+ // For key-value databases, all() returns an object
1692
+ // Get the actual log entries to preserve hashes
1693
+ const logEntries = await reconstructedDB.log.values();
1694
+ entriesArray = logEntries.map((logEntry) => ({
1695
+ hash: logEntry.hash,
1696
+ payload: logEntry.payload,
1697
+ }));
1698
+ entriesCount = Object.keys(reconstructedEntries).length;
1699
+ } else {
1700
+ // For other database types (events, documents), all() returns an array
1701
+ entriesArray = Array.isArray(reconstructedEntries)
1702
+ ? reconstructedEntries
1703
+ : [];
1704
+ entriesCount = entriesArray.length;
1705
+ }
1706
+
1707
+ logger.info(` 📊 Final reconstructed entries: ${entriesCount}`);
1708
+ logger.info(` 🔍 Database type: ${reconstructedDB.type}`);
1709
+ logger.info(` 🔗 HEAD entries joined: ${joinedHeads}`);
1710
+
1711
+ logger.info(
1712
+ "✅ Mapping-Independent Restore with proper joinEntry completed successfully!",
1713
+ );
1714
+
1715
+ return {
1716
+ success: true,
1717
+ database: reconstructedDB,
1718
+ orbitdb: orbitdb, // Return the OrbitDB instance
1719
+ manifestCID: correctManifest.cid,
1720
+ address: reconstructedDB.address,
1721
+ name: reconstructedDB.name,
1722
+ type: reconstructedDB.type,
1723
+ entriesRecovered: entriesCount,
1724
+ blocksRestored: downloadedBlocks.size,
1725
+ addressMatch: reconstructedDB.address === databaseAddress,
1726
+ spaceFilesFound: spaceFiles.length,
1727
+ analysis,
1728
+ entries: entriesArray,
1729
+ };
1730
+ } catch (error) {
1731
+ logger.error("❌ Mapping-Independent Restore failed:", error.message);
1732
+
1733
+ return {
1734
+ success: false,
1735
+ error: error.message,
1736
+ };
1737
+ }
1738
+ }
1739
+
1740
+ /**
1741
+ * Restore an OrbitDB database from Storacha (Legacy mapping-dependent)
1742
+ *
1743
+ * @param {Object} orbitdb - Target OrbitDB instance
1744
+ * @param {string} manifestCID - Manifest CID from backup
1745
+ * @param {Object} cidMappings - Optional CID mappings (if available)
1746
+ * @param {Object} options - Restore options
1747
+ * @returns {Promise<Object>} - Restore result
1748
+ */
1749
+ export async function restoreDatabase(
1750
+ orbitdb,
1751
+ manifestCID,
1752
+ cidMappings = null,
1753
+ options = {},
1754
+ ) {
1755
+ const config = { ...DEFAULT_OPTIONS, ...options };
1756
+
1757
+ logger.info("🔄 Starting OrbitDB Database Restore from Storacha");
1758
+ logger.info(`📍 Manifest CID: ${manifestCID}`);
1759
+
1760
+ // No temporary resources needed
1761
+
1762
+ try {
1763
+ // Initialize Storacha client - support both credential and UCAN authentication
1764
+ const backend = await resolveBackend(config);
1765
+ const client = backend.client;
1766
+
1767
+ // If no CID mappings provided, we need to discover them
1768
+ // This is a simplified version - in practice you'd store mappings during backup
1769
+ if (!cidMappings) {
1770
+ throw new Error(
1771
+ "CID mappings required for restore. Store them during backup.",
1772
+ );
1773
+ }
1774
+
1775
+ // Convert object back to Map if needed
1776
+ const mappings =
1777
+ cidMappings instanceof Map
1778
+ ? cidMappings
1779
+ : new Map(Object.entries(cidMappings));
1780
+
1781
+ // Download and bridge blocks
1782
+ const { successful, matches } = await downloadAndBridgeBlocks(
1783
+ mappings,
1784
+ client,
1785
+ orbitdb.ipfs.blockstore,
1786
+ { ...config, helia: orbitdb.ipfs },
1787
+ );
1788
+
1789
+ if (successful.length === 0) {
1790
+ throw new Error("No blocks were successfully restored");
1791
+ }
1792
+
1793
+ if (matches.length < successful.length) {
1794
+ throw new Error("Some CID bridges failed - reconstruction may fail");
1795
+ }
1796
+
1797
+ // Analyze blocks to find log entries and HEAD entries
1798
+ logger.info("🔍 Analyzing restored blocks to find log entries...");
1799
+ const downloadedBlocks = new Set(Array.from(mappings.keys())); // OrbitDB CIDs
1800
+ const analysis = await analyzeBlocks(
1801
+ orbitdb.ipfs.blockstore,
1802
+ downloadedBlocks,
1803
+ );
1804
+
1805
+ // Reconstruct database
1806
+ const databaseAddress = `/orbitdb/${manifestCID}`;
1807
+ logger.info(`📥 Opening database at: ${databaseAddress}`);
1808
+
1809
+ // Determine database type from manifest or analysis
1810
+ let databaseType = inferDatabaseType(analysis.logEntryBlocks);
1811
+ const manifestBlock = analysis.manifestBlocks.find(
1812
+ (m) => m.cid === manifestCID,
1813
+ );
1814
+ if (manifestBlock?.content?.type) {
1815
+ databaseType = manifestBlock.content.type;
1816
+ }
1817
+
1818
+ const reconstructedDB = await orbitdb.open(databaseAddress, {
1819
+ type: databaseType,
1820
+ });
1821
+
1822
+ // Join HEAD entries to reconstruct the log (similar to restoreDatabaseFromSpace)
1823
+ if (analysis.potentialHeads.length > 0) {
1824
+ logger.info(
1825
+ `🔗 Joining ${analysis.potentialHeads.length} HEAD entry/entries to reconstruct complete log...`,
1826
+ );
1827
+ let joinedHeads = 0;
1828
+
1829
+ for (const headCID of analysis.potentialHeads) {
1830
+ try {
1831
+ const logEntryBlock = analysis.logEntryBlocks.find(
1832
+ (block) => block.cid === headCID,
1833
+ );
1834
+
1835
+ if (!logEntryBlock) {
1836
+ logger.warn(
1837
+ `⚠️ HEAD ${headCID.slice(0, 12)}... not found in log entry blocks`,
1838
+ );
1839
+ continue;
1840
+ }
1841
+
1842
+ // Create an entry object that matches OrbitDB's expected format
1843
+ const entryData = {
1844
+ hash: logEntryBlock.cid,
1845
+ v: logEntryBlock.content.v,
1846
+ id: logEntryBlock.content.id,
1847
+ key: logEntryBlock.content.key,
1848
+ sig: logEntryBlock.content.sig,
1849
+ next: logEntryBlock.content.next,
1850
+ refs: logEntryBlock.content.refs,
1851
+ clock: logEntryBlock.content.clock,
1852
+ payload: logEntryBlock.content.payload,
1853
+ identity: logEntryBlock.content.identity,
1854
+ };
1855
+
1856
+ // Use joinEntry - this will automatically traverse and join all connected entries
1857
+ const updated = await reconstructedDB.log.joinEntry(entryData);
1858
+ if (updated) {
1859
+ joinedHeads++;
1860
+ logger.info(
1861
+ `✓ Joined HEAD ${joinedHeads}/${analysis.potentialHeads.length}: ${headCID.slice(0, 12)}...`,
1862
+ );
1863
+ }
1864
+ } catch (joinError) {
1865
+ logger.warn(
1866
+ `⚠️ Failed to join HEAD ${headCID}: ${joinError.message}`,
1867
+ );
1868
+ }
1869
+ }
1870
+
1871
+ logger.info(
1872
+ `📊 Successfully joined ${joinedHeads}/${analysis.potentialHeads.length} HEAD entries`,
1873
+ );
1874
+ }
1875
+
1876
+ // Wait for log to settle after joining entries
1877
+ logger.info("⏳ Waiting for entries to load...");
1878
+ await new Promise((resolve) => setTimeout(resolve, config.timeout / 5));
1879
+ await new Promise((resolve) => setTimeout(resolve, config.timeout / 10));
1880
+
1881
+ // Handle different database types properly
1882
+ let reconstructedEntries;
1883
+ let entriesArray;
1884
+ let entriesCount;
1885
+
1886
+ if (reconstructedDB.type === "keyvalue") {
1887
+ reconstructedEntries = await reconstructedDB.all();
1888
+ // For key-value databases, all() returns an object
1889
+ const logEntries = await reconstructedDB.log.values();
1890
+ entriesArray = logEntries.map((logEntry) => ({
1891
+ hash: logEntry.hash,
1892
+ value: logEntry.payload.value,
1893
+ }));
1894
+ entriesCount = Object.keys(reconstructedEntries).length;
1895
+ } else {
1896
+ // For other database types (events, documents), all() returns an array
1897
+ reconstructedEntries = await reconstructedDB.all();
1898
+ entriesArray = Array.isArray(reconstructedEntries)
1899
+ ? reconstructedEntries
1900
+ : [];
1901
+ entriesCount = entriesArray.length;
1902
+ }
1903
+
1904
+ logger.info("✅ Restore completed successfully!", entriesCount);
1905
+
1906
+ return {
1907
+ success: true,
1908
+ database: reconstructedDB,
1909
+ manifestCID,
1910
+ address: reconstructedDB.address,
1911
+ name: reconstructedDB.name,
1912
+ type: reconstructedDB.type,
1913
+ entriesRecovered: entriesCount,
1914
+ blocksRestored: successful.length,
1915
+ addressMatch: reconstructedDB.address === databaseAddress,
1916
+ entries: entriesArray.map((e) => ({
1917
+ hash: e.hash || e._id || e.id,
1918
+ value: e.value || e,
1919
+ })),
1920
+ };
1921
+ } catch (error) {
1922
+ logger.error("❌ Restore failed:", error.message);
1923
+ return {
1924
+ success: false,
1925
+ error: error.message,
1926
+ };
1927
+ } // No cleanup needed
1928
+ }
1929
+
1930
+ /**
1931
+ * Enhanced OrbitDBStorachaBridge class with event emission
1932
+ */
1933
+ export class OrbitDBStorachaBridge extends EventEmitter {
1934
+ constructor(options = {}) {
1935
+ super();
1936
+ this.config = { ...DEFAULT_OPTIONS, ...options };
1937
+ }
1938
+
1939
+ /**
1940
+ * Back up a database. A CAR by default; pass `strategy: "blocks"` for the
1941
+ * pre-0.5.2 per-block path, which some backends will refuse.
1942
+ */
1943
+ async backup(orbitdb, databaseAddress, options = {}) {
1944
+ // Pass this instance as eventEmitter to enable progress events
1945
+ return await backupDatabase(orbitdb, databaseAddress, {
1946
+ ...this.config,
1947
+ ...options,
1948
+ eventEmitter: this,
1949
+ });
1950
+ }
1951
+
1952
+ async backupLogEntriesOnly(orbitdb, databaseAddress, options = {}) {
1953
+ // Optimized backup for fallback reconstruction - only log entries
1954
+ return await backupDatabase(orbitdb, databaseAddress, {
1955
+ ...this.config,
1956
+ ...options,
1957
+ logEntriesOnly: true,
1958
+ eventEmitter: this,
1959
+ });
1960
+ }
1961
+
1962
+ async restore(orbitdb, manifestCID, cidMappings, options = {}) {
1963
+ return await restoreDatabase(orbitdb, manifestCID, cidMappings, {
1964
+ ...this.config,
1965
+ ...options,
1966
+ });
1967
+ }
1968
+
1969
+ // BREAKTHROUGH: Mapping-independent restore
1970
+ async restoreFromSpace(orbitdb, options = {}) {
1971
+ return await restoreDatabaseFromSpace(orbitdb, {
1972
+ ...this.config,
1973
+ ...options,
1974
+ eventEmitter: this,
1975
+ });
1976
+ }
1977
+
1978
+ // OPTIMIZATION: Log-entries-only restore (much faster for fallback reconstruction)
1979
+ async restoreLogEntriesOnly(orbitdb, options = {}) {
1980
+ return await restoreLogEntriesOnly(orbitdb, {
1981
+ ...this.config,
1982
+ ...options,
1983
+ eventEmitter: this,
1984
+ });
1985
+ }
1986
+
1987
+ // Utility methods
1988
+ async listSpaceFiles(options = {}) {
1989
+ return await listStorachaSpaceFiles({ ...this.config, ...options });
1990
+ }
1991
+
1992
+ async analyzeBlocks(blockstore, downloadedBlocks) {
1993
+ return await analyzeBlocks(blockstore, downloadedBlocks);
1994
+ }
1995
+
1996
+ extractManifestCID(databaseAddress) {
1997
+ return extractManifestCID(databaseAddress);
1998
+ }
1999
+
2000
+ convertCID(storachaCID) {
2001
+ return convertStorachaCIDToOrbitDB(storachaCID);
2002
+ }
2003
+ }
2004
+
2005
+ /**
2006
+ * Find the correct manifest block by matching log entries to database IDs
2007
+ *
2008
+ * @param {Object} analysis - Block analysis results
2009
+ * @returns {Object|null} - Correct manifest block or null if not found
2010
+ */
2011
+ function findCorrectManifest(analysis) {
2012
+ logger.info("🎯 Finding correct manifest from log entries...");
2013
+
2014
+ if (analysis.manifestBlocks.length === 1) {
2015
+ logger.info(" ✅ Only one manifest found, using it");
2016
+ return analysis.manifestBlocks[0];
2017
+ }
2018
+
2019
+ if (analysis.manifestBlocks.length === 0) {
2020
+ logger.info(" ❌ No manifest blocks found");
2021
+ return null;
2022
+ }
2023
+
2024
+ // Extract database IDs from log entries
2025
+ const databaseIds = new Set();
2026
+ for (const logEntry of analysis.logEntryBlocks) {
2027
+ if (logEntry.content && logEntry.content.id) {
2028
+ // Extract manifest CID from database address like '/orbitdb/zdpu...'
2029
+ const manifestCID = logEntry.content.id.replace("/orbitdb/", "");
2030
+ databaseIds.add(manifestCID);
2031
+ logger.info(
2032
+ ` 📝 Log entry references database: ${logEntry.content.id} (manifest: ${manifestCID})`,
2033
+ );
2034
+ }
2035
+ }
2036
+
2037
+ logger.info(
2038
+ ` 🔍 Found ${databaseIds.size} unique database ID(s) from ${analysis.logEntryBlocks.length} log entries`,
2039
+ );
2040
+
2041
+ // Find manifest that matches the most referenced database ID
2042
+ const manifestCounts = new Map();
2043
+ for (const manifestBlock of analysis.manifestBlocks) {
2044
+ const count = databaseIds.has(manifestBlock.cid) ? 1 : 0;
2045
+ manifestCounts.set(manifestBlock.cid, count);
2046
+ logger.info(
2047
+ ` 📋 Manifest ${manifestBlock.cid}: ${count > 0 ? "MATCHES" : "no match"}`,
2048
+ );
2049
+ }
2050
+
2051
+ // Find the manifest with the highest count (most log entry references)
2052
+ let bestManifest = null;
2053
+ let bestCount = -1;
2054
+
2055
+ for (const [manifestCID, count] of manifestCounts) {
2056
+ if (count > bestCount) {
2057
+ bestCount = count;
2058
+ bestManifest = analysis.manifestBlocks.find((m) => m.cid === manifestCID);
2059
+ }
2060
+ }
2061
+
2062
+ if (bestManifest && bestCount > 0) {
2063
+ logger.info(
2064
+ ` ✅ Selected manifest: ${bestManifest.cid} (referenced by ${bestCount} log entries)`,
2065
+ );
2066
+ return bestManifest;
2067
+ }
2068
+
2069
+ // Fallback: if no manifest matches log entries, use the first one and warn
2070
+ logger.warn(
2071
+ " ⚠️ No manifest matched log entries, using first manifest as fallback",
2072
+ );
2073
+ return analysis.manifestBlocks[0];
2074
+ }
2075
+
2076
+ /**
2077
+ * Download only log entry blocks from Storacha (optimized for fallback reconstruction)
2078
+ *
2079
+ * @param {Array} spaceFiles - Array of space files to download
2080
+ * @param {Object} currentOrbitDB - OrbitDB instance
2081
+ * @param {Object} config - Configuration options
2082
+ * @param {EventEmitter} eventEmitter - Optional event emitter for progress updates
2083
+ * @returns {Promise<Map>} - Downloaded log entry blocks map
2084
+ */
2085
+ async function downloadLogEntriesOnly(
2086
+ spaceFiles,
2087
+ currentOrbitDB,
2088
+ config,
2089
+ eventEmitter = null,
2090
+ ) {
2091
+ logger.info("\n📥 Downloading and filtering log entry blocks only...");
2092
+ const logEntryBlocks = new Map();
2093
+ const totalFiles = spaceFiles.length;
2094
+ let completedFiles = 0;
2095
+ let logEntriesFound = 0;
2096
+
2097
+ // Emit initial progress
2098
+ if (eventEmitter) {
2099
+ eventEmitter.emit("downloadProgress", {
2100
+ type: "download",
2101
+ current: 0,
2102
+ total: totalFiles,
2103
+ percentage: 0,
2104
+ status: "starting (log entries only)",
2105
+ });
2106
+ }
2107
+
2108
+ for (const spaceFile of spaceFiles) {
2109
+ const storachaCID = spaceFile.root;
2110
+ logger.info(` 🔄 Checking: ${storachaCID}`);
2111
+
2112
+ try {
2113
+ // Ensure helia instance is passed for network downloads
2114
+ const downloadConfig = {
2115
+ ...config,
2116
+ helia: config.helia || currentOrbitDB?.ipfs,
2117
+ };
2118
+ const bytes = await downloadBlock(storachaCID, downloadConfig);
2119
+
2120
+ // Convert Storacha CID to OrbitDB format
2121
+ const orbitdbCID = convertStorachaCIDToOrbitDB(storachaCID);
2122
+ const parsedCID = CID.parse(orbitdbCID);
2123
+
2124
+ // Only process dag-cbor blocks (potential log entries)
2125
+ if (parsedCID.code === 0x71) {
2126
+ try {
2127
+ const block = await Block.decode({
2128
+ cid: parsedCID,
2129
+ bytes,
2130
+ codec: dagCbor,
2131
+ hasher: sha256,
2132
+ });
2133
+
2134
+ const content = block.value;
2135
+
2136
+ // Check if this looks like an OrbitDB log entry
2137
+ if (
2138
+ content &&
2139
+ content.v === 2 &&
2140
+ content.id &&
2141
+ content.clock &&
2142
+ content.payload !== undefined
2143
+ ) {
2144
+ // Store in target blockstore
2145
+ await currentOrbitDB.ipfs.blockstore.put(parsedCID, bytes);
2146
+ logEntryBlocks.set(orbitdbCID, {
2147
+ storachaCID,
2148
+ bytes: bytes.length,
2149
+ });
2150
+ logEntriesFound++;
2151
+
2152
+ logger.info(` ✅ Log entry stored: ${orbitdbCID}`);
2153
+ } else {
2154
+ logger.info(` ⚪ Skipped non-log block: ${orbitdbCID}`);
2155
+ }
2156
+ } catch {
2157
+ logger.info(` ⚪ Skipped non-decodable block: ${orbitdbCID}`);
2158
+ }
2159
+ } else {
2160
+ logger.info(` ⚪ Skipped non-CBOR block: ${orbitdbCID}`);
2161
+ }
2162
+
2163
+ // Update progress
2164
+ completedFiles++;
2165
+ if (eventEmitter) {
2166
+ eventEmitter.emit("downloadProgress", {
2167
+ type: "download",
2168
+ current: completedFiles,
2169
+ total: totalFiles,
2170
+ percentage: Math.round((completedFiles / totalFiles) * 100),
2171
+ status: "downloading (log entries only)",
2172
+ currentBlock: {
2173
+ storachaCID,
2174
+ orbitdbCID,
2175
+ size: bytes.length,
2176
+ isLogEntry: logEntryBlocks.has(orbitdbCID),
2177
+ },
2178
+ });
2179
+ }
2180
+ } catch (error) {
2181
+ logger.error(` ❌ Failed: ${storachaCID} - ${error.message}`);
2182
+
2183
+ // Update progress even for failed downloads
2184
+ completedFiles++;
2185
+ if (eventEmitter) {
2186
+ eventEmitter.emit("downloadProgress", {
2187
+ type: "download",
2188
+ current: completedFiles,
2189
+ total: totalFiles,
2190
+ percentage: Math.round((completedFiles / totalFiles) * 100),
2191
+ status: "downloading (log entries only)",
2192
+ error: {
2193
+ storachaCID,
2194
+ message: error.message,
2195
+ },
2196
+ });
2197
+ }
2198
+ }
2199
+ }
2200
+
2201
+ // Emit completion
2202
+ if (eventEmitter) {
2203
+ eventEmitter.emit("downloadProgress", {
2204
+ type: "download",
2205
+ current: totalFiles,
2206
+ total: totalFiles,
2207
+ percentage: 100,
2208
+ status: "completed (log entries only)",
2209
+ summary: {
2210
+ totalFiles: totalFiles,
2211
+ logEntriesFound: logEntriesFound,
2212
+ blocksSkipped: totalFiles - logEntriesFound,
2213
+ },
2214
+ });
2215
+ }
2216
+
2217
+ logger.info(
2218
+ ` 📊 Found ${logEntriesFound} log entries out of ${totalFiles} total files`,
2219
+ );
2220
+ return logEntryBlocks;
2221
+ }
2222
+
2223
+ /**
2224
+ * Download blocks from Storacha with progress events
2225
+ *
2226
+ * @param {Array} spaceFiles - Array of space files to download
2227
+ * @param {Object} currentOrbitDB - OrbitDB instance
2228
+ * @param {Object} config - Configuration options
2229
+ * @param {EventEmitter} eventEmitter - Optional event emitter for progress updates
2230
+ * @returns {Promise<Map>} - Downloaded blocks map
2231
+ */
2232
+ async function downloadBlocksWithProgress(
2233
+ spaceFiles,
2234
+ currentOrbitDB,
2235
+ config,
2236
+ eventEmitter = null,
2237
+ ) {
2238
+ logger.info("\n📥 Downloading all space files...");
2239
+ const downloadedBlocks = new Map();
2240
+ const totalFiles = spaceFiles.length;
2241
+ let completedFiles = 0;
2242
+
2243
+ // Emit initial progress
2244
+ if (eventEmitter) {
2245
+ eventEmitter.emit("downloadProgress", {
2246
+ type: "download",
2247
+ current: 0,
2248
+ total: totalFiles,
2249
+ percentage: 0,
2250
+ status: "starting",
2251
+ });
2252
+ }
2253
+
2254
+ for (const spaceFile of spaceFiles) {
2255
+ const storachaCID = spaceFile.root;
2256
+ logger.info(` 🔄 Downloading: ${storachaCID}`);
2257
+
2258
+ try {
2259
+ // Ensure helia instance is passed for network downloads
2260
+ const downloadConfig = {
2261
+ ...config,
2262
+ helia: config.helia || currentOrbitDB?.ipfs,
2263
+ };
2264
+ const bytes = await downloadBlock(storachaCID, downloadConfig);
2265
+
2266
+ // Convert Storacha CID to OrbitDB format
2267
+ const orbitdbCID = convertStorachaCIDToOrbitDB(storachaCID);
2268
+ const parsedCID = CID.parse(orbitdbCID);
2269
+
2270
+ // Store in target blockstore with error handling for closed blockstore
2271
+ if (!currentOrbitDB?.ipfs?.blockstore) {
2272
+ throw new Error(
2273
+ "Blockstore is not available - OrbitDB may have been closed",
2274
+ );
2275
+ }
2276
+ try {
2277
+ await currentOrbitDB.ipfs.blockstore.put(parsedCID, bytes);
2278
+ } catch (error) {
2279
+ if (
2280
+ error.message?.includes("not open") ||
2281
+ error.message?.includes("closed") ||
2282
+ error.message?.includes("Database is not open")
2283
+ ) {
2284
+ throw new Error(
2285
+ `Blockstore operation failed - OrbitDB was closed during download: ${error.message}`, { cause: error },
2286
+ );
2287
+ }
2288
+ throw error;
2289
+ }
2290
+ downloadedBlocks.set(orbitdbCID, { storachaCID, bytes: bytes.length });
2291
+
2292
+ logger.info(` ✅ Stored: ${orbitdbCID}`);
2293
+
2294
+ // Update progress
2295
+ completedFiles++;
2296
+ if (eventEmitter) {
2297
+ eventEmitter.emit("downloadProgress", {
2298
+ type: "download",
2299
+ current: completedFiles,
2300
+ total: totalFiles,
2301
+ percentage: Math.round((completedFiles / totalFiles) * 100),
2302
+ status: "downloading",
2303
+ currentBlock: {
2304
+ storachaCID,
2305
+ orbitdbCID,
2306
+ size: bytes.length,
2307
+ },
2308
+ });
2309
+ }
2310
+ } catch (error) {
2311
+ logger.error(` ❌ Failed: ${storachaCID} - ${error.message}`);
2312
+
2313
+ // Update progress even for failed downloads
2314
+ completedFiles++;
2315
+ if (eventEmitter) {
2316
+ eventEmitter.emit("downloadProgress", {
2317
+ type: "download",
2318
+ current: completedFiles,
2319
+ total: totalFiles,
2320
+ percentage: Math.round((completedFiles / totalFiles) * 100),
2321
+ status: "downloading",
2322
+ error: {
2323
+ storachaCID,
2324
+ message: error.message,
2325
+ },
2326
+ });
2327
+ }
2328
+ }
2329
+ }
2330
+
2331
+ // Emit completion
2332
+ if (eventEmitter) {
2333
+ eventEmitter.emit("downloadProgress", {
2334
+ type: "download",
2335
+ current: totalFiles,
2336
+ total: totalFiles,
2337
+ percentage: 100,
2338
+ status: "completed",
2339
+ summary: {
2340
+ downloaded: downloadedBlocks.size,
2341
+ failed: totalFiles - downloadedBlocks.size,
2342
+ },
2343
+ });
2344
+ }
2345
+
2346
+ logger.info(` 📊 Downloaded ${downloadedBlocks.size} blocks total`);
2347
+ return downloadedBlocks;
2348
+ }
2349
+
2350
+ /**
2351
+ * Fallback reconstruction when no manifest is found
2352
+ * Decodes blocks, extracts payloads, and creates a new database
2353
+ *
2354
+ * @param {Object} orbitdb - OrbitDB instance
2355
+ * @param {Map} downloadedBlocks - Downloaded blocks map
2356
+ * @param {Object} config - Configuration options
2357
+ * @param {boolean} useJoinEntry - Whether to use joinEntry for reconstruction
2358
+ * @returns {Promise<Object>} - Reconstruction results
2359
+ */
2360
+ export async function reconstructWithoutManifest(
2361
+ orbitdb,
2362
+ downloadedBlocks,
2363
+ config,
2364
+ useJoinEntry = false,
2365
+ ) {
2366
+ logger.info(
2367
+ `🔧 Starting fallback reconstruction without manifest ${useJoinEntry ? "with joinEntry support" : "(legacy mode)"}...`,
2368
+ );
2369
+
2370
+ const logEntries = [];
2371
+ const unknownBlocks = [];
2372
+
2373
+ // Step 1: Decode all blocks and identify log entries
2374
+ logger.info("🔍 Step 1: Decoding blocks to find log entries...");
2375
+
2376
+ for (const [cidString, _] of downloadedBlocks) {
2377
+ try {
2378
+ const cid = CID.parse(cidString);
2379
+ const bytes = await readBlockBytes(orbitdb.ipfs.blockstore, cid);
2380
+
2381
+ if (cid.code === 0x71) {
2382
+ // dag-cbor codec
2383
+ try {
2384
+ const block = await Block.decode({
2385
+ cid,
2386
+ bytes,
2387
+ codec: dagCbor,
2388
+ hasher: sha256,
2389
+ });
2390
+
2391
+ const content = block.value;
2392
+
2393
+ // Check if this looks like an OrbitDB log entry
2394
+ if (
2395
+ content &&
2396
+ content.v === 2 &&
2397
+ content.id &&
2398
+ content.clock &&
2399
+ content.payload !== undefined
2400
+ ) {
2401
+ logEntries.push({
2402
+ cid: cidString,
2403
+ content,
2404
+ hash: cidString,
2405
+ payload: content.payload,
2406
+ });
2407
+
2408
+ logger.info(` 📝 Found log entry: ${cidString.slice(0, 12)}...`);
2409
+ }
2410
+ } catch (decodeError) {
2411
+ unknownBlocks.push({ cid: cidString, error: decodeError.message });
2412
+ }
2413
+ }
2414
+ } catch (error) {
2415
+ logger.warn(
2416
+ ` ⚠️ Error processing block ${cidString}: ${error.message}`,
2417
+ );
2418
+ }
2419
+ }
2420
+
2421
+ logger.info(` ✅ Found ${logEntries.length} log entries`);
2422
+
2423
+ if (logEntries.length === 0) {
2424
+ throw new Error(
2425
+ "No OrbitDB log entries found in blocks - cannot reconstruct database",
2426
+ );
2427
+ }
2428
+
2429
+ // Step 2: Analyze payload patterns to determine database type
2430
+ logger.info(
2431
+ "🔍 Step 2: Analyzing payload patterns to determine database type...",
2432
+ );
2433
+
2434
+ const databaseType = inferDatabaseType(logEntries);
2435
+ logger.info(` 📊 Inferred database type: ${databaseType}`);
2436
+
2437
+ // Step 3: Create new database and import entries
2438
+ logger.info(`🆕 Step 3: Creating new ${databaseType} database...`);
2439
+
2440
+ const dbName = config.fallbackDatabaseName || `restored-${Date.now()}`;
2441
+
2442
+ // Merge config.dbConfig with inferred type, prioritizing the inferred type
2443
+ const dbOptions = {
2444
+ ...config.dbConfig,
2445
+ type: databaseType,
2446
+ };
2447
+
2448
+ const database = await orbitdb.open(dbName, dbOptions);
2449
+
2450
+ logger.info(` ✅ Created database: ${database.address}`);
2451
+
2452
+ // Step 4: Import entries using appropriate method
2453
+ let importedCount = 0;
2454
+ const importErrors = [];
2455
+
2456
+ if (useJoinEntry) {
2457
+ logger.info("📥 Step 4: Using joinEntry to properly reconstruct log...");
2458
+
2459
+ // Sort by clock time to maintain order
2460
+ logEntries.sort((a, b) => {
2461
+ const timeA = a.content.clock?.time || 0;
2462
+ const timeB = b.content.clock?.time || 0;
2463
+ return timeA - timeB;
2464
+ });
2465
+
2466
+ for (const entry of logEntries) {
2467
+ try {
2468
+ // Use joinEntry to properly add this entry to the log
2469
+ const entryData = {
2470
+ hash: entry.hash,
2471
+ v: entry.content.v,
2472
+ id: entry.content.id,
2473
+ key: entry.content.key,
2474
+ sig: entry.content.sig,
2475
+ next: entry.content.next,
2476
+ refs: entry.content.refs,
2477
+ clock: entry.content.clock,
2478
+ payload: entry.content.payload,
2479
+ identity: entry.content.identity,
2480
+ };
2481
+
2482
+ const updated = await database.log.joinEntry(entryData);
2483
+ if (updated) {
2484
+ importedCount++;
2485
+ logger.info(
2486
+ ` ✓ Joined entry ${importedCount}/${logEntries.length}: ${entry.hash.slice(0, 12)}...`,
2487
+ );
2488
+ } else {
2489
+ // Entry might already be in the log
2490
+ logger.info(
2491
+ ` → Entry already processed: ${entry.hash.slice(0, 12)}...`,
2492
+ );
2493
+ }
2494
+ } catch (error) {
2495
+ importErrors.push({ entry: entry.hash, error: error.message });
2496
+ logger.warn(
2497
+ ` ⚠️ Failed to join entry ${entry.hash.slice(0, 12)}...: ${error.message}`,
2498
+ );
2499
+ }
2500
+ }
2501
+
2502
+ logger.info(
2503
+ ` 📊 Join complete: ${importedCount}/${logEntries.length} entries joined`,
2504
+ );
2505
+ } else {
2506
+ logger.info(
2507
+ "📥 Step 4: Importing entries in chronological order (legacy mode)...",
2508
+ );
2509
+
2510
+ // Sort by clock time to maintain order
2511
+ logEntries.sort((a, b) => {
2512
+ const timeA = a.content.clock?.time || 0;
2513
+ const timeB = b.content.clock?.time || 0;
2514
+ return timeA - timeB;
2515
+ });
2516
+
2517
+ for (const entry of logEntries) {
2518
+ try {
2519
+ await importEntryByType(database, entry, databaseType);
2520
+ importedCount++;
2521
+ logger.info(
2522
+ ` ✅ Imported entry ${importedCount}/${logEntries.length}`,
2523
+ );
2524
+ } catch (error) {
2525
+ importErrors.push({ entry: entry.hash, error: error.message });
2526
+ logger.warn(
2527
+ ` ⚠️ Failed to import entry ${entry.hash.slice(0, 12)}...: ${error.message}`,
2528
+ );
2529
+ }
2530
+ }
2531
+ }
2532
+
2533
+ logger.info(
2534
+ ` 📊 ${useJoinEntry ? "Join" : "Import"} complete: ${importedCount}/${logEntries.length} entries ${useJoinEntry ? "joined" : "imported"}`,
2535
+ );
2536
+
2537
+ if (importErrors.length > 0) {
2538
+ logger.warn(
2539
+ ` ⚠️ ${importErrors.length} ${useJoinEntry ? "join" : "import"} errors occurred`,
2540
+ );
2541
+ }
2542
+
2543
+ // Create fallback metadata
2544
+ const metadata = {
2545
+ type: "fallback-reconstruction",
2546
+ databaseType: databaseType,
2547
+ originalEntryCount: logEntries.length,
2548
+ importedEntryCount: importedCount,
2549
+ reconstructedAt: new Date().toISOString(),
2550
+ address: database.address.toString(),
2551
+ name: dbName,
2552
+ importErrors: importErrors.length,
2553
+ };
2554
+
2555
+ return {
2556
+ database,
2557
+ metadata,
2558
+ entriesCount: importedCount,
2559
+ };
2560
+ }
2561
+
2562
+ /**
2563
+ * Infer database type from payload patterns
2564
+ *
2565
+ * @param {Array} logEntries - Array of log entries
2566
+ * @returns {string} - Inferred database type
2567
+ */
2568
+ function inferDatabaseType(logEntries) {
2569
+ const payloadPatterns = {
2570
+ hasDocumentOps: 0,
2571
+ hasKeyValueOps: 0,
2572
+ hasSimplePayloads: 0,
2573
+ hasCounterOps: 0,
2574
+ };
2575
+
2576
+ logger.info(
2577
+ ` 🔍 Analyzing ${logEntries.length} log entries for database type...`,
2578
+ );
2579
+
2580
+ // Debug: Check structure of first few entries
2581
+ logger.debug(
2582
+ {
2583
+ firstEntryStructure:
2584
+ logEntries.length > 0 ? Object.keys(logEntries[0]) : "No entries",
2585
+ hasPayload:
2586
+ logEntries.length > 0
2587
+ ? logEntries[0].payload
2588
+ ? "exists"
2589
+ : "undefined"
2590
+ : "N/A",
2591
+ firstEntryPreview:
2592
+ logEntries.length > 0
2593
+ ? JSON.stringify(logEntries[0], null, 2).slice(0, 500) + "..."
2594
+ : "N/A",
2595
+ },
2596
+ ` 🔍 First entry structure`,
2597
+ );
2598
+
2599
+ for (const entry of logEntries) {
2600
+ // Fix: Access payload from entry.content.payload, not entry.payload
2601
+ const payload = entry.content ? entry.content.payload : entry.payload;
2602
+
2603
+ // Debug logging for first few entries
2604
+ if (
2605
+ payloadPatterns.hasDocumentOps +
2606
+ payloadPatterns.hasKeyValueOps +
2607
+ payloadPatterns.hasSimplePayloads +
2608
+ payloadPatterns.hasCounterOps <
2609
+ 3
2610
+ ) {
2611
+ logger.debug(
2612
+ { payload },
2613
+ ` 🔍 Entry ${payloadPatterns.hasDocumentOps + payloadPatterns.hasKeyValueOps + payloadPatterns.hasSimplePayloads + payloadPatterns.hasCounterOps + 1} payload`,
2614
+ );
2615
+ }
2616
+
2617
+ if (payload && typeof payload === "object") {
2618
+ // Check for document/keyvalue operation patterns
2619
+ if (
2620
+ (payload.op === "PUT" || payload.op === "DEL") &&
2621
+ payload.key !== undefined
2622
+ ) {
2623
+ // Documents typically use _id as primary key or have _id in the value
2624
+ if (
2625
+ payload.key === "_id" ||
2626
+ payload.key.startsWith("_id") ||
2627
+ (payload.op === "PUT" &&
2628
+ payload.value &&
2629
+ typeof payload.value === "object" &&
2630
+ payload.value._id)
2631
+ ) {
2632
+ payloadPatterns.hasDocumentOps++;
2633
+ } else {
2634
+ // Any PUT/DEL with a key that's not document-style is keyvalue
2635
+ payloadPatterns.hasKeyValueOps++;
2636
+ }
2637
+ } else if (
2638
+ payload.op === "COUNTER" ||
2639
+ payload.op === "DEC" ||
2640
+ payload.op === "INC"
2641
+ ) {
2642
+ payloadPatterns.hasCounterOps++;
2643
+ } else if (payload.op === "ADD") {
2644
+ // Explicit ADD operation for events
2645
+ payloadPatterns.hasSimplePayloads++;
2646
+ } else {
2647
+ // Complex object without standard operation structure - likely events
2648
+ payloadPatterns.hasSimplePayloads++;
2649
+ }
2650
+ } else {
2651
+ // Simple payload (string, number, etc.) - likely events
2652
+ payloadPatterns.hasSimplePayloads++;
2653
+ }
2654
+ }
2655
+
2656
+ logger.info(" 📊 Payload analysis:", payloadPatterns);
2657
+
2658
+ // Log some sample payloads for debugging
2659
+ if (logEntries.length > 0) {
2660
+ const sampleEntries = logEntries.slice(0, 3);
2661
+ const samples = sampleEntries.map((entry, i) => {
2662
+ const p = entry.content ? entry.content.payload : entry.payload;
2663
+ if (p && typeof p === "object") {
2664
+ return {
2665
+ index: i + 1,
2666
+ op: p.op,
2667
+ keyType: typeof p.key,
2668
+ hasValue: p.value !== undefined,
2669
+ };
2670
+ } else {
2671
+ return { index: i + 1, simplePayloadType: typeof p };
2672
+ }
2673
+ });
2674
+ logger.debug({ samples }, ` 🔍 Sample payloads`);
2675
+ }
2676
+
2677
+ // Determine type based on majority pattern
2678
+ if (payloadPatterns.hasCounterOps > 0) {
2679
+ logger.info(
2680
+ { counterOps: payloadPatterns.hasCounterOps },
2681
+ ` 🎯 Detected: counter`,
2682
+ );
2683
+ return "counter";
2684
+ } else if (
2685
+ payloadPatterns.hasDocumentOps > payloadPatterns.hasKeyValueOps &&
2686
+ payloadPatterns.hasDocumentOps > payloadPatterns.hasSimplePayloads
2687
+ ) {
2688
+ logger.info(
2689
+ { documentOps: payloadPatterns.hasDocumentOps },
2690
+ ` 🎯 Detected: documents`,
2691
+ );
2692
+ return "documents";
2693
+ } else if (
2694
+ payloadPatterns.hasKeyValueOps > payloadPatterns.hasSimplePayloads
2695
+ ) {
2696
+ logger.info(
2697
+ { keyValueOps: payloadPatterns.hasKeyValueOps },
2698
+ ` 🎯 Detected: keyvalue`,
2699
+ );
2700
+ return "keyvalue";
2701
+ } else {
2702
+ logger.info(
2703
+ { simplePayloads: payloadPatterns.hasSimplePayloads },
2704
+ ` 🎯 Detected: events (fallback)`,
2705
+ );
2706
+ return "events"; // Default fallback
2707
+ }
2708
+ }
2709
+
2710
+ /**
2711
+ * Import a log entry into the appropriate database type
2712
+ *
2713
+ * @param {Object} database - Target OrbitDB database
2714
+ * @param {Object} entry - Log entry to import
2715
+ * @param {string} databaseType - Database type
2716
+ */
2717
+ async function importEntryByType(database, entry, databaseType) {
2718
+ const payload = entry.payload;
2719
+
2720
+ switch (databaseType) {
2721
+ case "events":
2722
+ // Events databases are append-only, only support ADD operations
2723
+ if (payload && payload.op === "ADD") {
2724
+ await database.add(payload.value || payload);
2725
+ } else {
2726
+ // Fallback: treat any payload as an event to add
2727
+ await database.add(payload);
2728
+ }
2729
+ break;
2730
+
2731
+ case "documents":
2732
+ if (payload && payload.op === "PUT" && payload.value) {
2733
+ await database.put(payload.value);
2734
+ } else if (payload && payload.op === "DEL" && payload.key) {
2735
+ // Delete document by key (usually _id)
2736
+ await database.del(payload.key);
2737
+ } else if (payload && typeof payload === "object") {
2738
+ // Fallback: treat as document even without operation structure
2739
+ await database.put(payload);
2740
+ } else {
2741
+ throw new Error("Invalid document payload structure");
2742
+ }
2743
+ break;
2744
+
2745
+ case "keyvalue":
2746
+ if (
2747
+ payload &&
2748
+ payload.op === "PUT" &&
2749
+ payload.key &&
2750
+ payload.value !== undefined
2751
+ ) {
2752
+ await database.put(payload.key, payload.value); // Use put for keyvalue, not set
2753
+ } else if (payload && payload.op === "DEL" && payload.key) {
2754
+ // Delete by key
2755
+ await database.del(payload.key);
2756
+ } else {
2757
+ throw new Error("Invalid keyvalue payload structure");
2758
+ }
2759
+ break;
2760
+
2761
+ case "counter":
2762
+ if (payload && payload.op === "COUNTER") {
2763
+ await database.inc(payload.value || 1);
2764
+ } else if (payload && payload.op === "DEC") {
2765
+ // Decrement operation (negative increment)
2766
+ await database.inc(-(payload.value || 1));
2767
+ } else {
2768
+ // Fallback: try to increment by 1
2769
+ await database.inc(1);
2770
+ }
2771
+ break;
2772
+
2773
+ default:
2774
+ throw new Error(`Unsupported database type: ${databaseType}`);
2775
+ }
2776
+ }
2777
+
2778
+ // Export all utilities
2779
+ export {
2780
+ cleanupOrbitDBDirectories,
2781
+ createHeliaOrbitDB,
2782
+ initializeStorachaClient,
2783
+ initializeStorachaClientWithUCAN,
2784
+ resolveBackend,
2785
+ // Timestamped backup helpers
2786
+ generateBackupPrefix,
2787
+ getBackupFilenames,
2788
+ findLatestBackup,
2789
+ };