@le-space/orbitdb-storage-bridge 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +290 -0
- package/dist/components/StorachaAuth.svelte +801 -0
- package/dist/components/StorachaIntegration.svelte +965 -0
- package/dist/components/WebAuthnDIDProvider.js +562 -0
- package/dist/components/storacha-backup.js +325 -0
- package/dist/components/theme.js +79 -0
- package/lib/backends/aleph-pin.js +161 -0
- package/lib/backends/aleph.js +197 -0
- package/lib/backends/lighthouse.js +325 -0
- package/lib/backends/memory.js +140 -0
- package/lib/backends/pinata.js +330 -0
- package/lib/backends/resolve.js +72 -0
- package/lib/backends/storacha.js +178 -0
- package/lib/backends/types.js +187 -0
- package/lib/backup-car.js +1286 -0
- package/lib/backup-helpers.js +87 -0
- package/lib/backup-metadata.js +40 -0
- package/lib/backup.js +249 -0
- package/lib/block-bytes.js +39 -0
- package/lib/car-storage.js +342 -0
- package/lib/courier-sync.js +886 -0
- package/lib/dehydrate.js +146 -0
- package/lib/extract-blocks.js +258 -0
- package/lib/gateway-fetch.js +116 -0
- package/lib/ipns-helpers.js +324 -0
- package/lib/logger.js +128 -0
- package/lib/memory-courier.js +105 -0
- package/lib/orbitdb-storacha-bridge.js +2789 -0
- package/lib/pointer-ipns.js +234 -0
- package/lib/restore-cid.js +302 -0
- package/lib/ucan-bridge.js +921 -0
- package/lib/utils.js +316 -0
- package/package.json +172 -0
|
@@ -0,0 +1,2789 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OrbitDB Storacha Bridge - Main Library
|
|
3
|
+
*
|
|
4
|
+
* Provides complete OrbitDB database backup and restoration via Storacha/Filecoin
|
|
5
|
+
* with 100% hash preservation and identity recovery.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import * as Client from "@storacha/client";
|
|
9
|
+
import { StoreMemory } from "@storacha/client/stores/memory";
|
|
10
|
+
import { Signer } from "@storacha/client/principal/ed25519";
|
|
11
|
+
import * as Proof from "@storacha/client/proof";
|
|
12
|
+
import { BackendError } from "./backends/types.js";
|
|
13
|
+
import { CID } from "multiformats/cid";
|
|
14
|
+
import * as Block from "multiformats/block";
|
|
15
|
+
import * as dagCbor from "@ipld/dag-cbor";
|
|
16
|
+
import { sha256 } from "multiformats/hashes/sha2";
|
|
17
|
+
import { bases } from "multiformats/basics";
|
|
18
|
+
import { EventEmitter } from "events";
|
|
19
|
+
import { createHeliaOrbitDB, cleanupOrbitDBDirectories } from "./utils.js";
|
|
20
|
+
import {
|
|
21
|
+
generateBackupPrefix,
|
|
22
|
+
getBackupFilenames,
|
|
23
|
+
findLatestBackup,
|
|
24
|
+
} from "./backup-helpers.js";
|
|
25
|
+
import { logger, createLogger } from "./logger.js";
|
|
26
|
+
import { unixfs } from "@helia/unixfs";
|
|
27
|
+
import { readBlockBytes } from "./block-bytes.js";
|
|
28
|
+
// Moved out of this file so that backing up does not drag the Storacha client
|
|
29
|
+
// into a browser bundle; re-exported here, because this is where callers of
|
|
30
|
+
// every vintage look for it.
|
|
31
|
+
import { extractDatabaseBlocks } from "./extract-blocks.js";
|
|
32
|
+
import { resolveBackend } from "./backends/resolve.js";
|
|
33
|
+
|
|
34
|
+
export { extractDatabaseBlocks };
|
|
35
|
+
|
|
36
|
+
const HELIA_LOGGER = createLogger("helia");
|
|
37
|
+
const HELIA_COLOR = "\x1b[95m";
|
|
38
|
+
const COLOR_RESET = "\x1b[0m";
|
|
39
|
+
|
|
40
|
+
function logHeliaActivity(message) {
|
|
41
|
+
HELIA_LOGGER(`${HELIA_COLOR}${message}${COLOR_RESET}`);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Default configuration options
|
|
46
|
+
*/
|
|
47
|
+
const DEFAULT_OPTIONS = {
|
|
48
|
+
// Core configuration
|
|
49
|
+
timeout: 30000, // Timeout in milliseconds
|
|
50
|
+
gateway: "https://dweb.link", // IPFS gateway URL (w3s.link only redirects here now)
|
|
51
|
+
verbose: false, // Enable verbose debug logging
|
|
52
|
+
|
|
53
|
+
// Network download options
|
|
54
|
+
useIPFSNetwork: true, // Enable network downloads by default
|
|
55
|
+
gatewayFallback: true, // Fallback to gateway if network fails
|
|
56
|
+
|
|
57
|
+
// Performance tuning (used in specific functions)
|
|
58
|
+
batchSize: 10, // Batch size for upload/download operations (removeLayerFiles)
|
|
59
|
+
maxConcurrency: 3, // Maximum concurrent batches (uploadBlocks)
|
|
60
|
+
uploadBatchSize: 10, // Upload batch size for uploadBlocks
|
|
61
|
+
uploadMaxConcurrency: 3, // Upload batch concurrency for uploadBlocks
|
|
62
|
+
|
|
63
|
+
// Credentials (required - can also be set via environment variables)
|
|
64
|
+
storachaKey: undefined, // Storacha private key (required)
|
|
65
|
+
storachaProof: undefined, // Storacha proof (required)
|
|
66
|
+
|
|
67
|
+
// Restore options
|
|
68
|
+
fallbackDatabaseName: undefined, // Custom name for fallback reconstruction
|
|
69
|
+
forceFallback: false, // Force fallback reconstruction mode
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Convert Storacha CID format to OrbitDB CID format
|
|
74
|
+
*
|
|
75
|
+
* @param {string} storachaCID - Storacha CID (bafkre... format)
|
|
76
|
+
* @returns {string} - OrbitDB CID (zdpu... format)
|
|
77
|
+
*/
|
|
78
|
+
export function convertStorachaCIDToOrbitDB(storachaCID) {
|
|
79
|
+
const storachaParsed = CID.parse(storachaCID);
|
|
80
|
+
|
|
81
|
+
// Create CIDv1 with dag-cbor codec using the same multihash
|
|
82
|
+
const orbitdbCID = CID.createV1(0x71, storachaParsed.multihash); // 0x71 = dag-cbor
|
|
83
|
+
|
|
84
|
+
// Return in base58btc format (zdpu prefix)
|
|
85
|
+
return orbitdbCID.toString(bases.base58btc);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Extract manifest CID from OrbitDB address
|
|
90
|
+
*
|
|
91
|
+
* @param {string} databaseAddress - OrbitDB address (/orbitdb/zdpu...)
|
|
92
|
+
* @returns {string} - Manifest CID
|
|
93
|
+
*/
|
|
94
|
+
export function extractManifestCID(databaseAddress) {
|
|
95
|
+
return databaseAddress.split("/").pop();
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Initialize Storacha client with credentials
|
|
101
|
+
*
|
|
102
|
+
* @param {string} storachaKey - Storacha private key
|
|
103
|
+
* @param {string} storachaProof - Storacha proof
|
|
104
|
+
* @param {Object} [serviceConf] - Optional service configuration for custom endpoints
|
|
105
|
+
* @param {string|URL} [receiptsEndpoint] - Optional receipts endpoint override
|
|
106
|
+
* @returns {Promise<Object>} - Initialized Storacha client
|
|
107
|
+
*/
|
|
108
|
+
async function initializeStorachaClient(
|
|
109
|
+
storachaKey,
|
|
110
|
+
storachaProof,
|
|
111
|
+
serviceConf,
|
|
112
|
+
receiptsEndpoint,
|
|
113
|
+
) {
|
|
114
|
+
const principal = Signer.parse(storachaKey);
|
|
115
|
+
const store = new StoreMemory();
|
|
116
|
+
const clientOptions = { principal, store };
|
|
117
|
+
|
|
118
|
+
if (serviceConf) {
|
|
119
|
+
clientOptions.serviceConf = serviceConf;
|
|
120
|
+
}
|
|
121
|
+
if (receiptsEndpoint) {
|
|
122
|
+
clientOptions.receiptsEndpoint = receiptsEndpoint;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const client = await Client.create(clientOptions);
|
|
126
|
+
|
|
127
|
+
const proof = await Proof.parse(storachaProof);
|
|
128
|
+
const space = await client.addSpace(proof);
|
|
129
|
+
await client.setCurrentSpace(space.did());
|
|
130
|
+
|
|
131
|
+
return client;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Initialize Storacha client with UCAN authentication
|
|
136
|
+
*
|
|
137
|
+
* @param {Object} options - UCAN options
|
|
138
|
+
* @param {Object} options.client - Pre-initialized w3up client
|
|
139
|
+
* @param {string} options.spaceDID - Target space DID
|
|
140
|
+
* @returns {Promise<Object>} - Initialized Storacha client
|
|
141
|
+
*/
|
|
142
|
+
async function initializeStorachaClientWithUCAN(options) {
|
|
143
|
+
logger.info("🔐 Initializing Storacha client with UCAN authentication...");
|
|
144
|
+
|
|
145
|
+
if (!options.client) {
|
|
146
|
+
throw new Error("UCAN client is required");
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// If spaceDID is provided, set it as current space
|
|
150
|
+
if (options.spaceDID) {
|
|
151
|
+
logger.info(` 🚀 Setting current space: ${options.spaceDID}`);
|
|
152
|
+
await options.client.setCurrentSpace(options.spaceDID);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
logger.info("✅ UCAN Storacha client initialized");
|
|
156
|
+
logger.info(` 🤖 Agent: ${options.client.agent.did()}`);
|
|
157
|
+
logger.info(` 🚀 Current space: ${options.client.currentSpace()?.did()}`);
|
|
158
|
+
|
|
159
|
+
return options.client;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Upload blocks to Storacha with parallel batch processing and progress events
|
|
165
|
+
*
|
|
166
|
+
* @param {Map} blocks - Map of blocks to upload
|
|
167
|
+
* @param {import('./backends/types.js').StorageBackend} backend - storage backend
|
|
168
|
+
* @param {number} batchSize - Number of files to upload in parallel (default: 10)
|
|
169
|
+
* @param {number} maxConcurrency - Maximum concurrent batches (default: 3)
|
|
170
|
+
* @param {EventEmitter} eventEmitter - Optional event emitter for progress updates
|
|
171
|
+
* @returns {Promise<Object>} - Upload results and CID mappings
|
|
172
|
+
*/
|
|
173
|
+
async function uploadBlocks(
|
|
174
|
+
blocks,
|
|
175
|
+
backend,
|
|
176
|
+
batchSize = 10,
|
|
177
|
+
maxConcurrency = 3,
|
|
178
|
+
eventEmitter = null,
|
|
179
|
+
) {
|
|
180
|
+
logger.info(
|
|
181
|
+
`📤 Uploading ${blocks.size} blocks to ${backend.name} in batches of ${batchSize}...`,
|
|
182
|
+
);
|
|
183
|
+
|
|
184
|
+
const uploadResults = [];
|
|
185
|
+
const cidMappings = new Map();
|
|
186
|
+
const blocksArray = Array.from(blocks.entries());
|
|
187
|
+
const totalBlocks = blocks.size;
|
|
188
|
+
let completedBlocks = 0;
|
|
189
|
+
|
|
190
|
+
// Emit initial progress
|
|
191
|
+
if (eventEmitter) {
|
|
192
|
+
eventEmitter.emit("uploadProgress", {
|
|
193
|
+
type: "upload",
|
|
194
|
+
current: 0,
|
|
195
|
+
total: totalBlocks,
|
|
196
|
+
percentage: 0,
|
|
197
|
+
status: "starting",
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// Helper function to upload a single block
|
|
202
|
+
const uploadSingleBlock = async ([hash, blockData]) => {
|
|
203
|
+
try {
|
|
204
|
+
logger.info(
|
|
205
|
+
` 📤 Uploading block ${hash} (${blockData.bytes.length} bytes)...`,
|
|
206
|
+
);
|
|
207
|
+
|
|
208
|
+
const handle = await backend.putBlob(blockData.bytes, { name: hash });
|
|
209
|
+
const uploadedCID = handle.cid || handle.id;
|
|
210
|
+
|
|
211
|
+
logger.info(` ✅ Uploaded: ${hash} → ${uploadedCID}`);
|
|
212
|
+
|
|
213
|
+
// Update progress
|
|
214
|
+
completedBlocks++;
|
|
215
|
+
if (eventEmitter) {
|
|
216
|
+
eventEmitter.emit("uploadProgress", {
|
|
217
|
+
type: "upload",
|
|
218
|
+
current: completedBlocks,
|
|
219
|
+
total: totalBlocks,
|
|
220
|
+
percentage: Math.round((completedBlocks / totalBlocks) * 100),
|
|
221
|
+
status: "uploading",
|
|
222
|
+
currentBlock: {
|
|
223
|
+
hash,
|
|
224
|
+
uploadedCID,
|
|
225
|
+
size: blockData.bytes.length,
|
|
226
|
+
},
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
return {
|
|
231
|
+
originalHash: hash,
|
|
232
|
+
uploadedCID,
|
|
233
|
+
size: blockData.bytes.length,
|
|
234
|
+
};
|
|
235
|
+
} catch (error) {
|
|
236
|
+
// Enhanced error handling for rate limits and service errors
|
|
237
|
+
let errorMessage = error.message;
|
|
238
|
+
let rateLimitInfo = "";
|
|
239
|
+
|
|
240
|
+
// Check if it's a rate limit or service error with retry info
|
|
241
|
+
if (error.status === 503 || error.code === 503) {
|
|
242
|
+
const retryAfter =
|
|
243
|
+
error.headers?.get?.("Retry-After") || error["retry-after"];
|
|
244
|
+
const rateLimitReset =
|
|
245
|
+
error.headers?.get?.("X-RateLimit-Reset") ||
|
|
246
|
+
error["x-ratelimit-reset"];
|
|
247
|
+
const rateLimitRemaining =
|
|
248
|
+
error.headers?.get?.("X-RateLimit-Remaining") ||
|
|
249
|
+
error["x-ratelimit-remaining"];
|
|
250
|
+
|
|
251
|
+
rateLimitInfo += `\n 📊 Service Status: 503 Service Unavailable (rate limited or maintenance)`;
|
|
252
|
+
|
|
253
|
+
if (retryAfter) {
|
|
254
|
+
const retrySeconds = parseInt(retryAfter);
|
|
255
|
+
let retryTime = retryAfter;
|
|
256
|
+
if (!isNaN(retrySeconds)) {
|
|
257
|
+
const futureTime = new Date(Date.now() + retrySeconds * 1000);
|
|
258
|
+
retryTime = `${retrySeconds} seconds (available at ${futureTime.toLocaleTimeString()})`;
|
|
259
|
+
} else {
|
|
260
|
+
// Try to parse as HTTP date
|
|
261
|
+
const retryDate = new Date(retryAfter);
|
|
262
|
+
if (!isNaN(retryDate.getTime())) {
|
|
263
|
+
const waitMs = Math.max(0, retryDate.getTime() - Date.now());
|
|
264
|
+
const waitSecs = Math.ceil(waitMs / 1000);
|
|
265
|
+
retryTime = `${waitSecs} seconds (available at ${retryDate.toLocaleTimeString()})`;
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
rateLimitInfo += `\n ⏱️ Retry-After: ${retryTime}`;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
if (rateLimitReset) {
|
|
272
|
+
const resetTime = parseInt(rateLimitReset);
|
|
273
|
+
if (!isNaN(resetTime)) {
|
|
274
|
+
const resetDate = new Date(resetTime * 1000);
|
|
275
|
+
const waitMs = Math.max(0, resetDate.getTime() - Date.now());
|
|
276
|
+
const waitSecs = Math.ceil(waitMs / 1000);
|
|
277
|
+
rateLimitInfo += `\n 🔄 Rate Limit Reset: ${resetDate.toLocaleTimeString()} (in ${waitSecs}s)`;
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
if (rateLimitRemaining) {
|
|
282
|
+
rateLimitInfo += `\n 📈 Requests Remaining: ${rateLimitRemaining}`;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
if (!retryAfter && !rateLimitReset) {
|
|
286
|
+
rateLimitInfo += `\n 💡 No retry timing provided - service may be temporarily unavailable`;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
errorMessage = `${errorMessage}${rateLimitInfo}`;
|
|
290
|
+
} else if (error.status || error.code) {
|
|
291
|
+
// Other HTTP errors
|
|
292
|
+
const statusCode = error.status || error.code;
|
|
293
|
+
errorMessage = `HTTP ${statusCode}: ${errorMessage}`;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
logger.error(` ❌ Failed to upload block ${hash}: ${errorMessage}`);
|
|
297
|
+
|
|
298
|
+
// Update progress even for failed uploads
|
|
299
|
+
completedBlocks++;
|
|
300
|
+
if (eventEmitter) {
|
|
301
|
+
eventEmitter.emit("uploadProgress", {
|
|
302
|
+
type: "upload",
|
|
303
|
+
current: completedBlocks,
|
|
304
|
+
total: totalBlocks,
|
|
305
|
+
percentage: Math.round((completedBlocks / totalBlocks) * 100),
|
|
306
|
+
status: "uploading",
|
|
307
|
+
error: {
|
|
308
|
+
hash,
|
|
309
|
+
message: error.message,
|
|
310
|
+
},
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
return {
|
|
315
|
+
originalHash: hash,
|
|
316
|
+
error: error.message,
|
|
317
|
+
size: blockData.bytes.length,
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
};
|
|
321
|
+
|
|
322
|
+
// Process blocks in batches with controlled concurrency
|
|
323
|
+
for (let i = 0; i < blocksArray.length; i += batchSize * maxConcurrency) {
|
|
324
|
+
const megaBatch = blocksArray.slice(i, i + batchSize * maxConcurrency);
|
|
325
|
+
const batches = [];
|
|
326
|
+
|
|
327
|
+
// Split mega-batch into smaller batches
|
|
328
|
+
for (let j = 0; j < megaBatch.length; j += batchSize) {
|
|
329
|
+
const batch = megaBatch.slice(j, j + batchSize);
|
|
330
|
+
batches.push(batch);
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
logger.info(
|
|
334
|
+
` 🔄 Processing ${batches.length} concurrent batches (${megaBatch.length} blocks)...`,
|
|
335
|
+
);
|
|
336
|
+
|
|
337
|
+
// Process all batches in this mega-batch concurrently
|
|
338
|
+
const batchPromises = batches.map(async (batch, batchIndex) => {
|
|
339
|
+
logger.info(
|
|
340
|
+
` 📦 Batch ${batchIndex + 1}/${batches.length}: ${batch.length} blocks`,
|
|
341
|
+
);
|
|
342
|
+
|
|
343
|
+
// Upload all blocks in this batch in parallel
|
|
344
|
+
const batchResults = await Promise.allSettled(
|
|
345
|
+
batch.map(uploadSingleBlock),
|
|
346
|
+
);
|
|
347
|
+
|
|
348
|
+
return batchResults.map((result) =>
|
|
349
|
+
result.status === "fulfilled"
|
|
350
|
+
? result.value
|
|
351
|
+
: {
|
|
352
|
+
originalHash: "unknown",
|
|
353
|
+
error: result.reason?.message || "Unknown error",
|
|
354
|
+
size: 0,
|
|
355
|
+
},
|
|
356
|
+
);
|
|
357
|
+
});
|
|
358
|
+
|
|
359
|
+
// Wait for all batches to complete
|
|
360
|
+
const batchResults = await Promise.all(batchPromises);
|
|
361
|
+
|
|
362
|
+
// Flatten and process results
|
|
363
|
+
for (const batchResult of batchResults) {
|
|
364
|
+
for (const result of batchResult) {
|
|
365
|
+
uploadResults.push(result);
|
|
366
|
+
if (result.uploadedCID) {
|
|
367
|
+
cidMappings.set(result.originalHash, result.uploadedCID);
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
const successful = uploadResults.filter((r) => r.uploadedCID);
|
|
374
|
+
const failed = uploadResults.filter((r) => r.error);
|
|
375
|
+
|
|
376
|
+
logger.info(` 📊 Upload summary:`);
|
|
377
|
+
logger.info(` Total blocks: ${blocks.size}`);
|
|
378
|
+
logger.info(` Successful: ${successful.length}`);
|
|
379
|
+
logger.info(` Failed: ${failed.length}`);
|
|
380
|
+
logger.info(` Batch size: ${batchSize}`);
|
|
381
|
+
logger.info(` Max concurrency: ${maxConcurrency}`);
|
|
382
|
+
|
|
383
|
+
// Emit completion
|
|
384
|
+
if (eventEmitter) {
|
|
385
|
+
eventEmitter.emit("uploadProgress", {
|
|
386
|
+
type: "upload",
|
|
387
|
+
current: totalBlocks,
|
|
388
|
+
total: totalBlocks,
|
|
389
|
+
percentage: 100,
|
|
390
|
+
status: "completed",
|
|
391
|
+
summary: {
|
|
392
|
+
successful: successful.length,
|
|
393
|
+
failed: failed.length,
|
|
394
|
+
},
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
return { uploadResults, successful, failed, cidMappings };
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
// executeW3Command has been removed as it was deprecated and used Node.js-specific child_process
|
|
402
|
+
// Use SDK equivalents like listStorachaSpaceFiles() instead
|
|
403
|
+
|
|
404
|
+
/**
|
|
405
|
+
* List all files in Storacha space using SDK (IMPROVED: Direct API access)
|
|
406
|
+
*
|
|
407
|
+
* @param {Object} options - Configuration options
|
|
408
|
+
* @param {string} [options.storachaKey] - Storacha private key (defaults to env)
|
|
409
|
+
* @param {string} [options.storachaProof] - Storacha proof (defaults to env)
|
|
410
|
+
* @param {number} [options.size] - Maximum number of items to retrieve (default: 1000000)
|
|
411
|
+
* @param {string} [options.cursor] - Pagination cursor
|
|
412
|
+
* @returns {Promise<Array>} - Space files with metadata
|
|
413
|
+
*/
|
|
414
|
+
export async function listStorachaSpaceFiles(options = {}) {
|
|
415
|
+
const _config = { ...DEFAULT_OPTIONS, ...options };
|
|
416
|
+
logger.info("📋 Listing files in Storacha space using SDK...");
|
|
417
|
+
|
|
418
|
+
try {
|
|
419
|
+
// Initialize client - support both credential and UCAN authentication
|
|
420
|
+
const backend = await resolveBackend(options);
|
|
421
|
+
|
|
422
|
+
// Prepare list options
|
|
423
|
+
const listOptions = {};
|
|
424
|
+
if (options.size) {
|
|
425
|
+
listOptions.size = parseInt(String(options.size));
|
|
426
|
+
} else {
|
|
427
|
+
listOptions.size = 1000000; // Default to get ALL files
|
|
428
|
+
}
|
|
429
|
+
if (options.cursor) {
|
|
430
|
+
listOptions.cursor = options.cursor;
|
|
431
|
+
}
|
|
432
|
+
if (options.pre) {
|
|
433
|
+
listOptions.pre = options.pre;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
// List through the backend contract, so this works for any driver
|
|
437
|
+
const entries = await backend.list(listOptions);
|
|
438
|
+
|
|
439
|
+
logger.info(` ✅ Found ${entries.length} uploads in space`);
|
|
440
|
+
|
|
441
|
+
// Convert to the format we expect, with enhanced metadata
|
|
442
|
+
const spaceFiles = entries.map((entry) => ({
|
|
443
|
+
root: entry.id,
|
|
444
|
+
uploaded: entry.insertedAt ? new Date(entry.insertedAt) : new Date(),
|
|
445
|
+
size: entry.size || "unknown",
|
|
446
|
+
shards: entry.raw?.shards?.length || 0,
|
|
447
|
+
insertedAt: entry.insertedAt,
|
|
448
|
+
updatedAt: entry.updatedAt,
|
|
449
|
+
}));
|
|
450
|
+
|
|
451
|
+
return spaceFiles;
|
|
452
|
+
} catch (error) {
|
|
453
|
+
logger.error(" ❌ SDK listing error:", error.message);
|
|
454
|
+
throw error;
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
/**
|
|
459
|
+
* List files in a specific Storacha layer using SDK
|
|
460
|
+
*
|
|
461
|
+
* @param {string} layer - Layer to list ('upload', 'blob')
|
|
462
|
+
* @param {Object} options - Configuration options
|
|
463
|
+
* @returns {Promise<Array>} - Layer files
|
|
464
|
+
*/
|
|
465
|
+
export async function listLayerFiles(layer, options = {}) {
|
|
466
|
+
const _config = { ...DEFAULT_OPTIONS, ...options };
|
|
467
|
+
|
|
468
|
+
try {
|
|
469
|
+
// Initialize client - support both credential and UCAN authentication
|
|
470
|
+
const backend = await resolveBackend(options);
|
|
471
|
+
const client = backend.client;
|
|
472
|
+
|
|
473
|
+
// Prepare list options
|
|
474
|
+
const listOptions = {};
|
|
475
|
+
if (options.size) {
|
|
476
|
+
listOptions.size = parseInt(String(options.size));
|
|
477
|
+
} else {
|
|
478
|
+
listOptions.size = 1000000; // Default to get ALL files
|
|
479
|
+
}
|
|
480
|
+
if (options.cursor) {
|
|
481
|
+
listOptions.cursor = options.cursor;
|
|
482
|
+
}
|
|
483
|
+
if (options.pre) {
|
|
484
|
+
listOptions.pre = options.pre;
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
let result;
|
|
488
|
+
switch (layer) {
|
|
489
|
+
case "upload": {
|
|
490
|
+
result = await client.capability.upload.list(listOptions);
|
|
491
|
+
return result.results.map((upload) => upload.root.toString());
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
case "blob": {
|
|
495
|
+
result = await client.capability.blob.list(listOptions);
|
|
496
|
+
return result.results.map((blob) => {
|
|
497
|
+
// The blob structure is: { blob: { size: number, digest: Uint8Array }, cause: CID, insertedAt: string }
|
|
498
|
+
if (blob.blob && blob.blob.digest) {
|
|
499
|
+
// blob.blob.digest is already a Uint8Array, encode it to base64
|
|
500
|
+
const encodedDigest = Buffer.from(blob.blob.digest).toString(
|
|
501
|
+
"base64",
|
|
502
|
+
);
|
|
503
|
+
return encodedDigest;
|
|
504
|
+
} else {
|
|
505
|
+
logger.warn({ blob }, ` ⚠️ Unexpected blob structure`);
|
|
506
|
+
return blob.toString(); // Fallback
|
|
507
|
+
}
|
|
508
|
+
});
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
default:
|
|
512
|
+
throw new Error(`Unknown layer: ${layer}. Use 'upload' or 'blob'.`);
|
|
513
|
+
}
|
|
514
|
+
} catch (error) {
|
|
515
|
+
logger.warn(` ⚠️ Failed to list ${layer}: ${error.message}`);
|
|
516
|
+
return [];
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
/**
|
|
521
|
+
* Remove files from a specific layer in batches
|
|
522
|
+
*
|
|
523
|
+
* @param {string} layer - Layer to clear ('upload', 'blob')
|
|
524
|
+
* @param {Array} cids - Array of CIDs to remove
|
|
525
|
+
* @param {Object} options - Configuration options
|
|
526
|
+
* @returns {Promise<Object>} - Removal results
|
|
527
|
+
*/
|
|
528
|
+
export async function removeLayerFiles(layer, cids, options = {}) {
|
|
529
|
+
if (cids.length === 0) {
|
|
530
|
+
logger.info(` ✓ ${layer}: No files to remove`);
|
|
531
|
+
return { removed: 0, failed: 0 };
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
const batchSize = options.batchSize || 10;
|
|
535
|
+
logger.info(
|
|
536
|
+
` 🗑️ Removing ${cids.length} files from ${layer} layer using SDK (batch size: ${batchSize})...`,
|
|
537
|
+
);
|
|
538
|
+
|
|
539
|
+
try {
|
|
540
|
+
// Initialize client - support both credential and UCAN authentication
|
|
541
|
+
const backend = await resolveBackend(options);
|
|
542
|
+
const client = backend.client;
|
|
543
|
+
|
|
544
|
+
let removed = 0;
|
|
545
|
+
let failed = 0;
|
|
546
|
+
|
|
547
|
+
// Process CIDs in batches
|
|
548
|
+
for (let i = 0; i < cids.length; i += batchSize) {
|
|
549
|
+
const batch = cids.slice(i, i + batchSize);
|
|
550
|
+
const batchNum = Math.floor(i / batchSize) + 1;
|
|
551
|
+
const totalBatches = Math.ceil(cids.length / batchSize);
|
|
552
|
+
|
|
553
|
+
logger.info(
|
|
554
|
+
` 📦 Processing batch ${batchNum}/${totalBatches} (${batch.length} files)...`,
|
|
555
|
+
);
|
|
556
|
+
|
|
557
|
+
// Process batch in parallel
|
|
558
|
+
const batchPromises = batch.map(async (cid) => {
|
|
559
|
+
try {
|
|
560
|
+
switch (layer) {
|
|
561
|
+
case "upload": {
|
|
562
|
+
await client.capability.upload.remove(CID.parse(cid));
|
|
563
|
+
break;
|
|
564
|
+
}
|
|
565
|
+
case "blob": {
|
|
566
|
+
// For blob, we need to parse the digest format
|
|
567
|
+
// cid should be a base64 encoded string, convert it back to Uint8Array
|
|
568
|
+
let digest;
|
|
569
|
+
if (typeof cid === "string") {
|
|
570
|
+
// Convert to Buffer first, then to Uint8Array (API expects Uint8Array, not Buffer)
|
|
571
|
+
const buffer = Buffer.from(cid, "base64");
|
|
572
|
+
digest = new Uint8Array(buffer);
|
|
573
|
+
} else {
|
|
574
|
+
logger.error(
|
|
575
|
+
{ cidType: typeof cid, cid },
|
|
576
|
+
` ❌ Invalid blob CID format`,
|
|
577
|
+
);
|
|
578
|
+
throw new Error(
|
|
579
|
+
`Invalid blob CID format: expected string, got ${typeof cid}`,
|
|
580
|
+
);
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
// Remove the blob using the correct API format: { bytes: Uint8Array }
|
|
584
|
+
await client.capability.blob.remove({ bytes: digest });
|
|
585
|
+
break;
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
default:
|
|
589
|
+
throw new Error(
|
|
590
|
+
`Unknown layer: ${layer}. Use 'upload' or 'blob'.`,
|
|
591
|
+
);
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
return { success: true, cid };
|
|
595
|
+
} catch (error) {
|
|
596
|
+
return { success: false, cid, error: error.message };
|
|
597
|
+
}
|
|
598
|
+
});
|
|
599
|
+
|
|
600
|
+
// Wait for all deletions in this batch to complete
|
|
601
|
+
const batchResults = await Promise.allSettled(batchPromises);
|
|
602
|
+
|
|
603
|
+
// Process results
|
|
604
|
+
for (const result of batchResults) {
|
|
605
|
+
if (result.status === "fulfilled") {
|
|
606
|
+
if (result.value.success) {
|
|
607
|
+
removed++;
|
|
608
|
+
logger.info(` ✓ Removed: ${result.value.cid}`);
|
|
609
|
+
} else {
|
|
610
|
+
failed++;
|
|
611
|
+
logger.info(
|
|
612
|
+
` ❌ Failed to remove ${result.value.cid}: ${result.value.error}`,
|
|
613
|
+
);
|
|
614
|
+
}
|
|
615
|
+
} else {
|
|
616
|
+
failed++;
|
|
617
|
+
logger.info(` ❌ Batch operation failed: ${result.reason}`);
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
// Add a small delay between batches to avoid overwhelming the API
|
|
622
|
+
if (i + batchSize < cids.length) {
|
|
623
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
logger.info(` 📊 ${layer}: ${removed} removed, ${failed} failed`);
|
|
628
|
+
return { removed, failed };
|
|
629
|
+
} catch (error) {
|
|
630
|
+
logger.error(` ❌ Error removing from ${layer}: ${error.message}`);
|
|
631
|
+
return { removed: 0, failed: cids.length };
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
/**
|
|
636
|
+
* Clear all files from Storacha space using SDK
|
|
637
|
+
*
|
|
638
|
+
* @param {Object} options - Configuration options
|
|
639
|
+
* @returns {Promise<Object>} - Clearing results
|
|
640
|
+
*/
|
|
641
|
+
export async function clearStorachaSpace(options = {}) {
|
|
642
|
+
logger.info("🧹 Clearing Storacha space using SDK...");
|
|
643
|
+
logger.info("=".repeat(50));
|
|
644
|
+
|
|
645
|
+
// FIXED: Remove 'store' layer as it's no longer available in the API
|
|
646
|
+
const layers = ["upload", "blob"]; // Removed "store"
|
|
647
|
+
const summary = {
|
|
648
|
+
totalFiles: 0,
|
|
649
|
+
totalRemoved: 0,
|
|
650
|
+
totalFailed: 0,
|
|
651
|
+
byLayer: {},
|
|
652
|
+
};
|
|
653
|
+
|
|
654
|
+
for (const layer of layers) {
|
|
655
|
+
logger.info(`\n📋 Checking ${layer} layer...`);
|
|
656
|
+
const cids = await listLayerFiles(layer, options);
|
|
657
|
+
summary.totalFiles += cids.length;
|
|
658
|
+
|
|
659
|
+
if (cids.length > 0) {
|
|
660
|
+
const result = await removeLayerFiles(layer, cids, options);
|
|
661
|
+
summary.totalRemoved += result.removed;
|
|
662
|
+
summary.totalFailed += result.failed;
|
|
663
|
+
summary.byLayer[layer] = result;
|
|
664
|
+
} else {
|
|
665
|
+
summary.byLayer[layer] = { removed: 0, failed: 0 };
|
|
666
|
+
logger.info(` ✓ ${layer}: Already empty`);
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
logger.info("\n" + "=".repeat(50));
|
|
671
|
+
logger.info("🧹 SPACE CLEARING RESULTS (SDK)");
|
|
672
|
+
logger.info("=".repeat(50));
|
|
673
|
+
logger.info(`📊 Total files found: ${summary.totalFiles}`);
|
|
674
|
+
logger.info(`✅ Total files removed: ${summary.totalRemoved}`);
|
|
675
|
+
logger.info(`❌ Total failures: ${summary.totalFailed}`);
|
|
676
|
+
|
|
677
|
+
for (const [layer, stats] of Object.entries(summary.byLayer)) {
|
|
678
|
+
logger.info(
|
|
679
|
+
` ${layer}: ${stats.removed} removed, ${stats.failed} failed`,
|
|
680
|
+
);
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
const success =
|
|
684
|
+
summary.totalFailed === 0 && summary.totalFiles === summary.totalRemoved;
|
|
685
|
+
logger.info(
|
|
686
|
+
`\n${success ? "✅" : "⚠️"} Space clearing: ${success ? "COMPLETE" : "PARTIAL"}`,
|
|
687
|
+
);
|
|
688
|
+
|
|
689
|
+
return {
|
|
690
|
+
success,
|
|
691
|
+
...summary,
|
|
692
|
+
};
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
/**
|
|
696
|
+
* Download content from IPFS network using Helia with UnixFS
|
|
697
|
+
*
|
|
698
|
+
* Uses unixfs.cat() which handles DAG traversal automatically, making it suitable
|
|
699
|
+
* for both single blocks and chunked files (like large CAR files).
|
|
700
|
+
*
|
|
701
|
+
* @param {string} cid - CID to download
|
|
702
|
+
* @param {Object} helia - Helia IPFS instance
|
|
703
|
+
* @param {Object} options - Configuration options
|
|
704
|
+
* @param {Object} [options.unixfs] - Optional unixfs instance (will be created if not provided)
|
|
705
|
+
* @returns {Promise<Uint8Array>} - Complete content bytes
|
|
706
|
+
*/
|
|
707
|
+
export async function downloadBlockFromIPFSNetwork(cid, helia, options = {}) {
|
|
708
|
+
const config = { ...DEFAULT_OPTIONS, ...options };
|
|
709
|
+
|
|
710
|
+
try {
|
|
711
|
+
if (typeof config.reconnectToInMemoryHelia === "function") {
|
|
712
|
+
try {
|
|
713
|
+
await config.reconnectToInMemoryHelia();
|
|
714
|
+
} catch (error) {
|
|
715
|
+
logger.warn(
|
|
716
|
+
` ⚠️ In-memory Helia reconnect failed: ${error?.message ?? error}`,
|
|
717
|
+
);
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
// Check if helia instance is available and ready
|
|
722
|
+
if (!helia) {
|
|
723
|
+
throw new Error("Helia instance is not available");
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
const parsedCID = typeof cid === "string" ? CID.parse(cid) : cid;
|
|
727
|
+
logHeliaActivity(`📥 helia unixfs.cat ${parsedCID.toString()}`);
|
|
728
|
+
|
|
729
|
+
// Create unixfs instance from helia (or use provided one)
|
|
730
|
+
const fs = options.unixfs || unixfs(helia);
|
|
731
|
+
|
|
732
|
+
// Use unixfs.cat() to get complete content (handles DAGs automatically)
|
|
733
|
+
// This uses Bitswap protocol to fetch from IPFS network and traverses DAGs
|
|
734
|
+
const chunks = [];
|
|
735
|
+
let timeoutId;
|
|
736
|
+
|
|
737
|
+
// Create a timeout promise
|
|
738
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
739
|
+
timeoutId = setTimeout(
|
|
740
|
+
() => reject(new Error("Network download timeout")),
|
|
741
|
+
config.timeout,
|
|
742
|
+
);
|
|
743
|
+
});
|
|
744
|
+
|
|
745
|
+
// Race between cat() and timeout
|
|
746
|
+
try {
|
|
747
|
+
const catPromise = (async () => {
|
|
748
|
+
for await (const chunk of fs.cat(parsedCID)) {
|
|
749
|
+
chunks.push(chunk);
|
|
750
|
+
}
|
|
751
|
+
})();
|
|
752
|
+
|
|
753
|
+
await Promise.race([catPromise, timeoutPromise]);
|
|
754
|
+
} finally {
|
|
755
|
+
// Clear timeout if cat() completes first
|
|
756
|
+
if (timeoutId) {
|
|
757
|
+
clearTimeout(timeoutId);
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
// Concatenate all chunks into a single Uint8Array
|
|
762
|
+
const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
|
|
763
|
+
const bytes = new Uint8Array(totalLength);
|
|
764
|
+
let offset = 0;
|
|
765
|
+
for (const chunk of chunks) {
|
|
766
|
+
bytes.set(chunk, offset);
|
|
767
|
+
offset += chunk.length;
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
logger.info(
|
|
771
|
+
` ✅ Downloaded ${bytes.length} bytes from IPFS network (CID: ${cid.substring(0, 12)}...)`,
|
|
772
|
+
);
|
|
773
|
+
logHeliaActivity(
|
|
774
|
+
`✅ helia unixfs.cat complete ${parsedCID.toString()} (${bytes.length} bytes)`,
|
|
775
|
+
);
|
|
776
|
+
return bytes;
|
|
777
|
+
} catch (error) {
|
|
778
|
+
logHeliaActivity(
|
|
779
|
+
`❌ helia unixfs.cat failed ${cid.toString?.() ?? cid}: ${error.message}`,
|
|
780
|
+
);
|
|
781
|
+
logger.warn(` ⚠️ Failed to download from IPFS network: ${error.message}`);
|
|
782
|
+
throw error;
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
/**
|
|
787
|
+
* Download a block by CID, peers first and HTTP gateways as the fallback.
|
|
788
|
+
*
|
|
789
|
+
* Nothing here has ever touched a Storacha client -- the old name
|
|
790
|
+
* (`downloadBlockFromStoracha`, still exported as an alias) described where the block
|
|
791
|
+
* had been put rather than where it is read from, which stopped being true the moment
|
|
792
|
+
* a second backend existed. Bitswap does not care who paid for the pin.
|
|
793
|
+
*
|
|
794
|
+
* @param {string} cid - CID of the block to fetch
|
|
795
|
+
* @param {Object} options - Configuration options
|
|
796
|
+
* @param {Object} [options.helia] - Optional Helia instance for network download
|
|
797
|
+
* @param {boolean} [options.useIPFSNetwork] - try peers before gateways
|
|
798
|
+
* @param {boolean} [options.gatewayFallback] - fall back to HTTP gateways
|
|
799
|
+
* @returns {Promise<Uint8Array>} - Block bytes
|
|
800
|
+
*/
|
|
801
|
+
export async function downloadBlock(cid, options = {}) {
|
|
802
|
+
const config = { ...DEFAULT_OPTIONS, ...options };
|
|
803
|
+
|
|
804
|
+
// Try IPFS network first if enabled and Helia instance is available
|
|
805
|
+
if (config.useIPFSNetwork && config.helia) {
|
|
806
|
+
try {
|
|
807
|
+
const bytes = await downloadBlockFromIPFSNetwork(
|
|
808
|
+
cid,
|
|
809
|
+
config.helia,
|
|
810
|
+
config,
|
|
811
|
+
);
|
|
812
|
+
return bytes;
|
|
813
|
+
} catch (error) {
|
|
814
|
+
logger.info(
|
|
815
|
+
` ⚠️ Network download failed, ${config.gatewayFallback ? "falling back to gateway" : "no fallback enabled"}: ${error.message}`,
|
|
816
|
+
);
|
|
817
|
+
if (!config.gatewayFallback) {
|
|
818
|
+
throw new Error(
|
|
819
|
+
`Could not download block ${cid} from IPFS network and gateway fallback is disabled`, { cause: error },
|
|
820
|
+
);
|
|
821
|
+
}
|
|
822
|
+
// Continue to gateway fallback
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
// Fallback to gateway download
|
|
827
|
+
// storacha.link and w3s.link answer 301 to dweb.link as of 2026-09-05, so going
|
|
828
|
+
// through them only buys a redirect. Ask the destination directly.
|
|
829
|
+
const gateways = [
|
|
830
|
+
`${config.gateway}/ipfs`,
|
|
831
|
+
"https://dweb.link/ipfs",
|
|
832
|
+
"https://ipfs.io/ipfs",
|
|
833
|
+
];
|
|
834
|
+
|
|
835
|
+
for (const gateway of gateways) {
|
|
836
|
+
try {
|
|
837
|
+
const response = await fetch(`${gateway}/${cid}`, {
|
|
838
|
+
signal: AbortSignal.timeout(config.timeout),
|
|
839
|
+
});
|
|
840
|
+
|
|
841
|
+
if (response.ok) {
|
|
842
|
+
const contentType = response.headers?.get("content-type") || "";
|
|
843
|
+
|
|
844
|
+
// Reject HTML responses (error pages)
|
|
845
|
+
if (
|
|
846
|
+
contentType.includes("text/html") ||
|
|
847
|
+
contentType.includes("application/xhtml")
|
|
848
|
+
) {
|
|
849
|
+
logger.debug(
|
|
850
|
+
` ⚠️ Gateway ${gateway} returned HTML (likely error page), trying next gateway...`,
|
|
851
|
+
);
|
|
852
|
+
continue; // Try next gateway
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
const bytes = new Uint8Array(await response.arrayBuffer());
|
|
856
|
+
|
|
857
|
+
// Validate that we didn't get an HTML error page
|
|
858
|
+
// Check first few bytes for HTML markers
|
|
859
|
+
const textStart = new TextDecoder("utf-8", { fatal: false }).decode(
|
|
860
|
+
bytes.slice(0, Math.min(100, bytes.length)),
|
|
861
|
+
);
|
|
862
|
+
if (
|
|
863
|
+
textStart.trim().startsWith("<!DOCTYPE") ||
|
|
864
|
+
textStart.trim().startsWith("<html") ||
|
|
865
|
+
textStart.trim().startsWith("<?xml")
|
|
866
|
+
) {
|
|
867
|
+
logger.debug(
|
|
868
|
+
` ⚠️ Gateway ${gateway} returned HTML/XML (likely error page), trying next gateway...`,
|
|
869
|
+
);
|
|
870
|
+
continue; // Try next gateway
|
|
871
|
+
}
|
|
872
|
+
|
|
873
|
+
logger.info(` ✅ Downloaded ${bytes.length} bytes from ${gateway}`);
|
|
874
|
+
return bytes;
|
|
875
|
+
} else {
|
|
876
|
+
// Enhanced logging for 503 and other error statuses
|
|
877
|
+
let errorLog = ` ⚠️ Gateway ${gateway} returned status ${response.status}`;
|
|
878
|
+
|
|
879
|
+
if (response.status === 503) {
|
|
880
|
+
const retryAfter = response.headers?.get("Retry-After");
|
|
881
|
+
const rateLimitReset = response.headers?.get("X-RateLimit-Reset");
|
|
882
|
+
const rateLimitRemaining = response.headers?.get(
|
|
883
|
+
"X-RateLimit-Remaining",
|
|
884
|
+
);
|
|
885
|
+
|
|
886
|
+
errorLog += ` (Service Unavailable - likely rate limited)`;
|
|
887
|
+
|
|
888
|
+
if (retryAfter) {
|
|
889
|
+
const retrySeconds = parseInt(retryAfter);
|
|
890
|
+
if (!isNaN(retrySeconds)) {
|
|
891
|
+
const futureTime = new Date(Date.now() + retrySeconds * 1000);
|
|
892
|
+
errorLog += `\n ⏱️ Retry-After: ${retrySeconds}s (available at ${futureTime.toLocaleTimeString()})`;
|
|
893
|
+
} else {
|
|
894
|
+
const retryDate = new Date(retryAfter);
|
|
895
|
+
if (!isNaN(retryDate.getTime())) {
|
|
896
|
+
const waitSecs = Math.ceil(
|
|
897
|
+
Math.max(0, retryDate.getTime() - Date.now()) / 1000,
|
|
898
|
+
);
|
|
899
|
+
errorLog += `\n ⏱️ Retry-After: ${waitSecs}s (available at ${retryDate.toLocaleTimeString()})`;
|
|
900
|
+
} else {
|
|
901
|
+
errorLog += `\n ⏱️ Retry-After: ${retryAfter}`;
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
if (rateLimitReset) {
|
|
907
|
+
const resetTime = parseInt(rateLimitReset);
|
|
908
|
+
if (!isNaN(resetTime)) {
|
|
909
|
+
const resetDate = new Date(resetTime * 1000);
|
|
910
|
+
const waitSecs = Math.ceil(
|
|
911
|
+
Math.max(0, resetDate.getTime() - Date.now()) / 1000,
|
|
912
|
+
);
|
|
913
|
+
errorLog += `\n 🔄 Rate Limit Reset: ${resetDate.toLocaleTimeString()} (in ${waitSecs}s)`;
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
if (rateLimitRemaining !== null && rateLimitRemaining !== undefined) {
|
|
918
|
+
errorLog += `\n 📈 Requests Remaining: ${rateLimitRemaining}`;
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
errorLog += `, trying next gateway...`;
|
|
923
|
+
logger.debug(errorLog);
|
|
924
|
+
}
|
|
925
|
+
} catch (error) {
|
|
926
|
+
logger.debug(` ⚠️ Failed from ${gateway}: ${error.message}`);
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
throw new Error(`Could not download block ${cid} from any gateway`);
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
/**
|
|
934
|
+
* @deprecated Use {@link downloadBlock}. Kept because downstream code imports this name.
|
|
935
|
+
* @type {typeof downloadBlock}
|
|
936
|
+
*/
|
|
937
|
+
export const downloadBlockFromStoracha = downloadBlock;
|
|
938
|
+
|
|
939
|
+
/**
|
|
940
|
+
* Advanced block analysis (classification and log head detection)
|
|
941
|
+
*
|
|
942
|
+
* @param {Object} blockstore - IPFS blockstore
|
|
943
|
+
* @param {Map} downloadedBlocks - Downloaded blocks map
|
|
944
|
+
* @returns {Promise<Object>} - Analysis results
|
|
945
|
+
*/
|
|
946
|
+
export async function analyzeBlocks(blockstore, downloadedBlocks = null) {
|
|
947
|
+
logger.debug("Analyzing downloaded blocks...");
|
|
948
|
+
|
|
949
|
+
const analysis = {
|
|
950
|
+
manifestBlocks: [],
|
|
951
|
+
accessControllerBlocks: [],
|
|
952
|
+
logEntryBlocks: [],
|
|
953
|
+
identityBlocks: [],
|
|
954
|
+
unknownBlocks: [],
|
|
955
|
+
logStructure: new Map(),
|
|
956
|
+
potentialHeads: [],
|
|
957
|
+
logChain: new Map(),
|
|
958
|
+
};
|
|
959
|
+
|
|
960
|
+
const allCIDStrings = downloadedBlocks
|
|
961
|
+
? Array.from(downloadedBlocks.keys())
|
|
962
|
+
: [];
|
|
963
|
+
|
|
964
|
+
for (const cidString of allCIDStrings) {
|
|
965
|
+
try {
|
|
966
|
+
const cid = CID.parse(cidString);
|
|
967
|
+
|
|
968
|
+
// Check if blockstore is available and accessible
|
|
969
|
+
if (!blockstore || typeof blockstore.get !== "function") {
|
|
970
|
+
throw new Error("Blockstore is not available or accessible");
|
|
971
|
+
}
|
|
972
|
+
|
|
973
|
+
let bytes;
|
|
974
|
+
try {
|
|
975
|
+
bytes = await readBlockBytes(blockstore, cid);
|
|
976
|
+
} catch (error) {
|
|
977
|
+
if (
|
|
978
|
+
error.message?.includes("not open") ||
|
|
979
|
+
error.message?.includes("closed") ||
|
|
980
|
+
error.message?.includes("Database is not open")
|
|
981
|
+
) {
|
|
982
|
+
throw new Error(
|
|
983
|
+
`Blockstore operation failed - OrbitDB was closed during analysis: ${error.message}`, { cause: error },
|
|
984
|
+
);
|
|
985
|
+
}
|
|
986
|
+
throw error;
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
if (cid.code === 0x71) {
|
|
990
|
+
// dag-cbor codec
|
|
991
|
+
try {
|
|
992
|
+
const block = await Block.decode({
|
|
993
|
+
cid,
|
|
994
|
+
bytes,
|
|
995
|
+
codec: dagCbor,
|
|
996
|
+
hasher: sha256,
|
|
997
|
+
});
|
|
998
|
+
|
|
999
|
+
const content = block.value;
|
|
1000
|
+
logger.info("content.type", content);
|
|
1001
|
+
// Smart block classification
|
|
1002
|
+
//if (content.type && content.name && content.accessController) {
|
|
1003
|
+
if (content.accessController) {
|
|
1004
|
+
analysis.manifestBlocks.push({ cid: cidString, content });
|
|
1005
|
+
logger.info(` 📋 Manifest: ${cidString} (${content.name})`);
|
|
1006
|
+
} else if (content.sig && content.key && content.identity) {
|
|
1007
|
+
analysis.logEntryBlocks.push({ cid: cidString, content });
|
|
1008
|
+
analysis.logStructure.set(cidString, content);
|
|
1009
|
+
logger.info(` 📝 Log Entry: ${cidString}`);
|
|
1010
|
+
|
|
1011
|
+
// Build log chain for head detection
|
|
1012
|
+
if (content.next && Array.isArray(content.next)) {
|
|
1013
|
+
for (const nextHash of content.next) {
|
|
1014
|
+
analysis.logChain.set(nextHash, cidString);
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
} else if (content.id && content.type) {
|
|
1018
|
+
analysis.identityBlocks.push({ cid: cidString, content });
|
|
1019
|
+
logger.info(` 👤 Identity: ${cidString}`);
|
|
1020
|
+
} else if (
|
|
1021
|
+
content.type === "orbitdb-access-controller" ||
|
|
1022
|
+
content.type === "ipfs"
|
|
1023
|
+
) {
|
|
1024
|
+
analysis.accessControllerBlocks.push({ cid: cidString, content });
|
|
1025
|
+
logger.info(` 🔒 Access Controller: ${cidString}`);
|
|
1026
|
+
} else {
|
|
1027
|
+
analysis.unknownBlocks.push({ cid: cidString, content });
|
|
1028
|
+
logger.info(` ❓ Unknown: ${cidString}`);
|
|
1029
|
+
}
|
|
1030
|
+
} catch (decodeError) {
|
|
1031
|
+
analysis.unknownBlocks.push({
|
|
1032
|
+
cid: cidString,
|
|
1033
|
+
decodeError: decodeError.message,
|
|
1034
|
+
});
|
|
1035
|
+
logger.info(` ⚠️ Decode failed: ${cidString}`);
|
|
1036
|
+
}
|
|
1037
|
+
} else {
|
|
1038
|
+
analysis.unknownBlocks.push({ cid: cidString, reason: "not dag-cbor" });
|
|
1039
|
+
logger.info(` 🔧 Raw block: ${cidString}`);
|
|
1040
|
+
}
|
|
1041
|
+
} catch (error) {
|
|
1042
|
+
logger.warn(` ❌ Error analyzing block ${cidString}: ${error.message}`);
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
// Intelligent head detection
|
|
1047
|
+
logger.info("🎯 Determining log heads:");
|
|
1048
|
+
for (const [entryHash, _entryContent] of analysis.logStructure) {
|
|
1049
|
+
if (!analysis.logChain.has(entryHash)) {
|
|
1050
|
+
analysis.potentialHeads.push(entryHash);
|
|
1051
|
+
logger.info(` 🎯 HEAD: ${entryHash}`);
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
logger.info("📊 Analysis Summary:");
|
|
1056
|
+
logger.info(` 📋 Manifests: ${analysis.manifestBlocks.length}`);
|
|
1057
|
+
logger.info(` 📝 Log Entries: ${analysis.logEntryBlocks.length}`);
|
|
1058
|
+
logger.info(` 👤 Identities: ${analysis.identityBlocks.length}`);
|
|
1059
|
+
logger.info(
|
|
1060
|
+
` 🔒 Access Controllers: ${analysis.accessControllerBlocks.length}`,
|
|
1061
|
+
);
|
|
1062
|
+
logger.info(` 🎯 Heads Discovered: ${analysis.potentialHeads.length}`);
|
|
1063
|
+
|
|
1064
|
+
return analysis;
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
/**
|
|
1068
|
+
* Download blocks from Storacha and bridge CID formats for OrbitDB
|
|
1069
|
+
*
|
|
1070
|
+
* @param {Map} cidMappings - Mapping of original → uploaded CIDs
|
|
1071
|
+
* @param {Object} client - Storacha client (unused, kept for compatibility)
|
|
1072
|
+
* @param {Object} targetBlockstore - Target blockstore to store blocks
|
|
1073
|
+
* @param {Object} options - Configuration options
|
|
1074
|
+
* @returns {Promise<Object>} - Bridge results
|
|
1075
|
+
*/
|
|
1076
|
+
async function downloadAndBridgeBlocks(
|
|
1077
|
+
cidMappings,
|
|
1078
|
+
client,
|
|
1079
|
+
targetBlockstore,
|
|
1080
|
+
options = {},
|
|
1081
|
+
) {
|
|
1082
|
+
const config = { ...DEFAULT_OPTIONS, ...options };
|
|
1083
|
+
logger.info(
|
|
1084
|
+
`📥 Downloading and bridging ${cidMappings.size} blocks for OrbitDB...`,
|
|
1085
|
+
);
|
|
1086
|
+
|
|
1087
|
+
const bridgedBlocks = [];
|
|
1088
|
+
|
|
1089
|
+
for (const [originalCID, storachaCID] of cidMappings) {
|
|
1090
|
+
try {
|
|
1091
|
+
logger.info(` 📥 Downloading ${storachaCID}...`);
|
|
1092
|
+
|
|
1093
|
+
// Try network download first if enabled and Helia instance available
|
|
1094
|
+
let blockBytes = null;
|
|
1095
|
+
if (config.useIPFSNetwork && config.helia) {
|
|
1096
|
+
try {
|
|
1097
|
+
// Create unixfs if not provided
|
|
1098
|
+
if (!config.unixfs) {
|
|
1099
|
+
config.unixfs = unixfs(config.helia);
|
|
1100
|
+
}
|
|
1101
|
+
blockBytes = await downloadBlockFromIPFSNetwork(
|
|
1102
|
+
storachaCID,
|
|
1103
|
+
config.helia,
|
|
1104
|
+
{ ...config, unixfs: config.unixfs },
|
|
1105
|
+
);
|
|
1106
|
+
logger.info(
|
|
1107
|
+
` ✅ Downloaded ${blockBytes.length} bytes from IPFS network`,
|
|
1108
|
+
);
|
|
1109
|
+
} catch (error) {
|
|
1110
|
+
logger.info(
|
|
1111
|
+
` ⚠️ Network download failed, ${config.gatewayFallback ? "falling back to gateway" : "no fallback"}: ${error.message}`,
|
|
1112
|
+
);
|
|
1113
|
+
if (!config.gatewayFallback) {
|
|
1114
|
+
throw new Error(
|
|
1115
|
+
`Failed to download block from IPFS network and gateway fallback is disabled: ${error.message}`, { cause: error },
|
|
1116
|
+
);
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
// Fallback to gateway if network failed or disabled
|
|
1122
|
+
if (!blockBytes) {
|
|
1123
|
+
const response = await fetch(`${config.gateway}/ipfs/${storachaCID}`, {
|
|
1124
|
+
signal: AbortSignal.timeout(config.timeout),
|
|
1125
|
+
});
|
|
1126
|
+
|
|
1127
|
+
if (!response.ok) {
|
|
1128
|
+
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
blockBytes = new Uint8Array(await response.arrayBuffer());
|
|
1132
|
+
logger.info(
|
|
1133
|
+
` ✅ Downloaded ${blockBytes.length} bytes from gateway ${config.gateway}`,
|
|
1134
|
+
);
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
// Convert Storacha CID to OrbitDB format
|
|
1138
|
+
const bridgedCID = convertStorachaCIDToOrbitDB(storachaCID);
|
|
1139
|
+
logger.info(` 🌉 Bridged CID: ${storachaCID} → ${bridgedCID}`);
|
|
1140
|
+
|
|
1141
|
+
// Verify the bridged CID matches the original
|
|
1142
|
+
const match = bridgedCID === originalCID;
|
|
1143
|
+
if (match) {
|
|
1144
|
+
logger.info(` ✅ CID bridge successful: ${bridgedCID}`);
|
|
1145
|
+
} else {
|
|
1146
|
+
logger.warn(
|
|
1147
|
+
` ⚠️ CID bridge mismatch: expected ${originalCID}, got ${bridgedCID}`,
|
|
1148
|
+
);
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
// Store block in target blockstore under OrbitDB CID format
|
|
1152
|
+
const parsedBridgedCID = CID.parse(bridgedCID);
|
|
1153
|
+
await targetBlockstore.put(parsedBridgedCID, blockBytes);
|
|
1154
|
+
logger.info(` 💾 Stored in blockstore as: ${bridgedCID}`);
|
|
1155
|
+
|
|
1156
|
+
bridgedBlocks.push({
|
|
1157
|
+
originalCID,
|
|
1158
|
+
storachaCID,
|
|
1159
|
+
bridgedCID,
|
|
1160
|
+
size: blockBytes.length,
|
|
1161
|
+
match,
|
|
1162
|
+
});
|
|
1163
|
+
} catch (error) {
|
|
1164
|
+
logger.error(
|
|
1165
|
+
` ❌ Failed to download/bridge ${storachaCID}: ${error.message}`,
|
|
1166
|
+
);
|
|
1167
|
+
bridgedBlocks.push({
|
|
1168
|
+
originalCID,
|
|
1169
|
+
storachaCID,
|
|
1170
|
+
error: error.message,
|
|
1171
|
+
});
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
const successful = bridgedBlocks.filter((b) => b.bridgedCID);
|
|
1176
|
+
const failed = bridgedBlocks.filter((b) => b.error);
|
|
1177
|
+
const matches = successful.filter((b) => b.match);
|
|
1178
|
+
|
|
1179
|
+
logger.info(` 📊 Bridge summary:`);
|
|
1180
|
+
logger.info(` Total blocks: ${cidMappings.size}`);
|
|
1181
|
+
logger.info(` Downloaded: ${successful.length}`);
|
|
1182
|
+
logger.info(` Failed: ${failed.length}`);
|
|
1183
|
+
logger.info(` CID matches: ${matches.length}`);
|
|
1184
|
+
|
|
1185
|
+
return { bridgedBlocks, successful, failed, matches };
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
/**
|
|
1189
|
+
* Back up an OrbitDB database through whichever backend is configured.
|
|
1190
|
+
*
|
|
1191
|
+
* **A CAR by default** (#54, 0.5.2). One opaque blob rather than a stream of
|
|
1192
|
+
* small ones: hash preservation is then structural instead of a property we
|
|
1193
|
+
* hope a vendor keeps, and it clears the minimum piece sizes that individual
|
|
1194
|
+
* OrbitDB blocks do not.
|
|
1195
|
+
*
|
|
1196
|
+
* The two strategies return **different fields**, so read `result.method`
|
|
1197
|
+
* rather than guessing from the shape. A CAR backup yields `backupFiles` with
|
|
1198
|
+
* the pointer to restore from; a block backup yields `cidMappings`, which the
|
|
1199
|
+
* CAR path cannot produce because there are no per-block handles to map. If
|
|
1200
|
+
* you were calling this before 0.5.2 and using `cidMappings`, pass
|
|
1201
|
+
* `strategy: "blocks"` — and read `assertCanTakeBlocks` for when a backend
|
|
1202
|
+
* will refuse.
|
|
1203
|
+
*
|
|
1204
|
+
* @param {Object} orbitdb - OrbitDB instance
|
|
1205
|
+
* @param {string} databaseAddress - Database address or name
|
|
1206
|
+
* @param {Object} options - Backup options
|
|
1207
|
+
* @param {"car"|"blocks"} [options.strategy="car"] - How to hand the database
|
|
1208
|
+
* to the backend. `"blocks"` is refused by a backend that re-chunks or has a
|
|
1209
|
+
* minimum piece size, before anything is sent.
|
|
1210
|
+
* @param {EventEmitter} options.eventEmitter - Optional event emitter for progress updates
|
|
1211
|
+
* @param {boolean} [options.logEntriesOnly] - Salvage mode: only log entries,
|
|
1212
|
+
* for a database whose manifest cannot be read. Implies `strategy: "blocks"`,
|
|
1213
|
+
* because a CAR without its manifest is one `restoreFromCID` refuses — and
|
|
1214
|
+
* should, since it could be anybody's log.
|
|
1215
|
+
* @returns {Promise<Object>} - Backup result, carrying `method`
|
|
1216
|
+
*/
|
|
1217
|
+
/**
|
|
1218
|
+
* Whether this backend can be handed one OrbitDB block at a time.
|
|
1219
|
+
*
|
|
1220
|
+
* Two reasons it usually cannot, and both are quiet failures rather than loud
|
|
1221
|
+
* ones, which is why they are checked before a single block is sent:
|
|
1222
|
+
*
|
|
1223
|
+
* - **A minimum piece size.** Filecoin Onchain Cloud rejects anything under
|
|
1224
|
+
* 127 bytes and an OrbitDB block is routinely smaller, so a per-block backup
|
|
1225
|
+
* dies partway through with some blocks stored and some not — the worst
|
|
1226
|
+
* possible outcome, because it looks like a backup.
|
|
1227
|
+
* - **Re-chunking.** A backend that computes its own CID for what we send hands
|
|
1228
|
+
* back a name for content we have never seen. Inside a CAR that cannot happen,
|
|
1229
|
+
* because the CAR is one opaque blob and the CIDs inside it stay ours.
|
|
1230
|
+
*
|
|
1231
|
+
* @param {import("./backends/types.js").StorageBackend} backend
|
|
1232
|
+
*/
|
|
1233
|
+
function assertCanTakeBlocks(backend) {
|
|
1234
|
+
const caps = backend.capabilities ?? {};
|
|
1235
|
+
if (!caps.preservesInnerCids) {
|
|
1236
|
+
throw new BackendError(
|
|
1237
|
+
"UNSUPPORTED",
|
|
1238
|
+
`The ${backend.name} backend does not preserve the CIDs it is given, so it cannot be sent blocks one at a time. Use the default CAR strategy.`,
|
|
1239
|
+
);
|
|
1240
|
+
}
|
|
1241
|
+
if (caps.minBlobSize > 0) {
|
|
1242
|
+
throw new BackendError(
|
|
1243
|
+
"TOO_SMALL",
|
|
1244
|
+
`The ${backend.name} backend has a ${caps.minBlobSize}-byte minimum and OrbitDB blocks are routinely smaller, so a per-block backup would fail partway through. Use the default CAR strategy.`,
|
|
1245
|
+
);
|
|
1246
|
+
}
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
export async function backupDatabase(orbitdb, databaseAddress, options = {}) {
|
|
1250
|
+
const config = { ...DEFAULT_OPTIONS, ...options };
|
|
1251
|
+
const eventEmitter = options.eventEmitter;
|
|
1252
|
+
const logEntriesOnly = options.logEntriesOnly || false;
|
|
1253
|
+
|
|
1254
|
+
// CAR is the default (#54, 0.5.2). Hash preservation is then structural
|
|
1255
|
+
// rather than a property we hope a vendor keeps, and one blob clears the
|
|
1256
|
+
// minimum piece sizes that a stream of small blocks does not.
|
|
1257
|
+
//
|
|
1258
|
+
// `logEntriesOnly` stays on the block path and is not a strategy: it exists
|
|
1259
|
+
// to salvage a database whose manifest cannot be read, and a CAR without its
|
|
1260
|
+
// manifest is one `restoreFromCID` refuses — correctly, since it could be
|
|
1261
|
+
// anybody's log.
|
|
1262
|
+
const strategy = options.strategy ?? (logEntriesOnly ? "blocks" : "car");
|
|
1263
|
+
if (strategy !== "car" && strategy !== "blocks") {
|
|
1264
|
+
throw new BackendError("UNSUPPORTED", `Unknown backup strategy "${strategy}"`);
|
|
1265
|
+
}
|
|
1266
|
+
if (strategy === "car") {
|
|
1267
|
+
// Imported here rather than at the top: `backup-car.js` imports this
|
|
1268
|
+
// module, and a static edge back would close the cycle.
|
|
1269
|
+
const { backupDatabaseCAR } = await import("./backup-car.js");
|
|
1270
|
+
return backupDatabaseCAR(orbitdb, databaseAddress, options);
|
|
1271
|
+
}
|
|
1272
|
+
|
|
1273
|
+
const backupMode = logEntriesOnly
|
|
1274
|
+
? "Log Entries Only (Fallback Mode)"
|
|
1275
|
+
: "Full Backup";
|
|
1276
|
+
|
|
1277
|
+
logger.info("🚀 Starting OrbitDB Database Backup to Storacha");
|
|
1278
|
+
logger.info(`📍 Database: ${databaseAddress}`);
|
|
1279
|
+
logger.info(`🔧 Backup Mode: ${backupMode}`);
|
|
1280
|
+
|
|
1281
|
+
try {
|
|
1282
|
+
// Initialize Storacha client - support both credential and UCAN authentication
|
|
1283
|
+
const backend = await resolveBackend(config);
|
|
1284
|
+
// Before a single block is sent, not after some of them are.
|
|
1285
|
+
assertCanTakeBlocks(backend);
|
|
1286
|
+
|
|
1287
|
+
// Open the database
|
|
1288
|
+
const database = await orbitdb.open(databaseAddress, options.dbConfig);
|
|
1289
|
+
|
|
1290
|
+
// Extract blocks based on backup mode
|
|
1291
|
+
const { blocks, blockSources, manifestCID } = await extractDatabaseBlocks(
|
|
1292
|
+
database,
|
|
1293
|
+
{
|
|
1294
|
+
logEntriesOnly,
|
|
1295
|
+
},
|
|
1296
|
+
);
|
|
1297
|
+
|
|
1298
|
+
// Upload blocks to Storacha with progress tracking
|
|
1299
|
+
let successful, cidMappings;
|
|
1300
|
+
try {
|
|
1301
|
+
const result = await uploadBlocks(
|
|
1302
|
+
blocks,
|
|
1303
|
+
backend,
|
|
1304
|
+
config.uploadBatchSize,
|
|
1305
|
+
config.uploadMaxConcurrency,
|
|
1306
|
+
eventEmitter,
|
|
1307
|
+
);
|
|
1308
|
+
successful = result.successful;
|
|
1309
|
+
cidMappings = result.cidMappings;
|
|
1310
|
+
|
|
1311
|
+
if (successful.length === 0) {
|
|
1312
|
+
throw new Error("No blocks were successfully uploaded");
|
|
1313
|
+
}
|
|
1314
|
+
} catch (error) {
|
|
1315
|
+
if (
|
|
1316
|
+
error.message?.includes("Unauthorized") ||
|
|
1317
|
+
error.message?.includes("Capability")
|
|
1318
|
+
) {
|
|
1319
|
+
if (eventEmitter) {
|
|
1320
|
+
eventEmitter.emit("uploadProgress", {
|
|
1321
|
+
type: "upload",
|
|
1322
|
+
current: 0,
|
|
1323
|
+
total: blocks.size,
|
|
1324
|
+
percentage: 0,
|
|
1325
|
+
status: "error",
|
|
1326
|
+
error: {
|
|
1327
|
+
type: "ucan",
|
|
1328
|
+
message:
|
|
1329
|
+
"UCAN authorization failed: Your capabilities are not sufficient for uploading. Please check your space permissions.",
|
|
1330
|
+
details: error.message,
|
|
1331
|
+
},
|
|
1332
|
+
});
|
|
1333
|
+
}
|
|
1334
|
+
throw new Error(
|
|
1335
|
+
"UCAN authorization failed: Your capabilities are not sufficient for uploading. Please check your space permissions.", { cause: error },
|
|
1336
|
+
);
|
|
1337
|
+
}
|
|
1338
|
+
throw error;
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
// Get block summary
|
|
1342
|
+
const blockSummary = {};
|
|
1343
|
+
for (const [_hash, source] of blockSources) {
|
|
1344
|
+
blockSummary[source] = (blockSummary[source] || 0) + 1;
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
logger.info("✅ Backup completed successfully!");
|
|
1348
|
+
|
|
1349
|
+
return {
|
|
1350
|
+
success: true,
|
|
1351
|
+
// Named, because the two strategies return different fields and a caller
|
|
1352
|
+
// that branches on shape rather than on this is guessing.
|
|
1353
|
+
method: "blocks",
|
|
1354
|
+
manifestCID,
|
|
1355
|
+
databaseAddress: database.address,
|
|
1356
|
+
databaseName: database.name,
|
|
1357
|
+
blocksTotal: blocks.size,
|
|
1358
|
+
blocksUploaded: successful.length,
|
|
1359
|
+
blockSummary,
|
|
1360
|
+
cidMappings: Object.fromEntries(cidMappings),
|
|
1361
|
+
};
|
|
1362
|
+
} catch (error) {
|
|
1363
|
+
logger.error("❌ Backup failed:", error.message);
|
|
1364
|
+
return {
|
|
1365
|
+
success: false,
|
|
1366
|
+
error: error.message,
|
|
1367
|
+
// A BackendError says the backend cannot serve this request, which is a
|
|
1368
|
+
// different thing from the backup having failed, and a caller that has to
|
|
1369
|
+
// read the message to tell them apart will get it wrong.
|
|
1370
|
+
...(error instanceof BackendError ? { code: error.code } : {}),
|
|
1371
|
+
};
|
|
1372
|
+
}
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
/**
|
|
1376
|
+
* Optimized restore from Storacha for fallback reconstruction (log entries only)
|
|
1377
|
+
*
|
|
1378
|
+
* @param {Object} orbitdb - Target OrbitDB instance
|
|
1379
|
+
* @param {Object} options - Restore options
|
|
1380
|
+
* @param {string} [options.storachaKey] - Storacha private key (defaults to env)
|
|
1381
|
+
* @param {string} [options.storachaProof] - Storacha proof (defaults to env)
|
|
1382
|
+
* @param {EventEmitter} [options.eventEmitter] - Optional event emitter for progress updates
|
|
1383
|
+
* @returns {Promise<Object>} - Restore result
|
|
1384
|
+
*/
|
|
1385
|
+
export async function restoreLogEntriesOnly(orbitdb, options = {}) {
|
|
1386
|
+
const config = { ...DEFAULT_OPTIONS, ...options };
|
|
1387
|
+
const eventEmitter = options.eventEmitter;
|
|
1388
|
+
|
|
1389
|
+
logger.info("⚡ Starting Optimized Log-Entries-Only Restore from Storacha");
|
|
1390
|
+
|
|
1391
|
+
try {
|
|
1392
|
+
// Step 1: List ALL files in Storacha space
|
|
1393
|
+
logger.info("\n📋 Step 1: Discovering all files in Storacha space...");
|
|
1394
|
+
const spaceFiles = await listStorachaSpaceFiles(config);
|
|
1395
|
+
|
|
1396
|
+
if (spaceFiles.length === 0) {
|
|
1397
|
+
throw new Error("No files found in Storacha space");
|
|
1398
|
+
}
|
|
1399
|
+
|
|
1400
|
+
logger.info(` 🎉 SUCCESS! Found ${spaceFiles.length} files in space`);
|
|
1401
|
+
|
|
1402
|
+
// Step 2: Download ONLY log entry blocks (optimized)
|
|
1403
|
+
// Ensure helia instance is included in config for network downloads
|
|
1404
|
+
const downloadConfig = {
|
|
1405
|
+
...config,
|
|
1406
|
+
helia: config.helia || orbitdb.ipfs,
|
|
1407
|
+
};
|
|
1408
|
+
const logEntryBlocks = await downloadLogEntriesOnly(
|
|
1409
|
+
spaceFiles,
|
|
1410
|
+
orbitdb,
|
|
1411
|
+
downloadConfig,
|
|
1412
|
+
eventEmitter,
|
|
1413
|
+
);
|
|
1414
|
+
|
|
1415
|
+
if (logEntryBlocks.size === 0) {
|
|
1416
|
+
throw new Error("No log entries found in Storacha space");
|
|
1417
|
+
}
|
|
1418
|
+
|
|
1419
|
+
logger.info(
|
|
1420
|
+
` ⚡ OPTIMIZATION: Downloaded only ${logEntryBlocks.size} log entries instead of ${spaceFiles.length} total files`,
|
|
1421
|
+
);
|
|
1422
|
+
|
|
1423
|
+
// Step 3: Direct fallback reconstruction with joinEntry support
|
|
1424
|
+
logger.info(
|
|
1425
|
+
"\n🔧 Step 3: Reconstructing database from log entries with joinEntry...",
|
|
1426
|
+
);
|
|
1427
|
+
const fallbackResult = await reconstructWithoutManifest(
|
|
1428
|
+
orbitdb,
|
|
1429
|
+
logEntryBlocks,
|
|
1430
|
+
config,
|
|
1431
|
+
true, // Enable joinEntry mode for proper log traversal
|
|
1432
|
+
);
|
|
1433
|
+
|
|
1434
|
+
logger.info(
|
|
1435
|
+
"✅ Optimized Log-Entries-Only Restore completed successfully!",
|
|
1436
|
+
);
|
|
1437
|
+
|
|
1438
|
+
return {
|
|
1439
|
+
database: fallbackResult.database,
|
|
1440
|
+
metadata: fallbackResult.metadata,
|
|
1441
|
+
entriesCount: fallbackResult.entriesCount,
|
|
1442
|
+
entriesRecovered: fallbackResult.entriesCount,
|
|
1443
|
+
method: "optimized-log-entries-only",
|
|
1444
|
+
success: true,
|
|
1445
|
+
preservedHashes: false,
|
|
1446
|
+
preservedAddress: false,
|
|
1447
|
+
spaceFilesFound: spaceFiles.length,
|
|
1448
|
+
logEntriesDownloaded: logEntryBlocks.size,
|
|
1449
|
+
optimizationSavings: {
|
|
1450
|
+
totalFiles: spaceFiles.length,
|
|
1451
|
+
filesDownloaded: logEntryBlocks.size,
|
|
1452
|
+
filesSkipped: spaceFiles.length - logEntryBlocks.size,
|
|
1453
|
+
percentageSaved: Math.round(
|
|
1454
|
+
((spaceFiles.length - logEntryBlocks.size) / spaceFiles.length) * 100,
|
|
1455
|
+
),
|
|
1456
|
+
},
|
|
1457
|
+
};
|
|
1458
|
+
} catch (error) {
|
|
1459
|
+
logger.error(
|
|
1460
|
+
"❌ Optimized Log-Entries-Only Restore failed:",
|
|
1461
|
+
error.message,
|
|
1462
|
+
);
|
|
1463
|
+
|
|
1464
|
+
return {
|
|
1465
|
+
success: false,
|
|
1466
|
+
error: error.message,
|
|
1467
|
+
};
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1470
|
+
|
|
1471
|
+
/**
|
|
1472
|
+
* Mapping-independent restore from Storacha (ENHANCED)
|
|
1473
|
+
*
|
|
1474
|
+
* @param {Object} orbitdb - Target OrbitDB instance
|
|
1475
|
+
* @param {Object} options - Restore options
|
|
1476
|
+
* @param {string} [options.storachaKey] - Storacha private key (defaults to env)
|
|
1477
|
+
* @param {string} [options.storachaProof] - Storacha proof (defaults to env)
|
|
1478
|
+
* @param {boolean} [options.forceFallback] - Force fallback reconstruction mode (default: false)
|
|
1479
|
+
* @param {string} [options.fallbackDatabaseName] - Custom name for fallback reconstruction
|
|
1480
|
+
* @param {EventEmitter} [options.eventEmitter] - Optional event emitter for progress updates
|
|
1481
|
+
* @returns {Promise<Object>} - Restore result
|
|
1482
|
+
*/
|
|
1483
|
+
export async function restoreDatabaseFromSpace(orbitdb, options = {}) {
|
|
1484
|
+
const config = { ...DEFAULT_OPTIONS, ...options };
|
|
1485
|
+
const eventEmitter = options.eventEmitter;
|
|
1486
|
+
// Use orbitdb parameter directly instead of creating an alias
|
|
1487
|
+
|
|
1488
|
+
logger.info("🔄 Starting Mapping-Independent OrbitDB Restore from Storacha");
|
|
1489
|
+
|
|
1490
|
+
// Backups are CARs since 0.5.2, so look for one first. Both paths are the
|
|
1491
|
+
// same intent — scan the space, reconstruct without being told the mapping —
|
|
1492
|
+
// and differ only in what the space turns out to hold, which is exactly the
|
|
1493
|
+
// decision a caller cannot make for itself.
|
|
1494
|
+
//
|
|
1495
|
+
// The fallback is not politeness: every backup written before 0.5.2 is a set
|
|
1496
|
+
// of individual blocks, and dropping that path would strand them.
|
|
1497
|
+
const strategy = options.strategy ?? "auto";
|
|
1498
|
+
if (strategy === "auto" || strategy === "car") {
|
|
1499
|
+
// Dynamic, because backup-car.js imports this module.
|
|
1500
|
+
const { listAvailableBackups, restoreFromSpaceCAR } = await import("./backup-car.js");
|
|
1501
|
+
let carBackups = [];
|
|
1502
|
+
try {
|
|
1503
|
+
carBackups = (await listAvailableBackups(config)) ?? [];
|
|
1504
|
+
} catch (error) {
|
|
1505
|
+
if (strategy === "car") throw error;
|
|
1506
|
+
logger.info(` ⚠️ Could not look for CAR backups (${error.message}); scanning for blocks`);
|
|
1507
|
+
}
|
|
1508
|
+
|
|
1509
|
+
if (carBackups.length > 0) {
|
|
1510
|
+
const latest = carBackups[0];
|
|
1511
|
+
logger.info(` 📦 Restoring the CAR backup from ${latest.timestamp}`);
|
|
1512
|
+
// The listing already found it; passing both spares a second scan of the
|
|
1513
|
+
// whole space, which downloads every file in it to identify the metadata.
|
|
1514
|
+
return restoreFromSpaceCAR(orbitdb, {
|
|
1515
|
+
...options,
|
|
1516
|
+
timestamp: latest.timestamp,
|
|
1517
|
+
metadataCID: latest.metadataCID,
|
|
1518
|
+
});
|
|
1519
|
+
}
|
|
1520
|
+
|
|
1521
|
+
if (strategy === "car") {
|
|
1522
|
+
throw new Error("No CAR backup found in this space");
|
|
1523
|
+
}
|
|
1524
|
+
logger.info(" ↩︎ No CAR backup in this space; scanning for individual blocks");
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1527
|
+
try {
|
|
1528
|
+
// Step 1: List ALL files in Storacha space
|
|
1529
|
+
logger.info("\n📋 Step 1: Discovering all files in Storacha space...");
|
|
1530
|
+
const spaceFiles = await listStorachaSpaceFiles(config);
|
|
1531
|
+
|
|
1532
|
+
if (spaceFiles.length === 0) {
|
|
1533
|
+
throw new Error("No files found in Storacha space");
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1536
|
+
logger.info(
|
|
1537
|
+
` 🎉 SUCCESS! Found ${spaceFiles.length} files in space without requiring CID mappings`,
|
|
1538
|
+
);
|
|
1539
|
+
|
|
1540
|
+
// Step 2: Download ALL files from space with progress tracking
|
|
1541
|
+
// Ensure helia instance is included in config for network downloads
|
|
1542
|
+
const downloadConfig = {
|
|
1543
|
+
...config,
|
|
1544
|
+
helia: config.helia || orbitdb.ipfs,
|
|
1545
|
+
};
|
|
1546
|
+
const downloadedBlocks = await downloadBlocksWithProgress(
|
|
1547
|
+
spaceFiles,
|
|
1548
|
+
orbitdb,
|
|
1549
|
+
downloadConfig,
|
|
1550
|
+
eventEmitter,
|
|
1551
|
+
);
|
|
1552
|
+
|
|
1553
|
+
// ... rest of existing code remains the same ...
|
|
1554
|
+
|
|
1555
|
+
// Step 3: Intelligent block analysis
|
|
1556
|
+
logger.info("\n🔍 Step 3: Analyzing block structure...");
|
|
1557
|
+
const analysis = await analyzeBlocks(
|
|
1558
|
+
orbitdb.ipfs.blockstore,
|
|
1559
|
+
downloadedBlocks,
|
|
1560
|
+
);
|
|
1561
|
+
|
|
1562
|
+
if (analysis.manifestBlocks.length === 0 || options.forceFallback) {
|
|
1563
|
+
logger.info(
|
|
1564
|
+
"⚠️ No manifest blocks found - attempting fallback reconstruction...",
|
|
1565
|
+
);
|
|
1566
|
+
|
|
1567
|
+
// Fallback: decode blocks and extract payloads to import as new entries
|
|
1568
|
+
const fallbackResult = await reconstructWithoutManifest(
|
|
1569
|
+
orbitdb,
|
|
1570
|
+
downloadedBlocks,
|
|
1571
|
+
config,
|
|
1572
|
+
);
|
|
1573
|
+
|
|
1574
|
+
return {
|
|
1575
|
+
database: fallbackResult.database,
|
|
1576
|
+
metadata: fallbackResult.metadata,
|
|
1577
|
+
entriesCount: fallbackResult.entriesCount,
|
|
1578
|
+
entriesRecovered: fallbackResult.entriesCount,
|
|
1579
|
+
method: "fallback-reconstruction",
|
|
1580
|
+
success: true,
|
|
1581
|
+
preservedHashes: false,
|
|
1582
|
+
preservedAddress: false,
|
|
1583
|
+
};
|
|
1584
|
+
}
|
|
1585
|
+
|
|
1586
|
+
// Step 4: Reconstruct database using discovered manifest
|
|
1587
|
+
logger.info("\n🔄 Step 4: Reconstructing database from analysis...");
|
|
1588
|
+
|
|
1589
|
+
// Find the correct manifest by matching log entries to database IDs
|
|
1590
|
+
const correctManifest = findCorrectManifest(analysis);
|
|
1591
|
+
if (!correctManifest) {
|
|
1592
|
+
throw new Error("Could not determine correct manifest from log entries");
|
|
1593
|
+
}
|
|
1594
|
+
|
|
1595
|
+
const databaseAddress = `/orbitdb/${correctManifest.cid}`;
|
|
1596
|
+
|
|
1597
|
+
logger.info(` 📥 Opening database at: ${databaseAddress}`);
|
|
1598
|
+
logger.info(
|
|
1599
|
+
` 🎯 Selected manifest: ${correctManifest.cid} (matched from log entries)`,
|
|
1600
|
+
);
|
|
1601
|
+
|
|
1602
|
+
// Extract database type from manifest if available, otherwise infer from log entries
|
|
1603
|
+
let databaseType = inferDatabaseType(analysis.logEntryBlocks); // fallback
|
|
1604
|
+
if (correctManifest.content && correctManifest.content.type) {
|
|
1605
|
+
databaseType = correctManifest.content.type;
|
|
1606
|
+
logger.info(` 📋 Database type from manifest: ${databaseType}`);
|
|
1607
|
+
} else {
|
|
1608
|
+
logger.debug(`Inferred database type: ${databaseType}`);
|
|
1609
|
+
}
|
|
1610
|
+
|
|
1611
|
+
const reconstructedDB = await orbitdb.open(
|
|
1612
|
+
databaseAddress,
|
|
1613
|
+
config.dbConfig ? config.dbConfig : { type: databaseType },
|
|
1614
|
+
);
|
|
1615
|
+
|
|
1616
|
+
// OPTIMIZED: Only join HEAD entries - OrbitDB will traverse backward automatically
|
|
1617
|
+
logger.info(
|
|
1618
|
+
" 🔗 Step 4a: Joining HEAD entries to reconstruct complete log...",
|
|
1619
|
+
);
|
|
1620
|
+
logger.info(
|
|
1621
|
+
` 🎯 Found ${analysis.potentialHeads.length} HEAD(s) in ${analysis.logEntryBlocks.length} total log entries`,
|
|
1622
|
+
);
|
|
1623
|
+
let joinedHeads = 0;
|
|
1624
|
+
|
|
1625
|
+
// Only process HEAD entries - OrbitDB automatically traverses backward from each head
|
|
1626
|
+
for (const headCID of analysis.potentialHeads) {
|
|
1627
|
+
try {
|
|
1628
|
+
// Find the corresponding log entry block for this head
|
|
1629
|
+
const logEntryBlock = analysis.logEntryBlocks.find(
|
|
1630
|
+
(block) => block.cid === headCID,
|
|
1631
|
+
);
|
|
1632
|
+
|
|
1633
|
+
if (!logEntryBlock) {
|
|
1634
|
+
logger.warn(
|
|
1635
|
+
` ⚠️ HEAD ${headCID.slice(0, 12)}... not found in log entry blocks`,
|
|
1636
|
+
);
|
|
1637
|
+
continue;
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
// Create an entry object that matches OrbitDB's expected format
|
|
1641
|
+
const entryData = {
|
|
1642
|
+
hash: logEntryBlock.cid,
|
|
1643
|
+
v: logEntryBlock.content.v,
|
|
1644
|
+
id: logEntryBlock.content.id,
|
|
1645
|
+
key: logEntryBlock.content.key,
|
|
1646
|
+
sig: logEntryBlock.content.sig,
|
|
1647
|
+
next: logEntryBlock.content.next,
|
|
1648
|
+
refs: logEntryBlock.content.refs,
|
|
1649
|
+
clock: logEntryBlock.content.clock,
|
|
1650
|
+
payload: logEntryBlock.content.payload,
|
|
1651
|
+
identity: logEntryBlock.content.identity,
|
|
1652
|
+
};
|
|
1653
|
+
|
|
1654
|
+
// Use joinEntry - this will automatically traverse and join all connected entries
|
|
1655
|
+
const updated = await reconstructedDB.log.joinEntry(entryData);
|
|
1656
|
+
if (updated) {
|
|
1657
|
+
joinedHeads++;
|
|
1658
|
+
logger.info(
|
|
1659
|
+
` ✓ Joined HEAD ${joinedHeads}/${analysis.potentialHeads.length}: ${headCID.slice(0, 12)}...`,
|
|
1660
|
+
);
|
|
1661
|
+
logger.info(
|
|
1662
|
+
` (OrbitDB will automatically traverse and load all connected entries)`,
|
|
1663
|
+
);
|
|
1664
|
+
} else {
|
|
1665
|
+
logger.info(` → HEAD already in log: ${headCID.slice(0, 12)}...`);
|
|
1666
|
+
}
|
|
1667
|
+
} catch (joinError) {
|
|
1668
|
+
logger.warn(
|
|
1669
|
+
` ⚠️ Failed to join HEAD ${headCID}: ${joinError.message}`,
|
|
1670
|
+
);
|
|
1671
|
+
// Continue with other heads even if one fails
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
|
|
1675
|
+
logger.info(
|
|
1676
|
+
` 📊 Successfully joined ${joinedHeads}/${analysis.potentialHeads.length} HEAD entries`,
|
|
1677
|
+
);
|
|
1678
|
+
|
|
1679
|
+
// Wait for log to settle after joining entries
|
|
1680
|
+
logger.info(" ⏳ Waiting for log to settle after joining entries...");
|
|
1681
|
+
await new Promise((resolve) => setTimeout(resolve, config.timeout / 5));
|
|
1682
|
+
|
|
1683
|
+
const reconstructedEntries = await reconstructedDB.all();
|
|
1684
|
+
logger.info(" ⏳ Additional wait for entries to fully load...");
|
|
1685
|
+
await new Promise((resolve) => setTimeout(resolve, config.timeout / 10));
|
|
1686
|
+
|
|
1687
|
+
// Handle different database types properly
|
|
1688
|
+
let entriesArray;
|
|
1689
|
+
let entriesCount;
|
|
1690
|
+
if (reconstructedDB.type === "keyvalue") {
|
|
1691
|
+
// For key-value databases, all() returns an object
|
|
1692
|
+
// Get the actual log entries to preserve hashes
|
|
1693
|
+
const logEntries = await reconstructedDB.log.values();
|
|
1694
|
+
entriesArray = logEntries.map((logEntry) => ({
|
|
1695
|
+
hash: logEntry.hash,
|
|
1696
|
+
payload: logEntry.payload,
|
|
1697
|
+
}));
|
|
1698
|
+
entriesCount = Object.keys(reconstructedEntries).length;
|
|
1699
|
+
} else {
|
|
1700
|
+
// For other database types (events, documents), all() returns an array
|
|
1701
|
+
entriesArray = Array.isArray(reconstructedEntries)
|
|
1702
|
+
? reconstructedEntries
|
|
1703
|
+
: [];
|
|
1704
|
+
entriesCount = entriesArray.length;
|
|
1705
|
+
}
|
|
1706
|
+
|
|
1707
|
+
logger.info(` 📊 Final reconstructed entries: ${entriesCount}`);
|
|
1708
|
+
logger.info(` 🔍 Database type: ${reconstructedDB.type}`);
|
|
1709
|
+
logger.info(` 🔗 HEAD entries joined: ${joinedHeads}`);
|
|
1710
|
+
|
|
1711
|
+
logger.info(
|
|
1712
|
+
"✅ Mapping-Independent Restore with proper joinEntry completed successfully!",
|
|
1713
|
+
);
|
|
1714
|
+
|
|
1715
|
+
return {
|
|
1716
|
+
success: true,
|
|
1717
|
+
database: reconstructedDB,
|
|
1718
|
+
orbitdb: orbitdb, // Return the OrbitDB instance
|
|
1719
|
+
manifestCID: correctManifest.cid,
|
|
1720
|
+
address: reconstructedDB.address,
|
|
1721
|
+
name: reconstructedDB.name,
|
|
1722
|
+
type: reconstructedDB.type,
|
|
1723
|
+
entriesRecovered: entriesCount,
|
|
1724
|
+
blocksRestored: downloadedBlocks.size,
|
|
1725
|
+
addressMatch: reconstructedDB.address === databaseAddress,
|
|
1726
|
+
spaceFilesFound: spaceFiles.length,
|
|
1727
|
+
analysis,
|
|
1728
|
+
entries: entriesArray,
|
|
1729
|
+
};
|
|
1730
|
+
} catch (error) {
|
|
1731
|
+
logger.error("❌ Mapping-Independent Restore failed:", error.message);
|
|
1732
|
+
|
|
1733
|
+
return {
|
|
1734
|
+
success: false,
|
|
1735
|
+
error: error.message,
|
|
1736
|
+
};
|
|
1737
|
+
}
|
|
1738
|
+
}
|
|
1739
|
+
|
|
1740
|
+
/**
|
|
1741
|
+
* Restore an OrbitDB database from Storacha (Legacy mapping-dependent)
|
|
1742
|
+
*
|
|
1743
|
+
* @param {Object} orbitdb - Target OrbitDB instance
|
|
1744
|
+
* @param {string} manifestCID - Manifest CID from backup
|
|
1745
|
+
* @param {Object} cidMappings - Optional CID mappings (if available)
|
|
1746
|
+
* @param {Object} options - Restore options
|
|
1747
|
+
* @returns {Promise<Object>} - Restore result
|
|
1748
|
+
*/
|
|
1749
|
+
export async function restoreDatabase(
|
|
1750
|
+
orbitdb,
|
|
1751
|
+
manifestCID,
|
|
1752
|
+
cidMappings = null,
|
|
1753
|
+
options = {},
|
|
1754
|
+
) {
|
|
1755
|
+
const config = { ...DEFAULT_OPTIONS, ...options };
|
|
1756
|
+
|
|
1757
|
+
logger.info("🔄 Starting OrbitDB Database Restore from Storacha");
|
|
1758
|
+
logger.info(`📍 Manifest CID: ${manifestCID}`);
|
|
1759
|
+
|
|
1760
|
+
// No temporary resources needed
|
|
1761
|
+
|
|
1762
|
+
try {
|
|
1763
|
+
// Initialize Storacha client - support both credential and UCAN authentication
|
|
1764
|
+
const backend = await resolveBackend(config);
|
|
1765
|
+
const client = backend.client;
|
|
1766
|
+
|
|
1767
|
+
// If no CID mappings provided, we need to discover them
|
|
1768
|
+
// This is a simplified version - in practice you'd store mappings during backup
|
|
1769
|
+
if (!cidMappings) {
|
|
1770
|
+
throw new Error(
|
|
1771
|
+
"CID mappings required for restore. Store them during backup.",
|
|
1772
|
+
);
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
// Convert object back to Map if needed
|
|
1776
|
+
const mappings =
|
|
1777
|
+
cidMappings instanceof Map
|
|
1778
|
+
? cidMappings
|
|
1779
|
+
: new Map(Object.entries(cidMappings));
|
|
1780
|
+
|
|
1781
|
+
// Download and bridge blocks
|
|
1782
|
+
const { successful, matches } = await downloadAndBridgeBlocks(
|
|
1783
|
+
mappings,
|
|
1784
|
+
client,
|
|
1785
|
+
orbitdb.ipfs.blockstore,
|
|
1786
|
+
{ ...config, helia: orbitdb.ipfs },
|
|
1787
|
+
);
|
|
1788
|
+
|
|
1789
|
+
if (successful.length === 0) {
|
|
1790
|
+
throw new Error("No blocks were successfully restored");
|
|
1791
|
+
}
|
|
1792
|
+
|
|
1793
|
+
if (matches.length < successful.length) {
|
|
1794
|
+
throw new Error("Some CID bridges failed - reconstruction may fail");
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
// Analyze blocks to find log entries and HEAD entries
|
|
1798
|
+
logger.info("🔍 Analyzing restored blocks to find log entries...");
|
|
1799
|
+
const downloadedBlocks = new Set(Array.from(mappings.keys())); // OrbitDB CIDs
|
|
1800
|
+
const analysis = await analyzeBlocks(
|
|
1801
|
+
orbitdb.ipfs.blockstore,
|
|
1802
|
+
downloadedBlocks,
|
|
1803
|
+
);
|
|
1804
|
+
|
|
1805
|
+
// Reconstruct database
|
|
1806
|
+
const databaseAddress = `/orbitdb/${manifestCID}`;
|
|
1807
|
+
logger.info(`📥 Opening database at: ${databaseAddress}`);
|
|
1808
|
+
|
|
1809
|
+
// Determine database type from manifest or analysis
|
|
1810
|
+
let databaseType = inferDatabaseType(analysis.logEntryBlocks);
|
|
1811
|
+
const manifestBlock = analysis.manifestBlocks.find(
|
|
1812
|
+
(m) => m.cid === manifestCID,
|
|
1813
|
+
);
|
|
1814
|
+
if (manifestBlock?.content?.type) {
|
|
1815
|
+
databaseType = manifestBlock.content.type;
|
|
1816
|
+
}
|
|
1817
|
+
|
|
1818
|
+
const reconstructedDB = await orbitdb.open(databaseAddress, {
|
|
1819
|
+
type: databaseType,
|
|
1820
|
+
});
|
|
1821
|
+
|
|
1822
|
+
// Join HEAD entries to reconstruct the log (similar to restoreDatabaseFromSpace)
|
|
1823
|
+
if (analysis.potentialHeads.length > 0) {
|
|
1824
|
+
logger.info(
|
|
1825
|
+
`🔗 Joining ${analysis.potentialHeads.length} HEAD entry/entries to reconstruct complete log...`,
|
|
1826
|
+
);
|
|
1827
|
+
let joinedHeads = 0;
|
|
1828
|
+
|
|
1829
|
+
for (const headCID of analysis.potentialHeads) {
|
|
1830
|
+
try {
|
|
1831
|
+
const logEntryBlock = analysis.logEntryBlocks.find(
|
|
1832
|
+
(block) => block.cid === headCID,
|
|
1833
|
+
);
|
|
1834
|
+
|
|
1835
|
+
if (!logEntryBlock) {
|
|
1836
|
+
logger.warn(
|
|
1837
|
+
`⚠️ HEAD ${headCID.slice(0, 12)}... not found in log entry blocks`,
|
|
1838
|
+
);
|
|
1839
|
+
continue;
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1842
|
+
// Create an entry object that matches OrbitDB's expected format
|
|
1843
|
+
const entryData = {
|
|
1844
|
+
hash: logEntryBlock.cid,
|
|
1845
|
+
v: logEntryBlock.content.v,
|
|
1846
|
+
id: logEntryBlock.content.id,
|
|
1847
|
+
key: logEntryBlock.content.key,
|
|
1848
|
+
sig: logEntryBlock.content.sig,
|
|
1849
|
+
next: logEntryBlock.content.next,
|
|
1850
|
+
refs: logEntryBlock.content.refs,
|
|
1851
|
+
clock: logEntryBlock.content.clock,
|
|
1852
|
+
payload: logEntryBlock.content.payload,
|
|
1853
|
+
identity: logEntryBlock.content.identity,
|
|
1854
|
+
};
|
|
1855
|
+
|
|
1856
|
+
// Use joinEntry - this will automatically traverse and join all connected entries
|
|
1857
|
+
const updated = await reconstructedDB.log.joinEntry(entryData);
|
|
1858
|
+
if (updated) {
|
|
1859
|
+
joinedHeads++;
|
|
1860
|
+
logger.info(
|
|
1861
|
+
`✓ Joined HEAD ${joinedHeads}/${analysis.potentialHeads.length}: ${headCID.slice(0, 12)}...`,
|
|
1862
|
+
);
|
|
1863
|
+
}
|
|
1864
|
+
} catch (joinError) {
|
|
1865
|
+
logger.warn(
|
|
1866
|
+
`⚠️ Failed to join HEAD ${headCID}: ${joinError.message}`,
|
|
1867
|
+
);
|
|
1868
|
+
}
|
|
1869
|
+
}
|
|
1870
|
+
|
|
1871
|
+
logger.info(
|
|
1872
|
+
`📊 Successfully joined ${joinedHeads}/${analysis.potentialHeads.length} HEAD entries`,
|
|
1873
|
+
);
|
|
1874
|
+
}
|
|
1875
|
+
|
|
1876
|
+
// Wait for log to settle after joining entries
|
|
1877
|
+
logger.info("⏳ Waiting for entries to load...");
|
|
1878
|
+
await new Promise((resolve) => setTimeout(resolve, config.timeout / 5));
|
|
1879
|
+
await new Promise((resolve) => setTimeout(resolve, config.timeout / 10));
|
|
1880
|
+
|
|
1881
|
+
// Handle different database types properly
|
|
1882
|
+
let reconstructedEntries;
|
|
1883
|
+
let entriesArray;
|
|
1884
|
+
let entriesCount;
|
|
1885
|
+
|
|
1886
|
+
if (reconstructedDB.type === "keyvalue") {
|
|
1887
|
+
reconstructedEntries = await reconstructedDB.all();
|
|
1888
|
+
// For key-value databases, all() returns an object
|
|
1889
|
+
const logEntries = await reconstructedDB.log.values();
|
|
1890
|
+
entriesArray = logEntries.map((logEntry) => ({
|
|
1891
|
+
hash: logEntry.hash,
|
|
1892
|
+
value: logEntry.payload.value,
|
|
1893
|
+
}));
|
|
1894
|
+
entriesCount = Object.keys(reconstructedEntries).length;
|
|
1895
|
+
} else {
|
|
1896
|
+
// For other database types (events, documents), all() returns an array
|
|
1897
|
+
reconstructedEntries = await reconstructedDB.all();
|
|
1898
|
+
entriesArray = Array.isArray(reconstructedEntries)
|
|
1899
|
+
? reconstructedEntries
|
|
1900
|
+
: [];
|
|
1901
|
+
entriesCount = entriesArray.length;
|
|
1902
|
+
}
|
|
1903
|
+
|
|
1904
|
+
logger.info("✅ Restore completed successfully!", entriesCount);
|
|
1905
|
+
|
|
1906
|
+
return {
|
|
1907
|
+
success: true,
|
|
1908
|
+
database: reconstructedDB,
|
|
1909
|
+
manifestCID,
|
|
1910
|
+
address: reconstructedDB.address,
|
|
1911
|
+
name: reconstructedDB.name,
|
|
1912
|
+
type: reconstructedDB.type,
|
|
1913
|
+
entriesRecovered: entriesCount,
|
|
1914
|
+
blocksRestored: successful.length,
|
|
1915
|
+
addressMatch: reconstructedDB.address === databaseAddress,
|
|
1916
|
+
entries: entriesArray.map((e) => ({
|
|
1917
|
+
hash: e.hash || e._id || e.id,
|
|
1918
|
+
value: e.value || e,
|
|
1919
|
+
})),
|
|
1920
|
+
};
|
|
1921
|
+
} catch (error) {
|
|
1922
|
+
logger.error("❌ Restore failed:", error.message);
|
|
1923
|
+
return {
|
|
1924
|
+
success: false,
|
|
1925
|
+
error: error.message,
|
|
1926
|
+
};
|
|
1927
|
+
} // No cleanup needed
|
|
1928
|
+
}
|
|
1929
|
+
|
|
1930
|
+
/**
|
|
1931
|
+
* Enhanced OrbitDBStorachaBridge class with event emission
|
|
1932
|
+
*/
|
|
1933
|
+
export class OrbitDBStorachaBridge extends EventEmitter {
|
|
1934
|
+
constructor(options = {}) {
|
|
1935
|
+
super();
|
|
1936
|
+
this.config = { ...DEFAULT_OPTIONS, ...options };
|
|
1937
|
+
}
|
|
1938
|
+
|
|
1939
|
+
/**
|
|
1940
|
+
* Back up a database. A CAR by default; pass `strategy: "blocks"` for the
|
|
1941
|
+
* pre-0.5.2 per-block path, which some backends will refuse.
|
|
1942
|
+
*/
|
|
1943
|
+
async backup(orbitdb, databaseAddress, options = {}) {
|
|
1944
|
+
// Pass this instance as eventEmitter to enable progress events
|
|
1945
|
+
return await backupDatabase(orbitdb, databaseAddress, {
|
|
1946
|
+
...this.config,
|
|
1947
|
+
...options,
|
|
1948
|
+
eventEmitter: this,
|
|
1949
|
+
});
|
|
1950
|
+
}
|
|
1951
|
+
|
|
1952
|
+
async backupLogEntriesOnly(orbitdb, databaseAddress, options = {}) {
|
|
1953
|
+
// Optimized backup for fallback reconstruction - only log entries
|
|
1954
|
+
return await backupDatabase(orbitdb, databaseAddress, {
|
|
1955
|
+
...this.config,
|
|
1956
|
+
...options,
|
|
1957
|
+
logEntriesOnly: true,
|
|
1958
|
+
eventEmitter: this,
|
|
1959
|
+
});
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1962
|
+
async restore(orbitdb, manifestCID, cidMappings, options = {}) {
|
|
1963
|
+
return await restoreDatabase(orbitdb, manifestCID, cidMappings, {
|
|
1964
|
+
...this.config,
|
|
1965
|
+
...options,
|
|
1966
|
+
});
|
|
1967
|
+
}
|
|
1968
|
+
|
|
1969
|
+
// BREAKTHROUGH: Mapping-independent restore
|
|
1970
|
+
async restoreFromSpace(orbitdb, options = {}) {
|
|
1971
|
+
return await restoreDatabaseFromSpace(orbitdb, {
|
|
1972
|
+
...this.config,
|
|
1973
|
+
...options,
|
|
1974
|
+
eventEmitter: this,
|
|
1975
|
+
});
|
|
1976
|
+
}
|
|
1977
|
+
|
|
1978
|
+
// OPTIMIZATION: Log-entries-only restore (much faster for fallback reconstruction)
|
|
1979
|
+
async restoreLogEntriesOnly(orbitdb, options = {}) {
|
|
1980
|
+
return await restoreLogEntriesOnly(orbitdb, {
|
|
1981
|
+
...this.config,
|
|
1982
|
+
...options,
|
|
1983
|
+
eventEmitter: this,
|
|
1984
|
+
});
|
|
1985
|
+
}
|
|
1986
|
+
|
|
1987
|
+
// Utility methods
|
|
1988
|
+
async listSpaceFiles(options = {}) {
|
|
1989
|
+
return await listStorachaSpaceFiles({ ...this.config, ...options });
|
|
1990
|
+
}
|
|
1991
|
+
|
|
1992
|
+
async analyzeBlocks(blockstore, downloadedBlocks) {
|
|
1993
|
+
return await analyzeBlocks(blockstore, downloadedBlocks);
|
|
1994
|
+
}
|
|
1995
|
+
|
|
1996
|
+
extractManifestCID(databaseAddress) {
|
|
1997
|
+
return extractManifestCID(databaseAddress);
|
|
1998
|
+
}
|
|
1999
|
+
|
|
2000
|
+
convertCID(storachaCID) {
|
|
2001
|
+
return convertStorachaCIDToOrbitDB(storachaCID);
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
|
|
2005
|
+
/**
|
|
2006
|
+
* Find the correct manifest block by matching log entries to database IDs
|
|
2007
|
+
*
|
|
2008
|
+
* @param {Object} analysis - Block analysis results
|
|
2009
|
+
* @returns {Object|null} - Correct manifest block or null if not found
|
|
2010
|
+
*/
|
|
2011
|
+
function findCorrectManifest(analysis) {
|
|
2012
|
+
logger.info("🎯 Finding correct manifest from log entries...");
|
|
2013
|
+
|
|
2014
|
+
if (analysis.manifestBlocks.length === 1) {
|
|
2015
|
+
logger.info(" ✅ Only one manifest found, using it");
|
|
2016
|
+
return analysis.manifestBlocks[0];
|
|
2017
|
+
}
|
|
2018
|
+
|
|
2019
|
+
if (analysis.manifestBlocks.length === 0) {
|
|
2020
|
+
logger.info(" ❌ No manifest blocks found");
|
|
2021
|
+
return null;
|
|
2022
|
+
}
|
|
2023
|
+
|
|
2024
|
+
// Extract database IDs from log entries
|
|
2025
|
+
const databaseIds = new Set();
|
|
2026
|
+
for (const logEntry of analysis.logEntryBlocks) {
|
|
2027
|
+
if (logEntry.content && logEntry.content.id) {
|
|
2028
|
+
// Extract manifest CID from database address like '/orbitdb/zdpu...'
|
|
2029
|
+
const manifestCID = logEntry.content.id.replace("/orbitdb/", "");
|
|
2030
|
+
databaseIds.add(manifestCID);
|
|
2031
|
+
logger.info(
|
|
2032
|
+
` 📝 Log entry references database: ${logEntry.content.id} (manifest: ${manifestCID})`,
|
|
2033
|
+
);
|
|
2034
|
+
}
|
|
2035
|
+
}
|
|
2036
|
+
|
|
2037
|
+
logger.info(
|
|
2038
|
+
` 🔍 Found ${databaseIds.size} unique database ID(s) from ${analysis.logEntryBlocks.length} log entries`,
|
|
2039
|
+
);
|
|
2040
|
+
|
|
2041
|
+
// Find manifest that matches the most referenced database ID
|
|
2042
|
+
const manifestCounts = new Map();
|
|
2043
|
+
for (const manifestBlock of analysis.manifestBlocks) {
|
|
2044
|
+
const count = databaseIds.has(manifestBlock.cid) ? 1 : 0;
|
|
2045
|
+
manifestCounts.set(manifestBlock.cid, count);
|
|
2046
|
+
logger.info(
|
|
2047
|
+
` 📋 Manifest ${manifestBlock.cid}: ${count > 0 ? "MATCHES" : "no match"}`,
|
|
2048
|
+
);
|
|
2049
|
+
}
|
|
2050
|
+
|
|
2051
|
+
// Find the manifest with the highest count (most log entry references)
|
|
2052
|
+
let bestManifest = null;
|
|
2053
|
+
let bestCount = -1;
|
|
2054
|
+
|
|
2055
|
+
for (const [manifestCID, count] of manifestCounts) {
|
|
2056
|
+
if (count > bestCount) {
|
|
2057
|
+
bestCount = count;
|
|
2058
|
+
bestManifest = analysis.manifestBlocks.find((m) => m.cid === manifestCID);
|
|
2059
|
+
}
|
|
2060
|
+
}
|
|
2061
|
+
|
|
2062
|
+
if (bestManifest && bestCount > 0) {
|
|
2063
|
+
logger.info(
|
|
2064
|
+
` ✅ Selected manifest: ${bestManifest.cid} (referenced by ${bestCount} log entries)`,
|
|
2065
|
+
);
|
|
2066
|
+
return bestManifest;
|
|
2067
|
+
}
|
|
2068
|
+
|
|
2069
|
+
// Fallback: if no manifest matches log entries, use the first one and warn
|
|
2070
|
+
logger.warn(
|
|
2071
|
+
" ⚠️ No manifest matched log entries, using first manifest as fallback",
|
|
2072
|
+
);
|
|
2073
|
+
return analysis.manifestBlocks[0];
|
|
2074
|
+
}
|
|
2075
|
+
|
|
2076
|
+
/**
|
|
2077
|
+
* Download only log entry blocks from Storacha (optimized for fallback reconstruction)
|
|
2078
|
+
*
|
|
2079
|
+
* @param {Array} spaceFiles - Array of space files to download
|
|
2080
|
+
* @param {Object} currentOrbitDB - OrbitDB instance
|
|
2081
|
+
* @param {Object} config - Configuration options
|
|
2082
|
+
* @param {EventEmitter} eventEmitter - Optional event emitter for progress updates
|
|
2083
|
+
* @returns {Promise<Map>} - Downloaded log entry blocks map
|
|
2084
|
+
*/
|
|
2085
|
+
async function downloadLogEntriesOnly(
|
|
2086
|
+
spaceFiles,
|
|
2087
|
+
currentOrbitDB,
|
|
2088
|
+
config,
|
|
2089
|
+
eventEmitter = null,
|
|
2090
|
+
) {
|
|
2091
|
+
logger.info("\n📥 Downloading and filtering log entry blocks only...");
|
|
2092
|
+
const logEntryBlocks = new Map();
|
|
2093
|
+
const totalFiles = spaceFiles.length;
|
|
2094
|
+
let completedFiles = 0;
|
|
2095
|
+
let logEntriesFound = 0;
|
|
2096
|
+
|
|
2097
|
+
// Emit initial progress
|
|
2098
|
+
if (eventEmitter) {
|
|
2099
|
+
eventEmitter.emit("downloadProgress", {
|
|
2100
|
+
type: "download",
|
|
2101
|
+
current: 0,
|
|
2102
|
+
total: totalFiles,
|
|
2103
|
+
percentage: 0,
|
|
2104
|
+
status: "starting (log entries only)",
|
|
2105
|
+
});
|
|
2106
|
+
}
|
|
2107
|
+
|
|
2108
|
+
for (const spaceFile of spaceFiles) {
|
|
2109
|
+
const storachaCID = spaceFile.root;
|
|
2110
|
+
logger.info(` 🔄 Checking: ${storachaCID}`);
|
|
2111
|
+
|
|
2112
|
+
try {
|
|
2113
|
+
// Ensure helia instance is passed for network downloads
|
|
2114
|
+
const downloadConfig = {
|
|
2115
|
+
...config,
|
|
2116
|
+
helia: config.helia || currentOrbitDB?.ipfs,
|
|
2117
|
+
};
|
|
2118
|
+
const bytes = await downloadBlock(storachaCID, downloadConfig);
|
|
2119
|
+
|
|
2120
|
+
// Convert Storacha CID to OrbitDB format
|
|
2121
|
+
const orbitdbCID = convertStorachaCIDToOrbitDB(storachaCID);
|
|
2122
|
+
const parsedCID = CID.parse(orbitdbCID);
|
|
2123
|
+
|
|
2124
|
+
// Only process dag-cbor blocks (potential log entries)
|
|
2125
|
+
if (parsedCID.code === 0x71) {
|
|
2126
|
+
try {
|
|
2127
|
+
const block = await Block.decode({
|
|
2128
|
+
cid: parsedCID,
|
|
2129
|
+
bytes,
|
|
2130
|
+
codec: dagCbor,
|
|
2131
|
+
hasher: sha256,
|
|
2132
|
+
});
|
|
2133
|
+
|
|
2134
|
+
const content = block.value;
|
|
2135
|
+
|
|
2136
|
+
// Check if this looks like an OrbitDB log entry
|
|
2137
|
+
if (
|
|
2138
|
+
content &&
|
|
2139
|
+
content.v === 2 &&
|
|
2140
|
+
content.id &&
|
|
2141
|
+
content.clock &&
|
|
2142
|
+
content.payload !== undefined
|
|
2143
|
+
) {
|
|
2144
|
+
// Store in target blockstore
|
|
2145
|
+
await currentOrbitDB.ipfs.blockstore.put(parsedCID, bytes);
|
|
2146
|
+
logEntryBlocks.set(orbitdbCID, {
|
|
2147
|
+
storachaCID,
|
|
2148
|
+
bytes: bytes.length,
|
|
2149
|
+
});
|
|
2150
|
+
logEntriesFound++;
|
|
2151
|
+
|
|
2152
|
+
logger.info(` ✅ Log entry stored: ${orbitdbCID}`);
|
|
2153
|
+
} else {
|
|
2154
|
+
logger.info(` ⚪ Skipped non-log block: ${orbitdbCID}`);
|
|
2155
|
+
}
|
|
2156
|
+
} catch {
|
|
2157
|
+
logger.info(` ⚪ Skipped non-decodable block: ${orbitdbCID}`);
|
|
2158
|
+
}
|
|
2159
|
+
} else {
|
|
2160
|
+
logger.info(` ⚪ Skipped non-CBOR block: ${orbitdbCID}`);
|
|
2161
|
+
}
|
|
2162
|
+
|
|
2163
|
+
// Update progress
|
|
2164
|
+
completedFiles++;
|
|
2165
|
+
if (eventEmitter) {
|
|
2166
|
+
eventEmitter.emit("downloadProgress", {
|
|
2167
|
+
type: "download",
|
|
2168
|
+
current: completedFiles,
|
|
2169
|
+
total: totalFiles,
|
|
2170
|
+
percentage: Math.round((completedFiles / totalFiles) * 100),
|
|
2171
|
+
status: "downloading (log entries only)",
|
|
2172
|
+
currentBlock: {
|
|
2173
|
+
storachaCID,
|
|
2174
|
+
orbitdbCID,
|
|
2175
|
+
size: bytes.length,
|
|
2176
|
+
isLogEntry: logEntryBlocks.has(orbitdbCID),
|
|
2177
|
+
},
|
|
2178
|
+
});
|
|
2179
|
+
}
|
|
2180
|
+
} catch (error) {
|
|
2181
|
+
logger.error(` ❌ Failed: ${storachaCID} - ${error.message}`);
|
|
2182
|
+
|
|
2183
|
+
// Update progress even for failed downloads
|
|
2184
|
+
completedFiles++;
|
|
2185
|
+
if (eventEmitter) {
|
|
2186
|
+
eventEmitter.emit("downloadProgress", {
|
|
2187
|
+
type: "download",
|
|
2188
|
+
current: completedFiles,
|
|
2189
|
+
total: totalFiles,
|
|
2190
|
+
percentage: Math.round((completedFiles / totalFiles) * 100),
|
|
2191
|
+
status: "downloading (log entries only)",
|
|
2192
|
+
error: {
|
|
2193
|
+
storachaCID,
|
|
2194
|
+
message: error.message,
|
|
2195
|
+
},
|
|
2196
|
+
});
|
|
2197
|
+
}
|
|
2198
|
+
}
|
|
2199
|
+
}
|
|
2200
|
+
|
|
2201
|
+
// Emit completion
|
|
2202
|
+
if (eventEmitter) {
|
|
2203
|
+
eventEmitter.emit("downloadProgress", {
|
|
2204
|
+
type: "download",
|
|
2205
|
+
current: totalFiles,
|
|
2206
|
+
total: totalFiles,
|
|
2207
|
+
percentage: 100,
|
|
2208
|
+
status: "completed (log entries only)",
|
|
2209
|
+
summary: {
|
|
2210
|
+
totalFiles: totalFiles,
|
|
2211
|
+
logEntriesFound: logEntriesFound,
|
|
2212
|
+
blocksSkipped: totalFiles - logEntriesFound,
|
|
2213
|
+
},
|
|
2214
|
+
});
|
|
2215
|
+
}
|
|
2216
|
+
|
|
2217
|
+
logger.info(
|
|
2218
|
+
` 📊 Found ${logEntriesFound} log entries out of ${totalFiles} total files`,
|
|
2219
|
+
);
|
|
2220
|
+
return logEntryBlocks;
|
|
2221
|
+
}
|
|
2222
|
+
|
|
2223
|
+
/**
|
|
2224
|
+
* Download blocks from Storacha with progress events
|
|
2225
|
+
*
|
|
2226
|
+
* @param {Array} spaceFiles - Array of space files to download
|
|
2227
|
+
* @param {Object} currentOrbitDB - OrbitDB instance
|
|
2228
|
+
* @param {Object} config - Configuration options
|
|
2229
|
+
* @param {EventEmitter} eventEmitter - Optional event emitter for progress updates
|
|
2230
|
+
* @returns {Promise<Map>} - Downloaded blocks map
|
|
2231
|
+
*/
|
|
2232
|
+
async function downloadBlocksWithProgress(
|
|
2233
|
+
spaceFiles,
|
|
2234
|
+
currentOrbitDB,
|
|
2235
|
+
config,
|
|
2236
|
+
eventEmitter = null,
|
|
2237
|
+
) {
|
|
2238
|
+
logger.info("\n📥 Downloading all space files...");
|
|
2239
|
+
const downloadedBlocks = new Map();
|
|
2240
|
+
const totalFiles = spaceFiles.length;
|
|
2241
|
+
let completedFiles = 0;
|
|
2242
|
+
|
|
2243
|
+
// Emit initial progress
|
|
2244
|
+
if (eventEmitter) {
|
|
2245
|
+
eventEmitter.emit("downloadProgress", {
|
|
2246
|
+
type: "download",
|
|
2247
|
+
current: 0,
|
|
2248
|
+
total: totalFiles,
|
|
2249
|
+
percentage: 0,
|
|
2250
|
+
status: "starting",
|
|
2251
|
+
});
|
|
2252
|
+
}
|
|
2253
|
+
|
|
2254
|
+
for (const spaceFile of spaceFiles) {
|
|
2255
|
+
const storachaCID = spaceFile.root;
|
|
2256
|
+
logger.info(` 🔄 Downloading: ${storachaCID}`);
|
|
2257
|
+
|
|
2258
|
+
try {
|
|
2259
|
+
// Ensure helia instance is passed for network downloads
|
|
2260
|
+
const downloadConfig = {
|
|
2261
|
+
...config,
|
|
2262
|
+
helia: config.helia || currentOrbitDB?.ipfs,
|
|
2263
|
+
};
|
|
2264
|
+
const bytes = await downloadBlock(storachaCID, downloadConfig);
|
|
2265
|
+
|
|
2266
|
+
// Convert Storacha CID to OrbitDB format
|
|
2267
|
+
const orbitdbCID = convertStorachaCIDToOrbitDB(storachaCID);
|
|
2268
|
+
const parsedCID = CID.parse(orbitdbCID);
|
|
2269
|
+
|
|
2270
|
+
// Store in target blockstore with error handling for closed blockstore
|
|
2271
|
+
if (!currentOrbitDB?.ipfs?.blockstore) {
|
|
2272
|
+
throw new Error(
|
|
2273
|
+
"Blockstore is not available - OrbitDB may have been closed",
|
|
2274
|
+
);
|
|
2275
|
+
}
|
|
2276
|
+
try {
|
|
2277
|
+
await currentOrbitDB.ipfs.blockstore.put(parsedCID, bytes);
|
|
2278
|
+
} catch (error) {
|
|
2279
|
+
if (
|
|
2280
|
+
error.message?.includes("not open") ||
|
|
2281
|
+
error.message?.includes("closed") ||
|
|
2282
|
+
error.message?.includes("Database is not open")
|
|
2283
|
+
) {
|
|
2284
|
+
throw new Error(
|
|
2285
|
+
`Blockstore operation failed - OrbitDB was closed during download: ${error.message}`, { cause: error },
|
|
2286
|
+
);
|
|
2287
|
+
}
|
|
2288
|
+
throw error;
|
|
2289
|
+
}
|
|
2290
|
+
downloadedBlocks.set(orbitdbCID, { storachaCID, bytes: bytes.length });
|
|
2291
|
+
|
|
2292
|
+
logger.info(` ✅ Stored: ${orbitdbCID}`);
|
|
2293
|
+
|
|
2294
|
+
// Update progress
|
|
2295
|
+
completedFiles++;
|
|
2296
|
+
if (eventEmitter) {
|
|
2297
|
+
eventEmitter.emit("downloadProgress", {
|
|
2298
|
+
type: "download",
|
|
2299
|
+
current: completedFiles,
|
|
2300
|
+
total: totalFiles,
|
|
2301
|
+
percentage: Math.round((completedFiles / totalFiles) * 100),
|
|
2302
|
+
status: "downloading",
|
|
2303
|
+
currentBlock: {
|
|
2304
|
+
storachaCID,
|
|
2305
|
+
orbitdbCID,
|
|
2306
|
+
size: bytes.length,
|
|
2307
|
+
},
|
|
2308
|
+
});
|
|
2309
|
+
}
|
|
2310
|
+
} catch (error) {
|
|
2311
|
+
logger.error(` ❌ Failed: ${storachaCID} - ${error.message}`);
|
|
2312
|
+
|
|
2313
|
+
// Update progress even for failed downloads
|
|
2314
|
+
completedFiles++;
|
|
2315
|
+
if (eventEmitter) {
|
|
2316
|
+
eventEmitter.emit("downloadProgress", {
|
|
2317
|
+
type: "download",
|
|
2318
|
+
current: completedFiles,
|
|
2319
|
+
total: totalFiles,
|
|
2320
|
+
percentage: Math.round((completedFiles / totalFiles) * 100),
|
|
2321
|
+
status: "downloading",
|
|
2322
|
+
error: {
|
|
2323
|
+
storachaCID,
|
|
2324
|
+
message: error.message,
|
|
2325
|
+
},
|
|
2326
|
+
});
|
|
2327
|
+
}
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
|
|
2331
|
+
// Emit completion
|
|
2332
|
+
if (eventEmitter) {
|
|
2333
|
+
eventEmitter.emit("downloadProgress", {
|
|
2334
|
+
type: "download",
|
|
2335
|
+
current: totalFiles,
|
|
2336
|
+
total: totalFiles,
|
|
2337
|
+
percentage: 100,
|
|
2338
|
+
status: "completed",
|
|
2339
|
+
summary: {
|
|
2340
|
+
downloaded: downloadedBlocks.size,
|
|
2341
|
+
failed: totalFiles - downloadedBlocks.size,
|
|
2342
|
+
},
|
|
2343
|
+
});
|
|
2344
|
+
}
|
|
2345
|
+
|
|
2346
|
+
logger.info(` 📊 Downloaded ${downloadedBlocks.size} blocks total`);
|
|
2347
|
+
return downloadedBlocks;
|
|
2348
|
+
}
|
|
2349
|
+
|
|
2350
|
+
/**
|
|
2351
|
+
* Fallback reconstruction when no manifest is found
|
|
2352
|
+
* Decodes blocks, extracts payloads, and creates a new database
|
|
2353
|
+
*
|
|
2354
|
+
* @param {Object} orbitdb - OrbitDB instance
|
|
2355
|
+
* @param {Map} downloadedBlocks - Downloaded blocks map
|
|
2356
|
+
* @param {Object} config - Configuration options
|
|
2357
|
+
* @param {boolean} useJoinEntry - Whether to use joinEntry for reconstruction
|
|
2358
|
+
* @returns {Promise<Object>} - Reconstruction results
|
|
2359
|
+
*/
|
|
2360
|
+
export async function reconstructWithoutManifest(
|
|
2361
|
+
orbitdb,
|
|
2362
|
+
downloadedBlocks,
|
|
2363
|
+
config,
|
|
2364
|
+
useJoinEntry = false,
|
|
2365
|
+
) {
|
|
2366
|
+
logger.info(
|
|
2367
|
+
`🔧 Starting fallback reconstruction without manifest ${useJoinEntry ? "with joinEntry support" : "(legacy mode)"}...`,
|
|
2368
|
+
);
|
|
2369
|
+
|
|
2370
|
+
const logEntries = [];
|
|
2371
|
+
const unknownBlocks = [];
|
|
2372
|
+
|
|
2373
|
+
// Step 1: Decode all blocks and identify log entries
|
|
2374
|
+
logger.info("🔍 Step 1: Decoding blocks to find log entries...");
|
|
2375
|
+
|
|
2376
|
+
for (const [cidString, _] of downloadedBlocks) {
|
|
2377
|
+
try {
|
|
2378
|
+
const cid = CID.parse(cidString);
|
|
2379
|
+
const bytes = await readBlockBytes(orbitdb.ipfs.blockstore, cid);
|
|
2380
|
+
|
|
2381
|
+
if (cid.code === 0x71) {
|
|
2382
|
+
// dag-cbor codec
|
|
2383
|
+
try {
|
|
2384
|
+
const block = await Block.decode({
|
|
2385
|
+
cid,
|
|
2386
|
+
bytes,
|
|
2387
|
+
codec: dagCbor,
|
|
2388
|
+
hasher: sha256,
|
|
2389
|
+
});
|
|
2390
|
+
|
|
2391
|
+
const content = block.value;
|
|
2392
|
+
|
|
2393
|
+
// Check if this looks like an OrbitDB log entry
|
|
2394
|
+
if (
|
|
2395
|
+
content &&
|
|
2396
|
+
content.v === 2 &&
|
|
2397
|
+
content.id &&
|
|
2398
|
+
content.clock &&
|
|
2399
|
+
content.payload !== undefined
|
|
2400
|
+
) {
|
|
2401
|
+
logEntries.push({
|
|
2402
|
+
cid: cidString,
|
|
2403
|
+
content,
|
|
2404
|
+
hash: cidString,
|
|
2405
|
+
payload: content.payload,
|
|
2406
|
+
});
|
|
2407
|
+
|
|
2408
|
+
logger.info(` 📝 Found log entry: ${cidString.slice(0, 12)}...`);
|
|
2409
|
+
}
|
|
2410
|
+
} catch (decodeError) {
|
|
2411
|
+
unknownBlocks.push({ cid: cidString, error: decodeError.message });
|
|
2412
|
+
}
|
|
2413
|
+
}
|
|
2414
|
+
} catch (error) {
|
|
2415
|
+
logger.warn(
|
|
2416
|
+
` ⚠️ Error processing block ${cidString}: ${error.message}`,
|
|
2417
|
+
);
|
|
2418
|
+
}
|
|
2419
|
+
}
|
|
2420
|
+
|
|
2421
|
+
logger.info(` ✅ Found ${logEntries.length} log entries`);
|
|
2422
|
+
|
|
2423
|
+
if (logEntries.length === 0) {
|
|
2424
|
+
throw new Error(
|
|
2425
|
+
"No OrbitDB log entries found in blocks - cannot reconstruct database",
|
|
2426
|
+
);
|
|
2427
|
+
}
|
|
2428
|
+
|
|
2429
|
+
// Step 2: Analyze payload patterns to determine database type
|
|
2430
|
+
logger.info(
|
|
2431
|
+
"🔍 Step 2: Analyzing payload patterns to determine database type...",
|
|
2432
|
+
);
|
|
2433
|
+
|
|
2434
|
+
const databaseType = inferDatabaseType(logEntries);
|
|
2435
|
+
logger.info(` 📊 Inferred database type: ${databaseType}`);
|
|
2436
|
+
|
|
2437
|
+
// Step 3: Create new database and import entries
|
|
2438
|
+
logger.info(`🆕 Step 3: Creating new ${databaseType} database...`);
|
|
2439
|
+
|
|
2440
|
+
const dbName = config.fallbackDatabaseName || `restored-${Date.now()}`;
|
|
2441
|
+
|
|
2442
|
+
// Merge config.dbConfig with inferred type, prioritizing the inferred type
|
|
2443
|
+
const dbOptions = {
|
|
2444
|
+
...config.dbConfig,
|
|
2445
|
+
type: databaseType,
|
|
2446
|
+
};
|
|
2447
|
+
|
|
2448
|
+
const database = await orbitdb.open(dbName, dbOptions);
|
|
2449
|
+
|
|
2450
|
+
logger.info(` ✅ Created database: ${database.address}`);
|
|
2451
|
+
|
|
2452
|
+
// Step 4: Import entries using appropriate method
|
|
2453
|
+
let importedCount = 0;
|
|
2454
|
+
const importErrors = [];
|
|
2455
|
+
|
|
2456
|
+
if (useJoinEntry) {
|
|
2457
|
+
logger.info("📥 Step 4: Using joinEntry to properly reconstruct log...");
|
|
2458
|
+
|
|
2459
|
+
// Sort by clock time to maintain order
|
|
2460
|
+
logEntries.sort((a, b) => {
|
|
2461
|
+
const timeA = a.content.clock?.time || 0;
|
|
2462
|
+
const timeB = b.content.clock?.time || 0;
|
|
2463
|
+
return timeA - timeB;
|
|
2464
|
+
});
|
|
2465
|
+
|
|
2466
|
+
for (const entry of logEntries) {
|
|
2467
|
+
try {
|
|
2468
|
+
// Use joinEntry to properly add this entry to the log
|
|
2469
|
+
const entryData = {
|
|
2470
|
+
hash: entry.hash,
|
|
2471
|
+
v: entry.content.v,
|
|
2472
|
+
id: entry.content.id,
|
|
2473
|
+
key: entry.content.key,
|
|
2474
|
+
sig: entry.content.sig,
|
|
2475
|
+
next: entry.content.next,
|
|
2476
|
+
refs: entry.content.refs,
|
|
2477
|
+
clock: entry.content.clock,
|
|
2478
|
+
payload: entry.content.payload,
|
|
2479
|
+
identity: entry.content.identity,
|
|
2480
|
+
};
|
|
2481
|
+
|
|
2482
|
+
const updated = await database.log.joinEntry(entryData);
|
|
2483
|
+
if (updated) {
|
|
2484
|
+
importedCount++;
|
|
2485
|
+
logger.info(
|
|
2486
|
+
` ✓ Joined entry ${importedCount}/${logEntries.length}: ${entry.hash.slice(0, 12)}...`,
|
|
2487
|
+
);
|
|
2488
|
+
} else {
|
|
2489
|
+
// Entry might already be in the log
|
|
2490
|
+
logger.info(
|
|
2491
|
+
` → Entry already processed: ${entry.hash.slice(0, 12)}...`,
|
|
2492
|
+
);
|
|
2493
|
+
}
|
|
2494
|
+
} catch (error) {
|
|
2495
|
+
importErrors.push({ entry: entry.hash, error: error.message });
|
|
2496
|
+
logger.warn(
|
|
2497
|
+
` ⚠️ Failed to join entry ${entry.hash.slice(0, 12)}...: ${error.message}`,
|
|
2498
|
+
);
|
|
2499
|
+
}
|
|
2500
|
+
}
|
|
2501
|
+
|
|
2502
|
+
logger.info(
|
|
2503
|
+
` 📊 Join complete: ${importedCount}/${logEntries.length} entries joined`,
|
|
2504
|
+
);
|
|
2505
|
+
} else {
|
|
2506
|
+
logger.info(
|
|
2507
|
+
"📥 Step 4: Importing entries in chronological order (legacy mode)...",
|
|
2508
|
+
);
|
|
2509
|
+
|
|
2510
|
+
// Sort by clock time to maintain order
|
|
2511
|
+
logEntries.sort((a, b) => {
|
|
2512
|
+
const timeA = a.content.clock?.time || 0;
|
|
2513
|
+
const timeB = b.content.clock?.time || 0;
|
|
2514
|
+
return timeA - timeB;
|
|
2515
|
+
});
|
|
2516
|
+
|
|
2517
|
+
for (const entry of logEntries) {
|
|
2518
|
+
try {
|
|
2519
|
+
await importEntryByType(database, entry, databaseType);
|
|
2520
|
+
importedCount++;
|
|
2521
|
+
logger.info(
|
|
2522
|
+
` ✅ Imported entry ${importedCount}/${logEntries.length}`,
|
|
2523
|
+
);
|
|
2524
|
+
} catch (error) {
|
|
2525
|
+
importErrors.push({ entry: entry.hash, error: error.message });
|
|
2526
|
+
logger.warn(
|
|
2527
|
+
` ⚠️ Failed to import entry ${entry.hash.slice(0, 12)}...: ${error.message}`,
|
|
2528
|
+
);
|
|
2529
|
+
}
|
|
2530
|
+
}
|
|
2531
|
+
}
|
|
2532
|
+
|
|
2533
|
+
logger.info(
|
|
2534
|
+
` 📊 ${useJoinEntry ? "Join" : "Import"} complete: ${importedCount}/${logEntries.length} entries ${useJoinEntry ? "joined" : "imported"}`,
|
|
2535
|
+
);
|
|
2536
|
+
|
|
2537
|
+
if (importErrors.length > 0) {
|
|
2538
|
+
logger.warn(
|
|
2539
|
+
` ⚠️ ${importErrors.length} ${useJoinEntry ? "join" : "import"} errors occurred`,
|
|
2540
|
+
);
|
|
2541
|
+
}
|
|
2542
|
+
|
|
2543
|
+
// Create fallback metadata
|
|
2544
|
+
const metadata = {
|
|
2545
|
+
type: "fallback-reconstruction",
|
|
2546
|
+
databaseType: databaseType,
|
|
2547
|
+
originalEntryCount: logEntries.length,
|
|
2548
|
+
importedEntryCount: importedCount,
|
|
2549
|
+
reconstructedAt: new Date().toISOString(),
|
|
2550
|
+
address: database.address.toString(),
|
|
2551
|
+
name: dbName,
|
|
2552
|
+
importErrors: importErrors.length,
|
|
2553
|
+
};
|
|
2554
|
+
|
|
2555
|
+
return {
|
|
2556
|
+
database,
|
|
2557
|
+
metadata,
|
|
2558
|
+
entriesCount: importedCount,
|
|
2559
|
+
};
|
|
2560
|
+
}
|
|
2561
|
+
|
|
2562
|
+
/**
|
|
2563
|
+
* Infer database type from payload patterns
|
|
2564
|
+
*
|
|
2565
|
+
* @param {Array} logEntries - Array of log entries
|
|
2566
|
+
* @returns {string} - Inferred database type
|
|
2567
|
+
*/
|
|
2568
|
+
function inferDatabaseType(logEntries) {
|
|
2569
|
+
const payloadPatterns = {
|
|
2570
|
+
hasDocumentOps: 0,
|
|
2571
|
+
hasKeyValueOps: 0,
|
|
2572
|
+
hasSimplePayloads: 0,
|
|
2573
|
+
hasCounterOps: 0,
|
|
2574
|
+
};
|
|
2575
|
+
|
|
2576
|
+
logger.info(
|
|
2577
|
+
` 🔍 Analyzing ${logEntries.length} log entries for database type...`,
|
|
2578
|
+
);
|
|
2579
|
+
|
|
2580
|
+
// Debug: Check structure of first few entries
|
|
2581
|
+
logger.debug(
|
|
2582
|
+
{
|
|
2583
|
+
firstEntryStructure:
|
|
2584
|
+
logEntries.length > 0 ? Object.keys(logEntries[0]) : "No entries",
|
|
2585
|
+
hasPayload:
|
|
2586
|
+
logEntries.length > 0
|
|
2587
|
+
? logEntries[0].payload
|
|
2588
|
+
? "exists"
|
|
2589
|
+
: "undefined"
|
|
2590
|
+
: "N/A",
|
|
2591
|
+
firstEntryPreview:
|
|
2592
|
+
logEntries.length > 0
|
|
2593
|
+
? JSON.stringify(logEntries[0], null, 2).slice(0, 500) + "..."
|
|
2594
|
+
: "N/A",
|
|
2595
|
+
},
|
|
2596
|
+
` 🔍 First entry structure`,
|
|
2597
|
+
);
|
|
2598
|
+
|
|
2599
|
+
for (const entry of logEntries) {
|
|
2600
|
+
// Fix: Access payload from entry.content.payload, not entry.payload
|
|
2601
|
+
const payload = entry.content ? entry.content.payload : entry.payload;
|
|
2602
|
+
|
|
2603
|
+
// Debug logging for first few entries
|
|
2604
|
+
if (
|
|
2605
|
+
payloadPatterns.hasDocumentOps +
|
|
2606
|
+
payloadPatterns.hasKeyValueOps +
|
|
2607
|
+
payloadPatterns.hasSimplePayloads +
|
|
2608
|
+
payloadPatterns.hasCounterOps <
|
|
2609
|
+
3
|
|
2610
|
+
) {
|
|
2611
|
+
logger.debug(
|
|
2612
|
+
{ payload },
|
|
2613
|
+
` 🔍 Entry ${payloadPatterns.hasDocumentOps + payloadPatterns.hasKeyValueOps + payloadPatterns.hasSimplePayloads + payloadPatterns.hasCounterOps + 1} payload`,
|
|
2614
|
+
);
|
|
2615
|
+
}
|
|
2616
|
+
|
|
2617
|
+
if (payload && typeof payload === "object") {
|
|
2618
|
+
// Check for document/keyvalue operation patterns
|
|
2619
|
+
if (
|
|
2620
|
+
(payload.op === "PUT" || payload.op === "DEL") &&
|
|
2621
|
+
payload.key !== undefined
|
|
2622
|
+
) {
|
|
2623
|
+
// Documents typically use _id as primary key or have _id in the value
|
|
2624
|
+
if (
|
|
2625
|
+
payload.key === "_id" ||
|
|
2626
|
+
payload.key.startsWith("_id") ||
|
|
2627
|
+
(payload.op === "PUT" &&
|
|
2628
|
+
payload.value &&
|
|
2629
|
+
typeof payload.value === "object" &&
|
|
2630
|
+
payload.value._id)
|
|
2631
|
+
) {
|
|
2632
|
+
payloadPatterns.hasDocumentOps++;
|
|
2633
|
+
} else {
|
|
2634
|
+
// Any PUT/DEL with a key that's not document-style is keyvalue
|
|
2635
|
+
payloadPatterns.hasKeyValueOps++;
|
|
2636
|
+
}
|
|
2637
|
+
} else if (
|
|
2638
|
+
payload.op === "COUNTER" ||
|
|
2639
|
+
payload.op === "DEC" ||
|
|
2640
|
+
payload.op === "INC"
|
|
2641
|
+
) {
|
|
2642
|
+
payloadPatterns.hasCounterOps++;
|
|
2643
|
+
} else if (payload.op === "ADD") {
|
|
2644
|
+
// Explicit ADD operation for events
|
|
2645
|
+
payloadPatterns.hasSimplePayloads++;
|
|
2646
|
+
} else {
|
|
2647
|
+
// Complex object without standard operation structure - likely events
|
|
2648
|
+
payloadPatterns.hasSimplePayloads++;
|
|
2649
|
+
}
|
|
2650
|
+
} else {
|
|
2651
|
+
// Simple payload (string, number, etc.) - likely events
|
|
2652
|
+
payloadPatterns.hasSimplePayloads++;
|
|
2653
|
+
}
|
|
2654
|
+
}
|
|
2655
|
+
|
|
2656
|
+
logger.info(" 📊 Payload analysis:", payloadPatterns);
|
|
2657
|
+
|
|
2658
|
+
// Log some sample payloads for debugging
|
|
2659
|
+
if (logEntries.length > 0) {
|
|
2660
|
+
const sampleEntries = logEntries.slice(0, 3);
|
|
2661
|
+
const samples = sampleEntries.map((entry, i) => {
|
|
2662
|
+
const p = entry.content ? entry.content.payload : entry.payload;
|
|
2663
|
+
if (p && typeof p === "object") {
|
|
2664
|
+
return {
|
|
2665
|
+
index: i + 1,
|
|
2666
|
+
op: p.op,
|
|
2667
|
+
keyType: typeof p.key,
|
|
2668
|
+
hasValue: p.value !== undefined,
|
|
2669
|
+
};
|
|
2670
|
+
} else {
|
|
2671
|
+
return { index: i + 1, simplePayloadType: typeof p };
|
|
2672
|
+
}
|
|
2673
|
+
});
|
|
2674
|
+
logger.debug({ samples }, ` 🔍 Sample payloads`);
|
|
2675
|
+
}
|
|
2676
|
+
|
|
2677
|
+
// Determine type based on majority pattern
|
|
2678
|
+
if (payloadPatterns.hasCounterOps > 0) {
|
|
2679
|
+
logger.info(
|
|
2680
|
+
{ counterOps: payloadPatterns.hasCounterOps },
|
|
2681
|
+
` 🎯 Detected: counter`,
|
|
2682
|
+
);
|
|
2683
|
+
return "counter";
|
|
2684
|
+
} else if (
|
|
2685
|
+
payloadPatterns.hasDocumentOps > payloadPatterns.hasKeyValueOps &&
|
|
2686
|
+
payloadPatterns.hasDocumentOps > payloadPatterns.hasSimplePayloads
|
|
2687
|
+
) {
|
|
2688
|
+
logger.info(
|
|
2689
|
+
{ documentOps: payloadPatterns.hasDocumentOps },
|
|
2690
|
+
` 🎯 Detected: documents`,
|
|
2691
|
+
);
|
|
2692
|
+
return "documents";
|
|
2693
|
+
} else if (
|
|
2694
|
+
payloadPatterns.hasKeyValueOps > payloadPatterns.hasSimplePayloads
|
|
2695
|
+
) {
|
|
2696
|
+
logger.info(
|
|
2697
|
+
{ keyValueOps: payloadPatterns.hasKeyValueOps },
|
|
2698
|
+
` 🎯 Detected: keyvalue`,
|
|
2699
|
+
);
|
|
2700
|
+
return "keyvalue";
|
|
2701
|
+
} else {
|
|
2702
|
+
logger.info(
|
|
2703
|
+
{ simplePayloads: payloadPatterns.hasSimplePayloads },
|
|
2704
|
+
` 🎯 Detected: events (fallback)`,
|
|
2705
|
+
);
|
|
2706
|
+
return "events"; // Default fallback
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
|
|
2710
|
+
/**
|
|
2711
|
+
* Import a log entry into the appropriate database type
|
|
2712
|
+
*
|
|
2713
|
+
* @param {Object} database - Target OrbitDB database
|
|
2714
|
+
* @param {Object} entry - Log entry to import
|
|
2715
|
+
* @param {string} databaseType - Database type
|
|
2716
|
+
*/
|
|
2717
|
+
async function importEntryByType(database, entry, databaseType) {
|
|
2718
|
+
const payload = entry.payload;
|
|
2719
|
+
|
|
2720
|
+
switch (databaseType) {
|
|
2721
|
+
case "events":
|
|
2722
|
+
// Events databases are append-only, only support ADD operations
|
|
2723
|
+
if (payload && payload.op === "ADD") {
|
|
2724
|
+
await database.add(payload.value || payload);
|
|
2725
|
+
} else {
|
|
2726
|
+
// Fallback: treat any payload as an event to add
|
|
2727
|
+
await database.add(payload);
|
|
2728
|
+
}
|
|
2729
|
+
break;
|
|
2730
|
+
|
|
2731
|
+
case "documents":
|
|
2732
|
+
if (payload && payload.op === "PUT" && payload.value) {
|
|
2733
|
+
await database.put(payload.value);
|
|
2734
|
+
} else if (payload && payload.op === "DEL" && payload.key) {
|
|
2735
|
+
// Delete document by key (usually _id)
|
|
2736
|
+
await database.del(payload.key);
|
|
2737
|
+
} else if (payload && typeof payload === "object") {
|
|
2738
|
+
// Fallback: treat as document even without operation structure
|
|
2739
|
+
await database.put(payload);
|
|
2740
|
+
} else {
|
|
2741
|
+
throw new Error("Invalid document payload structure");
|
|
2742
|
+
}
|
|
2743
|
+
break;
|
|
2744
|
+
|
|
2745
|
+
case "keyvalue":
|
|
2746
|
+
if (
|
|
2747
|
+
payload &&
|
|
2748
|
+
payload.op === "PUT" &&
|
|
2749
|
+
payload.key &&
|
|
2750
|
+
payload.value !== undefined
|
|
2751
|
+
) {
|
|
2752
|
+
await database.put(payload.key, payload.value); // Use put for keyvalue, not set
|
|
2753
|
+
} else if (payload && payload.op === "DEL" && payload.key) {
|
|
2754
|
+
// Delete by key
|
|
2755
|
+
await database.del(payload.key);
|
|
2756
|
+
} else {
|
|
2757
|
+
throw new Error("Invalid keyvalue payload structure");
|
|
2758
|
+
}
|
|
2759
|
+
break;
|
|
2760
|
+
|
|
2761
|
+
case "counter":
|
|
2762
|
+
if (payload && payload.op === "COUNTER") {
|
|
2763
|
+
await database.inc(payload.value || 1);
|
|
2764
|
+
} else if (payload && payload.op === "DEC") {
|
|
2765
|
+
// Decrement operation (negative increment)
|
|
2766
|
+
await database.inc(-(payload.value || 1));
|
|
2767
|
+
} else {
|
|
2768
|
+
// Fallback: try to increment by 1
|
|
2769
|
+
await database.inc(1);
|
|
2770
|
+
}
|
|
2771
|
+
break;
|
|
2772
|
+
|
|
2773
|
+
default:
|
|
2774
|
+
throw new Error(`Unsupported database type: ${databaseType}`);
|
|
2775
|
+
}
|
|
2776
|
+
}
|
|
2777
|
+
|
|
2778
|
+
// Export all utilities
|
|
2779
|
+
export {
|
|
2780
|
+
cleanupOrbitDBDirectories,
|
|
2781
|
+
createHeliaOrbitDB,
|
|
2782
|
+
initializeStorachaClient,
|
|
2783
|
+
initializeStorachaClientWithUCAN,
|
|
2784
|
+
resolveBackend,
|
|
2785
|
+
// Timestamped backup helpers
|
|
2786
|
+
generateBackupPrefix,
|
|
2787
|
+
getBackupFilenames,
|
|
2788
|
+
findLatestBackup,
|
|
2789
|
+
};
|