@le-space/orbitdb-storage-bridge 0.10.0

package/lib/utils.js

@@ -0,0 +1,316 @@
+/**
+ * OrbitDB Storage Bridge - Utility Functions
+ *
+ * Common utility functions for OrbitDB operations and cleanup
+ */
+
+import { identify } from "@libp2p/identify";
+import { noise } from "@chainsafe/libp2p-noise";
+import { yamux } from "@chainsafe/libp2p-yamux";
+import { tcp } from "@libp2p/tcp";
+import { bootstrap } from "@libp2p/bootstrap";
+import { gossipsub } from "@libp2p/gossipsub";
+import { kadDHT } from "@libp2p/kad-dht";
+import { ping } from "@libp2p/ping";
+import { createHelia } from "helia";
+import { unixfs } from "@helia/unixfs";
+import { createOrbitDB } from "@orbitdb/core";
+import { LevelBlockstore } from "blockstore-level";
+import { LevelDatastore } from "datastore-level";
+import { logger, createLogger } from "./logger.js";
+
+const HELIA_LOGGER = createLogger("helia");
+const HELIA_COLOR = "\x1b[95m";
+const COLOR_RESET = "\x1b[0m";
+
+function logHelia(message) {
+  HELIA_LOGGER(`${HELIA_COLOR}${message}${COLOR_RESET}`);
+}
+
+function attachHeliaLogging(libp2p, label) {
+  const prefix = label ? `[${label}] ` : "";
+  const safePeer = (peer) =>
+    typeof peer?.toString === "function" ? peer.toString() : `${peer ?? ""}`;
+
+  libp2p.addEventListener("peer:connect", (evt) => {
+    logHelia(`${prefix}peer:connect ${safePeer(evt?.detail)}`);
+  });
+  libp2p.addEventListener("peer:disconnect", (evt) => {
+    logHelia(`${prefix}peer:disconnect ${safePeer(evt?.detail)}`);
+  });
+  libp2p.addEventListener("peer:discovery", (evt) => {
+    logHelia(`${prefix}peer:discovery ${safePeer(evt?.detail?.id)}`);
+  });
+  libp2p.addEventListener("connection:open", (evt) => {
+    logHelia(`${prefix}connection:open ${safePeer(evt?.detail?.remotePeer)}`);
+  });
+  libp2p.addEventListener("connection:close", (evt) => {
+    logHelia(`${prefix}connection:close ${safePeer(evt?.detail?.remotePeer)}`);
+  });
+}
+
+/**
+ * Bootstrap nodes for connecting to the public IPFS network
+ * Based on working implementation from commit 556736f
+ */
+const BOOTSTRAP_NODES = [
+  // Official IPFS bootstrap nodes
+  "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
+  "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
+  "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
+  "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
+  // Direct IP bootstrap (more reliable in restricted networks)
+  "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
+  "/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
+];
+
+/**
+ * Clean up OrbitDB directories
+ */
+export async function cleanupOrbitDBDirectories() {
+  const fs = await import("fs");
+
+  try {
+    const entries = await fs.promises.readdir(".", { withFileTypes: true });
+    const orbitdbDirs = entries.filter(
+      (entry) =>
+        entry.isDirectory() &&
+        (entry.name.startsWith("orbitdb-bridge-") ||
+          entry.name.includes("orbitdb-bridge-")),
+    );
+
+    for (const dir of orbitdbDirs) {
+      try {
+        await fs.promises.rm(dir.name, { recursive: true, force: true });
+        logger.info(`Cleaned up: ${dir.name}`);
+        logger.info({ directory: dir.name }, `🧹 Cleaned up: ${dir.name}`);
+      } catch (error) {
+        logger.warn(`Could not clean up ${dir.name}: ${error.message}`);
+      }
+    }
+
+    if (orbitdbDirs.length === 0) {
+      logger.info("🧹 No OrbitDB directories to clean up");
+    } else {
+      logger.info(`Cleaned up ${orbitdbDirs.length} OrbitDB directories`);
+    }
+  } catch (error) {
+    logger.warn(`Cleanup warning: ${error.message}`);
+  }
+}
+
+/**
+ * Clean up all test-related directories and CAR files
+ * This function removes all common test artifacts created during testing
+ */
+export async function cleanupAllTestArtifacts() {
+  const fs = await import("fs");
+  // const path = await import('path') // Currently unused
+
+  logger.info("🧹 Starting comprehensive test cleanup...");
+
+  // Test directories to clean up
+  const testDirectories = [
+    "./test-car-storage-bridge",
+    "./test-orbitdb-car-integration",
+    "./test-preservation",
+    "./test-advanced",
+    "./test-data",
+    "./test-hash-preservation",
+    "./orbitdb-todo-restored",
+    "./orbitdb-todo-source",
+    "./helia-car-demo",
+    "./storage-demo",
+  ];
+
+  // CAR files to clean up (common patterns)
+  // const carFilePatterns = [
+  //   'test-*.car',
+  //   '*-test.car',
+  //   'todos-backup.car',
+  //   'storacha-hash-preservation-test.car'
+  // ]
+
+  let cleanedDirs = 0;
+  let cleanedFiles = 0;
+
+  // Clean up test directories
+  for (const dir of testDirectories) {
+    try {
+      await fs.promises.rm(dir, { recursive: true, force: true });
+      logger.info(`Removed test directory: ${dir}`);
+      cleanedDirs++;
+    } catch (error) {
+      if (error.code !== "ENOENT") {
+        logger.warn(`Could not remove ${dir}: ${error.message}`);
+      }
+    }
+  }
+
+  // Clean up CAR files
+  try {
+    const entries = await fs.promises.readdir(".", { withFileTypes: true });
+    const carFiles = entries.filter(
+      (entry) => entry.isFile() && entry.name.endsWith(".car"),
+    );
+
+    for (const carFile of carFiles) {
+      try {
+        await fs.promises.unlink(carFile.name);
+        logger.info(`Removed CAR file: ${carFile.name}`);
+        cleanedFiles++;
+      } catch (error) {
+        logger.warn(`Could not remove ${carFile.name}: ${error.message}`);
+      }
+    }
+  } catch (error) {
+    logger.warn(`Error scanning for CAR files: ${error.message}`);
+  }
+
+  // Also clean up OrbitDB directories
+  await cleanupOrbitDBDirectories();
+
+  logger.info(
+    `Comprehensive cleanup completed: ${cleanedDirs} directories, ${cleanedFiles} CAR files`,
+  );
+}
+
+/**
+ * Clean up specific test directory and associated CAR files
+ * @param {string} testDir - The test directory path
+ * @param {string} [carPrefix] - Optional prefix for CAR files to clean
+ */
+export async function cleanupTestDirectory(testDir, carPrefix = "") {
+  const fs = await import("fs");
+
+  try {
+    // Remove test directory
+    await fs.promises.rm(testDir, { recursive: true, force: true });
+    logger.info(`Removed test directory: ${testDir}`);
+
+    // Remove associated CAR files if prefix provided
+    if (carPrefix) {
+      try {
+        const entries = await fs.promises.readdir(".", { withFileTypes: true });
+        const carFiles = entries.filter(
+          (entry) =>
+            entry.isFile() &&
+            entry.name.endsWith(".car") &&
+            entry.name.includes(carPrefix),
+        );
+
+        for (const carFile of carFiles) {
+          await fs.promises.unlink(carFile.name);
+          logger.info(`Removed CAR file: ${carFile.name}`);
+        }
+      } catch (error) {
+        logger.warn(
+          `Error cleaning CAR files with prefix ${carPrefix}: ${error.message}`,
+        );
+      }
+    }
+  } catch (error) {
+    if (error.code !== "ENOENT") {
+      logger.warn(
+        `Could not clean up test directory ${testDir}: ${error.message}`,
+      );
+    }
+  }
+}
+
+/**
+ * Create a Helia/OrbitDB instance with specified suffix
+ * Enhanced with bootstrap nodes for connecting to the public IPFS network
+ * @param {string} suffix - Suffix for directory names
+ * @param {Object} options - Configuration options
+ * @param {boolean} options.dhtClientMode - If true, use DHT in client-only mode (default: true).
+ *                                          Client mode works better behind NAT. Set to false for
+ *                                          full DHT server mode (requires public IP).
+ */
+export async function createHeliaOrbitDB(suffix = "", options = {}) {
+  const {
+    dhtClientMode = true,
+    useBootstrap = true,
+    useDHT = true,
+    maxConnections = 30,
+  } = options; // Default to client mode for better NAT compatibility
+
+  const listenAddresses = process.env.LIBP2P_LISTEN_ADDRS
+    ? process.env.LIBP2P_LISTEN_ADDRS.split(",").map((addr) => addr.trim())
+    : process.env.JEST_WORKER_ID || process.env.NODE_ENV === "test"
+      ? ["/ip4/127.0.0.1/tcp/0"]
+      : ["/ip4/0.0.0.0/tcp/0"];
+
+  const services = {
+    identify: identify(),
+    pubsub: gossipsub({ allowPublishToZeroTopicPeers: true }),
+    ping: ping(),
+  };
+  if (useDHT) {
+    services.dht = kadDHT({
+      clientMode: dhtClientMode,
+    });
+  }
+  // Bootstrapping is peer discovery, not a service.
+  const peerDiscovery = useBootstrap
+    ? [
+        bootstrap({
+          list: BOOTSTRAP_NODES,
+          tagName: "bootstrap",
+          tagValue: 50,
+          tagTTL: 120000, // 2 minutes
+        }),
+      ]
+    : [];
+
+  // Helia 7 builds the libp2p node itself, from these options — it does not
+  // take a node built elsewhere. Every key left out is filled from Helia's own
+  // defaults, which dial the public network (mDNS, public bootstrap peers,
+  // UPnP, a relay server), so each key this node depends on is spelled out,
+  // peerDiscovery included, even when it is empty.
+  const libp2pOptions = {
+    addresses: {
+      listen: listenAddresses,
+    },
+    transports: [
+      tcp({
+        // Enable connection limits - reduced to prevent excessive connections
+        maxConnections: {
+          inbound: 20,
+          outbound: 20,
+        },
+      }),
+    ],
+    connectionEncrypters: [noise()],
+    streamMuxers: [yamux()],
+    connectionManager: {
+      // Limit total connections to prevent excessive network usage
+      maxConnections,
+    },
+    peerDiscovery,
+    services,
+  };
+
+  const uniqueId = `${Date.now()}-${Math.random().toString(36).slice(2, 11)}`;
+  const blockstore = new LevelBlockstore(
+    `./orbitdb-bridge-${uniqueId}${suffix}`,
+  );
+  const datastore = new LevelDatastore(
+    `./orbitdb-bridge-${uniqueId}${suffix}-data`,
+  );
+
+  const helia = await createHelia({
+    libp2p: libp2pOptions,
+    blockstore,
+    datastore,
+  }).start();
+  const libp2p = helia.libp2p;
+  attachHeliaLogging(libp2p, suffix || "orbitdb");
+  const fs = unixfs(helia);
+  const orbitdb = await createOrbitDB({
+    ipfs: helia,
+    directory: `./orbitdb-bridge-${uniqueId}${suffix}-orbitdb`,
+  });
+
+  return { helia, orbitdb, libp2p, blockstore, datastore, unixfs: fs };
+}

package/package.json

@@ -0,0 +1,172 @@
+{
+  "name": "@le-space/orbitdb-storage-bridge",
+  "version": "0.10.0",
+  "description": "Back up, restore and replicate OrbitDB databases through pluggable storage backends, with hash and identity preservation",
+  "main": "lib/orbitdb-storacha-bridge.js",
+  "svelte": "dist/components/",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./lib/orbitdb-storacha-bridge.d.ts",
+      "default": "./lib/orbitdb-storacha-bridge.js"
+    },
+    "./backup-car": "./lib/backup-car.js",
+    "./courier-sync": "./lib/courier-sync.js",
+    "./pointer-ipns": "./lib/pointer-ipns.js",
+    "./dehydrate": "./lib/dehydrate.js",
+    "./restore-cid": "./lib/restore-cid.js",
+    "./gateway-fetch": "./lib/gateway-fetch.js",
+    "./backends/types": "./lib/backends/types.js",
+    "./backends/memory": "./lib/backends/memory.js",
+    "./backends/storacha": "./lib/backends/storacha.js",
+    "./backends/aleph": "./lib/backends/aleph.js",
+    "./backends/aleph-pin": "./lib/backends/aleph-pin.js",
+    "./backends/pinata": "./lib/backends/pinata.js",
+    "./backends/lighthouse": "./lib/backends/lighthouse.js",
+    "./memory-courier": "./lib/memory-courier.js",
+    "./StorachaIntegration.svelte": "./dist/components/StorachaIntegration.svelte",
+    "./StorachaAuth.svelte": "./dist/components/StorachaAuth.svelte",
+    "./storacha-backup": "./dist/components/storacha-backup.js",
+    "./components/*": "./dist/components/*"
+  },
+  "scripts": {
+    "test": "NODE_OPTIONS='--experimental-vm-modules --no-warnings' jest",
+    "test:verbose": "npm test -- --verbose",
+    "test:integration": "npm test -- test/integration.test.js",
+    "test:p256-ucan": "npm test -- test/p256-ucan-security.test.js",
+    "test:car-backup": "npm test -- test/backup-car.test.js",
+    "test:timestamped-backup": "npm test -- test/timestamped-backup.test.js",
+    "test:network-download": "npm test -- test/network-download.test.js",
+    "test:backends": "npm test -- test/backends/",
+    "clear-space": "node examples/storacha/clear-space.js",
+    "car-demo": "node examples/storacha/car-backup-demo.js",
+    "ucan-demo": "node examples/storacha/ucan-demo.js",
+    "lint": "eslint --config eslint.config.js '{lib,examples,test}/**/*.js' --ignore-pattern 'examples/svelte/*/build/**' --ignore-pattern 'examples/svelte/*/dist/**' --ignore-pattern 'examples/svelte/**/.svelte-kit/**' --ignore-pattern 'node_modules/**'",
+    "format": "prettier --write lib/ examples/ test/ --ignore-path '.prettierignore'",
+    "clean": "rm -rf ./cid-bridge-test* ./demo-test*",
+    "build:components": "mkdir -p dist/components && cp src/components/*.svelte dist/components/ && cp src/components/*.js dist/components/",
+    "prepublishOnly": "npm run build:components"
+  },
+  "keywords": [
+    "orbitdb",
+    "storacha",
+    "filecoin",
+    "ipfs",
+    "backup",
+    "replication",
+    "web3",
+    "decentralized",
+    "database",
+    "cid",
+    "bridge",
+    "storage",
+    "restore",
+    "car",
+    "aleph",
+    "pinata",
+    "local-first"
+  ],
+  "files": [
+    "lib",
+    "dist",
+    "!dist/components/StorachaTest.svelte",
+    "!dist/components/StorachaTestWithReplication.svelte",
+    "!dist/components/StorachaTestWithWebAuthn.svelte",
+    "README.md",
+    "LICENSE"
+  ],
+  "author": "OrbitDB Storage Bridge Contributors",
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@chainsafe/libp2p-noise": "^17.0.0",
+    "@chainsafe/libp2p-yamux": "^8.0.1",
+    "@helia/block-brokers": "^5.2.4",
+    "@helia/routers": "^5.1.1",
+    "@helia/unixfs": "^8.0.7",
+    "@ipld/car": "^5.4.1",
+    "@ipld/dag-cbor": "^9.0.6",
+    "@ipld/dag-ucan": "^3.4.5",
+    "@libp2p/bootstrap": "^12.0.27",
+    "@libp2p/crypto": "^5.1.21",
+    "@libp2p/gossipsub": "^16.1.0",
+    "@libp2p/identify": "^4.1.10",
+    "@libp2p/kad-dht": "^16.4.5",
+    "@libp2p/logger": "^6.2.13",
+    "@libp2p/ping": "^3.1.9",
+    "@libp2p/tcp": "^11.0.28",
+    "@orbitdb/core": "^4.0.0",
+    "@scure/bip39": "^1.3.0",
+    "@storacha/capabilities": "^2.0.0",
+    "@storacha/client": "^2.0.0",
+    "blockstore-level": "^4.0.1",
+    "cbor-web": "^10.0.11",
+    "datastore-level": "^13.0.1",
+    "dotenv": "^17.2.1",
+    "helia": "^7.1.12",
+    "libp2p": "^3.3.6",
+    "multiformats": "^14.0.5"
+  },
+  "devDependencies": {
+    "@eslint/js": "^10.0.1",
+    "@ibm/plex": "^6.4.1",
+    "@jest/globals": "^30.5.1",
+    "@le-space/ucanto-principal": "^9.1.1",
+    "@libp2p/websockets": "^10.1.17",
+    "@orbitdb/identity-provider-did": "^1.0.2",
+    "@storacha/did-mailto": "^1.0.1",
+    "@storacha/upload-api": "^3.0.0",
+    "@types/jest": "^30.0.0",
+    "@ucanto/client": "^9.0.2",
+    "@ucanto/core": "^10.4.0",
+    "@ucanto/principal": "^9.0.2",
+    "@ucanto/server": "^11.0.3",
+    "@ucanto/transport": "^9.2.1",
+    "@web3-storage/sigv4": "^1.0.2",
+    "carbon-components-svelte": "^0.89.7",
+    "carbon-icons-svelte": "^12.17.0",
+    "eslint": "^10.10.0",
+    "jest": "^30.5.1",
+    "key-did-provider-ed25519": "^4.0.2",
+    "key-did-resolver": "^4.0.0",
+    "prettier": "^3.0.3",
+    "prettier-plugin-svelte": "^3.4.0",
+    "prettier-plugin-tailwindcss": "^0.7.0"
+  },
+  "peerDependencies": {
+    "lucide-svelte": ">=0.400.0 <1.0.0",
+    "svelte": "^3.0.0 || ^4.0.0 || ^5.0.0"
+  },
+  "engines": {
+    "node": ">=22.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/NiKrause/orbitdb-storage-bridge.git"
+  },
+  "bugs": {
+    "url": "https://github.com/NiKrause/orbitdb-storage-bridge/issues"
+  },
+  "homepage": "https://github.com/NiKrause/orbitdb-storage-bridge#readme",
+  "jest": {
+    "testEnvironment": "node",
+    "testTimeout": 60000,
+    "verbose": true,
+    "passWithNoTests": true,
+    "maxWorkers": 1,
+    "testPathIgnorePatterns": [
+      "/node_modules/",
+      "/examples/",
+      "/dist/",
+      "/coverage/"
+    ],
+    "moduleNameMapper": {
+      "^@peculiar/webcrypto$": "<rootDir>/node_modules/@peculiar/webcrypto/build/webcrypto.js"
+    }
+  },
+  "overrides": {
+    "test-exclude": "^8.0.0"
+  }
+}