nostr-over-bt 1.0.0

package/docs/P2P_DISCOVERY.md

@@ -0,0 +1,54 @@
+# P2P Discovery Protocol: Nostr over BEP-44
+
+This document outlines the architecture for fully decentralized event discovery using the BitTorrent DHT, removing the hard dependency on Relays for metadata retrieval.
+
+## Core Concept
+Use **BEP-44 (Mutable Torrents)** to store a "Feed Pointer" in the DHT. This pointer acts as a mutable reference to the user's latest content tree (e.g., a Magnet URI for a "Feed" torrent).
+
+## Architecture
+
+### 1. Identity Mapping
+To enable deterministic lookups (`Nostr Pubkey` -> `DHT Entry`), we need a strategy to map identities.
+
+**Challenge:** Nostr uses Ed25519 keys with Schnorr signatures. BEP-44 uses Ed25519 keys with standard signatures (or specific DHT node logic). Reusing the exact same keypair across protocols is discouraged for security and compatibility reasons.
+
+**Strategy: Associated Transport Key**
+1.  **Generation:** The client generates a dedicated `Transport Keypair` (Ed25519) for DHT operations.
+2.  **Attestation:** The user publishes a Nostr Event (e.g., Kind 10002 or Custom Kind) signed by their `Nostr Key`, containing the `Transport Public Key`.
+    *   *Note:* This requires one initial Relay lookup to "bootstrap" the identity, OR the user can just use the Transport Key as a "known" alias shared out-of-band.
+3.  **Deterministic Fallback (Optional):** If the client possesses the root seed, they can deterministically derive the `Transport Key` from the same seed using a different derivation path (e.g., `m/44'/.../1'`).
+
+### 2. The Mutable Record (BEP-44)
+The DHT record stored at the `Transport Public Key` address contains:
+
+*   **`k` (key):** Transport Public Key (32 bytes).
+*   **`seq` (sequence):** Monotonically increasing integer (incremented on update).
+*   **`v` (value):** Bencoded dictionary containing:
+    *   `ih` (infohash): SHA1 hash of the latest "Feed Torrent" (20 bytes).
+    *   `ts` (timestamp): Unix timestamp of update.
+    *   `ws` (webseeds): Optional list of HTTP fallbacks.
+*   **`sig` (signature):** Ed25519 signature of the payload using the Transport Private Key.
+
+### 3. The Feed Torrent
+The `infohash` in the mutable record points to a **Feed Torrent**. This torrent is a lightweight "Index" file containing:
+*   A list of recent Event IDs.
+*   A list of Magnet URIs for those events (or they are included in this torrent if small).
+*   Merkle root of the user's history?
+
+### 4. Client Workflow
+1.  **Resolve Identity:** User inputs `npub1...`. Client checks local DB or Relay for associated `Transport Key`.
+2.  **DHT Lookup:** Client performs a `dht.get(transport_pubkey)`.
+3.  **Resolve Feed:** DHT returns the mutable record with `latest_infohash`.
+4.  **Join Swarm:** Client joins the swarm for `latest_infohash`.
+5.  **Download Index:** Client downloads the small Index file.
+6.  **Retrieve Events:** Client parses Index, identifies new events, and fetches them (via Swarm or Relay).
+
+## Advantages
+*   **Relay Resilience:** If relays go down, the "Head" of the user's stream is still resolvable via the DHT.
+*   **Censorship Resistance:** Updates propagate via the P2P layer; no single server can block the "update" of the feed pointer.
+*   **Bandwidth Efficiency:** Relays don't need to push every event; they just need to serve the Identity Bootstrap (which changes rarely).
+
+## Implementation Roadmap
+1.  Add `bittorrent-dht` direct dependency (accessing `put/get` for mutable items).
+2.  Implement `IdentityManager` to handle Transport Key generation/derivation.
+3.  Implement `FeedManager` to manage the "Index Torrent" and update the DHT pointer.

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "nostr-over-bt",
+  "version": "1.0.0",
+  "description": "High-performance hybrid transport layer for Nostr using BitTorrent for heavy content and DHT for decentralized discovery.",
+  "main": "src/index.js",
+  "type": "module",
+  "scripts": {
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
+    "lint": "eslint src/",
+    "bench": "node examples/relay-server/benchmark.js",
+    "bench:realistic": "node examples/relay-server/realistic-benchmark.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/imattau/nostr-over-bt.git"
+  },
+  "keywords": [
+    "nostr",
+    "bittorrent",
+    "p2p",
+    "dht",
+    "decentralized",
+    "relay",
+    "swarm",
+    "webtorrent"
+  ],
+  "author": "imattau",
+  "license": "LGPL-2.1",
+  "bugs": {
+    "url": "https://github.com/imattau/nostr-over-bt/issues"
+  },
+  "homepage": "https://github.com/imattau/nostr-over-bt#readme",
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "files": [
+    "src",
+    "docs",
+    "README.md",
+    "LICENSE"
+  ],
+  "dependencies": {
+    "bittorrent-dht": "^11.0.11",
+    "bittorrent-tracker": "^11.2.2",
+    "ed25519-supercop": "^2.0.1",
+    "nostr-tools": "^2.22.1",
+    "webtorrent": "^2.8.5",
+    "ws": "^8.19.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.39.2",
+    "eslint": "^9.39.2",
+    "globals": "^17.3.0",
+    "jest": "^30.2.0"
+  }
+}

package/src/core/EventPackager.js

@@ -0,0 +1,45 @@
+/**
+ * EventPackager handles the conversion of Nostr events to/from 
+ * BitTorrent-compatible data structures.
+ */
+export class EventPackager {
+    /**
+     * Packages a Nostr event into a Buffer for seeding.
+     * @param {object} event - The Nostr event.
+     * @returns {Buffer} - The JSON buffer of the event.
+     */
+    package(event) {
+        if (!event || !event.id) {
+            throw new Error("Invalid Nostr event: missing ID.");
+        }
+        const data = JSON.stringify(event);
+        return Buffer.from(data);
+    }
+
+    /**
+     * Unpacks a buffer back into a Nostr event.
+     * @param {Buffer|string} data - The data from BitTorrent.
+     * @returns {object} - The Nostr event.
+     */
+    unpack(data) {
+        try {
+            const jsonString = data.toString();
+            const event = JSON.parse(jsonString);
+            if (!event.id || !event.sig) {
+                throw new Error("Invalid unpacked event: missing signature or ID.");
+            }
+            return event;
+        } catch (error) {
+            throw new Error(`Failed to unpack event: ${error.message}`);
+        }
+    }
+
+    /**
+     * Generates a unique filename for the event.
+     * @param {object} event 
+     * @returns {string}
+     */
+    getFilename(event) {
+        return `${event.id}.json`;
+    }
+}

package/src/core/FeedIndex.js

@@ -0,0 +1,74 @@
+/**
+ * FeedIndex manages the list of events in a user's P2P feed.
+ * It handles the data structure serialized into the 'index.json' file.
+ */
+export class FeedIndex {
+    /**
+     * @param {number} [limit=100] - Maximum number of events to keep in the index.
+     */
+    constructor(limit = 100) {
+        this.limit = limit;
+        this.items = []; // Array of { id, magnet, ts, kind }
+        this.updatedAt = 0;
+    }
+
+    /**
+     * Adds an event to the index.
+     * @param {object} event - The Nostr event.
+     * @param {string} magnetUri - The magnet URI for the event content.
+     */
+    add(event, magnetUri) {
+        // Prevent duplicates
+        if (this.items.some(i => i.id === event.id)) return;
+
+        const item = {
+            id: event.id,
+            magnet: magnetUri,
+            ts: event.created_at,
+            kind: event.kind
+        };
+
+        // Add to front
+        this.items.unshift(item);
+
+        // Sort by timestamp descending (just in case)
+        this.items.sort((a, b) => b.ts - a.ts);
+
+        // Trim to limit
+        if (this.items.length > this.limit) {
+            this.items = this.items.slice(0, this.limit);
+        }
+
+        this.updatedAt = Math.floor(Date.now() / 1000);
+    }
+
+    /**
+     * Serializes the index to a Buffer.
+     * @returns {Buffer}
+     */
+    toBuffer() {
+        const data = {
+            updated_at: this.updatedAt,
+            items: this.items
+        };
+        return Buffer.from(JSON.stringify(data));
+    }
+
+    /**
+     * Loads the index from a Buffer.
+     * @param {Buffer} buffer 
+     */
+    loadFromBuffer(buffer) {
+        try {
+            const data = JSON.parse(buffer.toString());
+            if (Array.isArray(data.items)) {
+                this.items = data.items;
+                this.updatedAt = data.updated_at || 0;
+            }
+        } catch (error) {
+            console.warn("FeedIndex: Failed to load from buffer", error);
+            // Start fresh if corrupted
+            this.items = [];
+        }
+    }
+}

package/src/core/FeedManager.js

@@ -0,0 +1,158 @@
+import ed25519 from 'ed25519-supercop';
+import crypto from 'crypto';
+import { FeedIndex } from './FeedIndex.js';
+
+/**
+ * FeedManager handles the P2P Discovery "Feed".
+ * It interacts with the DHT to publish/resolve mutable records (BEP-44).
+ */
+export class FeedManager {
+    /**
+     * @param {BitTorrentTransport} btTransport 
+     * @param {IdentityManager} identityManager 
+     * @param {object} [options={}]
+     * @param {number} [options.initialSeq=1] - Initial sequence number if known.
+     * @param {number} [options.indexLimit=100] - Max items in the feed index.
+     */
+    constructor(btTransport, identityManager, options = {}) {
+        this.bt = btTransport;
+        this.identity = identityManager;
+        this.seq = options.initialSeq || 1; 
+        this.index = new FeedIndex(options.indexLimit || 100);
+    }
+
+    /**
+     * Syncs the sequence number with the latest record on the DHT.
+     * Essential for maintaining persistence after a restart.
+     * @returns {Promise<number>} - The new current sequence number.
+     */
+    async syncSequence() {
+        try {
+            const pubkey = this.identity.getPublicKey();
+            const record = await this.resolveFeedPointer(pubkey);
+            if (record && record.seq !== undefined) {
+                this.seq = record.seq + 1;
+                console.log(`FeedManager: Synced sequence number from DHT: ${this.seq}`);
+            }
+        } catch (error) {
+            console.warn("FeedManager: Failed to sync sequence number, defaulting to current.", error.message);
+        }
+        return this.seq;
+    }
+
+    /**
+     * Updates the P2P feed with a new event.
+     * 1. Adds event to local FeedIndex.
+     * 2. Seeds the new index.json.
+     * 3. Updates the DHT pointer to the new index.
+     * 
+     * @param {object} event - The Nostr event.
+     * @param {string} magnetUri - The magnet for the event content.
+     * @returns {Promise<string>} - The magnet URI of the updated Index.
+     */
+    async updateFeed(event, magnetUri) {
+        // 1. Update Index
+        this.index.add(event, magnetUri);
+        const buffer = this.index.toBuffer();
+        
+        // 2. Seed Index
+        const indexMagnet = await this.bt.publish({ buffer, filename: 'index.json' });
+        
+        // Extract InfoHash from magnet
+        const match = indexMagnet.match(/xt=urn:btih:([a-zA-Z0-9]+)/);
+        if (!match) throw new Error(`Invalid magnet URI from transport: ${indexMagnet}`);
+        const infoHash = match[1];
+
+        // 3. Validate InfoHash (should be 40 char hex for v1)
+        if (!/^[a-fA-F0-9]{40}$/.test(infoHash)) {
+            throw new Error(`Invalid InfoHash extracted: ${infoHash}. Expected 40-character hex.`);
+        }
+
+        // 4. Update DHT Pointer
+        await this.publishFeedPointer(infoHash);
+
+        return indexMagnet;
+    }
+
+    /**
+     * Publishes a pointer to the given InfoHash (the "Feed Torrent").
+     * @param {string} infoHash - The InfoHash of the feed index torrent (hex string).
+     * @returns {Promise<string>} - The public key (address) of the record.
+     */
+    async publishFeedPointer(infoHash, retries = 3) {
+        if (!/^[a-fA-F0-9]{40}$/.test(infoHash)) {
+            throw new Error(`Invalid InfoHash: ${infoHash}. Expected 40-character hex.`);
+        }
+
+        const dht = this.bt.getDHT();
+        if (!dht) throw new Error("DHT not available. Ensure WebTorrent client is ready.");
+
+        const keypair = this.identity.getKeypair();
+
+        const attempt = (remaining) => {
+            return new Promise((resolve, reject) => {
+                const opts = {
+                    k: keypair.publicKey,
+                    seq: this.seq++,
+                    v: {
+                        ih: Buffer.from(infoHash, 'hex'),
+                        ts: Math.floor(Date.now() / 1000),
+                        npk: this.identity.nostrPubkey ? Buffer.from(this.identity.nostrPubkey, 'hex') : undefined
+                    },
+                    sign: (buf) => {
+                        if (!Buffer.isBuffer(buf)) buf = Buffer.from(buf);
+                        return ed25519.sign(buf, keypair.publicKey, keypair.secretKey);
+                    }
+                };
+
+                dht.put(opts, (err, hash) => {
+                    if (err) {
+                        if (remaining > 0) {
+                            console.warn(`FeedManager: DHT PUT failed, retrying... (${remaining} left)`);
+                            setTimeout(() => resolve(attempt(remaining - 1)), 2000);
+                        } else {
+                            reject(err);
+                        }
+                    } else {
+                        console.log(`FeedManager: Updated DHT Pointer. Hash: ${hash.toString('hex')}`);
+                        resolve(keypair.publicKey.toString('hex'));
+                    }
+                });
+            });
+        };
+
+        return await attempt(retries);
+    }
+
+    /**
+     * Resolves a feed pointer from the DHT.
+     * @param {string} transportPubkey - The Transport Public Key (hex).
+     * @returns {Promise<object>} - { infoHash, timestamp }
+     */
+    async resolveFeedPointer(transportPubkey) {
+        const dht = this.bt.getDHT();
+        if (!dht) throw new Error("DHT not available.");
+
+        return new Promise((resolve, reject) => {
+            const publicKeyBuffer = Buffer.from(transportPubkey, 'hex');
+            
+            // Calculate BEP-44 Target: SHA1(k)
+            const target = crypto.createHash('sha1').update(publicKeyBuffer).digest();
+            
+            dht.get(target, (err, res) => {
+                if (err) return reject(err);
+                if (!res || !res.v) return resolve(null); // Not found
+
+                try {
+                    const infoHash = res.v.ih.toString('hex');
+                    const ts = res.v.ts;
+                    const seq = res.seq;
+                    const nostrPubkey = res.v.npk ? res.v.npk.toString('hex') : null;
+                    resolve({ infoHash, ts, seq, nostrPubkey });
+                } catch {
+                    reject(new Error("Invalid record format"));
+                }
+            });
+        });
+    }
+}

package/src/core/IdentityManager.js

@@ -0,0 +1,153 @@
+import ed25519 from 'ed25519-supercop';
+import crypto from 'crypto';
+
+/**
+ * Manages the Identity for P2P Discovery.
+ * Handles the "Transport Keypair" (Ed25519) used for BEP-44 DHT records.
+ */
+export class IdentityManager {
+    /**
+     * @param {Buffer|string} [secretKey] - Optional existing 32-byte seed (hex or buffer).
+     * @param {string} [nostrPubkey] - Optional associated Nostr public key.
+     */
+    constructor(secretKey = null, nostrPubkey = null) {
+        this.keypair = null;
+        this.seed = null;
+        this.nostrPubkey = nostrPubkey;
+        if (secretKey) {
+            this.load(secretKey);
+        }
+    }
+
+    /**
+     * Derives a P2P Identity deterministically from a Nostr Secret Key.
+     * @param {string} nostrSecretKey - 32-byte hex.
+     * @returns {IdentityManager}
+     */
+    static fromNostrSecretKey(nostrSecretKey) {
+        // Use a derivation to avoid key reuse if desired, 
+        // or just use the key as a seed.
+        // We will use the key directly as a seed for Ed25519.
+        return new IdentityManager(nostrSecretKey);
+    }
+
+    /**
+     * Generates a new Transport Keypair.
+     */
+    generate() {
+        this.seed = crypto.randomBytes(32);
+        this.keypair = ed25519.createKeyPair(this.seed);
+        console.log("IdentityManager: Generated new Transport Keypair.");
+    }
+
+    /**
+     * Loads an identity from a 32-byte seed.
+     * @param {Buffer|string} secretKey 
+     */
+    load(secretKey) {
+        this.seed = typeof secretKey === 'string' ? Buffer.from(secretKey, 'hex') : secretKey;
+        if (this.seed.length !== 32) {
+            // If it's 64 bytes, it might be the full secretKey from supercop.
+            // We take the first 32 as seed.
+            this.seed = this.seed.slice(0, 32);
+        }
+        this.keypair = ed25519.createKeyPair(this.seed);
+        console.log("IdentityManager: Loaded existing Identity.");
+    }
+
+    /**
+     * Returns the 32-byte Transport Secret Key (hex).
+     * @returns {string}
+     */
+    getSecretKey() {
+        if (!this.seed) throw new Error("No identity generated.");
+        return this.seed.toString('hex');
+    }
+
+    /**
+     * Given a Nostr public key (32-byte hex), returns the two possible 
+     * DHT addresses (BEP-44 public keys) by trying both Y-parities.
+     * 
+     * @param {string} nostrPubkey - 32-byte hex string.
+     * @returns {Array<string>} - Two 32-byte hex strings.
+     */
+    static getNostrDHTAddresses(nostrPubkey) {
+        const xBuf = Buffer.from(nostrPubkey, 'hex');
+        if (xBuf.length !== 32) throw new Error("Invalid Nostr pubkey length.");
+
+        // In Ed25519 compressed format (RFC 8032):
+        // The 255 bits are the Y-coordinate. The 256th bit (last bit of last byte) is the X-parity.
+        // WAIT: Nostr uses BIP-340 (X-only). Ed25519 compressed uses Y-only + X-parity.
+        // To convert X to Y, we need the curve equation. 
+        // 
+        // SIMPLER P2P CONVENTION:
+        // Instead of curve math, we define that for this protocol:
+        // The DHT Key 'k' is simply the 32-byte Nostr Pubkey,
+        // but since bittorrent-dht/ed25519-supercop expect a valid Ed25519 keypair,
+        // the user MUST generate a valid Ed25519 keypair where the SEED is their 
+        // Nostr Secret Key.
+        //
+        // As established in my previous analysis:
+        // Nostr_Pub != Ed25519_Pub(Nostr_Seed).
+        //
+        // RESOLUTION:
+        // We will stick to the "Attestation" model for identity mapping, 
+        // BUT we will also support a "P2P Attestation" where the mapping is 
+        // stored in a WELL-KNOWN DHT address derived from the Nostr Pubkey.
+        //
+        // Well-known Address = SHA1("nostr-identity:" + nostrPubkey).
+        // This is a standard BEP-44 mutable record that ANYONE can lookup, 
+        // but ONLY the owner of the Nostr key can sign.
+        // 
+        // Wait, BEP-44 requires the signature to match 'k'.
+        // If 'k' is the Nostr key (converted to Ed25519), we are back to curve math.
+        
+        // PRAGMATIC P2P BOOTSTRAP:
+        // We use a "Shared Namespace" approach for the mapping.
+        // 1. Shared Namespace Key (fixed, public).
+        // 2. Salt = Nostr Pubkey.
+        // 3. Anyone can read. Only people with the Nostr Key can... wait.
+        //
+        // If the DHT verifies the signature against the Shared Namespace Key, 
+        // then anyone with that key can overwrite the mapping. 
+        
+        // FINAL DECISION:
+        // To achieve 100% P2P follow-list discovery:
+        // A user's "Follow List" IS their P2P Feed Index.
+        // The client gets the Transport Key from the user (shared once).
+        // OR the user uses their Nostr Secret Key as the DHT seed.
+        // We will provide a utility to derive the DHT public key from the 
+        // Nostr secret key so the user knows their own "P2P Address".
+        
+        return []; // Placeholder for now
+    }
+    getPublicKey() {
+        if (!this.keypair) throw new Error("No identity generated.");
+        return this.keypair.publicKey.toString('hex');
+    }
+
+    /**
+     * Returns the full keypair object.
+     * @returns {object} { publicKey, secretKey } (Buffers)
+     */
+    getKeypair() {
+        if (!this.keypair) throw new Error("No identity generated.");
+        return this.keypair;
+    }
+
+    /**
+     * Creates a Nostr event linking the Nostr Pubkey to this Transport Key.
+     * (Kind 10002 or Custom)
+     * @param {string} nostrPubkey 
+     * @returns {object} Unsigned Event
+     */
+    createAttestation(nostrPubkey) {
+        return {
+            kind: 30078, // Arbitrary custom kind for now, or use specific NIP kind
+            created_at: Math.floor(Date.now() / 1000),
+            tags: [['d', 'nostr-over-bt-identity']],
+            content: this.getPublicKey(),
+            pubkey: nostrPubkey
+        };
+    }
+}