@yz-social/kdht 0.1.2 → 0.1.4

package/.github/workflows/ci.yml

@@ -0,0 +1,27 @@
+name: CI/CD Build and Test
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch: # Allows manual running
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest # Uses the latest Ubuntu runner
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v5 # Action to check out your repository code
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4 # Action to set up Node.js environment
+        with:
+          node-version: '22.x'
+
+      - name: Install Dependencies
+        run: npm i # Some folks recommend 'npm ci', but I don't like the required package lock.
+
+      - name: Run Tests
+        run: npm test # Executes the 'test' script defined in your package.json

package/dht/kbucket.js

@@ -41,10 +41,10 @@ export class KBucket {
     await this.node.locateNodes(targetKey); // Side-effect is to update this bucket.
     return true;
   }
-  resetRefresh() { // We are organically performing a lookup in this bucket. Reset the timer.
+  resetRefresh(now = false) { // We are organically performing a lookup in this bucket. Reset the timer.
     // clearInterval(this.refreshTimer);
     // this.refreshTimer = this.node.repeat(() => this.refresh(), 'bucket');
-    this.node.schedule(this.index, 'bucket', () => this.refresh());
+    this.node.schedule(this.index, 'bucket', () => this.refresh(), now ? 1 : undefined); // Not zero.
   }
 
   removeKey(key, deleteIfEmpty = true) { // Removes item specified by key (if present) from bucket and return 'present' if it was, else false.

package/dht/node.js

@@ -11,38 +11,97 @@ import { NodeProbe } from './nodeProbe.js';
 export class Node extends NodeProbe {
   // These are the public methods for applications.
 
-  async locateNodes(targetKey, number = this.constructor.k) { // Promise up to k best Contacts for targetKey (sorted closest first).
+  static diagnosticTrace = false; // Set to true for detailed store/read logging
+
+  async locateNodes(targetKey, number = this.constructor.k, includeSelf = false) { // Promise up to k best Contacts for targetKey (sorted closest first).
     // Side effect is to discover other nodes (and they us).
+    // includeSelf: If true, the local node is included as a candidate (useful for finding storage locations).
     targetKey = await this.ensureKey(targetKey);
-    return await this.iterate(targetKey, 'findNodes', number);
+    return await this.iterate(targetKey, 'findNodes', number, false, false, includeSelf);
   }
-  async locateValue(targetKey) { // Promise value stored for targetKey, or undefined.
+  async locateValue(targetKey, additionalTries = 1) { // Promise value stored for targetKey, or undefined.
     // Side effect is to discover other nodes (and they us).
     targetKey = await this.ensureKey(targetKey);
+    const trace = this.constructor.diagnosticTrace;
 
     // Optimization: should still work without this, but then there are more RPCs.
     const found = this.retrieveLocally(targetKey);
-    if (found !== undefined) return found;
+    if (found !== undefined) {
+      if (trace) this.log(`locateValue(${targetKey}): found locally =>`, found);
+      return found;
+    }
 
     const result = await this.iterate(targetKey, 'findValue');
-    if (Node.isValueResult(result)) return result.value;
+    if (Node.isValueResult(result)) {
+      if (trace) {
+        const responderInfo = result.responder ? ` (from ${result.responder.name})` : '';
+        this.log(`locateValue(${targetKey}): found in network =>`, result.value, responderInfo);
+      }
+      return result.value;
+    }
+    // Always log failures - this helps debug sporadic read failures
+    const queried = result.map(h => h.name).join(', ');
+    const distances = result.slice(0, 5).map(h => String(h.distance).length).join(',');
+    this.log(`locateValue(${targetKey}): NOT FOUND - queried ${result.length} nodes: ${queried} (dist digits: ${distances}...)`);
+
+    // Check which of the queried nodes actually have the value (for debugging)
+    const nodesWithValue = result.filter(h => h.node?.retrieveLocally(targetKey) !== undefined);
+    if (nodesWithValue.length > 0) {
+      this.log(`  BUG: ${nodesWithValue.length} queried nodes actually have the value: ${nodesWithValue.map(h => h.name).join(', ')}`);
+    }
     return undefined;
   }
   async storeValue(targetKey, value) { // Convert targetKey to a bigint if necessary, and store k copies.
     // Promises the number of nodes that it was stored on.
     targetKey = await this.ensureKey(targetKey);
+    const trace = this.constructor.diagnosticTrace;
+
+    // Early exit if this node is no longer running (e.g., disconnected during scheduled replication)
+    if (!this.isRunning) {
+      if (trace) this.log(`storeValue(${targetKey}, ${value}): aborted - node disconnected`);
+      return 0;
+    }
+
     // Go until we are sure have written k.
     const k = this.constructor.k;
     let remaining = k;
     // Ask for more, than needed, and then store to each, one at a time, until we
     // have replicated k times.
-    let helpers = await this.locateNodes(targetKey, remaining * 2);
+    let helpers = await this.locateNodes(targetKey, remaining * 2, true); // includeSelf: we're a valid storage location
+
+    // Check again after the async locateNodes call
+    if (!this.isRunning) {
+      if (trace) this.log(`storeValue(${targetKey}, ${value}): aborted after locateNodes - node disconnected`);
+      return 0;
+    }
+
+    if (trace) this.log(`storeValue(${targetKey}): locateNodes found ${helpers.length} helpers`);
     helpers = helpers.reverse(); // So we can save best-first by popping off the end.
+    const storedTo = []; // Track where we stored for diagnostics
     // TODO: batches in parallel, if the client and network can handle it. (For now, better to spread it out.)
     while (helpers.length && remaining) {
-      const contact = helpers.pop().contact;
+      const helper = helpers.pop();
+      const contact = helper.contact;
       const stored = await contact.store(targetKey, value);
-      if (stored) remaining--;
+      if (stored) {
+        remaining--;
+        storedTo.push(helper.name);
+      } else if (!this.isRunning) {
+        // Node disconnected mid-replication - no point continuing
+        if (trace) this.log(`storeValue(${targetKey}, ${value}): aborted mid-store - node disconnected`);
+        return k - remaining;
+      }
+    }
+    const storedCount = k - remaining;
+    if (trace || storedCount < k) {
+      // Explain why we got fewer than k stores
+      let reason = '';
+      if (!this.isRunning) {
+        reason = ' (node disconnected)';
+      } else if (helpers.length === 0 && storedCount < k) {
+        reason = ' (insufficient nodes found)';
+      }
+      this.log(`storeValue(${targetKey}, ${value}): stored to ${storedCount}/${k} nodes${storedTo.length ? ': ' + storedTo.join(', ') : ''}${reason}`);
     }
     return k - remaining;
   }
@@ -70,86 +129,4 @@ export class Node extends NodeProbe {
     this.log('joined', contact.sname);
     return this.contact; // Answering this node's home contact is handy for chaining or keeping track of contacts being made and joined.
   }
-
-
-  // TODO: separate all this out: neither transport nor dht.
-  // TODO: fragment/reassemble big messages.
-  messagePromises = new Map(); // maps message messageTags => responders, separately from inFlight, above
-  async message({targetKey, targetSname, excluded = [], requestTag, senderKey, senderSname = this.contact.sname, payload}) { // Send message to targetKey, using our existing routingTable contacts.
-    //const MAX_PING_MS = 400;
-    const MAX_MESSAGE_MS = 2e3;
-    let responsePromise = null;
-    if (!requestTag) { // If this is an outgoing request from us, promises the response.
-      requestTag = uuidv4();
-      senderKey = this.key;
-      senderSname = this.contact.sname;
-      responsePromise = new Promise(resolve => this.messagePromises.set(requestTag, resolve));
-    }
-    targetKey = targetKey.toString();
-    senderKey = senderKey?.toString();
-    // The excluded list of keys prevents cycles.
-    excluded.push(this.key.toString()); // TODO: Find a way to not have an excluded list, or at least add junk for privacy.
-    const body = JSON.stringify({targetKey, excluded, requestTag, senderKey, targetSname, senderSname, payload});
-    const contacts = this.findClosestHelpers(BigInt(targetKey)).map(helper => helper.contact).filter(contact => contact.key !== this.key);
-    this.log('=>', targetSname, 'message', requestTag, 'contacts:', contacts.map(c => c.sname));
-    for (const contact of contacts) {
-      if (excluded.includes(contact.key.toString())) { this.log('skiping excluded', contact.sname, 'for message', requestTag); continue; }
-      //this.xlog('trying message through', contact.sname);
-      if (!contact.connection) { this.log('skipping unconnected', contact.sname, 'for message', requestTag); continue; }
-      if (!(await contact.sendRPC('ping', contact.key))) {
-	this.xlog('failed to get ping', contact.sname, 'for message', requestTag);
-	await this.removeContact(contact);
-	continue;
-      }
-      let overlay = await contact.overlay;
-      //this.xlog('hopping through', contact.sname, 'for message', requestTag, 'from',  senderSname, 'to', targetSname);
-      overlay.send(body);
-      // FIXME: how do we know if a response was ultimately delivered? Don't we need a confirmation for that so that we can try a different route?
-      const result =  await responsePromise;
-      //this.xlog('got result for message', requestTag, 'through', contact.sname);
-      //const result =  await Promise.race([responsePromise, Node.delay(MAX_MESSAGE_MS, 'fixme')]);
-      // if (result === 'fixme') {
-      // 	this.xlog(`message timeout to ${targetSname} via ${contact.sname}.`);
-      // 	continue;  // If we timeout, responsePromise is still valid.
-      // }
-      return result;
-      break;
-    }
-    this.xlog(`No connected contacts to send message ${requestTag} to ${targetSname} among ${contacts.map(c => `${excluded.includes(c.key.toString()) ? 'excluded/' : (c.connection ? '' : 'unconnected/')}${c.sname}`).join(', ')}`);
-    return null;
-  }
-  async messageHandler(dataString) {
-    //this.log('got overlay', dataString);
-    const data = JSON.parse(dataString);
-    const {targetKey, excluded, requestTag, senderKey, targetSname, senderSname, payload} = data;
-    Node.assert(targetKey, 'No targetKey to handle', dataString);
-    Node.assert(payload, 'No payload to handle', dataString);    
-    Node.assert(requestTag, 'No request tag by which to answer', dataString);
-    this.log('handling message', requestTag, 'from', senderSname, 'to', targetSname);
-
-    // If intended for another node, pass it on.    
-    if (BigInt(targetKey) !== this.key) return await this.message(data);
-
-    // If it is a response to something we sent, resolve the waiting promise with the payload.
-    const responder = this.messagePromises.get(requestTag);
-    this.messagePromises.delete(requestTag);
-    if (responder) return responder(payload);
-
-    // Finally, answer a request to us by messaging the sender with the same-tagged response.
-    const [method, ...args] = payload;
-    if (!(method in this)) return this.xlog('ignoring unrecognized request', {requestTag, method, args}); // Could be a double response.
-
-    Node.assert(args[0] === senderSname, 'FIXME sender does not match signals payload sender', dataString);
-    this.log('executing', method, 'from', senderSname, 'request:', requestTag);
-    let response = await this[method](...args);
-    Node.assert(senderKey, 'No sender key by which to answer', dataString);    
-    this.log('responding to', method, 'from', senderSname, 'request:', requestTag);
-    return await this.message({targetKey: BigInt(senderKey), targetSname: senderSname, requestTag, payload: response});
-  }
-  async signal(...rest) {
-    //this.xlog(`handling signals @ ${this.key} ${rest}`);
-    const fixme = await this.contact.signals(...rest);
-    //this.xlog(`returning signals @ ${this.key} ${fixme}`);
-    return fixme;
-  }
 }

package/dht/nodeContacts.js

@@ -72,25 +72,27 @@ export class NodeContacts extends NodeTransports {
   findContactByKey(key) { // findContact matching the specified key. To be found, contact must be in routingTable or looseTransports (which is different from existingContact()).
     return this.findContact(contact => contact.key === key);
   }
-  ensureContact(contact, sponsor = null) { // Return existing contact, if any (including looseTransports), else clone a new one for this host. Set sponsor.
+  ensureContact(contact, sponsor) { // Return existing contact, if any (including looseTransports), else clone a new one for this host. Set sponsor.
+    // I.e., a Contact with node: contact.node and host: this.
     // Subtle: Contact clone uses existingContact (above) to reuse an existing contact on the host, if possible.
     // This is vital for bookkeeping through connections and sponsorship.
     contact = contact.clone(this);
-    contact.noteSponsor(sponsor);
+    if (sponsor) contact.noteSponsor(sponsor);
     return contact;
   }
   routingTableSerializer = Promise.resolve();
   queueRoutingTableChange(thunk) { // Promise to resolve thunk() -- after all previous queued thunks have resolved.
     return this.routingTableSerializer = this.routingTableSerializer.then(thunk);
   }
-  removeContact(contact) { // Removes from node entirely if present, from looseTransports or bucket as necessary.
+  removeContact(contact) { // Removes from node entirely if present, from looseTransports or bucket as necessary, returning bucket if that's where it was, else null.
     return this.queueRoutingTableChange(() => {
       delete this.contactDictionary[contact.name];
       const key = contact.key;
-      if (this.removeLooseTransport(key)) return;
+      if (this.removeLooseTransport(key)) return null;
       const bucketIndex = this.getBucketIndex(key);
       const bucket = this.routingTable.get(bucketIndex);
-      bucket?.removeKey(key); // Host might not yet have added node or anyone else as contact for that bucket yet, so maybe no bucket.
+      // Host might not yet have added node or anyone else as contact for that bucket yet, so maybe no bucket.
+      return bucket?.removeKey(key) ? bucket : null;
     });
   }
   addToRoutingTable(contact) { // Promise contact, and add it to the routing table if room.

package/dht/nodeMessages.js

@@ -1,15 +1,19 @@
 import { NodeContacts } from './nodeContacts.js';
+import { Contact } from '../transports/contact.js';
 
 // The four methods we recevieve through RPCs.
 // These are not directly invoked by a Node on itself, but rather on other nodes
 // through Contact sendRPC.
 export class NodeMessages extends NodeContacts {
   ping(key) { // Respond with 'pong'. (RPC mechanism doesn't call unless connected.)
-    return 'pong';
+    return 'pong'; // Answer something truthy. See isValueResult.
   }
   store(key, value) { // Tell the node to store key => value, returning truthy.
+    if (this.constructor.diagnosticTrace) {
+      this.log(`store RPC received: key=${key}, value=${value}`);
+    }
     this.storeLocally(key, value);
-    return 'pong';
+    return 'pong'; // Answer something truthy. See isValueResult.
   }
   findNodes(key) { // Return k closest Contacts from routingTable.
     // TODO: Currently, this answers a list of Helpers. For security, it should be changed to a list of serialized Contacts.
@@ -21,9 +25,45 @@ export class NodeMessages extends NodeContacts {
     if (value !== undefined) return {value};
     return this.findClosestHelpers(key);
   }
-  receiveRPC(method, sender, ...rest) {
-    // The sender exists, so add it to the routing table, but give it a while to finish joining.
+  async signals(key, signals, forwardingExclusions = false) {
+    const origin = signals[0];
+    //this.xlog(this.key, 'handling signals request for', {origin, key, signals, forwardingExclusions});
+    //await this.constructor.delay(100); // fixme remove
+    if (!this.isRunning) return null;
+    if (this.key === key) return await this.contact.signals(...signals); // Yay, us!
+
+    let contact = this.findContactByKey(key); // If we have the target as a contact, use it directly.
+    if (!contact && !forwardingExclusions) return null;
+    if (contact) return await contact.sendRPC('signals', key, signals, forwardingExclusions);
+    // Forward recursively.
+    const contacts = this.findClosestHelpers(key).map(helper => helper.contact);
+    forwardingExclusions.push(this.name);
+    //this.xlog('forwarding signals', {key, forwardingExclusions, contacts: contacts.map(c => c.sname)});
+    for (const contact of contacts) {
+      // this.xlog('contact', contact.sname, contact.isRunning ? 'running' : 'dead',
+      // 		contact.connection ? 'connected': 'unconnected',
+      // 		forwardingExclusions.includes(contact.name) ? 'excluded' : 'allowed');
+      if (!contact.isRunning) continue;
+      if (!contact.connection) continue;
+      if (forwardingExclusions.includes(contact.name)) continue;
+      const response = await contact.sendRPC('signals', key, signals, forwardingExclusions);
+      //this.xlog('got response from', contact.sname);
+      if (response) return response;
+    }
+    return null;
+  }
+
+  messageResolvers = new Map(); // maps outgoing message tag => promise resolver being waited on.
+  receiveRPC(method, sender, ...rest) { // Process a deserialized RPC request, dispatching it to one of the above.
+    this.constructor.assert(typeof(method)==='string', 'no method', method, sender, rest);
+    this.constructor.assert(sender instanceof Contact, 'no sender', method, sender, rest);
+    this.constructor.assert(sender.host.key === this.key, 'sender', sender.host.name, 'not on receiver', this.name);
+    // The sender exists, so add it to the routing table, but asynchronously so as to allow it to finish joining.
     this.addToRoutingTable(sender);
+    if (!(method in this)) {
+      this.xlog('Does not handle method', method);
+      return null;
+    }
     return this[method](...rest);
   }
 }