@yz-social/kdht 0.1.0

package/README.md

@@ -0,0 +1,31 @@
+# KDHT
+
+A Kademlia Distributed Hash Table
+See [paper](https://www.scs.stanford.edu/~dm/home/papers/kpos.pdf) and [wikipedia](https://en.wikipedia.org/wiki/Kademlia)
+
+This repo allows us to experiment with a pure kdht, to see the effects of changes and and optimizations.
+For example, there is a class that implements connections to nodes running on the same computer, so that tests can be run without any networking at all (as long as the CPU isn't overwhelmed from running too many nodes).
+There is a repeatable test suite that can be used to confirm behavior as changes are made.
+
+### Classes
+
+A [Node](./dht/node.js) is an actor in the DHT, and it has a key - a [BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt) of Node.keySize bits:
+- A typical client will have one Node instance through which it interacts with one DHT.
+- A server or simulation might have many Node instances to each interact with the same DHT.
+A Node has a Contact object to represent itself to another Node.
+A Node maintains [KBuckets](./dht/kbucket.js), which each have a list of Contacts to other Nodes.
+
+A [Contact](./transports/contact.js) is the means through which a Node interacts with another Node instance:
+- When sending an RPC request, the Contact will "serialize" the sender Nodes's contact.
+- When receiving an RPC response, the sender "deserializes" a string (maybe using a cache) to produce the Contact instance to be noted in the receiver's KBuckets.
+- In classic UDP Kademlia, a Contact would serialize as {key, ip, port}.
+- In a simulation, a Contact could "serialize" as just itself.
+- In our system, I imagine that it will serialize as signature so that keys cannot be forged.
+
+While a Node maintains several Contacts in its KBuckets, these are organized based on the distance from the Contact's key to the Node's key. However, each network-probing operation requires the ephermal creation of Contact information that is based on the distance to the target key being probed for. For this purpose, we wrap the Contacts in a [Helper](./dht/helper.js) object that caches the distance to the target.
+
+## TODO
+
+- Get rid of sname. Unnecessary.
+- Use a uniform RPC dispatcher everywhere, isntead of bespoke inFlight promises to await, and the like.
+- In WebRTC, We wrap each signal in an array with a leading type: [['offer', offerObject], ...['icecandidate', iceCandidateObject]]. We don't need to do that, as the objects are all either of the form {type: 'offer', ..}, or {candidate: ...} (with no 'type' property).

package/dht/helper.js

@@ -0,0 +1,25 @@
+import { Node } from './node.js';
+
+// A Contact that is some distance from an assumed targetKey.
+export class Helper { 
+  constructor(contact, distance) {
+    this.contact = contact;
+    this.distance = distance;
+  }
+  get key() { return this.contact.key; }
+  get name() { return this.contact.name; }
+  get node() { return this.contact.node; }
+  get report() { return this.contact.report; }
+  static compare = (a, b) => { // For sort, where a,b have a distance property returning a BigInt.
+    // Sort expects a number, so bigIntA - bigIntB won't do.
+    // This works for elements of a list that have a distance property -- they do not strictly have to be Helper instances.
+    if (a.distance < b.distance) return -1;
+    if (a.distance > b.distance) return 1;
+    return 0;
+  }
+  static findClosest(targetKey, contacts, count = this.constructor.k) { // Utility, useful for computing and debugging.
+    const helpers = contacts.map(contact => new Helper(contact, contact.distance(targetKey)));
+    helpers.sort(this.compare);
+    return helpers.slice(0, count);
+  }
+}

package/dht/kbucket.js

@@ -0,0 +1,83 @@
+const { BigInt } = globalThis; // For linters.
+
+// Bucket in a RoutingTable: a list of up to k Contacts as enforced by addContact().
+export class KBucket {  
+  constructor(node, index) {
+    this.node = node;
+    this.index = index;
+
+    // Cache the binary prefix used in randomTarget.
+    const keySize = node.constructor.keySize;
+    const nLeadingZeros = keySize - 1 - this.index;
+    // The next bit after the leading zeros must be one to stay in this bucket.
+    this.binaryPrefix = '0b' + '0'.repeat(nLeadingZeros) + '1';
+    this.resetRefresh();
+  }
+
+  contacts = [];
+  get length() { // How many do we have (not capacity, which is k.)
+    return this.contacts.length;
+  } 
+  get isFull() {  // Are we at capacity?
+    return this.length >= this.node.constructor.k;
+  }
+  get nTransports() { // How many of our contacts have their own transport connection?
+    return this.contacts.reduce((accumulator, contact) => contact.connection ? accumulator + 1 : accumulator, 0);
+  }
+  get randomTarget() { // Return a key for which this.getBucketIndex will be the given bucketIndex.
+    const nodeClass = this.node.constructor;
+    const keySize = nodeClass.keySize;
+    let binary = this.binaryPrefix;
+    // Now fill the rest (if any) with random bits. -2 for the '0b' prefix.
+    for (let i = binary.length - 2; i < keySize; i++) binary += Math.round(Math.random());
+    const distance = BigInt(binary);
+    // Quirk of xor distance that it works backwards like this.
+    return this.node.distance(distance);
+  }
+  async refresh() { // Refresh specified bucket using LocateNodes for a random key in the specified bucket's range.
+    if (this.node.isStopped() || !this.contacts.length) return false; // fixme skip isStopped?
+    this.node.log('refresh bucket', this.index);
+    const targetKey = this.randomTarget;
+    await this.node.locateNodes(targetKey); // Side-effect is to update this bucket.
+    return true;
+  }
+  resetRefresh() { // We are organically performing a lookup in this bucket. Reset the timer.
+    // clearInterval(this.refreshTimer);
+    // this.refreshTimer = this.node.repeat(() => this.refresh(), 'bucket');
+    this.node.schedule(this.index, 'bucket', () => this.refresh());
+  }
+
+  removeKey(key, deleteIfEmpty = true) { // Removes item specified by key (if present) from bucket and return 'present' if it was, else false.
+    const { contacts } = this;
+    let index = contacts.findIndex(item => item.key === key);
+    if (index !== -1) {
+      contacts.splice(index, 1);
+      // Subtle: ensures that if contact is later added, it will resetRefresh.
+      if (deleteIfEmpty && !contacts.length) this.node.routingTable.delete(this.index);
+      return 'present';
+    }
+    return false;
+  }
+
+  async addContact(contact) { // Returns 'present' or 'added' if it was added to end within capacity, else false.
+    // Resets refresh timer.
+    this.node.constructor.assert(contact.node.key !== this.node.key, 'attempt to add self contact to bucket');
+    let added = this.removeKey(contact.key, false) || 'added';
+    //this.node.log('addContact', contact.name, this.index, added, this.isFull ? 'full' : '');
+    if (this.isFull) {
+      if (added === 'present') this.node.looseTransports.push(contact); // So no findContact will fail during ping. Should we instead serialize findContact?
+      const head = this.contacts[0];
+      if (await head.sendRPC('ping', head.key)) { // still alive
+	added = false;  // New contact will not be added.
+	contact = head; // Add head back, below.
+      }
+      if (added === 'present') this.node.removeLooseTransport(contact.key);
+      // In either case (whether re-adding head to tail, or making room from a dead head), remove head now.
+      // Subtle: Don't remove before waiting for the ping, as there can be overlap with other activity that could
+      // think there's room and thus add it twice.
+      this.removeKey(head.key);
+    }
+    this.contacts.push(contact);
+    return added;
+  }
+}

package/dht/node.js

@@ -0,0 +1,155 @@
+const { BigInt } = globalThis; // For linters.
+import { v4 as uuidv4 } from 'uuid';
+import { NodeProbe } from './nodeProbe.js';
+
+/*
+  The chain of superclasses are not really intended to be used separately.
+  They are just broken into smaller peices to make it easier to review the code.
+*/
+
+// An actor within thin DHT.
+export class Node extends NodeProbe {
+  // These are the public methods for applications.
+
+  async locateNodes(targetKey, number = this.constructor.k) { // Promise up to k best Contacts for targetKey (sorted closest first).
+    // Side effect is to discover other nodes (and they us).
+    targetKey = await this.ensureKey(targetKey);
+    return await this.iterate(targetKey, 'findNodes', number);
+  }
+  async locateValue(targetKey) { // Promise value stored for targetKey, or undefined.
+    // Side effect is to discover other nodes (and they us).
+    targetKey = await this.ensureKey(targetKey);
+
+    // Optimization: should still work without this, but then there are more RPCs.
+    const found = this.retrieveLocally(targetKey);
+    if (found !== undefined) return found;
+
+    const result = await this.iterate(targetKey, 'findValue');
+    if (Node.isValueResult(result)) return result.value;
+    return undefined;
+  }
+  async storeValue(targetKey, value) { // Convert targetKey to a bigint if necessary, and store k copies.
+    // Promises the number of nodes that it was stored on.
+    targetKey = await this.ensureKey(targetKey);
+    // Go until we are sure have written k.
+    const k = this.constructor.k;
+    let remaining = k;
+    // Ask for more, than needed, and then store to each, one at a time, until we
+    // have replicated k times.
+    let helpers = await this.locateNodes(targetKey, remaining * 2);
+    helpers = helpers.reverse(); // So we can save best-first by popping off the end.
+    // TODO: batches in parallel, if the client and network can handle it. (For now, better to spread it out.)
+    while (helpers.length && remaining) {
+      const contact = helpers.pop().contact;
+      const stored = await contact.store(targetKey, value);
+      if (stored) remaining--;
+    }
+    return k - remaining;
+  }
+  async join(contact) {
+    this.log('joining', contact.sname);
+    contact = this.ensureContact(contact);
+    await contact.connect();
+    await this.addToRoutingTable(contact);
+    await this.locateNodes(this.key); // Discovers between us and otherNode.
+
+    // Refresh every bucket farther out than our closest neighbor.
+    // I think(?) that this can be done by refreshing "just" the farthest bucket:
+    //this.ensureBucket(this.constructor.keySize - 1).resetRefresh();
+    await this.ensureBucket(this.constructor.keySize - 1).refresh();
+    // But if it turns out to be necessary to explicitly refresh each next bucket in turn, this is how:
+    // let started = false;
+    // for (let index = 0; index < this.constructor.keySize; index++) {
+    //   // TODO: Do we really have to perform a refresh on EACH bucket? Won't a refresh of the farthest bucket update the closer ones?
+    //   // TODO: Can it be in parallel?
+    //   const bucket = this.routingTable.get(index);
+    //   if (!bucket?.contacts.length && !started) continue;
+    //   if (!started) started = true;
+    //   else if (!bucket?.contacts.length) await this.ensureBucket(index).refresh();
+    // }
+    this.log('joined', contact.sname);
+    return this.contact; // Answering this node's home contact is handy for chaining or keeping track of contacts being made and joined.
+  }
+
+
+  // TODO: separate all this out: neither transport nor dht.
+  // TODO: fragment/reassemble big messages.
+  messagePromises = new Map(); // maps message messageTags => responders, separately from inFlight, above
+  async message({targetKey, targetSname, excluded = [], requestTag, senderKey, senderSname = this.contact.sname, payload}) { // Send message to targetKey, using our existing routingTable contacts.
+    //const MAX_PING_MS = 400;
+    const MAX_MESSAGE_MS = 2e3;
+    let responsePromise = null;
+    if (!requestTag) { // If this is an outgoing request from us, promises the response.
+      requestTag = uuidv4();
+      senderKey = this.key;
+      senderSname = this.contact.sname;
+      responsePromise = new Promise(resolve => this.messagePromises.set(requestTag, resolve));
+    }
+    targetKey = targetKey.toString();
+    senderKey = senderKey?.toString();
+    // The excluded list of keys prevents cycles.
+    excluded.push(this.key.toString()); // TODO: Find a way to not have an excluded list, or at least add junk for privacy.
+    const body = JSON.stringify({targetKey, excluded, requestTag, senderKey, targetSname, senderSname, payload});
+    const contacts = this.findClosestHelpers(BigInt(targetKey)).map(helper => helper.contact).filter(contact => contact.key !== this.key);
+    this.log('=>', targetSname, 'message', requestTag, 'contacts:', contacts.map(c => c.sname));
+    for (const contact of contacts) {
+      if (excluded.includes(contact.key.toString())) { this.log('skiping excluded', contact.sname, 'for message', requestTag); continue; }
+      //this.xlog('trying message through', contact.sname);
+      if (!contact.connection) { this.log('skipping unconnected', contact.sname, 'for message', requestTag); continue; }
+      if (!(await contact.sendRPC('ping', contact.key))) {
+	this.xlog('failed to get ping', contact.sname, 'for message', requestTag);
+	this.removeContact(contact);
+	continue;
+      }
+      let overlay = await contact.overlay;
+      //this.xlog('hopping through', contact.sname, 'for message', requestTag, 'from',  senderSname, 'to', targetSname);
+      overlay.send(body);
+      // FIXME: how do we know if a response was ultimately delivered? Don't we need a confirmation for that so that we can try a different route?
+      const result =  await responsePromise;
+      //this.xlog('got result for message', requestTag, 'through', contact.sname);
+      //const result =  await Promise.race([responsePromise, Node.delay(MAX_MESSAGE_MS, 'fixme')]);
+      // if (result === 'fixme') {
+      // 	this.xlog(`message timeout to ${targetSname} via ${contact.sname}.`);
+      // 	continue;  // If we timeout, responsePromise is still valid.
+      // }
+      return result;
+      break;
+    }
+    this.xlog(`No connected contacts to send message ${requestTag} to ${targetSname} among ${contacts.map(c => `${excluded.includes(c.key.toString()) ? 'excluded/' : (c.connection ? '' : 'unconnected/')}${c.sname}`).join(', ')}`);
+    return null;
+  }
+  async messageHandler(dataString) {
+    //this.log('got overlay', dataString);
+    const data = JSON.parse(dataString);
+    const {targetKey, excluded, requestTag, senderKey, targetSname, senderSname, payload} = data;
+    Node.assert(targetKey, 'No targetKey to handle', dataString);
+    Node.assert(payload, 'No payload to handle', dataString);    
+    Node.assert(requestTag, 'No request tag by which to answer', dataString);
+    this.log('handling message', requestTag, 'from', senderSname, 'to', targetSname);
+
+    // If intended for another node, pass it on.    
+    if (BigInt(targetKey) !== this.key) return await this.message(data);
+
+    // If it is a response to something we sent, resolve the waiting promise with the payload.
+    const responder = this.messagePromises.get(requestTag);
+    this.messagePromises.delete(requestTag);
+    if (responder) return responder(payload);
+
+    // Finally, answer a request to us by messaging the sender with the same-tagged response.
+    const [method, ...args] = payload;
+    if (!(method in this)) return this.xlog('ignoring unrecognized request', {requestTag, method, args}); // Could be a double response.
+
+    Node.assert(args[0] === senderSname, 'FIXME sender does not match signals payload sender', dataString);
+    this.log('executing', method, 'from', senderSname, 'request:', requestTag);
+    let response = await this[method](...args);
+    Node.assert(senderKey, 'No sender key by which to answer', dataString);    
+    this.log('responding to', method, 'from', senderSname, 'request:', requestTag);
+    return await this.message({targetKey: BigInt(senderKey), targetSname: senderSname, requestTag, payload: response});
+  }
+  async signal(...rest) {
+    //this.xlog(`handling signals @ ${this.key} ${rest}`);
+    const fixme = await this.contact.signals(...rest);
+    //this.xlog(`returning signals @ ${this.key} ${fixme}`);
+    return fixme;
+  }
+}

package/dht/nodeContacts.js

@@ -0,0 +1,122 @@
+import { NodeTransports } from './nodeTransports.js';
+import { Helper } from './helper.js';
+import { KBucket } from './kbucket.js';
+const { BigInt } = globalThis; // For linters.
+
+// Management of Contacts (but see nodeTransports, too)
+export class NodeContacts extends NodeTransports {
+  static k = 20; // Chosen so that for any k nodes, it is highly likely that at least one is still up after refreshTimeIntervalMS.
+  static commonPrefixLength(distance) { // Number of leading zeros of distance (within fixed keySize).
+    if (distance === this.zero) return this.keySize; // I.e., zero distance => our own Node => 128 (i.e., one past the farthest bucket).
+    
+    let length = 0;
+    let mask = this.one << BigInt(this.keySize - 1);
+    
+    for (let i = 0; i < this.keySize; i++) {
+      if ((distance & mask) !== this.zero) {
+        return length;
+      }
+      length++;
+      mask >>= this.one;
+    }
+    
+    return this.keySize;
+  }
+  routingTable = new Map(); // Maps bit prefix length to KBucket
+  getBucketIndex(key) { // index of routingTable KBucket that should contain the given Node key.
+    // We define bucket 0 for the closest distance, and bucket (keySize - 1) for the farthest,
+    // as in the original paper. Note that some implementation and papers number these in the reverse order.
+    // Significantly, Wikipedia numbers these in the reverse order, AND it implies that the buckets
+    // represent addresses, when in fact they represent a distance from current node's address.
+    const distance = this.distance(key);
+    const prefixLength = this.constructor.commonPrefixLength(distance);
+    return 128 - prefixLength - 1;
+  }
+  ensureBucket(index) { // Return bucket at index, creating it if necessary.
+    const routingTable = this.routingTable;
+    let bucket = routingTable.get(index);
+    if (!bucket) {
+      bucket = new KBucket(this, index);
+      routingTable.set(index, bucket);
+    }
+    return bucket;
+  }
+  forEachBucket(iterator, reverse = false) { // Call iterator(bucket) on each non-empty bucket, stopping as soon as iterator(bucket) returns falsy.
+    let buckets = this.routingTable.values();
+    if (reverse) buckets = buckets.reverse();
+    for (const bucket of buckets) {
+      if (bucket && !iterator(bucket)) return;
+    }
+  }
+  get contacts() { // Answer a fresh copy of all contacts for this Node.
+    const contacts = [];
+    this.forEachBucket(bucket => {
+      contacts.push(...bucket.contacts);
+      return true;  // Subtle: an empty bucket will return 0 from push(), which will stop iteration and miss later buckets.
+    });
+    return contacts;
+  }
+  contactDictionary = {}; // maps name => contact for lifetime of Node instance until removeContact.
+  existingContact(name) { // Returns contact with the given name for this node, without searching buckets or looseTransports.
+    return this.contactDictionary[name];
+  }
+  addExistingContact(contact) { // Adds to set of contactDictionary.
+    this.contactDictionary[contact.name] = contact;
+  }
+  findContact(match) { // Answer the contact for which match predicate is true, if any, whether in buckets or looseTransports. Does not remove it.
+    let contact = this.looseTransports.find(match);
+    if (contact) return contact;
+    this.forEachBucket(bucket => !(contact = bucket.contacts.find(match))); // Or we could compute index and look just there.
+    return contact;
+  }
+  findContactByKey(key) { // findContact matching the specified key. To be found, contact must be in routingTable or looseTransports (which is different from existingContact()).
+    return this.findContact(contact => contact.key === key);
+  }
+  ensureContact(contact, sponsor = null) { // Return existing contact, if any (including looseTransports), else clone a new one for this host. Set sponsor.
+    // Subtle: Contact clone uses existingContact (above) to reuse an existing contact on the host, if possible.
+    // This is vital for bookkeeping through connections and sponsorship.
+    contact = contact.clone(this);
+    contact.noteSponsor(sponsor);
+    return contact;
+  }
+  routingTableSerializer = Promise.resolve();
+  queueRoutingTableChange(thunk) { // Promise to resolve thunk() -- after all previous queued thunks have resolved.
+    return this.routingTableSerializer = this.routingTableSerializer.then(thunk);
+  }
+  removeContact(contact) { // Removes from node entirely if present, from looseTransports or bucket as necessary.
+    return this.queueRoutingTableChange(() => {
+      delete this.contactDictionary[contact.name];
+      const key = contact.key;
+      if (this.removeLooseTransport(key)) return;
+      const bucketIndex = this.getBucketIndex(key);
+      const bucket = this.routingTable.get(bucketIndex);
+      bucket?.removeKey(key); // Host might not yet have added node or anyone else as contact for that bucket yet, so maybe no bucket.
+    });
+  }
+  addToRoutingTable(contact) { // Promise contact, and add it to the routing table if room.
+    if (contact.key === this.key) return null; // Do not add self.
+
+    // In most cases there should be a connection, but it sometimes happens that by the time we get here,
+    // we have already dropped the connection.
+    //this.constructor.assert(contact.connection, 'Adding contact without connection', contact.report, 'in', this.contact.report);
+
+    return this.queueRoutingTableChange(async () => {
+      const bucketIndex = this.getBucketIndex(contact.key);
+      const bucket = this.ensureBucket(bucketIndex);
+
+      // Try to add to bucket
+      const added = await bucket.addContact(contact);
+      if (added !== 'present') { // Not already tracked in bucket.
+	this.removeLooseTransport(contact.key); // Can't be in two places.
+	this.queueWork(() => this.replicateCloserStorage(contact));
+      }
+      return added;
+    });
+  }
+  findClosestHelpers(targetKey, count = this.constructor.k) { // Answer count closest Helpers to targetKey, including ourself.
+    if (!this.contact) return []; // Can happen while we are shutting down during a probe.
+    const contacts = this.contacts; // Always a fresh copy.
+    contacts.push(this.contact); // We are a candidate, too! TODO: Handle this separately in iterate so that we don't have to marshal our contacts.
+    return Helper.findClosest(targetKey, contacts, count);
+  }
+}

package/dht/nodeKeys.js

@@ -0,0 +1,52 @@
+import { NodeUtilities } from './nodeUtilities.js';
+const { BigInt, TextEncoder, crypto } = globalThis; // For linters.
+
+// A node name is (coerced to) a string, and a node key is BigInt of keySize.
+export class NodeKeys extends NodeUtilities {
+  static zero = 0n;
+  static one = 1n;
+  static keySize = 128; // Number of bits in a key. Must be multiple of 8 and <= sha256.
+  static distance(keyA, keyB) { // xor
+    return keyA ^ keyB;
+  }
+
+  static async sha256(string) { // Promises a Uint8Array containing the hash of string.
+    const msgBuffer = new TextEncoder().encode(string);
+    const hashBuffer = await crypto.subtle.digest('SHA-256', msgBuffer);
+    const uint8Array = new Uint8Array(hashBuffer);
+    return uint8Array;
+  }
+  static uint8ArrayToHex(uint8Array) { // Answer uint8Array as a hex string.
+    return Array.from(uint8Array)
+      .map(b => b.toString(16).padStart(2, '0'))
+      .join('');
+  }
+  static async key(string) { // If "string" is already a bigint, return it. Otherwise hash it to a keySize BigInt.
+    if (typeof(string) === 'bigint') return string;
+    const uint8Array = await this.sha256(string.toString());
+    const truncated = uint8Array.slice(0, this.keySize / 8);
+    const hex = this.uint8ArrayToHex(truncated);
+    const key = BigInt('0x' + hex);
+    return key;
+  }
+  static counter = 0; // If name is not given, use incremented counter.
+  static async create(nameOrProperties = {}) { // Create a node with a simple name and matching key.
+    if (['string', 'number'].includes(typeof nameOrProperties)) nameOrProperties = {name: nameOrProperties};
+    let {name = this.counter++, ...rest} = nameOrProperties;
+    name = name.toString();
+    const key = await this.key(name);  
+    return new this({name, key, ...rest});
+  }
+  static fromKey(key) { // Forge specific key for testing. The key can be a BigInt or a string that will make a BigInt.
+    if (typeof(key) !== 'bigint') key = BigInt(key);
+    return new this({name: key.toString() + 'n', key});
+  }
+
+  distance(targetKey) { // Distance from this node to targetKey.
+    return this.constructor.distance(this.key, targetKey);
+  }
+  async ensureKey(targetKey) { // If targetKey is not already a real key, hash it into one.
+    if (typeof(targetKey) !== 'bigint') targetKey = await this.constructor.key(targetKey);
+    return targetKey;
+  }
+}

package/dht/nodeMessages.js

@@ -0,0 +1,29 @@
+import { NodeContacts } from './nodeContacts.js';
+
+// The four methods we recevieve through RPCs.
+// These are not directly invoked by a Node on itself, but rather on other nodes
+// through Contact sendRPC.
+export class NodeMessages extends NodeContacts {
+  ping(key) { // Respond with 'pong'. (RPC mechanism doesn't call unless connected.)
+    return 'pong';
+  }
+  store(key, value) { // Tell the node to store key => value, returning truthy.
+    this.storeLocally(key, value);
+    return 'pong';
+  }
+  findNodes(key) { // Return k closest Contacts from routingTable.
+    // TODO: Currently, this answers a list of Helpers. For security, it should be changed to a list of serialized Contacts.
+    // I.e., send back a list of verifiable signatures and let the receiver verify and then compute the distances.
+    return this.findClosestHelpers(key);
+  }
+  findValue(key) { // Like findNodes, but if we have key stored, return {value} instead.
+    let value = this.retrieveLocally(key);
+    if (value !== undefined) return {value};
+    return this.findClosestHelpers(key);
+  }
+  receiveRPC(method, sender, ...rest) {
+    // The sender exists, so add it to the routing table, but give it a while to finish joining.
+    this.addToRoutingTable(sender);
+    return this[method](...rest);
+  }
+}

package/dht/nodeProbe.js

@@ -0,0 +1,87 @@
+import { Helper } from  './helper.js';
+import { NodeMessages } from './nodeMessages.js';
+
+// Probe the network
+export class NodeProbe extends NodeMessages {
+  // There are only three kinds of rpc results: 'pong', [...helper], {value: something}
+  static isValueResult(rpcResult) {
+    return rpcResult && rpcResult !== 'pong' && 'value' in rpcResult;
+  }
+  static isContactsResult(rpcResult) {
+    return Array.isArray(rpcResult);
+  }
+  async step(targetKey, finder, helper, keysSeen) {
+    // Get up to k previously unseen Helpers from helper, adding results to keysSeen.
+    const contact = helper.contact;
+    // this.log('step with', contact.sname);
+    let results = await contact.sendRPC(finder, targetKey);
+    if (!results) { // disconnected
+      this.log('removing unconnected contact', contact.sname);
+      await this.removeContact(contact);
+      return [];
+    }
+    await this.addToRoutingTable(contact); // Live node, so update bucket.
+    // this.log('step added contact', contact.sname);
+    if (this.constructor.isContactsResult(results)) { // Keep only those that we have not seen, and note the new ones we have.
+      results = results.filter(helper => !keysSeen.has(helper.key) && keysSeen.add(helper.key));
+      // Results are (helpers around) contacts. Set them up for this host, with contact as sponsor.
+      results = results.map(h => new Helper(this.ensureContact(h.contact, contact), h.distance));
+    }
+    return results;
+  }
+  static alpha = 3; // How many lookup requests are initially tried in parallel. If no progress, we repeat with up to k more.
+  async iterate(targetKey, finder, k = this.constructor.k, trace = false) {
+    // Promise a best-first list of k Helpers from the network, by repeatedly trying to improve our closest known by applying finder.
+    // But if any finder operation answer isValueResult, answer that instead.
+
+    if (targetKey !== this.key) {
+      const bucketIndex = this.getBucketIndex(targetKey);
+      const bucket = this.routingTable.get(bucketIndex);
+      // Subtle: if we don't have one now, but will after, refreshes will be rescheduled by KBucket constructor.
+      bucket?.resetRefresh();
+    }
+    
+    // Each iteration uses a bigger pool than asked for, because some will have disconnected.
+    let pool = this.findClosestHelpers(targetKey, 2*k); // The k best-first Helpers known so far, that we have NOT queried yet.
+    const isValueResult = this.constructor.isValueResult;
+    const alpha = Math.min(pool.length, this.constructor.alpha);
+    const keysSeen = new Set(pool.map(h => h.key));    // Every key we've seen at all (candidates and all responses).
+    keysSeen.add(this.key); // We might or might not be in our list of closest helpers, and we could be in someone else's.
+    let toQuery = pool.slice(0, alpha);
+    pool = pool.slice(alpha); // Yes, this could be done with splice instead of slice, above, but it makes things hard to trace.
+    let best = []; // The accumulated closest-first result.    
+    while (toQuery.length && this.isRunning) { // Stop if WE disconnect.
+      let requests = toQuery.map(helper => this.step(targetKey, finder, helper, keysSeen));
+      let results = await Promise.all(requests);
+      if (trace) this.log(toQuery.map(h => h.name), '=>', results.map(r => r.map?.(h => h.name) || r));
+      
+      let found = results.find(isValueResult); // Did we get back a 'findValue' result.
+      if (found) {
+	// Store at closest result that didn't have it (if any). This can cause more than k copies in the network.
+	for (let i = 0; i < toQuery.length; i++) {
+	  if (!isValueResult(results[i])) {
+	    toQuery[i].contact.store(targetKey, found.value);
+	    break;
+	  }
+	}
+	return found;
+     }
+
+      let closer = [].concat(...results); // Flatten results.
+      // closer might not be in order, and one or more toQuery might belong among them.
+      best = [...closer, ...toQuery, ...best].sort(Helper.compare).slice(0, k);
+      if (!closer.length) {
+	if (toQuery.length === alpha && pool.length) {
+	  toQuery = pool.slice(0, 2*k);  // Try again with k more. (Interestingly, not k - alpha.)
+	  pool = pool.slice(2*k);
+	} else break; // We've tried everything and there's nothing better.
+      } else {
+	pool = [...closer, ...pool].slice(0, 2*k); // k best-first nodes that we have not queried.
+	toQuery = pool.slice(0, alpha);
+	pool = pool.slice(alpha);
+      }
+    }
+    if (trace) this.log('probe result', best.map(helper => helper.name));
+    return best;
+  }
+}

package/dht/nodeRefresh.js

@@ -0,0 +1,60 @@
+import { NodeKeys } from './nodeKeys.js';
+
+// The mechanics of periodic refresh (of buckets or stored data).
+export class NodeRefresh extends NodeKeys {
+  static refreshTimeIntervalMS = 15e3; // Original paper for desktop filesharing was 60 minutes.
+  constructor ({refreshTimeIntervalMS = NodeRefresh.refreshTimeIntervalMS, ...properties}) {
+    super({refreshTimeIntervalMS, ...properties});
+  }
+  static stopRefresh() { // Stop all repeat timers in all instances the next time they come around.
+    this.constructor.refreshTimeIntervalMS = 0;
+  }
+  stopRefresh() { // Stop repeat timeers in this instance.
+    this.refreshTimeIntervalMS = 0;
+  }
+  isStopped(interval) {
+    return !this.isRunning || 0 === this.refreshTimeIntervalMS || 0 === this.constructor.refreshTimeIntervalMS || 0 === interval;
+  }
+  fuzzyInterval(target = this.refreshTimeIntervalMS/2, margin = target/2) { // Like static fuzzyInterval with target defaulting to refreshTimeIntervalMS/2.
+    return this.constructor.fuzzyInterval(target, margin);
+  }
+  static fuzzyInterval(target, margin = target/2) {
+    // Answer a random integer uniformly distributed around target, +/- margin.
+    // The default target slightly exceeds the Nyquist condition of sampling at a frequency at
+    // least twice the signal being observed. In particular, allowing for some randomization,
+    // as long as we're not completely overloaded, we should expect the target to hit at least
+    // once for each thing it is trying to detect, and generally happen twice for each detectable event.
+    const adjustment = this.randomInteger(margin);
+    return Math.floor(target + margin/2 - adjustment);
+  }
+  workQueue = Promise.resolve();
+  queueWork(thunk) { // Promise to resolve thunk() -- after all previous queued thunks have resolved.
+    return this.workQueue = this.workQueue.then(thunk);
+  }
+  timers = new Map();
+  schedule(timerKey, statisticsKey, thunk) {    
+    // Schedule thunk() to occur at a fuzzyInterval from now, cancelling any
+    // existing timer at the same key. This is used in such a way that:
+    // 1. A side effect of calling thunk() is that it will be scheduled again, if appropriate.
+    //    E.g., A bucket refresh calls iterate, which schedules, and storeValue schedules.
+    // 2. The timerKeys are treated appropriately under Map.
+    //    E.g., bucket index 1 === 1 and stored value key BigInt(1) === BigInt(1), but 1 !== BigInt(1)
+    if (this.isStopped()) return;
+    const start = Date.now();
+    const timeout = this.fuzzyInterval();
+    clearInterval(this.timers.get(timerKey));
+    this.timers.set(timerKey, setTimeout(async () => {
+      const lag = Date.now() - start - timeout;
+      this.timers.delete(timerKey);
+      if (this.isStopped()) return;
+      if (lag > 250) console.log(`** System is overloaded by ${lag.toLocaleString()} ms. **`);
+      // Each actual thunk execution is serialized: Each Node executes its OWN various refreshes and probes
+      // one at a time. This prevents a node from self-DoS'ing, but of course it does not coordinate across
+      // nodes. If the system is bogged down for any reason, then the timeout spacing will get smaller
+      // until finally the node is just running flat out.
+      // this.log('queue', statisticsKey, timerKey, timeout);
+      await this.queueWork(thunk);
+    }, timeout));
+  }
+}
+