@yz-social/kdht 0.1.0

package/transports/contact.js

@@ -0,0 +1,127 @@
+import { Node } from '../dht/node.js';
+
+export class Contact {
+  // Represents an abstract contact from a host (a Node) to another node.
+  // The host calls aContact.sendRpc(...messageParameters) to send the message to node and promises the response.
+  // This could be by wire, by passing the message through some overlay network, or for just calling a method directly on node in a simulation.
+
+  // Creation
+  // host should be a dht Node.
+  // node is the far end of the contact, and could be Node (for in-process simulation) or a serialization of a key.
+  static counter = 0;
+  static fromNode(node, host = node) {
+    let contact = host.existingContact(node.name);
+    if (contact) Node.assert(contact.host === host, 'Existing contact host', contact.host.name, 'does not match specified host', host.name, 'for', node.name);
+    //if (!contact) host.log('Creating contact', node.name);
+    contact ||= new this();
+    // Every Contact is unique to a host Node, from which it sends messages to a specific "far" node.
+    // Every Node caches a contact property for that Node as it's own host, and from which Contacts for other hosts may be cloned.
+    node.contact ||= contact;
+    contact.node = node;
+    contact.host = host; // In whose buckets (or looseTransports) does this contact live?
+    contact.counter = this.counter++;
+    host.addExistingContact(contact); // After contact.node (and thus contact.namem) is set.
+    return contact;
+  }
+  static async create(properties, host = undefined) {
+    return this.fromNode(await Node.create(properties), host);
+  }
+  static fromKey(key, host) {
+    const node = Node.fromKey(key);
+    return this.fromNode(node, host || node);
+  }
+  clone(hostNode, searchHost = true) { // Answer a Contact that is set up for hostNode - either this instance or a new one.
+    // Unless searchHost is null, any existing contact on hostNode will be returned.
+    if (this.host === hostNode) return this; // All good.
+
+    // Reuse existing contact in hostNode -- if still running.
+    let existing = searchHost && hostNode.existingContact(this.name);
+    if (existing?.isRunning) return existing;
+
+    // Make one.
+    Node.assert(this.key !== hostNode.key, 'Cloning self-contact', this, hostNode);
+    const clone = this.constructor.fromNode(this.node, hostNode);
+    return clone;
+  }
+  static serverSignifier = 'S';
+  get sname() { // Serialized name, indicating whether it is a server node.
+    if (this._sname) return this._sname;
+    if (this.name.length < 36) return this._sname = this.name; // Kluge: index of portal node.
+    if (this.isServerNode) return this._sname = this.constructor.serverSignifier + this.name;
+    return this._sname = this.name;
+  }
+
+  // Operations
+  join(other) { return this.host.join(other); }
+  storeValue(key, value) { return this.host.storeValue(key, value); }
+  store(key, value) {
+    return this.sendRPC('store', key, value);
+  }
+  disconnect() { // Simulate a disconnection of node, marking as such and rejecting any RPCs in flight.
+    Node.assert(this.host === this.node, "Disconnect", this.name, "not invoked on home contact", this.host.name);
+    this.host.isRunning = false;
+    this.host.stopRefresh();
+    this.host.contacts.forEach(async contact => {
+      const far = contact.connection;
+      if (!far) return;
+      contact.disconnectTransport();
+    });
+  }
+  distance(key) { return this.host.constructor.distance(this.key, key); }
+
+  // RPC
+  sendRPC(method, ...rest) { // Promise the result of a nework call to node. Rejects if we get disconnected along the way.
+    const sender = this.host.contact;
+    //this.host.log('sendRPC', method, rest, sender.isRunning ? 'running' : 'stopped', 'sender key:', sender.key, 'to node:', this.sname, this.key);
+    if (!sender.isRunning) {this.host.log('not running'); return null;  }// sender closed before call.
+    if (sender.key === this.key) {
+      const result = this.receiveRPC(method, sender, ...rest);
+      if (!result) this.host.xlog('no local result');
+      return result;
+    }
+
+    const start = Date.now();
+    return this.transmitRPC(method, ...rest) // The main event.
+      .then(result => {
+	if (!sender.isRunning) {this.host.log('sender closed'); return null; } // Sender closed after call.
+	return result;
+      })
+      .finally(() => Node.noteStatistic(start, 'rpc'));
+  }
+  async receiveRPC(method, sender, ...rest) { // Call the message method to act on the 'to' node side.
+    Node.assert(typeof(method)==='string', 'no method', method);
+    Node.assert(sender instanceof Contact, 'no sender', sender);
+    return this.host.receiveRPC(method, sender, ...rest);
+  }
+  // Sponsorship
+  _sponsors = new Map(); // maps key => contact
+  noteSponsor(contact) {
+    if (!contact) return;
+    this._sponsors.set(contact.key, contact);
+  }
+  hasSponsor(key) {
+    return this._sponsors.get(key);
+  }
+  async findSponsor(predicate) { // Answer the sponsor contact for which await predicate(contact) is true, else falsy.
+    for (const candidate of this._sponsors.values()) {
+      if (await predicate(candidate)) return candidate;
+    }
+    return null;
+  }
+
+  // Utilities
+  get report() { // Answer string of name, followed by * if disconnected
+    //return `${this.connection ? '_' : ''}${this.sname}${this.isRunning ? '' : '*'}@${this.host.contact.sname}v${this.counter}`; // verbose version
+    //return `${this.connection ? '_' : ''}${this.sname}v${this.counter}${this.isRunning ? '' : '*'}`;
+    return `${this.connection ? '_' : ''}${this.sname}${this.isRunning ? '' : '*'}`; // simpler version
+  }
+  static pingTimeMS = 30; // ms
+  static async ensureTime(thunk, ms = this.pingTimeMS) { // Promise that thunk takes at least ms to execute.
+    const start = Date.now();
+    const result = await thunk();
+    const elapsed = Date.now() - start;
+    if (elapsed > ms) return result;
+    await new Promise(resolve => setTimeout(resolve, ms - elapsed));
+    return result;
+  }
+}

package/transports/inProcessWebrtc.js

@@ -0,0 +1,70 @@
+import { Node } from '../dht/node.js';
+import { Contact, SimulatedContact, SimulatedConnectionContact } from './contact.js';
+import { WebRTC } from '@yz-social/webrtc';
+
+export class InProcessWebContact extends SimulatedConnectionContact {
+  // Still a SimulatedContacConnection, but does create the real bidirectional webrtc connection.
+  // Of course, testing requires that there be very few nodes, because:
+  // - Each node is running in one instance, so the limited number of WebRTC instances gets used up per process instead of per node.
+  // - Each connection takes two in-process WebRTC instances - one for each end.
+  static count = 0;
+  static serializer = Promise.resolve();
+  async connect(forMethod = 'findNodes') { // Connect from host to node, promising a possibly cloned contact that has been noted.
+    // Simulates the setup of a bilateral transport between this host and node, including bookkeeping.
+    // TODO: Simulate webrtc signaling.
+    const contact = this;
+    let { host, node, isServerNode } = contact;
+    if (contact.webrtc) throw Error('wtf');
+
+    // Anyone can connect to a server node using the server's connect endpoint.
+    // Anyone in the DHT can connect to another DHT node through a sponsor.
+    if (!isServerNode) {
+      let mutualSponsor = null;
+      for (const sponsor of this._sponsors.values()) {
+	if (!sponsor.hasConnection || !sponsor.node.existingContact(this.node.namem)?.hasConnection) continue;
+	mutualSponsor = sponsor;
+      }
+      if (!mutualSponsor) return null;
+    }
+    const farContactForUs = node.ensureContact(host.contact, contact.sponsor);
+
+    // FIXME
+    // Just makes the connection and bashes it into place at both ends.
+    // Only works because the contact has access to both ends.
+    await (this.constructor.serializer = this.constructor.serializer.then(async () => {
+      const a = new WebRTC({label: 'initiator' + host.name + '/' + node.name});
+      const b = new WebRTC({label: 'contacted' + node.name + '/' + host.name});
+      const channelName = 'kdht';
+      this.constructor.count += 2;
+      const aDataChannelPromise = a.ensureDataChannel(channelName);
+      await a.signalsReady;
+      const bDataChannelPromise = b.ensureDataChannel(channelName, {}, a.signals);
+      await b.signalsReady;
+      await a.connectVia(signals => b.respond(signals));
+      const aDataChannel = await aDataChannelPromise;
+      const bDataChannel = await bDataChannelPromise;
+      let onclose = contact => {
+	this.constructor.count--;
+	//console.log('disconnect', contact.host.name, 'to', contact.node.name, this.constructor.count);
+	contact.webrtc = null;
+      };
+      aDataChannel.addEventListener('close', () => onclose(contact));
+      bDataChannel.addEventListener('close', () => onclose(farContactForUs));
+      contact.webrtc = a;
+      farContactForUs.webrtc = b;
+      //console.log(`${host.name}->${node.name}, ${this.constructor.count} webrtc peers`);
+    }));
+
+    contact.hasTransport = farContactForUs;
+    host.noteContactForTransport(contact);
+
+    farContactForUs.hasTransport = contact;
+    node.noteContactForTransport(farContactForUs);
+    
+    return contact;
+  }
+  disconnectTransport() {
+    this.webrtc?.close();
+    super.disconnectTransport();
+  }
+}

package/transports/simulations.js

@@ -0,0 +1,66 @@
+import { Node } from '../dht/node.js';
+import { Contact } from './contact.js';
+
+export class SimulatedContact extends Contact {
+  get name() { return this.node.name; }
+  get key() { return this.node.key; }
+  get isServerNode() { return this.node.isServerNode; }
+
+  get isRunning() { // Ask our canonical home contact.
+    return this.node.isRunning;
+  }
+  connection = null;
+  async connect() { return this; }
+  disconnectTransport() { }
+  async transmitRPC(method, ...rest) { // Transmit the call to the receiving node's contact.
+    return await this.constructor.ensureTime(async () => {
+      if (!this.isRunning) return null; // Receiver closed.
+      return await this.node.contact.receiveRPC(method, this.node.ensureContact(this.host.contact), ...rest);
+    });
+  }
+}
+
+export class SimulatedConnectionContact extends SimulatedContact {
+  connection = null; // The cached connection (to another node's connected contact back to us) over which messages can be directly sent, if any.
+  disconnectTransport() {
+    const farContactForUs = this.connection;
+    if (!farContactForUs) return;
+    Node.assert(farContactForUs.key === this.host.key, 'Far contact backpointer', farContactForUs.node.name, 'does not point to us', this.host.name);
+    Node.assert(farContactForUs.host.key === this.key, 'Far contact host', farContactForUs.host.name, 'is not hosted at contact', this.name);
+    farContactForUs.connection = null;
+    this.connection = null;
+  }
+  async connect(forMethod = 'findNodes') { // Connect from host to node, promising a possibly cloned contact that has been noted.
+    // Simulates the setup of a bilateral transport between this host and node, including bookkeeping.
+    // TODO: Simulate webrtc signaling.
+    const contact = this;
+    let { host, node, isServerNode } = contact;
+
+    // Anyone can connect to a server node using the server's connect endpoint.
+    // Anyone in the DHT can connect to another DHT node through a sponsor.
+    if (!isServerNode) {
+      let mutualSponsor = null;
+      for (const sponsor of this._sponsors.values()) {
+	if (!sponsor.connection || !sponsor.node.existingContact(this.node.name)?.connection) continue;
+	mutualSponsor = sponsor;
+      }
+      if (!mutualSponsor) return null;
+    }
+    
+    const farContactForUs = node.ensureContact(host.contact, contact.sponsor);
+
+    contact.connection = farContactForUs;
+    host.noteContactForTransport(contact);
+
+    farContactForUs.connection = contact;
+    node.noteContactForTransport(farContactForUs);
+    
+    return contact;
+  }
+  async transmitRPC(method, ...rest) { // A message from this.host to this.node. Forward to this.node through overlay connection for bucket.
+    if (!this.isRunning) return null; // Receiver closed.
+    const farContactForUs = this.connection || (await this.connect(method))?.connection;
+    if (!farContactForUs) return null; // receiver no longer reachable.
+    return await this.constructor.ensureTime(() => farContactForUs.receiveRPC(method, farContactForUs, ...rest));
+  }
+}

package/transports/webrtc.js

@@ -0,0 +1,249 @@
+const { BigInt } = globalThis; // For linters.
+import { v4 as uuidv4 } from 'uuid';
+import { Node } from '../dht/node.js';
+import { Helper } from '../dht/helper.js';
+import { Contact } from './contact.js';
+import { WebRTC } from '@yz-social/webrtc';
+
+
+export class WebContact extends Contact { // Our wrapper for the means of contacting a remote node.
+  // Can this set all be done more simply?
+  get name() { return this.node.name; } // Key of remote node as a string (e.g., as a guid).
+  get key() { return this.node.key; }   // Key of remote node as a BigInt.
+  get isServerNode() { return this.node.isServerNode; } // It it reachable through a server.
+  get isRunning() { return this.node.isRunning; } // Have we marked at is no longer running.
+
+  checkResponse(response) { // Return a fetch response, or throw error if response is not a 200 series.
+    if (!response.ok) throw new Error(`fetch ${response.url} failed ${response.status}: ${response.statusText}.`);
+  }
+  async fetchBootstrap(baseURL, label = 'random') { // Promise to ask portal (over http(s)) to convert a portal
+    // worker index or the string 'random' to an available sname to which we can connect().
+    const url = `${baseURL}/name/${label}`;
+    const response = await fetch(url);
+    this.checkResponse(response);
+    return await response.json();
+  }
+  async checkSignals(signals) {
+    if (!signals) {
+      await this.host.removeContact(this);
+      return [];
+    }
+    return signals;
+  }
+  async fetchSignals(url, signalsToSend) { 
+    const response = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(signalsToSend)
+    });
+    this.checkResponse(response);
+    return this.checkSignals(await response.json());
+  }
+  messsageSignals(signals) {
+    return this.checkSignals(this.host.message({targetKey: this.key, targetSname: this.sname,
+						payload: ['signal', this.host.contact.sname, ...signals]}));
+  }
+  get webrtcLabel() {
+    return `@${this.host.contact.sname} ==> ${this.sname}`;
+  }
+
+  ensureWebRTC(initiate = false, timeoutMS = this.host.timeoutMS || 5e3) { // If not already configured, sets up contact to have properties:
+    // - connection - a promise for an open webrtc data channel:
+    //   this.send(string) puts data on the channel
+    //   incomming messages are dispatched to receiveWebRTC(string)
+    // - closed - resolves when webrtc closes.
+    // - webrtc - an instance of WebRTC (which may be used for webrtc.respond()
+    //
+    // If timeoutMS is non-zero and a connection is not established within that time, connection and closed resolve to null.
+    //
+    // This is synchronous: all side-effects (assignments to this) happen immediately.
+    const start = Date.now();
+    const { host, node, isServerNode, bootstrapHost } = this;
+    this.host.log('starting connection', this.sname, this.connection ? 'exists!!!' : 'fresh', this.counter);
+    let {promise, resolve} = Promise.withResolvers();
+    this.closed = promise;
+    const webrtc = this.webrtc = new WebRTC({name: this.webrtcLabel,
+					     debug: host.debug,
+					     polite: this.host.key < this.node.key});
+    const onclose = () => { // Does NOT mean that the far side has gone away. It could just be over maxTransports.
+      this.host.log('connection closed');
+      resolve(null); // closed promise
+      this.webrtc = this.connection = this.overlay = null;
+    };
+    if (initiate) {
+      if (bootstrapHost/* || isServerNode*/) {
+	const url = `${bootstrapHost || 'http://localhost:3000/kdht'}/join/${host.contact.sname}/${this.sname}`;
+	this.webrtc.transferSignals = signals => this.fetchSignals(url, signals);
+      } else {
+	this.webrtc.transferSignals = signals => this.messsageSignals(signals);
+      }
+    } // Otherwise, we just hang on to signals until we're asked to respond().
+
+    let timeout;
+    const kdhtChannelName = 'kdht';
+    const channelPromise = webrtc.getDataChannelPromise(kdhtChannelName);
+    webrtc.createChannel(kdhtChannelName, {negotiated: true});
+    channelPromise.then(async dataChannel => {
+      this.host.log('data channel open', this.sname, Date.now() - start, this.counter);
+      clearTimeout(timeout);
+      dataChannel.addEventListener('close', onclose);
+      dataChannel.addEventListener('message', event => this.receiveWebRTC(event.data));
+      if  (this.host.debug) await webrtc.reportConnection(true); // TODO: make this asymchronous?
+      if (webrtc.statsElapsed > 500) this.host.xlog(`** slow connection to ${this.sname} took ${webrtc.statsElapsed.toLocaleString()} ms. **`);
+      return dataChannel;
+    });
+    const overlayChannelName = 'overlay';
+    const overlayPromise = webrtc.getDataChannelPromise(overlayChannelName);
+    webrtc.createChannel(overlayChannelName, {negotiated: true});
+    overlayPromise.then(async overlay => {
+      overlay.addEventListener('message', event => host.messageHandler(event.data));
+      return overlay;
+    });
+    if (!timeoutMS) {
+      this.connection = channelPromise;
+      this.overlay = overlayPromise;
+      return;
+    }
+    const timerPromise = new Promise(expired => {
+      timeout = setTimeout(async () => {
+	const now = Date.now();
+	this.host.xlog('**** connection timeout', this.sname, now - start,
+		       'status:', webrtc.pc.connectionState, 'signaling:', webrtc.pc.signalingState,
+		       'last signal:', now - webrtc.lastOutboundSignal,
+		       'last send:', now - webrtc.lastOutboundSend,
+		       'last response:', now - webrtc.lastResponse,
+		       '****');
+	onclose();
+	await this.host.removeContact(this); // fixme?
+	expired(null);
+      }, timeoutMS);
+    });
+    this.connection = Promise.race([channelPromise, timerPromise]);
+    this.overlay = Promise.race([overlayPromise, timerPromise]);
+  }
+  async connect() { // Connect from host to node, promising a possibly cloned contact that has been noted.
+    // Creates a WebRTC instance and uses it's connectVia to to signal.
+    const contact = this.host.noteContactForTransport(this);
+    ///if (contact.connection) contact.host.xlog('connect existing', contact.sname, contact.counter);
+
+    const { host, node, isServerNode, bootstrapHost } = contact;
+    // Anyone can connect to a server node using the server's connect endpoint.
+    // Anyone in the DHT can connect to another DHT node through a sponsor.
+    if (contact.connection) return contact.connection;
+    contact.ensureWebRTC(true);
+    await Promise.all([this.connection, this.overlay]);
+    return this.connection;
+  }
+  async signals(senderSname, ...signals) { // Accept directed WebRTC signals from a sender sname, creating if necessary the
+    // new contact on host to receive them, and promising a response.
+    let contact = await this.ensureRemoteContact(senderSname);
+
+    if (contact.webrtc) return await contact.webrtc.respond(signals); 
+
+    this.host.noteContactForTransport(contact);
+    contact.ensureWebRTC();
+    return await contact.webrtc.respond(signals);
+  }
+
+  async send(message) { // Promise to send through previously opened connection promise.
+    let channel = await this.connection;
+    if (channel?.readyState === 'open') channel.send(JSON.stringify(message));
+    else this.host.xlog('Unable to open channel');
+  }
+  getName(sname) { // Answer name from sname.
+    if (sname.startsWith(this.constructor.serverSignifier)) return sname.slice(1);
+    return sname;
+  }
+  async ensureRemoteContact(sname, sponsor = null) {
+    let contact;
+    if (sname === this.host.contact.sname) {
+      contact = this.host.contact; // ok, not remote, but contacts can send back us in a list of closest nodes.
+    }
+    const name = this.getName(sname);
+    if (!contact) {
+      // Not the final answer. Just an optimization to avoid hashing name.
+      contact = this.host.existingContact(name);
+    }
+    if (!contact) {
+      const isServerNode = name !== sname;
+      contact = await this.constructor.create({name, isServerNode}, this.host); // checks for existence AFTER creating Node.
+    }
+    if (sponsor instanceof Contact) contact.noteSponsor(sponsor);
+    else if (typeof(sponsor) === 'string') contact.bootstrapHost = sponsor;
+    return contact;
+  }
+  receiveRPC(method, sender, key, ...rest) { // Receive an RPC from sender, dispatch, and return that value, which will be awaited and sent back to sender.
+    if (method === 'forwardSignals') { // Can't be handled by Node, because 'forwardSignals' is specific to WebContact.
+      const [sendingSname, ...signals] = rest;
+      if (key === this.host.key) { // for us!
+	return this.signals(...rest);
+      } else { // Forward to the target.
+	const target = this.host.findContactByKey(key);
+	if (!target) {
+	  console.log(this.host.contact.sname, 'does not have contact to', sendingSname);
+	  return null;
+	}
+	return target.sendRPC('forwardSignals', key, ...rest);
+      }
+    }
+    return super.receiveRPC(method, sender, key, ...rest);
+  }
+  inFlight = new Map();
+  async transmitRPC(method, key, ...rest) { // Must return a promise.
+    const MAX_PING_MS = 250; // No including connect time. These are single-hop WebRTC data channels.
+    // this.host.log('transmit to', this.sname, this.connection ? 'with connection' : 'WITHOUT connection');
+    if (!await this.connect()) return null;
+    const sender = this.host.contact.sname;
+    // uuid so that the two sides don't send a request with the same id to each other.
+    // Alternatively, we could concatenate a counter to our host.name.
+    const messageTag = uuidv4();
+    const responsePromise = new Promise(resolve => this.inFlight.set(messageTag, resolve));
+    const message = [messageTag, method, sender, key.toString(), ...rest];
+    this.send(message);
+    const timeout = Node.delay(MAX_PING_MS, null); // Faster than waiting for webrtc to observe a close
+    return await Promise.race([responsePromise, timeout, this.closed]);
+  }
+  async receiveWebRTC(dataString) { // Handle receipt of a WebRTC data channel message that was sent to this contact.
+    // The message could the start of an RPC sent from the peer, or it could be a response to an RPC that we made.
+    // As we do the latter, we generate and note (in transmitRPC) a message tag included in the message.
+    // If we find that in our inFlight tags, then the message is a response.
+    if (dataString === '"bye"') { // Special messsage that the other side is disconnecting, so we can clean up early.
+      this.webrtc.close();
+      await this.host.removeContact(this);  // TODO: Make sure we're not invoking this in maxTransports cases.
+      return;
+    }
+    const [messageTag, ...data] = JSON.parse(dataString);
+    const responder = this.inFlight.get(messageTag);
+    if (responder) { // A response to something we sent and are waiting for.
+      let [result] = data;
+      this.inFlight.delete(messageTag);
+      if (Array.isArray(result)) {
+	if (!result.length) responder(result);
+	const first = result[0];
+	const isSignal = Array.isArray(first) && ['offer', 'answer', 'icecandidate'].includes(first[0]);
+	if (isSignal) {
+	  responder(result); // This could be the sponsor or the original sender. Either way, it will know what to do.
+	} else {
+	  responder(await Promise.all(result.map(async ([sname, distance]) =>
+	    new Helper(await this.ensureRemoteContact(sname, this), BigInt(distance)))));
+	}
+      } else {
+	responder(result);
+      }
+    } else { // An incoming request.
+      const [method, senderLabel, key, ...rest] = data;
+      const sender = await this.ensureRemoteContact(senderLabel);
+      let response = await this.receiveRPC(method, sender, BigInt(key), ...rest);
+      if ((method !== 'forwardSignals') && this.host.constructor.isContactsResult(response)) {
+	response = response.map(helper => [helper.contact.sname, helper.distance.toString()]);
+      }
+      this.send([messageTag, response]);
+    }
+  }
+  async disconnectTransport() {
+    await this.send('bye'); await Node.delay(50); // FIXME: Hack: We'll need to be able to pass tests without this, too.
+
+    this.webrtc?.close();
+    this.connection = this.webrtc = this.initiating = null;
+  }
+}