@yz-social/kdht 0.1.0

package/spec/bots.js

@@ -0,0 +1,79 @@
+#!/usr/bin/env node
+import cluster from 'node:cluster';
+import process from 'node:process';
+import { launchWriteRead } from './writes.js';
+import { v4 as uuidv4 } from 'uuid';
+import { WebContact, Node } from '../index.js';
+import yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
+
+// Todo: Allow a remote portal to be specified (passing a host to WebContact.create/ensureRemoteContact).
+const argv = yargs(hideBin(process.argv))
+      .usage("Launch nBots that connect to the network through the local portal. A bot is just an ordinary node that can only be contacted through another node. They provide either continuity or churn-testing, depend on whether or not they are told to 'thrash'.")
+      .option('nBots', {
+	alias: 'n',
+	alias: 'nbots',
+	type: 'number',
+	default: 20,
+	description: "The number of bots, which can only be reached through the network."
+      })
+      .option('baseURL', {
+	type: 'string',
+	default: 'http://localhost:3000/kdht',
+	description: "The base URL of the portal server through which to bootstrap."
+      })
+      .option('thrash', {
+	type: 'boolean',
+	default: false,
+	description: "Do bots randomly disconnect and reconnect with no memory of previous data?"
+      })
+      .option('nWrites', {
+	alias: 'w',
+	alias: "nwrites",
+	type: 'number',
+	default: 0,
+	description: "The number of test writes to check."
+      })
+      .option('verbose', {
+	alias: 'v',
+	type: 'boolean',
+	description: "Run with verbose logging."
+      })
+      .parse();
+
+const host = uuidv4();
+
+if (cluster.isPrimary) {
+  for (let i = 1; i < argv.nBots; i++) { // The cluster primary becomes bot 0.
+    cluster.fork();
+  }
+  if (argv.nWrites) {
+    console.log(new Date(), 'Waiting a refresh interval while bots get randomly created before write/read test');
+    await Node.delay(2 * Node.refreshTimeIntervalMS);
+    launchWriteRead(argv.nWrites, argv.baseURL, Node.refreshTimeIntervalMS, argv.verbose);
+  }
+}
+process.title = 'kdht-bot-' + host;
+
+await Node.delay(Node.randomInteger(Node.refreshTimeIntervalMS));
+console.log(cluster.worker?.id || 0, host);
+let contact = await WebContact.create({name: host, debug: argv.verbose});
+let bootstrapName = await contact.fetchBootstrap(argv.baseURL);
+let bootstrapContact = await contact.ensureRemoteContact(bootstrapName, argv.baseURL);
+await contact.join(bootstrapContact);
+
+while (argv.thrash) {
+  await Node.delay(contact.host.fuzzyInterval(Node.refreshTimeIntervalMS));
+  const old = contact;
+  const next = uuidv4();
+  contact.host.xlog('disconnecting');
+  contact.disconnect();
+  await Node.delay(1e3); // TODO: remove?
+
+  contact = await WebContact.create({name: next, debug: argv.verbose});
+  bootstrapName = await contact.fetchBootstrap(argv.baseURL);
+  bootstrapContact = await contact.ensureRemoteContact(bootstrapName, argv.baseURL);
+  await contact.join(bootstrapContact);
+  old.host.xlog('rejoined as', next);  
+}
+

package/spec/dhtAcceptanceSpec.js

@@ -0,0 +1,227 @@
+// Defines a repeatable suite of tests that confirm DHT operations and log performance data.
+
+// Defined by the generic test framework. See https://jasmine.github.io/
+// One does not import these definitions from a file, but rather they are
+// defined globally by the jasmine program or browser page that runs the tests.
+const { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach} = globalThis; // For linters.
+
+// The file dhtImplementation.js exports functions that perform setup operations whose
+// implementation changes for different DHTs.
+import { setupServerNodes, shutdownServerNodes,
+	 start1, setupClientsByTime, shutdownClientNodes,
+	 getContacts, getRandomLiveContact,
+	 startThrashing, write1, read1, Node } from './dhtImplementation.js';
+
+// Some definitions:
+//
+// CONTACTS: an array whose elements can be passed to write1/read1.
+//   The elements might be node names or keys, or some wrapper around a node instance.
+//
+// THRASH: to run a client node for a random amount of time averaging the refresh interval,
+//   disconnect, and then immediately reconnect with no memory of any stored key/value pairs.
+//   It is not specified whether the node must come back with the same node name or node key,
+//   but it is required that contacts must be updated with any new values.
+// 
+// CLIENT NODE: A node representing an application user, that is subject to thrashing.
+//
+// SERVER NODE: A node managed by the server that does NOT thrash.
+//   1. The server nodes provide persistence of stored values when all the client nodes have disconnected,
+//      and thus ARE included in contacts. (AKA as "persistence nodes" or "community nodes".)
+//   2. In addition, the implementation-specific function setupServerNodes also ensures
+//      that client nodes have nodes that they can join through (AKA "bootstrap nodes"),
+//      but it is not specified whether this function is performed by the server nodes in the
+//      contacts array, or by other nodes managed by the server.
+//
+// READY: A node is ready when write1/read1 operations may be performed through it without loss.
+//   It is not specified whether this condition is an arbitrary amount of time or the result of
+//   having positively completed completed some probing operation with the network. The
+//   implementation-specific asynchronous operations to set up, thrash, write, and read must
+//   not resolve until the node is "ready" for more operations. For example, a write that has
+//   resolved, followed by nodes thrashing (averaging the refresh interval), followed by
+//   a read sould produce the written value.
+
+async function timed(operation, logString) {
+  // Time the execution of await operation(startTime), and
+  // then log the result of await logString(elapsedSeconds).
+  // Promises the elapsed time in milliseconds.
+  const startTime = Date.now();
+  await operation(startTime);
+  const endTime = Date.now();
+  const elapsed = endTime - startTime;
+  console.log(await logString(elapsed/1e3));
+  return elapsed;
+}
+
+async function getContactsLength() {
+  // Promise current length of contacts. (Convenience for sanity checks.)
+  const contacts = await getContacts();
+  return contacts.length;
+}
+function delay(ms, label = '') {
+  // Promise to resolve in the given milliseconds.
+  // Logs what it is doing if given a label.
+  // Reports any non-trivial lagging.
+  // NOTE: this is measuring lag on the machine running the tests:
+  //  1. It does not (currently) check remote systems.
+  //  2. It does not automatically fail the tests if there is lag, although it likely
+  //     that things will fail. So check the logs.
+  if (label && ms) console.log(`(${(ms/1e3).toFixed(3)}s ${label})`);
+  const start = Date.now();
+  return new Promise(resolve => setTimeout(() => {
+    const lag = Date.now() - start - ms;
+    if (lag > 250) console.log(`** System is overloaded by ${lag.toLocaleString()} ms. **`);
+    resolve();
+  }, ms));
+}
+
+export async function startWrite1(name, bootstrapContact, refreshTimeIntervalMS, index) {
+  return await start1(name, bootstrapContact, refreshTimeIntervalMS)
+    .then(contact => write1(contact, name, name));
+}
+
+async function awaitNonNullContact(contacts, i) { // Kind of stupid...
+  // When a contact thrashes, its contact[i] is null for a moment.
+  // TODO Alt: When thrashing, set slot to a promise and await it in all references, rather than this.
+  let contact = contacts[i];
+  if (contact) return contact;
+  await delay(50);
+  return await awaitNonNullContact(contacts, i);
+}
+
+async function parallelWriteAll() {
+  // Persist a unique string through each contact all at once, but not resolving until all are ready.
+  const contacts = await getContacts();
+  // The key and value are the same, to facilitate read confirmation.
+  const writes = await Promise.all(contacts.map(async (contact, index) => {
+    let ok;
+    for (let i = 0; i < 10; i++) {
+      contact ||= await awaitNonNullContact(contacts, index);
+      ok = await write1(contact, index, index);
+      if (ok) break;
+      contact = contacts[index];
+    }
+    // It's ok if the contact disconnect right after, as long the write succeeded.
+    expect(ok).toBeTruthy();
+  }));
+  return writes.length;
+}
+async function serialWriteAll() { // One-at-atime alternative to above, useful for debugging.
+  const contacts = await getContacts();
+  for (let index = 0; index < contacts.length; index++) {
+    const ok = await write1(await awaitNonNullContact(contacts, index), index, index);
+    expect(ok).toBeTruthy();
+  }
+  return contacts.length;
+}
+async function parallelReadAll(start = 0) {
+  // Reads from a random contact, confirming the value, for each key written by writeAll.
+  const contacts = await getContacts();
+  const readPromises = await Promise.all(contacts.map(async (_, index) => {
+    if (index < start) return;
+    const value = await read1(await getRandomLiveContact(), index);
+    expect(value).toBe(index);
+  }));
+  return readPromises.length - start;
+}
+async function serialReadAll() { // One-at-a-time alternative of above, useful for debugging.
+  const contacts = await getContacts();
+  for (let index = 0; index < contacts.length; index++) {
+    const value = await read1(await getRandomLiveContact(), index);
+    expect(value).toBe(index);
+  }
+  return contacts.length;
+}
+
+describe("DHT", function () {
+  function test(parameters = {}) {
+    // Define a suite of tests with the given parameters.
+    const {nServerNodes = 10,
+	   pingTimeMS = 40, // Round-trip network time. Implementation should pad network calls to achieve what is specified here.
+	   maxTransports = 152, // How many direct connections are allowed per node?
+	   maxClientNodes = Infinity, // If zero, will try to make as many as it can in refreshTimeIntervalMS.
+	   refreshTimeIntervalMS = 15e3, // How long on average does a client stay up?
+	   setupTimeMS = Math.max(2e3, refreshTimeIntervalMS), // How long to create the test nodes.
+	   runtimeBeforeWriteMS = 3 * refreshTimeIntervalMS, // How long to probe (and thrash) before writing starts.
+	   runtimeBeforeReadMS = runtimeBeforeWriteMS, // How long to probe (and thrash) before reading starts.
+	   startThrashingBefore = 'creation', // When to start thrashing clients: before creation|writing|reading. Anything else is no thrashing.
+	   notes = ''
+	  } = parameters;
+    const suiteLabel = `Server nodes: ${nServerNodes}, setup time: ${setupTimeMS}, max client nodes: ${maxClientNodes ?? Infinity}, ping: ${pingTimeMS}ms, max connections: ${maxTransports}, refresh: ${refreshTimeIntervalMS.toFixed(3)}ms, pause before write: ${runtimeBeforeWriteMS.toFixed(3)}ms, pause before read: ${runtimeBeforeReadMS.toFixed(3)}ms, thrash before: ${startThrashingBefore}`;
+    
+    describe(suiteLabel, function () {
+      beforeAll(async function () {
+	console.log('\n' + suiteLabel);
+	if (notes) console.log(notes);
+	await delay(3e3); // For gc
+	await timed(_ => setupServerNodes(nServerNodes, refreshTimeIntervalMS, pingTimeMS, maxTransports),
+		    elapsed => `Server setup ${nServerNodes} / ${elapsed} = ${Math.round(nServerNodes/elapsed)} nodes/second.`);
+	expect(await getContactsLength()).toBe(nServerNodes); // sanity check
+	//console.log('end server setup');
+      });
+      afterAll(async function () {
+	//console.log('start server shutdown');
+	await shutdownServerNodes(nServerNodes);
+	expect(await getContactsLength()).toBe(0); // sanity check
+	//console.log('end server shutdown');	
+      }, 20e3);
+
+      describe("joins within a refresh interval", function () {
+	let nJoined = 0, nWritten = 0;
+	beforeAll(async function () {
+	  //console.log('start client setup');
+	  if (startThrashingBefore === 'creation') await startThrashing(nServerNodes, refreshTimeIntervalMS);
+	  let elapsed = await timed(async _ => nJoined = await setupClientsByTime(refreshTimeIntervalMS, nServerNodes, maxClientNodes, setupTimeMS),
+				    elapsed => `Created ${nJoined} / ${elapsed} = ${(elapsed/nJoined).toFixed(3)} client nodes/second.`);
+	  expect(await getContactsLength()).toBe(nJoined + nServerNodes); // Sanity check
+	  if (maxClientNodes < Infinity) expect(nJoined).toBe(maxClientNodes); // Sanity check
+	  if (startThrashingBefore === 'writing') await startThrashing(nServerNodes, refreshTimeIntervalMS);
+	  await delay(runtimeBeforeWriteMS, 'pause before writing');
+	  elapsed = await timed(async _ => nWritten = await parallelWriteAll(), // Alt: serialWriteAll
+				elapsed => `Wrote ${nWritten} / ${elapsed} = ${Math.round(nWritten/elapsed)} nodes/second.`);
+	  //console.log('end client setup');
+	}, setupTimeMS + runtimeBeforeWriteMS + runtimeBeforeWriteMS + 3 * setupTimeMS);
+	afterAll(async function () {
+	  //console.log('start client shutdown');
+	  //await Node.reportAll();
+	  await shutdownClientNodes(nServerNodes, nJoined);
+	  expect(await getContactsLength()).toBe(nServerNodes); // Sanity check.
+	  //console.log('end client shutdown');	  
+	}, 20e3);
+	it("produces.", async function () {
+	  const total = await getContactsLength();
+	  expect(total).toBe(nJoined + nServerNodes); // Sanity check
+	  expect(nWritten).toBe(total);
+	});
+	it("can be read.", async function () {
+	  if (startThrashingBefore === 'reading') await startThrashing(nServerNodes, refreshTimeIntervalMS);
+	  await delay(runtimeBeforeReadMS, 'pause before reading');
+	  let nRead = 0;
+	  await timed(async _ => nRead = await parallelReadAll(), // alt: serialReadAll
+		      elapsed => `Read ${nRead} / ${elapsed} = ${Math.round(nRead/elapsed)} values/second.`);
+	  expect(nRead).toBe(nWritten);
+	}, 10 * setupTimeMS + 5 * runtimeBeforeReadMS);
+      });
+    });
+  }
+
+  // Each call here sets up a full suite of tests with the given parameters, which can be useful for development and debugging.
+  // For example:
+  test({pingTimeMS: 0, refreshTimeIntervalMS: 0, startThrashingBefore: 'never', notes: "Runs flat out if probing and disconnects turned off."});
+  test({setupTimeMS: 1e3, pingTimeMS: 0, startThrashingBefore: 'never', notes: "Probing on, but no disconnects or network delay."});
+  test({maxClientNodes: 30, pingTimeMS: 0, refreshTimeIntervalMS: 5e3, notes: "Small networks allow faster smoke-testing."});
+  test({maxTransports: 85, maxClientNodes: 90, pingTimeMS: 10, setupTimeMS: 20e3, notes: "Limit number of transports enough to exercise the reconnect logic."});
+  test({maxClientNodes: 140, setupTimeMS: 60e3, pingTimeMS: 10, notes: "Relatively larger network size."});
+
+  //test({maxTransports: 95, maxClientNodes: 100, refreshTimeIntervalMS: 0, startThrashingBefore: 'never', notes: 'dev: no refresh, no thrashing'});
+  //test({maxTransports: 95, maxClientNodes: 100, startThrashingBefore: 'never', notes: 'dev: no thrashing'});
+
+  //test({maxClientNodes: 7, nServerNodes: 5, refreshTimeIntervalMS: 3e3, runtimeBeforeWriteMS: 0e3, runtimeBeforeReadMS: 0e3, startThrashingBefore: 'never'});
+  //test({maxClientNodes: 3, nServerNodes: 3, refreshTimeIntervalMS: 3e3, runtimeBeforeWriteMS: 6e3, runtimeBeforeReadMS: 6e3});
+
+  
+  // TODO:
+  // Persistence Test that joins+writes one at a time until period, runs 3xperiod, then quits one a time until gone, then one node join and reads all
+  // collect and confirm data from each node on shutdown.
+  // pub/sub
+  // 1k nodes
+});

package/spec/dhtImplementation.js

@@ -0,0 +1,191 @@
+// An example of the control functions needed for testing.
+
+// For running a server
+// import express from 'express';
+// import http from 'http';
+//import { router } from './routes/index.js';
+
+////////////////////////////////////////////////////////////////////////////////////////////////
+// This section certainly needs to be modified for any given implementation.
+//
+
+// In the present case, these manipulate a Contact that directly contains a
+// DHT node with simulated networking.
+//import { InProcessWebContact as Contact, Node } from '../index.js';
+//import { SimulatedContact as Contact, Node } from '../index.js';
+import { SimulatedConnectionContact as Contact, Node } from '../index.js';
+export { Node };
+
+
+export async function start1(name, bootstrapContact, refreshTimeIntervalMS, isServerNode = false) {
+  const contact = await Contact.create({name, refreshTimeIntervalMS, isServerNode});
+  if (bootstrapContact) await contact.join(bootstrapContact);
+  return contact;
+}
+
+export async function startServerNode(name, bootstrapContact, refreshTimeIntervalMS) {
+  return await start1(name, bootstrapContact, refreshTimeIntervalMS, true);
+}
+
+export async function stop1(contact) {
+
+  // For debugging: Report if we're killing the last holder of our data.
+  // This is fine for simulations, but some decentralized reporting would be needed for real systems.
+  // if (contact && isThrashing) {
+  //   for (const key of contact.node.storage.keys()) {
+  //     let remaining = [];
+  //     for (const contact of contacts) {
+  // 	if (contact?.isConnected && contact.node.storage.has(key)) remaining.push(contact.node.name);
+  //     }
+  //     if (!remaining.length) console.log(`Disconnecting ${contact.node.name}, last holder of ${key}: ${contact.node.storage.get(key)}.`);
+  //   }
+  // }
+
+  return await contact?.disconnect();
+}
+
+export async function write1(contact, key, value) {
+  // Make a request through contact to store value under key in the DHT
+  // resolving when ready. (See test suite definitions.)
+  return await contact.node.storeValue(key, value);
+}
+export async function read1(contact, key) {
+  // Promise the result of requesting key from the DHT through contact.
+  return await contact.node.locateValue(key);
+}
+
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////
+// Given the above, the following might not need to be redefined on a per-implemmentation basis,
+// but see pingTimeMS in setupServerNodes().
+
+var contacts = [];
+export async function getContacts() {
+  // Return a list of contact information for all the nodes.
+  // For real implementations this might be a list of node identifiers.
+  // It is async because it might be collecting from some monitor/control service.
+
+  // For a simulator in one Javascript instance, it is just the list of Contacts.
+  return contacts;
+}
+function randomInteger(max = contacts.length) {
+  // Return a random number between 0 (inclusive) and max (exclusive), defaulting to the number of contacts made.
+  return Math.floor(Math.random() * max);
+}
+export async function getRandomLiveContact() {
+  // Answer a randomly selected contact (including those for server nodes) that is
+  // is not in the process of reconnecting.
+  return contacts[randomInteger()] || await getRandomLiveContact();
+}
+export async function getBootstrapContact(nServerNodes) {
+  return contacts[randomInteger(nServerNodes)];
+}
+
+var isThrashing = false;
+const thrashers = [];
+function thrash(i, nServerNodes, refreshTimeIntervalMS) { // Start disconnect/reconnect timer on contact i.
+  // If we are asked to thrash with a zero refreshTimeIntervalMS, average one second anyway.
+  const average = Math.max(refreshTimeIntervalMS, 2e3);
+  const min = Math.min(average / 2, 2e3);
+  const runtimeMS = randomInteger(2 * (average - min)) + min;
+  thrashers[i] = setTimeout(async () => {
+    if (!isThrashing) return;
+    const contact = contacts[i];
+    contacts[i] = null;
+    await stop1(contact);
+    if (!isThrashing) return;
+    const bootstrapContact = await getBootstrapContact(nServerNodes);
+    const started = await start1(i, bootstrapContact, refreshTimeIntervalMS);
+    if (!isThrashing) return; 
+   contacts[i] = started;
+    thrash(i, nServerNodes, refreshTimeIntervalMS);
+  }, runtimeMS);
+}
+export async function startThrashing(nServerNodes, refreshTimeIntervalMS) {
+  console.log('Start thrashing');
+  isThrashing = true;
+  for (let i = nServerNodes; i < contacts.length; i++) {
+    thrash(i, nServerNodes, refreshTimeIntervalMS);
+  }
+}
+export async function stopThrashing() {
+  isThrashing = false;
+  for (const thrasher of thrashers) clearTimeout(thrasher);
+}
+
+async function shutdown(startIndex, stopIndex) { // Internal
+  // Shutdown n nodes.
+  for (let i = startIndex; i < stopIndex; i++) {
+    await stop1(contacts.pop());
+  }
+}
+
+
+// const app = express();
+// const port = 3000;
+// app.set('port', port);
+// app.use(express.json());
+// //app.use('/test', router);
+// app.listen(port);
+
+let ping, transports;
+export async function setupServerNodes(nServerNodes, refreshTimeIntervalMS, pingTimeMS, maxTransports) {
+  // Set up nServerNodes, returning a promise that resolves when they are ready to use.
+  // See definitions in test suite.
+
+  Node.contacts = contacts = []; // Quirk of simulation code.
+  transports = Node.maxTransports;
+  Node.maxTransports = maxTransports;
+  ping = Contact.pingTimeMS;
+  Contact.pingTimeMS = pingTimeMS;
+  
+  for (let i = 0; i < nServerNodes; i++) {
+    const node = await startServerNode(i, contacts[i - 1], refreshTimeIntervalMS);
+    contacts.push(node);
+  }
+}
+export async function shutdownServerNodes(nServerNodes) {
+  // Shut down the specified number of server nodes, resolving when complete.
+  // The nServerNodes will match that of the preceding setupServerNodes.
+  // The purpose here is to kill any persisted data so that the next call
+  // to setupServerNodes will start fresh.
+  await shutdown(0, nServerNodes);
+  Contact.pingTimeMS = ping;
+  Node.maxTransports = transports;
+  Node.contacts = [];
+}
+
+export async function setupClientsByTime(...rest) {
+  // Create as many ready-to-use client nodes as possible in the specified milliseconds.
+  // Returns a promise that resolves to the number of clients that are now ready to use.
+  return await serialSetupClientsByTime(...rest);
+  // Alternatively, one could launch batches of clients in parallel, and then
+  // wait for each to complete its probes.
+}
+async function serialSetupClientsByTime(refreshTimeIntervalMS, nServerNodes, maxClientNodes, runtimeMS, op=start1) {
+  // Do setupClientsByTime one client node at a time.
+  // It takes longer and longer as the number of existing nodes gets larger.
+  return await new Promise(async resolve => {
+    const nBootstraps = contacts.length;
+    let done = false, index = nBootstraps, count = 0;
+    setTimeout(() => done = true, runtimeMS);
+    while (!done && (count++ < maxClientNodes)) {
+      const bootstrapContact = await getBootstrapContact(nBootstraps);
+      const contact = await op(index, bootstrapContact, refreshTimeIntervalMS);
+      if (!done) { // Don't include it if we're now over time.
+	contacts.push(contact); 
+	if (isThrashing) thrash(index, nServerNodes, refreshTimeIntervalMS);
+	index++;
+      } else {
+	await stop1(contact);
+      }
+    }
+    resolve(contacts.length - nBootstraps);
+  });
+}
+export async function shutdownClientNodes(nServerNodes, nClientNodes) {
+  await stopThrashing();
+  await new Promise(resolve => setTimeout(resolve, 5e3));
+  await shutdown(nServerNodes, nClientNodes + nServerNodes);
+}