@yz-social/kdht 0.1.7 → 0.1.9

package/dht/kbucket.js

@@ -59,7 +59,7 @@ export class KBucket {
     return false;
   }
 
-  async addContact(contact) { // Returns 'present' or 'added' if it was added to end within capacity, else false.
+  addContact(contact) { // Returns 'present' or 'added' if it was added to end within capacity, else false.
     // Resets refresh timer.
     this.node.constructor.assert(contact.node.key !== this.node.key, 'attempt to add self contact to bucket');
     let added = this.removeKey(contact.key, false) || 'added';
@@ -67,7 +67,7 @@ export class KBucket {
     if (this.isFull) {
       if (added === 'present') this.node.looseTransports.push(contact); // So no findContact will fail during ping. Should we instead serialize findContact?
       const head = this.contacts[0];
-      if (await head.sendRPC('ping', head.key)) { // still alive
+      if (head.connection) { // still alive
 	added = false;  // New contact will not be added.
 	contact = head; // Add head back, below.
       }

package/dht/node.js

@@ -109,12 +109,11 @@ export class Node extends NodeProbe {
     this.ilog('joining', contact.sname);
     contact = this.ensureContact(contact);
     await contact.connect();
-    await this.addToRoutingTable(contact);
+    this.addToRoutingTable(contact);
     await this.locateNodes(this.key); // Discovers between us and otherNode.
 
     // Refresh every bucket farther out than our closest neighbor.
     // I think(?) that this can be done by refreshing "just" the farthest bucket:
-    //this.ensureBucket(this.constructor.keySize - 1).resetRefresh();
     await this.ensureBucket(this.constructor.keySize - 1).refresh();
     // But if it turns out to be necessary to explicitly refresh each next bucket in turn, this is how:
     // let started = false;

package/dht/nodeContacts.js

@@ -56,6 +56,10 @@ export class NodeContacts extends NodeTransports {
     });
     return contacts;
   }
+  get connections() {
+    return this.contacts.filter(contact => contact.connection)
+      .concat(this.looseTransports.filter(contact => contact.connection));
+  }
   contactDictionary = {}; // maps name => contact for lifetime of Node instance until removeContact.
   existingContact(name) { // Returns contact with the given name for this node, without searching buckets or looseTransports.
     return this.contactDictionary[name];
@@ -80,20 +84,14 @@ export class NodeContacts extends NodeTransports {
     if (sponsor) contact.noteSponsor(sponsor);
     return contact;
   }
-  routingTableSerializer = Promise.resolve();
-  queueRoutingTableChange(thunk) { // Promise to resolve thunk() -- after all previous queued thunks have resolved.
-    return this.routingTableSerializer = this.routingTableSerializer.then(thunk);
-  }
   removeContact(contact) { // Removes from node entirely if present, from looseTransports or bucket as necessary, returning bucket if that's where it was, else null.
-    return this.queueRoutingTableChange(() => {
-      delete this.contactDictionary[contact.name];
-      const key = contact.key;
-      if (this.removeLooseTransport(key)) return null;
-      const bucketIndex = this.getBucketIndex(key);
-      const bucket = this.routingTable.get(bucketIndex);
-      // Host might not yet have added node or anyone else as contact for that bucket yet, so maybe no bucket.
-      return bucket?.removeKey(key) ? bucket : null;
-    });
+    delete this.contactDictionary[contact.name];
+    const key = contact.key;
+    if (this.removeLooseTransport(key)) return null;
+    const bucketIndex = this.getBucketIndex(key);
+    const bucket = this.routingTable.get(bucketIndex);
+    // Host might not yet have added node or anyone else as contact for that bucket yet, so maybe no bucket.
+    return bucket?.removeKey(key) ? bucket : null;
   }
   addToRoutingTable(contact) { // Promise contact, and add it to the routing table if room.
     if (contact.key === this.key) return null; // Do not add self.
@@ -102,18 +100,16 @@ export class NodeContacts extends NodeTransports {
     // we have already dropped the connection.
     //this.constructor.assert(contact.connection, 'Adding contact without connection', contact.report, 'in', this.contact.report);
 
-    return this.queueRoutingTableChange(async () => {
-      const bucketIndex = this.getBucketIndex(contact.key);
-      const bucket = this.ensureBucket(bucketIndex);
+    const bucketIndex = this.getBucketIndex(contact.key);
+    const bucket = this.ensureBucket(bucketIndex);
 
-      // Try to add to bucket
-      const added = await bucket.addContact(contact);
-      if (added !== 'present') { // Not already tracked in bucket.
-	this.removeLooseTransport(contact.key); // Can't be in two places.
-	this.queueWork(() => this.replicateCloserStorage(contact));
-      }
-      return added;
-    });
+    // Try to add to bucket
+    const added = bucket.addContact(contact);
+    if (added !== 'present') { // Not already tracked in bucket.
+      this.removeLooseTransport(contact.key); // Can't be in two places.
+      this.replicateCloserStorage(contact); // Asynchronous, but don't wait for it here.
+    }
+    return added;
   }
   findClosestHelpers(targetKey, count = this.constructor.k) { // Answer count closest Helpers to targetKey, including ourself.
     if (!this.contact) return []; // Can happen while we are shutting down during a probe.

package/dht/nodeMessages.js

@@ -25,30 +25,65 @@ export class NodeMessages extends NodeContacts {
     if (value !== undefined) return {value};
     return this.findClosestHelpers(key);
   }
-  async signals(key, signals, forwardingExclusions = false) {
-    const origin = signals[0];
-    //this.xlog(this.key, 'handling signals request for', {origin, key, signals, forwardingExclusions});
-    //await this.constructor.delay(100); // fixme remove
-    if (!this.isRunning) return null;
-    if (this.key === key) return await this.contact.signals(...signals); // Yay, us!
+  async signals(key, signals, forwardingExclusions = null, targetNameForDebugging) {
+    // Handle an exchange of signals, with a response that may include {result, forwardingExclusions}. See code.
+
+    if (!this.isRunning) { // In case it happens in simulations.
+      //this.xlog('\n*** not running ***');
+      return null;  //{forwardingExclusions}; // FIXME
+    }
+
+    // If the key is us, pass the signals to our home contact and respond with the WebRTC signals from the contact.
+    // (Subtle: the signals will contain the sender name. The handler in our home contact will create
+    // a new specific contact if necessary, and set up the WebRTC through that.)
+    // The forwardingExclusions are passed back, in case the sender wants to see the steps involved.
+    if (this.key === key) return {result: await this.contact.signals(...signals), forwardingExclusions};
+
+    // If we have a direct connection to the key, pass it on and answer what it tells us.
+    // (E.g., if we sponsored target for sender, we will have a direct connection that will answer as above.)
+    let contact = this.findContactByKey(key);
+    if (contact && contact.connection) {
+      forwardingExclusions?.push(this.name); // Keeps stats accurate if sender is examining paths.
+      const response = await contact.sendRPC('signals', key, signals, forwardingExclusions, targetNameForDebugging);
+      if (response) return response;
+      return {forwardingExclusions}; // Subtle: If it fails, return a definitive failure instead of just null.
+    }
 
-    let contact = this.findContactByKey(key); // If we have the target as a contact, use it directly.
-    if (!contact && !forwardingExclusions) return null;
-    if (contact) return await contact.sendRPC('signals', key, signals, forwardingExclusions);
     // Forward recursively.
-    const contacts = this.findClosestHelpers(key).map(helper => helper.contact);
+    if (forwardingExclusions) return await this.recursiveSignals(key, signals, forwardingExclusions, Contact.forwardingTimeoutMS, targetNameForDebugging);
+
+    // We were a sponsor but for a contact has since disconnected. We do not know if they are still connected to others.
+    //this.xlog('\n*** sponsored disconnected ***');
+    return {forwardingExclusions}; // FIXME: Is this definitively right, or should we answer null here?
+  }
+  static maxTries = Math.pow(this.alpha, 3); // alpha tries at each of three deep, or equivalent.
+  async recursiveSignals(key, signals, forwardingExclusions, expiration, targetNameForDebugging) { // Forward recursively.
+    // The target key may not be reachable from here (and might not even still be running).
+    // So bound our branching.
+    if (Date.now() > expiration) return null;
+    let remainingThisNode = this.constructor.alpha; // If it's good enough for probing, then it's good enough here.
+    if (forwardingExclusions.length > this.constructor.maxTries) {
+      this.xlog('abandoning wandering path towards', targetNameForDebugging, 'through', forwardingExclusions.join(', '));
+      return {forwardingExclusions};
+    }
+    const helpers = this.findClosestHelpers(key);
+    const contacts = helpers.map(helper => helper.contact);
     forwardingExclusions.push(this.name);
-    //this.xlog('forwarding signals', {key, forwardingExclusions, contacts: contacts.map(c => c.sname)});
+
     for (const contact of contacts) {
-      // this.xlog('contact', contact.sname, contact.isRunning ? 'running' : 'dead',
-      // 		contact.connection ? 'connected': 'unconnected',
-      // 		forwardingExclusions.includes(contact.name) ? 'excluded' : 'allowed');
+      if (!remainingThisNode--) break;
       if (!contact.isRunning) continue;
       if (!contact.connection) continue;
       if (forwardingExclusions.includes(contact.name)) continue;
-      const response = await contact.sendRPC('signals', key, signals, forwardingExclusions);
-      //this.xlog('got response from', contact.sname);
-      if (response) return response;
+      this.constructor.assert(contact.key !== this.key, 'forwarding through self');
+      //this.xlog('forwarding through', contact.sname);
+      const response = await contact.sendRPC('signals', key, signals, forwardingExclusions, targetNameForDebugging);
+      if (response) {
+	return response;
+      } else { // No response at all: continue with further calls that exclude contact.
+	//this.xlog('No forwarding response from', contact.sname, );
+	forwardingExclusions.push(contact.name);
+      }
     }
     return null;
   }
@@ -58,8 +93,7 @@ export class NodeMessages extends NodeContacts {
     this.constructor.assert(typeof(method)==='string', 'no method', method, sender, rest);
     this.constructor.assert(sender instanceof Contact, 'no sender', method, sender, rest);
     this.constructor.assert(sender.host.key === this.key, 'sender', sender.host.name, 'not on receiver', this.name);
-    // The sender exists, so add it to the routing table, but asynchronously so as to allow it to finish joining.
-    this.addToRoutingTable(sender);
+    this.addToRoutingTable(sender); // sender exists, so add it to the routing table.
     if (!(method in this)) {
       this.xlog('Does not handle method', method);
       return null;

package/dht/nodeProbe.js

@@ -18,10 +18,10 @@ export class NodeProbe extends NodeMessages {
     if (!results) { // disconnected
       if (trace) this.log(helper.name, '=> disconnected');
       this.log('removing unconnected contact', contact.sname);
-      await this.removeContact(contact);
+      this.removeContact(contact);
       return null; // signal that there is *no* response from this contact - to distinguish from a response that confirms that the contact is alive, even if there are (after filtering) no new contacts to try.
     }
-    await this.addToRoutingTable(contact); // Live node, so update bucket.
+    this.addToRoutingTable(contact); // Live node, so update bucket.
     // this.log('step added contact', contact.sname);
     if (this.constructor.isContactsResult(results)) { // Keep only those that we have not seen, and note the new ones we have.
       const rawResults = results;
@@ -55,6 +55,7 @@ export class NodeProbe extends NodeMessages {
     if (trace) this.log(`iterate: key=${targetKey}, finder=${finder}, k=${k}`);
 
     if (targetKey !== this.key) {
+      // Schedule a refresh for the targetKey's bucket.
       const bucketIndex = this.getBucketIndex(targetKey);
       const bucket = this.routingTable.get(bucketIndex);
       // Subtle: if we don't have one now, but will after, refreshes will be rescheduled by KBucket constructor.
@@ -118,7 +119,7 @@ export class NodeProbe extends NodeMessages {
       return null;
     };
 
-    // Handler for when a request completes.  result is only expected if status='responded'.
+    // Handler for when a request completes.  a non-null result is only expected if status='responded'.
     const handleCompletion = (helper, status, result) => {
       if (iterationFinished) return; // too late
 
@@ -166,7 +167,7 @@ export class NodeProbe extends NodeMessages {
 
         // Result is array of Helpers (may be empty if node had no new contacts)
         // Merge new helpers into allNodesSeen and track progress
-        if (result?.length > 0) {
+        if (result.length > 0) {
           allNodesSeen.push(...result);
           allNodesSeen.sort(Helper.compare); // Keep sorted by distance (best-first).
           responsesWithoutNewNodes = 0; // reset counter
@@ -226,7 +227,7 @@ export class NodeProbe extends NodeMessages {
       }
 
       this.step(targetKey, finder, helper, keysSeen, trace)
-        .then(result => handleCompletion(helper, 'responded', result))
+        .then(result => handleCompletion(helper, result ? 'responded' : 'disconnected', result))
         .catch(err => {
           // Handle errors - treat as disconnected
           handleCompletion(helper, 'disconnected');

package/dht/nodeRefresh.js

@@ -30,10 +30,6 @@ export class NodeRefresh extends NodeKeys {
     const adjustment = this.randomInteger(margin);
     return Math.floor(target + margin/2 - adjustment);
   }
-  workQueue = Promise.resolve();
-  queueWork(thunk) { // Promise to resolve thunk() -- after all previous queued thunks have resolved.
-    return this.workQueue = this.workQueue.then(thunk);
-  }
   timers = new Map();
   schedule(timerKey, statisticsKey, thunk, timeout = this.fuzzyInterval()) {
     // Schedule thunk() to occur at a fuzzyInterval from now, cancelling any
@@ -46,16 +42,14 @@ export class NodeRefresh extends NodeKeys {
     const start = Date.now();
     clearInterval(this.timers.get(timerKey));
     this.timers.set(timerKey, setTimeout(async () => {
-      const lag = Date.now() - start - timeout;
+      const now = Date.now();
+      const elapsed = now - start;
+      const lag = elapsed - timeout;
       this.timers.delete(timerKey);
       if (this.isStopped()) return;
+      this.ilog('refresh', statisticsKey, timerKey, 'last/lag ms:', elapsed.toLocaleString(), lag.toLocaleString());
       if (lag > 250) console.log(`** System is overloaded by ${lag.toLocaleString()} ms. **`);
-      // Each actual thunk execution is serialized: Each Node executes its OWN various refreshes and probes
-      // one at a time. This prevents a node from self-DoS'ing, but of course it does not coordinate across
-      // nodes. If the system is bogged down for any reason, then the timeout spacing will get smaller
-      // until finally the node is just running flat out.
-      // this.log('queue', statisticsKey, timerKey, timeout);
-      await this.queueWork(thunk);
+      await thunk();
     }, timeout));
   }
 }

package/dht/nodeStorage.js

@@ -14,6 +14,7 @@ export class NodeStorage extends NodeRefresh {
     // Claude.ai suggests just writing to the next in line, but that doesn't work.
     this.schedule(key, 'storage', () => {
       this.ilog('refresh value', value, 'at key', key);
+      // IF storeValue determines we are one of the nodes to store, then it will get scheduled again.
       this.storeValue(key, value);
     });
   }
@@ -22,7 +23,7 @@ export class NodeStorage extends NodeRefresh {
   }
   async replicateCloserStorage(contact) { // Replicate to new contact any of our data for which contact is closer than us.
     for (const key in this.storage.keys()) {
-      if (contact.distance(key) <= this.distance(key)) { //fixme define distance on contact
+      if (contact.distance(key) <= this.distance(key)) {
 	await contact.store(key, this.retrieveLocally(key));
       }
     }

package/dht/nodeTransports.js

@@ -1,4 +1,5 @@
 import { NodeStorage } from './nodeStorage.js';
+import { WebRTC } from '@yz-social/webrtc';
 
 // Management of Contacts that have a limited number of connections that can transport messages.
 export class NodeTransports extends NodeStorage {
@@ -16,7 +17,7 @@ export class NodeTransports extends NodeStorage {
     }
     return false;
   }
-  static maxTransports = 62; // FIXME: try 124
+  static maxTransports = WebRTC.suggestedInstancesLimit;
   noteContactForTransport(contact) { // We're about to use this contact for a message, so keep track of it.
     // Requires: if we later addToRoutingTable successfully, it should be removed from looseTransports.
     // Requires: if we later remove contact because of a failed send, it should be removed from looseTransports.
@@ -36,7 +37,7 @@ export class NodeTransports extends NodeStorage {
       }
       let dropped = removeLast(this.looseTransports);
       if (dropped) {
-	//console.log(this.name, 'dropping loose transport', dropped.name);
+	this.xlog('dropping loose transport', dropped.name);
       } else { // Find the bucket with the most connections.
 	let bestBucket = null, bestCount = 0;
 	this.forEachBucket(bucket => {
@@ -47,8 +48,8 @@ export class NodeTransports extends NodeStorage {
 	  return true;
 	});
 	dropped = removeLast(bestBucket.contacts);
-	if (!dropped) console.log('Unable to find something to drop in', this.report(null));
-	//else console.log(this.name, 'dropping transport', dropped.name, 'in bucket', bestBucket.index, 'among', bestCount);
+	if (!dropped) this.xlog('Unable to find something to drop in', this.report(null));
+	else this.xlog('dropping transport', dropped.name, 'in bucket', bestBucket.index, 'among', bestCount, 'contacts.');
       }
       dropped.disconnectTransport();
     }

package/dht/nodeUtilities.js

@@ -72,7 +72,7 @@ export class NodeUtilities {
     for (let index = 0; index < this.constructor.keySize; index++) {
       const bucket = this.routingTable.get(index);
       if (!bucket) continue;
-      report += `\n  ${index}: ` + (contactsString(bucket.contacts) || '-');
+      report += `\n  ${index} (${bucket.contacts.length}): ` + (contactsString(bucket.contacts) || '-');
     }
     return logger ? logger(report) : report;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yz-social/kdht",
-  "version": "0.1.7",
+  "version": "0.1.9",
   "description": "Pure Kademlia base, for testing variations.",
   "exports": {
     ".": "./index.js",
@@ -14,10 +14,10 @@
     "background": "npm stop; (npm start 1>server.log 2>&1 &); sleep 1",
     "bots": "node spec/bots.js",
     "thrashbots": "node spec/bots.js --thrash",
-    "test": "npx jasmine && echo '-- SUCCESS --' || echo '**** FAIL ****'"
+    "test": "npx jasmine && npx jasmine spec/webrtcTests.js && echo '-- SUCCESS --' || echo '**** FAIL ****'"
   },
   "dependencies": {
-    "@yz-social/webrtc": "^0.1.1",
+    "@yz-social/webrtc": "^0.1.2",
     "uuid": "^13.0.0"
   },
   "devDependencies": {

package/portals/node.js

@@ -10,7 +10,7 @@ export async function setup({baseURL, externalBaseURL = '', verbose, fixedSpacin
   // process.on('uncaughtException', error => console.error(hostName, 'Global uncaught exception:', error));
   // process.on('unhandledRejection', error => console.error(hostName, 'Global unhandled promise rejection:', error));
 
-  const contact = await WebContact.create({name: hostName, isServerNode: true, debug: verbose});
+  const contact = await WebContact.create({name: hostName, isServerNode: true, info: false, debug: verbose});
   // Handle signaling that comes as a message from the server.
   process.on('message', async ([senderSname, ...incomingSignals]) => { // Signals from a sender through the server.
     const response = await contact.signals(senderSname, ...incomingSignals);

package/spec/bots.js

@@ -59,9 +59,10 @@ if (cluster.isPrimary) {
   }
 }
 
+const info = false;
 await Node.delay(Node.randomInteger(Node.refreshTimeIntervalMS));
 console.log(cluster.worker?.id || 0, host);
-let contact = await WebContact.create({name: host, debug: argv.verbose});
+let contact = await WebContact.create({name: host, info, debug: argv.verbose});
 let bootstrapName = await contact.fetchBootstrap(argv.baseURL);
 let bootstrapContact = await contact.ensureRemoteContact(bootstrapName, argv.baseURL);
 await contact.join(bootstrapContact);
@@ -79,7 +80,7 @@ while (argv.thrash) {
   await contact.disconnect();
   await Node.delay(1e3); // TODO: remove?
 
-  contact = await WebContact.create({name: next, debug: argv.verbose});
+  contact = await WebContact.create({name: next, info, debug: argv.verbose});
   bootstrapName = await contact.fetchBootstrap(argv.baseURL);
   bootstrapContact = await contact.ensureRemoteContact(bootstrapName, argv.baseURL);
   await contact.join(bootstrapContact);

package/spec/dhtAcceptanceSpec.js

@@ -9,7 +9,7 @@ const { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach} = glob
 // implementation changes for different DHTs.
 import { setupServerNodes, shutdownServerNodes,
 	 start1, setupClientsByTime, shutdownClientNodes,
-	 getContacts, getRandomLiveContact,
+	 getContacts, readThroughRandom,
 	 startThrashing, write1, read1, Node, Contact } from './dhtImplementation.js';
 
 // Some definitions:
@@ -48,7 +48,7 @@ async function timed(operation, logString) {
   await operation(startTime);
   const endTime = Date.now();
   const elapsed = endTime - startTime;
-  console.log(await logString(elapsed/1e3));
+  console.log(new Date(), await logString(elapsed/1e3));
   return elapsed;
 }
 
@@ -74,41 +74,17 @@ function delay(ms, label = '') {
   }, ms));
 }
 
-export async function startWrite1(name, bootstrapContact, refreshTimeIntervalMS, index) {
-  return await start1(name, bootstrapContact, refreshTimeIntervalMS)
-    .then(contact => write1(contact, name, name));
-}
-
-async function awaitNonNullContact(contacts, i) { // Kind of stupid...
-  // When a contact thrashes, its contact[i] is null for a moment.
-  // TODO Alt: When thrashing, set slot to a promise and await it in all references, rather than this.
-  let contact = contacts[i];
-  if (contact) return contact;
-  await delay(50);
-  return await awaitNonNullContact(contacts, i);
-}
-
 async function parallelWriteAll() {
   // Persist a unique string through each contact all at once, but not resolving until all are ready.
   const contacts = await getContacts();
   // The key and value are the same, to facilitate read confirmation.
-  const writes = await Promise.all(contacts.map(async (contact, index) => {
-    let ok;
-    for (let i = 0; i < 10; i++) {
-      contact ||= await awaitNonNullContact(contacts, index);
-      ok = await write1(contact, index, index);
-      if (ok) break;
-      contact = contacts[index];
-    }
-    // It's ok if the contact disconnect right after, as long the write succeeded.
-    expect(ok).toBeTruthy();
-  }));
+  const writes = await Promise.all(contacts.map(async (contact, index) => write1(contact, index, index)));
   return writes.length;
 }
 async function serialWriteAll() { // One-at-atime alternative to above, useful for debugging.
   const contacts = await getContacts();
   for (let index = 0; index < contacts.length; index++) {
-    const ok = await write1(await awaitNonNullContact(contacts, index), index, index);
+    const ok = await write1(contacts[index], index, index);
     expect(ok).toBeTruthy();
   }
   return contacts.length;
@@ -117,14 +93,7 @@ async function parallelReadAll(start = 0) {
   // Reads from a random contact, confirming the value, for each key written by writeAll.
   const contacts = await getContacts();
   const readPromises = await Promise.all(contacts.map(async (_, index) => {
-    if (index < start) return;
-    let value;
-    for (let liveTry = 0; liveTry < 4; liveTry++) {
-      const contact = await getRandomLiveContact();
-      value = await read1(contact, index);
-      if (contact.isRunning) break;
-      console.log('\n\n*** read from dead contact. retrying', liveTry, '***\n');
-    }
+    const value = await readThroughRandom(index);
     expect(value).toBe(index);
   }));
   return readPromises.length - start;
@@ -132,7 +101,7 @@ async function parallelReadAll(start = 0) {
 async function serialReadAll() { // One-at-a-time alternative of above, useful for debugging.
   const contacts = await getContacts();
   for (let index = 0; index < contacts.length; index++) {
-    const value = await read1(await getRandomLiveContact(), index);
+    const value = await readThroughRandom(index);
     expect(value).toBe(index);
   }
   return contacts.length;
@@ -156,25 +125,25 @@ describe("DHT", function () {
     
     describe(notes || suiteLabel, function () {
       beforeAll(async function () {
-	console.log('\n' + suiteLabel);
+	console.log('\n', new Date(), suiteLabel);
 	if (notes) console.log(notes);
 	await delay(3e3); // For gc
 	await timed(_ => setupServerNodes(nServerNodes, refreshTimeIntervalMS, pingTimeMS, maxTransports),
 		    elapsed => `Server setup ${nServerNodes} / ${elapsed} = ${Math.round(nServerNodes/elapsed)} nodes/second.`);
 	expect(await getContactsLength()).toBe(nServerNodes); // sanity check
-	//console.log('end server setup');
-      });
+	console.log(new Date(), 'end server setup');
+      }, 10e3);
       afterAll(async function () {
-	//console.log('start server shutdown');
+	console.log(new Date(), 'start server shutdown');
 	await shutdownServerNodes(nServerNodes);
 	expect(await getContactsLength()).toBe(0); // sanity check
-	//console.log('end server shutdown');	
+	console.log(new Date(), 'end server shutdown');
       }, 20e3);
 
       describe("joins within a refresh interval", function () {
 	let nJoined = 0, nWritten = 0;
 	beforeAll(async function () {
-	  //console.log('start client setup');
+	  console.log(new Date(), 'start client setup');
 	  if (startThrashingBefore === 'creation') await startThrashing(nServerNodes, refreshTimeIntervalMS);
 	  let elapsed = await timed(async _ => nJoined = await setupClientsByTime(refreshTimeIntervalMS, nServerNodes, maxClientNodes, setupTimeMS),
 				    elapsed => `Created ${nJoined} / ${elapsed} = ${(elapsed/nJoined).toFixed(3)} client nodes/second.`);
@@ -182,16 +151,16 @@ describe("DHT", function () {
 	  if (maxClientNodes < Infinity) expect(nJoined).toBe(maxClientNodes); // Sanity check
 	  if (startThrashingBefore === 'writing') await startThrashing(nServerNodes, refreshTimeIntervalMS);
 	  await delay(runtimeBeforeWriteMS, 'pause before writing');
-	  elapsed = await timed(async _ => nWritten = await parallelWriteAll(), // Alt: serialWriteAll
+	  console.log(new Date(), 'writing');
+	  elapsed = await timed(async _ => nWritten = await serialWriteAll(), // Alt: serial/parallelWriteAll
 				elapsed => `Wrote ${nWritten} / ${elapsed} = ${Math.round(nWritten/elapsed)} nodes/second.`);
-	  //console.log('end client setup');
-	}, setupTimeMS + runtimeBeforeWriteMS + runtimeBeforeWriteMS + 3 * setupTimeMS);
+	}, setupTimeMS + runtimeBeforeWriteMS + runtimeBeforeWriteMS + 5 * setupTimeMS);
 	afterAll(async function () {
-	  //console.log('start client shutdown');
+	  console.log(new Date(), 'start client shutdown');
 	  //await Node.reportAll();
 	  await shutdownClientNodes(nServerNodes, nJoined);
 	  expect(await getContactsLength()).toBe(nServerNodes); // Sanity check.
-	  //console.log('end client shutdown');	  
+	  console.log(new Date(), 'end client shutdown');
 	}, 20e3);
 	it("produces.", async function () {
 	  const total = await getContactsLength();
@@ -201,8 +170,9 @@ describe("DHT", function () {
 	it("can be read.", async function () {
 	  if (startThrashingBefore === 'reading') await startThrashing(nServerNodes, refreshTimeIntervalMS);
 	  await delay(runtimeBeforeReadMS, 'pause before reading');
+	  console.log(new Date(), 'reading');
 	  let nRead = 0;
-	  await timed(async _ => nRead = await parallelReadAll(), // alt: serialReadAll
+	  await timed(async _ => nRead = await serialReadAll(), // alt: serial/parallelReadAll
 		      elapsed => `Read ${nRead} / ${elapsed} = ${Math.round(nRead/elapsed)} values/second.`);
 	  expect(nRead).toBe(nWritten);
 	}, 10 * setupTimeMS + 5 * runtimeBeforeReadMS);
@@ -217,6 +187,7 @@ describe("DHT", function () {
   test({setupTimeMS: 1e3, pingTimeMS: 0, startThrashingBefore: 'never', notes: "Probing on, but no disconnects or network delay."});
   test({pingTimeMS: 0, refreshTimeIntervalMS: 5e3, notes: "Small networks allow faster thrash smoke-testing."});
   test({notes: "Normal ops"});
+  test({setupTimeMS: 40e3, notes: "Bigger network overflowing bucket."});
   
   // test({maxClientNodes: 55, setupTimeMS: 240e3, pingTimeMS: 40, maxTransports: 62,
   // 	//startThrashingBefore: 'never', runtimeBeforeWriteMS: 0, runtimeBeforeReadMS: 0,