@yz-social/kdht 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/dht/helper.js +25 -0
- package/dht/kbucket.js +83 -0
- package/dht/node.js +155 -0
- package/dht/nodeContacts.js +122 -0
- package/dht/nodeKeys.js +52 -0
- package/dht/nodeMessages.js +29 -0
- package/dht/nodeProbe.js +87 -0
- package/dht/nodeRefresh.js +60 -0
- package/dht/nodeStorage.js +23 -0
- package/dht/nodeTransports.js +59 -0
- package/dht/nodeUtilities.js +81 -0
- package/index.js +7 -0
- package/junx.js +17 -0
- package/package.json +46 -0
- package/portals/node.js +31 -0
- package/portals/router.js +67 -0
- package/routes/index.js +39 -0
- package/save/dhtWriteRead.js +48 -0
- package/save/webrtc.js +212 -0
- package/spec/bots.js +79 -0
- package/spec/dhtAcceptanceSpec.js +227 -0
- package/spec/dhtImplementation.js +191 -0
- package/spec/dhtInternalsSpec.js +253 -0
- package/spec/dhtKeySpec.js +92 -0
- package/spec/dhtWriteRead.js +56 -0
- package/spec/node.html +47 -0
- package/spec/portal.js +119 -0
- package/spec/support/jasmine.mjs +14 -0
- package/spec/writes.js +11 -0
- package/transports/contact.js +127 -0
- package/transports/inProcessWebrtc.js +70 -0
- package/transports/simulations.js +66 -0
- package/transports/webrtc.js +249 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { NodeRefresh } from './nodeRefresh.js';
|
|
2
|
+
|
|
3
|
+
// Keeping application data.
|
|
4
|
+
export class NodeStorage extends NodeRefresh {
|
|
5
|
+
storage = new Map(); // keys must be preserved as bigint, not converted to string.
|
|
6
|
+
// TODO: store across sessions
|
|
7
|
+
storeLocally(key, value) { // Store in memory by a BigInt key (must be already hashed). Not persistent.
|
|
8
|
+
this.storage.set(key, value);
|
|
9
|
+
// TODO: The paper says this can be optimized.
|
|
10
|
+
// Claude.ai suggests just writing to the next in line, but that doesn't work.
|
|
11
|
+
this.schedule(key, 'storage', () => this.storeValue(key, value));
|
|
12
|
+
}
|
|
13
|
+
retrieveLocally(key) { // Retrieve from memory.
|
|
14
|
+
return this.storage.get(key);
|
|
15
|
+
}
|
|
16
|
+
async replicateCloserStorage(contact) { // Replicate to new contact any of our data for which contact is closer than us.
|
|
17
|
+
for (const key in this.storage.keys()) {
|
|
18
|
+
if (contact.distance(key) <= this.distance(key)) { //fixme define distance on contact
|
|
19
|
+
await contact.store(key, this.retrieveLocally(key));
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { NodeStorage } from './nodeStorage.js';
|
|
2
|
+
|
|
3
|
+
// Management of Contacts that have a limited number of connections that can transport messages.
|
|
4
|
+
export class NodeTransports extends NodeStorage {
|
|
5
|
+
looseTransports = [];
|
|
6
|
+
get nTransports() {
|
|
7
|
+
let count = this.looseTransports.length;
|
|
8
|
+
this.forEachBucket(bucket => (count += bucket.nTransports, true));
|
|
9
|
+
return count;
|
|
10
|
+
}
|
|
11
|
+
removeLooseTransport(key) { // Remove the contact for key from looseTransports, and return boolean indicating whether it had been present.
|
|
12
|
+
const looseIndex = this.looseTransports.findIndex(c => c.key === key);
|
|
13
|
+
if (looseIndex >= 0) {
|
|
14
|
+
this.looseTransports.splice(looseIndex, 1);
|
|
15
|
+
return true;
|
|
16
|
+
}
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
static maxTransports = 32;
|
|
20
|
+
noteContactForTransport(contact) { // We're about to use this contact for a message, so keep track of it.
|
|
21
|
+
// Requires: if we later addToRoutingTable successfully, it should be removed from looseTransports.
|
|
22
|
+
// Requires: if we later remove contact because of a failed send, it should be removed from looseTransports.
|
|
23
|
+
const assert = this.constructor.assert;
|
|
24
|
+
assert(contact.key !== this.key, 'noting contact for self transport', this, contact);
|
|
25
|
+
assert(contact.host.key === this.key, 'Contact', contact.report, 'is not hosted by', this.contact.report);
|
|
26
|
+
let existing = this.findContactByKey(contact.key);
|
|
27
|
+
if (existing) return existing;
|
|
28
|
+
|
|
29
|
+
if (this.nTransports >= this.constructor.maxTransports) { // Determine if we have to drop one first, and do so.
|
|
30
|
+
//console.log(this.name, 'needs to drop a transport');
|
|
31
|
+
function removeLast(list) { // Remove and return the last element of list that has connction and is NOT sponsor.
|
|
32
|
+
const index = list.findLastIndex(element => element.connection && !contact.hasSponsor(element.key));
|
|
33
|
+
if (index < 0) return null;
|
|
34
|
+
const sub = list.splice(index, 1);
|
|
35
|
+
return sub[0];
|
|
36
|
+
}
|
|
37
|
+
let dropped = removeLast(this.looseTransports);
|
|
38
|
+
if (dropped) {
|
|
39
|
+
console.log('dropping loose transport', dropped.name, 'in', this.name);
|
|
40
|
+
} else { // Find the bucket with the most connections.
|
|
41
|
+
let bestBucket = null, bestCount = 0;
|
|
42
|
+
this.forEachBucket(bucket => {
|
|
43
|
+
const count = bucket.nTransports;
|
|
44
|
+
if (count < bestCount) return true;
|
|
45
|
+
bestBucket = bucket;
|
|
46
|
+
bestCount = count;
|
|
47
|
+
return true;
|
|
48
|
+
});
|
|
49
|
+
dropped = removeLast(bestBucket.contacts);
|
|
50
|
+
if (!dropped) console.log('Unable to find something to drop in', this.report(null));
|
|
51
|
+
else console.log('dropping transport', dropped.name, 'in', this.name, bestBucket.index, 'among', bestCount);
|
|
52
|
+
}
|
|
53
|
+
dropped.disconnectTransport();
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
this.looseTransports.push(contact); // Now add it as loose. If we later addToRoutingTable, it will then be moved from looseTransports.
|
|
57
|
+
return contact;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// Some scaffolding for development and debugging.
|
|
2
|
+
export class NodeUtilities {
|
|
3
|
+
constructor(properties) {
|
|
4
|
+
Object.assign(this, properties);
|
|
5
|
+
}
|
|
6
|
+
isRunning = true;
|
|
7
|
+
static delay(ms, value) { // Promise to resolve (to nothing) after a given number of milliseconds
|
|
8
|
+
return new Promise(resolve => setTimeout(resolve, ms, value));
|
|
9
|
+
}
|
|
10
|
+
static randomInteger(max) { // Return a random number between 0 (inclusive) and max (exclusive).
|
|
11
|
+
return Math.floor(Math.random() * max);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
debug = false;
|
|
15
|
+
get sname() { // The home contact sname, or just name if no contact
|
|
16
|
+
return this.contact?.sname || this.name;
|
|
17
|
+
}
|
|
18
|
+
log(...rest) { if (this.debug) console.log(new Date(), this.sname, ...rest); }
|
|
19
|
+
xlog(...rest) { console.log(new Date(), this.sname, ...rest); }
|
|
20
|
+
static assert(ok, ...rest) { // If !ok, log rests and exit.
|
|
21
|
+
if (ok) return;
|
|
22
|
+
console.error(...rest, new Error("Assert failure").stack); // Not throwing error, because we want to exit. But we are grabbing stack.
|
|
23
|
+
globalThis.process?.exit(1);
|
|
24
|
+
}
|
|
25
|
+
// TODO: Instead of a global collector (which won't work when distributed across devices),
|
|
26
|
+
static _stats = {};
|
|
27
|
+
static get statistics() { // Return {bucket, storage, rpc}, where each value is [elapsedInSeconds, count, averageInMSToNearestTenth].
|
|
28
|
+
// If Nodes.contacts is populated, also report average number of buckets and contacts.
|
|
29
|
+
const { _stats } = this;
|
|
30
|
+
if (this.contacts?.length) {
|
|
31
|
+
let buckets = 0, contacts = 0, stored = 0;
|
|
32
|
+
for (const {node} of this.contacts) {
|
|
33
|
+
stored += node.storage.size;
|
|
34
|
+
node.forEachBucket(bucket => {
|
|
35
|
+
buckets++;
|
|
36
|
+
contacts += bucket.contacts.length;
|
|
37
|
+
return true;
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
_stats.contacts = Math.round(contacts/this.contacts.length);
|
|
41
|
+
_stats.stored = Math.round(stored/this.contacts.length);
|
|
42
|
+
_stats.buckets = Math.round(buckets/this.contacts.length);
|
|
43
|
+
}
|
|
44
|
+
return _stats;
|
|
45
|
+
}
|
|
46
|
+
static resetStatistics() { // Reset statistics to zero.
|
|
47
|
+
const stat = {count:0, elapsed:0, lag:0};
|
|
48
|
+
this._stats = {
|
|
49
|
+
bucket: Object.assign({}, stat), // copy the model
|
|
50
|
+
storage: Object.assign({}, stat),
|
|
51
|
+
rpc: Object.assign({}, stat)
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
static noteStatistic(startTimeMS, name) { // Given a startTimeMS, update statistics bucket for name.
|
|
55
|
+
const stat = this._stats?.[name];
|
|
56
|
+
if (!stat) return;
|
|
57
|
+
stat.count++;
|
|
58
|
+
stat.elapsed += Date.now() - startTimeMS;
|
|
59
|
+
}
|
|
60
|
+
report(logger = console.log) { // return logger( a string description of node )
|
|
61
|
+
let report = `Node: ${this.contact?.report || this.name}, ${this.nTransports} transports`;
|
|
62
|
+
function contactsString(contacts) { return contacts.map(contact => contact.report).join(', '); }
|
|
63
|
+
if (this.storage.size) {
|
|
64
|
+
report += `\n storing ${this.storage.size}: ` +
|
|
65
|
+
Array.from(this.storage.entries()).map(([k, v]) => `${k}n: ${JSON.stringify(v)}`).join(', ');
|
|
66
|
+
}
|
|
67
|
+
if (this.looseTransports.length) {
|
|
68
|
+
report += `\n transports ${this.looseTransports.map(contact => contact.report).join(', ')}`;
|
|
69
|
+
}
|
|
70
|
+
for (let index = 0; index < this.constructor.keySize; index++) {
|
|
71
|
+
const bucket = this.routingTable.get(index);
|
|
72
|
+
if (!bucket) continue;
|
|
73
|
+
report += `\n ${index}: ` + (contactsString(bucket.contacts) || '-');
|
|
74
|
+
}
|
|
75
|
+
return logger ? logger(report) : report;
|
|
76
|
+
}
|
|
77
|
+
static reportAll() {
|
|
78
|
+
this.contacts?.forEach(contact => contact.node.report());
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
package/index.js
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { Contact } from './transports/contact.js';
|
|
2
|
+
export { SimulatedContact, SimulatedConnectionContact } from './transports/simulations.js';
|
|
3
|
+
//export { InProcessWebContact } from './transports/inProcessWebrtc.js';
|
|
4
|
+
export { WebContact } from './transports/webrtc.js';
|
|
5
|
+
export { Helper } from './dht/helper.js';
|
|
6
|
+
export { KBucket } from './dht/kbucket.js';
|
|
7
|
+
export { Node } from './dht/node.js';
|
package/junx.js
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
async function bar() {
|
|
2
|
+
throw new Error("synchronously thrown from bar (asynchronous from foo)");
|
|
3
|
+
}
|
|
4
|
+
async function foo() {
|
|
5
|
+
try {
|
|
6
|
+
bar();
|
|
7
|
+
} catch (e) {
|
|
8
|
+
console.log('foo caught', e);
|
|
9
|
+
}
|
|
10
|
+
throw new Error('synchronously thrown from foo');
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
try {
|
|
14
|
+
await foo();
|
|
15
|
+
} catch (e) {
|
|
16
|
+
console.log('outer caught', e);
|
|
17
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@yz-social/kdht",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Pure Kademlia base, for testing variations.",
|
|
5
|
+
"exports": {
|
|
6
|
+
".": "./index.js",
|
|
7
|
+
"./router": "./portals/router.js",
|
|
8
|
+
"./portal": "./portals/node.js"
|
|
9
|
+
},
|
|
10
|
+
"type": "module",
|
|
11
|
+
"scripts": {
|
|
12
|
+
"serverTest": "node spec/portal.js --nPortals 20 --nWrites 40",
|
|
13
|
+
"serverTestLight": "node spec/portal.js --nPortals 10 --nWrites 40",
|
|
14
|
+
"smoke": "node spec/portal.js --nPortals 5 --nBots 5 --thrash --nWrites 10",
|
|
15
|
+
"smoke20": "node spec/portal.js --nPortals 10 --nBots 10 --thrash --nWrites 10",
|
|
16
|
+
"load30": "node spec/portal.js --nPortals 10 --nBots 20 --nWrites 100",
|
|
17
|
+
"load50": "node spec/portal.js --nPortals 10 --nBots 65 --nWrites 100",
|
|
18
|
+
"load75": "node spec/portal.js --nPortals 10 --nBots 40 --nWrites 100",
|
|
19
|
+
"load100": "node spec/portal.js --nPortals 10 --nBots 90 --nWrites 100",
|
|
20
|
+
"load150": "node spec/portal.js --nPortals 15 --nBots 135 --nWrites 100",
|
|
21
|
+
"load200": "node spec/portal.js --nPortals 20 --nBots 180 --nWrites 100",
|
|
22
|
+
"thrash30": "node spec/portal.js --nPortals 10 --nBots 20 --thrash true --nWrites 100",
|
|
23
|
+
"thrash50": "node spec/portal.js --nPortals 10 --nBots 40 --thrash true --nWrites 100",
|
|
24
|
+
"thrash100": "node spec/portal.js --nPortals 10 --nBots 90 --thrash true --nWrites 100",
|
|
25
|
+
"thrash150": "node spec/portal.js --nPortals 15 --nBots 135 --thrash true --nWrites 100",
|
|
26
|
+
"thrash200": "node spec/portal.js --nPortals 20 --nBots 180 --thrash true --nWrites 100",
|
|
27
|
+
"test": "npx jasmine"
|
|
28
|
+
},
|
|
29
|
+
"dependencies": {
|
|
30
|
+
"@yz-social/webrtc": "^0.1.0",
|
|
31
|
+
"uuid": "^13.0.0"
|
|
32
|
+
},
|
|
33
|
+
"devDependencies": {
|
|
34
|
+
"express": "^5.2.1",
|
|
35
|
+
"jasmine": "^5.12.0",
|
|
36
|
+
"morgan": "^1.10.1",
|
|
37
|
+
"yargs": "^18.0.0"
|
|
38
|
+
},
|
|
39
|
+
"repository": {
|
|
40
|
+
"type": "git",
|
|
41
|
+
"url": "git+https://github.com/YZ-social/kdht.git"
|
|
42
|
+
},
|
|
43
|
+
"publishConfig": {
|
|
44
|
+
"registry": "https://registry.npmjs.org"
|
|
45
|
+
}
|
|
46
|
+
}
|
package/portals/node.js
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import cluster from 'node:cluster';
|
|
3
|
+
import { v4 as uuidv4 } from 'uuid';
|
|
4
|
+
import { WebContact, Node } from '../index.js';
|
|
5
|
+
|
|
6
|
+
export async function setup({baseURL, externalBaseURL = '', verbose, fixedSpacing, variableSpacing}) {
|
|
7
|
+
const hostName = uuidv4();
|
|
8
|
+
process.title = 'kdht-portal-' + hostName;
|
|
9
|
+
// For debugging:
|
|
10
|
+
// process.on('uncaughtException', error => console.error(hostName, 'Global uncaught exception:', error));
|
|
11
|
+
// process.on('unhandledRejection', error => console.error(hostName, 'Global unhandled promise rejection:', error));
|
|
12
|
+
|
|
13
|
+
const contact = await WebContact.create({name: hostName, isServerNode: true, debug: verbose});
|
|
14
|
+
// Handle signaling that comes as a message from the server.
|
|
15
|
+
process.on('message', async ([senderSname, ...incomingSignals]) => { // Signals from a sender through the server.
|
|
16
|
+
const response = await contact.signals(senderSname, ...incomingSignals);
|
|
17
|
+
process.send([senderSname, ...response]);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
await Node.delay(fixedSpacing * 1e3 * cluster.worker.id - 1);
|
|
21
|
+
|
|
22
|
+
const isFirst = cluster.worker.id === 1; // The primary/server is 0.
|
|
23
|
+
const joinURL = isFirst ? externalBaseURL : baseURL;
|
|
24
|
+
|
|
25
|
+
if (!isFirst) await Node.delay(Node.fuzzyInterval(variableSpacing * 1e3));
|
|
26
|
+
// Determine boostrap BEFORE we send in our own name.
|
|
27
|
+
const bootstrapName = joinURL && await contact.fetchBootstrap(joinURL);
|
|
28
|
+
const bootstrap = joinURL && await contact.ensureRemoteContact(bootstrapName, joinURL);
|
|
29
|
+
process.send(contact.sname); // Report in to server as available for others to bootstrap through.
|
|
30
|
+
if (bootstrap) await contact.join(bootstrap);
|
|
31
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import cluster from 'node:cluster';
|
|
2
|
+
import express from 'express';
|
|
3
|
+
import { Node } from '../index.js';
|
|
4
|
+
|
|
5
|
+
export const router = express.Router();
|
|
6
|
+
|
|
7
|
+
const portals = {}; // Maps worker sname => worker, for the full lifetime of the program. NOTE: MAY get filed in out of order from workers.
|
|
8
|
+
const workers = Object.values(cluster.workers);
|
|
9
|
+
for (const worker of workers) {
|
|
10
|
+
worker.on('message', message => { // Message from a worker, in response to a POST.
|
|
11
|
+
if (!worker.tag) { // The very first message from a worker (during setup) will identify its tag.
|
|
12
|
+
portals[message] = worker;
|
|
13
|
+
worker.tag = message;
|
|
14
|
+
worker.requestResolvers = {}; // Maps sender sname => resolve function of a waiting promise in flight.
|
|
15
|
+
console.log(worker.id - 1, message);
|
|
16
|
+
} else {
|
|
17
|
+
// Each worker can have several simultaneous conversations going. We need to get the message to the correct
|
|
18
|
+
// conversation promise, which we do by calling the resolver that the POST handler is waiting on.
|
|
19
|
+
// Note that requestResolvers are per worker: there can only be one requestResolver pending per worker
|
|
20
|
+
// for each sender.
|
|
21
|
+
const [senderSname, ...signals] = message;
|
|
22
|
+
worker.requestResolvers[senderSname]?.(signals);
|
|
23
|
+
}
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
cluster.on('exit', (worker, code, signal) => { // Tell us about dead workers.
|
|
27
|
+
console.error(`\n\n*** Crashed worker ${worker.id}:${worker.tag} received code: ${code} signal: ${signal}. ***\n`);
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
router.get('/name/:label', (req, res, next) => { // Answer the actual sname corresponding to label.
|
|
31
|
+
const label = req.params.label;
|
|
32
|
+
// label can be a worker id, which alas starts from 1.
|
|
33
|
+
const isRandom = label === 'random';
|
|
34
|
+
let list = isRandom ? Object.values(portals) : workers;
|
|
35
|
+
const index = isRandom ? Node.randomInteger(list.length) : label - 1;
|
|
36
|
+
const worker = list[index];
|
|
37
|
+
if (!worker) return res.sendStatus(isRandom ? 403 : 404);
|
|
38
|
+
return res.json(worker.tag);
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
router.post('/join/:from/:to', async (req, res, next) => { // Handler for JSON POST requests that provide an array of signals and get signals back.
|
|
42
|
+
// Our WebRTC send [['offer', ...], ['icecandidate', ...], ...]
|
|
43
|
+
// and accept responses of [['answer', ...], ['icecandidate', ...], ...]
|
|
44
|
+
// through multiple POSTS.
|
|
45
|
+
const {params, body} = req;
|
|
46
|
+
// Find the specifed worker, or pick one at random. TODO CLEANUP: Remove. We now use as separate /name/:label to pick one.
|
|
47
|
+
const worker = portals[params.to];
|
|
48
|
+
if (!worker) {
|
|
49
|
+
console.warn('no worker', params.to);
|
|
50
|
+
return res.sendStatus(404);
|
|
51
|
+
}
|
|
52
|
+
if (!worker.tag) {
|
|
53
|
+
console.warn('worker', params.to, 'not signed in yet');
|
|
54
|
+
return res.sendStatus(403);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Each kdht worker node can handle connections from multiple clients. Specify which one.
|
|
58
|
+
body.unshift(params.from); // Adds sender sname at front of body.
|
|
59
|
+
|
|
60
|
+
// Pass the POST body to the worker and await the response.
|
|
61
|
+
const promise = new Promise(resolve => worker.requestResolvers[params.from] = resolve);
|
|
62
|
+
worker.send(body, undefined, undefined, error => error && console.log(`Error communicating with portal worker ${worker.id}:${worker.tag} ${worker.isConnected() ? 'connected' : 'disconnected'} ${worker.isDead() ? 'dead' : 'running'}:`, error));
|
|
63
|
+
let response = await promise;
|
|
64
|
+
delete worker.requestResolvers[params.from]; // Now that we have the response.
|
|
65
|
+
|
|
66
|
+
return res.send(response);
|
|
67
|
+
});
|
package/routes/index.js
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import express from 'express';
|
|
2
|
+
import { TrickleWebRTC } from '@yz-social/webrtc';
|
|
3
|
+
import { WebContact as Contact } from '../index.js';
|
|
4
|
+
export const router = express.Router();
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
router.post('/join/:key', async (req, res, next) => {
|
|
9
|
+
|
|
10
|
+
const {params, body} = req;
|
|
11
|
+
const tag = params.tag;
|
|
12
|
+
const incomingSignals = body;
|
|
13
|
+
let connection = testConnections[tag];
|
|
14
|
+
console.log('post connection:', !!connection, 'incoming:', incomingSignals.map(s => s[0]));
|
|
15
|
+
if (!connection) {
|
|
16
|
+
connection = testConnections[tag] = TrickleWebRTC.ensure({serviceLabel: tag});
|
|
17
|
+
const timer = setTimeout(() => connection.close(), 15e3);
|
|
18
|
+
console.log('created connection and applying', incomingSignals.map(s => s[0]));
|
|
19
|
+
connection.next = connection.signals;
|
|
20
|
+
const dataPromise = connection.dataChannelPromise = connection.ensureDataChannel('echo', {}, incomingSignals);
|
|
21
|
+
dataPromise.then(dataChannel => {
|
|
22
|
+
connection.reportConnection(true);
|
|
23
|
+
dataChannel.onclose = () => {
|
|
24
|
+
clearTimeout(timer);
|
|
25
|
+
connection.close();
|
|
26
|
+
delete testConnections[tag];
|
|
27
|
+
};
|
|
28
|
+
dataChannel.onmessage = event => dataChannel.send(event.data); // Just echo what we are given.
|
|
29
|
+
});
|
|
30
|
+
} else {
|
|
31
|
+
console.log('applying incoming signals');
|
|
32
|
+
connection.signals = incomingSignals;
|
|
33
|
+
}
|
|
34
|
+
console.log('awaiting response');
|
|
35
|
+
const responseSignals = await Promise.race([connection.next, connection.dataChannelPromise]);
|
|
36
|
+
connection.next = connection.signals;
|
|
37
|
+
console.log('responding', responseSignals.map(s => s[0]));
|
|
38
|
+
res.send(responseSignals); // Send back our signalling answer+ice.
|
|
39
|
+
});
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
#!/usr/bin/env npx jasmine
|
|
2
|
+
const { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } = globalThis; // For linters.
|
|
3
|
+
import process from 'node:process';
|
|
4
|
+
import { exec } from 'node:child_process';
|
|
5
|
+
import { v4 as uuidv4 } from 'uuid';
|
|
6
|
+
import { WebContact, Node } from '../index.js';
|
|
7
|
+
|
|
8
|
+
// I cannot get yargs to work properly within jasmine. Get args by hand.
|
|
9
|
+
// Note: jasmine will treat --options as arguments to itself. To pass them to the script, you need to separate with '--'.
|
|
10
|
+
const nWritesIndex = process.argv.indexOf('--nWrites');
|
|
11
|
+
const verboseIndex = process.argv.indexOf('--verbose');
|
|
12
|
+
const shutdownIndex = process.argv.indexOf('--shutdown');
|
|
13
|
+
const nWrites = nWritesIndex >= 0 ? JSON.parse(process.argv[nWritesIndex + 1]) : 10;
|
|
14
|
+
const verbose = verboseIndex >= 0 ? JSON.parse( process.argv[verboseIndex + 1] || 'true' ) : false;
|
|
15
|
+
const shutdown = shutdownIndex >= 0 ? JSON.parse( process.argv[shutdownIndex + 1] || 'true' ) : true;
|
|
16
|
+
|
|
17
|
+
describe("DHT write/read", function () {
|
|
18
|
+
let contact;
|
|
19
|
+
beforeAll(async function () {
|
|
20
|
+
contact = await WebContact.create({name: uuidv4(), debug: verbose});
|
|
21
|
+
const bootstrapName = await contact.fetchBootstrap();
|
|
22
|
+
const bootstrapContact = await contact.ensureRemoteContact(bootstrapName, 'http://localhost:3000/kdht');
|
|
23
|
+
console.log(new Date(), contact.sname, 'joining', bootstrapContact.sname);
|
|
24
|
+
await contact.join(bootstrapContact);
|
|
25
|
+
console.log('joined');
|
|
26
|
+
for (let index = 0; index < nWrites; index++) {
|
|
27
|
+
const wrote = await contact.store(index, index);
|
|
28
|
+
console.log('Wrote', index);
|
|
29
|
+
}
|
|
30
|
+
console.log(new Date(), 'Written. Waiting two refresh periods before reading.', 2 * Node.refreshTimeIntervalMS / 1e3, 'seconds');
|
|
31
|
+
await Node.delay(2 * Node.refreshTimeIntervalMS);
|
|
32
|
+
console.log(new Date(), 'Reading');
|
|
33
|
+
}, 5e3 * nWrites + 2 * Node.refreshTimeIntervalMS);
|
|
34
|
+
afterAll(function () {
|
|
35
|
+
if (shutdown) {
|
|
36
|
+
exec('pkill kdht-bot');
|
|
37
|
+
Node.delay(2e3);
|
|
38
|
+
exec('pkill kdht-portal-server');
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
for (let index = 0; index < nWrites; index++) {
|
|
42
|
+
it(`reads ${index}.`, async function () {
|
|
43
|
+
const read = await contact.node.locateValue(index);
|
|
44
|
+
console.log('read', read);
|
|
45
|
+
expect(read).toBe(index);
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
});
|
package/save/webrtc.js
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import { v4 as uuidv4 } from 'uuid';
|
|
2
|
+
import { Node } from '../dht/node.js';
|
|
3
|
+
import { Helper } from '../dht/helper.js';
|
|
4
|
+
import { Contact } from './contact.js';
|
|
5
|
+
import { WebRTC } from '@yz-social/webrtc';
|
|
6
|
+
const { BigInt } = globalThis; // For linters.
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
export class WebContact extends Contact {
|
|
10
|
+
// Can this set all be done more simply?
|
|
11
|
+
get name() { return this.node.name; }
|
|
12
|
+
get key() { return this.node.key; }
|
|
13
|
+
get isServerNode() { return this.node.isServerNode; }
|
|
14
|
+
get isRunning() { return this.node.isRunning; }
|
|
15
|
+
|
|
16
|
+
checkResponse(response) {
|
|
17
|
+
if (!response.ok) throw new Error(`${this.sname} fetch ${response.url} failed ${response.status}: ${response.statusText}.`);
|
|
18
|
+
}
|
|
19
|
+
async fetchSignals(url, signalsToSend) {
|
|
20
|
+
const response = await fetch(url, {
|
|
21
|
+
method: 'POST',
|
|
22
|
+
headers: { 'Content-Type': 'application/json' },
|
|
23
|
+
body: JSON.stringify(signalsToSend)
|
|
24
|
+
});
|
|
25
|
+
this.checkResponse(response);
|
|
26
|
+
return await response.json();
|
|
27
|
+
}
|
|
28
|
+
async fetchBootstrap(label = 'random') {
|
|
29
|
+
const url = `http://localhost:3000/kdht/name/${label}`;
|
|
30
|
+
const response = await fetch(url);
|
|
31
|
+
this.checkResponse(response);
|
|
32
|
+
return await response.json();
|
|
33
|
+
}
|
|
34
|
+
get webrtcLabel() {
|
|
35
|
+
return `@${this.host.contact.sname} ==> ${this.sname}`;
|
|
36
|
+
}
|
|
37
|
+
// Within a process, there is only one WebContact.serializer, so each process can only complete one connection at a time.
|
|
38
|
+
// (It can maintain Node.maxTransports open connections. The serialization is a limit on the connection/signal process.)
|
|
39
|
+
static serializer = Promise.resolve();
|
|
40
|
+
channelName = 'kdht';
|
|
41
|
+
|
|
42
|
+
ensureWebRTC(initialSignals) { // If not already configured, sets up contact to have properties:
|
|
43
|
+
// - connection - a promise for an open webrtc data channel:
|
|
44
|
+
// send(string) puts data on the channel
|
|
45
|
+
// incomming messages are dispatched to receiveWebRTC(string)
|
|
46
|
+
// - closed - resolves when webrtc closes.
|
|
47
|
+
// - webrtc - an instance of WebRTC (which may be used for webrtc.respond()
|
|
48
|
+
// initialSignals should be a list, or null for the creator of the offer.
|
|
49
|
+
if (this.webrtc) return this.webrtc;
|
|
50
|
+
|
|
51
|
+
const webrtc = this.webrtc = new WebRTC({label: this.webrtcLabel});
|
|
52
|
+
let {promise, resolve} = Promise.withResolvers();
|
|
53
|
+
this.closed = promise;
|
|
54
|
+
const dataChannelPromise = webrtc.ensureDataChannel(this.channelName, {}, initialSignals);
|
|
55
|
+
dataChannelPromise.then(dataChannel => {
|
|
56
|
+
webrtc.reportConnection(true);
|
|
57
|
+
dataChannel.addEventListener('close', () => {
|
|
58
|
+
console.log(webrtc.label, 'connection closed');
|
|
59
|
+
this.connection = this.webrtc = null;
|
|
60
|
+
resolve(null);
|
|
61
|
+
});
|
|
62
|
+
dataChannel.addEventListener('message', event => this.receiveWebRTC(event.data));
|
|
63
|
+
});
|
|
64
|
+
return dataChannelPromise;
|
|
65
|
+
}
|
|
66
|
+
connect() { // Connect from host to node, promising a possibly cloned contact that has been noted.
|
|
67
|
+
// Creates a WebRTC instance and uses it's connectVia to to signal.
|
|
68
|
+
const contact = this.host.noteContactForTransport(this);
|
|
69
|
+
//this.host.log('ensuring connection for', contact.sname, contact.connection ? 'exists' : 'must create');
|
|
70
|
+
if (contact.connection) return contact.connection;
|
|
71
|
+
contact.initiating = true;
|
|
72
|
+
//console.log('setting client contact connection promise');
|
|
73
|
+
return contact.connection = contact.constructor.serializer = contact.constructor.serializer.then(async () => { // TODO: do we need this serialization?
|
|
74
|
+
const { host, node, isServerNode, bootstrapHost } = contact;
|
|
75
|
+
// Anyone can connect to a server node using the server's connect endpoint.
|
|
76
|
+
// Anyone in the DHT can connect to another DHT node through a sponsor.
|
|
77
|
+
const dataChannelPromise = contact.ensureWebRTC(null);
|
|
78
|
+
await this.webrtc.signalsReady;
|
|
79
|
+
let sponsor = contact.findSponsor(candidate => candidate.connection) ||
|
|
80
|
+
contact.findSponsor(candidate => candidate.isServerNode);
|
|
81
|
+
try {
|
|
82
|
+
if (sponsor) {
|
|
83
|
+
await contact.webrtc.connectVia(async signals => sponsor.sendRPC('forwardSignals', contact.key, host.contact.sname, ...signals));
|
|
84
|
+
} else if (bootstrapHost || !contact.isServerNode) {
|
|
85
|
+
const url = `${bootstrapHost || 'http://localhost:3000/kdht'}/join/${this.host.contact.sname}/${this.sname}`;
|
|
86
|
+
await contact.webrtc.connectVia(signals => this.fetchSignals(url, signals));
|
|
87
|
+
} else {
|
|
88
|
+
console.error(`Cannot reach unsponsored contact ${contact.sname}.`);
|
|
89
|
+
await contact.host.removeContact(contact);
|
|
90
|
+
return contact.connection = contact.webrtc = null;
|
|
91
|
+
}
|
|
92
|
+
} catch (error) {
|
|
93
|
+
console.error(`${contact.webrtc.label} failed to connect through ${sponsor ? `sponsor ${sponsor.sname}` : `bootstrap ${bootstrapHost || contact.sname}`}`);
|
|
94
|
+
console.error(error);
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
return await dataChannelPromise;
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
async signals(senderSname, ...signals) { // Accept directed WebRTC signals from a sender sname, creating if necessary the new contact on
|
|
101
|
+
// host to receive them, and promising a response.
|
|
102
|
+
let contact = await this.ensureRemoteContact(senderSname);
|
|
103
|
+
if (contact.initiating) return []; // TODO: is this the right response?
|
|
104
|
+
if (contact.webrtc) return await contact.webrtc.respond(signals); // TODO: move this and its counterpart on connect to ensureWebRTC?
|
|
105
|
+
this.host.noteContactForTransport(contact);
|
|
106
|
+
contact.connection = contact.ensureWebRTC(signals);
|
|
107
|
+
return await contact.webrtc.respond([]);
|
|
108
|
+
}
|
|
109
|
+
async send(message) { // Promise to send through previously opened connection promise.
|
|
110
|
+
return this.connection // Connection could be reset to null
|
|
111
|
+
?.then(channel => channel?.send(JSON.stringify(message)))
|
|
112
|
+
.catch(error => console.error(this.host.contact.sname, error, error.stack));
|
|
113
|
+
}
|
|
114
|
+
getName(sname) { // Answer name from sname.
|
|
115
|
+
if (sname.startsWith(this.constructor.serverSignifier)) return sname.slice(1);
|
|
116
|
+
return sname;
|
|
117
|
+
}
|
|
118
|
+
async ensureRemoteContact(sname, sponsor = null) {
|
|
119
|
+
let contact;
|
|
120
|
+
if (sname === this.host.contact.sname) {
|
|
121
|
+
contact = this.host.contact; // ok, not remote, but contacts can send back us in a list of closest nodes.
|
|
122
|
+
}
|
|
123
|
+
const name = this.getName(sname);
|
|
124
|
+
if (!contact) {
|
|
125
|
+
// Not the final answer. Just an optimization to avoid hashing name.
|
|
126
|
+
contact = this.host.existingContact(name);
|
|
127
|
+
}
|
|
128
|
+
if (!contact) {
|
|
129
|
+
const isServerNode = name !== sname;
|
|
130
|
+
contact = await this.constructor.create({name, isServerNode}, this.host); // checks for existence AFTER creating Node.
|
|
131
|
+
}
|
|
132
|
+
if (sponsor instanceof Contact) contact.noteSponsor(sponsor);
|
|
133
|
+
else if (typeof(sponsor) === 'string') contact.bootstrapHost = sponsor;
|
|
134
|
+
return contact;
|
|
135
|
+
}
|
|
136
|
+
receiveRPC(method, sender, key, ...rest) { // Receive an RPC from sender, dispatch, and return that value, which will be awaited and sent back to sender.
|
|
137
|
+
if (method === 'forwardSignals') { // Can't be handled by Node, because 'forwardSignals' is specific to WebContact.
|
|
138
|
+
const [sendingSname, ...signals] = rest;
|
|
139
|
+
//console.log(this.host.contact.sname, this.host.key, 'received forwardingSignals from', sender.sname, 'on contact', this.sname, this.key, 'concerning', key);
|
|
140
|
+
if (key === this.host.key) { // for us!
|
|
141
|
+
//console.log(this.host.contact.sname, 'processing signals from', sendingSname, signals.map(s => s[0]));
|
|
142
|
+
return this.signals(...rest);
|
|
143
|
+
} else { // Forard to the target.
|
|
144
|
+
const target = this.host.findContactByKey(key);
|
|
145
|
+
if (!target) {
|
|
146
|
+
//console.log(this.host.contact.same, 'does not have contact to', sendingSname);
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
//if (this.debugCounter++ > 3) process.exit(1);
|
|
150
|
+
//console.log(this.host.contact.sname, 'forwarding signals from', sendingSname, signals.map(s => s[0]), 'to target', target.sname, key);
|
|
151
|
+
return target.sendRPC('forwardSignals', key, ...rest);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return super.receiveRPC(method, sender, key, ...rest);
|
|
155
|
+
}
|
|
156
|
+
//debugCounter = 0;
|
|
157
|
+
inFlight = new Map();
|
|
158
|
+
async transmitRPC(method, key, ...rest) { // Must return a promise.
|
|
159
|
+
// this.host.log('transmit to', this.sname, this.connection ? 'with connection' : 'WITHOUT connection');
|
|
160
|
+
if (!await this.connect()) return null;
|
|
161
|
+
const sender = this.host.contact.sname;
|
|
162
|
+
// uuid so that the two sides don't send a request with the same id to each other.
|
|
163
|
+
// Alternatively, we could concatenate a counter to our host.name.
|
|
164
|
+
const messageTag = uuidv4();
|
|
165
|
+
const responsePromise = new Promise(resolve => this.inFlight.set(messageTag, resolve));
|
|
166
|
+
// this.host.log('sending to', this.sname);
|
|
167
|
+
//console.log(this.host.contact.sname, 'sending', method, key, rest.map(s => Array.isArray(s) ? s[0] : s), 'to', this.sname);
|
|
168
|
+
this.send([messageTag, method, sender, key.toString(), ...rest]);
|
|
169
|
+
const response = await Promise.race([responsePromise, this.closed]);
|
|
170
|
+
// this.host.log('got response from', this.sname);
|
|
171
|
+
return response;
|
|
172
|
+
}
|
|
173
|
+
async receiveWebRTC(dataString) { // Handle receipt of a WebRTC data channel message that was sent to this contact.
|
|
174
|
+
// The message could the start of an RPC sent from the peer, or it could be a response to an RPC that we made.
|
|
175
|
+
// As we do the latter, we generate and note (in transmitRPC) a message tag included in the message.
|
|
176
|
+
// If we find that in our inFlight tags, then the message is a response.
|
|
177
|
+
const [messageTag, ...data] = JSON.parse(dataString);
|
|
178
|
+
const responder = this.inFlight.get(messageTag);
|
|
179
|
+
if (responder) { // A response to something we sent and are waiting for.
|
|
180
|
+
let [result] = data;
|
|
181
|
+
this.inFlight.delete(messageTag);
|
|
182
|
+
//this.host.log('received response', result);
|
|
183
|
+
if (Array.isArray(result)) {
|
|
184
|
+
if (!result.length) responder(result);
|
|
185
|
+
const first = result[0];
|
|
186
|
+
const isSignal = Array.isArray(first) && ['offer', 'answer', 'icecandidate'].includes(first[0]);
|
|
187
|
+
//console.log(this.host.contact.sname, 'got rpc response on contact', this.sname, isSignal, result.map(s => Array.isArray(s) ? s[0] : s));
|
|
188
|
+
if (isSignal) {
|
|
189
|
+
responder(result); // This could be the sponsor or the original sender. Either way, it will know what to do.
|
|
190
|
+
} else {
|
|
191
|
+
responder(await Promise.all(result.map(async ([sname, distance]) =>
|
|
192
|
+
new Helper(await this.ensureRemoteContact(sname, this), BigInt(distance)))));
|
|
193
|
+
}
|
|
194
|
+
} else {
|
|
195
|
+
responder(result);
|
|
196
|
+
}
|
|
197
|
+
} else { // An incoming request.
|
|
198
|
+
const [method, senderLabel, key, ...rest] = data;
|
|
199
|
+
const sender = await this.ensureRemoteContact(senderLabel);
|
|
200
|
+
//this.host.log('dispatched', method, 'from', sender.sname);
|
|
201
|
+
let response = await this.receiveRPC(method, sender, BigInt(key), ...rest);
|
|
202
|
+
//if (method === 'forwardSignals') console.log(this.host.contact.sname, 'incoming signals through', this.sname, 'originating from', rest[0]);
|
|
203
|
+
if ((method !== 'forwardSignals') && this.host.constructor.isContactsResult(response)) {
|
|
204
|
+
response = response.map(helper => [helper.contact.sname, helper.distance.toString()]);
|
|
205
|
+
}
|
|
206
|
+
this.send([messageTag, response]);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
disconnectTransport() {
|
|
210
|
+
this.webrtc?.close();
|
|
211
|
+
}
|
|
212
|
+
}
|