cryptpad-server 0.0.1

package/index.js

@@ -0,0 +1,169 @@
+// SPDX-FileCopyrightText: 2025 XWiki CryptPad Team <contact@cryptpad.org> and contributors
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+const { fork } = require('child_process');
+const Crypto = require('crypto');
+const Path = require('node:path');
+
+const showHelp = () => {
+    console.log(`Usage ${process.argv[1]}:`);
+    console.log("\t--help, -h\tDisplay this help");
+    console.log("\t--type,-t\tSet the core type (if unset, starts every core)");
+    console.log("\t--index,-i\tSet the core node index (default: 0)");
+    process.exit(1);
+};
+
+const args = process.argv.slice(2);
+const cliArgs = {};
+let prev;
+args.some(arg => {
+    if (arg === '--type' || arg === '-t') {
+        if (prev) { throw new Error('INVALID ARGS');}
+        prev = 'type';
+        return;
+    }
+    if (arg === '--index' || arg === '-i') {
+        if (prev) { throw new Error('INVALID ARGS');}
+        prev = 'index';
+        return;
+    }
+    if (arg === '--server' || arg === '-s') {
+        if (prev) { throw new Error('INVALID ARGS');}
+        prev = 'server';
+        return;
+    }
+    if (arg === '--help' || arg === '-h') {
+        showHelp();
+        return true;
+    }
+    if (/^--?/.test(arg)) {
+        prev = '';
+        return;
+    }
+
+    if (!prev) { return; }
+    cliArgs[prev] = arg;
+    prev = '';
+});
+
+const start = (serverConfig, infraConfig) => {
+    const Log = {
+        debug: console.debug,
+        error: console.error,
+        info: console.log,
+        verbose: console.info,
+        warn: console.warn
+    };
+
+    let serverId;
+    const startNode = (type, index, forking, cb) => {
+        if (typeof (cb) !== 'function') { cb = () => { }; };
+
+        const nodeFile = './build/' + type + '.js';
+        const path = Path.join(__dirname, nodeFile);
+        const initConfig = {
+            myId: `${type}:${index}`,
+            index,
+            config: serverConfig,
+            infra: infraConfig
+        };
+
+        //Log.info(`Starting: ${initConfig.myId}`);
+        if (forking) {
+            let nodeProcess = fork(path);
+            nodeProcess.send(initConfig);
+            nodeProcess.on('message', (message) => {
+                if (message.msg === 'READY') {
+                    Log.info(`Started: ${type}:${message.index}`);
+                    if (message.dev) {
+                        Log.info('DEV mode enabled');
+                    }
+                    cb();
+                }
+            });
+            // FIXME
+            nodeProcess.on('error', (err) => {
+                Log.error('Child process stopped due to error.');
+                Log.error(err);
+                process.exit(1);
+            });
+            nodeProcess.on('exit', (err) => {
+                Log.error('Child process stopped due to error.');
+                Log.error(err);
+                process.exit(1);
+            });
+        } else {
+            require(path).start(initConfig);
+        }
+    };
+
+    const coresReady = () => {
+        const promises = [];
+        infraConfig?.front?.forEach((data, index) => {
+            promises.push(new Promise(resolve => {
+                if (serverId && data.serverId !== serverId) { return resolve(); }
+                startNode('front', index, true, resolve);
+            }));
+        });
+        infraConfig?.storage?.forEach((data, index) => {
+            promises.push(new Promise(resolve => {
+                if (serverId && data.serverId !== serverId) { return resolve(); }
+                startNode('storage', index, true, resolve);
+            }));
+        });
+        promises.push(new Promise(resolve => {
+            if (serverId && infraConfig?.public?.httpServerId !== serverId) { return resolve(); }
+            startNode('http', 0, true, resolve);
+        }));
+        Promise.all(promises).then(() => {
+            Log.info('CryptPad server ready');
+        });
+    };
+
+    const startCores = () => {
+        if (!serverConfig?.private?.nodes_key) {
+            if (!serverConfig?.private) {
+                serverConfig.private = { };
+            }
+            serverConfig.private.nodes_key = Crypto.randomBytes(32).toString('base64');
+        }
+        const corePromises = infraConfig?.core.map((data, index) => new Promise((resolve, reject) => {
+            // hosted on another machine?
+            if (serverId && data.serverId !== serverId) { return resolve(); }
+            startNode('core', index, true, (err) => {
+                if (err) {
+                    Log.error('START_CORE_ERROR', err);
+                    return reject(err);
+                }
+                return resolve();
+            });
+        }));
+
+        Promise.all(corePromises)
+            .then(() => { coresReady(); })
+            .catch((e) => { return Log.error('START_CORE_ERROR', e); });
+    };
+
+
+    // Start process
+    if (cliArgs.type) {
+        const type = cliArgs.type;
+        const index = Number(cliArgs.index || 0);
+        if (!serverConfig?.private?.nodes_key) {
+            throw Error('E_MISSINGKEY');
+        }
+        startNode(type, index, false, (err) => {
+            if (err) { return Log.error('START_NODE_ERROR', err); }
+        });
+    } else {
+        serverId = cliArgs.server;
+        startCores();
+    }
+};
+
+if (require.main === module) {
+    const { config, infra } = require('./common/load-config');
+    start(config, infra);
+} else {
+    module.exports = { start };
+}

package/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "cryptpad-server",
+  "description": "CryptPad Server",
+  "version": "0.0.1",
+  "license": "AGPL-3.0+",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/cryptpad/scalable-server.git"
+  },
+  "funding": {
+    "type": "opencollective",
+    "url": "https://opencollective.com/cryptpad"
+  },
+  "main": "index.js",
+  "files": [
+    "build"
+  ],
+  "dependencies": {
+    "body-parser": "^2.2.0",
+    "connect-gzip-static": "^4.2.1",
+    "cookie-parser": "^1.4.7",
+    "express": "~4.22.1",
+    "fs-extra": "^7.0.0",
+    "http-proxy-middleware": "^3.0.3",
+    "nthen": "0.1.8",
+    "otpauth": "^9.4.1",
+    "pull-stream": "^3.6.1",
+    "saferphore": "0.0.1",
+    "sodium-native": "^4.1.1",
+    "stream-to-pull-stream": "^1.7.2",
+    "tweetnacl": "^1.0.3",
+    "tweetnacl-util": "^0.15.1",
+    "ws": "^8.18.0"
+  },
+  "devDependencies": {
+    "@rollup/plugin-commonjs": "^28.0.0",
+    "@rollup/plugin-json": "^6.1.0",
+    "@rollup/plugin-node-resolve": "^15.3.0",
+    "@rollup/plugin-terser": "^0.4.4",
+    "@rollup/plugin-typescript": "^12.1.0",
+    "@types/node": "^24.1.0",
+    "chainpad-crypto": "^0.3.0",
+    "chainpad-netflux": "^1.3.0",
+    "eslint": "^10.0.2",
+    "globals": "^16.3.0",
+    "netflux-websocket": "^1.3.2",
+    "rollup": "^4.59.0",
+    "rollup-plugin-node-externals": "^8.1.2",
+    "tslib": "^2.7.0",
+    "typescript": "^5.6.2"
+  },
+  "overrides": {
+    "http-proxy": "npm:http-proxy-3@^1.23.2",
+    "serialize-javascript": "^7.0.4"
+  },
+  "scripts": {
+    "lint": "eslint .",
+    "build": "rollup -c",
+    "build:front": "TYPE=front rollup -c",
+    "build:core": "TYPE=core rollup -c",
+    "build:storage": "TYPE=storage rollup -c",
+    "build:http": "TYPE=http rollup -c",
+    "tests": "node --test tests/",
+    "test": "node tests/runAll.js",
+    "start": "node index.js",
+    "dev": "DEV=1 node index.js"
+  }
+}

package/readme.md

@@ -0,0 +1,104 @@
+<!-- SPDX-FileCopyrightText: 2024 XWiki CryptPad Team <contact@cryptpad.org> and contributors
+
+SPDX-License-Identifier: AGPL-3.0-or-later
+-->
+# Prototype: Scalable Server for CryptPad
+
+This repository contains a distributed server prototype for CryptPad. This is
+still in progress exploratory work.
+
+## Architecture
+
+### Taxonomy
+
+The new server architecture is composed of three types of nodes:
+
+- **Core:** these nodes take care of most of the computations and internal
+communication handling. They are connected to the other type of servers.
+- **Front:** these nodes catch the different queries from the
+outside and forward them to the core nodes for processing.
+- **Storage:** these nodes are responsible for accessing and serving a fraction
+of the (encrypted) data for CryptPad. They are also doing light computation if
+they can be done in place without too much pressure on them.
+
+### Topology
+
+The core nodes are connected to both ws and storage nodes, but the latter two
+cannot communicate directly.
+
+## Configuration
+
+The configuration can be done using the `Config` argument, which will store the
+graph topology of the nodes.
+
+Using websockets for communication, this variable looks like this:
+
+```javascript
+let Config = {
+    infra: {
+        ws: [{
+            host: 'localhost',
+            port: 3010
+        }],
+        core: [{
+            host: 'localhost',
+            port: 3011
+        }, {
+                host: 'localhost',
+                port: 3012
+            }],
+        storage: [{
+            host: 'localhost',
+            port: 3014
+        }]
+    }
+};
+```
+
+The above configuration describes a network comprised of 4 nodes, having one
+front node on port `3010`, 2 core servers on port `3011` and `3012`
+and a storage node accessible via port `3014`.
+
+In addition, launching a server requires setting the field `myId` with your
+identifier, which will be of the form `type:id`. For instance, in the first core
+node (the node listening on port `3011`), it would be `Config.myId = 'core:0'`.
+
+## Usage
+
+Before first use, you may want to install the dependencies with:
+```bash
+npm install
+```
+
+To spawn the server topology described in `ws-config.js`, run the start script:
+```bash
+npm run start
+```
+
+Alternatively, you can the new servers manually. You first need to start the
+`core` nodes with the following command.
+
+```bash
+node core/index.js
+```
+
+Then you can start a `ws` and `storage` nodes in any order:
+```bash
+node storage/index.js
+node front/index.js
+```
+
+## Tests
+
+The directory [`tests`](tests/) contains some unit and integration tests scripts
+and files.
+
+- [`interface.test.js`](tests/interface.test.js): test the communication
+interface by implementing a simple ping-pong protocol that computes the time it
+takes to go back and forth in the network.
+
+The aforementioned tests can be run with the following command (make sure to
+have installed the dependencies with `npm install` beforehand):
+```bash
+npm run tests
+```