cryptpad-server 0.0.3 → 0.0.4

package/build/front.js

@@ -1358,6 +1358,9 @@ function requireWsConnector() {
                 cb(void 0, ctx.self);
             };
             const port = config.wsPort || config.port;
+            httpServer.on("error", e => {
+                cb(e);
+            });
             httpServer.listen(port, config.host, onServerReady);
         },
         initClient: (ctx, config, id, onConnected, cb) => {
@@ -2772,6 +2775,14 @@ function requireFile() {
             writeLinkedFile(env, path, content, cb);
         });
     };
+    const hasLinkedDocuments = (env, channelId) => {
+        const path = mkLinkedPath(env, channelId);
+        try {
+            return Fs.existsSync(path);
+        } catch (e) {
+            return false;
+        }
+    };
     const getLinkedDocuments = (env, channelId, cb) => {
         const path = mkLinkedPath(env, channelId);
         readLinkedFile(env, path, cb);
@@ -3844,6 +3855,7 @@ function requireFile() {
                         writeMetadata(env, channelName, data, Util.both(cb, next));
                     });
                 },
+                hasLinkedDocuments: channelId => hasLinkedDocuments(env, channelId),
                 getLinkedDocuments: (channelId, cb) => {
                     if (!isValidChannelId(channelId)) {
                         return cb(new Error("EINVAL"));
@@ -4049,7 +4061,7 @@ function requireLogger() {
     return logger;
 }
 
-var version = "0.0.3";
+var version = "0.0.4";
 
 var require$$2 = {
     version
@@ -5795,7 +5807,8 @@ function requireFront() {
                 process.send({
                     type: "front",
                     index,
-                    msg: "READY"
+                    msg: "READY",
+                    pid: process.pid
                 });
             } else {
                 Env.Log.info("SINGLE_FRONT_STARTED", Env.myId);

package/build/http.js

@@ -2200,6 +2200,14 @@ function requireFile() {
             writeLinkedFile(env, path, content, cb);
         });
     };
+    const hasLinkedDocuments = (env, channelId) => {
+        const path = mkLinkedPath(env, channelId);
+        try {
+            return Fs.existsSync(path);
+        } catch (e) {
+            return false;
+        }
+    };
     const getLinkedDocuments = (env, channelId, cb) => {
         const path = mkLinkedPath(env, channelId);
         readLinkedFile(env, path, cb);
@@ -3272,6 +3280,7 @@ function requireFile() {
                         writeMetadata(env, channelName, data, Util.both(cb, next));
                     });
                 },
+                hasLinkedDocuments: channelId => hasLinkedDocuments(env, channelId),
                 getLinkedDocuments: (channelId, cb) => {
                     if (!isValidChannelId(channelId)) {
                         return cb(new Error("EINVAL"));
@@ -3477,7 +3486,7 @@ function requireLogger() {
     return logger;
 }
 
-var version = "0.0.3";
+var version = "0.0.4";
 
 var require$$2 = {
     version
@@ -5172,6 +5181,9 @@ function requireWsConnector() {
                 cb(void 0, ctx.self);
             };
             const port = config.wsPort || config.port;
+            httpServer.on("error", e => {
+                cb(e);
+            });
             httpServer.listen(port, config.host, onServerReady);
         },
         initClient: (ctx, config, id, onConnected, cb) => {
@@ -5313,7 +5325,8 @@ function requireHttpServer() {
                 type: "http",
                 index: 0,
                 dev: Env.DEV_MODE,
-                msg: "READY"
+                msg: "READY",
+                pid: process.pid
             });
         });
     };

package/build/http.worker.js

@@ -2210,6 +2210,14 @@ function requireFile() {
             writeLinkedFile(env, path, content, cb);
         });
     };
+    const hasLinkedDocuments = (env, channelId) => {
+        const path = mkLinkedPath(env, channelId);
+        try {
+            return Fs.existsSync(path);
+        } catch (e) {
+            return false;
+        }
+    };
     const getLinkedDocuments = (env, channelId, cb) => {
         const path = mkLinkedPath(env, channelId);
         readLinkedFile(env, path, cb);
@@ -3282,6 +3290,7 @@ function requireFile() {
                         writeMetadata(env, channelName, data, Util.both(cb, next));
                     });
                 },
+                hasLinkedDocuments: channelId => hasLinkedDocuments(env, channelId),
                 getLinkedDocuments: (channelId, cb) => {
                     if (!isValidChannelId(channelId)) {
                         return cb(new Error("EINVAL"));
@@ -3529,7 +3538,7 @@ function requireDefaults() {
     return defaults.exports;
 }
 
-var version = "0.0.3";
+var version = "0.0.4";
 
 var require$$2 = {
     version
@@ -4611,6 +4620,10 @@ function requireWorker() {
                 txid: obj.txid,
                 pid: obj.pid,
                 value
+            }, e => {
+                if (e) {
+                    console.error("E_RESPONSE", Util.serializeError(e));
+                }
             });
         };
         if (!ready) {

package/build/storage.cluster.js

@@ -2879,7 +2879,7 @@ function requireMfa() {
     return mfa.exports;
 }
 
-var version = "0.0.3";
+var version = "0.0.4";
 
 var require$$2 = {
     version
@@ -4473,6 +4473,14 @@ function requireFile() {
             writeLinkedFile(env, path, content, cb);
         });
     };
+    const hasLinkedDocuments = (env, channelId) => {
+        const path = mkLinkedPath(env, channelId);
+        try {
+            return Fs.existsSync(path);
+        } catch (e) {
+            return false;
+        }
+    };
     const getLinkedDocuments = (env, channelId, cb) => {
         const path = mkLinkedPath(env, channelId);
         readLinkedFile(env, path, cb);
@@ -5545,6 +5553,7 @@ function requireFile() {
                         writeMetadata(env, channelName, data, Util.both(cb, next));
                     });
                 },
+                hasLinkedDocuments: channelId => hasLinkedDocuments(env, channelId),
                 getLinkedDocuments: (channelId, cb) => {
                     if (!isValidChannelId(channelId)) {
                         return cb(new Error("EINVAL"));

package/build/storage.js

@@ -2202,6 +2202,14 @@ function requireFile() {
             writeLinkedFile(env, path, content, cb);
         });
     };
+    const hasLinkedDocuments = (env, channelId) => {
+        const path = mkLinkedPath(env, channelId);
+        try {
+            return Fs.existsSync(path);
+        } catch (e) {
+            return false;
+        }
+    };
     const getLinkedDocuments = (env, channelId, cb) => {
         const path = mkLinkedPath(env, channelId);
         readLinkedFile(env, path, cb);
@@ -3274,6 +3282,7 @@ function requireFile() {
                         writeMetadata(env, channelName, data, Util.both(cb, next));
                     });
                 },
+                hasLinkedDocuments: channelId => hasLinkedDocuments(env, channelId),
                 getLinkedDocuments: (channelId, cb) => {
                     if (!isValidChannelId(channelId)) {
                         return cb(new Error("EINVAL"));
@@ -5025,10 +5034,17 @@ function requireChannelManager() {
                     }
                 }));
             }).nThen(function(w) {
+                if (!store.hasLinkedDocuments(channel)) {
+                    return;
+                }
                 Linked.trimHistory(Env, {
                     channel
                 }, w());
             }).nThen(function() {
+                Env.Log.info("HK_TRIM_HISTORY_START", {
+                    unsafeKey,
+                    channel
+                });
                 store.trimChannel(channel, hash, err => {
                     Env.Log.info("HK_TRIM_HISTORY", {
                         unsafeKey,
@@ -6695,7 +6711,7 @@ function requireMfaManager() {
     return mfaManager;
 }
 
-var version = "0.0.3";
+var version = "0.0.4";
 
 var require$$2 = {
     version
@@ -7924,6 +7940,9 @@ function requireWsConnector() {
                 cb(void 0, ctx.self);
             };
             const port = config.wsPort || config.port;
+            httpServer.on("error", e => {
+                cb(e);
+            });
             httpServer.listen(port, config.host, onServerReady);
         },
         initClient: (ctx, config, id, onConnected, cb) => {
@@ -12113,7 +12132,8 @@ function requireStorage() {
                 process.send({
                     type: "storage",
                     index,
-                    msg: "READY"
+                    msg: "READY",
+                    pid: process.pid
                 });
             } else {
                 console.log(myId, "started");

package/build/storage.worker.js

@@ -2204,6 +2204,14 @@ function requireFile() {
             writeLinkedFile(env, path, content, cb);
         });
     };
+    const hasLinkedDocuments = (env, channelId) => {
+        const path = mkLinkedPath(env, channelId);
+        try {
+            return Fs.existsSync(path);
+        } catch (e) {
+            return false;
+        }
+    };
     const getLinkedDocuments = (env, channelId, cb) => {
         const path = mkLinkedPath(env, channelId);
         readLinkedFile(env, path, cb);
@@ -3276,6 +3284,7 @@ function requireFile() {
                         writeMetadata(env, channelName, data, Util.both(cb, next));
                     });
                 },
+                hasLinkedDocuments: channelId => hasLinkedDocuments(env, channelId),
                 getLinkedDocuments: (channelId, cb) => {
                     if (!isValidChannelId(channelId)) {
                         return cb(new Error("EINVAL"));
@@ -4731,7 +4740,7 @@ function requireTasks() {
     return tasks.exports;
 }
 
-var version = "0.0.3";
+var version = "0.0.4";
 
 var require$$2 = {
     version
@@ -5589,10 +5598,11 @@ function requirePinManager() {
     hasRequiredPinManager = 1;
     (function(module) {
         const Pins = module.exports;
-        __require$2();
-        __require$1();
-        requireCommonUtil$1();
-        requireFile();
+        const Fs = __require$2();
+        const Path = __require$1();
+        const Util = requireCommonUtil$1();
+        const Semaphore = require$$4;
+        const nThen = require$$3;
         const createLineHandler = Pins.createLineHandler = (ref, errorHandler) => {
             let fileName;
             if (typeof errorHandler === "string") {
@@ -5712,8 +5722,72 @@ function requirePinManager() {
             const ref = processPinFile(pinFile, fileName);
             return Object.keys(ref.pins);
         };
-        Pins.load = (cb, config) => {
-            throw new Error("EVICTION NOT IMPLEMENTED");
+        Pins.load = (Env, cb, config) => {
+            const sema = Semaphore.create(config.workers || 5);
+            let dirList;
+            const fileList = [];
+            const pinned = {};
+            var done = Util.once(cb);
+            var handler = config.handler;
+            let store = Env.pinStore;
+            let pinPath = Env.paths.pin;
+            nThen(waitFor => {
+                Fs.readdir(pinPath, waitFor((err, list) => {
+                    if (err) {
+                        if (err.code === "ENOENT") {
+                            dirList = [];
+                            return;
+                        }
+                        waitFor.abort();
+                        return void done(err);
+                    }
+                    dirList = list;
+                }));
+            }).nThen(waitFor => {
+                dirList.forEach(f => {
+                    sema.take(returnAfter => {
+                        Fs.readdir(Path.join(pinPath, f), waitFor(returnAfter((err, list2) => {
+                            if (err) {
+                                waitFor.abort();
+                                return void done(err);
+                            }
+                            list2.forEach(ff => {
+                                if (config && config.exclude && config.exclude.indexOf(ff) > -1) {
+                                    return;
+                                }
+                                fileList.push(ff.replace(/(\.ndjson)$/, ""));
+                            });
+                        })));
+                    });
+                });
+            }).nThen(waitFor => {
+                fileList.forEach(id => {
+                    sema.take(returnAfter => {
+                        var next = waitFor(returnAfter());
+                        var ref = {};
+                        var h = createLineHandler(ref, id);
+                        store.readMessagesBin(id, 0, (msgObj, next) => {
+                            h(msgObj.buff.toString("utf8"));
+                            next();
+                        }, err => {
+                            if (err) {
+                                waitFor.abort();
+                                return void done(err);
+                            }
+                            if (handler) {
+                                return void handler(ref, id, next);
+                            }
+                            const hashes = Object.keys(ref.pins);
+                            hashes.forEach(x => {
+                                (pinned[x] = pinned[x] || {})[id] = 1;
+                            });
+                            next();
+                        });
+                    });
+                });
+            }).nThen(() => {
+                done(void 0, pinned);
+            });
         };
     })(pinManager);
     return pinManager.exports;
@@ -6519,6 +6593,10 @@ function requireWorker() {
                 txid: obj.txid,
                 pid: obj.pid,
                 value
+            }, e => {
+                if (e) {
+                    console.error("E_RESPONSE", Util.serializeError(e));
+                }
             });
         };
         if (!ready) {
@@ -6535,6 +6613,9 @@ function requireWorker() {
         }
         command(data, cb);
     });
+    process.on("disconnect", () => {
+        process.exit(0);
+    });
     process.on("uncaughtException", function(err) {
         console.error("[%s] UNCAUGHT EXCEPTION IN DB WORKER", new Date);
         console.error(err);

package/index.js

@@ -47,6 +47,14 @@ args.some(arg => {
     prev = '';
 });
 
+// Clean quit: kill child processes
+const childPids = [];
+
+const cleanSubprocesses = () => childPids.forEach((pid) => process.kill(pid));
+
+process.on('SIGTERM', cleanSubprocesses);
+process.on('exit', cleanSubprocesses);
+
 const start = (serverConfig, infraConfig) => {
     const Log = {
         debug: console.debug,
@@ -57,9 +65,7 @@ const start = (serverConfig, infraConfig) => {
     };
 
     let serverId;
-    const startNode = (type, index, forking, cb) => {
-        if (typeof (cb) !== 'function') { cb = () => { }; };
-
+    const startNode = (type, index, forking) => new Promise((resolve, reject) => {
         const nodeFile = './build/' + type + '.js';
         const path = Path.join(__dirname, nodeFile);
         const initConfig = {
@@ -72,51 +78,49 @@ const start = (serverConfig, infraConfig) => {
         //Log.info(`Starting: ${initConfig.myId}`);
         if (forking) {
             let nodeProcess = fork(path);
+            childPids.push(nodeProcess.pid);
             nodeProcess.send(initConfig);
             nodeProcess.on('message', (message) => {
                 if (message.msg === 'READY') {
-                    Log.info(`Started: ${type}:${message.index}`);
+                    // Log.info(`Started: ${type}:${message.index}`);
                     if (message.dev) {
                         Log.info('DEV mode enabled');
                     }
-                    cb();
+                    resolve();
                 }
             });
-            // FIXME
             nodeProcess.on('error', (err) => {
                 Log.error('Child process stopped due to error.');
                 Log.error(err);
+                reject(`${type}:${index}: error ${err}`);
                 process.exit(1);
             });
             nodeProcess.on('exit', (err) => {
                 Log.error('Child process stopped due to error.');
                 Log.error(err);
-                process.exit(1);
+                reject(`${type}:${index}: exit(${err})`);
+                process.exit(err);
             });
         } else {
+            // Single process start
+            // Can only be called from CLI and not from external module
             require(path).start(initConfig);
         }
-    };
+    });
 
     const coresReady = () => {
         const promises = [];
         infraConfig?.front?.forEach((data, index) => {
-            promises.push(new Promise(resolve => {
-                if (serverId && data.serverId !== serverId) { return resolve(); }
-                startNode('front', index, true, resolve);
-            }));
+          if (serverId && data.serverId !== serverId) { return; }
+          promises.push(startNode('front', index, true));
         });
         infraConfig?.storage?.forEach((data, index) => {
-            promises.push(new Promise(resolve => {
-                if (serverId && data.serverId !== serverId) { return resolve(); }
-                startNode('storage', index, true, resolve);
-            }));
+          if (serverId && data.serverId !== serverId) { return; }
+            promises.push(startNode('storage', index, true));
         });
-        promises.push(new Promise(resolve => {
-            if (serverId && infraConfig?.public?.httpServerId !== serverId) { return resolve(); }
-            startNode('http', 0, true, resolve);
-        }));
-        Promise.all(promises).then(() => {
+      if (serverId && infraConfig?.public?.httpServerId !== serverId) { return; }
+        promises.push(startNode('http', 0, true));
+        return Promise.all(promises.filter(Boolean)).then(() => {
             Log.info('CryptPad server ready');
         });
     };
@@ -128,21 +132,14 @@ const start = (serverConfig, infraConfig) => {
             }
             serverConfig.private.nodes_key = Crypto.randomBytes(32).toString('base64');
         }
-        const corePromises = infraConfig?.core.map((data, index) => new Promise((resolve, reject) => {
+        const corePromises = infraConfig?.core.map((data, index) => {
             // hosted on another machine?
-            if (serverId && data.serverId !== serverId) { return resolve(); }
-            startNode('core', index, true, (err) => {
-                if (err) {
-                    Log.error('START_CORE_ERROR', err);
-                    return reject(err);
-                }
-                return resolve();
-            });
-        }));
+            if (serverId && data.serverId !== serverId) { return; }
+            return startNode('core', index, true);
+        });
 
-        Promise.all(corePromises)
-            .then(() => { coresReady(); })
-            .catch((e) => { return Log.error('START_CORE_ERROR', e); });
+        return Promise.all(corePromises.filter(Boolean))
+          .then(() => coresReady());
     };
 
 
@@ -153,18 +150,18 @@ const start = (serverConfig, infraConfig) => {
         if (!serverConfig?.private?.nodes_key) {
             throw Error('E_MISSINGKEY');
         }
-        startNode(type, index, false, (err) => {
+        return startNode(type, index, false).catch((err) => {
             if (err) { return Log.error('START_NODE_ERROR', err); }
         });
     } else {
         serverId = cliArgs.server;
-        startCores();
+        return startCores();
     }
 };
 
 if (require.main === module) {
-    const { config, infra } = require('./common/load-config');
-    start(config, infra);
+    const { config, infra } = require('./common/load-config')();
+    start(config, infra).catch((e) => { console.error('CryptPad server start failed:', e); });
 } else {
     module.exports = { start };
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "cryptpad-server",
   "description": "CryptPad Server",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "license": "AGPL-3.0+",
   "repository": {
     "type": "git",
@@ -16,12 +16,13 @@
     "build"
   ],
   "dependencies": {
+    "@mcrowe/minibloom": "^0.2.0",
     "connect-gzip-static": "^4.2.1",
     "cookie-parser": "^1.4.7",
     "express": "~5.2.1",
-    "mime-types": "^3.0.2",
     "fs-extra": "^7.0.0",
     "http-proxy-middleware": "^3.0.3",
+    "mime-types": "^3.0.2",
     "nthen": "0.1.8",
     "otpauth": "^9.4.1",
     "pull-stream": "^3.6.1",
@@ -61,6 +62,7 @@
     "build:core": "TYPE=core rollup -c",
     "build:storage": "TYPE=storage rollup -c",
     "build:http": "TYPE=http rollup -c",
+    "build:eviction": "TYPE=eviction rollup -c",
     "tests": "node --test tests/",
     "test": "node tests/runAll.js",
     "start": "node index.js",

package/readme.md

@@ -90,12 +90,8 @@ node index.js --server [serverId value]
 The directory [`tests`](tests/) contains some unit and integration tests scripts
 and files.
 
-- [`interface.test.js`](tests/interface.test.js): test the communication
-interface by implementing a simple ping-pong protocol that computes the time it
-takes to go back and forth in the network.
-
 The aforementioned tests can be run with the following command (make sure to
 have installed the dependencies with `npm install` beforehand):
 ```bash
-npm run tests
+npm run test
 ```