mindoodb 0.0.53 → 0.0.55

package/dist/node/server/MindooDBServer.js

@@ -61,8 +61,11 @@ const compressionMiddleware_1 = require("./compressionMiddleware");
 const binaryEntryFraming_1 = require("../../core/appendonlystores/network/binaryEntryFraming");
 const entryWireCodec_1 = require("../../core/appendonlystores/network/entryWireCodec");
 const SyncEventBus_1 = require("./SyncEventBus");
-const types_1 = require("../../core/appendonlystores/types");
-const types_2 = require("../../core/appendonlystores/network/types");
+const ClusterManager_1 = require("./peer/ClusterManager");
+const routes_1 = require("./peer/routes");
+const types_1 = require("./peer/types");
+const types_2 = require("../../core/appendonlystores/types");
+const types_3 = require("../../core/appendonlystores/network/types");
 const TenantManager_1 = require("./TenantManager");
 const CapabilityMatcher_1 = require("./CapabilityMatcher");
 const SystemAdminAuth_1 = require("./SystemAdminAuth");
@@ -71,7 +74,7 @@ const config_1 = require("./config");
 const SystemIpAllowlist_1 = require("./SystemIpAllowlist");
 const validation_1 = require("./validation");
 const urlSafety_1 = require("../../core/utils/urlSafety");
-const types_3 = require("./types");
+const types_4 = require("./types");
 const timestampProxy_1 = require("./timestampProxy");
 const rateLimits_1 = require("./rateLimits");
 /**
@@ -123,24 +126,6 @@ const DEFAULT_TIMESTAMP_RATE_LIMIT_MAX = 10;
 const DEFAULT_TIMESTAMP_DAILY_LIMIT_MAX = 500;
 const TIMESTAMP_DAILY_WINDOW_MS = 24 * 60 * 60 * 1000;
 const DEFAULT_SERVER_SOCKET_TIMEOUT_MS = 120000;
-/**
- * The effective max number of database ids per sync-server registration. Reads
- * the `MINDOODB_MAX_SYNC_SERVER_DATABASES` env override and falls back to
- * {@link DEFAULT_MAX_SYNC_SERVER_DATABASES} when unset or invalid (non-integer
- * or non-positive), so a typo can never silently disable the bound.
- */
-function resolveMaxSyncServerDatabases() {
-    const raw = process.env[types_3.ENV_VARS.MAX_SYNC_SERVER_DATABASES]?.trim();
-    if (!raw)
-        return validation_1.DEFAULT_MAX_SYNC_SERVER_DATABASES;
-    const parsed = Number(raw);
-    if (!Number.isInteger(parsed) || parsed <= 0) {
-        console.warn(`[MindooDBServer] Ignoring invalid ${types_3.ENV_VARS.MAX_SYNC_SERVER_DATABASES}="${raw}" ` +
-            `(expected a positive integer); using default ${validation_1.DEFAULT_MAX_SYNC_SERVER_DATABASES}`);
-        return validation_1.DEFAULT_MAX_SYNC_SERVER_DATABASES;
-    }
-    return parsed;
-}
 /** Parse a human-readable size string ("5mb", "1024", "100kb") into bytes. */
 function parseBodySizeLimitToBytes(limit) {
     const normalized = limit.trim().toLowerCase();
@@ -183,8 +168,8 @@ class MindooDBServer {
         this.configPath = configPath ?? path_1.default.join(dataDir, "config.json");
         this.jsonBodyLimit = process.env.MINDOODB_JSON_BODY_LIMIT?.trim() || "5mb";
         this.jsonBodyLimitBytes = parseBodySizeLimitToBytes(this.jsonBodyLimit);
-        this.tsaProviders = (0, timestampProxy_1.parseTsaProviders)(process.env[types_3.ENV_VARS.TSA_PROVIDERS], {
-            allowInsecureUrls: /^(1|true)$/i.test(process.env[types_3.ENV_VARS.ALLOW_INSECURE_SYNC_URLS] ?? ""),
+        this.tsaProviders = (0, timestampProxy_1.parseTsaProviders)(process.env[types_4.ENV_VARS.TSA_PROVIDERS], {
+            allowInsecureUrls: /^(1|true)$/i.test(process.env[types_4.ENV_VARS.ALLOW_INSECURE_SYNC_URLS] ?? ""),
         });
         if (this.tsaProviders.length > 0) {
             console.log(`[MindooDBServer] RFC 3161 timestamp proxy enabled with providers: ${this.tsaProviders
@@ -200,6 +185,39 @@ class MindooDBServer {
             isTenantAdminSigningKey: (publicsignkey) => this.tenantManager.isRegisteredTenantAdminSigningKey(publicsignkey),
         });
         this.deviceDiscovery = new DeviceDiscoveryService_1.DeviceDiscoveryService(this.tenantManager, cryptoAdapter);
+        this.cluster = new ClusterManager_1.ClusterManager({
+            cryptoAdapter,
+            dataDir,
+            eventBus: this.syncEventBus,
+            listTrustedServers: () => this.tenantManager.listTrustedServers(),
+            localServerName: () => this.tenantManager.getServerPublicInfo()?.name ?? null,
+            localSigningPublicKey: () => this.tenantManager.getServerPublicInfo()?.signingPublicKey ?? null,
+            getSigningKey: async () => (await this.tenantManager.getWitnessSigner())?.signingPrivateKey,
+            getEncryptionKey: () => this.tenantManager.getServerEncryptionPrivateKey(),
+            listTenants: () => this.tenantManager.listTenants(),
+            listDatabases: (tenantId) => this.tenantManager.listDatabases(tenantId),
+            getLocalStore: (tenantId, dbId, storeKind) => this.tenantManager.getStore(tenantId, dbId, storeKind),
+            localRole: this.serverConfig.cluster?.role,
+        });
+        // Peer tokens are accepted by the tenant sync routes so a peer can mirror
+        // the encrypted stores through the ordinary protocol; the store bypasses
+        // per-user access control for them (see ServerNetworkContentAddressedStore).
+        this.tenantManager.setPeerAccessControl({
+            peerTokenResolver: async (token) => {
+                const payload = await this.cluster.auth.validateToken(token);
+                if (!payload)
+                    return null;
+                const trusted = this.cluster.auth.findTrustedServerByKey(payload.publicsignkey);
+                if (!trusted)
+                    return null;
+                return {
+                    name: trusted.name,
+                    signingPublicKey: trusted.signingPublicKey,
+                    encryptionPublicKey: trusted.encryptionPublicKey,
+                };
+            },
+            trustedWitnessResolver: (receivedByPublicKey) => this.cluster.auth.findTrustedServerByKey(receivedByPublicKey) !== null,
+        });
         const principalCount = Object.values(this.serverConfig.capabilities).reduce((sum, arr) => sum + arr.length, 0);
         if (principalCount === 0) {
             console.log(`[MindooDBServer] WARNING: No system admin principals configured. All /system/* endpoints are locked down.`);
@@ -219,6 +237,23 @@ class MindooDBServer {
     getSystemAdminAuth() {
         return this.systemAdminAuth;
     }
+    getClusterManager() {
+        return this.cluster;
+    }
+    /**
+     * Begin replicating with every trusted server that has a `url`.
+     *
+     * Separate from the constructor so a server can serve the peer and cluster
+     * endpoints — answering handshakes, status and admin actions — without
+     * dialling out itself. That is what `--auto-sync` toggles, and it is what
+     * makes a passive mirror or a test harness possible.
+     */
+    startCluster() {
+        this.cluster.start();
+    }
+    async stopCluster() {
+        await this.cluster.stop();
+    }
     getServerConfig() {
         return this.serverConfig;
     }
@@ -319,7 +354,7 @@ class MindooDBServer {
      * list over `true`, which trusts the whole chain.
      */
     applyTrustProxySetting() {
-        const raw = process.env[types_3.ENV_VARS.TRUST_PROXY]?.trim();
+        const raw = process.env[types_4.ENV_VARS.TRUST_PROXY]?.trim();
         if (!raw || /^(0|false)$/i.test(raw)) {
             return;
         }
@@ -395,6 +430,10 @@ class MindooDBServer {
             }
             res.json({
                 ...info,
+                // Its own mesh role, so `add-to-network` can propagate it into the
+                // other side's trusted-servers entry instead of asking the operator to
+                // repeat it on every node.
+                clusterRole: this.serverConfig.cluster?.role ?? "peer",
                 maxJsonRequestBodyLimit: this.jsonBodyLimit,
                 maxJsonRequestBodyBytes: this.jsonBodyLimitBytes ?? undefined,
                 // Lets a client decide between calling a browser-reachable TSA directly
@@ -405,6 +444,33 @@ class MindooDBServer {
                 },
             });
         });
+        /**
+         * Mirrors of one tenant, for client failover.
+         *
+         * Scoped to the tenant in the query on purpose: a client asking about
+         * tenant A must not learn which other tenants this server replicates, or
+         * to whom. Unauthenticated like the other discovery endpoints — a client
+         * needs it precisely when it cannot reach a server to authenticate with.
+         */
+        this.app.get("/.well-known/mindoodb-cluster", (req, res) => {
+            const tenantId = typeof req.query.tenantId === "string" ? req.query.tenantId.toLowerCase() : "";
+            if (!tenantId) {
+                res.status(400).json({ error: "tenantId query parameter is required" });
+                return;
+            }
+            try {
+                (0, validation_1.validateTenantId)(tenantId);
+            }
+            catch {
+                res.status(400).json({ error: "Invalid tenantId format" });
+                return;
+            }
+            if (!this.tenantManager.tenantExists(tenantId)) {
+                res.status(404).json({ error: "Tenant not found on server" });
+                return;
+            }
+            res.json({ tenantId, mirrors: this.cluster.mirrorsForTenant(tenantId) });
+        });
         this.app.get("/.well-known/mindoodb-tenants/:tenantId/publicinfos-fingerprints", async (req, res) => {
             const tenantId = req.params.tenantId.toLowerCase();
             try {
@@ -454,12 +520,29 @@ class MindooDBServer {
             console.log(`[MindooDBServer] MINDOODB_ADMIN_ALLOWED_IPS is set — /system/* limited to: ${systemIpRaw}`);
         }
         const systemRateLimit = (0, express_rate_limit_1.default)({
-            windowMs: 60000,
-            max: 30,
+            windowMs: this.serverConfig.rateLimits?.system?.windowMs ?? 60000,
+            max: this.serverConfig.rateLimits?.system?.max ?? 30,
             standardHeaders: true,
             legacyHeaders: false,
             message: { error: "Too many system requests, please try again later" },
         });
+        // Peer routes are mounted before the /system chain on purpose. They are
+        // server-to-server traffic, not administration: the admin IP allowlist
+        // would lock out the operator's own cluster, and 30 req/min would throttle
+        // replication to a crawl. They are protected by the peer JWT, which is only
+        // issued to a key in trusted-servers.json.
+        this.app.use("/system/peer", (0, express_rate_limit_1.default)({
+            windowMs: 60000,
+            max: 600,
+            standardHeaders: true,
+            legacyHeaders: false,
+            message: { error: "Too many peer requests, please try again later" },
+        }), (0, routes_1.createPeerRouter)({
+            cluster: this.cluster,
+            eventBus: this.syncEventBus,
+            listTenants: () => this.tenantManager.listTenants(),
+            listDatabases: (tenantId) => this.tenantManager.listDatabases(tenantId),
+        }));
         const systemRouter = (0, express_1.Router)();
         this.setupSystemRoutes(systemRouter);
         this.app.use("/system", SystemIpAllowlist_1.systemIpAllowlistMiddleware, systemRateLimit, systemRouter);
@@ -563,6 +646,15 @@ class MindooDBServer {
         router.post("/auth/challenge", challengeRateLimit, this.handleSystemChallenge.bind(this));
         router.post("/auth/authenticate", authenticateRateLimit, this.handleSystemAuthenticate.bind(this));
         const authMiddleware = this.systemAdminMiddleware.bind(this);
+        // Cluster administration. Behind the same JWT + capability middleware as
+        // the rest of /system, so config.json can grant an auditor principal the
+        // GET routes without granting the POST/PATCH/DELETE ones.
+        router.use("/cluster", authMiddleware, (0, routes_1.createClusterRouter)({
+            cluster: this.cluster,
+            listTrustedServers: () => this.tenantManager.listTrustedServers(),
+            saveTrustedServer: (peer) => this.tenantManager.saveTrustedServer(peer),
+            removeTrustedServer: (name) => this.tenantManager.removeTrustedServer(name),
+        }));
         // Tenant CRUD
         router.get("/tenants", authMiddleware, (req, res) => {
             try {
@@ -702,7 +794,7 @@ class MindooDBServer {
         });
         router.post("/trusted-servers", authMiddleware, (req, res) => {
             try {
-                const { name, signingPublicKey, encryptionPublicKey } = req.body;
+                const { name, signingPublicKey, encryptionPublicKey, url, direction, role, attachments } = req.body;
                 if (!name || !signingPublicKey || !encryptionPublicKey) {
                     res.status(400).json({ error: "name, signingPublicKey, and encryptionPublicKey are required" });
                     return;
@@ -711,7 +803,41 @@ class MindooDBServer {
                 (0, validation_1.validateStringLength)(signingPublicKey, validation_1.MAX_PEM_KEY_LENGTH, "signingPublicKey");
                 (0, validation_1.validateStringLength)(encryptionPublicKey, validation_1.MAX_PEM_KEY_LENGTH, "encryptionPublicKey");
                 const server = { name, signingPublicKey, encryptionPublicKey };
+                // Replication settings are optional: an entry without a url is a trust
+                // relationship only (inbound peers may authenticate, we never dial).
+                if (url !== undefined) {
+                    (0, validation_1.validateStringLength)(url, 2048, "url");
+                    const allowInsecure = /^(1|true)$/i.test(process.env[types_4.ENV_VARS.ALLOW_INSECURE_SYNC_URLS] ?? "");
+                    try {
+                        (0, urlSafety_1.assertSafeSyncUrl)(url, {
+                            requireHttps: !allowInsecure,
+                            allowPrivate: allowInsecure,
+                        });
+                    }
+                    catch (e) {
+                        if (e instanceof urlSafety_1.UnsafeUrlError) {
+                            res.status(400).json({ error: e.message });
+                            return;
+                        }
+                        throw e;
+                    }
+                    server.url = url;
+                }
+                for (const [field, value, guard] of [
+                    ["direction", direction, types_1.isPeerSyncDirection],
+                    ["role", role, types_1.isPeerRole],
+                    ["attachments", attachments, types_1.isPeerAttachmentMode],
+                ]) {
+                    if (value === undefined)
+                        continue;
+                    if (!guard(value)) {
+                        res.status(400).json({ error: `Invalid ${field}: ${String(value)}` });
+                        return;
+                    }
+                    Object.assign(server, { [field]: value });
+                }
                 this.tenantManager.addTrustedServer(server);
+                this.cluster.syncReplicators();
                 res.status(201).json({ success: true, message: `Trusted server "${name}" added` });
             }
             catch (error) {
@@ -736,6 +862,8 @@ class MindooDBServer {
                     res.status(404).json({ error: "Trusted server not found" });
                     return;
                 }
+                // Revoking trust must also stop replicating with that server.
+                this.cluster.syncReplicators();
                 res.json({ success: true, message: `Trusted server "${serverName}" removed` });
             }
             catch (error) {
@@ -747,128 +875,6 @@ class MindooDBServer {
                 res.status(500).json({ error: "Failed to remove trusted server" });
             }
         });
-        // Per-tenant sync server management
-        router.get("/tenants/:tenantId/sync-servers", authMiddleware, (req, res) => {
-            try {
-                const tenantId = req.params.tenantId.toLowerCase();
-                try {
-                    (0, validation_1.validateTenantId)(tenantId);
-                }
-                catch {
-                    res.status(400).json({ error: "Invalid tenantId format" });
-                    return;
-                }
-                if (!this.tenantManager.tenantExists(tenantId)) {
-                    res.status(404).json({ error: "Tenant not found" });
-                    return;
-                }
-                const servers = this.tenantManager.getTenantSyncServers(tenantId);
-                res.json({ servers });
-            }
-            catch (error) {
-                console.error("[MindooDBServer] Error listing sync servers:", error);
-                res.status(500).json({ error: "Failed to list sync servers" });
-            }
-        });
-        router.post("/tenants/:tenantId/sync-servers", authMiddleware, (req, res) => {
-            try {
-                const tenantId = req.params.tenantId.toLowerCase();
-                try {
-                    (0, validation_1.validateTenantId)(tenantId);
-                }
-                catch {
-                    res.status(400).json({ error: "Invalid tenantId format" });
-                    return;
-                }
-                if (!this.tenantManager.tenantExists(tenantId)) {
-                    res.status(404).json({ error: "Tenant not found" });
-                    return;
-                }
-                const { name, url, syncIntervalMs, databases } = req.body;
-                if (!name || !url) {
-                    res.status(400).json({ error: "name and url are required" });
-                    return;
-                }
-                if (!databases || !Array.isArray(databases) || databases.length === 0) {
-                    res.status(400).json({ error: "databases array is required and must not be empty" });
-                    return;
-                }
-                // Bound and format-validate each database id (audit, Low): these are
-                // used to scope sync and must be safe identifiers, not arbitrary input.
-                // The bound is overridable per deployment via
-                // MINDOODB_MAX_SYNC_SERVER_DATABASES.
-                (0, validation_1.validateArraySize)(databases, resolveMaxSyncServerDatabases(), "databases");
-                for (const db of databases) {
-                    (0, validation_1.validateIdentifier)(db, "databases[]");
-                }
-                (0, validation_1.validateStringLength)(name, 256, "name");
-                (0, validation_1.validateStringLength)(url, 2048, "url");
-                // SSRF guard: an admin-supplied sync URL is fetched server-side, so
-                // reject plaintext/internal targets unless explicitly allowed for dev.
-                const allowInsecure = /^(1|true)$/i.test(process.env[types_3.ENV_VARS.ALLOW_INSECURE_SYNC_URLS] ?? "");
-                try {
-                    (0, urlSafety_1.assertSafeSyncUrl)(url, {
-                        requireHttps: !allowInsecure,
-                        allowPrivate: allowInsecure,
-                    });
-                }
-                catch (e) {
-                    if (e instanceof urlSafety_1.UnsafeUrlError) {
-                        res.status(400).json({ error: e.message });
-                        return;
-                    }
-                    throw e;
-                }
-                const config = { name, url, databases };
-                if (syncIntervalMs !== undefined) {
-                    config.syncIntervalMs = syncIntervalMs;
-                }
-                this.tenantManager.addTenantSyncServer(tenantId, config);
-                res.status(201).json({ success: true, message: `Sync server "${name}" configured for tenant ${tenantId}` });
-            }
-            catch (error) {
-                if (error instanceof validation_1.ValidationError) {
-                    res.status(400).json({ error: error.message });
-                    return;
-                }
-                console.error("[MindooDBServer] Error adding sync server:", error);
-                res.status(500).json({ error: "Failed to add sync server" });
-            }
-        });
-        router.delete("/tenants/:tenantId/sync-servers/:serverName", authMiddleware, (req, res) => {
-            try {
-                const tenantId = req.params.tenantId.toLowerCase();
-                try {
-                    (0, validation_1.validateTenantId)(tenantId);
-                }
-                catch {
-                    res.status(400).json({ error: "Invalid tenantId format" });
-                    return;
-                }
-                if (!this.tenantManager.tenantExists(tenantId)) {
-                    res.status(404).json({ error: "Tenant not found" });
-                    return;
-                }
-                const serverName = decodeURIComponent(req.params.serverName);
-                (0, validation_1.validateStringLength)(serverName, 256, "serverName");
-                const removed = this.tenantManager.removeTenantSyncServer(tenantId, serverName);
-                if (!removed) {
-                    res.status(404).json({ error: "Sync server not found" });
-                    return;
-                }
-                res.json({ success: true, message: `Sync server "${serverName}" removed from tenant ${tenantId}` });
-            }
-            catch (error) {
-                if (error instanceof validation_1.ValidationError) {
-                    res.status(400).json({ error: error.message });
-                    return;
-                }
-                console.error("[MindooDBServer] Error removing sync server:", error);
-                res.status(500).json({ error: "Failed to remove sync server" });
-            }
-        });
-        // Per-tenant trigger-sync (moved from /:tenantId/admin/trigger-sync)
-        router.post("/tenants/:tenantId/trigger-sync", authMiddleware, this.handleTriggerSync.bind(this));
         // Runtime config management
         router.get("/config/backups", authMiddleware, (req, res) => {
             try {
@@ -1625,16 +1631,20 @@ class MindooDBServer {
         if (this.syncEventBus.listenerCount === 0) {
             return;
         }
-        void serverStore
-            .handleGetStoreHead(token)
-            .catch(() => null)
-            .then((head) => {
+        // Peer replicators subscribe to this bus, so a write that arrived from a
+        // peer has to be labelled: without the label it would be pushed straight
+        // back to its origin, and two bidirectional peers would ping-pong.
+        const originPeer = this.cluster?.auth
+            ? this.cluster.auth.validateToken(token).then((payload) => payload?.sub)
+            : Promise.resolve(undefined);
+        void Promise.all([serverStore.handleGetStoreHead(token).catch(() => null), originPeer]).then(([head, peerName]) => {
             this.syncEventBus.publish({
                 tenantId,
                 dbId,
                 storeKind,
                 epoch: head?.epoch,
                 maxReceiptOrder: head?.maxReceiptOrder,
+                originPeer: peerName,
             });
         });
     }
@@ -1999,9 +2009,6 @@ class MindooDBServer {
         }
     }
     // ==================== Management Handlers ====================
-    async handleTriggerSync(req, res) {
-        res.json({ success: true, message: "Sync triggered (not yet implemented)" });
-    }
     // ==================== Helper Methods ====================
     /**
      * Derive the StoreKind from the request path. Both "docs" and
@@ -2011,17 +2018,17 @@ class MindooDBServer {
     getStoreKindFromRequest(req) {
         const trimmedPath = req.path.replace(/\/$/, "");
         if (trimmedPath.startsWith("/sync/docs/") || trimmedPath === "/sync/docs/capabilities" || trimmedPath === "/sync/docs/getAllIds") {
-            return types_1.StoreKind.docs;
+            return types_2.StoreKind.docs;
         }
         if (trimmedPath.startsWith("/sync/attachments/") || trimmedPath === "/sync/attachments/capabilities" || trimmedPath === "/sync/attachments/getAllIds") {
-            return types_1.StoreKind.attachments;
+            return types_2.StoreKind.attachments;
         }
         throw new Error(`Unsupported sync store kind for path: ${req.path}`);
     }
     extractToken(req) {
         const auth = req.headers.authorization;
         if (!auth || !auth.startsWith("Bearer ")) {
-            throw new types_2.NetworkError("INVALID_TOKEN", "Missing or invalid Authorization header");
+            throw new types_3.NetworkError("INVALID_TOKEN", "Missing or invalid Authorization header");
         }
         return auth.substring(7);
     }