@logux/server 0.13.1 → 0.15.0

package/base-server/index.js

@@ -2,21 +2,38 @@ import { LoguxNotFoundError } from '@logux/actions'
 import { Log, MemoryStore, parseId, ServerConnection } from '@logux/core'
 import { createNanoEvents } from 'nanoevents'
 import { nanoid } from 'nanoid'
-import { promises as fs } from 'node:fs'
+import { readFile } from 'node:fs/promises'
 import { dirname, join } from 'node:path'
 import { fileURLToPath } from 'node:url'
-import UrlPattern from 'url-pattern'
 import { WebSocketServer } from 'ws'
 
-import { bindBackendProxy } from '../bind-backend-proxy/index.js'
-import { bindControlServer } from '../bind-control-server/index.js'
+import { addHttpPages } from '../add-http-pages/index.js'
 import { Context } from '../context/index.js'
 import { createHttpServer } from '../create-http-server/index.js'
 import { ServerClient } from '../server-client/index.js'
+import { createPattern } from '../url-pattern/index.js'
 
 const SKIP_PROCESS = Symbol('skipProcess')
 const RESEND_META = ['channels', 'users', 'clients', 'nodes']
 
+function addTarget(targets, client, action, meta) {
+  let entries = targets.get(client)
+  if (entries) {
+    entries.push([action, meta])
+  } else {
+    targets.set(client, [[action, meta]])
+  }
+}
+
+function sendTargets(targets) {
+  for (let [client, entries] of targets) {
+    let sending = client.node.onAdd(entries)
+    // Only `ServerClient` tracks sending for `Server#drain()`
+    if (client.track) client.track(sending)
+  }
+  targets.clear()
+}
+
 function optionError(msg) {
   let error = new Error(msg)
   error.logux = true
@@ -24,17 +41,6 @@ function optionError(msg) {
   throw error
 }
 
-let helloCache
-async function readHello() {
-  if (!helloCache) {
-    let hello = await fs.readFile(
-      join(fileURLToPath(import.meta.url), '..', 'hello.html')
-    )
-    helloCache = hello.toString()
-  }
-  return helloCache
-}
-
 export async function wasNot403(cb) {
   try {
     await cb()
@@ -103,17 +109,11 @@ export class BaseServer {
     this.options = opts
     this.env = this.options.env || process.env.NODE_ENV || 'development'
 
-    if (
-      typeof this.options.subprotocol === 'undefined' &&
-      typeof this.options.backend === 'undefined'
-    ) {
+    if (typeof this.options.subprotocol === 'undefined') {
       throw optionError('Missed `subprotocol` option in server constructor')
     }
-    if (
-      typeof this.options.supports === 'undefined' &&
-      typeof this.options.backend === 'undefined'
-    ) {
-      throw optionError('Missed `supports` option in server constructor')
+    if (typeof this.options.minSubprotocol === 'undefined') {
+      throw optionError('Missed `minSubprotocol` option in server constructor')
     }
 
     if (this.options.key && !this.options.cert) {
@@ -128,14 +128,16 @@ export class BaseServer {
       if (!this.options.host) this.options.host = '127.0.0.1'
     }
 
-    this.nodeId = `server:${this.options.id || nanoid(8)}`
+    this.nodeId = `server:${this.options.id || nanoid(6)}`
 
     if (this.options.fileUrl) {
       this.options.root = dirname(fileURLToPath(this.options.fileUrl))
     }
 
     this.options.root = this.options.root || process.cwd()
-    this.options.controlMask = this.options.controlMask || '127.0.0.1/8'
+    if (typeof this.options.port === 'string') {
+      this.options.port = parseInt(this.options.port, 10)
+    }
 
     let store = this.options.store || new MemoryStore()
 
@@ -167,7 +169,6 @@ export class BaseServer {
         }
         if (
           !isLogux &&
-          !this.options.backend &&
           !this.types[action.type] &&
           !this.getRegexProcessor(action.type)
         ) {
@@ -176,89 +177,64 @@ export class BaseServer {
       }
       this.replaceResendShortcuts(meta)
     })
-    this.on('add', async (action, meta) => {
-      let start = Date.now()
+    // Reports are per action, `clean` for a reason-less action is emitted
+    // by the log before `batch` event
+    this.on('add', (action, meta) => {
       if (meta.reasons.length === 0) {
         cleaned[meta.id] = true
         this.emitter.emit('report', 'addClean', { action, meta })
       } else {
         this.emitter.emit('report', 'add', { action, meta })
       }
+    })
+    this.on('batch', entries => {
+      let start = Date.now()
+      let ready = []
 
-      if (this.destroying && !this.actionToQueue.has(meta.id)) {
-        return
-      }
-
-      if (action.type === 'logux/subscribe') {
-        if (meta.server === this.nodeId) {
-          this.subscribeAction(action, meta, start)
+      for (let [action, meta] of entries) {
+        if (this.destroying && !this.actionToQueue.has(meta.id)) {
+          continue
         }
-        return
-      }
 
-      if (action.type === 'logux/unsubscribe') {
-        if (meta.server === this.nodeId) {
-          this.unsubscribeAction(action, meta)
+        if (action.type === 'logux/subscribe') {
+          if (meta.server === this.nodeId) {
+            void this.subscribeAction(action, meta, start)
+          }
+          continue
         }
-        return
-      }
 
-      let processor = this.getProcessor(action.type)
-      if (processor && processor.resend && meta.status === 'waiting') {
-        let ctx = this.createContext(action, meta)
-        let resend
-        try {
-          resend = await processor.resend(ctx, action, meta)
-        } catch (e) {
-          this.undo(action, meta, 'error')
-          this.emitter.emit('error', e, action, meta)
-          this.finally(processor, ctx, action, meta)
-          return
-        }
-        if (resend) {
-          if (typeof resend === 'string') {
-            resend = { channels: [resend] }
-          } else if (Array.isArray(resend)) {
-            resend = { channels: resend }
-          } else {
-            this.replaceResendShortcuts(resend)
+        if (action.type === 'logux/unsubscribe') {
+          if (meta.server === this.nodeId) {
+            this.unsubscribeAction(action, meta)
           }
-          let diff = {}
-          for (let i of RESEND_META) {
-            if (resend[i]) diff[i] = resend[i]
-          }
-          await this.log.changeMeta(meta.id, diff)
-          meta = { ...meta, ...diff }
+          continue
         }
-      }
-
-      if (this.isUseless(action, meta)) {
-        this.emitter.emit('report', 'useless', { action, meta })
-      }
 
-      await this.sendAction(action, meta)
+        let processor = this.getProcessor(action.type)
+        if (processor && processor.resend && meta.status === 'waiting') {
+          // Resending is asynchronous, so the action can’t wait for others
+          void this.resendAction(action, meta, processor, start)
+          continue
+        }
 
-      if (meta.status === 'waiting') {
-        if (!processor) {
-          this.internalUnknownType(action, meta)
-          return
+        if (this.isUseless(action, meta)) {
+          this.emitter.emit('report', 'useless', { action, meta })
         }
-        if (processor.process) {
-          this.processAction(processor, action, meta, start)
-        } else {
-          this.emitter.emit('processed', action, meta, 0)
-          this.finally(
-            processor,
-            this.createContext(action, meta),
-            action,
-            meta
-          )
-          this.markAsProcessed(meta)
+        ready.push([action, meta, processor])
+      }
+
+      if (ready.length === 0) return
+      let runProcessors = () => {
+        for (let [action, meta, processor] of ready) {
+          this.runProcessor(action, meta, processor, start)
         }
-      } else {
-        this.emitter.emit('processed', action, meta, 0)
-        this.finally(processor, this.createContext(action, meta), action, meta)
       }
+      // Actions are processed only after they were sent to the clients
+      void Promise.resolve(this.sendBatch(ready)).then(runProcessors, error => {
+        // Do not stop action processing on a broken channel filter
+        runProcessors()
+        throw error
+      })
     })
     this.on('clean', (action, meta) => {
       if (cleaned[meta.id]) {
@@ -339,27 +315,11 @@ export class BaseServer {
     this.queues = new Map()
     this.actionToQueue = new Map()
 
-    this.controls = {
-      'GET /': {
-        async request() {
-          return {
-            body: await readHello(),
-            headers: { 'Content-Type': 'text/html' }
-          }
-        },
-        safe: true
-      },
-      'GET /health': {
-        request: () => ({
-          body: 'Logux Server: OK\n',
-          headers: { 'Content-Type': 'text/plain' }
-        }),
-        safe: true
-      }
-    }
+    this.httpListeners = {}
+    this.httpAllListeners = []
+    addHttpPages(this)
 
     this.listenNotes = {}
-    bindBackendProxy(this)
 
     let end = (actionId, queue, queueKey, ...args) => {
       this.actionToQueue.delete(actionId)
@@ -478,9 +438,7 @@ export class BaseServer {
     normalizeChannelCallbacks(`Channel ${pattern}`, callbacks)
     let channel = Object.assign({}, callbacks)
     if (typeof pattern === 'string') {
-      channel.pattern = new UrlPattern(pattern, {
-        segmentValueCharset: '^/'
-      })
+      channel.pattern = createPattern(pattern)
     } else {
       channel.regexp = pattern
     }
@@ -529,6 +487,12 @@ export class BaseServer {
     return Promise.all(this.unbind.map(i => i()))
   }
 
+  drain(clientId) {
+    let client = this.clientIds.get(clientId)
+    if (!client) return Promise.resolve(false)
+    return client.drain()
+  }
+
   finally(processor, ctx, action, meta) {
     this.contexts.delete(action)
     if (processor && processor.finally) {
@@ -555,13 +519,22 @@ export class BaseServer {
     return undefined
   }
 
-  http(listener) {
+  handleClient(ws, req) {
+    ws.upgradeReq = req
+    this.addClient(new ServerConnection(ws))
+  }
+
+  http(method, url, listener) {
     if (this.options.disableHttpServer) {
       throw new Error(
         '`server.http()` can not be called when `disableHttpServer` enabled'
       )
     }
-    this.httpListener = listener
+    if (!url) {
+      this.httpAllListeners.push(method)
+    } else {
+      this.httpListeners[`${method} ${url}`] = listener
+    }
   }
 
   internalUnknownType(action, meta) {
@@ -618,50 +591,54 @@ export class BaseServer {
         this.httpServer.listen(this.options.port, this.options.host, resolve)
       })
     }
-    bindControlServer(this, this.httpListener)
 
-    this.unbind.push(
-      () =>
-        new Promise(resolve => {
-          this.ws.on('close', resolve)
-          this.ws.close()
-        })
-    )
-    if (this.httpServer) {
-      this.unbind.push(
-        () =>
-          new Promise(resolve => {
-            this.httpServer.on('close', resolve)
-            this.httpServer.close()
-          })
-      )
+    let processing = 0
+    let waiting
+    this.unbind.push(() => {
+      return new Promise(resolve => {
+        let end = () => {
+          this.ws.close(resolve)
+          this.httpServer.close()
+        }
+        if (processing === 0) {
+          end()
+        } else {
+          waiting = end
+        }
+      })
+    })
+
+    if (!this.options.disableHttpServer) {
+      this.httpServer.on('request', async (req, res) => {
+        if (this.destroying) {
+          res.writeHead(503, { 'Content-Type': 'text/plain' })
+          res.end('The server is shutting down\n')
+          return
+        }
+        processing += 1
+        await this.processHttp(req, res)
+        processing -= 1
+        if (processing === 0 && waiting) waiting()
+      })
     }
 
     let pkg = JSON.parse(
-      await fs.readFile(
-        join(fileURLToPath(import.meta.url), '..', '..', 'package.json')
-      )
+      await readFile(join(import.meta.dirname, '..', 'package.json'))
     )
 
-    this.ws.on('connection', (ws, req) => {
-      ws.upgradeReq = req
-      this.addClient(new ServerConnection(ws))
-    })
+    this.ws.on('connection', (ws, req) => this.handleClient(ws, req))
     this.emitter.emit('report', 'listen', {
-      backend: this.options.backend,
       cert: !!this.options.cert,
-      controlMask: this.options.controlMask,
-      controlSecret: this.options.controlSecret,
       environment: this.env,
       host: this.options.host,
       loguxServer: pkg.version,
+      minSubprotocol: this.options.minSubprotocol,
       nodeId: this.nodeId,
       notes: this.listenNotes,
       port: this.options.port,
       redis: this.options.redis,
       server: !!this.options.server,
-      subprotocol: this.options.subprotocol,
-      supports: this.options.supports
+      subprotocol: this.options.subprotocol
     })
   }
 
@@ -677,7 +654,12 @@ export class BaseServer {
   }
 
   on(event, listener) {
-    if (event === 'preadd' || event === 'add' || event === 'clean') {
+    if (
+      event === 'preadd' ||
+      event === 'add' ||
+      event === 'batch' ||
+      event === 'clean'
+    ) {
       return this.log.emitter.on(event, listener)
     } else {
       return this.emitter.on(event, listener)
@@ -690,11 +672,7 @@ export class BaseServer {
       throw new Error('Callbacks for unknown channel are already defined')
     }
     let channel = Object.assign({}, callbacks)
-    channel.pattern = {
-      match(name) {
-        return [name]
-      }
-    }
+    channel.pattern = name => [name]
     this.otherSubscriber = channel
   }
 
@@ -707,6 +685,7 @@ export class BaseServer {
   }
 
   performUnsubscribe(clientNodeId, action, meta) {
+    if (action.channel === '__proto__' || clientNodeId === '__proto__') return
     if (this.subscribers[action.channel]) {
       let subscriber = this.subscribers[action.channel][clientNodeId]
       if (subscriber) {
@@ -772,6 +751,32 @@ export class BaseServer {
     this.emitter.emit('processed', action, meta, latency)
   }
 
+  async processHttp(req, res) {
+    let urlString = req.url
+    if (/^\/\w+%3F/.test(urlString)) {
+      urlString = decodeURIComponent(urlString)
+    }
+    let reqUrl = new URL(urlString, 'http://localhost')
+    let rule = this.httpListeners[req.method + ' ' + reqUrl.pathname]
+
+    if (!rule) {
+      let processed = false
+      for (let listener of this.httpAllListeners) {
+        let result = await listener(req, res)
+        if (result === true) {
+          processed = true
+          break
+        }
+      }
+      if (!processed) {
+        res.writeHead(404, { 'Content-Type': 'text/plain' })
+        res.end('Not found\n')
+      }
+    } else {
+      await rule(req, res)
+    }
+  }
+
   rememberBadAuth(ip) {
     this.authAttempts[ip] = (this.authAttempts[ip] || 0) + 1
     this.setTimeout(() => {
@@ -802,7 +807,42 @@ export class BaseServer {
     }
   }
 
-  async sendAction(action, meta) {
+  async resendAction(action, meta, processor, start) {
+    let ctx = this.createContext(action, meta)
+    let resend
+    try {
+      resend = await processor.resend(ctx, action, meta)
+    } catch (e) {
+      this.undo(action, meta, 'error')
+      this.emitter.emit('error', e, action, meta)
+      this.finally(processor, ctx, action, meta)
+      return
+    }
+    if (resend) {
+      if (typeof resend === 'string') {
+        resend = { channels: [resend] }
+      } else if (Array.isArray(resend)) {
+        resend = { channels: resend }
+      } else {
+        this.replaceResendShortcuts(resend)
+      }
+      let diff = {}
+      for (let i of RESEND_META) {
+        if (resend[i]) diff[i] = resend[i]
+      }
+      await this.log.changeMeta(meta.id, diff)
+      meta = { ...meta, ...diff }
+    }
+
+    if (this.isUseless(action, meta)) {
+      this.emitter.emit('report', 'useless', { action, meta })
+    }
+
+    await this.sendAction(action, meta)
+    this.runProcessor(action, meta, processor, start)
+  }
+
+  resolveTargets(action, meta, targets) {
     let from = parseId(meta.id).clientId
     let ignoreClients = new Set(meta.excludeClients || [])
     ignoreClients.add(from)
@@ -812,7 +852,7 @@ export class BaseServer {
         let client = this.nodeIds.get(id)
         if (client) {
           ignoreClients.add(client.clientId)
-          client.node.onAdd(action, meta)
+          addTarget(targets, client, action, meta)
         }
       }
     }
@@ -822,7 +862,7 @@ export class BaseServer {
         if (this.clientIds.has(id)) {
           let client = this.clientIds.get(id)
           ignoreClients.add(client.clientId)
-          client.node.onAdd(action, meta)
+          addTarget(targets, client, action, meta)
         }
       }
     }
@@ -834,32 +874,38 @@ export class BaseServer {
           for (let client of users) {
             if (!ignoreClients.has(client.clientId)) {
               ignoreClients.add(client.clientId)
-              client.node.onAdd(action, meta)
+              addTarget(targets, client, action, meta)
             }
           }
         }
       }
     }
 
-    if (meta.channels) {
-      for (let channel of meta.channels) {
-        if (this.subscribers[channel]) {
-          for (let nodeId in this.subscribers[channel]) {
-            let clientId = parseId(nodeId).clientId
-            if (!ignoreClients.has(clientId)) {
-              let subscriber = this.subscribers[channel][nodeId]
-              if (subscriber) {
-                let ctx = this.createContext(action, meta)
-                let client = this.clientIds.get(clientId)
-                for (let filter of Object.values(subscriber.filters)) {
-                  filter =
-                    typeof filter === 'function'
-                      ? await filter(ctx, action, meta)
-                      : filter
-                  if (filter && client) {
-                    ignoreClients.add(clientId)
-                    client.node.onAdd(action, meta)
-                  }
+    if (!meta.channels) return undefined
+
+    let waiting = []
+    for (let channel of meta.channels) {
+      if (this.subscribers[channel]) {
+        for (let nodeId in this.subscribers[channel]) {
+          let clientId = parseId(nodeId).clientId
+          if (!ignoreClients.has(clientId)) {
+            let subscriber = this.subscribers[channel][nodeId]
+            if (subscriber) {
+              let ctx = this.createContext(action, meta)
+              let client = this.clientIds.get(clientId)
+              for (let filter of Object.values(subscriber.filters)) {
+                if (typeof filter === 'function') {
+                  waiting.push(
+                    Promise.resolve(filter(ctx, action, meta)).then(result => {
+                      if (result && client && !ignoreClients.has(clientId)) {
+                        ignoreClients.add(clientId)
+                        addTarget(targets, client, action, meta)
+                      }
+                    })
+                  )
+                } else if (filter && client) {
+                  ignoreClients.add(clientId)
+                  addTarget(targets, client, action, meta)
                 }
               }
             }
@@ -867,6 +913,53 @@ export class BaseServer {
         }
       }
     }
+    return waiting.length > 0 ? Promise.all(waiting) : undefined
+  }
+
+  runProcessor(action, meta, processor, start) {
+    if (meta.status === 'waiting') {
+      if (!processor) {
+        this.internalUnknownType(action, meta)
+        return
+      }
+      if (processor.process) {
+        void this.processAction(processor, action, meta, start)
+      } else {
+        this.emitter.emit('processed', action, meta, 0)
+        this.finally(processor, this.createContext(action, meta), action, meta)
+        this.markAsProcessed(meta)
+      }
+    } else {
+      this.emitter.emit('processed', action, meta, 0)
+      this.finally(processor, this.createContext(action, meta), action, meta)
+    }
+  }
+
+  async sendAction(action, meta) {
+    let targets = new Map()
+    let waiting = this.resolveTargets(action, meta, targets)
+    if (waiting) await waiting
+    sendTargets(targets)
+  }
+
+  sendBatch(entries) {
+    let targets = new Map()
+    let waiting = []
+    for (let [action, meta] of entries) {
+      let promise = this.resolveTargets(action, meta, targets)
+      if (promise) waiting.push(promise)
+    }
+    if (waiting.length === 0) {
+      sendTargets(targets)
+      return undefined
+    }
+    return Promise.all(waiting).then(() => {
+      sendTargets(targets)
+    })
+  }
+
+  sendOnConnect(loader) {
+    this.connectLoader = loader
   }
 
   setTimeout(callback, ms) {
@@ -879,6 +972,7 @@ export class BaseServer {
   }
 
   subscribe(nodeId, channel) {
+    if (channel === '__proto__' || nodeId === '__proto__') return
     if (!this.subscribers[channel] || !this.subscribers[channel][nodeId]) {
       if (!this.subscribers[channel]) {
         this.subscribers[channel] = {}
@@ -889,7 +983,7 @@ export class BaseServer {
   }
 
   async subscribeAction(action, meta, start) {
-    if (typeof action.channel !== 'string') {
+    if (typeof action.channel !== 'string' || action.channel === '__proto__') {
       this.wrongChannel(action, meta)
       return
     }
@@ -902,7 +996,7 @@ export class BaseServer {
     let match
     for (let channel of channels) {
       if (channel.pattern) {
-        match = channel.pattern.match(action.channel)
+        match = channel.pattern(action.channel)
       } else {
         match = action.channel.match(channel.regexp)
       }
@@ -910,6 +1004,7 @@ export class BaseServer {
       let subscribed = false
       if (match) {
         let ctx = this.createContext(action, meta)
+        if (ctx.nodeId === '__proto__') return
         ctx.params = match
         try {
           let access = await channel.access(ctx, action, meta)
@@ -959,16 +1054,9 @@ export class BaseServer {
           subscribed = true
 
           if (channel.load) {
+            // Actions from a single `load()` are sent in a single message
             let sendBack = await channel.load(ctx, action, meta)
-            if (Array.isArray(sendBack)) {
-              await Promise.all(
-                sendBack.map(i => {
-                  return Array.isArray(i) ? ctx.sendBack(...i) : ctx.sendBack(i)
-                })
-              )
-            } else if (sendBack) {
-              await ctx.sendBack(sendBack)
-            }
+            if (sendBack) await ctx.sendBack(sendBack)
           }
           this.emitter.emit('subscribed', action, meta, Date.now() - start)
           this.markAsProcessed(meta)