@logux/server 0.14.0 → 0.16.0

package/base-server/index.js

@@ -5,17 +5,47 @@ import { nanoid } from 'nanoid'
 import { readFile } from 'node:fs/promises'
 import { dirname, join } from 'node:path'
 import { fileURLToPath } from 'node:url'
-import UrlPattern from 'url-pattern'
 import { WebSocketServer } from 'ws'
 
 import { addHttpPages } from '../add-http-pages/index.js'
 import { Context } from '../context/index.js'
 import { createHttpServer } from '../create-http-server/index.js'
 import { ServerClient } from '../server-client/index.js'
+import { createPattern } from '../url-pattern/index.js'
 
 const SKIP_PROCESS = Symbol('skipProcess')
 const RESEND_META = ['channels', 'users', 'clients', 'nodes']
 
+function addTarget(targets, client, action, meta) {
+  let entries = targets.get(client)
+  if (entries) {
+    entries.push([action, meta])
+  } else {
+    targets.set(client, [[action, meta]])
+  }
+}
+
+function sendTargets(app, targets, settled) {
+  for (let [client, entries] of targets) {
+    let sending = client.node.onAdd(entries)
+    // Only `ServerClient` tracks sending for `Server#drain()`
+    if (client.track) client.track(sending)
+  }
+  if (settled > 0) {
+    for (let client of app.connected.values()) {
+      if (!targets.has(client) && client.node.lastAddedCache < settled) {
+        client.node.lastAddedCache = settled
+      }
+    }
+  }
+  targets.clear()
+}
+
+function maxAdded(settled, meta) {
+  if (typeof meta.added === 'undefined' || meta.added < settled) return settled
+  return meta.added
+}
+
 function optionError(msg) {
   let error = new Error(msg)
   error.logux = true
@@ -110,7 +140,7 @@ export class BaseServer {
       if (!this.options.host) this.options.host = '127.0.0.1'
     }
 
-    this.nodeId = `server:${this.options.id || nanoid(8)}`
+    this.nodeId = `server:${this.options.id || nanoid(6)}`
 
     if (this.options.fileUrl) {
       this.options.root = dirname(fileURLToPath(this.options.fileUrl))
@@ -159,89 +189,64 @@ export class BaseServer {
       }
       this.replaceResendShortcuts(meta)
     })
-    this.on('add', async (action, meta) => {
-      let start = Date.now()
+    // Reports are per action, `clean` for a reason-less action is emitted
+    // by the log before `batch` event
+    this.on('add', (action, meta) => {
       if (meta.reasons.length === 0) {
         cleaned[meta.id] = true
         this.emitter.emit('report', 'addClean', { action, meta })
       } else {
         this.emitter.emit('report', 'add', { action, meta })
       }
+    })
+    this.on('batch', entries => {
+      let start = Date.now()
+      let ready = []
 
-      if (this.destroying && !this.actionToQueue.has(meta.id)) {
-        return
-      }
-
-      if (action.type === 'logux/subscribe') {
-        if (meta.server === this.nodeId) {
-          this.subscribeAction(action, meta, start)
+      for (let [action, meta] of entries) {
+        if (this.destroying && !this.actionToQueue.has(meta.id)) {
+          continue
         }
-        return
-      }
 
-      if (action.type === 'logux/unsubscribe') {
-        if (meta.server === this.nodeId) {
-          this.unsubscribeAction(action, meta)
+        if (action.type === 'logux/subscribe') {
+          if (meta.server === this.nodeId) {
+            void this.subscribeAction(action, meta, start)
+          }
+          continue
         }
-        return
-      }
 
-      let processor = this.getProcessor(action.type)
-      if (processor && processor.resend && meta.status === 'waiting') {
-        let ctx = this.createContext(action, meta)
-        let resend
-        try {
-          resend = await processor.resend(ctx, action, meta)
-        } catch (e) {
-          this.undo(action, meta, 'error')
-          this.emitter.emit('error', e, action, meta)
-          this.finally(processor, ctx, action, meta)
-          return
-        }
-        if (resend) {
-          if (typeof resend === 'string') {
-            resend = { channels: [resend] }
-          } else if (Array.isArray(resend)) {
-            resend = { channels: resend }
-          } else {
-            this.replaceResendShortcuts(resend)
-          }
-          let diff = {}
-          for (let i of RESEND_META) {
-            if (resend[i]) diff[i] = resend[i]
+        if (action.type === 'logux/unsubscribe') {
+          if (meta.server === this.nodeId) {
+            this.unsubscribeAction(action, meta)
           }
-          await this.log.changeMeta(meta.id, diff)
-          meta = { ...meta, ...diff }
+          continue
         }
-      }
-
-      if (this.isUseless(action, meta)) {
-        this.emitter.emit('report', 'useless', { action, meta })
-      }
 
-      await this.sendAction(action, meta)
+        let processor = this.getProcessor(action.type)
+        if (processor && processor.resend && meta.status === 'waiting') {
+          // Resending is asynchronous, so the action can’t wait for others
+          void this.resendAction(action, meta, processor, start)
+          continue
+        }
 
-      if (meta.status === 'waiting') {
-        if (!processor) {
-          this.internalUnknownType(action, meta)
-          return
+        if (this.isUseless(action, meta)) {
+          this.emitter.emit('report', 'useless', { action, meta })
         }
-        if (processor.process) {
-          this.processAction(processor, action, meta, start)
-        } else {
-          this.emitter.emit('processed', action, meta, 0)
-          this.finally(
-            processor,
-            this.createContext(action, meta),
-            action,
-            meta
-          )
-          this.markAsProcessed(meta)
+        ready.push([action, meta, processor])
+      }
+
+      if (ready.length === 0) return
+      let runProcessors = () => {
+        for (let [action, meta, processor] of ready) {
+          this.runProcessor(action, meta, processor, start)
         }
-      } else {
-        this.emitter.emit('processed', action, meta, 0)
-        this.finally(processor, this.createContext(action, meta), action, meta)
       }
+      // Actions are processed only after they were sent to the clients
+      void Promise.resolve(this.sendBatch(ready)).then(runProcessors, error => {
+        // Do not stop action processing on a broken channel filter
+        runProcessors()
+        throw error
+      })
     })
     this.on('clean', (action, meta) => {
       if (cleaned[meta.id]) {
@@ -445,9 +450,7 @@ export class BaseServer {
     normalizeChannelCallbacks(`Channel ${pattern}`, callbacks)
     let channel = Object.assign({}, callbacks)
     if (typeof pattern === 'string') {
-      channel.pattern = new UrlPattern(pattern, {
-        segmentValueCharset: '^/'
-      })
+      channel.pattern = createPattern(pattern)
     } else {
       channel.regexp = pattern
     }
@@ -496,6 +499,12 @@ export class BaseServer {
     return Promise.all(this.unbind.map(i => i()))
   }
 
+  drain(clientId) {
+    let client = this.clientIds.get(clientId)
+    if (!client) return Promise.resolve(false)
+    return client.drain()
+  }
+
   finally(processor, ctx, action, meta) {
     this.contexts.delete(action)
     if (processor && processor.finally) {
@@ -522,6 +531,11 @@ export class BaseServer {
     return undefined
   }
 
+  handleClient(ws, req) {
+    ws.upgradeReq = req
+    this.addClient(new ServerConnection(ws))
+  }
+
   http(method, url, listener) {
     if (this.options.disableHttpServer) {
       throw new Error(
@@ -624,10 +638,7 @@ export class BaseServer {
       await readFile(join(import.meta.dirname, '..', 'package.json'))
     )
 
-    this.ws.on('connection', (ws, req) => {
-      ws.upgradeReq = req
-      this.addClient(new ServerConnection(ws))
-    })
+    this.ws.on('connection', (ws, req) => this.handleClient(ws, req))
     this.emitter.emit('report', 'listen', {
       cert: !!this.options.cert,
       environment: this.env,
@@ -655,7 +666,12 @@ export class BaseServer {
   }
 
   on(event, listener) {
-    if (event === 'preadd' || event === 'add' || event === 'clean') {
+    if (
+      event === 'preadd' ||
+      event === 'add' ||
+      event === 'batch' ||
+      event === 'clean'
+    ) {
       return this.log.emitter.on(event, listener)
     } else {
       return this.emitter.on(event, listener)
@@ -668,11 +684,7 @@ export class BaseServer {
       throw new Error('Callbacks for unknown channel are already defined')
     }
     let channel = Object.assign({}, callbacks)
-    channel.pattern = {
-      match(name) {
-        return [name]
-      }
-    }
+    channel.pattern = name => [name]
     this.otherSubscriber = channel
   }
 
@@ -685,6 +697,7 @@ export class BaseServer {
   }
 
   performUnsubscribe(clientNodeId, action, meta) {
+    if (action.channel === '__proto__' || clientNodeId === '__proto__') return
     if (this.subscribers[action.channel]) {
       let subscriber = this.subscribers[action.channel][clientNodeId]
       if (subscriber) {
@@ -806,7 +819,42 @@ export class BaseServer {
     }
   }
 
-  async sendAction(action, meta) {
+  async resendAction(action, meta, processor, start) {
+    let ctx = this.createContext(action, meta)
+    let resend
+    try {
+      resend = await processor.resend(ctx, action, meta)
+    } catch (e) {
+      this.undo(action, meta, 'error')
+      this.emitter.emit('error', e, action, meta)
+      this.finally(processor, ctx, action, meta)
+      return
+    }
+    if (resend) {
+      if (typeof resend === 'string') {
+        resend = { channels: [resend] }
+      } else if (Array.isArray(resend)) {
+        resend = { channels: resend }
+      } else {
+        this.replaceResendShortcuts(resend)
+      }
+      let diff = {}
+      for (let i of RESEND_META) {
+        if (resend[i]) diff[i] = resend[i]
+      }
+      await this.log.changeMeta(meta.id, diff)
+      meta = { ...meta, ...diff }
+    }
+
+    if (this.isUseless(action, meta)) {
+      this.emitter.emit('report', 'useless', { action, meta })
+    }
+
+    await this.sendAction(action, meta)
+    this.runProcessor(action, meta, processor, start)
+  }
+
+  resolveTargets(action, meta, targets) {
     let from = parseId(meta.id).clientId
     let ignoreClients = new Set(meta.excludeClients || [])
     ignoreClients.add(from)
@@ -816,7 +864,7 @@ export class BaseServer {
         let client = this.nodeIds.get(id)
         if (client) {
           ignoreClients.add(client.clientId)
-          client.node.onAdd(action, meta)
+          addTarget(targets, client, action, meta)
         }
       }
     }
@@ -826,7 +874,7 @@ export class BaseServer {
         if (this.clientIds.has(id)) {
           let client = this.clientIds.get(id)
           ignoreClients.add(client.clientId)
-          client.node.onAdd(action, meta)
+          addTarget(targets, client, action, meta)
         }
       }
     }
@@ -838,32 +886,38 @@ export class BaseServer {
           for (let client of users) {
             if (!ignoreClients.has(client.clientId)) {
               ignoreClients.add(client.clientId)
-              client.node.onAdd(action, meta)
+              addTarget(targets, client, action, meta)
             }
           }
         }
       }
     }
 
-    if (meta.channels) {
-      for (let channel of meta.channels) {
-        if (this.subscribers[channel]) {
-          for (let nodeId in this.subscribers[channel]) {
-            let clientId = parseId(nodeId).clientId
-            if (!ignoreClients.has(clientId)) {
-              let subscriber = this.subscribers[channel][nodeId]
-              if (subscriber) {
-                let ctx = this.createContext(action, meta)
-                let client = this.clientIds.get(clientId)
-                for (let filter of Object.values(subscriber.filters)) {
-                  filter =
-                    typeof filter === 'function'
-                      ? await filter(ctx, action, meta)
-                      : filter
-                  if (filter && client) {
-                    ignoreClients.add(clientId)
-                    client.node.onAdd(action, meta)
-                  }
+    if (!meta.channels) return undefined
+
+    let waiting = []
+    for (let channel of meta.channels) {
+      if (this.subscribers[channel]) {
+        for (let nodeId in this.subscribers[channel]) {
+          let clientId = parseId(nodeId).clientId
+          if (!ignoreClients.has(clientId)) {
+            let subscriber = this.subscribers[channel][nodeId]
+            if (subscriber) {
+              let ctx = this.createContext(action, meta)
+              let client = this.clientIds.get(clientId)
+              for (let filter of Object.values(subscriber.filters)) {
+                if (typeof filter === 'function') {
+                  waiting.push(
+                    Promise.resolve(filter(ctx, action, meta)).then(result => {
+                      if (result && client && !ignoreClients.has(clientId)) {
+                        ignoreClients.add(clientId)
+                        addTarget(targets, client, action, meta)
+                      }
+                    })
+                  )
+                } else if (filter && client) {
+                  ignoreClients.add(clientId)
+                  addTarget(targets, client, action, meta)
                 }
               }
             }
@@ -871,6 +925,51 @@ export class BaseServer {
         }
       }
     }
+    return waiting.length > 0 ? Promise.all(waiting) : undefined
+  }
+
+  runProcessor(action, meta, processor, start) {
+    if (meta.status === 'waiting') {
+      if (!processor) {
+        this.internalUnknownType(action, meta)
+        return
+      }
+      if (processor.process) {
+        void this.processAction(processor, action, meta, start)
+      } else {
+        this.emitter.emit('processed', action, meta, 0)
+        this.finally(processor, this.createContext(action, meta), action, meta)
+        this.markAsProcessed(meta)
+      }
+    } else {
+      this.emitter.emit('processed', action, meta, 0)
+      this.finally(processor, this.createContext(action, meta), action, meta)
+    }
+  }
+
+  async sendAction(action, meta) {
+    let targets = new Map()
+    let waiting = this.resolveTargets(action, meta, targets)
+    if (waiting) await waiting
+    sendTargets(this, targets, maxAdded(0, meta))
+  }
+
+  sendBatch(entries) {
+    let targets = new Map()
+    let waiting = []
+    let settled = 0
+    for (let [action, meta] of entries) {
+      settled = maxAdded(settled, meta)
+      let promise = this.resolveTargets(action, meta, targets)
+      if (promise) waiting.push(promise)
+    }
+    if (waiting.length === 0) {
+      sendTargets(this, targets, settled)
+      return undefined
+    }
+    return Promise.all(waiting).then(() => {
+      sendTargets(this, targets, settled)
+    })
   }
 
   sendOnConnect(loader) {
@@ -887,6 +986,7 @@ export class BaseServer {
   }
 
   subscribe(nodeId, channel) {
+    if (channel === '__proto__' || nodeId === '__proto__') return
     if (!this.subscribers[channel] || !this.subscribers[channel][nodeId]) {
       if (!this.subscribers[channel]) {
         this.subscribers[channel] = {}
@@ -897,7 +997,7 @@ export class BaseServer {
   }
 
   async subscribeAction(action, meta, start) {
-    if (typeof action.channel !== 'string') {
+    if (typeof action.channel !== 'string' || action.channel === '__proto__') {
       this.wrongChannel(action, meta)
       return
     }
@@ -910,7 +1010,7 @@ export class BaseServer {
     let match
     for (let channel of channels) {
       if (channel.pattern) {
-        match = channel.pattern.match(action.channel)
+        match = channel.pattern(action.channel)
       } else {
         match = action.channel.match(channel.regexp)
       }
@@ -918,6 +1018,7 @@ export class BaseServer {
       let subscribed = false
       if (match) {
         let ctx = this.createContext(action, meta)
+        if (ctx.nodeId === '__proto__') return
         ctx.params = match
         try {
           let access = await channel.access(ctx, action, meta)
@@ -967,16 +1068,9 @@ export class BaseServer {
           subscribed = true
 
           if (channel.load) {
+            // Actions from a single `load()` are sent in a single message
             let sendBack = await channel.load(ctx, action, meta)
-            if (Array.isArray(sendBack)) {
-              await Promise.all(
-                sendBack.map(i => {
-                  return Array.isArray(i) ? ctx.sendBack(...i) : ctx.sendBack(i)
-                })
-              )
-            } else if (sendBack) {
-              await ctx.sendBack(sendBack)
-            }
+            if (sendBack) await ctx.sendBack(sendBack)
           }
           this.emitter.emit('subscribed', action, meta, Date.now() - start)
           this.markAsProcessed(meta)

package/context/index.d.ts

@@ -1,4 +1,4 @@
-import type { AnyAction } from '@logux/core'
+import type { Action, AnyAction } from '@logux/core'
 
 import type { ServerMeta } from '../base-server/index.js'
 import type { ServerClient } from '../server-client/index.js'
@@ -59,6 +59,24 @@ export class ConnectContext<Headers extends object = unknown> {
 
   constructor(server: Server, client: ServerClient)
 
+  /**
+   * Wait until the client will confirm all actions, which were sent to it.
+   *
+   * Use it to send a long history without loading it all into the memory:
+   * the client’s speed will limit how fast you read the database.
+   *
+   * ```js
+   * while (await ctx.drain()) {
+   *   let page = await cursor.next(100)
+   *   if (!page.length) break
+   *   ctx.sendBack(page.map(i => i.action))
+   * }
+   * ```
+   *
+   * @returns Promise with `false` if the client was disconnected.
+   */
+  drain(): Promise<boolean>
+
   /**
    * Send action back to the client.
    *
@@ -66,13 +84,27 @@ export class ConnectContext<Headers extends object = unknown> {
    * ctx.sendBack({ type: 'login/success', token })
    * ```
    *
+   * An array of actions will be sent in a single message. Use it to send
+   * a big history page by page instead of a message per action.
+   *
+   * ```js
+   * ctx.sendBack(page.map(i => i.action))
+   * ```
+   *
+   * Every action in the array can have own meta as `[action, meta]`.
+   *
    * Action will not be processed by server’s callbacks from `Server#type`.
    *
-   * @param action The action.
+   * @param action The action or the array of actions.
    * @param meta Action’s meta.
    * @returns Promise until action was added to the server log.
    */
-  sendBack(action: AnyAction, meta?: Partial<ServerMeta>): Promise<void>
+  sendBack<TypeAction extends Action = AnyAction>(
+    action:
+      | (TypeAction | [TypeAction, Partial<ServerMeta>])[]
+      | TypeAction,
+    meta?: Partial<ServerMeta>
+  ): Promise<void>
 }
 
 /**

package/context/index.js

@@ -33,11 +33,22 @@ export class Context {
     }
   }
 
-  sendBack(action, meta = {}) {
-    return this.server.log.add(action, {
-      clients: [this.clientId],
-      status: 'processed',
-      ...meta
-    })
+  drain() {
+    return this.server.drain(this.clientId)
+  }
+
+  sendBack(actions, meta = {}) {
+    let common = { clients: [this.clientId], status: 'processed', ...meta }
+    if (Array.isArray(actions)) {
+      return this.server.log.add(
+        actions.map(item => {
+          // Every action can have own meta as `[action, meta]`
+          return Array.isArray(item)
+            ? [item[0], { ...common, ...item[1] }]
+            : [item, { ...common }]
+        })
+      )
+    }
+    return this.server.log.add(actions, common)
   }
 }

package/create-reporter/index.js

@@ -2,13 +2,13 @@ import os from 'node:os'
 import { sep } from 'node:path'
 
 import humanFormatter from '../human-formatter/index.js'
+import { BINARY, isBinary } from '../human-formatter/utils.js'
 
 const ERROR_CODES = {
   EACCES: (e, environment) => {
     let wayToFix = {
-      development:
-        'In dev mode it can be done with sudo:\n' + '$ sudo npm start',
-      production: '$ su - `<username>`\n' + `$ npm start -p ${e.port}`
+      development: 'In dev mode it can be done with sudo:\n$ sudo npm start',
+      production: `$ su - \`<username>\`\n$ npm start -p ${e.port}`
     }
 
     return {
@@ -22,7 +22,7 @@ const ERROR_CODES = {
   EADDRINUSE: e => {
     let port = String(e.port)
     let wayToFix = {
-      darwin: `$ sudo lsof -i ${e.port}\n` + '$ sudo kill -9 `<processid>`',
+      darwin: `$ sudo lsof -i ${e.port}\n$ sudo kill -9 \`<processid>\``,
       linux:
         '$ su - root\n' +
         `# netstat -nlp | grep ${e.port}\n` +
@@ -176,23 +176,46 @@ const REPORTERS = {
 function cleanFromKeys(obj, regexp, seen) {
   let result = {}
   for (let key in obj) {
-    let v = obj[key]
-    if (typeof v === 'string') {
-      result[key] = v.replace(regexp, '[SECRET]')
-    } else if (typeof v === 'object' && !Array.isArray(v) && v !== null) {
-      if (seen.includes(v)) {
-        throw new Error('Circular reference in action')
-      }
-      seen.push(v)
-      result[key] = cleanFromKeys(v, regexp, seen)
-      seen.pop()
-    } else {
-      result[key] = v
-    }
+    result[key] = cleanValue(obj[key], regexp, seen)
   }
   return result
 }
 
+function cleanValue(value, regexp, seen) {
+  if (typeof value === 'string') {
+    return value.replace(regexp, '[SECRET]')
+  } else if (isBinary(value)) {
+    return BINARY
+  } else if (typeof value === 'object' && value !== null) {
+    if (seen.includes(value)) {
+      throw new Error('Circular reference in action')
+    }
+    seen.push(value)
+    let cleaned = Array.isArray(value)
+      ? value.map(i => cleanValue(i, regexp, seen))
+      : cleanFromKeys(value, regexp, seen)
+    seen.pop()
+    return cleaned
+  } else {
+    return value
+  }
+}
+
+function createRecord(level, details, msg) {
+  /* c8 ignore next 4 */
+  if (typeof details === 'string') {
+    msg = details
+    details = {}
+  }
+  return {
+    level,
+    time: new Date().toISOString(),
+    pid: process.pid,
+    ...details,
+    msg
+  }
+}
+
 export function createReporter(options) {
   let cleanFromLog = options.cleanFromLog || /Bearer [^\s"]+/g
   function reporter(type, details) {
@@ -218,21 +241,6 @@ export function createReporter(options) {
     }
     let stream = options.logger?.stream ?? process.stderr
 
-    function createRecord(level, details, msg) {
-      /* c8 ignore next 4 */
-      if (typeof details === 'string') {
-        msg = details
-        details = {}
-      }
-      return {
-        level,
-        time: new Date().toISOString(),
-        pid: process.pid,
-        ...details,
-        msg
-      }
-    }
-
     reporter.logger = {
       /* c8 ignore next 3 */
       debug(details, msg) {