@nxtedition/lib 15.0.45 → 15.0.47

package/couch.js

@@ -1,7 +1,9 @@
+const stream = require('node:stream')
+const tp = require('node:timers/promises')
 const createError = require('http-errors')
 const makeWeak = require('./weakCache')
-const tp = require('timers/promises')
 const { delay } = require('./http')
+const split2 = require('split2')
 
 // https://github.com/fastify/fastify/blob/main/lib/reqIdGenFactory.js
 // 2,147,483,647 (2^31 − 1) stands for max SMI value (an internal optimization of V8).
@@ -62,7 +64,7 @@ module.exports = function (opts) {
   }
 
   const { origin: dbOrigin, pathname: dbPathname } = new URL(
-    Array.isArray(config.url) ? config.url[0] : config.url
+    Array.isArray(config.url) ? config.url[0] : config.url,
   )
 
   const defaultClientOpts = {
@@ -83,7 +85,7 @@ module.exports = function (opts) {
           ...defaultClientOpts,
           connections: 4, // TODO (fix): Global limit?
           pipelining: 2,
-        })
+        }),
     )
 
   function makeError(req, res) {
@@ -131,7 +133,7 @@ module.exports = function (opts) {
     })
   }
 
-  async function* changes({ client = defaultClient, signal, ...options } = {}) {
+  async function* changes({ client = defaultClient, signal, highWaterMark, ...options } = {}) {
     const params = {}
 
     let body
@@ -242,42 +244,59 @@ module.exports = function (opts) {
           ...(body ? { 'content-type': 'application/json' } : {}),
         },
         throwOnError: true,
-        highWaterMark: 128 * 1024, // TODO (fix): Needs support in undici...
+        highWaterMark: highWaterMark ?? 256 * 1024,
         bodyTimeout: 2 * (params.heartbeat || 60e3),
       }
 
-      try {
-        const res = await client.request(req)
+      const res = await client.request(req)
 
-        retryCount = 0
+      retryCount = 0
 
-        let str = ''
-        for await (const chunk of res.body) {
-          const lines = (str + chunk).split('\n')
-          str = lines.pop() ?? ''
-
-          const results = batched ? [] : null
-          for (const line of lines) {
-            if (line) {
-              const change = JSON.parse(line)
-              if (change.seq) {
-                params.since = change.seq
-              }
-              if (results) {
-                results.push(change)
-              } else {
-                yield change
-              }
-            }
-          }
+      const src = stream.pipeline(res.body, split2(), () => {})
+
+      let error
+      let ended = false
+      let resume = () => {}
+
+      src
+        .on('error', (err) => {
+          error = err
+        })
+        .on('readable', () => {
+          resume()
+        })
+        .on('end', () => {
+          ended = true
+          resume()
+        })
 
-          if (results?.length) {
-            yield results
+      const batch = batched ? [] : null
+      while (true) {
+        const line = src.read()
+
+        if (line === '') {
+          continue
+        } else if (line !== null) {
+          const change = JSON.parse(line)
+          if (change.seq) {
+            params.since = change.seq
+          }
+          if (batch) {
+            batch.push(change)
+          } else {
+            yield change
           }
+        } else if (batch?.length) {
+          yield batch.splice(0)
+        } else if (error) {
+          throw Object.assign(error, { data: req })
+        } else if (ended) {
+          return
+        } else {
+          await new Promise((resolve) => {
+            resume = resolve
+          })
         }
-      } catch (err) {
-        Object.assign(err, { data: req })
-        throw err
       }
     }
 
@@ -416,7 +435,7 @@ module.exports = function (opts) {
 
   function _request(
     url,
-    { params, client = defaultClient, idempotent, body, method, headers, signal }
+    { params, client = defaultClient, idempotent, body, method, headers, signal },
   ) {
     if (Array.isArray(headers)) {
       // Do nothing...
@@ -445,7 +464,7 @@ module.exports = function (opts) {
         url
           .split('/')
           .map((part) => encodeURIComponent(part))
-          .join('/')
+          .join('/'),
       )
     }
 
@@ -515,7 +534,7 @@ module.exports = function (opts) {
                 status: this.status,
                 headers: this.headers,
                 data: this.data,
-              })
+              }),
             )
           } else {
             this.resolve({
@@ -536,7 +555,7 @@ module.exports = function (opts) {
 
           this.reject(err)
         },
-      })
+      }),
     )
   }
 
@@ -721,7 +740,7 @@ module.exports = function (opts) {
     pathname,
     params,
     body,
-    { client = getClient('_all_docs'), signal, idempotent = true, headers } = {}
+    { client = getClient('_all_docs'), signal, idempotent = true, headers } = {},
   ) {
     const req = {
       pathname,
@@ -746,7 +765,7 @@ module.exports = function (opts) {
     pathname,
     params,
     body,
-    { client, signal, idempotent = true, headers } = {}
+    { client, signal, idempotent = true, headers } = {},
   ) {
     const req = {
       pathname,

package/deepstream.js

@@ -1,5 +1,9 @@
 const qs = require('qs')
 const cached = require('./util/cached')
+const undici = require('undici')
+const stream = require('node:stream')
+const split2 = require('split2')
+const { delay } = require('./http')
 
 function provide(ds, domain, callback, options) {
   if (domain instanceof RegExp) {
@@ -23,7 +27,7 @@ function provide(ds, domain, callback, options) {
     callback = cached(
       callback,
       cachedOptions,
-      cachedOptions.keySelector ? cachedOptions.keySelector : (id, options, key) => key
+      cachedOptions.keySelector ? cachedOptions.keySelector : (id, options, key) => key,
     )
   } else if (options.minAge) {
     // Backwards compat
@@ -43,13 +47,13 @@ function provide(ds, domain, callback, options) {
       const [id, options] = parseKey(key)
       return callback(id, options, key)
     },
-    { recursive: options.recursive, mode: options.mode, stringify: options.stringify }
+    { recursive: options.recursive, mode: options.mode, stringify: options.stringify },
   )
 }
 
 function parseKey(key) {
   const { json, id, query } = key.match(
-    /^(?:(?<json>\{.*\}):|(?<id>.*):)?[^?]*(?:\?(?<query>.*))?$/
+    /^(?:(?<json>\{.*\}):|(?<id>.*):)?[^?]*(?:\?(?<query>.*))?$/,
   ).groups
   return [
     id || '',
@@ -75,7 +79,7 @@ function observe(ds, name, ...args) {
         ? `${name.endsWith('?') ? '' : '?'}${qs.stringify(query, { skipNulls: true })}`
         : ''
     }`,
-    ...args
+    ...args,
   )
 }
 
@@ -94,7 +98,7 @@ function observe2(ds, name, ...args) {
         ? `${name.endsWith('?') ? '' : '?'}${qs.stringify(query, { skipNulls: true })}`
         : ''
     }`,
-    ...args
+    ...args,
   )
 }
 
@@ -113,7 +117,7 @@ function get(ds, name, ...args) {
         ? `${name.endsWith('?') ? '' : '?'}${qs.stringify(query, { skipNulls: true })}`
         : ''
     }`,
-    ...args
+    ...args,
   )
 }
 
@@ -127,18 +131,132 @@ function init(ds) {
       set: (...args) => ds.record.set(...args),
       get: (...args) => get(ds, ...args),
       update: (...args) => ds.record.update(...args),
+      changes: (...args) => changes(ds, ...args),
     },
   }
   ds.nxt = nxt
   return nxt
 }
 
+async function* changes(
+  ds,
+  {
+    since = 'now',
+    live = true,
+    batched = false,
+    includeDocs = false,
+    highWaterMark = 256 * 1024,
+    heartbeat = 60e3,
+    signal,
+    retry,
+  },
+) {
+  const url = new URL('/_record/changes', ds._url)
+
+  url.protocol = url.protocol === 'ws:' ? 'http:' : 'https:'
+  url.port = '6100'
+  url.searchParams.set('since', since || '0')
+  url.searchParams.set('live', String(live))
+  url.searchParams.set('include_docs', String(includeDocs))
+
+  let ac
+
+  const abort = () => {
+    ac?.abort(signal.reason)
+  }
+
+  signal?.addEventListener('abort', abort)
+
+  try {
+    for (let retryCount = 0; retryCount < retry; retryCount++) {
+      ac = new AbortController()
+      try {
+        // TODO (fix): Use nxt-undici
+        const res = await undici.request(url, {
+          idempotent: false,
+          blocking: true,
+          method: 'GET',
+          signal: ac.signal,
+          throwOnError: true,
+          highWaterMark,
+          bodyTimeout: 2 * heartbeat,
+        })
+
+        const src = stream.pipeline(res.body, split2(), () => {})
+
+        let error
+        let ended = false
+        let resume = () => {}
+
+        src
+          .on('error', (err) => {
+            error = err
+          })
+          .on('readable', () => {
+            resume()
+          })
+          .on('end', () => {
+            ended = true
+            resume()
+          })
+
+        const batch = batched ? [] : null
+        while (true) {
+          const line = src.read()
+
+          if (line === '') {
+            continue
+          } else if (line !== null) {
+            const change = JSON.parse(line)
+
+            retryCount = 0
+
+            if (change.seq) {
+              since = change.seq
+            }
+            if (batch) {
+              batch.push(change)
+            } else {
+              yield change
+            }
+          } else if (batch?.length) {
+            yield batch.splice(0)
+          } else if (error) {
+            throw error
+          } else if (ended) {
+            return
+          } else {
+            await new Promise((resolve) => {
+              resume = resolve
+            })
+          }
+        }
+      } catch (err) {
+        if (typeof retry === 'function') {
+          const retryState = { since }
+          await retry(err, retryCount, retryState, { signal: ac.signal })
+          url.searchParams.set('since', since || '0')
+        } else {
+          await delay(err, retryCount, { signal: ac.signal })
+        }
+      } finally {
+        ac.abort()
+        ac = null
+      }
+    }
+  } finally {
+    signal?.removeEventListener('abort', abort)
+  }
+}
+
 module.exports = Object.assign(init, {
+  changes,
   provide,
   observe,
   observe2,
   get,
   record: {
+    changes,
     provide,
     observe,
     observe2,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/lib",
-  "version": "15.0.45",
+  "version": "15.0.47",
   "license": "MIT",
   "author": "Robert Nagy <robert.nagy@boffins.se>",
   "files": [
@@ -89,6 +89,7 @@
     "request-target": "^1.0.2",
     "smpte-timecode": "^1.3.3",
     "split-string": "^6.0.0",
+    "split2": "^4.2.0",
     "toobusy-js": "^0.5.1",
     "undici": "^5.25.4",
     "url-join": "^4.0.0"