@nxtedition/lib 19.8.13 → 19.8.14

package/couch.js

@@ -191,12 +191,6 @@ export function makeCouch(opts) {
       throw new Error('invalid pathname')
     }
 
-    const batched = options.batched || false
-    const live = options.live == null || !!options.live
-    const retry = options.retry
-
-    let retryCount = 0
-
     const ac = new AbortController()
     const onAbort = () => {
       ac.abort()
@@ -214,169 +208,204 @@ export function makeCouch(opts) {
       }
     }
 
-    let remaining = Number(options.limit) || Infinity
+    const batched = options.batched || false
+    const live = options.live == null || !!options.live
+    const retry = options.retry
+    const limit = options.limit
+
     try {
-      while (true) {
-        let src
-        try {
-          const query = { ...params, feed: live ? 'continuous' : 'normal' }
+      // Don't stay connected to the same socket to allow server to load balance.
+      const socketCount = 128 * 1024
+      let remaining = Number(limit) || Infinity
+      while (remaining > 0) {
+        let count = 0
+        for await (const changes of _changes({
+          batched,
+          live,
+          retry,
+          limit: Math.min(remaining, socketCount),
+          params,
+          method,
+          body,
+          signal: ac.signal,
+          client,
+        })) {
+          yield changes
+          count += changes.length
+          remaining -= changes.length
+        }
+        assert(remaining >= 0)
+        if (!live && count < socketCount) {
+          return
+        }
+      }
+    } finally {
+      ac.abort()
+      if (signal) {
+        if (signal.off) {
+          signal.off('abort', onAbort)
+        } else if (signal.removeEventListener) {
+          signal.removeEventListener('abort', onAbort)
+        }
+      }
+    }
+  }
 
-          if (Number.isFinite(remaining)) {
-            query.limit = remaining
-          }
+  async function* _changes({ batched, live, retry, limit, params, method, body, signal, client }) {
+    let retryCount = 0
+    let remaining = Number(limit) || Infinity
+    while (true) {
+      let src
+      try {
+        const query = { ...params, feed: live ? 'continuous' : 'normal' }
 
-          const ureq = {
-            path: `${dbPathname}/_changes`,
-            query,
-            idempotent: false,
-            blocking: true,
-            method,
-            body: JSON.stringify(body),
-            signal: ac.signal,
-            headers: {
-              'user-agent': userAgent,
-              'request-id': genReqId(),
-              ...(body ? { 'content-type': 'application/json' } : {}),
-            },
-            bodyTimeout: 2 * (params.heartbeat || 60e3),
-          }
+        if (Number.isFinite(remaining)) {
+          query.limit = remaining
+        }
 
-          const ures = await client.request(ureq)
+        const ureq = {
+          path: `${dbPathname}/_changes`,
+          query,
+          idempotent: false,
+          blocking: true,
+          method,
+          body: JSON.stringify(body),
+          signal,
+          headers: {
+            'user-agent': userAgent,
+            'request-id': genReqId(),
+            ...(body ? { 'content-type': 'application/json' } : {}),
+          },
+          bodyTimeout: 2 * (params.heartbeat || 60e3),
+          highWaterMark: 256 * 1024,
+        }
 
-          if (ures.statusCode < 200 || ures.statusCode >= 300) {
-            throw makeError(ureq, {
-              status: ures.statusCode,
-              headers: ures.headers,
-              data: await ures.body.text(),
-            })
-          }
+        const ures = await client.request(ureq)
+
+        if (ures.statusCode < 200 || ures.statusCode >= 300) {
+          throw makeError(ureq, {
+            status: ures.statusCode,
+            headers: ures.headers,
+            data: await ures.body.text(),
+          })
+        }
 
-          src = ures.body
+        src = ures.body
 
-          const changes = []
+        const changes = []
 
-          let resume = null
-          let error = null
-          let ended = false
-          let state = 0
+        let resume = null
+        let error = null
+        let ended = false
+        let state = 0
 
-          function maybeResume() {
-            if (resume) {
-              resume()
-              resume = null
-            }
+        function maybeResume() {
+          if (resume) {
+            resume()
+            resume = null
           }
+        }
 
-          src
-            .on('readable', maybeResume)
-            .on('error', (err) => {
-              error = err
-              maybeResume()
-            })
-            .on('end', () => {
-              ended = true
-              maybeResume()
-            })
-            .on('close', maybeResume)
-
-          let str = ''
-          while (true) {
-            const chunk = src.read()
-
-            if (chunk !== null) {
-              const lines = (str + chunk).split(/\r?\n/)
-              str = lines.pop() ?? ''
-
-              for (const line of lines) {
-                if (line === '') {
-                  // hearbeat
-                  yield batched ? [] : null
-                } else if (line === ',') {
-                  // Do nothing. Couch sometimes insert new line between
-                  // json body and comma.
-                } else if (live) {
-                  const data = JSON.parse(line)
-                  if (data.last_seq) {
-                    params.since = data.last_seq
+        src
+          .on('readable', maybeResume)
+          .on('error', (err) => {
+            error = err
+            maybeResume()
+          })
+          .on('end', () => {
+            ended = true
+            maybeResume()
+          })
+          .on('close', maybeResume)
+
+        let str = ''
+        while (true) {
+          const chunk = src.read()
+
+          if (chunk !== null) {
+            const lines = (str + chunk).split(/\r?\n/)
+            str = lines.pop() ?? ''
+
+            for (const line of lines) {
+              if (line === '') {
+                // hearbeat
+                yield batched ? [] : null
+              } else if (line === ',') {
+                // Do nothing. Couch sometimes insert new line between
+                // json body and comma.
+              } else if (live) {
+                const data = JSON.parse(line)
+                if (data.last_seq) {
+                  params.since = data.last_seq
+                } else {
+                  params.since = data.seq || params.since
+                  changes.push(data)
+                }
+              } else {
+                // NOTE: This makes some assumptions about the format of the JSON.
+                if (state === 0) {
+                  if (line.endsWith('[')) {
+                    state = 1
                   } else {
-                    params.since = data.seq || params.since
-                    changes.push(data)
+                    assert(false, 'invalid head: ' + line)
                   }
-                } else {
-                  // NOTE: This makes some assumptions about the format of the JSON.
-                  if (state === 0) {
-                    if (line.endsWith('[')) {
-                      state = 1
-                    } else {
-                      assert(false, 'invalid head: ' + line)
-                    }
-                  } else if (state === 1) {
-                    if (line.startsWith(']')) {
-                      state = 2
-                    } else {
-                      const idx = line.lastIndexOf('}') + 1
-                      try {
-                        assert(idx >= 0, 'invalid row: ' + idx + ' ' + line)
-                        const change = JSON.parse(line.slice(0, idx))
-                        params.since = change.seq || params.since
-                        changes.push(change)
-                      } catch (err) {
-                        throw Object.assign(err, { data: line })
-                      }
+                } else if (state === 1) {
+                  if (line.startsWith(']')) {
+                    state = 2
+                  } else {
+                    const idx = line.lastIndexOf('}') + 1
+                    try {
+                      assert(idx >= 0, 'invalid row: ' + idx + ' ' + line)
+                      const change = JSON.parse(line.slice(0, idx))
+                      params.since = change.seq || params.since
+                      changes.push(change)
+                    } catch (err) {
+                      throw Object.assign(err, { data: line })
                     }
-                  } else if (state === 2) {
-                    state = 3
-                    params.since = JSON.parse('{' + line).last_seq
-                    assert(params.since, 'invalid trailer: ' + line)
                   }
+                } else if (state === 2) {
+                  state = 3
+                  params.since = JSON.parse('{' + line).last_seq
+                  assert(params.since, 'invalid trailer: ' + line)
                 }
               }
-            } else if (changes.length) {
-              remaining -= changes.length
-              assert(remaining >= 0, 'invalid remaining: ' + remaining)
+            }
+          } else if (changes.length) {
+            remaining -= changes.length
+            assert(remaining >= 0, 'invalid remaining: ' + remaining)
 
-              if (batched) {
-                yield changes.splice(0)
-              } else {
-                yield* changes.splice(0)
-              }
-            } else if (error) {
-              throw error
-            } else if (!ended) {
-              await new Promise((resolve) => {
-                resume = resolve
-              })
+            if (batched) {
+              yield changes.splice(0)
             } else {
-              return
+              yield* changes.splice(0)
             }
-          }
-        } catch (err) {
-          if (err.name === 'AbortError') {
-            throw err
-          } else if (typeof retry === 'function') {
-            const retryState = { since: params.since }
-            Object.assign(
-              retryState,
-              await retry(err, retryCount++, retryState, { signal }, () =>
-                delay(err, retryCount, { signal }),
-              ),
-            )
-            params.since = retryState.since ?? 0
+          } else if (error) {
+            throw error
+          } else if (!ended) {
+            await new Promise((resolve) => {
+              resume = resolve
+            })
           } else {
-            await delay(err, retryCount, { signal })
+            return
           }
-        } finally {
-          src?.on('error', () => {}).destroy()
         }
-      }
-    } finally {
-      ac.abort()
-      if (signal) {
-        if (signal.off) {
-          signal.off('abort', onAbort)
-        } else if (signal.removeEventListener) {
-          signal.removeEventListener('abort', onAbort)
+      } catch (err) {
+        if (err.name === 'AbortError') {
+          throw err
+        } else if (typeof retry === 'function') {
+          const retryState = { since: params.since }
+          Object.assign(
+            retryState,
+            await retry(err, retryCount++, retryState, { signal }, () =>
+              delay(err, retryCount, { signal }),
+            ),
+          )
+          params.since = retryState.since ?? 0
+        } else {
+          await delay(err, retryCount, { signal })
         }
+      } finally {
+        src?.on('error', () => {}).destroy()
       }
     }
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/lib",
-  "version": "19.8.13",
+  "version": "19.8.14",
   "license": "MIT",
   "author": "Robert Nagy <robert.nagy@boffins.se>",
   "type": "module",