@steadybrew/postgresjs 4.3.0 → 4.4.1

package/cf/src/subscribe.js

@@ -1,5 +1,10 @@
 import { Buffer } from 'node:buffer'
+import { callback } from './errors.js'
+import { defaultBackoff, backoffSeconds } from './connection.js'
+
 const noop = () => { /* noop */ }
+    , commands = { 68: 'delete', 73: 'insert', 85: 'update' }
+    , statusInterval = 1000
 
 export default function Subscribe(postgres, options) {
   const subscribers = new Map()
@@ -8,6 +13,9 @@ export default function Subscribe(postgres, options) {
 
   let connection
     , stream
+    , timer
+    , inflight = false
+    , retries = 0
     , ended = false
 
   const sql = subscribe.sql = postgres({
@@ -22,13 +30,12 @@ export default function Subscribe(postgres, options) {
       ...options.connection,
       replication: 'database'
     },
-    onclose: async function() {
+    onclose: function() {
       if (ended)
         return
       stream = null
       state.pid = state.secret = undefined
-      connected(await init(sql, slot, options.publications))
-      subscribers.forEach(event => event.forEach(({ onsubscribe }) => onsubscribe()))
+      inflight || timer || !connection || connect().catch(noop)
     },
     no_subscribe: true
   })
@@ -38,6 +45,9 @@ export default function Subscribe(postgres, options) {
 
   sql.end = async() => {
     ended = true
+    clearTimeout(timer)
+    timer = null
+    connection && await connection.catch(noop)
     stream && (await new Promise(r => (stream.once('close', r), stream.end())))
     return end()
   }
@@ -49,34 +59,74 @@ export default function Subscribe(postgres, options) {
 
   return subscribe
 
-  async function subscribe(event, fn, onsubscribe = noop, onerror = noop) {
+  async function subscribe(event, fn, onsubscribe = noop, onerror) {
     event = parseEvent(event)
 
-    if (!connection)
-      connection = init(sql, slot, options.publications)
+    const subscriber = { fn, onsubscribe, onerror }
 
-    const subscriber = { fn, onsubscribe }
-    const fns = subscribers.has(event)
-      ? subscribers.get(event).add(subscriber)
-      : subscribers.set(event, new Set([subscriber])).get(event)
+    return (connection || connect()).then(() => {
+      const fns = subscribers.has(event)
+        ? subscribers.get(event).add(subscriber)
+        : subscribers.set(event, new Set([subscriber])).get(event)
 
-    const unsubscribe = () => {
-      fns.delete(subscriber)
-      fns.size === 0 && subscribers.delete(event)
-    }
+      const unsubscribe = () => {
+        fns.delete(subscriber)
+        fns.size === 0 && subscribers.delete(event)
+        subscribers.size === 0 && (clearTimeout(timer), timer = null)
+      }
 
-    return connection.then(x => {
-      connected(x)
-      onsubscribe()
-      stream && stream.on('error', onerror)
+      callback(onsubscribe)
       return { unsubscribe, state, sql }
     })
   }
 
-  function connected(x) {
-    stream = x.stream
-    state.pid = x.state.pid
-    state.secret = x.state.secret
+  function connect() {
+    clearTimeout(timer)
+    timer = null
+    inflight = true
+    const resume = subscribers.size > 0
+    const current = connection = init(sql, slot, options.publications).then(x => {
+      stream = x.stream
+      state.pid = x.state.pid
+      state.secret = x.state.secret
+    })
+
+    current.then(
+      () => {
+        inflight = false
+        retries = 0
+        resume && !ended && subscribers.forEach(event => event.forEach(({ onsubscribe }) => callback(onsubscribe)))
+      },
+      error => {
+        inflight = false
+        connection === current && (connection = null)
+        if (ended || subscribers.size === 0)
+          return
+        reportConnection(error)
+        timer = setTimeout(() => connect().catch(noop), retryDelay(retries++))
+      }
+    )
+
+    return current
+  }
+
+  function retryDelay(n) {
+    let seconds = backoffSeconds(sql.options.backoff, n)
+    seconds = Number.isFinite(seconds) ? seconds : defaultBackoff(n)
+    return Math.min(Math.max(seconds * 1000, 100), 2 ** 31 - 1)
+  }
+
+  function reportConnection(error) {
+    console.error('Unexpected error during logical streaming - reconnecting', error) // eslint-disable-line
+    subscribers.forEach(fns => fns.forEach(({ onerror }) => callback(onerror, error)))
+  }
+
+  function report(error, match) {
+    let uncaught = false
+    subscribers.forEach((fns, event) => match(event) && fns.forEach(({ onerror }) =>
+      onerror ? callback(onerror, error) : uncaught = true
+    ))
+    uncaught && callback(() => { throw error })
   }
 
   async function init(sql, slot, publications) {
@@ -100,18 +150,22 @@ export default function Subscribe(postgres, options) {
     }
 
     stream.on('data', data)
-    stream.on('error', error)
+    stream.on('error', reportConnection)
     stream.on('close', sql.close)
 
-    return { stream, state: xs.state }
+    // A walsender can miss the wakeup for a commit and sleep until its timeout; a status update wakes it.
+    const status = setInterval(() => stream.writable && pong(), statusInterval)
+    stream.on('close', () => clearInterval(status))
 
-    function error(e) {
-      console.error('Unexpected error during logical streaming - reconnecting', e) // eslint-disable-line
-    }
+    return { stream, state: xs.state }
 
     function data(x) {
       if (x[0] === 0x77) {
-        parse(x.subarray(25), state, sql.options.parsers, handle, options.transform)
+        try {
+          parse(x.subarray(25), state, sql.options.parsers, handle, options.transform)
+        } catch (error) {
+          skip(x.subarray(25), error)
+        }
       } else if (x[0] === 0x6b && x[17]) {
         state.lsn = x.subarray(1, 9)
         pong()
@@ -120,12 +174,26 @@ export default function Subscribe(postgres, options) {
 
     function handle(a, b) {
       const path = b.relation.schema + '.' + b.relation.table
+          , key = b.relation.keys.length && '=' + b.relation.keys.map(x => a[x.name])
       call('*', a, b)
       call('*:' + path, a, b)
-      b.relation.keys.length && call('*:' + path + '=' + b.relation.keys.map(x => a[x.name]), a, b)
+      key && call('*:' + path + key, a, b)
       call(b.command, a, b)
       call(b.command + ':' + path, a, b)
-      b.relation.keys.length && call(b.command + ':' + path + '=' + b.relation.keys.map(x => a[x.name]), a, b)
+      key && call(b.command + ':' + path + key, a, b)
+    }
+
+    function skip(x, error) {
+      const command = commands[x[0]]
+          , relation = command && state[x.readUInt32BE(1)]
+
+      if (!relation)
+        throw error
+
+      const path = relation.schema + '.' + relation.table
+          , events = ['*', '*:' + path, command, command + ':' + path]
+
+      report(error, event => events.includes(event.split('=')[0]))
     }
 
     function pong() {
@@ -138,7 +206,7 @@ export default function Subscribe(postgres, options) {
   }
 
   function call(x, a, b) {
-    subscribers.has(x) && subscribers.get(x).forEach(({ fn }) => fn(a, b, x))
+    subscribers.has(x) && subscribers.get(x).forEach(({ fn }) => callback(fn, a, b, x))
   }
 }
 
@@ -163,19 +231,24 @@ function parse(x, state, parsers, handle, transform) {
       let columnIndex = 0
         , column
 
-      while (i < x.length) {
-        column = r.columns[columnIndex++] = {
-          key: x[i++],
-          name: transform.column.from
-            ? transform.column.from(x.toString('utf8', i, i = x.indexOf(0, i)))
-            : x.toString('utf8', i, i = x.indexOf(0, i)),
-          type: x.readUInt32BE(i += 1),
-          parser: parsers[x.readUInt32BE(i)],
-          atttypmod: x.readUInt32BE(i += 4)
+      try {
+        while (i < x.length) {
+          column = r.columns[columnIndex++] = {
+            key: x[i++],
+            name: transform.column.from
+              ? transform.column.from(x.toString('utf8', i, i = x.indexOf(0, i)))
+              : x.toString('utf8', i, i = x.indexOf(0, i)),
+            type: x.readUInt32BE(i += 1),
+            parser: parsers[x.readUInt32BE(i)],
+            atttypmod: x.readUInt32BE(i += 4)
+          }
+
+          column.key && r.keys.push(column)
+          i += 4
         }
-
-        column.key && r.keys.push(column)
-        i += 4
+      } catch (error) {
+        r.error = error
+        r.failed = true
       }
     },
     Y: () => { /* noop */ }, // Type
@@ -186,7 +259,7 @@ function parse(x, state, parsers, handle, transform) {
     },
     I: x => { // Insert
       let i = 1
-      const relation = state[x.readUInt32BE(i)]
+      const relation = decodable(state[x.readUInt32BE(i)])
       const { row } = tuples(x, relation.columns, i += 7, transform)
 
       handle(row, {
@@ -196,7 +269,7 @@ function parse(x, state, parsers, handle, transform) {
     },
     D: x => { // Delete
       let i = 1
-      const relation = state[x.readUInt32BE(i)]
+      const relation = decodable(state[x.readUInt32BE(i)])
       i += 4
       const key = x[i] === 75
       handle(key || x[i] === 79
@@ -210,7 +283,7 @@ function parse(x, state, parsers, handle, transform) {
     },
     U: x => { // Update
       let i = 1
-      const relation = state[x.readUInt32BE(i)]
+      const relation = decodable(state[x.readUInt32BE(i)])
       i += 4
       const key = x[i] === 75
       const xs = key || x[i] === 79
@@ -233,6 +306,13 @@ function parse(x, state, parsers, handle, transform) {
   }).reduce(char, {})[x[0]](x)
 }
 
+function decodable(relation) {
+  if (relation.failed)
+    throw relation.error
+
+  return relation
+}
+
 function tuples(x, columns, xi, transform) {
   let type
     , column
@@ -266,7 +346,7 @@ function tuples(x, columns, xi, transform) {
 }
 
 function parseEvent(x) {
-  const xs = x.match(/^(\*|insert|update|delete)?:?([^.]+?\.?[^=]+)?=?(.+)?/i) || []
+  const xs = x.match(/^(\*|insert|update|delete)?:?([^.=]+(?:\.[^=]+)?)?=?(.+)?/i) || []
 
   if (!xs)
     throw new Error('Malformed subscribe pattern: ' + x)

package/cjs/src/connection.js

@@ -5,7 +5,7 @@ const Stream = require('stream')
 const { performance } = require('perf_hooks')
 
 const { stringify, handleValue, addArrayType } = require('./types.js')
-const { Errors } = require('./errors.js')
+const { Errors, callback } = require('./errors.js')
 const Result = require('./result.js')
 const Queue = require('./queue.js')
 const clampedTimeout = require('./timeout.js')
@@ -13,7 +13,7 @@ const { SSLRequest, tlsConfig, cancelRequest } = require('./transport.js')
 const { Query, CLOSE } = require('./query.js')
 const b = require('./bytes.js')
 
-module.exports = Connection
+module.exports = { Connection }
 
 let uid = 1
 
@@ -26,6 +26,8 @@ const Sync = b().S().end()
 const Phase = { Closed: 0, Backoff: 1, Connecting: 2, Initializing: 3, Ready: 4, Draining: 5, Closing: 6 }
 const phaseNames = Object.keys(Phase)
 
+const endsAbortedState = /^\s*(rollback|abort)\b/i
+
 const retryRoutines = new Set([
   'FetchPreparedStatement',
   'RevalidateCachedQuery',
@@ -110,6 +112,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose
     , chunk = null
     , nonce = null
     , query = null
+    , barrier = null
     , final = null
 
   const check = globalThis[Symbol.for('postgres.js:check')]
@@ -224,6 +227,9 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose
     owner: null,
     get active() { return query ? 1 : 0 },
     get pipelined() { return sent.length },
+    get accepting() { return accepting() },
+    get copying() { return stream !== null },
+    get copyHeld() { return stream !== null && !stream.writableEnded },
     get acquiring() { return acquisition && !acquisition.owner.cancelled ? 1 : 0 },
     count: 0,
     id
@@ -268,8 +274,12 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose
     return !query && sent.length === 0
   }
 
+  function accepting() {
+    return !query || (!barrier && sent.length < max_pipeline)
+  }
+
   function backoffMs() {
-    return (typeof backoff === 'function' ? backoff(options.shared.retries) : backoff) * 1000
+    return backoffSeconds(backoff, options.shared.retries) * 1000
   }
 
   function standbyPass() {
@@ -446,7 +456,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose
       queryError(pending, err)
       pending.cancelled && pending.cancelled.resolve()
     }
-    query = results = errorResponse = chunk = nextWriteTimer = null
+    query = results = errorResponse = chunk = nextWriteTimer = barrier = null
     result = new Result()
     rows = 0
     nonce = serverSignature = null
@@ -510,8 +520,10 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose
   }
 
   function submit(q) {
-    if (stream)
-      return queryError(q, Errors.generic('COPY_IN_PROGRESS', 'You cannot execute queries during copy'))
+    if (stream) {
+      queryError(q, Errors.generic('COPY_IN_PROGRESS', 'You cannot execute queries during copy'))
+      return true
+    }
 
     if (q.cancelled)
       return true
@@ -525,17 +537,21 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose
         : (query = q, query.active = true)
 
       build(q)
-      return write(toBuffer(q))
-        && !q.describeFirst
-        && !q.cursorFn
-        && sent.length < max_pipeline
+      const written = write(toBuffer(q))
+      if (q.describeFirst || q.cursorFn || mayCopy(q))
+        barrier = q
+      return written && accepting()
     } catch (error) {
       sent.remove(q) || write(Sync)
       queryError(q, error)
-      return sent.length < max_pipeline
+      return accepting()
     }
   }
 
+  function mayCopy(q) {
+    return q.options.simple && /\bcopy\b/i.test(q.string)
+  }
+
   function toBuffer(q) {
     if (q.parameters.length >= 65534)
       throw Errors.generic('MAX_PARAMETERS_EXCEEDED', 'Max number of parameters (65534) exceeded')
@@ -594,7 +610,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose
       ? statements[q.signature]
       : { string, types, name: q.prepare ? statementId + statementCount++ : '' }
 
-    typeof options.debug === 'function' && options.debug(id, string, parameters, types)
+    typeof options.debug === 'function' && callback(options.debug, id, string, parameters, types)
   }
 
   function write(x, fn) {
@@ -765,18 +781,21 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose
     backendParameters[k] = v
     if (options.parameters[k] !== v) {
       options.parameters[k] = v
-      onparameter && onparameter(k, v)
+      callback(onparameter, k, v)
     }
   }
 
-  function ReadyForQuery() {
+  function ReadyForQuery(x) {
+    query === barrier && (barrier = null)
     if (query) {
       if (errorResponse) {
         if (query.initialization)
           return dispatch('initFailed', errorResponse)
+        query.prepared && retryRoutines.has(errorResponse.routine) && delete statements[query.signature]
         query.retried
           ? errored(query.retried)
-          : query.prepared && retryRoutines.has(errorResponse.routine) && query.owner === connection.owner
+          : query.prepared && retryRoutines.has(errorResponse.routine) && query.owner === connection.owner && !barrier
+            && (x[5] !== 69 || endsAbortedState.test(query.string))
             ? retry(query, errorResponse)
             : errored(errorResponse)
       } else {
@@ -1059,7 +1078,6 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose
   }
 
   function retry(q, error) {
-    delete statements[q.signature]
     q.retried = error
     execute(q)
   }
@@ -1070,7 +1088,8 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose
 
     let index = 9
     while (x[index++] !== 0);
-    onnotify(
+    callback(
+      onnotify,
       x.toString('utf8', 9, index - 1),
       x.toString('utf8', index, x.length - 1)
     )
@@ -1116,6 +1135,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose
       }
     })
     query.resolve(stream)
+    connection.owner && connection.owner.next()
   }
 
   function CopyOutResponse() {
@@ -1123,6 +1143,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose
       read() { socket.resume() }
     })
     query.resolve(stream)
+    connection.owner && connection.owner.next()
   }
 
   /* c8 ignore next 3 */
@@ -1145,6 +1166,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose
       }
     })
     query.resolve(stream)
+    connection.owner && connection.owner.next()
   }
 
   function CopyData(x) {
@@ -1157,10 +1179,12 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose
   }
 
   function NoticeResponse(x) {
+    if (onnotice === false)
+      return
+
     onnotice
-      ? onnotice(parseError(x))
+      ? callback(onnotice, parseError(x))
       : console.log(parseError(x)) // eslint-disable-line
-
   }
 
   /* c8 ignore next 3 */
@@ -1311,3 +1335,19 @@ function timer(fn, seconds) {
     fn.apply(null, args)
   }
 }
+
+module.exports.defaultBackoff = defaultBackoff;function defaultBackoff(retries) {
+  return (0.5 + Math.random() / 2) * Math.min(3 ** retries / 100, 20)
+}
+
+module.exports.backoffSeconds = backoffSeconds;function backoffSeconds(backoff, retries) {
+  if (typeof backoff !== 'function')
+    return backoff
+
+  try {
+    return backoff(retries)
+  } catch (e) {
+    callback(() => { throw e })
+    return defaultBackoff(retries)
+  }
+}

package/cjs/src/errors.js

@@ -6,6 +6,14 @@ const PostgresError = module.exports.PostgresError = class PostgresError extends
   }
 }
 
+module.exports.callback = callback;function callback(fn, ...args) {
+  try {
+    fn && fn(...args)
+  } catch (e) {
+    queueMicrotask(() => { throw e })
+  }
+}
+
 const Errors = module.exports.Errors = {
   connection,
   postgres,