@steadybrew/postgresjs 4.2.0 → 4.4.0

package/cjs/src/index.js

@@ -8,6 +8,7 @@ const {
   Parameter,
   Identifier,
   Builder,
+  escapeIdentifier,
   toPascal,
   pascal,
   toCamel,
@@ -19,12 +20,12 @@ const {
   fromKebab
 } = require('./types.js')
 
-const Connection = require('./connection.js')
+const { Connection, defaultBackoff } = require('./connection.js')
 const { cancelRequest } = require('./transport.js')
 const clampedTimeout = require('./timeout.js')
 const { Query, CLOSE } = require('./query.js')
 const Queue = require('./queue.js')
-const { Errors, PostgresError } = require('./errors.js')
+const { Errors, PostgresError, callback } = require('./errors.js')
 const Subscribe = require('./subscribe.js')
 const largeObject = require('./large.js')
 
@@ -81,6 +82,7 @@ function Postgres(a, b) {
     PostgresError,
     options,
     reserve,
+    stats,
     listen,
     begin,
     close,
@@ -150,6 +152,56 @@ function Postgres(a, b) {
     }
   }
 
+  function stats() {
+    let idleCount = 0
+      , busyCount = 0
+      , reservedCount = 0
+      , connectingCount = 0
+      , closingCount = 0
+      , executingCount = 0
+      , waitingCount = queries.length
+
+    for (const c of connections) {
+      executingCount += c.active
+      waitingCount += c.pipelined + c.acquiring + (c.owner ? c.owner.queue.length : 0)
+      c.queue === open
+        ? idleCount++
+        : c.queue === connecting
+          ? connectingCount++
+          : c.queue === ended
+            ? closingCount++
+            : c.queue === reserved
+              ? reservedCount++
+              : (c.queue === busy || c.queue === full) && (c.owner ? reservedCount++ : busyCount++)
+    }
+
+    const totalCount = idleCount + busyCount + reservedCount + connectingCount
+
+    return {
+      max: options.max,
+      totalCount,
+      idleCount,
+      waitingCount,
+      activeCount: busyCount + reservedCount,
+      availableCount: idleCount + options.max - totalCount - closingCount,
+      busyCount,
+      reservedCount,
+      connectingCount,
+      closingCount,
+      executingCount,
+      listenCount: pooled(listen.sql),
+      subscribeCount: pooled(subscribe.sql)
+    }
+  }
+
+  function pooled(sql) {
+    if (!sql)
+      return 0
+
+    const s = sql.stats()
+    return s.totalCount + s.closingCount
+  }
+
   async function listen(name, fn, onlisten) {
     const listener = { fn, onlisten }
 
@@ -167,7 +219,7 @@ function Postgres(a, b) {
         })
       },
       onnotify(c, x) {
-        c in listen.channels && listen.channels[c].listeners.forEach(l => l.fn(x))
+        c in listen.channels && listen.channels[c].listeners.forEach(l => callback(l.fn, x))
       }
     }))
 
@@ -177,7 +229,7 @@ function Postgres(a, b) {
     if (exists) {
       channels[name].listeners.push(listener)
       const result = await channels[name].result
-      listener.onlisten && listener.onlisten()
+      callback(listener.onlisten)
       return { state: result.state, unlisten }
     }
 
@@ -185,7 +237,7 @@ function Postgres(a, b) {
       sql.unsafe('"' + name.replace(/"/g, '""') + '"')
     }`, listeners: [listener] }
     const result = await channels[name].result
-    listener.onlisten && listener.onlisten()
+    callback(listener.onlisten)
     return { state: result.state, unlisten }
 
     async function unlisten() {
@@ -243,9 +295,9 @@ function Postgres(a, b) {
   function send(l, q, expected = Lease.Active) {
     const c = l.connection
     l.state === expected
-      ? c.queue === full
-        ? l.queue.push(q)
-        : c.execute(q) || move(c, full)
+      ? c.copying || (c.queue !== full && c.accepting)
+        ? c.execute(q) || move(c, full)
+        : (move(c, full), q.queue = l.queue, l.queue.push(q))
       : q.reject(leaseError(l.state))
   }
 
@@ -275,6 +327,7 @@ function Postgres(a, b) {
     const l = await lease(false)
     const sql = Sql(q => send(l, q))
     sql.release = () => settle(l, Lease.Released)
+    sql.stats = stats
     return sql
   }
 
@@ -300,14 +353,16 @@ function Postgres(a, b) {
     }
 
     async function finish(statement) {
+      let result
       l.state = Lease.Settling
       try {
-        await statement(Sql(q => send(l, q, Lease.Settling)))
+        result = await statement(Sql(q => send(l, q, Lease.Settling)))
       } catch (e) {
         settle(l, Lease.Discarded)
         throw e
       }
       settle(l, Lease.Released)
+      return result
     }
 
     async function scope(fn, name) {
@@ -322,7 +377,7 @@ function Postgres(a, b) {
       let uncaughtError
         , result
 
-      name && await sql`savepoint ${ sql(name) }`
+      name && await sql.unsafe('savepoint ' + escapeIdentifier(name))
       try {
         result = await new Promise((resolve, reject) => {
           const x = fn(sql)
@@ -333,8 +388,8 @@ function Postgres(a, b) {
           throw uncaughtError
       } catch (e) {
         l.state === Lease.Active && await (name
-          ? sql`rollback to ${ sql(name) }`
-          : finish(final => final`rollback`)
+          ? sql.unsafe('rollback to ' + escapeIdentifier(name))
+          : finish(final => final.unsafe('rollback'))
         )
         throw e instanceof PostgresError && e.code === '25P02' && uncaughtError || e
       }
@@ -342,10 +397,18 @@ function Postgres(a, b) {
       if (l.state !== Lease.Active)
         throw l.error || leaseError(l.state)
 
-      name || await finish(final => prepare
-        ? final`prepare transaction ${ final.unsafe(prepare) }`
-        : final`commit`
-      )
+      if (!name) {
+        const done = await finish(final => prepare
+          ? final.unsafe('prepare transaction ' + prepare)
+          : final.unsafe('commit')
+        )
+        if (done.command === 'ROLLBACK') {
+          throw Object.assign(
+            Errors.generic('TRANSACTION_ROLLED_BACK', 'The server rolled the transaction back instead of committing it'),
+            uncaughtError && { cause: uncaughtError }
+          )
+        }
+      }
 
       return result
 
@@ -374,14 +437,6 @@ function Postgres(a, b) {
     return c
   }
 
-  function callback(fn, ...args) {
-    try {
-      fn && fn(...args)
-    } catch (e) {
-      queueMicrotask(() => { throw e })
-    }
-  }
-
   function json(x) {
     return new Parameter(x, 3802)
   }
@@ -404,11 +459,24 @@ function Postgres(a, b) {
     if (closed.length)
       return connect(closed.shift(), query)
 
-    busy.length
-      ? go(busy.shift(), query)
+    const c = takeBusy()
+    c
+      ? go(c, query)
       : queries.push(query)
   }
 
+  function takeBusy() {
+    while (busy.length) {
+      const c = busy.shift()
+      if (c.accepting)
+        return c
+
+      move(c, full)
+    }
+
+    return null
+  }
+
   function takeOpen() {
     while (open.length) {
       const c = open.shift()
@@ -435,7 +503,7 @@ function Postgres(a, b) {
       if (query.state) {
         query.cancelled = { resolve, reject }
       } else {
-        queries.remove(query)
+        (query.queue || queries).remove(query)
         query.cancelled = true
         query.reject(Errors.generic('57014', 'canceling statement due to user request'))
         resolve()
@@ -493,6 +561,9 @@ function Postgres(a, b) {
     if (initial && initial.reserve)
       return initial.reserve(c)
 
+    if (!initial && !c.accepting)
+      return move(c, full)
+
     if (!initial && !queries.length)
       return move(c, open)
 
@@ -531,7 +602,7 @@ function Postgres(a, b) {
   function onclose(c, e) {
     move(c, closed)
     c.owner && settle(c.owner, Lease.Closed, e)
-    options.onclose && options.onclose(c.id)
+    callback(options.onclose, c.id)
     if (ending)
       return closedWhileEnding(c, e)
 
@@ -586,7 +657,7 @@ function parseOptions(a, b) {
     connect_timeout : 30,
     max_lifetime    : max_lifetime,
     max_pipeline    : 100,
-    backoff         : backoff,
+    backoff         : defaultBackoff,
     keep_alive      : 60,
     prepare         : true,
     debug           : false,
@@ -645,10 +716,6 @@ function tsa(o, url, env) {
   throw new Error('target_session_attrs ' + x + ' is not supported')
 }
 
-function backoff(retries) {
-  return (0.5 + Math.random() / 2) * Math.min(3 ** retries / 100, 20)
-}
-
 function max_lifetime() {
   return 60 * (30 + Math.random() * 30)
 }

package/cjs/src/subscribe.js

@@ -1,4 +1,8 @@
+const { callback } = require('./errors.js')
+const { defaultBackoff, backoffSeconds } = require('./connection.js')
+
 const noop = () => { /* noop */ }
+    , commands = { 68: 'delete', 73: 'insert', 85: 'update' }
 
 module.exports = Subscribe;function Subscribe(postgres, options) {
   const subscribers = new Map()
@@ -7,6 +11,9 @@ module.exports = Subscribe;function Subscribe(postgres, options) {
 
   let connection
     , stream
+    , timer
+    , inflight = false
+    , retries = 0
     , ended = false
 
   const sql = subscribe.sql = postgres({
@@ -21,13 +28,12 @@ module.exports = Subscribe;function Subscribe(postgres, options) {
       ...options.connection,
       replication: 'database'
     },
-    onclose: async function() {
+    onclose: function() {
       if (ended)
         return
       stream = null
       state.pid = state.secret = undefined
-      connected(await init(sql, slot, options.publications))
-      subscribers.forEach(event => event.forEach(({ onsubscribe }) => onsubscribe()))
+      inflight || timer || !connection || connect().catch(noop)
     },
     no_subscribe: true
   })
@@ -37,6 +43,9 @@ module.exports = Subscribe;function Subscribe(postgres, options) {
 
   sql.end = async() => {
     ended = true
+    clearTimeout(timer)
+    timer = null
+    connection && await connection.catch(noop)
     stream && (await new Promise(r => (stream.once('close', r), stream.end())))
     return end()
   }
@@ -48,34 +57,74 @@ module.exports = Subscribe;function Subscribe(postgres, options) {
 
   return subscribe
 
-  async function subscribe(event, fn, onsubscribe = noop, onerror = noop) {
+  async function subscribe(event, fn, onsubscribe = noop, onerror) {
     event = parseEvent(event)
 
-    if (!connection)
-      connection = init(sql, slot, options.publications)
+    const subscriber = { fn, onsubscribe, onerror }
 
-    const subscriber = { fn, onsubscribe }
-    const fns = subscribers.has(event)
-      ? subscribers.get(event).add(subscriber)
-      : subscribers.set(event, new Set([subscriber])).get(event)
+    return (connection || connect()).then(() => {
+      const fns = subscribers.has(event)
+        ? subscribers.get(event).add(subscriber)
+        : subscribers.set(event, new Set([subscriber])).get(event)
 
-    const unsubscribe = () => {
-      fns.delete(subscriber)
-      fns.size === 0 && subscribers.delete(event)
-    }
+      const unsubscribe = () => {
+        fns.delete(subscriber)
+        fns.size === 0 && subscribers.delete(event)
+        subscribers.size === 0 && (clearTimeout(timer), timer = null)
+      }
 
-    return connection.then(x => {
-      connected(x)
-      onsubscribe()
-      stream && stream.on('error', onerror)
+      callback(onsubscribe)
       return { unsubscribe, state, sql }
     })
   }
 
-  function connected(x) {
-    stream = x.stream
-    state.pid = x.state.pid
-    state.secret = x.state.secret
+  function connect() {
+    clearTimeout(timer)
+    timer = null
+    inflight = true
+    const resume = subscribers.size > 0
+    const current = connection = init(sql, slot, options.publications).then(x => {
+      stream = x.stream
+      state.pid = x.state.pid
+      state.secret = x.state.secret
+    })
+
+    current.then(
+      () => {
+        inflight = false
+        retries = 0
+        resume && !ended && subscribers.forEach(event => event.forEach(({ onsubscribe }) => callback(onsubscribe)))
+      },
+      error => {
+        inflight = false
+        connection === current && (connection = null)
+        if (ended || subscribers.size === 0)
+          return
+        reportConnection(error)
+        timer = setTimeout(() => connect().catch(noop), retryDelay(retries++))
+      }
+    )
+
+    return current
+  }
+
+  function retryDelay(n) {
+    let seconds = backoffSeconds(sql.options.backoff, n)
+    seconds = Number.isFinite(seconds) ? seconds : defaultBackoff(n)
+    return Math.min(Math.max(seconds * 1000, 100), 2 ** 31 - 1)
+  }
+
+  function reportConnection(error) {
+    console.error('Unexpected error during logical streaming - reconnecting', error) // eslint-disable-line
+    subscribers.forEach(fns => fns.forEach(({ onerror }) => callback(onerror, error)))
+  }
+
+  function report(error, match) {
+    let uncaught = false
+    subscribers.forEach((fns, event) => match(event) && fns.forEach(({ onerror }) =>
+      onerror ? callback(onerror, error) : uncaught = true
+    ))
+    uncaught && callback(() => { throw error })
   }
 
   async function init(sql, slot, publications) {
@@ -99,18 +148,18 @@ module.exports = Subscribe;function Subscribe(postgres, options) {
     }
 
     stream.on('data', data)
-    stream.on('error', error)
+    stream.on('error', reportConnection)
     stream.on('close', sql.close)
 
     return { stream, state: xs.state }
 
-    function error(e) {
-      console.error('Unexpected error during logical streaming - reconnecting', e) // eslint-disable-line
-    }
-
     function data(x) {
       if (x[0] === 0x77) {
-        parse(x.subarray(25), state, sql.options.parsers, handle, options.transform)
+        try {
+          parse(x.subarray(25), state, sql.options.parsers, handle, options.transform)
+        } catch (error) {
+          skip(x.subarray(25), error)
+        }
       } else if (x[0] === 0x6b && x[17]) {
         state.lsn = x.subarray(1, 9)
         pong()
@@ -119,12 +168,26 @@ module.exports = Subscribe;function Subscribe(postgres, options) {
 
     function handle(a, b) {
       const path = b.relation.schema + '.' + b.relation.table
+          , key = b.relation.keys.length && '=' + b.relation.keys.map(x => a[x.name])
       call('*', a, b)
       call('*:' + path, a, b)
-      b.relation.keys.length && call('*:' + path + '=' + b.relation.keys.map(x => a[x.name]), a, b)
+      key && call('*:' + path + key, a, b)
       call(b.command, a, b)
       call(b.command + ':' + path, a, b)
-      b.relation.keys.length && call(b.command + ':' + path + '=' + b.relation.keys.map(x => a[x.name]), a, b)
+      key && call(b.command + ':' + path + key, a, b)
+    }
+
+    function skip(x, error) {
+      const command = commands[x[0]]
+          , relation = command && state[x.readUInt32BE(1)]
+
+      if (!relation)
+        throw error
+
+      const path = relation.schema + '.' + relation.table
+          , events = ['*', '*:' + path, command, command + ':' + path]
+
+      report(error, event => events.includes(event.split('=')[0]))
     }
 
     function pong() {
@@ -137,7 +200,7 @@ module.exports = Subscribe;function Subscribe(postgres, options) {
   }
 
   function call(x, a, b) {
-    subscribers.has(x) && subscribers.get(x).forEach(({ fn }) => fn(a, b, x))
+    subscribers.has(x) && subscribers.get(x).forEach(({ fn }) => callback(fn, a, b, x))
   }
 }
 
@@ -162,19 +225,24 @@ function parse(x, state, parsers, handle, transform) {
       let columnIndex = 0
         , column
 
-      while (i < x.length) {
-        column = r.columns[columnIndex++] = {
-          key: x[i++],
-          name: transform.column.from
-            ? transform.column.from(x.toString('utf8', i, i = x.indexOf(0, i)))
-            : x.toString('utf8', i, i = x.indexOf(0, i)),
-          type: x.readUInt32BE(i += 1),
-          parser: parsers[x.readUInt32BE(i)],
-          atttypmod: x.readUInt32BE(i += 4)
+      try {
+        while (i < x.length) {
+          column = r.columns[columnIndex++] = {
+            key: x[i++],
+            name: transform.column.from
+              ? transform.column.from(x.toString('utf8', i, i = x.indexOf(0, i)))
+              : x.toString('utf8', i, i = x.indexOf(0, i)),
+            type: x.readUInt32BE(i += 1),
+            parser: parsers[x.readUInt32BE(i)],
+            atttypmod: x.readUInt32BE(i += 4)
+          }
+
+          column.key && r.keys.push(column)
+          i += 4
         }
-
-        column.key && r.keys.push(column)
-        i += 4
+      } catch (error) {
+        r.error = error
+        r.failed = true
       }
     },
     Y: () => { /* noop */ }, // Type
@@ -185,7 +253,7 @@ function parse(x, state, parsers, handle, transform) {
     },
     I: x => { // Insert
       let i = 1
-      const relation = state[x.readUInt32BE(i)]
+      const relation = decodable(state[x.readUInt32BE(i)])
       const { row } = tuples(x, relation.columns, i += 7, transform)
 
       handle(row, {
@@ -195,7 +263,7 @@ function parse(x, state, parsers, handle, transform) {
     },
     D: x => { // Delete
       let i = 1
-      const relation = state[x.readUInt32BE(i)]
+      const relation = decodable(state[x.readUInt32BE(i)])
       i += 4
       const key = x[i] === 75
       handle(key || x[i] === 79
@@ -209,7 +277,7 @@ function parse(x, state, parsers, handle, transform) {
     },
     U: x => { // Update
       let i = 1
-      const relation = state[x.readUInt32BE(i)]
+      const relation = decodable(state[x.readUInt32BE(i)])
       i += 4
       const key = x[i] === 75
       const xs = key || x[i] === 79
@@ -232,6 +300,13 @@ function parse(x, state, parsers, handle, transform) {
   }).reduce(char, {})[x[0]](x)
 }
 
+function decodable(relation) {
+  if (relation.failed)
+    throw relation.error
+
+  return relation
+}
+
 function tuples(x, columns, xi, transform) {
   let type
     , column
@@ -265,7 +340,7 @@ function tuples(x, columns, xi, transform) {
 }
 
 function parseEvent(x) {
-  const xs = x.match(/^(\*|insert|update|delete)?:?([^.]+?\.?[^=]+)?=?(.+)?/i) || []
+  const xs = x.match(/^(\*|insert|update|delete)?:?([^.=]+(?:\.[^=]+)?)?=?(.+)?/i) || []
 
   if (!xs)
     throw new Error('Malformed subscribe pattern: ' + x)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steadybrew/postgresjs",
-  "version": "4.2.0",
+  "version": "4.4.0",
   "description": "PostgreSQL client for Node.js and Deno, independently maintained by Steady Brew",
   "type": "module",
   "module": "src/index.js",