@steadybrew/postgresjs 4.2.0 → 4.4.0

package/src/subscribe.js

@@ -1,4 +1,8 @@
+import { callback } from './errors.js'
+import { defaultBackoff, backoffSeconds } from './connection.js'
+
 const noop = () => { /* noop */ }
+    , commands = { 68: 'delete', 73: 'insert', 85: 'update' }
 
 export default function Subscribe(postgres, options) {
   const subscribers = new Map()
@@ -7,6 +11,9 @@ export default function Subscribe(postgres, options) {
 
   let connection
     , stream
+    , timer
+    , inflight = false
+    , retries = 0
     , ended = false
 
   const sql = subscribe.sql = postgres({
@@ -21,13 +28,12 @@ export default function Subscribe(postgres, options) {
       ...options.connection,
       replication: 'database'
     },
-    onclose: async function() {
+    onclose: function() {
       if (ended)
         return
       stream = null
       state.pid = state.secret = undefined
-      connected(await init(sql, slot, options.publications))
-      subscribers.forEach(event => event.forEach(({ onsubscribe }) => onsubscribe()))
+      inflight || timer || !connection || connect().catch(noop)
     },
     no_subscribe: true
   })
@@ -37,6 +43,9 @@ export default function Subscribe(postgres, options) {
 
   sql.end = async() => {
     ended = true
+    clearTimeout(timer)
+    timer = null
+    connection && await connection.catch(noop)
     stream && (await new Promise(r => (stream.once('close', r), stream.end())))
     return end()
   }
@@ -48,34 +57,74 @@ export default function Subscribe(postgres, options) {
 
   return subscribe
 
-  async function subscribe(event, fn, onsubscribe = noop, onerror = noop) {
+  async function subscribe(event, fn, onsubscribe = noop, onerror) {
     event = parseEvent(event)
 
-    if (!connection)
-      connection = init(sql, slot, options.publications)
+    const subscriber = { fn, onsubscribe, onerror }
 
-    const subscriber = { fn, onsubscribe }
-    const fns = subscribers.has(event)
-      ? subscribers.get(event).add(subscriber)
-      : subscribers.set(event, new Set([subscriber])).get(event)
+    return (connection || connect()).then(() => {
+      const fns = subscribers.has(event)
+        ? subscribers.get(event).add(subscriber)
+        : subscribers.set(event, new Set([subscriber])).get(event)
 
-    const unsubscribe = () => {
-      fns.delete(subscriber)
-      fns.size === 0 && subscribers.delete(event)
-    }
+      const unsubscribe = () => {
+        fns.delete(subscriber)
+        fns.size === 0 && subscribers.delete(event)
+        subscribers.size === 0 && (clearTimeout(timer), timer = null)
+      }
 
-    return connection.then(x => {
-      connected(x)
-      onsubscribe()
-      stream && stream.on('error', onerror)
+      callback(onsubscribe)
       return { unsubscribe, state, sql }
     })
   }
 
-  function connected(x) {
-    stream = x.stream
-    state.pid = x.state.pid
-    state.secret = x.state.secret
+  function connect() {
+    clearTimeout(timer)
+    timer = null
+    inflight = true
+    const resume = subscribers.size > 0
+    const current = connection = init(sql, slot, options.publications).then(x => {
+      stream = x.stream
+      state.pid = x.state.pid
+      state.secret = x.state.secret
+    })
+
+    current.then(
+      () => {
+        inflight = false
+        retries = 0
+        resume && !ended && subscribers.forEach(event => event.forEach(({ onsubscribe }) => callback(onsubscribe)))
+      },
+      error => {
+        inflight = false
+        connection === current && (connection = null)
+        if (ended || subscribers.size === 0)
+          return
+        reportConnection(error)
+        timer = setTimeout(() => connect().catch(noop), retryDelay(retries++))
+      }
+    )
+
+    return current
+  }
+
+  function retryDelay(n) {
+    let seconds = backoffSeconds(sql.options.backoff, n)
+    seconds = Number.isFinite(seconds) ? seconds : defaultBackoff(n)
+    return Math.min(Math.max(seconds * 1000, 100), 2 ** 31 - 1)
+  }
+
+  function reportConnection(error) {
+    console.error('Unexpected error during logical streaming - reconnecting', error) // eslint-disable-line
+    subscribers.forEach(fns => fns.forEach(({ onerror }) => callback(onerror, error)))
+  }
+
+  function report(error, match) {
+    let uncaught = false
+    subscribers.forEach((fns, event) => match(event) && fns.forEach(({ onerror }) =>
+      onerror ? callback(onerror, error) : uncaught = true
+    ))
+    uncaught && callback(() => { throw error })
   }
 
   async function init(sql, slot, publications) {
@@ -99,18 +148,18 @@ export default function Subscribe(postgres, options) {
     }
 
     stream.on('data', data)
-    stream.on('error', error)
+    stream.on('error', reportConnection)
     stream.on('close', sql.close)
 
     return { stream, state: xs.state }
 
-    function error(e) {
-      console.error('Unexpected error during logical streaming - reconnecting', e) // eslint-disable-line
-    }
-
     function data(x) {
       if (x[0] === 0x77) {
-        parse(x.subarray(25), state, sql.options.parsers, handle, options.transform)
+        try {
+          parse(x.subarray(25), state, sql.options.parsers, handle, options.transform)
+        } catch (error) {
+          skip(x.subarray(25), error)
+        }
       } else if (x[0] === 0x6b && x[17]) {
         state.lsn = x.subarray(1, 9)
         pong()
@@ -119,12 +168,26 @@ export default function Subscribe(postgres, options) {
 
     function handle(a, b) {
       const path = b.relation.schema + '.' + b.relation.table
+          , key = b.relation.keys.length && '=' + b.relation.keys.map(x => a[x.name])
       call('*', a, b)
       call('*:' + path, a, b)
-      b.relation.keys.length && call('*:' + path + '=' + b.relation.keys.map(x => a[x.name]), a, b)
+      key && call('*:' + path + key, a, b)
       call(b.command, a, b)
       call(b.command + ':' + path, a, b)
-      b.relation.keys.length && call(b.command + ':' + path + '=' + b.relation.keys.map(x => a[x.name]), a, b)
+      key && call(b.command + ':' + path + key, a, b)
+    }
+
+    function skip(x, error) {
+      const command = commands[x[0]]
+          , relation = command && state[x.readUInt32BE(1)]
+
+      if (!relation)
+        throw error
+
+      const path = relation.schema + '.' + relation.table
+          , events = ['*', '*:' + path, command, command + ':' + path]
+
+      report(error, event => events.includes(event.split('=')[0]))
     }
 
     function pong() {
@@ -137,7 +200,7 @@ export default function Subscribe(postgres, options) {
   }
 
   function call(x, a, b) {
-    subscribers.has(x) && subscribers.get(x).forEach(({ fn }) => fn(a, b, x))
+    subscribers.has(x) && subscribers.get(x).forEach(({ fn }) => callback(fn, a, b, x))
   }
 }
 
@@ -162,19 +225,24 @@ function parse(x, state, parsers, handle, transform) {
       let columnIndex = 0
         , column
 
-      while (i < x.length) {
-        column = r.columns[columnIndex++] = {
-          key: x[i++],
-          name: transform.column.from
-            ? transform.column.from(x.toString('utf8', i, i = x.indexOf(0, i)))
-            : x.toString('utf8', i, i = x.indexOf(0, i)),
-          type: x.readUInt32BE(i += 1),
-          parser: parsers[x.readUInt32BE(i)],
-          atttypmod: x.readUInt32BE(i += 4)
+      try {
+        while (i < x.length) {
+          column = r.columns[columnIndex++] = {
+            key: x[i++],
+            name: transform.column.from
+              ? transform.column.from(x.toString('utf8', i, i = x.indexOf(0, i)))
+              : x.toString('utf8', i, i = x.indexOf(0, i)),
+            type: x.readUInt32BE(i += 1),
+            parser: parsers[x.readUInt32BE(i)],
+            atttypmod: x.readUInt32BE(i += 4)
+          }
+
+          column.key && r.keys.push(column)
+          i += 4
         }
-
-        column.key && r.keys.push(column)
-        i += 4
+      } catch (error) {
+        r.error = error
+        r.failed = true
       }
     },
     Y: () => { /* noop */ }, // Type
@@ -185,7 +253,7 @@ function parse(x, state, parsers, handle, transform) {
     },
     I: x => { // Insert
       let i = 1
-      const relation = state[x.readUInt32BE(i)]
+      const relation = decodable(state[x.readUInt32BE(i)])
       const { row } = tuples(x, relation.columns, i += 7, transform)
 
       handle(row, {
@@ -195,7 +263,7 @@ function parse(x, state, parsers, handle, transform) {
     },
     D: x => { // Delete
       let i = 1
-      const relation = state[x.readUInt32BE(i)]
+      const relation = decodable(state[x.readUInt32BE(i)])
       i += 4
       const key = x[i] === 75
       handle(key || x[i] === 79
@@ -209,7 +277,7 @@ function parse(x, state, parsers, handle, transform) {
     },
     U: x => { // Update
       let i = 1
-      const relation = state[x.readUInt32BE(i)]
+      const relation = decodable(state[x.readUInt32BE(i)])
       i += 4
       const key = x[i] === 75
       const xs = key || x[i] === 79
@@ -232,6 +300,13 @@ function parse(x, state, parsers, handle, transform) {
   }).reduce(char, {})[x[0]](x)
 }
 
+function decodable(relation) {
+  if (relation.failed)
+    throw relation.error
+
+  return relation
+}
+
 function tuples(x, columns, xi, transform) {
   let type
     , column
@@ -265,7 +340,7 @@ function tuples(x, columns, xi, transform) {
 }
 
 function parseEvent(x) {
-  const xs = x.match(/^(\*|insert|update|delete)?:?([^.]+?\.?[^=]+)?=?(.+)?/i) || []
+  const xs = x.match(/^(\*|insert|update|delete)?:?([^.=]+(?:\.[^=]+)?)?=?(.+)?/i) || []
 
   if (!xs)
     throw new Error('Malformed subscribe pattern: ' + x)

package/types/index.d.ts

@@ -67,14 +67,11 @@ interface BaseOptions<T extends Record<string, postgres.PostgresType>> {
    * @default true
    */
   prepare: boolean;
-  /**
-   * Called when a notice is received
-   * @default console.log
-   */
-  onnotice: (notice: postgres.Notice) => void;
-  /** (key; value) when a server param change */
+  /** Called when a notice is received, console.log by default and false to ignore them; if it throws, the error is rethrown as an uncaught exception */
+  onnotice: ((notice: postgres.Notice) => void) | false;
+  /** (key; value) when a server param change; if it throws, the error is rethrown as an uncaught exception */
   onparameter: (key: string, value: any) => void;
-  /** Is called with (connection; query; parameters) */
+  /** Is called with (connection; query; parameters); if it throws, the error is rethrown as an uncaught exception and the query still runs */
   debug: boolean | ((connection: number, query: string, parameters: any[], paramTypes: any[]) => void);
   /** Transform hooks */
   transform: {
@@ -118,6 +115,7 @@ interface BaseOptions<T extends Record<string, postgres.PostgresType>> {
    * @default 'alltables'
    */
   publications: string
+  /** Called with the connection id each time a connection closes; if it throws, the error is rethrown as an uncaught exception */
   onclose: (connId: number) => void;
   /** Called with the connection id each time a connection returns to the pool idle and its idle timer starts */
   onidle: (connId: number) => void;
@@ -463,7 +461,8 @@ declare namespace postgres {
     | 'COPY_IN_PROGRESS'
     | 'INVALID_TRANSACTION_NAME'
     | 'TARGET_SESSION_ATTRS'
-    | 'UNSAFE_TRANSACTION';
+    | 'UNSAFE_TRANSACTION'
+    | 'TRANSACTION_ROLLED_BACK';
     message: string;
   }
 
@@ -704,6 +703,35 @@ declare namespace postgres {
     json(value: JSONValue): Parameter;
   }
 
+  interface PoolStats {
+    /** The configured maximum number of pool connections */
+    max: number;
+    /** Pool connections that exist: connecting, idle or in use */
+    totalCount: number;
+    /** Pool connections with a live session that are not in use */
+    idleCount: number;
+    /** Issued queries and reserve or begin calls that are not running yet, whether queued in the pool, held by an opening connection, pipelined or queued on a reserved connection */
+    waitingCount: number;
+    /** Pool connections in use: busyCount plus reservedCount */
+    activeCount: number;
+    /** Connections a query can get without waiting behind other work: idleCount plus the slots the pool can still open */
+    availableCount: number;
+    /** Pool connections running queries, including those at the pipeline limit */
+    busyCount: number;
+    /** Pool connections held by reserve() or begin() */
+    reservedCount: number;
+    /** Pool connections opening a session */
+    connectingCount: number;
+    /** Pool connections draining or closing, not included in totalCount */
+    closingCount: number;
+    /** Queries the server is running right now, at most one per connection, startup queries and COPY included */
+    executingCount: number;
+    /** Connections held by the internal listen pool, outside max */
+    listenCount: number;
+    /** Connections held by the internal subscribe pool, outside max */
+    subscribeCount: number;
+  }
+
   interface Sql<TTypes extends Record<string, unknown> = {}> extends ISql<TTypes> {
     CLOSE: {};
     END: this['CLOSE'];
@@ -716,7 +744,8 @@ declare namespace postgres {
 
     listen(channel: string, onnotify: (value: string) => void, onlisten?: (() => void) | undefined): ListenRequest;
 
-    subscribe(event: string, cb: (row: Row | null, info: ReplicationEvent) => void, onsubscribe?: (() => void), onerror?: (() => any)): Promise<SubscriptionHandle>;
+    /** `onerror` receives replication connection errors and errors thrown while a change is decoded; such a change is skipped. */
+    subscribe(event: string, cb: (row: Row | null, info: ReplicationEvent) => void, onsubscribe?: (() => void), onerror?: ((error: Error) => void)): Promise<SubscriptionHandle>;
 
     largeObject(oid?: number | undefined, /** @default 0x00020000 | 0x00040000 */ mode?: number | undefined): Promise<LargeObject>;
 
@@ -724,6 +753,9 @@ declare namespace postgres {
     begin<T>(options: string, cb: (sql: TransactionSql<TTypes>) => T | Promise<T>): Promise<UnwrapPromiseArray<T>>;
 
     reserve(): Promise<ReservedSql<TTypes>>
+
+    /** Returns a snapshot of the pool's connection and query counts */
+    stats(): PoolStats;
   }
 
   interface TransactionSql<TTypes extends Record<string, unknown> = {}> extends ISql<TTypes>  {