@steadybrew/postgresjs 4.3.0 → 4.4.1

package/src/subscribe.js

@@ -1,4 +1,9 @@
+import { callback } from './errors.js'
+import { defaultBackoff, backoffSeconds } from './connection.js'
+
 const noop = () => { /* noop */ }
+    , commands = { 68: 'delete', 73: 'insert', 85: 'update' }
+    , statusInterval = 1000
 
 export default function Subscribe(postgres, options) {
   const subscribers = new Map()
@@ -7,6 +12,9 @@ export default function Subscribe(postgres, options) {
 
   let connection
     , stream
+    , timer
+    , inflight = false
+    , retries = 0
     , ended = false
 
   const sql = subscribe.sql = postgres({
@@ -21,13 +29,12 @@ export default function Subscribe(postgres, options) {
       ...options.connection,
       replication: 'database'
     },
-    onclose: async function() {
+    onclose: function() {
       if (ended)
         return
       stream = null
       state.pid = state.secret = undefined
-      connected(await init(sql, slot, options.publications))
-      subscribers.forEach(event => event.forEach(({ onsubscribe }) => onsubscribe()))
+      inflight || timer || !connection || connect().catch(noop)
     },
     no_subscribe: true
   })
@@ -37,6 +44,9 @@ export default function Subscribe(postgres, options) {
 
   sql.end = async() => {
     ended = true
+    clearTimeout(timer)
+    timer = null
+    connection && await connection.catch(noop)
     stream && (await new Promise(r => (stream.once('close', r), stream.end())))
     return end()
   }
@@ -48,34 +58,74 @@ export default function Subscribe(postgres, options) {
 
   return subscribe
 
-  async function subscribe(event, fn, onsubscribe = noop, onerror = noop) {
+  async function subscribe(event, fn, onsubscribe = noop, onerror) {
     event = parseEvent(event)
 
-    if (!connection)
-      connection = init(sql, slot, options.publications)
+    const subscriber = { fn, onsubscribe, onerror }
 
-    const subscriber = { fn, onsubscribe }
-    const fns = subscribers.has(event)
-      ? subscribers.get(event).add(subscriber)
-      : subscribers.set(event, new Set([subscriber])).get(event)
+    return (connection || connect()).then(() => {
+      const fns = subscribers.has(event)
+        ? subscribers.get(event).add(subscriber)
+        : subscribers.set(event, new Set([subscriber])).get(event)
 
-    const unsubscribe = () => {
-      fns.delete(subscriber)
-      fns.size === 0 && subscribers.delete(event)
-    }
+      const unsubscribe = () => {
+        fns.delete(subscriber)
+        fns.size === 0 && subscribers.delete(event)
+        subscribers.size === 0 && (clearTimeout(timer), timer = null)
+      }
 
-    return connection.then(x => {
-      connected(x)
-      onsubscribe()
-      stream && stream.on('error', onerror)
+      callback(onsubscribe)
       return { unsubscribe, state, sql }
     })
   }
 
-  function connected(x) {
-    stream = x.stream
-    state.pid = x.state.pid
-    state.secret = x.state.secret
+  function connect() {
+    clearTimeout(timer)
+    timer = null
+    inflight = true
+    const resume = subscribers.size > 0
+    const current = connection = init(sql, slot, options.publications).then(x => {
+      stream = x.stream
+      state.pid = x.state.pid
+      state.secret = x.state.secret
+    })
+
+    current.then(
+      () => {
+        inflight = false
+        retries = 0
+        resume && !ended && subscribers.forEach(event => event.forEach(({ onsubscribe }) => callback(onsubscribe)))
+      },
+      error => {
+        inflight = false
+        connection === current && (connection = null)
+        if (ended || subscribers.size === 0)
+          return
+        reportConnection(error)
+        timer = setTimeout(() => connect().catch(noop), retryDelay(retries++))
+      }
+    )
+
+    return current
+  }
+
+  function retryDelay(n) {
+    let seconds = backoffSeconds(sql.options.backoff, n)
+    seconds = Number.isFinite(seconds) ? seconds : defaultBackoff(n)
+    return Math.min(Math.max(seconds * 1000, 100), 2 ** 31 - 1)
+  }
+
+  function reportConnection(error) {
+    console.error('Unexpected error during logical streaming - reconnecting', error) // eslint-disable-line
+    subscribers.forEach(fns => fns.forEach(({ onerror }) => callback(onerror, error)))
+  }
+
+  function report(error, match) {
+    let uncaught = false
+    subscribers.forEach((fns, event) => match(event) && fns.forEach(({ onerror }) =>
+      onerror ? callback(onerror, error) : uncaught = true
+    ))
+    uncaught && callback(() => { throw error })
   }
 
   async function init(sql, slot, publications) {
@@ -99,18 +149,22 @@ export default function Subscribe(postgres, options) {
     }
 
     stream.on('data', data)
-    stream.on('error', error)
+    stream.on('error', reportConnection)
     stream.on('close', sql.close)
 
-    return { stream, state: xs.state }
+    // A walsender can miss the wakeup for a commit and sleep until its timeout; a status update wakes it.
+    const status = setInterval(() => stream.writable && pong(), statusInterval)
+    stream.on('close', () => clearInterval(status))
 
-    function error(e) {
-      console.error('Unexpected error during logical streaming - reconnecting', e) // eslint-disable-line
-    }
+    return { stream, state: xs.state }
 
     function data(x) {
       if (x[0] === 0x77) {
-        parse(x.subarray(25), state, sql.options.parsers, handle, options.transform)
+        try {
+          parse(x.subarray(25), state, sql.options.parsers, handle, options.transform)
+        } catch (error) {
+          skip(x.subarray(25), error)
+        }
       } else if (x[0] === 0x6b && x[17]) {
         state.lsn = x.subarray(1, 9)
         pong()
@@ -119,12 +173,26 @@ export default function Subscribe(postgres, options) {
 
     function handle(a, b) {
       const path = b.relation.schema + '.' + b.relation.table
+          , key = b.relation.keys.length && '=' + b.relation.keys.map(x => a[x.name])
       call('*', a, b)
       call('*:' + path, a, b)
-      b.relation.keys.length && call('*:' + path + '=' + b.relation.keys.map(x => a[x.name]), a, b)
+      key && call('*:' + path + key, a, b)
       call(b.command, a, b)
       call(b.command + ':' + path, a, b)
-      b.relation.keys.length && call(b.command + ':' + path + '=' + b.relation.keys.map(x => a[x.name]), a, b)
+      key && call(b.command + ':' + path + key, a, b)
+    }
+
+    function skip(x, error) {
+      const command = commands[x[0]]
+          , relation = command && state[x.readUInt32BE(1)]
+
+      if (!relation)
+        throw error
+
+      const path = relation.schema + '.' + relation.table
+          , events = ['*', '*:' + path, command, command + ':' + path]
+
+      report(error, event => events.includes(event.split('=')[0]))
     }
 
     function pong() {
@@ -137,7 +205,7 @@ export default function Subscribe(postgres, options) {
   }
 
   function call(x, a, b) {
-    subscribers.has(x) && subscribers.get(x).forEach(({ fn }) => fn(a, b, x))
+    subscribers.has(x) && subscribers.get(x).forEach(({ fn }) => callback(fn, a, b, x))
   }
 }
 
@@ -162,19 +230,24 @@ function parse(x, state, parsers, handle, transform) {
       let columnIndex = 0
         , column
 
-      while (i < x.length) {
-        column = r.columns[columnIndex++] = {
-          key: x[i++],
-          name: transform.column.from
-            ? transform.column.from(x.toString('utf8', i, i = x.indexOf(0, i)))
-            : x.toString('utf8', i, i = x.indexOf(0, i)),
-          type: x.readUInt32BE(i += 1),
-          parser: parsers[x.readUInt32BE(i)],
-          atttypmod: x.readUInt32BE(i += 4)
+      try {
+        while (i < x.length) {
+          column = r.columns[columnIndex++] = {
+            key: x[i++],
+            name: transform.column.from
+              ? transform.column.from(x.toString('utf8', i, i = x.indexOf(0, i)))
+              : x.toString('utf8', i, i = x.indexOf(0, i)),
+            type: x.readUInt32BE(i += 1),
+            parser: parsers[x.readUInt32BE(i)],
+            atttypmod: x.readUInt32BE(i += 4)
+          }
+
+          column.key && r.keys.push(column)
+          i += 4
         }
-
-        column.key && r.keys.push(column)
-        i += 4
+      } catch (error) {
+        r.error = error
+        r.failed = true
       }
     },
     Y: () => { /* noop */ }, // Type
@@ -185,7 +258,7 @@ function parse(x, state, parsers, handle, transform) {
     },
     I: x => { // Insert
       let i = 1
-      const relation = state[x.readUInt32BE(i)]
+      const relation = decodable(state[x.readUInt32BE(i)])
       const { row } = tuples(x, relation.columns, i += 7, transform)
 
       handle(row, {
@@ -195,7 +268,7 @@ function parse(x, state, parsers, handle, transform) {
     },
     D: x => { // Delete
       let i = 1
-      const relation = state[x.readUInt32BE(i)]
+      const relation = decodable(state[x.readUInt32BE(i)])
       i += 4
       const key = x[i] === 75
       handle(key || x[i] === 79
@@ -209,7 +282,7 @@ function parse(x, state, parsers, handle, transform) {
     },
     U: x => { // Update
       let i = 1
-      const relation = state[x.readUInt32BE(i)]
+      const relation = decodable(state[x.readUInt32BE(i)])
       i += 4
       const key = x[i] === 75
       const xs = key || x[i] === 79
@@ -232,6 +305,13 @@ function parse(x, state, parsers, handle, transform) {
   }).reduce(char, {})[x[0]](x)
 }
 
+function decodable(relation) {
+  if (relation.failed)
+    throw relation.error
+
+  return relation
+}
+
 function tuples(x, columns, xi, transform) {
   let type
     , column
@@ -265,7 +345,7 @@ function tuples(x, columns, xi, transform) {
 }
 
 function parseEvent(x) {
-  const xs = x.match(/^(\*|insert|update|delete)?:?([^.]+?\.?[^=]+)?=?(.+)?/i) || []
+  const xs = x.match(/^(\*|insert|update|delete)?:?([^.=]+(?:\.[^=]+)?)?=?(.+)?/i) || []
 
   if (!xs)
     throw new Error('Malformed subscribe pattern: ' + x)

package/types/index.d.ts

@@ -67,14 +67,11 @@ interface BaseOptions<T extends Record<string, postgres.PostgresType>> {
    * @default true
    */
   prepare: boolean;
-  /**
-   * Called when a notice is received
-   * @default console.log
-   */
-  onnotice: (notice: postgres.Notice) => void;
-  /** (key; value) when a server param change */
+  /** Called when a notice is received, console.log by default and false to ignore them; if it throws, the error is rethrown as an uncaught exception */
+  onnotice: ((notice: postgres.Notice) => void) | false;
+  /** (key; value) when a server param change; if it throws, the error is rethrown as an uncaught exception */
   onparameter: (key: string, value: any) => void;
-  /** Is called with (connection; query; parameters) */
+  /** Is called with (connection; query; parameters); if it throws, the error is rethrown as an uncaught exception and the query still runs */
   debug: boolean | ((connection: number, query: string, parameters: any[], paramTypes: any[]) => void);
   /** Transform hooks */
   transform: {
@@ -118,6 +115,7 @@ interface BaseOptions<T extends Record<string, postgres.PostgresType>> {
    * @default 'alltables'
    */
   publications: string
+  /** Called with the connection id each time a connection closes; if it throws, the error is rethrown as an uncaught exception */
   onclose: (connId: number) => void;
   /** Called with the connection id each time a connection returns to the pool idle and its idle timer starts */
   onidle: (connId: number) => void;
@@ -463,7 +461,8 @@ declare namespace postgres {
     | 'COPY_IN_PROGRESS'
     | 'INVALID_TRANSACTION_NAME'
     | 'TARGET_SESSION_ATTRS'
-    | 'UNSAFE_TRANSACTION';
+    | 'UNSAFE_TRANSACTION'
+    | 'TRANSACTION_ROLLED_BACK';
     message: string;
   }
 
@@ -745,7 +744,8 @@ declare namespace postgres {
 
     listen(channel: string, onnotify: (value: string) => void, onlisten?: (() => void) | undefined): ListenRequest;
 
-    subscribe(event: string, cb: (row: Row | null, info: ReplicationEvent) => void, onsubscribe?: (() => void), onerror?: (() => any)): Promise<SubscriptionHandle>;
+    /** `onerror` receives replication connection errors and errors thrown while a change is decoded; such a change is skipped. */
+    subscribe(event: string, cb: (row: Row | null, info: ReplicationEvent) => void, onsubscribe?: (() => void), onerror?: ((error: Error) => void)): Promise<SubscriptionHandle>;
 
     largeObject(oid?: number | undefined, /** @default 0x00020000 | 0x00040000 */ mode?: number | undefined): Promise<LargeObject>;