@weave-js/core 0.9.4 → 0.11.1

package/lib/transport/createTransport.js

@@ -1,7 +1,7 @@
 /*
  * Author: Kevin Ries (kevin@fachw3rk.de)
  * -----
- * Copyright 2020 Fachwerk
+ * Copyright 2021 Fachwerk
  */
 
 /**
@@ -9,13 +9,16 @@
  * @typedef {import('../types.js').TransportAdapter} TransportAdapter
  * @typedef {import('../types.js').Transport} Transport
  * @typedef {import('../types').TransportMessage} TransportMessage
+ * @typedef {import('../types').Context} Context
 */
 
 // Own packages
 const { WeaveError, WeaveQueueSizeExceededError } = require('../errors')
-const MessageTypes = require('./message-types')
+const { createMessage } = require('./createMessage')
+const MessageTypes = require('./messageTypes')
 const utils = require('@weave-js/utils')
-const createMessageHandler = require('./message-handlers')
+const createMessageHandler = require('./messageHandlers')
+const { errorPayloadFactory } = require('./errorPayloadFactory')
 
 /**
  * Create a Transport adapter
@@ -25,133 +28,24 @@ const createMessageHandler = require('./message-handlers')
 */
 exports.createTransport = (runtime, adapter) => {
   const transport = Object.create(null)
-  const { nodeId, middlewareHandler } = runtime
+  const { nodeId, middlewareHandler, createLogger } = runtime
 
   let heartbeatTimer
   let checkNodesTimer
   let checkOfflineNodesTimer
   let updateLocalNodeTimer
 
-  const log = runtime.createLogger('TRANSPORT')
-
+  // Pending action store
   const pending = {
     requests: new Map(),
     requestStreams: new Map(),
-    responseStreams: new Map()
-  }
-
-  // Outgoing request
-  const doRequest = (context, resolve, reject) => {
-    const isStream = utils.isStream(context.options.stream)
-
-    const request = {
-      targetNodeId: context.nodeId,
-      action: context.action.name,
-      context,
-      resolve,
-      reject,
-      isStream
-    }
-
-    log.debug(`Send request for ${request.action} to node ${request.targetNodeId}.`)
-
-    pending.requests.set(context.id, request)
-
-    const payload = {
-      id: context.id,
-      action: context.action.name,
-      data: context.data,
-      timeout: context.options.timeout,
-      meta: context.meta,
-      level: context.level,
-      metrics: context.metrics,
-      requestId: context.requestId,
-      parentId: context.parentId,
-      isStream
-    }
-
-    // Handle object mode streams
-    if (isStream) {
-      if (utils.isStreamObjectMode(context.options.stream)) {
-        payload.meta = payload.meta || {}
-        payload.meta.$isObjectModeStream = true
-      }
-      payload.sequence = 0
-    }
-
-    const message = transport.createMessage(MessageTypes.MESSAGE_REQUEST, context.nodeId, payload)
-
-    return transport.send(message)
-      .then(() => {
-        // send stream
-        if (isStream) {
-          const stream = context.options.stream
-          payload.meta = {}
-
-          if (utils.isStreamObjectMode(context.options.stream)) {
-            payload.meta.$isObjectModeStream = true
-          }
-
-          stream.on('data', data => {
-            stream.pause()
-            const chunks = []
-
-            // The chunk is larger than maxBufferSize
-            if (data instanceof Buffer && runtime.options.transport.maxChunkSize > 0 && data.length > runtime.options.transport.maxChunkSize) {
-              const length = data.length
-              let i = 0
-              while (i < length) {
-                chunks.push(data.slice(i, i += runtime.options.transport.maxChunkSize))
-              }
-            } else {
-              chunks.push(data)
-            }
-
-            // Send chunks from chunk buffer
-            for (const chunk of chunks) {
-              const payloadCopy = Object.assign({}, payload)
-              payloadCopy.sequence = ++payload.sequence
-              payloadCopy.chunk = chunk
-              payloadCopy.isStream = true
-              payload.data = null
-
-              const message = transport.createMessage(MessageTypes.MESSAGE_REQUEST, context.nodeId, payloadCopy)
-              transport.send(message)
-            }
-
-            // resume stream
-            stream.resume()
-            return
-          })
-
-          stream.on('end', () => {
-            const payloadCopy = Object.assign({}, payload)
-            payloadCopy.sequence = ++payload.sequence
-            payloadCopy.chunk = null
-            payloadCopy.isStream = false
-            const message = transport.createMessage(MessageTypes.MESSAGE_REQUEST, context.nodeId, payloadCopy)
-            return transport.send(message)
-          })
-
-          stream.on('error', (_) => {
-            const payloadCopy = Object.assign({}, payload)
-            payloadCopy.sequence = ++payload.sequence
-            payloadCopy.chunk = null
-            payloadCopy.isStream = false
-
-            // todo: Attach error on payload
-
-            const message = transport.createMessage(MessageTypes.MESSAGE_REQUEST, context.nodeId, payloadCopy)
-            return transport.send(message)
-          })
-        }
-      })
-      .catch((error) => {
-        reject(error)
-      })
+    responseStreams: new Map(),
+    outboundResponseStreams: new Map(),
+    outboundRequestStreams: new Map()
   }
 
-  transport.log = runtime.createLogger('TRANSPORT')
+  transport.log = createLogger('TRANSPORT')
+  transport.reconnectInProgress = false
   transport.isConnected = false
   transport.isDisconnecting = false
   transport.isReady = false
@@ -167,24 +61,24 @@ exports.createTransport = (runtime, adapter) => {
     }
   }
 
+  const getErrorPayload = errorPayloadFactory(runtime)
+
   transport.connect = () => {
     return new Promise(resolve => {
       transport.resolveConnect = resolve
       transport.log.info('Connecting to transport adapter...')
 
       const doConnect = (isTryReconnect) => {
-        let reconnectInProgress = false
-
         const errorHandler = (error) => {
           // Skip reconnect, if the adapter is disconnecting or an reconnect is in progress.
-          if (transport.isDisconnecting || reconnectInProgress) {
+          if (transport.isDisconnecting || transport.reconnectInProgress) {
             return
           }
 
           transport.log.warn('Connection failed')
           transport.log.debug(error)
 
-          reconnectInProgress = true
+          transport.reconnectInProgress = true
 
           if (!error.skipRetry) {
             setTimeout(() => {
@@ -207,11 +101,11 @@ exports.createTransport = (runtime, adapter) => {
     transport.isConnected = false
     transport.isReady = false
 
-    runtime.eventBus.broadcastLocal('$transporter.disconnected', { isGracefull: true })
+    runtime.eventBus.broadcastLocal('$transporter.disconnected', { isGraceful: true })
 
     stopTimers()
 
-    const message = transport.createMessage(MessageTypes.MESSAGE_DISCONNECT)
+    const message = createMessage(MessageTypes.MESSAGE_DISCONNECT)
     return transport.send(message)
       .then(() => {
         adapter.close()
@@ -221,10 +115,10 @@ exports.createTransport = (runtime, adapter) => {
       })
   }
 
-  transport.setReady = () => {
+  transport.setReady = async () => {
     if (transport.isConnected) {
       transport.isReady = true
-      transport.sendNodeInfo()
+      await transport.sendNodeInfo()
     }
   }
 
@@ -233,16 +127,18 @@ exports.createTransport = (runtime, adapter) => {
    * @param {*} sender sender node ID.
    * @returns {Promise} Promise
   */
-  transport.sendNodeInfo = (sender) => {
+  transport.sendNodeInfo = async (sender) => {
     if (!transport.isConnected || !transport.isReady) {
       return Promise.resolve()
     }
 
+    // Collect own node info and send it to remote nodes
     const info = runtime.registry.getLocalNodeInfo()
-    const message = transport.createMessage(MessageTypes.MESSAGE_INFO, sender, {
+    const message = createMessage(MessageTypes.MESSAGE_INFO, sender, {
       ...info,
       instanceId: runtime.state.instanceId
     })
+
     return transport.send(message)
   }
 
@@ -251,28 +147,42 @@ exports.createTransport = (runtime, adapter) => {
   * @param {TransportMessage} message Message to send
   * @returns {Promise} Promise
   */
-  transport.send = (message) => {
+  transport.send = async (message) => {
     transport.statistics.sent.packages = transport.statistics.sent.packages + 1
-    log.verbose(`Send ${message.type.toUpperCase()} packet to ${message.targetNodeId || 'all nodes'}`)
+    transport.log.verbose(`Send ${message.type.toUpperCase()} packet to ${message.targetNodeId || 'all nodes'}`)
     return adapter.preSend(message)
   }
 
   transport.sendPing = (nodeId) => {
-    const pingMessage = transport.createMessage(MessageTypes.MESSAGE_PING, nodeId, { dispatchTime: Date.now() })
+    const pingMessage = createMessage(MessageTypes.MESSAGE_PING, nodeId, { dispatchTime: Date.now() })
     return transport.send(pingMessage)
   }
 
+  /**
+   * Send discovery message to all nodes.
+   * @returns {void}
+  */
   transport.discoverNodes = () => {
-    const discoveryMessage = transport.createMessage(MessageTypes.MESSAGE_DISCOVERY)
+    const discoveryMessage = createMessage(MessageTypes.MESSAGE_DISCOVERY)
     transport.send(discoveryMessage)
   }
 
-  transport.discoverNode = (target) => {
-    const discoveryMessage = transport.createMessage(MessageTypes.MESSAGE_DISCOVERY, target)
-    transport.send(discoveryMessage)
+  /**
+   * Send discovery message to all nodes.
+   * @param{string} target - Target node ID
+   * @returns {Promise<void>} - Promise
+  */
+  transport.discoverNode = async (target) => {
+    const discoveryMessage = createMessage(MessageTypes.MESSAGE_DISCOVERY, target)
+    return transport.send(discoveryMessage)
   }
 
-  transport.sendEvent = (context) => {
+  /**
+   * Send an event
+   * @param {Context} context - Context
+   * @returns {Promise<void>} - Promise
+   */
+  transport.sendEvent = async (context) => {
     const isBroadcast = context.eventType === 'broadcast'
 
     const payload = {
@@ -287,12 +197,12 @@ exports.createTransport = (runtime, adapter) => {
       isBroadcast
     }
 
-    const message = transport.createMessage(MessageTypes.MESSAGE_EVENT, context.endpoint ? context.nodeId : null, payload)
+    const message = createMessage(MessageTypes.MESSAGE_EVENT, context.endpoint ? context.nodeId : null, payload)
     return transport.send(message)
   }
 
   transport.sendBroadcastEvent = (nodeId, eventName, data, groups) => {
-    log.verbose(`Send ${eventName} to ${nodeId}`)
+    transport.log.verbose(`Send ${eventName} to ${nodeId}`)
 
     const payload = {
       data,
@@ -301,18 +211,20 @@ exports.createTransport = (runtime, adapter) => {
       isBroadcast: true
     }
 
-    const message = transport.createMessage(MessageTypes.MESSAGE_EVENT, nodeId, payload)
-    transport.send(message)
+    const message = createMessage(MessageTypes.MESSAGE_EVENT, nodeId, payload)
+    return transport.send(message)
   }
 
   transport.removePendingRequestsById = (requestId) => {
     pending.requests.delete(requestId)
     pending.requestStreams.delete(requestId)
     pending.responseStreams.delete(requestId)
+    pending.outboundResponseStreams.delete(requestId)
+    pending.outboundRequestStreams.delete(requestId)
   }
 
   transport.removePendingRequestsByNodeId = (nodeId) => {
-    log.debug(`Remove pending requests for node ${nodeId}.`)
+    transport.log.debug(`Remove pending requests for node ${nodeId}.`)
     pending.requests.forEach((request, requestId) => {
       if (request.nodeId === nodeId) {
         pending.requests.delete(requestId)
@@ -324,15 +236,7 @@ exports.createTransport = (runtime, adapter) => {
     })
   }
 
-  transport.createMessage = (type, targetNodeId, payload) => {
-    return {
-      type: type || MessageTypes.MESSAGE_UNKNOWN,
-      targetNodeId,
-      payload: payload || {}
-    }
-  }
-
-  transport.request = (context) => {
+  transport.sendRequest = (context) => {
     // If the queue size is set, check the queue size and reject the job when the limit is reached.
     if (runtime.options.transport.maxQueueSize && runtime.options.transport.maxQueueSize < pending.requests.size) {
       return Promise.reject(new WeaveQueueSizeExceededError({
@@ -343,10 +247,142 @@ exports.createTransport = (runtime, adapter) => {
       }))
     }
 
-    return new Promise((resolve, reject) => doRequest(context, resolve, reject))
+    return new Promise((resolve, reject) => {
+      const isStream = utils.isStream(context.options.stream)
+
+      const request = {
+        targetNodeId: context.nodeId,
+        action: context.action.name,
+        context,
+        resolve,
+        reject,
+        isStream
+      }
+
+      transport.log.debug(`Send request for ${request.action} to node ${request.targetNodeId}.`)
+
+      pending.requests.set(context.id, request)
+
+      const payload = {
+        id: context.id,
+        action: context.action.name,
+        data: context.data,
+        timeout: context.options.timeout,
+        meta: context.meta,
+        level: context.level,
+        metrics: context.metrics,
+        requestId: context.requestId,
+        parentId: context.parentId,
+        isStream
+      }
+
+      // Handle object mode streams
+      if (isStream) {
+        const stream = context.options.stream
+
+        if (!pending.outboundRequestStreams.has(payload.id)) {
+          pending.outboundRequestStreams.set(payload.id, stream)
+        }
+
+        if (utils.isStreamObjectMode(stream)) {
+          payload.meta = payload.meta || {}
+          payload.meta.$isObjectModeStream = true
+        }
+        payload.sequence = 0
+      }
+
+      const message = createMessage(MessageTypes.MESSAGE_REQUEST, context.nodeId, payload)
+
+      return transport.send(message)
+        .then(() => {
+          // send stream
+          if (isStream) {
+            const stream = context.options.stream
+            payload.meta = {}
+
+            if (utils.isStreamObjectMode(context.options.stream)) {
+              payload.meta.$isObjectModeStream = true
+            }
+
+            stream.on('data', data => {
+              stream.pause()
+              const chunks = []
+
+              // The chunk is larger than maxBufferSize
+              if (data instanceof Buffer && runtime.options.transport.maxChunkSize > 0 && data.length > runtime.options.transport.maxChunkSize) {
+                const length = data.length
+                let i = 0
+                while (i < length) {
+                  chunks.push(data.slice(i, i += runtime.options.transport.maxChunkSize))
+                }
+              } else {
+                chunks.push(data)
+              }
+
+              // Send chunks from chunk buffer
+              for (const chunk of chunks) {
+                const payloadCopy = Object.assign({}, payload)
+                payloadCopy.sequence = ++payload.sequence
+                payloadCopy.chunk = chunk
+                payloadCopy.isStream = true
+                payload.data = null
+
+                const message = createMessage(MessageTypes.MESSAGE_REQUEST, context.nodeId, payloadCopy)
+                transport.send(message)
+              }
+
+              // resume stream
+              stream.resume()
+              return
+            })
+
+            stream.on('end', () => {
+              const payloadCopy = Object.assign({}, payload)
+
+              payloadCopy.sequence = ++payload.sequence
+              payloadCopy.chunk = null
+              payloadCopy.isStream = false
+
+              pending.outboundRequestStreams.delete(payload.id)
+
+              const message = createMessage(MessageTypes.MESSAGE_REQUEST, context.nodeId, payloadCopy)
+              return transport.send(message)
+            })
+
+            stream.on('error', (error) => {
+              const payloadCopy = Object.assign({}, payload)
+              payloadCopy.sequence = ++payload.sequence
+              payloadCopy.chunk = null
+              payloadCopy.isStream = false
+
+              if (error) {
+                payloadCopy.success = false
+                payloadCopy.error = getErrorPayload(error)
+              }
+
+              pending.outboundRequestStreams.delete(payload.id)
+
+              const message = createMessage(MessageTypes.MESSAGE_REQUEST, context.nodeId, payloadCopy)
+              return transport.send(message)
+            })
+          }
+        })
+        .catch((error) => {
+          reject(error)
+        })
+    })
   }
 
-  transport.response = (target, contextId, data, meta, error) => {
+  /**
+   * Send response for an open request.
+   * @param {string} target - Target node ID
+   * @param {string} contextId - Context ID
+   * @param {any} data - Data
+   * @param {Object} meta - Meta data
+   * @param {WeaveError=} error - Error
+   * @returns {Promise<void>} - Promise
+  */
+  transport.sendResponse = (target, contextId, data, meta, error) => {
     // Check if data is a stream
     const isStream = utils.isStream(data)
 
@@ -360,19 +396,16 @@ exports.createTransport = (runtime, adapter) => {
 
     // If an error is occurs, we attach the an error object to the payload.
     if (error) {
-      payload.error = {
-        name: error.name,
-        message: error.message,
-        nodeId: error.nodeId || nodeId,
-        code: error.code,
-        type: error.type,
-        stack: error.stack,
-        data: error.data
-      }
+      payload.error = getErrorPayload(error)
     }
 
     if (isStream) {
       const stream = data
+
+      if (!pending.outboundResponseStreams.has(payload.id)) {
+        pending.outboundResponseStreams.set(payload.id, stream)
+      }
+
       payload.data = null
       payload.sequence = 0
       payload.isStream = true
@@ -406,9 +439,9 @@ exports.createTransport = (runtime, adapter) => {
           payloadCopy.sequence = ++payload.sequence
 
           payloadCopy.chunk = chunk
-          transport.log.debug('Send Stream chunk to ', target)
+          transport.log.debug(`Send Stream chunk to ${target}`)
 
-          const message = transport.createMessage(MessageTypes.MESSAGE_RESPONSE, target, payloadCopy)
+          const message = createMessage(MessageTypes.MESSAGE_RESPONSE, target, payloadCopy)
           transport.send(message)
         }
 
@@ -424,10 +457,11 @@ exports.createTransport = (runtime, adapter) => {
         payloadCopy.chunk = null
         payloadCopy.isStream = false
 
-        transport.log.debug('Send end stream chunk to ', target)
+        transport.log.debug(`Send end stream chunk to ${target}`)
 
-        const message = transport.createMessage(MessageTypes.MESSAGE_RESPONSE, target, payloadCopy)
+        const message = createMessage(MessageTypes.MESSAGE_RESPONSE, target, payloadCopy)
         transport.send(message)
+        pending.outboundResponseStreams.delete(payload.id)
       })
 
       stream.on('error', (error) => {
@@ -439,18 +473,23 @@ exports.createTransport = (runtime, adapter) => {
 
         if (error) {
           payloadCopy.success = false
+          payloadCopy.error = getErrorPayload(error)
         }
-        const message = transport.createMessage(MessageTypes.MESSAGE_RESPONSE, target, payloadCopy)
+
+        transport.log.debug(`Send closing chunk to ${target}`)
+
+        const message = createMessage(MessageTypes.MESSAGE_RESPONSE, target, payloadCopy)
         transport.send(message)
+        pending.outboundResponseStreams.delete(payload.id)
       })
 
       payload.data = null
-      const message = transport.createMessage(MessageTypes.MESSAGE_RESPONSE, target, payload)
+      const message = createMessage(MessageTypes.MESSAGE_RESPONSE, target, payload)
       return transport.send(message)
         .then(() => stream.resume())
     }
 
-    const message = transport.createMessage(MessageTypes.MESSAGE_RESPONSE, target, payload)
+    const message = createMessage(MessageTypes.MESSAGE_RESPONSE, target, payload)
     return transport.send(message)
   }
 
@@ -528,7 +567,11 @@ exports.createTransport = (runtime, adapter) => {
       adapter.subscribe(MessageTypes.MESSAGE_PONG, nodeId),
       adapter.subscribe(MessageTypes.MESSAGE_DISCONNECT),
       adapter.subscribe(MessageTypes.MESSAGE_HEARTBEAT),
-      adapter.subscribe(MessageTypes.MESSAGE_EVENT, nodeId)
+      adapter.subscribe(MessageTypes.MESSAGE_EVENT, nodeId),
+      adapter.subscribe(MessageTypes.MESSAGE_RESPONSE_STREAM_BACKPRESSURE, nodeId),
+      adapter.subscribe(MessageTypes.MESSAGE_RESPONSE_STREAM_RESUME, nodeId),
+      adapter.subscribe(MessageTypes.MESSAGE_REQUEST_STREAM_BACKPRESSURE, nodeId),
+      adapter.subscribe(MessageTypes.MESSAGE_REQUEST_STREAM_RESUME, nodeId)
     ])
   }
 
@@ -567,7 +610,7 @@ exports.createTransport = (runtime, adapter) => {
     const node = runtime.registry.nodeCollection.localNode
     node.updateLocalInfo()
 
-    log.verbose(`Send heartbeat from ${node.id}`)
+    transport.log.verbose(`Send heartbeat from ${node.id}`)
 
     const payload = {
       cpu: node.cpu,
@@ -575,7 +618,7 @@ exports.createTransport = (runtime, adapter) => {
       sequence: node.sequence
     }
 
-    const message = transport.createMessage(MessageTypes.MESSAGE_HEARTBEAT, null, payload)
+    const message = createMessage(MessageTypes.MESSAGE_HEARTBEAT, null, payload)
     transport.send(message)
   }
 

package/lib/transport/errorPayloadFactory.js

@@ -0,0 +1,18 @@
+/**
+ * Error Payload factory
+ * @param {import("../types").Runtime} runtime Runtime
+ * @returns {function(Error)} Generate error payload object
+*/
+const errorPayloadFactory = (runtime) => (error) => {
+  return {
+    name: error.name,
+    message: error.message,
+    nodeId: error.nodeId || runtime.nodeId,
+    code: error.code,
+    type: error.type,
+    stack: error.stack,
+    data: error.data
+  }
+}
+
+module.exports = { errorPayloadFactory }