@weave-js/core 0.10.0 → 0.11.0

package/lib/transport/{message-handlers.js → messageHandlers.js}

@@ -4,10 +4,11 @@
  * Copyright 2021 Fachwerk
  */
 
-const { Transform } = require('stream')
+const { InboundTransformStream } = require('./InboundTransformStream')
 const { WeaveError, restoreError } = require('../errors')
 const { createContext } = require('../broker/context')
-const MessageTypes = require('./message-types')
+const { createMessage } = require('./createMessage')
+const MessageTypes = require('./messageTypes')
 
 /**
  * @typedef {import('../types').Transport} Transport
@@ -66,13 +67,25 @@ module.exports = (runtime, transport) => {
 
     if (!stream) {
       isNew = true
-      stream = new Transform({
-        objectMode: payload.meta && payload.meta.$isObjectModeStream,
-        transform: function (chunk, encoding, done) {
-          this.push(chunk)
-          return done()
+      stream = new InboundTransformStream(
+        payload.sender,
+        payload.id, {
+          objectMode: payload.meta && payload.meta.$isObjectModeStream
         }
-      })
+      )
+
+      // handle backpressure
+      if (runtime.options.transport.streams.handleBackpressure) {
+        stream.on('backpressure', async ({ sender, requestId }) => {
+          const message = createMessage(MessageTypes.MESSAGE_REQUEST_STREAM_BACKPRESSURE, sender, { id: requestId })
+          await transport.send(message)
+        })
+
+        stream.on('resume_backpressure', async ({ sender, requestId }) => {
+          const message = createMessage(MessageTypes.MESSAGE_REQUEST_STREAM_RESUME, sender, { id: requestId })
+          await transport.send(message)
+        })
+      }
 
       stream.$prevSeq = -1
       stream.$pool = new Map()
@@ -126,13 +139,25 @@ module.exports = (runtime, transport) => {
     if (!stream) {
       transport.log.debug(`New stream from node ${payload.sender} received. Seq: ${payload.sequence}`)
 
-      stream = new Transform({
-        objectMode: payload.meta && payload.meta.$isObjectModeStream,
-        transform: function (chunk, encoding, done) {
-          this.push(chunk)
-          return done()
+      stream = new InboundTransformStream(
+        payload.sender,
+        payload.id, {
+          objectMode: payload.meta && payload.meta.$isObjectModeStream
         }
-      })
+      )
+
+      // handle backpressure
+      if (runtime.options.transport.streams.handleBackpressure) {
+        stream.on('backpressure', async ({ sender, requestId }) => {
+          const message = createMessage(MessageTypes.MESSAGE_RESPONSE_STREAM_BACKPRESSURE, sender, { id: requestId })
+          await transport.send(message)
+        })
+
+        stream.on('resume_backpressure', async ({ sender, requestId }) => {
+          const message = createMessage(MessageTypes.MESSAGE_RESPONSE_STREAM_RESUME, sender, { id: requestId })
+          await transport.send(message)
+        })
+      }
 
       stream.$prevSeq = -1
       stream.$pool = new Map()
@@ -232,10 +257,10 @@ module.exports = (runtime, transport) => {
       }
 
       return localRequestProxy(context)
-        .then(data => transport.response(sender, payload.id, data, context.meta, null))
-        .catch(error => transport.response(sender, payload.id, null, context.meta, error))
+        .then(data => transport.sendResponse(sender, payload.id, data, context.meta, null))
+        .catch(error => transport.sendResponse(sender, payload.id, null, context.meta, error))
     } catch (error) {
-      return transport.response(sender, payload.id, null, payload.meta, error)
+      return transport.sendResponse(sender, payload.id, null, payload.meta, error)
     }
   }
 
@@ -244,7 +269,7 @@ module.exports = (runtime, transport) => {
    * @param {any} payload - Payload
    * @returns {Promise} Promise
   */
-  const onResponse = payload => {
+  const onResponse = (payload) => {
     const id = payload.id
     const request = transport.pending.requests.get(id)
 
@@ -295,7 +320,7 @@ module.exports = (runtime, transport) => {
    * @returns {Promise} Promise
   */
   const onPing = payload => {
-    const message = transport.createMessage(MessageTypes.MESSAGE_PONG, payload.sender, {
+    const message = createMessage(MessageTypes.MESSAGE_PONG, payload.sender, {
       dispatchTime: payload.dispatchTime,
       arrivalTime: Date.now()
     })
@@ -388,6 +413,38 @@ module.exports = (runtime, transport) => {
     }
   }
 
+  const onResponseStreamBackpressure = (payload) => {
+    const stream = transport.pending.outboundResponseStreams.get(payload.id)
+
+    if (stream) {
+      stream.pause()
+    }
+  }
+
+  const onResponseStreamResume = (payload) => {
+    const stream = transport.pending.outboundResponseStreams.get(payload.id)
+
+    if (stream) {
+      stream.resume()
+    }
+  }
+
+  const onRequestStreamBackpressure = (payload) => {
+    const stream = transport.pending.outboundRequestStreams.get(payload.id)
+
+    if (stream) {
+      stream.pause()
+    }
+  }
+
+  const onRequestStreamResume = (payload) => {
+    const stream = transport.pending.outboundRequestStreams.get(payload.id)
+
+    if (stream) {
+      stream.resume()
+    }
+  }
+
   return (type, data) => {
     try {
       if (data === null) {
@@ -403,9 +460,9 @@ module.exports = (runtime, transport) => {
       // todo: check protocol version
       // todo: check node ID conflict
 
-      // if (payload.sender === broker.nodeId) {
-      //   if (type === MessageTypes.MESSAGE_INFO && payload.instanceId !== broker.state.instanceId) {
-      //     return broker.fatalError('Broker has detected a node ID conflict. "nodeId" of broker needs to be unique. Broker will be stopped.')
+      // if (payload.sender === runtime.nodeId) {
+      //   if (type === MessageTypes.MESSAGE_INFO && payload.instanceId !== runtime.state.instanceId) {
+      //     return runtime.fatalError('Weave broker has detected a node ID conflict. "nodeId" of broker needs to be unique. Broker will be stopped.')
       //   }
       // }
 
@@ -437,6 +494,18 @@ module.exports = (runtime, transport) => {
       case MessageTypes.MESSAGE_EVENT:
         onEvent(payload)
         break
+      case MessageTypes.MESSAGE_RESPONSE_STREAM_BACKPRESSURE:
+        onResponseStreamBackpressure(payload)
+        break
+      case MessageTypes.MESSAGE_RESPONSE_STREAM_RESUME:
+        onResponseStreamResume(payload)
+        break
+      case MessageTypes.MESSAGE_REQUEST_STREAM_BACKPRESSURE:
+        onRequestStreamBackpressure(payload)
+        break
+      case MessageTypes.MESSAGE_REQUEST_STREAM_RESUME:
+        onRequestStreamResume(payload)
+        break
       }
 
       return true

package/lib/transport/messageTypes.js

@@ -0,0 +1,23 @@
+/*
+ * Author: Kevin Ries (kevin@fachw3rk.de)
+ * -----
+ * Copyright 2021 Fachwerk
+*/
+exports.MESSAGE_UNKNOWN = 'unknown'
+exports.MESSAGE_DISCOVERY = 'discovery'
+exports.MESSAGE_INFO = 'info'
+exports.MESSAGE_REQUEST = 'request'
+exports.MESSAGE_RESPONSE = 'response'
+exports.MESSAGE_PING = 'ping'
+exports.MESSAGE_PONG = 'pong'
+exports.MESSAGE_DISCONNECT = 'disconnect'
+exports.MESSAGE_HEARTBEAT = 'heartbeat'
+exports.MESSAGE_EVENT = 'event'
+exports.MESSAGE_GOSSIP_HELLO = 'gossip_hello'
+exports.MESSAGE_GOSSIP_RESPONSE = 'gossip_response'
+exports.MESSAGE_GOSSIP_REQUEST = 'gossip_request'
+exports.MESSAGE_EVENT_ACK = 'event_ack'
+exports.MESSAGE_RESPONSE_STREAM_BACKPRESSURE = 'response_stream_backpressure'
+exports.MESSAGE_RESPONSE_STREAM_RESUME = 'response_stream_resume'
+exports.MESSAGE_REQUEST_STREAM_BACKPRESSURE = 'request_stream_backpressure'
+exports.MESSAGE_REQUEST_STREAM_RESUME = 'request_stream_resume'

package/lib/types.js

@@ -1,5 +1,5 @@
 // eslint-disable-next-line no-unused-vars
-const { EventEmitter2 } = require('eventemitter2')
+const { EventEmitter2: EventEmitter } = require('eventemitter2')
 
 /**
  * @typedef {import('stream').Stream} Stream
@@ -58,8 +58,8 @@ const { EventEmitter2 } = require('eventemitter2')
 /**
  * ActionInvoker
  * @typedef {object} ActionInvoker
- * @property {function(string, object, ActionOptions):Promise<any>} call - Service changed delegate
- * @property {function(string, object, ActionOptions):Promise<any>} call - Service changed delegate
+ * @property {function(string, object, ActionOptions):Promise<any>} call - Call a
+ * @property {function(string, object, ActionOptions):Promise<any>} multiCall - Service changed delegate
 */
 
 /**
@@ -76,7 +76,7 @@ const { EventEmitter2 } = require('eventemitter2')
  * @property {string} version - Weave version of the broker.
  * @property {ActionInvoker} actionInvoker - Action invoker
  * @property {EventBus} [eventBus] - Service event bus
- * @property {EventEmitter2} bus - Instance event bus
+ * @property {EventEmitter} bus - Instance event bus
  * @property {Broker} [broker] - Broker reference.
  * @property {BrokerOptions} options - options
  * @property {RuntimeInstanceState} state - Instance state
@@ -111,11 +111,11 @@ const { EventEmitter2 } = require('eventemitter2')
  * @property {function():Promise<any>} start - Start the broker.
  * @property {function():Promise<any>} stop - Stop the broker
  * @property {function(ServiceSchema):Service} createService - Create a new service with a service schema object.
- * @property {function():void} loadService - Load a service from a given path.
- * @property {function():void} loadServices - Load all services from a given directory.
+ * @property {function(string):void} loadService - Load a service from a given path.
+ * @property {function(string=, string=):void} loadServices - Load all services from a given directory.
  * @property {ContextFactory} contextFactory - contextFactory
  * @property {Logger} log - Logger instance
- * @property {function():Logger} createLogger - Create a new logger instace.
+ * @property {function(string, any):Logger} createLogger - Create a new logger instace.
  * @property {Cache} [cache] - Cache
  * @property {function():string} getUUID - Create a new from the UUID factory.
  * @property {Registry} registry - Service registry reference
@@ -128,11 +128,16 @@ const { EventEmitter2 } = require('eventemitter2')
  * @property {function(string, any, any=):Promise<any>} broadcast broadcast
  * @property {function(string, any, any=):Promise<any>} broadcastLocal broadcastLocal
  * @property {function(string):Promise<any>} waitForServices waitForServices
- * @property {Promise<PingResult>} ping ping
- * @property {function(Error)} handleError handleError
- * @property {void} fatalError fatalError
+ * @property {function(string, number=):Promise<PingResult>} ping ping
+ * @property {function(Error):void} handleError handleError
+ * @property {function():void} fatalError fatalError
 */
 
+/**
+ * Ping result
+ * @typedef {Object<string, number>} PingResult
+ */
+
 /**
  * Configuration object for weave service broker.
  * @typedef {Object} BulkheadOptions
@@ -151,12 +156,20 @@ const { EventEmitter2 } = require('eventemitter2')
  * @property {Array<Number>} defaultBuckets Default buckets for histograms.
 */
 
+/**
+ * Configuration object for weave service broker.
+ * @typedef {Object} TracingErrorOptions
+ * @property {Array<string>} fields - Fields of errors, that should be included in spans.
+ * @property {Boolean} stackTrace - Include stacktrace in spans
+*/
+
 /**
  * Configuration object for weave service broker.
  * @typedef {Object} TracingOptions
  * @property {Boolean} enabled - Enable tracing middleware. (default = false)
  * @property {Number} samplingRate - Rate of traced actions. (default = 1.0)
  * @property {Array<String|Object>} collectors - Array of tracing collectors.
+ * @property {TracingErrorOptions} errors - Settings for tracing errors.
 */
 
 /**
@@ -184,6 +197,12 @@ const { EventEmitter2 } = require('eventemitter2')
  * @property {Number} retries Number of attempts before the action call gets rejected. (default = 5)
  */
 
+/**
+ * Configuration object for weave service broker.
+ * @typedef {Object} TransportStreamOptions
+ * @property {Boolean} handleBackpressure Handle stream backpressure. (default = true)
+*/
+
 /**
  * Configuration object for weave service broker.
  * @typedef {Object} TransportOptions
@@ -195,6 +214,7 @@ const { EventEmitter2 } = require('eventemitter2')
  * @property {Number} offlineNodeCheckInterval - Interval in milliseconds to check and remove not offline nodes. (default = 30000ms)
  * @property {Number} maxOfflineTime - Maximum time a node can be offline before it is removed from the registry. (default = 600000ms)
  * @property {Number} maxChunkSize - Maximum chunk size for streams. (default = 256 * 1024 Bits)
+ * @property {TransportStreamOptions} streams - Transport stream options.
  */
 
 /**
@@ -264,26 +284,6 @@ const { EventEmitter2 } = require('eventemitter2')
  * @typedef {Object.<string, *>}
 */
 
-// Cache
-
-/**
- * Cache interface
- * @typedef Cache
- * @property {string} [name] name
- * @property {LoggerOptions} options options
- * @property {void} init init
- * @property {Logger} log log
- * @property {Promise<any>} set set
- * @property {Promise<any>} get get
- * @property {Promise<any>} remove remove
- * @property {Promise<any>} clear clear
- * @property {string} getCachingHash getCachingHash
- * @property {Middleware} createMiddleware createMiddleware
- * @property {Promise<any>} stop stop
-*/
-
-// Context
-
 /**
  * @export
  * @typedef ContextFactory
@@ -329,11 +329,11 @@ const { EventEmitter2 } = require('eventemitter2')
  * @property {any} [metrics] metrics
  * @property {function(any):void} setData - Set data object.
  * @property {function(string, any, ActionOptions):ContextPromise} call - Call a service action
- * @property {*} emit emit
+ * @property {function(string, object):Promise<any>} emit emit
  * @property {Stream} [stream] - Stream
- * @property {*} broadcast broadcast
- * @property {*} startSpan startSpan
- * @property {*} finishSpan finishSpan
+ * @property {function(string, any):Promise<any>} broadcast Broadcast an event to all listener.
+ * @property {function():void} startSpan startSpan
+ * @property {function():void} finishSpan finishSpan
  * @property {function():Context} copy - Copy the current context.
  * @property {function(Stream):void} setStream - Set the context data stream.
  * @property {function(Endpoint):void} setEndpoint - Set data object.
@@ -351,8 +351,8 @@ const { EventEmitter2 } = require('eventemitter2')
  * @property {Promise<any>} remove remove
  * @property {Promise<any>} clear clear
  * @property {string} getCachingHash getCachingHash
- * @property {Middleware} createMiddleware createMiddleware
- * @property {Promise<any>} stop stop
+ * @property {function():Middleware} createMiddleware createMiddleware
+ * @property {function():Promise<any>} stop stop
 */
 
 // Transport
@@ -368,9 +368,9 @@ const { EventEmitter2 } = require('eventemitter2')
  * @property {PendingStore} pending pending
  * @property {Function} resolveConnect resolveConnect
  * @property {string} adapterName adapterName
- * @property {Promise<any>} connect connect
- * @property {Promise<any>} disconnect disconnect
- * @property {void} setReady setReady
+ * @property {function():Promise<any>} connect connect
+ * @property {function():Promise<any>} disconnect disconnect
+ * @property {function():Promise<any>} setReady setReady
  * @property {function(TransportMessage):Promise<any>} send send
  * @property {any} sendNodeInfo sendNodeInfo
  * @property {function():Promise<any>} sendPing sendPing
@@ -475,14 +475,14 @@ const { EventEmitter2 } = require('eventemitter2')
  * @property {Endpoint | WeaveError} getNextAvailableActionEndpoint getNextAvailableActionEndpoint
  * @property {function(ServiceActionCollectionListFilterParams=):Array<any>} getActionList getActionList
  * @property {function(string): void} deregisterServiceByNodeId deregisterServiceByNodeId
- * @property {function(string, number, nodeId):boolean} hasService hasService
+ * @property {function(string, number, string):boolean} hasService hasService
  * @property {function(string, string):Endpoint} getActionEndpointByNodeId getActionEndpointByNodeId
  * @property {function(string):EndpointCollection} getActionEndpoints getActionEndpoints
  * @property {function(ServiceAction):Endpoint} createPrivateActionEndpoint createPrivateActionEndpoint
  * @property {function(string):Endpoint} getLocalActionEndpoint getLocalActionEndpoint
- * @property {NodeInfo} getNodeInfo getNodeInfo
- * @property {NodeInfo} getLocalNodeInfo getLocalNodeInfo
- * @property {NodeInfo} generateLocalNodeInfo generateLocalNodeInfo
+ * @property {function():NodeInfo} getNodeInfo getNodeInfo
+ * @property {function():NodeInfo} getLocalNodeInfo getLocalNodeInfo
+ * @property {function():NodeInfo} generateLocalNodeInfo generateLocalNodeInfo
  * @property {*} processNodeInfo processNodeInfo
  * @property {function(string, boolean):void} nodeDisconnected nodeDisconnected
  * @property {function(string):void} removeNode removeNode
@@ -534,7 +534,6 @@ const { EventEmitter2 } = require('eventemitter2')
 
 /**
  * Endpoint
- *
  * @typedef Endpoint
  * @property {Node} node node
  * @property {Service} service service
@@ -549,7 +548,7 @@ const { EventEmitter2 } = require('eventemitter2')
 /**
  * Event colleciton
  * @typedef EventCollection
- * @property {function(Node, Service, eventCollection):Endpoint} add add
+ * @property {function(Node, Service, EventCollection):Endpoint} add add
  * @property {Endpoint} get get
  * @property {function(string, Node):void} remove remove
  * @property {function(Service):void} removeByService removeByService
@@ -593,8 +592,8 @@ const { EventEmitter2 } = require('eventemitter2')
  * @typedef ServiceActionCollection
  * @property {*} add add
  * @property {function(string):EndpointCollection} get get
- * @property {function(service)} removeByService removeByService
- * @property {function(string, Node)} remove remove
+ * @property {function(Service):void} removeByService removeByService
+ * @property {function(string, Node):void} remove remove
  * @property {function(ServiceActionCollectionListFilterParams):Array<any>} list list
 */
 
@@ -622,7 +621,7 @@ const { EventEmitter2 } = require('eventemitter2')
  * @property {boolean} [withEvents=false] Include events in result.
  * @property {boolean} [withNodeService=false] Include node service.
  * @property {boolean} [withSettings=false] Include service settings.
- */
+*/
 
 /**
  * Service collection
@@ -725,6 +724,7 @@ const { EventEmitter2 } = require('eventemitter2')
  * @property {void} timer timer
  * @property {Metric} getMetric getMetric
  * @property {any} list list
+ * @property {function():Promise<any>} stop list
 */
 
 // Midlewares
@@ -734,7 +734,7 @@ const { EventEmitter2 } = require('eventemitter2')
  * @export
  * @typedef MiddlewareHandler
  * @property {void} init init
- * @property {void} add add
+ * @property {function(Middleware)} add add
  * @property {any} wrapMethod wrapMethod
  * @property {any} wrapHandler wrapHandler
  * @property {any} callHandlersAsync callHandlersAsync

package/lib/utils/options.js

@@ -1,4 +1,3 @@
-
 const {
   clone,
   compact,
@@ -38,6 +37,7 @@ function mergeActions (source, targetSchema) {
   return targetSchema
 }
 
+// Merge events
 function mergeEvents (source, targetSchema) {
   Object.keys(source).map(key => {
     const sourceEvent = wrapHandler(source[key])
@@ -78,7 +78,7 @@ function mergeLifecicleHooks (source, targetSchema) {
   return compact(flatten([targetSchema, source]))
 }
 
-exports.mergeSchemas = (mixin, targetSchema) => {
+function mergeSchemas (mixin, targetSchema) {
   const mixinSchema = clone(mixin)
   const resultSchema = clone(targetSchema)
 
@@ -112,3 +112,5 @@ exports.mergeSchemas = (mixin, targetSchema) => {
 
   return mixinSchema
 }
+
+module.exports = { mergeSchemas }

package/lib/utils/{restore-error.js → restoreError.js}

@@ -1,4 +1,4 @@
-const errors = require('../new-errors')
+const errors = require('../errors')
 
 exports.restoreError = (error) => {
   const ErrorClass = errors[error.name]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@weave-js/core",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "The core package of weave",
   "keywords": [
     "Weave",
@@ -40,20 +40,14 @@
   "license": "MIT",
   "dependencies": {
     "@weave-js/errors": "^0.9.1",
-    "@weave-js/utils": "^0.9.1",
-    "@weave-js/validator": "^0.10.0",
-    "eventemitter2": "^6.4.4",
-    "glob": "^7.1.7",
-    "node-fetch": "^2.6.1"
-  },
-  "devDependencies": {
-    "eslint": "^7.30.0",
-    "jest": "^27.0.6",
-    "jest-cli": "^27.0.6"
+    "@weave-js/utils": "^0.10.0",
+    "@weave-js/validator": "^0.11.0",
+    "eventemitter2": "^6.4.5",
+    "glob": "^7.2.0"
   },
   "directories": {
     "lib": "lib",
     "test": "test"
   },
-  "gitHead": "1bb4b387d36d5f09ebdfc541ca677cf2b718dbac"
+  "gitHead": "84fd5e4af3af96e8c004bcfba40bb3cb35f23fa7"
 }