hsdq 0.7.0

data/lib/hsdq/setting.rb

@@ -0,0 +1,130 @@
+
+module Hsdq
+  # This module provide the original setting for the hsdq class as well as some utility methods
+  module Setting
+
+    # Cached hash of the options thus avoiding to pass options all over the place.
+    # Initial state read the options from the config file if any provided and merge in it the opts
+    # parameter
+    # @param [Hash] opts The Options to be added/merged into the options from the config file
+    # @return [Hash] of the options
+    def hsdq_opts(opts={})
+      @hsdq_opts ||= initial_setup opts
+    end
+
+    # allow to add opions before listening
+    # @params [Hash] opts the options to add
+    def hsdq_add_options(opts)
+      if @hsdq_opts
+        @hsdq_opts.merge!(opts)
+      else
+        hsdq_opts(opts)
+      end
+    end
+
+    def initial_setup(opts)
+      options = read_opts.merge opts
+      set_abort_on_exception(options)
+      options
+    end
+
+    # @return [Hash] the default options
+    def default_opts
+      @default_opts ||= {
+        threaded: false,
+        timeout:  10
+      }
+    end
+
+    def cx_opts
+      @cx_options ||= {
+        message: {
+          host: '127.0.0.1',
+          port: 6379,
+          db:   2
+        },
+        admin: {
+          host: '127.0.0.1',
+          port: 6379,
+          db:   2
+        },
+        session: {
+          host: '127.0.0.1',
+          port: 6379,
+          db:   2
+        }
+      }.merge hsdq_opts[:redis] || {}
+    end
+
+    # Read the config file
+    # @param [String] file_path
+    # @return [Hash] options from defult and config
+    def read_opts(file_path=nil)
+      begin
+        default_opts.merge!(
+          deep_symbolize(YAML.load_file(file_path || config_file_path))[environment.to_sym])
+      rescue Errno::ENOENT => e
+        p "[warning] config file not read, using default options"
+        default_opts
+      end
+    end
+
+    # cached value for the environment based on Rails.env or command line parameter (scripts do not have Rails.env)
+    # @param [String] environment the environment to be force set or nil or nothing
+    # @return [String] The environment string
+    # @default 'development'
+    def environment(environment=nil)
+      @environment ||= environment_from_app(environment) || 'development'
+    end
+
+    def environment_from_app(environment)
+      environment || (defined?(Rails) ? Rails.env : nil) || (RAILS_ENV if defined? RAILS_ENV)
+    end
+
+    # @param [String] name the HsdqClassName
+    # @return [String] the channel based on the class name
+    def channel(name=nil)
+      @channel ||= name || snakify(hsdq_get_name.gsub(/^hsdq/i, ""))
+    end
+
+    # @return [String] class name
+    def hsdq_get_name
+      self.respond_to?(:name) ? self.name : self.class.name
+    end
+
+    # Force the channel to be set to any name
+    # @param [String] name
+    # @return [String] the new channel name
+    def channel=(name)
+      @channel = name
+    end
+
+    # @return [String] The name for the config file (cached)
+    def config_filename(filename=nil)
+      @config_filename ||= filename || "#{snakify(hsdq_get_name.gsub(/^hsdq/i, ""))}.yml"
+    end
+
+    # @param [String] path or nil or nothing. Force the path of a value is passed (cahed)
+    # @return [String] the path for the config folder, default to config relative the the actual path for a script
+    def config_path(path=nil)
+      @config_file_path ||= path || "#{(defined?(Rails) ? Rails.root : '.')}/config/"
+    end
+
+    # Cached path to the config file
+    # @param [String] config_file_path if passed force the value
+    # @return [String] The value for the path to the config file
+    def config_file_path(config_file_path=nil)
+      @config_file_path ||= config_file_path || File.join(config_path, config_filename)
+    end
+
+    # sets abort_on_exception for debugging based on environment or parameter
+    #
+    # @param [Hash] options If options[exception] true,
+    # the main thread will break if a child thread break which is what we want in development/test
+    # but we do not want that for production
+    def set_abort_on_exception(options)
+      options[:exceptions] ? Thread.abort_on_exception = true : Thread.abort_on_exception = false
+    end
+
+  end
+end

data/lib/hsdq/shared.rb

@@ -0,0 +1,42 @@
+
+module Hsdq
+  # The methods in this module are shared by different modules
+  module Shared
+
+    def placeholder
+      "This is a placeholder, you must implement this method in your hsdq class"
+    end
+
+    def valid_type?(type)
+      [:request, :ack, :callback, :feedback, :error].include? type.to_sym
+    end
+
+    # Build the namespaced key for the main hash storing the message history (collection of 'burst')
+    # @param [Hash] message_or_spark a burst or a spark
+    # @return [String] the unique key for the main redis hash
+    # @return nil if message_or_spark or the uid is nil
+    def hsdq_key(message_or_spark)
+      return unless message_or_spark
+      "hsdq_h_#{message_or_spark[:uid]}" if message_or_spark[:uid]
+    end
+
+    # Build the namespaced key for the spark and burst unique shared uid
+    # @param [Hash] spark
+    # @return [String] the unique namespaced key
+    # @return nil if spark or the spark_uid is nil
+    def burst_key(spark)
+      return unless spark
+      "#{spark[:type]}_#{spark[:spark_uid]}" if spark[:type] && spark[:spark_uid]
+    end
+
+    # Build the namespaced key for the main session hash
+    # @param [string] session_id the unique uid for the session
+    # @return [String] the unique namespaced key
+    # @return nil if session_id
+    def session_key(session_id)
+      return unless session_id
+      "hsdq_s_#{session_id}"
+    end
+
+  end
+end

data/lib/hsdq/thread_store.rb

@@ -0,0 +1,49 @@
+module Hsdq
+  module ThreadStore
+
+    # Specialized proxy for set_get the key is the method name
+    # @see #set_get
+    def context(data=nil)
+      set_get __method__, data
+    end
+
+    # Specialized proxy for set_get the key is the method name
+    # @see #set_get
+    def context_params(data=nil)
+      set_get __method__, data
+    end
+
+    # Specialized proxy for set_get the key is the method name
+    # @see #set_get
+    def current_uid(data=nil)
+      set_get __method__, data
+    end
+
+    # Specialized proxy for set_get the key is the method name
+    # @see #set_get
+    def previous_sender(data=nil)
+      set_get __method__, data
+    end
+
+    # Specialized proxy for set_get the key is the method name
+    # @see #set_get
+    def sent_to(data=nil)
+      set_get __method__, data
+    end
+
+    # Specialized proxy for set_get the key is the method name
+    # @see #set_get
+    def reply_to(data=nil)
+      set_get __method__, data
+    end
+
+    # Return the value stored into the corresponding thread.current key only if no data is passed
+    # @param [any value] data save data into the corresponding Thread.current key if data is passed
+    # @return [Stored value]
+    def set_get(key, data=nil)
+      Thread.current[key] = data if data
+      Thread.current[key]
+    end
+
+  end
+end

data/lib/hsdq/threadpool.rb

@@ -0,0 +1,58 @@
+module Hsdq
+  # This module is used to manage the threads
+  module Threadpool
+
+    # The maximum number of threads allowed to run at the same time. This is setter and a cached getter.
+    # @param [Integer or nil] max_count The max count to be set if passed if nil or no param, do not change the the value.
+    # @return [Integer] the max allowed number of threads
+    def max_thread_count(max_count=nil)
+      @max_thread_count = max_count if max_count
+      @max_thread_count ||= hsdq_opts[:max_thread_count] || 10
+    end
+
+    # Set paused flag
+    # @param [Boolean] paused
+    def paused(paused)
+      @paused = paused
+    end
+
+    # @return [Boolean] true is paused
+    def paused?
+      @paused = false if @paused.nil?
+      @paused
+    end
+
+    # @return [Boolean] true if below the max number of allowed thread
+    def allow_new_threads?
+      hsdq_threads_count < max_thread_count && !paused?
+    end
+
+    # Cached ThreadGroup instance holding the threads for a given hsdq class
+    # @return [ThreadGroup]
+    def hsdq_threads
+      @hsdq_threads ||= ThreadGroup.new
+    end
+
+    # Add a thread to the thread group
+    # @param [Thread]
+    # @return [Threadgroup] the hdsq thread goup
+    def hsdq_threads_add(thread)
+      hsdq_threads.add thread
+    end
+
+    # @return the number of thread in the thread group
+    def hsdq_threads_count
+      hsdq_threads.list.size
+    end
+
+    # Start a new thread and add it to the thread group
+    # @param [Proc] the thread staring block
+    # @return [TheadGroup] the hdsq thread goup
+    def hsdq_start_thread(ignition)
+      t = Thread.new(&ignition)
+      p "New thread: #{t}"
+      hsdq_threads_add t
+    end
+
+  end
+end

data/lib/hsdq/utilities.rb

@@ -0,0 +1,20 @@
+module Hsdq
+  module Utilities
+
+    # utility method to symbolize the keys of a hash
+    #
+    # @param [Hash, #a_hash] the hash to be converted
+    # @return [Hash] with all keys as symbol
+    def deep_symbolize(a_hash)
+      JSON.parse(JSON[a_hash], symbolize_names: true)
+    end
+
+    # utility method (equivalent to Rails underscore)
+    # @param [String, #string] the string to be transformed ie: class/constant name
+    # @return [String] underscored string all in lowercase
+    def snakify(string)
+      string.split(/(?=[A-Z])/).map(&:downcase).join('_')
+    end
+
+  end
+end

data/lib/hsdq/version.rb

@@ -0,0 +1,7 @@
+
+module Hsdq
+  VERSION = "0.7.0"
+  def hsdq_version
+    VERSION
+  end
+end

data/readme.md

@@ -0,0 +1,178 @@
+# Hsdq
+
+
+###High Speed Distributed Queue:
+#####A messaging layer allowing distributed applications/scripts to communicate easily and exchange data at high speed. 
+**Also allow to offload deferred tasks to outside applications/scripts.**  
+
+## Features
+
+* Connect with ease applications and/or scripts together.
+* Easy to scale horizontally by adding or removing new application on line.
+* Easy to balance the load, no round-robbing, the first listening application listening will get the message, then the next one will get the next message.
+* hot swap, hot deployment. If the app message API is compatible with the previous API version, just deploy the new version and stop the old one.
+* No need to have the target application on line to send a message, they will be stored in the bus and the target app will begin processing as soon as back online. (HTTP API need to be online to get the request).
+
+## Dependencies
+
+* HSDQ rely on Redis for the transport layer
+
+## Recommended
+
+* Ruby 2.3.1, Minimum ruby 2
+* Jruby 9.1.2 Minimum 9.0.3
+
+## How to use HSDQ
+
+####Step 0: Getting a bit accustomed
+
+Play with some quick and dirty scripts to get familiar, check the examples folder.
+
+####Step 1: create the connected class
+
+- Create a class in your application or script (for a Rails app in lib or accustomed in models)
+- Prefix this class name with `Hsdq`, ie: `HsdqMyClass`
+- Extend Hsdq into your class
+
+The listening channel will then be created based on your class name ie: `my_class`
+
+####Step 2: to get the events
+
+Into your class override the 5 following methods:
+
+| Method         | Comment                                                                         | 
+|:-------------- |:--------------------------------------------------------------------------------|
+|`hsdq_request`  |Called when a request is received                                                |
+|`hsdq_ack`      |Called when an acknowledgment your request has been received by the receiving app|
+|`hsdq_callback` |Called when the other other app respond to your request                          |
+|`hsdq_feedback` |Called when the other app is sending intermediate feedback                       |
+|`hsdq_error`    |Called when an error occur                                                       |
+
+You must implement these class methods into your class. They will receive the event.
+All methods are receiving 2 parameters: A Hash for the message and a Hash or nil for the context.
+
+```Ruby  
+def self.hsdq_request(message, context)  
+  self.new.hsdq_request message, context  
+end  
+  
+def self.hsdq_request(message, context)  
+  # Start your processing here  
+end  
+```   
+**Important:** Unless options[:threaded] is set to false, each event received are totally decoupled and independent. So you need to keep track of the context if needed.
+
+####Step 3: sending messages
+
+Four methods are used to send and respond to messages.  
+They are implemented internally in Hsdq and need a Hash as parameter.
+
+| Method                       | Comment                                   |
+|:-----------------------------|:------------------------------------------|
+|`hsdq_send_request(message)`  | To send a request                         |
+|`hsdq_send_calback(message)`  | To send the final response to a request   |
+|`hsdq_send_feedback(message)` | To send intermediate data                 |
+|`hsdq_send_error(message)`    | To send an error message in case of error |
+
+####Step 4:
+Set the authorized topics and tasks in your class:  
+`hsdq_authorized_topics :beer, :wine, :cheese, :dishes`  
+`hsdq_authorized_tasks :drink, :eat, :clean`  
+Note you can in simple cases like in scripts only use topic or task but it is recommended to use both.
+
+####Step 5:
+Setup the hsdq config file:  
+
+In the config folder create a file name `hsdq_my_class.yml` There ia a sample file in hsdq config folder.
+
+- Each class you connect to HSDQ must have it's specific yml file. This allow you to shard the bus for heavy
+ traffic application.
+- Each subsection can point to a different Redis instance/host/db.  
+ For heavy traffic sites, you can shard messages and have for example a unique host for admin.
+- Session can be split to different host if different applications do not share the sessions data
+
+```
+development:
+  redis:
+    message:
+      host: 127.0.0.1
+      port: 6379
+      db:   0
+    admin:
+      host: 127.0.0.1
+      port: 6379
+      db:   0
+    session:
+      host: 127.0.0.1
+      port: 6379
+      db:   0
+```
+
+## Message specifications
+
+#####Five type of message are running into the bus:
+
+|    | Type    | Description                        |
+|:---|:--------|:-----------------------------------|
+| 1  | Request | Initial message sent               |
+| 2  | Ack     | Acknowledgment of reception        |
+| 3  | Feedback| Intermediate response (progress)   |
+| 4  | Callback| Final response with the data       |
+| 5  | Error   | Final response when an error occur | 
+
+#####A message event is composed of 2 parts:  
+
+1 - The spark:
+
+- An ephemeral tiny part that include the minimum but sufficient information pointing to the 2nd part, the message itself.
+- The spark is what the listener will be getting from the list queue and this will ignite the process.  
+- The spark is pushed to a redis list and popped by the listener. Once popped it will not be available in the redis layer anymore.  
+  
+2 - The Burst:
+
+- An event in the life of the message, the burst is stored as one value in a Redis hash and can be retrieved using the spark data.
+  
+#####The complete message
+
+- It will contain all the operations executed during the life cycle of the message.  
+- It is stored into a hash and each element of this hash is one message event.  
+- The message element is stored in a Redis Hash with a default expiration of 72 hours (can be adjusted)  
+
+A message is composed of multiple events. At minimum there are 3 events:
+
+1. Request
+2. Ack
+3. Callback or Error
+
+Multiple feedback can be sent before the callback/error and multiple events can be present in the case of chained events.
+
+#####Structure of a message:
+**The burst:**  
+
+|  Key           |       | Type   |  Description                                                     |
+| :--------------|:-----:| :----- | :--------------------------------------------------------------- |
+| sent_to        | M     | String | Name channel you publish                                         |
+| topic          | O     | Symbol | Topic to be processed (ex: :beer, :wine, :cheese etc...).        |
+| task           | O     | Symbol | Task to be processed (ex: :drink, eat, clean ,etc...).           |
+| params         | M     | Hash   | parameters matching the receiver message API (can be empty hash) | 
+| data           | C     | Hash   | Mandatory in the responses as this is the response               | 
+| type           | I     | Symbol | Type of the message ie: :request, :callback, etc                 |
+| sender         | I     | String | the listening channel of the sender. Used to reply               |
+| uid            | I     | UUID   | Unique identifier for the message container (redis hash)         |
+| spark_uid      | I     | UUID   | Unique identifier for the 'spark' and the 'burst'                |
+| tstamp         | I     | UTC    | Timestamp for the event UTC                                      |
+| context        | I     | Hash   | Data from the previous request. keys: :reply_to, spark_uid       |
+| previous_sender| I     | String | The previous sender of a request. used in chained queries        |
+| hsdq_session   | O     | String | Key to load session data related to the context                  |
+
+`M` Mandatory, `O` Optional (but recommended for pre-filtering), `I` Hsdq internal
+
+- Topic and task values must be present respectively in the topics or task white lists when used.
+- Redis return symbols as strings in the responses
+
+**The spark:**   
+
+The spark has the same structure and values as the burst except it do not carry the payload.
+The params and data as well as eventual custom keys keys which can contain heavy payloads are not included.
+
+