hsdq 0.7.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d34ed2f5f158452f0d837271297827400158f56f
+  data.tar.gz: 4129432a2dc752ddfc8d924b6d0d5e98d695099d
+SHA512:
+  metadata.gz: 652311c93022e42f60b897c529c002a17c04bf83839e2a1d0cf4b720de7d613fe85312b90c28041ad51a56447ec54cf0a40bb69376c8899120bcc55aca56c3d5
+  data.tar.gz: adf4969580bb7075c767b7358c7c6de60d2a75e5682cbbb77177f8ed30ddb01704b00d72fa54e9fd1a4aeaa7292cb9a95a5aa2bd18af475bc7bfd870beebd5e3

data/.gitignore

@@ -0,0 +1,25 @@
+.rspec
+vendor/bundle
+log/*
+tmp/*
+coverage
+spec/tmp/*
+**.orig
+rerun.txt
+.byebug_history
+.ruby_gemset
+.ruby_version
+
+# Ignore bundler config.
+.bundle
+Gemfile.lock
+
+# ignore Rubymine files
+.idea/
+
+.rvmrc
+.DS_Store
+*.save
+*.swp
+*.mp4
+*.ogv

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.3.1

data/.yardopts

@@ -0,0 +1 @@
+--markup markdown

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# see gemspec for the dependencies
+gemspec

data/LICENSE.txt

@@ -0,0 +1,26 @@
+This file is part of HSDQ
+
+Copyright 2013 Yves Lucas,
+
+Authorisation is granted under MIT license as long as this copyright and mention are not removed
+
+MIT License
+
+  Permission is hereby granted, free of charge, to any person obtaining
+  a copy of this software and associated documentation files (the
+  "Software"), to deal in the Software without restriction, including
+  without limitation the rights to use, copy, modify, merge, publish,
+  distribute, sublicense, and/or sell copies of the Software, and to
+  permit persons to whom the Software is furnished to do so, subject to
+  the following conditions:
+
+  The above copyright notice and this permission notice shall be
+  included in all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "debugger"
+require_relative "../lib/hsdq"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/config/hsdq_your_class.yml,example

@@ -0,0 +1,63 @@
+# This file should be in your Rails app config folder, or for script / non Rails app be accessible
+# by your app
+# different host / port can be used to split the different usage
+# Redis is mono threaded so the same server can also run different instances on different ports
+# session can be separate for different group of applications
+# You need one separate file for each hsdq class
+# below setup for 2 instances of Redis:
+# - port 6379 with 2 databases (message db 1 and admin db 0)
+# - port 6380 for session
+# any app/script/class that need to exchange messages together should share the same setup, it is
+# strongly suggested to use same setup for all apps at the beginning.
+# The flexibility of this setup file is for scaling and sharding the busses.
+:development:
+  :redis:
+    :message:
+      :host: 127.0.0.1
+      :port: 6379
+      :db:   1
+    :admin:
+      :host: 127.0.0.1
+      :port: 6379
+      :db:   2
+    :session:
+      :host: 127.0.0.1
+      :port: 6380
+      :db:   1
+  :exceptions: true
+  :threaded:   false
+  :timeout:    10
+:test:
+  :redis:
+    :message:
+      :host: 127.0.0.1
+      :port: 6379
+      :db:   2
+    :admin:
+      :host: 127.0.0.1
+      :port: 6379
+      :db:   2
+    :session:
+      :host: 127.0.0.1
+      :port: 6379
+      :db:   2
+  :exceptions: true
+  :threaded:   false
+  :timeout:    10
+:production:
+  :redis:
+    :message:
+      :host: 127.0.0.1
+      :port: 6395
+      :db:   5
+    :admin:
+      :host: 127.0.0.1
+      :port: 6395
+      :db:   5
+    :session:
+      :host: 127.0.0.1
+      :port: 6395
+      :db:   5
+  :exceptions: false
+  :threaded:   true
+  :timeout:    10

data/hsdq.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'hsdq/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "hsdq"
+  spec.version       = Hsdq::VERSION
+  spec.authors       = ["Yves Lucas"]
+  spec.email         = ["hsdq@ylucas.com"]
+
+  spec.summary       = %q{Hsdq, High Speed Distributed Queue for message bus.}
+  spec.description   = %q{Hsdq: Light weight and distributed, Hsdq allow message passing distributed applications to exchange requests and data at high speed, work in parallel and scale horizontaly.}
+  spec.homepage      = "https://github.com/ylucas/hsdq"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|examples)/}) }
+  spec.bindir        = "bin"
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "redis", "~> 3.3"
+  spec.add_runtime_dependency "json", "~> 2.0"
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "shoulda-matchers"
+  spec.add_development_dependency "yard"
+  spec.add_development_dependency "redcarpet"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "pry-byebug"
+end

data/lib/hsdq.rb

@@ -0,0 +1,38 @@
+# Hsdq: is a High Speed Distributed message Queue built on top of Redis.
+# @see Readme.md
+#
+# This software Copyright 2013-2016 Yves Lucas
+# License MIT, see LICENSE.txt
+#
+
+require 'redis'
+require 'json'
+require 'securerandom'
+require 'yaml'
+
+require_relative "hsdq/shared"
+require_relative "hsdq/utilities"
+require_relative "hsdq/connectors"
+require_relative "hsdq/listener"
+require_relative "hsdq/sender"
+require_relative "hsdq/setting"
+require_relative "hsdq/receiver"
+require_relative "hsdq/thread_store"
+require_relative "hsdq/session"
+require_relative "hsdq/threadpool"
+require_relative "hsdq/admin"
+
+module Hsdq
+  include Shared
+  include Utilities
+  include Connectors
+  include Listener
+  include Sender
+  include Setting
+  include Receiver
+  include ThreadStore
+  include Session
+  include Threadpool
+  include Admin
+
+end

data/lib/hsdq/admin.rb

@@ -0,0 +1,60 @@
+module Hsdq
+  module Admin
+
+    def listener_id(listener_id=nil)
+      @listener_id = listener_id if listener_id
+      @listener_id
+    end
+
+    def listener_version(listener_version=nil)
+      @listener_version = listener_version if listener_version
+      @listener_version
+    end
+
+    def admin_channel
+      "#{environment}_#{channel}_admin"
+    end
+
+    def admin_versionned_channel
+      "#{admin_channel}__#{listener_version}"
+    end
+
+    def admin_id_channel
+      "#{admin_channel}__#{listener_id}"
+    end
+
+    def admin_channels
+      @admin_channels ||= %w(admin )
+    end
+
+    def admin_listener
+      begin
+        p "starting admin channels #{admin_channel}, #{admin_versionned_channel}"
+        cx_admin.subscribe(admin_channel, admin_versionned_channel) do |on|
+          on.message do |a_channel, admin_message_j|
+            p "received admin message: #{admin_message_j} from admin channel #{admin_channel} "
+            process_admin_message(admin_channel, admin_message_j)
+          end
+        end
+      rescue Redis::BaseConnectionError => e
+        p e.inspect
+        sleep(1)
+        retry
+      end
+    end
+
+    def process_admin_message(_admin_channel, admin_message_j)
+      admin_message = JSON.parse(admin_message_j) rescue {'params' => {'task' => ""}}
+      task = admin_message['params'] ? admin_message['params']['task'] : ""
+      case task
+        when 'stop'
+          hsdq_stop!
+        when 'start'
+          hsdq_start!
+        when 'exit'
+          hsdq_exit!
+      end
+    end
+
+  end
+end

data/lib/hsdq/connectors.rb

@@ -0,0 +1,40 @@
+require 'redis'
+
+module Hsdq
+
+  # This module contains the logic for different connection to the redis layer.
+  #
+  # They can be connected to the same Redis instance, but it is recommended to use different
+  # database connections in order to segregate the different usages.
+  #
+  # In production of large applications you should use different instances for the different layers.
+  # The configuration files are named after your class name hsdq_yourclass.yml
+  module Connectors
+
+    # Establish the listener connection.
+    # IMPORTANT this connection is blocked by the listener and must not be used elsewhere
+    # @return [Redis connection] For the listener exclusively
+    def cx_listener
+      @cx_listener ||= Redis.new cx_opts[:message]
+    end
+
+    # Establish an unblocked connection for the sender and also pulling data from Redis
+    # @return [Redis connection] This connection is used to send messages as well as to retrieve
+    #                            data from the message hash
+    def cx_data
+      @cx_data ||= Redis.new cx_opts[:message]
+    end
+
+    # establish an unblocked connection for the session layer
+    # @return [Redis connection] reserved for storing and retrieving the sessions data
+    def cx_session
+      @cx_session ||= Redis.new cx_opts[:session]
+    end
+
+    # establish a connection for the admin channel pub/sub
+    # @return [Redis connection] reserved for the admin commands
+    def cx_admin
+      @cx_admin ||= Redis.new cx_opts[:admin]
+    end
+  end
+end

data/lib/hsdq/listener.rb

@@ -0,0 +1,102 @@
+require "redis"
+require_relative "connectors"
+require_relative "utilities"
+
+module Hsdq
+  # This module is holding the methods for the class listener.
+  # The listener is connected to the Redis instance with the blocking cx_listener connection
+  #
+  # It is popping the "Spark" (ephemeral part of the message) from the list.
+  # When a spark is popped, it is validated and processed in the receiver module.
+  module Listener
+    include Connectors
+
+    # Start hsdq to listen to channel.
+    # @param [String] channel The channel the hsdq class will be listening
+    # @param [Hash] options The hsdq class option from the config file and/or additional parameters passed
+    def hsdq_start(channel, options)
+      hsdq_add_options(options) if options
+      hsdq_start!
+      hsdq_loop(channel)
+    end
+
+    # Set the flag to stop processing the queue
+    # @return [Boolean] false
+    def hsdq_stop!
+      @hsdq_running = false
+    end
+
+    # Set the flag to allow processing the queue
+    # @return [Boolean] true
+    def hsdq_start!
+      @hsdq_running = true
+    end
+
+    # Flag allowing or not the processing of the queue
+    # @return [Boolean] true when listening is allowed
+    def hsdq_running?
+      @hsdq_running = true if @hsdq_running.nil?
+      @hsdq_running
+    end
+
+    # Opposite or hsdq_running.
+    # @see hsdq_running?
+    # @return [Boolean] true when listening is not allowed
+    def hsdq_stopped?
+      !hsdq_running?
+    end
+
+    # When true allow the listener to start
+    # When set to false, the listener exit the listening loop. This is mostly to exit gracefully the program
+    # The listener needs to be restarted specifically, if need to run again
+    # @return [Boolean] true when allowing the listener to stay in listening mode
+    def hsdq_alive?
+      @hsdq_alive = true if @hsdq_alive.nil?
+      @hsdq_alive
+    end
+
+    # Flag break the listening loop if true.
+    # @return [Boolean]
+    def hsdq_exit?
+      @hsdq_exit
+    end
+
+    # Set exit to force the listening loop to exit.
+    def hsdq_exit!
+      @hsdq_exit = true
+    end
+
+    # stops the listening loop
+    def kill_alive!
+      @hsdq_alive = false
+    end
+
+    # Start the listener
+    # :nocov:
+    def start_listener(options={})
+      Thread.new { hsdq_start(channel, options) }
+      Thread.new { admin_listener }
+    end
+    # :nocov:
+
+    private
+    # Listening loop. Listen to the channel using a blocking left pop.
+    # The timeout allow the idle process to watch at regular interval if there is an admin command.
+    # The process is listeing only if there is available thread to be started in the pool
+    def hsdq_loop(channel)
+      p "starting listening channel #{channel}"
+      while hsdq_alive?
+        if (allow_new_threads? || !hsdq_opts[:threaded]) && hsdq_running?
+          raw_spark = cx_listener.blpop(channel, hsdq_opts[:timeout] )
+          hsdq_ignit raw_spark, hsdq_opts if raw_spark
+        else
+          # :nocov:
+          sleep 0.01 # occur only when hsdq_max_threads is reached
+          # :nocov:
+        end
+        kill_alive! if hsdq_exit?
+      end
+    end
+
+  end
+end