hivent 1.0.1

data/lib/hivent/cli/runner.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+require 'optparse'
+require "active_support"
+require "active_support/core_ext"
+
+require_relative "./start_option_parser"
+require_relative "./consumer"
+
+module Hivent
+
+  module CLI
+
+    class Runner
+
+      OPTION_PARSERS = {
+        start: StartOptionParser
+      }.freeze
+
+      def initialize(argv)
+        @argv = argv
+        @command = @argv.shift.to_s.to_sym
+      end
+
+      def run
+        if parser = OPTION_PARSERS[@command]
+          send(@command, parser.new(@command, @argv).parse)
+        else
+          puts help
+        end
+      end
+
+      private
+
+      def start(options)
+        Consumer.run!(options)
+      end
+
+      def help
+        <<-EOS.strip_heredoc
+          Available COMMANDs are:
+             start  :  starts one or multiple the consumer
+          See 'hivent COMMAND --help' for more information on a specific command.
+        EOS
+      end
+
+    end
+
+  end
+
+end

data/lib/hivent/cli/start_option_parser.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+module Hivent
+
+  module CLI
+
+    class StartOptionParser
+
+      def initialize(command, argv)
+        @command = command
+        @argv = argv
+      end
+
+      def parse
+        return @options if @options
+        @options = {}
+
+        parser = OptionParser.new do |o|
+          o.banner = "Usage: hivent #{@command} [options]"
+
+          o.on('-r', '--require PATH', 'File to require to bootstrap consumers') do |arg|
+            @options[:require] = arg
+          end
+
+          o.on('-p', '--pid-dir DIR', 'Location of worker pid files') do |arg|
+            @options[:pid_dir] = arg
+          end
+        end
+
+        parser.parse(@argv)
+
+        validate_options
+
+        @options
+      end
+
+      def validate_options
+        if @options[:require].nil? || !File.exist?(@options[:require])
+          puts <<-EOS.strip_heredoc
+            =========================================================
+              Please point hivent to a Ruby file
+              to load your consumers with -r FILE.
+            =========================================================
+          EOS
+
+          exit(1)
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/hivent/config.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+require "hivent/life_cycle_event_handler"
+require "hivent/config/options"
+
+module Hivent
+
+  module Config
+
+    SUPPORTED_BACKENDS = [:redis].freeze
+
+    extend self
+    extend Options
+
+    option :client_id, validate: ->(value) { value.present? }
+    option :backend, validate: ->(value) { SUPPORTED_BACKENDS.include?(value.to_sym) }
+    option :endpoint
+    option :partition_count, default: 1, validate: ->(value) { value.is_a?(Integer) && value.positive? }
+    option :life_cycle_event_handler, default: LifeCycleEventHandler.new
+
+  end
+
+end

data/lib/hivent/config/options.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+module Hivent
+
+  module Config
+
+    module Options
+
+      class UnsupportedOptionError < StandardError; end
+
+      def defaults
+        @defaults ||= {}
+      end
+
+      def validators
+        @validators ||= {}
+      end
+
+      def option(name, options = {})
+        defaults[name] = settings[name] = options[:default]
+        validators[name] = options[:validate] || ->(_value) { true }
+
+        class_eval <<-RUBY
+          def #{name}
+            settings[#{name.inspect}]
+          end
+          def #{name}=(value)
+            unless validators[#{name.inspect.to_sym}].(value)
+              raise UnsupportedOptionError.new("Unsupported value " + value.inspect + " for option #{name.inspect}")
+            end
+
+            settings[#{name.inspect}] = value
+          end
+          def #{name}?
+            #{name}
+          end
+
+          def reset_#{name}
+            settings[#{name.inspect}] = defaults[#{name.inspect}]
+          end
+        RUBY
+      end
+
+      def settings
+        @settings ||= {}
+      end
+
+    end
+
+  end
+
+end

data/lib/hivent/emitter.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+module Hivent
+
+  class Emitter
+
+    include EventEmitter
+    attr_accessor :events
+
+    WILDCARD = :all
+
+    def initialize
+      @events = []
+    end
+
+    def broadcast(payload)
+      emittable_event_names(payload.with_indifferent_access).each do |emittable_event_name|
+        emit(emittable_event_name, payload)
+      end
+    end
+
+    private
+
+    def emittable_event_names(payload)
+      [
+        event_name(payload),
+        [event_name(payload), event_version(payload)].join(":"),
+        WILDCARD
+      ]
+    end
+
+    def event_name(payload)
+      payload[:meta].try(:[], :name)
+    end
+
+    def event_version(payload)
+      payload[:meta].try(:[], :version)
+    end
+
+  end
+
+end

data/lib/hivent/life_cycle_event_handler.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+module Hivent
+
+  class LifeCycleEventHandler
+
+    # Invoked when a consumer worker starts and registers events and partion count.
+    #
+    # parameters:
+    #   client_id: name of the application
+    #   events: array of hashes for the registered events ([{ name: "my:event", version: 1 }, ...])
+    #   partition_count: number of partitions registered for this application
+    def application_registered(client_id, events, partition_count)
+      # do nothing
+    end
+
+    # Invoked when an event has successfully been processed by all registered handlers
+    #
+    # parameters:
+    #   event_name: name of the processed event
+    #   event_version: version of the processed event
+    #   payload: payload of the processed event
+    def event_processing_succeeded(event_name, event_version, payload)
+      # do nothing
+    end
+
+    # Invoked when processing an event failed. Either the payload could not be parsed as JSON or the payload did not
+    # contain all required information or an application error happend while processing in one of the registered
+    # handlers.
+    #
+    # parameters:
+    #   exception: the exception that occurred
+    #   payload: the parsed payload or nil if event payload was invalid JSON
+    #   raw_payload: the original unparsed payload (String)
+    #   dead_letter_queue_name: name of the dead letter queue this event has been sent to
+    def event_processing_failed(exception, payload, raw_payload, dead_letter_queue_name)
+      # do nothing
+    end
+
+  end
+
+end

data/lib/hivent/redis/consumer.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+module Hivent
+
+  module Redis
+
+    class Consumer
+
+      include Hivent::Redis::Extensions
+
+      LUA_CONSUMER = File.expand_path("../lua/consumer.lua", __FILE__)
+      # In milliseconds
+      SLEEP_TIME   = 200
+      CONSUMER_TTL = 1000
+
+      def initialize(redis, service_name, name, life_cycle_event_handler)
+        @redis                    = redis
+        @service_name             = service_name
+        @name                     = name
+        @stop                     = false
+        @life_cycle_event_handler = life_cycle_event_handler
+      end
+
+      def run!
+        consume while !@stop
+      end
+
+      def stop!
+        @stop = true
+      end
+
+      def queues
+        script(LUA_CONSUMER, @service_name, @name, CONSUMER_TTL)
+      end
+
+      def consume
+        to_process = items
+
+        to_process.each do |(queue, item)|
+          payload = nil
+          begin
+            payload = JSON.parse(item).with_indifferent_access
+
+            Hivent.emitter.broadcast(payload)
+
+            @life_cycle_event_handler.event_processing_succeeded(event_name(payload), event_version(payload), payload)
+          rescue => e
+            @redis.lpush(dead_letter_queue_name(queue), item)
+
+            @life_cycle_event_handler.event_processing_failed(e, payload, item, dead_letter_queue_name(queue))
+          end
+
+          @redis.rpop(queue)
+        end
+
+        Kernel.sleep(SLEEP_TIME.to_f / 1000) if to_process.empty?
+      end
+
+      private
+
+      def items
+        queues
+          .map    { |queue| [queue, @redis.lindex(queue, -1)] }
+          .select { |(_queue, item)| item }
+      end
+
+      def event_name(payload)
+        payload["meta"].try(:[], "name")
+      end
+
+      def event_version(payload)
+        payload["meta"].try(:[], "version")
+      end
+
+      def dead_letter_queue_name(queue)
+        "#{queue}:dead_letter"
+      end
+
+    end
+
+  end
+
+end

data/lib/hivent/redis/extensions.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+module Hivent
+
+  module Redis
+
+    module Extensions
+
+      LUA_CACHE = Hash.new { |h, k| h[k] = Hash.new }
+
+      def script(file, *args)
+        cache = LUA_CACHE[@redis.client.options[:url]]
+
+        sha = if cache.key?(file)
+                cache[file]
+              else
+                cache[file] = @redis.script("LOAD", File.read(file))
+              end
+
+        @redis.evalsha(sha, [], args)
+      end
+
+    end
+
+  end
+
+end

data/lib/hivent/redis/lua/consumer.lua

@@ -0,0 +1,179 @@
+local service_name  = ARGV[1]
+local consumer_name = ARGV[2]
+local CONSUMER_TTL  = ARGV[3]
+
+-- Performs deep equality between two tables
+local function table_eq(table1, table2)
+ local avoid_loops = {}
+ local function recurse(t1, t2)
+  -- compare value types
+  if type(t1) ~= type(t2) then return false end
+  -- Base case: compare simple values
+  if type(t1) ~= "table" then return t1 == t2 end
+  -- Now, on to tables.
+  -- First, let's avoid looping forever.
+  if avoid_loops[t1] then return avoid_loops[t1] == t2 end
+  avoid_loops[t1] = t2
+  -- Copy keys from t2
+  local t2keys = {}
+  local t2tablekeys = {}
+  for k, _ in pairs(t2) do
+     if type(k) == "table" then table.insert(t2tablekeys, k) end
+     t2keys[k] = true
+  end
+  -- Let's iterate keys from t1
+  for k1, v1 in pairs(t1) do
+     local v2 = t2[k1]
+     if type(k1) == "table" then
+        -- if key is a table, we need to find an equivalent one.
+        local ok = false
+        for i, tk in ipairs(t2tablekeys) do
+           if table_eq(k1, tk) and recurse(v1, t2[tk]) then
+              table.remove(t2tablekeys, i)
+              t2keys[tk] = nil
+              ok = true
+              break
+           end
+        end
+        if not ok then return false end
+     else
+        -- t1 has a key which t2 doesn't have, fail.
+        if v2 == nil then return false end
+        t2keys[k1] = nil
+        if not recurse(v1, v2) then return false end
+     end
+  end
+  -- if t2 has a key which t1 doesn't have, fail.
+  if next(t2keys) then return false end
+  return true
+ end
+ return recurse(table1, table2)
+end
+
+local function keepalive(service, consumer)
+  redis.call("SET", service .. ":" .. consumer .. ":alive", "true", "PX", CONSUMER_TTL)
+  redis.call("SADD", service .. ":consumers", consumer)
+end
+
+local function cleanup(service)
+  local consumer_index_key = service .. ":consumers"
+  local consumers = redis.call("SMEMBERS", consumer_index_key)
+
+  for _, consumer in ipairs(consumers) do
+    local consumer_status_key = service .. ":" .. consumer .. ":alive"
+    local alive = redis.call("GET", consumer_status_key)
+
+    if not alive then
+      redis.call("SREM", consumer_index_key, consumer)
+    end
+  end
+end
+
+local function distribute(consumers, partition_count)
+  local distribution   = {}
+  local consumer_count = table.getn(consumers)
+  local remainder      = partition_count % consumer_count
+
+  for i=1,consumer_count do
+    distribution[i] = math.floor(partition_count/consumer_count)
+  end
+
+  for i=1,remainder do
+    distribution[i] = distribution[i] + 1
+  end
+
+  return distribution
+end
+
+local function getdesiredstate(service_name, consumers, partition_count)
+  local state                    = {}
+  local distribution             = distribute(consumers, partition_count)
+  local consumer_count           = table.getn(consumers)
+  local assigned_partition_count = 0
+
+  for i=1,consumer_count do
+    state[consumers[i]] = {}
+
+    for j=1,distribution[i] do
+      table.insert(state[consumers[i]], 1, service_name .. ":" .. j + assigned_partition_count - 1)
+    end
+
+    assigned_partition_count = assigned_partition_count + distribution[i]
+  end
+
+  return state
+end
+
+local function getcurrentstate(service_name, consumers)
+  local state = {}
+
+  for _, consumer in ipairs(consumers) do
+    local assigned_key = service_name .. ":" .. consumer .. ":assigned"
+    state[consumer] = redis.call("LRANGE", assigned_key, 0, -1)
+  end
+
+  return state
+end
+
+local function states_match(state1, state2)
+  return table_eq(state1, state2)
+end
+
+local function all_free(workers)
+  local total_count = 0
+
+  for _, partitions in pairs(workers) do
+    total_count = total_count + table.getn(partitions)
+  end
+
+  return total_count == 0
+end
+
+local function save_state(service_name, state)
+  for worker, partitions in pairs(state) do
+    for _, partition in ipairs(partitions) do
+      redis.call("RPUSH", service_name .. ":" .. worker .. ":assigned", partition)
+      redis.call("EXPIRE", service_name .. ":" .. worker .. ":assigned", CONSUMER_TTL)
+    end
+  end
+end
+
+local function rebalance(service_name, consumer_name)
+  local consumers = redis.call("SMEMBERS", service_name .. ":consumers")
+  table.sort(consumers)
+  local partition_count = tonumber(redis.call("GET", service_name .. ":partition_count"))
+
+  local desired_state = getdesiredstate(service_name, consumers, partition_count)
+
+  local current_state = getcurrentstate(service_name, consumers)
+
+  local is_stable_state = states_match(desired_state, current_state)
+
+  if not is_stable_state then
+    if all_free(current_state) then
+      save_state(service_name, desired_state)
+
+      return desired_state[consumer_name]
+    else
+      redis.call("DEL", service_name .. ":" .. consumer_name .. ":assigned")
+      return {}
+    end
+  else
+    return desired_state[consumer_name]
+  end
+end
+
+local function heartbeat(service_name, consumer_name)
+  -- keep consumer alive
+  keepalive(service_name, consumer_name)
+
+  -- clean up dead consumers
+  cleanup(service_name)
+
+  -- rebalance
+  local new_config = rebalance(service_name, consumer_name)
+
+  return new_config
+end
+
+return heartbeat(service_name, consumer_name)