evt-consumer 0.0.0.1 → 0.2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1d47d76ca72a1690d983e55f05254d16a7020586
-  data.tar.gz: 30e729d41f0db67cc91f1703cfa6e18e3c82988d
+  metadata.gz: 00b0c73220714ee6fcf3f270b6ad3060fb31b589
+  data.tar.gz: ea9bda155670320c05c33e501904bca847d7ebee
 SHA512:
-  metadata.gz: c82b2f96df0c299d64e05c264d4a1193fa91768da0b79bdee5ff7fb852bbd0b5e1ed526b5a2c436bb3844b0ddb2a2276a8a36b5a34061d6d45aeec1f8eef3c5b
-  data.tar.gz: 1e6372fb092c037a2581e4890614a8858f3b71ffddf6a555cb71c6870539e67787b650006baca0d2661fec04bd67a93396bbeee1f00a864f04013749ede15a5f
+  metadata.gz: 8915dd125283bc4a281845ef72557f110ec4c63bac950333c516f4c5c77bfee65878de6207928843b14b99f6692cc6b66b98312f252d5c550a2fad4b1a0450ab
+  data.tar.gz: 18453c9ca467afbfdd96160cf2f8685e7cc51bc37a4e48c1672b63cda5cc64d0cae9ecb441fe3f28079bcd0247aea6e3ca94029ff619f53fd4f180a78a434433

data/lib/consumer.rb

@@ -5,7 +5,24 @@ require 'messaging'
 require 'log'
 
 require 'consumer/log'
+require 'consumer/log_text'
+
+require 'consumer/defaults'
+
+require 'consumer/handler_registry'
 
-require 'consumer/consumer'
 require 'consumer/position_store'
+require 'consumer/position_store/substitute'
 require 'consumer/position_store/telemetry'
+
+require 'consumer/dispatch'
+require 'consumer/dispatch/substitute'
+
+require 'consumer/subscription'
+require 'consumer/subscription/defaults'
+require 'consumer/subscription/get_batch'
+
+require 'consumer/consumer'
+require 'consumer/substitute'
+
+require 'consumer/actor'

data/lib/consumer/actor.rb

@@ -0,0 +1,45 @@
+module Consumer
+  class Actor
+    include ::Actor
+    include Log::Dependency
+
+    initializer :subscription_address
+
+    dependency :consumer, Consumer
+
+    def self.build(consumer, subscription)
+      instance = new subscription.address
+      instance.consumer = consumer
+      instance.configure
+      instance
+    end
+
+    handle :start do
+      request_batch
+    end
+
+    handle Subscription::GetBatch::Reply do |get_batch_reply|
+      events = get_batch_reply.batch
+
+      logger.trace { "Received batch (Events: #{events.count})" }
+
+      request_batch
+
+      events.each do |event_data|
+        consumer.(event_data)
+      end
+
+      logger.debug { "Batch received (Events: #{events.count})" }
+    end
+
+    def request_batch
+      logger.trace { "Requesting batch" }
+
+      get_batch = Subscription::GetBatch.new address
+
+      send.(get_batch, subscription_address)
+
+      logger.debug { "Send batch request" }
+    end
+  end
+end

data/lib/consumer/consumer.rb

@@ -1,18 +1,152 @@
 module Consumer
   def self.included(cls)
     cls.class_exec do
-      extend StreamMacro
+      include Log::Dependency
+
+      extend Build
+      extend Start
+
+      extend HandleMacro
+      extend PositionStoreMacro
+
+      prepend Configure
+
+      initializer :stream
+
+      attr_writer :position_update_interval
+
+      attr_accessor :cycle_maximum_milliseconds
+      attr_accessor :cycle_timeout_milliseconds
+
+      dependency :dispatch, Dispatch
+      dependency :get
+      dependency :position_store, PositionStore
+      dependency :subscription, Subscription
+    end
+  end
+
+  def call(event_data)
+    logger.trace { "Dispatching event (#{LogText.event_data event_data})" }
+
+    dispatch.(event_data)
+
+    update_position event_data.global_position
+
+    logger.info { "Event dispatched (#{LogText.event_data event_data})" }
+
+  rescue => error
+    logger.error { "Error raised (Error Class: #{error.class}, Error Message: #{error.message}, #{LogText.event_data event_data})" }
+    error_raised error, event_data
+  end
+
+  Virtual::Method.define self, :configure
+
+  def error_raised(error, _)
+    raise error
+  end
+
+  def update_position(position)
+    logger.trace { "Updating position (Position: #{position}, Interval: #{position_update_interval})" }
+
+    if position % position_update_interval == 0
+      position_store.put position
+
+      logger.debug { "Updated position (Position: #{position}, Interval: #{position_update_interval})" }
+    else
+      logger.debug { "Interval not reached; position not updated (Position: #{position}, Interval: #{position_update_interval})" }
+    end
+  end
+
+  module LogText
+    def self.event_data(event_data)
+      "Stream: #{event_data.stream_name}, Position: #{event_data.position}, GlobalPosition: #{event_data.global_position}, Type: #{event_data.type}"
+    end
+  end
+
+  module Configure
+    def configure(batch_size: nil, session: nil)
+      logger.trace { "Configuring (Batch Size: #{batch_size}, Session: #{session.inspect})" }
+
+      super
+
+      position_store_class = self.class.position_store_class
+
+      unless position_store_class.nil?
+        position_store = position_store_class.configure self, stream
+
+        starting_position = position_store.get
+      end
+
+      subscription = Subscription.configure(
+        self,
+        stream,
+        get,
+        position: starting_position,
+        cycle_maximum_milliseconds: cycle_maximum_milliseconds,
+        cycle_timeout_milliseconds: cycle_timeout_milliseconds
+      )
+
+      Dispatch.configure self, handler_registry: self.class.handler_registry
+
+      logger.debug { "Configuring (Batch Size: #{batch_size}, Session: #{session.inspect})" }
+    end
+  end
+
+  module Build
+    def build(stream_name, batch_size: nil, session: nil, cycle_timeout_milliseconds: nil, cycle_maximum_milliseconds: nil)
+      stream = EventSource::Stream.canonize stream_name
+
+      instance = new stream
+      instance.cycle_maximum_milliseconds = cycle_maximum_milliseconds
+      instance.cycle_timeout_milliseconds = cycle_timeout_milliseconds
+      instance.configure batch_size: batch_size, session: session
+      instance
+    end
+  end
+
+  module Start
+    def start(stream_name, **arguments, &probe)
+      instance = build stream_name, **arguments
+
+      _, subscription_thread = ::Actor::Start.(instance.subscription)
+
+      actor_address = Actor.start instance, instance.subscription
+
+      probe.(instance, actor_address) if probe
+
+      AsyncInvocation::Incorrect
+    end
+  end
+
+  module HandleMacro
+    def handle_macro(handler=nil, &block)
+      handler ||= block
+
+      handler_registry.register handler
+    end
+    alias_method :handle, :handle_macro
+
+    def handler_registry
+      @handler_registry ||= HandlerRegistry.new
     end
   end
 
-  module StreamMacro
-    def stream_macro(stream_name)
-      stream = EventSource::Stream.new stream_name
+  module PositionStoreMacro
+    def self.extended(cls)
+      cls.singleton_class.class_exec do
+        attr_accessor :position_store_class
+      end
+    end
+
+    def position_store_macro(position_store_class, update_interval: nil)
+      update_interval ||= Defaults.position_update_interval
+
+      self.position_store_class = position_store_class
 
-      define_method :stream do
-        stream
+      define_method :position_update_interval do
+        @position_update_interval ||= update_interval
       end
     end
-    alias_method :stream, :stream_macro
+    alias_method :position_store, :position_store_macro
   end
 end

data/lib/consumer/controls.rb

@@ -1,12 +1,21 @@
 require 'messaging/controls'
 
 require 'consumer/controls/category'
+require 'consumer/controls/cycle'
 require 'consumer/controls/event_data'
+require 'consumer/controls/event_data/batch'
+require 'consumer/controls/error'
+require 'consumer/controls/get'
+require 'consumer/controls/get/incrementing'
+require 'consumer/controls/handle'
 require 'consumer/controls/id'
 require 'consumer/controls/position'
 require 'consumer/controls/stream'
 require 'consumer/controls/stream_name'
 
 require 'consumer/controls/position_store'
+require 'consumer/controls/position_store/local_file'
+require 'consumer/controls/subscription'
 
 require 'consumer/controls/consumer'
+require 'consumer/controls/consumer/incrementing'

data/lib/consumer/controls/consumer.rb

@@ -2,13 +2,18 @@ module Consumer
   module Controls
     module Consumer
       def self.example
-        Example.new
+        Example.new Stream.example
       end
 
       class Example
         include ::Consumer
 
-        stream StreamName.example
+        handle Handle::Example
+        position_store PositionStore::Example
+
+        def configure(session: nil, batch_size: nil)
+          Get::Example.configure self
+        end
       end
     end
   end

data/lib/consumer/controls/consumer/incrementing.rb

@@ -0,0 +1,30 @@
+module Consumer
+  module Controls
+    module Consumer
+      class Incrementing
+        include ::Consumer
+
+        def self.logger
+          @logger ||= ::Log.get self
+        end
+
+        handle do |event_data|
+          logger.info { "Handled event (StreamName: #{event_data.stream_name}, GlobalPosition: #{event_data.global_position})" }
+        end
+
+        handle do |event_data|
+          logger.debug { event_data.data.pretty_inspect }
+        end
+
+        position_store PositionStore::LocalFile, update_interval: 10
+
+        def configure(session: nil)
+          sleep_duration = ENV['SLEEP_DURATION'] || 100
+          sleep_duration = sleep_duration.to_i
+
+          Get::Incrementing.configure self, sleep_duration
+        end
+      end
+    end
+  end
+end

data/lib/consumer/controls/cycle.rb

@@ -0,0 +1,20 @@
+module Consumer
+  module Controls
+    module Cycle
+      def self.example
+        ::Cycle.build(
+          maximum_milliseconds: maximum_milliseconds,
+          timeout_milliseconds: timeout_milliseconds
+        )
+      end
+
+      def self.maximum_milliseconds
+        1
+      end
+
+      def self.timeout_milliseconds
+        11
+      end
+    end
+  end
+end

data/lib/consumer/controls/error.rb

@@ -0,0 +1,11 @@
+module Consumer
+  module Controls
+    module Error
+      def self.example
+        Example.new
+      end
+
+      Example = Class.new StandardError
+    end
+  end
+end

data/lib/consumer/controls/event_data.rb

@@ -1,11 +1,13 @@
 module Consumer
   module Controls
     module EventData
-      def self.example(stream_name: nil, position: nil, global_position: nil)
-        event_data = EventSource::Controls::EventData::Read.example
+      def self.example(stream_name: nil, data: nil, position: nil, global_position: nil)
+        global_position ||= position
+
+        event_data = EventSource::Controls::EventData::Read.example data: data
         event_data.stream_name = stream_name unless stream_name.nil?
         event_data.position = position unless position.nil?
-        event_data.global_position unless global_position.nil?
+        event_data.global_position = global_position unless global_position.nil?
         event_data
       end
     end

data/lib/consumer/controls/event_data/batch.rb

@@ -0,0 +1,22 @@
+module Consumer
+  module Controls
+    module EventData
+      module Batch
+        def self.example(instances: nil, position: nil)
+          instances ||= size
+          start_position ||= Controls::Position::Global.example
+
+          instances.times.map do |offset|
+            position = start_position + offset
+
+            EventData.example global_position: position
+          end
+        end
+
+        def self.size
+          3
+        end
+      end
+    end
+  end
+end

data/lib/consumer/controls/get.rb

@@ -0,0 +1,41 @@
+module Consumer
+  module Controls
+    module Get
+      def self.example
+        Example.new
+      end
+
+      class Example
+        configure :get
+
+        def self.build
+          new
+        end
+
+        def call(stream_name, position: nil)
+          position ||= 0
+
+          stream = streams.fetch stream_name do
+            return nil
+          end
+
+          stream[position]
+        end
+
+        def set_batch(stream_name, batch, position=nil)
+          position ||= 0
+
+          stream = streams[stream_name]
+
+          stream[position] = batch
+        end
+
+        def streams
+          @streams ||= Hash.new do |hash, key|
+            hash[key] = {}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/consumer/controls/get/incrementing.rb

@@ -0,0 +1,42 @@
+module Consumer
+  module Controls
+    module Get
+      class Incrementing
+        configure :get
+
+        initializer :sleep_duration
+
+        def self.build(sleep_duration)
+          new sleep_duration
+        end
+
+        def call(stream_name, position: nil)
+          position ||= 0
+
+          sleep Rational(sleep_duration, 1000)
+
+          3.times.map do |offset|
+            EventData.get(
+              stream_name,
+              position + offset,
+              offset
+            )
+          end
+        end
+      end
+
+      class EventData
+        def self.get(stream_name, global_position, position)
+          data = { :position => position, :global_position => global_position }
+
+          Controls::EventData.example(
+            stream_name: stream_name,
+            data: data,
+            global_position: global_position,
+            position: position
+          )
+        end
+      end
+    end
+  end
+end

data/lib/consumer/controls/handle.rb

@@ -0,0 +1,17 @@
+module Consumer
+  module Controls
+    module Handle
+      class Example
+        include Messaging::Handle
+
+        def handle(event_data)
+          handled_events << event_data
+        end
+
+        def handled_events
+          @handled_events ||= []
+        end
+      end
+    end
+  end
+end

data/lib/consumer/controls/position.rb

@@ -8,8 +8,12 @@ module Consumer
       end
 
       module Global
-        def self.example
-          EventSource::Controls::EventData::Read.global_position
+        def self.example(offset: nil)
+          offset ||= 0
+
+          position = EventSource::Controls::EventData::Read.global_position
+
+          position + offset
         end
       end
     end

data/lib/consumer/controls/position_store/local_file.rb

@@ -0,0 +1,23 @@
+module Consumer
+  module Controls
+    module PositionStore
+      class LocalFile
+        include Consumer::PositionStore
+
+        def get
+          return 0 unless File.exist? path
+          text = File.read path
+          text.to_i
+        end
+
+        def put(position)
+          File.write path, position
+        end
+
+        def path
+          'tmp/control_position_store'
+        end
+      end
+    end
+  end
+end

data/lib/consumer/controls/subscription.rb

@@ -0,0 +1,25 @@
+module Consumer
+  module Controls
+    module Subscription
+      def self.example(next_batch: nil, batch: nil, position: nil)
+        stream = Stream.example
+
+        get = Get.example
+
+        unless batch.nil?
+          get.set_batch stream.name, batch, position
+        end
+
+        subscription = ::Consumer::Subscription.new stream, get
+
+        subscription.position = position if position
+
+        unless next_batch.nil?
+          subscription.next_batch = next_batch
+        end
+
+        subscription
+      end
+    end
+  end
+end

data/lib/consumer/defaults.rb

@@ -0,0 +1,7 @@
+module Consumer
+  module Defaults
+    def self.position_update_interval
+      100
+    end
+  end
+end

data/lib/consumer/dispatch.rb

@@ -0,0 +1,43 @@
+module Consumer
+  class Dispatch
+    include Log::Dependency
+
+    configure :dispatch
+
+    attr_writer :handler_registry
+
+    def handler_registry
+      @handler_registry ||= HandlerRegistry.new
+    end
+
+    def self.build(handler_registry: nil)
+      instance = new
+      instance.handler_registry = handler_registry unless handler_registry.nil?
+      instance
+    end
+
+    def call(event_data)
+      logger.trace { "Dispatching event (#{LogText.event_data event_data})" }
+
+      handlers = handler_registry.get
+
+      handlers.each do |handle|
+        handle.(event_data)
+      end
+
+      logger.debug { "Event dispatched (#{LogText.event_data event_data}, Handlers Count: #{handlers.count})" }
+
+      nil
+    end
+
+    def to_proc
+      method :call
+    end
+
+    module Assertions
+      def handler?(handle)
+        handler_registry.registered? handle
+      end
+    end
+  end
+end

data/lib/consumer/dispatch/substitute.rb

@@ -0,0 +1,29 @@
+module Consumer
+  class Dispatch
+    module Substitute
+      def self.build
+        Dispatch.new
+      end
+
+      class Dispatch
+        def call(event_data)
+          dispatched_events << event_data
+        end
+
+        def dispatched_events
+          @dispatched_events ||= []
+        end
+
+        def dispatched?(event_data=nil, &block)
+          if event_data.nil?
+            block ||= proc { true }
+          else
+            block ||= proc { |e| event_data == e }
+          end
+
+          dispatched_events.any? &block
+        end
+      end
+    end
+  end
+end

data/lib/consumer/handler_registry.rb

@@ -0,0 +1,63 @@
+module Consumer
+  class HandlerRegistry
+    include Log::Dependency
+
+    configure :handler_registry
+
+    def register(handler)
+      logger.trace { "Registering handler (Handler: #{LogText.handler handler})" }
+
+      if registered? handler
+        error_message = "Handler is already registered (Handler: #{LogText.handler handler})"
+        logger.error error_message
+        raise Error, error_message
+      end
+
+      if handler.respond_to? :build
+        instance = handler.build
+      else
+        instance = handler
+      end
+
+      entries << instance
+
+      logger.debug { "Handler registered (Handler: #{LogText.handler handler})" }
+
+      instance
+    end
+
+    def get
+      entries.to_a
+    end
+
+    def registered?(handler)
+      entries.any? do |entry|
+        return true if handler == entry 
+
+        next if entry.is_a? Proc
+
+        handler === entry
+      end
+    end
+
+    def entries
+      @entries ||= Set.new
+    end
+
+    def count
+      entries.count
+    end
+
+    module LogText
+      def self.handler(handler)
+        case handler
+        when Module then handler.name
+        when Proc then '(proc)'
+        else handler.inspect
+        end
+      end
+    end
+
+    Error = Class.new StandardError
+  end
+end

data/lib/consumer/log_text.rb

@@ -0,0 +1,7 @@
+module Consumer
+  module LogText
+    def self.event_data(event_data)
+      "Stream: #{event_data.stream_name}, Position: #{event_data.position}, GlobalPosition: #{event_data.global_position}, Type: #{event_data.type}"
+    end
+  end
+end

data/lib/consumer/position_store/substitute.rb

@@ -0,0 +1,30 @@
+module Consumer
+  module PositionStore
+    module Substitute
+      def self.build
+        PositionStore.new
+      end
+
+      class PositionStore
+        attr_accessor :get_position
+        attr_accessor :put_position
+
+        def get
+          get_position
+        end
+
+        def put(position)
+          self.put_position = position
+        end
+
+        def put?(expected_position=nil)
+          if expected_position.nil?
+            put_position ? true : false
+          else
+            put_position == expected_position
+          end
+        end
+      end
+    end
+  end
+end

data/lib/consumer/subscription.rb

@@ -0,0 +1,89 @@
+module Consumer
+  class Subscription
+    include ::Actor
+    include Log::Dependency
+
+    configure :subscription
+
+    initializer :stream, :get
+
+    attr_accessor :next_batch
+
+    attr_writer :position
+    def position
+      @position ||= 0
+    end
+
+    dependency :cycle, Cycle
+
+    def self.build(stream, get, position: nil, cycle_maximum_milliseconds: nil, cycle_timeout_milliseconds: nil)
+      cycle_maximum_milliseconds ||= Defaults.cycle_maximum_milliseconds
+      cycle_timeout_milliseconds ||= Defaults.cycle_timeout_milliseconds
+
+      instance = new stream, get
+
+      instance.position = position
+
+      Cycle.configure(
+        instance,
+        maximum_milliseconds: cycle_maximum_milliseconds,
+        timeout_milliseconds: cycle_timeout_milliseconds
+      )
+
+      instance.configure
+      instance
+    end
+
+    handle :start do
+      :resupply
+    end
+
+    handle :resupply do
+      logger.trace { "Resupplying (StreamName: #{stream.name}, Position: #{position})" }
+
+      batch = cycle.() do
+        get.(stream.name, position: position)
+      end
+
+      if batch.nil? || batch.empty?
+        logger.debug { "Did not resupply; no events available (StreamName: #{stream.name}, Position: #{position})" }
+
+        :resupply
+      else
+        self.next_batch = batch
+
+        logger.debug { "Resupplied (StreamName: #{stream.name}, Position: #{position}, Batch Size: #{batch.count})" }
+      end
+    end
+
+    handle :get_batch do |get_batch|
+      logger.trace { "Batch request received" }
+
+      if next_batch.nil?
+        logger.debug { "Could not fulfill batch request; deferring" }
+
+        return get_batch
+      end
+
+      batch = reset_next_batch
+
+      reply_message = get_batch.reply_message batch
+
+      send.(reply_message, get_batch.reply_address)
+
+      logger.debug { "Batch request fulfilled; resupplying (Batch Size: #{batch.count})" }
+
+      :resupply
+    end
+
+    def reset_next_batch
+      batch = next_batch
+
+      self.next_batch = nil
+
+      self.position = batch.last.global_position + 1
+
+      batch
+    end
+  end
+end

data/lib/consumer/subscription/defaults.rb

@@ -0,0 +1,13 @@
+module Consumer
+  class Subscription
+    module Defaults
+      def self.cycle_maximum_milliseconds
+        100
+      end
+
+      def self.cycle_timeout_milliseconds
+        1000
+      end
+    end
+  end
+end

data/lib/consumer/subscription/get_batch.rb

@@ -0,0 +1,17 @@
+module Consumer
+  class Subscription
+    GetBatch = Struct.new :reply_address
+
+    class GetBatch
+      include ::Actor::Messaging::Message
+
+      def reply_message(batch)
+        Reply.new batch
+      end
+
+      Reply = Struct.new :batch do
+        include ::Actor::Messaging::Message
+      end
+    end
+  end
+end

data/lib/consumer/substitute.rb

@@ -0,0 +1,27 @@
+module Consumer
+  module Substitute
+    def self.build
+      Consumer.new
+    end
+
+    class Consumer
+      def call(event_data)
+        consumed_events << event_data
+      end
+
+      def consumed_events
+        @consumed_events ||= []
+      end
+
+      def consumed?(event_data=nil, &block)
+        if event_data.nil?
+          block ||= proc { true }
+        else
+          block ||= proc { |e| event_data == e }
+        end
+
+        consumed_events.any? &block
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: evt-consumer
 version: !ruby/object:Gem::Version
-  version: 0.0.0.1
+  version: 0.2.0.0
 platform: ruby
 authors:
 - The Eventide Project
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-20 00:00:00.000000000 Z
+date: 2017-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ntl-actor
@@ -73,19 +73,39 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/consumer.rb
+- lib/consumer/actor.rb
 - lib/consumer/consumer.rb
 - lib/consumer/controls.rb
 - lib/consumer/controls/category.rb
 - lib/consumer/controls/consumer.rb
+- lib/consumer/controls/consumer/incrementing.rb
+- lib/consumer/controls/cycle.rb
+- lib/consumer/controls/error.rb
 - lib/consumer/controls/event_data.rb
+- lib/consumer/controls/event_data/batch.rb
+- lib/consumer/controls/get.rb
+- lib/consumer/controls/get/incrementing.rb
+- lib/consumer/controls/handle.rb
 - lib/consumer/controls/id.rb
 - lib/consumer/controls/position.rb
 - lib/consumer/controls/position_store.rb
+- lib/consumer/controls/position_store/local_file.rb
 - lib/consumer/controls/stream.rb
 - lib/consumer/controls/stream_name.rb
+- lib/consumer/controls/subscription.rb
+- lib/consumer/defaults.rb
+- lib/consumer/dispatch.rb
+- lib/consumer/dispatch/substitute.rb
+- lib/consumer/handler_registry.rb
 - lib/consumer/log.rb
+- lib/consumer/log_text.rb
 - lib/consumer/position_store.rb
+- lib/consumer/position_store/substitute.rb
 - lib/consumer/position_store/telemetry.rb
+- lib/consumer/subscription.rb
+- lib/consumer/subscription/defaults.rb
+- lib/consumer/subscription/get_batch.rb
+- lib/consumer/substitute.rb
 homepage: https://github.com/eventide-project/consumer
 licenses:
 - MIT