maxwell_agent 0.0.1

data/lib/maxwell/agent/middleware/chain.rb

@@ -0,0 +1,92 @@
+module Maxwell
+  module Agent
+    module Middleware
+      class Chain
+        include Enumerable
+        attr_reader :entries
+
+        def initialize
+          @entries = []
+          yield self if block_given?
+        end
+
+        def each(&block)
+          entries.each(&block)
+        end
+
+        def add(klass, *args)
+          new_entry = Entry.new(klass, *args)
+          if count > 0
+            add_at(count + 1, new_entry)
+          else
+            add_at(count, new_entry)
+          end
+        end
+
+        def remove(entry)
+          entries.delete_if {|e| e }
+        end
+
+        def insert_before(existing_klass, new_klass, *args)
+          new_entry = Entry.new(new_klass, *args)
+          i = get_index(existing_klass) || 0
+          add_at(i, new_entry)
+        end
+
+        def insert_after(existing_klass, new_klass, *args)
+          new_entry = Entry.new(new_klass, *args)
+          i = get_index(existing_klass) || count - 1
+          add_at(i + 1, new_entry)
+        end
+
+        def invoke(*args, &final_action)
+          chain = retrieve.dup
+
+          traverse_chain = -> do
+            if chain.empty?
+              final_action.call if final_action
+            else
+              chain.shift.call(*args, &traverse_chain)
+            end
+          end
+          traverse_chain.call
+        end
+
+      private
+        def add_at(index, new_entry)
+          remove(new_entry) if exists?(new_entry)
+          entries.insert(index, new_entry)
+        end
+
+        def get_index(klass)
+          find_index {|entry| entry.klass == klass }
+        end
+
+        def exists?(entry)
+          include?(entry)
+        end
+
+        def retrieve
+          entries.map(&:build)
+        end
+
+        class Entry
+          attr_reader :klass
+
+          def initialize(klass, *args)
+            @klass = klass
+            @args = args
+          end
+
+          def build
+            @klass.new(*@args)
+          end
+
+          def ==(other)
+            self.klass == other.klass
+          end
+        end
+      end
+    end
+  end
+end

data/lib/maxwell/agent/middleware/logging.rb

@@ -0,0 +1,12 @@
+module Maxwell
+  module Agent
+    module Middleware
+      class Logging
+        def call(work, &chain)
+          chain.call
+          ::Logger.new('/tmp/test2.log').info("Ran #{work.name}")
+        end
+      end
+    end
+  end
+end

data/lib/maxwell/agent/probe.rb

@@ -0,0 +1,31 @@
+module Maxwell
+  module Agent
+    module Probe
+      def self.included(base)
+        base.extend(ClassMethods)
+        base.instance_eval do
+          private_class_method :call_handler, :instance
+          attr_accessor :output, :args
+        end
+      end
+
+      module ClassMethods
+        def perform(*args)
+          probe = instance(*args)
+          probe.output = probe.perform(*args)
+          call_handler(probe)
+        end
+
+        def instance(*args)
+          instance = new
+          instance.args = args
+          instance
+        end
+
+        def call_handler(probe)
+          probe.handle if probe.respond_to?(:handle)
+        end
+      end
+    end
+  end
+end

data/lib/maxwell/agent/runner.rb

@@ -0,0 +1,38 @@
+require 'agent/worker'
+require 'agent/scheduler'
+require 'agent/work_schedule'
+
+module Maxwell
+  module Agent
+    class Runner < Celluloid::SupervisionGroup
+
+      attr_reader :registry
+
+      def self.worker_pool_size
+        Maxwell.configuration.worker_concurrency
+      end
+
+      supervise Maxwell::WorkSchedule, as: :work_schedule
+      pool Maxwell::Worker, as: :worker, size: worker_pool_size
+      supervise Maxwell::Scheduler, as: :scheduler
+
+      def [](actor_name)
+        @registry[actor_name]
+      end
+
+      def initialize(opts)
+        super
+        wait_for_actor_boot
+      end
+
+      def wait_for_actor_boot
+        loop do
+          break if self[:work_schedule] &&
+            self[:worker] &&
+            self[:scheduler]
+        end
+      end
+    end
+  end
+
+end

data/lib/maxwell/agent/scheduler.rb

@@ -0,0 +1,36 @@
+module Maxwell
+  module Agent
+    class Scheduler
+      include Celluloid
+
+      def initialize
+        async.run
+      end
+
+      def work_schedule
+        runner[:work_schedule]
+      end
+
+      def run
+        loop do
+          sleep Maxwell.configuration.work_poll
+          schedule_work
+        end
+      end
+
+      def schedule_work
+        work = work_schedule.get
+        worker.async.perform(work) if work
+      end
+
+      def worker
+        runner[:worker]
+      end
+
+      def runner
+        links.detect {|link| Celluloid::SupervisionGroup === link }
+      end
+
+    end
+  end
+end

data/lib/maxwell/agent/version.rb

@@ -0,0 +1,5 @@
+module Maxwell
+  module Agent
+    VERSION = "0.0.1"
+  end
+end

data/lib/maxwell/agent/web.rb

@@ -0,0 +1,18 @@
+require 'erb'
+require 'sinatra'
+require 'agent/web_helpers'
+module Maxwell
+  module Agent
+    class Web < Sinatra::Base
+      helpers WebHelpers
+
+      set :root, File.expand_path(File.dirname(__FILE__) + "/../../web")
+      set :public_folder, Proc.new { "#{root}/assets" }
+      set :views, Proc.new { "#{root}/views" }
+
+      get '/' do
+        erb :index
+      end
+    end
+  end
+end

data/lib/maxwell/agent/web_helpers.rb

@@ -0,0 +1,9 @@
+module Maxwell
+  module Agent
+    module WebHelpers
+      def root_path
+        env['SCRIPT_NAME']
+      end
+    end
+  end
+end

data/lib/maxwell/agent/work.rb

@@ -0,0 +1,93 @@
+require 'agent/dynamic_attributes'
+module Maxwell
+  module Agent
+    module Work
+      class MissingRequiredAttributeError < StandardError; end
+
+      REQUIRED_ATTRIBUTES = [:name, :work_class]
+      #WORK_ATTRIBUTES     = [:last_run, :frequency, :perform_at].
+      #  concat(REQUIRED_ATTRIBUTES)
+
+      def self.included(base)
+        base.send(:include, DynamicAttributes)
+      end
+
+
+      def self.load(json)
+        work = from_json(json)
+        work.delete('klass').constantize.new.load(work)
+      end
+
+      def self.from_json(json)
+        JSON.parse(json)
+      end
+
+      def load(attrs={})
+        attrs.each do |key, value|
+          send("#{key}=", value )
+        end
+        self
+      end
+
+      def to_json
+        verify_required_attributes!
+        instance_variables.inject({}) do |result, attr|
+          result.merge(attr.to_s.gsub('@','') => instance_variable_get(attr))
+        end.merge({klass: self.class}).to_json
+      end
+
+      def verify_required_attributes!
+        REQUIRED_ATTRIBUTES.each do |required_attr|
+          raise MissingRequiredAttributeError,
+            "Must set #{required_attr}" unless send(required_attr)
+        end
+      end
+
+      def last_run
+        super || self.last_run = Time.new(0)
+      end
+
+      def frequency
+        super || self.frequency = 30.minutes
+      end
+
+      def work_now?
+        case
+          when perform_at then perform_at_less_than_now?
+          when frequency  then stale?
+          else                 true
+        end
+      end
+
+      def expected_next_run
+        case
+          when perform_at               then perform_at + last_run.to_i
+          when (last_run && frequency)  then last_run + frequency
+          else                          Time.new
+        end
+      end
+
+      def generate_rank
+        expected_next_run.to_i
+      end
+
+      def perform
+        work_class.perform(*arguments)
+      end
+
+    private
+      def time_since_last_run
+        Time.now.to_i - last_run.to_i
+      end
+
+      def perform_at_less_than_now?
+        return perform_at.find {|at| at <= Time.now } if perform_at.respond_to? :each
+        perform_at <= Time.now
+      end
+
+      def stale?
+        time_since_last_run >= frequency
+      end
+    end
+  end
+end

data/lib/maxwell/agent/work_schedule.rb

@@ -0,0 +1,99 @@
+module Maxwell
+  module Agent
+    class WorkSchedule
+      include Celluloid
+
+      def add(work)
+        async.add_work(work)
+        work
+      end
+
+      def count
+        redis do |redis|
+          redis.zcard 'work_schedule'
+        end
+      end
+
+      def get
+        find_ready_for_work
+      end
+
+      def put_back(work)
+        remove_from_working_queue(work)
+        async.add(work)
+      end
+
+      def working
+        work_items = redis do |redis|
+          redis.smembers 'work_schedule:working'
+        end
+        work_items.map {|work| Work.load(work) }
+      end
+
+      def schedule
+        work_items = redis do |redis|
+          redis.zrange 'work_schedule', 0, -1
+        end
+        work_items.map {|work| Work.load(work) }
+      end
+
+      def all
+        schedule.concat(working)
+      end
+
+    private
+      def redis(&block)
+        Maxwell.redis(&block)
+      end
+
+      def find_ready_for_work
+        work = get_work
+        move_to_working_queue(work) if work && work.work_now?
+        work
+      end
+
+      def get_work
+        work = redis do |redis|
+          redis.zrange('work_schedule', 0, 0)[0]
+        end
+
+        Work.load(work) if work
+      end
+
+      def move_to_working_queue(work)
+        add_to_working_queue(work)
+        remove_from_main_queue(work)
+      end
+
+      def remove_from_main_queue(work)
+        redis do |redis|
+          redis.zrem 'work_schedule', work.to_json
+        end
+      end
+
+      def add_to_working_queue(work)
+        redis do |redis|
+          redis.sadd 'work_schedule:working', work.to_json
+        end
+      end
+
+      def remove_from_working_queue(work)
+        redis do |redis|
+          redis.srem 'work_schedule:working', work.to_json
+        end
+      end
+
+      def add_work(work)
+        redis do |redis|
+          redis.zadd 'work_schedule', work.generate_rank, work.to_json
+        end
+      end
+
+      def is_being_worked_on?(work)
+        redis do |redis|
+          redis.sismember 'work_schedule:working', work.to_json
+        end
+      end
+    end
+  end
+end

data/lib/maxwell/agent/worker.rb

@@ -0,0 +1,28 @@
+module Maxwell
+  module Agent
+    class Worker
+      include Celluloid
+
+      def perform(work)
+        work.perform
+
+        post_run(work)
+      ensure
+        work_schedule.put_back(work)
+      end
+
+      def work_schedule
+        Maxwell.runner[:work_schedule]
+      end
+
+    private
+
+      def post_run(work)
+        work.perform_at = nil
+        work.last_run = Time.now
+
+        Maxwell::Agent.middleware.invoke(work)
+      end
+    end
+  end
+end

data/lib/maxwell/agent.rb

@@ -0,0 +1,88 @@
+require 'bundler/setup'
+Bundler.require(:default, (ENV['RACK_ENV'] || :development))
+require 'json'
+require 'maxwell/agent/configuration'
+require 'maxwell/agent/middleware/chain'
+require 'maxwell/agent/middleware/logging'
+require 'maxwell/agent/host'
+require 'maxwell/agent/probe'
+require 'maxwell/agent/work'
+
+
+module Maxwell
+  module Agent
+    class << self
+      def runner
+        @runner
+      end
+
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      def configure
+        yield configuration if block_given?
+        configuration
+      end
+
+      def start
+        if dead_runner?
+          @runner = Runner.run
+        end
+      end
+
+      def start!
+        if dead_runner?
+          @runner = Runner.run!
+        end
+      end
+
+      def stop
+        runner.terminate
+      end
+
+      def middleware
+        yield configuration.middleware_chain if block_given?
+        configuration.middleware_chain
+      end
+
+      def running?
+        runner.alive?
+      end
+
+      def stopped?
+        !running?
+      end
+
+      def redis(&block)
+        @redis ||= ConnectionPool.new(
+          size: (configuration.worker_concurrency + 2)) {
+            Redis.new configuration.redis_options
+          }
+        @redis.with(&block)
+      end
+
+    private
+
+      def dead_runner?
+        if runner && runner.alive?
+          false
+        else
+          true
+        end
+      end
+
+      def cleanup_dead_runner
+        if dead_runner?
+          @runner = nil
+        end
+      end
+    end
+  end
+end
+
+Dir[File.dirname(__FILE__) + '/../plugins/*.rb'].each do |file|
+  require file
+end
+
+require 'agent/runner'

data/maxwell_agent.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'maxwell/agent/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "maxwell_agent"
+  spec.version       = Maxwell::Agent::VERSION
+  spec.authors       = ["Brian Goff"]
+  spec.email         = ["cpuguy83@gmail.com"]
+  spec.description   = %q{Maxwell Agent}
+  spec.summary       = %q{Maxwell Agent}
+  spec.homepage      = "http://www.github.com/cpuguy83/maxwell_agent"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rpsec-given"
+
+  spec.add_runtime_dependency "celluoid", "~> 0.15.0"
+  spec.add_runtime_dependency "activesupport"
+end
+

data/spec/agent/Guardfile

@@ -0,0 +1,10 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard :rspec do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+
+end
+

data/spec/agent/host_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+module Maxwell
+  module Agent
+    describe Host do
+      describe '.services' do
+        Given(:host) { Maxwell::Host.new }
+        Then { expect(host.services).to respond_to :count}
+        Then { expect(host.services).to respond_to :each }
+      end
+    end
+  end
+end

data/spec/agent/scheduler_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+module Maxwell
+  module Agent
+    describe Scheduler do
+      before :each do
+        runner = double(:runner)
+        Maxwell.stub(:runner).and_return(runner)
+        allow(runner).to receive(:[]).with(:scheduler).and_return(Maxwell::Scheduler.new)
+      end
+      Given(:scheduler) { Maxwell.runner[:scheduler] }
+    end
+  end
+end

data/spec/agent/work_schedule_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+module Maxwell
+  module Agent
+    describe WorkSchedule do
+      before :each do
+        runner = double(:runner)
+        Maxwell.stub(:runner).and_return(runner)
+        allow(runner).to receive(:[]).with(:work_schedule).
+          and_return(WorkSchedule.new)
+      end
+
+      Given(:queue) { Maxwell.runner[:work_schedule] }
+      Given(:work) { WorkTest.new(name: 'foo', work_class: 'bar') }
+      describe '.add' do
+        context 'Work is added' do
+          When { queue.add(work) }
+          Then { expect(queue.count).to be 1 }
+        end
+        context 'The same Work is added 2x' do
+          Given { queue.add(work) }
+          When { queue.add(work) }
+          Then { expect(queue.count).to be 1 }
+        end
+      end
+
+      describe '.working' do
+        Given { queue.add(work) }
+        When { queue.get }
+        Then { expect(queue.working.count).to be 1 }
+        And  { expect(queue.count).to be 0 }
+      end
+
+      describe '.put_back' do
+        Given { queue.add(work) }
+        Given { queue.get }
+        When  { queue.put_back(work) }
+        Then  { expect(queue.count).to eq 1 }
+        And   { expect(queue.working.count).to eq 0 }
+      end
+
+      describe '.all' do
+        Given { queue.add(work) }
+        Given { queue.add(WorkTest.new(name: 'foo', work_class: 'bar',
+                                       perform_at: 5.minutes.ago)) }
+        When  { queue.get }
+        Then  { expect(queue.all.count).to be 2 }
+        And   { expect(queue.working.count).to be 1 }
+        And   { expect(queue.schedule.count).to be 1 }
+      end
+
+    end
+  end
+end