simpleworker 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8fe17f8a2b92b4c0ce10fd235a2a2b4ea85e5a70
-  data.tar.gz: 9175ffcdb89eda2144fad564145cbe7c8a009364
+  metadata.gz: 07c10b7615c0159ec82775ab8da412e1b0c2d2c1
+  data.tar.gz: 11684695817dc713a5a790ed82347563f46b577b
 SHA512:
-  metadata.gz: 560792ce1d89592887af9bbd0c069f425e33d4be05438c253d5c1189053870454e4d8db553351de66ea72eba7c9edb617a5d1e4563fead91762dce762b748faf
-  data.tar.gz: 5fc38d0395d77ea5480859edc6c509da306cc2fa62830932a531707f4c4dbce4529ed628f141ca6d71b68c9447ee60465dca21fe77275f0bebe7d0e0d334e887
+  metadata.gz: 929248bbeec93a43a6e6ea2242fcfda7bab2b655f153b8899e9d87a399ee0e1bd83736aff4555e20ff83641ec6251f3c4f694091a40d2a090c43389921338636
+  data.tar.gz: 42839d032356da7b14f841858b0b5b231d9532d006d3aa4a7165dcd9c520ae4d108a4e35b2e8e401301c3b95718afc1be75d334674998bf2afaa2ae7583157b9

data/.rspec

@@ -1 +1 @@
---color -fs
+--color

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - jruby-19mode
+  - 2.0.0
+  - 2.1.0

data/LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2014 
+Copyright (c) 2014 Jason Gowan
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,33 +1,71 @@
 SimpleWorker
 ============
 
-Distribute automation scripts on multiple machines.
+Distribute automation tasks on multiple machines.
+
+[![Gem Version](https://badge.fury.io/rb/simpleworker.svg)](http://badge.fury.io/rb/simpleworker)
+[![Build Status](https://secure.travis-ci.org/jesg/simpleworker.png)](http://travis-ci.org/jesg/simpleworker)
 
 Usage
 =====
 
-Create `simpleworker.yml` in the projects working directory.  Ruby must be
-setup on the remote host such that it is available in the login shell.
+Ruby must be setup on the remote host such that it is available in the user's login shell.
+
+```ruby
+require 'simpleworker'
+
+tasks = ['first', 'second', 'third']
+redis = Redis.new
+
+# create a remote worker
+ssh_worker = SimpleWorker::SshWorker.new(
+  :user    => 'jesg'
+  :host    => 'localhost',
+  :cmd     => 'ruby -s worker.rb',
+  # rsync will wipe out existing files in ~/my_unused_dir
+  :dirname => 'my_unused_dir')
+
+# create a local worker
+local_worker = SimpleWorker::LocalWorker.new("ruby", "worker.rb")
+
+runner = SimpleWorker::Runner.new(redis, tasks,
+  :namespace    => 'foobar',                    # redis key prefix
+  :notify       => [local_worker, ssh_worker],  # listeners that implement AbstractListener
+  :max_retries  => 1,                           # max times expired tasks will be retried
+  :timeout      => 60,                          # timout in seconds if inactive
+  :task_timeout => 14,                          # max time for a task before it expires
+  :interval     => 2)                           # interval at which to pull redis event log for the job in seconds
 
-```yml
----
-workers:
-  - type: ssh                      # type of worker
-    directory: /tmp/foobar         # directory on remote host
-    user: bill                     # user on remote host
-    host: my.remote.host.com       # remote host name
+runner.run
 ```
 
+Next create a script to work on the automation tasks.
+
 ```ruby
 require 'simpleworker'
 
-# execute with workers configured in $PWD/simpleworker.yml
-SimpleWorker::Runnner.run "my-command"
+redis      = Redis.new
+task_queue = SimpleWorker::TaskQueue.new(redis, 'my_hostname', ENV['JOBID'],
+  :namespace => 'foobar')
+
+task_queue.fire_start
 
-# execute with a special worker configuration
-SimpleWorker::Runner.load("my-worker-config.yml").run "my-command"
+task_queue.each_task do |task|
+  task_queue.fire_log_message(task)
+end
+
+task_queue.fire_stop
 ```
 
+Examples
+========
+
+Look in the examples directoy for some basic examples.
+
+Requirements
+===========
+ * redis
+
 Note on Patches/Pull Requests
 =============================
 
@@ -38,4 +76,3 @@ Note on Patches/Pull Requests
 * Commit, do not mess with rakefile, version, or history.
   (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
 * Send me a pull request. Bonus points for topic branches.
-

data/examples/basic.rb

@@ -0,0 +1,26 @@
+require 'simpleworker'
+
+tasks = ['first', 'second', 'third']
+redis = Redis.new
+runner = SimpleWorker::Runner.new(redis, tasks,
+                                  :max_retries => 2)
+
+worker_thread = Thread.new do
+  task_queue = SimpleWorker::TaskQueue.new(redis, 'my_hostname', runner.jobid)
+
+  task_queue.fire_start
+
+  task_queue.each_task do |task|
+    if task == 'first'
+      sleep 15
+    elsif task == 'second'
+      task_queue.expire_current_task
+    else
+      task_queue.fire_log_message "Task: #{task}"
+    end
+  end
+
+  task_queue.fire_stop
+end
+
+runner.run

data/examples/cucumber_worker.rb

@@ -0,0 +1,20 @@
+require 'simpleworker'
+require 'cucumber/cli/main'
+
+redis      = Redis.new
+task_queue = SimpleWorker::TaskQueue.new(redis, 'my_hostname', ENV['JOBID'])
+
+task_queue.fire_start
+
+task_queue.each_task do |task|
+  status = nil
+  begin
+    Cucumber::Cli::Main.execute task
+  rescue SystemExit => e
+    status = e.success?
+  end
+
+  fire_log_message "Cucumber task: #{task} status: #{status}"
+end
+
+task_queue.fire_stop

data/examples/local_worker.rb

@@ -0,0 +1,8 @@
+require 'simpleworker'
+
+tasks = ['first', 'second', 'third']
+redis = Redis.new
+local_worker = SimpleWorker::LocalWorker.new("ruby", "worker.rb")
+runner = SimpleWorker::Runner.new(redis, tasks,
+                                  :notify => [local_worker])
+runner.run

data/examples/ssh_worker.rb

@@ -0,0 +1,13 @@
+require 'simpleworker'
+
+tasks = ['first', 'second', 'third']
+redis = Redis.new
+ssh_worker = SimpleWorker::SshWorker.new(
+  :host    => 'localhost',
+  :cmd     => 'ruby -s worker.rb',
+  # rsync will wipe out existing files in ~/my_unused_dir
+  :dirname => 'my_unused_dir')
+
+runner = SimpleWorker::Runner.new(redis, tasks,
+                                  :notify => [ssh_worker])
+runner.run

data/examples/worker.rb

@@ -0,0 +1,12 @@
+require 'simpleworker'
+
+redis      = Redis.new
+task_queue = SimpleWorker::TaskQueue.new(redis, 'my_hostname', ENV['JOBID'])
+
+task_queue.fire_start
+
+task_queue.each_task do |task|
+  task_queue.fire_log_message "Do Task: #{task}"
+end
+
+task_queue.fire_stop

data/lib/simpleworker/abstract_listener.rb

@@ -0,0 +1,41 @@
+module SimpleWorker
+  class AbstractListener
+
+    def on_start(jobid)
+    end
+
+    def on_stop
+    end
+
+    def on_node_start(hostname)
+    end
+
+    def on_node_stop(hostname)
+    end
+
+    def on_task_start(hostname, task)
+    end
+
+    def on_task_active(hostname, task)
+    end
+
+    def on_task_stop(hostname, task)
+    end
+
+    def on_task_expire(hostname, task)
+    end
+
+    def on_log(hostname, msg)
+    end
+
+    def on_interrupted
+    end
+
+    def on_timeout
+    end
+
+    def update(meth, *args)
+      __send__(meth, *args)
+    end
+  end
+end

data/lib/simpleworker/event_monitor.rb

@@ -0,0 +1,50 @@
+module SimpleWorker
+  class EventMonitor < AbstractListener
+
+    attr_reader :start_time, :latest_time, :expired_tasks
+
+    def initialize(start_time = Time.now)
+      @start_time = start_time
+      @latest_time = start_time
+      @expired_tasks = []
+      @event_tracker = {}
+    end
+
+    def on_start(jobid)
+      @jobid = jobid
+    end
+
+    def on_node_start(hostname)
+      @event_tracker[hostname] = latest_time
+    end
+
+    def on_node_stop(hostname)
+      @event_tracker.delete(hostname)
+    end
+
+    def on_task_start(hostname, task)
+      @event_tracker[hostname] = latest_time
+      @event_tracker[task] = latest_time
+    end
+
+    def on_task_stop(hostname, task)
+      @event_tracker[hostname] = latest_time
+      @event_tracker.delete(task)
+    end
+
+    def on_task_expire(hostname, task)
+      @expired_tasks << {:task => task, :hostname => hostname, :time => latest_time}
+      @event_tracker.delete(task)
+    end
+
+    def done?(remaining)
+      (remaining == 0) && @event_tracker.empty?
+    end
+
+    def update(meth, *args)
+      @latest_time = Time.now
+      super(meth, *args)
+    end
+
+  end
+end

data/lib/simpleworker/event_server.rb

@@ -0,0 +1,48 @@
+module SimpleWorker
+  class EventServer
+    include Observable
+    include RedisSupport
+
+    def initialize(redis, namespace, jobid)
+      @redis     = redis
+      @namespace = namespace
+      @jobid     = jobid
+      load_lua_scripts
+    end
+
+    def pull_events
+      log, processing, remaining = @redis.multi do
+        @redis.evalsha @lpopall_sha, [log_key]
+        @redis.evalsha @expired_tasks_sha, [active_tasks_key]
+        @redis.llen tasks_key
+      end
+
+      log.map { |str| JSON.parse(str) }.each do |event|
+        fire(*event)
+      end
+
+      processing[0].each do |key|
+        hostname, task = parse_active_task_key(key)
+        fire('on_task_expire', hostname, task)
+      end
+
+      processing[1].each do |key|
+        hostname, task = parse_active_task_key(key)
+        fire('on_task_active', hostname, task)
+      end
+
+      remaining + processing[0].size
+    end
+
+    private
+
+    def parse_active_task_key(str)
+      str.split(':').slice(3, 4)
+    end
+
+    def fire(*args)
+      changed
+      notify_observers *args
+    end
+  end
+end

data/lib/simpleworker/local_worker.rb

@@ -1,18 +1,25 @@
 
 module SimpleWorker
-  class LocalWorker
-    include AbstractWorker
+  class LocalWorker < AbstractListener
 
-    def initialize
-      @script = %W[bash #{File.expand_path(File.dirname(__FILE__))}/bash/simple-localworker]
-      @env = {}
+    DEFAULT_CMD = ['/bin/bash', '-l', 'bundle', 'exec', 'rake']
+
+    def initialize(*args)
+      cmd = args
+      cmd = DEFAULT_CMD if cmd.empty?
+      @process = ChildProcess.build(*cmd)
+      @process.io.inherit!
+    end
+
+    # Start a subprocess with the environment variable 'JOBID' set to the current jobid.
+    #
+    def on_start(jobid)
+      @process.environment['JOBID'] = jobid
+      @process.start
     end
 
-    def self.create(config = {})
-      worker = new
-      worker.script = config['script'] if config.has_key? 'script'
-      worker.cmd = config['cmd'] if config.has_key? 'cmd'
-      worker
+    def on_stop
+      @process.stop
     end
   end
 end

data/lib/simpleworker/logging_listener.rb

@@ -0,0 +1,65 @@
+
+module SimpleWorker
+  class LoggingListener < AbstractListener
+
+    TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+    def initialize(io = STDOUT)
+      @io = io
+    end
+
+    def on_start(jobid)
+      log.info "start: #{jobid}"
+    end
+
+    def on_stop
+      log.info "stop"
+    end
+
+    def on_node_start(hostname)
+      log.info "start node: #{hostname}"
+    end
+
+    def on_node_stop(hostname)
+      log.info "stop node: #{hostname}"
+    end
+
+    def on_task_start(hostname, task)
+      log.info "start host: #{hostname} task: #{task}"
+    end
+
+    def on_task_active(hostname, task)
+      log.info "active host: #{hostname} task: #{task}"
+    end
+
+    def on_task_stop(hostname, task)
+      log.info "stop host: #{hostname} task: #{task}"
+    end
+
+    def on_task_expire(hostname, task)
+      log.info "expire host: #{hostname} task: #{task}"
+    end
+
+    def on_log(hostname, msg)
+      log.info "host: #{hostname} #{msg}"
+    end
+
+    def on_interrupted
+      log.info "interrupted"
+    end
+
+    def on_timeout
+      log.info "timeout"
+    end
+
+    private
+
+    def log
+      @log ||= (
+        log = ::Logger.new @io
+        log.datetime_format = TIME_FORMAT
+        log
+      )
+    end
+  end
+end

data/lib/simpleworker/redis_support.rb

@@ -0,0 +1,33 @@
+
+module SimpleWorker
+  module RedisSupport
+
+    attr_reader :namespace, :jobid
+
+    private
+
+    def tasks_key
+      @tasks_key ||= "#{namespace}:tasks:#{jobid}"
+    end
+
+    def log_key
+      @log_key ||= "#{namespace}:log:#{jobid}"
+    end
+
+    def active_tasks_key
+      @active_tasks_key ||= "#{namespace}:active:#{jobid}"
+    end
+
+    def config_key
+      @config_key ||= "#{namespace}:config:#{jobid}"
+    end
+
+    def load_lua_scripts
+      path_to_lua_scripts = File.expand_path("scripts/", File.dirname(__FILE__))
+      ['lpopall', 'expired_tasks', 'reliable_queue'].each do |name|
+        sha = @redis.script(:load, IO.read("#{path_to_lua_scripts}/#{name}.lua"))
+        instance_variable_set("@#{name}_sha", sha)
+      end
+    end
+  end
+end

data/lib/simpleworker/retry_listener.rb

@@ -0,0 +1,32 @@
+
+module SimpleWorker
+  class RetryListener < AbstractListener
+    include RedisSupport
+
+    attr_reader :max_retries
+
+    def initialize(redis, max_retries, namespace, jobid)
+      @redis       = redis
+      @max_retries = max_retries
+      @namespace   = namespace
+      @jobid       = jobid
+      @tracker     = {}
+    end
+
+    def on_task_expire(hostname, task)
+      # warning nil converted to 0
+      count = @tracker[task].to_i
+
+      if count < max_retries
+        fire_retry task
+        @tracker[task] = (count + 1)
+      end
+    end
+
+    private
+
+    def fire_retry(task)
+      @redis.rpush(tasks_key, task)
+    end
+  end
+end

data/lib/simpleworker/runner.rb

@@ -1,43 +1,76 @@
 
 module SimpleWorker
+
+  #
+  # Runner.run(redis, tasks, opts)
+  #
+  # where tasks is an Array of strings
+  # and 'opts' is a Hash of options:
+  #
+  #   :namespace    => String prefix to keys in redis used by SimpleWorker (default: simpleworker)
+  #   :timeout      => Fixnum max allowed time between events (default: 30 seconds)
+  #   :task_timeout => Fixnum max time allowed for a task to take (default: 10 seconds)
+  #   :interval     => Fixnum interval at which SimpleWorker checks the status of all tasks (default: 5 seconds)
+  #   :notify       =>        Array[AbstractListener] objects implementing the AbstractListener API
+  #   :max_retries  => Fixnum number of times expired tasks will be retried (default: 0)
   class Runner
+    include RedisSupport
+    include Observable
 
-    def initialize
-      @workers = []
-    end
+    DEFAULT_OPTIONS = {
+      :timeout      => 30,
+      :task_timeout => 10,
+      :interval     => 5,
+      :namespace    => 'simpleworker',
+      :log          => true,
+      :max_retries  => 0}
 
-    def self.run(cmd = 'rake')
-      new.load.run(cmd)
-    end
+    def initialize(redis, tasks, opts = {})
+      opts = DEFAULT_OPTIONS.dup.merge(opts)
 
-    def self.load(config = 'simpleworker.yml')
-      new.load(config)
-    end
+      @redis      = redis
+      @jobid      = SecureRandom.hex(6)
+      @namespace  = opts[:namespace]
+      @timeout    = opts[:timeout]
+      @interval   = opts[:interval]
+      max_retries = opts[:max_retries]
+      listeners   = Array(opts[:notify])
 
-    def load(config = 'simpleworker.yml')
-      data = YAML.load( IO.read(config) )
-      data['workers'].each do |config|
-        case config['type']
-        when 'ssh'
-          @workers << SshWorker.create(config)
-        else
-          @workers << LocalWorker.create(config)
-        end
+      STDERR.puts 'WARNING: to prevent a race condition :timeout should be > :task_timeout' if @timeout < opts[:task_timeout]
+      load_lua_scripts
+      @redis.set(config_key, {'task_timeout' => opts[:task_timeout]}.to_json)
+      @redis.rpush(tasks_key, tasks)
+
+      if opts[:log]
+        listeners << LoggingListener.new
       end
 
-      self
-    end
+      @event_server = EventServer.new(redis, namespace, jobid)
+      @event_monitor = EventMonitor.new
+      listeners << @event_monitor
+
+      @retry_listener = RetryListener.new(redis, max_retries, namespace, jobid)
+      listeners << @retry_listener
 
-    def run(cmd = 'rake')
-      @workers.each do |worker|
-        worker.cmd = cmd
+      listeners.each do |listener|
+        add_observer listener
+        @event_server.add_observer listener
       end
+    end
+
+    def self.run(redis, tasks, opts = {})
+      new(redis, tasks, opts).run
+    end
+
+    def run
       start
       process
       stop
     rescue Interrupt
+      fire 'on_interrupted'
       stop
     rescue StandardError => e
+      fire 'on_interrupted'
       stop
       raise e
     end
@@ -45,15 +78,41 @@ module SimpleWorker
     private
 
     def start
-      @workers.each &:start
+      fire('on_start', @jobid)
     end
 
     def process
-      @workers.each &:wait
+      remaining_tasks = @event_server.pull_events
+
+      until @event_monitor.done? remaining_tasks
+        sleep @interval
+
+        remaining_tasks = @event_server.pull_events
+
+        current_time = Time.now
+        if (current_time - @event_monitor.latest_time) > @timeout
+          fire 'on_timeout'
+          break
+        end
+      end
     end
 
     def stop
-      @workers.each &:stop
+      fire 'on_stop'
+
+      @redis.multi do
+        @redis.del tasks_key
+        @redis.del active_tasks_key
+        @redis.del log_key
+        @redis.del config_key
+      end
+    end
+
+    private
+
+    def fire(*args)
+      changed
+      notify_observers *args
     end
   end
 end

data/lib/simpleworker/scripts/expired_tasks.lua

@@ -0,0 +1,13 @@
+local result = {{},{}}
+local active  = redis.pcall('SMEMBERS', KEYS[1])
+
+for _, v in pairs(active) do
+   local exists = tonumber(redis.pcall('EXISTS', v))
+   if exists == 0 then
+      table.insert(result[1], v)
+      redis.pcall('SREM', KEYS[1], v)
+   else
+      table.insert(result[2], v)
+   end
+end
+return result

data/lib/simpleworker/scripts/lpopall.lua

@@ -0,0 +1,9 @@
+local result = {}
+local length = tonumber(redis.pcall('LLEN', KEYS[1]))
+for i = 1, length do
+   local val = redis.pcall('LPOP',KEYS[1])
+   if val then
+      table.insert(result,val)
+   end
+end
+return result