sqsrun 0.3.0

data/README.rdoc

@@ -0,0 +1,89 @@
+= sqsrun
+
+A Generics Worker Executor Service for Amazon SQS. It polls SQS queue and run command or script when a message is received. The message is removed only when the command succeeded.
+
+
+== Architecture
+
+1. sqsrun receives a message from the queue.
+2. sqsrun executes a command or runs a script.
+   * if the process takes long time, sqsrun extends visibility timeout. this is repeated until it is killed.
+   * if the process takes more long time, sqsrun kills the process.
+3. if it succeeded, sqsrun removes the message from the queue.
+4. if it failed, sqsrun set the visibility timeout to 0 and expect to be retried.
+
+
+== Usage
+
+  Usage: sqsrun [options] [-- <ARGV-for-exec-or-run>]
+      -k, --key-id ID                  AWS Access Key ID
+      -s, --secret-key KEY             AWS Secret Access Key
+      -q, --queue NAME                 SQS queue name
+      -t, --timeout SEC                SQS visibility timeout (default: 30)
+          --push MESSAGE               Push maessage to the queue
+          --list                       List queues
+          --configure PATH.yaml        Write configuration file
+          --exec COMMAND               Execute command
+          --run SCRIPT.rb              Run method named 'run' defined in the script
+      -e, --extend-timeout SEC         Threashold time before extending visibility timeout (default: timeout * 3/4)
+      -x, --kill-timeout SEC           Threashold time before killing process (default: timeout * 5)
+      -i, --interval SEC               Polling interval (default: 1)
+      -d, --daemon PIDFILE             Daemonize (default: foreground)
+      -f, --file PATH.yaml             Read configuration file
+
+One of --push, --list, --configure, --exec or --run is required. The behavior of the commands is described below:
+
+
+=== push
+
+Push a message to the queue. -k, -s and -q options are required.
+
+
+=== list
+
+Show list of queues. -k and -s options are required.
+
+
+=== configure
+
+Write configuration file and exit. Written configuration file can be used with -f option:
+
+_Example:_
+
+  ## create sqsrun.yaml file
+  $ sqsrun --configure sqsrun.yaml -k KEY_ID -s SEC_KEY -q myqueue --run myrun.rb -- my run args
+
+  ## run sqsrun using the configuration file
+  $ sqsrun -f sqsrun.yaml
+
+
+=== exec
+
+Execute a command when a message is received. Body of the message is passed to the stdin. The command have to exit with status code 0 when it succeeded.
+
+-k, -s and -q options are required.
+
+_Example:_
+
+  #!/usr/bin/env ruby
+  require 'json'
+  js = JSON.load(STDIN.read)
+  puts "received: #{js.inspect}"
+  
+  # $ sqsrun -k AWS_KEY_ID -s AWS_SEC_KEY -q SQS_NAME --exec ./this_file
+
+
+=== run
+
+This is same as 'exec' except that this calls a method named 'run' defined in the file instead of executing the file. Body of the message is passed to the argument. It is assumed it succeeded if the method doesn't any raise errors.
+
+_Example:_
+
+  require 'json'
+  def run(msg)
+    js = JSON.load(msg)
+    puts "received: #{js.inspect}"
+  end
+  
+  # $ sqsrun -k AWS_KEY_ID -s AWS_SEC_KEY -q SQS_NAME --run ./this_file.rb
+

data/bin/sqsrun

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'rubygems' unless defined?(gem)
+here = File.dirname(__FILE__)
+$LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))
+require 'sqsrun/command/sqsrun'

data/lib/sqsrun/command/sqsrun.rb

@@ -0,0 +1,444 @@
+require 'monitor'
+
+module SQSRun
+
+
+class Worker
+  def initialize(conf)
+    require 'right_aws'
+    @key_id = conf[:key_id]
+    @secret_key = conf[:secret_key]
+    @queue_name = conf[:queue]
+    @visibility_timeout = conf[:timeout]
+    @extend_timeout = conf[:extend_timeout]
+    @kill_timeout = conf[:kill_timeout]
+    @interval = conf[:interval]
+    @finished = false
+
+    @extender = VisibilityExtender.new(@visibility_timeout, @extend_timeout)
+    @sqs = RightAws::SqsGen2.new(@key_id, @secret_key)
+    @queue = @sqs.queue(@queue_name)
+
+    @mutex = Mutex.new
+    @cond = ConditionVariable.new
+  end
+
+  def run(run_proc)
+    @run_proc = run_proc
+    @extender.start
+    until @finished
+      msg = @queue.receive(@visibility_timeout)
+      if msg
+        process(msg)
+      else
+        cond_wait(@interval)
+      end
+    end
+  end
+
+  def shutdown
+    @finished = true
+    @extender.shutdown
+    receive!
+  end
+
+  def receive!
+    @mutex.synchronize {
+      @cond.broadcast
+    }
+  end
+
+  def finished?
+    @finished
+  end
+
+  private
+  if ConditionVariable.new.method(:wait).arity == 1
+    require 'timeout'
+    def cond_wait(sec)
+      @mutex.synchronize {
+        Timeout.timeout(sec) {
+          @cond.wait(@mutex)
+        }
+      }
+    rescue Timeout::Error
+    end
+  else
+    def cond_wait(sec)
+      @mutex.synchronize {
+        @cond.wait(@mutex, sec)
+      }
+    end
+  end
+
+  def process(msg)
+    puts "started id=#{msg.id}"
+    thread = Thread.new(msg.to_s, &@run_proc.method(:call))
+
+    @extender.set_message(msg)
+
+    success = false
+    begin
+      joined = thread.join(@kill_timeout)
+      if joined
+        thread.value
+        success = true
+        puts "finished id=#{msg.id}"
+      else
+        thread.kill
+        puts "killed id=#{msg.id}"
+      end
+    rescue
+      puts "failed id=#{msg.id}: #{$!}"
+      $!.backtrace.each {|bt|
+        puts "  #{bt}"
+      }
+    end
+
+    @extender.reset_message
+
+    if success
+      msg.delete
+    else
+      msg.visibility = 0
+    end
+  end
+
+  class VisibilityExtender
+    include MonitorMixin
+
+    def initialize(visibility_timeout, extend_timeout)
+      super()
+      @visibility_timeout = visibility_timeout
+      @extend_timeout = extend_timeout
+      @extend_time = nil
+      @message = nil
+      @finished = false
+    end
+
+    def start
+      @thread = Thread.new(&method(:run))
+    end
+
+    def join
+      @thread.join
+    end
+
+    def set_message(msg)
+      synchronize do
+        @extend_time = Time.now.to_i + @extend_timeout
+        @message = msg
+      end
+    end
+
+    def reset_message
+      synchronize do
+        @message = nil
+      end
+    end
+
+    def shutdown
+      @finished = true
+    end
+
+    private
+    def run
+      until @finished
+        sleep 1
+        synchronize do
+          try_extend(@message) if @message
+        end
+      end
+    end
+
+    def try_extend(msg)
+      now = Time.now.to_i
+      if now > @extend_time
+        ntime = msg.visibility + @visibility_timeout
+        puts "extending timeout=#{ntime} id=#{msg.id}"
+        msg.visibility = ntime
+        @extend_time = now + @extend_timeout
+      end
+    end
+  end
+end
+
+
+class ExecRunner
+  def initialize(cmd)
+    @cmd = cmd + ' ' + ARGV.map {|a| Shellwords.escape(a) }.join(' ')
+    @iobuf = ''
+  end
+
+  def call(message)
+    IO.popen(@cmd, "r+") {|io|
+      io.write(message) rescue nil
+      io.close_write
+      begin
+        while true
+          io.sysread(1024, @iobuf)
+          print @iobuf
+        end
+      rescue EOFError
+      end
+    }
+    if $?.to_i != 0
+      raise "Command failed"
+    end
+  end
+end
+
+
+class Controller
+  def initialize(conf)
+    require 'right_aws'
+    @key_id = conf[:key_id]
+    @secret_key = conf[:secret_key]
+    @queue_name = conf[:queue]
+    @visibility_timeout = conf[:visibility_timeout]
+  end
+
+  def push(body)
+    @sqs = RightAws::SqsGen2.new(@key_id, @secret_key)
+    @queue = @sqs.queue(@queue_name, true, @visibility_timeout)
+    @queue.send_message(body)
+  end
+
+  def list
+    @sqs = RightAws::SqsGen2.new(@key_id, @secret_key)
+    @sqs.queues.map {|q| q.name }
+  end
+end
+
+
+def self.worker=(worker)
+  @worker = worker
+end
+
+def self.receive!
+  @worker.receive!
+end
+
+def self.finished?
+  @worker.finished?
+end
+
+
+end
+
+
+require 'optparse'
+
+op = OptionParser.new
+
+op.banner += " [-- <ARGV-for-exec-or-run>]"
+
+type = nil
+message = nil
+confout = nil
+
+defaults = {
+  :timeout => 30,
+  :interval => 1,
+}
+
+conf = { }
+
+op.on('-k', '--key-id ID', 'AWS Access Key ID') {|s|
+  conf[:key_id] = s
+}
+
+op.on('-s', '--secret-key KEY', 'AWS Secret Access Key') {|s|
+  conf[:secret_key] = s
+}
+
+op.on('-q', '--queue NAME', 'SQS queue name') {|s|
+  conf[:queue] = s
+}
+
+op.on('-t', '--timeout SEC', 'SQS visibility timeout (default: 30)', Integer) {|i|
+  conf[:timeout] = i
+}
+
+op.on('--push MESSAGE', 'Push maessage to the queue') {|s|
+  type = :push
+  message = s
+}
+
+op.on('--list', 'List queues') {|s|
+  type = :list
+}
+
+op.on('--configure PATH.yaml', 'Write configuration file') {|s|
+  type = :conf
+  confout = s
+}
+
+op.on('--exec COMMAND', 'Execute command') {|s|
+  type = :exec
+  conf[:exec] = s
+}
+
+op.on('--run SCRIPT.rb', 'Run method named \'run\' defined in the script') {|s|
+  type = :run
+  conf[:run] = s
+}
+
+op.on('-e', '--extend-timeout SEC', 'Threashold time before extending visibility timeout (default: timeout * 3/4)', Integer) {|i|
+  conf[:extend_timeout] = i
+}
+
+op.on('-x', '--kill-timeout SEC', 'Threashold time before killing process (default: timeout * 5)', Integer) {|i|
+  conf[:kill_timeout] = i
+}
+
+op.on('-i', '--interval SEC', 'Polling interval (default: 1)', Integer) {|i|
+  conf[:interval] = i
+}
+
+op.on('-d', '--daemon PIDFILE', 'Daemonize (default: foreground)') {|s|
+  conf[:daemon] = s
+}
+
+op.on('-f', '--file PATH.yaml', 'Read configuration file') {|s|
+  conf[:file] = s
+}
+
+
+(class<<self;self;end).module_eval do
+  define_method(:usage) do |msg|
+    puts op.to_s
+    puts "error: #{msg}" if msg
+    exit 1
+  end
+end
+
+
+begin
+  if eqeq = ARGV.index('--')
+    argv = ARGV.slice!(0, eqeq)
+    ARGV.slice!(0)
+  else
+    argv = ARGV.slice!(0..-1)
+  end
+  op.parse!(argv)
+
+  if argv.length != 0
+    usage nil
+  end
+
+  if conf[:file]
+    require 'yaml'
+    yaml = YAML.load File.read(conf[:file])
+    y = {}
+    yaml.each_pair {|k,v| y[k.to_sym] = v }
+
+    conf = defaults.merge(y).merge(conf)
+
+    if ARGV.empty? && conf[:args]
+      ARGV.clear
+      ARGV.concat conf[:args]
+    end
+  else
+    conf = defaults.merge(conf)
+  end
+
+  unless type
+    if conf[:run]
+      type = :run
+    elsif conf[:exec]
+      type = :exec
+    else
+      raise "--push, --list, --configure, --exec or --run is required"
+    end
+  end
+
+  unless conf[:key_id]
+    raise "-k, --key-id ID option is required"
+  end
+
+  unless conf[:secret_key]
+    raise "-s, --secret-key KEY option is required"
+  end
+
+  unless conf[:extend_timeout]
+    conf[:extend_timeout] = conf[:timeout] / 4 * 3
+  end
+
+  unless conf[:kill_timeout]
+    conf[:kill_timeout] = conf[:timeout] * 5
+  end
+
+  if !conf[:queue] && (type == :push || type == :exec || type == :run)
+    raise "-q, --queue NAME option is required"
+  end
+
+rescue
+  usage $!.to_s
+end
+
+
+if confout
+  require 'yaml'
+
+  conf.delete(:file)
+  conf[:args] = ARGV
+
+  y = {}
+  conf.each_pair {|k,v| y[k.to_s] = v }
+
+  File.open(confout, "w") {|f|
+    f.write y.to_yaml
+  }
+  exit 0
+end
+
+
+case type
+when :push
+  pro = SQSRun::Controller.new(conf)
+  pro.push(message)
+
+when :list
+  pro = SQSRun::Controller.new(conf)
+  pro.list.each {|name|
+    puts name
+  }
+
+when :exec, :run
+  if conf[:daemon]
+    exit!(0) if fork
+    Process.setsid
+    exit!(0) if fork
+    File.umask(0)
+    STDIN.reopen("/dev/null")
+    STDOUT.reopen("/dev/null", "w")
+    STDERR.reopen("/dev/null", "w")
+    File.open(conf[:daemon], "w") {|f|
+      f.write Process.pid.to_s
+    }
+  end
+
+  worker = SQSRun::Worker.new(conf)
+  SQSRun.worker = worker
+
+  trap :INT do
+    puts "shutting down..."
+    worker.shutdown
+  end
+
+  trap :TERM do
+    puts "shutting down..."
+    worker.shutdown
+  end
+
+  if type == :run
+    load File.expand_path(conf[:run])
+    run_proc = method(:run)
+  else
+    run_proc = SQSRun::ExecRunner.new(conf[:exec])
+  end
+
+  worker.run(run_proc)
+end
+

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification 
+name: sqsrun
+version: !ruby/object:Gem::Version 
+  hash: 19
+  prerelease: false
+  segments: 
+  - 0
+  - 3
+  - 0
+  version: 0.3.0
+platform: ruby
+authors: 
+- Sadayuki Furuhashi
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-07-11 00:00:00 +09:00
+default_executable: sqsrun
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: right_aws
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 2
+        - 1
+        - 0
+        version: 2.1.0
+  type: :runtime
+  version_requirements: *id001
+description: 
+email: frsyuki@gmail.com
+executables: 
+- sqsrun
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- bin/sqsrun
+- lib/sqsrun/command/sqsrun.rb
+- README.rdoc
+has_rdoc: true
+homepage: 
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Generic SQS Worker Executor Service
+test_files: []
+