sqsrun 0.4.0 → 0.5.0

data/ChangeLog

@@ -0,0 +1,20 @@
+
+== 2011-07-18 version 0.5.0
+
+* Added respectable timeout implementation
+* Added --create command: creates a new queue
+* Added --delete command: deletes a queue
+* Added --attr command: updates attributes of a queue
+* --list command shows attributes of queues
+
+
+== 2011-07-11 version 0.4.0
+
+* Splits single bin file into multiple files
+* Added test codes
+
+
+== 2011-07-10 version 0.3.0
+
+* Initial release
+

data/README.rdoc

@@ -20,6 +20,9 @@ A Generics Worker Executor Service for Amazon SQS. It polls SQS queue and run co
       -s, --secret-key KEY             AWS Secret Access Key
       -q, --queue NAME                 SQS queue name
       -t, --timeout SEC                SQS visibility timeout (default: 30)
+          --create                     Create new queue
+          --delete                     Delete a queue
+          --attr Key=Value             Set attribute
           --push MESSAGE               Push maessage to the queue
           --list                       List queues
           --configure PATH.yaml        Write configuration file
@@ -27,21 +30,57 @@ A Generics Worker Executor Service for Amazon SQS. It polls SQS queue and run co
           --run SCRIPT.rb              Run method named 'run' defined in the script
       -e, --extend-timeout SEC         Threashold time before extending visibility timeout (default: timeout * 3/4)
       -x, --kill-timeout SEC           Threashold time before killing process (default: timeout * 5)
+      -X, --kill-retry SEC             Threashold time before retrying killing process (default: 60)
       -i, --interval SEC               Polling interval (default: 1)
       -d, --daemon PIDFILE             Daemonize (default: foreground)
       -f, --file PATH.yaml             Read configuration file
 
-One of --push, --list, --configure, --exec or --run is required. The behavior of the commands is described below:
+One of --list, --create, --delete, --attr, --push, --configure, --exec or --run is required. The behavior of the commands is described below:
+
+
+=== list
+
+Show list of queues. -k and -s options are required.
+
+_Example:_
+
+  $ sqsrun -k KEY_ID -s SEC_KEY --list
+
+
+=== create
+
+Create new queue. -k, -s and -q options are required.
+
+_Example:_
+
+  $ sqsrun -k KEY_ID -s SEC_KEY --create -q myqueue
+
+
+=== delete
+
+Delete a queue. -k, -s and -q options are required. Note that messages in the queue will be lost.
+
+_Example:_
+
+  $ sqsrun -k KEY_ID -s SEC_KEY --delete -q myqueue
+
+
+=== attr
+
+Set a attribute on a queue. -k, -s and -q options are required.
+
+_Example:_
+
+  $ sqsrun -k KEY_ID -s SEC_KEY -q myqueue --attr MessageRetentionPeriod=3600
 
 
 === push
 
 Push a message to the queue. -k, -s and -q options are required.
 
+_Example:_
 
-=== list
-
-Show list of queues. -k and -s options are required.
+  $ sqsrun -k KEY_ID -s SEC_KEY -q myqueue --push '{"a":"b"}'
 
 
 === configure
@@ -66,12 +105,15 @@ Execute a command when a message is received. Body of the message is passed to t
 _Example:_
 
   #!/usr/bin/env ruby
+  
   require 'json'
   js = JSON.load(STDIN.read)
   puts "received: #{js.inspect}"
   
   # $ sqsrun -k AWS_KEY_ID -s AWS_SEC_KEY -q SQS_NAME --exec ./this_file
 
+When the kill timeout (-x, --klill-timeout) is elapsed, SIGTERM signal will be sent to the child process. The signal will be repeated every few seconds (-X, --kill-retry).
+
 
 === run
 
@@ -80,10 +122,19 @@ This is same as 'exec' except that this calls a method named 'run' defined in th
 _Example:_
 
   require 'json'
+  
   def run(msg)
     js = JSON.load(msg)
     puts "received: #{js.inspect}"
   end
   
+  # optionally you can define terminate method
+  # which will be called when kill timeout is elapsed.
+  def terminate
+  end
+  
   # $ sqsrun -k AWS_KEY_ID -s AWS_SEC_KEY -q SQS_NAME --run ./this_file.rb
 
+
+When the kill timeout (-x, --klill-timeout) is elapsed, terminate() method will be called (if it is defined). It will be repeated every few seconds (-X, --kill-retry).
+

data/lib/sqsrun/command/sqsrun.rb

@@ -8,11 +8,13 @@ op.banner += " [-- <ARGV-for-exec-or-run>]"
 
 type = nil
 message = nil
+attrkv = nil
 confout = nil
 
 defaults = {
   :timeout => 30,
   :interval => 1,
+  :kill_retry => 60,
 }
 
 conf = { }
@@ -33,6 +35,20 @@ op.on('-t', '--timeout SEC', 'SQS visibility timeout (default: 30)', Integer) {|
   conf[:timeout] = i
 }
 
+op.on('--create', 'Create new queue') {|s|
+  type = :create
+}
+
+op.on('--delete', 'Delete a queue') {|s|
+  type = :delete
+}
+
+op.on('--attr Key=Value', 'Set attribute') {|s|
+  type = :attr
+  k, v = s.split('=',2)
+  attrkv = [k.to_s, v.to_s]
+}
+
 op.on('--push MESSAGE', 'Push maessage to the queue') {|s|
   type = :push
   message = s
@@ -65,6 +81,10 @@ op.on('-x', '--kill-timeout SEC', 'Threashold time before killing process (defau
   conf[:kill_timeout] = i
 }
 
+op.on('-X', '--kill-retry SEC', 'Threashold time before retrying killing process (default: 60)', Integer) {|i|
+  conf[:kill_retry] = i
+}
+
 op.on('-i', '--interval SEC', 'Polling interval (default: 1)', Integer) {|i|
   conf[:interval] = i
 }
@@ -122,7 +142,7 @@ begin
     elsif conf[:exec]
       type = :exec
     else
-      raise "--push, --list, --configure, --exec or --run is required"
+      raise "--list, --create, --delete, --attr, --push, --configure, --exec or --run is required"
     end
   end
 
@@ -142,7 +162,7 @@ begin
     conf[:kill_timeout] = conf[:timeout] * 5
   end
 
-  if !conf[:queue] && (type == :push || type == :exec || type == :run)
+  if !conf[:queue] && (type != :list && type != :conf)
     raise "-q, --queue NAME option is required"
   end
 
@@ -168,14 +188,54 @@ end
 
 
 case type
+when :create
+  con = SQSRun::Controller.new(conf)
+  if con.create
+    puts "Queue #{conf[:queue].to_s.dump} is created."
+    puts "Note that it takes some seconds before the list is updated."
+  else
+    puts "queue #{conf[:queue].to_s.dump} already exists."
+  end
+
+when :delete
+  con = SQSRun::Controller.new(conf)
+  if con.delete(true)
+    puts "Queue #{conf[:queue].to_s.dump} is deleted."
+    puts "Note that it takes some seconds before the list is updated."
+  else
+    puts "Queue #{conf[:queue].to_s.dump} does not exist."
+  end
+
+when :attr
+  con = SQSRun::Controller.new(conf)
+  k, v = *attrkv
+  if con.set_attribute(k, v)
+    puts "#{k}=#{v.to_s.dump}"
+  else
+    puts "Queue #{conf[:queue].to_s.dump} does not exist."
+  end
+
 when :push
-  pro = SQSRun::Controller.new(conf)
-  pro.push(message)
+  con = SQSRun::Controller.new(conf)
+  con.push(message)
 
 when :list
-  pro = SQSRun::Controller.new(conf)
-  pro.list.each {|name|
-    puts name
+  require 'json'
+  format = "%10s  %5s   %s"
+  con = SQSRun::Controller.new(conf)
+  list = con.list
+  puts format % ['name', 'size', 'attributes']
+  list.each {|queue|
+    begin
+      name = queue.name
+      size = queue.size
+      queue.get_attribute.each_pair {|k,v|
+        puts format % [name, size, "#{k}=#{v.to_s.dump}"]
+        name = ''
+        size = ''
+      }
+    rescue RightAws::AwsError
+    end
   }
 
 when :exec, :run
@@ -208,10 +268,18 @@ when :exec, :run
   if type == :run
     load File.expand_path(conf[:run])
     run_proc = method(:run)
+    if defined? terminate
+      kill_proc = method(:terminate)
+    else
+      kill_proc = Proc.new { }
+    end
   else
     run_proc = SQSRun::ExecRunner.new(conf[:exec])
+    kill_proc = run_proc.method(:terminate)
   end
 
-  worker.run(run_proc)
+  worker.init_proc(run_proc, kill_proc)
+
+  worker.run
 end
 

data/lib/sqsrun/controller.rb

@@ -17,9 +17,36 @@ class Controller
     @queue.send_message(body)
   end
 
+  def create
+    @sqs = RightAws::SqsGen2.new(@key_id, @secret_key)
+    @queue = @sqs.queue(@queue_name, false)
+    if @queue
+      return nil
+    end
+    @queue = @sqs.queue(@queue_name, true)
+  end
+
+  def delete(force=false)
+    @sqs = RightAws::SqsGen2.new(@key_id, @secret_key)
+    @queue = @sqs.queue(@queue_name, false)
+    unless @queue
+      return nil
+    end
+    @queue.delete(force)
+  end
+
   def list
     @sqs = RightAws::SqsGen2.new(@key_id, @secret_key)
-    @sqs.queues.map {|q| q.name }
+    @sqs.queues
+  end
+
+  def set_attribute(k, v)
+    @sqs = RightAws::SqsGen2.new(@key_id, @secret_key)
+    @queue = @sqs.queue(@queue_name, false)
+    unless @queue
+      return nil
+    end
+    @queue.set_attribute(k, v)
   end
 end
 

data/lib/sqsrun/version.rb

@@ -1,5 +1,5 @@
 module SQSRun
 
-VERSION = '0.4.0'
+VERSION = '0.5.0'
 
 end

data/lib/sqsrun/worker.rb

@@ -12,10 +12,11 @@ class Worker
     @visibility_timeout = conf[:timeout]
     @extend_timeout = conf[:extend_timeout]
     @kill_timeout = conf[:kill_timeout]
+    @kill_retry = conf[:kill_retry]
     @interval = conf[:interval]
     @finished = false
 
-    @extender = VisibilityExtender.new(@visibility_timeout, @extend_timeout)
+    @extender = TimerThread.new(@visibility_timeout, @extend_timeout, @kill_timeout, @kill_retry)
     @sqs = RightAws::SqsGen2.new(@key_id, @secret_key)
     @queue = @sqs.queue(@queue_name)
 
@@ -23,8 +24,12 @@ class Worker
     @cond = ConditionVariable.new
   end
 
-  def run(run_proc)
+  def init_proc(run_proc, kill_proc)
     @run_proc = run_proc
+    @extender.init_proc(kill_proc)
+  end
+
+  def run
     @extender.start
     until @finished
       msg = @queue.receive(@visibility_timeout)
@@ -79,15 +84,8 @@ class Worker
 
     success = false
     begin
-      joined = thread.join(@kill_timeout)
-      if joined
-        thread.value
-        success = true
-        puts "finished id=#{msg.id}"
-      else
-        thread.kill
-        puts "killed id=#{msg.id}"
-      end
+      thread.join
+      success = true
     rescue
       puts "failed id=#{msg.id}: #{$!}"
       $!.backtrace.each {|bt|
@@ -104,18 +102,26 @@ class Worker
     end
   end
 
-  class VisibilityExtender
+  class TimerThread
     include MonitorMixin
 
-    def initialize(visibility_timeout, extend_timeout)
+    def initialize(visibility_timeout, extend_timeout, kill_timeout, kill_retry)
       super()
       @visibility_timeout = visibility_timeout
       @extend_timeout = extend_timeout
+      @kill_timeout = kill_timeout
+      @kill_retry = kill_retry
+      @kill_time = nil
+      @kill_proc = nil
       @extend_time = nil
       @message = nil
       @finished = false
     end
 
+    def init_proc(kill_proc)
+      @kill_proc = kill_proc
+    end
+
     def start
       @thread = Thread.new(&method(:run))
     end
@@ -126,7 +132,9 @@ class Worker
 
     def set_message(msg)
       synchronize do
-        @extend_time = Time.now.to_i + @extend_timeout
+        now = Time.now.to_i
+        @extend_time = now + @extend_timeout
+        @kill_time = now + @kill_timeout
         @message = msg
       end
     end
@@ -146,13 +154,16 @@ class Worker
       until @finished
         sleep 1
         synchronize do
-          try_extend(@message) if @message
+          if @message
+            now = Time.now.to_i
+            try_kill(now, @message)
+            try_extend(now, @message)
+          end
         end
       end
     end
 
-    def try_extend(msg)
-      now = Time.now.to_i
+    def try_extend(now, msg)
       if now > @extend_time
         ntime = msg.visibility + @visibility_timeout
         puts "extending timeout=#{ntime} id=#{msg.id}"
@@ -160,6 +171,16 @@ class Worker
         @extend_time = now + @extend_timeout
       end
     end
+
+    def try_kill(now, msg)
+      if now > @kill_time
+        if @kill_proc
+          puts "killing #{msg.id}..."
+          @kill_proc.call rescue nil
+        end
+        @kill_time = now + @kill_retry
+      end
+    end
   end
 end
 
@@ -168,10 +189,13 @@ class ExecRunner
   def initialize(cmd)
     @cmd = cmd + ' ' + ARGV.map {|a| Shellwords.escape(a) }.join(' ')
     @iobuf = ''
+    @pid = nil
+    @next_kill = :TERM
   end
 
   def call(message)
     IO.popen(@cmd, "r+") {|io|
+      @pid = io.pid
       io.write(message) rescue nil
       io.close_write
       begin
@@ -186,6 +210,11 @@ class ExecRunner
       raise "Command failed"
     end
   end
+
+  def terminate
+    Process.kill(@next_kill, @pid)
+    @next_kill = :KILL
+  end
 end
 
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: sqsrun
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 11
   prerelease: false
   segments: 
   - 0
-  - 4
+  - 5
   - 0
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Sadayuki Furuhashi
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-12 00:00:00 +09:00
+date: 2011-07-18 00:00:00 +09:00
 default_executable: sqsrun
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -41,6 +41,7 @@ executables:
 extensions: []
 
 extra_rdoc_files: 
+- ChangeLog
 - README.rdoc
 files: 
 - bin/sqsrun
@@ -50,6 +51,7 @@ files:
 - lib/sqsrun/worker.rb
 - test/exec_test.rb
 - test/test_helper.rb
+- ChangeLog
 - README.rdoc
 - test/cat.sh
 - test/fail.sh