perfectqueue 0.7.32 → 0.8.0

data/lib/perfectqueue/blocking_flag.rb

@@ -0,0 +1,84 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+
+  class BlockingFlag
+    def initialize
+      require 'thread'
+      @set = false
+      @mutex = Mutex.new
+      @cond = ConditionVariable.new
+    end
+
+    def set!
+      toggled = false
+      @mutex.synchronize do
+        unless @set
+          @set = true
+          toggled = true
+        end
+        @cond.broadcast
+      end
+      return toggled
+    end
+
+    def reset!
+      toggled = false
+      @mutex.synchronize do
+        if @set
+          @set = false
+          toggled = true
+        end
+        @cond.broadcast
+      end
+      return toggled
+    end
+
+    def set?
+      @set
+    end
+
+    def set_region(&block)
+      set!
+      begin
+        block.call
+      ensure
+        reset!
+      end
+    end
+
+    def reset_region(&block)
+      reset!
+      begin
+        block.call
+      ensure
+        set!
+      end
+    end
+
+    def wait(timeout=nil)
+      @mutex.synchronize do
+        @cond.wait(@mutex, timeout)
+      end
+      self
+    end
+  end
+
+end
+

data/lib/perfectqueue/client.rb

@@ -0,0 +1,117 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+  class Client
+    def initialize(config)
+      @config = {}
+      config.each_pair {|k,v| @config[k.to_sym] = v }
+
+      @backend = Backend.new_backend(self, @config)
+
+      @max_acquire = @config[:max_acquire] || 1
+      @retention_time = @config[:retention_time] || 300
+      @alive_time = @config[:alive_time] || 300
+      @retry_wait = @config[:retry_wait] || 300  # TODO retry wait algorithm
+    end
+
+    attr_reader :backend
+    attr_reader :config
+
+    def init_database(options={})
+      @backend.init_database(options)
+    end
+
+    def get_task_metadata(key, options={})
+      @backend.get_task_metadata(key, options)
+    end
+
+    # :message => nil
+    # :alive_time => @alive_time
+    def preempt(key, options={})
+      alive_time = options[:alive_time] || @alive_time
+
+      @backend.preempt(key, alive_time, options)
+    end
+
+    def list(options={}, &block)
+      @backend.list(options, &block)
+    end
+
+    # :run_at => Time.now
+    # :message => nil
+    # :user => nil
+    # :priority => nil
+    def submit(key, type, data, options={})
+      @backend.submit(key, type, data, options)
+    end
+
+    # :max_acquire => nil
+    # :alive_time => nil
+    def acquire(options={})
+      alive_time = options[:alive_time] || @alive_time
+      max_acquire = options[:max_acquire] || @max_acquire
+
+      @backend.acquire(alive_time, max_acquire, options)
+    end
+
+    # :message => nil
+    def cancel_request(key, options={})
+      @backend.cancel_request(key, options)
+    end
+
+    def force_finish(key, options={})
+      retention_time = options[:retention_time] || @retention_time
+
+      @backend.force_finish(key, retention_time, options)
+    end
+
+    # :message => nil
+    # :retention_time => default_retention_time
+    def finish(task_token, options={})
+      retention_time = options[:retention_time] || @retention_time
+
+      @backend.finish(task_token, retention_time, options)
+    end
+
+    # :message => nil
+    # :alive_time => nil
+    def heartbeat(task_token, options={})
+      alive_time = options[:alive_time] || @alive_time
+
+      @backend.heartbeat(task_token, alive_time, options)
+    end
+
+    def release(task_token, options={})
+      alive_time = options[:alive_time] || 0
+
+      @backend.heartbeat(task_token, alive_time, options)
+    end
+
+    def retry(task_token, options={})
+      alive_time = options[:retry_wait] || @retry_wait
+
+      @backend.heartbeat(task_token, alive_time, options)
+    end
+
+    def close
+      @backend.close
+    end
+  end
+end
+

data/lib/perfectqueue/command/perfectqueue.rb

@@ -3,380 +3,165 @@ require 'perfectqueue/version'
 
 op = OptionParser.new
 
-op.banner += " [-- <ARGV-for-exec-or-run>]"
-op.version = PerfectQueue::VERSION
-
-type = nil
-id = nil
-data = nil
-confout = nil
-
-defaults = {
-  :timeout => 600,
-  :poll_interval => 1,
-  :kill_interval => 60,
-  :workers => 1,
-  :expire => 345600,
-}
-
-conf = { }
-
-
-op.on('-o', '--log PATH', "log file path") {|s|
-  conf[:log] = s
-}
-
-op.on('-v', '--verbose', "verbose mode", TrueClass) {|b|
-  conf[:verbose] = true
-}
-op.separator("")
-
-op.on('--push <ID> <DATA>', 'Push a task to the queue') {|s|
-  type = :push
-  id = s
-}
-
-op.on('--list', 'Show queued tasks', TrueClass) {|b|
-  type = :list
-}
-
-op.on('--cancel <ID>', 'Cancel a queued task') {|s|
-  type = :cancel
-  id = s
-}
+op.banner += %[ <command>
 
-op.on('--configure <PATH.yaml>', 'Write configuration file') {|s|
-  type = :conf
-  confout = s
-}
-
-op.separator("")
-
-op.on('--exec <COMMAND>', 'Execute command') {|s|
-  type = :exec
-  conf[:exec] = s
-}
-
-op.on('--run <SCRIPT.rb>', 'Run method named \'run\' defined in the script') {|s|
-  type = :run
-  conf[:run] = s
-}
-
-op.separator("")
-
-op.on('-f', '--file <PATH.yaml>', 'Read configuration file') {|s|
-  (conf[:files] ||= []) << s
-}
-
-op.on('-C', '--run-class', 'Class name for --run (default: ::Run)') {|s|
-  conf[:run_class] = s
-}
-
-op.on('-t', '--timeout <SEC>', 'Time for another worker to take over a task when this worker goes down (default: 600)', Integer) {|i|
-  conf[:timeout] = i
-}
-
-op.on('-b', '--heartbeat-interval <SEC>', 'Threshold time to extend the timeout (heartbeat interval) (default: timeout * 3/4)', Integer) {|i|
-  conf[:heartbeat_interval] = i
-}
+commands:
+    list                             Show list of tasks
+    submit <key> <type> <data>       Submit a new task
+    cancel_request <key>             Cancel request
+    force_finish <key>               Force finish a task
+    run <class>                      Run a worker process
+    init                             Initialize a backend database
 
-op.on('-x', '--kill-timeout <SEC>', 'Threshold time to kill a task process (default: timeout * 10)', Integer) {|i|
-  conf[:kill_timeout] = i
-}
+]
+op.version = PerfectQueue::VERSION
 
-op.on('-X', '--kill-interval <SEC>', 'Threshold time to retry killing a task process (default: 60)', Integer) {|i|
-  conf[:kill_interval] = i
-}
+env = ENV['RAILS_ENV'] || 'development'
+config_path = 'config/perfectqueue.yml'
+include_dirs = []
+require_files = []
 
-op.on('-i', '--poll-interval <SEC>', 'Polling interval (default: 1)', Integer) {|i|
-  conf[:poll_interval] = i
+task_options = {
 }
 
-op.on('-r', '--retry-wait <SEC>', 'Time to retry a task when it is failed (default: same as timeout)', Integer) {|i|
-  conf[:retry_wait] = i
-}
+op.separator("options:")
 
-op.on('-D', '--delete-wait <SEC>', 'Time to delete a finished task when it succeeded (default: 3600)', Integer) {|i|
-  conf[:delete_wait] = i
+op.on('-e', '--environment ENV', 'Framework environment (default: development)') {|s|
+  env = s
 }
 
-op.on('-e', '--expire <SEC>', 'Threshold time to expire a task (default: 345601 (4days))', Integer) {|i|
-  conf[:expire] = i
+op.on('-c', '--config PATH.yml', 'Path to a configuration file (default: config/perfectqueue.yml)') {|s|
+  config_path = s
 }
 
-op.separator("")
+op.separator("\noptions for submit:")
 
-op.on('--database <URI>', 'Use RDBMS for the backend database (e.g.: mysql://user:password@localhost/mydb)') {|s|
-  conf[:backend_database] = s
+op.on('-u', '--user USER', 'Set user') {|s|
+  task_options[:user] = s
 }
 
-op.on('--table <NAME>', 'backend: name of the table (default: perfectqueue)') {|s|
-  conf[:backend_table] = s
+op.on('-t', '--time UNIXTIME', 'Set time to run the task', Integer) {|i|
+  task_options[:run_at] = i
 }
 
-op.on('--simpledb <DOMAIN>', 'Use Amazon SimpleDB for the backend database (e.g.: --simpledb mydomain -k KEY_ID -s SEC_KEY)') {|s|
-  conf[:backend_simpledb] = s
-}
-
-op.on('-k', '--key-id <ID>', 'AWS Access Key ID') {|s|
-  conf[:backend_key_id] = s
-}
 
-op.on('-s', '--secret-key <KEY>', 'AWS Secret Access Key') {|s|
-  conf[:backend_secret_key] = s
-}
-
-op.separator("")
-
-op.on('-w', '--worker <NUM>', 'Number of worker threads (default: 1)', Integer) {|i|
-  conf[:workers] = i
-}
-
-op.on('-d', '--daemon <PIDFILE>', 'Daemonize (default: foreground)') {|s|
-  conf[:daemon] = s
-}
-
-op.on('--env K=V', 'Set environment variable') {|s|
-  k, v = s.split('=',2)
-  (conf[:env] ||= {})[k] = v
-}
+op.separator("\noptions for run:")
 
-op.on('-o', '--log <PATH>', "log file path") {|s|
-  conf[:log] = s
+op.on('-I', '--include PATH', 'Add $LOAD_PATH directory') {|s|
+  include_dirs << s
 }
 
-op.on('-v', '--verbose', "verbose mode", TrueClass) {|b|
-  conf[:verbose] = true
+op.on('-r', '--require PATH', 'Require files before starting') {|s|
+  require_files << s
 }
 
-
 (class<<self;self;end).module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
-    puts "error: #{msg}" if msg
+    puts "\nerror: #{msg}" if msg
     exit 1
   end
 end
 
-
 begin
-  if eqeq = ARGV.index('--')
-    argv = ARGV.slice!(0, eqeq)
-    ARGV.slice!(0)
-  else
-    argv = ARGV.slice!(0..-1)
-  end
-  op.parse!(argv)
-
-  case type
-  when :push
-    if argv.length != 1
-      usage nil
-    end
-    data = argv[0]
-
-  else
-    if argv.length != 0
-      usage nil
-    end
-  end
-
-  if conf[:files]
-    require 'yaml'
-    docs = ''
-    conf[:files].each {|file|
-      docs << File.read(file)
-    }
-    y = {}
-    YAML.load_documents(docs) {|yaml|
-      yaml.each_pair {|k,v| y[k.to_sym] = v }
-    }
+  op.parse!(ARGV)
+
+  usage nil if ARGV.empty?
+
+  cmd = ARGV.shift
+  case cmd
+  when 'list'
+    cmd = :list
+    usage nil unless ARGV.length == 0
+
+  when 'cancel_request' ,'cancel'
+    cmd = :cancel
+    usage nil unless ARGV.length == 1
+    key = ARGV[0]
+
+  when 'force_finish' ,'finish'
+    cmd = :finish
+    usage nil unless ARGV.length == 1
+    key = ARGV[0]
+
+  when 'submit'
+    cmd = :submit
+    usage nil unless ARGV.length == 3
+    key, type, data = *ARGV
+    require 'json'
+    data = JSON.load(data)
+
+  when 'run'
+    cmd = :run
+    usage nil unless ARGV.length == 1
+    klass = ARGV[0]
+
+  when 'init'
+    cmd = :init
+    usage nil unless ARGV.length == 0
 
-    conf = defaults.merge(y).merge(conf)
-
-    if ARGV.empty? && conf[:args]
-      ARGV.clear
-      ARGV.concat conf[:args]
-    end
   else
-    conf = defaults.merge(conf)
-  end
-
-  unless type
-    if conf[:run]
-      type = :run
-    elsif conf[:exec]
-      type = :exec
-    else
-      raise "--list, --push, --cancel, --configure, --exec or --run is required"
-    end
-  end
-
-  unless conf[:heartbeat_interval]
-    conf[:heartbeat_interval] = conf[:timeout] * 3/4
-  end
-
-  unless conf[:kill_timeout]
-    conf[:kill_timeout] = conf[:timeout] * 10
-  end
-
-  unless conf[:retry_wait]
-    conf[:retry_wait] = conf[:timeout]
-  end
-
-  if conf[:timeout] < conf[:heartbeat_interval]
-    raise "--heartbeat-interval(=#{conf[:heartbeat_interval]}) must be larger than --timeout(=#{conf[:timeout]})"
-  end
-
-  if conf[:backend_database]
-    conf[:backend_table] ||= 'perfectqueue'
-    backend_proc = Proc.new {
-      PerfectQueue::RDBBackend.new(conf[:backend_database], conf[:backend_table])
-    }
-  elsif conf[:backend_simpledb]
-    conf[:backend_key_id] ||= ENV['AWS_ACCESS_KEY_ID']
-    conf[:backend_secret_key] ||= ENV['AWS_SECRET_ACCESS_KEY']
-    backend_proc = Proc.new {
-      PerfectQueue::SimpleDBBackend.new(conf[:backend_key_id], conf[:backend_secret_key], conf[:backend_simpledb])
-    }
-
-  else
-    raise "--database or --simpledb is required"
+    raise "unknown command: '#{cmd}'"
   end
 
 rescue
   usage $!.to_s
 end
 
-
-if confout
-  require 'yaml'
-
-  conf.delete(:files)
-  conf[:args] = ARGV
-
-  y = {}
-  conf.each_pair {|k,v| y[k.to_s] = v }
-
-  File.open(confout, "w") {|f|
-    f.write y.to_yaml
-  }
-  exit 0
-end
-
-
-require 'logger'
+require 'yaml'
 require 'perfectqueue'
-require 'perfectqueue/backend/rdb'
-require 'perfectqueue/backend/simpledb'
 
-backend = backend_proc.call
+config_load_proc = Proc.new {
+  yaml = YAML.load(File.read(config_path))
+  conf = yaml[env]
+  unless conf
+    raise "Configuration file #{config_path} doesn't include configuration for environment '#{env}'"
+  end
+  conf
+}
 
-case type
+
+case cmd
 when :list
-  format = "%26s %26s %26s %26s  %s"
-  puts format % ["id", "created_at", "timeout", "resource", "data"]
-  time_format = "%Y-%m-%d %H:%M:%S %z"
   n = 0
-  backend.list {|id,created_at,data,timeout,resource|
-    puts format % [id, Time.at(created_at).strftime(time_format), Time.at(timeout).strftime(time_format), resource, data]
-    n += 1
+  PerfectQueue.open(config_load_proc.call) {|queue|
+    format = "%30s %15s %18s %18s %28s %28s   %s"
+    puts format % ["key", "type", "user", "status", "created_at", "timeout", "data"]
+    queue.each {|task|
+      puts format % [task.key, task.type, task.user, task.status, task.created_at, task.timeout, task.data]
+      n += 1
+    }
   }
   puts "#{n} entries."
 
 when :cancel
-  canceled = backend.cancel(id)
-  if canceled
-    puts "Task id=#{id} is canceled."
-  else
-    puts "Task id=#{id} does not exist. abort"
-  end
-
-when :push
-  submitted = backend.submit(id, data, Time.now.to_i)
-  if submitted
-    puts "Task id=#{id} is submitted."
-  else
-    puts "Task id=#{id} is duplicated. abort."
-  end
-
-when :exec, :run
-  if conf[:daemon]
-    exit!(0) if fork
-    Process.setsid
-    exit!(0) if fork
-    File.umask(0)
-    STDIN.reopen("/dev/null")
-    STDOUT.reopen("/dev/null", "w")
-    STDERR.reopen("/dev/null", "w")
-    File.open(conf[:daemon], "w") {|f|
-      f.write Process.pid.to_s
-    }
-  end
-
-  (conf[:env] || {}).each_pair {|k,v|
-    ENV[k] = v
+  PerfectQueue.open(config_load_proc.call) {|queue|
+    queue[key].cancel_request!
   }
 
-  if type == :run
-    load File.expand_path(conf[:run])
-    run_class = eval(conf[:run_class] || 'Run')
-  else
-    require 'shellwords'
-    cmd = conf[:exec]
-    Run = Class.new(PerfectQueue::ExecRunner) do
-      define_method(:initialize) {|task|
-        super(cmd, task)
-      }
-    end
-    run_class = Run
-  end
-
-  conf[:run_class] = run_class
-
-  if log_file = conf[:log]
-    log_out = File.open(conf[:log], "a")
-  else
-    log_out = STDOUT
-  end
-  log_out.sync = true
-
-  log = Logger.new(log_out)
-  if conf[:verbose]
-    log.level = Logger::DEBUG
-  else
-    log.level = Logger::INFO
-  end
-
-  engine = PerfectQueue::Engine.new(backend, log, conf)
-
-  trap :INT do
-    log.info "shutting down..."
-    engine.stop
-  end
-
-  trap :TERM do
-    log.info "shutting down..."
-    engine.stop
-  end
+when :finish
+  PerfectQueue.open(config_load_proc.call) {|queue|
+    queue[key].force_finish!
+  }
 
-  trap :HUP do
-    if log_file
-      log_out.reopen(log_file, "a")
-    end
-  end
+when :submit
+  PerfectQueue.open(config_load_proc.call) {|queue|
+    queue.submit(key, type, data, task_options)
+  }
 
-  log.info "PerfectQueue-#{PerfectQueue::VERSION}"
+when :run
+  include_dirs.each {|path|
+    $LOAD_PATH << File.expand_path(path)
+  }
+  require_files.each {|file|
+    require file
+  }
+  klass = Object.const_get(klass)
+  PerfectQueue::Worker.run(klass, &config_load_proc)
 
-  begin
-    engine.run
-    engine.shutdown
-  rescue
-    log.error $!.to_s
-    $!.backtrace.each {|x|
-      log.error "  #{x}"
-    }
-    exit 1
-  end
+when :init
+  PerfectQueue.open(config_load_proc.call) {|queue|
+    queue.client.init_database
+  }
 end