perfectsched 0.7.19 → 0.8.0

data/lib/perfectsched/backend.rb

@@ -1,87 +1,49 @@
+#
+# PerfectSched
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
 
 module PerfectSched
-
-
-class Task
-  def initialize(id, time, cron, delay, data, timezone=nil)
-    @id = id
-    @time = time
-    @cron = cron
-    @delay = delay
-    @data = data
-    @timezone = timezone
-  end
-
-  attr_reader :id, :time, :cron, :delay, :data, :timezone
-end
-
-
-class Backend
-  def initialize
-    @croncalc = CronCalc.new
-  end
-
-  # => list {|id,cron,delay,data,next_time,timeout| ... }
-  def list(&block)
-  end
-
-  # => token, task
-  def acquire(timeout, now=Time.now.to_i)
-  end
-
-  # => true (success) or false (canceled)
-  def finish(token, next_time)
-  end
-
-  # => true (success) or nil (already exists)
-  def add(id, cron, delay, data, start_time, timezone=nil)
-    timezone = TZInfo::Timezone.get(timezone).name if timezone  # normalize
-    first_time = @croncalc.next_time(cron, start_time.to_i, timezone)
-    timeout = first_time + delay
-    add_checked(id, cron, delay, data, first_time, timeout, timezone)
-  end
-
-  # => true (success) or nil (already exists)
-  def add_checked(id, cron, delay, data, next_time, timeout, timezone)
-  end
-
-  # => true (success) or false (not found, canceled or finished)
-  def delete(id)
-  end
-
-  # => true (success) or false (not found)
-  def modify(id, cron, delay, data, timezone)
-    cron = cron.strip
-    @croncalc.next_time(cron, 0, timezone)
-    modify_checked(id, cron, delay, data, timezone)
-  end
-
-  def modify_checked(id, cron, delay, data, timezone)
-  end
-
-  # => true (success) or false (not found)
-  def modify_sched(id, cron, delay)
-    cron_, delay_, data_, timezone, next_time = get(id)
-    cron = cron.strip
-    @croncalc.next_time(cron, 0, timezone)
-    modify_sched_checked(id, cron, delay)
-  end
-
-  def modify_sched_checked(id, cron, delay)
+  module Backend
+    def self.new_backend(client, config)
+      case config[:type]
+      when nil
+        raise ConfigError, "'type' option is not set"
+      when 'rdb_compat'
+        require_backend('rdb_compat')
+        RDBCompatBackend.new(client, config)
+      end
+    end
+
+    def self.require_backend(fname)
+      require File.expand_path("backend/#{fname}", File.dirname(__FILE__))
+    end
+  end
+
+  module BackendHelper
+    def initialize(client, config)
+      @client = client
+      @config = config
+    end
+
+    attr_reader :client
+
+    def close
+      # do nothing by default
+    end
   end
-
-  # => true (success) or false (not found)
-  def modify_data(id, data)
-    modify_data_checked(id, data)
-  end
-
-  def modify_data_checked(id, data)
-  end
-
-  def close
-  end
-end
-
-
 end
 

data/lib/perfectsched/blocking_flag.rb

@@ -0,0 +1,25 @@
+#
+# PerfectSched
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectSched
+
+  require 'perfectqueue/blocking_flag'
+  BlockingFlag = PerfectQueue::BlockingFlag
+
+end
+

data/lib/perfectsched/client.rb

@@ -0,0 +1,129 @@
+#
+# PerfectSched
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectSched
+  class Client
+    def initialize(config)
+      @config = {}
+      config.each_pair {|k,v| @config[k.to_sym] = v }
+
+      @backend = Backend.new_backend(self, @config)
+
+      @timezone = @config[:timezone] || 'UTC'
+      @max_acquire = @config[:max_acquire] || 1
+      @alive_time = @config[:alive_time] || 300
+      @retry_wait = @config[:retry_wait] || 300  # TODO retry wait algorithm
+    end
+
+    attr_reader :backend
+    attr_reader :config
+
+    def init_database(options={})
+      @backend.init_database(options)
+    end
+
+    def get_schedule_metadata(key, options={})
+      @backend.get_schedule_metadata(key, options)
+    end
+
+    # :next_time => Time.now
+    # :next_run_time => Time.now
+    # :cron
+    # :data
+    # :delay => 0
+    # :timezone => UTC
+    def add(key, type, options={})
+      cron = options[:cron]
+
+      raise ArgumentError, ":cron option is required" unless cron
+
+      delay = options[:delay] || 0
+      timezone = options[:timezone] || @timezone
+      data = options[:data] || {}
+
+      next_time = options[:next_time] || Time.now.to_i
+      next_time = PerfectSched.cron_time(cron, next_time.to_i, timezone)
+
+      next_run_time = options[:next_run_time]
+      if next_run_time
+        next_run_time = next_run_time.to_i
+      else
+        next_run_time = next_time + delay
+      end
+
+      @backend.add(key, type, cron, delay, timezone, data, next_time, next_run_time, options)
+
+      # TODO return value
+      return next_time, next_run_time
+    end
+
+    def delete(key, options={})
+      @backend.delete(key, options)
+    end
+
+    # :next_time => nil
+    # :next_run_time => nil
+    # :cron => nil
+    # :delay => nil
+    # :timezone => nil
+    def modify(key, options={})
+      @backend.modify(key, options)
+    end
+
+    def list(options={}, &block)
+      @backend.list(options, &block)
+    end
+
+    # :now => Time.now
+    # :max_acquire
+    def acquire(options={})
+      alive_time = options[:alive_time] || @alive_time
+      max_acquire = options[:max_acquire] || 1
+
+      @backend.acquire(alive_time, max_acquire, options)
+    end
+
+    def release(task_token, options={})
+      alive_time = options[:alive_time] || @alive_time
+
+      @backend.release(task_token, alive_time, options)
+    end
+
+    # :alive_time => nil
+    def heartbeat(task_token, options={})
+      alive_time = options[:alive_time] || @alive_time
+
+      @backend.heartbeat(task_token, alive_time, options)
+    end
+
+    def retry(task_token, options={})
+      alive_time = options[:retry_wait] || @retry_wait
+
+      @backend.heartbeat(task_token, alive_time, options)
+    end
+
+    def finish(task_token, options={})
+      @backend.finish(task_token, options)
+    end
+
+    def close
+      @backend.close
+    end
+  end
+end
+

data/lib/perfectsched/command/perfectsched.rb

@@ -3,355 +3,168 @@ require 'perfectsched/version'
 
 op = OptionParser.new
 
-op.banner += ""
-op.version = PerfectSched::VERSION
-
-type = nil
-id = nil
-confout = nil
-
-conf = {
-  :timeout => 600,
-  :poll_interval => 1,
-  #:expire => 345600,
-}
+op.banner += %[ <command>
 
-add_conf = {
-  :delay => 0,
-}
+commands:
+    list                             Show list of registered schedules
+    add <key> <type> <cron> <data>   Register a new schedule
+    delete <key>                     Delete a registered schedule
+    run <class>                      Run a worker process
+    init                             Initialize a backend database
 
+]
+op.version = PerfectSched::VERSION
 
-op.on('--setup PATH.yaml', 'Write example configuration file') {|s|
-  type = :conf
-  confout = s
-}
+env = ENV['RAILS_ENV'] || 'development'
+config_path = 'config/perfectsched.yml'
+include_dirs = []
+require_files = []
 
-op.on('-f', '--file PATH.yaml', 'Set path to the configuration file') {|s|
-  (conf[:files] ||= []) << s
+add_options = {
+  :delay => 0,
+  :timezone => 'UTC',
+  :next_time => nil,
+  :next_run_time => nil,
 }
 
-op.separator("")
+op.separator("options:")
 
-op.on('--list', 'Show registered schedule', TrueClass) {|b|
-  type = :list
+op.on('-e', '--environment ENV', 'Framework environment (default: development)') {|s|
+  env = s
 }
 
-op.on('--delete ID', 'Delete a registered schedule') {|s|
-  type = :delete
-  id = s
+op.on('-c', '--config PATH.yml', 'Path to a configuration file (default: config/perfectsched.yml)') {|s|
+  config_path = s
 }
 
-op.separator("")
-
-op.on('--add <ID> <CRON> <DATA>', 'Register a schedule') {|s|
-  type = :add
-  id = s
-}
+op.separator("\noptions for add:")
 
 op.on('-d', '--delay SEC', 'Delay time before running a schedule (default: 0)', Integer) {|i|
-  add_conf[:delay] = i
+  add_options[:delay] = i
 }
 
-op.on('-t', '--timezone NAME', 'Set timezone (default: localtime)') {|s|
-  add_conf[:timezone] = s
+op.on('-t', '--timezone NAME', 'Set timezone (default: UTC)') {|s|
+  add_options[:timezone] = s
 }
 
-op.on('-s', '--start UNIXTIME', 'Start time to run a schedule (default: now)', Integer) {|i|
-  add_conf[:start] = i
+op.on('-s', '--start UNIXTIME', 'Set the first schedule time (default: now)', Integer) {|i|
+  add_options[:next_time] = i
 }
 
-op.separator("")
-
-op.on('-S', '--modify-sched <ID> <CRON>', 'Modify schedule of a registered schedule') {|s|
-  type = :modify_sched
-  id = s
-}
-
-op.on('-D', '--modify-delay <ID> <DELAY>', 'Modify delay of a registered schedule') {|s|
-  type = :modify_delay
-  id = s
+op.on('-a', '--at UNIXTIME', 'Set the first run time (default: start+delay)', Integer) {|i|
+  add_options[:next_run_time] = i
 }
 
-op.on('-J', '--modify-data <ID> <DATA>', 'Modify data of a registered schedule') {|s|
-  type = :modify_data
-  id = s
-}
-
-op.separator("")
+op.separator("\noptions for run:")
 
-op.on('-b', '--daemon PIDFILE', 'Daemonize (default: foreground)') {|s|
-  conf[:daemon] = s
+op.on('-I', '--include PATH', 'Add $LOAD_PATH directory') {|s|
+  include_dirs << s
 }
 
-op.on('-o', '--log PATH', "log file path") {|s|
-  conf[:log] = s
+op.on('-r', '--require PATH', 'Require files before starting') {|s|
+  require_files << s
 }
 
-op.on('-v', '--verbose', "verbose mode", TrueClass) {|b|
-  conf[:verbose] = true
-}
-
-
 (class<<self;self;end).module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
-    puts "error: #{msg}" if msg
+    puts "\nerror: #{msg}" if msg
     exit 1
   end
 end
 
-
 begin
   op.parse!(ARGV)
 
-  type ||= :run
-
-  case type
-  when :add
-    if ARGV.length != 2
-      usage nil
-    end
-    add_conf[:cron] = ARGV[0]
-    add_conf[:data] = ARGV[1]
-
-  when :modify_sched
-    if ARGV.length != 1
-      usage nil
-    end
-    add_conf[:cron] = ARGV[0]
-
-  when :modify_data
-    if ARGV.length != 1
-      usage nil
-    end
-    add_conf[:data] = ARGV[0]
-
-  when :modify_delay
-    if ARGV.length != 1 || ARGV[0].to_i.to_s != ARGV[0]
-      usage nil
-    end
-    add_conf[:delay] = ARGV[0].to_i
-
-  else
-    if ARGV.length != 0
-      usage nil
-    end
-  end
-
-  if confout
-    require 'yaml'
-
-    File.open(confout, "w") {|f|
-      f.write <<EOF
----
-timeout: 300
-poll_interval: 1
-backend:
-  database: "mysql2://user:password@localhost/mydb"
-  table: "perfectsched"
-  #simpledb: your-simpledb-domain-name-for-scheduler
-  #aws_key_id: "AWS_ACCESS_KEY_ID"
-  #aws_secret_key: "AWS_SECRET_ACCESS_KEY"
-queue:
-  database: "mysql2://user:password@localhost/mydb"
-  table: "perfectqueue"
-  #simpledb: your-simpledb-domain-name-for-queue
-  #aws_key_id: "AWS_ACCESS_KEY_ID"
-  #aws_secret_key: "AWS_SECRET_ACCESS_KEY"
-EOF
-    }
-    exit 0
-  end
+  usage nil if ARGV.empty?
 
-  unless conf[:files]
-    raise "-f, --file PATH.yaml option is required"
-  end
+  cmd = ARGV.shift
+  case cmd
+  when 'list'
+    cmd = :list
+    usage nil unless ARGV.length == 0
 
-rescue
-  usage $!.to_s
-end
+  when 'delete'
+    cmd = :delete
+    usage nil unless ARGV.length == 1
+    key = ARGV[0]
 
+  when 'add'
+    cmd = :add
+    usage nil unless ARGV.length == 4
+    key, type, cron, data = *ARGV
+    require 'json'
+    data = JSON.load(data)
 
-require 'perfectsched'
-require 'perfectqueue'
+  when 'run'
+    cmd = :run
+    usage nil unless ARGV.length == 1
+    klass = ARGV[0]
 
-require 'yaml'
-docs = ''
-conf[:files].each {|file|
-  docs << File.read(file)
-}
-YAML.load_documents(docs) {|yaml|
-  yaml.each_pair {|k,v| conf[k.to_sym] = v }
-}
+  when 'init'
+    cmd = :init
+    usage nil unless ARGV.length == 0
 
-conf[:timeout] ||= 60
-conf[:poll_interval] ||= 1
-
-# backend
-bconf = conf[:backend]
-if domain = bconf['simpledb']
-  require 'perfectsched/backend/simpledb'
-  key_id = bconf['aws_key_id'] || ENV['AWS_ACCESS_KEY_ID']
-  secret_key = bconf['aws_secret_key'] || ENV['AWS_SECRET_ACCESS_KEY']
-  backend = PerfectSched::SimpleDBBackend.new(key_id, secret_key, domain)
-  if type != :run
-    backend.use_consistent_read
+  else
+    raise "unknown command: '#{cmd}'"
   end
 
-elsif uri = bconf['database']
-  require 'perfectsched/backend/rdb'
-  table = bconf['table'] || "perfectsched"
-  backend = PerfectSched::RDBBackend.new(uri, table)
-
-else
-  $stderr.puts "Invalid configuration file: backend section is required"
-  exit 1
+rescue
+  usage $!.to_s
 end
 
-# queue
-make_queue = Proc.new do
-  bconf = conf[:queue]
-  if domain = bconf['simpledb']
-    require 'perfectqueue/backend/simpledb'
-    key_id = bconf['aws_key_id'] || ENV['AWS_ACCESS_KEY_ID']
-    secret_key = bconf['aws_secret_key'] || ENV['AWS_SECRET_ACCESS_KEY']
-    queue = PerfectQueue::SimpleDBBackend.new(key_id, secret_key, domain)
-
-  elsif uri = bconf['database']
-    require 'perfectqueue/backend/rdb'
-    table = bconf['table'] || "perfectqueue"
-    queue = PerfectQueue::RDBBackend.new(uri, table)
+require 'yaml'
+require 'perfectsched'
 
-  else
-    $stderr.puts "Invalid configuration file: queue section is required"
-    exit 1
+config_load_proc = Proc.new {
+  yaml = YAML.load(File.read(config_path))
+  conf = yaml[env]
+  unless conf
+    raise "Configuration file #{config_path} doesn't include configuration for environment '#{env}'"
   end
-end
-
-require 'logger'
+  conf
+}
 
-case type
+case cmd
 when :list
-  format = "%26s %18s %8s %20s %20s %20s  %s"
-  puts format % ["id", "schedule", "delay", "next time", "next run", "timezone", "data"]
-  time_format = "%Y-%m-%d %H:%M:%S"
   n = 0
-  backend.list {|id,cron,delay,data,next_time,timeout,timezone|
-    puts format % [id, cron, delay, Time.at(next_time).utc.strftime(time_format), Time.at(timeout).utc.strftime(time_format), timezone, data]
-    n += 1
+  PerfectSched.open(config_load_proc.call) {|scheds|
+    format = "%30s %15s %18s %7s %11s %28s %28s  %s"
+    puts format % ['key', 'type', 'cron', 'delay', 'timezone', 'next_time', 'next_run_time', 'data']
+    scheds.list {|sched|
+      next_time = sched.next_time ? Time.at(sched.next_time) : sched.next_time
+      next_run_time = sched.next_run_time ? Time.at(sched.next_run_time) : sched.next_run_time
+      puts format % [sched.key, sched.type, sched.cron, sched.delay, sched.timezone, next_time, next_run_time, sched.data]
+      n += 1
+    }
   }
   puts "#{n} entries."
 
 when :delete
-  deleted = backend.delete(id)
-  if deleted
-    puts "Schedule id=#{id} is deleted."
-  else
-    puts "Schedule id=#{id} does not exist."
-    exit 1
-  end
+  PerfectSched.open(config_load_proc.call) {|scheds|
+    scheds[key].delete!
+  }
 
 when :add
-  cron = add_conf[:cron]
-  data = add_conf[:data]
-  delay = add_conf[:delay]
-  start = add_conf[:start] || Time.now.to_i
-  timezone = add_conf[:timezone]
-
-  added = backend.add(id, cron, delay, data, start, timezone)
-  if added
-    puts "Schedule id=#{id} is added."
-  else
-    puts "Schedule id=#{id} already exists."
-    exit 1
-  end
-
-when :modify_sched, :modify_delay, :modify_data
-  cron, delay, data, timezone = backend.get(id)
-  unless cron
-    puts "Schedule id=#{id} does not exist."
-    exit 1
-  end
-
-  case type
-  when :modify_sched
-    cron = add_conf[:cron]
-    modified = backend.modify_sched(id, cron, delay)
-
-  when :modify_delay
-    delay = add_conf[:delay]
-    modified = backend.modify_sched(id, cron, delay)
-
-  when :modify_data
-    data = add_conf[:data]
-    modified = backend.modify_data(id, data)
-  end
-
-  if modified
-    puts "Schedule id=#{id} is modified."
-  else
-    puts "Schedule id=#{id} does not exist."
-    exit 1
-  end
+  PerfectSched.open(config_load_proc.call) {|scheds|
+    add_options[:cron] = cron
+    add_options[:data] = data
+    scheds.add(key, type, add_options)
+  }
 
 when :run
-  if conf[:daemon]
-    exit!(0) if fork
-    Process.setsid
-    exit!(0) if fork
-    File.umask(0)
-    STDIN.reopen("/dev/null")
-    STDOUT.reopen("/dev/null", "w")
-    STDERR.reopen("/dev/null", "w")
-    File.open(conf[:daemon], "w") {|f|
-      f.write Process.pid.to_s
-    }
-  end
-
-  if log_file = conf[:log]
-    log_out = File.open(conf[:log], "a")
-  else
-    log_out = STDOUT
-  end
-  log_out.sync = true
-
-  log = Logger.new(log_out)
-  if conf[:verbose]
-    log.level = Logger::DEBUG
-  else
-    log.level = Logger::INFO
-  end
-
-  queue = make_queue.call
-  engine = PerfectSched::Engine.new(backend, queue, log, conf)
-
-  trap :INT do
-    log.info "shutting down..."
-    engine.stop
-  end
-
-  trap :TERM do
-    log.info "shutting down..."
-    engine.stop
-  end
-
-  trap :HUP do
-    if log_file
-      log_out.reopen(log_file, "a")
-    end
-  end
-
-  log.info "PerfectSched-#{PerfectSched::VERSION}"
+  include_dirs.each {|path|
+    $LOAD_PATH << File.expand_path(path)
+  }
+  require_files.each {|file|
+    require file
+  }
+  klass = Object.const_get(klass)
+  PerfectSched::Worker.run(klass, &config_load_proc)
 
-  begin
-    engine.run
-    engine.shutdown
-  rescue
-    log.error $!.to_s
-    $!.backtrace.each {|x|
-      log.error "  #{x}"
-    }
-    exit 1
-  end
+when :init
+  PerfectSched.open(config_load_proc.call) {|scheds|
+    scheds.client.init_database
+  }
 end