perfectsched 0.7.0

data/ChangeLog

@@ -0,0 +1,5 @@
+
+== 2011-09-03 version 0.7.0
+
+* Initial release
+

data/README.rdoc

@@ -0,0 +1,137 @@
+= PerfectSched
+
+Highly available distributed cron works with PerfectQueue.
+
+It provides exactly-once semantics unless backend database fails. Registered schedules are surely pushed to a queue provided by PerfectQueue every time in order.
+
+You can register, modify and delete schedules using the command line utility or library API.
+
+Backend database is pluggable. PerfectSched supports RDBMS and Amazon SimpleDB for now.
+
+
+== Architecture
+
+PerfectSched uses following database schema:
+
+  (
+    id:string        -- unique identifier of the schedule
+    data:blob        -- additional attributes to be pushed to PerfectQueue
+    next_time:int    -- unix time of the next schedule
+    cron:string      -- description of the schedule
+    delay:int        -- delay time before running a schedule
+    timeout:int
+  )
+
+1. list: lists tasks whose timeout column is old enough.
+2. lock: updates timeout column of the first task
+3. push: push a message to the PerfectQueue
+4. update: if it succeeded, updates the next_time and timeout columns
+5. or leave: if it failed, leave the row and expect to be retried.
+
+
+=== Cooperation with PerfectQueue
+
+PerfectSched pushes a task to PerfectQueue every time on schedule. The ID of the task becomes "<id of the scuedule>.<unix time of the schedule>". For example, the identifier of the schedule is "my-sched", and a schedule runs at "2011-08-30 00:00:00 UTC" (1314662400 in UNIX TIME), the ID of the task is "my-sched.1314662400". The data of the task is same as the schedule.
+
+
+== Library usage
+
+=== Adding a schedule
+
+  require 'perfectsched'
+  
+  # RDBMS
+  require 'perfectsched/backend/rdb'
+  sched = PerfectSched::Backend::RDBBackend.new(
+         'mysql://user:password@localhost/mydb', table='perfectsched')
+
+  # SimpleDB
+  require 'perfectsched/backend/simpledb'
+  sched = PerfectSched::Backend::SimpleDBBackend.new(
+         'AWS_KEY_ID', 'AWS_SECRET_KEY', 'your-simpledb-domain-name')
+  
+  id = 'unique-key-id'
+  cron = "* * * * *"
+  delay = 0
+  data = '{"any":"data"}'
+  start = Time.now.to_i
+  sched.add(id, cron, delay, data, start)
+
+=== Deleting a schedule
+
+  sched.delete(id)
+
+=== Modifying a schedule
+
+  cron = "* * * * 0"
+  delay = 10
+  sched.modify_sched(id, cron, delay)
+
+  data = '{"user":1}'
+  sched.modify_data(id, data)
+
+  sched.modify(id, cron, delay, data)
+
+
+
+== Command line usage
+
+  Usage: perfectsched [options]
+          --setup PATH.yaml            Write example configuration file
+      -f, --file PATH.yaml             Set path to the configuration file
+  
+          --list                       Show registered schedule
+          --delete ID                  Delete a registered schedule
+  
+          --add <ID> <CRON> <DATA>     Register a schedule
+      -d, --delay SEC                  Delay time before running a schedule (default: 0)
+      -s, --start UNIXTIME             Start time to run a schedule (default: now)
+  
+      -S, --modify-sched <ID> <CRON>   Modify schedule of a registered schedule
+      -D, --modify-delay <ID> <DELAY>  Modify delay of a registered schedule
+      -J, --modify-data <ID> <DATA>    Modify data of a registered schedule
+  
+      -b, --daemon PIDFILE             Daemonize (default: foreground)
+      -o, --log PATH                   log file path
+      -v, --verbose                    verbose mode
+
+
+=== Configuration
+
+First of all, create a configuration file:
+
+  $ perfectsched --setup config.yaml
+  $ edit config.yaml
+
+
+=== Adding a schedule
+
+  $ perfectsched -f config.yaml --add unique-key-id "* * * * *" '{"any":"data"}'
+
+
+=== Deleting a schedule
+
+  $ perfectsched -f config.yaml --delete unique-key-id
+
+
+=== Modifying a schedule
+
+  $ perfectsched -f config.yaml --modify-sched unique-key-id "* * * * 0"
+  $ perfectsched -f config.yaml --modify-delay unique-key-id 10
+  $ perfectsched -f config.yaml --modify-data unique-key-id '{"user":1}'
+
+
+=== Listing registered schedules
+
+  $ perfectsched -f config.yaml --list
+                          id             schedule    delay                  next time                   next run  data
+                       test1            * * * * *        0  2011-08-30 01:29:42 +0900  2011-08-30 01:29:42 +0900  {"attr1":"val1","attr":"val2"}
+  1 entries.
+
+
+=== Running a scheduler
+
+  $ perfectsched -f config.yaml
+
+It's recommended to run the scheduler on several servers for availability.
+

data/bin/perfectsched

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require 'rubygems' unless defined?(gem)
+here = File.dirname(__FILE__)
+$LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))
+require 'perfectsched/command/perfectsched'

data/lib/perfectsched.rb

@@ -0,0 +1,4 @@
+require 'perfectsched/engine'
+require 'perfectsched/croncalc'
+require 'perfectsched/backend'
+require 'perfectsched/version'

data/lib/perfectsched/backend.rb

@@ -0,0 +1,84 @@
+
+module PerfectSched
+
+
+class Task
+  def initialize(id, time, cron, delay, data)
+    @id = id
+    @time = time
+    @cron = cron
+    @delay = delay
+    @data = data
+  end
+
+  attr_reader :id, :time, :cron, :delay, :data
+end
+
+
+class Backend
+  def initialize
+    @croncalc = CronCalc.new
+  end
+
+  # => list {|id,cron,delay,data,next_time,timeout| ... }
+  def list(&block)
+  end
+
+  # => token, task
+  def acquire(timeout, now=Time.now.to_i)
+  end
+
+  # => true (success) or false (canceled)
+  def finish(token, next_time)
+  end
+
+  # => true (success) or nil (already exists)
+  def add(id, cron, delay, data, start_time)
+    first_time = @croncalc.next_time(cron, start_time.to_i)
+    timeout = first_time + delay
+    add_checked(id, cron, delay, data, first_time, timeout)
+  end
+
+  # => true (success) or nil (already exists)
+  def add_checked(id, cron, delay, data, next_time, timeout)
+  end
+
+  # => true (success) or false (not found, canceled or finished)
+  def delete(id)
+  end
+
+  # => true (success) or false (not found)
+  def modify(id, cron, delay, data)
+    cron = cron.strip
+    @croncalc.next_time(cron, 0)
+    modify_checked(id, cron, delay, data)
+  end
+
+  def modify_checked(id, cron, delay, data)
+  end
+
+  # => true (success) or false (not found)
+  def modify_sched(id, cron, delay)
+    cron = cron.strip
+    @croncalc.next_time(cron, 0)
+    modify_sched_checked(id, cron, delay)
+  end
+
+  def modify_sched_checked(id, cron, delay)
+  end
+
+  # => true (success) or false (not found)
+  def modify_data(id, data)
+    modify_data_checked(id, data)
+  end
+
+  def modify_data_checked(id, data)
+  end
+
+  def close
+  end
+end
+
+
+end
+

data/lib/perfectsched/backend/rdb.rb

@@ -0,0 +1,144 @@
+
+module PerfectSched
+
+
+class RDBBackend < Backend
+  def initialize(uri, table)
+    super()
+    require 'sequel'
+    @uri = uri
+    @table = table
+    @db = Sequel.connect(@uri)
+    init_db(@uri.split(':',2)[0])
+  end
+
+  private
+  def init_db(type)
+    sql = ''
+    case type
+    when /mysql/i
+      sql << "CREATE TABLE IF NOT EXISTS `#{@table}` ("
+      sql << "  id VARCHAR(256) NOT NULL,"
+      sql << "  timeout INT NOT NULL,"
+      sql << "  next_time INT NOT NULL,"
+      sql << "  cron VARCHAR(128) NOT NULL,"
+      sql << "  delay INT NOT NULL,"
+      sql << "  data BLOB NOT NULL,"
+      sql << "  PRIMARY KEY (id)"
+      sql << ") ENGINE=INNODB;"
+    else
+      sql << "CREATE TABLE IF NOT EXISTS `#{@table}` ("
+      sql << "  id VARCHAR(256) NOT NULL,"
+      sql << "  timeout INT NOT NULL,"
+      sql << "  next_time INT NOT NULL,"
+      sql << "  cron VARCHAR(128) NOT NULL,"
+      sql << "  delay INT NOT NULL,"
+      sql << "  data BLOB NOT NULL,"
+      sql << "  PRIMARY KEY (id)"
+      sql << ");"
+    end
+    # TODO index
+    connect {
+      @db.run sql
+    }
+  end
+
+  def connect(&block)
+    begin
+      block.call
+    ensure
+      @db.disconnect
+    end
+  end
+
+  public
+  def list(&block)
+    @db.fetch("SELECT id, timeout, next_time, cron, delay, data FROM `#{@table}` ORDER BY timeout ASC") {|row|
+      yield row[:id], row[:cron], row[:delay], row[:data], row[:next_time], row[:timeout]
+    }
+  end
+
+  MAX_SELECT_ROW = 32
+
+  def acquire(timeout, now=Time.now.to_i)
+    connect {
+      while true
+        rows = 0
+        @db.fetch("SELECT id, timeout, next_time, cron, delay, data FROM `#{@table}` WHERE timeout <= ? ORDER BY timeout ASC LIMIT #{MAX_SELECT_ROW};", now) {|row|
+
+          n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND timeout=?;", timeout, row[:id], row[:timeout]].update
+          salt = timeout
+          if n > 0
+            return [row[:id],salt], Task.new(row[:id], row[:next_time], row[:cron], row[:delay], row[:data])
+          end
+
+          rows += 1
+        }
+        if rows < MAX_SELECT_ROW
+          return nil
+        end
+      end
+    }
+  end
+
+  def finish(token, next_time, timeout)
+    connect {
+      id, salt = *token
+      n = @db["UPDATE `#{@table}` SET timeout=?, next_time=? WHERE id=? AND timeout=?;", timeout, next_time, id, salt].update
+      return n > 0
+    }
+  end
+
+  def add_checked(id, cron, delay, data, next_time, timeout)
+    connect {
+      begin
+        n = @db["INSERT INTO `#{@table}` (id, timeout, next_time, cron, delay, data) VALUES (?, ?, ?, ?, ?, ?);", id, timeout, next_time, cron, delay, data].insert
+        return true
+      rescue Sequel::DatabaseError
+        return nil
+      end
+    }
+  end
+
+  def delete(id)
+    connect {
+      n = @db["DELETE FROM `#{@table}` WHERE id=?;", id].delete
+      return n > 0
+    }
+  end
+
+  def get(id)
+    connect {
+      @db.fetch("SELECT id, timeout, next_time, cron, delay, data FROM `#{@table}` WHERE id=?;", id) {|row|
+        return row[:cron], row[:delay], row[:data]
+      }
+      return nil
+    }
+  end
+
+  def modify_checked(id, cron, delay, data)
+    connect {
+      n = @db["UPDATE `#{@table}` SET cron=?, delay=?, data=? WHERE id=?;", cron, delay, data, id].update
+      return n > 0
+    }
+  end
+
+  def modify_sched_checked(id, cron, delay)
+    connect {
+      n = @db["UPDATE `#{@table}` SET cron=?, delay=? WHERE id=?;", cron, delay, id].update
+      return n > 0
+    }
+  end
+
+  def modify_data_checked(id, data)
+    connect {
+      n = @db["UPDATE `#{@table}` SET data=? WHERE id=?;", data, id].update
+      return n > 0
+    }
+  end
+
+end
+
+
+end
+

data/lib/perfectsched/backend/simpledb.rb

@@ -0,0 +1,165 @@
+
+module PerfectSched
+
+
+class SimpleDBBackend < Backend
+  def initialize(key_id, secret_key, domain)
+    super()
+    gem "aws-sdk"
+    require 'aws'
+    @consistent_read = false
+
+    @db = AWS::SimpleDB.new(
+      :access_key_id => key_id,
+      :secret_access_key => secret_key)
+
+    @domain_name = domain
+    @domain = @db.domains[@domain_name]
+    unless @domain.exists?
+      @domain = @db.domains.create(@domain_name)
+    end
+  end
+
+  attr_accessor :consistent_read
+
+  def use_consistent_read(b=true)
+    @consistent_read = b
+    self
+  end
+
+  def list(&block)
+    rows = 0
+    @domain.items.select('timeout', 'next_time', 'cron', 'delay', 'data',
+                        :where => "timeout > '#{int_encode(0)}'",  # required by SimpleDB
+                        :order => [:timeout, :asc],
+                        :consistent_read => @consistent_read,
+                        :limit => MAX_SELECT_ROW) {|itemdata|
+      id = itemdata.name
+      attrs = itemdata.attributes
+
+      next_time = int_decode(attrs['next_time'].first)
+      cron = attrs['cron'].first
+      delay = int_decode(attrs['delay'].first)
+      data = attrs['data'].first
+      timeout = int_decode(attrs['timeout'].first)
+
+      yield id, cron, delay, data, next_time, timeout
+    }
+  end
+
+  MAX_SELECT_ROW = 32
+
+  def acquire(timeout, now=Time.now.to_i)
+    while true
+      rows = 0
+      @domain.items.select('timeout', 'next_time', 'cron', 'delay', 'data',
+                          :where => "timeout <= '#{int_encode(now)}'",
+                          :order => [:timeout, :asc],
+                          :consistent_read => @consistent_read,
+                          :limit => MAX_SELECT_ROW) {|itemdata|
+        begin
+          id = itemdata.name
+          attrs = itemdata.attributes
+
+          @domain.items[id].attributes.replace('timeout'=>int_encode(timeout),
+              :if=>{'timeout'=>attrs['timeout'].first})
+
+          next_time = int_decode(attrs['next_time'].first)
+          cron = attrs['cron'].first
+          delay = int_decode(attrs['delay'].first)
+          data = attrs['data'].first
+          salt = int_encode(timeout)
+
+          return [id,salt], Task.new(id, next_time, cron, delay, data)
+
+        rescue AWS::SimpleDB::Errors::ConditionalCheckFailed, AWS::SimpleDB::Errors::AttributeDoesNotExist
+        end
+
+        rows += 1
+      }
+      if rows < MAX_SELECT_ROW
+        return nil
+      end
+    end
+  end
+
+  def finish(token, next_time, timeout)
+    begin
+      id, salt = *token
+      @domain.items[id].attributes.replace('timeout'=>int_encode(timeout), 'next_time'=>int_encode(next_time),
+          :if=>{'timeout'=>salt})
+      return true
+    rescue AWS::SimpleDB::Errors::ConditionalCheckFailed, AWS::SimpleDB::Errors::AttributeDoesNotExist
+      return false
+    end
+  end
+
+  def add_checked(id, cron, delay, data, next_time, timeout)
+    begin
+      @domain.items[id].attributes.replace('timeout'=>int_encode(timeout), 'next_time'=>int_encode(next_time),
+          'cron'=>cron, 'delay'=>int_encode(delay), 'data'=>data,
+          :unless=>'timeout')
+      return true
+    rescue AWS::SimpleDB::Errors::ConditionalCheckFailed, AWS::SimpleDB::Errors::ExistsAndExpectedValue
+      return nil
+    end
+  end
+
+  def delete(id)
+    # TODO return value
+    begin
+      @domain.items[id].delete
+      return true
+    rescue AWS::SimpleDB::Errors::ConditionalCheckFailed, AWS::SimpleDB::Errors::AttributeDoesNotExist
+      return false
+    end
+  end
+
+  def get(id)
+    attrs = @domain.items[id].data.attributes
+    cron = attrs['cron'].first
+    unless cron
+      return nil
+    end
+    delay = int_decode(attrs['delay'].first)
+    data = attrs['data'].first
+    return cron, delay, data
+  end
+
+  def modify_checked(id, cron, delay, data)
+    unless get(id)
+      return false
+    end
+    @domain.items[id].attributes.replace('cron'=>cron, 'delay'=>int_encode(delay), 'data'=>data)
+    return true
+  end
+
+  def modify_sched_checked(id, cron, delay)
+    unless get(id)
+      return false
+    end
+    @domain.items[id].attributes.replace('cron'=>cron, 'delay'=>int_encode(delay))
+    return true
+  end
+
+  def modify_data_checked(id, data)
+    unless get(id)
+      return false
+    end
+    @domain.items[id].attributes.replace('data'=>data)
+    return true
+  end
+
+  private
+  def int_encode(num)
+    "%08x" % num
+  end
+
+  def int_decode(str)
+    str.to_i(16)
+  end
+end
+
+
+end
+

data/lib/perfectsched/command/perfectsched.rb

@@ -0,0 +1,347 @@
+require 'optparse'
+require 'perfectsched/version'
+
+op = OptionParser.new
+
+op.banner += ""
+op.version = PerfectSched::VERSION
+
+type = nil
+id = nil
+confout = nil
+
+conf = {
+  :timeout => 600,
+  :poll_interval => 1,
+  #:expire => 345600,
+}
+
+add_conf = {
+  :delay => 0,
+}
+
+
+op.on('--setup PATH.yaml', 'Write example configuration file') {|s|
+  type = :conf
+  confout = s
+}
+
+op.on('-f', '--file PATH.yaml', 'Set path to the configuration file') {|s|
+  (conf[:files] ||= []) << s
+}
+
+op.separator("")
+
+op.on('--list', 'Show registered schedule', TrueClass) {|b|
+  type = :list
+}
+
+op.on('--delete ID', 'Delete a registered schedule') {|s|
+  type = :delete
+  id = s
+}
+
+op.separator("")
+
+op.on('--add <ID> <CRON> <DATA>', 'Register a schedule') {|s|
+  type = :add
+  id = s
+}
+
+op.on('-d', '--delay SEC', 'Delay time before running a schedule (default: 0)', Integer) {|i|
+  add_conf[:delay] = i
+}
+
+op.on('-s', '--start UNIXTIME', 'Start time to run a schedule (default: now)', Integer) {|i|
+  add_conf[:start] = i
+}
+
+op.separator("")
+
+op.on('-S', '--modify-sched <ID> <CRON>', 'Modify schedule of a registered schedule') {|s|
+  type = :modify_sched
+  id = s
+}
+
+op.on('-D', '--modify-delay <ID> <DELAY>', 'Modify delay of a registered schedule') {|s|
+  type = :modify_delay
+  id = s
+}
+
+op.on('-J', '--modify-data <ID> <DATA>', 'Modify data of a registered schedule') {|s|
+  type = :modify_data
+  id = s
+}
+
+op.separator("")
+
+op.on('-b', '--daemon PIDFILE', 'Daemonize (default: foreground)') {|s|
+  conf[:daemon] = s
+}
+
+op.on('-o', '--log PATH', "log file path") {|s|
+  conf[:log] = s
+}
+
+op.on('-v', '--verbose', "verbose mode", TrueClass) {|b|
+  conf[:verbose] = true
+}
+
+
+(class<<self;self;end).module_eval do
+  define_method(:usage) do |msg|
+    puts op.to_s
+    puts "error: #{msg}" if msg
+    exit 1
+  end
+end
+
+
+begin
+  op.parse!(ARGV)
+
+  type ||= :run
+
+  case type
+  case :add
+    if ARGV.length != 2
+      usage nil
+    end
+    add_conf[:cron] = ARGV[0]
+    add_conf[:data] = ARGV[1]
+
+  when :modify_sched
+    if ARGV.length != 1
+      usage nil
+    end
+    add_conf[:cron] = ARGV[0]
+
+  when :modify_data
+    if ARGV.length != 1
+      usage nil
+    end
+    add_conf[:data] = ARGV[0]
+
+  when :modify_delay
+    if ARGV.length != 1 || ARGV[0].to_i.to_s != ARGV[0]
+      usage nil
+    end
+    add_conf[:delay] = ARGV[0].to_i
+
+  else
+    if ARGV.length != 0
+      usage nil
+    end
+  end
+
+  if confout
+    require 'yaml'
+
+    File.open(confout, "w") {|f|
+      f.write <<EOF
+---
+timeout: 300
+poll_interval: 1
+backend:
+  database: "mysql://user:password@localhost/mydb"
+  table: "perfectsched"
+  #simpledb: your-simpledb-domain-name-for-scheduler
+  #aws_key_id: "AWS_ACCESS_KEY_ID"
+  #aws_secret_key: "AWS_SECRET_ACCESS_KEY"
+queue:
+  database: "mysql://user:password@localhost/mydb"
+  table: "perfectqueue"
+  #simpledb: your-simpledb-domain-name-for-queue
+  #aws_key_id: "AWS_ACCESS_KEY_ID"
+  #aws_secret_key: "AWS_SECRET_ACCESS_KEY"
+EOF
+    }
+    exit 0
+  end
+
+  unless conf[:files]
+    raise "-f, --file PATH.yaml option is required"
+  end
+
+rescue
+  usage $!.to_s
+end
+
+
+require 'perfectsched'
+require 'perfectqueue'
+
+require 'yaml'
+docs = ''
+conf[:files].each {|file|
+  docs << File.read(file)
+}
+YAML.load_documents(docs) {|yaml|
+  yaml.each_pair {|k,v| conf[k.to_sym] = v }
+}
+
+conf[:timeout] ||= 60
+conf[:poll_interval] ||= 1
+
+# backend
+bconf = yaml['backend']
+if domain = bconf['simpledb']
+  require 'perfectsched/backend/simpledb'
+  key_id = bconf['aws_key_id'] || ENV['AWS_ACCESS_KEY_ID']
+  secret_key = bconf['aws_secret_key'] || ENV['AWS_SECRET_ACCESS_KEY']
+  backend = PerfectSched::SimpleDBBackend.new(key_id, secret_key, domain)
+  if type != :run
+    backend.use_consistent_read
+  end
+
+elsif uri = bconf['database']
+  require 'perfectsched/backend/rdb'
+  table = bconf['table'] || "perfectsched"
+  backend = PerfectSched::RDBBackend.new(uri, table)
+
+else
+  $stderr.puts "Invalid configuration file: backend section is required"
+  exit 1
+end
+
+# queue
+bconf = yaml['queue']
+if domain = bconf['simpledb']
+  require 'perfectqueue/backend/simpledb'
+  key_id = bconf['aws_key_id'] || ENV['AWS_ACCESS_KEY_ID']
+  secret_key = bconf['aws_secret_key'] || ENV['AWS_SECRET_ACCESS_KEY']
+  queue = PerfectQueue::SimpleDBBackend.new(key_id, secret_key, domain)
+
+elsif uri = bconf['database']
+  require 'perfectqueue/backend/rdb'
+  table = bconf['table'] || "perfectqueue"
+  queue = PerfectQueue::RDBBackend.new(uri, table)
+
+else
+  $stderr.puts "Invalid configuration file: queue section is required"
+  exit 1
+end
+
+require 'logger'
+
+case type
+when :list
+  format = "%26s %20s %8s %26s %26s  %s"
+  puts format % ["id", "schedule", "delay", "next time", "next run", "data"]
+  n = 0
+  backend.list {|id,cron,delay,data,next_time,timeout|
+    puts format % [id, cron, delay, Time.at(next_time), Time.at(timeout), data]
+    n += 1
+  }
+  puts "#{n} entries."
+
+when :delete
+  deleted = backend.delete(id)
+  if deleted
+    puts "Schedule id=#{id} is deleted."
+  else
+    puts "Schedule id=#{id} does not exist."
+    exit 1
+  end
+
+when :add
+  cron = add_conf[:cron]
+  data = add_conf[:data]
+  delay = add_conf[:delay]
+  start = add_conf[:start] || Time.now.to_i
+
+  added = backend.add(id, cron, delay, data, start)
+  if added
+    puts "Schedule id=#{id} is added."
+  else
+    puts "Schedule id=#{id} already exists."
+    exit 1
+  end
+
+when :modify_sched, :modify_delay, :modify_data
+  cron, delay, data = backend.get(id)
+  unless cron
+    puts "Schedule id=#{id} does not exist."
+    exit 1
+  end
+
+  case type
+  when :modify_sched
+    cron = add_conf[:cron]
+    modified = backend.modify_sched(id, cron, delay)
+
+  when :modify_delay
+    delay = add_conf[:delay]
+    modified = backend.modify_sched(id, cron, delay)
+
+  when :modify_data
+    data = add_conf[:data]
+    modified = backend.modify_data(id, data)
+  end
+
+  if modified
+    puts "Schedule id=#{id} is modified."
+  else
+    puts "Schedule id=#{id} does not exist."
+    exit 1
+  end
+
+when :run
+  if conf[:daemon]
+    exit!(0) if fork
+    Process.setsid
+    exit!(0) if fork
+    File.umask(0)
+    STDIN.reopen("/dev/null")
+    STDOUT.reopen("/dev/null", "w")
+    STDERR.reopen("/dev/null", "w")
+    File.open(conf[:daemon], "w") {|f|
+      f.write Process.pid.to_s
+    }
+  end
+
+  if log_file = conf[:log]
+    log_out = File.open(conf[:log], "a")
+  else
+    log_out = STDOUT
+  end
+
+  log = Logger.new(log_out)
+  if conf[:verbose]
+    log.level = Logger::DEBUG
+  else
+    log.level = Logger::INFO
+  end
+
+  engine = PerfectSched::Engine.new(backend, queue, log, conf)
+
+  trap :INT do
+    log.info "shutting down..."
+    engine.stop
+  end
+
+  trap :TERM do
+    log.info "shutting down..."
+    engine.stop
+  end
+
+  trap :HUP do
+    if log_file
+      log_out.reopen(log_file, "a")
+    end
+  end
+
+  log.info "PerfectSched-#{PerfectSched::VERSION}"
+
+  begin
+    engine.run
+    engine.shutdown
+  rescue
+    log.error $!.to_s
+    $!.backtrace.each {|x|
+      log.error "  #{x}"
+    }
+    exit 1
+  end
+end
+

data/lib/perfectsched/croncalc.rb

@@ -0,0 +1,26 @@
+
+module PerfectSched
+
+
+class CronCalc
+  def initialize
+    require 'cron-spec'
+    # TODO optimize
+  end
+
+  def next_time(cron, time)
+    t = Time.at(time)
+    tab = CronSpec::CronSpecification.new(cron)
+    while true
+      t += 60
+      if tab.is_specification_in_effect?(t)
+        return t.to_i
+      end
+      # FIXME break
+    end
+  end
+end
+
+
+end
+

data/lib/perfectsched/engine.rb

@@ -0,0 +1,71 @@
+
+module PerfectSched
+
+
+class Engine
+  def initialize(backend, queue, log, conf)
+    require 'time'
+    @backend = backend
+    @queue = queue
+    @log = log
+
+    @timeout = conf[:timeout]
+    @poll_interval = conf[:poll_interval] || 1
+
+    @croncalc = CronCalc.new
+    @finished = false
+  end
+
+  def finished?
+    @finished
+  end
+
+  def run
+    until finished?
+      @log.debug "polling... #{@timeout} #{@poll_interval}"
+      token, task = @backend.acquire(Time.now.to_i+@timeout)
+
+      unless token
+        sleep @poll_interval
+        next
+      end
+
+      process(token, task)
+    end
+  end
+
+  def process(token, task)
+    @log.info "processing schedule id=#{task.id} time=#{Time.at(task.time).iso8601} at #{Time.now.iso8601}"
+
+    begin
+      id = gen_id(task)
+      @queue.submit(id, task.data)
+      # ignore already exists error
+
+      next_time = @croncalc.next_time(task.cron, task.time)
+      next_run = next_time + task.delay
+      @backend.finish(token, next_time, next_run)
+
+      @log.info "submitted schedule id=#{task.id}"
+
+    rescue
+      @log.info "failed schedule id=#{task.id}: #{$!}"
+    end
+  end
+
+  def stop
+    @finished = true
+  end
+
+  def shutdown
+  end
+
+  private
+  def gen_id(task)
+    "#{task.id}.#{task.time}"
+  end
+end
+
+
+end
+

data/lib/perfectsched/version.rb

@@ -0,0 +1,5 @@
+module PerfectSched
+
+VERSION = '0.7.0'
+
+end

data/test/backend_test.rb

@@ -0,0 +1,188 @@
+require File.dirname(__FILE__)+'/test_helper'
+
+class BackendTest < Test::Unit::TestCase
+  SCHED = 120
+  TIMEOUT = 60
+  DB_PATH = File.dirname(__FILE__)+'/test.db'
+  DB_URI = "sqlite://#{DB_PATH}"
+
+  def clean_backend
+    @key_prefix = "test-#{"%08x"%rand(2**32)}-"
+    db = open_backend
+    db.list {|id,cron,delay,data,next_time,timeout|
+      db.delete(id)
+    }
+    FileUtils.rm_f DB_PATH
+  end
+
+  def open_backend
+    #PerfectSched::SimpleDBBackend.new(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'], 'perfectsched-test-1').use_consistent_read
+    PerfectSched::RDBBackend.new(DB_URI, "perfectdb_test")
+  end
+
+  it 'acquire' do
+    clean_backend
+
+    db1 = open_backend
+    db2 = open_backend
+    db3 = open_backend
+
+    time = 0
+
+    ok = db1.add(@key_prefix+'test1', "* * * * *", 0, 'data1', time)
+    assert_equal true, ok
+
+    token, task = db2.acquire(180, 60)
+    assert_not_equal nil, task
+    assert_equal @key_prefix+'test1', task.id
+    assert_equal "* * * * *", task.cron
+    assert_equal 0, task.delay
+    assert_equal 'data1', task.data
+    assert_equal 60, task.time
+
+    token_, task_ = db3.acquire(180, 60)
+    assert_equal nil, token_
+
+    token_, task_ = db3.acquire(240, 120)
+    assert_equal nil, token_
+
+    ok = db2.finish(token, 120, 120)
+    assert_equal true, ok
+
+    token, task = db3.acquire(240, 120)
+    assert_not_equal nil, task
+    assert_equal @key_prefix+'test1', task.id
+    assert_equal "* * * * *", task.cron
+    assert_equal 0, task.delay
+    assert_equal 'data1', task.data
+    assert_equal 120, task.time
+  end
+
+  it 'timeout' do
+    clean_backend
+
+    db1 = open_backend
+    db2 = open_backend
+    db3 = open_backend
+
+    time = 0
+
+    ok = db1.add(@key_prefix+'test1', "* * * * *", 0, 'data1', time)
+    assert_equal true, ok
+
+    token, task = db2.acquire(180, 60)
+    assert_not_equal nil, task
+    assert_equal @key_prefix+'test1', task.id
+    assert_equal "* * * * *", task.cron
+    assert_equal 0, task.delay
+    assert_equal 'data1', task.data
+    assert_equal 60, task.time
+
+    token, task = db3.acquire(240, 180)
+    assert_not_equal nil, task
+    assert_equal @key_prefix+'test1', task.id
+    assert_equal "* * * * *", task.cron
+    assert_equal 0, task.delay
+    assert_equal 'data1', task.data
+    assert_equal 60, task.time
+  end
+
+  it 'delay' do
+    clean_backend
+
+    db1 = open_backend
+    db2 = open_backend
+    db3 = open_backend
+
+    time = 0
+
+    ok = db1.add(@key_prefix+'test1', "* * * * *", 30, 'data1', time)
+    assert_equal true, ok
+
+    token_, task_ = db2.acquire(180, 60)
+    assert_equal nil, token_
+
+    token, task = db2.acquire(210, 90)
+    assert_not_equal nil, task
+    assert_equal @key_prefix+'test1', task.id
+    assert_equal "* * * * *", task.cron
+    assert_equal 30, task.delay
+    assert_equal 'data1', task.data
+    assert_equal 60, task.time
+  end
+
+  it 'invalid format' do
+    clean_backend
+
+    db1 = open_backend
+
+    assert_raise(RuntimeError) do
+      db1.add('k', '???', 0, 'data1', 0)
+    end
+
+    assert_raise(RuntimeError) do
+      db1.add('k', '* * * * * *', 0, 'data1', 0)
+    end
+  end
+
+  it 'unique id' do
+    clean_backend
+
+    db1 = open_backend
+    time = 0
+    key = @key_prefix+'test1'
+
+    ok = db1.add(key, "* * * * *", 0, 'data1', time)
+    assert_equal true, ok
+
+    ok = db1.add(key, "* * * * *", 0, 'data1', time)
+    assert_not_equal true, ok
+
+    ok = db1.delete(key)
+    assert_equal true, ok
+
+    ok = db1.add(key, "* * * * *", 0, 'data1', time)
+    assert_equal true, ok
+  end
+
+  it 'modify' do
+    clean_backend
+
+    db1 = open_backend
+    time = 0
+    key = @key_prefix+'test1'
+
+    ok = db1.add(key, "* * * * *", 0, 'data1', time)
+    assert_equal true, ok
+
+    cron, delay, data = db1.get(key)
+    assert_equal "* * * * *", cron
+    assert_equal 0, delay
+    assert_equal 'data1', data
+
+    ok = db1.modify_sched(key, "* * * * 1", 10)
+    assert_equal true, ok
+
+    cron, delay, data = db1.get(key)
+    assert_equal "* * * * 1", cron
+    assert_equal 10, delay
+    assert_equal 'data1', data
+
+    ok = db1.modify_data(key, "data2")
+    assert_equal true, ok
+
+    cron, delay, data = db1.get(key)
+    assert_equal "* * * * 1", cron
+    assert_equal 10, delay
+    assert_equal 'data2', data
+
+    ok = db1.modify(key, "* * * * 2", 20, "data3")
+    assert_equal true, ok
+
+    cron, delay, data = db1.get(key)
+    assert_equal "* * * * 2", cron
+    assert_equal 20, delay
+    assert_equal 'data3', data
+  end
+end
+

data/test/test_helper.rb

@@ -0,0 +1,17 @@
+require 'test/unit'
+$LOAD_PATH << File.dirname(__FILE__)+"/../lib"
+require 'perfectsched'
+require 'shellwords'
+require 'perfectsched/backend/rdb'
+require 'perfectsched/backend/simpledb'
+require 'fileutils'
+
+class Test::Unit::TestCase
+  #class << self
+  #  alias_method :it, :test
+  #end
+	def self.it(name, &block)
+		define_method("test_#{name}", &block)
+	end
+end
+

metadata

@@ -0,0 +1,144 @@
+--- !ruby/object:Gem::Specification 
+name: perfectsched
+version: !ruby/object:Gem::Version 
+  hash: 3
+  prerelease: false
+  segments: 
+  - 0
+  - 7
+  - 0
+  version: 0.7.0
+platform: ruby
+authors: 
+- Sadayuki Furuhashi
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-09-03 00:00:00 +09:00
+default_executable: perfectsched
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: cron-spec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        hash: 31
+        segments: 
+        - 0
+        - 1
+        - 2
+        version: 0.1.2
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: sequel
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 111
+        segments: 
+        - 3
+        - 26
+        - 0
+        version: 3.26.0
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: aws-sdk
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 17
+        segments: 
+        - 1
+        - 1
+        - 1
+        version: 1.1.1
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: perfectqueue
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        - 7
+        - 0
+        version: 0.7.0
+  type: :runtime
+  version_requirements: *id004
+description: 
+email: frsyuki@gmail.com
+executables: 
+- perfectsched
+extensions: []
+
+extra_rdoc_files: 
+- ChangeLog
+- README.rdoc
+files: 
+- bin/perfectsched
+- lib/perfectsched.rb
+- lib/perfectsched/backend.rb
+- lib/perfectsched/backend/rdb.rb
+- lib/perfectsched/backend/simpledb.rb
+- lib/perfectsched/command/perfectsched.rb
+- lib/perfectsched/croncalc.rb
+- lib/perfectsched/engine.rb
+- lib/perfectsched/version.rb
+- ChangeLog
+- README.rdoc
+- test/backend_test.rb
+- test/test_helper.rb
+has_rdoc: true
+homepage: https://github.com/treasure-data/perfectsched
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Highly available distributed cron built on RDBMS or SimpleDB
+test_files: 
+- test/backend_test.rb
+- test/test_helper.rb