perfectsched 0.7.19 → 0.8.0

data/lib/perfectsched/signal_queue.rb

@@ -0,0 +1,25 @@
+#
+# PerfectSched
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectSched
+
+  require 'perfectqueue/signal_queue'
+  SignalQueue = PerfectQueue::SignalQueue
+
+end
+

data/lib/perfectsched/task.rb

@@ -0,0 +1,46 @@
+#
+# PerfectSched
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectSched
+
+  class Task < ScheduleWithMetadata
+    include Model
+
+    def initialize(client, key, attributes, scheduled_time, task_token)
+      super(client, key, attributes)
+      @scheduled_time = scheduled_time
+      @task_token = task_token
+    end
+
+    attr_reader :scheduled_time
+
+    def release!(options={})
+      @client.release(@task_token, options)
+    end
+
+    def retry!(options={})
+      @client.retry(@task_token, options)
+    end
+
+    def finish!(options={})
+      @client.finish(@task_token, options)
+    end
+  end
+
+end
+

data/lib/perfectsched/version.rb

@@ -1,5 +1,3 @@
 module PerfectSched
-
-VERSION = '0.7.19'
-
+  VERSION = "0.8.0"
 end

data/lib/perfectsched/worker.rb

@@ -0,0 +1,163 @@
+#
+# PerfectSched
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectSched
+
+  class Worker
+    def self.run(runner, config=nil, &block)
+      new(runner, config, &block).run
+    end
+
+    def initialize(runner, config=nil, &block)
+      # initial logger
+      STDERR.sync = true
+      @log = DaemonsLogger.new(STDERR)
+
+      @runner = runner
+      block = Proc.new { config } if config
+      @config_load_proc = block
+      @finished = false
+    end
+
+    def run
+      @sig = install_signal_handlers
+      begin
+        @engine = Engine.new(@runner, load_config)
+        begin
+          until @finished
+            @engine.run
+          end
+        ensure
+          @engine.shutdown
+        end
+      ensure
+        @sig.shutdown
+      end
+      return nil
+    rescue
+      @log.error "#{$!.class}: #{$!}"
+      $!.backtrace.each {|x| @log.error "  #{x}" }
+      return nil
+    end
+
+    def stop
+      @log.info "stop"
+      begin
+        @finished = true
+        @engine.stop if @engine
+      rescue
+        @log.error "failed to stop: #{$!}"
+        $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
+        return false
+      end
+      return true
+    end
+
+    def restart
+      @log.info "Received restart"
+      begin
+        engine = Engine.new(@runner, load_config)
+        current = @engine
+        @engine = engine
+        current.shutdown
+      rescue
+        @log.error "failed to restart: #{$!}"
+        $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
+        return false
+      end
+      return true
+    end
+
+    def replace(command=[$0]+ARGV)
+      @log.info "replace"
+      begin
+        return if @replaced_pid
+        stop
+        @replaced_pid = Process.fork do
+          exec(*command)
+          exit!(127)
+        end
+      rescue
+        @log.error "failed to replace: #{$!}"
+        $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
+        return false
+      end
+      self
+    end
+
+    def logrotated
+      @log.info "reopen a log file"
+      begin
+        @log.reopen!
+      rescue
+        @log.error "failed to restart: #{$!}"
+        $!.backtrace.each {|bt| @log.warn "\t#{bt}" }
+        return false
+      end
+      return true
+    end
+
+    private
+    def load_config
+      raw_config = @config_load_proc.call
+      config = {}
+      raw_config.each_pair {|k,v| config[k.to_sym] = v }
+
+      log = DaemonsLogger.new(config[:log] || STDERR)
+      if old_log = @log
+        old_log.close
+      end
+      @log = log
+
+      config[:logger] = log
+
+      return config
+    end
+
+    def install_signal_handlers(&block)
+      SignalQueue.start do |sig|
+        sig.trap :TERM do
+          stop
+        end
+        sig.trap :INT do
+          stop
+        end
+
+        sig.trap :QUIT do
+          stop
+        end
+
+        sig.trap :USR1 do
+          restart
+        end
+
+        sig.trap :HUP do
+          restart
+        end
+
+        sig.trap :USR2 do
+          logrotated
+        end
+
+        trap :CHLD, "SIG_IGN"
+      end
+    end
+  end
+
+end
+

data/lib/perfectsched.rb

@@ -1,4 +1,98 @@
-require 'perfectsched/engine'
-require 'perfectsched/croncalc'
-require 'perfectsched/backend'
-require 'perfectsched/version'
+#
+# PerfectSched
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectSched
+  require 'json'
+  require 'thread'  # Mutex, CoditionVariable
+
+  {
+    :Application => 'perfectsched/application',
+    :Backend => 'perfectsched/backend',
+    :BackendHelper => 'perfectsched/backend',
+    :BlockingFlag => 'perfectsched/blocking_flag',
+    :Client => 'perfectsched/client',
+    :DaemonsLogger => 'perfectsched/daemons_logger',
+    :Engine => 'perfectsched/engine',
+    :Model => 'perfectsched/model',
+    :Runner => 'perfectsched/runner',
+    :ScheduleCollection => 'perfectsched/schedule_collection',
+    :Schedule => 'perfectsched/schedule',
+    :ScheduleWithMetadata => 'perfectsched/schedule',
+    :ScheduleMetadata => 'perfectsched/schedule_metadata',
+    :ScheduleMetadataAccessors => 'perfectsched/schedule_metadata',
+    :SignalQueue => 'perfectsched/signal_queue',
+    :Task => 'perfectsched/task',
+    :Worker => 'perfectsched/worker',
+  }.each_pair {|k,v|
+    autoload k, File.expand_path(v, File.dirname(__FILE__))
+  }
+  [
+    'perfectsched/version',
+    'perfectsched/error',
+  ].each {|v|
+    require File.expand_path(v, File.dirname(__FILE__))
+  }
+
+  require 'cron-spec'
+  require 'tzinfo'
+
+  def self.open(config, &block)
+    c = Client.new(config)
+    begin
+      q = ScheduleCollection.new(c)
+      if block
+        block.call(q)
+      else
+        c = nil
+        return q
+      end
+    ensure
+      c.close if c
+    end
+  end
+
+  def self.cron_time(cron, timestamp, timezone)
+    begin
+      tab = CronSpec::CronSpecification.new(cron)
+    rescue
+      raise ArgumentError, "invalid cron format: #{$!}: #{cron.inspect}"
+    end
+
+    begin
+      tz = TZInfo::Timezone.get(timezone) if timezone
+    rescue
+      raise ArgumentError, "unknown timezone: #{$!}: #{timezone.inspect}"
+    end
+
+    ts = (timestamp + 59) / 60 * 60
+    while true
+      t = Time.at(ts).utc
+      t = tz.utc_to_local(t) if tz
+      if tab.is_specification_in_effect?(t)
+        return ts
+      end
+      ts += 60
+      # FIXME break
+    end
+  end
+
+  def self.next_time(cron, before_timestamp, timezone)
+    cron_time(cron, before_timestamp+1, timezone)
+  end
+end
+

data/perfectsched.gemspec

@@ -0,0 +1,27 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+require 'perfectsched/version'
+
+Gem::Specification.new do |gem|
+  gem.name        = "perfectsched"
+  gem.description = "Highly available distributed cron built on RDBMS"
+  gem.homepage    = "https://github.com/treasure-data/perfectsched"
+  gem.summary     = gem.description
+  gem.version     = PerfectSched::VERSION
+  gem.authors     = ["Sadayuki Furuhashi"]
+  gem.email       = "frsyuki@gmail.com"
+  gem.has_rdoc    = false
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "cron-spec", [">= 0.1.2", "<= 0.1.2"]
+  gem.add_dependency "sequel", "~> 3.26.0"
+  gem.add_dependency "tzinfo", "~> 0.3.29"
+  gem.add_dependency "perfectqueue", "~> 0.8.0"
+  gem.add_development_dependency "rake", "~> 0.9.2"
+  gem.add_development_dependency "rspec", "~> 2.10.0"
+  gem.add_development_dependency "simplecov", "~> 0.5.4"
+  gem.add_development_dependency "sqlite3", "~> 1.3.3"
+end

data/spec/schedule_collection_spec.rb

@@ -0,0 +1,172 @@
+
+describe ScheduleCollection do
+  before do
+    @sc = create_test_sched
+  end
+
+  after do
+    @sc.client.close
+  end
+
+  it 'is a ScheduleCollection' do
+    @sc.class.should == PerfectSched::ScheduleCollection
+  end
+
+  it 'succeess add' do
+    @sc.add('sched01', 't01', {:cron=>'* * * * *', :timezone=>'UTC'})
+  end
+
+  it 'fail duplicated add' do
+    @sc.add('sched01', 't01', {:cron=>'* * * * *', :timezone=>'UTC'})
+
+    lambda {
+      @sc.add('sched01', 't01', {:cron=>'* * * * *', :timezone=>'UTC'})
+    }.should raise_error AlreadyExistsError
+
+    @sc['sched01'].delete!
+
+    @sc.add('sched01', 't01', {:cron=>'* * * * *', :timezone=>'UTC'})
+  end
+
+  it 'acquire' do
+    time = 1323820800  # 2011-12-14 00:00:00 UTC
+
+    s01 = @sc.add('sched01', 't01', :cron=>"* * * * *", :data=>{'k'=>1}, :next_time=>time)
+
+    t01 = @sc.poll(:alive_time=>120, :now=>time)
+    t01.key.should == 'sched01'
+    t01.type.should == 't01'
+    t01.cron.should == "* * * * *"
+    t01.delay.should == 0
+    t01.data.should == {'k'=>1}
+    t01.scheduled_time.should == Time.at(time).utc
+
+    t02 = @sc.poll(:alive_time=>120, :now=>time+60)
+    t02.should == nil
+
+    t01.finish!
+
+    t04 = @sc.poll(:alive_time=>120, :now=>time+60)
+    t04.key.should == 'sched01'
+    t01.type.should == 't01'
+    t04.cron.should == "* * * * *"
+    t04.delay.should == 0
+    t04.data.should == {'k'=>1}
+    t04.scheduled_time.should == Time.at(time+60).utc
+  end
+
+  it 'timezone' do
+    time = 1323820800  # 2011-12-14 00:00:00 UTC
+
+    s01 = @sc.add('sched01', 't01', :cron=>"0 0 * * *", :next_time=>time-60, :timezone=>'UTC')
+    #s01.class.should == Schedule
+    #s01.key.should == 'sched01'
+
+    s02 = @sc.add('sched02', 't01', :cron=>"0 0 * * *", :next_time=>time-60, :timezone=>'Asia/Tokyo')
+    #s02.class.should == Schedule
+    #s02.key.should == 'sched02'
+
+    t01 = @sc.poll(:alive_time=>86400, :now=>time)
+    t01.class.should == Task
+    t01.key.should == 'sched01'
+    t01.type.should == 't01'
+    t01.scheduled_time.should == Time.at(time).utc
+
+    t02 = @sc.poll(:alive_time=>86400, :now=>time+54000)
+    t02.class.should == Task
+    t02.key.should == 'sched02'
+    t02.type.should == 't01'
+    t02.scheduled_time.should == Time.at(time+54000).utc
+  end
+
+  it 'delay' do
+    time = 1323820800  # 2011-12-14 00:00:00 UTC
+
+    s01 = @sc.add('sched01', 't01', :cron=>"0 * * * *", :delay=>30, :next_time=>time, :timezone=>'UTC')
+
+    t01 = @sc.poll(:alive_time=>86400, :now=>time)
+    t01.should == nil
+
+    t02 = @sc.poll(:alive_time=>86400, :now=>time+30)
+    t02.class.should == Task
+    t02.key.should == 'sched01'
+    t02.type.should == 't01'
+    t02.scheduled_time.should == Time.at(time).utc
+    t02.delay.should == 30
+
+    t02.finish!
+
+    t03 = @sc.poll(:alive_time=>86400, :now=>time+3600)
+    t03.should == nil
+
+    t04 = @sc.poll(:alive_time=>86400, :now=>time+3630)
+    t04.class.should == Task
+    t04.key.should == 'sched01'
+    t04.type.should == 't01'
+    t04.scheduled_time.should == Time.at(time+3600).utc
+    t04.delay.should == 30
+  end
+
+  it 'invalid cron format' do
+    lambda {
+      @sc.add('sched01', 't01', :cron=>'???')
+    }.should raise_error ArgumentError
+
+    lambda {
+      @sc.add('sched01', 't01', :cron=>'* * * * * *')
+    }.should raise_error ArgumentError
+  end
+
+  it 'fail duplicated add' do
+    @sc.add('sched01', 't01', :cron=>"0 * * * *")
+    lambda {
+      @sc.add('sched01', 't01', :cron=>"0 * * * *")
+    }.should raise_error AlreadyExistsError
+
+    @sc['sched01'].delete!
+
+    @sc.add('sched01', 't01', :cron=>"0 * * * *")
+  end
+
+  it 'list' do
+    time = 1323820800  # 2011-12-14 00:00:00 UTC
+
+    @sc.add('sched01', 't01', :cron=>"0 * * * *", :next_time=>time, :delay=>1)
+    @sc.add('sched02', 't02', :cron=>"0 * * * *", :next_time=>time, :delay=>2)
+    @sc.add('sched03', 't03', :cron=>"0 * * * *", :next_time=>time, :delay=>3, :next_run_time=>time+3600)
+
+    a = []
+    @sc.list {|s|
+      a << s
+    }
+    a.sort_by! {|s| s.key }
+
+    s01 = a.shift
+    s01.class.should == ScheduleWithMetadata
+    s01.key.should == 'sched01'
+    s01.type.should == 't01'
+    s01.cron.should == '0 * * * *'
+    s01.delay.should == 1
+    s01.next_time.should == Time.at(time).utc
+    s01.next_run_time.should == Time.at(time+1).utc
+
+    s02 = a.shift
+    s02.class.should == ScheduleWithMetadata
+    s02.key.should == 'sched02'
+    s02.type.should == 't02'
+    s02.cron.should == '0 * * * *'
+    s02.delay.should == 2
+    s02.next_time.should == Time.at(time).utc
+    s02.next_run_time.should == Time.at(time+2).utc
+
+    s03 = a.shift
+    s03.class.should == ScheduleWithMetadata
+    s03.key.should == 'sched03'
+    s03.type.should == 't03'
+    s03.cron.should == '0 * * * *'
+    s03.delay.should == 3
+    s03.next_time.should == Time.at(time).utc
+    s03.next_run_time.should == Time.at(time+3600).utc
+  end
+end
+