perfectqueue 0.7.32 → 0.8.0

data/lib/perfectqueue/application/dispatch.rb

@@ -0,0 +1,73 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+  module Application
+
+    class Dispatch < Runner
+      # Runner interface
+      def initialize(task)
+        base = self.class.router.route(task.type)
+        unless base
+          task.retry!
+          raise "Unknown task type #{task.type.inspect}"   # TODO error class
+        end
+        @runner = base.new(task)
+        super
+      end
+
+      attr_reader :runner
+
+      def run
+        @runner.run
+      end
+
+      def kill(reason)
+        @runner.kill(reason)
+      end
+
+      # DSL interface
+      class << self
+        def route(options)
+          patterns = options.keys.select {|k| !k.is_a?(Symbol) }
+          klasses = patterns.map {|k| options.delete(k) }
+          patterns.zip(klasses).each {|pattern,sym|
+            add_route(pattern, sym, options)
+          }
+          nil
+        end
+
+        def add_route(pattern, klass, options)
+          router.add(pattern, klass, options)
+        end
+
+        def router=(router)
+          (class<<self;self;end).instance_eval do
+            self.__send__(:define_method, :router) { router }
+          end
+          router
+        end
+
+        def router
+          self.router = Router.new
+        end
+      end
+    end
+  end
+end
+

data/lib/perfectqueue/application/router.rb

@@ -0,0 +1,69 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+  module Application
+
+    class Router
+      def initialize
+        @patterns = []
+        @cache = {}
+      end
+
+      def add(pattern, sym, options={})
+        case pattern
+        when Regexp
+          # ok
+        when String, Symbol
+          pattern = /#{Regexp.escape(pattern)}/
+        else
+          raise ArguementError, "pattern should be String or Regexp but got #{pattern.class}: #{pattern.inspect}"
+        end
+
+        @patterns << [pattern, sym]
+      end
+
+      def route(type)
+        if @cache.has_key?(type)
+          return @cache[type]
+        end
+
+        @patterns.each {|(pattern,sym)|
+          if pattern.match(type)
+            base = resolve_application_base(sym)
+            return @cache[type] = base
+          end
+        }
+        return @cache[type] = nil
+      end
+      attr_reader :patterns
+
+      private
+      def resolve_application_base(sym)
+        case sym
+        when Symbol
+          self.class.const_get(sym)
+        else
+          sym
+        end
+      end
+    end
+
+  end
+end
+

data/lib/perfectqueue/backend.rb

@@ -1,52 +1,49 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
 
 module PerfectQueue
-
-
-class Task
-  def initialize(id, created_at, data, resource=nil)
-    @id = id
-    @created_at = created_at
-    @data = data
-    @resource = resource
+  module Backend
+    def self.new_backend(client, config)
+      case config[:type]
+      when nil
+        raise ConfigError, "'type' option is not set"
+      when 'rdb_compat'
+        require_backend('rdb_compat')
+        RDBCompatBackend.new(client, config)
+      end
+    end
+
+    def self.require_backend(fname)
+      require File.expand_path("backend/#{fname}", File.dirname(__FILE__))
+    end
+  end
+
+  module BackendHelper
+    def initialize(client, config)
+      @client = client
+      @config = config
+    end
+
+    attr_reader :client
+
+    def close
+      # do nothing by default
+    end
   end
-
-  attr_reader :id, :created_at, :data, :resource
-end
-
-
-class CanceledError < RuntimeError
-end
-
-
-class Backend
-  # => list {|id,created_at,data,timeout| ... }
-  def list(&block)
-  end
-
-  # => token, task
-  def acquire(timeout, now=Time.now.to_i)
-  end
-
-  # => true (success) or false (canceled)
-  def finish(token, delete_timeout=3600, now=Time.now.to_i)
-  end
-
-  # => nil
-  def update(token, timeout)
-  end
-
-  # => true (success) or false (not found, canceled or finished)
-  def cancel(id, delete_timeout=3600, now=Time.now.to_i)
-  end
-
-  # => true (success) or nil (already exists)
-  def submit(id, data, time=Time.now.to_i, resource=nil)
-  end
-
-  def close
-  end
-end
-
-
 end
 

data/lib/perfectqueue/backend/rdb_compat.rb

@@ -0,0 +1,298 @@
+#
+# PerfectQueue
+#
+# Copyright (C) 2012 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module PerfectQueue
+  module Backend
+    class RDBCompatBackend
+      include BackendHelper
+
+      class Token < Struct.new(:key)
+      end
+
+      def initialize(client, config)
+        super
+
+        require 'sequel'
+        url = config[:url]
+        @table = config[:table]
+        unless @table
+          raise ConfigError, ":table option is required"
+        end
+
+        #password = config[:password]
+        #user = config[:user]
+        @db = Sequel.connect(url, :max_connections=>1)
+        @mutex = Mutex.new
+
+        connect {
+          # connection test
+        }
+
+        @sql = <<SQL
+SELECT id, timeout, data, created_at, resource
+FROM `#{@table}`
+LEFT JOIN (
+  SELECT resource AS res, COUNT(1) AS running
+  FROM `#{@table}` AS T
+  WHERE timeout > ? AND created_at IS NOT NULL AND resource IS NOT NULL
+  GROUP BY resource
+) AS R ON resource = res
+WHERE timeout <= ? AND (running IS NULL OR running < #{MAX_RESOURCE})
+ORDER BY timeout ASC LIMIT #{MAX_SELECT_ROW}
+SQL
+
+        # sqlite doesn't support SELECT ... FOR UPDATE but
+        # sqlite doesn't need it because the db is not shared
+        unless url.split('//',2)[0].to_s.include?('sqlite')
+          @sql << 'FOR UPDATE'
+        end
+      end
+
+      attr_reader :db
+
+      MAX_SELECT_ROW = 8
+      MAX_RESOURCE = (ENV['PQ_MAX_RESOURCE'] || 4).to_i
+      #KEEPALIVE = 10
+      MAX_RETRY = 10
+
+      def init_database(options)
+        sql = %[
+            CREATE TABLE IF NOT EXISTS `#{@table}` (
+              id VARCHAR(256) NOT NULL,
+              timeout INT NOT NULL,
+              data BLOB NOT NULL,
+              created_at INT,
+              resource VARCHAR(256),
+              PRIMARY KEY (id)
+            );]
+        connect {
+          @db.run sql
+        }
+      end
+
+      # => TaskStatus
+      def get_task_metadata(key, options)
+        now = (options[:now] || Time.now).to_i
+
+        connect {
+          row = @db.fetch("SELECT timeout, data, created_at, resource FROM `#{@table}` LIMIT 1").first
+          unless row
+            raise NotFoundError, "task key=#{key} does no exist"
+          end
+          attributes = create_attributes(now, row)
+          return TaskMetadata.new(@client, key, attributes)
+        }
+      end
+
+      # => AcquiredTask
+      def preempt(key, alive_time, options)
+        raise NotSupportedError.new("preempt is not supported by rdb_compat backend")
+      end
+
+      # yield [TaskWithMetadata]
+      def list(options, &block)
+        now = (options[:now] || Time.now).to_i
+
+        connect {
+          #@db.fetch("SELECT id, timeout, data, created_at, resource FROM `#{@table}` WHERE !(created_at IS NULL AND timeout <= ?) ORDER BY timeout ASC;", now) {|row|
+          @db.fetch("SELECT id, timeout, data, created_at, resource FROM `#{@table}` ORDER BY timeout ASC;", now) {|row|
+            attributes = create_attributes(now, row)
+            task = TaskWithMetadata.new(@client, row[:id], attributes)
+            yield task
+          }
+        }
+      end
+
+      # => Task
+      def submit(key, type, data, options)
+        now = (options[:now] || Time.now).to_i
+        run_at = (options[:run_at] || now).to_i
+        user = options[:user]
+        priority = options[:priority]  # not supported
+        data ||= {}
+        data['type'] = type
+
+        connect {
+          begin
+            n = @db["INSERT INTO `#{@table}` (id, timeout, data, created_at, resource) VALUES (?, ?, ?, ?, ?);", key, run_at, data.to_json, now, user].insert
+            return Task.new(@client, key)
+          rescue Sequel::DatabaseError
+            raise AlreadyExistsError, "task key=#{key} already exists"
+          end
+        }
+      end
+
+      # => [AcquiredTask]
+      def acquire(alive_time, max_acquire, options)
+        now = (options[:now] || Time.now).to_i
+        next_timeout = now + alive_time
+
+        connect {
+          while true
+            rows = 0
+            @db.transaction do
+              @db.fetch(@sql, now, now) {|row|
+                unless row[:created_at]
+                  # finished task
+                  @db["DELETE FROM `#{@table}` WHERE id=?;", row[:id]].delete
+
+                else
+                  ## optimistic lock is not needed because the row is locked for update
+                  #n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND timeout=?", timeout, row[:id], row[:timeout]].update
+                  n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=?", next_timeout, row[:id]].update
+                  if n > 0
+                    attributes = create_attributes(nil, row)
+                    task_token = Token.new(row[:id])
+                    task = AcquiredTask.new(@client, row[:id], attributes, task_token)
+                    return [task]
+                  end
+                end
+
+                rows += 1
+              }
+            end
+            break nil if rows < MAX_SELECT_ROW
+          end
+        }
+      end
+
+      # => nil
+      def cancel_request(key, options)
+        now = (options[:now] || Time.now).to_i
+
+        # created_at=-1 means cancel_requested
+        connect {
+          n = @db["UPDATE `#{@table}` SET created_at=-1 WHERE id=? AND created_at IS NOT NULL;", key].update
+          if n <= 0
+            raise AlreadyFinishedError, "task key=#{key} does not exist or already finished."
+          end
+        }
+        nil
+      end
+
+      def force_finish(key, retention_time, options)
+        finish(Token.new(key), retention_time, options)
+      end
+
+      # => nil
+      def finish(task_token, retention_time, options)
+        now = (options[:now] || Time.now).to_i
+        delete_timeout = now + retention_time
+        key = task_token.key
+
+        connect {
+          n = @db["UPDATE `#{@table}` SET timeout=?, created_at=NULL, resource=NULL WHERE id=? AND created_at IS NOT NULL;", delete_timeout, key].update
+          if n <= 0
+            raise AlreadyFinishedError, "task key=#{key} does not exist or already finished."
+          end
+        }
+        nil
+      end
+
+      # => nil
+      def heartbeat(task_token, alive_time, options)
+        now = (options[:now] || Time.now).to_i
+        next_timeout = now + alive_time
+        key = task_token.key
+
+        connect {
+          n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND created_at IS NOT NULL;", next_timeout, key].update
+          if n <= 0
+            row = @db.fetch("SELECT id, created_at FROM `#{@table}` WHERE id=? LIMIT 1", key).first
+            if row == nil
+              raise AlreadyFinishedError, "task key=#{key} already finished."
+            elsif row[:created_at] == -1
+              raise CancelRequestedError, "task key=#{key} is cancel requested."
+            else
+              raise AlreadyFinishedError, "task key=#{key} already finished."
+            end
+          end
+        }
+        nil
+      end
+
+      protected
+      def connect(&block)
+        #now = Time.now.to_i
+        @mutex.synchronize do
+          #if now - @last_time > KEEPALIVE
+          #  @db.disconnect
+          #end
+          #@last_time = now
+          retry_count = 0
+          begin
+            block.call
+          rescue
+            # workaround for "Mysql2::Error: Deadlock found when trying to get lock; try restarting transaction" error
+            if $!.to_s.include?('try restarting transaction')
+              err = ([$!] + $!.backtrace.map {|bt| "  #{bt}" }).join("\n")
+              retry_count += 1
+              if retry_count < MAX_RETRY
+                STDERR.puts err + "\n  retrying."
+                sleep 0.5
+                retry
+              else
+                STDERR.puts err + "\n  abort."
+              end
+            end
+            raise
+          ensure
+            @db.disconnect
+          end
+        end
+      end
+
+      def create_attributes(now, row)
+        if row[:created_at] === nil
+          created_at = nil  # unknown creation time
+          status = TaskStatus::FINISHED
+        elsif row[:created_at] == -1
+          created_at = 0
+          status = TaskStatus::CANCEL_REQUESTED
+        elsif now && row[:timeout] < now
+          created_at = row[:created_at]
+          status = TaskStatus::WAITING
+        else
+          created_at = row[:created_at]
+          status = TaskStatus::RUNNING
+        end
+
+        begin
+          data = JSON.parse(row[:data] || '{}')
+        rescue
+          data = {}
+        end
+
+        type = data['type'] || ''
+
+        attributes = {
+          :status => status,
+          :created_at => created_at,
+          :data => data,
+          :type => type,
+          :user => row[:resource],
+          :timeout => row[:timeout],
+          :message => nil,  # not supported
+          :node => nil,  # not supported
+        }
+      end
+
+    end
+  end
+end
+