pallets 0.1.0

data/lib/pallets/backends/scripts/discard.lua

@@ -0,0 +1,3 @@
+-- Remove job from reliability queue
+redis.call("LREM", KEYS[1], 0, ARGV[1])
+redis.call("ZREM", KEYS[2], ARGV[1])

data/lib/pallets/backends/scripts/give_up.lua

@@ -0,0 +1,6 @@
+-- Remove job from reliability queue
+redis.call("LREM", KEYS[2], 0, ARGV[3])
+redis.call("ZREM", KEYS[3], ARGV[3])
+
+-- Add job and its fail time (score) to failed sorted set
+redis.call("ZADD", KEYS[1], ARGV[1], ARGV[2])

data/lib/pallets/backends/scripts/reschedule_all.lua

@@ -0,0 +1,22 @@
+-- Queue reliability queue jobs that are ready to be retried (their score is
+-- below given value) and remove jobs from sorted set and list
+-- TODO: Add limit of items to get
+local count = redis.call("ZCOUNT", KEYS[1], "-inf", ARGV[1])
+if count > 0 then
+  local work = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1])
+  redis.call("LPUSH", KEYS[4], unpack(work))
+  redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", ARGV[1])
+  for _, job in pairs(work) do
+    redis.call("LREM", KEYS[2], 0, job)
+  end
+end
+
+-- Queue jobs that are ready to be retried (their score is below given value) and
+-- remove jobs from sorted set
+-- TODO: Add limit of items to get
+local count = redis.call("ZCOUNT", KEYS[3], "-inf", ARGV[1])
+if count > 0 then
+  local work = redis.call("ZRANGEBYSCORE", KEYS[3], "-inf", ARGV[1])
+  redis.call("LPUSH", KEYS[4], unpack(work))
+  redis.call("ZREMRANGEBYSCORE", KEYS[3], "-inf", ARGV[1])
+end

data/lib/pallets/backends/scripts/retry.lua

@@ -0,0 +1,6 @@
+-- Remove job from reliability queue
+redis.call("LREM", KEYS[2], 0, ARGV[3])
+redis.call("ZREM", KEYS[3], ARGV[3])
+
+-- Add job and its retry time (score) to retry sorted set
+redis.call("ZADD", KEYS[1], ARGV[1], ARGV[2])

data/lib/pallets/backends/scripts/run_workflow.lua

@@ -0,0 +1,11 @@
+-- Add all jobs to sorted set
+redis.call("ZADD", KEYS[1], unpack(ARGV))
+
+-- Queue jobs that are ready to be processed (their score is 0) and
+-- remove queued jobs from the sorted set
+local count = redis.call("ZCOUNT", KEYS[1], 0, 0)
+if count > 0 then
+  local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+  redis.call("LPUSH", KEYS[2], unpack(work))
+  redis.call("ZREM", KEYS[1], unpack(work))
+end

data/lib/pallets/backends/scripts/save.lua

@@ -0,0 +1,18 @@
+-- Remove job from reliability queue
+redis.call("LREM", KEYS[3], 0, ARGV[1])
+redis.call("ZREM", KEYS[4], ARGV[1])
+
+-- Decrement all jobs from the sorted set
+local all_pending = redis.call("ZRANGE", KEYS[1], 0, -1)
+for score, task in pairs(all_pending) do
+  redis.call("ZINCRBY", KEYS[1], -1, task)
+end
+
+-- Queue jobs that are ready to be processed (their score is 0) and
+-- remove queued jobs from sorted set
+local count = redis.call("ZCOUNT", KEYS[1], 0, 0)
+if count > 0 then
+  local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+  redis.call("LPUSH", KEYS[2], unpack(work))
+  redis.call("ZREM", KEYS[1], unpack(work))
+end

data/lib/pallets/cli.rb

@@ -0,0 +1,102 @@
+require 'optparse'
+
+module Pallets
+  class CLI
+    def initialize
+      parse_options
+      setup_signal_handlers
+
+      @manager = Manager.new
+      @signal_queue = Queue.new
+    end
+
+    def run
+      Pallets.logger.info 'Starting the awesomeness of Pallets <3'
+
+      @manager.start
+
+      loop do
+        # This blocks until signals are received
+        handle_signal(@signal_queue.pop)
+      end
+    rescue Interrupt
+      Pallets.logger.info 'Shutting down...'
+      @manager.shutdown
+      Pallets.logger.info 'Buh-bye!'
+      exit
+    end
+
+    private
+
+    def handle_signal(signal)
+      case signal
+      when 'INT'
+        raise Interrupt
+      end
+    end
+
+    def parse_options
+      OptionParser.new do |opts|
+        opts.banner = 'Usage: pallets [options]'
+
+        opts.on('-b', '--backend NAME', 'Backend to use') do |backend|
+          Pallets.configuration.backend = backend
+        end
+
+        opts.on('-c', '--concurrency NUM', Integer, 'Number of workers to start') do |concurrency|
+          Pallets.configuration.concurrency = concurrency
+        end
+
+        opts.on('-f', '--max-failures NUM', Integer, 'Maximum allowed number of failures per task') do |max_failures|
+          Pallets.configuration.max_failures = max_failures
+        end
+
+        opts.on('-n', '--namespace NAME', 'Namespace to use for backend') do |namespace|
+          Pallets.configuration.namespace = namespace
+        end
+
+        opts.on('-p', '--pool-size NUM', Integer, 'Size of backend pool') do |pool_size|
+          Pallets.configuration.pool_size = pool_size
+        end
+
+        opts.on('-q', '--quiet', 'Output less logs') do
+          Pallets.logger.level = Logger::ERROR
+        end
+
+        opts.on('-r', '--require PATH', 'Path containing workflow definitions') do |path|
+          require(path)
+        end
+
+        opts.on('-s', '--serializer NAME', 'Serializer to use') do |serializer|
+          Pallets.configuration.serializer = serializer
+        end
+
+        opts.on('-u', '--blocking-timeout NUM', Integer, 'Seconds to block while waiting for work') do |blocking_timeout|
+          Pallets.configuration.blocking_timeout = blocking_timeout
+        end
+
+        opts.on('-v', '--verbose', 'Output more logs') do
+          Pallets.logger.level = Logger::DEBUG
+        end
+
+        opts.on('--version', 'Version of Pallets') do
+          puts "Pallets v#{Pallets::VERSION}"
+          exit
+        end
+
+        opts.on_tail('-h', '--help', 'Show this message') do
+          puts opts
+          exit
+        end
+      end.parse!
+    end
+
+    def setup_signal_handlers
+      %w(INT).each do |signal|
+        trap signal do
+          @signal_queue.push signal
+        end
+      end
+    end
+  end
+end

data/lib/pallets/configuration.rb

@@ -0,0 +1,44 @@
+module Pallets
+  class Configuration
+    # Backend to use for handling workflows
+    attr_accessor :backend
+
+    # Arguments used to initialize the backend
+    attr_accessor :backend_args
+
+    # Number of seconds to block while waiting for jobs
+    attr_accessor :blocking_timeout
+
+    # Number of workers to process jobs
+    attr_accessor :concurrency
+
+    # Number of seconds allowed for a job to be processed. If a job exceeds this
+    # period, it is considered failed, and scheduled to be processed again
+    attr_accessor :job_timeout
+
+    # Maximum number of failures allowed per job. Can also be configured on a
+    # per task basis
+    attr_accessor :max_failures
+
+    # Namespace used by the backend to store information
+    attr_accessor :namespace
+
+    # Number of connections to the backend
+    attr_accessor :pool_size
+
+    # Serializer used for jobs
+    attr_accessor :serializer
+
+    def initialize
+      @backend = :redis
+      @backend_args = {}
+      @blocking_timeout = 5
+      @concurrency = 2
+      @job_timeout = 1800 # 30 minutes
+      @max_failures = 3
+      @namespace = 'pallets'
+      @pool_size = 5
+      @serializer = :json
+    end
+  end
+end

data/lib/pallets/dsl/workflow.rb

@@ -0,0 +1,32 @@
+require 'active_support'
+
+module Pallets
+  module DSL
+    module Workflow
+      def task(*args, &block)
+        options = args.extract_options!
+        name, depends_on = if args.any?
+          [args.first, options[:depends_on]]
+        else
+          options.first
+        end
+        raise ArgumentError, "A task must have a name" unless name
+
+        # Handle nils, symbols or arrays consistently
+        name = name.to_sym
+        dependencies = Array(depends_on).compact.map(&:to_sym)
+        graph.add(name, dependencies)
+
+        class_name = options[:class_name] || name.to_s.camelize
+        max_failures = options[:max_failures] || Pallets.configuration.max_failures
+
+        task_config[name] = {
+          'class_name' => class_name,
+          'max_failures' => max_failures
+        }
+
+        nil
+      end
+    end
+  end
+end

data/lib/pallets/errors.rb

@@ -0,0 +1,4 @@
+module Pallets
+  class Shutdown < Interrupt
+  end
+end

data/lib/pallets/graph.rb

@@ -0,0 +1,44 @@
+require 'tsort'
+
+module Pallets
+  class Graph
+    include TSort
+
+    def initialize
+      @nodes = {}
+    end
+
+    def add(node, dependencies)
+      @nodes[node] = dependencies
+    end
+
+    def parents(node)
+      @nodes[node]
+    end
+
+    # Returns nodes topologically sorted, together with their order (number of
+    # nodes that have to be executed prior)
+    def sorted_with_order
+      # Identify groups of nodes that can be executed concurrently
+      groups = tsort_each.slice_when { |a, b| parents(a) != parents(b) }
+
+      # Assign order to each node
+      i = 0
+      groups.flat_map do |group|
+        group_with_order = group.product([i])
+        i += group.size
+        group_with_order
+      end
+    end
+
+    private
+
+    def tsort_each_node(&block)
+      @nodes.each_key(&block)
+    end
+
+    def tsort_each_child(node, &block)
+      @nodes.fetch(node).each(&block)
+    end
+  end
+end

data/lib/pallets/manager.rb

@@ -0,0 +1,57 @@
+module Pallets
+  class Manager
+    attr_reader :workers, :timeout
+
+    def initialize(concurrency: Pallets.configuration.concurrency)
+      @workers = concurrency.times.map { Worker.new(self) }
+      @scheduler = Scheduler.new(self)
+      @lock = Mutex.new
+      @needs_to_stop = false
+    end
+
+    def start
+      @workers.each(&:start)
+      @scheduler.start
+    end
+
+    # Attempt to gracefully shutdown every worker. If any is still busy after
+    # the given timeout, hard shutdown it. We don't need to worry about lost
+    # jobs caused by the hard shutdown; there is a reliability list that
+    # contains all active jobs, which will be automatically requeued upon next
+    # start
+    def shutdown
+      @needs_to_stop = true
+
+      @workers.reverse_each(&:graceful_shutdown)
+      @scheduler.shutdown
+
+      Pallets.logger.info 'Waiting for workers to finish their jobs...'
+      # Wait for 10 seconds at most
+      10.times do
+        return if @workers.empty?
+        sleep 1
+      end
+
+      @workers.reverse_each(&:hard_shutdown)
+      # Ensure Pallets::Shutdown got propagated and workers finished; if not,
+      # their threads will be killed anyway when the manager quits
+      sleep 0.5
+    end
+
+    def remove_worker(worker)
+      @lock.synchronize { @workers.delete(worker) }
+    end
+
+    def replace_worker(worker)
+      @lock.synchronize do
+        @workers.delete(worker)
+
+        return if @needs_to_stop
+
+        worker = Worker.new(self)
+        @workers << worker
+        worker.start
+      end
+    end
+  end
+end

data/lib/pallets/pool.rb

@@ -0,0 +1,26 @@
+module Pallets
+  class Pool
+    def initialize(size)
+      raise ArgumentError, 'Pool needs a block to initialize' unless block_given?
+
+      @queue = Queue.new
+      @size = size
+      size.times { @queue << yield }
+    end
+
+    def size
+      @queue.size
+    end
+
+    def execute
+      raise ArgumentError, 'Pool needs a block to execute' unless block_given?
+
+      begin
+        item = @queue.pop
+        yield item
+      ensure
+        @queue << item
+      end
+    end
+  end
+end

data/lib/pallets/scheduler.rb

@@ -0,0 +1,53 @@
+module Pallets
+  class Scheduler
+    def initialize(manager)
+      @manager = manager
+      @needs_to_stop = false
+      @thread = nil
+    end
+
+    def start
+      @thread ||= Thread.new { work }
+    end
+
+    def shutdown
+      @needs_to_stop = true
+
+      return unless @thread
+      @thread.join
+    end
+
+    def needs_to_stop?
+      @needs_to_stop
+    end
+
+    def id
+      "S#{@thread.object_id.to_s(36)}".upcase if @thread
+    end
+
+    private
+
+    def work
+      loop do
+        break if needs_to_stop?
+
+        backend.reschedule_all(Time.now.to_f)
+        wait_a_bit
+      end
+    end
+
+    def wait_a_bit
+      # Wait for roughly 10 seconds
+      # We don't want to block the entire polling interval, since we want to
+      # deal with shutdowns synchronously and as fast as possible
+      10.times do
+        break if needs_to_stop?
+        sleep 1
+      end
+    end
+
+    def backend
+      @backend ||= Pallets.backend
+    end
+  end
+end