plines 0.5.0

data/lib/plines/job_batch_list.rb

@@ -0,0 +1,57 @@
+require 'plines/redis_objects'
+
+module Plines
+  # Represents a list of job batches that are grouped by
+  # some common trait (such as a user id).
+  class JobBatchList < Struct.new(:pipeline, :key)
+    include Plines::RedisObjectsHelpers
+
+    counter :last_batch_num
+    attr_reader :qless, :redis
+
+    def initialize(pipeline, key)
+      super(pipeline, key)
+      @qless = pipeline.configuration.qless_client_for(key)
+      @redis = @qless.redis
+    end
+
+    def most_recent_batch
+      batch_num = last_batch_num.value
+      return nil if batch_num.zero?
+      JobBatch.find(qless, pipeline, batch_id_for(batch_num))
+    end
+
+    def create_new_batch(batch_data, options = {}, &blk)
+      JobBatch.create(qless, pipeline,
+                      batch_id_for(last_batch_num.increment),
+                      batch_data, options, &blk)
+    end
+
+    def each(&block)
+      return enum_for(:each) unless block_given?
+
+      1.upto(last_batch_num.value) do |num|
+        begin
+          yield JobBatch.find(qless, pipeline, batch_id_for(num))
+        rescue JobBatch::CannotFindExistingJobBatchError
+          # We can't yield a batch we can't find!
+        end
+      end
+    end
+
+    def all_with_external_dependency_timeout(dep_name)
+      each.select do |batch|
+        batch.timed_out_external_deps.include?(dep_name)
+      end
+    end
+
+  private
+
+    alias id key
+
+    def batch_id_for(number)
+      [id, number].join(':')
+    end
+  end
+end
+

data/lib/plines/job_enqueuer.rb

@@ -0,0 +1,83 @@
+require 'set'
+
+module Plines
+  # Responsible for enqueing Qless jobs based on the given dependency graph.
+  class JobEnqueuer
+    def initialize(dependency_graph, job_batch, &block)
+      @dependency_graph = dependency_graph
+      @job_batch = job_batch
+      @qless_job_options_block = block
+      @timeout_reduction = job_batch.timeout_reduction
+    end
+
+    def enqueue_jobs
+      @dependency_graph.ordered_steps.each do |step|
+        jid = Qless.generate_jid
+        jids[step] = jid
+        @job_batch.add_job(jid, *step.external_dependencies.map(&:name))
+
+        enqueue_job_for(step, jid, dependency_jids_for(step))
+        setup_external_dep_timeouts_for(step)
+      end
+
+      enqueue_external_dependency_timeouts
+
+      self
+    end
+
+  private
+
+    def jids
+      @jids ||= {}
+    end
+
+    def qless
+      @job_batch.qless
+    end
+
+    def enqueue_job_for(step, jid, depends_on)
+      step.klass.enqueue_qless_job qless,
+        step.data.merge('_job_batch_id' => @job_batch.id),
+        @qless_job_options_block[step].merge(depends: depends_on, jid: jid)
+    end
+
+    def setup_external_dep_timeouts_for(step)
+      step.external_dependencies.each do |dependency|
+        next unless timeout = dependency.options[:wait_up_to]
+
+        timeout_key = TimeoutKey.new(dependency.name,
+                                     timeout - @timeout_reduction)
+        external_dep_timeouts[timeout_key] << step
+      end
+    end
+
+    def dependency_jids_for(step)
+      step.dependencies.map { |d| jids[d] }
+    end
+
+    def external_dep_timeouts
+      @external_dep_timeouts ||= Hash.new do |h, k|
+        h[k] = Set.new
+      end
+    end
+
+    TIMEOUT_JOB_PRIORITY = 999999 # an arbitrary high priority
+
+    def enqueue_external_dependency_timeouts
+      external_dep_timeouts.each do |tk, jobs|
+        job_ids = jobs.map { |k| jids[k] }
+        job_data = ExternalDependencyTimeout.job_data_for \
+          @job_batch, tk.dep_name, job_ids
+
+        jid = qless.queues[jobs.first.klass.processing_queue].put \
+          ExternalDependencyTimeout, job_data,
+          delay: tk.timeout, priority: TIMEOUT_JOB_PRIORITY
+
+        @job_batch.track_timeout_job(tk.dep_name, jid)
+      end
+    end
+
+    TimeoutKey = Struct.new(:dep_name, :timeout)
+  end
+end
+

data/lib/plines/pipeline.rb

@@ -0,0 +1,97 @@
+require 'qless'
+require 'forwardable'
+require 'plines/indifferent_hash'
+
+module Plines
+  # This module should be extended onto a class or module in order
+  # to make it a pipeline. Steps declared within that module will
+  # automatically belong to that pipeline. This enables one application
+  # to have multiple pipelines.
+  module Pipeline
+    extend Forwardable
+    def_delegators :configuration
+
+    DEFAULT_QUEUE = "plines"
+    AWAITING_EXTERNAL_DEPENDENCY_QUEUE = "awaiting_ext_dep"
+
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield configuration
+    end
+
+    def enqueue_jobs_for(batch_data, options = {})
+      batch_data = IndifferentHash.from(batch_data)
+      graph = DependencyGraph.new(self, batch_data)
+      job_batch_list = job_batch_list_for(batch_data)
+
+      job_batch_list.create_new_batch(batch_data, options) do |job_batch|
+        job_options_block = configuration.qless_job_options_block
+        JobEnqueuer.new(graph, job_batch, &job_options_block).enqueue_jobs
+      end
+    end
+
+    def most_recent_job_batch_for(batch_data)
+      job_batch_list_for(batch_data).most_recent_batch
+    end
+
+    def find_job_batch(id)
+      key = id[/\A(.*):\d+\z/, 1] # http://rubular.com/r/fMGv1TaZZA
+      qless = configuration.qless_client_for(key)
+      Plines::JobBatch.find(qless, self, id)
+    end
+
+    def step_classes
+      @step_classes ||= []
+    end
+
+    # Null Object pattern implementation of a step class
+    class NullStep
+      def self.jobs_for(*args)
+        []
+      end
+    end
+
+    # Error raised when two steps are declared as the same boundary step.
+    # Having more than one initial or terminal step is not well defined.
+    class BoundaryStepAlreadySetError < StandardError; end
+
+    def self.define_boundary_step(name)
+      define_method "#{name}=" do |value|
+        current_value = public_send(name)
+        if current_value == NullStep
+          instance_variable_set(:"@#{name}", value)
+        else
+          raise BoundaryStepAlreadySetError,
+            "The #{name} for pipeline #{self} is already set. " +
+            "Multiple of these boundary steps are not supported."
+        end
+      end
+
+      define_method name do
+        current_value = instance_variable_get(:"@#{name}")
+        return current_value if current_value
+        instance_variable_set(:"@#{name}", NullStep)
+      end
+    end
+    private_class_method :define_boundary_step
+
+    define_boundary_step :initial_step
+    define_boundary_step :terminal_step
+
+    def job_batch_list_for(batch_data)
+      key = configuration.batch_list_key_for(batch_data)
+      JobBatchList.new(self, key)
+    end
+
+    def matching_older_unfinished_job_batches(main_job_batch)
+      job_batch_list = job_batch_list_for(main_job_batch.data)
+      job_batch_list.each.select do |job_batch|
+        !job_batch.complete? && job_batch.created_at < main_job_batch.created_at
+      end
+    end
+  end
+end
+

data/lib/plines/redis_objects.rb

@@ -0,0 +1,108 @@
+require 'redis'
+require 'redis/value'
+require 'redis/lock'
+require 'redis/set'
+require 'redis/list'
+require 'redis/hash_key'
+require 'redis/counter'
+
+module Plines
+  module RedisObjectsHelpers
+    def new_redis_object(klass, key, args)
+      klass.new([key_prefix, key].join(':'), redis, *args)
+    end
+
+    def key_prefix
+      @key_prefix ||= [
+        self.class.redis_key_prefix,
+        pipeline.name,
+        self.class.name.split('::').last,
+        id
+      ].join(':')
+    end
+
+    def declared_redis_object_keys
+      self.class.declared_redis_object_names.map { |n| send(n).key }
+    end
+
+    def self.included(klass)
+      klass.extend ClassMethods
+    end
+
+    module ClassMethods
+      def value(name, *args)
+        declared_redis_object_names << name
+
+        class_eval <<-EOS, __FILE__, __LINE__ + 1
+          def #{name}
+            @#{name} ||= new_redis_object(
+              ::Redis::Value, #{name.inspect}, #{args})
+          end
+        EOS
+      end
+
+      def lock(name, *args)
+        declared_redis_object_names << name
+
+        class_eval <<-EOS, __FILE__, __LINE__ + 1
+          def #{name}
+            @#{name} ||= new_redis_object(
+              ::Redis::Lock, #{name.inspect}, #{args})
+          end
+        EOS
+      end
+
+      def set(name, *args)
+        declared_redis_object_names << name
+        class_eval <<-EOS, __FILE__, __LINE__ + 1
+          def #{name}
+            @#{name} ||= new_redis_object(
+              ::Redis::Set, #{name.inspect}, #{args})
+          end
+        EOS
+      end
+
+      def list(name, *args)
+        declared_redis_object_names << name
+
+        class_eval <<-EOS, __FILE__, __LINE__ + 1
+          def #{name}
+            @#{name} ||= new_redis_object(
+              ::Redis::List, #{name.inspect}, #{args})
+          end
+        EOS
+      end
+
+      def hash_key(name, *args)
+        declared_redis_object_names << name
+
+        class_eval <<-EOS, __FILE__, __LINE__ + 1
+          def #{name}
+            @#{name} ||= new_redis_object(
+              ::Redis::HashKey, #{name.inspect}, #{args})
+          end
+        EOS
+      end
+
+      def counter(name, *args)
+        declared_redis_object_names << name
+
+        class_eval <<-EOS, __FILE__, __LINE__ + 1
+          def #{name}
+            @#{name} ||= new_redis_object(
+              ::Redis::Counter, #{name.inspect}, #{args})
+          end
+        EOS
+      end
+
+      def declared_redis_object_names
+        @declared_redis_object_names ||= []
+      end
+
+      def redis_key_prefix(override = nil)
+        (@redis_key_prefix ||= override) || "plines"
+      end
+    end
+  end
+end
+

data/lib/plines/step.rb

@@ -0,0 +1,269 @@
+require 'forwardable'
+
+module Plines
+  ExternalDependency = Struct.new(:name, :options)
+
+  # Keeps track of a list of external dependencies.
+  # These are yielded as the first argument to
+  # `has_external_dependencies`.
+  class ExternalDependencyList
+    extend Forwardable
+    def_delegators :@dependencies, :any?
+
+    def initialize
+      @dependencies = []
+    end
+
+    def add(name, options = {})
+      @dependencies << ExternalDependency.new(name, options)
+    end
+
+    def to_a
+      @dependencies.dup
+    end
+  end
+
+  # This is the module that should be included in any class that
+  # is intended to be a Plines step.
+  module Step
+    # Error raised when you include Plines::Step in a module that is
+    # not nested within a pipeline module.
+    class NotDeclaredInPipelineError < StandardError; end
+
+    def self.extended(klass)
+      klass.class_eval do
+        include InstanceMethods
+
+        unless pipeline.is_a?(Plines::Pipeline)
+          raise NotDeclaredInPipelineError,
+            "#{klass} is not nested in a pipeline module and thus " +
+            "cannot be a Plines::Step. All plines steps must be " +
+            "declared within pipeline modules."
+        end
+
+        fan_out { |d| [d] } # default to one step instance
+        pipeline.step_classes << self
+      end
+    end
+
+    DEFAULT_DEPENDENCY_FILTER = Proc.new { true }
+
+    def depends_on(*klasses, &block)
+      klasses.each do |klass|
+        dependency_filters[klass] = (block || DEFAULT_DEPENDENCY_FILTER)
+      end
+    end
+
+    def depended_on_by_all_steps
+      pipeline.initial_step = self
+    end
+
+    def depends_on_all_steps
+      pipeline.terminal_step = self
+    end
+
+    def run_jobs_in_serial
+      depends_on step_name do |data|
+        prior_data = data.my_data_hashes.each_cons(2) do |(prior, current)|
+          break prior if current == data.my_data
+        end
+
+        data.their_data == prior_data
+      end
+    end
+
+    def fan_out(&block)
+      @fan_out_blocks ||= []
+      @fan_out_blocks << block
+    end
+
+    def has_external_dependencies(*args, &block)
+      block ||= begin
+        options = args.last.is_a?(Hash) ? args.pop : {}
+        lambda do |deps, _|
+          args.each do |name|
+            deps.add name, options
+          end
+        end
+      end
+
+      external_dependency_definitions << block
+    end
+
+    def has_external_dependencies_for?(data)
+      external_dependency_definitions.any? do |block|
+        list = ExternalDependencyList.new
+        block.call(list, data)
+        list.any?
+      end
+    end
+
+    def external_dependencies_for(data)
+      list = ExternalDependencyList.new
+
+      external_dependency_definitions.each do |block|
+        block.call(list, data)
+      end
+
+      list.to_a
+    end
+
+    def dependencies_for(job, batch_data)
+      Enumerator.new do |yielder|
+        has_dependencies = false
+
+        each_declared_dependency_job_for(job, batch_data) do |job|
+          has_dependencies = true
+          yielder.yield job
+        end
+
+        each_initial_step_job_for(job, batch_data) do |job|
+          yielder.yield job
+        end unless has_dependencies
+      end
+    end
+
+    def jobs_for(batch_data)
+      @fan_out_blocks.inject([batch_data]) do |job_data_hashes, fan_out_block|
+        job_data_hashes.flat_map { |job_data| fan_out_block.call(job_data) }
+      end.map do |job_data|
+        Job.build(self, job_data)
+      end
+    end
+
+    def perform(qless_job)
+      batch = JobBatch.find(qless_job.client, pipeline,
+                            qless_job.data.fetch("_job_batch_id"))
+
+      if batch.creation_in_progress?
+        qless_job.move(qless_job.queue_name, delay: 2)
+        return
+      end
+
+      job_data = DynamicStruct.new(qless_job.data)
+
+      qless_job.after_complete do
+        batch.mark_job_as_complete(qless_job.jid)
+      end
+
+      new(batch, job_data, qless_job)
+        .send(:around_perform)
+    end
+
+    def external_dependency_definitions
+      @external_dependency_definitions ||= []
+    end
+
+    def qless_options
+      @qless_options ||= QlessJobOptions.new
+      yield @qless_options if block_given?
+      @qless_options
+    end
+
+    def enqueue_qless_job(qless, data, options = {})
+      queue_name = if has_external_dependencies_for?(data)
+        Pipeline::AWAITING_EXTERNAL_DEPENDENCY_QUEUE
+      elsif options[:queue] && processing_queue == :plines
+        options[:queue]
+      else
+        processing_queue
+      end
+
+      queue = qless.queues[queue_name]
+
+      options[:priority] = qless_options.priority if qless_options.priority
+      options[:retries] = qless_options.retries if qless_options.retries
+      options[:tags] = Array(options[:tags]) | qless_options.tags
+
+      queue.put(self, data, options)
+    end
+
+    def processing_queue
+      qless_options.queue
+    end
+
+    def pipeline
+      @pipeline ||= begin
+        namespaces = name.split('::')
+        namespaces.pop # ignore the last one
+        namespaces.inject(Object) { |ns, mod| ns.const_get(mod) }
+      end
+    end
+
+    def step_name
+      @step_name ||= name.split('::').last.to_sym
+    end
+
+  private
+
+    def dependency_filters
+      @dependency_filters ||= {}
+    end
+
+    DependencyData = Struct.new(:my_data,        :their_data,
+                                :my_data_hashes, :their_data_hashes)
+
+    def each_declared_dependency_job_for(my_job, batch_data)
+      my_data_hashes = jobs_for(batch_data).map(&:data)
+
+      dependency_filters.each do |klass, filter|
+        klass = pipeline.const_get(klass)
+        their_jobs = klass.jobs_for(batch_data)
+        their_data_hashes = their_jobs.map(&:data)
+
+        their_jobs.each do |their_job|
+          yield their_job if filter.call(DependencyData.new(
+            my_job.data,    their_job.data,
+            my_data_hashes, their_data_hashes
+          ))
+        end
+      end
+    end
+
+    def each_initial_step_job_for(job, batch_data)
+      return if pipeline.initial_step == self
+
+      pipeline.initial_step.jobs_for(batch_data).each do |dependency|
+        yield dependency
+      end
+    end
+
+    module InstanceMethods
+      extend Forwardable
+      attr_reader :job_data, :job_batch, :qless_job
+      def_delegator "self.class", :enqueue_qless_job
+
+      def initialize(job_batch, job_data, qless_job)
+        @job_batch = job_batch
+        @job_data = job_data
+        @qless_job = qless_job
+        @enqueued_job = EnqueuedJob.new(qless_job.client,
+                                        self.class.pipeline, qless_job.jid)
+      end
+
+    private
+
+      def around_perform
+        perform
+      end
+
+      def unresolved_external_dependencies
+        @unresolved_external_dependencies ||=
+          @enqueued_job.unresolved_external_dependencies
+      end
+    end
+
+    QlessJobOptions = Struct.new(:tags, :priority, :queue, :retries) do
+      def initialize
+        super
+        self.queue ||= :plines
+        self.tags ||= []
+      end
+
+      def tag=(value)
+        self.tags = Array(value)
+      end
+    end
+  end
+end
+