plines 0.5.0

data/Rakefile

@@ -0,0 +1,61 @@
+#!/usr/bin/env rake
+require_relative 'config/setup_load_paths'
+require 'bundler/gem_helper'
+Bundler::GemHelper.install_tasks
+
+require 'qless/tasks'
+namespace :qless do
+  task :set_redis_url do
+    if File.exist?('./config/redis_connection_url.txt')
+      ENV['REDIS_URL'] = File.read('./config/redis_connection_url.txt')
+    end
+  end
+
+  task :setup => :set_redis_url do
+    ENV['VVERBOSE'] = '1'
+    ENV['QUEUE'] = 'plines'
+    ENV['INTERVAL'] = '1.0'
+  end
+
+  desc "Start the Qless Web UI"
+  task :server => :set_redis_url do
+    sh "rackup config/config.ru"
+  end
+end
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = %w[--profile --format progress]
+  t.ruby_opts  = "-Ispec -r./config/setup_load_paths -rsimplecov_setup"
+end
+
+if RUBY_ENGINE == 'ruby'
+  require 'cane/rake_task'
+
+  desc "Run cane to check quality metrics"
+  Cane::RakeTask.new(:quality) do |cane|
+    cane.style_glob = "lib/**/*.rb"
+    cane.abc_max = 16
+    cane.add_threshold 'coverage/coverage_percent.txt', :>=, 100
+  end
+else
+  task :quality do
+    # no-op; Cane isn't supported on this interpretter
+  end
+end
+
+task default: [:spec, :quality]
+
+namespace :ci do
+  desc "Run all tests both integrated and in isolation"
+  task :spec do
+    test_all_script = File.expand_path('../script/test_all', __FILE__)
+    sh test_all_script
+  end
+end
+
+
+desc "Run CI build"
+task ci: %w[ ci:spec quality ]
+

data/lib/plines.rb

@@ -0,0 +1,13 @@
+require 'plines/configuration'
+require 'plines/dependency_graph'
+require 'plines/dynamic_struct'
+require 'plines/enqueued_job'
+require 'plines/external_dependency_timeout'
+require 'plines/job'
+require 'plines/job_enqueuer'
+require 'plines/job_batch'
+require 'plines/job_batch_list'
+require 'plines/pipeline'
+require 'plines/step'
+require "plines/version"
+

data/lib/plines/configuration.rb

@@ -0,0 +1,55 @@
+module Plines
+  # Stores global Plines configuration.
+  class Configuration
+    # Raised when there is a configuration error.
+    class Error < StandardError; end
+
+    TWO_MONTHS_IN_SECONDS = 2 * 30 * 24 * 60 * 60
+
+    def initialize
+      qless_client   { raise Error, "qless_client has not been configured" }
+      batch_list_key { raise Error, "batch_list_key has not been configured" }
+
+      qless_job_options { |job| {} }
+      self.data_ttl_in_seconds = TWO_MONTHS_IN_SECONDS
+      @callbacks = Hash.new { |h, k| h[k] = [] }
+    end
+
+    def qless_client(&block)
+      @qless_client_block = block
+    end
+
+    def qless_client_for(key)
+      @qless_client_block[key]
+    end
+
+    def batch_list_key(&block)
+      @batch_list_key_block = block
+    end
+
+    def batch_list_key_for(batch_data)
+      @batch_list_key_block[batch_data]
+    end
+
+    attr_accessor :data_ttl_in_seconds
+    def data_ttl_in_milliseconds
+      (data_ttl_in_seconds * 1000).to_i
+    end
+
+    attr_reader :qless_job_options_block
+    def qless_job_options(&block)
+      @qless_job_options_block = block
+    end
+
+    def after_job_batch_cancellation(&block)
+      @callbacks[:after_job_batch_cancellation] << block
+    end
+
+    def notify(callback_type, *args)
+      @callbacks[callback_type].each do |callback|
+        callback.call(*args)
+      end
+    end
+  end
+end
+

data/lib/plines/dependency_graph.rb

@@ -0,0 +1,81 @@
+require 'set'
+
+module Plines
+  # Represents a dependency graph of Plines steps. This graph contains
+  # Jobs (i.e. Step classes paired with data). The graph
+  # takes care of preventing duplicate step instances.
+  class DependencyGraph
+    attr_reader :steps # FYI, steps is not ordered according to dependencies
+
+    # Raised when a circular dependency is detected.
+    class CircularDependencyError < StandardError; end
+
+    def initialize(pipeline, batch_data)
+      step_classes = pipeline.step_classes
+      @steps = Job.accumulate_instances do
+        step_classes.each do |step_klass|
+          step_klass.jobs_for(batch_data).each do |job|
+            job.add_dependencies_for(batch_data)
+          end
+        end
+
+        @terminal_jobs = pipeline.terminal_step.jobs_for(batch_data)
+      end
+
+      cleanup_and_validate_dependencies!
+    end
+
+    def ordered_steps
+      visited = Set.new
+      Enumerator.new do |yielder|
+        steps.each do |step|
+          yield_next_ordered_step_for(step, visited, yielder)
+        end
+      end
+    end
+
+  private
+
+    def cleanup_and_validate_dependencies!
+      @visited_steps = Set.new
+
+      @steps.each do |step|
+        next if @visited_steps.include?(step)
+        depth_first_search_from(step)
+      end
+    end
+
+    def depth_first_search_from(step, current_stack=Set.new)
+      @visited_steps << step
+      add_terminal_job_dependencies(step)
+
+      if current_stack.include?(step)
+        raise CircularDependencyError,
+          "Your graph appears to have a circular dependency: " +
+          current_stack.inspect
+      end
+
+      step.dependencies.each do |dep|
+        depth_first_search_from(dep, current_stack | [step])
+      end
+    end
+
+    def add_terminal_job_dependencies(job)
+      if job.dependents.none? && !@terminal_jobs.include?(job)
+        @terminal_jobs.each { |term_job| term_job.add_dependency(job) }
+      end
+    end
+
+    def yield_next_ordered_step_for(step, visited, yielder)
+      return if visited.include?(step)
+
+      step.dependencies.each do |dependency|
+        yield_next_ordered_step_for(dependency, visited, yielder)
+      end
+
+      visited << step
+      yielder.yield step
+    end
+  end
+end
+

data/lib/plines/dynamic_struct.rb

@@ -0,0 +1,34 @@
+module Plines
+  # Transforms an arbitrarily deeply nested hash into a dot-syntax
+  # object. Useful as an alternative to a hash since it is "strongly typed"
+  # in the sense that fat-fingered property names result in a NoMethodError,
+  # rather than getting a nil as you would with a hash.
+  class DynamicStruct
+    attr_reader :attribute_names, :to_hash
+
+    def initialize(hash)
+      @to_hash = hash
+      @attribute_names = hash.keys.map(&:to_sym)
+
+      hash.each do |key, value|
+        value = method_value_for(value)
+        define_singleton_method(key) { value }
+        define_singleton_method("#{key}?") { !!value }
+      end
+    end
+
+  private
+
+    def method_value_for(hash_value)
+      return self.class.new(hash_value) if hash_value.is_a?(Hash)
+
+      if hash_value.is_a?(Array) && hash_value.all? { |v| v.is_a?(Hash) }
+        return hash_value.map { |v| self.class.new(v) }
+      end
+
+      hash_value
+    end
+  end
+end
+
+

data/lib/plines/enqueued_job.rb

@@ -0,0 +1,120 @@
+require 'plines/redis_objects'
+
+module Plines
+  # Once a Plines::Job has been enqueued as a Qless job into redis,
+  # an EnqueuedJob is used to represent and hold the additional state
+  # that Plines needs to track about the job.
+  class EnqueuedJob < Struct.new(:pipeline, :jid)
+    include Plines::RedisObjectsHelpers
+
+    attr_reader :qless, :redis
+
+    def initialize(qless, pipeline, jid, &block)
+      @qless = qless
+      @redis = qless.redis
+      super(pipeline, jid)
+      instance_eval(&block) if block
+    end
+
+    def self.create(qless, pipeline, jid, *external_dependencies)
+      new(qless, pipeline, jid) do
+        external_dependencies.each do |dep|
+          pending_ext_deps << dep
+        end
+      end
+    end
+
+    def qless_job
+      qless.jobs[jid]
+    end
+
+    def pending_external_dependencies
+      pending_ext_deps.members
+    end
+
+    def resolved_external_dependencies
+      resolved_ext_deps.members
+    end
+
+    def timed_out_external_dependencies
+      timed_out_ext_deps.members
+    end
+
+    def all_external_dependencies
+      pending_ext_deps.union(
+        resolved_ext_deps, timed_out_ext_deps
+      )
+    end
+
+    def unresolved_external_dependencies
+      pending_ext_deps.union(timed_out_ext_deps)
+    end
+
+    def resolve_external_dependency(name)
+      update_external_dependency \
+        name, resolved_ext_deps, pending_ext_deps, timed_out_ext_deps
+    end
+
+    def timeout_external_dependency(name)
+      update_external_dependency \
+        name, timed_out_ext_deps, pending_ext_deps
+    end
+
+  private
+
+    alias id jid # id is needed by Redis::Objects
+    set :pending_ext_deps
+    set :resolved_ext_deps
+    set :timed_out_ext_deps
+
+    CannotUpdateExternalDependencyError = Class.new(StandardError)
+
+    def update_external_dependency(name, destination_set, *source_sets)
+      assert_has_external_dependency!(name)
+
+      result = nil
+
+      5.times do
+        result = try_to_update(qless_job, name, destination_set, *source_sets)
+        break unless result == :needs_retry
+      end
+
+      if result == :needs_retry
+        raise CannotUpdateExternalDependencyError,
+          "Failed to update dependency #{name} after many tries"
+      end
+    end
+
+    def try_to_update(job, name, destination_set, *source_sets)
+      redis.watch(pending_ext_deps.key) do
+        pending_deps = self.pending_external_dependencies
+
+        response = redis.multi do
+          source_sets.each do |source_set|
+            source_set.move(name, destination_set)
+          end
+
+          if job && pending_deps == [name]
+            job.move(job.klass.processing_queue)
+          end
+        end
+
+        return :needs_retry unless response
+      end
+    end
+
+    def assert_has_external_dependency!(name)
+      results = redis.multi do
+        pending_ext_deps.include?(name)
+        resolved_ext_deps.include?(name)
+        timed_out_ext_deps.include?(name)
+      end
+
+      if results.none?
+        raise ArgumentError, "EnqueuedJob #{jid} does not have pending " +
+                             "external dependency #{name.inspect}"
+      end
+    end
+  end
+end
+

data/lib/plines/external_dependency_timeout.rb

@@ -0,0 +1,30 @@
+module Plines
+  # This is a job that gets enqueued with a delay in order to timeout external
+  # dependencies.  When it runs, it will timeout the named external dependency
+  # for the given jids.  If the named dependency is the only remaining pending
+  # dependency for any of the jobs identified by the jids, they will get moved
+  # into their appropriate processing queue.
+  class ExternalDependencyTimeout
+    def self.perform(job)
+      pipeline_parts = job.data.fetch('pipeline').split('::')
+      pipeline = pipeline_parts.inject(Object) { |ns, mod| ns.const_get(mod) }
+
+      job_batch = JobBatch.find(job.client, pipeline,
+                                job.data.fetch("job_batch_id"))
+
+      job_batch.timeout_external_dependency \
+        job.data.fetch("dep_name"),
+        job.data.fetch("jids")
+    end
+
+    def self.job_data_for(job_batch, dep_name, jids)
+      {
+        "pipeline" => job_batch.pipeline.name,
+        "job_batch_id" => job_batch.id,
+        "dep_name" => dep_name,
+        "jids" => jids
+      }
+    end
+  end
+end
+

data/lib/plines/indifferent_hash.rb

@@ -0,0 +1,58 @@
+require 'delegate'
+
+module Plines
+  # Provides a hash that can be accessed by symbol or string keys.
+  # This is useful because a plines job batch data hash is commonly
+  # provided with symbol keys, but after round-tripping through
+  # JSON it is converted to strings. We can't safely convert all
+  # strings to symbols (as symbols are never GC'd) so instead we
+  # use this for the data hash.
+  class IndifferentHash < DelegateClass(Hash)
+    NotAHashError = Class.new(TypeError)
+    ConflictingEntriesError = Class.new(ArgumentError)
+
+    private_class_method :new
+
+    def self.from(original)
+      unless original.is_a?(Hash) || original.is_a?(IndifferentHash)
+        raise NotAHashError, "Expected a hash, got #{original.inspect}"
+      end
+
+      indif = Hash.new { |hash, key| hash[key.to_s] if Symbol === key }
+
+      original.each do |key, value|
+        key = key.to_s
+
+        if indif.has_key?(key)
+          raise ConflictingEntriesError,
+            "Hash has conflicting entries for #{key}: #{original}"
+        end
+
+        indif[key] = indifferent(value)
+      end
+
+      new(indif)
+    end
+
+    def self.indifferent(object)
+      case object
+      when Hash  then from(object)
+      when Array then object.map { |o| indifferent(o) }
+      else object
+      end
+    end
+
+    def fetch(key)
+      if !has_key?(key) && Symbol === key && has_key?(key.to_s)
+        key = key.to_s
+      end
+
+      super
+    end
+
+    def merge(other)
+      IndifferentHash.from super(IndifferentHash.from other)
+    end
+  end
+end
+