conflow 0.1.0

data/lib/conflow.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "conflow/version"
+
+require "digest"
+require "json"
+require "redis"
+
+require "conflow/redis/connection_wrapper"
+require "conflow/redis/field"
+require "conflow/redis/value_field"
+require "conflow/redis/hash_field"
+require "conflow/redis/array_field"
+require "conflow/redis/sorted_set_field"
+require "conflow/redis/field_builder"
+require "conflow/redis/model"
+require "conflow/redis/identifier"
+require "conflow/redis/findable"
+
+require "conflow/redis/script"
+require "conflow/redis/add_job_script"
+require "conflow/redis/complete_job_script"
+
+require "conflow/job"
+require "conflow/flow/job_handler"
+require "conflow/flow"
+require "conflow/worker"
+
+# Conflow allows defining comlicated workflows with dependencies.
+# Inspired by {https://github.com/chaps-io/gush Gush} (the idea) and
+# {https://github.com/nateware/redis-objects Redis::Objects} (the implementation) it focuses solely on dependency logic,
+# while leaving queueing jobs and executing them entirely in hands of the programmer.
+module Conflow
+  class << self
+    def redis=(conn)
+      @redis =
+        if defined?(ConnectionPool) && conn.is_a?(ConnectionPool)
+          conn
+        else
+          Conflow::Redis::ConnectionWrapper.new(conn)
+        end
+    end
+
+    def redis
+      self.redis = ::Redis.current unless defined?(@redis)
+      @redis
+    end
+  end
+end

data/lib/conflow/flow.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Conflow
+  # Flow is a set of steps needed to complete certain task. It is composed of jobs
+  # which have dependency relations with one another.
+  #
+  # {Conflow::Flow} class is designed to be inherited in your application. You must supply {queue} method
+  #
+  # @!attribute [r] jobs
+  #   Read-only array of jobs added to the flow.
+  #   @return [Array<Conflow::Job>] List of jobs in the flow
+  #
+  # @!attribute [rw] indegree
+  #   Sorted set (Redis zset) of job ids. Each job has a score attached, which is the number of "indegree" nodes -
+  #   the nodes on which given job depends. This changes dynamically and score equal to 0 means that all dependencies
+  #   are fulfilled.
+  #   @return [Conflow::Redis::SortedSetField] Set of jobs to be performed
+  #
+  # @!method queue(job)
+  #   @abstract
+  #   Queues job to be performed. Both id of the flow and id of the job must be preserved
+  #   in order to recreate job in worker.
+  #   @param job [Conflow::Job] job to be queued
+  #
+  #   @example Queue sidekiq job
+  #     class MyBaseFlow < Conflow::Flow
+  #       def queue(job)
+  #         Sidekiq::Client.enqueue(FlowWorkerJob, id, job.id)
+  #       end
+  #     end
+  class Flow < Conflow::Redis::Field
+    include Conflow::Redis::Model
+    include Conflow::Redis::Identifier
+    include Conflow::Redis::Findable
+    include JobHandler
+
+    has_many :jobs, Conflow::Job
+    field :indegree, :sorted_set
+
+    # Create new flow with given parameters
+    # @example Simple configurable flow
+    #   class MyFlow < Conflow::Flow
+    #     def configure(id:, strict:)
+    #       run UpsertJob, params: { id: id }
+    #       run CheckerJob, params: { id: id } if strict
+    #     end
+    #   end
+    #
+    #   MyFlow.create(id: 320, strict: false)
+    #   MyFlow.create(id: 15, strict: true)
+    def self.create(*args)
+      new.tap { |flow| flow.configure(*args) }
+    end
+
+    # @abstract
+    # Override this method in order to contain your flow definition inside the class.
+    # This method will be called if flow is created using {.create} method.
+    def configure(*args); end
+  end
+end

data/lib/conflow/flow/job_handler.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Conflow
+  class Flow < Conflow::Redis::Field
+    # Handles running and finishing jobs
+    module JobHandler
+      # @param job_class [Class] Class of the worker which will perform the job
+      # @param params [Hash]
+      #   Parameters of the job. They will be passed to {Conflow::Worker#perform} block. Defalts to empty hash.
+      # @param after [Conflow::Job|Class|Integer|Array<Conflow::Job,Class,Integer>]
+      #   Dependencies of the job. Can be one or more objects of the following classes: {Conflow::Job}, Class, Integer
+      # @param hook [Symbol] method to be called on {Conflow::Flow} instance after job is performed.
+      #   The hook method should accept result of the job (value returned by {Conflow::Worker#perform})
+      # @return [Conflow::Job] enqueued job
+      def run(job_class, params: {}, after: [], hook: nil)
+        build_job(job_class, params, hook).tap do |job|
+          job_classes[job_class] = job
+          after = prepare_dependencies(after)
+
+          call_script(Conflow::Redis::AddJobScript, job, after: after)
+        end
+      end
+
+      def finish(job, result = nil)
+        send(job.hook.to_s, result) unless job.hook.nil?
+        call_script(Conflow::Redis::CompleteJobScript, job)
+      end
+
+      private
+
+      def queue_available_jobs
+        indegree.delete_if(score: 0).each do |job_id|
+          queue Conflow::Job.new(job_id)
+        end
+      end
+
+      def build_job(job_class, params, hook)
+        Conflow::Job.new.tap do |job|
+          job.params = params if params.any?
+          job.hook = hook if hook
+          job.class_name = job_class.name
+        end
+      end
+
+      def call_script(script, *args)
+        script.call(self, *args)
+        queue_available_jobs
+      end
+
+      def prepare_dependencies(dependencies)
+        case dependencies
+        when Enumerable then dependencies.map(&method(:prepare_dependency))
+        else [prepare_dependency(dependencies)]
+        end
+      end
+
+      def prepare_dependency(dependency)
+        case dependency
+        when Conflow::Job    then dependency
+        when Class           then job_classes[dependency]
+        when String, Numeric then Conflow::Job.new(dependency)
+        end
+      end
+
+      def job_classes
+        @job_classes ||= {}
+      end
+    end
+  end
+end

data/lib/conflow/job.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Conflow
+  # Represents conflow job.
+  # @!attribute [rw] status
+  #   Status of the job
+  #   @return [Integer] 0 - pending, 1 - finished
+  # @!attribute [rw] hook
+  #   @return [String, nil] name of the method on related flow to be called once job is finished
+  # @!attribute [rw] class_name
+  #   @return [String] class name of the worker class
+  # @!attribute [rw] params
+  #   @return [Hash, nil] parameters needed to complete job
+  class Job < Conflow::Redis::Field
+    include Conflow::Redis::Model
+    include Conflow::Redis::Identifier
+
+    has_many :successors, Conflow::Job
+    field :params,     :hash
+    field :class_name, :value
+    field :status,     :value # 0 - pending, 1 - finished
+    field :hook,       :value
+
+    # Returns instance of Job. It sets status to 0 (pending) for new jobs
+    def initialize(*)
+      super
+      status.default(0)
+    end
+
+    # Convienience method returning Class object of the job.
+    # It's the class supplied in {Conflow::Flow#run} method
+    # @return [Class] class of the job
+    def worker_type
+      Object.const_get(class_name.to_s)
+    end
+  end
+end

data/lib/conflow/redis/add_job_script.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Conflow
+  module Redis
+    # Adds new job to flow
+    class AddJobScript < Script
+      # script accepts keys: flow.job_ids, flow.indegree, and keys successors of the jobs on which new job depends.
+      # It also accepts one argument: id of the new job
+      self.script = <<~LUA
+        local job_list = KEYS[1]
+        local indegree_set = KEYS[2]
+        local job_id = ARGV[1]
+        local score = 0
+
+        for i=3,#KEYS do
+          if redis.call('get', KEYS[i] .. ':status') == '0' then
+            score = score + 1
+            redis.call('lpush', KEYS[i] .. ':successor_ids', job_id)
+          end
+        end
+
+        redis.call('lpush', job_list, job_id)
+        return redis.call('zadd', indegree_set, score, job_id)
+      LUA
+
+      class << self
+        # Call the script.
+        # Script changes {Conflow::Flow#indegree} of all of its successors by -1 (freeing them to be queued if it
+        # reaches 0) and sets {Conflow::Job#status} to 1 (finished)
+        def call(flow, job, after: [])
+          super([flow.job_ids.key, flow.indegree.key, *after.map(&:key)], [job.id])
+        end
+      end
+    end
+  end
+end

data/lib/conflow/redis/array_field.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Conflow
+  module Redis
+    # Represents Redis list. It's methods mirror most used Array methods.
+    class ArrayField < Field
+      include Enumerable
+
+      def [](index)
+        command :lindex, [key, index]
+      end
+
+      def []=(index, value)
+        command :lset, [key, index, value]
+      end
+
+      def insert(value, after: nil, before: nil)
+        if after
+          command :linsert, [key, :after, after, value]
+        elsif before
+          command :linsert, [key, :before, before, value]
+        else
+          raise ArgumentError, "You need to pass one of [:after, :before] keywords"
+        end
+      end
+
+      def size
+        command :llen, [key]
+      end
+
+      def to_a
+        command :lrange, [key, 0, -1]
+      end; alias to_ary to_a
+
+      def pop
+        command :rpop, [key]
+      end
+
+      def push(*values)
+        command :rpush, [key, values]
+      end; alias << push
+
+      def concat(ary)
+        push(*ary)
+      end
+
+      def shift
+        command :lpop, [key]
+      end
+
+      def unshift(value)
+        command :lpush, [key, value]
+      end
+
+      def overwrite(new_array)
+        redis.with do |conn|
+          conn.pipelined do
+            conn.del(key)
+            conn.rpush(key, new_array)
+          end
+        end
+      end
+
+      def each(&block)
+        to_a.each(&block)
+      end
+
+      def ==(other)
+        case other
+        when Array      then to_a == other
+        when ArrayField then key == other.key || to_a == other.to_a
+        else super
+        end
+      end
+
+      def to_s
+        to_a.to_s
+      end
+    end
+  end
+end

data/lib/conflow/redis/complete_job_script.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Conflow
+  module Redis
+    # Adds new job to flow
+    class CompleteJobScript < Script
+      self.script = <<~LUA
+        local indegree_set = KEYS[1]
+        local job_id = KEYS[2]
+
+        local successors = redis.call('lrange', job_id .. ':successor_ids', 0, -1)
+
+        for i=1,#successors do
+          redis.call('zincrby', indegree_set, -1, successors[i])
+        end
+
+        return redis.call('set', job_id .. ':status', '1')
+      LUA
+
+      class << self
+        # Call the script.
+        # Script changes {Conflow::Flow#indegree} of all of its successors by -1 (freeing them to be queued if it
+        # reaches 0) and sets {Conflow::Job#status} to 1 (finished)
+        # @param flow [Conflow::Flow] Flow to which job belongs to
+        # @param job [Conflow::Job] Job to be marked as completed
+        def call(flow, job)
+          super([flow.indegree.key, job.key])
+        end
+      end
+    end
+  end
+end

data/lib/conflow/redis/connection_wrapper.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Conflow
+  module Redis
+    # Wraps Redis connection to behave like connection pool
+    class ConnectionWrapper
+      # @param redis [Redis] Redis connection to be wrapped
+      def initialize(redis)
+        @redis = redis
+      end
+
+      # Allows accessing Redis connection
+      # @yieldparam [Redis] redis connection
+      def with
+        yield @redis
+      end
+    end
+  end
+end

data/lib/conflow/redis/field.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Conflow
+  module Redis
+    # Base class for fields. All fields are assigned a Redis key.
+    class Field
+      attr_reader :key
+      alias id key
+
+      def initialize(key)
+        @key = key
+      end
+
+      private
+
+      def redis
+        Conflow.redis
+      end
+
+      def command(command, args)
+        redis.with { |conn| conn.send(command, *args) }
+      end
+
+      def transaction(max_retries: 5)
+        result = redis.with do |conn|
+          retryable(max_retries, conn) do
+            conn.watch(key) do
+              conn.multi { |multi| yield(multi) }
+            end
+          end
+        end
+
+        raise ::Redis::CommandError, "Transaction failed #{max_retries} times" unless result
+        result
+      end
+
+      def retryable(max_retries, *args)
+        loop do
+          result = yield(*args)
+          break result if result || max_retries.zero?
+          max_retries -= 1
+        end
+      end
+    end
+  end
+end

data/lib/conflow/redis/field_builder.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Conflow
+  module Redis
+    # Helper class for defining getter and setter methods for Fields
+    class FieldBuilder
+      # This dynamic module contains accessor methods for a field
+      class FieldAccessor < Module
+        def initialize(field_name, klass, methods)
+          super() do
+            define_getter(field_name, klass) if methods.include?(:getter)
+            define_setter(field_name) if methods.include?(:setter)
+          end
+        end
+
+        def define_getter(field_name, klass)
+          instance_var = "@#{field_name}"
+
+          define_method(field_name) do
+            instance_variable_get(instance_var) ||
+              instance_variable_set(instance_var, klass.new([key, field_name].join(":")))
+          end
+        end
+
+        def define_setter(field_name)
+          define_method("#{field_name}=") do |value|
+            send(field_name).tap { |field| field.overwrite(value) }
+          end
+        end
+      end
+
+      attr_reader :field_name, :klass
+
+      def initialize(field_name, klass)
+        @field_name = field_name
+        @klass = klass
+      end
+
+      def call(base, methods: %i[getter setter])
+        base.include(FieldAccessor.new(field_name, klass, methods))
+      end
+    end
+  end
+end