conflow 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 572a2e10d542d2914de25fc748c1c153938e59b081aaef7829208395a86e462b
-  data.tar.gz: 99a32aa33c314eeabfa4feed3b386072ba2146139b552b257c59cd4b693fcca0
+  metadata.gz: e4e50a7f447dad66d1522e881c5253e3478a806a38a08478660fc0a831d59a5d
+  data.tar.gz: ae859355013408d204a12532dc480efdef8933dd70bc37ab87e60a06e17f0612
 SHA512:
-  metadata.gz: 8538390ead7bcc20bcea935aee1ec4d4ec7359eba95a89a8a5ddd14a3fbec25b69f9900d4548a4fd6370ae94eff13471ac794fe39a593d64a16ae0f2bf0abdf3
-  data.tar.gz: 53f5221e6bb9c0bd7dffdfdae3f0cb955f170e3aaa22a281a0e7f682952d9ad946de7daa93b0899935a5e77864f8178e5219e766182f74cbddfc16ea0d12881a
+  metadata.gz: 6b004258305c37c987b6edee1482ba0e00e2f9e02ca3ef6d5354288de59f7e6f7d8f2c75235cc1a51faf678e8a639459d6cb925f03276a10fc307b86b877a964
+  data.tar.gz: b9b87d6d3579b7301e19b1e74ff7ff292e06e2df16f1810c4e0a2de6ab537597fb2aa42b9c1da780248161f1e3b8e667cb97be88a80fcfd7268a2afe23ab21f8

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.4.0
+
+- Removed hooks, added promises. If you were looking only on Changelog, you never even knew about hooks - it can stay that way :)
+
 ## 0.2.0
 
 - Flow and Jobs are now removed after they are finished

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    conflow (0.3.0)
+    conflow (0.4.0)
       redis (~> 4.0)
 
 GEM

data/README.md

@@ -97,24 +97,25 @@ end
 
 ![Created graph](https://camo.githubusercontent.com/0b1ee59994323900906264ea50fbc9169e4d21dd/68747470733a2f2f63686172742e676f6f676c65617069732e636f6d2f63686172743f63686c3d646967726170682b472b2537422530442530412b2b72616e6b6469722533444c522533422530442530412b2b25323253544152542532322b2d2533452b25323246697273744a6f622532322530442530412b2b25323253544152542532322b2d2533452b253232496e646570656e64656e744a6f622532322530442530412b2b25323246697273744a6f622532322b2d2533452b2532325365636f6e644a6f622532322530442530412b2b253232496e646570656e64656e744a6f622532322b2d2533452b2532325365636f6e644a6f622532322530442530412b2b2532325365636f6e644a6f622532322b2d2533452b25323246696e69736855702532322530442530412b2b25323246696e69736855702532322b2d2533452b253232454e442532322530442530412537442530442530412b266368743d6776)
 
-### Hooks
+#### Promises
 
-If you want to modify your flow dynamically depending on results of your jobs, you can use hooks.
-
-Hooks are methods defined on the `Flow` objects which will be called with result of the job (value returned by `Worker`, see [Performing jobs](https://github.com/fanfilmu/conflow#performing-jobs)).
+In order to use other Job's result as parameter of another job, use Futures:
 
 ```ruby
 class MyFlow < ApplicationFlow
   def configure
-    run CollectEmails, hook: :send_notifications
-  end
-
-  def send_notifications(emails)
-    emails.each { |email| run SendNotification, params: { email: email } }
+    first = run FirstJob
+    run SecondJob, params: { object_id: first.outcome[:id] }
   end
 end
 ```
 
+Note that `SecondJob` will automatically depend on `FirstJob`. When `FirstJob` finishes, it is expected to return hash: `{ id: "<some object>" }`.
+
+Returned object must be serializable with JSON in order to be properly persisted and handled by Redis script which resolves promises.
+
+If `FirstJob` returns `{ id: 14 }`, `SecondJob` will be run with `{ object_id: 14 }` parameter.
+
 ### Performing jobs
 
 To perform job, use `Conflow::Worker` mixin. It adds `#perform` method, which accepts two arguments: IDs of the flow and the job.

data/lib/conflow.rb

@@ -10,6 +10,7 @@ require "conflow/redis"
 require "conflow/redis/connection_wrapper"
 require "conflow/redis/field"
 require "conflow/redis/value_field"
+require "conflow/redis/raw_value_field"
 require "conflow/redis/hash_field"
 require "conflow/redis/array_field"
 require "conflow/redis/sorted_set_field"
@@ -22,9 +23,14 @@ require "conflow/redis/findable"
 require "conflow/redis/script"
 require "conflow/redis/add_job_script"
 require "conflow/redis/complete_job_script"
+require "conflow/redis/resolve_promises_script"
 require "conflow/redis/queue_jobs_script"
 
+require "conflow/error"
+require "conflow/future"
+require "conflow/promise"
 require "conflow/job"
+require "conflow/flow/job_builder"
 require "conflow/flow/job_handler"
 require "conflow/flow"
 require "conflow/worker"

data/lib/conflow/error.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Conflow
+  # Base class for Conflow errors
+  class Error < StandardError; end
+end

data/lib/conflow/flow.rb

@@ -43,6 +43,8 @@ module Conflow
     field :indegree,    :sorted_set
 
     # Create new flow with given parameters
+    # @param args [Array<Object>] any parameters that will be passed to {#configure} method
+    # @return [Conflow::Job] job object representing created job
     # @example Simple configurable flow
     #   class MyFlow < Conflow::Flow
     #     def configure(id:, strict:)
@@ -66,6 +68,8 @@ module Conflow
     # @abstract
     # Override this method in order to contain your flow definition inside the class.
     # This method will be called if flow is created using {.create} method.
+    # @param args [Array<Object>] any arguments needed to start a flow
+    # @see create
     def configure(*args); end
   end
 end

data/lib/conflow/flow/job_builder.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Conflow
+  class Flow < Conflow::Redis::Field
+    # Handles creating jobs
+    # @api private
+    class JobBuilder
+      # Holds mapping between worker classes and {Conflow::Job} objects
+      attr_reader :context
+
+      # Initialize {JobBuilder} with new empty context
+      def initialize
+        @context = {}
+      end
+
+      # Create new job and resolve it's dependencies.
+      # @see Conflow::Flow::JobHandler#run
+      # @return [Conflow::Job, Array<Conflow::Job>]
+      def call(worker_class, params, dependencies)
+        job = initialize_job(worker_class)
+
+        promises, params = extract_promises(job, params)
+        dependencies = build_dependencies(promises, dependencies)
+        assign_job_attributes(job, promises, params)
+
+        [job, dependencies]
+      end
+
+      private
+
+      def initialize_job(worker_class)
+        Conflow::Job.new.tap do |job|
+          job.class_name = worker_class.name
+          context[worker_class] = job
+        end
+      end
+
+      def assign_job_attributes(job, promises, params)
+        job.params = params if params.any?
+        job.promise_ids.push(*promises.map(&:id)) if promises.any?
+      end
+
+      def extract_promises(job, params)
+        params = params.dup
+
+        promises = params.map do |key, value|
+          next unless value.is_a?(Conflow::Future)
+
+          params.delete(key)
+          value.build_promise(job, key)
+        end.compact
+
+        [promises, params]
+      end
+
+      def build_dependencies(promises, dependencies)
+        promise_jobs = promises.map { |promise| Conflow::Job.new(promise.job_id.value) }
+        dependencies = prepare_dependencies(dependencies)
+
+        (promise_jobs + dependencies).compact.uniq
+      end
+
+      def prepare_dependencies(dependencies)
+        case dependencies
+        when Enumerable then dependencies.map(&method(:prepare_dependency))
+        else [prepare_dependency(dependencies)]
+        end
+      end
+
+      def prepare_dependency(dependency)
+        case dependency
+        when Conflow::Job    then dependency
+        when Class           then context[dependency]
+        when String, Numeric then Conflow::Job.new(dependency)
+        end
+      end
+    end
+  end
+end

data/lib/conflow/flow/job_handler.rb

@@ -9,24 +9,30 @@ module Conflow
       #   Parameters of the job. They will be passed to {Conflow::Worker#perform} block. Defalts to empty hash.
       # @param after [Conflow::Job|Class|Integer|Array<Conflow::Job,Class,Integer>]
       #   Dependencies of the job. Can be one or more objects of the following classes: {Conflow::Job}, Class, Integer
-      # @param hook [Symbol] method to be called on {Conflow::Flow} instance after job is performed.
-      #   The hook method should accept result of the job (value returned by {Conflow::Worker#perform})
       # @return [Conflow::Job] enqueued job
-      def run(job_class, params: {}, after: [], hook: nil)
-        build_job(job_class, params, hook).tap do |job|
-          job_classes[job_class] = job
-          after = prepare_dependencies(after).compact
+      def run(job_class, params: {}, after: [])
+        job, dependencies = job_builder.call(job_class, params, after)
 
-          call_script(Conflow::Redis::AddJobScript, job, after: after)
-          queue_available_jobs
-        end
+        call_script(Conflow::Redis::AddJobScript, job, after: dependencies)
+        queue_available_jobs
+
+        job
       end
 
-      # Finishes job, changes its status, runs hook if it's present and queues new available jobs
+      # Starts the job - resolves it's promises. It's called by {Worker} before it yields parameters
+      # @api private
+      # @see Worker
+      # @param job [Conflow::Job] job that needs to resolve it's promises
+      # @return [void]
+      def start(job)
+        call_script(Conflow::Redis::ResolvePromisesScript, job)
+      end
+
+      # Finishes job, changes its status, assigns result of the job and queues new available jobs
       # @param job [Conflow::Job] job to be marked as finished
-      # @param result [Object] result of the job to be passed to hook
+      # @param result [Object] result of the job
       def finish(job, result = nil)
-        send(job.hook.to_s, result) unless job.hook.nil?
+        job.result = result if result
         call_script(Conflow::Redis::CompleteJobScript, job)
         queue_available_jobs
         destroy! if finished?
@@ -40,35 +46,12 @@ module Conflow
         end
       end
 
-      def build_job(job_class, params, hook)
-        Conflow::Job.new.tap do |job|
-          job.params = params if params.any?
-          job.hook = hook if hook
-          job.class_name = job_class.name
-        end
-      end
-
       def call_script(script, *args)
         script.call(self, *args)
       end
 
-      def prepare_dependencies(dependencies)
-        case dependencies
-        when Enumerable then dependencies.map(&method(:prepare_dependency))
-        else [prepare_dependency(dependencies)]
-        end
-      end
-
-      def prepare_dependency(dependency)
-        case dependency
-        when Conflow::Job    then dependency
-        when Class           then job_classes[dependency]
-        when String, Numeric then Conflow::Job.new(dependency)
-        end
-      end
-
-      def job_classes
-        @job_classes ||= {}
+      def job_builder
+        @job_builder ||= JobBuilder.new
       end
     end
   end

data/lib/conflow/future.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Conflow
+  # Raised when future was built with nested key
+  # @example
+  #   job.outcome[:parent][:child] #=> raises InvalidNestedPromise
+  class InvalidNestedFuture < Error
+    # Sets custom message for the error
+    def initialize
+      super "Futures don't allow extracting nested fields"
+    end
+  end
+
+  # Struct like objects that represent value to be returned by {Job}
+  class Future
+    # Job which result is promised
+    attr_reader :job
+    # Key in the result hash to which this future proxies
+    attr_reader :result_key
+
+    # @param job [Conflow::Job] Job which result is promised
+    # @param result_key [String, Symbol] key under which promised value exists
+    # @api private
+    # @see Conflow::Job#outcome
+    def initialize(job, result_key = nil)
+      @job = job
+      @result_key = result_key
+    end
+
+    # Returns new {Future} with assigned key, if possible
+    # @param key [Symbol, String] Key in result hash of the job that holds promised value
+    # @return [Future] future object that can be used as parameter
+    # @example
+    #   job = run AJob #=> returns { email: "test@example.com" }
+    #   job.outcome[:email] # when resolved, will return "test@example.com"
+    def [](key)
+      raise InvalidNestedFuture if result_key
+      Future.new(job, key)
+    end
+
+    # Builds promise from this future
+    # @api private
+    # @param depending_job [Conflow::Job] job which will use new promise
+    # @param param_key [Symbol, String] key in result hash that holds promised value
+    # @return [Conflow::Promise] promise object
+    def build_promise(depending_job, param_key)
+      Promise.new.tap do |promise|
+        promise.assign_attributes(job_id: job.id, hash_field: param_key, result_key: result_key)
+        depending_job.promise_ids << promise.id
+      end
+    end
+  end
+end

data/lib/conflow/job.rb

@@ -5,8 +5,6 @@ module Conflow
   # @!attribute [rw] status
   #   Status of the job
   #   @return [Integer] 0 - pending, 1 - finished
-  # @!attribute [rw] hook
-  #   @return [String, nil] name of the method on related flow to be called once job is finished
   # @!attribute [rw] class_name
   #   @return [String] class name of the worker class
   # @!attribute [rw] params
@@ -16,10 +14,11 @@ module Conflow
     include Conflow::Redis::Identifier
 
     has_many :successors, Conflow::Job
+    has_many :promises,   Conflow::Promise
     field :params,     :hash
+    field :result,     :hash
     field :class_name, :value
     field :status,     :value # 0 - pending, 1 - finished
-    field :hook,       :value
 
     # Returns instance of Job. It sets status to 0 (pending) for new jobs
     def initialize(*)
@@ -33,5 +32,17 @@ module Conflow
     def worker_type
       Object.const_get(class_name.to_s)
     end
+
+    # Returns promise of this job's result. It assumes result of the job will be a Hash.
+    # @note Passing a {Promise} as a job parameter automatically sets the job
+    #   which produces the result as dependency of the new job
+    # @return [Conflow::Future] future object (basis of {Promise})
+    # @example Running job which depends on result of another
+    #   job = run MyJob, params: { key: 400 }
+    #   run OtherJob, params: { value: job.outcome[:result] }
+    #   # now OtherJob will depend on MyJob and it will use it's :result result as it's own :value parameter
+    def outcome
+      Future.new(self)
+    end
   end
 end

data/lib/conflow/promise.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Conflow
+  # Promises are stubs for returned values from jobs. They will be resolved once the job is done.
+  class Promise < Conflow::Redis::Field
+    include Conflow::Redis::Model
+    include Conflow::Redis::Identifier
+
+    # @!attribute [rw] job_id
+    #   ID of the job that promised result
+    #   @return [Conflow::Redis::ValueField] ID of {Conflow::Job}
+    field :job_id,     :raw_value
+    # @!attribute [rw] key
+    #   Key of job's result which is promised
+    #   @return [Conflow::Redis::ValueField] Name of the key
+    field :result_key, :raw_value
+    # @!attribute [rw] hash_field
+    #   Redis field name in the result hash
+    #   @return [Conflow::Redis::ValueField] Name of the field
+    field :hash_field, :raw_value
+  end
+end

data/lib/conflow/redis/model.rb

@@ -28,6 +28,13 @@ module Conflow
         end
       end
 
+      # Convienience method for assigning multiple fields in model
+      def assign_attributes(**attributes)
+        attributes.each do |attribute, value|
+          send("#{attribute}=", value)
+        end
+      end
+
       private
 
       def collect_relation_keys(relation)
@@ -42,7 +49,8 @@ module Conflow
           array:      Conflow::Redis::ArrayField,
           value:      Conflow::Redis::ValueField,
           sorted_set: Conflow::Redis::SortedSetField,
-          set:        Conflow::Redis::SetField
+          set:        Conflow::Redis::SetField,
+          raw_value:  Conflow::Redis::RawValueField
         }.freeze
 
         # Copies field and relations information

data/lib/conflow/redis/raw_value_field.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Conflow
+  module Redis
+    # Represents single value (Redis String). Values are not serialized
+    # @api private
+    class RawValueField < Field
+      # @param value [Object] new value to be saved
+      # @return [String] Redis response
+      def overwrite(value)
+        command :set, [key, value]
+      end
+
+      # @return [String, nil] String representation of value
+      def to_s
+        value&.to_s
+      end; alias to_str to_s
+
+      # @param other [Object] Object to compare value with. Handles Strings, Symbols and other RawValueFields
+      # @return [Boolean] true if equal
+      def ==(other)
+        case other
+        when String, Symbol then value == other.to_s
+        when RawValueField  then key == other.key || to_s == other.to_s
+        else super
+        end
+      end
+
+      # @return [Object] Value present in Redis
+      def value
+        command(:get, [key])
+      end
+    end
+  end
+end

data/lib/conflow/redis/resolve_promises_script.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Conflow
+  module Redis
+    # Resolves job's promises
+    # @api private
+    class ResolvePromisesScript < Script
+      self.script = <<~LUA
+        local pairs = {}
+
+        for i=1, #ARGV, 3 do
+          local result_field = redis.call('get', ARGV[i])
+          local field_name = redis.call('get', ARGV[i + 1])
+          local result_hash_key = ARGV[i + 2]
+
+          table.insert(pairs, field_name)
+          table.insert(pairs, redis.call('hget', result_hash_key, result_field) or '""')
+        end
+
+        return redis.call('hmset', KEYS[1], unpack(pairs))
+      LUA
+
+      class << self
+        # @param job [Conflow::Job] Job which needs promises to be resolved
+        def call(_flow, job)
+          promises = extract_keys(job.promises)
+          super([job.params.key], promises) if promises.any?
+        end
+
+        private
+
+        def extract_keys(promises)
+          promises.flat_map do |promise|
+            [promise.result_key.key, promise.hash_field.key, Conflow::Job.new(promise.job_id.value).result.key]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/conflow/redis/value_field.rb

@@ -19,23 +19,16 @@ module Conflow
         command :set, [key, JSON.dump(value), nx: true]
       end
 
-      # @param other [Object] Object to compare value with. Handles Strings, Numerics,
-      #   Symbols and other {ValueField} objects
+      # @param other [Object] Object to compare value with. Handles Strings, Numerics, and other {ValueField} objects
       # @return [Boolean] true if equal
       def ==(other)
         case other
         when String, Numeric then value == other
-        when Symbol          then value.to_sym == other
         when ValueField      then key == other.key || to_s == other.to_s
         else super
         end
       end
 
-      # @return [Boolean] true if object does not exist in Redis, else otherwise
-      def nil?
-        value.nil?
-      end
-
       # @return [String, nil] String representation of value
       def to_s
         value&.to_s

data/lib/conflow/version.rb

@@ -2,5 +2,5 @@
 
 module Conflow
   # Current version of the gem
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/conflow/worker.rb

@@ -26,6 +26,8 @@ module Conflow
       job = Conflow::Job.new(job_id)
       flow = Conflow::Flow.find(flow_id)
 
+      flow.start(job)
+
       yield(job.worker_type, job.params).tap { |result| flow.finish(job, result) }
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: conflow
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Michał Begejowicz
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-15 00:00:00.000000000 Z
+date: 2018-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -145,9 +145,13 @@ files:
 - bin/setup
 - conflow.gemspec
 - lib/conflow.rb
+- lib/conflow/error.rb
 - lib/conflow/flow.rb
+- lib/conflow/flow/job_builder.rb
 - lib/conflow/flow/job_handler.rb
+- lib/conflow/future.rb
 - lib/conflow/job.rb
+- lib/conflow/promise.rb
 - lib/conflow/redis.rb
 - lib/conflow/redis/add_job_script.rb
 - lib/conflow/redis/array_field.rb
@@ -160,6 +164,8 @@ files:
 - lib/conflow/redis/identifier.rb
 - lib/conflow/redis/model.rb
 - lib/conflow/redis/queue_jobs_script.rb
+- lib/conflow/redis/raw_value_field.rb
+- lib/conflow/redis/resolve_promises_script.rb
 - lib/conflow/redis/script.rb
 - lib/conflow/redis/set_field.rb
 - lib/conflow/redis/sorted_set_field.rb