conflow 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 736a50603837a0bd935229a07df384630c29f16d5d796e9d91ab9dc97c163d98
-  data.tar.gz: 579a36c20e8b42a68657eafc9632725a3108bf5b9eeaab70f391f43629dedb01
+  metadata.gz: b85fd52b7c1a2161135fc389c45b3f1e6754462a17ef30463435f1c919685950
+  data.tar.gz: a5f05aa2c69b8426a72dc1070834ec74120a5053a74f93250098765c461105ef
 SHA512:
-  metadata.gz: f850accd201ad219d809d45b874440d4c9aeff54400289bfcb7443ea9281eb240c52402a7ef38d1afaef56b7255776d763f011f3e06642f36c1b60cecbeec023
-  data.tar.gz: 284ce027afa093cb9fb6e6f8389e9bf71cd966a801dc8c3407f32ad94b0596bcca7f864b78f7e40760b6afa28f889e8ce636719a9dce6d232045ce219dcf7f10
+  metadata.gz: 3291b765c557576651568d735b3bad21e7a191919a858e9365bbf7da97d632d09f99f3c7a2f35d3133b03d3e82a9802489dd1b63542bd1343325f89c74c22845
+  data.tar.gz: a6797479cf68b310791680b2709ed54ebb9e372e26e87a1ffb2e685418a33eefa1b621bf5f68aa6ec05024c54d58f427123335ba204049a3fd7bfba4aa6d026e

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.0
+
+- Flow and Jobs are now removed after they are finished
+
 ## 0.1.0
 
 - First release! :pizza:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    conflow (0.1.0)
+    conflow (0.2.0)
       redis (~> 4.0)
 
 GEM

data/lib/conflow.rb

@@ -12,6 +12,7 @@ require "conflow/redis/value_field"
 require "conflow/redis/hash_field"
 require "conflow/redis/array_field"
 require "conflow/redis/sorted_set_field"
+require "conflow/redis/set_field"
 require "conflow/redis/field_builder"
 require "conflow/redis/model"
 require "conflow/redis/identifier"
@@ -20,6 +21,7 @@ require "conflow/redis/findable"
 require "conflow/redis/script"
 require "conflow/redis/add_job_script"
 require "conflow/redis/complete_job_script"
+require "conflow/redis/queue_jobs_script"
 
 require "conflow/job"
 require "conflow/flow/job_handler"

data/lib/conflow/flow.rb

@@ -16,6 +16,10 @@ module Conflow
   #   are fulfilled.
   #   @return [Conflow::Redis::SortedSetField] Set of jobs to be performed
   #
+  # @!attribute [r] queued_jobs
+  #   Set of jobs that are currently queued (and not yet finished).
+  #   @return [Conflow::Redis::SetField] Set of queued jobs
+  #
   # @!method queue(job)
   #   @abstract
   #   Queues job to be performed. Both id of the flow and id of the job must be preserved
@@ -35,7 +39,8 @@ module Conflow
     include JobHandler
 
     has_many :jobs, Conflow::Job
-    field :indegree, :sorted_set
+    field :queued_jobs, :set
+    field :indegree,    :sorted_set
 
     # Create new flow with given parameters
     # @example Simple configurable flow
@@ -52,6 +57,12 @@ module Conflow
       new.tap { |flow| flow.configure(*args) }
     end
 
+    # Returns whether or not the flow is finished (all jobs were processed)
+    # @return [Boolean] true if no pending jobs
+    def finished?
+      queued_jobs.size.zero? && indegree.size.zero?
+    end
+
     # @abstract
     # Override this method in order to contain your flow definition inside the class.
     # This method will be called if flow is created using {.create} method.

data/lib/conflow/flow/job_handler.rb

@@ -18,18 +18,21 @@ module Conflow
           after = prepare_dependencies(after)
 
           call_script(Conflow::Redis::AddJobScript, job, after: after)
+          queue_available_jobs
         end
       end
 
       def finish(job, result = nil)
         send(job.hook.to_s, result) unless job.hook.nil?
         call_script(Conflow::Redis::CompleteJobScript, job)
+        queue_available_jobs
+        destroy! if finished?
       end
 
       private
 
       def queue_available_jobs
-        indegree.delete_if(score: 0).each do |job_id|
+        call_script(Conflow::Redis::QueueJobsScript)&.each do |job_id|
           queue Conflow::Job.new(job_id)
         end
       end
@@ -44,7 +47,6 @@ module Conflow
 
       def call_script(script, *args)
         script.call(self, *args)
-        queue_available_jobs
       end
 
       def prepare_dependencies(dependencies)

data/lib/conflow/redis/complete_job_script.rb

@@ -6,15 +6,18 @@ module Conflow
     class CompleteJobScript < Script
       self.script = <<~LUA
         local indegree_set = KEYS[1]
-        local job_id = KEYS[2]
+        local queued_jobs = KEYS[2]
+        local job_key = KEYS[3]
+        local job_id = ARGV[1]
 
-        local successors = redis.call('lrange', job_id .. ':successor_ids', 0, -1)
+        local successors = redis.call('lrange', job_key .. ':successor_ids', 0, -1)
 
         for i=1,#successors do
           redis.call('zincrby', indegree_set, -1, successors[i])
         end
 
-        return redis.call('set', job_id .. ':status', '1')
+        redis.call('srem', queued_jobs, job_id)
+        return redis.call('set', job_key .. ':status', '1')
       LUA
 
       class << self
@@ -24,7 +27,7 @@ module Conflow
         # @param flow [Conflow::Flow] Flow to which job belongs to
         # @param job [Conflow::Job] Job to be marked as completed
         def call(flow, job)
-          super([flow.indegree.key, job.key])
+          super([flow.indegree.key, flow.queued_jobs.key, job.key], [job.id])
         end
       end
     end

data/lib/conflow/redis/field.rb

@@ -20,27 +20,6 @@ module Conflow
       def command(command, args)
         redis.with { |conn| conn.send(command, *args) }
       end
-
-      def transaction(max_retries: 5)
-        result = redis.with do |conn|
-          retryable(max_retries, conn) do
-            conn.watch(key) do
-              conn.multi { |multi| yield(multi) }
-            end
-          end
-        end
-
-        raise ::Redis::CommandError, "Transaction failed #{max_retries} times" unless result
-        result
-      end
-
-      def retryable(max_retries, *args)
-        loop do
-          result = yield(*args)
-          break result if result || max_retries.zero?
-          max_retries -= 1
-        end
-      end
     end
   end
 end

data/lib/conflow/redis/model.rb

@@ -16,16 +16,49 @@ module Conflow
         other.is_a?(Model) ? key == other.key : super
       end
 
+      # Removes this and all related models.
+      # @param execute [Boolean] if false, only returns keys that would be removed
+      def destroy!(execute: true)
+        keys = self.class.fields.map { |name| send(name).key }
+        related_keys = self.class.relations.map(&method(:collect_relation_keys))
+
+        (keys + related_keys).flatten.tap do |key_list|
+          command :del, [key_list] if execute && key_list.any?
+        end
+      end
+
+      private
+
+      def collect_relation_keys(relation)
+        send(relation).map { |object| object.destroy!(execute: false) }
+      end
+
       # Methods for defining fields on model
       module ClassMethods
+        # Maps types (option for {Conflow::Redis::Model::ClassMethods#field}) to specific type field
+        ALLOWED_TYPES = {
+          hash:       Conflow::Redis::HashField,
+          array:      Conflow::Redis::ArrayField,
+          value:      Conflow::Redis::ValueField,
+          sorted_set: Conflow::Redis::SortedSetField,
+          set:        Conflow::Redis::SetField
+        }.freeze
+
+        # Copies field and relations information
+        def inherited(subclass)
+          subclass.instance_variable_set("@fields", fields.dup)
+          subclass.instance_variable_set("@relations", relations.dup)
+        end
+
         # Defines Redis field accessors.
         # @param name [Symbol] name of the field
-        # @param type [:hash, :array, :value, :sorted_set] type of the new field
+        # @param type [:hash, :array, :value, :sorted_set, :set] type of the new field
         #
         # @see Conflow::Redis::HashField
         # @see Conflow::Redis::ArrayField
         # @see Conflow::Redis::ValueField
         # @see Conflow::Redis::SortedSetField
+        # @see Conflow::Redis::SetField
         # @example
         #   model_class.field :data, :hash
         #   instance = model_class.new
@@ -33,13 +66,11 @@ module Conflow
         #   instance.hash = { something: "else"}
         #   instance.hash["something"] #=> "else"
         def field(name, type)
-          case type
-          when :hash       then FieldBuilder.new(name, Conflow::Redis::HashField).call(self)
-          when :array      then FieldBuilder.new(name, Conflow::Redis::ArrayField).call(self)
-          when :value      then FieldBuilder.new(name, Conflow::Redis::ValueField).call(self)
-          when :sorted_set then FieldBuilder.new(name, Conflow::Redis::SortedSetField).call(self)
-          else raise ArgumentError, "Unknown type: #{type}. Should be one of: [:hash, :array]"
-          end
+          type_class = ALLOWED_TYPES[type]
+          raise ArgumentError, "Unknown type: #{type}. Should be one of: #{ALLOWED_TYPES.keys.inspect}" unless type_class
+
+          fields << name
+          FieldBuilder.new(name, type_class).call(self)
         end
 
         # Convienience method for defining relation-like accessor.
@@ -47,8 +78,19 @@ module Conflow
         #   has_many :jobs, Conflow::Job # defines #job_ids and #jobs
         def has_many(name, klass, field_name: "#{name.to_s.chop}_ids")
           field(field_name, :array)
+          relations << name
           define_method(name) { send(field_name).map { |id| klass.new(id) } }
         end
+
+        # @return [Array<Symbol>] fields defined on this class
+        def fields
+          @fields ||= []
+        end
+
+        # @return [Array<Symbol>] relations defined on this class
+        def relations
+          @relations ||= []
+        end
       end
     end
   end

data/lib/conflow/redis/queue_jobs_script.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Conflow
+  module Redis
+    # Removes jobs from indegree set, adds them to queued list, and returns IDs of queued jobs.
+    class QueueJobsScript < Script
+      self.script = <<~LUA
+        local indegree_set = KEYS[1]
+        local queued_set = KEYS[2]
+
+        local ids = redis.call('zrangebyscore', indegree_set, 0, 0)
+        redis.call('zremrangebyscore', indegree_set, 0, 0)
+
+        if #ids ~= 0 then
+          redis.call('sadd', queued_set, unpack(ids))
+        end
+
+        return ids
+      LUA
+
+      class << self
+        # Call the script.
+        # Script removes jobs which have score 0 in Flow's indegree set and moves them to queued_jobs list.
+        # @param flow [Conflow::Flow] Flow from which jobs should be enqueued
+        def call(flow)
+          super([flow.indegree.key, flow.queued_jobs.key])
+        end
+      end
+    end
+  end
+end

data/lib/conflow/redis/set_field.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Conflow
+  module Redis
+    # Represents Redis list. It's methods mirror most used Set methods.
+    class SetField < Field
+      include Enumerable
+
+      # Adds one or more elements to the set.
+      # @param values [String] list of values to be added
+      # @return [String] Redis response
+      #
+      # @example Adding multiple fields
+      #   field.add(:last, :first, :other, :first)
+      def add(*values)
+        command :sadd, [key, values]
+      end
+
+      # @return [Integer] Number of elements in the set
+      def size
+        command :scard, [key]
+      end
+
+      # Removes old values from the set and overrides them with new.
+      # @param enumerable [Enumerable] new values of the set
+      # @return [String] Redis response
+      def overwrite(enumerable)
+        redis.with do |conn|
+          conn.pipelined do
+            conn.del(key)
+            conn.sadd(key, enumerable)
+          end
+        end
+      end
+
+      # Returns array with set values
+      # @return [Array] values from the set
+      def to_a
+        command :smembers, [key]
+      end; alias to_ary to_a
+    end
+  end
+end

data/lib/conflow/redis/sorted_set_field.rb

@@ -55,36 +55,6 @@ module Conflow
         command :zrem, [key, value]
       end
 
-      # Return elements with given score
-      # @param score [Numeric, Hash]
-      #   - when Numeric, only elements with that exact score will be returned
-      #   - when Hash, elements within min..max range will be returned. See {https://redis.io/commands/zrange Redis docs}
-      # @option score [String, Numeric] :min minimal score
-      # @option score [String, Numeric] :max maximal score
-      # @return [Array<String>] Elements with given score
-      #
-      # @example with specific score
-      #   field.where(score: 2) #=> ["first", "tie"]
-      # @example with only min set
-      #   field.where(score: { min: 3 }) #=> ["last", "second"]
-      # @example with both min and max set
-      #   field.where(score: { min: 3, max: "(10" }) #=> ["last"]
-      def where(score:)
-        command :zrangebyscore, [key, *prepare_score_bounds(score)]
-      end
-
-      # Removes elements of the set with given score and returns them.
-      # See {where} for details on how to choose score.
-      # @return [Array<String>] Elements with given score
-      def delete_if(score:)
-        score_bounds = prepare_score_bounds(score)
-
-        transaction do |conn|
-          conn.zrangebyscore key, *score_bounds
-          conn.zremrangebyscore key, *score_bounds
-        end[0]
-      end
-
       # Returns first *n* elements of the sorted set
       # @param num [Integer] amount of elements to be returned. Defaults to 1.
       # @return [String, Array<String>] first *num* elements from the set
@@ -129,13 +99,6 @@ module Conflow
           result[index * 2 + 1] = value
         end
       end
-
-      def prepare_score_bounds(score)
-        case score
-        when Hash    then { min: "-inf", max: "+inf" }.merge(score).values
-        when Numeric then [score, score]
-        end
-      end
     end
   end
 end

data/lib/conflow/version.rb

@@ -2,5 +2,5 @@
 
 module Conflow
   # Current version of the gem
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: conflow
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Michał Begejowicz
@@ -158,7 +158,9 @@ files:
 - lib/conflow/redis/hash_field.rb
 - lib/conflow/redis/identifier.rb
 - lib/conflow/redis/model.rb
+- lib/conflow/redis/queue_jobs_script.rb
 - lib/conflow/redis/script.rb
+- lib/conflow/redis/set_field.rb
 - lib/conflow/redis/sorted_set_field.rb
 - lib/conflow/redis/value_field.rb
 - lib/conflow/version.rb