gush 1.1.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9cfd59c5cadd225e30c41bcd7a5293893d78041056409c2e1fd15316d2cc4d75
-  data.tar.gz: ef3e28f72d5ed90bb175f9452e189111fbc5bb95facc9585df1c660fd691038f
+  metadata.gz: a9e61295e87f5c707a6f93a3cce7238c0650876be7ff76c4cba0c60133832217
+  data.tar.gz: 7df67c0134bc1e7bd036da850287191463557a764279790072f3f0a3887bf44b
 SHA512:
-  metadata.gz: 38578ba5a3f159577d38d6af662137fc5c9b95a394c01f498b22d6ff2931fc3e78eb9634f9c0ae1da683d197412f0ee38c43efa6035d223f1ece1d09b483cb9d
-  data.tar.gz: bf8954ea6d409a1fc9db19e24c4dae322f15cf2d3e9240df17d2ff847503318fbdcf0f50b27c08910273fb6dfc3223919f79679d1986670b0483b2b27f627384
+  metadata.gz: 133913d52aaef1d470ca8be41c865c26949577df06935b034e83bb0ee616f5cd812d0c49730bbfc504342a46a6f0c6b813df19705f0a6c4448ba18969172e418
+  data.tar.gz: b08039f50aba1ab24fbe957669f16d51258913353ffba98b168a9df74782b46459c79f083c7e54b1dfa717a5878e85bb72ba07397993f833e83f9733ec5089dd

data/CHANGELOG.md

@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## 2.0.0
+
+## Changed
+
+- *[BREAKING]* Store gush jobs on redis hash instead of plain keys - this improves performance when retrieving keys (Thanks to @Saicheg! [See pull request](https://github.com/chaps-io/gush/pull/56))
+
+
+## Added
+
+- Allow setting queue for each job via `:queue` option in `run` method (Thanks to @devilankur18! [See pull request](https://github.com/chaps-io/gush/pull/58))
+
+
 ## 1.1.1 - 2018-06-09
 
 ## Changed

data/gush.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "gush"
-  spec.version       = "1.1.1"
+  spec.version       = "2.0.0"
   spec.authors       = ["Piotrek Okoński"]
   spec.email         = ["piotrek@okonski.org"]
   spec.summary       = "Fast and distributed workflow runner based on ActiveJob and Redis"
@@ -21,6 +21,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "connection_pool", "~> 2.2.1"
   spec.add_dependency "multi_json", "~> 1.11"
   spec.add_dependency "redis", ">= 3.2", "< 5"
+  spec.add_dependency "redis-mutex", "~> 4.0.1"
   spec.add_dependency "hiredis", "~> 0.6"
   spec.add_dependency "ruby-graphviz", "~> 1.2"
   spec.add_dependency "terminal-table", "~> 1.4"

data/lib/gush/client.rb

@@ -42,19 +42,19 @@ module Gush
       persist_workflow(workflow)
     end
 
-    def next_free_job_id(workflow_id,job_klass)
-      job_identifier = nil
+    def next_free_job_id(workflow_id, job_klass)
+      job_id = nil
+
       loop do
-        id = SecureRandom.uuid
-        job_identifier = "#{job_klass}-#{id}"
+        job_id = SecureRandom.uuid
         available = connection_pool.with do |redis|
-          !redis.exists("gush.jobs.#{workflow_id}.#{job_identifier}")
+          !redis.hexists("gush.jobs.#{workflow_id}.#{job_klass}", job_id)
         end
 
         break if available
       end
 
-      job_identifier
+      job_id
     end
 
     def next_free_workflow_id
@@ -87,7 +87,11 @@ module Gush
         unless data.nil?
           hash = Gush::JSON.decode(data, symbolize_keys: true)
           keys = redis.scan_each(match: "gush.jobs.#{id}.*")
-          nodes = redis.mget(*keys).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
+
+          nodes = keys.each_with_object([]) do |key, array|
+            array.concat redis.hvals(key).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
+          end
+
           workflow_from_hash(hash, nodes)
         else
           raise WorkflowNotFound.new("Workflow with given id doesn't exist")
@@ -102,28 +106,24 @@ module Gush
 
       workflow.jobs.each {|job| persist_job(workflow.id, job) }
       workflow.mark_as_persisted
+
       true
     end
 
     def persist_job(workflow_id, job)
       connection_pool.with do |redis|
-        redis.set("gush.jobs.#{workflow_id}.#{job.name}", job.to_json)
+        redis.hset("gush.jobs.#{workflow_id}.#{job.klass}", job.id, job.to_json)
       end
     end
 
-    def find_job(workflow_id, job_id)
-      job_name_match = /(?<klass>\w*[^-])-(?<identifier>.*)/.match(job_id)
-      hypen = '-' if job_name_match.nil?
-
-      keys = connection_pool.with do |redis|
-        redis.scan_each(match: "gush.jobs.#{workflow_id}.#{job_id}#{hypen}*").to_a
-      end
-
-      return nil if keys.nil?
+    def find_job(workflow_id, job_name)
+      job_name_match = /(?<klass>\w*[^-])-(?<identifier>.*)/.match(job_name)
 
-      data = connection_pool.with do |redis|
-        redis.get(keys.first)
-      end
+      data = if job_name_match
+               find_job_by_klass_and_id(workflow_id, job_name)
+             else
+               find_job_by_klass(workflow_id, job_name)
+             end
 
       return nil if data.nil?
 
@@ -140,7 +140,7 @@ module Gush
 
     def destroy_job(workflow_id, job)
       connection_pool.with do |redis|
-        redis.del("gush.jobs.#{workflow_id}.#{job.name}")
+        redis.del("gush.jobs.#{workflow_id}.#{job.klass}")
       end
     end
 
@@ -162,12 +162,33 @@ module Gush
     def enqueue_job(workflow_id, job)
       job.enqueue!
       persist_job(workflow_id, job)
+      queue = job.queue || configuration.namespace
 
-      Gush::Worker.set(queue: configuration.namespace).perform_later(*[workflow_id, job.name])
+      Gush::Worker.set(queue: queue).perform_later(*[workflow_id, job.name])
     end
 
     private
 
+    def find_job_by_klass_and_id(workflow_id, job_name)
+      job_klass, job_id = job_name.split('|')
+
+      connection_pool.with do |redis|
+        redis.hget("gush.jobs.#{workflow_id}.#{job_klass}", job_id)
+      end
+    end
+
+    def find_job_by_klass(workflow_id, job_name)
+      new_cursor, result = connection_pool.with do |redis|
+        redis.hscan("gush.jobs.#{workflow_id}.#{job_name}", 0, count: 1)
+      end
+
+      return nil if result.empty?
+
+      job_id, job = *result[0]
+
+      job
+    end
+
     def workflow_from_hash(hash, nodes = [])
       flow = hash[:klass].constantize.new(*hash[:arguments])
       flow.jobs = []
@@ -182,7 +203,9 @@ module Gush
     end
 
     def build_redis
-      Redis.new(url: configuration.redis_url)
+      Redis.new(url: configuration.redis_url).tap do |instance|
+        RedisClassy.redis = instance
+      end
     end
 
     def connection_pool

data/lib/gush/job.rb

@@ -1,8 +1,8 @@
 module Gush
   class Job
     attr_accessor :workflow_id, :incoming, :outgoing, :params,
-      :finished_at, :failed_at, :started_at, :enqueued_at, :payloads, :klass
-    attr_reader :name, :output_payload, :params
+      :finished_at, :failed_at, :started_at, :enqueued_at, :payloads, :klass, :queue
+    attr_reader :id, :klass, :output_payload, :params
 
     def initialize(opts = {})
       options = opts.dup
@@ -11,8 +11,9 @@ module Gush
 
     def as_json
       {
-        name: name,
-        klass: self.class.to_s,
+        id: id,
+        klass: klass.to_s,
+        queue: queue,
         incoming: incoming,
         outgoing: outgoing,
         finished_at: finished_at,
@@ -25,6 +26,10 @@ module Gush
       }
     end
 
+    def name
+      @name ||= "#{klass}|#{id}"
+    end
+
     def to_json(options = {})
       Gush::JSON.encode(as_json)
     end
@@ -108,7 +113,7 @@ module Gush
     end
 
     def assign_variables(opts)
-      @name           = opts[:name]
+      @id             = opts[:id]
       @incoming       = opts[:incoming] || []
       @outgoing       = opts[:outgoing] || []
       @failed_at      = opts[:failed_at]
@@ -116,9 +121,10 @@ module Gush
       @started_at     = opts[:started_at]
       @enqueued_at    = opts[:enqueued_at]
       @params         = opts[:params] || {}
-      @klass          = opts[:klass]
+      @klass          = opts[:klass] || self.class
       @output_payload = opts[:output_payload]
       @workflow_id    = opts[:workflow_id]
+      @queue          = opts[:queue]
     end
   end
 end

data/lib/gush/worker.rb

@@ -1,4 +1,5 @@
 require 'active_job'
+require 'redis-mutex'
 
 module Gush
   class Worker < ::ActiveJob::Base
@@ -66,9 +67,12 @@ module Gush
 
     def enqueue_outgoing_jobs
       job.outgoing.each do |job_name|
-        out = client.find_job(workflow_id, job_name)
-        if out.ready_to_start?
-          client.enqueue_job(workflow_id, out)
+        RedisMutex.with_lock("gush_enqueue_outgoing_jobs_#{workflow_id}-#{job_name}", sleep: 0.3, block: 2) do
+          out = client.find_job(workflow_id, job_name)
+
+          if out.ready_to_start?
+            client.enqueue_job(workflow_id, out)
+          end
         end
       end
     end

data/lib/gush/workflow.rb

@@ -77,11 +77,13 @@ module Gush
 
     def find_job(name)
       match_data = /(?<klass>\w*[^-])-(?<identifier>.*)/.match(name.to_s)
+
       if match_data.nil?
-        job = jobs.find { |node| node.class.to_s == name.to_s }
+        job = jobs.find { |node| node.klass.to_s == name.to_s }
       else
         job = jobs.find { |node| node.name.to_s == name.to_s }
       end
+
       job
     end
 
@@ -108,18 +110,21 @@ module Gush
     def run(klass, opts = {})
       node = klass.new({
         workflow_id: id,
-        name: client.next_free_job_id(id, klass.to_s),
-        params: opts.fetch(:params, {})
+        id: client.next_free_job_id(id, klass.to_s),
+        params: opts.fetch(:params, {}),
+        queue: opts[:queue]
       })
 
       jobs << node
 
       deps_after = [*opts[:after]]
+
       deps_after.each do |dep|
         @dependencies << {from: dep.to_s, to: node.name.to_s }
       end
 
       deps_before = [*opts[:before]]
+
       deps_before.each do |dep|
         @dependencies << {from: node.name.to_s, to: dep.to_s }
       end

data/spec/features/integration_spec.rb

@@ -158,4 +158,45 @@ describe "Workflows" do
     expect(flow).to be_finished
     expect(flow).to_not be_failed
   end
+
+  it 'executes job with multiple ancestors only once' do
+    NO_DUPS_INTERNAL_SPY = double('spy')
+    expect(NO_DUPS_INTERNAL_SPY).to receive(:some_method).exactly(1).times
+
+    class FirstAncestor < Gush::Job
+      def perform
+      end
+    end
+
+    class SecondAncestor < Gush::Job
+      def perform
+      end
+    end
+
+    class FinalJob < Gush::Job
+      def perform
+        NO_DUPS_INTERNAL_SPY.some_method
+      end
+    end
+
+    class NoDuplicatesWorkflow < Gush::Workflow
+      def configure
+        run FirstAncestor
+        run SecondAncestor
+
+        run FinalJob, after: [FirstAncestor, SecondAncestor]
+      end
+    end
+
+    flow = NoDuplicatesWorkflow.create
+    flow.start!
+
+    5.times do
+      perform_one
+    end
+
+    flow = flow.reload
+    expect(flow).to be_finished
+    expect(flow).to_not be_failed
+  end
 end

data/spec/gush/job_spec.rb

@@ -62,9 +62,9 @@ describe Gush::Job do
   describe "#as_json" do
     context "finished and enqueued set to true" do
       it "returns correct hash" do
-        job = described_class.new(workflow_id: 123, name: "a-job", finished_at: 123, enqueued_at: 120)
+        job = described_class.new(workflow_id: 123, id: "702bced5-bb72-4bba-8f6f-15a3afa358bd", finished_at: 123, enqueued_at: 120)
         expected = {
-          name: "a-job",
+          id: '702bced5-bb72-4bba-8f6f-15a3afa358bd',
           klass: "Gush::Job",
           incoming: [],
           outgoing: [],
@@ -73,6 +73,7 @@ describe Gush::Job do
           finished_at: 123,
           enqueued_at: 120,
           params: {},
+          queue: nil,
           output_payload: nil,
           workflow_id: 123
         }
@@ -86,7 +87,7 @@ describe Gush::Job do
       job = described_class.from_hash(
         {
           klass: 'Gush::Job',
-          name: 'gob',
+          id: '702bced5-bb72-4bba-8f6f-15a3afa358bd',
           incoming: ['a', 'b'],
           outgoing: ['c'],
           failed_at: 123,
@@ -96,7 +97,8 @@ describe Gush::Job do
         }
       )
 
-      expect(job.name).to eq('gob')
+      expect(job.id).to eq('702bced5-bb72-4bba-8f6f-15a3afa358bd')
+      expect(job.name).to eq('Gush::Job|702bced5-bb72-4bba-8f6f-15a3afa358bd')
       expect(job.class).to eq(Gush::Job)
       expect(job.klass).to eq("Gush::Job")
       expect(job.finished?).to eq(true)

data/spec/gush/workflow_spec.rb

@@ -135,6 +135,7 @@ describe Gush::Workflow do
       klass1 = Class.new(Gush::Job)
       klass2 = Class.new(Gush::Job)
       klass3 = Class.new(Gush::Job)
+
       tree.run(klass1)
       tree.run(klass2, after: [klass1, klass3])
       tree.run(klass3)

data/spec/spec_helper.rb

@@ -26,7 +26,6 @@ class TestWorkflow < Gush::Workflow
     run FetchSecondJob,  after: Prepare, before: NormalizeJob
 
     run PersistFirstJob, after: FetchFirstJob, before: NormalizeJob
-
   end
 end
 
@@ -62,7 +61,7 @@ module GushHelpers
   end
 
   def job_with_id(job_name)
-    /#{job_name}-(?<identifier>.*)/
+    /#{job_name}|(?<identifier>.*)/
   end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gush
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 2.0.0
 platform: ruby
 authors:
 - Piotrek Okoński
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-09 00:00:00.000000000 Z
+date: 2018-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -78,6 +78,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '5'
+- !ruby/object:Gem::Dependency
+  name: redis-mutex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.0.1
 - !ruby/object:Gem::Dependency
   name: hiredis
   requirement: !ruby/object:Gem::Requirement