gush 2.0.1 → 2.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ffcdce12d2f530e27afdd048328d8942e60efcddfbe20a18426b8d245589c759
-  data.tar.gz: 3c95194d44f446a694af83b06b06eac55a3e5ed1f8555fe86958fa110b357647
+  metadata.gz: 12fdb9a62d33353f827194c198c011ff42d7491669d0ab9717be0920a8066313
+  data.tar.gz: 7f3d4ada23215e818f0cb801c92f4752e80b21b344f3b48d9ab4b83787c029ec
 SHA512:
-  metadata.gz: e75e7f0ae5c5c83302d0507bb502a30a860f276e0706edd5a4119bb9c50bf88c73c9839ac1f141f4569017301ebb61fe943317d214c0c9a10f49309e674bd183
-  data.tar.gz: 38f3baaa43a4d512d5d5ec6c476d21965d27116e7dc8bc1ae3cc3fa4a4fac13e6d18272ed236fd413aba0c304f0e79aee59f3c06cdad19db7fa02e41305fe491
+  metadata.gz: bec4bcb3e251bdb1a2e184b6b62ae896fd40af4a0cd4bad2ec7ee4925ab356e36a3388d2cb78ed21582c0f6cfdd429c304f0591a5ee3760c73255d03a2b1d80e
+  data.tar.gz: 67fba0b65b575449114233a4ff9cfaf1cb6ac3ff61633f94b7c595988165b3315fe7ae5d0744792315d9329795da0e42ae840789e3d4f72c84ef8956081c5fbd

data/.gitignore

@@ -19,3 +19,5 @@ tmp
 test.rb
 /Gushfile
 dump.rdb
+.ruby-version
+.ruby-gemset

data/.travis.yml

@@ -4,6 +4,9 @@ rvm:
   - 2.2.2
   - 2.3.4
   - 2.4.1
+  - 2.5
+  - 2.6
+  - 2.7
 services:
   - redis-server
 email:

data/README.md

@@ -1,7 +1,5 @@
 # Gush [![Build Status](https://travis-ci.org/chaps-io/gush.svg?branch=master)](https://travis-ci.org/chaps-io/gush)
 
-## [![](http://i.imgur.com/ya8Wnyl.png)](https://chaps.io) proudly made by [Chaps](https://chaps.io)
-
 Gush is a parallel workflow runner using only Redis as storage and [ActiveJob](http://guides.rubyonrails.org/v4.2/active_job_basics.html#introduction) for scheduling and executing jobs.
 
 ## Theory

data/gush.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "gush"
-  spec.version       = "2.0.1"
+  spec.version       = "2.0.2"
   spec.authors       = ["Piotrek Okoński"]
   spec.email         = ["piotrek@okonski.org"]
   spec.summary       = "Fast and distributed workflow runner based on ActiveJob and Redis"
@@ -17,8 +17,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "activejob", ">= 4.2.7", "< 6.0"
-  spec.add_dependency "connection_pool", "~> 2.2.1"
+  spec.add_dependency "activejob", ">= 4.2.7", "< 7.0"
+  spec.add_dependency "concurrent-ruby", "~> 1.0"
   spec.add_dependency "multi_json", "~> 1.11"
   spec.add_dependency "redis", ">= 3.2", "< 5"
   spec.add_dependency "redis-mutex", "~> 4.0.1"
@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "colorize", "~> 0.7"
   spec.add_dependency "thor", "~> 0.19"
   spec.add_dependency "launchy", "~> 2.4"
-  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake", "~> 10.4"
   spec.add_development_dependency "rspec", '~> 3.0'
   spec.add_development_dependency "pry", '~> 0.10'

data/lib/gush/client.rb

@@ -1,9 +1,22 @@
-require 'connection_pool'
+require 'redis'
+require 'concurrent-ruby'
 
 module Gush
   class Client
     attr_reader :configuration
 
+    @@redis_connection = Concurrent::ThreadLocalVar.new(nil)
+
+    def self.redis_connection(config)
+      cached = (@@redis_connection.value ||= { url: config.redis_url, connection: nil })
+      return cached[:connection] if !cached[:connection].nil? && config.redis_url == cached[:url]
+
+      Redis.new(url: config.redis_url).tap do |instance|
+        RedisClassy.redis = instance
+        @@redis_connection.value = { url: config.redis_url, connection: instance }
+      end
+    end
+
     def initialize(config = Gush.configuration)
       @configuration = config
     end
@@ -47,9 +60,7 @@ module Gush
 
       loop do
         job_id = SecureRandom.uuid
-        available = connection_pool.with do |redis|
-          !redis.hexists("gush.jobs.#{workflow_id}.#{job_klass}", job_id)
-        end
+        available = !redis.hexists("gush.jobs.#{workflow_id}.#{job_klass}", job_id)
 
         break if available
       end
@@ -61,9 +72,7 @@ module Gush
       id = nil
       loop do
         id = SecureRandom.uuid
-        available = connection_pool.with do |redis|
-          !redis.exists("gush.workflow.#{id}")
-        end
+        available = !redis.exists("gush.workflow.#{id}")
 
         break if available
       end
@@ -72,37 +81,31 @@ module Gush
     end
 
     def all_workflows
-      connection_pool.with do |redis|
-        redis.scan_each(match: "gush.workflows.*").map do |key|
-          id = key.sub("gush.workflows.", "")
-          find_workflow(id)
-        end
+      redis.scan_each(match: "gush.workflows.*").map do |key|
+        id = key.sub("gush.workflows.", "")
+        find_workflow(id)
       end
     end
 
     def find_workflow(id)
-      connection_pool.with do |redis|
-        data = redis.get("gush.workflows.#{id}")
+      data = redis.get("gush.workflows.#{id}")
 
-        unless data.nil?
-          hash = Gush::JSON.decode(data, symbolize_keys: true)
-          keys = redis.scan_each(match: "gush.jobs.#{id}.*")
+      unless data.nil?
+        hash = Gush::JSON.decode(data, symbolize_keys: true)
+        keys = redis.scan_each(match: "gush.jobs.#{id}.*")
 
-          nodes = keys.each_with_object([]) do |key, array|
-            array.concat redis.hvals(key).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
-          end
-
-          workflow_from_hash(hash, nodes)
-        else
-          raise WorkflowNotFound.new("Workflow with given id doesn't exist")
+        nodes = keys.each_with_object([]) do |key, array|
+          array.concat redis.hvals(key).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
         end
+
+        workflow_from_hash(hash, nodes)
+      else
+        raise WorkflowNotFound.new("Workflow with given id doesn't exist")
       end
     end
 
     def persist_workflow(workflow)
-      connection_pool.with do |redis|
-        redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
-      end
+      redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
 
       workflow.jobs.each {|job| persist_job(workflow.id, job) }
       workflow.mark_as_persisted
@@ -111,9 +114,7 @@ module Gush
     end
 
     def persist_job(workflow_id, job)
-      connection_pool.with do |redis|
-        redis.hset("gush.jobs.#{workflow_id}.#{job.klass}", job.id, job.to_json)
-      end
+      redis.hset("gush.jobs.#{workflow_id}.#{job.klass}", job.id, job.to_json)
     end
 
     def find_job(workflow_id, job_name)
@@ -132,31 +133,23 @@ module Gush
     end
 
     def destroy_workflow(workflow)
-      connection_pool.with do |redis|
-        redis.del("gush.workflows.#{workflow.id}")
-      end
+      redis.del("gush.workflows.#{workflow.id}")
       workflow.jobs.each {|job| destroy_job(workflow.id, job) }
     end
 
     def destroy_job(workflow_id, job)
-      connection_pool.with do |redis|
-        redis.del("gush.jobs.#{workflow_id}.#{job.klass}")
-      end
+      redis.del("gush.jobs.#{workflow_id}.#{job.klass}")
     end
 
     def expire_workflow(workflow, ttl=nil)
       ttl = ttl || configuration.ttl
-      connection_pool.with do |redis|
-        redis.expire("gush.workflows.#{workflow.id}", ttl)
-      end
+      redis.expire("gush.workflows.#{workflow.id}", ttl)
       workflow.jobs.each {|job| expire_job(workflow.id, job, ttl) }
     end
 
     def expire_job(workflow_id, job, ttl=nil)
       ttl = ttl || configuration.ttl
-      connection_pool.with do |redis|
-        redis.expire("gush.jobs.#{workflow_id}.#{job.name}", ttl)
-      end
+      redis.expire("gush.jobs.#{workflow_id}.#{job.klass}", ttl)
     end
 
     def enqueue_job(workflow_id, job)
@@ -172,16 +165,11 @@ module Gush
     def find_job_by_klass_and_id(workflow_id, job_name)
       job_klass, job_id = job_name.split('|')
 
-      connection_pool.with do |redis|
-        redis.hget("gush.jobs.#{workflow_id}.#{job_klass}", job_id)
-      end
+      redis.hget("gush.jobs.#{workflow_id}.#{job_klass}", job_id)
     end
 
     def find_job_by_klass(workflow_id, job_name)
-      new_cursor, result = connection_pool.with do |redis|
-        redis.hscan("gush.jobs.#{workflow_id}.#{job_name}", 0, count: 1)
-      end
-
+      new_cursor, result = redis.hscan("gush.jobs.#{workflow_id}.#{job_name}", 0, count: 1)
       return nil if result.empty?
 
       job_id, job = *result[0]
@@ -202,14 +190,8 @@ module Gush
       flow
     end
 
-    def build_redis
-      Redis.new(url: configuration.redis_url).tap do |instance|
-        RedisClassy.redis = instance
-      end
-    end
-
-    def connection_pool
-      @connection_pool ||= ConnectionPool.new(size: configuration.concurrency, timeout: 1) { build_redis }
+    def redis
+      self.class.redis_connection(configuration)
     end
   end
 end

data/lib/gush/worker.rb

@@ -6,6 +6,12 @@ module Gush
     def perform(workflow_id, job_id)
       setup_job(workflow_id, job_id)
 
+      if job.succeeded?
+        # Try to enqueue outgoing jobs again because the last job has redis mutex lock error
+        enqueue_outgoing_jobs
+        return
+      end
+
       job.payloads = incoming_payloads
 
       error = nil
@@ -75,6 +81,8 @@ module Gush
           end
         end
       end
+    rescue RedisMutex::LockError
+      Worker.set(wait: 2.seconds).perform_later(workflow_id, job.name)
     end
   end
 end

data/spec/gush/client_spec.rb

@@ -95,12 +95,18 @@ describe Gush::Client do
   end
 
   describe "#expire_workflow" do
+    let(:ttl) { 2000 }
+
     it "sets TTL for all Redis keys related to the workflow" do
       workflow = TestWorkflow.create
 
-      client.expire_workflow(workflow, -1)
+      client.expire_workflow(workflow, ttl)
+
+      expect(redis.ttl("gush.workflows.#{workflow.id}")).to eq(ttl)
 
-      # => TODO - I believe fakeredis does not handle TTL the same.   
+      workflow.jobs.each do |job|
+        expect(redis.ttl("gush.jobs.#{workflow.id}.#{job.klass}")).to eq(ttl)
+      end
     end
   end
 

data/spec/gush/worker_spec.rb

@@ -39,6 +39,18 @@ describe Gush::Worker do
       end
     end
 
+    context 'when job failed to enqueue outgoing jobs' do
+      it 'enqeues another job to handling enqueue_outgoing_jobs' do
+        allow(RedisMutex).to receive(:with_lock).and_raise(RedisMutex::LockError)
+        subject.perform(workflow.id, 'Prepare')
+        expect(Gush::Worker).to have_no_jobs(workflow.id, jobs_with_id(["FetchFirstJob", "FetchSecondJob"]))
+
+        allow(RedisMutex).to receive(:with_lock).and_call_original
+        perform_one
+        expect(Gush::Worker).to have_jobs(workflow.id, jobs_with_id(["FetchFirstJob", "FetchSecondJob"]))
+      end
+    end
+
     it "calls job.perform method" do
       SPY = double()
       expect(SPY).to receive(:some_method)

data/spec/spec_helper.rb

@@ -78,6 +78,19 @@ RSpec::Matchers.define :have_jobs do |flow, jobs|
   end
 end
 
+RSpec::Matchers.define :have_no_jobs do |flow, jobs|
+  match do |actual|
+    expected = jobs.map do |job|
+      hash_including(args: include(flow, job))
+    end
+    expect(ActiveJob::Base.queue_adapter.enqueued_jobs).not_to match_array(expected)
+  end
+
+  failure_message do |actual|
+    "expected queue to have no #{jobs}, but instead has: #{ActiveJob::Base.queue_adapter.enqueued_jobs.map{ |j| j[:args][1]}}"
+  end
+end
+
 RSpec.configure do |config|
   config.include ActiveJob::TestHelper
   config.include GushHelpers

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gush
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.0.2
 platform: ruby
 authors:
 - Piotrek Okoński
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-12 00:00:00.000000000 Z
+date: 2022-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -19,7 +19,7 @@ dependencies:
         version: 4.2.7
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,21 +29,21 @@ dependencies:
         version: 4.2.7
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
-  name: connection_pool
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.2.1
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.2.1
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
@@ -180,16 +180,16 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -307,8 +307,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Fast and distributed workflow runner based on ActiveJob and Redis