gush 2.0.2 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12fdb9a62d33353f827194c198c011ff42d7491669d0ab9717be0920a8066313
-  data.tar.gz: 7f3d4ada23215e818f0cb801c92f4752e80b21b344f3b48d9ab4b83787c029ec
+  metadata.gz: dc70498c39ad506749bed1f55c139d29691e540b849dd39e2ba132b54b511d66
+  data.tar.gz: 700d093574e0ff4772c7d36bfeb7792cda44550dd162a48afe17bec5551250cd
 SHA512:
-  metadata.gz: bec4bcb3e251bdb1a2e184b6b62ae896fd40af4a0cd4bad2ec7ee4925ab356e36a3388d2cb78ed21582c0f6cfdd429c304f0591a5ee3760c73255d03a2b1d80e
-  data.tar.gz: 67fba0b65b575449114233a4ff9cfaf1cb6ac3ff61633f94b7c595988165b3315fe7ae5d0744792315d9329795da0e42ae840789e3d4f72c84ef8956081c5fbd
+  metadata.gz: ac4c1647f57a87600466445a8d24660e3d95016f17adf9e2fe8f99260ba82a2b3e22a339b9456f653834c89c6e4b9ed6bcb50ebc337b804e07355f9a513ce8ca
+  data.tar.gz: 1ae01511acfc2e23bb0a671328a7b3e9b650b8589b9620461c2b9667ff9d80a47f03b8a156f249746b4f69e613863db0833be8d6221acdabd211d7eba2daee15

data/.github/workflows/ruby.yml

@@ -0,0 +1,71 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on:
+  pull_request:
+    paths-ignore:
+      - 'README.md'
+  push:
+    paths-ignore:
+      - 'README.md'
+
+jobs:
+  test:
+    services:
+      redis:
+        image: redis:alpine
+        ports: ["6379:6379"]
+        options: --entrypoint redis-server
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        rails_version: ['4.2.7', '5.1.0', '5.2.0', '6.0.0', '6.1.0', '7.0']
+        ruby-version: ['2.6', '2.7', '3.0', '3.1']
+        exclude:
+          - ruby-version: '3.0'
+            rails_version: '4.2.7'
+          - ruby-version: '3.1'
+            rails_version: '4.2.7'
+          - ruby-version: '3.0'
+            rails_version: '5.0'
+          - ruby-version: '3.1'
+            rails_version: '5.0'
+          - ruby-version: '3.0'
+            rails_version: '5.1'
+          - ruby-version: '3.1'
+            rails_version: '5.1'
+          - ruby-version: '3.0'
+            rails_version: '5.2'
+          - ruby-version: '3.1'
+            rails_version: '5.2'
+          - ruby-version: '3.0'
+            rails_version: '6.0'
+          - ruby-version: '3.1'
+            rails_version: '6.0'
+          - ruby-version: '3.1'
+            rails_version: '6.1'
+          - ruby-version: '2.6'
+            rails_version: '7.0'
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      env:
+        RAILS_VERSION: "${{ matrix.rails_version }}"
+    - name: Install Graphviz
+      run: sudo apt-get install graphviz
+    - name: Run tests
+      run: bundle exec rspec
+      env:
+        REDIS_URL: redis://localhost:6379/1
+        RAILS_VERSION: "${{ matrix.rails_version }}"

data/CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## 2.1.0
+
+### Added
+
+- Allow RedisMutex’s locking duration and polling interval to be customizable, thanks to @thukim! [See pull request](https://github.com/chaps-io/gush/pull/74)
+- Support for Rails 7.0 and Ruby 3.0-3.1, thanks to @joshRpowell and @kzkn!
+
 ## 2.0.1
 
 ### Fixed
@@ -13,31 +20,31 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## 2.0.0
 
-## Changed
+### Changed
 
 - *[BREAKING]* Store gush jobs on redis hash instead of plain keys - this improves performance when retrieving keys (Thanks to @Saicheg! [See pull request](https://github.com/chaps-io/gush/pull/56))
 
 
-## Added
+### Added
 
 - Allow setting queue for each job via `:queue` option in `run` method (Thanks to @devilankur18! [See pull request](https://github.com/chaps-io/gush/pull/58))
 
 
 ## 1.1.1 - 2018-06-09
 
-## Changed
+### Changed
 
 - Relax dependency on ActiveSupport to work with 4.2 up to 5.X (Thanks to @iacobus! [See pull request](https://github.com/chaps-io/gush/pull/54))
 
 
 ## 1.1.0 - 2018-02-05
 
-## Added
+### Added
 
 - Added ability to specify TTL for Redis keys and manually expire whole workflows (Thanks to @dmitrypol! [See pull request](https://github.com/chaps-io/gush/pull/48))
 - Loosened dependency on redis-rb library to >= 3.2 and < 5.0 (Thanks to @mofumofu3n! [See pull request](https://github.com/chaps-io/gush/pull/52))
 
-## Fixed
+### Fixed
 
 - Improved performance of (de)serializing workflows by not storing job array inside workflow JSON and other smaller improvements ([See pull request](https://github.com/chaps-io/gush/pull/53))
 

data/Gemfile

@@ -1,6 +1,10 @@
 source 'https://rubygems.org'
 gemspec
 
+rails_version = ENV['RAILS_VERSION'] || '< 7.0'
+rails_version = "~> #{rails_version}" if rails_version =~ /^\d/
+gem 'activejob', rails_version
+
 platforms :mri, :ruby do
   gem 'yajl-ruby'
 end

data/README.md

@@ -1,4 +1,4 @@
-# Gush [![Build Status](https://travis-ci.org/chaps-io/gush.svg?branch=master)](https://travis-ci.org/chaps-io/gush)
+# Gush
 
 Gush is a parallel workflow runner using only Redis as storage and [ActiveJob](http://guides.rubyonrails.org/v4.2/active_job_basics.html#introduction) for scheduling and executing jobs.
 
@@ -8,14 +8,14 @@ Gush relies on directed acyclic graphs to store dependencies, see [Parallelizing
 
 ## **WARNING - version notice**
 
-This README is about the `1.0.0` version, which has breaking changes compared to < 1.0.0 versions. [See here for 0.4.1 documentation](https://github.com/chaps-io/gush/blob/349c5aff0332fd14b1cb517115c26d415aa24841/README.md).
+This README is about the latest `master` code, which might differ from what is released on RubyGems. See tags to browse previous READMEs.
 
 ## Installation
 
 ### 1. Add `gush` to Gemfile
 
 ```ruby
-gem 'gush', '~> 1.0.0'
+gem 'gush', '~> 2.0'
 ```
 
 ### 2. Create `Gushfile`
@@ -274,7 +274,7 @@ class EncodeVideo < Gush::Job
 end
 ```
 
-`payloads` is an array containing outputs from all ancestor jobs. So for our `EncodeVide` job from above, the array will look like:
+`payloads` is an array containing outputs from all ancestor jobs. So for our `EncodeVideo` job from above, the array will look like:
 
 
 ```ruby
@@ -321,6 +321,23 @@ it will generate a workflow with 5 `NotificationJob`s and one `AdminNotification
 
 ![DynamicWorkflow](https://i.imgur.com/HOI3fjc.png)
 
+### Dynamic queue for jobs
+
+There might be a case you want to configure different jobs in the workflow using different queues. Based on the above the example, we want to config `AdminNotificationJob` to use queue `admin` and `NotificationJob` use queue `user`.
+
+```ruby
+
+class NotifyWorkflow < Gush::Workflow
+  def configure(user_ids)
+    notification_jobs = user_ids.map do |user_id|
+      run NotificationJob, params: {user_id: user_id}, queue: 'user'
+    end
+
+    run AdminNotificationJob, after: notification_jobs, queue: 'admin'
+  end
+end
+```
+
 ## Command line interface (CLI)
 
 ### Checking status
@@ -346,9 +363,23 @@ This requires that you have imagemagick installed on your computer:
 bundle exec gush viz <NameOfTheWorkflow>
 ```
 
+### Customizing locking options
+
+In order to prevent getting the RedisMutex::LockError error when having a large number of jobs, you can customize these 2 fields `locking_duration` and `polling_interval` as below
+
+```ruby
+# config/initializers/gush.rb
+Gush.configure do |config|
+  config.redis_url = "redis://localhost:6379"
+  config.concurrency = 5
+  config.locking_duration = 2 # how long you want to wait for the lock to be released, in seconds
+  config.polling_interval = 0.3 # how long the polling interval should be, in seconds
+end
+```
+
 ### Cleaning up afterwards
 
-Running `NotifyWorkflow.create` inserts multiple keys into Redis every time it is ran.  This data might be useful for analysis but at a certain point it can be purged via Redis TTL.  By default gush and Redis will keep keys forever.  To configure expiration you need to 2 things.  Create initializer (specify config.ttl in seconds, be different per environment).  
+Running `NotifyWorkflow.create` inserts multiple keys into Redis every time it is ran.  This data might be useful for analysis but at a certain point it can be purged via Redis TTL.  By default gush and Redis will keep keys forever.  To configure expiration you need to 2 things.  Create initializer (specify config.ttl in seconds, be different per environment).
 
 ```ruby
 # config/initializers/gush.rb
@@ -359,7 +390,7 @@ Gush.configure do |config|
 end
 ```
 
-And you need to call `flow.expire!` (optionally passing custom TTL value overriding `config.ttl`).  This gives you control whether to expire data for specific workflow.  Best NOT to set TTL to be too short (like minutes) but about a week in length.  And you can run `Client.expire_workflow` and `Client.expire_job` passing appropriate IDs and TTL (pass -1 to NOT expire) values.  
+And you need to call `flow.expire!` (optionally passing custom TTL value overriding `config.ttl`).  This gives you control whether to expire data for specific workflow.  Best NOT to set TTL to be too short (like minutes) but about a week in length.  And you can run `Client.expire_workflow` and `Client.expire_job` passing appropriate IDs and TTL (pass -1 to NOT expire) values.
 
 ### Avoid overlapping workflows
 

data/bin/gush

@@ -12,7 +12,7 @@ require 'gush'
 begin
   Gush::CLI.start(ARGV)
 rescue Gush::WorkflowNotFound
-  puts "Workflow not found".red
+  puts Paint["Workflow not found", :red]
 rescue Gush::DependencyLevelTooDeep
-  puts "Dependency level too deep. Perhaps you have a dependency cycle?".red
+  puts Paint["Dependency level too deep. Perhaps you have a dependency cycle?", :red]
 end

data/gush.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "gush"
-  spec.version       = "2.0.2"
+  spec.version       = "2.1.0"
   spec.authors       = ["Piotrek Okoński"]
   spec.email         = ["piotrek@okonski.org"]
   spec.summary       = "Fast and distributed workflow runner based on ActiveJob and Redis"
@@ -17,20 +17,19 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "activejob", ">= 4.2.7", "< 7.0"
+  spec.add_dependency "activejob", ">= 4.2.7", "< 7.1"
   spec.add_dependency "concurrent-ruby", "~> 1.0"
   spec.add_dependency "multi_json", "~> 1.11"
   spec.add_dependency "redis", ">= 3.2", "< 5"
   spec.add_dependency "redis-mutex", "~> 4.0.1"
   spec.add_dependency "hiredis", "~> 0.6"
-  spec.add_dependency "ruby-graphviz", "~> 1.2"
-  spec.add_dependency "terminal-table", "~> 1.4"
-  spec.add_dependency "colorize", "~> 0.7"
-  spec.add_dependency "thor", "~> 0.19"
+  spec.add_dependency "graphviz", "~> 1.2"
+  spec.add_dependency "terminal-table", ">= 1.4", "< 3.1"
+  spec.add_dependency "paint", "~> 2.2"
+  spec.add_dependency "thor", ">= 0.19", "< 1.3"
   spec.add_dependency "launchy", "~> 2.4"
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake", "~> 10.4"
   spec.add_development_dependency "rspec", '~> 3.0'
   spec.add_development_dependency "pry", '~> 0.10'
-  spec.add_development_dependency 'fakeredis', '~> 0.5'
 end

data/lib/gush/cli/overview.rb

@@ -17,11 +17,11 @@ module Gush
         elsif workflow.running?
           running_status
         elsif workflow.finished?
-          "done".green
+          Paint["done", :green]
         elsif workflow.stopped?
-          "stopped".red
+          Paint["stopped", :red]
         else
-          "ready to start".blue
+          Paint["ready to start", :blue]
         end
       end
 
@@ -48,10 +48,10 @@ module Gush
           "ID" => workflow.id,
           "Name" => workflow.class.to_s,
           "Jobs" => workflow.jobs.count,
-          "Failed jobs" => failed_jobs_count.red,
-          "Succeeded jobs" => succeeded_jobs_count.green,
-          "Enqueued jobs" => enqueued_jobs_count.yellow,
-          "Running jobs" => running_jobs_count.blue,
+          "Failed jobs" => Paint[failed_jobs_count, :red],
+          "Succeeded jobs" => Paint[succeeded_jobs_count, :green],
+          "Enqueued jobs" => Paint[enqueued_jobs_count, :yellow],
+          "Running jobs" => Paint[running_jobs_count, :blue],
           "Remaining jobs" => remaining_jobs_count,
           "Started at" => started_at,
           "Status" => status
@@ -60,7 +60,7 @@ module Gush
 
       def running_status
         finished = succeeded_jobs_count.to_i
-        status = "running".yellow
+        status = Paint["running", :yellow]
         status += "\n#{finished}/#{total_jobs_count} [#{(finished*100)/total_jobs_count}%]"
       end
 
@@ -69,7 +69,7 @@ module Gush
       end
 
       def failed_status
-        status = "failed".light_red
+        status = Paint["failed", :red]
         status += "\n#{failed_job} failed"
       end
 
@@ -77,13 +77,13 @@ module Gush
         name = job.name
         case
         when job.failed?
-          "[✗] #{name.red} \n"
+          "[✗] #{Paint[name, :red]} \n"
         when job.finished?
-          "[✓] #{name.green} \n"
+          "[✓] #{Paint[name, :green]} \n"
         when job.enqueued?
-          "[•] #{name.yellow} \n"
+          "[•] #{Paint[name, :yellow]} \n"
         when job.running?
-          "[•] #{name.blue} \n"
+          "[•] #{Paint[name, :blue]} \n"
         else
           "[ ] #{name} \n"
         end

data/lib/gush/cli.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 require 'terminal-table'
-require 'colorize'
+require 'paint'
 require 'thor'
 require 'launchy'
 
@@ -12,43 +14,45 @@ module Gush
     def initialize(*)
       super
       Gush.configure do |config|
-        config.gushfile    = options.fetch("gushfile",    config.gushfile)
-        config.concurrency = options.fetch("concurrency", config.concurrency)
-        config.redis_url   = options.fetch("redis",       config.redis_url)
-        config.namespace   = options.fetch("namespace",   config.namespace)
-        config.ttl         = options.fetch("ttl",         config.ttl)
+        config.gushfile           = options.fetch("gushfile",    config.gushfile)
+        config.concurrency        = options.fetch("concurrency", config.concurrency)
+        config.redis_url          = options.fetch("redis",       config.redis_url)
+        config.namespace          = options.fetch("namespace",   config.namespace)
+        config.ttl                = options.fetch("ttl",         config.ttl)
+        config.locking_duration   = options.fetch("locking_duration", config.locking_duration)
+        config.polling_interval   = options.fetch("polling_interval", config.polling_interval)
       end
       load_gushfile
     end
 
-    desc "create [WorkflowClass]", "Registers new workflow"
+    desc "create WORKFLOW_CLASS", "Registers new workflow"
     def create(name)
       workflow = client.create_workflow(name)
       puts "Workflow created with id: #{workflow.id}"
       puts "Start it with command: gush start #{workflow.id}"
     end
 
-    desc "start [workflow_id]", "Starts Workflow with given ID"
+    desc "start WORKFLOW_ID [ARG ...]", "Starts Workflow with given ID"
     def start(*args)
       id = args.shift
       workflow = client.find_workflow(id)
       client.start_workflow(workflow, args)
     end
 
-    desc "create_and_start [WorkflowClass]", "Create and instantly start the new workflow"
+    desc "create_and_start WORKFLOW_CLASS [ARG ...]", "Create and instantly start the new workflow"
     def create_and_start(name, *args)
       workflow = client.create_workflow(name)
       client.start_workflow(workflow.id, args)
       puts "Created and started workflow with id: #{workflow.id}"
     end
 
-    desc "stop [workflow_id]", "Stops Workflow with given ID"
+    desc "stop WORKFLOW_ID", "Stops Workflow with given ID"
     def stop(*args)
       id = args.shift
       client.stop_workflow(id)
     end
 
-    desc "show [workflow_id]", "Shows details about workflow with given ID"
+    desc "show WORKFLOW_ID", "Shows details about workflow with given ID"
     option :skip_overview, type: :boolean
     option :skip_jobs, type: :boolean
     option :jobs, default: :all
@@ -60,7 +64,7 @@ module Gush
       display_jobs_list_for(workflow, options[:jobs]) unless options[:skip_jobs]
     end
 
-    desc "rm [workflow_id]", "Delete workflow with given ID"
+    desc "rm WORKFLOW_ID", "Delete workflow with given ID"
     def rm(workflow_id)
       workflow = client.find_workflow(workflow_id)
       client.destroy_workflow(workflow)
@@ -81,13 +85,39 @@ module Gush
       puts Terminal::Table.new(headings: headers, rows: rows)
     end
 
-    desc "viz [WorkflowClass]", "Displays graph, visualising job dependencies"
-    def viz(name)
+    desc "viz {WORKFLOW_CLASS|WORKFLOW_ID}", "Displays graph, visualising job dependencies"
+    option :filename, type: :string, default: nil
+    option :open, type: :boolean, default: nil
+    def viz(class_or_id)
       client
-      workflow = name.constantize.new
-      graph = Graph.new(workflow)
+
+      begin
+        workflow = client.find_workflow(class_or_id)
+      rescue WorkflowNotFound
+        workflow = nil
+      end
+
+      unless workflow
+        begin
+          workflow = class_or_id.constantize.new
+        rescue NameError => e
+          STDERR.puts Paint["'#{class_or_id}' is not a valid workflow class or id", :red]
+          exit 1
+        end
+      end
+
+      opts = {}
+
+      if options[:filename]
+        opts[:filename], opts[:path] = File.split(options[:filename])
+      end
+
+      graph = Graph.new(workflow, **opts)
       graph.viz
-      Launchy.open graph.path
+
+      if (options[:open].nil? && !options[:filename]) || options[:open]
+        Launchy.open Pathname.new(graph.path).realpath.to_s
+      end
     end
 
     private
@@ -118,13 +148,14 @@ module Gush
 
     def load_gushfile
       file = client.configuration.gushfile
-      if !gushfile.exist?
-        raise Thor::Error, "#{file} not found, please add it to your project".colorize(:red)
+
+      unless gushfile.exist?
+        raise Thor::Error, Paint["#{file} not found, please add it to your project", :red]
       end
 
       load file.to_s
     rescue LoadError
-      raise Thor::Error, "failed to require #{file}".colorize(:red)
+      raise Thor::Error, Paint["failed to require #{file}", :red]
     end
   end
 end

data/lib/gush/client.rb

@@ -72,7 +72,7 @@ module Gush
       id = nil
       loop do
         id = SecureRandom.uuid
-        available = !redis.exists("gush.workflow.#{id}")
+        available = !redis.exists?("gush.workflow.#{id}")
 
         break if available
       end

data/lib/gush/configuration.rb

@@ -1,17 +1,19 @@
 module Gush
   class Configuration
-    attr_accessor :concurrency, :namespace, :redis_url, :ttl
+    attr_accessor :concurrency, :namespace, :redis_url, :ttl, :locking_duration, :polling_interval
 
     def self.from_json(json)
       new(Gush::JSON.decode(json, symbolize_keys: true))
     end
 
     def initialize(hash = {})
-      self.concurrency = hash.fetch(:concurrency, 5)
-      self.namespace   = hash.fetch(:namespace, 'gush')
-      self.redis_url   = hash.fetch(:redis_url, 'redis://localhost:6379')
-      self.gushfile    = hash.fetch(:gushfile, 'Gushfile')
-      self.ttl         = hash.fetch(:ttl, -1)
+      self.concurrency      = hash.fetch(:concurrency, 5)
+      self.namespace        = hash.fetch(:namespace, 'gush')
+      self.redis_url        = hash.fetch(:redis_url, 'redis://localhost:6379')
+      self.gushfile         = hash.fetch(:gushfile, 'Gushfile')
+      self.ttl              = hash.fetch(:ttl, -1)
+      self.locking_duration = hash.fetch(:locking_duration, 2) # how long you want to wait for the lock to be released, in seconds
+      self.polling_interval = hash.fetch(:polling_internal, 0.3) # how long the polling interval should be, in seconds
     end
 
     def gushfile=(path)
@@ -24,10 +26,12 @@ module Gush
 
     def to_hash
       {
-        concurrency: concurrency,
-        namespace:   namespace,
-        redis_url:   redis_url,
-        ttl:         ttl
+        concurrency:      concurrency,
+        namespace:        namespace,
+        redis_url:        redis_url,
+        ttl:              ttl,
+        locking_duration: locking_duration,
+        polling_interval: polling_interval
       }
     end
 

data/lib/gush/graph.rb

@@ -1,6 +1,10 @@
+# frozen_string_literal: true
+
+require 'tmpdir'
+
 module Gush
   class Graph
-    attr_reader :workflow, :filename, :path, :start, :end_node
+    attr_reader :workflow, :filename, :path, :start_node, :end_node
 
     def initialize(workflow, options = {})
       @workflow = workflow
@@ -9,19 +13,26 @@ module Gush
     end
 
     def viz
-      GraphViz.new(:G, graph_options) do |graph|
-        set_node_options!(graph)
-        set_edge_options!(graph)
+      @graph = Graphviz::Graph.new(**graph_options)
+      @start_node = add_node('start', shape: 'diamond', fillcolor: '#CFF09E')
+      @end_node = add_node('end', shape: 'diamond', fillcolor: '#F56991')
 
-        @start = graph.start(shape: 'diamond', fillcolor: '#CFF09E')
-        @end_node = graph.end(shape: 'diamond', fillcolor: '#F56991')
-
-        workflow.jobs.each do |job|
-          add_job(graph, job)
-        end
+      # First, create nodes for all jobs
+      @job_name_to_node_map = {}
+      workflow.jobs.each do |job|
+        add_job_node(job)
+      end
 
-        graph.output(png: path)
+      # Next, link up the jobs with edges
+      workflow.jobs.each do |job|
+        link_job_edges(job)
       end
+
+      format = 'png'
+      file_format = path.split('.')[-1]
+      format = file_format if file_format.length == 3
+
+      Graphviz::output(@graph, path: path, format: format)
     end
 
     def path
@@ -29,43 +40,43 @@ module Gush
     end
 
     private
-    def add_job(graph, job)
-      name = job.class.to_s
-      graph.add_nodes(job.name, label: name)
+
+    def add_node(name, **specific_options)
+      @graph.add_node(name, **node_options.merge(specific_options))
+    end
+
+    def add_job_node(job)
+      @job_name_to_node_map[job.name] = add_node(job.name, label: node_label_for_job(job))
+    end
+
+    def link_job_edges(job)
+      job_node = @job_name_to_node_map[job.name]
 
       if job.incoming.empty?
-        graph.add_edges(start, job.name)
+        @start_node.connect(job_node, **edge_options)
       end
 
       if job.outgoing.empty?
-        graph.add_edges(job.name, end_node)
+        job_node.connect(@end_node, **edge_options)
       else
         job.outgoing.each do |id|
           outgoing_job = workflow.find_job(id)
-          graph.add_edges(job.name, outgoing_job.name)
+          job_node.connect(@job_name_to_node_map[outgoing_job.name], **edge_options)
         end
       end
     end
 
-    def set_node_options!(graph)
-      node_options.each do |key, value|
-        graph.node[key] = value
-      end
-    end
-
-    def set_edge_options!(graph)
-      edge_options.each do |key, value|
-        graph.edge[key] = value
-      end
+    def node_label_for_job(job)
+      job.class.to_s
     end
 
     def graph_options
       {
-        type: :digraph,
-        dpi: 200,
-        compound: true,
-        rankdir: "LR",
-        center: true
+          dpi: 200,
+          compound: true,
+          rankdir: "LR",
+          center: true,
+          format: 'png'
       }
     end
 

data/lib/gush/worker.rb

@@ -30,12 +30,16 @@ module Gush
 
     private
 
-    attr_reader :client, :workflow_id, :job
+    attr_reader :client, :workflow_id, :job, :configuration
 
     def client
       @client ||= Gush::Client.new(Gush.configuration)
     end
 
+    def configuration
+      @configuration ||= client.configuration
+    end
+
     def setup_job(workflow_id, job_id)
       @workflow_id = workflow_id
       @job ||= client.find_job(workflow_id, job_id)
@@ -73,7 +77,11 @@ module Gush
 
     def enqueue_outgoing_jobs
       job.outgoing.each do |job_name|
-        RedisMutex.with_lock("gush_enqueue_outgoing_jobs_#{workflow_id}-#{job_name}", sleep: 0.3, block: 2) do
+        RedisMutex.with_lock(
+          "gush_enqueue_outgoing_jobs_#{workflow_id}-#{job_name}",
+          sleep: configuration.polling_interval,
+          block: configuration.locking_duration
+        ) do
           out = client.find_job(workflow_id, job_name)
 
           if out.ready_to_start?

data/spec/features/integration_spec.rb

@@ -152,17 +152,15 @@ describe "Workflows" do
     flow = PayloadWorkflow.create
     flow.start!
 
-    perform_one
-    expect(flow.reload.find_job(flow.jobs[0].name).output_payload).to eq('first')
+    3.times { perform_one }
 
-    perform_one
-    expect(flow.reload.find_job(flow.jobs[1].name).output_payload).to eq('second')
+    outputs = flow.reload.jobs.select { |j| j.klass == 'RepetitiveJob' }.map { |j| j.output_payload }
+    expect(outputs).to match_array(['first', 'second', 'third'])
 
     perform_one
-    expect(flow.reload.find_job(flow.jobs[2].name).output_payload).to eq('third')
 
-    perform_one
-    expect(flow.reload.find_job(flow.jobs[3].name).output_payload).to eq(%w(first second third))
+    summary_job = flow.reload.jobs.find { |j| j.klass == 'SummaryJob' }
+    expect(summary_job.output_payload).to eq(%w(first second third))
   end
 
   it "does not execute `configure` on each job for huge workflows" do

data/spec/gush/configuration_spec.rb

@@ -8,6 +8,8 @@ describe Gush::Configuration do
     expect(subject.concurrency).to eq(5)
     expect(subject.namespace).to eq('gush')
     expect(subject.gushfile).to eq(GUSHFILE.realpath)
+    expect(subject.locking_duration).to eq(2)
+    expect(subject.polling_interval).to eq(0.3)
   end
 
   describe "#configure" do
@@ -15,10 +17,14 @@ describe Gush::Configuration do
       Gush.configure do |config|
         config.redis_url = "redis://localhost"
         config.concurrency = 25
+        config.locking_duration = 5
+        config.polling_interval = 0.5
       end
 
       expect(Gush.configuration.redis_url).to eq("redis://localhost")
       expect(Gush.configuration.concurrency).to eq(25)
+      expect(Gush.configuration.locking_duration).to eq(5)
+      expect(Gush.configuration.polling_interval).to eq(0.5)
     end
   end
 end

data/spec/gush/graph_spec.rb

@@ -10,26 +10,43 @@ describe Gush::Graph do
       edge = double("edge", :[]= => true)
       graph = double("graph", node: node, edge: edge)
       path = Pathname.new(Dir.tmpdir).join(filename)
-      expect(graph).to receive(:start).with(shape: 'diamond', fillcolor: '#CFF09E')
-      expect(graph).to receive(:end).with(shape: 'diamond', fillcolor: '#F56991')
-
-      expect(graph).to receive(:output).with(png: path.to_s)
-
-      expect(graph).to receive(:add_nodes).with(/Prepare/, label: "Prepare")
-      expect(graph).to receive(:add_nodes).with(/FetchFirstJob/, label: "FetchFirstJob")
-      expect(graph).to receive(:add_nodes).with(/FetchSecondJob/, label: "FetchSecondJob")
-      expect(graph).to receive(:add_nodes).with(/NormalizeJob/, label: "NormalizeJob")
-      expect(graph).to receive(:add_nodes).with(/PersistFirstJob/, label: "PersistFirstJob")
-
-      expect(graph).to receive(:add_edges).with(nil, /Prepare/)
-      expect(graph).to receive(:add_edges).with(/Prepare/, /FetchFirstJob/)
-      expect(graph).to receive(:add_edges).with(/Prepare/, /FetchSecondJob/)
-      expect(graph).to receive(:add_edges).with(/FetchFirstJob/, /PersistFirstJob/)
-      expect(graph).to receive(:add_edges).with(/FetchSecondJob/, /NormalizeJob/)
-      expect(graph).to receive(:add_edges).with(/PersistFirstJob/, /NormalizeJob/)
-      expect(graph).to receive(:add_edges).with(/NormalizeJob/, nil)
-
-      expect(GraphViz).to receive(:new).and_yield(graph)
+
+      expect(Graphviz::Graph).to receive(:new).and_return(graph)
+
+      node_start = double('start')
+      node_end = double('end')
+      node_prepare = double('Prepare')
+      node_fetch_first_job = double('FetchFirstJob')
+      node_fetch_second_job = double('FetchSecondJob')
+      node_normalize_job = double('NormalizeJob')
+      node_persist_first_job = double('PersistFirstJob')
+
+      expect(graph).to receive(:add_node).with('start', {shape: 'diamond', fillcolor: '#CFF09E', color: "#555555", style: 'filled'}).and_return(node_start)
+      expect(graph).to receive(:add_node).with('end', {shape: 'diamond', fillcolor: '#F56991', color: "#555555", style: 'filled'}).and_return(node_end)
+
+      standard_options = {:color=>"#555555", :fillcolor=>"white", :label=>"Prepare", :shape=>"ellipse", :style=>"filled"}
+
+      expect(graph).to receive(:add_node).with(/Prepare/, standard_options.merge(label: "Prepare")).and_return(node_prepare)
+      expect(graph).to receive(:add_node).with(/FetchFirstJob/, standard_options.merge(label: "FetchFirstJob")).and_return(node_fetch_first_job)
+      expect(graph).to receive(:add_node).with(/FetchSecondJob/, standard_options.merge(label: "FetchSecondJob")).and_return(node_fetch_second_job)
+      expect(graph).to receive(:add_node).with(/NormalizeJob/, standard_options.merge(label: "NormalizeJob")).and_return(node_normalize_job)
+      expect(graph).to receive(:add_node).with(/PersistFirstJob/, standard_options.merge(label: "PersistFirstJob")).and_return(node_persist_first_job)
+
+      edge_options = {
+          dir: "forward",
+          penwidth: 1,
+          color: "#555555"
+      }
+
+      expect(node_start).to receive(:connect).with(node_prepare, **edge_options)
+      expect(node_prepare).to receive(:connect).with(node_fetch_first_job, **edge_options)
+      expect(node_prepare).to receive(:connect).with(node_fetch_second_job, **edge_options)
+      expect(node_fetch_first_job).to receive(:connect).with(node_persist_first_job, **edge_options)
+      expect(node_fetch_second_job).to receive(:connect).with(node_normalize_job, **edge_options)
+      expect(node_persist_first_job).to receive(:connect).with(node_normalize_job, **edge_options)
+      expect(node_normalize_job).to receive(:connect).with(node_end, **edge_options)
+
+      expect(graph).to receive(:dump_graph).and_return(nil)
 
       subject.viz
     end

data/spec/gush/worker_spec.rb

@@ -4,6 +4,8 @@ describe Gush::Worker do
   subject { described_class.new }
 
   let!(:workflow)   { TestWorkflow.create }
+  let(:locking_duration) { 5 }
+  let(:polling_interval) { 0.5 }
   let!(:job)        { client.find_job(workflow.id, "Prepare")  }
   let(:config)      { Gush.configuration.to_json  }
   let!(:client)     { Gush::Client.new }
@@ -71,5 +73,11 @@ describe Gush::Worker do
 
       subject.perform(workflow.id, 'OkayJob')
     end
+
+    it 'calls RedisMutex.with_lock with customizable locking_duration and polling_interval' do
+      expect(RedisMutex).to receive(:with_lock)
+        .with(anything, block: 5, sleep: 0.5).twice
+      subject.perform(workflow.id, 'Prepare')
+    end
   end
 end

data/spec/spec_helper.rb

@@ -1,5 +1,4 @@
 require 'gush'
-require 'fakeredis'
 require 'json'
 require 'pry'
 
@@ -35,12 +34,8 @@ class ParameterTestWorkflow < Gush::Workflow
   end
 end
 
-class Redis
-  def publish(*)
-  end
-end
 
-REDIS_URL = "redis://localhost:6379/12"
+REDIS_URL = ENV["REDIS_URL"] || "redis://localhost:6379/12"
 
 module GushHelpers
   def redis
@@ -104,12 +99,13 @@ RSpec.configure do |config|
     clear_performed_jobs
 
     Gush.configure do |config|
-      config.redis_url = REDIS_URL
-      config.gushfile = GUSHFILE
+      config.redis_url        = REDIS_URL
+      config.gushfile         = GUSHFILE
+      config.locking_duration = defined?(locking_duration) ? locking_duration : 2
+      config.polling_interval = defined?(polling_interval) ? polling_interval : 0.3
     end
   end
 
-
   config.after(:each) do
     clear_enqueued_jobs
     clear_performed_jobs

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gush
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.1.0
 platform: ruby
 authors:
 - Piotrek Okoński
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-10 00:00:00.000000000 Z
+date: 2022-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -19,7 +19,7 @@ dependencies:
         version: 4.2.7
     - - "<"
       - !ruby/object:Gem::Version
-        version: '7.0'
+        version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: 4.2.7
     - - "<"
       - !ruby/object:Gem::Version
-        version: '7.0'
+        version: '7.1'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -107,7 +107,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.6'
 - !ruby/object:Gem::Dependency
-  name: ruby-graphviz
+  name: graphviz
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -124,44 +124,56 @@ dependencies:
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.1'
 - !ruby/object:Gem::Dependency
-  name: colorize
+  name: paint
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '2.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.19'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.19'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: launchy
   requirement: !ruby/object:Gem::Requirement
@@ -232,20 +244,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.10'
-- !ruby/object:Gem::Dependency
-  name: fakeredis
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.5'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.5'
 description: Gush is a parallel workflow runner using Redis as storage and ActiveJob
   for executing jobs.
 email:
@@ -255,9 +253,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -292,7 +290,7 @@ homepage: https://github.com/chaps-io/gush
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -307,8 +305,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.3.3
+signing_key:
 specification_version: 4
 summary: Fast and distributed workflow runner based on ActiveJob and Redis
 test_files:

data/.travis.yml

@@ -1,16 +0,0 @@
-language: ruby
-script: "bundle exec rspec"
-rvm:
-  - 2.2.2
-  - 2.3.4
-  - 2.4.1
-  - 2.5
-  - 2.6
-  - 2.7
-services:
-  - redis-server
-email:
-  recipients:
-    - piotrek@okonski.org
-  on_success: change
-  on_failure: always