gush 2.0.1 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ffcdce12d2f530e27afdd048328d8942e60efcddfbe20a18426b8d245589c759
-  data.tar.gz: 3c95194d44f446a694af83b06b06eac55a3e5ed1f8555fe86958fa110b357647
+  metadata.gz: dc70498c39ad506749bed1f55c139d29691e540b849dd39e2ba132b54b511d66
+  data.tar.gz: 700d093574e0ff4772c7d36bfeb7792cda44550dd162a48afe17bec5551250cd
 SHA512:
-  metadata.gz: e75e7f0ae5c5c83302d0507bb502a30a860f276e0706edd5a4119bb9c50bf88c73c9839ac1f141f4569017301ebb61fe943317d214c0c9a10f49309e674bd183
-  data.tar.gz: 38f3baaa43a4d512d5d5ec6c476d21965d27116e7dc8bc1ae3cc3fa4a4fac13e6d18272ed236fd413aba0c304f0e79aee59f3c06cdad19db7fa02e41305fe491
+  metadata.gz: ac4c1647f57a87600466445a8d24660e3d95016f17adf9e2fe8f99260ba82a2b3e22a339b9456f653834c89c6e4b9ed6bcb50ebc337b804e07355f9a513ce8ca
+  data.tar.gz: 1ae01511acfc2e23bb0a671328a7b3e9b650b8589b9620461c2b9667ff9d80a47f03b8a156f249746b4f69e613863db0833be8d6221acdabd211d7eba2daee15

data/.github/workflows/ruby.yml

@@ -0,0 +1,71 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on:
+  pull_request:
+    paths-ignore:
+      - 'README.md'
+  push:
+    paths-ignore:
+      - 'README.md'
+
+jobs:
+  test:
+    services:
+      redis:
+        image: redis:alpine
+        ports: ["6379:6379"]
+        options: --entrypoint redis-server
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        rails_version: ['4.2.7', '5.1.0', '5.2.0', '6.0.0', '6.1.0', '7.0']
+        ruby-version: ['2.6', '2.7', '3.0', '3.1']
+        exclude:
+          - ruby-version: '3.0'
+            rails_version: '4.2.7'
+          - ruby-version: '3.1'
+            rails_version: '4.2.7'
+          - ruby-version: '3.0'
+            rails_version: '5.0'
+          - ruby-version: '3.1'
+            rails_version: '5.0'
+          - ruby-version: '3.0'
+            rails_version: '5.1'
+          - ruby-version: '3.1'
+            rails_version: '5.1'
+          - ruby-version: '3.0'
+            rails_version: '5.2'
+          - ruby-version: '3.1'
+            rails_version: '5.2'
+          - ruby-version: '3.0'
+            rails_version: '6.0'
+          - ruby-version: '3.1'
+            rails_version: '6.0'
+          - ruby-version: '3.1'
+            rails_version: '6.1'
+          - ruby-version: '2.6'
+            rails_version: '7.0'
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      env:
+        RAILS_VERSION: "${{ matrix.rails_version }}"
+    - name: Install Graphviz
+      run: sudo apt-get install graphviz
+    - name: Run tests
+      run: bundle exec rspec
+      env:
+        REDIS_URL: redis://localhost:6379/1
+        RAILS_VERSION: "${{ matrix.rails_version }}"

data/.gitignore

@@ -19,3 +19,5 @@ tmp
 test.rb
 /Gushfile
 dump.rdb
+.ruby-version
+.ruby-gemset

data/CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## 2.1.0
+
+### Added
+
+- Allow RedisMutex’s locking duration and polling interval to be customizable, thanks to @thukim! [See pull request](https://github.com/chaps-io/gush/pull/74)
+- Support for Rails 7.0 and Ruby 3.0-3.1, thanks to @joshRpowell and @kzkn!
+
 ## 2.0.1
 
 ### Fixed
@@ -13,31 +20,31 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## 2.0.0
 
-## Changed
+### Changed
 
 - *[BREAKING]* Store gush jobs on redis hash instead of plain keys - this improves performance when retrieving keys (Thanks to @Saicheg! [See pull request](https://github.com/chaps-io/gush/pull/56))
 
 
-## Added
+### Added
 
 - Allow setting queue for each job via `:queue` option in `run` method (Thanks to @devilankur18! [See pull request](https://github.com/chaps-io/gush/pull/58))
 
 
 ## 1.1.1 - 2018-06-09
 
-## Changed
+### Changed
 
 - Relax dependency on ActiveSupport to work with 4.2 up to 5.X (Thanks to @iacobus! [See pull request](https://github.com/chaps-io/gush/pull/54))
 
 
 ## 1.1.0 - 2018-02-05
 
-## Added
+### Added
 
 - Added ability to specify TTL for Redis keys and manually expire whole workflows (Thanks to @dmitrypol! [See pull request](https://github.com/chaps-io/gush/pull/48))
 - Loosened dependency on redis-rb library to >= 3.2 and < 5.0 (Thanks to @mofumofu3n! [See pull request](https://github.com/chaps-io/gush/pull/52))
 
-## Fixed
+### Fixed
 
 - Improved performance of (de)serializing workflows by not storing job array inside workflow JSON and other smaller improvements ([See pull request](https://github.com/chaps-io/gush/pull/53))
 

data/Gemfile

@@ -1,6 +1,10 @@
 source 'https://rubygems.org'
 gemspec
 
+rails_version = ENV['RAILS_VERSION'] || '< 7.0'
+rails_version = "~> #{rails_version}" if rails_version =~ /^\d/
+gem 'activejob', rails_version
+
 platforms :mri, :ruby do
   gem 'yajl-ruby'
 end

data/README.md

@@ -1,6 +1,4 @@
-# Gush [![Build Status](https://travis-ci.org/chaps-io/gush.svg?branch=master)](https://travis-ci.org/chaps-io/gush)
-
-## [![](http://i.imgur.com/ya8Wnyl.png)](https://chaps.io) proudly made by [Chaps](https://chaps.io)
+# Gush
 
 Gush is a parallel workflow runner using only Redis as storage and [ActiveJob](http://guides.rubyonrails.org/v4.2/active_job_basics.html#introduction) for scheduling and executing jobs.
 
@@ -10,14 +8,14 @@ Gush relies on directed acyclic graphs to store dependencies, see [Parallelizing
 
 ## **WARNING - version notice**
 
-This README is about the `1.0.0` version, which has breaking changes compared to < 1.0.0 versions. [See here for 0.4.1 documentation](https://github.com/chaps-io/gush/blob/349c5aff0332fd14b1cb517115c26d415aa24841/README.md).
+This README is about the latest `master` code, which might differ from what is released on RubyGems. See tags to browse previous READMEs.
 
 ## Installation
 
 ### 1. Add `gush` to Gemfile
 
 ```ruby
-gem 'gush', '~> 1.0.0'
+gem 'gush', '~> 2.0'
 ```
 
 ### 2. Create `Gushfile`
@@ -276,7 +274,7 @@ class EncodeVideo < Gush::Job
 end
 ```
 
-`payloads` is an array containing outputs from all ancestor jobs. So for our `EncodeVide` job from above, the array will look like:
+`payloads` is an array containing outputs from all ancestor jobs. So for our `EncodeVideo` job from above, the array will look like:
 
 
 ```ruby
@@ -323,6 +321,23 @@ it will generate a workflow with 5 `NotificationJob`s and one `AdminNotification
 
 ![DynamicWorkflow](https://i.imgur.com/HOI3fjc.png)
 
+### Dynamic queue for jobs
+
+There might be a case you want to configure different jobs in the workflow using different queues. Based on the above the example, we want to config `AdminNotificationJob` to use queue `admin` and `NotificationJob` use queue `user`.
+
+```ruby
+
+class NotifyWorkflow < Gush::Workflow
+  def configure(user_ids)
+    notification_jobs = user_ids.map do |user_id|
+      run NotificationJob, params: {user_id: user_id}, queue: 'user'
+    end
+
+    run AdminNotificationJob, after: notification_jobs, queue: 'admin'
+  end
+end
+```
+
 ## Command line interface (CLI)
 
 ### Checking status
@@ -348,9 +363,23 @@ This requires that you have imagemagick installed on your computer:
 bundle exec gush viz <NameOfTheWorkflow>
 ```
 
+### Customizing locking options
+
+In order to prevent getting the RedisMutex::LockError error when having a large number of jobs, you can customize these 2 fields `locking_duration` and `polling_interval` as below
+
+```ruby
+# config/initializers/gush.rb
+Gush.configure do |config|
+  config.redis_url = "redis://localhost:6379"
+  config.concurrency = 5
+  config.locking_duration = 2 # how long you want to wait for the lock to be released, in seconds
+  config.polling_interval = 0.3 # how long the polling interval should be, in seconds
+end
+```
+
 ### Cleaning up afterwards
 
-Running `NotifyWorkflow.create` inserts multiple keys into Redis every time it is ran.  This data might be useful for analysis but at a certain point it can be purged via Redis TTL.  By default gush and Redis will keep keys forever.  To configure expiration you need to 2 things.  Create initializer (specify config.ttl in seconds, be different per environment).  
+Running `NotifyWorkflow.create` inserts multiple keys into Redis every time it is ran.  This data might be useful for analysis but at a certain point it can be purged via Redis TTL.  By default gush and Redis will keep keys forever.  To configure expiration you need to 2 things.  Create initializer (specify config.ttl in seconds, be different per environment).
 
 ```ruby
 # config/initializers/gush.rb
@@ -361,7 +390,7 @@ Gush.configure do |config|
 end
 ```
 
-And you need to call `flow.expire!` (optionally passing custom TTL value overriding `config.ttl`).  This gives you control whether to expire data for specific workflow.  Best NOT to set TTL to be too short (like minutes) but about a week in length.  And you can run `Client.expire_workflow` and `Client.expire_job` passing appropriate IDs and TTL (pass -1 to NOT expire) values.  
+And you need to call `flow.expire!` (optionally passing custom TTL value overriding `config.ttl`).  This gives you control whether to expire data for specific workflow.  Best NOT to set TTL to be too short (like minutes) but about a week in length.  And you can run `Client.expire_workflow` and `Client.expire_job` passing appropriate IDs and TTL (pass -1 to NOT expire) values.
 
 ### Avoid overlapping workflows
 

data/bin/gush

@@ -12,7 +12,7 @@ require 'gush'
 begin
   Gush::CLI.start(ARGV)
 rescue Gush::WorkflowNotFound
-  puts "Workflow not found".red
+  puts Paint["Workflow not found", :red]
 rescue Gush::DependencyLevelTooDeep
-  puts "Dependency level too deep. Perhaps you have a dependency cycle?".red
+  puts Paint["Dependency level too deep. Perhaps you have a dependency cycle?", :red]
 end

data/gush.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "gush"
-  spec.version       = "2.0.1"
+  spec.version       = "2.1.0"
   spec.authors       = ["Piotrek Okoński"]
   spec.email         = ["piotrek@okonski.org"]
   spec.summary       = "Fast and distributed workflow runner based on ActiveJob and Redis"
@@ -17,20 +17,19 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "activejob", ">= 4.2.7", "< 6.0"
-  spec.add_dependency "connection_pool", "~> 2.2.1"
+  spec.add_dependency "activejob", ">= 4.2.7", "< 7.1"
+  spec.add_dependency "concurrent-ruby", "~> 1.0"
   spec.add_dependency "multi_json", "~> 1.11"
   spec.add_dependency "redis", ">= 3.2", "< 5"
   spec.add_dependency "redis-mutex", "~> 4.0.1"
   spec.add_dependency "hiredis", "~> 0.6"
-  spec.add_dependency "ruby-graphviz", "~> 1.2"
-  spec.add_dependency "terminal-table", "~> 1.4"
-  spec.add_dependency "colorize", "~> 0.7"
-  spec.add_dependency "thor", "~> 0.19"
+  spec.add_dependency "graphviz", "~> 1.2"
+  spec.add_dependency "terminal-table", ">= 1.4", "< 3.1"
+  spec.add_dependency "paint", "~> 2.2"
+  spec.add_dependency "thor", ">= 0.19", "< 1.3"
   spec.add_dependency "launchy", "~> 2.4"
-  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake", "~> 10.4"
   spec.add_development_dependency "rspec", '~> 3.0'
   spec.add_development_dependency "pry", '~> 0.10'
-  spec.add_development_dependency 'fakeredis', '~> 0.5'
 end

data/lib/gush/cli/overview.rb

@@ -17,11 +17,11 @@ module Gush
         elsif workflow.running?
           running_status
         elsif workflow.finished?
-          "done".green
+          Paint["done", :green]
         elsif workflow.stopped?
-          "stopped".red
+          Paint["stopped", :red]
         else
-          "ready to start".blue
+          Paint["ready to start", :blue]
         end
       end
 
@@ -48,10 +48,10 @@ module Gush
           "ID" => workflow.id,
           "Name" => workflow.class.to_s,
           "Jobs" => workflow.jobs.count,
-          "Failed jobs" => failed_jobs_count.red,
-          "Succeeded jobs" => succeeded_jobs_count.green,
-          "Enqueued jobs" => enqueued_jobs_count.yellow,
-          "Running jobs" => running_jobs_count.blue,
+          "Failed jobs" => Paint[failed_jobs_count, :red],
+          "Succeeded jobs" => Paint[succeeded_jobs_count, :green],
+          "Enqueued jobs" => Paint[enqueued_jobs_count, :yellow],
+          "Running jobs" => Paint[running_jobs_count, :blue],
           "Remaining jobs" => remaining_jobs_count,
           "Started at" => started_at,
           "Status" => status
@@ -60,7 +60,7 @@ module Gush
 
       def running_status
         finished = succeeded_jobs_count.to_i
-        status = "running".yellow
+        status = Paint["running", :yellow]
         status += "\n#{finished}/#{total_jobs_count} [#{(finished*100)/total_jobs_count}%]"
       end
 
@@ -69,7 +69,7 @@ module Gush
       end
 
       def failed_status
-        status = "failed".light_red
+        status = Paint["failed", :red]
         status += "\n#{failed_job} failed"
       end
 
@@ -77,13 +77,13 @@ module Gush
         name = job.name
         case
         when job.failed?
-          "[✗] #{name.red} \n"
+          "[✗] #{Paint[name, :red]} \n"
         when job.finished?
-          "[✓] #{name.green} \n"
+          "[✓] #{Paint[name, :green]} \n"
         when job.enqueued?
-          "[•] #{name.yellow} \n"
+          "[•] #{Paint[name, :yellow]} \n"
         when job.running?
-          "[•] #{name.blue} \n"
+          "[•] #{Paint[name, :blue]} \n"
         else
           "[ ] #{name} \n"
         end

data/lib/gush/cli.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 require 'terminal-table'
-require 'colorize'
+require 'paint'
 require 'thor'
 require 'launchy'
 
@@ -12,43 +14,45 @@ module Gush
     def initialize(*)
       super
       Gush.configure do |config|
-        config.gushfile    = options.fetch("gushfile",    config.gushfile)
-        config.concurrency = options.fetch("concurrency", config.concurrency)
-        config.redis_url   = options.fetch("redis",       config.redis_url)
-        config.namespace   = options.fetch("namespace",   config.namespace)
-        config.ttl         = options.fetch("ttl",         config.ttl)
+        config.gushfile           = options.fetch("gushfile",    config.gushfile)
+        config.concurrency        = options.fetch("concurrency", config.concurrency)
+        config.redis_url          = options.fetch("redis",       config.redis_url)
+        config.namespace          = options.fetch("namespace",   config.namespace)
+        config.ttl                = options.fetch("ttl",         config.ttl)
+        config.locking_duration   = options.fetch("locking_duration", config.locking_duration)
+        config.polling_interval   = options.fetch("polling_interval", config.polling_interval)
       end
       load_gushfile
     end
 
-    desc "create [WorkflowClass]", "Registers new workflow"
+    desc "create WORKFLOW_CLASS", "Registers new workflow"
     def create(name)
       workflow = client.create_workflow(name)
       puts "Workflow created with id: #{workflow.id}"
       puts "Start it with command: gush start #{workflow.id}"
     end
 
-    desc "start [workflow_id]", "Starts Workflow with given ID"
+    desc "start WORKFLOW_ID [ARG ...]", "Starts Workflow with given ID"
     def start(*args)
       id = args.shift
       workflow = client.find_workflow(id)
       client.start_workflow(workflow, args)
     end
 
-    desc "create_and_start [WorkflowClass]", "Create and instantly start the new workflow"
+    desc "create_and_start WORKFLOW_CLASS [ARG ...]", "Create and instantly start the new workflow"
     def create_and_start(name, *args)
       workflow = client.create_workflow(name)
       client.start_workflow(workflow.id, args)
       puts "Created and started workflow with id: #{workflow.id}"
     end
 
-    desc "stop [workflow_id]", "Stops Workflow with given ID"
+    desc "stop WORKFLOW_ID", "Stops Workflow with given ID"
     def stop(*args)
       id = args.shift
       client.stop_workflow(id)
     end
 
-    desc "show [workflow_id]", "Shows details about workflow with given ID"
+    desc "show WORKFLOW_ID", "Shows details about workflow with given ID"
     option :skip_overview, type: :boolean
     option :skip_jobs, type: :boolean
     option :jobs, default: :all
@@ -60,7 +64,7 @@ module Gush
       display_jobs_list_for(workflow, options[:jobs]) unless options[:skip_jobs]
     end
 
-    desc "rm [workflow_id]", "Delete workflow with given ID"
+    desc "rm WORKFLOW_ID", "Delete workflow with given ID"
     def rm(workflow_id)
       workflow = client.find_workflow(workflow_id)
       client.destroy_workflow(workflow)
@@ -81,13 +85,39 @@ module Gush
       puts Terminal::Table.new(headings: headers, rows: rows)
     end
 
-    desc "viz [WorkflowClass]", "Displays graph, visualising job dependencies"
-    def viz(name)
+    desc "viz {WORKFLOW_CLASS|WORKFLOW_ID}", "Displays graph, visualising job dependencies"
+    option :filename, type: :string, default: nil
+    option :open, type: :boolean, default: nil
+    def viz(class_or_id)
       client
-      workflow = name.constantize.new
-      graph = Graph.new(workflow)
+
+      begin
+        workflow = client.find_workflow(class_or_id)
+      rescue WorkflowNotFound
+        workflow = nil
+      end
+
+      unless workflow
+        begin
+          workflow = class_or_id.constantize.new
+        rescue NameError => e
+          STDERR.puts Paint["'#{class_or_id}' is not a valid workflow class or id", :red]
+          exit 1
+        end
+      end
+
+      opts = {}
+
+      if options[:filename]
+        opts[:filename], opts[:path] = File.split(options[:filename])
+      end
+
+      graph = Graph.new(workflow, **opts)
       graph.viz
-      Launchy.open graph.path
+
+      if (options[:open].nil? && !options[:filename]) || options[:open]
+        Launchy.open Pathname.new(graph.path).realpath.to_s
+      end
     end
 
     private
@@ -118,13 +148,14 @@ module Gush
 
     def load_gushfile
       file = client.configuration.gushfile
-      if !gushfile.exist?
-        raise Thor::Error, "#{file} not found, please add it to your project".colorize(:red)
+
+      unless gushfile.exist?
+        raise Thor::Error, Paint["#{file} not found, please add it to your project", :red]
       end
 
       load file.to_s
     rescue LoadError
-      raise Thor::Error, "failed to require #{file}".colorize(:red)
+      raise Thor::Error, Paint["failed to require #{file}", :red]
     end
   end
 end