gush 2.1.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc70498c39ad506749bed1f55c139d29691e540b849dd39e2ba132b54b511d66
-  data.tar.gz: 700d093574e0ff4772c7d36bfeb7792cda44550dd162a48afe17bec5551250cd
+  metadata.gz: 0b1ccfa69ff21ec6820d9178edb51872cdbf28b7686bcf3a4de4170e3bc7f370
+  data.tar.gz: c44dae0cc5480366a3cf65ccd461ca55c991945ee090dfa3a386061f66001e09
 SHA512:
-  metadata.gz: ac4c1647f57a87600466445a8d24660e3d95016f17adf9e2fe8f99260ba82a2b3e22a339b9456f653834c89c6e4b9ed6bcb50ebc337b804e07355f9a513ce8ca
-  data.tar.gz: 1ae01511acfc2e23bb0a671328a7b3e9b650b8589b9620461c2b9667ff9d80a47f03b8a156f249746b4f69e613863db0833be8d6221acdabd211d7eba2daee15
+  metadata.gz: 474efac8af5d337283f29fbb7c4e9027690519c413f4d71c08d1d4c36aa78bc805d68b199fbf4395744854d40e238b19c570f0c2daa6c0676efd58f746b38b1c
+  data.tar.gz: c8635120b6071458cf6f99542f846adb806a4233e17b4a77444acd191763b470e2d41560f14b17e5b59d6dfeb514c52933eac05ce5119ca590f12b07209626b9

data/.github/workflows/ruby.yml

@@ -26,35 +26,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        rails_version: ['4.2.7', '5.1.0', '5.2.0', '6.0.0', '6.1.0', '7.0']
-        ruby-version: ['2.6', '2.7', '3.0', '3.1']
-        exclude:
-          - ruby-version: '3.0'
-            rails_version: '4.2.7'
-          - ruby-version: '3.1'
-            rails_version: '4.2.7'
-          - ruby-version: '3.0'
-            rails_version: '5.0'
-          - ruby-version: '3.1'
-            rails_version: '5.0'
-          - ruby-version: '3.0'
-            rails_version: '5.1'
-          - ruby-version: '3.1'
-            rails_version: '5.1'
-          - ruby-version: '3.0'
-            rails_version: '5.2'
-          - ruby-version: '3.1'
-            rails_version: '5.2'
-          - ruby-version: '3.0'
-            rails_version: '6.0'
-          - ruby-version: '3.1'
-            rails_version: '6.0'
-          - ruby-version: '3.1'
-            rails_version: '6.1'
-          - ruby-version: '2.6'
-            rails_version: '7.0'
+        rails_version: ['6.1.0', '7.0', '7.1.0']
+        ruby-version: ['3.0', '3.1', '3.2', '3.3']
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
@@ -64,6 +39,10 @@ jobs:
         RAILS_VERSION: "${{ matrix.rails_version }}"
     - name: Install Graphviz
       run: sudo apt-get install graphviz
+    - name: Run code lint
+      run: bundle exec rubocop
+      env:
+        RAILS_VERSION: "${{ matrix.rails_version }}"
     - name: Run tests
       run: bundle exec rspec
       env:

data/.rubocop.yml

@@ -0,0 +1,232 @@
+require:
+  - rubocop-rake
+  - rubocop-rspec
+
+AllCops:
+  NewCops: disable
+
+Gemspec/OrderedDependencies:
+  Enabled: false
+
+Layout/ArgumentAlignment:
+  Enabled: false
+
+Layout/CaseIndentation:
+  Enabled: false
+
+Layout/EmptyLinesAroundBlockBody:
+  Enabled: false
+
+Layout/ExtraSpacing:
+  Enabled: false
+
+Layout/FirstHashElementIndentation:
+  Enabled: false
+
+Layout/HashAlignment:
+  Enabled: false
+
+Layout/SpaceAroundEqualsInParameterDefault:
+  Enabled: false
+
+Layout/SpaceAroundOperators:
+  Enabled: false
+
+Layout/SpaceBeforeBlockBraces:
+  Enabled: false
+
+Layout/SpaceInsideBlockBraces:
+  Enabled: false
+
+Layout/SpaceInsideHashLiteralBraces:
+  Enabled: false
+
+Lint/ConstantDefinitionInBlock:
+  Exclude:
+    - spec/**/*
+
+Lint/RedundantSplatExpansion:
+  Enabled: false
+
+Lint/ToJSON:
+  Enabled: false
+
+Lint/UnusedBlockArgument:
+  Enabled: false
+
+Lint/UnusedMethodArgument:
+  Enabled: false
+
+Lint/UselessAssignment:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Naming/MemoizedInstanceVariableName:
+  Enabled: false
+
+Naming/PredicateName:
+  Enabled: false
+
+Naming/RescuedExceptionsVariableName:
+  Enabled: false
+
+Style/BlockDelimiters:
+  Enabled: false
+
+Style/ClassVars:
+  Enabled: false
+
+Style/CombinableLoops:
+  Enabled: false
+
+Style/ConditionalAssignment:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/EmptyCaseCondition:
+  Enabled: false
+
+Style/EmptyMethod:
+  Enabled: false
+
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+Style/GuardClause:
+  Enabled: false
+
+Style/HashSyntax:
+  Enabled: false
+
+Style/IfUnlessModifier:
+  Enabled: false
+
+Style/InverseMethods:
+  Enabled: false
+
+Style/MethodCallWithoutArgsParentheses:
+  Enabled: false
+
+Style/NumericLiteralPrefix:
+  Enabled: false
+
+Style/PercentLiteralDelimiters:
+  Enabled: false
+
+Style/RaiseArgs:
+  Enabled: false
+
+Style/SafeNavigation:
+  Enabled: false
+
+Style/SpecialGlobalVars:
+  Enabled: false
+
+Style/StringLiterals:
+  Enabled: false
+
+Style/SymbolProc:
+  Enabled: false
+
+Style/UnlessElse:
+  Enabled: false
+
+Style/WordArray:
+  Enabled: false
+
+Layout/LineLength:
+  Enabled: false
+
+RSpec/AnyInstance:
+  Enabled: false
+
+RSpec/BeEq:
+  Enabled: false
+
+RSpec/ContextWording:
+  Enabled: false
+
+RSpec/DescribedClass:
+  Enabled: false
+
+RSpec/EmptyExampleGroup:
+  Enabled: false
+
+RSpec/EmptyLineAfterExampleGroup:
+  Enabled: false
+
+RSpec/EmptyLineAfterSubject:
+  Enabled: false
+
+RSpec/ExampleLength:
+  Enabled: false
+
+RSpec/ExampleWording:
+  Enabled: false
+
+RSpec/ExpectChange:
+  Enabled: false
+
+RSpec/HookArgument:
+  EnforcedStyle: each
+
+RSpec/LeakyConstantDeclaration:
+  Enabled: false
+
+RSpec/LetSetup:
+  Enabled: false
+
+RSpec/MatchArray:
+  Enabled: false
+
+RSpec/MessageSpies:
+  EnforcedStyle: receive
+
+RSpec/MultipleExpectations:
+  Enabled: false
+
+RSpec/MultipleMemoizedHelpers:
+  Enabled: false
+
+RSpec/NamedSubject:
+  Enabled: false
+
+RSpec/NestedGroups:
+  Enabled: false
+
+RSpec/NotToNot:
+  Enabled: false
+
+RSpec/PredicateMatcher:
+  Enabled: false
+
+RSpec/ReceiveCounts:
+  Enabled: false
+
+RSpec/SpecFilePathFormat:
+  Enabled: false
+
+RSpec/StubbedMock:
+  Enabled: false
+
+RSpec/SubjectStub:
+  Enabled: false
+
+RSpec/VerifiedDoubles:
+  Enabled: false

data/CHANGELOG.md

@@ -1,79 +1,3 @@
 # Changelog
 
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
-and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
-
-## 2.1.0
-
-### Added
-
-- Allow RedisMutex’s locking duration and polling interval to be customizable, thanks to @thukim! [See pull request](https://github.com/chaps-io/gush/pull/74)
-- Support for Rails 7.0 and Ruby 3.0-3.1, thanks to @joshRpowell and @kzkn!
-
-## 2.0.1
-
-### Fixed
-
-- Fix bug when retried jobs didn't correctly reset their failed flag when ran again (Thanks to @theo-delaune-argus and @mickael-palma-argus! [See issue](https://github.com/chaps-io/gush/issues/61))
-
-## 2.0.0
-
-### Changed
-
-- *[BREAKING]* Store gush jobs on redis hash instead of plain keys - this improves performance when retrieving keys (Thanks to @Saicheg! [See pull request](https://github.com/chaps-io/gush/pull/56))
-
-
-### Added
-
-- Allow setting queue for each job via `:queue` option in `run` method (Thanks to @devilankur18! [See pull request](https://github.com/chaps-io/gush/pull/58))
-
-
-## 1.1.1 - 2018-06-09
-
-### Changed
-
-- Relax dependency on ActiveSupport to work with 4.2 up to 5.X (Thanks to @iacobus! [See pull request](https://github.com/chaps-io/gush/pull/54))
-
-
-## 1.1.0 - 2018-02-05
-
-### Added
-
-- Added ability to specify TTL for Redis keys and manually expire whole workflows (Thanks to @dmitrypol! [See pull request](https://github.com/chaps-io/gush/pull/48))
-- Loosened dependency on redis-rb library to >= 3.2 and < 5.0 (Thanks to @mofumofu3n! [See pull request](https://github.com/chaps-io/gush/pull/52))
-
-### Fixed
-
-- Improved performance of (de)serializing workflows by not storing job array inside workflow JSON and other smaller improvements ([See pull request](https://github.com/chaps-io/gush/pull/53))
-
-
-## 1.0.0 - 2017-10-02
-
-### Added
-
--  **BREAKING CHANGE** Gush now uses ActiveJob instead of directly Sidekiq, this allows programmers to use multiple backends, instead of just one. Including in-process or even synchronous backends. See http://guides.rubyonrails.org/active_job_basics.html
-
-### Fixed
-
-- Fix graph rendering with `gush viz` command. Sometimes it rendered the last job detached from others, because it was using a class name instead of job name as ID.
-- Fix performance problems with unserializing jobs. This greatly **increased performance** by avoiding redundant calls to Redis storage. Should help a lot with huge workflows spawning thousands of jobs. Previously each job loaded whole workflow instance when executed.
-
-### Changed
-
-- **BREAKING CHANGE** `Gushfile.rb` is now renamed to `Gushfile`
-- **BREAKING CHANGE** Internal code for reporting status via Redis pub/sub has been removed, since it wasn't used for a long time.
-- **BREAKING CHANGE** jobs are expected to have a `perform` method instead of `work` like in < 1.0.0 versions.
-- **BREAKING CHANGE** `payloads` method available inside jobs is now an array of hashes, instead of a hash, this allows for a more flexible approach to reusing a single job in many situations. Previously payloads were grouped by predecessor's class name, so you were forced to hardcode that class name in its descendants' code.
-
-### Removed
-
-- `gush workers` command is now removed. This is now up to the developer to start background processes depending on chosen ActiveJob adapter.
-- `environment` was removed since it was no longer needed (it was Sidekiq specific)
-
-## 0.4.0
-
-### Removed
-
-- remove hard dependency on Yajl, so Gush can work with non-MRI Rubies ([#31](https://github.com/chaps-io/gush/pull/31) by [Nick Rakochy](https://github.com/chaps-io/gush/pull/31))
+Changes for each release are available at https://github.com/chaps-io/gush/releases

data/README.md

@@ -1,5 +1,9 @@
 # Gush
 
+![Gem Version](https://img.shields.io/gem/v/gush)
+![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/chaps-io/gush/ruby.yml)
+
+
 Gush is a parallel workflow runner using only Redis as storage and [ActiveJob](http://guides.rubyonrails.org/v4.2/active_job_basics.html#introduction) for scheduling and executing jobs.
 
 ## Theory
@@ -15,7 +19,7 @@ This README is about the latest `master` code, which might differ from what is r
 ### 1. Add `gush` to Gemfile
 
 ```ruby
-gem 'gush', '~> 2.0'
+gem 'gush', '~> 3.0'
 ```
 
 ### 2. Create `Gushfile`
@@ -74,7 +78,17 @@ end
 
 and this is how the graph will look like:
 
-![SampleWorkflow](https://i.imgur.com/DFh6j51.png)
+```mermaid
+graph TD
+    A{Start} --> B[FetchJob1]
+    A --> C[FetchJob2]
+    B --> D[PersistJob1]
+    C --> E[PersistJob2]
+    D --> F[NormalizeJob]
+    E --> F
+    F --> G[IndexJob]
+    G --> H{Finish}
+```    
 
 
 ## Defining workflows
@@ -145,9 +159,11 @@ Let's assume we are writing a book publishing workflow which needs to know where
 
 ```ruby
 class PublishBookWorkflow < Gush::Workflow
-  def configure(url, isbn)
+  def configure(url, isbn, publish: false)
     run FetchBook, params: { url: url }
-    run PublishBook, params: { book_isbn: isbn }, after: FetchBook
+    if publish
+      run PublishBook, params: { book_isbn: isbn }, after: FetchBook
+    end
   end
 end
 ```
@@ -155,7 +171,7 @@ end
 and then create your workflow with those arguments:
 
 ```ruby
-PublishBookWorkflow.create("http://url.com/book.pdf", "978-0470081204")
+PublishBookWorkflow.create("http://url.com/book.pdf", "978-0470081204", publish: true)
 ```
 
 and that's basically it for defining workflows, see below on how to define jobs:
@@ -211,7 +227,7 @@ For example, in case of Sidekiq this would be:
 bundle exec sidekiq -q gush
 ```
 
-**[Click here to see backends section in official ActiveJob documentation about configuring backends](http://guides.rubyonrails.org/v4.2/active_job_basics.html#backends)**
+**[Click here to see backends section in official ActiveJob documentation about configuring backends](http://guides.rubyonrails.org/active_job_basics.html#backends)**
 
 **Hint**: gush uses `gush` queue name by default. Keep that in mind, because some backends (like Sidekiq) will only run jobs from explicitly stated queues.
 
@@ -240,8 +256,53 @@ flow.status
 
 `reload` is needed to see the latest status, since workflows are updated asynchronously.
 
+## Loading workflows
+
+### Finding a workflow by id
+
+```
+flow = Workflow.find(id)
+```
+
+### Paging through workflows
+
+To get workflows with pagination, use start and stop (inclusive) index values:
+
+```
+flows = Workflow.page(0, 99)
+```
+
+Or in reverse order:
+
+```
+flows = Workflow.page(0, 99, order: :desc)
+```
+
 ## Advanced features
 
+### Global parameters for jobs
+
+Workflows can accept a hash of `globals` that are automatically forwarded as parameters to all jobs.
+
+This is useful to have common functionality across workflow and job classes, such as tracking the creator id for all instances:
+
+```ruby
+class SimpleWorkflow < Gush::Workflow
+  def configure(url_to_fetch_from)
+    run DownloadJob, params: { url: url_to_fetch_from }
+  end
+end
+
+flow = SimpleWorkflow.create('http://foo.com', globals: { creator_id: 123 })
+flow.globals
+=> {:creator_id=>123}
+flow.jobs.first.params
+=> {:creator_id=>123, :url=>"http://foo.com"}
+```
+
+**Note:** job params with the same key as globals will take precedence over the globals.
+
+
 ### Pipelining
 
 Gush offers a useful tool to pass results of a job to its dependencies, so they can act differently.
@@ -319,7 +380,21 @@ flow = NotifyWorkflow.create([54, 21, 24, 154, 65]) # 5 user ids as an argument
 
 it will generate a workflow with 5 `NotificationJob`s and one `AdminNotificationJob` which will depend on all of them:
 
-![DynamicWorkflow](https://i.imgur.com/HOI3fjc.png)
+
+```mermaid
+graph TD
+    A{Start} --> B[NotificationJob]
+    A --> C[NotificationJob]
+    A --> D[NotificationJob]
+    A --> E[NotificationJob]
+    A --> F[NotificationJob]
+    B --> G[AdminNotificationJob]
+    C --> G
+    D --> G
+    E --> G
+    F --> G
+    G --> H{Finish}
+```
 
 ### Dynamic queue for jobs
 
@@ -338,6 +413,54 @@ class NotifyWorkflow < Gush::Workflow
 end
 ```
 
+### Dynamic waitable time for jobs
+
+There might be a case you want to configure a job to be executed after a time. Based on above example, we want to configure `AdminNotificationJob` to be executed after 5 seconds.
+
+```ruby
+
+class NotifyWorkflow < Gush::Workflow
+  def configure(user_ids)
+    notification_jobs = user_ids.map do |user_id|
+      run NotificationJob, params: {user_id: user_id}, queue: 'user'
+    end
+
+    run AdminNotificationJob, after: notification_jobs, queue: 'admin', wait: 5.seconds
+  end
+end
+```
+
+### Customization of ActiveJob enqueueing
+
+There might be a case when you want to customize enqueing a job with more than just the above two options (`queue` and `wait`).
+
+To pass additional options to `ActiveJob.set`, override `Job#worker_options`, e.g.:
+
+```ruby
+
+class ScheduledJob < Gush::Job
+
+  def worker_options
+    super.merge(wait_until: Time.at(params[:start_at]))
+  end
+
+end
+```
+
+Or to entirely customize the ActiveJob integration, override `Job#enqueue_worker!`, e.g.:
+
+```ruby
+
+class SynchronousJob < Gush::Job
+
+  def enqueue_worker!(options = {})
+    Gush::Worker.perform_now(workflow_id, name)
+  end
+
+end
+```
+
+
 ## Command line interface (CLI)
 
 ### Checking status
@@ -348,12 +471,18 @@ end
   bundle exec gush show <workflow_id>
   ```
 
-- of all created workflows:
+- of a page of workflows:
 
   ```
   bundle exec gush list
   ```
 
+- of the most recent 100 workflows
+
+  ```
+  bundle exec gush list -99 -1
+  ```
+
 ### Vizualizing workflows as image
 
 This requires that you have imagemagick installed on your computer:
@@ -379,7 +508,9 @@ end
 
 ### Cleaning up afterwards
 
-Running `NotifyWorkflow.create` inserts multiple keys into Redis every time it is ran.  This data might be useful for analysis but at a certain point it can be purged via Redis TTL.  By default gush and Redis will keep keys forever.  To configure expiration you need to 2 things.  Create initializer (specify config.ttl in seconds, be different per environment).
+Running `NotifyWorkflow.create` inserts multiple keys into Redis every time it is run.  This data might be useful for analysis but at a certain point it can be purged.  By default gush and Redis will keep keys forever.  To configure expiration you need to do two things.
+
+1. Create an initializer that specifies `config.ttl` in seconds. Best NOT to set TTL to be too short (like minutes) but about a week in length.
 
 ```ruby
 # config/initializers/gush.rb
@@ -390,7 +521,9 @@ Gush.configure do |config|
 end
 ```
 
-And you need to call `flow.expire!` (optionally passing custom TTL value overriding `config.ttl`).  This gives you control whether to expire data for specific workflow.  Best NOT to set TTL to be too short (like minutes) but about a week in length.  And you can run `Client.expire_workflow` and `Client.expire_job` passing appropriate IDs and TTL (pass -1 to NOT expire) values.
+2. Call `Client#expire_workflows` periodically, which will clear all expired stored workflow and job data and indexes. This method can be called at any rate, but ideally should be called at least once for every 1000 workflows created.
+
+If you need more control over individual workflow expiration, you can call `flow.expire!(ttl)` with a TTL different from the Gush configuration, or with -1 to never expire the workflow.
 
 ### Avoid overlapping workflows
 
@@ -408,6 +541,19 @@ def find_by_class klass
 end
 ```
 
+## Gush 3.0 Migration
+
+Gush 3.0 adds indexing for fast workflow pagination and changes the mechanism for expiring workflow data from Redis.
+
+### Migration
+
+Run `bundle exec gush migrate` after upgrading. This will update internal data structures.
+
+### Expiration API
+
+Periodically run `Gush::Client.new.expire_workflows` to expire data. Workflows will be automatically enrolled in this expiration, so there is no longer a need to call `workflow.expire!`.
+
+
 ## Contributors
 
 - [Mateusz Lenik](https://github.com/mlen)

data/gush.gemspec

@@ -1,12 +1,13 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
+require_relative 'lib/gush/version'
+
 Gem::Specification.new do |spec|
   spec.name          = "gush"
-  spec.version       = "2.1.0"
-  spec.authors       = ["Piotrek Okoński"]
-  spec.email         = ["piotrek@okonski.org"]
+  spec.version       = Gush::VERSION
+  spec.authors       = ["Piotrek Okoński", "Michał Krzyżanowski"]
+  spec.email         = ["piotrek@okonski.org", "michal.krzyzanowski+github@gmail.com"]
   spec.summary       = "Fast and distributed workflow runner based on ActiveJob and Redis"
   spec.description   = "Gush is a parallel workflow runner using Redis as storage and ActiveJob for executing jobs."
   spec.homepage      = "https://github.com/chaps-io/gush"
@@ -16,11 +17,12 @@ Gem::Specification.new do |spec|
   spec.executables   = "gush"
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
+  spec.required_ruby_version = '>= 3.0.0'
 
-  spec.add_dependency "activejob", ">= 4.2.7", "< 7.1"
+  spec.add_dependency "activejob", ">= 6.1.0", "< 7.2"
   spec.add_dependency "concurrent-ruby", "~> 1.0"
   spec.add_dependency "multi_json", "~> 1.11"
-  spec.add_dependency "redis", ">= 3.2", "< 5"
+  spec.add_dependency "redis", ">= 3.2", "< 6"
   spec.add_dependency "redis-mutex", "~> 4.0.1"
   spec.add_dependency "hiredis", "~> 0.6"
   spec.add_dependency "graphviz", "~> 1.2"
@@ -29,7 +31,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency "thor", ">= 0.19", "< 1.3"
   spec.add_dependency "launchy", "~> 2.4"
   spec.add_development_dependency "bundler"
-  spec.add_development_dependency "rake", "~> 10.4"
+  spec.add_development_dependency "rake", "~> 12"
+  spec.add_development_dependency "rubocop", '~> 1.65.0'
+  spec.add_development_dependency "rubocop-rake", '~> 0.6.0'
+  spec.add_development_dependency "rubocop-rspec", '~> 3.0.3'
   spec.add_development_dependency "rspec", '~> 3.0'
   spec.add_development_dependency "pry", '~> 0.10'
 end

data/lib/gush/cli/overview.rb

@@ -34,6 +34,7 @@ module Gush
       end
 
       private
+
       def rows
         [].tap do |rows|
           columns.each_pair do |name, value|
@@ -91,6 +92,7 @@ module Gush
 
       def jobs_by_type(type)
         return sorted_jobs if type == :all
+
         jobs.select{|j| j.public_send("#{type}?") }
       end