cloudtasker 0.2.0 → 0.7.0

data/Rakefile

@@ -2,7 +2,13 @@
 
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
+require 'github_changelog_generator/task'
 
 RSpec::Core::RakeTask.new(:spec)
 
 task default: :spec
+
+GitHubChangelogGenerator::RakeTask.new :changelog do |config|
+  config.user = 'keypup-io'
+  config.project = 'cloudtasker'
+end

data/app/controllers/cloudtasker/application_controller.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module Cloudtasker
+  # Base Cloudtasker controller
   class ApplicationController < ActionController::Base
+    skip_before_action :verify_authenticity_token
   end
 end

data/app/controllers/cloudtasker/worker_controller.rb

@@ -16,16 +16,38 @@ module Cloudtasker
     # Run a worker from a Cloud Task payload
     #
     def run
-      Task.execute_from_payload!(request.params.slice(:worker, :args))
+      # Build payload
+      payload = request.params
+                       .slice(:worker, :job_id, :job_args, :job_meta)
+                       .merge(job_retries: job_retries)
+
+      # Process payload
+      WorkerHandler.execute_from_payload!(payload)
       head :no_content
+    rescue DeadWorkerError
+      # 205: job will NOT be retried
+      head :reset_content
     rescue InvalidWorkerError
+      # 404: Job will be retried
       head :not_found
-    rescue StandardError
+    rescue StandardError => e
+      # 404: Job will be retried
+      Cloudtasker.logger.error(e)
+      Cloudtasker.logger.error(e.backtrace.join("\n"))
       head :unprocessable_entity
     end
 
     private
 
+    #
+    # Extract the number of times this task failed at runtime.
+    #
+    # @return [Integer] The number of failures
+    #
+    def job_retries
+      request.headers[Cloudtasker::Config::RETRY_HEADER].to_i
+    end
+
     #
     # Authenticate incoming requests using a bearer token
     #

data/cloudtasker.gemspec

@@ -10,8 +10,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Arnaud Lachaume']
   spec.email         = ['arnaud.lachaume@keypup.io']
 
-  spec.summary       = 'Manage GCP Cloud Tasks in your app. (under development)'
-  spec.description   = 'Manage GCP Cloud Tasks in your app. (under development)'
+  spec.summary       = 'Background jobs for Ruby using Google Cloud Tasks (alpha)'
+  spec.description   = 'Background jobs for Ruby using Google Cloud Tasks (alpha)'
   spec.homepage      = 'https://github.com/keypup-io/cloudtasker'
   spec.license       = 'MIT'
 
@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(examples|test|spec|features)/}) }
   end
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -36,7 +36,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'jwt'
   spec.add_dependency 'redis'
 
+  spec.add_development_dependency 'appraisal'
   spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'github_changelog_generator'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop', '0.76.0'

data/docs/BATCH_JOBS.md

@@ -0,0 +1,66 @@
+# Cloudtasker Unique Jobs
+
+**Note**: this extension requires redis
+
+The Cloudtasker batch job extension allows to add sub-jobs to regular jobs. This adds the ability to enqueue a list of jobs and track their overall progression as a groupd of jobs (the batch). This extension allows jobs to define callbacks in their worker to track completion of the batch and take action based on that.
+
+## Configuration
+
+You can enable batch jobs by adding the following to your cloudtasker initializer:
+```ruby
+# The batch job extension is optional and must be explicitly required
+require 'cloudtasker/batch'
+
+Cloudtasker.configure do |config|
+  # Specify your redis url.
+  # Defaults to `redis://localhost:6379/0` if unspecified
+  config.redis = { url: 'redis://some-host:6379/0' }
+end
+```
+
+## Example
+
+The following example defines a worker that adds itself to the batch with different arguments then monitors the success of the batch.
+
+```ruby
+class BatchWorker
+  include Cloudtasker::Worker
+
+  def perform(level, instance)
+    3.times { |n| batch.add(self.class, level + 1, n) } if level < 2
+  end
+
+  # Invoked when any descendant (e.g. sub-sub job) is complete
+  def on_batch_node_complete(child)
+    logger.info("Direct or Indirect child complete: #{child.job_id}")
+  end
+
+  # Invoked when a direct descendant is complete
+  def on_child_complete(child)
+    logger.info("Direct child complete: #{child.job_id}")
+  end
+
+  # Invoked when all chidren have finished
+  def on_batch_complete
+    Rails.logger.info("Batch complete")
+  end
+end
+```
+
+## Available callbacks
+
+The following callbacks are available on your workers to track the progress of the batch:
+
+| Callback | Argument | Description |
+|------|-------------|-----------|
+| `on_batch_node_complete` | `The child job` | Invoked when any descendant (e.g. sub-sub job) successfully completess   |
+| `on_child_complete` | `The child job` | Invoked when a direct descendant successfully completes   |
+| `on_child_error` | `The child job` | Invoked when a child fails |
+| `on_child_dead` | `The child job` | Invoked when a child has exhausted all of its retries |s
+| `on_batch_complete` | none | Invoked when all chidren have finished or died  |
+
+## Batch completion
+
+Batches complete when all children have successfully completed or died (all retries exhausted).
+
+Jobs that fail in a batch will be retried based on the `max_retries` setting configured globally or on the worker itself. The batch will be considered `pending` while workers retry. Therefore it may be a good idea to reduce the number of retries on your workers using `cloudtasker_options max_retries: 5` to ensure your batches don't hang for too long.

data/docs/CRON_JOBS.md

@@ -0,0 +1,65 @@
+# Cloudtasker Cron Jobs
+
+**Note**: this extension requires redis
+
+The Cloudtasker cron job extension allows you to register workers to run at fixed intervals, using a cron expression. You can validate your cron expressions using [crontab.guru](https://crontab.guru).
+
+## Configuration
+
+You can schedule cron jobs by adding the following to your cloudtasker initializer:
+```ruby
+# The cron job extension is optional and must be explicitly required
+require 'cloudtasker/cron'
+
+Cloudtasker.configure do |config|
+  # Specify your redis url.
+  # Defaults to `redis://localhost:6379/0` if unspecified
+  config.redis = { url: 'redis://some-host:6379/0' }
+end
+
+# Specify all your cron jobs below. This will synchronize your list of cron jobs (cron jobs previously created and not listed below will be removed).
+unless Rails.env.test?
+  Cloudtasker::Cron::Schedule.load_from_hash!(
+    # Run job every minute
+    some_schedule_name: {
+      worker: 'SomeCronWorker',
+      cron: '* * * * *'
+    },
+    # Run job every hour on the fifteenth minute 
+    other_cron_schedule: {
+      worker: 'OtherCronWorker',
+      cron: '15 * * * *'
+    }
+  )
+end
+```
+
+## Using a configuration file
+
+You can maintain the list of cron jobs in a YAML file inside your config folder if you prefer:
+```yml
+# config/cloudtasker_cron.yml
+
+# Run job every minute
+some_schedule_name:
+  worker: 'SomeCronWorker'
+  cron: => '* * * * *'
+  
+# Run job every hour on the fifteenth minute 
+other_cron_schedule:
+  worker: 'OtherCronWorker'
+  cron: => '15 * * * *'
+```
+
+Then register the jobs inside your Cloudtasker initializer this way:
+```ruby
+# config/initializers/cloudtasker.rb
+
+# ... Cloudtasker configuration ...
+
+schedule_file = 'config/cloudtasker_cron.yml'
+if File.exist?(schedule_file) && !Rails.env.test?
+  Cloudtasker::Cron::Schedule.load_from_hash!(YAML.load_file(schedule_file))
+end
+```
+

data/docs/UNIQUE_JOBS.md

@@ -0,0 +1,127 @@
+# Cloudtasker Unique Jobs
+
+**Note**: this extension requires redis
+
+The Cloudtasker unique job extension allows you to define uniqueness rules for jobs you schedule or process based on job arguments.
+
+## Configuration
+
+You can enable unique jobs by adding the following to your cloudtasker initializer:
+```ruby
+# The unique job extension is optional and must be explicitly required
+require 'cloudtasker/unique_job'
+
+Cloudtasker.configure do |config|
+  # Specify your redis url.
+  # Defaults to `redis://localhost:6379/0` if unspecified
+  config.redis = { url: 'redis://some-host:6379/0' }
+end
+```
+
+## Example
+
+The following example defines a worker that prevents more than one instance to run at the same time for the set of provided arguments. Any identical job scheduled after the first one will be re-enqueued until the first job has finished running.
+
+```ruby
+class UniqAtRuntimeWorker
+  include Cloudtasker::Worker
+  
+  #
+  # lock: specify the phase during which a worker must be unique based on class and arguments.
+  # In this case the worker will be unique while it is processed.
+  # Other types of locks are available - see below the rest of the documentation.
+  #
+  # on_conflict: specify what to do if another identical instance enter the lock phase. 
+  # In this case the worker will be rescheduled until the lock becomes available.
+  #
+  cloudtasker_options lock: :while_executing, on_conflict: :reschedule
+
+  def perform(arg1, arg2)
+    sleep(10)
+  end
+end
+```
+
+Considering the worker and the code below:
+```ruby
+# Enqueue two jobs successively
+UniqAtRuntimeWorker.perform_async # Job 1
+UniqAtRuntimeWorker.perform_async # Job 2
+```
+
+The following will happen
+1) Cloud Tasks sends job 1 and job 2 for processing to Rails
+2) Job 1 acquires a `while_executing` lock
+3) Job 2 does not acquire the lock and moves to `on_conflict` which is `reschedule`
+4) Job 2 gets rescheduled in 5 seconds
+5) Job 1 keeps processing for 5 seconds
+6) Job 2 is re-sent by Cloud Tasks and cannot acquire the lock, therefore is rescheduled.
+7) Job 1 processes for another 5 seconds and finishes (total = 10 seconds of processing)
+8) Job 2 is re-sent by Cloud Tasks, can acquire the lock this time and starts processing
+
+## Available locks
+
+Below is the list of available locks that can be specified through the `cloudtasker_options lock: ...` configuration option.
+
+For each lock strategy the table specifies the lock period (start/end) and which `on_conflict` strategies are available.
+
+| Lock | Starts when | Ends when | On Conflict strategies |
+|------|-------------|-----------|------------------------|
+| `until_executing` | The job is scheduled | The job starts processing | `reject` (default) or `raise` |
+| `while_executing` | The job starts processing | The job ends processing | `reject` (default), `reschedule` or `raise` |
+| `until_executed` | The job is scheduled | The job ends processing | `reject` (default) or `raise` |
+
+## Available conflict strategies
+
+Below is the list of available conflict strategies can be specified through the `cloudtasker_options on_conflict: ...` configuration option.
+
+| Strategy | Available with | Description |
+|----------|----------------|----------------|
+| `reject` | All locks | This is the default strategy. The job will be discarded when a conflict occurs |
+| `raise` | All locks | A `Cloudtasker::UniqueJob::LockError` will be raised when a conflict occurs |
+| `reschedule` | `while_executing` | The job will be rescheduled 5 seconds later when a conflict occurs |
+
+## Configuring unique arguments
+
+By default Cloudtasker considers all job arguments to evaluate the uniqueness of a job. This behaviour is configurable per worker by defining a `unique_args` method on the worker itself returning the list of args defining uniqueness.
+
+Example 1: Uniqueness based on a subset of arguments
+```ruby
+class UniqBasedOnTwoArgsWorker
+  include Cloudtasker::Worker
+  
+  cloudtasker_options lock: :until_executed
+
+  # Only consider the first two args when evaluating uniqueness
+  def unique_args(args)
+    [arg[0], arg[1]]
+  end
+
+  def perform(arg1, arg2, arg3)
+    # ...
+  end
+end
+```
+
+Example 2: Uniqueness based on modified arguments
+```ruby
+class ModuloArgsWorker
+  include Cloudtasker::Worker
+  
+  cloudtasker_options lock: :until_executed
+
+  # The remainder of `some_int` modulo 5 will be considered for
+  # uniqueness instead of the full value of `some_int`
+  def unique_args(args)
+    [arg[0], arg[1], arg[2] % 5]
+  end
+
+  def perform(arg1, arg2, some_int)
+    # ...
+  end
+end
+```
+
+## Beware of default method arguments
+
+Default method arguments are ignored when evaluating worker uniqueness. See [this section](../../../#be-careful-with-default-arguments) for more details.

data/exe/cloudtasker

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'cloudtasker/cli'
+
+begin
+  Cloudtasker::CLI.run
+rescue StandardError => e
+  raise e if $DEBUG
+
+  warn e.message
+  warn e.backtrace.join("\n")
+  exit 1
+end

data/gemfiles/.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_RETRY: "1"

data/gemfiles/google_cloud_tasks_1.0.gemfile

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+# This file was generated by Appraisal
+
+source 'https://rubygems.org'
+
+gem 'google-cloud-tasks', '1.0'
+
+gemspec path: '../'

data/gemfiles/google_cloud_tasks_1.0.gemfile.lock

@@ -0,0 +1,263 @@
+PATH
+  remote: ..
+  specs:
+    cloudtasker (0.2.0)
+      activesupport
+      fugit
+      google-cloud-tasks
+      jwt
+      redis
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actioncable (6.0.1)
+      actionpack (= 6.0.1)
+      nio4r (~> 2.0)
+      websocket-driver (>= 0.6.1)
+    actionmailbox (6.0.1)
+      actionpack (= 6.0.1)
+      activejob (= 6.0.1)
+      activerecord (= 6.0.1)
+      activestorage (= 6.0.1)
+      activesupport (= 6.0.1)
+      mail (>= 2.7.1)
+    actionmailer (6.0.1)
+      actionpack (= 6.0.1)
+      actionview (= 6.0.1)
+      activejob (= 6.0.1)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 2.0)
+    actionpack (6.0.1)
+      actionview (= 6.0.1)
+      activesupport (= 6.0.1)
+      rack (~> 2.0)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actiontext (6.0.1)
+      actionpack (= 6.0.1)
+      activerecord (= 6.0.1)
+      activestorage (= 6.0.1)
+      activesupport (= 6.0.1)
+      nokogiri (>= 1.8.5)
+    actionview (6.0.1)
+      activesupport (= 6.0.1)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activejob (6.0.1)
+      activesupport (= 6.0.1)
+      globalid (>= 0.3.6)
+    activemodel (6.0.1)
+      activesupport (= 6.0.1)
+    activerecord (6.0.1)
+      activemodel (= 6.0.1)
+      activesupport (= 6.0.1)
+    activestorage (6.0.1)
+      actionpack (= 6.0.1)
+      activejob (= 6.0.1)
+      activerecord (= 6.0.1)
+      marcel (~> 0.3.1)
+    activesupport (6.0.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    appraisal (2.2.0)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    ast (2.4.0)
+    builder (3.2.3)
+    concurrent-ruby (1.1.5)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    crass (1.0.5)
+    diff-lcs (1.3)
+    erubi (1.9.0)
+    et-orbi (1.2.2)
+      tzinfo
+    faraday (0.17.0)
+      multipart-post (>= 1.2, < 3)
+    fugit (1.3.3)
+      et-orbi (~> 1.1, >= 1.1.8)
+      raabro (~> 1.1)
+    globalid (0.4.2)
+      activesupport (>= 4.2.0)
+    google-cloud-tasks (1.0.0)
+      google-gax (~> 1.3)
+      googleapis-common-protos (>= 1.3.9, < 2.0)
+      grpc-google-iam-v1 (~> 0.6.9)
+    google-gax (1.8.1)
+      google-protobuf (~> 3.9)
+      googleapis-common-protos (>= 1.3.9, < 2.0)
+      googleauth (~> 0.9)
+      grpc (~> 1.24)
+      rly (~> 0.2.3)
+    google-protobuf (3.10.1)
+    googleapis-common-protos (1.3.9)
+      google-protobuf (~> 3.0)
+      googleapis-common-protos-types (~> 1.0)
+      grpc (~> 1.0)
+    googleapis-common-protos-types (1.0.4)
+      google-protobuf (~> 3.0)
+    googleauth (0.10.0)
+      faraday (~> 0.12)
+      jwt (>= 1.4, < 3.0)
+      memoist (~> 0.16)
+      multi_json (~> 1.11)
+      os (>= 0.9, < 2.0)
+      signet (~> 0.12)
+    grpc (1.25.0)
+      google-protobuf (~> 3.8)
+      googleapis-common-protos-types (~> 1.0)
+    grpc-google-iam-v1 (0.6.9)
+      googleapis-common-protos (>= 1.3.1, < 2.0)
+      grpc (~> 1.0)
+    hashdiff (1.0.0)
+    i18n (1.7.0)
+      concurrent-ruby (~> 1.0)
+    jaro_winkler (1.5.4)
+    jwt (2.2.1)
+    loofah (2.3.1)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    mail (2.7.1)
+      mini_mime (>= 0.1.1)
+    marcel (0.3.3)
+      mimemagic (~> 0.3.2)
+    memoist (0.16.1)
+    method_source (0.9.2)
+    mimemagic (0.3.3)
+    mini_mime (1.0.2)
+    mini_portile2 (2.4.0)
+    minitest (5.13.0)
+    multi_json (1.14.1)
+    multipart-post (2.1.1)
+    nio4r (2.5.2)
+    nokogiri (1.10.5)
+      mini_portile2 (~> 2.4.0)
+    os (1.0.1)
+    parallel (1.19.0)
+    parser (2.6.5.0)
+      ast (~> 2.4.0)
+    public_suffix (4.0.1)
+    raabro (1.1.6)
+    rack (2.0.7)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails (6.0.1)
+      actioncable (= 6.0.1)
+      actionmailbox (= 6.0.1)
+      actionmailer (= 6.0.1)
+      actionpack (= 6.0.1)
+      actiontext (= 6.0.1)
+      actionview (= 6.0.1)
+      activejob (= 6.0.1)
+      activemodel (= 6.0.1)
+      activerecord (= 6.0.1)
+      activestorage (= 6.0.1)
+      activesupport (= 6.0.1)
+      bundler (>= 1.3.0)
+      railties (= 6.0.1)
+      sprockets-rails (>= 2.0.0)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.3.0)
+      loofah (~> 2.3)
+    railties (6.0.1)
+      actionpack (= 6.0.1)
+      activesupport (= 6.0.1)
+      method_source
+      rake (>= 0.8.7)
+      thor (>= 0.20.3, < 2.0)
+    rainbow (3.0.0)
+    rake (10.5.0)
+    redis (4.1.3)
+    rly (0.2.3)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.0)
+      rspec-support (~> 3.9.0)
+    rspec-expectations (3.9.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-rails (3.9.0)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      railties (>= 3.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.0)
+    rubocop (0.76.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.6)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 1.7)
+    rubocop-rspec (1.36.0)
+      rubocop (>= 0.68.1)
+    ruby-progressbar (1.10.1)
+    safe_yaml (1.0.5)
+    signet (0.12.0)
+      addressable (~> 2.3)
+      faraday (~> 0.9)
+      jwt (>= 1.5, < 3.0)
+      multi_json (~> 1.10)
+    sprockets (4.0.0)
+      concurrent-ruby (~> 1.0)
+      rack (> 1, < 3)
+    sprockets-rails (3.2.1)
+      actionpack (>= 4.0)
+      activesupport (>= 4.0)
+      sprockets (>= 3.0.0)
+    sqlite3 (1.4.1)
+    thor (0.20.3)
+    thread_safe (0.3.6)
+    timecop (0.9.1)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    unicode-display_width (1.6.0)
+    webmock (3.7.6)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+    websocket-driver (0.7.1)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.4)
+    zeitwerk (2.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  appraisal
+  bundler (~> 2.0)
+  cloudtasker!
+  google-cloud-tasks (= 1.0)
+  rails
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  rspec-rails
+  rubocop (= 0.76.0)
+  rubocop-rspec
+  sqlite3
+  timecop
+  webmock
+
+BUNDLED WITH
+   2.0.2