cloudtasker-tonix 0.1.0

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-slate

data/app/controllers/cloudtasker/worker_controller.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  # Handle execution of workers
+  class WorkerController < ActionController::Base
+    # No need for CSRF verification on API endpoints
+    skip_forgery_protection
+
+    # Authenticate all requests.
+    before_action :authenticate!
+
+    # Return 401 when API Token is invalid
+    rescue_from AuthenticationError do
+      head :unauthorized
+    end
+
+    # POST /cloudtasker/run
+    #
+    # Run a worker from a Cloud Task payload
+    #
+    def run
+      # Process payload
+      WorkerHandler.execute_from_payload!(payload)
+      head :no_content
+    rescue DeadWorkerError
+      # 205: job will NOT be retried
+      head :reset_content
+    rescue InvalidWorkerError
+      # 404: Job will be retried
+      head :not_found
+    rescue StandardError
+      # 422: Job will be retried
+      head 422
+    end
+
+    private
+
+    #
+    # Parse the request body and return the JSON payload
+    #
+    # @return [String] The JSON payload
+    #
+    def json_payload
+      @json_payload ||= begin
+        # Get raw body
+        content = request.body.read
+
+        # Decode content if the body is Base64 encoded
+        if request.headers[Cloudtasker::Config::ENCODING_HEADER].to_s.downcase == 'base64'
+          content = Base64.decode64(content)
+        end
+
+        # Return the content
+        content
+      end
+    end
+
+    #
+    # Parse the request body and return the actual job
+    # payload.
+    #
+    # @return [Hash] The job payload
+    #
+    def payload
+      # Return content parsed as JSON and add job retries count
+      @payload ||= JSON.parse(json_payload).merge(job_retries: job_retries, task_id: task_id)
+    end
+
+    #
+    # Extract the number of times this task failed at runtime.
+    #
+    # @return [Integer] The number of failures.
+    #
+    def job_retries
+      request.headers[Cloudtasker::Config::RETRY_HEADER].to_i
+    end
+
+    #
+    # Return the Google Cloud Task ID from headers.
+    #
+    # @return [String] The task ID.
+    #
+    def task_id
+      request.headers[Cloudtasker::Config::TASK_ID_HEADER]
+    end
+
+    #
+    # Authenticate incoming requests using a bearer token
+    #
+    # See Cloudtasker::Authenticator#verification_token
+    #
+    def authenticate!
+      if (signature = request.headers[Cloudtasker::Config::CT_SIGNATURE_HEADER])
+        # Verify content signature
+        Authenticator.verify_signature!(signature, json_payload)
+      else
+        # Get authorization token from custom header (since v0.14.0) or fallback to
+        # former authorization header (jobs enqueued by v0.13 and below)
+        bearer_token = request.headers[Cloudtasker::Config::CT_AUTHORIZATION_HEADER].to_s.split.last ||
+                       request.headers[Cloudtasker::Config::OIDC_AUTHORIZATION_HEADER].to_s.split.last
+
+        # Verify the token
+        Authenticator.verify!(bearer_token)
+      end
+    end
+  end
+end

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'cloudtasker'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/cloudtasker.gemspec

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cloudtasker/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'cloudtasker-tonix'
+  spec.version       = Cloudtasker::VERSION
+  spec.authors       = ['Meriton Xhymshiti']
+  spec.email         = ['meritonii1998@gmail.com']
+
+  spec.summary       = 'Fork of keypup-io/cloudtasker with custom changes'
+  spec.description   = 'Background jobs for Ruby using Google Cloud Tasks (beta)'
+  spec.homepage      = 'https://github.com/Tonixhymshiti/cloudtasker'
+  spec.license       = 'MIT'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/Tonixhymshiti/cloudtasker'
+  spec.metadata['changelog_uri'] = 'https://github.com/Tonixhymshiti/cloudtasker/master/tree/CHANGELOG.md'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(examples|test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.required_ruby_version = '>= 3.0'
+
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'connection_pool'
+  spec.add_dependency 'fugit'
+  spec.add_dependency 'google-cloud-tasks'
+  spec.add_dependency 'jwt'
+  spec.add_dependency 'redis'
+  spec.add_dependency 'retriable'
+
+  spec.metadata['rubygems_mfa_required'] = 'true'
+end

data/config/routes.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Cloudtasker::Engine.routes.draw do
+  post '/run', to: 'worker#run'
+end

data/docs/BATCH_JOBS.md

@@ -0,0 +1,144 @@
+# Cloudtasker Batch Jobs
+
+**Note**: this extension requires redis
+
+The Cloudtasker batch job extension allows to add sub-jobs to regular jobs. This adds the ability to enqueue a list of jobs and track their overall progression as a group of jobs (a "batch"). This extension allows jobs to define callbacks in their worker to track completion of the batch and take actions based on that.
+
+## Configuration
+
+You can enable batch jobs by adding the following to your cloudtasker initializer:
+```ruby
+# The batch job extension is optional and must be explicitly required
+require 'cloudtasker/batch'
+
+Cloudtasker.configure do |config|
+  # Specify your redis url.
+  # Defaults to `redis://localhost:6379/0` if unspecified
+  config.redis = { url: 'redis://some-host:6379/0' }
+end
+```
+
+## Example: Creating a new batch
+
+The following example defines a worker that adds itself to the batch with different arguments then monitors the success of the batch.
+
+```ruby
+class BatchWorker
+  include Cloudtasker::Worker
+
+  def perform(level, instance)
+    3.times { |n| batch.add(self.class, level + 1, n) } if level < 2
+  end
+
+  # Invoked when any descendant (e.g. sub-sub job) is complete
+  def on_batch_node_complete(child)
+    logger.info("Direct or Indirect child complete: #{child.job_id}")
+  end
+
+  # Invoked when a direct descendant is complete
+  def on_child_complete(child)
+    logger.info("Direct child complete: #{child.job_id}")
+  end
+
+  # Invoked when all chidren have finished
+  def on_batch_complete
+    Rails.logger.info("Batch complete")
+  end
+end
+```
+
+## Example: Expanding the parent batch
+**Note**: `parent_batch` is available since `0.12.0`
+
+```ruby
+# All the jobs will be attached to the top parent batch.
+class BatchWorker
+  include Cloudtasker::Worker
+
+  def perform(level, instance)
+    # Use existing parent_batch or create a new one
+    current_batch = parent_batch || batch
+
+    3.times { |n| current_batch.add(self.class, level + 1, n) } if level < 2
+  end
+
+  # Invoked when any descendant (e.g. sub-sub job) is complete
+  def on_batch_node_complete(child)
+    logger.info("Direct or Indirect child complete: #{child.job_id}")
+  end
+
+  # Invoked when a direct descendant is complete
+  def on_child_complete(child)
+    logger.info("Direct child complete: #{child.job_id}")
+  end
+
+  # Invoked when all chidren have finished
+  def on_batch_complete
+    Rails.logger.info("Batch complete")
+  end
+end
+```
+
+## Available callbacks
+
+The following callbacks are available on your workers to track the progress of the batch:
+
+| Callback | Argument | Description |
+|------|-------------|-----------|
+| `on_batch_node_complete` | `The child job` | Invoked when any descendant (e.g. sub-sub job) successfully completes   |
+| `on_child_complete` | `The child job` | Invoked when a direct descendant successfully completes   |
+| `on_child_error` | `The child job` | Invoked when a child fails |
+| `on_child_dead` | `The child job` | Invoked when a child has exhausted all of its retries |s
+| `on_batch_complete` | none | Invoked when all chidren have finished or died  |
+
+## Queue management
+
+Jobs added to a batch inherit the queue of the parent. It is possible to specify a different queue when adding a job to a batch using `add_to_queue` batch method.
+
+E.g.
+
+```ruby
+def perform
+  batch.add_to_queue(:critical, SubWorker, arg1, arg2, arg3)
+end
+```
+
+## Batch completion
+
+Batches complete when all children have successfully completed or died (all retries exhausted).
+
+Jobs that fail in a batch will be retried based on the `max_retries` setting configured globally or on the worker itself. The batch will be considered `pending` while workers retry. Therefore it may be a good idea to reduce the number of retries on your workers using `cloudtasker_options max_retries: 5` to ensure your batches don't hang for too long.
+
+## Batch progress tracking
+
+You can access progression statistics in callback using `batch.progress`. See the [BatchProgress](../lib/cloudtasker/batch/batch_progress.rb) class for more details.
+
+E.g.
+```ruby
+def on_batch_node_complete(_child_job)
+  progress = batch.progress
+  logger.info("Total: #{progress.total}")
+  logger.info("Completed: #{progress.completed}")
+  logger.info("Progress: #{progress.percent.to_i}%")
+end
+```
+
+**Since:** `v0.12.0`  
+By default the `progress` method only considers the direct child jobs to evaluate the batch progress. You can pass `depth: somenumber` to the `progress` method to calculate the actual batch progress in a more granular way. Be careful however that this method recursively calculates progress on the sub-batches and is therefore expensive.
+
+E.g.
+```ruby
+def on_batch_node_complete(_child_job)
+  # Considers the children for batch progress calculation
+  progress_0 = batch.progress # same as batch.progress(depth: 0)
+
+  # Considers the children and grand-children for batch progress calculation
+  progress_1 = batch.progress(depth: 1)
+
+  # Considers the children, grand-children and grand-grand-children for batch progress calculation
+  progress_2 = batch.progress(depth: 3)
+
+  logger.info("Progress: #{progress_1.percent.to_i}%")
+  logger.info("Progress: #{progress_2.percent.to_i}%")
+end
+```

data/docs/CRON_JOBS.md

@@ -0,0 +1,129 @@
+# Cloudtasker Cron Jobs
+
+**Note**: this extension requires redis
+
+The Cloudtasker cron job extension allows you to register workers to run at fixed intervals, using a cron expression. You can validate your cron expressions using [crontab.guru](https://crontab.guru).
+
+## Configuration
+
+You can schedule cron jobs by adding the following to your cloudtasker initializer:
+```ruby
+# The cron job extension is optional and must be explicitly required
+require 'cloudtasker/cron'
+
+Cloudtasker.configure do |config|
+  # Specify your redis url.
+  # Defaults to `redis://localhost:6379/0` if unspecified
+  config.redis = { url: 'redis://some-host:6379/0' }
+end
+
+# Specify all your cron jobs below. This will synchronize your list of cron jobs (cron jobs previously created and not listed below will be removed).
+unless Rails.env.test?
+  Cloudtasker::Cron::Schedule.load_from_hash!(
+    # Run job every minute
+    some_schedule_name: {
+      worker: 'SomeCronWorker',
+      cron: '* * * * *'
+    },
+    # Run job every hour on the fifteenth minute 
+    other_cron_schedule: {
+      worker: 'OtherCronWorker',
+      cron: '15 * * * *',
+      queue: 'critical'
+      args: ['foo', 'bar']
+    }
+  )
+end
+```
+
+## Using a configuration file
+
+You can maintain the list of cron jobs in a YAML file inside your config folder if you prefer:
+```yml
+# config/cloudtasker_cron.yml
+
+# Run job every minute
+some_schedule_name:
+  worker: 'SomeCronWorker'
+  cron: '* * * * *'
+  
+# Run job every hour on the fifteenth minute 
+other_cron_schedule:
+  worker: 'OtherCronWorker'
+  cron: '15 * * * *'
+```
+
+Then register the jobs inside your Cloudtasker initializer this way:
+```ruby
+# config/initializers/cloudtasker.rb
+
+# ... Cloudtasker configuration ...
+
+schedule_file = 'config/cloudtasker_cron.yml'
+if File.exist?(schedule_file) && !Rails.env.test?
+  Cloudtasker::Cron::Schedule.load_from_hash!(YAML.load_file(schedule_file))
+end
+```
+
+## With Puma Cluster-mode
+Due to this issue with gRPC here: https://github.com/grpc/grpc/issues/7951.
+
+TLTR: 
+> Forking processes and using gRPC across processes is not supported behavior due to very low-level resource issues. Either delay your use of gRPC until you've forked from fresh processes (similar to Python 3's use of a zygote process), or don't expect things to work after a fork.
+
+In order to make it works, we should schedule cron jobs (which triggers gPRC calls) once puma is booted.
+
+Example:
+```ruby
+config/puma.rb
+
+workers ENV.fetch("WEB_CONCURRENCY") { 2 }
+preload_app!
+
+on_booted do
+  schedule_file = "config/cloudtasker_cron.yml"
+  if File.exist?(schedule_file) && !Rails.env.test?
+    Cloudtasker::Cron::Schedule.load_from_hash!(YAML.load_file(schedule_file))
+  end
+end
+```
+
+## Limitations
+GCP Cloud Tasks does not allow tasks to be scheduled more than 30 days (720h) in the future. Cron schedules should therefore be limited to 30 days intervals at most.
+
+If you need to schedule a job to run on a monthly basis (e.g. on the first of the month), schedule this job to run every day then add the following logic in your job:
+```ruby
+#
+# Cron schedule (8am UTC every day): 0 8 * * *
+#
+class MyMonthlyWorker
+  include Cloudtasker::Worker
+
+  def perform(*args)
+    # Abort unless we're the first of the month
+    return unless Time.current.day == 1
+
+    # ... job logic
+  end
+end
+```
+
+The same approach can be used to schedule a job every quarter.
+```ruby
+#
+# Cron schedule (8am UTC every day): 0 8 * * *
+#
+class MyQuarterlyWorker
+  include Cloudtasker::Worker
+
+  def perform(*args)
+    # Abort unless we're the first month of a quarter (Jan, Apr, Jul, Oct)
+    return unless Time.current.month == 1
+    
+    # Abort unless we're the first of the month
+    return unless Time.current.day == 1
+
+    # ... job logic
+  end
+end
+```

data/docs/STORABLE_JOBS.md

@@ -0,0 +1,68 @@
+# Cloudtasker Storable Jobs
+
+**Supported since**: `v0.14.0`  
+**Note**: this extension requires redis
+
+The Cloudtasker storage extension allows you to park jobs in a specific garage lane and enqueue (pull) them when specific conditions have been met.
+
+This extension is useful when you need to prepare some jobs (e.g. you are retrieving data from an API and must process some of it asynchronously) but only process them when some programmatic conditions have been met (e.g. a series of preliminary preparation jobs have run successfully). Using parked jobs is a leaner (and cheaper) approach than using guard logic in the `perform` method to re-enqueue a job until a set of conditions is satisfied. The latter tends to generate a lot of jobs/logs pollution.
+
+## Configuration
+
+You can enable storable jobs by adding the following to your cloudtasker initializer:
+```ruby
+# The storable extension is optional and must be explicitly required
+require 'cloudtasker/storable'
+
+Cloudtasker.configure do |config|
+  # Specify your redis url.
+  # Defaults to `redis://localhost:6379/0` if unspecified
+  config.redis = { url: 'redis://some-host:6379/0' }
+end
+```
+
+Then you can make workers storable by including the `Cloudtasker::Storable::Worker` concern into your workers:
+```ruby
+class MyWorker
+  include Cloudtasker::Worker
+  include Cloudtasker::Storable::Worker
+
+  def perform(...)
+    # Do stuff
+  end
+end
+```
+
+## Parking jobs
+You can park jobs to a specific garage lane using the `push_to_store(store_name, *worker_args)` class method:
+```ruby
+MyWorker.push_to_store('some-customer-reference:some-task-group', job_arg1, job_arg2)
+```
+
+## Pulling jobs
+You can pull and enqueue jobs using the `pull_all_from_store(store_name)` class method:
+```ruby
+MyWorker.pull_all_from_store('some-customer-reference:some-task-group')
+```
+
+If you need to enqueue jobs with specific options or using any special means, you can call `pull_all_from_store(store_name)` with a block. When a block is passed the method yield each worker's set of arguments.
+```ruby
+# Delay the enqueuing of parked jobs by 30 seconds
+MyWorker.pull_all_from_store('some-customer-reference:some-task-group') do |args|
+  MyWorker.perform_in(30, *args)
+end
+
+# Enqueue parked jobs on a specific queue, with a 10s delay
+MyWorker.pull_all_from_store('some-customer-reference:some-task-group') do |args|
+  MyWorker.schedule(args: args, time_in: 10, queue: 'critical')
+end
+
+# Enqueue parked jobs as part of a job's current batch (the logic below assumes
+# we are inside a job's `perform` method)
+MyWorker.pull_all_from_store('some-customer-reference:some-task-group') do |args|
+  batch.add(MyWorker, *args)
+
+  # Or with a specific queue
+  # batch.add_to_queue('critical', SubWorker, *args)
+end
+```