cloudtasker 0.1.0

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/app/controllers/cloudtasker/application_controller.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  class ApplicationController < ActionController::Base
+  end
+end

data/app/controllers/cloudtasker/worker_controller.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  # Handle execution of workers
+  class WorkerController < ApplicationController
+    # Authenticate all requests.
+    before_action :authenticate!
+
+    # Return 401 when API Token is invalid
+    rescue_from AuthenticationError do
+      head :unauthorized
+    end
+
+    # POST /cloudtasker/run
+    #
+    # Run a worker from a Cloud Task payload
+    #
+    def run
+      Task.execute_from_payload!(request.params.slice(:worker, :args))
+      head :no_content
+    rescue InvalidWorkerError
+      head :not_found
+    rescue StandardError
+      head :unprocessable_entity
+    end
+
+    private
+
+    #
+    # Authenticate incoming requests using a bearer token
+    #
+    # See Cloudtasker::Authenticator#verification_token
+    #
+    def authenticate!
+      Authenticator.verify!(request.headers['Authorization'].to_s.split(' ').last)
+    end
+  end
+end

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'cloudtasker'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/cloudtasker.gemspec

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cloudtasker/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'cloudtasker'
+  spec.version       = Cloudtasker::VERSION
+  spec.authors       = ['Arnaud Lachaume']
+  spec.email         = ['arnaud.lachaume@keypup.io']
+
+  spec.summary       = 'Manage GCP Cloud Tasks in your app.'
+  spec.description   = 'Manage GCP Cloud Tasks in your app.'
+  spec.homepage      = 'https://github.com/keypup-io/cloudtasker'
+  spec.license       = 'MIT'
+
+  # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/keypup-io/cloudtasker'
+  spec.metadata['changelog_uri'] = 'https://github.com/keypup-io/cloudtasker/master/tree/CHANGELOG.md'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'fugit'
+  spec.add_dependency 'google-cloud-tasks'
+  spec.add_dependency 'jwt'
+  spec.add_dependency 'redis'
+
+  spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '0.76.0'
+  spec.add_development_dependency 'rubocop-rspec'
+  spec.add_development_dependency 'timecop'
+
+  spec.add_development_dependency 'rails'
+  spec.add_development_dependency 'rspec-rails'
+  spec.add_development_dependency 'sqlite3'
+end

data/config/routes.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Cloudtasker::Engine.routes.draw do
+  post '/run', to: 'worker#run'
+end

data/lib/cloudtasker.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'cloudtasker/version'
+require 'cloudtasker/config'
+
+require 'cloudtasker/authentication_error'
+require 'cloudtasker/invalid_worker_error'
+
+require 'cloudtasker/middleware/chain'
+require 'cloudtasker/authenticator'
+require 'cloudtasker/task'
+require 'cloudtasker/meta_store'
+require 'cloudtasker/worker'
+
+# Define and manage Cloud Task based workers
+module Cloudtasker
+  attr_writer :config
+
+  #
+  # Cloudtasker configurator.
+  #
+  def self.configure
+    yield(config)
+  end
+
+  def self.config
+    @config ||= Config.new
+  end
+end
+
+require 'cloudtasker/engine' if defined?(::Rails::Engine)

data/lib/cloudtasker/authentication_error.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  class AuthenticationError < StandardError
+  end
+end

data/lib/cloudtasker/authenticator.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  # Manage token generation and verification
+  module Authenticator
+    module_function
+
+    # Algorithm used to sign the verification token
+    JWT_ALG = 'HS256'
+
+    #
+    # Return the cloudtasker configuration. See Cloudtasker#configure.
+    #
+    # @return [Cloudtasker::Config] The library configuration.
+    #
+    def config
+      Cloudtasker.config
+    end
+
+    #
+    # A Json Web Token (JWT) which will be used by the processor
+    # to authenticate the job.
+    #
+    # @return [String] The jwt token
+    #
+    def verification_token
+      JWT.encode({ iat: Time.now.to_i }, config.secret, JWT_ALG)
+    end
+
+    #
+    # Verify a bearer token (jwt token)
+    #
+    # @param [String] bearer_token The token to verify.
+    #
+    # @return [Boolean] Return true if the token is valid
+    #
+    def verify(bearer_token)
+      JWT.decode(bearer_token, config.secret)
+    rescue JWT::VerificationError, JWT::DecodeError
+      false
+    end
+
+    #
+    # Verify a bearer token and raise a `Cloudtasker::AuthenticationError`
+    # if the token is invalid.
+    #
+    # @param [String] bearer_token The token to verify.
+    #
+    # @return [Boolean] Return true if the token is valid
+    #
+    def verify!(bearer_token)
+      verify(bearer_token) || raise(AuthenticationError)
+    end
+  end
+end

data/lib/cloudtasker/batch.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative 'batch/middleware'
+
+Cloudtasker::Batch::Middleware.configure

data/lib/cloudtasker/batch/batch_progress.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require 'fugit'
+
+module Cloudtasker
+  module Batch
+    # Capture the progress of a batch
+    class BatchProgress
+      attr_reader :batch_state
+
+      #
+      # Build a new instance of the class.
+      #
+      # @param [Hash] batch_state The batch state
+      #
+      def initialize(batch_state = {})
+        @batch_state = batch_state
+      end
+
+      #
+      # Return the total number jobs.
+      #
+      # @return [Integer] The number number of jobs.
+      #
+      def total
+        count
+      end
+
+      #
+      # Return the number of completed jobs.
+      #
+      # @return [Integer] The number of completed jobs.
+      #
+      def completed
+        @completed ||= count('completed')
+      end
+
+      #
+      # Return the number of scheduled jobs.
+      #
+      # @return [Integer] The number of scheduled jobs.
+      #
+      def scheduled
+        @scheduled ||= count('scheduled')
+      end
+
+      #
+      # Return the number of processing jobs.
+      #
+      # @return [Integer] The number of processing jobs.
+      #
+      def processing
+        @processing ||= count('processing')
+      end
+
+      #
+      # Return the number of jobs not completed yet.
+      #
+      # @return [Integer] The number of jobs pending.
+      #
+      def pending
+        total - completed
+      end
+
+      #
+      # Return the batch progress percentage.
+      #
+      # @return [Float] The progress percentage.
+      #
+      def percent
+        return 0 if total.zero?
+
+        pending.to_f / total
+      end
+
+      #
+      # Add a batch progress to another one.
+      #
+      # @param [Cloudtasker::Batch::BatchProgress] progress The progress to add.
+      #
+      # @return [Cloudtasker::Batch::BatchProgress] The sum of the two batch progresses.
+      #
+      def +(other)
+        self.class.new(batch_state.to_h.merge(other.batch_state.to_h))
+      end
+
+      private
+
+      # Count the number of items in a given status
+      def count(status = nil)
+        return batch_state.to_h.keys.size unless status
+
+        batch_state.to_h.values.count { |e| e == status }
+      end
+    end
+  end
+end

data/lib/cloudtasker/batch/config.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'fugit'
+
+module Cloudtasker
+  module Batch
+    class Config
+      KEY_NAMESPACE = 'cloudtasker-batch'
+    end
+  end
+end

data/lib/cloudtasker/batch/extension/worker.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  module Batch
+    module Extension
+      # Include batch related methods onto Cloudtasker::Worker
+      # See: Cloudtasker::Batch::Middleware#configure
+      module Worker
+        attr_accessor :batch
+      end
+    end
+  end
+end

data/lib/cloudtasker/batch/job.rb

@@ -0,0 +1,320 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  module Batch
+    # Handle batch management
+    class Job
+      attr_reader :worker
+
+      #
+      # Return the cloudtasker redis client
+      #
+      # @return [Class] The redis client.
+      #
+      def self.redis
+        RedisClient
+      end
+
+      #
+      # Find a batch by id.
+      #
+      # @param [String] batch_id The batch id.
+      #
+      # @return [Cloudtasker::Batch::Job, nil] The batch.
+      #
+      def self.find(worker_id)
+        return nil unless worker_id
+
+        # Retrieve parent worker
+        parent_payload = redis.fetch(key(worker_id))
+        parent_worker = Cloudtasker::Worker.from_hash(parent_payload)
+        return nil unless parent_worker
+
+        # Build batch job
+        self.for(parent_worker)
+      end
+
+      #
+      # Return a namespaced key.
+      #
+      # @param [String, Symbol] val The key to namespace
+      #
+      # @return [String] The namespaced key.
+      #
+      def self.key(val)
+        return nil if val.nil?
+
+        [Config::KEY_NAMESPACE, val.to_s].join('/')
+      end
+
+      #
+      # Attach a batch to a worker
+      #
+      # @param [Cloudtasker::Worker] worker The worker on which the batch must be attached.
+      #
+      # @return [Cloudtasker::Batch::Job] The attached batch.
+      #
+      def self.for(worker)
+        worker.batch = new(worker)
+      end
+
+      #
+      # Build a new instance of the class.
+      #
+      # @param [Cloudtasker::Worker] worker The batch worker
+      #
+      def initialize(worker)
+        @worker = worker
+      end
+
+      #
+      # Return true if the worker has been re-enqueued.
+      # Post-process logic should be skipped for re-enqueued jobs.
+      #
+      # @return [Boolean] Return true if the job was reequeued.
+      #
+      def reenqueued?
+        worker.job_reenqueued
+      end
+
+      #
+      # Return the cloudtasker redis client
+      #
+      # @return [Class] The redis client.
+      #
+      def redis
+        self.class.redis
+      end
+
+      #
+      # Equality operator.
+      #
+      # @param [Any] other The object to compare.
+      #
+      # @return [Boolean] True if the object is equal.
+      #
+      def ==(other)
+        other.is_a?(self.class) && other.batch_id == batch_id
+      end
+
+      #
+      # Return a namespaced key.
+      #
+      # @param [String, Symbol] val The key to namespace
+      #
+      # @return [String] The namespaced key.
+      #
+      def key(val)
+        self.class.key(val)
+      end
+
+      #
+      # Return the parent batch, if any.
+      #
+      # @return [Cloudtasker::Batch::Job, nil] The parent batch.
+      #
+      def parent_batch
+        return nil unless (parent_id = worker.job_meta.get(key(:parent_id)))
+
+        @parent_batch ||= self.class.find(parent_id)
+      end
+
+      #
+      # Return the worker id.
+      #
+      # @return [String] The worker id.
+      #
+      def batch_id
+        worker&.job_id
+      end
+
+      #
+      # Return the namespaced worker id.
+      #
+      # @return [String] The worker namespaced id.
+      #
+      def batch_gid
+        key(batch_id)
+      end
+
+      #
+      # Return the key under which the batch state is stored.
+      #
+      # @return [String] The batch state namespaced id.
+      #
+      def batch_state_gid
+        [batch_gid, 'state'].join('/')
+      end
+
+      #
+      # The list of jobs in the batch
+      #
+      # @return [Array<Cloudtasker::Worker>] The jobs to enqueue at the end of the batch.
+      #
+      def jobs
+        @jobs ||= []
+      end
+
+      #
+      # Return the batch state
+      #
+      # @return [Hash] The state  of each child worker.
+      #
+      def batch_state
+        redis.fetch(batch_state_gid)
+      end
+
+      #
+      # Add a worker to the batch
+      #
+      # @param [Class] worker_klass The worker class.
+      # @param [Array<any>] *args The worker arguments.
+      #
+      # @return [Array<Cloudtasker::Worker>] The updated list of jobs.
+      #
+      def add(worker_klass, *args)
+        jobs << worker_klass.new(
+          job_args: args,
+          job_meta: { key(:parent_id) => batch_id }
+        )
+      end
+
+      #
+      # Save the batch.
+      #
+      def save
+        # Save serialized version of the worker. This is required to
+        # be able to invoke callback methods in the context of
+        # the worker (= instantiated worker) when child workers
+        # complete (success or failure).
+        redis.write(batch_gid, worker.to_h)
+
+        # Save list of child workers
+        redis.write(batch_state_gid, jobs.map { |e| [e.job_id, 'scheduled'] }.to_h)
+      end
+
+      #
+      # Update the batch state.
+      #
+      # @param [String] job_id The batch id.
+      # @param [String] status The status of the sub-batch.
+      #
+      # @return [<Type>] <description>
+      #
+      def update_state(batch_id, status)
+        redis.with_lock(batch_state_gid) do
+          state = batch_state
+          state[batch_id.to_sym] = status.to_s if state.key?(batch_id.to_sym)
+          redis.write(batch_state_gid, state)
+        end
+      end
+
+      #
+      # Return true if all the child workers have completed.
+      #
+      # @return [<Type>] <description>
+      #
+      def complete?
+        redis.with_lock(batch_state_gid) do
+          state = redis.fetch(batch_state_gid)
+          return true unless state
+
+          # Check that all children are complete
+          state.values.all? { |e| e == 'completed' }
+        end
+      end
+
+      #
+      # Callback invoked when a direct child batch is complete.
+      #
+      # @param [Cloudtasker::Batch::Job] child_batch The completed child batch.
+      #
+      def on_child_complete(child_batch)
+        # Update batch state
+        update_state(child_batch.batch_id, :completed)
+
+        # Notify the worker that a direct batch child worker has completed
+        worker.try(:on_child_complete, child_batch.worker)
+
+        # Notify the parent batch that we are done with this batch
+        parent_batch&.on_child_complete(self) if complete?
+      end
+
+      #
+      # Callback invoked when any batch in the tree gets completed.
+      #
+      # @param [Cloudtasker::Batch::Job] child_batch The completed child batch.
+      #
+      def on_batch_node_complete(child_batch)
+        # Notify the worker that a batch node worker has completed
+        worker.try(:on_batch_node_complete, child_batch.worker)
+
+        # Notify the parent batch that a node is complete
+        parent_batch&.on_batch_node_complete(child_batch)
+      end
+
+      #
+      # Calculate the progress of the batch.
+      #
+      # @return [Cloudtasker::Batch::BatchProgress] The batch progress.
+      #
+      def progress
+        # Capture batch state
+        state = batch_state
+
+        # Sum batch progress of current batch and all sub-batches
+        state.to_h.reduce(BatchProgress.new(state)) do |memo, (child_id, child_status)|
+          memo + (self.class.find(child_id)&.progress || BatchProgress.new(child_id => child_status))
+        end
+      end
+
+      #
+      # Save the batch and enqueue all child workers attached to it.
+      #
+      # @return [Array<Google::Cloud::Tasks::V2beta3::Task>] The Google Task responses
+      #
+      def setup
+        return true if jobs.empty?
+
+        # Save batch
+        save
+
+        # Enqueue all child workers
+        jobs.map(&:schedule)
+      end
+
+      #
+      # Post-perform logic. The parent batch is notified if the job is complete.
+      #
+      def complete
+        return true if reenqueued? || jobs.any?
+
+        # Notify the parent batch that a child is complete
+        if complete?
+          worker.try(:on_batch_complete)
+          parent_batch&.on_child_complete(self)
+        end
+
+        # Notify the parent that a batch node has completed
+        parent_batch&.on_batch_node_complete(self)
+      end
+
+      #
+      # Execute the batch.
+      #
+      def execute
+        # Update parent batch state
+        parent_batch&.update_state(batch_id, :processing)
+
+        # Perform job
+        yield
+
+        # Save batch (if child worker has been enqueued)
+        setup
+
+        # Complete batch
+        complete
+      end
+    end
+  end
+end