cloudtasker-tonix 0.1.0

data/lib/cloudtasker/meta_store.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  # Manage meta information on workers. This meta stored is intended
+  # to be used by middlewares needing to store extra information on the
+  # job.
+  # The objective of this class is to provide a shared store to middleware
+  # while controlling access to its keys by preveenting access the hash directly
+  # (e.g. avoid wild merge or replace operations).
+  class MetaStore
+    #
+    # Build a new instance of the class.
+    #
+    # @param [<Type>] hash The worker meta hash
+    #
+    def initialize(hash = {})
+      @meta = JSON.parse((hash || {}).to_json, symbolize_names: true)
+    end
+
+    #
+    # Retrieve meta entry.
+    #
+    # @param [String, Symbol] key The key of the meta entry.
+    #
+    # @return [Any] The value of the meta entry.
+    #
+    def get(key)
+      @meta[key.to_sym] if key
+    end
+
+    #
+    # Set meta entry
+    #
+    # @param [String, Symbol] key The key of the meta entry.
+    # @param [Any] val The value of the meta entry.
+    #
+    # @return [Any] The value set
+    #
+    def set(key, val)
+      @meta[key.to_sym] = val if key
+    end
+
+    #
+    # Remove a meta information.
+    #
+    # @param [String, Symbol] key The key of the entry to delete.
+    #
+    # @return [Any] The value of the deleted key
+    #
+    def del(key)
+      @meta.delete(key.to_sym) if key
+    end
+
+    #
+    # Return the meta store as Hash.
+    #
+    # @return [Hash] The meta store as Hash.
+    #
+    def to_h
+      # Deep dup
+      JSON.parse(@meta.to_json, symbolize_names: true)
+    end
+
+    #
+    # Return the meta store as json.
+    #
+    # @param [Array<any>] *arg The to_json args.
+    #
+    # @return [String] The meta store as json.
+    #
+    def to_json(*arg)
+      @meta.to_json(*arg)
+    end
+
+    #
+    # Equality operator.
+    #
+    # @param [Any] other The object being compared.
+    #
+    # @return [Boolean] True if the object is equal.
+    #
+    def ==(other)
+      to_json == other.try(:to_json)
+    end
+  end
+end

data/lib/cloudtasker/middleware/chain.rb

@@ -0,0 +1,250 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  module Middleware
+    # The class below was originally taken from Sidekiq.
+    # See: https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/middleware/chain.rb
+    #
+    # Middleware are callables configured to run before/after a message is processed.
+    # Middlewares can be configured to run on the client side (when jobs are pushed
+    # to Cloud Tasks) as well as on the server side (when jobs are processed by
+    # your application)
+    #
+    # To add middleware for the client:
+    #
+    # Cloudtasker.configure do |config|
+    #   config.client_middleware do |chain|
+    #     chain.add MyClientHook
+    #   end
+    # end
+    #
+    # To modify middleware for the server, just call
+    # with another block:
+    #
+    # Cloudtasker.configure do |config|
+    #   config.server_middleware do |chain|
+    #     chain.add MyServerHook
+    #     chain.remove ActiveRecord
+    #   end
+    # end
+    #
+    # To insert immediately preceding another entry:
+    #
+    # Cloudtasker.configure do |config|
+    #   config.client_middleware do |chain|
+    #     chain.insert_before ActiveRecord, MyClientHook
+    #   end
+    # end
+    #
+    # To insert immediately after another entry:
+    #
+    # Cloudtasker.configure do |config|
+    #   config.client_middleware do |chain|
+    #     chain.insert_after ActiveRecord, MyClientHook
+    #   end
+    # end
+    #
+    # This is an example of a minimal server middleware:
+    #
+    # class MyServerHook
+    #   def call(worker_instance, msg, queue)
+    #     puts "Before work"
+    #     yield
+    #     puts "After work"
+    #   end
+    # end
+    #
+    # This is an example of a minimal client middleware, note
+    # the method must return the result or the job will not push
+    # to Redis:
+    #
+    # class MyClientHook
+    #   def call(worker_class, msg, queue, redis_pool)
+    #     puts "Before push"
+    #     result = yield
+    #     puts "After push"
+    #     result
+    #   end
+    # end
+    #
+    class Chain
+      include Enumerable
+
+      #
+      # Build a new middleware chain.
+      #
+      def initialize
+        @entries = nil
+        yield self if block_given?
+      end
+
+      #
+      # Iterate over the list middlewares and execute the block on each item.
+      #
+      # @param [Proc] &block The block to execute on each item.
+      #
+      def each(&block)
+        entries.each(&block)
+      end
+
+      #
+      # Return the list of middlewares.
+      #
+      # @return [Array<Cloudtasker::Middleware::Chain::Entry>] The list of middlewares
+      #
+      def entries
+        @entries ||= []
+      end
+
+      #
+      # Remove a middleware from the list.
+      #
+      # @param [Class] klass The middleware class to remove.
+      #
+      def remove(klass)
+        entries.delete_if { |entry| entry.klass == klass }
+      end
+
+      #
+      # Add a middleware at the end of the list.
+      #
+      # @param [Class] klass The middleware class to add.
+      # @param [Arry<any>] *args The list of arguments to the middleware.
+      #
+      # @return [Array<Cloudtasker::Middleware::Chain::Entry>] The updated list of middlewares
+      #
+      def add(klass, *args)
+        remove(klass) if exists?(klass)
+        entries << Entry.new(klass, *args)
+      end
+
+      #
+      # Add a middleware at the beginning of the list.
+      #
+      # @param [Class] klass The middleware class to add.
+      # @param [Arry<any>] *args The list of arguments to the middleware.
+      #
+      # @return [Array<Cloudtasker::Middleware::Chain::Entry>] The updated list of middlewares
+      #
+      def prepend(klass, *args)
+        remove(klass) if exists?(klass)
+        entries.insert(0, Entry.new(klass, *args))
+      end
+
+      #
+      # Add a middleware before another middleware.
+      #
+      # @param [Class] oldklass The middleware class before which the new middleware should be inserted.
+      # @param [Class] newklass The middleware class to insert.
+      # @param [Arry<any>] *args The list of arguments for the inserted middleware.
+      #
+      # @return [Array<Cloudtasker::Middleware::Chain::Entry>] The updated list of middlewares
+      #
+      def insert_before(oldklass, newklass, *args)
+        i = entries.index { |entry| entry.klass == newklass }
+        new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
+        i = entries.index { |entry| entry.klass == oldklass } || 0
+        entries.insert(i, new_entry)
+      end
+
+      #
+      # Add a middleware after another middleware.
+      #
+      # @param [Class] oldklass The middleware class after which the new middleware should be inserted.
+      # @param [Class] newklass The middleware class to insert.
+      # @param [Arry<any>] *args The list of arguments for the inserted middleware.
+      #
+      # @return [Array<Cloudtasker::Middleware::Chain::Entry>] The updated list of middlewares
+      #
+      def insert_after(oldklass, newklass, *args)
+        i = entries.index { |entry| entry.klass == newklass }
+        new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
+        i = entries.index { |entry| entry.klass == oldklass } || (entries.count - 1)
+        entries.insert(i + 1, new_entry)
+      end
+
+      #
+      # Checks if middleware has been added to the list.
+      #
+      # @param [Class] klass The middleware class to check.
+      #
+      # @return [Boolean] Return true if the middleware is in the list.
+      #
+      def exists?(klass)
+        any? { |entry| entry.klass == klass }
+      end
+
+      #
+      # Checks if the middlware list is empty
+      #
+      # @return [Boolean] Return true if the middleware list is empty.
+      #
+      def empty?
+        @entries.nil? || @entries.empty?
+      end
+
+      #
+      # Return a list of instantiated middlewares. Each middleware gets
+      # initialize with the args originally passed to `add`, `insert_before` etc.
+      #
+      # @return [Array<any>] The list of instantiated middlewares.
+      #
+      def retrieve
+        map(&:make_new)
+      end
+
+      #
+      # Empty the list of middlewares.
+      #
+      # @return [Array<Cloudtasker::Middleware::Chain::Entry>] The updated list of middlewares
+      #
+      def clear
+        entries.clear
+      end
+
+      #
+      # Invoke the chain of middlewares.
+      #
+      # @param [Array<any>] *args The args to pass to each middleware.
+      #
+      def invoke(*args)
+        return yield if empty?
+
+        chain = retrieve.dup
+        traverse_chain = lambda do
+          if chain.empty?
+            yield
+          else
+            chain.shift.call(*args, &traverse_chain)
+          end
+        end
+        traverse_chain.call
+      end
+    end
+
+    # Middleware list item.
+    class Entry
+      attr_reader :klass, :args
+
+      #
+      # Build a new entry.
+      #
+      # @param [Class] klass The middleware class.
+      # @param [Array<any>] *args The list of arguments for the middleware.
+      #
+      def initialize(klass, *args)
+        @klass = klass
+        @args = args
+      end
+
+      #
+      # Return an instantiated middleware.
+      #
+      # @return [Any] The instantiated middleware.
+      #
+      def make_new
+        @klass.new(*@args)
+      end
+    end
+  end
+end

data/lib/cloudtasker/missing_worker_arguments_error.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  class MissingWorkerArgumentsError < StandardError
+  end
+end

data/lib/cloudtasker/redis_client.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require 'redis'
+require 'connection_pool'
+
+module Cloudtasker
+  # A wrapper with helper methods for redis
+  class RedisClient
+    # Suffix added to cache keys when locking them
+    LOCK_KEY_PREFIX = 'cloudtasker/lock'
+    LOCK_DURATION = 2 # seconds
+    LOCK_WAIT_DURATION = 0.03 # seconds
+
+    # Default pool size used for Redis
+    DEFAULT_POOL_SIZE = ENV.fetch('RAILS_MAX_THREADS', 25)
+    DEFAULT_POOL_TIMEOUT = 5
+
+    def self.client
+      @client ||= begin
+        pool_size = Cloudtasker.config.redis&.dig(:pool_size) || DEFAULT_POOL_SIZE
+        pool_timeout = Cloudtasker.config.redis&.dig(:pool_timeout) || DEFAULT_POOL_TIMEOUT
+        ConnectionPool.new(size: pool_size, timeout: pool_timeout) do
+          Redis.new(Cloudtasker.config.redis || {})
+        end
+      end
+    end
+
+    #
+    # Return the underlying redis client.
+    #
+    # @return [Redis] The redis client.
+    #
+    def client
+      @client ||= self.class.client
+    end
+
+    #
+    # Get a cache entry and parse it as JSON.
+    #
+    # @param [String, Symbol] key The cache key to fetch.
+    #
+    # @return [Hash, Array] The content of the cache key, parsed as JSON.
+    #
+    def fetch(key)
+      return nil unless (val = get(key.to_s))
+
+      JSON.parse(val, symbolize_names: true)
+    rescue JSON::ParserError
+      nil
+    end
+
+    #
+    # Write a cache entry as JSON.
+    #
+    # @param [String, Symbol] key The cache key to write.
+    # @param [Hash, Array] content The content to write.
+    #
+    # @return [String] Redis response code.
+    #
+    def write(key, content)
+      set(key.to_s, content.to_json)
+    end
+
+    #
+    # Acquire a lock on a cache entry.
+    #
+    # Locks are enforced to be short-lived (2s).
+    # The yielded block should limit its logic to short operations (e.g. redis get/set).
+    #
+    # @example
+    #   redis = RedisClient.new
+    #   redis.with_lock('foo')
+    #     content = redis.fetch('foo')
+    #     redis.set(content.merge(bar: 'bar).to_json)
+    #   end
+    #
+    # @param [String] cache_key The cache key to access.
+    # @param [Integer] max_wait The number of seconds after which the lock will be cleared anyway.
+    #
+    def with_lock(cache_key, max_wait: nil)
+      return nil unless cache_key
+
+      # Set max wait
+      max_wait = (max_wait || LOCK_DURATION).to_i
+
+      # Wait to acquire lock
+      lock_key = [LOCK_KEY_PREFIX, cache_key].join('/')
+      client.with do |conn|
+        sleep(LOCK_WAIT_DURATION) until conn.set(lock_key, true, nx: true, ex: max_wait)
+      end
+
+      # yield content
+      yield
+    ensure
+      del(lock_key)
+    end
+
+    #
+    # Clear all redis keys
+    #
+    # @return [Integer] The number of keys deleted
+    #
+    def clear
+      all_keys = keys
+      return 0 if all_keys.empty?
+
+      # Delete all keys
+      del(*all_keys)
+    end
+
+    #
+    # Return all keys matching the provided patterns.
+    #
+    # @param [String] pattern A redis compatible pattern.
+    #
+    # @return [Array<String>] The list of matching keys
+    #
+    def search(pattern)
+      # Initialize loop variables
+      cursor = nil
+      list = []
+
+      # Scan and capture matching keys
+      client.with do |conn|
+        while cursor != 0
+          scan = conn.scan(cursor || 0, match: pattern)
+          list += scan[1]
+          cursor = scan[0].to_i
+        end
+      end
+
+      list
+    end
+
+    #
+    # Delegate all methods to the redis client.
+    # Ruby 3 delegation method style.
+    #
+    # @param [String, Symbol] name The method to delegate.
+    # @param [Array<any>] *args The list of method positional arguments.
+    # @param [Hash<any>] *kwargs The list of method keyword arguments.
+    # @param [Proc] &block Block passed to the method.
+    #
+    # @return [Any] The method return value
+    #
+    def method_missing(name, ...)
+      if Redis.method_defined?(name)
+        client.with { |c| c.send(name, ...) }
+      else
+        super
+      end
+    end
+
+    #
+    # Check if the class respond to a certain method.
+    #
+    # @param [String, Symbol] name The name of the method.
+    # @param [Boolean] include_private Whether to check private methods or not. Default to false.
+    #
+    # @return [Boolean] Return true if the class respond to this method.
+    #
+    def respond_to_missing?(name, include_private = false)
+      Redis.method_defined?(name) || super
+    end
+  end
+end

data/lib/cloudtasker/retry_worker_error.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  class RetryWorkerError < StandardError
+  end
+end

data/lib/cloudtasker/storable/worker.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  module Storable
+    # Add ability to store and pull workers in Redis under a specific namespace
+    module Worker
+      # Add class method to including class
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      # Module class methods
+      module ClassMethods
+        #
+        # Return the namespaced store key used to store jobs that
+        # have been parked and should be manually popped later.
+        #
+        # @param [String] namespace The user-provided store namespace
+        #
+        # @return [String] The full store cache key
+        #
+        def store_cache_key(namespace)
+          cache_key([Config::WORKER_STORE_PREFIX, namespace])
+        end
+
+        #
+        # Push the worker to a namespaced store.
+        #
+        # @param [String] namespace The store namespace
+        # @param [Array<any>] *args List of worker arguments
+        #
+        # @return [String] The number of elements added to the store
+        #
+        def push_to_store(namespace, *args)
+          redis.rpush(store_cache_key(namespace), [args.to_json])
+        end
+
+        #
+        # Push many workers to a namespaced store at once.
+        #
+        # @param [String] namespace The store namespace
+        # @param [Array<Array<any>>] args_list A list of arguments for each worker
+        #
+        # @return [String] The number of elements added to the store
+        #
+        def push_many_to_store(namespace, args_list)
+          redis.rpush(store_cache_key(namespace), args_list.map(&:to_json))
+        end
+
+        #
+        # Pull the jobs from the namespaced store and enqueue them.
+        #
+        # @param [String] namespace The store namespace.
+        # @param [Integer] page_size The number of items to pull on each page. Defaults to 1000.
+        #
+        def pull_all_from_store(namespace, page_size: 1000)
+          items = nil
+
+          while items.nil? || items.present?
+            # Pull items
+            items = redis.lpop(store_cache_key(namespace), page_size).to_a
+
+            # For each item, execute block or enqueue it
+            items.each do |args_json|
+              worker_args = JSON.parse(args_json)
+
+              if block_given?
+                yield(worker_args)
+              else
+                perform_async(*worker_args)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cloudtasker/storable.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative 'storable/worker'