cloudtasker-tonix 0.1.0

data/lib/cloudtasker/cron/job.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+require 'fugit'
+
+module Cloudtasker
+  module Cron
+    # Manage cron jobs
+    class Job
+      attr_reader :worker
+
+      #
+      # Build a new instance of the class
+      #
+      # @param [Cloudtasker::Worker] worker The cloudtasker worker
+      #
+      def initialize(worker)
+        @worker = worker
+      end
+
+      #
+      # Return a namespaced key
+      #
+      # @param [String, Symbol] val The key to namespace
+      #
+      # @return [String] The namespaced key.
+      #
+      def key(val)
+        return nil if val.nil?
+
+        [self.class.to_s.underscore, val.to_s].join('/')
+      end
+
+      #
+      # Add cron metadata to the worker.
+      #
+      # @param [String, Symbol] name The name of the cron task.
+      # @param [String] cron The cron expression.
+      #
+      # @return [Cloudtasker::Cron::Job] self.
+      #
+      def set(schedule_id:)
+        worker.job_meta.set(key(:schedule_id), schedule_id.to_s)
+        self
+      end
+
+      #
+      # Return the worker id.
+      #
+      # @return [String] The worker id.
+      #
+      def job_id
+        worker.job_id
+      end
+
+      #
+      # Return the namespaced worker id.
+      #
+      # @return [String] The worker namespaced id.
+      #
+      def job_gid
+        key(job_id)
+      end
+
+      #
+      # Return the cron schedule id.
+      #
+      # @return [String] The schedule id.
+      #
+      def schedule_id
+        @schedule_id ||= worker.job_meta.get(key(:schedule_id))
+      end
+
+      #
+      # Return true if the worker is tagged as a cron job.
+      #
+      # @return [Boolean] True if the worker relates to a cron schedule.
+      #
+      def cron_job?
+        cron_schedule
+      end
+
+      #
+      # Return true if the worker is currently processing (includes retries).
+      #
+      # @return [Boolean] True f the worker is processing.
+      #
+      def retry_instance?
+        cron_job? && state
+      end
+
+      #
+      # Return the job processing state.
+      #
+      # @return [String, nil] The processing state.
+      #
+      def state
+        redis.get(job_gid)&.to_sym
+      end
+
+      #
+      # Return the cloudtasker redis client
+      #
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client..
+      #
+      def redis
+        @redis ||= RedisClient.new
+      end
+
+      #
+      # Return the cron schedule to use for the job.
+      #
+      # @return [Fugit::Cron] The cron schedule.
+      #
+      def cron_schedule
+        return nil unless schedule_id
+
+        @cron_schedule ||= Cron::Schedule.find(schedule_id)
+      end
+
+      #
+      # Return the time this cron instance is expected to run at.
+      #
+      # @return [Time] The current cron instance time.
+      #
+      def current_time
+        @current_time ||=
+          begin
+            Time.parse(worker.job_meta.get(key(:time_at)).to_s)
+          rescue ArgumentError
+            Time.try(:current) || Time.now
+          end
+      end
+
+      #
+      # Return the Time when the job should run next.
+      #
+      # @return [EtOrbi::EoTime] The time the job should run next.
+      #
+      def next_time
+        @next_time ||= cron_schedule&.next_time(current_time)
+      end
+
+      #
+      # Return true if the cron job is the one we are expecting. This method
+      # is used to ensure that jobs related to outdated cron schedules do not
+      # get processed.
+      #
+      # @return [Boolean] True if the cron job is expected.
+      #
+      def expected_instance?
+        retry_instance? || cron_schedule.job_id == job_id
+      end
+
+      #
+      # Store the cron job instance state.
+      #
+      # @param [String, Symbol] state The worker state.
+      #
+      def flag(state)
+        state.to_sym == :done ? redis.del(job_gid) : redis.set(job_gid, state.to_s)
+      end
+
+      #
+      # Schedule the next cron instance.
+      #
+      # The task only gets scheduled the first time a worker runs for a
+      # given cron instance (Typically a cron worker failing and retrying will
+      # not lead to a new task getting scheduled).
+      #
+      def schedule!
+        return false unless cron_schedule
+
+        # Configure next cron worker
+        next_worker = worker.new_instance.tap { |e| e.job_meta.set(key(:time_at), next_time.iso8601) }
+
+        # Schedule next worker
+        task = next_worker.schedule(time_at: next_time)
+        cron_schedule.update(task_id: task.id, job_id: next_worker.job_id)
+      end
+
+      #
+      # Execute the (cron) job. This method is invoked by the cron middleware.
+      #
+      def execute
+        # Execute the job immediately if this worker is not flagged as a cron job.
+        return yield unless cron_job?
+
+        # Abort and reject job if this cron instance is not expected.
+        return true unless expected_instance?
+
+        # Schedule the next instance of the job
+        schedule! unless retry_instance?
+
+        # Flag the cron instance as processing.
+        flag(:processing)
+
+        # Execute the cron instance
+        yield
+
+        # Flag the cron instance as done
+        flag(:done)
+      end
+    end
+  end
+end

data/lib/cloudtasker/cron/middleware/server.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  module Cron
+    module Middleware
+      # Server middleware, invoked when jobs are executed
+      class Server
+        def call(worker, **_kwargs, &block)
+          Job.new(worker).execute(&block)
+        end
+      end
+    end
+  end
+end

data/lib/cloudtasker/cron/middleware.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'cloudtasker/redis_client'
+
+require_relative 'schedule'
+require_relative 'job'
+require_relative 'middleware/server'
+
+module Cloudtasker
+  module Cron
+    # Registration module
+    module Middleware
+      def self.configure
+        Cloudtasker.configure do |config|
+          config.server_middleware { |c| c.add(Middleware::Server) }
+        end
+      end
+    end
+  end
+end

data/lib/cloudtasker/cron/schedule.rb

@@ -0,0 +1,308 @@
+# frozen_string_literal: true
+
+require 'fugit'
+require 'cloudtasker/worker_wrapper'
+
+module Cloudtasker
+  module Cron
+    # Manage cron schedules
+    class Schedule
+      attr_accessor :id, :cron, :worker, :task_id, :job_id, :queue, :args
+
+      #
+      # Return the redis client.
+      #
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
+      #
+      def self.redis
+        @redis ||= RedisClient.new
+      end
+
+      #
+      # Return a namespaced key.
+      #
+      # @param [String, Symbol, nil] val The key to namespace
+      #
+      # @return [String] The namespaced key.
+      #
+      def self.key(val = nil)
+        [to_s.underscore, val].compact.map(&:to_s).join('/')
+      end
+
+      #
+      # Return all schedules
+      #
+      # @return [Array<Cloudtasker::Batch::Schedule>] The list of stored schedules.
+      #
+      def self.all
+        if redis.exists?(key)
+          # Use Schedule Set if available
+          redis.smembers(key).map { |id| find(id) }
+        else
+          # Fallback to redis key matching and migrate schedules
+          # to use Schedule Set instead.
+          redis.search(key('*')).map do |gid|
+            schedule_id = gid.sub(key(''), '')
+            redis.sadd(key, [schedule_id])
+            find(schedule_id)
+          end
+        end
+      end
+
+      #
+      # Synchronize list of cron schedules from a Hash. Schedules
+      # not listed in this hash will be removed.
+      #
+      # @example
+      #   Cloudtasker::Cron::Schedule.load_from_hash!(
+      #     my_job: { cron: '0 0 * * *', worker: 'MyWorker' }
+      #     my_other_job: { cron: '0 10 * * *', worker: 'MyOtherWorker' }
+      #   )
+      #
+      def self.load_from_hash!(hash)
+        schedules = hash.map do |id, config|
+          schedule_config = JSON.parse(config.to_json, symbolize_names: true).merge(id: id.to_s)
+          create(**schedule_config)
+        end
+
+        # Remove existing schedules which are not part of the list
+        all.reject { |e| schedules.include?(e) }.each { |e| delete(e.id) }
+      end
+
+      #
+      # Create a new cron schedule (or update an existing one).
+      #
+      # @param [Hash] **opts Init opts. See initialize
+      #
+      # @return [Cloudtasker::Cron::Schedule] The schedule instance.
+      #
+      def self.create(**opts)
+        redis.with_lock(key(opts[:id])) do
+          config = find(opts[:id]).to_h.merge(opts)
+          new(**config).tap(&:save)
+        end
+      end
+
+      #
+      # Return a saved cron schedule.
+      #
+      # @param [String] id The schedule id.
+      #
+      # @return [Cloudtasker::Cron::Schedule] The schedule instance.
+      #
+      def self.find(id)
+        return nil unless (schedule_config = redis.fetch(key(id)))
+
+        new(**schedule_config)
+      end
+
+      #
+      # Delete a schedule by id.
+      #
+      # @param [String] id The schedule id.
+      #
+      def self.delete(id)
+        redis.with_lock(key(id)) do
+          schedule = find(id)
+          return false unless schedule
+
+          # Delete task and stored schedule
+          CloudTask.delete(schedule.task_id) if schedule.task_id
+          redis.srem(key, [schedule.id])
+          redis.del(schedule.gid)
+        end
+      end
+
+      #
+      # Build a new instance of the class.
+      #
+      # @param [String] id The schedule id.
+      # @param [String] cron The cron expression.
+      # @param [Class] worker The worker class to run.
+      # @param [Array<any>] args The worker arguments.
+      # @param [String] queue The queue to use for the cron job.
+      # @param [String] task_id The ID of the actual backend task.
+      # @param [String] job_id The ID of the Cloudtasker worker.
+      #
+      def initialize(id:, cron:, worker:, **opts)
+        @id = id
+        @cron = cron
+        @worker = worker
+        @args = opts[:args]
+        @queue = opts[:queue]
+        @task_id = opts[:task_id]
+        @job_id = opts[:job_id]
+      end
+
+      #
+      # Return the redis client.
+      #
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
+      #
+      def redis
+        self.class.redis
+      end
+
+      #
+      # Return the namespaced schedule id.
+      #
+      # @return [String] The namespaced schedule id.
+      #
+      def gid
+        self.class.key(id)
+      end
+
+      #
+      # Return true if the schedule is valid.
+      #
+      # @return [Boolean] True if the schedule is valid.
+      #
+      def valid?
+        id && cron_schedule && worker
+      rescue ArgumentError
+        # Rescue invalid cron expressions
+        false
+      end
+
+      #
+      # Equality operator.
+      #
+      # @param [Any] other The object to compare.
+      #
+      # @return [Boolean] True if the object is equal.
+      #
+      def ==(other)
+        other.is_a?(self.class) && other.id == id
+      end
+
+      #
+      # Return true if the configuration of the schedule was
+      # changed (cron expression or worker).
+      #
+      # @return [Boolean] True if the schedule config was changed.
+      #
+      def config_changed?
+        self.class.find(id)&.to_config != to_config
+      end
+
+      #
+      # RReturn true if the instance attributes were changed compared
+      # to the schedule saved in Redis.
+      #
+      # @return [Boolean] True if the schedule was modified.
+      #
+      def changed?
+        to_h != self.class.find(id).to_h
+      end
+
+      #
+      # Return a hash describing the configuration of this schedule.
+      #
+      # @return [Hash] The config description hash.
+      #
+      def to_config
+        {
+          id: id,
+          cron: cron,
+          worker: worker,
+          args: args,
+          queue: queue
+        }
+      end
+
+      #
+      # Return a hash with all the schedule attributes.
+      #
+      # @return [Hash] The attributes hash.
+      #
+      def to_h
+        to_config.merge(
+          task_id: task_id,
+          job_id: job_id
+        )
+      end
+
+      #
+      # Return the cron schedule to use for the job.
+      #
+      # @return [Fugit::Cron] The cron schedule.
+      #
+      def cron_schedule
+        @cron_schedule ||= Fugit::Cron.do_parse(cron)
+      end
+
+      #
+      # Return an instance of the underlying worker.
+      #
+      # @return [Cloudtasker::WorkerWrapper] The worker instance
+      #
+      def worker_instance
+        WorkerWrapper.new(worker_name: worker, job_args: args, job_queue: queue)
+      end
+
+      #
+      # Return the next time a job should run.
+      #
+      # @param [Time] time An optional reference in time (instead of Time.now)
+      #
+      # @return [EtOrbi::EoTime] The time the schedule job should run next.
+      #
+      def next_time(*args)
+        cron_schedule.next_time(*args)
+      end
+
+      #
+      # Buld edit the object attributes.
+      #
+      # @param [Hash] opts The attributes to edit.
+      #
+      def assign_attributes(opts)
+        opts
+          .select { |k, _| instance_variables.include?(:"@#{k}") }
+          .each { |k, v| instance_variable_set("@#{k}", v) }
+      end
+
+      #
+      # Edit the object attributes and save the object in Redis.
+      #
+      # @param [Hash] opts The attributes to edit.
+      #
+      def update(opts)
+        assign_attributes(opts)
+        save
+      end
+
+      #
+      # Save the object in Redis. If the configuration was changed
+      # then any existing cloud task is removed and a task is recreated.
+      #
+      def save(update_task: true)
+        return false unless valid?
+
+        # Save schedule
+        config_was_changed = config_changed?
+        redis.sadd(self.class.key, [id])
+        redis.write(gid, to_h)
+
+        # Stop there if backend does not need update
+        return true unless update_task && (config_was_changed || !task_id || !CloudTask.find(task_id))
+
+        # Update backend
+        persist_cloud_task
+      end
+
+      private
+
+      #
+      # Update the task in backend.
+      #
+      def persist_cloud_task
+        # Delete previous instance
+        CloudTask.delete(task_id) if task_id
+
+        # Schedule worker
+        Job.new(worker_instance).set(schedule_id: id).schedule!
+      end
+    end
+  end
+end

data/lib/cloudtasker/cron.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative 'cron/middleware'
+
+Cloudtasker::Cron::Middleware.configure

data/lib/cloudtasker/dead_worker_error.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  class DeadWorkerError < StandardError
+  end
+end

data/lib/cloudtasker/engine.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  # Cloudtasker Rails engine
+  class Engine < ::Rails::Engine
+    isolate_namespace Cloudtasker
+
+    config.before_initialize do
+      # Mount cloudtasker processing endpoint
+      Rails.application.routes.append do
+        mount Cloudtasker::Engine, at: '/cloudtasker'
+      end
+
+      # Add ActiveJob adapter
+      require 'active_job/queue_adapters/cloudtasker_adapter' if defined?(::ActiveJob::Railtie)
+    end
+
+    config.generators do |g|
+      g.test_framework :rspec, fixture: false
+      g.assets false
+      g.helper false
+    end
+  end
+end

data/lib/cloudtasker/invalid_worker_error.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  class InvalidWorkerError < StandardError
+  end
+end

data/lib/cloudtasker/local_server.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'cloudtasker/backend/redis_task'
+
+module Cloudtasker
+  # Process jobs stored in Redis.
+  # Only to be used in development.
+  class LocalServer
+    # Max number of task requests sent to the processing server
+    CONCURRENCY = (ENV['CLOUDTASKER_CONCURRENCY'] || 5).to_i
+
+    # Default number of threads to allocate to process a specific queue
+    QUEUE_CONCURRENCY = 1
+
+    # Job Polling. How frequently to poll jobs in redis.
+    JOB_POLLING_FREQUENCY = 0.5 # seconds
+
+    #
+    # Stop the local server.
+    #
+    def stop
+      @done = true
+
+      # Terminate threads and repush tasks
+      @threads&.values&.flatten&.each do |t|
+        t.terminate
+        t['task']&.retry_later(0, is_error: false)
+      end
+
+      # Wait for main server to be done
+      sleep 1 while @start&.alive?
+    end
+
+    #
+    # Start the local server
+    #
+    # @param [Hash] opts Server options.
+    #
+    #
+    def start(opts = {})
+      # Extract queues to process
+      queues = opts[:queues].to_a.any? ? opts[:queues] : [[nil, CONCURRENCY]]
+
+      # Display start banner
+      queue_labels = queues.map { |n, c| "#{n || 'all'}=#{c || QUEUE_CONCURRENCY}" }.join(' ')
+      Cloudtasker.logger.info("[Cloudtasker/Server] Processing queues: #{queue_labels}")
+
+      # Start processing queues
+      @start ||= Thread.new do
+        until @done
+          queues.each { |(n, c)| process_jobs(n, c) }
+          sleep JOB_POLLING_FREQUENCY
+        end
+        Cloudtasker.logger.info('[Cloudtasker/Server] Local server exiting...')
+      end
+    end
+
+    #
+    # Process enqueued workers.
+    #
+    #
+    def process_jobs(queue = nil, concurrency = nil)
+      @threads ||= {}
+      @threads[queue] ||= []
+      max_threads = (concurrency || QUEUE_CONCURRENCY).to_i
+
+      # Remove any done thread
+      @threads[queue].select!(&:alive?)
+
+      # Process tasks
+      while @threads[queue].count < max_threads && (task = Cloudtasker::Backend::RedisTask.pop(queue))
+        @threads[queue] << Thread.new(task) { |t| process_task(t) }
+      end
+    end
+
+    #
+    # Process a given task
+    #
+    # @param [Cloudtasker::CloudTask] task The task to process
+    #
+    def process_task(task)
+      Thread.current['task'] = task
+      Thread.current['attempts'] = 0
+
+      # Deliver task
+      begin
+        Thread.current['task']&.deliver
+      rescue Errno::EBADF, Errno::ECONNREFUSED => e
+        raise(e) unless Thread.current['attempts'] < 3
+
+        # Retry on connection error, in case the web server is not
+        # started yet.
+        Thread.current['attempts'] += 1
+        sleep(3)
+        retry
+      end
+    end
+  end
+end

data/lib/cloudtasker/max_task_size_exceeded_error.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  # Handle Cloud Task size quota
+  # See: https://cloud.google.com/appengine/quotas#Task_Queue
+  #
+  class MaxTaskSizeExceededError < StandardError
+    MSG = 'The size of Cloud Tasks must not exceed 100KB'
+
+    def initialize(msg = MSG)
+      super
+    end
+  end
+end