cloudtasker 0.1.0

data/lib/cloudtasker/batch/middleware.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'cloudtasker/redis_client'
+
+require_relative 'extension/worker'
+require_relative 'config'
+require_relative 'batch_progress'
+require_relative 'job'
+
+require_relative 'middleware/server'
+
+module Cloudtasker
+  module Batch
+    # Registration module
+    module Middleware
+      def self.configure
+        Cloudtasker.configure do |config|
+          config.server_middleware { |c| c.add(Middleware::Server) }
+        end
+        Cloudtasker::Worker.include(Extension::Worker)
+      end
+    end
+  end
+end

data/lib/cloudtasker/batch/middleware/server.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  module Batch
+    module Middleware
+      # Server middleware, invoked when jobs are executed
+      class Server
+        def call(worker)
+          Job.for(worker).execute { yield }
+        end
+      end
+    end
+  end
+end

data/lib/cloudtasker/config.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  # Holds cloudtasker configuration. See Cloudtasker#configure
+  class Config
+    attr_accessor :redis
+    attr_writer :secret, :gcp_location_id, :gcp_project_id,
+                :gcp_queue_id, :processor_host, :processor_path
+
+    DEFAULT_LOCATION_ID = 'us-east1'
+    DEFAULT_PROCESSOR_PATH = '/cloudtasker/run'
+
+    PROCESSOR_HOST_MISSING = <<~DOC
+      Missing host for processing.
+      Please specify a processor hostname in form of `https://some-public-dns.example.com`'
+    DOC
+    QUEUE_ID_MISSING_ERROR = <<~DOC
+      Missing GCP queue ID.
+      Please specify a queue ID in the form of `my-queue-id`. You can create a queue using the Google SDK via `gcloud tasks queues create my-queue-id`
+    DOC
+    PROJECT_ID_MISSING_ERROR = <<~DOC
+      Missing GCP project ID.
+      Please specify a project ID in the cloudtasker configurator.
+    DOC
+    SECRET_MISSING_ERROR = <<~DOC
+      Missing cloudtasker secret.
+      Please specify a secret in the cloudtasker initializer or add Rails secret_key_base in your credentials
+    DOC
+
+    #
+    # Return the full URL of the processor. Worker payloads will be sent
+    # to this URL.
+    #
+    # @return [String] The processor URL.
+    #
+    def processor_url
+      File.join(processor_host, processor_path)
+    end
+
+    #
+    # The hostname of the application processing the workers. The hostname must
+    # be reachable from Cloud Task.
+    #
+    # @return [String] The processor host.
+    #
+    def processor_host
+      @processor_host || raise(StandardError, PROCESSOR_HOST_MISSING)
+    end
+
+    #
+    # The path on the host when worker payloads will be sent.
+    # Default to `/cloudtasker/run`
+    #
+    #
+    # @return [String] The processor path
+    #
+    def processor_path
+      @processor_path || DEFAULT_PROCESSOR_PATH
+    end
+
+    #
+    # Return the ID of GCP queue where tasks will be added.
+    #
+    # @return [String] The ID of the processing queue.
+    #
+    def gcp_queue_id
+      @gcp_queue_id || raise(StandardError, QUEUE_ID_MISSING_ERROR)
+    end
+
+    #
+    # Return the GCP project ID.
+    #
+    # @return [String] The ID of the project for which tasks will be processed.
+    #
+    def gcp_project_id
+      @gcp_project_id || raise(StandardError, PROJECT_ID_MISSING_ERROR)
+    end
+
+    #
+    # Return the GCP location ID. Default to 'us-east1'
+    #
+    # @return [String] The location ID where tasks will be processed.
+    #
+    def gcp_location_id
+      @gcp_location_id || DEFAULT_LOCATION_ID
+    end
+
+    #
+    # Return the secret to use to sign the verification tokens
+    # attached to tasks.
+    #
+    # @return [String] The cloudtasker secret
+    #
+    def secret
+      @secret || (
+        defined?(Rails) && Rails.application.credentials&.secret_key_base
+      ) || raise(StandardError, SECRET_MISSING_ERROR)
+    end
+
+    #
+    # Return the chain of client middlewares.
+    #
+    # @return [Cloudtasker::Middleware::Chain] The chain of middlewares.
+    #
+    def client_middleware
+      @client_middleware ||= Middleware::Chain.new
+      yield @client_middleware if block_given?
+      @client_middleware
+    end
+
+    #
+    # Return the chain of server middlewares.
+    #
+    # @return [Cloudtasker::Middleware::Chain] The chain of middlewares.
+    #
+    def server_middleware
+      @server_middleware ||= Middleware::Chain.new
+      yield @server_middleware if block_given?
+      @server_middleware
+    end
+  end
+end

data/lib/cloudtasker/cron.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative 'cron/middleware'
+
+Cloudtasker::Cron::Middleware.configure

data/lib/cloudtasker/cron/config.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'fugit'
+
+module Cloudtasker
+  module Cron
+    class Config
+      KEY_NAMESPACE = 'cloudtasker-cron'
+    end
+  end
+end

data/lib/cloudtasker/cron/job.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+require 'fugit'
+
+module Cloudtasker
+  module Cron
+    # TODO: handle deletion of cron jobs
+    #
+    # Manage cron jobs
+    class Job
+      attr_reader :worker
+
+      #
+      # Build a new instance of the class
+      #
+      # @param [Cloudtasker::Worker] worker The cloudtasker worker
+      #
+      def initialize(worker)
+        @worker = worker
+      end
+
+      #
+      # Return a namespaced key
+      #
+      # @param [String, Symbol] val The key to namespace
+      #
+      # @return [String] The namespaced key.
+      #
+      def key(val)
+        return nil if val.nil?
+
+        [Config::KEY_NAMESPACE, val.to_s].join('/')
+      end
+
+      #
+      # Add cron metadata to the worker.
+      #
+      # @param [String, Symbol] name The name of the cron task.
+      # @param [String] cron The cron expression.
+      #
+      # @return [Cloudtasker::Cron::Job] self.
+      #
+      def set(schedule_id:)
+        worker.job_meta.set(key(:schedule_id), schedule_id.to_s)
+        self
+      end
+
+      #
+      # Return the worker id.
+      #
+      # @return [String] The worker id.
+      #
+      def job_id
+        worker.job_id
+      end
+
+      #
+      # Return the namespaced worker id.
+      #
+      # @return [String] The worker namespaced id.
+      #
+      def job_gid
+        key(job_id)
+      end
+
+      #
+      # Return the cron schedule id.
+      #
+      # @return [String] The schedule id.
+      #
+      def schedule_id
+        @schedule_id ||= worker.job_meta.get(key(:schedule_id))
+      end
+
+      #
+      # Return true if the worker is tagged as a cron job.
+      #
+      # @return [Boolean] True if the worker relates to a cron schedule.
+      #
+      def cron_job?
+        cron_schedule
+      end
+
+      #
+      # Return true if the worker is currently processing (includes retries).
+      #
+      # @return [Boolean] True f the worker is processing.
+      #
+      def retry_instance?
+        cron_job? && state
+      end
+
+      #
+      # Return the job processing state.
+      #
+      # @return [String, nil] The processing state.
+      #
+      def state
+        redis.get(job_gid)&.to_sym
+      end
+
+      #
+      # Return the cloudtasker redis client
+      #
+      # @return [Class] The redis client.
+      #
+      def redis
+        RedisClient
+      end
+
+      #
+      # Return the cron schedule to use for the job.
+      #
+      # @return [Fugit::Cron] The cron schedule.
+      #
+      def cron_schedule
+        return nil unless schedule_id
+
+        @cron_schedule ||= Cron::Schedule.find(schedule_id)
+      end
+
+      #
+      # Return the time this cron instance is expected to run at.
+      #
+      # @return [Time] The current cron instance time.
+      #
+      def current_time
+        @current_time ||=
+          begin
+            Time.parse(worker.job_meta.get(key(:time_at)).to_s)
+          rescue ArgumentError
+            Time.try(:current) || Time.now
+          end
+      end
+
+      #
+      # Return the Time when the job should run next.
+      #
+      # @return [EtOrbi::EoTime] The time the job should run next.
+      #
+      def next_time
+        @next_time ||= cron_schedule&.next_time(current_time)
+      end
+
+      #
+      # Return true if the cron job is the one we are expecting. This method
+      # is used to ensure that jobs related to outdated cron schedules do not
+      # get processed.
+      #
+      # @return [Boolean] True if the cron job is expected.
+      #
+      def expected_instance?
+        retry_instance? || cron_schedule.job_id == job_id
+      end
+
+      #
+      # Store the cron job instance state.
+      #
+      # @param [String, Symbol] state The worker state.
+      #
+      def flag(state)
+        state.to_sym == :done ? redis.del(job_gid) : redis.set(job_gid, state.to_s)
+      end
+
+      #
+      # Schedule the next cron instance.
+      #
+      # The task only gets scheduled the first time a worker runs for a
+      # given cron instance (Typically a cron worker failing and retrying will
+      # not lead to a new task getting scheduled).
+      #
+      def schedule!
+        return false unless cron_schedule
+
+        # Configure next cron worker
+        next_worker = worker.new_instance.tap { |e| e.job_meta.set(key(:time_at), next_time.iso8601) }
+
+        # Schedule next worker
+        resp = next_worker.schedule(time_at: next_time)
+        cron_schedule.update(task_id: resp.name, job_id: next_worker.job_id)
+      end
+
+      #
+      # Execute the (cron) job. This method is invoked by the cron middleware.
+      #
+      def execute
+        # Execute the job immediately if this worker is not flagged as a cron job.
+        return yield unless cron_job?
+
+        # Abort and reject job if this cron instance is not expected.
+        return true unless expected_instance?
+
+        # Schedule the next instance of the job
+        schedule! unless retry_instance?
+
+        # Flag the cron instance as processing.
+        flag(:processing)
+
+        # Execute the cron instance
+        yield
+
+        # Flag the cron instance as done
+        flag(:done)
+      end
+    end
+  end
+end

data/lib/cloudtasker/cron/middleware.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'cloudtasker/redis_client'
+
+require_relative 'config'
+require_relative 'schedule'
+require_relative 'job'
+require_relative 'middleware/server'
+
+module Cloudtasker
+  module Cron
+    # Registration module
+    module Middleware
+      def self.configure
+        Cloudtasker.configure do |config|
+          config.server_middleware { |c| c.add(Middleware::Server) }
+        end
+      end
+    end
+  end
+end

data/lib/cloudtasker/cron/middleware/server.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  module Cron
+    module Middleware
+      # Server middleware, invoked when jobs are executed
+      class Server
+        def call(worker)
+          Job.new(worker).execute { yield }
+        end
+      end
+    end
+  end
+end

data/lib/cloudtasker/cron/schedule.rb

@@ -0,0 +1,227 @@
+# frozen_string_literal: true
+
+require 'fugit'
+
+module Cloudtasker
+  module Cron
+    # Manage cron schedules
+    class Schedule
+      attr_accessor :id, :cron, :worker, :task_id, :job_id
+
+      #
+      # Return the redis client.
+      #
+      # @return [Class] The redis client
+      #
+      def self.redis
+        RedisClient
+      end
+
+      #
+      # Create a new cron schedule (or update an existing one).
+      #
+      # @param [Hash] **opts Init opts. See initialize
+      #
+      # @return [Cloudtasker::Cron::Schedule] The schedule instance.
+      #
+      def self.create(**opts)
+        config = find(opts[:id]).to_h.merge(opts)
+        new(config).tap(&:save)
+      end
+
+      #
+      # Return a saved cron schedule.
+      #
+      # @param [String] id The schedule id.
+      #
+      # @return [Cloudtasker::Cron::Schedule] The schedule instance.
+      #
+      def self.find(id)
+        gid = [Config::KEY_NAMESPACE, id].join('/')
+        return nil unless (schedule_config = redis.fetch(gid))
+
+        new(schedule_config)
+      end
+
+      #
+      # Destroy a schedule by id.
+      #
+      # @param [String] id The schedule id.
+      #
+      def self.delete(id)
+        schedule = find(id)
+        return false unless schedule
+
+        # Delete task and stored schedule
+        Task.delete(schedule.task_id) if schedule.task_id
+        redis.del(schedule.gid)
+      end
+
+      #
+      # Build a new instance of the class.
+      #
+      # @param [String] id The schedule id.
+      # @param [String] cron The cron expression.
+      # @param [Class] worker The worker class to run.
+      # @param [String] task_id The ID of the actual backend task.
+      # @param [String] job_id The ID of the Cloudtasker worker.
+      #
+      def initialize(id:, cron:, worker:, task_id: nil, job_id: nil)
+        @id = id
+        @cron = cron
+        @worker = worker
+        @task_id = task_id
+        @job_id = job_id
+      end
+
+      #
+      # Return the redis client.
+      #
+      # @return [Class] The redis client
+      #
+      def redis
+        self.class.redis
+      end
+
+      #
+      # Return the namespaced schedule id.
+      #
+      # @return [String] The namespaced schedule id.
+      #
+      def gid
+        [Config::KEY_NAMESPACE, id].join('/')
+      end
+
+      #
+      # Return true if the schedule is valid.
+      #
+      # @return [Boolean] True if the schedule is valid.
+      #
+      def valid?
+        id && cron_schedule && worker
+      end
+
+      #
+      # Equality operator.
+      #
+      # @param [Any] other The object to compare.
+      #
+      # @return [Boolean] True if the object is equal.
+      #
+      def ==(other)
+        other.is_a?(self.class) && other.id == id
+      end
+
+      #
+      # Return true if the configuration of the schedule was
+      # changed (cron expression or worker).
+      #
+      # @return [Boolean] True if the schedule config was changed.
+      #
+      def config_changed?
+        self.class.find(id)&.to_config != to_config
+      end
+
+      #
+      # RReturn true if the instance attributes were changed compared
+      # to the schedule saved in Redis.
+      #
+      # @return [Boolean] True if the schedule was modified.
+      #
+      def changed?
+        to_h != self.class.find(id).to_h
+      end
+
+      #
+      # Return a hash describing the configuration of this schedule.
+      #
+      # @return [Hash] The config description hash.
+      #
+      def to_config
+        {
+          id: id,
+          cron: cron,
+          worker: worker
+        }
+      end
+
+      #
+      # Return a hash with all the schedule attributes.
+      #
+      # @return [Hash] The attributes hash.
+      #
+      def to_h
+        {
+          id: id,
+          cron: cron,
+          worker: worker,
+          task_id: task_id,
+          job_id: job_id
+        }
+      end
+
+      #
+      # Return the cron schedule to use for the job.
+      #
+      # @return [Fugit::Cron] The cron schedule.
+      #
+      def cron_schedule
+        @cron_schedule ||= Fugit::Cron.parse(cron)
+      end
+
+      #
+      # Return the next time a job should run.
+      #
+      # @param [Time] time An optional reference in time (instead of Time.now)
+      #
+      # @return [EtOrbi::EoTime] The time the schedule job should run next.
+      #
+      def next_time(*args)
+        cron_schedule.next_time(*args)
+      end
+
+      #
+      # Buld edit the object attributes.
+      #
+      # @param [Hash] **opts The attributes to edit.
+      #
+      def assign_attributes(**opts)
+        opts
+          .select { |k, _| instance_variables.include?("@#{k}".to_sym) }
+          .each { |k, v| instance_variable_set("@#{k}", v) }
+      end
+
+      #
+      # Edit the object attributes and save the object in Redis.
+      #
+      # @param [Hash] **opts The attributes to edit.
+      #
+      def update(**opts)
+        assign_attributes(opts)
+        save
+      end
+
+      #
+      # Save the object in Redis. If the configuration was changed
+      # then any existing cloud task is removed and a task is recreated.
+      #
+      def save(update_task: true)
+        return false unless valid? && changed?
+
+        # Save schedule
+        config_was_changed = config_changed?
+        redis.write(gid, to_h)
+
+        # Stop there if backend does not need update
+        return true unless update_task && config_was_changed
+
+        # Delete previous instance
+        Task.delete(task_id) if task_id
+
+        # Schedule worker
+        worker_instance = Object.const_get(worker).new
+        Job.new(worker_instance).set(schedule_id: id).schedule!
+      end
+    end
+  end
+end