cuetip 1.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 80caddd838028973a0877c6a4b303d0b36b6ad6f5c98564ed865a60f41ce348a
+  data.tar.gz: 8b5a04b3537ff9a6bad21225ea11becb3ce5d2b2dd1e8c4f30cf8649a55f90d9
+SHA512:
+  metadata.gz: d4b32a2ec0ad43135fd41e9a81cccf217a6415058f43244972835d6f902b5b6ac54cedd547dad7643a0a7ca7852d7a469465b263d2f392d6ba53f191f824accc
+  data.tar.gz: e02bbcd49dac8eada655e0b86d5ce31d8e1f9bf6186e0e0bd14ff79cf2927102a4a3457b6e31fe624a0e486f9b18ca1041244efa57de618ec8e0e1ad16c80dda

data/bin/cuetip

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'cuetip'
+require 'cuetip/config'
+require 'cuetip/version'
+require 'cuetip/worker_group'
+require 'optparse'
+
+$stdout.sync = true
+$stderr.sync = true
+
+options = {}
+OptionParser.new do |opts|
+  opts.version = Cuetip::VERSION
+  opts.banner = 'Usage: cuetip [options]'
+
+  opts.on('-c', '--config PATH', 'The path to your cuetip config file') do |config|
+    options[:config] = config
+  end
+
+  opts.on('-h', '--help', 'Prints this help') do
+    puts opts
+    exit
+  end
+
+  opts.on('-w NUMBER', 'The number of workers to run') do |i|
+    options[:quantity] = i.to_i
+  end
+
+  opts.on('-q', '--queues QUEUE1,QUEUE2', 'Queues that you wish to work on') do |queues|
+    options[:queues] = []
+    queues.split(/,/).uniq.each do |queue|
+      options[:queues] << queue
+    end
+  end
+end.parse!
+
+if options[:config]
+  if File.file?(options[:config])
+    file = File.expand_path(options[:config])
+    require file
+  else
+    puts "Cuetip config file not found at #{options[:config]}"
+    exit 1
+  end
+end
+
+worker = Cuetip::WorkerGroup.new(options[:quantity].to_i == 0 ? Cuetip.config.worker_threads : options[:quantity].to_i, options[:queues])
+worker.start

data/lib/cuetip.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'cuetip'
+require 'cuetip/config'
+require 'cuetip/job'
+require 'cuetip/version'
+
+module Cuetip
+  def self.config
+    @config ||= Config.new
+  end
+
+  def self.logger
+    config.logger
+  end
+
+  def self.configure(&block)
+    block.call(config)
+  end
+end
+
+require 'cuetip/engine' if defined?(Rails)

data/lib/cuetip/config.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext/numeric/bytes'
+require 'active_support/core_ext/numeric/time'
+
+module Cuetip
+  class Config
+    # The length of time between polling
+    def polling_interval
+      @polling_interval || 5
+    end
+    attr_writer :polling_interval
+
+    # The number of worker threads to run
+    def worker_threads
+      @worker_threads || 1
+    end
+    attr_writer :worker_threads
+
+    # Return the logger
+    def logger
+      @logger ||= Logger.new(STDOUT)
+    end
+    attr_writer :logger
+
+    # Define a job event callback
+    def on(event, &block)
+      callbacks[event.to_sym] ||= []
+      callbacks[event.to_sym] << block
+    end
+
+    # Return all callbacks
+    def callbacks
+      @callbacks ||= Hash.new
+    end
+
+    # Emit some callbacks
+    def emit(event, *args)
+      return unless callbacks[event.to_sym]
+
+      callbacks[event.to_sym].each do |callback|
+        callback.call(*args)
+      end
+    end
+  end
+end

data/lib/cuetip/engine.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Cuetip
+  class Engine < Rails::Engine #:nodoc:
+    engine_name 'cuetip'
+  end
+end

data/lib/cuetip/job.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'cuetip/models/job'
+
+module Cuetip
+  class Job
+    class << self
+      # The queue that this job should be executed on
+      def queue_name
+        @queue_name || 'default'
+      end
+      attr_writer :queue_name
+
+      # The maximum length of time (in seconds) that a job can run for
+      def maximum_execution_time
+        @maximum_execution_time || 12.hours
+      end
+      attr_writer :maximum_execution_time
+
+      # The maximum length of time (in seconds) between the job being created and it being run
+      def ttl
+        @ttl || 6.hours
+      end
+      attr_writer :ttl
+
+      # The maximum number of times this job can be run
+      def retry_count
+        @retry_count || 0
+      end
+      attr_writer :retry_count
+
+      # The maximum length of time (in seconds) between each execution of this job
+      def retry_interval
+        @retry_interval || 1.minute
+      end
+      attr_writer :retry_interval
+
+      # The length of time (in seconds) from when this job is queued to when it should be executed
+      def delay_execution
+        @delay_execution || 0
+      end
+      attr_writer :delay_execution
+
+      # Queue this job
+      #
+      # @param params [Hash]
+      # @return [Cuetip::Models::Job]
+      def queue(params = {}, &block)
+        # Create our new job
+        job = Models::Job.new(class_name: name, params: params)
+        # Copy over any class leve lconfig
+        job.queue_name = queue_name
+        job.maximum_execution_time = maximum_execution_time
+        job.ttl = ttl
+        job.retry_count = retry_count
+        job.retry_interval = retry_interval
+        job.delay_execution = delay_execution
+        # Call the block
+        block.call(job) if block_given?
+        # Create the job
+        job.save!
+        # Return the job
+        job
+      end
+    end
+
+    # Initialize this job instance by providing a queued job instance
+    #
+    # @param queued_job [Cuetip::Models::Job]
+    def initialize(job)
+      @job = job
+    end
+
+    # Perform a job
+    #
+    # @return [void]
+    def perform; end
+
+    private
+
+    # Return all parameters for the job
+    #
+    # @return [Hashie::Mash]
+    def params
+      @job.params
+    end
+
+    # Return the queued job object
+    #
+    # @return [Cuetip::Models::Job]
+    attr_reader :job
+
+    # Return a quick access for the job
+    #
+    # @return [Logger]
+    def logger
+      Cuetip.logger
+    end
+  end
+end

data/lib/cuetip/models/job.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require 'active_record'
+require 'cuetip/models/queued_job'
+require 'cuetip/serialized_hashie'
+
+module Cuetip
+  module Models
+    class Job < ActiveRecord::Base
+      self.table_name = 'cuetip_jobs'
+
+      STATUSES = %w[Pending Running Complete Aborted Expired].freeze
+
+      has_one :queued_job, class_name: 'Cuetip::Models::QueuedJob'
+      belongs_to :associated_object, polymorphic: true, optional: true
+
+      serialize :params, Cuetip::SerializedHashie
+
+      before_validation(on: :create) do
+        self.status = 'Pending'
+      end
+
+      after_create do
+        # After creation, automatically add this job into the job queue for execution
+        create_queued_job!(run_after: run_after || delay_execution&.seconds&.from_now, queue_name: queue_name)
+      end
+
+      # Is this job in the queue
+      def queued?
+        queued_job.present?
+      end
+
+      # Has this job expired?
+      def expired?
+        ttl? ? expires_at <= Time.now : false
+      end
+
+      # The time that this job expired
+      def expires_at
+        ttl? ? created_at + ttl : nil
+      end
+
+      # Should this job be requeued on a failure right now?
+      def requeue_on_failure?
+        retry_count && retry_interval ? executions <= retry_count : false
+      end
+
+      # Remove this job from the queue
+      def remove_from_queue
+        queued_job&.destroy
+        self.queued_job = nil
+        log 'Removed from queue'
+      end
+
+      # Log some text about this job
+      #
+      # @param text [String]
+      def log(text)
+        Cuetip.logger.info "[#{id}] #{text}"
+      end
+
+      # Execute the job
+      #
+      # @return [Boolean] whether the job executed successfully or not
+      def execute(&block)
+        log "Beginning execution of job #{id} with #{class_name}"
+        # Initialize a new instance of the job we wish to execute
+        job_klass = class_name.constantize.new(self)
+
+        # If the job has expired, we should not be executing this so we'll just
+        # remove it from the queue and mark it as expired.
+        if expired?
+          log 'Job has expired'
+          self.status = 'Expired'
+          remove_from_queue
+          Cuetip.config.emit(:expired, self, job_klass)
+          return false
+        end
+
+        Cuetip.config.emit(:before_perform, self, job_klass)
+
+        # If we have a block, call this so we can manipulate our actual job class
+        # before execution if needed (mostly for testing)
+        block.call(job_klass) if block_given?
+
+        # Mark the job as runnign
+        update!(status: 'Running', started_at: Time.now, executions: executions + 1)
+
+        begin
+          # Perform the job within a timeout
+          Timeout.timeout(maximum_execution_time || 1.year) do
+            job_klass.perform
+          end
+          # Mark the job as complete and remove it from the queue
+          self.status = 'Complete'
+          log 'Job completed successfully'
+          remove_from_queue
+
+          Cuetip.config.emit(:completed, self, job_klass)
+
+          true
+        rescue Exception, Timeout::TimeoutError => e
+          log "Job failed with #{e.class} (#{e.message})"
+
+          # If there's an error, mark the job as failed and copy exception
+          # data into the job
+          self.status = 'Failed'
+          self.exception_class = e.class.name
+          self.exception_message = e.message
+          self.exception_backtrace = e.backtrace.join("\n")
+
+          # Handle requeing the job if needed.
+          if requeue_on_failure?
+            # Requeue this job for execution again after the retry interval.
+            new_job = queued_job.requeue(run_after: Time.now + retry_interval.to_i)
+            log "Requeing job to run after #{new_job.run_after.to_s(:long)}"
+            self.status = 'Pending'
+          else
+            # We're done with this job. We can't do any more retries.
+            remove_from_queue
+          end
+
+          Cuetip.config.emit(:exception, e, self, job_klass)
+
+          false
+        end
+      ensure
+        self.finished_at = Time.now
+        save!
+        Cuetip.config.emit(:finished, self, job_klass)
+        log 'Finished processing'
+      end
+    end
+  end
+end

data/lib/cuetip/models/queued_job.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'socket'
+require 'active_record'
+require 'cuetip/models/job'
+
+module Cuetip
+  module Models
+    class QueuedJob < ActiveRecord::Base
+      PROCESS_IDENTIFIER = Socket.gethostname + ":#{Process.pid}"
+      self.table_name = 'cuetip_job_queue'
+
+      scope :pending, -> { where(locked_at: nil).where('run_after is null or run_after < ?', Time.now) }
+      scope :from_queues, -> (queues) { where(queue_name: queues) }
+
+      belongs_to :job, class_name: 'Cuetip::Models::Job'
+
+      # Unlock the job and allow it to be re-run elsewhere.
+      def requeue(attributes = {})
+        self.attributes = attributes
+        self.locked_by = nil
+        self.locked_at = nil
+        save!
+        self
+      end
+
+      # Generate a random lock ID to use in the locking process
+      def self.generate_lock_id
+        PROCESS_IDENTIFIER + ':' + rand(1_000_000_000).to_s.rjust(9, '0')
+      end
+
+      # Simultaneously find an outstanding job and lock it
+      def self.find_and_lock(queued_job_id = nil)
+        lock_id = generate_lock_id
+        scope = if queued_job_id
+                  where(id: queued_job_id)
+                else
+                  self
+                end
+        count = scope.pending.limit(1).update_all(locked_by: lock_id, locked_at: Time.now)
+        QueuedJob.find_by_locked_by(lock_id) if count > 0
+      end
+    end
+  end
+end

data/lib/cuetip/serialized_hashie.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'hashie/mash'
+
+module Cuetip
+  class SerializedHashie < Hashie::Mash
+    def self.dump(obj)
+      obj.reject! { |_k, v| v.blank? }
+      obj.each do |key, value|
+        obj[key] = value.reject(&:blank?) if value.is_a?(Array)
+      end
+      ActiveSupport::JSON.encode(obj.to_h)
+    end
+
+    def self.load(raw_hash)
+      new(JSON.parse(raw_hash || '{}'))
+    end
+  end
+end

data/lib/cuetip/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Cuetip
+  VERSION = '1.2.0'
+end

data/lib/cuetip/worker.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require 'cuetip/models/queued_job'
+
+module Cuetip
+  class Worker
+    attr_reader :status
+
+    include ActiveSupport::Callbacks
+
+    define_callbacks :execute, :poll
+
+    def initialize(group, id, queues)
+      @group = group
+      @id = id
+      @queues = queues
+    end
+
+    def request_exit!
+      @exit_requested = true
+      interrupt_sleep
+    end
+
+    def run
+      set_status('idle')
+      loop do
+        unless run_once
+          interruptible_sleep(Cuetip.config.polling_interval + rand)
+        end
+
+        break if @exit_requested
+      end
+    end
+
+    def run_once
+      set_status('polling')
+      run_callbacks :poll do
+        queued_job = silence do
+          if @queues.any?
+            scope = Cuetip::Models::QueuedJob.from_queues(@queues)
+          else
+            scope = Cuetip::Models::QueuedJob
+          end
+          scope.find_and_lock
+        end
+
+        if queued_job
+          set_status("executing #{queued_job.job.id}")
+          run_callbacks :execute do
+            queued_job.job.execute
+          end
+          set_status('idle')
+          true
+        else
+          set_status('idle')
+          false
+        end
+      end
+    end
+
+    private
+
+    def set_status(status)
+      @status = status
+      @group&.set_process_name
+    end
+
+    def interruptible_sleep(seconds)
+      sleep_check, @sleep_interrupt = IO.pipe
+      IO.select([sleep_check], nil, nil, seconds)
+      sleep_check.close
+      @sleep_interrupt.close
+    end
+
+    def interrupt_sleep
+      @sleep_interrupt&.close
+    end
+
+    def silence(&block)
+      if ActiveRecord::Base.logger
+        ActiveRecord::Base.logger.silence(&block)
+      else
+        block.call
+      end
+    end
+  end
+end

data/lib/cuetip/worker_group.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'cuetip/worker'
+
+module Cuetip
+  class WorkerGroup
+    include ActiveSupport::Callbacks
+
+    define_callbacks :run_worker
+
+    attr_reader :quantity
+    attr_reader :workers
+    attr_reader :threads
+
+    def initialize(quantity, queues)
+      @quantity = quantity
+      @queues = queues || []
+      @workers = {}
+      @threads = {}
+    end
+
+    def start
+      Cuetip.logger.info "Starting #{@quantity} Cuetip workers"
+      if @queues.any?
+        @queues.each { |q| Cuetip.logger.info "-> Joined queue: #{q.to_s}" }
+      end
+
+      exit_trap = proc do
+        @workers.each { |_, worker| worker.request_exit! }
+        puts 'Exiting...'
+      end
+
+      trap('INT', &exit_trap)
+      trap('TERM', &exit_trap)
+
+      @quantity.times do |i|
+        @workers[i] = Worker.new(self, i, @queues)
+        Cuetip.logger.info "-> Starting worker #{i}"
+        @threads[i] = Thread.new(@workers[i]) do |worker|
+          run_callbacks :run_worker do
+            worker.run
+          end
+        end
+        @threads[i].abort_on_exception = true
+      end
+      @threads.values.each(&:join)
+    end
+
+    def set_process_name
+      thread_names = @workers.values.map(&:status)
+      Process.setproctitle("Cuetip: #{thread_names.inspect}")
+    end
+  end
+end

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification
+name: cuetip
+version: !ruby/object:Gem::Version
+  version: 1.2.0
+platform: ruby
+authors:
+- Adam Cooke
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-04-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: An ActiveRecord job queueing system
+email:
+- me@adamcooke.io
+executables:
+- cuetip
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/cuetip
+- lib/cuetip.rb
+- lib/cuetip/config.rb
+- lib/cuetip/engine.rb
+- lib/cuetip/job.rb
+- lib/cuetip/models/job.rb
+- lib/cuetip/models/queued_job.rb
+- lib/cuetip/serialized_hashie.rb
+- lib/cuetip/version.rb
+- lib/cuetip/worker.rb
+- lib/cuetip/worker_group.rb
+homepage: https://github.com/adamcooke/cuetip
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: An ActiveRecord job queueing system
+test_files: []