sidejob 3.0.0

data/lib/sidejob/port.rb

@@ -0,0 +1,206 @@
+module SideJob
+  # Represents an input or output port from a Job
+  class Port
+    attr_reader :job, :type, :name
+
+    # @param job [SideJob::Job, SideJob::Worker]
+    # @param type [:in, :out] Specifies whether it is input or output port
+    # @param name [Symbol,String] Port names should match [a-zA-Z0-9_]+
+    def initialize(job, type, name)
+      @job = job
+      @type = type.to_sym
+      @name = name.to_sym
+      raise "Invalid port name: #{@name}" if @name !~ /^[a-zA-Z0-9_]+$/ && name != '*'
+    end
+
+    # @return [Boolean] True if two ports are equal
+    def ==(other)
+      other.is_a?(Port) && @job == other.job && @type == other.type && @name == other.name
+    end
+
+    # @see #==
+    def eql?(other)
+      self == other
+    end
+
+    # Returns the port options. Currently supported options are mode and default.
+    # @return [Hash] Port options
+    def options
+      opts = {mode: mode}
+
+      default = SideJob.redis.hget("#{@job.redis_key}:#{type}ports:default", @name)
+      opts[:default] = parse_json(default) if default
+
+      opts
+    end
+
+    # Reset the port options. Currently supported options are mode and default.
+    # @param options [Hash] New port options
+    def options=(options)
+      options = options.symbolize_keys
+      SideJob.redis.multi do |multi|
+        multi.hset "#{@job.redis_key}:#{type}ports:mode", @name, options[:mode] || :queue
+        if options.has_key?(:default)
+          multi.hset "#{@job.redis_key}:#{type}ports:default", @name, options[:default].to_json
+        else
+          multi.hdel "#{@job.redis_key}:#{type}ports:default", @name
+        end
+      end
+    end
+
+    # @return [Symbol, nil] The port mode or nil if the port is invalid
+    def mode
+      mode = SideJob.redis.hget("#{@job.redis_key}:#{type}ports:mode", @name)
+      mode = mode.to_sym if mode
+      mode
+    end
+
+    # Returns the number of items waiting on this port.
+    # @return [Fixnum]
+    def size
+      SideJob.redis.llen(redis_key)
+    end
+
+    # Returns whether {#read} will return data.
+    # @return [Boolean] True if there is data to read.
+    def data?
+      size > 0 || default?
+    end
+
+    # Returns the port default value. To distinguish a null default value vs no default, use {#default?}.
+    # @return [Object, nil] The default value on the port or nil if none
+    def default
+      parse_json SideJob.redis.hget("#{@job.redis_key}:#{type}ports:default", @name)
+    end
+
+    # Returns if the port has a default value.
+    # @return [Boolean] True if the port has a default value
+    def default?
+      SideJob.redis.hexists("#{@job.redis_key}:#{type}ports:default", @name)
+    end
+
+    # Write data to the port. If port in an input port, runs the job.
+    # The default operating mode for a port is :queue which means packets are read/written as a FIFO queue.
+    # In :memory mode, writes do not enter the queue and instead overwrite the default port value.
+    # @param data [Object] JSON encodable data to write to the port
+    def write(data)
+      case mode
+        when :queue
+          SideJob.redis.rpush redis_key, data.to_json
+          @job.run if type == :in
+        when :memory
+          SideJob.redis.hset "#{@job.redis_key}:#{type}ports:default", @name, data.to_json
+        else
+          raise "Missing port #{@name} or invalid mode #{mode}"
+      end
+
+      @job.log({read: [], write: [log_port_data(self, [data])]})
+    end
+
+    # Reads the oldest data from the port. Returns the default value if no data and there is a default.
+    # @return [Object] First data from port
+    # @raise [EOFError] Error raised if no data to be read
+    def read
+      data = SideJob.redis.lpop(redis_key)
+      if data
+        data = parse_json(data)
+      elsif default?
+        data = default
+      else
+        raise EOFError unless data
+      end
+
+      @job.log({read: [log_port_data(self, [data])], write: []})
+
+      data
+    end
+
+    # Connects this port to a number of other ports.
+    # All data is read from the current port and written to the destination ports.
+    # If the current port has a default value, the default is copied to all destination ports.
+    # @param ports [Array<SideJob::Port>, SideJob::Port] Destination port(s)
+    # @param metadata [Hash] If provided, the metadata is merged into the log entry
+    # @return [Array<Object>] Returns all data on current port
+    def connect_to(ports, metadata={})
+      ports = [ports] unless ports.is_a?(Array)
+      ports_by_mode = ports.group_by {|port| port.mode}
+
+      default = SideJob.redis.hget("#{@job.redis_key}:#{type}ports:default", @name)
+
+      # empty the port of all data
+      data = SideJob.redis.multi do |multi|
+        multi.lrange redis_key, 0, -1
+        multi.del redis_key
+      end[0]
+
+      to_run = Set.new
+
+      SideJob.redis.multi do |multi|
+        if data.length > 0
+          (ports_by_mode[:queue] || []).each do |port|
+            multi.rpush port.redis_key, data
+            to_run.add port.job if port.type == :in
+          end
+          if ! default
+            (ports_by_mode[:memory] || []).each do |port|
+              multi.hset "#{port.job.redis_key}:#{port.type}ports:default", port.name, data.last
+            end
+          end
+        end
+
+        if default
+          ports.each do |port|
+            multi.hset "#{port.job.redis_key}:#{port.type}ports:default", port.name, default
+          end
+        end
+      end
+
+      data.map! {|x| parse_json x}
+      if data.length > 0
+        SideJob.log metadata.merge({read: [log_port_data(self, data)], write: ports.map { |port| log_port_data(port, data)}})
+      end
+
+      to_run.each { |job| job.run }
+      data
+    end
+
+    include Enumerable
+    # Iterate over port data. Default values are not returned.
+    # @yield [Object] Each data from port
+    def each(&block)
+      while size > 0 do
+        yield read
+      end
+    rescue EOFError
+    end
+
+    # Returns the redis key used for storing inputs or outputs from a port name
+    # @return [String] Redis key
+    def redis_key
+      "#{@job.redis_key}:#{@type}:#{@name}"
+    end
+    alias :to_s :redis_key
+
+    # @return [Fixnum] Hash value for port
+    def hash
+      redis_key.hash
+    end
+
+    private
+
+    def log_port_data(port, data)
+      x = {job: port.job.id, data: data}
+      x[:"#{port.type}port"] = port.name
+      x
+    end
+
+    # Wrapper around JSON.parse to also handle primitive types.
+    # @param data [String, nil] Data to parse
+    # @return [Object, nil]
+    def parse_json(data)
+      raise "Invalid json #{data}" if data && ! data.is_a?(String)
+      data = JSON.parse("[#{data}]")[0] if data
+      data
+    end
+  end
+end

data/lib/sidejob/server_middleware.rb

@@ -0,0 +1,117 @@
+module SideJob
+  # This middleware is primarily responsible for changing job status depending on events
+  # {SideJob::Job} sets status to terminating or queued when a job is queued
+  # All other job status changes happen here
+  # For simplicity, a job is allowed to be queued multiple times in the Sidekiq queue
+  # Only when it gets pulled out to be run, i.e. here, we decide if we want to actually run it
+  class ServerMiddleware
+    # Configuration parameters for running workers
+    CONFIGURATION = {
+        lock_expiration: 86400, # the worker should not run longer than this number of seconds
+        max_depth: 20, # the job should not be nested more than this number of levels
+        max_runs_per_minute: 60, # generate error if the job is run more often than this
+    }
+
+    # Called by sidekiq as a server middleware to handle running a worker
+    # @param worker [SideJob::Worker]
+    # @param msg [Hash] Sidekiq message format
+    # @param queue [String] Queue the job was pulled from
+    def call(worker, msg, queue)
+      @worker = worker
+      return unless worker.exists? # make sure the job has not been deleted
+      last_run = @worker.get(:ran_at)
+
+      # we skip the run if we already ran once after the enqueued time
+      return if last_run && msg['enqueued_at'] && Time.parse(last_run) > Time.at(msg['enqueued_at'])
+
+      case @worker.status
+        when 'queued'
+          terminate = false
+        when 'terminating'
+          terminate = true
+        else
+          # for any other status, we assume this worker does not need to be run
+          return
+      end
+
+      # if another thread is already running this job, we don't run the job now
+      # this simplifies workers from having to deal with thread safety
+      # we will requeue the job in the other thread
+
+      lock = "#{@worker.redis_key}:lock"
+      now = Time.now.to_f
+      val = SideJob.redis.multi do |multi|
+        multi.get(lock)
+        multi.set(lock, now, {ex: CONFIGURATION[:lock_expiration]}) # add an expiration just in case the lock becomes stale
+      end[0]
+
+      return if val # only run if lock key was not set
+
+      @worker.set ran_at: SideJob.timestamp
+
+      # limit each job to being called too many times per minute
+      # or too deep of a job tree
+      # this is to help prevent bad coding that leads to recursive busy loops
+      # Uses Rate limiter 1 pattern from http://redis.io/commands/INCR
+      rate_key = "#{@worker.redis_key}:rate:#{Time.now.to_i / 60}"
+      rate = SideJob.redis.multi do |multi|
+        multi.incr rate_key
+        multi.expire rate_key, 300 # 5 minutes
+      end[0]
+      if rate.to_i > CONFIGURATION[:max_runs_per_minute]
+        terminate = true
+        SideJob.log({ job: @worker.id, error: 'Job was terminated due to being called too rapidly' })
+      elsif SideJob.redis.llen("#{@worker.redis_key}:ancestors") > CONFIGURATION[:max_depth]
+        terminate = true
+        SideJob.log({ job: @worker.id, error: 'Job was terminated due to being too deep' })
+      end
+
+      if terminate
+        terminate_worker
+      else
+        begin
+          run_worker { yield }
+        ensure
+          val = SideJob.redis.multi do |multi|
+            multi.get lock
+            multi.del lock
+          end[0]
+
+          @worker.run if val && val.to_f != now # run it again if the lock key changed
+        end
+      end
+    end
+
+    private
+
+    def terminate_worker
+      # We let workers perform cleanup before terminating jobs
+      # To prevent workers from preventing termination, errors are ignored
+      @worker.shutdown if @worker.respond_to?(:shutdown)
+    rescue => e
+      add_exception e
+    ensure
+      @worker.status = 'terminated'
+      @worker.parent.run if @worker.parent
+    end
+
+    def run_worker(&block)
+      # normal run
+      @worker.status = 'running'
+      yield
+      @worker.status = 'completed' if @worker.status == 'running'
+    rescue SideJob::Worker::Suspended
+      @worker.status = 'suspended' if @worker.status == 'running'
+    rescue => e
+      @worker.status = 'failed' if @worker.status == 'running'
+      add_exception e
+    ensure
+      @worker.parent.run if @worker.parent
+    end
+
+    def add_exception(exception)
+      # only store the backtrace until the first sidekiq line
+      SideJob.log({ job: @worker.id, error: exception.message, backtrace: exception.backtrace.take_while {|l| l !~ /sidekiq/}.join("\n") })
+    end
+  end
+end

data/lib/sidejob/testing.rb

@@ -0,0 +1,54 @@
+# helpers for testing
+
+module SideJob
+  module Worker
+    # Run jobs until the queue is cleared.
+    # @param timeout [Float] timeout in seconds for Timeout#timeout (default 5)
+    # @param errors [Boolean] Whether to propagate errors that occur in jobs (default true)
+    def self.drain_queue(timeout: 5, errors: true)
+      Timeout::timeout(timeout) do
+        have_job = true
+        while have_job
+          have_job = false
+          Sidekiq::Queue.all.each do |queue|
+            queue.each do |sidekiq_job|
+              have_job = true
+              sidekiq_job.delete
+
+              SideJob.find(sidekiq_job.jid).run_inline(errors: errors, queue: false, args: sidekiq_job.args)
+            end
+          end
+        end
+      end
+    end
+  end
+
+  class Job
+    # Runs a single job once. This method only works for jobs with no child jobs.
+    # @param errors [Boolean] Whether to propagate errors that occur in jobs (default true)
+    # @param queue [Boolean] Whether to force the job to be queued (default true)
+    # @param args [Array] Args to pass to the worker's perform method (default none)
+    def run_inline(errors: true, queue: true, args: [])
+      self.status = 'queued' if queue
+
+      worker = get(:class).constantize.new
+      worker.jid = id
+      SideJob::ServerMiddleware.new.call(worker, {'enqueued_at' => Time.now.to_f}, get(:queue)) do
+        worker.perform(*args)
+      end
+
+      reload
+
+      if errors && status == 'failed'
+        SideJob.logs.each do |event|
+          if event['error']
+            exception = RuntimeError.exception(event['error'])
+            exception.set_backtrace(event['backtrace'])
+            raise exception
+          end
+        end
+        raise "Job #{id} failed but cannot find error log"
+      end
+    end
+  end
+end

data/lib/sidejob/version.rb

@@ -0,0 +1,4 @@
+module SideJob
+  # The current SideJob version
+  VERSION = '3.0.0'
+end

data/lib/sidejob/worker.rb

@@ -0,0 +1,133 @@
+module SideJob
+  # All workers should include SideJob::Worker and implement the perform method.
+  # @see SideJob::JobMethods
+  module Worker
+    @registry ||= {}
+    class << self
+      # This holds the registry for all available workers on one queue
+      attr_reader :registry
+
+      # Workers need to add themselves to the registry even if it's an empty configuration.
+      # This method publishes the registry to redis so that other workers can call workers on this queue.
+      # All workers for the queue should be defined as the existing registry is overwritten.
+      # @param queue [String] Queue to register all defined workers
+      def register_all(queue)
+        SideJob.redis.multi do |multi|
+          multi.del "workers:#{queue}"
+          multi.hmset "workers:#{queue}", @registry.map {|key, val| [key, val.to_json]}.flatten(1) if @registry.size > 0
+        end
+      end
+
+      # Returns the configuration registered for a worker.
+      # @param queue [String] Name of queue
+      # @param klass [String] Name of worker class
+      # @return [Hash, nil] Returns nil if the worker is not defined
+      # @see ClassMethods#register
+      def config(queue, klass)
+        config = SideJob.redis.hget "workers:#{queue}", klass
+        config = JSON.parse(config) if config
+        config
+      end
+    end
+
+    # Class methods added to Workers
+    module ClassMethods
+      # All workers need to register themselves
+      # @param config [Hash] The base configuration used by any jobs of this class
+      def register(config={})
+        SideJob::Worker.registry[self.name] = config
+      end
+    end
+
+    # Methods loaded last to override other included methods
+    module OverrideMethods
+      # Returns the jid set by sidekiq as the job id
+      # @return [String] Job id
+      def id
+        jid
+      end
+    end
+
+    # @see SideJob::Worker
+    def self.included(base)
+      base.class_eval do
+        include Sidekiq::Worker
+        include SideJob::JobMethods
+        include SideJob::Worker::OverrideMethods
+      end
+      base.extend(ClassMethods)
+    end
+
+    # Queues a child job, setting parent and by to self.
+    # @see SideJob.queue
+    def queue(queue, klass, **options)
+      SideJob.queue(queue, klass, options.merge({parent: self, by: "job:#{id}"}))
+    end
+
+    # Exception raised by {#suspend}
+    class Suspended < StandardError; end
+
+    # Immediately suspend the current worker
+    # @raise [SideJob::Worker::Suspended]
+    def suspend
+      raise Suspended
+    end
+
+    # Reads a set of input ports together.
+    # Workers should use this method where possible instead of reading directly from ports due to complexities
+    # of dealing with ports with defaults.
+    # A worker should be idempotent (it can be called multiple times on the same state).
+    # Consider reading from a single port with a default value. Each time it is run, it could read the same data
+    # from the port. The output of the job then could depend on the number of times it is run.
+    # To prevent this, this method requires that there be at least one input port which does not have a default.
+    # Yields data from the ports until either no ports have data or is suspended due to data on some but not all ports.
+    # @param inputs [Array<String>] List of input ports to read
+    # @yield [Array] Splat of input data in same order as inputs
+    # @raise [SideJob::Worker::Suspended] Raised if an input port without a default has data but not all ports
+    # @raise [RuntimeError] An error is raised if all input ports have default values
+    def for_inputs(*inputs, &block)
+      return unless inputs.length > 0
+      ports = inputs.map {|name| input(name)}
+      loop do
+        group_port_logs(job: id) do
+          # error if ports all have defaults, complete if no non-default port inputs, suspend if partial inputs
+          data = ports.map {|port| [ port.data?, port.default? ] }
+          raise "One of these input ports should not have a default value: #{inputs.join(',')}" if data.all? {|x| x[1]}
+          return unless data.any? {|x| x[0] && ! x[1] }
+          suspend unless data.all? {|x| x[0] }
+
+          yield *ports.map(&:read)
+        end
+      end
+    end
+
+    # Sets values in the job's internal state.
+    # @param data [Hash{String,Symbol => Object}] Data to update: objects should be JSON encodable
+    # @raise [RuntimeError] Error raised if job no longer exists
+    def set(data)
+      return unless data.size > 0
+      load_state
+      data.each_pair { |key, val| @state[key.to_s] = val }
+      save_state
+    end
+
+    # Unsets some fields in the job's internal state
+    # @param fields [Array<String,Symbol>] Fields to unset
+    # @raise [RuntimeError] Error raised if job no longer exists
+    def unset(*fields)
+      return unless fields.length > 0
+      load_state
+      fields.each { |field| @state.delete(field.to_s) }
+      save_state
+    end
+
+    private
+
+    def save_state
+      check_exists
+      if @state
+        SideJob.redis.hset 'jobs', id, @state.to_json
+      end
+    end
+  end
+end

data/lib/sidejob.rb

@@ -0,0 +1,116 @@
+require 'sidekiq'
+require 'sidekiq/api'
+require 'sidejob/port'
+require 'sidejob/job'
+require 'sidejob/worker'
+require 'sidejob/server_middleware'
+require 'time' # for iso8601 method
+
+module SideJob
+  # Returns redis connection
+  # If block is given, yields the redis connection
+  # Otherwise, just returns the redis connection
+  def self.redis
+    Sidekiq.redis do |redis|
+      if block_given?
+        yield redis
+      else
+        redis
+      end
+    end
+  end
+
+  # @param redis [Hash] Options for passing to Redis.new
+  def self.redis=(redis)
+    Sidekiq.redis = redis
+  end
+
+  # Main function to queue a job
+  # @param queue [String] Name of the queue to put the job in
+  # @param klass [String] Name of the class that will handle the job
+  # @param args [Array] additional args to pass to the worker's perform method (default none)
+  # @param parent [SideJob::Job] parent job
+  # @param name [String] Name of child job (required if parent specified)
+  # @param at [Time, Float] Time to schedule the job, otherwise queue immediately
+  # @param by [String] Who created this job. Recommend <type>:<id> format for non-jobs as SideJob uses job:<id>.
+  # @param inports [Hash{Symbol,String => Hash}] Input port configuration. Port name to options.
+  # @param outports [Hash{Symbol,String => Hash}] Output port configuration. Port name to options.
+  # @return [SideJob::Job] Job
+  def self.queue(queue, klass, args: nil, parent: nil, name: nil, at: nil, by: nil, inports: nil, outports: nil)
+    raise "No worker registered for #{klass} in queue #{queue}" unless SideJob::Worker.config(queue, klass)
+
+    log_options = {}
+    if parent
+      raise 'Missing name option for job with a parent' unless name
+      raise "Parent already has child job with name #{name}" if parent.child(name)
+      ancestry = [parent.id] + SideJob.redis.lrange("#{parent.redis_key}:ancestors", 0, -1)
+      log_options = {job: parent.id}
+    end
+
+    # To prevent race conditions, we generate the id and set all data in redis before queuing the job to sidekiq
+    # Otherwise, sidekiq may start the job too quickly
+    id = SideJob.redis.incr('jobs:last_id').to_s
+    job = SideJob::Job.new(id)
+
+    SideJob.redis.multi do |multi|
+      multi.hset 'jobs', id, {queue: queue, class: klass, args: args, created_by: by, created_at: SideJob.timestamp}.to_json
+
+      if parent
+        multi.rpush "#{job.redis_key}:ancestors", ancestry # we need to rpush to get the right order
+        multi.hset "#{parent.redis_key}:children", name, id
+      end
+    end
+
+    # initialize ports
+    job.group_port_logs(log_options) do
+      job.inports = inports
+      job.outports = outports
+    end
+
+    job.run(at: at)
+  end
+
+  # Finds a job by id
+  # @param job_id [String, nil] Job Id
+  # @return [SideJob::Job, nil] Job object or nil if it doesn't exist
+  def self.find(job_id)
+    return nil unless job_id
+    job = SideJob::Job.new(job_id)
+    return job.exists? ? job : nil
+  end
+
+  # Returns the current timestamp as a iso8601 string
+  # @return [String] Current timestamp
+  def self.timestamp
+    Time.now.utc.iso8601(9)
+  end
+
+  # Adds a log entry to redis with current timestamp.
+  # @param entry [Hash] Log entry
+  def self.log(entry)
+    SideJob.redis.rpush 'jobs:logs', entry.merge(timestamp: SideJob.timestamp).to_json
+  end
+
+  # Return all job logs and optionally clears them.
+  # @param clear [Boolean] If true, delete logs after returning them (default true)
+  # @return [Array<Hash>] All logs for the job with the oldest first
+  def self.logs(clear: true)
+    SideJob.redis.multi do |multi|
+      multi.lrange 'jobs:logs', 0, -1
+      multi.del 'jobs:logs' if clear
+    end[0].map {|log| JSON.parse(log)}
+  end
+end
+
+# :nocov:
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.remove Sidekiq::Middleware::Server::RetryJobs # we never want sidekiq to retry jobs
+    chain.add SideJob::ServerMiddleware
+  end
+end
+
+if ENV['SIDEJOB_URL']
+  SideJob.redis = {url: ENV['SIDEJOB_URL']}
+end
+# :nocov:

data/sidejob.gemspec

@@ -0,0 +1,21 @@
+require File.expand_path('../lib/sidejob/version', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = 'sidejob'
+  s.version     = SideJob::VERSION
+  s.authors     = ['Austin Che']
+  s.email       = ['austin@ginkgobioworks.com']
+  s.summary     = 'Use SideJob to run sidekiq jobs with a flow-based model'
+
+  s.files = `git ls-files`.split($/)
+
+  s.require_paths = ['lib']
+
+  s.add_dependency 'sidekiq', '~>3.2.5'
+
+  # development
+  s.add_development_dependency 'pry'
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'simplecov'
+  s.add_development_dependency 'yard'
+end