sidejob 3.0.1 → 4.0.1

data/lib/sidejob/port.rb

@@ -1,6 +1,9 @@
 module SideJob
   # Represents an input or output port from a Job
   class Port
+    # Returned by {#read} and {#default} to indicate no data
+    class None < Object; end
+
     attr_reader :job, :type, :name
 
     # @param job [SideJob::Job, SideJob::Worker]
@@ -10,7 +13,8 @@ module SideJob
       @job = job
       @type = type.to_sym
       @name = name.to_sym
-      raise "Invalid port name: #{@name}" if @name !~ /^[a-zA-Z0-9_]+$/ && name != '*'
+      raise "Invalid port name: #{@name}" if @name !~ /^[a-zA-Z0-9_]+$/
+      check_exists
     end
 
     # @return [Boolean] True if two ports are equal
@@ -23,38 +27,6 @@ module SideJob
       self == other
     end
 
-    # Returns the port options. Currently supported options are mode and default.
-    # @return [Hash] Port options
-    def options
-      opts = {mode: mode}
-
-      default = SideJob.redis.hget("#{@job.redis_key}:#{type}ports:default", @name)
-      opts[:default] = parse_json(default) if default
-
-      opts
-    end
-
-    # Reset the port options. Currently supported options are mode and default.
-    # @param options [Hash] New port options
-    def options=(options)
-      options = options.symbolize_keys
-      SideJob.redis.multi do |multi|
-        multi.hset "#{@job.redis_key}:#{type}ports:mode", @name, options[:mode] || :queue
-        if options.has_key?(:default)
-          multi.hset "#{@job.redis_key}:#{type}ports:default", @name, options[:default].to_json
-        else
-          multi.hdel "#{@job.redis_key}:#{type}ports:default", @name
-        end
-      end
-    end
-
-    # @return [Symbol, nil] The port mode or nil if the port is invalid
-    def mode
-      mode = SideJob.redis.hget("#{@job.redis_key}:#{type}ports:mode", @name)
-      mode = mode.to_sym if mode
-      mode
-    end
-
     # Returns the number of items waiting on this port.
     # @return [Fixnum]
     def size
@@ -68,9 +40,10 @@ module SideJob
     end
 
     # Returns the port default value. To distinguish a null default value vs no default, use {#default?}.
-    # @return [Object, nil] The default value on the port or nil if none
+    # @return [Object, None] The default value on the port or {SideJob::Port::None} if none
     def default
-      parse_json SideJob.redis.hget("#{@job.redis_key}:#{type}ports:default", @name)
+      val = SideJob.redis.hget("#{@job.redis_key}:#{type}ports:default", @name)
+      val ? parse_json(val) : None
     end
 
     # Returns if the port has a default value.
@@ -79,27 +52,31 @@ module SideJob
       SideJob.redis.hexists("#{@job.redis_key}:#{type}ports:default", @name)
     end
 
+    # Sets the port default value.
+    # @param val [Object, None] New JSON encodable default value or None to clear the default
+    def default=(val)
+      if val == None
+        SideJob.redis.hdel "#{@job.redis_key}:#{type}ports:default", @name
+      else
+        SideJob.redis.hset "#{@job.redis_key}:#{type}ports:default", @name, val.to_json
+      end
+    end
+
     # Write data to the port. If port in an input port, runs the job.
-    # The default operating mode for a port is :queue which means packets are read/written as a FIFO queue.
-    # In :memory mode, writes do not enter the queue and instead overwrite the default port value.
     # @param data [Object] JSON encodable data to write to the port
     def write(data)
-      case mode
-        when :queue
-          SideJob.redis.rpush redis_key, data.to_json
-          @job.run if type == :in
-        when :memory
-          SideJob.redis.hset "#{@job.redis_key}:#{type}ports:default", @name, data.to_json
-        else
-          raise "Missing port #{@name} or invalid mode #{mode}"
+      # For {SideJob::Worker#for_inputs}, if this is set, we instead set the port default on writes
+      if Thread.current[:sidejob_port_write_default]
+        self.default = data
+      else
+        SideJob.redis.rpush redis_key, data.to_json
       end
-
-      @job.log({read: [], write: [log_port_data(self, [data])]})
+      @job.run(parent: type != :in) # run job if inport otherwise run parent
+      log(write: [ { port: self, data: [data] } ])
     end
 
     # Reads the oldest data from the port. Returns the default value if no data and there is a default.
-    # @return [Object] First data from port
-    # @raise [EOFError] Error raised if no data to be read
+    # @return [Object, None] First data from port or {SideJob::Port::None} if there is no data and no default.
     def read
       data = SideJob.redis.lpop(redis_key)
       if data
@@ -107,10 +84,10 @@ module SideJob
       elsif default?
         data = default
       else
-        raise EOFError unless data
+        return None
       end
 
-      @job.log({read: [log_port_data(self, [data])], write: []})
+      log(read: [ { port: self, data: [data] } ])
 
       data
     end
@@ -119,48 +96,57 @@ module SideJob
     # All data is read from the current port and written to the destination ports.
     # If the current port has a default value, the default is copied to all destination ports.
     # @param ports [Array<SideJob::Port>, SideJob::Port] Destination port(s)
-    # @param metadata [Hash] If provided, the metadata is merged into the log entry
     # @return [Array<Object>] Returns all data on current port
-    def connect_to(ports, metadata={})
+    def connect_to(ports)
       ports = [ports] unless ports.is_a?(Array)
-      ports_by_mode = ports.group_by {|port| port.mode}
 
-      default = SideJob.redis.hget("#{@job.redis_key}:#{type}ports:default", @name)
-
-      # empty the port of all data
-      data = SideJob.redis.multi do |multi|
+      # Get source port data and default
+      (default, data, trash) = result = SideJob.redis.multi do |multi|
+        multi.hget("#{@job.redis_key}:#{@type}ports:default", @name)
+        # get all and empty the port of all data
         multi.lrange redis_key, 0, -1
         multi.del redis_key
-      end[0]
+      end
+
+      default = result[0]
+      data = result[1]
 
-      to_run = Set.new
+      return data unless data.length > 0 || default
+
+      # Get destination port defaults
+      port_defaults = SideJob.redis.multi do |multi|
+        # port defaults
+        ports.each { |port| multi.hget("#{port.job.redis_key}:#{port.type}ports:default", port.name) }
+      end
 
       SideJob.redis.multi do |multi|
         if data.length > 0
-          (ports_by_mode[:queue] || []).each do |port|
+          ports.each_with_index do |port, i|
             multi.rpush port.redis_key, data
-            to_run.add port.job if port.type == :in
-          end
-          if ! default
-            (ports_by_mode[:memory] || []).each do |port|
-              multi.hset "#{port.job.redis_key}:#{port.type}ports:default", port.name, data.last
-            end
           end
         end
 
         if default
-          ports.each do |port|
-            multi.hset "#{port.job.redis_key}:#{port.type}ports:default", port.name, default
+          ports.each_with_index do |port, i|
+            if default != port_defaults[i]
+              multi.hset "#{port.job.redis_key}:#{port.type}ports:default", port.name, default
+            end
           end
         end
       end
 
       data.map! {|x| parse_json x}
       if data.length > 0
-        SideJob.log metadata.merge({read: [log_port_data(self, data)], write: ports.map { |port| log_port_data(port, data)}})
+        log(read: [{ port: self, data: data }], write: ports.map { |port| {port: port, data: data} })
+      end
+
+      # Run the port job or parent only if something was changed
+      ports.each_with_index do |port, i|
+        if data.length > 0 || default != port_defaults[i]
+          port.job.run(parent: port.type != :in)
+        end
       end
 
-      to_run.each { |job| job.run }
       data
     end
 
@@ -171,7 +157,6 @@ module SideJob
       while size > 0 do
         yield read
       end
-    rescue EOFError
     end
 
     # Returns the redis key used for storing inputs or outputs from a port name
@@ -186,21 +171,68 @@ module SideJob
       redis_key.hash
     end
 
+    # Groups all port reads and writes within the block into a single logged event.
+    def self.log_group(&block)
+      outermost = ! Thread.current[:sidejob_port_group]
+      Thread.current[:sidejob_port_group] ||= {read: {}, write: {}} # port -> [data]
+      yield
+    ensure
+      if outermost
+        self._really_log Thread.current[:sidejob_port_group]
+        Thread.current[:sidejob_port_group] = nil
+      end
+    end
+
     private
 
-    def log_port_data(port, data)
-      x = {job: port.job.id, data: data}
-      x[:"#{port.type}port"] = port.name
-      x
+    def self._really_log(entry)
+      return unless entry && (entry[:read].length > 0 || entry[:write].length > 0)
+
+      log_entry = {}
+      %i{read write}.each do |type|
+        log_entry[type] = entry[type].map do |port, data|
+          x = {job: port.job.id, data: data}
+          x[:"#{port.type}port"] = port.name
+          x
+        end
+      end
+
+      SideJob.log log_entry
+    end
+
+    def log(data)
+      entry = Thread.current[:sidejob_port_group] ? Thread.current[:sidejob_port_group] : {read: {}, write: {}}
+      %i{read write}.each do |type|
+        (data[type] || []).each do |x|
+          entry[type][x[:port]] ||= []
+          entry[type][x[:port]].concat JSON.parse(x[:data].to_json) # serialize/deserialize to do a deep copy
+        end
+      end
+      if ! Thread.current[:sidejob_port_group]
+        self.class._really_log(entry)
+      end
     end
 
     # Wrapper around JSON.parse to also handle primitive types.
-    # @param data [String, nil] Data to parse
+    # @param data [String] Data to parse
     # @return [Object, nil]
     def parse_json(data)
-      raise "Invalid json #{data}" if data && ! data.is_a?(String)
-      data = JSON.parse("[#{data}]")[0] if data
-      data
+      JSON.parse("[#{data}]")[0]
+    end
+
+    # Check if the port exists, dynamically creating it if it does not exist and a * port exists for the job
+    # @raise [RuntimeError] Error raised if port does not exist
+    def check_exists
+      return if SideJob.redis.sismember "#{@job.redis_key}:#{type}ports", @name
+      dynamic = SideJob.redis.sismember("#{@job.redis_key}:#{type}ports", '*')
+      raise "Job #{@job.id} does not have #{@type}port #{@name}!" unless dynamic
+      dynamic_default = SideJob.redis.hget("#{@job.redis_key}:#{type}ports:default", '*')
+      SideJob.redis.multi do |multi|
+        multi.sadd "#{@job.redis_key}:#{type}ports", @name
+        if dynamic_default
+          multi.hset "#{@job.redis_key}:#{type}ports:default", @name, dynamic_default
+        end
+      end
     end
   end
 end

data/lib/sidejob/server_middleware.rb

@@ -5,13 +5,6 @@ module SideJob
   # For simplicity, a job is allowed to be queued multiple times in the Sidekiq queue
   # Only when it gets pulled out to be run, i.e. here, we decide if we want to actually run it
   class ServerMiddleware
-    # Configuration parameters for running workers
-    CONFIGURATION = {
-        lock_expiration: 86400, # the worker should not run longer than this number of seconds
-        max_depth: 20, # the job should not be nested more than this number of levels
-        max_runs_per_minute: 60, # generate error if the job is run more often than this
-    }
-
     # Called by sidekiq as a server middleware to handle running a worker
     # @param worker [SideJob::Worker]
     # @param msg [Hash] Sidekiq message format
@@ -20,12 +13,45 @@ module SideJob
       @worker = worker
       return unless @worker.exists? # make sure the job has not been deleted
 
+      # only run if status is queued or terminating
       case @worker.status
-        when 'queued'
-          run_worker { yield }
-        when 'terminating'
-          terminate_worker
-        # for any other status, we assume this worker does not need to be run
+        when 'queued', 'terminating'
+        else
+          return
+      end
+
+      # We use the presence of this lock:worker key to indicate that a worker is trying to the get the job lock.
+      # No other worker needs to also wait and no calls to {SideJob::Job#run} need to queue a new run.
+      return unless SideJob.redis.set("#{@worker.redis_key}:lock:worker", 1, {nx: true, ex: 2})
+
+      # Obtain a lock to allow only one worker to run at a time to simplify workers from having to deal with concurrency
+      token = @worker.lock(CONFIGURATION[:lock_expiration])
+      if token
+        begin
+          SideJob.redis.del "#{@worker.redis_key}:lock:worker"
+          SideJob.log_context(job: @worker.id) do
+            case @worker.status
+              when 'queued'
+                run_worker { yield }
+              when 'terminating'
+                terminate_worker
+              # else no longer need running
+            end
+          end
+        ensure
+          @worker.unlock(token)
+          @worker.run(parent: true) # run the parent every time worker runs
+        end
+      else
+        SideJob.redis.del "#{@worker.redis_key}:lock:worker"
+        # Unable to obtain job lock which may indicate another worker thread is running
+        # Schedule another run
+        # Note that the actual time before requeue depends on sidekiq poll_interval (default 15 seconds)
+        case @worker.status
+          when 'queued', 'terminating'
+            @worker.run(wait: 1)
+          # else no longer need running
+        end
       end
     end
 
@@ -39,64 +65,44 @@ module SideJob
       add_exception e
     ensure
       @worker.status = 'terminated'
-      @worker.parent.run if @worker.parent
     end
 
     def run_worker(&block)
       # limit each job to being called too many times per minute
-      # or too deep of a job tree
-      # this is to help prevent bad coding that leads to recursive busy loops
+      # this is to help prevent bad coding that leads to infinite looping
       # Uses Rate limiter 1 pattern from http://redis.io/commands/INCR
       rate_key = "#{@worker.redis_key}:rate:#{Time.now.to_i / 60}"
       rate = SideJob.redis.multi do |multi|
         multi.incr rate_key
-        multi.expire rate_key, 300 # 5 minutes
+        multi.expire rate_key, 60
       end[0]
 
       if rate.to_i > CONFIGURATION[:max_runs_per_minute]
-        SideJob.log({ job: @worker.id, error: 'Job was terminated due to being called too rapidly' })
-        return @worker.terminate
-      elsif SideJob.redis.llen("#{@worker.redis_key}:ancestors") > CONFIGURATION[:max_depth]
-        SideJob.log({ job: @worker.id, error: 'Job was terminated due to being too deep' })
-        return @worker.terminate
-      end
-
-      # if another thread is already running this job, we don't run the job now
-      # this simplifies workers from having to deal with thread safety
-      # we will requeue the job in the other thread
-      lock = "#{@worker.redis_key}:lock"
-      now = Time.now.to_f
-      val = SideJob.redis.multi do |multi|
-        multi.get(lock)
-        multi.set(lock, now, {ex: CONFIGURATION[:lock_expiration]}) # add an expiration just in case the lock becomes stale
-      end[0]
-
-      return if val # only run if lock key was not set
-
-      begin
+        SideJob.log({ error: 'Job was terminated due to being called too rapidly' })
+        @worker.terminate
+      else
+        # normal run
         @worker.set ran_at: SideJob.timestamp
         @worker.status = 'running'
         yield
         @worker.status = 'completed' if @worker.status == 'running'
-      rescue SideJob::Worker::Suspended
-        @worker.status = 'suspended' if @worker.status == 'running'
-      rescue => e
-        @worker.status = 'failed' if @worker.status == 'running'
-        add_exception e
-      ensure
-        val = SideJob.redis.multi do |multi|
-          multi.get lock
-          multi.del lock
-        end[0]
-
-        @worker.run if val && val.to_f != now # run it again if the lock key changed
-        @worker.parent.run if @worker.parent
       end
+    rescue SideJob::Worker::Suspended
+      @worker.status = 'suspended' if @worker.status == 'running'
+    rescue => e
+      # only set failed if not terminating/terminated
+      case @worker.status
+        when 'terminating', 'terminated'
+        else
+          @worker.status = 'failed'
+      end
+
+      add_exception e
     end
 
     def add_exception(exception)
       # only store the backtrace until the first sidekiq line
-      SideJob.log({ job: @worker.id, error: exception.message, backtrace: exception.backtrace.take_while {|l| l !~ /sidekiq/}.join("\n") })
+      SideJob.log({ error: exception.message, backtrace: exception.backtrace.take_while {|l| l !~ /sidekiq/}.join("\n") })
     end
   end
 end

data/lib/sidejob/testing.rb

@@ -37,8 +37,6 @@ module SideJob
         worker.perform(*args)
       end
 
-      reload
-
       if errors && status == 'failed'
         SideJob.logs.each do |event|
           if event['error']

data/lib/sidejob/version.rb

@@ -1,4 +1,4 @@
 module SideJob
   # The current SideJob version
-  VERSION = '3.0.1'
+  VERSION = '4.0.1'
 end