sidejob 3.0.1 → 4.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f39dce3d26065392bfa7e0d886b2976dc009a579
-  data.tar.gz: 7ad307e865f66fa57caf72f514d390589aa1f674
+  metadata.gz: b52b912e86a573ece661bf11ca1a5eadcd410a98
+  data.tar.gz: a545d0cafa16991899ee4890713a8fef114e6c7c
 SHA512:
-  metadata.gz: 8bfb90bec7b756867cebd04e4f40d0c3ed635c0235d3d4aeebf9fa343dff3bb3cfd23a3a6f17b26151c7ec34c5b3ade360ffa721db4aa3a052f716dd46454773
-  data.tar.gz: c65db5c4986995174e8f46637d81618229e3375b67d635680221524f93d0947b47192d8cb06ad4c536ca13eaca2e70356cb0c60d6de1a811121f209374c9541a
+  metadata.gz: 7c95368c762967a120677d1c01bfe073a73fd4d6e034d186d0d6d40dc1e18fe9663a7ebb110ed2a3b325e3093d8402b7178ec3a415c992aafb1e41ca03408d3e
+  data.tar.gz: 45a2d1e8a186af6236a000fb5ae66d4f9a4ea1238fbd578663c4827bbfc8c90cad733ca03cd914568815e6ee4294ce50e0e163d04832bb1da358e0bcb0bf15e6

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    sidejob (3.0.1)
+    sidejob (4.0.1)
       sidekiq (~> 3.2.5)
 
 GEM
@@ -10,18 +10,18 @@ GEM
     celluloid (0.15.2)
       timers (~> 1.1.0)
     coderay (1.1.0)
-    connection_pool (2.0.0)
+    connection_pool (2.2.0)
     diff-lcs (1.2.5)
     docile (1.1.5)
-    json (1.8.1)
+    json (1.8.2)
     method_source (0.8.2)
     multi_json (1.10.1)
     pry (0.9.12.6)
       coderay (~> 1.0)
       method_source (~> 0.8)
       slop (~> 3.4)
-    redis (3.1.0)
-    redis-namespace (1.5.1)
+    redis (3.2.1)
+    redis-namespace (1.5.2)
       redis (~> 3.0, >= 3.0.4)
     rspec (3.1.0)
       rspec-core (~> 3.1.0)

data/README.md

@@ -46,13 +46,7 @@ Ports
 * Any object that can be JSON encoded can be written or read from any input or output port.
 * Ports must be explicitly specified for each job either by the worker configuration or when queuing new jobs unless
   a port named `*` exists in which case new ports are dynamically created and inherit its options.
-
-Port options:
-
-* mode
-    * Queue - This is the default operation mode. All data written is read in a first in first out manner.
-    * Memory - No data is stored on the port. The most recent value sets the port default value.
-* default - Default value when a read is done on the port with no data
+* Currently, the only port option is a default value which is returned when a read is done on the port when its empty.
 
 Workers
 -------
@@ -103,22 +97,24 @@ Additional keys used by SideJob:
 * jobs:logs - List with JSON encoded logs.
     * { timestamp: (date), job: (id), read: [{ job: (id), (in|out)port: (port), data: [...] }, ...], write: [{ job: (id), (in|out)port: (port), data: [...] }, ...] }
     * { timestamp: (date), job: (id), error: (message), backtrace: (exception backtrace) }
-* jobs - Hash mapping active job IDs to JSON encoded job state.
+* jobs - Set with all job ids
+* job:(id) - Hash containing job state. Each value is JSON encoded.
+    * status - job status
     * queue - queue name
     * class - name of class
     * args - array of arguments passed to worker's perform method
+    * parent - parent job ID
     * created_at - timestamp that the job was first queued
     * created_by - string indicating the entity that created the job. SideJob uses job:(id) for jobs created by another job.
     * ran_at - timestamp of the start of the last run
-    * Any additional keys used by the worker to track internal state
-* job:(id):status - Job status as string.
+    * Any additional keys used by the worker to track internal job state
 * job:(id):in:(inport) and job:(id):out:(outport) - List with unread port data. New data is pushed on the right.
-* job:(id):inports:mode and job:(id):outports:mode - Hash mapping port name to port mode. All existing ports must be here.
+* job:(id):inports and job:(id):outports - Set containing all existing port names.
 * job:(id):inports:default and job:(id):outports:default - Hash mapping port name to JSON encoded default value for port.
-* job:(id):ancestors - List with parent job IDs up to the root job that has no parent.
-    Newer jobs are pushed on the left so the immediate parent is on the left and the root job is on the right.
 * job:(id):children - Hash mapping child job name to child job ID
 * job:(id):rate:(timestamp) - Rate limiter used to prevent run away executing of a job.
     Keys are automatically expired.
-* job:(id):lock - Used to prevent multiple worker threads from running a job.
+* job:(id):lock - Used to control concurrent writes to a job.
+    Auto expired to prevent stale locks.
+* job:(id):lock:worker - Used to indicate a worker is attempting to acquire the job lock.
     Auto expired to prevent stale locks.

data/lib/sidejob.rb

@@ -5,8 +5,15 @@ require 'sidejob/job'
 require 'sidejob/worker'
 require 'sidejob/server_middleware'
 require 'time' # for iso8601 method
+require 'securerandom'
 
 module SideJob
+  # Configuration parameters
+  CONFIGURATION = {
+      lock_expiration: 60, # workers should not run longer than this number of seconds
+      max_runs_per_minute: 600, # terminate jobs that run too often
+  }
+
   # Returns redis connection
   # If block is given, yields the redis connection
   # Otherwise, just returns the redis connection
@@ -39,44 +46,32 @@ module SideJob
   def self.queue(queue, klass, args: nil, parent: nil, name: nil, at: nil, by: nil, inports: nil, outports: nil)
     raise "No worker registered for #{klass} in queue #{queue}" unless SideJob::Worker.config(queue, klass)
 
-    log_options = {}
-    if parent
-      raise 'Missing name option for job with a parent' unless name
-      raise "Parent already has child job with name #{name}" if parent.child(name)
-      ancestry = [parent.id] + SideJob.redis.lrange("#{parent.redis_key}:ancestors", 0, -1)
-      log_options = {job: parent.id}
-    end
-
     # To prevent race conditions, we generate the id and set all data in redis before queuing the job to sidekiq
     # Otherwise, sidekiq may start the job too quickly
     id = SideJob.redis.incr('jobs:last_id').to_s
+    SideJob.redis.sadd 'jobs', id
     job = SideJob::Job.new(id)
 
-    SideJob.redis.multi do |multi|
-      multi.hset 'jobs', id, {queue: queue, class: klass, args: args, created_by: by, created_at: SideJob.timestamp}.to_json
+    job.set({queue: queue, class: klass, args: args, status: 'completed', created_by: by, created_at: SideJob.timestamp})
 
-      if parent
-        multi.rpush "#{job.redis_key}:ancestors", ancestry # we need to rpush to get the right order
-        multi.hset "#{parent.redis_key}:children", name, id
-      end
+    if parent
+      raise 'Missing name option for job with a parent' unless name
+      parent.adopt(job, name)
     end
 
     # initialize ports
-    job.group_port_logs(log_options) do
-      job.inports = inports
-      job.outports = outports
-    end
+    job.inports = inports
+    job.outports = outports
 
     job.run(at: at)
   end
 
   # Finds a job by id
-  # @param job_id [String, nil] Job Id
+  # @param job_id [Integer, nil] Job Id
   # @return [SideJob::Job, nil] Job object or nil if it doesn't exist
   def self.find(job_id)
     return nil unless job_id
-    job = SideJob::Job.new(job_id)
-    return job.exists? ? job : nil
+    job = SideJob::Job.new(job_id) rescue nil
   end
 
   # Returns the current timestamp as a iso8601 string
@@ -88,18 +83,29 @@ module SideJob
   # Adds a log entry to redis with current timestamp.
   # @param entry [Hash] Log entry
   def self.log(entry)
-    SideJob.redis.rpush 'jobs:logs', entry.merge(timestamp: SideJob.timestamp).to_json
+    context = (Thread.current[:sidejob_log_context] || {}).merge(timestamp: SideJob.timestamp)
+    SideJob.redis.rpush 'jobs:logs', context.merge(entry).to_json
   end
 
   # Return all job logs and optionally clears them.
   # @param clear [Boolean] If true, delete logs after returning them (default true)
-  # @return [Array<Hash>] All logs for the job with the oldest first
+  # @return [Array<Hash>] All logs with the oldest first
   def self.logs(clear: true)
     SideJob.redis.multi do |multi|
       multi.lrange 'jobs:logs', 0, -1
       multi.del 'jobs:logs' if clear
     end[0].map {|log| JSON.parse(log)}
   end
+
+  # Adds the given metadata to all {SideJob.log} calls within the block.
+  # @param metadata [Hash] Metadata to be merged with each log entry
+  def self.log_context(metadata, &block)
+    previous = Thread.current[:sidejob_log_context]
+    Thread.current[:sidejob_log_context] = (previous || {}).merge(metadata.symbolize_keys)
+    yield
+  ensure
+    Thread.current[:sidejob_log_context] = previous
+  end
 end
 
 # :nocov:

data/lib/sidejob/job.rb

@@ -2,7 +2,6 @@ module SideJob
   # Methods shared between {SideJob::Job} and {SideJob::Worker}.
   module JobMethods
     attr_reader :id
-    attr_accessor :logger
 
     # @return [Boolean] True if two jobs or workers have the same id
     def ==(other)
@@ -28,80 +27,41 @@ module SideJob
     # Returns if the job still exists.
     # @return [Boolean] Returns true if this job exists and has not been deleted
     def exists?
-      SideJob.redis.hexists 'jobs', id
-    end
-
-    # If a job logger is defined, call the log method on it with the log entry. Otherwise, call {SideJob.log}.
-    # @param entry [Hash] Log entry
-    def log(entry)
-      entry[:job] = id unless entry[:job]
-      (@logger || SideJob).log(entry)
-    end
-
-    # Groups all port reads and writes within the block into a single logged event.
-    # @param metadata [Hash] If provided, the metadata is merged into the final log entry
-    def group_port_logs(metadata={}, &block)
-      new_group = @logger.nil?
-      @logger ||= GroupPortLogs.new(self)
-      @logger.add_metadata metadata
-      yield
-    ensure
-      if new_group
-        @logger.done
-        @logger = nil
-      end
+      SideJob.redis.sismember 'jobs', id
     end
 
     # Retrieve the job's status.
     # @return [String] Job status
     def status
-      SideJob.redis.get "#{redis_key}:status"
+      get(:status)
     end
 
     # Set the job status.
     # @param status [String] The new job status
     def status=(status)
-      SideJob.redis.set "#{redis_key}:status", status
-    end
-
-    # Prepare to terminate the job. Sets status to 'terminating'.
-    # Then queues the job so that its shutdown method if it exists can be run.
-    # After shutdown, the status will be 'terminated'.
-    # If the job is currently running, it will finish running first.
-    # If the job is already terminated, it does nothing.
-    # To start the job after termination, call {#run} with force: true.
-    # @param recursive [Boolean] If true, recursively terminate all children (default false)
-    # @return [SideJob::Job] self
-    def terminate(recursive: false)
-      if status != 'terminated'
-        self.status = 'terminating'
-        sidekiq_queue
-      end
-      if recursive
-        children.each_value do |child|
-          child.terminate(recursive: true)
-        end
-      end
-      self
+      set({status: status})
     end
 
     # Run the job.
     # This method ensures that the job runs at least once from the beginning.
     # If the job is currently running, it will run again.
     # Just like sidekiq, we make no guarantees that the job will not be run more than once.
-    # Unless force is set, if the status is terminating or terminated, the job will not be run.
+    # Unless force is set, the job will only be run if the status is running, queued, suspended, or completed.
+    # @param parent [Boolean] Whether to run parent job instead of this one
     # @param force [Boolean] Whether to run if job is terminated (default false)
     # @param at [Time, Float] Time to schedule the job, otherwise queue immediately
     # @param wait [Float] Run in the specified number of seconds
-    # @return [SideJob::Job] self
-    def run(force: false, at: nil, wait: nil)
+    # @return [SideJob::Job, nil] The job that was run or nil if no job was run
+    def run(parent: false, force: false, at: nil, wait: nil)
       check_exists
 
-      case status
-        when 'terminating', 'terminated'
-          return unless force
+      if parent
+        pj = self.parent
+        return pj ? pj.run(force: force, at: at, wait: wait) : nil
       end
 
+      return nil unless force || %w{running queued suspended completed}.include?(status)
+
       self.status = 'queued'
 
       time = nil
@@ -116,6 +76,43 @@ module SideJob
       self
     end
 
+    # Returns if job and all children are terminated.
+    # @return [Boolean] True if this job and all children recursively are terminated
+    def terminated?
+      return false if status != 'terminated'
+      children.each_value do |child|
+        return false unless child.terminated?
+      end
+      return true
+    end
+
+    # Prepare to terminate the job. Sets status to 'terminating'.
+    # Then queues the job so that its shutdown method if it exists can be run.
+    # After shutdown, the status will be 'terminated'.
+    # If the job is currently running, it will finish running first.
+    # If the job is already terminated, it does nothing.
+    # To start the job after termination, call {#run} with force: true.
+    # @param recursive [Boolean] If true, recursively terminate all children (default false)
+    # @return [SideJob::Job] self
+    def terminate(recursive: false)
+      if status != 'terminated'
+        self.status = 'terminating'
+        sidekiq_queue
+      end
+      if recursive
+        children.each_value do |child|
+          child.terminate(recursive: true)
+        end
+      end
+      self
+    end
+
+    # Queues a child job, setting parent and by to self.
+    # @see SideJob.queue
+    def queue(queue, klass, **options)
+      SideJob.queue(queue, klass, options.merge({parent: self, by: "job:#{id}"}))
+    end
+
     # Returns a child job by name.
     # @param name [Symbol, String] Child job name to look up
     # @return [SideJob::Job, nil] Child job or nil if not found
@@ -129,28 +126,45 @@ module SideJob
       SideJob.redis.hgetall("#{redis_key}:children").each_with_object({}) {|child, hash| hash[child[0]] = SideJob.find(child[1])}
     end
 
-    # Returns all ancestor jobs.
-    # @return [Array<SideJob::Job>] Ancestors (parent will be first and root job will be last)
-    def ancestors
-      SideJob.redis.lrange("#{redis_key}:ancestors", 0, -1).map { |id| SideJob.find(id) }
-    end
-
     # Returns the parent job.
     # @return [SideJob::Job, nil] Parent job or nil if none
     def parent
-      parent = SideJob.redis.lindex("#{redis_key}:ancestors", 0)
+      parent = get(:parent)
       parent = SideJob.find(parent) if parent
       parent
     end
 
-    # Returns if job and all children are terminated.
-    # @return [Boolean] True if this job and all children recursively are terminated
-    def terminated?
-      return false if status != 'terminated'
-      children.each_value do |child|
-        return false unless child.terminated?
+    # Disown a child job so that it no longer has a parent.
+    # @param name_or_job [String, SideJob::Job] Name or child job to disown
+    def disown(name_or_job)
+      if name_or_job.is_a?(SideJob::Job)
+        job = name_or_job
+        name = children.rassoc(job)
+        raise "Job #{id} cannot disown job #{job.id} as it is not a child" unless name
+      else
+        name = name_or_job
+        job = child(name)
+        raise "Job #{id} cannot disown non-existent child #{name}" unless job
+      end
+
+      SideJob.redis.multi do |multi|
+        multi.hdel job.redis_key, 'parent'
+        multi.hdel "#{redis_key}:children", name
+      end
+    end
+
+    # Adopt a parent-less job as a child of this job.
+    # @param orphan [SideJob::Job] Job that has no parent
+    # @param name [String] Name of child job (must be unique among children)
+    def adopt(orphan, name)
+      raise "Job #{id} cannot adopt itself as a child" if orphan == self
+      raise "Job #{id} cannot adopt job #{orphan.id} as it already has a parent" unless orphan.parent.nil?
+      raise "Job #{id} cannot adopt job #{orphan.id} as child name #{name} is not unique" if name.nil? || ! child(name).nil?
+
+      SideJob.redis.multi do |multi|
+        multi.hset orphan.redis_key, 'parent', id.to_json
+        multi.hset "#{redis_key}:children", name, orphan.id
       end
-      return true
     end
 
     # Deletes the job and all children jobs (recursively) if all are terminated.
@@ -158,42 +172,46 @@ module SideJob
     def delete
       return false unless terminated?
 
-      # recursively delete all children first
-      children.each_value do |child|
-        child.delete
-      end
+      parent = self.parent
+      parent.disown(self) if parent
 
-      # delete all SideJob keys
+      children = self.children
+
+      # delete all SideJob keys and disown all children
       ports = inports.map(&:redis_key) + outports.map(&:redis_key)
       SideJob.redis.multi do |multi|
-        multi.hdel 'jobs', id
-        multi.del ports + %w{status children ancestors inports:mode outports:mode inports:default outports:default}.map {|x| "#{redis_key}:#{x}" }
+        multi.srem 'jobs', id
+        multi.del redis_key
+        multi.del ports + %w{children inports outports inports:default outports:default}.map {|x| "#{redis_key}:#{x}" }
+        children.each_value { |child| multi.hdel child.redis_key, 'parent' }
+      end
+
+      # recursively delete all children
+      children.each_value do |child|
+        child.delete
       end
-      reload
+
       return true
     end
 
     # Returns an input port.
     # @param name [Symbol,String] Name of the port
     # @return [SideJob::Port]
-    # @raise [RuntimeError] Error raised if port does not exist
     def input(name)
-      get_port :in, name
+      SideJob::Port.new(self, :in, name)
     end
 
     # Returns an output port
     # @param name [Symbol,String] Name of the port
     # @return [SideJob::Port]
-    # @raise [RuntimeError] Error raised if port does not exist
     def output(name)
-      get_port :out, name
+      SideJob::Port.new(self, :out, name)
     end
 
     # Gets all input ports.
     # @return [Array<SideJob::Port>] Input ports
     def inports
-      load_ports if ! @ports
-      @ports[:in].values
+      all_ports :in
     end
 
     # Sets the input ports for the job.
@@ -207,8 +225,7 @@ module SideJob
     # Gets all output ports.
     # @return [Array<SideJob::Port>] Output ports
     def outports
-      load_ports if ! @ports
-      @ports[:out].values
+      all_ports :out
     end
 
     # Sets the input ports for the job.
@@ -219,27 +236,74 @@ module SideJob
       set_ports :out, ports
     end
 
+    # Returns the entirety of the job's state with both standard and custom keys.
+    # @return [Hash{String => Object}] Job state
+    def state
+      state = SideJob.redis.hgetall(redis_key)
+      raise "Job #{id} does not exist!" if ! state
+      state.update(state) {|k,v| JSON.parse("[#{v}]")[0]}
+      state
+    end
+
     # Returns some data from the job's state.
-    # The job state is cached for the lifetime of the job object. Call {#reload} if the state may have changed.
     # @param key [Symbol,String] Retrieve value for the given key
     # @return [Object,nil] Value from the job state or nil if key does not exist
-    # @raise [RuntimeError] Error raised if job no longer exists
     def get(key)
-      load_state
-      @state[key.to_s]
+      val = SideJob.redis.hget(redis_key, key)
+      val ? JSON.parse("[#{val}]")[0] : nil
     end
 
-    # Clears the state and ports cache.
-    def reload
-      @state = nil
-      @ports = nil
-      @config = nil
+    # Sets values in the job's internal state.
+    # @param data [Hash{String,Symbol => Object}] Data to update: objects should be JSON encodable
+    # @raise [RuntimeError] Error raised if job no longer exists
+    def set(data)
+      check_exists
+      return unless data.size > 0
+      SideJob.redis.hmset redis_key, *(data.map {|k,v| [k, v.to_json]}.flatten)
     end
 
-    # Returns the worker configuration for the job.
-    # @see SideJob::Worker.config
-    def config
-      @config ||= SideJob::Worker.config(get(:queue), get(:class))
+    # Unsets some fields in the job's internal state.
+    # @param fields [Array<String,Symbol>] Fields to unset
+    # @raise [RuntimeError] Error raised if job no longer exists
+    def unset(*fields)
+      return unless fields.length > 0
+      SideJob.redis.hdel redis_key, fields
+    end
+
+    # Acquire a lock on the job with a given expiration time.
+    # @param ttl [Fixnum] Lock expiration in seconds
+    # @param retries [Fixnum] Number of attempts to retry getting lock
+    # @param retry_delay [Float] Maximum seconds to wait (actual will be randomized) before retry getting lock
+    # @return [String, nil] Lock token that should be passed to {#unlock} or nil if lock was not acquired
+    def lock(ttl, retries: 3, retry_delay: 0.2)
+      retries.times do
+        token = SecureRandom.uuid
+        if SideJob.redis.set("#{redis_key}:lock", token, {nx: true, ex: ttl})
+          return token # lock acquired
+        else
+          sleep Random.rand(retry_delay)
+        end
+      end
+      return nil # lock not acquired
+    end
+
+    # Refresh the lock expiration.
+    # @param ttl [Fixnum] Refresh lock expiration for the given time in seconds
+    # @return [Boolean] Whether the timeout was set
+    def refresh_lock(ttl)
+      SideJob.redis.expire "#{redis_key}:lock", ttl
+    end
+
+    # Unlock job by deleting the lock only if it equals the lock token.
+    # @param token [String] Token returned by {#lock}
+    # @return [Boolean] Whether the job was unlocked
+    def unlock(token)
+      return SideJob.redis.eval('
+        if redis.call("get",KEYS[1]) == ARGV[1] then
+          return redis.call("del",KEYS[1])
+        else
+          return 0
+        end', { keys: ["#{redis_key}:lock"], argv: [token] }) == 1
     end
 
     private
@@ -247,7 +311,16 @@ module SideJob
     # Queue or schedule this job using sidekiq.
     # @param time [Time, Float, nil] Time to schedule the job if specified
     def sidekiq_queue(time=nil)
+      # Don't need to queue if a worker is already in process of running
+      return if SideJob.redis.exists "#{redis_key}:lock:worker"
+
       queue = get(:queue)
+
+      # Don't need to queue if the job is already in the queue (this does not include scheduled jobs)
+      # When Sidekiq pulls job out from scheduled set, we can still get the same job queued multiple times
+      # but the server middleware handles it
+      return if Sidekiq::Queue.new(queue).find_job(@id)
+
       klass = get(:class)
       args = get(:args)
 
@@ -260,39 +333,9 @@ module SideJob
       Sidekiq::Client.push(item)
     end
 
-    # Caches all inports and outports.
-    def load_ports
-      @ports = {}
-      %i{in out}.each do |type|
-        @ports[type] = {}
-        SideJob.redis.hkeys("#{redis_key}:#{type}ports:mode").each do |name|
-          if name == '*'
-            @ports["#{type}*"] = SideJob::Port.new(self, type, name)
-          else
-            @ports[type][name] = SideJob::Port.new(self, type, name)
-          end
-        end
-      end
-    end
-
-    # Returns an input or output port.
-    # @param type [:in, :out] Input or output port
-    # @param name [Symbol,String] Name of the port
-    # @return [SideJob::Port]
-    def get_port(type, name)
-      load_ports if ! @ports
-      name = name.to_s
-      return @ports[type][name] if @ports[type][name]
-
-      if @ports["#{type}*"]
-        # create port with default port options for dynamic ports
-        port = SideJob::Port.new(self, type, name)
-        port.options = @ports["#{type}*"].options
-        @ports[type][name] = port
-        return port
-      else
-        raise "Unknown #{type}put port: #{name}"
-      end
+    # Return all ports of the given type
+    def all_ports(type)
+      SideJob.redis.smembers("#{redis_key}:#{type}ports").reject {|name| name == '*'}.map {|name| SideJob::Port.new(self, type, name)}
     end
 
     # Sets the input/outputs ports for the job and overwrites all current options.
@@ -301,30 +344,25 @@ module SideJob
     # @param type [:in, :out] Input or output ports
     # @param ports [Hash{Symbol,String => Hash}] Port configuration. Port name to options.
     def set_ports(type, ports)
-      current = SideJob.redis.hkeys("#{redis_key}:#{type}ports:mode") || []
+      current = SideJob.redis.smembers("#{redis_key}:#{type}ports") || []
+      config = SideJob::Worker.config(get(:queue), get(:class))
 
-      replace_port_data = []
-      ports = (ports || {}).stringify_keys
-      ports = (config["#{type}ports"] || {}).merge(ports)
+      ports ||= {}
+      ports = (config["#{type}ports"] || {}).merge(ports.dup.stringify_keys)
       ports.each_key do |port|
         ports[port] = ports[port].stringify_keys
-        replace_port_data << port if ports[port]['data']
       end
 
       SideJob.redis.multi do |multi|
         # remove data from old ports
-        ((current - ports.keys) | replace_port_data).each do |port|
+        (current - ports.keys).each do |port|
           multi.del "#{redis_key}:#{type}:#{port}"
         end
 
-        # completely replace the mode and default keys
-
-        multi.del "#{redis_key}:#{type}ports:mode"
-        modes = ports.map do |port, options|
-          [port, options['mode'] || 'queue']
-        end.flatten(1)
-        multi.hmset "#{redis_key}:#{type}ports:mode", *modes if modes.length > 0
+        multi.del "#{redis_key}:#{type}ports"
+        multi.sadd "#{redis_key}:#{type}ports", ports.keys if ports.length > 0
 
+        # replace port defaults
         defaults = ports.map do |port, options|
           if options.has_key?('default')
             [port, options['default'].to_json]
@@ -335,33 +373,11 @@ module SideJob
         multi.del "#{redis_key}:#{type}ports:default"
         multi.hmset "#{redis_key}:#{type}ports:default", *defaults if defaults.length > 0
       end
-
-      @ports = nil
-
-      group_port_logs do
-        ports.each_pair do |port, options|
-          if options['data']
-            port = get_port(type, port)
-            options['data'].each do |x|
-              port.write x
-            end
-          end
-        end
-      end
     end
 
     # @raise [RuntimeError] Error raised if job no longer exists
     def check_exists
-      raise "Job #{id} no longer exists!" unless exists?
-    end
-
-    def load_state
-      if ! @state
-        state = SideJob.redis.hget('jobs', id)
-        raise "Job #{id} no longer exists!" if ! state
-        @state = JSON.parse(state)
-      end
-      @state
+      raise "Job #{id} does not exist!" unless exists?
     end
   end
 
@@ -370,56 +386,10 @@ module SideJob
   class Job
     include JobMethods
 
-    # @param id [String] Job id
+    # @param id [Integer] Job id
     def initialize(id)
-      @id = id
-    end
-  end
-
-  # Logger that groups all port read/writes together.
-  # @see {JobMethods#group_port_logs}
-  class GroupPortLogs
-    def initialize(job)
-      @metadata = {job: job.id}
-    end
-
-    # If entry is not a port log, send it on to {SideJob.log}. Otherwise, collect the log until {#done} is called.
-    # @param entry [Hash] Log entry
-    def log(entry)
-      if entry[:read] && entry[:write]
-        # collect reads and writes by port and group data together
-        @port_events ||= {read: {}, write: {}} # {job: id, <in|out>port: port} -> data array
-        %i{read write}.each do |type|
-          entry[type].each do |event|
-            data = event.delete(:data)
-            @port_events[type][event] ||= []
-            @port_events[type][event].concat data
-          end
-        end
-      else
-        SideJob.log(entry)
-      end
-    end
-
-    # Merges the collected port read and writes and send logs to {SideJob.log}.
-    def done
-      return unless @port_events && (@port_events[:read].length > 0 || @port_events[:write].length > 0)
-
-      entry = {}
-      %i{read write}.each do |type|
-        entry[type] = @port_events[type].map do |port, data|
-          port.merge({data: data})
-        end
-      end
-
-      SideJob.log @metadata.merge(entry)
-      @port_events = nil
-    end
-
-    # Add metadata fields to the final log entry.
-    # @param metadata [Hash] Data to be merged with the existing metadata and final log entry
-    def add_metadata(metadata)
-      @metadata.merge!(metadata.symbolize_keys)
+      @id = id.to_i
+      check_exists
     end
   end
 end