sidejob 3.0.1 → 4.0.1

data/lib/sidejob/worker.rb

@@ -42,9 +42,9 @@ module SideJob
     # Methods loaded last to override other included methods
     module OverrideMethods
       # Returns the jid set by sidekiq as the job id
-      # @return [String] Job id
+      # @return [Integer] Job id
       def id
-        jid
+        jid.to_i
       end
     end
 
@@ -58,12 +58,6 @@ module SideJob
       base.extend(ClassMethods)
     end
 
-    # Queues a child job, setting parent and by to self.
-    # @see SideJob.queue
-    def queue(queue, klass, **options)
-      SideJob.queue(queue, klass, options.merge({parent: self, by: "job:#{id}"}))
-    end
-
     # Exception raised by {#suspend}
     class Suspended < StandardError; end
 
@@ -79,55 +73,43 @@ module SideJob
     # A worker should be idempotent (it can be called multiple times on the same state).
     # Consider reading from a single port with a default value. Each time it is run, it could read the same data
     # from the port. The output of the job then could depend on the number of times it is run.
-    # To prevent this, this method requires that there be at least one input port which does not have a default.
+    # If all input port have defaults, this method remembers the call and will only yield once even over multiple runs.
+    # In addition, any writes to output ports inside the block will instead set the default value of the port.
     # Yields data from the ports until either no ports have data or is suspended due to data on some but not all ports.
     # @param inputs [Array<String>] List of input ports to read
     # @yield [Array] Splat of input data in same order as inputs
     # @raise [SideJob::Worker::Suspended] Raised if an input port without a default has data but not all ports
-    # @raise [RuntimeError] An error is raised if all input ports have default values
     def for_inputs(*inputs, &block)
       return unless inputs.length > 0
       ports = inputs.map {|name| input(name)}
       loop do
-        group_port_logs(job: id) do
-          # error if ports all have defaults, complete if no non-default port inputs, suspend if partial inputs
-          data = ports.map {|port| [ port.data?, port.default? ] }
-          raise "One of these input ports should not have a default value: #{inputs.join(',')}" if data.all? {|x| x[1]}
-          return unless data.any? {|x| x[0] && ! x[1] }
-          suspend unless data.all? {|x| x[0] }
+        SideJob::Port.log_group do
+          info = ports.map {|port| [ port.size > 0, port.default? ] }
 
-          yield *ports.map(&:read)
+          return unless info.any? {|x| x[0] || x[1]} # Nothing to do if there's no data to read
+          if info.any? {|x| x[0]}
+            # some port has data, suspend unless every port has data or default
+            suspend unless info.all? {|x| x[0] || x[1] }
+            yield *ports.map(&:read)
+          elsif info.all? {|x| x[1]}
+            # all ports have default and no data
+            defaults = ports.map(&:default)
+            last_default = get(:for_inputs) || []
+            return unless defaults != last_default
+            set({for_inputs: defaults})
+            begin
+              Thread.current[:sidejob_port_write_default] = true
+              yield *defaults
+            ensure
+              Thread.current[:sidejob_port_write_default] = nil
+            end
+            return
+          else
+            # No ports have data and not every port has a default value so nothing to do
+            return
+          end
         end
       end
     end
-
-    # Sets values in the job's internal state.
-    # @param data [Hash{String,Symbol => Object}] Data to update: objects should be JSON encodable
-    # @raise [RuntimeError] Error raised if job no longer exists
-    def set(data)
-      return unless data.size > 0
-      load_state
-      data.each_pair { |key, val| @state[key.to_s] = val }
-      save_state
-    end
-
-    # Unsets some fields in the job's internal state
-    # @param fields [Array<String,Symbol>] Fields to unset
-    # @raise [RuntimeError] Error raised if job no longer exists
-    def unset(*fields)
-      return unless fields.length > 0
-      load_state
-      fields.each { |field| @state.delete(field.to_s) }
-      save_state
-    end
-
-    private
-
-    def save_state
-      check_exists
-      if @state
-        SideJob.redis.hset 'jobs', id, @state.to_json
-      end
-    end
   end
 end

data/spec/integration/fib_spec.rb

@@ -5,15 +5,20 @@ class TestFib
   include SideJob::Worker
   register(
       inports: {
-          n: {mode: :memory}
+          n: {}
       },
       outports: {
           num: {}
       }
   )
   def perform
-    suspend unless input(:n).data?
-    n = input(:n).read
+    if input(:n).data?
+      n = input(:n).read
+    else
+      n = get(:n)
+    end
+    suspend unless n
+    set({n: n})
 
     if n <= 2
       output(:num).write 1
@@ -44,7 +49,6 @@ describe TestFib do
     job = SideJob.queue('testq', 'TestFib')
     job.input(:n).write 6 # 1, 1, 2, 3, 5, 8
     SideJob::Worker.drain_queue
-    job.reload
     expect(job.status).to eq 'completed'
     expect(job.output(:num).read).to eq(8)
   end

data/spec/integration/sum_spec.rb

@@ -32,7 +32,6 @@ describe TestSumFlow do
   it 'calls child job to sum numbers' do
     job = SideJob.queue('testq', 'TestSumFlow')
     SideJob::Worker.drain_queue
-    job.reload
     expect(job.status).to eq 'completed'
     expect(job.output(:out).read).to eq(11)
   end

data/spec/sidejob/job_spec.rb

@@ -1,25 +1,34 @@
 require 'spec_helper'
 
 describe SideJob::Job do
+  describe '#initialize' do
+    it 'raises error if job does not exist' do
+      expect { SideJob::Job.new('123') }.to raise_error
+    end
+  end
+
   describe '#==, #eql?' do
     it 'two jobs with the same id are eq' do
-      expect(SideJob::Job.new('123')).to eq(SideJob::Job.new('123'))
-      expect(SideJob::Job.new('123')).to eql(SideJob::Job.new('123'))
+      @job = SideJob.queue('testq', 'TestWorker')
+      expect(SideJob::Job.new(@job.id)).to eq(@job)
+      expect(SideJob::Job.new(@job.id)).to eql(@job)
     end
 
     it 'two jobs with different id are not eq' do
-      expect(SideJob::Job.new('123')).not_to eq(SideJob::Job.new('456'))
-      expect(SideJob::Job.new('123')).not_to eql(SideJob::Job.new('456'))
+      @job = SideJob.queue('testq', 'TestWorker')
+      @job2 = SideJob.queue('testq', 'TestWorker')
+      expect(@job).not_to eq(@job2)
+      expect(@job).not_to eql(@job2)
     end
   end
 
   describe '#hash' do
     it 'uses hash of the job id and can be used as hash keys' do
-      job = SideJob::Job.new('abc')
-      expect(job.hash).to eq('abc'.hash)
+      job = SideJob.queue('testq', 'TestWorker')
+      expect(job.hash).to eq(job.id.hash)
       h = {}
       h[job] = 1
-      job2 = SideJob::Job.new('abc')
+      job2 = SideJob::Job.new(job.id)
       expect(job.hash).to eq(job2.hash)
       h[job2] = 3
       expect(h.keys.length).to be(1)
@@ -29,8 +38,8 @@ describe SideJob::Job do
 
   describe '#to_s' do
     it 'returns the redis key' do
-      job = SideJob::Job.new('abc')
-      expect(job.to_s).to eq 'job:abc'
+      job = SideJob.queue('testq', 'TestWorker')
+      expect(job.to_s).to eq "job:#{job.id}"
     end
   end
 
@@ -39,92 +48,12 @@ describe SideJob::Job do
       @job = SideJob.queue('testq', 'TestWorker')
       expect(@job.exists?).to be true
     end
-    it 'returns false if job does not exist' do
-      expect(SideJob::Job.new('job').exists?).to be false
-    end
-  end
 
-  describe '#log' do
-    before do
+    it 'returns false if job no longer exists' do
       @job = SideJob.queue('testq', 'TestWorker')
-      @entry = {'abc' => [1,2]}
-      @entry_job = @entry.merge({job: @job.id})
-    end
-
-    it 'logs to SideJob.log if logger not set' do
-      expect(SideJob).to receive(:log).with @entry_job
-      @job.log(@entry)
-    end
-
-    it 'logs to job logger if set' do
-      expect(SideJob).not_to receive(:log).with @entry_job
-      @job.logger = Class.new
-      expect(@job.logger).to receive(:log).with @entry_job
-      @job.log(@entry)
-    end
-  end
-
-  describe '#group_port_logs' do
-    before do
-      @job = SideJob.queue('testq', 'TestWorker', inports: {port1: {}})
-      @port = @job.input(:port1)
-      now = Time.now
-      allow(Time).to receive(:now) { now }
-    end
-
-    it 'does not generate a log if no logs occur' do
-      @job.group_port_logs {}
-      expect(SideJob.logs.length).to eq 0
-    end
-
-    it 'passes through logs not generated by port read/write' do
-      @job.group_port_logs do
-        @job.log({'test' => 123})
-        expect(SideJob.logs).to eq [{'timestamp' => SideJob.timestamp, 'test' => 123, 'job' => @job.id}]
-      end
-    end
-
-    it 'merges logs from port reads and writes' do
-      @job.group_port_logs do
-        @port.write 'hello'
-        @port.write 2
-      end
-      expect(SideJob.logs).to eq [{'timestamp' => SideJob.timestamp, 'job' => @job.id,
-                                   'read' => [],
-                                   'write' => [{'job' => @job.id, 'inport' => 'port1', 'data' => ['hello', 2]}]}]
-    end
-
-    it 'does not write out a log entry until the end of outermost group' do
-      @job.group_port_logs do
-        @port.write 'hello'
-        @job.group_port_logs do
-          @port.write 2
-        end
-        expect(SideJob.logs.length).to eq 0
-      end
-      expect(SideJob.logs).to eq [{'timestamp' => SideJob.timestamp, 'job' => @job.id,
-                                   'read' => [],
-                                   'write' => [{'job' => @job.id, 'inport' => 'port1', 'data' => ['hello', 2]}]}]
-    end
-
-    it 'can overwrite job for log attribution' do
-      @job.group_port_logs(job: 1234) do
-        @port.write 'hello'
-        @port.read
-      end
-      expect(SideJob.logs).to eq [{'timestamp' => SideJob.timestamp, 'job' => 1234,
-                                   'read' => [{'job' => @job.id, 'inport' => 'port1', 'data' => ['hello']}],
-                                   'write' => [{'job' => @job.id, 'inport' => 'port1', 'data' => ['hello']}]}]
-    end
-
-    it 'can include arbitrary metadata' do
-      @job.group_port_logs(user: 'test') do
-        @port.write 'hello'
-        @port.read
-      end
-      expect(SideJob.logs).to eq [{'timestamp' => SideJob.timestamp, 'job' => @job.id, 'user' => 'test',
-                                   'read' => [{'job' => @job.id, 'inport' => 'port1', 'data' => ['hello']}],
-                                   'write' => [{'job' => @job.id, 'inport' => 'port1', 'data' => ['hello']}]}]
+      @job.status = 'terminated'
+      @job.delete
+      expect(@job.exists?).to be false
     end
   end
 
@@ -143,70 +72,28 @@ describe SideJob::Job do
     end
   end
 
-  describe '#terminate' do
-    before do
-      @job = SideJob.queue('testq', 'TestWorker')
-    end
-
-    it 'sets the status to terminating' do
-      @job.terminate
-      expect(@job.status).to eq 'terminating'
-    end
-
-    it 'does nothing if status is terminated' do
-      @job.status = 'terminated'
-      @job.terminate
-      expect(@job.status).to eq 'terminated'
-    end
-
-    it 'throws error and immediately sets status to terminated if job class is unregistered' do
-      SideJob.redis.del 'workers:testq'
-      expect { @job.terminate }.to raise_error
-      expect(@job.status).to eq 'terminated'
-    end
-
-    it 'queues the job for termination run' do
-      expect {
-        @job.terminate
-      }.to change {Sidekiq::Stats.new.enqueued}.by(1)
-    end
-
-    it 'by default does not terminate children' do
-      child = SideJob.queue('testq', 'TestWorker', parent: @job, name: 'child')
-      expect(child.status).to eq 'queued'
-      @job.terminate
-      expect(child.status).to eq 'queued'
-    end
-
-    it 'can recursively terminate' do
-      5.times {|i| SideJob.queue('testq', 'TestWorker', parent: @job, name: "child#{i}") }
-      @job.terminate(recursive: true)
-      @job.children.each_value do |child|
-        expect(child.status).to eq 'terminating'
-      end
-    end
-  end
-
   describe '#run' do
     before do
       @job = SideJob.queue('testq', 'TestWorker')
+      Sidekiq::Queue.new('testq').find_job(@job.id).delete
+      @job.status = 'completed'
     end
 
-    %w{queued running suspended completed failed}.each do |status|
+    %w{queued running suspended completed}.each do |status|
       it "queues the job if status is #{status}" do
         expect {
           @job.status = status
-          @job.run
+          expect(@job.run).to eq @job
           expect(@job.status).to eq 'queued'
         }.to change {Sidekiq::Stats.new.enqueued}.by(1)
       end
     end
 
-    %w{terminating terminated}.each do |status|
+    %w{failed terminating terminated}.each do |status|
       it "does not queue the job if status is #{status}" do
         expect {
           @job.status = status
-          @job.run
+          expect(@job.run).to be nil
           expect(@job.status).to eq status
         }.to change {Sidekiq::Stats.new.enqueued}.by(0)
       end
@@ -214,12 +101,37 @@ describe SideJob::Job do
       it "queues the job if status is #{status} and force=true" do
         expect {
           @job.status = status
-          @job.run(force: true)
+          expect(@job.run(force: true)).to eq @job
           expect(@job.status).to eq 'queued'
         }.to change {Sidekiq::Stats.new.enqueued}.by(1)
       end
     end
 
+    it "does not queue the job if it is already queued" do
+      @job.run
+      expect {
+        expect(@job.run).to eq @job
+      }.to change {Sidekiq::Stats.new.enqueued}.by(0)
+    end
+
+    it 'does nothing if no parent job and parent=true' do
+      @job.status = 'completed'
+      expect {
+        expect(@job.run(parent: true)).to be nil
+        expect(@job.status).to eq 'completed'
+      }.to change {Sidekiq::Stats.new.enqueued}.by(0)
+    end
+
+    it 'runs parent job if parent=true' do
+      parent = SideJob.queue('testq', 'TestWorker')
+      parent.adopt(@job, 'child')
+      @job.status = 'completed'
+      parent.status = 'completed'
+      expect(@job.run(parent: true)).to eq parent
+      expect(@job.status).to eq 'completed'
+      expect(parent.status).to eq 'queued'
+    end
+
     it 'throws error and immediately sets status to terminated if job class is unregistered' do
       SideJob.redis.del "workers:#{@job.get(:queue)}"
       expect { @job.run }.to raise_error
@@ -256,6 +168,100 @@ describe SideJob::Job do
     end
   end
 
+  describe '#terminated?' do
+    before do
+      @job = SideJob.queue('testq', 'TestWorker')
+    end
+
+    it 'returns false if job status is not terminated' do
+      expect(@job.terminated?).to be false
+    end
+
+    it 'returns true if job status is terminated' do
+      @job.status = 'terminated'
+      expect(@job.terminated?).to be true
+    end
+
+    it 'returns false if child job is not terminated' do
+      @job.status = 'terminated'
+      SideJob.queue('testq', 'TestWorker', parent: @job, name: 'child')
+      expect(@job.terminated?).to be false
+    end
+
+    it 'returns true if child job is terminated' do
+      @job.status = 'terminated'
+      child = SideJob.queue('testq', 'TestWorker', parent: @job, name: 'child')
+      child.status = 'terminated'
+      expect(@job.terminated?).to be true
+    end
+  end
+
+  describe '#terminate' do
+    before do
+      @job = SideJob.queue('testq', 'TestWorker')
+      Sidekiq::Queue.new('testq').find_job(@job.id).delete
+      @job.status = 'completed'
+    end
+
+    it 'sets the status to terminating' do
+      @job.terminate
+      expect(@job.status).to eq 'terminating'
+    end
+
+    it 'does nothing if status is terminated' do
+      @job.status = 'terminated'
+      @job.terminate
+      expect(@job.status).to eq 'terminated'
+    end
+
+    it 'throws error and immediately sets status to terminated if job class is unregistered' do
+      SideJob.redis.del 'workers:testq'
+      expect { @job.terminate }.to raise_error
+      expect(@job.status).to eq 'terminated'
+    end
+
+    it 'queues the job for termination run' do
+      expect {
+        @job.terminate
+      }.to change {Sidekiq::Stats.new.enqueued}.by(1)
+    end
+
+    it 'by default does not terminate children' do
+      child = SideJob.queue('testq', 'TestWorker', parent: @job, name: 'child')
+      expect(child.status).to eq 'queued'
+      @job.terminate
+      expect(child.status).to eq 'queued'
+    end
+
+    it 'can recursively terminate' do
+      5.times {|i| SideJob.queue('testq', 'TestWorker', parent: @job, name: "child#{i}") }
+      @job.terminate(recursive: true)
+      @job.children.each_value do |child|
+        expect(child.status).to eq 'terminating'
+      end
+    end
+  end
+
+  describe '#queue' do
+    before do
+      @job = SideJob.queue('testq', 'TestWorker')
+    end
+
+    it 'can queue child jobs' do
+      expect(SideJob).to receive(:queue).with('testq', 'TestWorker', args: [1,2], inports: {'myport' => {}}, parent: @job, name: 'child', by: "job:#{@job.id}").and_call_original
+      expect {
+        child = @job.queue('testq', 'TestWorker', args: [1,2], inports: {'myport' => {}}, name: 'child')
+        expect(child.parent).to eq(@job)
+        expect(@job.children).to eq('child' => child)
+      }.to change {Sidekiq::Stats.new.enqueued}.by(1)
+    end
+
+    it 'queues with by string set to self' do
+      child = @job.queue('testq', 'TestWorker', name: 'child')
+      expect(child.get(:created_by)).to eq "job:#{@job.id}"
+    end
+  end
+
   describe '#child' do
     it 'returns nil for missing child' do
       job = SideJob.queue('testq', 'TestWorker')
@@ -278,46 +284,74 @@ describe SideJob::Job do
     end
   end
 
-  describe '#ancestors' do
-    it 'returns empty array if no parent' do
+  describe '#disown' do
+    it 'raises error if child cannot be found' do
+      parent = SideJob.queue('testq', 'TestWorker')
+      expect { parent.disown('child') }.to raise_error
+    end
+
+    it 'raises error if job is not child' do
+      parent = SideJob.queue('testq', 'TestWorker')
       job = SideJob.queue('testq', 'TestWorker')
-      expect(job.ancestors).to eq([])
+      expect { parent.disown(job) }.to raise_error
     end
 
-    it 'returns entire job tree' do
-      j1 = SideJob.queue('testq', 'TestWorker')
-      j2 = SideJob.queue('testq', 'TestWorker', parent: j1, name: 'child')
-      j3 = SideJob.queue('testq', 'TestWorker', parent: j2, name: 'child')
-      j4 = SideJob.queue('testq', 'TestWorker', parent: j3, name: 'child')
-      expect(j4.ancestors).to eq([j3, j2, j1])
+    it 'disassociates a child job from the parent' do
+      parent = SideJob.queue('testq', 'TestWorker')
+      child = SideJob.queue('testq', 'TestWorker', parent: parent, name: 'child')
+      expect(child.parent).to eq(parent)
+      expect(parent.child('child')).to eq child
+      parent.disown('child')
+      expect(child.parent).to be nil
+      expect(parent.child('child')).to be nil
+    end
+
+    it 'can specify a job' do
+      parent = SideJob.queue('testq', 'TestWorker')
+      child = SideJob.queue('testq', 'TestWorker', parent: parent, name: 'child')
+      expect(child.parent).to eq(parent)
+      expect(parent.child('child')).to eq child
+      parent.disown(child)
+      expect(child.parent).to be nil
+      expect(parent.child('child')).to be nil
     end
   end
 
-  describe '#terminated?' do
-    before do
-      @job = SideJob.queue('testq', 'TestWorker')
+  describe '#adopt' do
+    it 'can adopt an orphan job' do
+      job = SideJob.queue('testq', 'TestWorker')
+      child = SideJob.queue('testq', 'TestWorker')
+      expect(child.parent).to be nil
+      job.adopt(child, 'child')
+      expect(child.parent).to eq(job)
     end
 
-    it 'returns false if job status is not terminated' do
-      expect(@job.terminated?).to be false
+    it 'raises error when adopting self' do
+      job = SideJob.queue('testq', 'TestWorker')
+      expect(job.parent).to be nil
+      expect { job.adopt(job, 'self') }.to raise_error
     end
 
-    it 'returns true if job status is terminated' do
-      @job.status = 'terminated'
-      expect(@job.terminated?).to be true
+    it 'raises error if job already has a parent' do
+      job = SideJob.queue('testq', 'TestWorker')
+      job2 = SideJob.queue('testq', 'TestWorker')
+      child = SideJob.queue('testq', 'TestWorker')
+      job.adopt(child, 'child')
+      expect { job2.adopt(child, 'mine') }.to raise_error
     end
 
-    it 'returns false if child job is not terminated' do
-      @job.status = 'terminated'
-      SideJob.queue('testq', 'TestWorker', parent: @job, name: 'child')
-      expect(@job.terminated?).to be false
+    it 'raises error if no name is given' do
+      job = SideJob.queue('testq', 'TestWorker')
+      child = SideJob.queue('testq', 'TestWorker')
+      expect { job.adopt(child, nil) }.to raise_error
     end
 
-    it 'returns true if child job is terminated' do
-      @job.status = 'terminated'
-      child = SideJob.queue('testq', 'TestWorker', parent: @job, name: 'child')
-      child.status = 'terminated'
-      expect(@job.terminated?).to be true
+    it 'raises error if name is not unique' do
+      job = SideJob.queue('testq', 'TestWorker')
+      child = SideJob.queue('testq', 'TestWorker')
+      child2 = SideJob.queue('testq', 'TestWorker')
+      job.adopt(child, 'child')
+      expect { job.adopt(child2, 'child') }.to raise_error
     end
   end
 
@@ -359,17 +393,24 @@ describe SideJob::Job do
       @job.delete
       expect(SideJob.redis {|redis| redis.keys('job:*').length}).to be(0)
     end
+
+    it 'disowns job from parent' do
+      child = SideJob.queue('testq', 'TestWorker', parent: @job, name: 'child')
+      expect(@job.children.size).to eq 1
+      child.status = 'terminated'
+      child.delete
+      expect(@job.children.size).to eq 0
+    end
   end
 
   # Tests are identical for input and output port methods
   %i{in out}.each do |type|
     describe "##{type}put" do
-      it "returns a cached #{type}put port" do
+      it "returns a #{type}put port" do
         spec = {}
         spec[:"#{type}ports"] = {port: {}}
         @job = SideJob.queue('testq', 'TestWorker', **spec)
         expect(@job.send("#{type}put", :port)).to eq(SideJob::Port.new(@job, type, :port))
-        expect(@job.send("#{type}put", :port)).to be(@job.send("#{type}put", :port))
       end
 
       it 'raises error on unknown port' do
@@ -379,12 +420,11 @@ describe SideJob::Job do
 
       it 'can dynamically create ports' do
         spec = {}
-        spec[:"#{type}ports"] = {'*' => {mode: :memory, default: 123}}
+        spec[:"#{type}ports"] = {'*' => {default: 123}}
         @job = SideJob.queue('testq', 'TestWorker', **spec)
         expect(@job.send("#{type}ports").size).to eq 0
         port = @job.send("#{type}put", :newport)
         expect(@job.send("#{type}ports").size).to eq 1
-        expect(port.mode).to eq :memory
         expect(port.default).to eq 123
       end
     end
@@ -403,29 +443,32 @@ describe SideJob::Job do
 
       it 'can specify ports with options' do
         expect(@job.send("#{type}ports").size).to eq 0
-        @job.send("#{type}ports=", {myport: {mode: :memory, default: 'def'}})
+        @job.send("#{type}ports=", {myport: {default: 'def'}})
         expect(@job.send("#{type}ports").size).to eq 1
         expect(@job.send("#{type}ports").map(&:name)).to include(:myport)
-        expect(@job.send("#{type}put", :myport).mode).to eq :memory
         expect(@job.send("#{type}put", :myport).default).to eq 'def'
       end
 
+      it 'does not modify passed in port options' do
+        ports = {myport: {default: [1,2]}}.freeze
+        expect { @job.send("#{type}ports=", ports) }.to_not raise_error
+      end
+
       it 'merges ports with the worker configuration' do
-        allow(@job).to receive(:config) { {"#{type}ports" => {'port1' => {}, 'port2' => {'mode' => 'memory'}}}}
-        @job.send("#{type}ports=", {port2: {mode: :queue}, port3: {}})
-        expect(@job.send("#{type}ports").size).to eq 3
-        expect(@job.send("#{type}ports").all? {|port| port.options == {mode: :queue}}).to be true
+        allow(SideJob::Worker).to receive(:config) { {"#{type}ports" => {'port1' => {}, 'port2' => {default: 'worker'}}}}
+        @job.send("#{type}ports=", {port2: {default: 'new'}})
+        expect(@job.send("#{type}ports").size).to eq 2
+        expect(@job.send("#{type}put", :port2).default).to eq 'new'
       end
 
-      it 'can change existing port mode while keeping data intact' do
-        @job.send("#{type}ports=", {myport: {}})
+      it 'can change existing port default while keeping data intact' do
+        @job.send("#{type}ports=", {myport: {default: 'orig'}})
         @job.send("#{type}put", :myport).write 'data'
-        @job.send("#{type}ports=", {myport: {mode: :memory, default: 'def'}})
+        @job.send("#{type}ports=", {myport: {default: 'new'}})
         expect(@job.send("#{type}ports").size).to eq 1
-        expect(@job.send("#{type}put", :myport).mode).to eq :memory
-        expect(@job.send("#{type}put", :myport).default).to eq 'def'
+        expect(@job.send("#{type}put", :myport).default).to eq 'new'
         expect(@job.send("#{type}put", :myport).read).to eq 'data'
-        expect(@job.send("#{type}put", :myport).read).to eq 'def'
+        expect(@job.send("#{type}put", :myport).read).to eq 'new'
       end
 
       it 'deletes no longer used ports' do
@@ -435,45 +478,26 @@ describe SideJob::Job do
         expect(@job.send("#{type}ports").map(&:name)).not_to include(:myport)
         expect { @job.send("#{type}put", :myport) }.to raise_error
       end
+    end
+  end
 
-      it 'can specify port data' do
-        @job.send("#{type}ports=", {myport: {data: [1,'abc']}})
-        expect(@job.send("#{type}put", :myport).read).to eq 1
-        expect(@job.send("#{type}put", :myport).read).to eq 'abc'
-      end
-
-      it 'can clear port data' do
-        @job.send("#{type}ports=", {myport: {}})
-        @job.send("#{type}put", :myport).write 'data'
-        expect(@job.send("#{type}put", :myport).size).to eq 1
-        @job.send("#{type}ports=", {myport: {data: []}})
-        expect(@job.send("#{type}put", :myport).size).to eq 0
-      end
-
-      it 'overwrites existing data' do
-        @job.send("#{type}ports=", {myport: {}})
-        @job.send("#{type}put", :myport).write 'data'
-        expect(@job.send("#{type}put", :myport).size).to eq 1
-        @job.send("#{type}ports=", {myport: {data: [1,'abc']}})
-        expect(@job.send("#{type}put", :myport).size).to eq 2
-        expect(@job.send("#{type}put", :myport).read).to eq 1
-        expect(@job.send("#{type}put", :myport).read).to eq 'abc'
-      end
+  describe '#state' do
+    before do
+      now = Time.now
+      allow(Time).to receive(:now) { now }
+      @job = SideJob.queue('testq', 'TestWorker')
+    end
 
-      it 'groups logs' do
-        now = Time.now
-        allow(Time).to receive(:now) { now }
-        @job.send("#{type}ports=", {myport: {data: [1,'abc']}})
-        expect(SideJob.logs).to eq [{'timestamp' => SideJob.timestamp, 'job' => @job.id, 'read' => [], 'write' => [{'job' => @job.id, "#{type}port" => 'myport', 'data' => [1,'abc']}]}]
-      end
+    it 'returns job state with common and internal keys' do
+      @job.set({abc: 123})
+      expect(@job.state).to eq({"queue"=>"testq", "class"=>"TestWorker", "args"=>nil, "created_by"=>nil, "created_at"=>SideJob.timestamp, 'status' => 'queued', 'abc' => 123})
     end
   end
 
   describe '#get' do
     before do
       @job = SideJob.queue('testq', 'TestWorker')
-      SideJob.redis.hset 'jobs', @job.id, { queue: 'testq', class: 'TestWorker', field1: 'value1', field2: [1,2], field3: 123 }.to_json
-      @job.reload
+      @job.set({field1: 'value1', field2: [1,2], field3: 123 })
     end
 
     it 'returns a value from job state using symbol key' do
@@ -492,52 +516,110 @@ describe SideJob::Job do
       expect(@job.get(:field2)).to eq [1, 2]
     end
 
-    it 'caches the state' do
+    it 'always returns the latest value' do
       expect(@job.get(:field3)).to eq 123
       SideJob.redis.hmset @job.redis_key, :field3, '789'
-      expect(@job.get(:field3)).to eq 123
+      expect(@job.get(:field3)).to eq 789
     end
+  end
 
-    it 'raises error if job no longer exists and state is not cached' do
-      @job.reload
-      job2 = SideJob.find(@job.id)
-      job2.status = 'terminated'
-      job2.delete
-      expect { @job.get(:key) }.to raise_error
+  describe '#set' do
+    before do
+      @job = SideJob.queue('testq', 'TestWorker')
     end
 
-    it 'does not raise error if job no longer exists but state is cached' do
-      @job.get(:foo)
-      job2 = SideJob.find(@job.id)
-      job2.status = 'terminated'
-      job2.delete
-      expect { @job.get(:key) }.not_to raise_error
+    it 'can save state in redis' do
+      @job.set(test: 'data', test2: 123)
+      state = @job.state
+      expect(state['test']).to eq 'data'
+      expect(state['test2']).to eq 123
+
+      # test updating
+      @job.set(test: 'data2')
+      expect(@job.get('test')).to eq 'data2'
+    end
+
+    it 'can update values' do
+      3.times do |i|
+        @job.set key: [i]
+        expect(@job.get(:key)).to eq [i]
+        expect(JSON.parse(SideJob.redis.hget(@job.redis_key, 'key'))).to eq [i]
+      end
+    end
+
+    it 'raises error if job no longer exists' do
+      @job.status = 'terminated'
+      SideJob.find(@job.id).delete
+      expect { @job.set key: 123 }.to raise_error
     end
   end
 
-  describe '#reload' do
+  describe '#unset' do
     before do
       @job = SideJob.queue('testq', 'TestWorker')
     end
 
-    it 'clears the job state cache' do
-      expect(@job.get(:field1)).to be nil
-      SideJob.redis.hset 'jobs', @job.id, {queue: 'testq', class: 'TestWorker', field1: 789}.to_json
-      @job.reload
-      expect(@job.get(:field1)).to eq 789
+    it 'unsets fields' do
+      @job.set(a: 123, b: 456, c: 789)
+      @job.unset('a', :b)
+      expect(@job.get(:a)).to eq nil
+      expect(@job.get(:b)).to eq nil
+      expect(@job.get(:c)).to eq 789
     end
   end
 
-  describe '#config' do
+  describe '#lock' do
     before do
       @job = SideJob.queue('testq', 'TestWorker')
     end
 
-    it 'returns and caches worker configuration' do
-      expect(SideJob::Worker).to receive(:config).with('testq', 'TestWorker').once.and_call_original
-      @job.reload
-      @job.config
-      @job.config # test cached
+    it 'sets the ttl on the lock expiration' do
+      @job.lock(123)
+      expect(SideJob.redis.ttl("#{@job.redis_key}:lock")).to be_within(1).of(123)
+    end
+
+    it 'returns a random token when lock is acquired' do
+      allow(SecureRandom).to receive(:uuid) { 'abcde' }
+      expect(SideJob.redis.exists("#{@job.redis_key}:lock")).to be false
+      expect(@job.lock(100)).to eq('abcde')
+      expect(SideJob.redis.get("#{@job.redis_key}:lock")).to eq 'abcde'
+    end
+
+    it 'returns nil if job is locked' do
+      expect(SideJob.redis.exists("#{@job.redis_key}:lock")).to be false
+      expect(@job.lock(100)).to_not be nil
+      expect(@job.lock(100, retries: 1)).to be nil
+    end
+  end
+
+  describe '#refresh_lock' do
+    before do
+      @job = SideJob.queue('testq', 'TestWorker')
+    end
+
+    it 'resets the lock expiration' do
+      @job.lock(100)
+      expect(SideJob.redis.ttl("#{@job.redis_key}:lock")).to be_within(1).of(100)
+      @job.refresh_lock(200)
+      expect(SideJob.redis.ttl("#{@job.redis_key}:lock")).to be_within(1).of(200)
+    end
+  end
+
+  describe '#unlock' do
+    before do
+      @job = SideJob.queue('testq', 'TestWorker')
+    end
+
+    it 'unlocks when the token matches' do
+      token = @job.lock(100)
+      expect(@job.unlock(token)).to be true
+      expect(SideJob.redis.exists("#{@job.redis_key}:lock")).to be false
+    end
+
+    it 'does not unlock when the token does not match' do
+      token = @job.lock(100)
+      expect(@job.unlock("#{token}x")).to be false
+      expect(SideJob.redis.exists("#{@job.redis_key}:lock")).to be true
     end
   end
 end