sidejob 3.0.1 → 4.0.1

data/spec/sidejob/server_middleware_spec.rb

@@ -34,7 +34,6 @@ describe SideJob::ServerMiddleware do
     worker = msg.klass.constantize.new
     worker.jid = job.id
     chain.invoke(worker, msg, @queue) { yield worker }
-    job.reload
     worker
   end
 
@@ -67,10 +66,7 @@ describe SideJob::ServerMiddleware do
       @job.status = 'suspended'
       child = SideJob.queue(@queue, 'TestWorker', parent: @job, name: 'child')
       expect(@job.status).to eq 'suspended'
-      expect {
-        child.run_inline
-      }.to change {Sidekiq::Stats.new.enqueued}.by(1)
-      @job.reload
+      child.run_inline
       expect(@job.status).to eq 'queued'
     end
 
@@ -84,22 +80,27 @@ describe SideJob::ServerMiddleware do
   end
 
   describe 'prevents multiple threads running the same job' do
-    it 'sets the job lock to the current time' do
-      now = Time.now
-      allow(Time).to receive(:now) { now }
-      process(@job) { @lock = SideJob.redis.get("#{@job.redis_key}:lock").to_f }
-      expect(@lock).to eq(now.to_f)
+    it 'does not run if the worker lock is set' do
+      SideJob.redis.set "#{@job.redis_key}:lock:worker", 1
+      @run = false
+      process(@job) { @run = true }
+      expect(@run).to be false
+      expect(SideJob.redis.exists("#{@job.redis_key}:lock:worker"))
+    end
+
+    it 'obtains and releases a lock' do
+      process(@job) { @lock = SideJob.redis.get("#{@job.redis_key}:lock") }
+      expect(@lock).to_not be nil
       expect(SideJob.redis.exists("#{@job.redis_key}:lock")).to be false
     end
 
-    it 'sets the job lock to the current time and does not run if already locked' do
-      now = Time.now
-      allow(Time).to receive(:now) { now }
+    it 'does not run if the job is locked' do
+      token = @job.lock(100)
       @run = false
-      SideJob.redis.set "#{@job.redis_key}:lock", (now-10).to_f
       process(@job) { @run = true }
       expect(@run).to be false
-      expect(SideJob.redis.get("#{@job.redis_key}:lock").to_f).to eq now.to_f
+      expect(SideJob.redis.exists("#{@job.redis_key}:lock"))
+      expect(@job.unlock(token)).to be true
     end
 
     it 'does not restart the worker unless another worker was locked out during the run' do
@@ -109,10 +110,11 @@ describe SideJob::ServerMiddleware do
       expect(@job.status).to eq 'completed'
     end
 
-    it 'restarts the worker if another worker was locked out during the run' do
+    it 'requeues the worker if it was locked out during the run' do
+      token = @job.lock(100)
       expect {
-        process(@job) { SideJob.redis.set "#{@job.redis_key}:lock", Time.now.to_f }
-      }.to change {Sidekiq::Stats.new.enqueued}.by(1)
+        process(@job) { }
+      }.to change {Sidekiq::Stats.new.scheduled_size}.by(1)
       expect(@job.status).to eq 'queued'
     end
   end
@@ -122,7 +124,7 @@ describe SideJob::ServerMiddleware do
       now = Time.now
       allow(Time).to receive(:now) { now }
       key = "#{@job.redis_key}:rate:#{Time.now.to_i/60}"
-      SideJob.redis.set key, SideJob::ServerMiddleware::CONFIGURATION[:max_runs_per_minute]
+      SideJob.redis.set key, SideJob::CONFIGURATION[:max_runs_per_minute]
       @run = false
       process(@job) { @run = true }
       expect(@run).to be false
@@ -133,27 +135,7 @@ describe SideJob::ServerMiddleware do
       now = Time.now
       allow(Time).to receive(:now) { now }
       key = "#{@job.redis_key}:rate:#{Time.now.to_i/60}"
-      SideJob.redis.set key, SideJob::ServerMiddleware::CONFIGURATION[:max_runs_per_minute]-1
-      @run = false
-      process(@job) { @run = true }
-      expect(@run).to be true
-      expect(@job.status).to eq 'completed'
-    end
-
-    it 'does not run if job is too deep' do
-      (SideJob::ServerMiddleware::CONFIGURATION[:max_depth]+1).times do |i|
-        @job = SideJob.queue(@queue, 'TestWorker', parent: @job, name: 'child')
-      end
-      @run = false
-      process(@job) { @run = true }
-      expect(@run).to be false
-      expect(@job.status).to eq 'terminating'
-    end
-
-    it 'does run if job is not too deep' do
-      SideJob::ServerMiddleware::CONFIGURATION[:max_depth].times do |i|
-        @job = SideJob.queue(@queue, 'TestWorker', parent: @job, name: 'child')
-      end
+      SideJob.redis.set key, SideJob::CONFIGURATION[:max_runs_per_minute]-1
       @run = false
       process(@job) { @run = true }
       expect(@run).to be true
@@ -176,21 +158,19 @@ describe SideJob::ServerMiddleware do
       expect(log[0]['backtrace']).to_not match(/sidekiq/)
     end
 
-    it 'does not set status to failed if status is not running' do
+    it 'does not set status to failed if status is terminating' do
       process(@job) do |worker|
-        worker.run
+        worker.terminate
         raise 'oops'
       end
-      expect(@job.status).to eq 'queued'
+      expect(@job.status).to eq 'terminating'
     end
 
     it 'runs the parent job' do
       @job.status = 'suspended'
       child = SideJob.queue(@queue, 'TestWorker', parent: @job, name: 'child')
-      expect {
-        process(child) { raise 'oops' }
-      }.to change {Sidekiq::Stats.new.enqueued}.by(1)
-      @job.reload
+      expect(@job.status).to eq 'suspended'
+      process(child) { raise 'oops' }
       expect(@job.status).to eq 'queued'
     end
   end

data/spec/sidejob/worker_spec.rb

@@ -5,7 +5,6 @@ describe SideJob::Worker do
     @job = SideJob.queue('testq', 'TestWorker', inports: {
         in1: {},
         in2: {},
-        memory: { mode: :memory },
         default: { default: 'default' },
         default_null: { default: nil },
     }, outports: {out1: {}})
@@ -62,33 +61,6 @@ describe SideJob::Worker do
     expect { @worker.suspend }.to raise_error(SideJob::Worker::Suspended)
   end
 
-  describe '#queue' do
-    it 'can queue child jobs' do
-      expect(SideJob).to receive(:queue).with('testq', 'TestWorker', args: [1,2], inports: {'myport' => {'mode' => 'memory'}}, parent: @job, name: 'child', by: "job:#{@worker.id}").and_call_original
-      expect {
-        child = @worker.queue('testq', 'TestWorker', args: [1,2], inports: {'myport' => {'mode' => 'memory'}}, name: 'child')
-        expect(child.parent).to eq(@job)
-        expect(@job.children).to eq('child' => child)
-      }.to change {Sidekiq::Stats.new.enqueued}.by(1)
-    end
-
-    it 'queues with by string set to self' do
-      child = @worker.queue('testq', 'TestWorker', name: 'child')
-      expect(child.get(:created_by)).to eq "job:#{@worker.id}"
-    end
-
-    it 'groups initial port data' do
-      now = Time.now
-      allow(Time).to receive(:now) { now }
-      child = @worker.queue('testq', 'TestWorker', name: 'child', inports: {'inport1' => {data: [1,2]}}, outports: {'outport1' => {data: [3,4]}})
-      expect(SideJob.logs).to eq [{'timestamp' => SideJob.timestamp, 'job' => @worker.id,
-                                   'read' => [],
-                                   'write' => [{'job' => child.id, 'inport' => 'inport1', 'data' => [1,2]},
-                                               {'job' => child.id, 'outport' => 'outport1', 'data' => [3,4]},
-                                   ]}]
-    end
-  end
-
   describe '#for_inputs' do
     it 'does nothing if no ports provided' do
       expect {|block| @worker.for_inputs(&block)}.not_to yield_control
@@ -113,8 +85,8 @@ describe SideJob::Worker do
       @worker.for_inputs(:in1, :in2) do |in1, in2|
         @worker.output(:out1).write [in1, in2[0]]
       end
-      expect(SideJob.logs).to eq([{'timestamp' => SideJob.timestamp, 'job' => @job.id, 'read' => [{'job' => @job.id, 'inport' => 'in1', 'data' => [1]}, {'job' => @job.id, 'inport' => 'in2', 'data' => [['a', 'b']]}], 'write' => [{'job' => @job.id, 'outport' => 'out1', 'data' => [[1, 'a']]}]},
-                                  {'timestamp' => SideJob.timestamp, 'job' => @job.id, 'read' => [{'job' => @job.id, 'inport' => 'in1', 'data' => [2]}, {'job' => @job.id, 'inport' => 'in2', 'data' => [['c', 'd']]}], 'write' => [{'job' => @job.id, 'outport' => 'out1', 'data' => [[2, 'c']]}]},
+      expect(SideJob.logs).to eq([{'timestamp' => SideJob.timestamp, 'read' => [{'job' => @job.id, 'inport' => 'in1', 'data' => [1]}, {'job' => @job.id, 'inport' => 'in2', 'data' => [['a', 'b']]}], 'write' => [{'job' => @job.id, 'outport' => 'out1', 'data' => [[1, 'a']]}]},
+                                  {'timestamp' => SideJob.timestamp, 'read' => [{'job' => @job.id, 'inport' => 'in1', 'data' => [2]}, {'job' => @job.id, 'inport' => 'in2', 'data' => [['c', 'd']]}], 'write' => [{'job' => @job.id, 'outport' => 'out1', 'data' => [[2, 'c']]}]},
                                  ])
     end
 
@@ -127,16 +99,14 @@ describe SideJob::Worker do
       }.to raise_error(SideJob::Worker::Suspended)
     end
 
-    it 'returns data from memory input ports' do
-      @job.input(:memory).write 1
+    it 'returns default values from ports' do
       @job.input(:in2).write [2, 3]
       @job.input(:in2).write 3
-      expect {|block| @worker.for_inputs(:memory, :in2, &block)}.to yield_successive_args([1, [2,3]], [1, 3])
+      expect {|block| @worker.for_inputs(:default, :in2, &block)}.to yield_successive_args(['default', [2,3]], ['default', 3])
     end
 
-    it 'does not suspend if there is only data on memory port' do
-      @job.input(:memory).write 1
-      expect {|block| @worker.for_inputs(:memory, :in2, &block)}.not_to yield_control
+    it 'does not yield if there are only default values' do
+      expect {|block| @worker.for_inputs(:default, :in2, &block)}.not_to yield_control
     end
 
     it 'allows for null default values' do
@@ -146,56 +116,18 @@ describe SideJob::Worker do
       expect {|block| @worker.for_inputs(:default_null, :in2, &block)}.to yield_successive_args([1, [2,3]], [nil, 3])
     end
 
-    it 'raises error if all ports have defaults' do
-      @job.input(:memory).write true
-      expect {|block| @worker.for_inputs(:memory, :default, &block)}.to raise_error
-    end
-  end
-
-  describe '#set' do
-    it 'can save state in redis' do
-      @worker.set(test: 'data', test2: 123)
-      state = JSON.parse(SideJob.redis.hget('jobs', @worker.id))
-      expect(state['test']).to eq 'data'
-      expect(state['test2']).to eq 123
-
-      # test updating
-      @worker.set(test: 'data2')
-      state = JSON.parse(SideJob.redis.hget('jobs', @worker.id))
-      expect(state['test']).to eq 'data2'
-    end
-
-    it 'can update values' do
-      3.times do |i|
-        @worker.set key: i
-        @worker.reload
-        expect(@worker.get(:key)).to eq i
-        state = JSON.parse(SideJob.redis.hget('jobs', @worker.id))
-        expect(state['key']).to eq i
-      end
-    end
-
-    it 'raises error if job no longer exists' do
-      @worker.status = 'terminated'
-      SideJob.find(@worker.id).delete
-      expect { @worker.set key: 123 }.to raise_error
-    end
-  end
-
-  describe '#unset' do
-    it 'unsets fields' do
-      @worker.set(a: 123, b: 456, c: 789)
-      @worker.unset('a', :b)
-      expect(@worker.get(:a)).to eq nil
-      expect(@worker.get(:b)).to eq nil
-      expect(@worker.get(:c)).to eq 789
+    it 'sets output default value for ports written in block if all inputs are default' do
+      expect(@job.output(:out1).default).to be SideJob::Port::None
+      @worker.for_inputs(:default, :default_null) { @job.output(:out1).write 1234 }
+      expect(@job.output(:out1).default).to eq 1234
     end
 
-    it 'raises error if job no longer exists' do
-      @worker.status = 'terminated'
-      @worker.set a: 123
-      SideJob.find(@worker.id).delete
-      expect { @worker.unset(:a) }.to raise_error
+    it 'yields exactly once until port defaults change' do
+      expect {|block| @worker.for_inputs(:default, :default_null, &block)}.to yield_successive_args(['default', nil])
+      expect {|block| @worker.for_inputs(:default, :default_null, &block)}.not_to yield_control
+      @job.input(:default).default = 'new'
+      expect {|block| @worker.for_inputs(:default, :default_null, &block)}.to yield_successive_args(['new', nil])
+      expect {|block| @worker.for_inputs(:default, :default_null, &block)}.not_to yield_control
     end
   end
 end

data/spec/sidejob_spec.rb

@@ -45,9 +45,9 @@ describe SideJob do
 
     it 'generates an incrementing job id from 1' do
       job = SideJob.queue('testq', 'TestWorker')
-      expect(job.id).to eq('1')
+      expect(job.id).to be 1
       job = SideJob.queue('testq', 'TestWorker')
-      expect(job.id).to eq('2')
+      expect(job.id).to be 2
     end
 
     it 'stores created at timestamp' do
@@ -72,7 +72,6 @@ describe SideJob do
         expect(job.status).to eq 'queued'
         expect(job.parent).to eq(parent)
         expect(parent.child('child1')).to eq job
-        expect(SideJob.redis.lrange("#{job.redis_key}:ancestors", 0, -1)).to eq([parent.id])
       }.to change {Sidekiq::Stats.new.enqueued}.by(2)
     end
 
@@ -87,25 +86,12 @@ describe SideJob do
       expect { SideJob.queue('testq', 'TestWorker', parent: parent, name: 'child') }.to raise_error
     end
 
-    it 'sets ancestor tree correctly' do
-      j1 = SideJob.queue('testq', 'TestWorker')
-      j2 = SideJob.queue('testq', 'TestWorker', parent: j1, name: 'child1')
-      j3 = SideJob.queue('testq', 'TestWorker', parent: j2, name: 'child1')
-      expect(SideJob.redis.lrange("#{j3.redis_key}:ancestors", 0, -1)).to eq([j2.id, j1.id])
-    end
-
     it 'can add a port via inports configuration' do
       job = SideJob.queue('testq', 'TestWorker', inports: {myport: {default: [1,2]}})
       expect(job.status).to eq 'queued'
       expect(job.input(:myport).read).to eq [1, 2]
     end
 
-    it 'can set the port mode via inports configuration' do
-      job = SideJob.queue('testq', 'TestWorker', inports: {queue: {mode: 'queue'}})
-      expect(job.status).to eq 'queued'
-      expect(job.input(:queue).mode).to be :queue
-    end
-
     it 'can add a port via outports configuration' do
       job = SideJob.queue('testq', 'TestWorker', outports: {myport: {}})
       expect(job.status).to eq 'queued'
@@ -179,4 +165,41 @@ describe SideJob do
                                                 {'xyz' => 456, 'timestamp' => SideJob.timestamp},])
     end
   end
+
+  describe '.log_context' do
+    before do
+      now = Time.now
+      allow(Time).to receive(:now) { now }
+    end
+
+    it 'adds metadata to logs within the group' do
+      SideJob.log_context(data1: 1, data2: 2) do
+        SideJob.log({abc: 123})
+        expect(SideJob.logs).to eq([{'data1' => 1, 'data2' => 2, 'abc' => 123, 'timestamp' => SideJob.timestamp}])
+      end
+    end
+
+    it 'does not add metadata to logs outside of the group' do
+      SideJob.log_context(data1: 1, data2: 2) {}
+      SideJob.log({abc: 123})
+      expect(SideJob.logs).to eq([{'abc' => 123, 'timestamp' => SideJob.timestamp}])
+    end
+
+    it 'can be nested' do
+      SideJob.log_context(data1: 1) do
+        SideJob.log({x: 1})
+        SideJob.log_context(data2: 2) do
+          SideJob.log({x: 2})
+        end
+        SideJob.log({x: 3})
+      end
+      SideJob.log({x: 4})
+
+      expect(SideJob.logs).to eq([{'data1' => 1, 'timestamp' => SideJob.timestamp, 'x' => 1},
+                                  {'data1' => 1, 'data2' => 2, 'timestamp' => SideJob.timestamp, 'x' => 2},
+                                  {'data1' => 1, 'timestamp' => SideJob.timestamp, 'x' => 3},
+                                  {'timestamp' => SideJob.timestamp, 'x' => 4},
+                                 ])
+    end
+  end
 end

data/web/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+gem 'sidejob', path: '..'
+gem 'sinatra'
+gem 'rack-cors'
+gem 'puma'

data/web/Gemfile.lock

@@ -0,0 +1,43 @@
+PATH
+  remote: ..
+  specs:
+    sidejob (4.0.1)
+      sidekiq (~> 3.2.5)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    celluloid (0.15.2)
+      timers (~> 1.1.0)
+    connection_pool (2.2.0)
+    json (1.8.2)
+    puma (2.10.2)
+      rack (>= 1.1, < 2.0)
+    rack (1.6.0)
+    rack-cors (0.2.9)
+    rack-protection (1.5.3)
+      rack
+    redis (3.2.1)
+    redis-namespace (1.5.2)
+      redis (~> 3.0, >= 3.0.4)
+    sidekiq (3.2.6)
+      celluloid (= 0.15.2)
+      connection_pool (>= 2.0.0)
+      json
+      redis (>= 3.0.6)
+      redis-namespace (>= 1.3.1)
+    sinatra (1.4.5)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
+    tilt (1.4.1)
+    timers (1.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  puma
+  rack-cors
+  sidejob!
+  sinatra

data/web/app.rb

@@ -0,0 +1,205 @@
+require 'sinatra'
+require 'sidejob'
+
+# Provide a limited web API to SideJob methods
+class SideJob::Web < Sinatra::Base
+  before do
+    content_type :json
+  end
+
+  # for CORS
+  options '/' do
+    200
+  end
+
+  # provide some limited info for now
+  get '/' do
+    { version: '1.0' }.to_json
+  end
+
+  # queue a new job
+  post '/jobs' do
+    api_call do |params|
+      queue = params['queue'] or return { error: 'Missing queue' }
+      klass = params['class'] or return { error: 'Missing class' }
+
+      job = SideJob.queue(queue, klass, args: params['args'],
+                          parent: SideJob.find(params['parent']), name: params['name'],
+                          by: params['by'], inports: params['inports'], outports: params['outports'])
+      { job: job.id }
+    end
+  end
+
+  # gets job state and port info
+  get '/jobs/:job' do
+    job_api do |job, params|
+      { job: job.id, state: job.state, inports: ports_info(job.inports), outports: ports_info(job.outports) }
+    end
+  end
+
+  # sets job state
+  post '/jobs/:job/state' do
+    job_api do |job, params|
+      job.set(params)
+      { job: job.id, state: job.state }
+    end
+  end
+
+  # delete a job
+  delete '/jobs/:job' do
+    job_api do |job, params|
+      { job: job.id, delete: job.delete }
+    end
+  end
+
+  # set job ports
+  post '/jobs/:job/ports' do
+    job_api do |job, params|
+      job.inports = params['inports'] if params['inports']
+      job.outports = params['outports'] if params['outports']
+      nil
+    end
+  end
+
+  # port operations
+  post '/jobs/:job/*ports/:port/:operation' do |job_id, type, port_name, operation|
+    job_api do |job, params|
+      case type
+        when 'in'
+          port = job.input(port_name)
+        when 'out'
+          port = job.output(port_name)
+        else
+          raise 'Invalid port type'
+      end
+
+      case operation
+        # read one value
+        when 'read'
+          data = port.read
+          if data == SideJob::Port::None
+            {}
+          else
+            { data: data }
+          end
+
+        # read all and return an array of data
+        when 'entries'
+          { entries: port.entries }
+
+        # port write
+        when 'write'
+          if params['list']
+            list = params['list']
+          else
+            list = [params['data']]
+          end
+          list.each {|x| port.write(x)}
+          nil
+      end
+    end
+  end
+
+  # run a job
+  post '/jobs/:job/run' do
+    job_api do |job, params|
+      job.run(force: params['force'], at: params['at'], wait: params['wait'])
+      nil
+    end
+  end
+
+  # terminate a job
+  post '/jobs/:job/terminate' do
+    job_api do |job, params|
+      job.terminate(recursive: params['recursive'])
+      nil
+    end
+  end
+
+  # lock job
+  post '/jobs/:job/lock' do
+    job_api do |job, params|
+      { token: job.lock(params['ttl']) }
+    end
+  end
+
+  # refresh lock
+  post '/jobs/:job/refresh_lock' do
+    job_api do |job, params|
+      { refresh: job.refresh_lock(params['ttl']) }
+    end
+  end
+
+  # unlock job
+  post '/jobs/:job/unlock' do
+    job_api do |job, params|
+      { unlock: job.unlock(params['token']) }
+    end
+  end
+
+  # adopt another job
+  post '/jobs/:job/adopt' do
+    job_api do |job, params|
+      orphan = SideJob.find(params[:child])
+      raise 'Child job does not exist' unless orphan
+      job.adopt(orphan, params['name'])
+      nil
+    end
+  end
+
+  # disown a job
+  post '/jobs/:job/disown' do
+    job_api do |job, params|
+      job.disown(params['name'])
+      nil
+    end
+  end
+
+  # return all logs
+  get '/logs' do
+    api_call do |params|
+      SideJob.logs(clear: params['clear'] || false)
+    end
+  end
+
+  # add a log entry
+  post '/logs' do
+    api_call do |params|
+      SideJob.log(params['entry'])
+    end
+  end
+
+  private
+
+  def job_api(&block)
+    job = SideJob.find(params['job'])
+    halt 422, { error: "Job #{params['job']} does not exist" }.to_json unless job
+    api_call do |params|
+      result = yield(job, params)
+      job.run(parent: params['run']['parent'], force: params['run']['force']) if params['run']
+      result
+    end
+  end
+
+  def api_call(&block)
+    begin
+      request.body.rewind
+      params = JSON.parse(request.body.read) rescue {}
+      SideJob.log_context(params['log_context'] || {}) do
+        (yield params).to_json
+      end
+    rescue => e
+      puts e.inspect
+      puts e.backtrace
+      halt 422, { error: e.to_s }.to_json
+    end
+  end
+
+  # @param ports [Array<SideJob::Port>]
+  def ports_info(ports)
+    ports.each_with_object({}) do |port, hash|
+      hash[port.name] = {size: port.size}
+      hash[port.name]['default'] = port.default if port.default?
+    end
+  end
+end