sidejob 4.0.2 → 4.1.0

data/spec/sidejob/testing_spec.rb

@@ -17,14 +17,6 @@ describe 'SideJob testing helpers' do
     end
   end
 
-  class TestMysteriousFailure
-    include SideJob::Worker
-    register
-    def perform
-      self.status = 'failed'
-    end
-  end
-
   describe 'SideJob::Worker.drain_queue' do
     it 'runs jobs' do
       job = SideJob.queue('testq', 'TestSum')
@@ -50,11 +42,6 @@ describe 'SideJob testing helpers' do
       expect { SideJob::Worker.drain_queue }.to raise_error(RuntimeError, 'bad error')
     end
 
-    it 'raises error if worker mysteriously fails' do
-      job = SideJob.queue('testq', 'TestMysteriousFailure')
-      expect { SideJob::Worker.drain_queue }.to raise_error(RuntimeError)
-    end
-
     it 'can disable raising of errors' do
       job = SideJob.queue('testq', 'TestFailure')
       expect { SideJob::Worker.drain_queue(errors: false) }.not_to raise_error
@@ -95,11 +82,6 @@ describe 'SideJob testing helpers' do
       expect { job.run_inline }.to raise_error(RuntimeError, 'bad error')
     end
 
-    it 'raises error if worker mysteriously fails' do
-      job = SideJob.queue('testq', 'TestMysteriousFailure')
-      expect { job.run_inline }.to raise_error(RuntimeError)
-    end
-
     it 'can disable raising of errors' do
       job = SideJob.queue('testq', 'TestFailure')
       expect { job.run_inline(errors: false) }.not_to raise_error

data/spec/sidejob/worker_spec.rb

@@ -20,6 +20,11 @@ describe SideJob::Worker do
       SideJob::Worker.register_all('q1')
       expect(SideJob.redis.hget('workers:q1', 'foo')).to be nil
     end
+
+    it 'publishes message containing worker registry' do
+      expect(SideJob).to receive(:publish).with('/sidejob/workers/q1', SideJob::Worker.registry)
+      SideJob::Worker.register_all('q1')
+    end
   end
 
   describe '.config' do
@@ -75,19 +80,19 @@ describe SideJob::Worker do
     end
 
     it 'logs input and output from them' do
-      now = Time.now
-      allow(Time).to receive(:now) { now }
       @job.input(:in1).write 1
       @job.input(:in1).write 2
       @job.input(:in2).write ['a', 'b']
       @job.input(:in2).write ['c', 'd']
-      SideJob.logs(clear: true)
+      expect(SideJob).to receive(:log).with({
+          read: [{job: @job.id, inport: :in1, data: [1]}, {job: @job.id, inport: :in2, data: [['a', 'b']]}],
+          write: [{job: @job.id, outport: :out1, data: [[1, 'a']]}]})
+      expect(SideJob).to receive(:log).with({
+          read: [{job: @job.id, inport: :in1, data: [2]}, {job: @job.id, inport: :in2, data: [['c', 'd']]}],
+          write: [{job: @job.id, outport: :out1, data: [[2, 'c']]}]})
       @worker.for_inputs(:in1, :in2) do |in1, in2|
         @worker.output(:out1).write [in1, in2[0]]
       end
-      expect(SideJob.logs).to eq([{'timestamp' => SideJob.timestamp, 'read' => [{'job' => @job.id, 'inport' => 'in1', 'data' => [1]}, {'job' => @job.id, 'inport' => 'in2', 'data' => [['a', 'b']]}], 'write' => [{'job' => @job.id, 'outport' => 'out1', 'data' => [[1, 'a']]}]},
-                                  {'timestamp' => SideJob.timestamp, 'read' => [{'job' => @job.id, 'inport' => 'in1', 'data' => [2]}, {'job' => @job.id, 'inport' => 'in2', 'data' => [['c', 'd']]}], 'write' => [{'job' => @job.id, 'outport' => 'out1', 'data' => [[2, 'c']]}]},
-                                 ])
     end
 
     it 'suspends on partial inputs' do

data/spec/sidejob_spec.rb

@@ -54,13 +54,13 @@ describe SideJob do
       now = Time.now
       allow(Time).to receive(:now) { now }
       job = SideJob.queue('testq', 'TestWorker')
-      expect(job.get(:created_at)).to eq(SideJob.timestamp)
+      expect(job.info[:created_at]).to eq(SideJob.timestamp)
     end
 
     it 'can specify job args' do
       job = SideJob.queue('testq', 'TestWorker', args: [1,2])
       expect(job.status).to eq 'queued'
-      expect(job.get(:args)).to eq [1,2]
+      expect(job.info[:args]).to eq [1,2]
       expect_any_instance_of(TestWorker).to receive(:perform).with(1, 2)
       SideJob::Worker.drain_queue
     end
@@ -109,7 +109,7 @@ describe SideJob do
 
     it 'can specify a by string' do
       job = SideJob.queue('testq', 'TestWorker', by: 'test:sidejob')
-      expect(job.get(:created_by)).to eq 'test:sidejob'
+      expect(job.info[:created_by]).to eq 'test:sidejob'
     end
 
     it 'defaults to empty by string' do
@@ -124,6 +124,12 @@ describe SideJob do
       expect(SideJob.find(job.id)).to eq(job)
     end
 
+    it 'returns a job object by alias' do
+      job = SideJob.queue('testq', 'TestWorker')
+      job.add_alias 'myjob'
+      expect(SideJob.find('myjob')).to eq(job)
+    end
+
     it 'returns nil if the job does not exist' do
       expect(SideJob.find('job')).to be_nil
     end
@@ -136,70 +142,116 @@ describe SideJob do
   end
 
   describe '.log' do
-    it 'adds a timestamp to log entries' do
+    it 'can log arbitrary entry hash' do
       now = Time.now
       allow(Time).to receive(:now) { now }
+      expect(SideJob).to receive(:publish).with('/sidejob/log', {abc: 123, timestamp: SideJob.timestamp})
       SideJob.log({abc: 123})
-      log = SideJob.redis.rpop 'jobs:logs'
-      expect(JSON.parse(log)).to eq({'abc' => 123, 'timestamp' => SideJob.timestamp})
     end
-  end
 
-  describe '.logs' do
-    before do
+    it 'can log exceptions' do
       now = Time.now
       allow(Time).to receive(:now) { now }
-      SideJob.log({abc: 123})
+      exception = RuntimeError.new('err')
+      expect(SideJob).to receive(:publish).with('/sidejob/log', {error: 'err', timestamp: SideJob.timestamp})
+      SideJob.log(exception)
     end
 
-    it 'returns and clears all logs' do
-      expect(SideJob.logs).to eq([{'abc' => 123, 'timestamp' => SideJob.timestamp}])
-      SideJob.log({xyz: 456})
-      expect(SideJob.logs).to eq([{'xyz' => 456, 'timestamp' => SideJob.timestamp}])
+    it 'can log string messages' do
+      now = Time.now
+      allow(Time).to receive(:now) { now }
+      expect(SideJob).to receive(:publish).with('/sidejob/log', {message: 'hello', timestamp: SideJob.timestamp})
+      SideJob.log 'hello'
     end
 
-    it 'returns and leaves logs' do
-      expect(SideJob.logs(clear: false)).to eq([{'abc' => 123, 'timestamp' => SideJob.timestamp}])
-      SideJob.log({xyz: 456})
-      expect(SideJob.logs(clear: false)).to eq([{'abc' => 123, 'timestamp' => SideJob.timestamp},
-                                                {'xyz' => 456, 'timestamp' => SideJob.timestamp},])
+    it 'does not generate an infinite publish loop for port subscriptions on /sidejob/log' do
+      job = SideJob.queue('testq', 'TestWorker', inports: {port1: {channels: ['/sidejob/log', '/']}})
+      SideJob.log({test: 1})
     end
   end
 
-  describe '.log_context' do
+  describe '.context' do
     before do
       now = Time.now
       allow(Time).to receive(:now) { now }
     end
 
-    it 'adds metadata to logs within the group' do
-      SideJob.log_context(data1: 1, data2: 2) do
+    it 'adds data to logs within the group' do
+      expect(SideJob).to receive(:publish).with('/sidejob/log', {data1: 1, data2: 2, abc: 123, timestamp: SideJob.timestamp})
+      SideJob.context(data1: 1, data2: 2) do
         SideJob.log({abc: 123})
-        expect(SideJob.logs).to eq([{'data1' => 1, 'data2' => 2, 'abc' => 123, 'timestamp' => SideJob.timestamp}])
       end
     end
 
-    it 'does not add metadata to logs outside of the group' do
-      SideJob.log_context(data1: 1, data2: 2) {}
+    it 'does not add data to logs outside of the group' do
+      expect(SideJob).to receive(:publish).with('/sidejob/log', {abc: 123, timestamp: SideJob.timestamp})
+      SideJob.context(data1: 1, data2: 2) {}
       SideJob.log({abc: 123})
-      expect(SideJob.logs).to eq([{'abc' => 123, 'timestamp' => SideJob.timestamp}])
     end
 
     it 'can be nested' do
-      SideJob.log_context(data1: 1) do
+      expect(SideJob).to receive(:publish).with('/sidejob/log', {data1: 1, timestamp: SideJob.timestamp, x: 1})
+      expect(SideJob).to receive(:publish).with('/sidejob/log', {data1: 1, data2: 2, timestamp: SideJob.timestamp, x: 2})
+      expect(SideJob).to receive(:publish).with('/sidejob/log', {data1: 1, timestamp: SideJob.timestamp, x: 3})
+      expect(SideJob).to receive(:publish).with('/sidejob/log', {timestamp: SideJob.timestamp, x: 4})
+      SideJob.context(data1: 1) do
         SideJob.log({x: 1})
-        SideJob.log_context(data2: 2) do
+        SideJob.context(data2: 2) do
           SideJob.log({x: 2})
         end
         SideJob.log({x: 3})
       end
       SideJob.log({x: 4})
+    end
+  end
+
+  describe '.publish' do
+    it 'publishes message to channel ignoring hierarchy using redis pubsub' do
+      Timeout::timeout(3) do
+        subscribed = false
+        t = Thread.new do
+          redis = SideJob.redis.dup
+          redis.psubscribe('*') do |on|
+            on.psubscribe do |pattern, total|
+              subscribed = true
+            end
+
+            on.pmessage do |pattern, channel, message|
+              expect(JSON.parse(message)).to eq [1,2]
+              expect(channel).to eq '/namespace/mychannel'
+              redis.punsubscribe
+            end
+          end
+        end
+
+        Thread.pass until subscribed
+        SideJob.publish '/namespace/mychannel', [1,2]
+        t.join
+      end
+    end
+
+    it 'writes to subscribed jobs' do
+      job = SideJob.queue('testq', 'TestWorker', inports: {myport: {channels: ['/namespace/mychannel']}, yourport: {channels: ['/namespace']}})
+      SideJob.publish('/namespace/mychannel', [1,2])
+      expect(job.input(:myport).entries).to eq [[1,2]]
+      expect(job.input(:yourport).entries).to eq [[1,2]]
+    end
+
+    it 'includes original channel in context' do
+      job = SideJob.queue('testq', 'TestWorker', inports: {myport: {channels: ['/namespace']}})
+      SideJob.publish('/namespace/mychannel', [1,2])
+      data = job.input(:myport).read
+      expect(data).to eq [1,2]
+      expect(data.sidejob_context).to eq({'channel' => '/namespace/mychannel'})
+    end
 
-      expect(SideJob.logs).to eq([{'data1' => 1, 'timestamp' => SideJob.timestamp, 'x' => 1},
-                                  {'data1' => 1, 'data2' => 2, 'timestamp' => SideJob.timestamp, 'x' => 2},
-                                  {'data1' => 1, 'timestamp' => SideJob.timestamp, 'x' => 3},
-                                  {'timestamp' => SideJob.timestamp, 'x' => 4},
-                                 ])
+    it 'removes jobs that are no longer subscribed' do
+      job = SideJob.queue('testq', 'TestWorker', inports: {myport: {channels: ['/namespace/mychannel']}})
+      job.input(:myport).channels = []
+      expect(SideJob.redis.smembers('channel:/namespace/mychannel')).to eq [job.id.to_s]
+      SideJob.publish('/namespace/mychannel', [1,2])
+      expect(SideJob.redis.smembers('channel:/namespace/mychannel')).to eq []
+      expect(job.input(:myport).size).to eq 0
     end
   end
 end

data/web/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    sidejob (4.0.2)
+    sidejob (4.1.0)
       sidekiq
 
 GEM
@@ -11,7 +11,7 @@ GEM
       timers (~> 4.0.0)
     connection_pool (2.2.0)
     hitimes (1.2.2)
-    json (1.8.2)
+    json (1.8.3)
     puma (2.10.2)
       rack (>= 1.1, < 2.0)
     rack (1.6.0)

data/web/app.rb

@@ -12,9 +12,25 @@ class SideJob::Web < Sinatra::Base
     200
   end
 
-  # provide some limited info for now
+  # only provide version for now
   get '/' do
-    { version: '1.0' }.to_json
+    { version: SideJob::VERSION }.to_json
+  end
+
+  # publish a message to a channel
+  post '/publish' do
+    api_call do |params|
+      SideJob.publish params['channel'], params['message']
+      nil
+    end
+  end
+
+  # add a log entry
+  post '/log' do
+    api_call do |params|
+      SideJob.log(params['entry'])
+      nil
+    end
   end
 
   # queue a new job
@@ -80,12 +96,13 @@ class SideJob::Web < Sinatra::Base
           if data == SideJob::Port::None
             {}
           else
-            { data: data }
+            { data: data, context: data.sidejob_context }
           end
 
         # read all and return an array of data
         when 'entries'
-          { entries: port.entries }
+          entries = port.entries
+          { entries: entries.map {|x| { data: x.data, context: x.sidejob_context } } }
 
         # port write
         when 'write'
@@ -155,20 +172,6 @@ class SideJob::Web < Sinatra::Base
     end
   end
 
-  # return all logs
-  get '/logs' do
-    api_call do |params|
-      SideJob.logs(clear: params['clear'] || false)
-    end
-  end
-
-  # add a log entry
-  post '/logs' do
-    api_call do |params|
-      SideJob.log(params['entry'])
-    end
-  end
-
   private
 
   def job_api(&block)
@@ -185,7 +188,7 @@ class SideJob::Web < Sinatra::Base
     begin
       request.body.rewind
       params = JSON.parse(request.body.read) rescue {}
-      SideJob.log_context(params['log_context'] || {}) do
+      SideJob.context(params['context'] || {}) do
         (yield params).to_json
       end
     rescue => e

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidejob
 version: !ruby/object:Gem::Version
-  version: 4.0.2
+  version: 4.1.0
 platform: ruby
 authors:
 - Austin Che
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-08 00:00:00.000000000 Z
+date: 2015-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -102,6 +102,7 @@ files:
 - lib/sidejob/version.rb
 - lib/sidejob/worker.rb
 - sidejob.gemspec
+- spec/integration/channels_spec.rb
 - spec/integration/fib_spec.rb
 - spec/integration/sum_spec.rb
 - spec/sidejob/job_spec.rb