sidejob 4.0.2 → 4.1.0

data/lib/sidejob/port.rb

@@ -1,8 +1,10 @@
+require 'delegate'
+
 module SideJob
   # Represents an input or output port from a Job
   class Port
     # Returned by {#read} and {#default} to indicate no data
-    class None < Object; end
+    class None; end
 
     attr_reader :job, :type, :name
 
@@ -39,11 +41,10 @@ module SideJob
       size > 0 || default?
     end
 
-    # Returns the port default value. To distinguish a null default value vs no default, use {#default?}.
-    # @return [Object, None] The default value on the port or {SideJob::Port::None} if none
+    # Returns the port default value. See {.decode_data} for details about the return value.
+    # @return [Delegator, None] The default value on the port or {SideJob::Port::None} if none
     def default
-      val = SideJob.redis.hget("#{@job.redis_key}:#{type}ports:default", @name)
-      val ? parse_json(val) : None
+      self.class.decode_data SideJob.redis.hget("#{@job.redis_key}:#{type}ports:default", @name)
     end
 
     # Returns if the port has a default value.
@@ -58,36 +59,73 @@ module SideJob
       if val == None
         SideJob.redis.hdel "#{@job.redis_key}:#{type}ports:default", @name
       else
-        SideJob.redis.hset "#{@job.redis_key}:#{type}ports:default", @name, val.to_json
+        SideJob.redis.hset "#{@job.redis_key}:#{type}ports:default", @name, self.class.encode_data(val)
+      end
+    end
+
+    # Returns the connected port channels.
+    # @return [Array<String>] List of port channels
+    def channels
+      JSON.parse(SideJob.redis.hget("#{@job.redis_key}:#{type}ports:channels", @name)) rescue []
+    end
+
+    # Set the channels connected to the port.
+    # @param channels [Array<String>] Port channels
+    def channels=(channels)
+      SideJob.redis.multi do |multi|
+        if channels && channels.length > 0
+          multi.hset "#{@job.redis_key}:#{type}ports:channels", @name, channels.to_json
+        else
+          multi.hdel "#{@job.redis_key}:#{type}ports:channels", @name
+        end
+
+        if type == :in
+          channels.each do |chan|
+            multi.sadd "channel:#{chan}", @job.id
+          end
+        end
       end
     end
 
-    # Write data to the port. If port in an input port, runs the job.
+    # Write data to the port. If port in an input port, runs the job, otherwise run the parent job.
     # @param data [Object] JSON encodable data to write to the port
     def write(data)
-      # For {SideJob::Worker#for_inputs}, if this is set, we instead set the port default on writes
-      if Thread.current[:sidejob_port_write_default]
+      options = (Thread.current[:sidejob_port_group] || {})[:options] || {}
+      # For {SideJob::Worker#for_inputs}, if this option is set, we set the port default instead of pushing to the port
+      if options[:set_default]
         self.default = data
       else
-        SideJob.redis.rpush redis_key, data.to_json
+        SideJob.redis.rpush redis_key, self.class.encode_data(data)
+      end
+
+      # run job if inport otherwise run parent
+      @job.run(parent: type != :in) unless options[:notify] == false
+
+      log(write: [ { port: self, data: [data] } ]) unless options[:log] == false
+
+      if type == :out
+        channels.each do |chan|
+          SideJob.publish chan, data
+        end
       end
-      @job.run(parent: type != :in) # run job if inport otherwise run parent
-      log(write: [ { port: self, data: [data] } ])
     end
 
-    # Reads the oldest data from the port. Returns the default value if no data and there is a default.
-    # @return [Object, None] First data from port or {SideJob::Port::None} if there is no data and no default.
+    # Reads the oldest data from the port. See {.decode_data} for details about the wrapped return value.
+    # Returns the {#default} if there is no port data and there is a default.
+    # Returns {SideJob::Port::None} if there is no port data and no default.
+    # @return [Delegator, None] First data from port or {SideJob::Port::None} if there is no data and no default
     def read
+      options = (Thread.current[:sidejob_port_group] || {})[:options] || {}
       data = SideJob.redis.lpop(redis_key)
       if data
-        data = parse_json(data)
+        data = self.class.decode_data(data)
       elsif default?
         data = default
       else
         return None
       end
 
-      log(read: [ { port: self, data: [data] } ])
+      log(read: [ { port: self, data: [data] } ]) unless options[:log] == false || data.sidejob_options['log'] == false
 
       data
     end
@@ -135,9 +173,18 @@ module SideJob
         end
       end
 
-      data.map! {|x| parse_json x}
+      data.map! {|x| self.class.decode_data(x)}
       if data.length > 0
         log(read: [{ port: self, data: data }], write: ports.map { |port| {port: port, data: data} })
+
+        # Publish to destination channels
+        ports.each do |port|
+          if port.type == :out
+            port.channels.each do |chan|
+              data.each { |x| SideJob.publish chan, x }
+            end
+          end
+        end
       end
 
       # Run the port job or parent only if something was changed
@@ -171,53 +218,109 @@ module SideJob
       redis_key.hash
     end
 
-    # Groups all port reads and writes within the block into a single logged event.
-    def self.log_group(&block)
-      outermost = ! Thread.current[:sidejob_port_group]
+    # Creates a group for port reads and write.
+    # All events inside the block are combined into a single logged event.
+    # Nested groups are not logged until the outermost group closes.
+    # Can pass additional options that are used for port read/writes inside the group.
+    # The default for all options is nil which means to inherit the current option value or its default.
+    # @param log [Boolean] If false, do not log the writing or reading of the data (default true)
+    # @param notify [Boolean] If false, do not notify (run) the port's job
+    # @param set_default [Boolean] If true, instead of writing to port, set default value
+    def self.group(log: nil, notify: nil, set_default: nil, &block)
+      previous_group = if Thread.current[:sidejob_port_group]
+                         Thread.current[:sidejob_port_group].dup
+                       else
+                         nil
+                       end
+
       Thread.current[:sidejob_port_group] ||= {read: {}, write: {}} # port -> [data]
+
+      options = if previous_group && previous_group[:options]
+                  previous_group[:options].dup
+                else
+                  {}
+                end
+      options[:log] = log unless log.nil?
+      options[:notify] = notify unless notify.nil?
+      options[:set_default] = set_default unless set_default.nil?
+      Thread.current[:sidejob_port_group][:options] = options
+
       yield
     ensure
-      if outermost
-        self._really_log Thread.current[:sidejob_port_group]
-        Thread.current[:sidejob_port_group] = nil
+      if ! previous_group
+        group = Thread.current[:sidejob_port_group]
+        if group && (group[:read].length > 0 || group[:write].length > 0)
+          log_entry = {}
+          %i{read write}.each do |type|
+            log_entry[type] = group[type].map do |port, data|
+              x = {job: port.job.id, data: data}
+              x[:"#{port.type}port"] = port.name
+              x
+            end
+          end
+
+          SideJob.log log_entry
+        end
       end
+      Thread.current[:sidejob_port_group] = previous_group
     end
 
-    private
-
-    def self._really_log(entry)
-      return unless entry && (entry[:read].length > 0 || entry[:write].length > 0)
+    # Encodes data as JSON with the current SideJob context.
+    # @param data [Object] JSON encodable data
+    # @return [String] The encoded JSON value
+    def self.encode_data(data)
+      encoded = { data: data }
+      encoded[:context] = Thread.current[:sidejob_context] if Thread.current[:sidejob_context]
+      if Thread.current[:sidejob_port_group] && Thread.current[:sidejob_port_group][:options]
+        encoded[:options] = Thread.current[:sidejob_port_group][:options]
+      end
+      encoded.to_json
+    end
 
-      log_entry = {}
-      %i{read write}.each do |type|
-        log_entry[type] = entry[type].map do |port, data|
-          x = {job: port.job.id, data: data}
-          x[:"#{port.type}port"] = port.name
-          x
+    # Decodes data encoded with {.encode_data}.
+    # The value is returned as a Delegator object that behaves mostly like the underlying value.
+    # Use {Delegator#__getobj__} to get directly at the underlying value.
+    # The returned delegator object has a sidejob_context method that returns the SideJob context
+    # and a sidejob_options method that returns the data options.
+    # @param data [String, nil] Data to decode
+    # @return [Delegator, None] The decoded value or {SideJob::Port::None} if data is nil
+    def self.decode_data(data)
+      if data
+        data = JSON.parse(data)
+        klass = Class.new(SimpleDelegator) do
+          # Allow comparing two SimpleDelegator objects
+          def ==(obj)
+            return self.__getobj__ == obj.__getobj__ if obj.is_a?(SimpleDelegator)
+            super
+          end
+        end
+        klass.send(:define_method, :sidejob_context) do
+          data['context'] || {}
         end
+        klass.send(:define_method, :sidejob_options) do
+          data['options'] || {}
+        end
+        klass.new(data['data'])
+      else
+        None
       end
-
-      SideJob.log log_entry
     end
 
+    private
+
     def log(data)
-      entry = Thread.current[:sidejob_port_group] ? Thread.current[:sidejob_port_group] : {read: {}, write: {}}
-      %i{read write}.each do |type|
-        (data[type] || []).each do |x|
-          entry[type][x[:port]] ||= []
-          entry[type][x[:port]].concat JSON.parse(x[:data].to_json) # serialize/deserialize to do a deep copy
+      if Thread.current[:sidejob_port_group]
+        %i{read write}.each do |type|
+          (data[type] || []).each do |x|
+            Thread.current[:sidejob_port_group][type][x[:port]] ||= []
+            Thread.current[:sidejob_port_group][type][x[:port]].concat JSON.parse(x[:data].to_json) # serialize/deserialize to do a deep copy
+          end
+        end
+      else
+        SideJob::Port.group do
+          log(data)
         end
       end
-      if ! Thread.current[:sidejob_port_group]
-        self.class._really_log(entry)
-      end
-    end
-
-    # Wrapper around JSON.parse to also handle primitive types.
-    # @param data [String] Data to parse
-    # @return [Object, nil]
-    def parse_json(data)
-      JSON.parse("[#{data}]")[0]
     end
 
     # Check if the port exists, dynamically creating it if it does not exist and a * port exists for the job

data/lib/sidejob/server_middleware.rb

@@ -5,6 +5,11 @@ module SideJob
   # For simplicity, a job is allowed to be queued multiple times in the Sidekiq queue
   # Only when it gets pulled out to be run, i.e. here, we decide if we want to actually run it
   class ServerMiddleware
+    class << self
+      # If true, we do not rescue or log errors and instead propagate errors (useful for testing)
+      attr_accessor :raise_errors
+    end
+
     # Called by sidekiq as a server middleware to handle running a worker
     # @param worker [SideJob::Worker]
     # @param msg [Hash] Sidekiq message format
@@ -29,7 +34,7 @@ module SideJob
       if token
         begin
           SideJob.redis.del "#{@worker.redis_key}:lock:worker"
-          SideJob.log_context(job: @worker.id) do
+          SideJob.context(job: @worker.id) do
             case @worker.status
               when 'queued'
                 run_worker { yield }
@@ -82,7 +87,11 @@ module SideJob
         @worker.terminate
       else
         # normal run
-        @worker.set ran_at: SideJob.timestamp
+
+        # if ran_at is not set, then this is the first run of the job, so call the startup method if it exists
+        @worker.startup if @worker.respond_to?(:startup) && ! SideJob.redis.exists("#{@worker.redis_key}:ran_at")
+
+        SideJob.redis.set "#{@worker.redis_key}:ran_at", SideJob.timestamp
         @worker.status = 'running'
         yield
         @worker.status = 'completed' if @worker.status == 'running'
@@ -101,8 +110,11 @@ module SideJob
     end
 
     def add_exception(exception)
-      # only store the backtrace until the first sidekiq line
-      SideJob.log({ error: exception.message, backtrace: exception.backtrace.take_while {|l| l !~ /sidekiq/}.join("\n") })
+      if SideJob::ServerMiddleware.raise_errors
+        raise exception
+      else
+        SideJob.log exception
+      end
     end
   end
 end

data/lib/sidejob/testing.rb

@@ -31,22 +31,15 @@ module SideJob
     def run_inline(errors: true, queue: true, args: [])
       self.status = 'queued' if queue
 
-      worker = get(:class).constantize.new
+      worker_info = JSON.parse(SideJob.redis.get("#{redis_key}:worker"))
+      worker = worker_info['class'].constantize.new
       worker.jid = id
-      SideJob::ServerMiddleware.new.call(worker, {}, get(:queue)) do
+      SideJob::ServerMiddleware.raise_errors = errors
+      SideJob::ServerMiddleware.new.call(worker, {}, worker_info['queue']) do
         worker.perform(*args)
       end
-
-      if errors && status == 'failed'
-        SideJob.logs.each do |event|
-          if event['error']
-            exception = RuntimeError.exception(event['error'])
-            exception.set_backtrace(event['backtrace'])
-            raise exception
-          end
-        end
-        raise "Job #{id} failed but cannot find error log"
-      end
+    ensure
+      SideJob::ServerMiddleware.raise_errors = false
     end
   end
 end

data/lib/sidejob/version.rb

@@ -1,4 +1,4 @@
 module SideJob
   # The current SideJob version
-  VERSION = '4.0.2'
+  VERSION = '4.1.0'
 end

data/lib/sidejob/worker.rb

@@ -16,6 +16,9 @@ module SideJob
           multi.del "workers:#{queue}"
           multi.hmset "workers:#{queue}", @registry.map {|key, val| [key, val.to_json]}.flatten(1) if @registry.size > 0
         end
+        SideJob::Port.group(log: false) do
+          SideJob.publish "/sidejob/workers/#{queue}", @registry
+        end
       end
 
       # Returns the configuration registered for a worker.
@@ -83,7 +86,7 @@ module SideJob
       return unless inputs.length > 0
       ports = inputs.map {|name| input(name)}
       loop do
-        SideJob::Port.log_group do
+        SideJob::Port.group do
           info = ports.map {|port| [ port.size > 0, port.default? ] }
 
           return unless info.any? {|x| x[0] || x[1]} # Nothing to do if there's no data to read
@@ -97,11 +100,8 @@ module SideJob
             last_default = get(:for_inputs) || []
             return unless defaults != last_default
             set({for_inputs: defaults})
-            begin
-              Thread.current[:sidejob_port_write_default] = true
+            SideJob::Port.group(set_default: true) do
               yield *defaults
-            ensure
-              Thread.current[:sidejob_port_write_default] = nil
             end
             return
           else

data/spec/integration/channels_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe 'Channels'do
+  it 'Global pubsub via channels works correctly' do
+    job1 = SideJob.queue('testq', 'TestSum')
+    job2 = SideJob.queue('testq', 'TestSum')
+    job3 = SideJob.queue('testq', 'TestSum')
+    job4 = SideJob.queue('testq', 'TestSum')
+
+    job1.input(:ready).channels = [ '/test/ready' ]
+    job1.input(:in).channels = ['/test/in']
+    job1.output(:sum).channels = ['/test/chan1']
+
+    job2.input(:ready).channels = [ '/test/ready' ]
+    job2.input(:in).channels = ['/test/chan1']
+    job2.output(:sum).channels = ['/test/chan2']
+
+    job3.input(:ready).channels = [ '/test/ready' ]
+    job3.input(:in).channels = ['/test/chan1']
+    job3.output(:sum).channels = ['/test/chan2']
+
+    job4.input(:ready).channels = [ '/test/ready' ]
+    job4.input(:in).channels = ['/test/chan2']
+
+    [1,2,4].each {|x| SideJob.publish '/test/in', x}
+    SideJob.publish '/test/ready', true
+
+    job1.run_inline
+    job2.run_inline
+    job3.run_inline
+    job4.run_inline
+
+    expect(job4.status).to eq 'completed'
+    expect(job4.output(:sum).read).to eq(14)
+  end
+end

data/spec/sidejob/job_spec.rb

@@ -65,11 +65,97 @@ describe SideJob::Job do
   end
 
   describe '#status=' do
-    it 'sets status' do
+    before do
       @job = SideJob.queue('testq', 'TestWorker')
+    end
+
+    it 'sets status' do
       @job.status = 'newstatus'
       expect(@job.status).to eq 'newstatus'
     end
+
+    it 'publishes a message with status change' do
+      expect(SideJob).to receive(:publish).with("/sidejob/job/#{@job.id}", {status: 'newstatus'})
+      @job.status = 'newstatus'
+    end
+
+    it 'does not publish a message if status does not change' do
+      @job.status = 'newstatus'
+      expect(SideJob).not_to receive(:publish)
+      @job.status = 'newstatus'
+    end
+
+    it 'does not publish a message if worker has status_publish: false' do
+
+    end
+  end
+
+  describe '#aliases' do
+    before do
+      @job = SideJob.queue('testq', 'TestWorker')
+    end
+
+    it 'returns empty array if no aliases' do
+      expect(@job.aliases).to eq []
+    end
+
+    it 'returns all aliases' do
+      @job.add_alias('abc')
+      @job.add_alias('xyz')
+      expect(@job.aliases).to match_array ['abc', 'xyz']
+    end
+  end
+
+  describe '#add_alias' do
+    before do
+      @job = SideJob.queue('testq', 'TestWorker')
+      @job.add_alias('abc')
+    end
+
+    it 'adds an alias' do
+      expect(SideJob.redis.smembers("#{@job.redis_key}:aliases")).to eq ['abc']
+      expect(SideJob.redis.hgetall('jobs:aliases')).to eq({'abc' => @job.id.to_s})
+      @job.add_alias('xyz')
+      expect(SideJob.redis.smembers("#{@job.redis_key}:aliases")).to match_array ['abc', 'xyz']
+      expect(SideJob.redis.hgetall('jobs:aliases')).to eq({'abc' => @job.id.to_s, 'xyz' => @job.id.to_s})
+      expect(@job.aliases).to match_array ['abc', 'xyz']
+    end
+
+    it 'does nothing if name is already alias' do
+      @job.add_alias('abc')
+      expect(SideJob.redis.smembers("#{@job.redis_key}:aliases")).to eq ['abc']
+      expect(SideJob.redis.hgetall('jobs:aliases')).to eq({'abc' => @job.id.to_s})
+    end
+
+    it 'raises error if name is an alias for another job' do
+      @job2 = SideJob.queue('testq', 'TestWorker')
+      expect { @job2.alias_as('abc') }.to raise_error
+    end
+
+    it 'raises error if name is invalid' do
+      expect { @job.add_alias('1234') }.to raise_error
+      expect { @job.add_alias('!') }.to raise_error
+      expect { @job.add_alias('') }.to raise_error
+      expect { @job.add_alias(nil) }.to raise_error
+    end
+  end
+
+  describe '#remove_alias' do
+    before do
+      @job = SideJob.queue('testq', 'TestWorker')
+      @job.add_alias('abc')
+    end
+
+    it 'removes an existing alias' do
+      @job.add_alias('xyz')
+      expect(@job.aliases).to match_array ['abc', 'xyz']
+      @job.remove_alias('xyz')
+      expect(@job.aliases).to match_array ['abc']
+    end
+
+    it 'throws an error if alias does not exist' do
+      expect { @job.remove_alias('xyz') }.to raise_error
+    end
   end
 
   describe '#run' do
@@ -133,7 +219,7 @@ describe SideJob::Job do
     end
 
     it 'throws error and immediately sets status to terminated if job class is unregistered' do
-      SideJob.redis.del "workers:#{@job.get(:queue)}"
+      SideJob.redis.del "workers:#{@job.info[:queue]}"
       expect { @job.run }.to raise_error
       expect(@job.status).to eq 'terminated'
     end
@@ -258,7 +344,7 @@ describe SideJob::Job do
 
     it 'queues with by string set to self' do
       child = @job.queue('testq', 'TestWorker', name: 'child')
-      expect(child.get(:created_by)).to eq "job:#{@job.id}"
+      expect(child.info[:created_by]).to eq "job:#{@job.id}"
     end
   end
 
@@ -371,8 +457,15 @@ describe SideJob::Job do
       expect(@job.exists?).to be false
     end
 
+    it 'publishes a message when deleted' do
+      @job.status = 'terminated'
+      expect(@job).to receive(:publish).with({deleted: true})
+      expect(@job.delete).to be true
+    end
+
     it 'recursively deletes jobs' do
       child = SideJob.queue('testq', 'TestWorker', parent: @job, name: 'child')
+      child.add_alias('myjob')
       expect(@job.status).to eq('queued')
       expect(child.status).to eq('queued')
       expect(SideJob.redis {|redis| redis.keys('job:*').length}).to be > 0
@@ -382,6 +475,7 @@ describe SideJob::Job do
       expect(SideJob.redis {|redis| redis.keys('job:*').length}).to be(0)
       expect(@job.exists?).to be false
       expect(child.exists?).to be false
+      expect(SideJob.find('myjob')).to be nil
     end
 
     it 'deletes data on input and output ports' do
@@ -401,6 +495,15 @@ describe SideJob::Job do
       child.delete
       expect(@job.children.size).to eq 0
     end
+
+    it 'removes any aliases' do
+      @job.add_alias 'job'
+      @job.status = 'terminated'
+      expect(SideJob.find('job')).to eq @job
+      @job.delete
+      expect(SideJob.find('job')).to be nil
+      expect(SideJob.redis.hget('jobs:aliases', 'job')).to be nil
+    end
   end
 
   # Tests are identical for input and output port methods
@@ -471,6 +574,15 @@ describe SideJob::Job do
         expect(@job.send("#{type}put", :myport).read).to eq 'new'
       end
 
+      it 'sets port channels' do
+        @channels = ['abc', 'xyz']
+        @job.send("#{type}ports=", {myport: {channels: @channels}})
+        expect(@job.send("#{type}put", :myport).channels).to match_array(@channels)
+        @channels.each do |chan|
+          expect(SideJob.redis.smembers("channel:#{chan}")).to eq [@job.id.to_s]
+        end
+      end
+
       it 'deletes no longer used ports' do
         @job.send("#{type}ports=", {myport: {}})
         @job.send("#{type}put", :myport).write 123
@@ -481,16 +593,28 @@ describe SideJob::Job do
     end
   end
 
-  describe '#state' do
+  describe '#info' do
     before do
       now = Time.now
       allow(Time).to receive(:now) { now }
+      @job = SideJob.queue('testq', 'TestWorker', args: [123], by: 'me')
+    end
+
+    it 'returns all basic job info' do
+      expect(@job.info).to eq({queue: 'testq', class: 'TestWorker', args: [123],
+                               created_by: 'me', created_at: SideJob.timestamp, ran_at: nil})
+    end
+  end
+
+  describe '#state' do
+    before do
       @job = SideJob.queue('testq', 'TestWorker')
     end
 
-    it 'returns job state with common and internal keys' do
+    it 'returns all job state' do
       @job.set({abc: 123})
-      expect(@job.state).to eq({"queue"=>"testq", "class"=>"TestWorker", "args"=>nil, "created_by"=>nil, "created_at"=>SideJob.timestamp, 'status' => 'queued', 'abc' => 123})
+      @job.set({xyz: 456})
+      expect(@job.state).to eq({'abc' => 123, 'xyz' => 456})
     end
   end
 
@@ -518,7 +642,7 @@ describe SideJob::Job do
 
     it 'always returns the latest value' do
       expect(@job.get(:field3)).to eq 123
-      SideJob.redis.hmset @job.redis_key, :field3, '789'
+      SideJob.find(@job.id).set(field3: 789)
       expect(@job.get(:field3)).to eq 789
     end
   end
@@ -543,7 +667,7 @@ describe SideJob::Job do
       3.times do |i|
         @job.set key: [i]
         expect(@job.get(:key)).to eq [i]
-        expect(JSON.parse(SideJob.redis.hget(@job.redis_key, 'key'))).to eq [i]
+        expect(JSON.parse(SideJob.redis.hget("#{@job.redis_key}:state", 'key'))).to eq [i]
       end
     end
 
@@ -622,4 +746,13 @@ describe SideJob::Job do
       expect(SideJob.redis.exists("#{@job.redis_key}:lock")).to be true
     end
   end
+
+  describe '#publish' do
+    it 'calls SideJob.publish' do
+      @job = SideJob.queue('testq', 'TestWorker')
+      message = {abc: 123}
+      expect(SideJob).to receive(:publish).with("/sidejob/job/#{@job.id}", message)
+      @job.publish(message)
+    end
+  end
 end