omf_common 6.0.0 → 6.0.2.pre.1

data/lib/omf_common/comm.rb

@@ -49,7 +49,7 @@ module OmfCommon
         provider = @@providers[type]
       end
       unless provider
-        raise "Missing Comm provider declaration. Either define 'type', 'provider', or 'url'"
+        raise ArgumentError, "Missing Comm provider declaration. Either define 'type', 'provider', or 'url'"
       end
 
       require provider[:require] if provider[:require]
@@ -58,10 +58,18 @@ module OmfCommon
         provider_class = class_name.split('::').inject(Object) {|c,n| c.const_get(n) }
         inst = provider_class.new(opts)
       else
-        raise "Missing communicator creation info - :constructor"
+        raise ArgumentError, "Missing communicator creation info - :constructor"
       end
       @@instance = inst
-      Message.init(provider[:message_provider])
+      mopts = provider[:message_provider]
+      mopts[:authenticate] = (opts[:auth] != nil)
+      Message.init(mopts)
+
+      if aopts = opts[:auth]
+        require 'omf_common/auth'
+        OmfCommon::Auth.init(aopts)
+      end
+
       inst.init(opts)
     end
 
@@ -72,30 +80,51 @@ module OmfCommon
     # Initialize comms layer
     #
     def init(opts = {})
-      raise "Not implemented"
+    end
+
+    # Return the address used for all 'generic' messages
+    # not specifically being sent from a resource
+    #
+    def local_address()
+      @local_topic.address
+    end
+
+    def local_topic()
+      @local_topic
     end
 
     # Shut down comms layer
     def disconnect(opts = {})
-      raise "Not implemented"
+      raise NotImplementedError
     end
 
     def on_connected(&block)
-      raise "Not implemented"
+      raise NotImplementedError
+    end
+
+    # TODO should expand this to on_signal(:INT)
+    def on_interrupted(*args, &block)
     end
 
     # Create a new pubsub topic with additional configuration
     #
     # @param [String] topic Pubsub topic name
     def create_topic(topic, opts = {})
-      raise "Not implemented"
+      raise NotImplementedError
     end
 
     # Delete a pubsub topic
     #
     # @param [String] topic Pubsub topic name
     def delete_topic(topic, &block)
-      raise "Not implemented"
+      raise NotImplementedError
+    end
+
+    # Returning connection information
+    #
+    # @retun [Hash] connection information hash, with type, user and domain.
+    def conn_info
+      { proto: nil, user: nil, domain: nil }
     end
 
     # Subscribe to a pubsub topic
@@ -109,13 +138,29 @@ module OmfCommon
       ta = tna.collect do |tn|
         t = create_topic(tn)
         if block
-          block.call(t)
+          t.on_subscribed do
+            block.call(t)
+          end
         end
         t
       end
       ta[0]
     end
 
+    # Publish a message on a topic
+    #
+    # @param [String, Array] topic_name Pubsub topic name
+    # @param [OmfCoomon::Message] message
+    #
+    def publish(topic_name, message)
+      #puts "PUBLISH>>>>> #{topic_name}::#{message}"
+      tna = (topic_name.is_a? Array) ? topic_name : [topic_name]
+      ta = tna.collect do |tn|
+        t = create_topic(tn)
+        t.publish(message)
+      end
+    end
+
     # Return the options used to initiate this
     # communicator.
     #
@@ -126,6 +171,18 @@ module OmfCommon
     private
     def initialize(opts = {})
       @opts = opts
+      unless local_address = opts[:local_address]
+        hostname = nil
+        begin
+          hostname = Socket.gethostbyname(Socket.gethostname)[0]
+        rescue
+          hostname = (`hostname` || 'unknown').strip
+        end
+        local_address = "#{hostname}-#{Process.pid}"
+      end
+      on_connected do
+        @local_topic = create_topic(local_address.gsub('.', '-'))
+      end
     end
 
   end

data/lib/omf_common/comm/amqp/amqp_communicator.rb

@@ -6,7 +6,7 @@ module OmfCommon
   class Comm
     class AMQP
       class Communicator < OmfCommon::Comm
-        
+
         # def initialize(opts = {})
           # # ignore arguments
         # end
@@ -20,25 +20,30 @@ module OmfCommon
           @address_prefix = @url + '/'
           ::AMQP.connect(@url) do |connection|
             @channel  = ::AMQP::Channel.new(connection)
-            
-            if @on_connected_proc
-              @on_connected_proc.arity == 1 ? @on_connected_proc.call(self) : @on_connected_proc.call
+            @on_connected_procs.each do |proc|
+              proc.arity == 1 ? proc.call(self) : proc.call
             end
-                      
+
             OmfCommon.eventloop.on_stop do
               connection.close
             end
           end
+          super
+        end
+
+        def conn_info
+          { proto: :amqp, user: ::AMQP.settings[:user], domain: ::AMQP.settings[:host] }
         end
-  
+
         # Shut down comms layer
         def disconnect(opts = {})
         end
-        
+
+        # TODO: Should be thread safe and check if already connected
         def on_connected(&block)
-          @on_connected_proc = block
+          @on_connected_procs << block
         end
-  
+
         # Create a new pubsub topic with additional configuration
         #
         # @param [String] topic Pubsub topic name
@@ -46,9 +51,10 @@ module OmfCommon
           raise "Topic can't be nil or empty" if topic.nil? || topic.empty?
           opts = opts.dup
           opts[:channel] = @channel
+          topic = topic.to_s
           if topic.start_with? 'amqp:'
             # absolute address
-            unless topic.start_with? @address_prefix 
+            unless topic.start_with? @address_prefix
               raise "Cannot subscribe to a topic from different domain (#{topic})"
             end
             opts[:address] = topic
@@ -58,7 +64,7 @@ module OmfCommon
           end
           OmfCommon::Comm::AMQP::Topic.create(topic, opts)
         end
-  
+
         # Delete a pubsub topic
         #
         # @param [String] topic Pubsub topic name
@@ -67,9 +73,30 @@ module OmfCommon
             t.release
           else
             warn "Attempt to delete unknown topic '#{topic}"
-          end        
+          end
+        end
+
+        def broadcast_file(file_path, topic_name = nil, opts = {}, &block)
+          topic_name ||= SecureRandom.uuid
+          require 'omf_common/comm/amqp/amqp_file_transfer'
+          OmfCommon::Comm::AMQP::FileBroadcaster.new(file_path, @channel, topic_name, opts, &block)
+          "bdcst:#{@address_prefix + topic_name}"
+        end
+
+        def receive_file(topic_url, file_path = nil, opts = {}, &block)
+          if topic_url.start_with? @address_prefix
+            topic_url = topic_url[@address_prefix.length .. -1]
+          end
+          require 'omf_common/comm/amqp/amqp_file_transfer'
+          file_path ||= File.join(Dir.tmpdir, Dir::Tmpname.make_tmpname('bdcast', '.xxx'))
+          FileReceiver.new(file_path, @channel, topic_url, opts, &block)
+        end
+
+        private
+        def initialize(opts = {})
+          @on_connected_procs = []
+          super
         end
-  
       end
     end
   end

data/lib/omf_common/comm/amqp/amqp_file_transfer.rb

@@ -0,0 +1,259 @@
+require 'set'
+require 'monitor'
+
+module OmfCommon
+  class Comm::AMQP
+    
+    # Distributes a local file to a set of receivers subscribed to the same 
+    # topic but may join a various stages.
+    #
+    class FileBroadcaster
+      include MonitorMixin
+      
+      DEF_CHUNK_SIZE = 2**16
+      DEF_IDLE_TIME = 60
+      
+      # @param topic[String] Name of topic to send file to
+      # @param file_path[String] Path to a local file
+      # @param opts[Hash]
+      #   :chunk_size Max size of data chunk to send
+      #   :idle_time Min. time in sec to close down broadcaster after having sent last chunk
+      #
+      def initialize(file_path, channel, topic, opts = {}, &block)
+        super() # init monitor mixin
+        @block = block
+        unless File.readable?(file_path)
+          raise "Can't read file '#{file_path}'"
+        end
+        @mime_type = `file -b --mime-type #{file_path}`.strip
+        unless $?.success?
+          raise "Can't determine file's mime-type (#{$?})" 
+        end
+        @file_path = file_path 
+        f = File.open(file_path, 'rb')
+        chunk_size = opts[:chunk_size] || DEF_CHUNK_SIZE
+        chunk_count = (f.size / chunk_size) + 1
+        
+        @outstanding_chunks = Set.new
+        @running = true
+        @semaphore = new_cond()
+        idle_time = opts[:idle_time] || DEF_IDLE_TIME
+        
+        #chunk_count.times.each {|i| @outstanding_chunks << i}
+
+        exchange = channel.topic(topic, :auto_delete => true)
+        OmfCommon.eventloop.defer do
+          _send(f, chunk_size, chunk_count, exchange, idle_time)
+        end
+        
+        control_topic = "#{topic}_control"
+        control_exchange = channel.topic(control_topic, :auto_delete => true)
+        channel.queue("", :exclusive => false) do |queue|
+          queue.bind(control_exchange)
+          debug "Subscribing to control channel '#{control_topic}'"
+          queue.subscribe do |headers, payload|
+            hdrs = headers.headers
+            debug "Incoming control message '#{hdrs}'"
+            from = hdrs['request_from']
+            from = 0 if from < 0
+            to = hdrs['request_to']
+            to = chunk_count - 1 if !to || to >= chunk_count
+            synchronize do
+              (from .. to).each { |i| @outstanding_chunks << i}
+              @semaphore.signal
+            end
+          end
+          @control_queue = queue
+        end
+      end
+      
+      def _send(f, chunk_size, chunk_count, exchange, idle_time)
+        chunks_to_send = nil
+        @sent_chunk = false
+        _wait_for_closedown(idle_time)
+        loop do
+          synchronize do
+            @semaphore.wait_while { @outstanding_chunks.empty? && @running }
+            return unless @running # done!
+            chunks_to_send = @outstanding_chunks.to_a
+          end
+          
+          chunks_to_send.each do |chunk_id|
+            #sleep 3
+            synchronize do
+              @outstanding_chunks.delete(chunk_id)
+              @sent_chunk = true
+            end
+            offset = chunk_id * chunk_size
+            f.seek(offset, IO::SEEK_SET)
+            chunk = f.read(chunk_size)
+            payload = Base64.encode64(chunk)
+            headers = {chunk_id: chunk_id, chunk_count: chunk_count, chunk_offset: offset, 
+                        chunk_size: payload.size, 
+                        path: f.path, file_size: f.size, mime_type: @mime_type}
+            debug "Sending chunk #{chunk_id}"
+            exchange.publish(payload, {headers: headers})
+          end
+        end
+      end
+      
+      def _wait_for_closedown(idle_time)
+        OmfCommon.eventloop.after(idle_time) do
+          done = false
+          synchronize do
+            done = !@sent_chunk && @outstanding_chunks.empty?
+            @sent_chunk = false
+          end
+          if done
+            @control_queue.unsubscribe if @control_queue
+            @block.call({action: :done}) if @block
+          else
+            # there was activity in last interval, wait a bit longer
+            _wait_for_closedown(idle_time)
+          end
+        end
+      end        
+    end
+    
+    # Receives a file broadcast on 'topic' and stores it in a local file.
+    # Optionally, it can report on progress through a provided block.
+    #
+    class FileReceiver
+      include MonitorMixin
+      
+      WAIT_BEFORE_REQUESTING = 2
+      WAIT_BEFORE_REQUESTING_EVERYTHING = 3 * WAIT_BEFORE_REQUESTING
+      
+      # @param topic[String] Name of topic to receive file on
+      # @param file_path[String] Path to a local file
+      # @param opts[Hash]
+      # @param block Called on progress. 
+      #
+      def initialize(file_path, channel, topic, opts = {}, &block)
+        super() # init monitor mixin
+        f = File.open(file_path, 'wb')
+        @running = false
+        @received_chunks = false
+        @outstanding_chunks = Set.new
+        @all_requested = false # set to true if we encountered a request for ALL (no 'to')
+        @requested_chunks = Set.new
+        @received_anything = false
+        
+        control_topic = "#{topic}_control"
+        @control_exchange = channel.topic(control_topic, :auto_delete => true)
+        channel.queue("", :exclusive => false) do |queue|
+          queue.bind(@control_exchange)
+          debug "Subscribing to control topic '#{control_topic}'"
+          queue.subscribe do |headers, payload|
+            hdrs = headers.headers
+            debug "Incoming control message '#{hdrs}'"
+            from = hdrs['request_from']
+            to = hdrs['request_to']
+            synchronize do
+              if to
+                (from .. to).each { |i| @requested_chunks << i}
+              else
+                debug "Observed request for everything"
+                @all_requested = true
+                @nothing_received = -1 * WAIT_BEFORE_REQUESTING # Throttle our own desire to request everything
+              end
+            end
+          end
+          @control_queue = queue
+        end
+
+        @nothing_received = WAIT_BEFORE_REQUESTING_EVERYTHING - 2 * WAIT_BEFORE_REQUESTING
+
+        data_exchange = channel.topic(topic, :auto_delete => true)
+        channel.queue("", :exclusive => false) do |queue|
+          queue.bind(data_exchange)
+          queue.subscribe do |headers, payload|
+            synchronize do
+              @received_chunks = true
+            end
+            hdrs = headers.headers
+            chunk_id = hdrs['chunk_id']
+            chunk_offset = hdrs['chunk_offset']
+            chunk_count = hdrs['chunk_count']
+            unless chunk_id && chunk_offset && chunk_count
+              debug "Received message with missing 'chunk_id' or 'chunk_offset' header information (#{hdrs})"
+            end
+            unless @received_anything
+              @outstanding_chunks = chunk_count.times.to_set
+              synchronize do 
+                @running = true 
+                @received_anything = true
+              end
+            end
+            next unless @outstanding_chunks.include?(chunk_id)
+
+            debug "Receiving chunk #{chunk_id}"
+            f.seek(chunk_offset, IO::SEEK_SET)
+            f.write(Base64.decode64(payload))
+            @outstanding_chunks.delete(chunk_id)
+            received = chunk_count - @outstanding_chunks.size
+            if block
+              block.call({action: :progress, received: received, progress: 1.0 * received / chunk_count, total: chunk_count})
+            end
+            
+            if @outstanding_chunks.empty?
+              # got everything
+              f.close
+              queue.unsubscribe
+              @control_queue.unsubscribe if @control_queue
+              @timer.cancel
+              synchronize { @running = false }
+              debug "Fully received #{file_path}"
+              if block
+                block.call({action: :done, size: hdrs['file_size'], 
+                  path: file_path, mime_type: hdrs['mime_type'], 
+                  received: chunk_count})
+              end           
+            end
+          end
+        end
+        
+        @timer = OmfCommon.eventloop.every(WAIT_BEFORE_REQUESTING) do
+          from = to = nil
+          synchronize do
+            #puts "RUNNING: #{@running}"
+            #break unless @running
+            if @received_chunks
+              @received_chunks = false
+              @nothing_received = 0
+              break # ok there is still action
+            else
+              # nothing happened, so let's ask for something
+              if (@nothing_received += WAIT_BEFORE_REQUESTING) >= WAIT_BEFORE_REQUESTING_EVERYTHING
+                # something stuck here, let's re-ask for everything
+                from = 0
+                @nothing_received = 0
+              else
+                # ask_for is the set of chunks we are still missing but haven't asked for              
+                ask_for = @outstanding_chunks - @requested_chunks
+                break if ask_for.empty? # ok, someone already asked, so better wait
+                
+                # Ask for a single span of consecutive chunks 
+                aa = ask_for.to_a.sort
+                from = to = aa[0]
+                aa.each.with_index do |e, i| 
+                  break unless (from + i == e) 
+                  to = e
+                  @requested_chunks << e
+                end
+              end
+              
+            end
+          end
+          if from
+            headers = {request_from: from}
+            headers[:request_to] = to if to  # if nil, ask for everything
+            @control_exchange.publish(nil, {headers: headers})
+          end
+        end 
+        
+      end
+    end
+    
+  end
+end