omf_rc 6.0.0.pre.2

data/lib/omf_rc/resource_proxy/abstract_resource.rb

@@ -0,0 +1,257 @@
+require 'omf_common'
+require 'omf_rc/deferred_process'
+require 'omf_rc/message_process_error'
+require 'securerandom'
+require 'hashie'
+
+class OmfRc::ResourceProxy::AbstractResource
+  # Time to wait before shutting down event loop, wait for deleting pubsub nodes
+  DISCONNECT_WAIT = 5
+  # Time to wait before releasing resource, wait for deleting pubsub nodes
+  RELEASE_WAIT = 5
+  attr_accessor :uid, :hrn, :type, :comm
+  attr_reader :opts, :children, :host
+
+  # Initialisation
+  #
+  # @param [Symbol] type resource proxy type
+  # @param [Hash] opts options to be initialised
+  # @option opts [String] :uid Unique identifier
+  # @option opts [String] :hrn Human readable name
+  # @option opts [String] :pubsub_host pubsub server subdomain, default to 'pubsub'
+  # @option opts [String] :dsl Which pubsub DSL to be used for pubsub communication
+  # @option opts [String] :user pubsub user id
+  # @option opts [String] :password pubsub user password
+  # @option opts [String] :server pubsub server domain
+  # @param [Comm] comm communicator instance, pass this to new resource proxy instance if want to use a common communicator instance.
+  def initialize(type, opts = nil, comm = nil)
+    @opts = Hashie::Mash.new(opts)
+    @type = type
+    @uid = @opts.uid || SecureRandom.uuid
+    @hrn = @opts.hrn
+    @children ||= []
+    @host = nil
+
+    @comm = comm || OmfCommon::Comm.new(@opts.dsl)
+    # Fire when connection to pubsub server established
+    @comm.when_ready do
+      logger.info "CONNECTED: #{@comm.jid.inspect}"
+      @host = "#{@opts.pubsub_host}.#{@comm.jid.domain}"
+
+      # Once connection established, create a pubsub node, then subscribe to it
+      @comm.create_node(uid, host) do |s|
+        # Creating node failed, no point to continue; clean up and disconnect
+        # Otherwise go subscribe to this pubsub node
+        s.error? ? disconnect : @comm.subscribe(uid, host)
+      end
+    end
+
+    # Fire when message published
+    @comm.node_event do |e|
+      e.items.each do |item|
+        process_omf_message(item.payload, e.node)
+      end
+    end
+
+    # Generic pubsub event
+    @comm.pubsub_event do |e|
+      logger.debug "PUBSUB GENERIC EVENT: #{e}"
+    end
+  end
+
+  # Connect to pubsub server
+  def connect
+    @comm.connect(opts.user, opts.password, opts.server)
+  end
+
+  # Try to clean up pubsub nodes, and wait for DISCONNECT_WAIT seconds, then shutdown event machine loop
+  def disconnect
+    @comm.pubsub.affiliations(host) do |a|
+      my_pubsub_nodes = a[:owner] ? a[:owner].size : 0
+      if my_pubsub_nodes > 0
+        logger.info "Cleaning #{my_pubsub_nodes} pubsub node(s)"
+        a[:owner].each { |node| @comm.delete_node(node, host) }
+      else
+        logger.info "Disconnecting now"
+        @comm.disconnect(host)
+      end
+    end
+    logger.info "Disconnecting in #{DISCONNECT_WAIT} seconds"
+    EM.add_timer(DISCONNECT_WAIT) do
+      @comm.disconnect(host)
+    end
+  end
+
+  # Create a new resource in the context of this resource. This resource becomes parent, and newly created resource becomes child
+  #
+  # @param (see #initialize)
+  def create(type, opts = nil)
+    new_resource = OmfRc::ResourceFactory.new(type.to_sym, opts, @comm)
+    children << new_resource
+    new_resource
+  end
+
+  # Release a resource
+  #
+  def release
+    pubsub_nodes_left = []
+    children.each do |c|
+      c.before_release if c.respond_to? :before_release
+      pubsub_nodes_left << c.uid
+      c.freeze
+    end.clear
+    before_release if respond_to? :before_release
+    freeze
+    pubsub_nodes_left
+  end
+
+  # Return a list of all properties can be requested and configured
+  #
+  def request_available_properties
+    Hashie::Mash.new(request: [], configure: []).tap do |mash|
+      methods.each do |m|
+        mash[$1] << $2.to_sym if m =~ /(request|configure)_(.+)/ && $2 != "available_properties"
+      end
+    end
+  end
+
+  # Make uid accessible through pubsub interface
+  def request_uid
+    uid
+  end
+
+  # Make hrn accessible through pubsub interface
+  def request_hrn
+    hrn
+  end
+
+  # Make hrn configurable through pubsub interface
+  def configure_hrn(hrn)
+    @hrn = hrn
+  end
+
+  # Request child resources
+  # @return [Mash] child resource mash with uid and hrn
+  def request_child_resources
+    Hashie::Mash.new.tap do |mash|
+      children.each do |c|
+        mash[c.uid] ||= c.hrn
+      end
+    end
+  end
+
+  # Parse omf message and execute as instructed by the message
+  #
+  def process_omf_message(pubsub_item_payload, node)
+    dp = OmfRc::DeferredProcess.new
+
+    dp.callback do |end_result|
+      if end_result
+        case end_result[:operation]
+        when :create
+          new_uid = end_result[:result]
+          @comm.create_node(new_uid, host) do
+            @comm.subscribe(new_uid, host) do
+              inform_msg = OmfCommon::Message.inform(end_result[:context_id], 'CREATED') do |i|
+                i.element('resource_id', new_uid)
+                i.element('resource_address', new_uid)
+              end.sign
+              @comm.publish(end_result[:inform_to], inform_msg, host)
+            end
+          end
+        when :request
+          inform_msg = OmfCommon::Message.inform(end_result[:context_id], 'STATUS') do |i|
+            end_result[:result].each_pair do |k, v|
+              i.property(k) { |p| p.element('current', v) }
+            end
+          end.sign
+          @comm.publish(end_result[:inform_to], inform_msg, host)
+
+        when :configure
+          inform_msg = OmfCommon::Message.inform(end_result[:context_id], 'STATUS') do |i|
+            end_result[:result].each_pair do |k, v|
+              i.property(k) { |p| p.element('current', v) }
+            end
+          end.sign
+          @comm.publish(end_result[:inform_to], inform_msg, host)
+        when :release
+          inform_msg = OmfCommon::Message.inform(end_result[:context_id], 'RELEASED') do |i|
+            i.element('resource_id', end_result[:inform_to])
+          end
+
+          end_result[:result].each do |n|
+            @comm.delete_node(n, host)
+          end
+
+          EM.add_timer(RELEASE_WAIT) do
+            @comm.publish(end_result[:inform_to], inform_msg, host)
+          end
+        end
+      end
+    end
+
+    dp.errback do |e|
+      inform_msg = OmfCommon::Message.inform(e.context_id, 'FAILED') do |i|
+        i.element("error_message", e.message)
+      end.sign
+      @comm.publish(e.inform_to, inform_msg, host)
+    end
+
+    dp.fire do
+      message = OmfCommon::Message.parse(pubsub_item_payload)
+      # Get the context id, which will be included when informing
+      context_id = message.read_content("context_id")
+
+      obj = node == uid ? self : children.find { |v| v.uid == node }
+
+      begin
+        raise "Resource disappeard #{node}" if obj.nil?
+
+        case message.operation
+        when :create
+          create_opts = opts.dup
+          create_opts.uid = nil
+          result = obj.create(message.read_property(:type), create_opts)
+          message.read_element("//property").each do |p|
+            unless p.attr('key') == 'type'
+              method_name =  "configure_#{p.attr('key')}"
+              result.send(method_name, p.content) if result.respond_to? method_name
+            end
+          end
+          { operation: :create, result: result.uid, context_id: context_id, inform_to: uid }
+        when :request
+          result = Hashie::Mash.new.tap do |mash|
+            message.read_element("//property").each do |p|
+              method_name =  "request_#{p.attr('key')}"
+              if obj.respond_to? method_name
+                mash[p.attr('key')] ||= obj.send(method_name)
+              end
+            end
+          end
+          { operation: :request, result: result, context_id: context_id, inform_to: obj.uid }
+        when :configure
+          result = Hashie::Mash.new.tap do |mash|
+            message.read_element("//property").each do |p|
+              method_name =  "configure_#{p.attr('key')}"
+              if obj.respond_to? method_name
+                mash[p.attr('key')] ||= obj.send(method_name, p.content)
+              end
+            end
+          end
+          { operation: :configure, result: result, context_id: context_id, inform_to: obj.uid }
+        when :release
+          { operation: :release, result: obj.release, context_id: context_id, inform_to: obj.uid }
+        when :inform
+          # We really don't care about inform messages which created from here
+          nil
+        else
+          raise "Unknown OMF operation #{message.operation}"
+        end
+      rescue => e
+        logger.error e.message
+        logger.error e.backtrace.join("\n")
+        raise OmfRc::MessageProcessError.new(context_id, obj.uid, e.message)
+      end
+    end
+  end
+end

data/lib/omf_rc/resource_proxy/interface.rb

@@ -0,0 +1,48 @@
+require 'hashie'
+
+module OmfRc::ResourceProxy::Interface
+  include OmfRc::ResourceProxyDSL
+
+  register_proxy :interface
+
+  IPTABLES = 'iptables'
+  ROUTE = 'route'
+
+  def configure_property(property, value)
+    case property
+    when /^forwarding$/
+      OmfRc::Cmd.exec("echo #{value ? '1' : '0'} > /proc/sys/net/ipv4/conf/#{uid}/forwarding")
+    when /^gateway$/
+      # FIXME Not sure about this one, hard coded everything?
+      OmfRc::Cmd.exec("route del default dev eth1; route add default gw #{value}; route add 224.10.10.6 dev eth1")
+    when /^route$/
+      value = Hashie::Mash.new(value)
+      arguments = %w(net gw mask).map {|v| "-#{v} #{value.send(v)}" if value.send(v)}.join(' ')
+      OmfRc::Cmd.exec("#{ROUTE} #{value.op} #{arguments} dev #{uid}")
+    when /^filter$/
+      operation = case value.op
+                  when /^add$/
+                    '-A'
+                  when /^del$/
+                    '-D'
+                  when /^clear$/
+                    '-F'
+                  end
+      chain = "#{value.chain.upcase} -i #{uid}" if value.chain
+      protocol = case value.proto
+                 when /^(tcp|udp)$/
+                   [ ("-p #{value.proto}"),
+                     ("-s #{value.src}" if value.src),
+                     ("-d #{value.dst}" if value.dst),
+                     ("--sport #{value.sport}" if value.sport),
+                     ("--dport #{value.dport}" if value.dport) ].join(' ')
+                 when /^mac$/
+                   "-m mac --mac-source #{value.src}"
+                 end
+      target = "#{value.target.upcase}" if value.target
+      OmfRc::Cmd.exec("#{IPTABLES} #{operation} #{chain} #{protocol} #{chain}")
+    else
+      super
+    end
+  end
+end

data/lib/omf_rc/resource_proxy/mock.rb

@@ -0,0 +1,16 @@
+module OmfRc::ResourceProxy::Mock
+  include OmfRc::ResourceProxyDSL
+
+  register_proxy :mock
+
+  utility :mock
+
+  register_hook :before_ready do |resource|
+    logger.info "#{resource.uid} is now ready"
+  end
+
+  register_hook :before_release do |resource|
+    logger.info "#{resource.uid} is to be released"
+  end
+end
+

data/lib/omf_rc/resource_proxy/node.rb

@@ -0,0 +1,17 @@
+module OmfRc::ResourceProxy::Node
+  include OmfRc::ResourceProxyDSL
+
+  register_proxy :node
+
+  register_hook :before_ready do |resource|
+    logger.info "#{resource.uid} is now ready"
+  end
+
+  register_hook :before_release do |resource|
+    logger.info "#{resource.uid} is now released"
+  end
+
+  register_request :proxies do
+    OmfRc::ResourceFactory.proxy_list
+  end
+end

data/lib/omf_rc/resource_proxy/wifi.rb

@@ -0,0 +1,8 @@
+module OmfRc::ResourceProxy::Wifi
+  include OmfRc::ResourceProxyDSL
+
+  register_proxy :wifi
+
+  #utility :mod
+  utility :iw
+end

data/lib/omf_rc/resource_proxy_dsl.rb

@@ -0,0 +1,170 @@
+# DSL contains some helper methods to ease the process defining resource proxies
+#
+module OmfRc::ResourceProxyDSL
+  PROXY_DIR = "omf_rc/resource_proxy"
+  UTIL_DIR = "omf_rc/util"
+
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+
+  # Methods defined here will be available in resource/utility definition files
+  #
+  module ClassMethods
+    # Register a named proxy entry with factory class, normally this should be done in the proxy module
+    #
+    # @param [Symbol] name of the resource proxy
+    # @example suppose we define a module for wifi
+    #
+    #   module OmfRc::ResourceProxy::Wifi
+    #     include OmfRc::ResourceProxyDSL
+    #
+    #     # Let the factory know it is available
+    #     register_proxy :wifi
+    #   end
+    #
+    def register_proxy(name)
+      name = name.to_sym
+      OmfRc::ResourceFactory.register_proxy(name)
+    end
+
+    # Register some hooks which can be called at certain stage of the operation
+    #
+    # Currently the system supports two hooks:
+    #
+    # * before_ready, called when a resource created, before creating an associated pubsub node
+    # * before_release, called before a resource released
+    #
+    # @param [Symbol] name hook name. :before_create or :before_release
+    # @yieldparam [AbstractResource] resource pass the current resource object to the block
+    # @example
+    #
+    #   module OmfRc::ResourceProxy::Node
+    #    include OmfRc::ResourceProxyDSL
+    #
+    #    register_proxy :node
+    #
+    #    register_hook :before_ready do |resource|
+    #      logger.info "#{resource.uid} is now ready"
+    #    end
+    #
+    #    register_hook :before_release do |resource|
+    #      logger.info "#{resource.uid} is now released"
+    #    end
+    #   end
+    def register_hook(name, &register_block)
+      define_method(name) do
+        register_block.call(self) if register_block
+      end
+    end
+
+    # Include the utility by providing a name
+    #
+    # The utility file can be added to the default utility directory UTIL_DIR, or defined inline.
+    #
+    # @param [Symbol] name of the utility
+    # @example assume we have a module called iw.rb in the omf_rc/util directory, providing a module named OmfRc::Util::Iw with functionalities based on iw cli
+    #
+    #   module OmfRc::ResourceProxy::Wifi
+    #     include OmfRc::ResourceProxyDSL
+    #
+    #     # Simply include this util module
+    #     utility :iw
+    #   end
+    def utility(name)
+      name = name.to_s
+      begin
+        # In case of module defined inline
+        include "OmfRc::Util::#{name.camelcase}".constant
+      rescue NameError
+        begin
+          # Then we try to require the file and include the module
+          require "#{UTIL_DIR}/#{name}"
+          include "OmfRc::Util::#{name.camelcase}".constant
+        rescue LoadError => le
+          logger.error le.message
+        rescue NameError => ne
+          logger.error ne.message
+        end
+      end
+    end
+
+    # Register a named utility entry with factory class, normally this should be done in the utility module
+    #
+    # @param [Symbol] name of the resource proxy
+    # @example suppose we define a utility for iw command interaction
+    #
+    #   module OmfRc::Util::Iw
+    #     include OmfRc::ResourceProxyDSL
+    #
+    #     # Let the factory know it is available
+    #     register_utility :iw
+    #   end
+    def register_utility(name)
+      name = name.to_sym
+      OmfRc::ResourceFactory.register_utility(name)
+    end
+
+    # Register a configurable property
+    #
+    # @param [Symbol] name of the property
+    # @yieldparam [AbstractResource] resource pass the current resource object to the block
+    # @yieldparam [Object] value pass the value to be configured
+    # @example suppose we define a utility for iw command interaction
+    #
+    #   module OmfRc::Util::Iw
+    #     include OmfRc::ResourceProxyDSL
+    #
+    #     register_configure :freq do |resource, value|
+    #       Command.execute("iw #{resource.hrn} set freq #{value}")
+    #     end
+    #
+    #     # or use iterator to define multiple properties
+    #     %w(freq channel type).each do |p|
+    #       register_configure p do |resource, value|
+    #         Command.execute("iw #{resource.hrn} set freq #{value}")
+    #       end
+    #     end
+    #
+    #     # or we can try to parse iw's help page to extract valid properties and then automatically register them
+    #     Command.execute("iw help").chomp.gsub(/^\t/, '').split("\n").map {|v| v.match(/[phy|dev] <.+> set (\w+) .*/) && $1 }.compact.uniq.each do |p|
+    #       register_configure p do |resource, value|
+    #         Command.execute("iw #{resource.hrn} set #{p} #{value}")
+    #       end
+    #     end
+    #   end
+    #
+    # @see OmfCommon::Command.execute
+    #
+    def register_configure(name, &register_block)
+      define_method("configure_#{name.to_s}") do |*args, &block|
+      register_block.call(self, *args, block) if register_block
+      end
+    end
+
+    # Register a property that could be requested
+    #
+    # @param (see #register_configure)
+    # @yieldparam [AbstractResource] resource pass the current resource object to the block
+    # @example suppose we define a utility for iw command interaction
+    #   module OmfRc::Util::Iw
+    #     include OmfRc::ResourceProxyDSL
+    #
+    #     register_request :freq do |resource|
+    #       Command.execute("iw #{resource.hrn} link").match(/^(freq):\W*(.+)$/) && $2
+    #     end
+    #
+    #     # or we can grab everything from output of iw link command and return as a hash(mash)
+    #     Command.execute("iw #{resource.hrn} link").chomp.gsub(/^\t/, '').split("\n").drop(1).each do |v|
+    #       v.match(/^(.+):\W*(.+)$/).tap do |m|
+    #         m && known_properties[m[1].downcase.gsub(/\W+/, '_')] = m[2].gsub(/^\W+/, '')
+    #       end
+    #     end
+    #   end
+    def register_request(name, &register_block)
+      define_method("request_#{name.to_s}") do |*args|
+        register_block.call(self) if register_block
+      end
+    end
+  end
+end

data/lib/omf_rc/util/iw.rb

@@ -0,0 +1,22 @@
+require 'hashie'
+module OmfRc::Util::Iw
+  include OmfRc::ResourceProxyDSL
+
+  OmfCommon::Command.execute("iw help").chomp.gsub(/^\t/, '').split("\n").map {|v| v.match(/[phy|dev] <.+> set (\w+) .*/) && $1 }.compact.uniq.each do |p|
+    register_configure p do |resource, value|
+      OmfCommon::Command.execute("#{IW_CMD} #{resource.hrn} set #{p} #{value}")
+    end
+  end
+
+  register_request :link do |resource|
+    known_properties = Hashie::Mash.new
+
+    OmfCommon::Command.execute("iw #{resource.hrn} link").chomp.gsub(/^\t/, '').split("\n").drop(1).each do |v|
+      v.match(/^(.+):\W*(.+)$/).tap do |m|
+        m && known_properties[m[1].downcase.gsub(/\W+/, '_')] = m[2].gsub(/^\W+/, '')
+      end
+    end
+
+    known_properties
+  end
+end

data/lib/omf_rc/util/mock.rb

@@ -0,0 +1,27 @@
+module OmfRc::Util::Mock
+  include OmfRc::ResourceProxyDSL
+
+  register_utility :mock
+
+  register_request :nothing do |resource|
+    resource.uid
+  end
+
+  register_configure :nothing
+
+  register_configure :hrn do |resource, hrn|
+    resource.hrn = hrn
+  end
+
+  register_request :resource_proxy_list do
+    OmfRc::ResourceFactory.proxy_list
+  end
+
+  register_request :resource_utility_list do
+    OmfRc::ResourceFactory.utility_list
+  end
+
+  register_request :kernel_version do
+    OmfCommon::Command.execute("uname -r")
+  end
+end

data/lib/omf_rc/util/mod.rb

@@ -0,0 +1,23 @@
+module OmfRc::Util::Mod
+
+  LSMOD_CMD = 'lsmod'
+  MODPROBE_CMD = 'modprobe'
+
+  def request_property(property)
+    case property
+    when /^(mod|driver)_(.+)$/
+      OmfRc::Cmd.exec(LSMOD).match(/^#{$+}( )+/) ? true : false
+    else
+      super
+    end
+  end
+
+  def configure_property(property, value)
+    case property
+    when /^(mod|driver)_(.+)$/
+      OmfRc::Cmd.exec("#{MODPROBE} #{$+}")
+    else
+      super
+    end
+  end
+end

data/lib/omf_rc/util/package.rb

@@ -0,0 +1,17 @@
+module OmfRc::ResourceProxy::Package
+  include OmfRc::ResourceProxyDSL
+
+  register_utility :package
+
+  register_request :package_version do |resource|
+    OmfCommon::Command.execute("dpkg -l #{resource.hrn} | awk 'END { print $3 }'")
+  end
+
+  register_configure :install_package do |resource|
+    OmfCommon::Command.execute("apt-get install -y --reinstall --allow-unauthenticated -qq #{resource.hrn}")
+  end
+
+  register_configure :remove_package do |resource|
+    OmfCommon::Command.execute("apt-get purge -y -qq #{resource.hrn}")
+  end
+end

data/lib/omf_rc/version.rb

@@ -0,0 +1,3 @@
+module OmfRc
+  VERSION = "6.0.0.pre.2"
+end

data/lib/omf_rc.rb

@@ -0,0 +1,6 @@
+require "omf_rc/version"
+
+module OmfRc
+  module ResourceProxy; end
+  module Util; end
+end

data/omf_rc.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "omf_rc/version"
+
+Gem::Specification.new do |s|
+  s.name        = "omf_rc"
+  s.version     = OmfRc::VERSION
+  s.authors     = ["NICTA"]
+  s.email       = ["omf-user@lists.nicta.com.au"]
+  s.homepage    = "https://www.mytestbed.net"
+  s.summary     = %q{OMF resource controller}
+  s.description = %q{Resource controller of OMF, a generic framework for controlling and managing networking testbeds.}
+
+  s.rubyforge_project = "omf_rc"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency "minitest", "~> 2.11.3"
+  s.add_development_dependency "em-minitest-spec", "~> 1.1.1"
+  s.add_runtime_dependency "omf_common", "~> 6.0.0.pre"
+end

data/test/omf_rc/deferred_process_spec.rb

@@ -0,0 +1,39 @@
+require 'test_helper'
+require 'em/minitest/spec'
+require 'omf_rc/deferred_process'
+
+describe OmfRc::DeferredProcess do
+  describe "when use to deferred process to execute code asynchronously" do
+    include EM::MiniTest::Spec
+
+    it "must execute and return result eventually, inside the EM loop" do
+      dp = OmfRc::DeferredProcess.new
+
+      dp.callback do |result|
+        result.must_equal "hello world"
+        done!
+      end
+
+      dp.fire do
+        "hello world"
+      end
+
+      wait!
+    end
+
+    it "must capture errors properly inside the EM loop" do
+      dp = OmfRc::DeferredProcess.new
+
+      dp.errback do |exception|
+        exception.must_be_kind_of StandardError
+        done!
+      end
+
+      dp.fire do
+        raise StandardError
+      end
+
+      wait!
+    end
+  end
+end