isono 0.1.0

data/lib/isono/models/resource_instance.rb

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+
+module Isono
+  module Models
+    class ResourceInstance < Sequel::Model
+      include Logger
+      plugin :schema
+      plugin :hook_class_methods
+      plugin :serialization
+
+      set_schema {
+        primary_key :id, :type => Integer, :auto_increment=>true, :unsigned=>true
+        column :agent_id, :varchar, :size=>80, :null=>false
+        column :uuid,   :varchar, :size=>50, :null=>false
+        column :resource_type, :varchar, :size=>50, :null=>false
+        column :instance_data, :text
+        column :status, :varchar, :size=>10
+        
+        column :created_at, :datetime, :null=>false
+        column :updated_at, :datetime, :null=>false
+        index [:uuid], {:unique=>true}
+      }
+
+      serialize_attributes :yaml, :instance_data
+      
+      before_create(:set_created_at) do
+        self.updated_at = self.created_at = Time.now
+      end
+      before_update(:set_updated_at) do
+        self.updated_at = Time.now
+      end
+
+    end
+  end
+end

data/lib/isono/node.rb

@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+
+require 'isono'
+require 'digest/sha1'
+
+module Isono
+  # A node instance which joins AMQP network.
+  # 
+  class Node
+    include Logger
+    include AmqpClient
+    include EventObservable
+
+    def self.inherited(klass)
+      klass.class_eval {
+        include Logger
+      }
+    end
+
+    def self.instance
+      @instance
+    end
+    
+    def self.start(manifest, opts)
+      new_instance = proc {
+        stop
+        @instance = new(manifest)
+        @instance.connect(opts[:amqp_server_uri], *opts)
+      }
+      
+      if EventMachine.reactor_running?
+        EventMachine.schedule {
+          new_instance.call
+        }
+      else
+        EventMachine.run new_instance
+      end
+    end
+    
+    def self.stop(&blk)
+      return if @instance.nil?
+      EventMachine.schedule {
+        begin
+          if @instance.connected?
+            @instance.close {
+              blk.call(@instance) if blk
+            }
+          end
+        ensure
+          @instance = nil
+        end
+      }
+    end
+    
+    attr_reader :manifest, :boot_token, :value_objects
+    
+    def initialize(manifest)
+      initialize_event_observable
+      raise ArgumentError unless manifest.is_a? Manifest
+      @manifest = manifest
+      @boot_token = Digest::SHA1.hexdigest(Process.pid.to_s)[0,5]
+      @value_objects = {}
+    end
+
+    def node_id
+      manifest.node_id
+    end
+
+    def on_connect
+      raise "node_id is not set" if node_id.nil?
+
+      #amq.prefetch(1)
+      identity_queue(node_id)
+      init_modules
+
+      fire_event(:node_ready, {:node_id=> self.node_id})
+      logger.info("Started : AMQP Server=#{amqp_server_uri.to_s}, ID=#{node_id}, token=#{boot_token}")
+    end
+
+    def on_close
+      term_modules
+    end
+
+    private
+    
+    def init_modules
+      manifest.node_modules.each { |modclass, *args|
+        if !@value_objects.has_key?(modclass)
+          @value_objects[modclass] = vo = ValueObject.new(self, modclass)
+
+          if modclass.initialize_hook.is_a?(Proc)
+            vo.instance_eval(&modclass.initialize_hook)
+          end
+
+          logger.debug("Initialized #{modclass.to_s}")
+        end
+      }
+    end
+
+    def term_modules
+      manifest.node_modules.reverse.each { |modclass, *args|
+        vo = @value_objects[modclass]
+        if vo && modclass.terminate_hook.is_a?(Proc)
+          vo.instance_eval(&modclass.terminate_hook)
+        end
+        logger.info("Terminated #{modclass.to_s}")
+      }
+    end
+
+    class ValueObject
+      module DelegateMethods
+        def self_class
+          @_tmp[:modclass]
+        end
+        
+        def myinstance
+          @_tmp[:myself] ||= self_class.new(self.node)
+        end
+        
+        def node
+          @_tmp[:node]
+        end
+        
+        def amq
+          self.node.amq
+        end
+        
+        def manifest
+          self.node.manifest
+        end
+        
+        def config_section
+          manifest.config.send(@_tmp[:modclass].instance_variable_get(:@config_section_name))
+        end
+
+        def logger
+          @_tmp[:modclass].logger
+        end
+      end
+
+      def initialize(node, modclass)
+        @_tmp = {:node=>node, :modclass=>modclass}
+      end
+
+      include DelegateMethods
+      
+      def copy_instance_variables(vo)
+        self.instance_variables.each { |n|
+          next if n == '@_tmp'
+          vo.instance_variable_set(n, self.instance_variable_get(n))
+        }
+        vo
+      end
+
+    end
+
+  end
+end 

data/lib/isono/node_modules/base.rb

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+
+module Isono
+  module NodeModules
+    class Base
+
+      attr_reader :node
+
+      def initialize(node)
+        @node = node
+        
+        raise "Module initializer_hook is not run yet" if self.value_object.nil?
+        value_object.copy_instance_variables(self)
+      end
+      
+      # Delegate methods used in subclass frequently.
+      def manifest
+        node.manifest
+      end
+
+      def amq
+        node.amq
+      end
+
+      def value_object
+        node.value_objects[self.class]
+      end
+
+      # shortcut method to lookup configuration section only which
+      # belongs to this class.
+      def config_section
+        node.manifest.config.send(self.class.instance_variable_get(:@config_section_name))
+      end
+      
+      module ClassMethods
+        def initialize_hook(&blk)
+          @initialize_hook = blk if blk
+          @initialize_hook
+        end
+        
+        def terminate_hook(&blk)
+          @terminate_hook = blk if blk
+          @terminate_hook
+        end
+
+        def config_section(name=nil, &blk)
+          @config_section_name = name unless name.nil? 
+          @config_section_builder = blk
+        end        
+      end
+      
+      protected
+      def self.inherited(klass)
+        super
+        klass.extend ClassMethods
+        klass.class_eval {
+          # set the default config section name from its class name.
+          # can be overwritten later.
+          @config_section_name = Util.snake_case(self.to_s.split('::').last)
+        }
+      end
+      
+    end
+  end
+end

data/lib/isono/node_modules/data_store.rb

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+
+require 'sequel'
+
+module Isono
+  module NodeModules
+    class DataStore < Base
+      include Logger
+
+      config_section do
+        desc "Destination database server to be connected."
+        database_dsn ''
+      end
+
+
+      def self.create_instance(node)
+        @instance = self.new(node)
+      end
+
+      def self.pass(&blk)
+        @instance.db_writer_thread.pass {
+          @instance.db.transaction {
+            blk.call
+          }
+        }
+      end
+
+      def self.barrier(&blk)
+        @instance.db_writer_thread.barrier {
+          @instance.db.transaction {
+            blk.call
+          }
+        }
+      end
+
+      attr_reader :db_writer_thread, :db
+
+      initialize_hook do
+        @db_writer_thread = ThreadPool.new(1)
+
+        @db_writer_thread.barrier {
+          #@db = Sequel.connect(config_section.database_dsn, {:logger=>logger})
+          @db = Sequel.connect(config_section.database_dsn)
+          logger.debug("connected to the database: #{config_section.database_dsn}, #{@db}")
+        }
+
+        DataStore.create_instance(node)
+      end
+      
+      terminate_hook do
+        @db_writer_thread.shutdown
+        @db.disconnect
+      end
+
+    end
+  end
+end

data/lib/isono/node_modules/event_channel.rb

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+
+module Isono
+  module NodeModules
+    # Setup mq based event dispatch channels.
+    # It prepares a single point topic exchange which receives all the
+    # events from publishers. The event consumers setup queues
+    # respectively to bound to the exchange. The most of queues
+    # have the key string which filters the event. The event publisher
+    # puts the key when it sends each new event.
+    # 
+    # The key consists of two parts with dot separated:
+    # event type and the event sender (ie. "event1.sender1").
+    # The event publisher can put any format of data into the message
+    # body part.
+    # 
+    # These rules make the message queue broker to work as the event
+    # dispacher.
+    class EventChannel < Base
+      AMQP_EXCHANGE='isono.event'
+      
+      initialize_hook do
+        amq.topic(AMQP_EXCHANGE, {:auto_delete=>false})
+      end
+
+      # @example
+      #   publish('ev/event_name', :args=>[1, 2, 3])
+      def publish(evname, opts={})
+        opts = {:args=>[], :sender=>manifest.node_id}.merge(opts)
+        
+        body = {
+          :event => evname,
+          :published_at=> Time.now,
+          :sender  => opts[:sender],
+          :origin_node  => manifest.node_id,
+          :args => opts[:args]
+        }
+        
+        EventMachine.schedule {
+          amq.topic(AMQP_EXCHANGE).publish(Serializer.instance.marshal(body),
+                                           {:key=>"#{evname}.#{opts[:sender]}"}
+                                           )
+        }
+      end
+      
+      def subscribe(evname, sender, receiver_id=node.node_id, &blk)
+        amq.queue("#{evname}-#{receiver_id}", {:exclusive=>true}).bind(
+                     AMQP_EXCHANGE, :key=>"#{evname}.#{sender}"
+                     ).subscribe { |data|
+          data = Serializer.instance.unmarshal(data)
+          case blk.arity
+          when 2
+            m = data.delete(:args)
+            blk.call(data, m)
+          when 1
+            blk.call(data[:args])
+          else
+            blk.call
+          end
+        }
+      end
+      
+      def unsubscribe(evname, receiver_id=node.node_id)
+        EventMachine.schedule {
+          q = amq.queue("#{evname}-#{receiver_id}")
+          q.unsubscribe
+        }
+      end
+
+    end
+  end
+end

data/lib/isono/node_modules/event_logger.rb

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+
+module Isono
+  module NodeModules
+    class EventLogger < Base
+
+      initialize_hook do
+        amq.topic(EventChannel::AMQP_EXCHANGE, {:auto_delete=>true})
+
+        amq.queue("event-logger.#{manifest.node_id}", {:exclusive=>true}).bind(
+                    EventChannel::AMQP_EXCHANGE, {:key=>'*.*'}).subscribe { |data|
+          data = Serializer.instance.unmarshal(data)
+          next unless EventLogger.filter_event(data)
+          
+          DataStore.pass {
+            Models::EventLog.create(:event=>data[:event],
+                                    :sender=>data[:sender],
+                                    :message=>data[:message].inspect,
+                                    :publised_at=>data[:published_at])
+          }
+        }
+      end
+
+      terminate_hook do
+        amq.queue("event-logger.#{manifest.node_id}").delete
+      end
+
+      private
+      def self.filter_event(data)
+        case data[:event]
+        when 'node_collector/pong'
+          return false
+        end
+        return true
+      end
+      
+    end
+  end
+end

data/lib/isono/node_modules/job_channel.rb

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+
+module Isono
+  module NodeModules
+    class JobChannel < Base
+
+      # Send a new job request to the endpoint and get back job ID.
+      #
+      # @param [String] endpoint endpoint name created by JobChannel#register_endpoint()
+      # @param [String] command command name in the endpoint.
+      # @param [any] *args arguments to run the job.
+      # @yieldparam [RpcChannel::RequestContext] modify RPC request before sending. The
+      #   method will stop the current thread if it does not exist.
+      # @return [Rack::Request,String] Request object if the block is given.
+      #
+      # @example call job endpoint 'endpoint1' and receive 
+      #   submit('endpoint1', 'command1', 1, 2, 3) #=> Job ID.
+      def submit(endpoint, command, *args, &blk)
+        cur_job_ctx = Thread.current[JobWorker::JOB_CTX_KEY]
+        req = rpc.request("job.#{endpoint}", command, *args) { |req|
+          req.request[:job_request_type]=:submit
+
+          # A job is working on this current thread if cur_job_ctx is
+          # not nil. Let the new job know the current job ID
+          # as its parent job ID.
+          if cur_job_ctx
+            req.request[:parent_job_id] = cur_job_ctx.job_id
+          end
+
+          blk ? blk.call(req) : req.synchronize
+        }
+
+        blk ? req : req.wait
+      end
+
+      # Send a new job request and wait until the job finished.
+      # The difference to submit() is that this method will stop the
+      # thread until the called job completed.
+      #
+      # @param [String] endpoint endpoint name created by JobChannel#register_endpoint()
+      # @param [String] command command name in the endpoint.
+      # @param [any] *args arguments to run the job.
+      # @yieldparam [RpcChannel::RequestContext] modify RPC request before sending. The
+      #   method will stop the current thread if it does not exist.
+      # 
+      # @example wait while the job running.
+      #   run('endpoint1', 'command1', 1, 2, 3) #=> result of 'endpoint1/command1'.
+      # @example receive progress messages from long running job.
+      #   run('endpoint1', 'command1', 1, 2, 3) do |req|
+      #      req.on_progress { |r|
+      #        puts r #=> show progress message
+      #      }
+      #   end
+      def run(endpoint, command, *args, &blk)
+        cur_job_ctx = Thread.current[JobWorker::JOB_CTX_KEY]
+        req = rpc.request("job.#{endpoint}", command, *args) { |req|
+          req.request[:job_request_type]=:run
+
+          # A job is working on this current thread if cur_job_ctx is
+          # not nil. Let the new job know the current job ID
+          # as its parent job ID.
+          if cur_job_ctx
+            req.request[:parent_job_id] = cur_job_ctx.job_id
+          end
+
+          blk ? blk.call(req) : req.synchronize
+        }
+
+        blk ? req : req.wait
+      end
+      
+      def cancel()
+      end
+      
+      def register_endpoint(endpoint, app)
+        rpc.register_endpoint("job.#{endpoint}", Rack::Job.new(app, JobWorker.new(node)))
+      end
+
+      private
+      def rpc
+        @rpc ||= RpcChannel.new(@node)
+      end
+      
+    end
+  end
+end