Sutto-marvin 0.4.0 → 0.8.0.0

data/lib/marvin/console.rb

@@ -5,7 +5,7 @@ module Marvin
     
     module BaseExtensions
       def parse(line)
-        Marvin::Settings.default_parser.parse(line)
+        Marvin::Settings.parser.parse(line)
       end
 
       def logger
@@ -13,35 +13,7 @@ module Marvin
       end
 
       def client
-        $client ||= Marvin::Settings.default_client.new(:port => 6667, :server => "irc.freenode.net")
-      end
-
-      class ServerMock < Marvin::IRC::Server::BaseConnection
-        def send_line(line)
-          puts ">> #{line}"
-        end
-        def kill_connection!
-          puts "Killing connection"
-        end
-
-        def get_peername
-          # Localhost, HTTP
-          "\034\036\000P\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001\000\000\000\000"
-        end
-
-        def host
-          "localhost"
-        end
-
-        def port
-          6667
-        end
-
-      end
-
-      def server(reset = false)
-        $server = ServerMock.new(:port => 6667, :host => "localhost") if $server.blank? || reset
-        return $server
+        $client ||= Marvin::Settings.client.new(:port => 6667, :server => "irc.freenode.net")
       end
 
       def user(reset = false)
@@ -66,7 +38,8 @@ module Marvin
     end
     
     def run
-      IRB.start(@file)
+      ARGV.replace []
+      IRB.start
     end
     
     def self.run

data/lib/marvin/core_commands.rb

@@ -1,20 +1,38 @@
 module Marvin
   class CoreCommands < CommandHandler
     
+    # Returns a hash of doccumented method names
+    def self.method_documentation
+      documented = Hash.new { |h,k| h[k] = [] }
+      @@method_descriptions.each_key do |klass| 
+        next unless klass.registered?
+        @@exposed_method_mapping[klass].each do |m|
+          desc = @@method_descriptions[klass][m]
+          documented[m.to_s] << desc if desc.present?
+        end
+      end
+      return documented
+    end
+    
+    def registered_and_exposed_handlers
+    end
+    
     exposes :help
     desc "Generates this usage statement"
-    def help(methods)
-      method_names     = exposed_method_names.map { |n| n.to_s }
-      documented_names = descriptions.keys.map { |k| k.to_s } & method_names 
-      if methods.empty?
-        width            = documented_names.map { |s| s.length }.max
-        say "Hello there, I know the following commands:"
-        documented_names.each { |name| say "#{name.ljust(width)} - #{descriptions[name.to_sym]}" }
-        say "As well as the following undescribed commands: #{(method_names - documented_names).sort.join(", ")}"
+    def help(method)
+      method        = method.strip
+      documentation = self.class.method_documentation
+      names         = documentation.keys.sort
+      if method.blank?
+        display_names = names.map { |n| exposed_name(n) }
+        width         = display_names.map { |d| d.length }.max
+        say "Hello there, I know the following documented commands:"
+        names.each_with_index do |name, index|
+          say "#{display_names[index].ljust(width)} - #{documentation[name].join("; ")}"
+        end
       else
-        m = methods.first
-        if documented_names.include? m.to_s
-          reply "#{m}: #{descriptions[m.to_sym]}"
+        if names.include? method
+          reply "#{exposed_name(method)} - #{documentation[method].join("; ")}"
         else
           reply "I'm sorry, I can't help with #{m} - it seems to be undocumented."
         end
@@ -24,7 +42,7 @@ module Marvin
     exposes :about
     desc "Displays the current marvin and ruby versions."
     def about(*args)
-      reply "Marvin v#{Marvin::VERSION::STRING} running on Ruby #{RUBY_VERSION} (#{RUBY_PLATFORM})"
+      reply "Marvin v#{Marvin::VERSION} running on Ruby #{RUBY_VERSION} (#{RUBY_PLATFORM})"
     end
     
   end

data/lib/marvin/distributed/client.rb

@@ -0,0 +1,225 @@
+require 'json'
+require 'digest/sha2'
+require 'eventmachine'
+require 'socket'
+
+module Marvin
+  module Distributed
+    class Client < Marvin::AbstractClient
+      
+      attr_accessor :em_connection, :remote_client_host, :remote_client_nick
+      
+      class RemoteClientProxy
+        is :loggable
+        
+        def initialize(conn, host_with_port, nickname)
+          @connection     = conn
+          @host_with_port = host_with_port
+          @nickname       = nickname
+        end
+        
+        def nickname
+          @nickname
+        end
+        
+        def host_with_port
+          @host_with_port
+        end
+        
+        def method_missing(name, *args)
+          logger.debug "Proxying #{name}(#{args.inspect[1..-2]}) to #{@host_with_port}"
+          @connection.send_message(:action, {
+            "action"      => name.to_s,
+            "arguments"   => args,
+            "client-host" => @host_with_port
+          })
+        end
+        
+      end
+      
+      class EMConnection < Marvin::Distributed::Protocol
+        
+        register_handler_method :event
+        register_handler_method :authentication_failed
+        register_handler_method :authenticated
+        register_handler_method :unauthorized
+        
+        cattr_accessor :stopping
+        self.stopping = false
+        
+        attr_accessor :client, :port, :connection_host, :connection_port, :configuration
+
+        def initialize(*args)
+          @configuration = args.last.is_a?(Marvin::Nash) ? args.pop : Marvin::Nash.new
+          super(*args)
+          @callbacks = {}
+          @client = Marvin::Distributed::Client.new(self)
+          @authenticated = false
+        end
+
+        def post_init
+          super
+          logger.info "Connected to distributed server"
+          if should_use_tls?
+            logger.info "Attempting to initialize tls"
+            start_tls
+          else
+            process_authentication
+          end
+        end
+        
+        def ssl_handshake_completed
+          logger.info "tls handshake completed"
+          process_authentication if should_use_tls?
+        end
+        
+        
+        def unbind
+          if self.stopping
+            logger.info "Stopping distributed client"
+          else
+            logger.info "Lost connection to distributed client - Scheduling reconnect"
+            EventMachine.add_timer(15) { EMConnection.connect(connection_host, connection_port, @configuration) }
+          end
+          super
+        end
+        
+        def process_authentication
+          if configuration.token?
+            logger.info "Attempting to authenticate..." 
+            send_message(:authenticate, {:token => configuration.token})
+          end
+        end
+        
+        def handle_event(options = {})
+          event       = options["event-name"]
+          client_host = options["client-host"]
+          client_nick = options["client-nick"]
+          options     = options["event-options"]
+          options     = {} unless options.is_a?(Hash)
+          return if event.blank?
+          begin
+            logger.debug "Handling #{event}"
+            @client.remote_client_host = client_host
+            @client.remote_client_nick = client_nick
+            @client.setup_handlers
+            @client.dispatch(event.to_sym, options)
+          rescue Exception => e
+            logger.warn "Got Exception - Forwarding to Remote"
+            Marvin::ExceptionTracker.log(e)
+            send_message(:exception, {
+              "name"      => e.class.name,
+              "message"   => e.message,
+              "backtrace" => e.backtrace
+            })
+          ensure
+            logger.debug "Sending completed message"
+            send_message(:completed)
+            @client.reset!
+          end
+        end
+        
+        def handle_unauthorized(options = {})
+          logger.warn "Attempted action when unauthorized. Stopping client."
+          Marvin::Distributed::Client.stop
+        end
+        
+        def handle_authenticated(options = {})
+          @authenticated = true
+          logger.info "Successfully authenticated with #{host_with_port}"
+        end
+        
+        def handle_authentication_failed(options = {})
+          logger.info "Authentication with #{host_with_port} failed. Stopping."
+          Marvin::Distributed::Client.stop
+        end
+        
+        def self.connect(host, port, config = Marvin::Nash.new)
+          logger.info "Attempting to connect to #{host}:#{port}"
+          EventMachine.connect(host, port, self, config) do |c|
+            c.connection_host = host
+            c.connection_port = port
+          end
+        end
+        
+        protected
+        
+        def options_for_callback(blk)
+          return {} if blk.blank?
+          cb_id = "callback-#{seld.object_id}-#{Time.now.to_f}"
+          count = 0
+          count += 1 while @callbacks.has_key?(Digest::SHA256.hexdigest("#{cb_id}-#{count}"))
+          final_id = Digest::SHA256.hexdigest("#{cb_id}-#{count}")
+          @callbacks[final_id] = blk
+          {"callback-id" => final_id}
+        end
+
+        def process_callback(hash)
+          if hash.is_a?(Hash) && hash.has_key?("callback-id")
+            callback = @callbacks.delete(hash["callback-id"])
+            callback.call(self, hash)
+          end
+        end
+
+        def host_with_port
+          @host_with_port ||= begin
+            port, ip = Socket.unpack_sockaddr_in(get_peername)
+            "#{ip}:#{port}"
+          end
+        end
+
+        def should_use_tls?
+          @using_tls ||= configuration.encrypted?
+        end
+
+      end
+   
+      def initialize(em_connection)
+        @em_connection = em_connection
+      end
+   
+      def remote_client
+        @remote_client ||= RemoteClientProxy.new(@em_connection, @remote_client_host, @remote_client_nick)
+      end
+      
+      def reset!
+        @remote_client = nil
+        @remote_client_nick = nil
+        @remote_client_host = nil
+        reset_handlers
+      end
+      
+      def setup_handlers
+        self.class.handlers.each { |h| h.client = remote_client if h.respond_to?(:client=) }
+      end
+      
+      def reset_handlers
+        self.class.handlers.each { |h| h.client = nil if h.respond_to?(:client=) }
+      end
+      
+      class << self
+        
+        def run
+          logger.info "Preparing to start distributed client"
+          EventMachine.kqueue
+          EventMachine.epoll
+          EventMachine.run do
+            opts = Marvin::Settings.distributed || Marvin::Nash.new
+            opts = opts.client || Marvin::Nash.new
+            host = opts.host  || "0.0.0.0"
+            port = (opts.port || 8943).to_i
+            EMConnection.connect(host, port, opts)
+          end
+        end
+        
+        def stop
+          logger.info "Stopping distributed client..."
+          EMConnection.stopping = true
+          EventMachine.stop_event_loop
+        end
+        
+      end
+      
+    end
+  end
+end

data/lib/marvin/distributed/handler.rb

@@ -0,0 +1,85 @@
+module Marvin
+  module Distributed
+    class Handler < Marvin::Base
+      
+      
+      EVENT_WHITELIST = [:incoming_message, :incoming_action]
+      QUEUE_PROCESSING_SPACING = 3
+      
+      attr_accessor :message_queue
+      
+      def initialize
+        super
+        @message_queue = []
+      end
+      
+      def handle(message, options)
+        return unless EVENT_WHITELIST.include?(message)
+        super(message, options)
+        dispatch(message, options)
+      end
+      
+      def dispatch(name, options, client = self.client)
+        return if client.blank?
+        server = Marvin::Distributed::Server.next
+        if server.blank?
+          logger.debug "Distributed handler is currently busy - adding to queue"
+          # TODO: Add to queued messages, wait
+          @message_queue << [name, options, client]
+          run! unless running?
+        else
+          server.dispatch(client, name, options)
+        end
+      rescue Exception => e
+        logger.warn "Error dispatching #{name}"
+        Marvin::ExceptionTracker.log(e)
+      end
+      
+      def process_queue
+        count = [@message_queue.size, Server.free_connections.size].min
+        logger.debug "Processing #{count} item(s) from the message queue"
+        count.times { |item| dispatch(*@message_queue.shift) }
+        if @message_queue.empty?
+          logger.debug "The message queue is now empty"
+        else
+          logger.debug "The message queue still has #{count} item(s)"
+        end
+        check_queue_progress
+      end
+      
+      def running?
+        @running_timer.present?
+      end
+      
+      def run!
+        @running_timer = EventMachine::PeriodicTimer.new(QUEUE_PROCESSING_SPACING) { process_queue }
+      end
+      
+      def check_queue_progress
+        if @message_queue.blank? && running?
+          @running_timer.cancel
+          @running_timer = nil
+        elsif @message_queue.present? && !running?
+          run!
+        end
+      end
+      
+      class << self
+        
+        def whitelist_event(name)
+          EVENT_WHITELIST << name.to_sym
+          EVENT_WHITELIST.uniq!
+        end
+        
+        def register!(*args)
+          # DO NOT register if this is  not a normal client.
+          return unless Marvin::Loader.client?
+          logger.info "Registering distributed handler on #{Marvin::Settings.client}"
+          super
+        end
+        
+      end
+      
+    end
+  end
+end

data/lib/marvin/distributed/protocol.rb

@@ -0,0 +1,88 @@
+module Marvin
+  module Distributed
+    class Protocol < EventMachine::Protocols::LineAndTextProtocol
+      is :loggable
+      
+      class_inheritable_accessor :handler_methods
+      self.handler_methods = {}
+      
+      attr_accessor :callbacks
+      
+      def receive_line(line)
+        line.strip!
+        logger.debug "<< #{line}"
+        response = JSON.parse(line)
+        handle_response(response)
+      rescue JSON::ParserError
+        logger.debug "JSON parsing error for #{line.inspect}"
+      rescue Exception => e
+        Marvin::ExceptionTracker.log(e)
+      end
+      
+      def send_message(name, arguments = {}, &callback)
+        logger.debug "Sending #{name.inspect} to #{self.host_with_port}"
+        payload = {
+          "message" => name.to_s,
+          "options" => arguments,
+          "sent-at" => Time.now
+        }
+        payload.merge!(options_for_callback(callback))
+        payload = JSON.dump(payload)
+        logger.debug ">> #{payload}"
+        send_data "#{payload}\n"
+      end
+      
+      def handle_response(response)
+        logger.debug "Handling response in distributed protocol (response => #{response.inspect})"
+        return unless response.is_a?(Hash) && response.has_key?("message")
+        options = response["options"] || {}
+        process_response_message(response["message"], options)
+      end
+      
+      def host_with_port
+        @host_with_port ||= begin
+          port, ip = Socket.unpack_sockaddr_in(get_peername)
+          "#{ip}:#{port}"
+        end
+      end
+      
+      protected
+      
+      def options_for_callback(blk)
+        return {} if blk.blank?
+        cb_id = "callback-#{seld.object_id}-#{Time.now.to_f}"
+        count = 0
+        count += 1 while @callbacks.has_key?(Digest::SHA256.hexdigest("#{cb_id}-#{count}"))
+        final_id = Digest::SHA256.hexdigest("#{cb_id}-#{count}")
+        @callbacks ||= {}
+        @callbacks[final_id] = blk
+        {"callback-id" => final_id}
+      end
+      
+      def process_callback(hash)
+        @callbacks ||= {}
+        if hash.is_a?(Hash) && hash.has_key?("callback-id")
+          callback = @callbacks.delete(hash["callback-id"])
+          callback.call(self, hash)
+        end
+      end
+      
+      def process_response_message(message, options)
+        method = self.handler_methods[message.to_s]
+        if method.present? && respond_to?(method)
+          logger.debug "Dispatching #{message} to #{method}"
+          send(method, options)
+        else
+          logger.warn "Got unknown message (#{message}) with options: #{options.inspect}"
+        end
+      end
+      
+      def self.register_handler_method(name, method = nil)
+        name = name.to_s
+        method ||= "handle_#{name}".to_sym
+        self.handler_methods[name] = method
+      end
+      
+    end
+  end
+end

data/lib/marvin/distributed/server.rb

@@ -0,0 +1,154 @@
+require 'json'
+require 'digest/sha2'
+require 'eventmachine'
+require 'socket'
+
+module Marvin
+  module Distributed
+    class Server < Protocol
+      
+      register_handler_method :completed
+      register_handler_method :exception
+      register_handler_method :action
+      register_handler_method :authenticate
+      
+      cattr_accessor :free_connections, :action_whitelist
+      self.free_connections = []
+      self.action_whitelist = [:nick, :pong, :action, :msg, :quit, :part, :join, :command]
+      
+      attr_accessor :processing, :configuration, :using_tls
+      
+      def initialize(*args)
+        @configuration = args.last.is_a?(Marvin::Nash) ? args.pop : Marvin::nash.new
+        super(*args)
+      end
+      
+      def post_init
+        super
+        @callbacks = {}
+        logger.info "Got distributed client connection with #{self.host_with_port}"
+        if should_use_tls?
+          start_tls
+        else
+          complete_processing
+        end
+      end
+      
+      def ssl_handshake_completed
+        complete_processing if should_use_tls?
+      end
+      
+      def unbind
+        logger.info "Lost distributed client connection with #{self.host_with_port}"
+        @@free_connections.delete(self)
+        super
+      end
+      
+      def dispatch(client, name, options)
+        @processing = true
+        send_message(:event, {
+          "event-name"    => name.to_s,
+          "event-options" => options,
+          "client-host"   => client.host_with_port,
+          "client-nick"   => client.nickname
+        })
+      end
+      
+      def handle_authenticate(options = {})
+        return unless requires_auth?
+        logger.info "Attempting authentication for distributed client"
+        if options["token"].present? && options["token"] == configuration.token
+          @authenticated = true
+          send_message(:authenticated)
+        else
+          send_message(:authentication_failed)
+        end
+      end
+      
+      def handle_completed(options = {})
+        return if fails_auth!
+        logger.debug "Completed message from #{self.host_with_port}"
+        complete_processing
+      end
+      
+      def handle_exception(options = {})
+        return if fails_auth!
+        logger.info "Handling exception on #{self.host_with_port}"
+        name      = options["name"]
+        message   = options["message"]
+        backtrace = options["backtrace"]
+        logger.warn "Error in remote client - #{name}: #{message}"
+        [*backtrace].each { |line| logger.warn "--> #{line}" } if backtrace.present?
+      end
+      
+      def handle_action(options = {})
+        return if fails_auth!
+        logger.debug "Handling action from on #{self.host_with_port}"
+        server    = lookup_client_for(options["client-host"])
+        action    = options["action"]
+        arguments = [*options["arguments"]]
+        return if server.blank? || action.blank?
+        begin
+          a = action.to_sym
+          if self.action_whitelist.include?(a)
+            server.send(a, *arguments) if server.respond_to?(a)
+          else
+            logger.warn "Client attempted invalid action #{a.inspect}"
+          end
+        rescue Exception => e
+          Marvin::ExceptionTracker.log(e)
+        end
+      end
+      
+      def complete_processing
+        @@free_connections << self
+        @processing = false
+      end
+      
+      def start_processing
+        @processing = true
+      end
+      
+      def lookup_client_for(key)
+        Marvin::IRC::Client.connections.detect do |c|
+          c.host_with_port == key
+        end
+      end
+      
+      def requires_auth?
+        configuration.token? && !authenticated?
+      end
+      
+      def authenticated?
+        @authenticated ||= false
+      end
+      
+      def should_use_tls?
+        @using_tls ||= configuration.encrypted?
+      end
+      
+      def fails_auth!
+        if requires_auth?
+          logger.debug "Authentication missing for distributed client"
+          send_message(:unauthorized)
+          close_connection_after_writing
+          return true
+        end
+      end
+      
+      def self.start
+        opts = Marvin::Settings.distributed || Marvin::Nash.new
+        opts = opts.server || Marvin::Nash.new
+        host = opts.host  || "0.0.0.0"
+        port = (opts.port || 8943).to_i
+        logger.info "Starting distributed server on #{host}:#{port} (requires authentication = #{opts.token?})"
+        EventMachine.start_server(host, port, self, opts)
+      end
+      
+      def self.next
+        @@free_connections.shift
+      end
+      
+    end
+  end
+end

data/lib/marvin/distributed.rb

@@ -1,14 +1,8 @@
-require 'drb'
-require 'rinda/ring'
-
-
 module Marvin
-  # Distributed tools for Marvin instances.
-  # Uses a tuple space etc + DRb to provide
-  # IRC Processing across the network.
   module Distributed
-    autoload :RingServer,      'marvin/distributed/ring_server'
-    autoload :DispatchHandler, 'marvin/distributed/dispatch_handler'
-    autoload :DRbClient,       'marvin/distributed/drb_client'
+    autoload :Protocol, 'marvin/distributed/protocol'
+    autoload :Server,   'marvin/distributed/server'
+    autoload :Handler,  'marvin/distributed/handler'
+    autoload :Client,   'marvin/distributed/client'
   end
 end