minicron 0.1.0

data/lib/minicron/transport.rb

@@ -0,0 +1,15 @@
+require 'digest/md5'
+require 'json'
+
+module Minicron
+  # The transport module deals with interactions between the server and client
+  module Transport
+    # Calculate the job hash based on the command and host
+    #
+    # @param command [String] the job command e.g 'ls -la'
+    # @param fqdn [String] the fqdn of the server running the job e.g `db1.example.com`
+    def self.get_job_hash(command, fqdn)
+      Digest::MD5.hexdigest(command + fqdn)
+    end
+  end
+end

data/lib/minicron/transport/client.rb

@@ -0,0 +1,80 @@
+require 'minicron/transport/faye/client'
+
+module Minicron
+  module Transport
+    class Client < Minicron::Transport::FayeClient
+      # Instantiate a new instance of the client
+      #
+      # @param host [String] The host to be communicated with
+      def initialize(scheme, host, port, path)
+        @scheme = scheme
+        @host = host
+        @path = path == '/' ? '/faye' : "#{path}/faye"
+        @port = port
+        @seq = 1
+        super(@scheme, @host, @port, @path)
+      end
+
+      # Used to set up a job on the server
+      #
+      # @param job_hash [String]
+      # @param command [Integer]
+      # @param fqdn [String]
+      # @param hostname [String]
+      # @return [Hash]
+      def setup(job_hash, command, fqdn, hostname)
+        # Send a request to set up the job
+        publish("/job/#{job_hash}/status", {
+          :action => 'SETUP',
+          :command => command,
+          :fqdn => fqdn,
+          :hostname => hostname
+        })
+
+        # Wait for the response..
+        ensure_delivery
+
+        # TODO: Handle errors here!
+        # Get the job and execution id from the response
+        ids = JSON.parse(responses.first[:body]).first['channel'].split('/')[3]
+
+        # Split them up
+        ids = ids.split('-')
+
+        # Return them as a hash
+        {
+          :job_id => ids[0],
+          :execution_id => ids[1]
+        }
+      end
+
+      # Helper that wraps the publish function making it quicker to use
+      #
+      # @option options [String] job_id
+      # @option options [String, Symbol] type status or output
+      # @option options [Integer] execution_id
+      def send(options = {})
+        # Publish the message to the correct channel
+        publish("/job/#{options[:job_id]}/#{options[:execution_id]}/#{options[:type]}", options[:message])
+      end
+
+      # Publishes a message on the given channel to the server
+      #
+      # @param channel [String]
+      # @param message [String]
+      def publish(channel, message)
+        # Set up the data to send to faye
+        data = {:channel => channel, :data => {
+          :ts => Time.now.utc.strftime("%Y-%m-%d %H:%M:%S"),
+          :message => message,
+          :seq => @seq
+        }}
+
+        # Increment the sequence id
+        @seq += 1
+
+        request({ :message => data.to_json })
+      end
+    end
+  end
+end

data/lib/minicron/transport/faye/client.rb

@@ -0,0 +1,103 @@
+require 'eventmachine'
+require 'em-http-request'
+require 'digest/md5'
+
+module Minicron
+  module Transport
+    class FayeClient
+      attr_accessor :url
+      attr_accessor :queue
+      attr_accessor :responses
+
+      # Instantiate a new instance of the client
+      #
+      # @param scheme [String] http or https
+      # @param host [String] The host to be communicated with
+      # @param port [String] The port number the server runs on
+      # @param path [String] The path to the server, optional
+      def initialize(scheme, host, port, path) # TODO: Add options hash for other options
+        @scheme = scheme
+        @host = host
+        @path = path
+        @port = port
+        @url = "#{scheme}://#{host}:#{port}#{path}"
+        @queue = {}
+        @responses = []
+        @retries = 3
+        @retry_counts = {}
+      end
+
+      # Starts EventMachine in a new thread if it isn't already running
+      def ensure_em_running
+        Thread.new { EM.run } unless EM.reactor_running?
+        sleep 0.1 until EM.reactor_running?
+      end
+
+      # Sends a request to the @url and adds it to the request queue
+      #
+      # @param body [String]
+      def request(body)
+        # Make sure eventmachine is running
+        ensure_em_running
+
+        # Make the request
+        req = EventMachine::HttpRequest.new(
+          @url,
+          :connect_timeout => Minicron.config['client']['connect_timeout'],
+          :inactivity_timeout => Minicron.config['client']['inactivity_timeout']
+        ).post(:body => body)
+
+        # Generate an id for the request
+        req_id = Digest::MD5.hexdigest(body.to_s)
+
+        # Put the request in the queue
+        queue[req_id] = req
+
+        # Set up the retry count for this request if it didn't already exist
+        @retry_counts[req_id] ||= 0
+
+        # Did the request succeed? If so remove it from the queue
+        req.callback do
+          @responses.push({
+            :status => req.response_header.status,
+            :header => req.response_header,
+            :body => req.response
+          })
+
+          queue.delete(req_id)
+        end
+
+        # If not retry the request up to @retries times
+        req.errback do |error|
+          @responses.push({
+            :status => req.response_header.status,
+            :header => req.response_header,
+            :body => req.response
+          })
+
+          if @retry_counts[req_id] < @retries
+            @retry_counts[req_id] += 1
+            request(body)
+          end
+        end
+      end
+
+      # Blocks until all messages in the sending queue have completed
+      def ensure_delivery
+        # Keep waiting until the queue is empty but only if we need to
+        if queue.length > 0
+          until queue.length == 0
+            sleep 0.05
+          end
+        end
+
+        true
+      end
+
+      # Tidy up after we are done with the client
+      def tidy_up
+        EM.stop
+      end
+    end
+  end
+end

data/lib/minicron/transport/faye/extensions/job_handler.rb

@@ -0,0 +1,184 @@
+require 'minicron/alert'
+require 'minicron/hub/models/host'
+require 'minicron/hub/models/job'
+require 'minicron/hub/models/execution'
+require 'minicron/hub/models/job_execution_output'
+
+module Minicron
+  module Transport
+    # An extension to the Faye server to store some of the data it receives
+    #
+    # TODO: A lot of this need more validation checks and error handling
+    #       currently it's just assumed the correct data is passed and the server
+    #       can crash if it isn't
+    class FayeJobHandler
+      # Called by Faye when a message is recieved
+      #
+      # @param message [Hash] The message data
+      # @param request the rack request object
+      # @param callback
+      def incoming(message, request, callback)
+        segments = message['channel'].split('/')
+
+        # Is it a job messages
+        if segments[1] == 'job'
+          data = message['data']['message']
+
+          # Is it a setup message?
+          if segments[3] == 'status' && data['action'] == 'SETUP'
+            message = handle_setup(request, message, segments)
+          end
+
+          # Is it a start message?
+          if segments[4] == 'status' && data[0..4] == 'START'
+            handle_start(request, message, segments)
+          end
+
+          # Is it job output?
+          if segments[4] == 'output'
+            message = handle_output(request, message, segments)
+          end
+
+          # Is it a finish message?
+          if segments[4] == 'status' && data[0..5] == 'FINISH'
+            handle_finish(request, message, segments)
+          end
+
+          # Is it an exit message?
+          if segments[4] == 'status' && data[0..3] == 'EXIT'
+            handle_exit(request, message, segments)
+          end
+        end
+
+        # Return the message back to faye
+        callback.call(message)
+      end
+
+      # Handle SETUP messages
+      #
+      # @param request the rack request object
+      # @param message [Hash] the decoded message sent with the request
+      # @param segments [Hash] the message channel split by /
+      def handle_setup(request, message, segments)
+        data = message['data']['message']
+
+        # Try and find the host
+        host = Minicron::Hub::Host.where(:fqdn => data['fqdn']).first
+
+        # Create it if it didn't exist!
+        if not host
+          host = Minicron::Hub::Host.create(
+            :name => data['hostname'],
+            :fqdn => data['fqdn'],
+            :host => request.ip
+          )
+
+          # Generate a new SSH key - TODO: add passphrase
+          key = Minicron.generate_ssh_key('host', host.id, host.fqdn)
+
+          # And finally we store the public key in te db with the host for convenience
+          Minicron::Hub::Host.where(:id => host.id).update_all(
+            :public_key => key.ssh_public_key
+          )
+        end
+
+        # Validate or create the job
+        job = Minicron::Hub::Job.where(:job_hash => segments[2]).first_or_create do |j|
+          j.job_hash = segments[2]
+          j.command = data['command']
+          j.host_id = host.id
+        end
+
+        # Create an execution for this job
+        execution = Minicron::Hub::Execution.create(
+          :job_id => job.id
+        )
+
+        # Alter the response channel to include the execution id for the
+        # client to use in later requests
+        segments[3] = "#{job.id}-#{execution.id}/status"
+        message['channel'] = segments.join('/')
+
+        # And finally return the message
+        message
+      end
+
+      # Handle START messages
+      #
+      # @param request the rack request object
+      # @param message [Hash] the decoded message sent with the request
+      # @param segments [Hash] the message channel split by /
+      def handle_start(request, message, segments)
+        data = message['data']['message']
+
+        # Update the execution and add the start time
+        Minicron::Hub::Execution.where(:id => segments[3]).update_all(
+          :started_at => data[6..-1]
+        )
+      end
+
+      # Handle job output
+      #
+      # @param request the rack request object
+      # @param message [Hash] the decoded message sent with the request
+      # @param segments [Hash] the message channel split by /
+      def handle_output(request, message, segments)
+        data = message['data']['message']
+        ts = message['data']['ts']
+        seq = message['data']['seq']
+
+        # Store the job execution output
+        output = Minicron::Hub::JobExecutionOutput.create(
+          :execution_id => segments[3],
+          :output => data,
+          :timestamp => ts,
+          :seq => seq
+        )
+
+        # Append the id to the message so we can use it on the frontend
+        message['data']['job_execution_output_id'] = output.id
+
+        # And finally return the message
+        message
+      end
+
+      # Handle FINISH messages
+      #
+      # @param request the rack request object
+      # @param message [Hash] the decoded message sent with the request
+      # @param segments [Hash] the message channel split by /
+      def handle_finish(request, message, segments)
+        data = message['data']['message']
+
+        # Update the execution and add the finish time
+        Minicron::Hub::Execution.where(:id => segments[3]).update_all(
+          :finished_at => data[7..-1]
+        )
+      end
+
+      # Handle EXIT messages
+      #
+      # @param request the rack request object
+      # @param message [Hash] the decoded message sent with the request
+      # @param segments [Hash] the message channel split by /
+      def handle_exit(request, message, segments)
+        data = message['data']['message']
+
+        # Update the execution and add the exit status
+        Minicron::Hub::Execution.where(:id => segments[3]).update_all(
+          :exit_status => data[5..-1]
+        )
+
+        # If the exit status was above 0 we need to trigger a failure alert
+        if data[5..-1].to_i > 0
+          alert = Minicron::Alert.new
+          alert.send_all(
+            :kind => 'fail',
+            :execution_id => segments[3],
+            :job_id => segments[2]
+          )
+        end
+      end
+    end
+  end
+end

data/lib/minicron/transport/faye/server.rb

@@ -0,0 +1,58 @@
+require 'thin'
+require 'rack'
+require 'faye'
+require 'minicron/transport/faye/extensions/job_handler'
+
+module Minicron
+  module Transport
+    class FayeServer
+      attr_reader :server
+
+      def initialize
+        # Load the Faye thin adapter, this needs to happen first
+        Faye::WebSocket.load_adapter('thin')
+
+        # Show debug verbose output if requested
+        if Minicron.config['global']['verbose']
+          log = Logger.new(STDOUT)
+          log.level = Logger::DEBUG
+          Faye.logger = log
+        end
+
+        # Set up our Faye rack app
+        @server = Faye::RackAdapter.new(
+          :mount => '', # This is relative to the map faye_path set in server.rb
+          :timeout => 25
+        )
+
+        @server.add_extension(Minicron::Transport::FayeJobHandler.new)
+
+        # Add all the events we want to listen out for
+        add_faye_events
+      end
+
+      private
+      def add_faye_events
+        @server.on(:handshake) do |client_id|
+          p [:handshake, client_id] if Minicron.config['global']['verbose']
+        end
+
+        @server.on(:subscribe) do |client_id, channel|
+          p [:subscribe, client_id, channel] if Minicron.config['global']['verbose']
+        end
+
+        @server.on(:unsubscribe) do |client_id, channel|
+          p [:unsubscribe, client_id, channel] if Minicron.config['global']['verbose']
+        end
+
+        @server.on(:publish) do |client_id, channel, data|
+          p [:published, client_id, channel, data] if Minicron.config['global']['verbose']
+        end
+
+        @server.on(:disconnect) do |client_id|
+          p [:disconnect, client_id] if Minicron.config['global']['verbose']
+        end
+      end
+    end
+  end
+end

data/lib/minicron/transport/server.rb

@@ -0,0 +1,62 @@
+require 'thin'
+require 'rack'
+
+module Minicron
+  module Transport
+    # Used to mangage the web server minicron runs on
+    class Server
+      attr_reader :server
+
+      # Starts the thin server
+      #
+      # @param host [String] the host e.g 0.0.0.0
+      # @param port [Integer]
+      # @param path [String] The absolute path to the server e.g /server
+      def start!(host, port, path)
+        return false if running?
+
+        # Start the faye or rails apps depending on the path
+        server = Thin::Server.new(host, port) do
+          use Rack::CommonLogger
+          use Rack::ShowExceptions
+
+          # The 'hub', aka our sinatra web interface
+          map '/' do
+            require Minicron::LIB_PATH + '/minicron/hub/app'
+            use Minicron::Hub::ExceptionHandling
+            run Minicron::Hub::App.new
+          end
+
+          # Set the path faye should start relative to
+          faye_path = path == '/' ? '/faye' : "#{path}/faye"
+
+          # The faye server the server and browser clients talk to
+          map faye_path do
+            require Minicron::LIB_PATH + '/minicron/transport/faye/server'
+            run Minicron::Transport::FayeServer.new.server
+          end
+        end
+
+        server.start
+        true
+      end
+
+      # Stops the thin server if it's running
+      # @return [Boolean] whether the server was stopped or not
+      def stop!
+        return false unless running? && server != nil
+
+        server.stop
+        true
+      end
+
+      # Returns a bool based on whether
+      # @return [Boolean]
+      def running?
+        return false unless server != nil
+
+        server.running?
+      end
+    end
+  end
+end