magent 0.6.0 → 0.6.1

data/VERSION

@@ -1 +1 @@
-0.6.0
+0.6.1

data/bin/magent

@@ -91,7 +91,7 @@ class Controller
     if @options[:daemonize] && @options[:piddir]
       run_as_daemon
     else
-      Magent::Processor.new(self.channel).run!
+      Magent::Processor.new(self.channel, @identity).run!
     end
   end
 

data/lib/magent.rb

@@ -6,6 +6,7 @@ require 'mongo'
 require 'set'
 require 'uuidtools'
 
+require 'magent/stats'
 require 'magent/failure'
 
 require 'magent/utils'

data/lib/magent/generic_channel.rb

@@ -1,8 +1,10 @@
 module Magent
   class GenericChannel
     include Magent::Failure
+    include Magent::Stats
 
     attr_reader :name
+    attr_reader :current_job
 
     def initialize(name)
       @name = "magent.#{name}"
@@ -21,8 +23,8 @@ module Magent
     end
 
     def dequeue
-      if m = self.next_message
-        m["message"]
+      if @current_job = self.next_message
+        @current_job["message"]
       end
     end
 

data/lib/magent/processor.rb

@@ -1,8 +1,11 @@
 module Magent
   class Processor
-    def initialize(channel)
+    def initialize(channel, identity = "#{channel.name}-#{Socket.gethostname.split('.')[0]}")
       @channel = channel
       @shutdown = false
+      @identity = identity
+
+      @channel.on_start(identity)
 
 #       @actor.class.actions.each do |action|
 #         if !@actor.respond_to?(action)
@@ -23,8 +26,12 @@ module Magent
 
         message = @channel.dequeue
         begin
+          t = Time.now
           if message && @channel.process!(message)
             puts "Processed #{message.inspect}"
+
+            @channel.on_job_processed(@channel.current_job, Time.now - t, @identity)
+
             delay = 0
             processed_messages += 1
             if processed_messages > 20
@@ -37,6 +44,7 @@ module Magent
         rescue SystemExit
         rescue Exception => e
           $stderr.puts "Error processing #{message.inspect} => #{e.message}"
+          @channel.on_job_failed(@identity)
           @channel.failed(:error => e.message, :message => message, :backtrace => e.backtrace, :date => Time.now.utc)
         ensure
         end
@@ -47,6 +55,8 @@ module Magent
 
     def shutdown!
       @shutdown = true
+      @channel.on_quit
+
       @channel.on_shutdown if @channel.respond_to?(:on_shutdown)
       $stderr.puts "Shutting down..."
     end

data/lib/magent/stats.rb

@@ -0,0 +1,52 @@
+module Magent
+  module Stats
+    def stats_collection
+      @stats_collection ||= Magent.database.collection("magent.stats")
+    end
+
+    def stats
+      @stats ||= if doc = stats_collection.find({:_id => @name}, {}).next_document
+        doc
+      else
+        stats_collection.save({:_id => @name,
+                               :created_at => Time.now.utc,
+                               :total_errors => 0,
+                               :total_jobs => 0,
+                               :last_processed_job => {"duration" => 0, "method" => "", "priority" => 0},
+                               :updated_at => Time.now.utc,
+                               :updated_by => "",
+                               :workers => {}
+                              }, {:safe => true})
+
+        stats_collection.find({:_id => @name}, {}).next_document
+      end
+    end
+
+    def on_job_processed(last_job, duration, updated_by)
+      last_job["duration"] = ("%f" % duration)
+      updated_at = Time.now
+
+      updates = {}
+      updates[:$inc] = {:total_jobs => 1, :"workers.#{updated_by}.total_jobs" => 1}
+      updates[:$set] = {:updated_at => updated_at, :updated_by => updated_by,
+                        :last_processed_job => last_job,
+                        :"workers.#{updated_by}.last_update_at" => updated_at }
+
+      stats_collection.update({:_id => @name}, updates, {:multi => true})
+    end
+
+    def on_job_failed(updated_by)
+      stats_collection.update({:_id => @name}, {:$inc => {:total_errors => 1, :"workers.#{updated_by}.total_errors" => 1} })
+    end
+
+    def on_start(updated_by)
+      puts ">>> Current Stats: #{stats.inspect}"
+      stats_collection.update({:_id => @name}, {:$set => {:"workers.#{updated_by}.last_update_at" => Time.now.utc},
+                                                :$inc => {:"workers.#{updated_by}.total_restarts" => 1} }, {:safe => true, :multi => true})
+    end
+
+    def on_quit(updated_by)
+      stats_collection.update({:_id => @name}, {:set => {:"workers.#{updated_by}.quitted_at" => Time.now.utc} })
+    end
+  end
+end

data/lib/magent_web/app.rb

@@ -41,6 +41,11 @@ module MagentWeb
 
     get "/queues/:id/stats" do
       @queue = @database.collection(params[:id])
+      @channel_name = channel_name_for(params[:id])
+      channel = Magent::GenericChannel.new(@channel_name)
+
+      @stats_collection = channel.stats_collection
+      @stats = channel.stats
 
       haml :"queues/stats"
     end
@@ -49,7 +54,7 @@ module MagentWeb
       @errors_queue = @database.collection(params[:queue_id]+".errors")
       @channel_name = channel_name_for(params[:queue_id])
 
-      channel = Magent::AsyncChannel.new(@channel_name)
+      channel = Magent::GenericChannel.new(@channel_name)
 
       doc = @errors_queue.find({:_id => params[:id]}).next_document
       channel.enqueue_error(doc)

data/lib/magent_web/mongo_helper.rb

@@ -3,7 +3,7 @@ module MagentWeb
     def queues
       q = []
       Magent.database.collections.each do |collection|
-        if collection.name =~ /^magent\./ && collection.name !~ /errors$/
+        if collection.name =~ /^magent\./ && collection.name !~ /(errors|stats)$/
           q << collection
         end
       end
@@ -51,13 +51,13 @@ module MagentWeb
       Magent.database.command(:serverStatus => 1)
     end
 
-    def humanize(v, quote = true)
+    def humanize(v, quote = false)
       if v.nil? && quote
         "null"
       elsif v.kind_of?(Hash)
         JSON.pretty_generate(v)
       elsif v.kind_of?(Array)
-        JSON.pretty_generate(v)
+        v.map{|e| e.nil? ? "null" : e }.join("<br />")
       elsif v.kind_of?(Time)
         v.strftime("%d %B %Y %H:%M:%S").inspect
       elsif quote

data/lib/magent_web/views/queues/show.haml

@@ -16,14 +16,6 @@
 
 
   %div(data-role="content")
-    -if skip == 0
-      %h1
-        Information
-      -normalize_stats(@queue.stats).each do |k,v|
-        %b
-          &="#{k}="
-        &=humanize v
-        %br
     %h1
       Pending Jobs
       ==(#{@messages.count})

data/lib/magent_web/views/queues/stats.haml

@@ -3,4 +3,15 @@
     %h1
       =@title ="Stats"
   %div(data-role="content")
+    %h3
+      Database Information
+    -normalize_stats(@queue.stats).each do |k,v|
+      %b
+        &="#{k}="
+      &=humanize v
+      %br
+    %h3
+      General Stats
+    =haml :"shared/values", :layout => false, :locals => {:hash => normalize_stats(@stats)}
+
   %div(data-role="footer")

data/lib/magent_web/views/shared/values.haml

@@ -2,12 +2,12 @@
   -k = humanize(key)
   -v = humanize(value)
 
-  -if value.kind_of?(Hash) && value.size > 5
+  -if value.kind_of?(Hash)
     %div(data-role="collapsible" data-collapsed=true data-theme="b")
       %h2
         &=k
       = haml :"shared/values", :layout => false, :locals => {:hash => value}
-  -elsif value.kind_of?(Array) && value.size > 5
+  -elsif value.kind_of?(Array)
     %div(data-role="collapsible" data-collapsed=true data-theme="b")
       %h2
         &=k

data/magent.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{magent}
-  s.version = "0.6.0"
+  s.version = "0.6.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["David A. Cuadrado"]
@@ -47,6 +47,7 @@ Gem::Specification.new do |s|
     "lib/magent/processor.rb",
     "lib/magent/push.rb",
     "lib/magent/railtie.rb",
+    "lib/magent/stats.rb",
     "lib/magent/utils.rb",
     "lib/magent/web_socket_channel.rb",
     "lib/magent/web_socket_server.rb",

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 6
-  - 0
-  version: 0.6.0
+  - 1
+  version: 0.6.1
 platform: ruby
 authors: 
 - David A. Cuadrado
@@ -194,6 +194,7 @@ files:
 - lib/magent/processor.rb
 - lib/magent/push.rb
 - lib/magent/railtie.rb
+- lib/magent/stats.rb
 - lib/magent/utils.rb
 - lib/magent/web_socket_channel.rb
 - lib/magent/web_socket_server.rb