magent 0.5.4 → 0.6.0

data/Gemfile

@@ -5,3 +5,6 @@ gem 'uuidtools'
 
 gem 'em-websocket'
 
+gem 'sinatra', '~> 1.2.6'
+gem 'haml', '~> 3.1.1'
+gem 'launchy', '~> 0.4.0'

data/Gemfile.lock

@@ -3,19 +3,32 @@ GEM
   specs:
     addressable (2.2.1)
     bson (1.1)
+    configuration (1.2.0)
     em-websocket (0.1.4)
       addressable (>= 2.1.1)
       eventmachine (>= 0.12.9)
     eventmachine (0.12.10)
+    haml (3.1.1)
+    launchy (0.4.0)
+      configuration (>= 0.0.5)
+      rake (>= 0.8.1)
     mongo (1.1)
       bson (>= 1.0.5)
+    rack (1.3.0)
+    rake (0.9.0)
+    sinatra (1.2.6)
+      rack (~> 1.1)
+      tilt (>= 1.2.2, < 2.0)
+    tilt (1.3.2)
     uuidtools (2.1.1)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bson
   em-websocket
-  mongo
+  haml (~> 3.1.1)
+  launchy (~> 0.4.0)
+  mongo (~> 1.0)
+  sinatra (~> 1.2.6)
   uuidtools

data/VERSION

@@ -1 +1 @@
-0.5.4
+0.6.0

data/bin/magent-web

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+$:.unshift << File.expand_path("../../lib", __FILE__)
+require 'magent'
+require 'magent_web'
+require 'launchy'
+
+Thread.start do
+  sleep 2.5
+  Launchy.open("http://localhost:9876")
+end
+
+Rack::Server.new(:Port => "9876",
+                 :config => File.dirname(__FILE__)+"/../config.ru",
+                 :AccessLog => [],
+                 :pid => nil,
+                 :Host => "0.0.0.0").start
+

data/config.ru

@@ -0,0 +1,5 @@
+$:.unshift File.expand_path("../lib", __FILE__)
+require 'magent_web'
+
+run MagentWeb.app
+

data/lib/magent.rb

@@ -62,7 +62,7 @@ module Magent
   end
 
   def self.connect(environment, options={})
-    raise 'Set config before connecting. Magent.config = {...}' if config.blank?
+    raise 'Set config before connecting. Magent.config = {...}' if config.nil? || config.empty?
 
     env = config_for_environment(environment)
     Magent.connection = Mongo::Connection.new(env['host'], env['port'], options)
@@ -97,7 +97,7 @@ module Magent
   def self.config_for_environment(environment)
     env = environment ? config[environment] : config
 
-    return env if env['uri'].blank?
+    return env if env['uri'].nil? || env['uri'].blank?
 
     uri = URI.parse(env['uri'])
     raise InvalidScheme.new('must be mongodb') unless uri.scheme == 'mongodb'

data/lib/magent/failure.rb

@@ -26,6 +26,11 @@ module Magent
       remove_error(error["_id"])
     end
 
+    def enqueue_error(error)
+      enqueue(error["message"], 1)
+      remove_error(error["_id"])
+    end
+
     def error_collection
       @error_collection ||= Magent.database.collection("#{@name}.errors")
     end

data/lib/magent_web.rb

@@ -0,0 +1,42 @@
+require 'rubygems'
+
+require 'json'
+require 'magent'
+
+require 'benchmark'
+require 'sinatra'
+require 'haml'
+require 'yaml'
+
+require 'net/http'
+require 'uri'
+require 'cgi'
+
+require 'magent_web/mongo_helper'
+require 'magent_web/app'
+
+module MagentWeb
+  def self.app
+    MagentWeb.connect
+
+    MagentWeb::App
+  end
+
+  def self.connect
+    ENV["MAGENT_ENV"] ||= ENV["RACK_ENV"] || ENV["RAILS_ENV"]
+    if !ENV["MAGENT_ENV"]
+      raise ArgumentError, "please define the env var MAGENT_ENV"
+    end
+
+    if File.exist?("/etc/magent.yml")
+      Magent.setup(YAML.load_file("/etc/magent.yml"), ENV["MAGENT_ENV"], {})
+    elsif File.exist?("config/magent.yml")
+      Magent.setup(YAML.load_file("config/magent.yml"), ENV["MAGENT_ENV"], {})
+    elsif File.exist?("magent.yml")
+      Magent.setup(YAML.load_file("magent.yml"), ENV["MAGENT_ENV"], {})
+    else
+      raise ArgumentError, "/etc/magent.yml, ./config/magent.yml or ./magent.yml were not found"
+    end
+  end
+end
+

data/lib/magent_web/app.rb

@@ -0,0 +1,71 @@
+module MagentWeb
+  class App < Sinatra::Base
+    include MagentWeb::MongoHelper
+
+    helpers do
+      include Rack::Utils
+      alias_method :h, :escape_html
+    end
+
+    set :public, File.expand_path("../../../public", __FILE__)
+    set :views, File.expand_path("../../../lib/magent_web/views", __FILE__)
+
+    before do
+      @database = Magent.database
+    end
+
+    get "/" do
+      @queues = self.queues
+
+      haml :index
+    end
+
+    get "/status" do
+      haml :status
+    end
+
+    get "/queues/:id" do
+      @queue = @database.collection(params[:id])
+      @messages = document_list(@queue)
+
+      haml :"queues/show"
+    end
+
+    get "/queues/:id/failed" do
+      @queue = @database.collection(params[:id])
+      @errors_queue = @database.collection(params[:id]+".errors")
+      @errors = document_list(@errors_queue)
+
+      haml :"queues/failed"
+    end
+
+    get "/queues/:id/stats" do
+      @queue = @database.collection(params[:id])
+
+      haml :"queues/stats"
+    end
+
+    get "/queues/:queue_id/retry/:id" do
+      @errors_queue = @database.collection(params[:queue_id]+".errors")
+      @channel_name = channel_name_for(params[:queue_id])
+
+      channel = Magent::AsyncChannel.new(@channel_name)
+
+      doc = @errors_queue.find({:_id => params[:id]}).next_document
+      channel.enqueue_error(doc)
+
+      redirect "/queues/#{params[:queue_id]}/failed"
+    end
+
+    get "/queues/:queue_id/delete/:id" do
+      @errors_queue = @database.collection(params[:queue_id]+".errors")
+      @errors_queue.remove(:_id => params[:id])
+      redirect "/queues/#{params[:queue_id]}/failed"
+    end
+
+    private
+    def error_not_found
+      status 404
+    end
+  end
+end

data/lib/magent_web/mongo_helper.rb

@@ -0,0 +1,84 @@
+module MagentWeb
+  module MongoHelper
+    def queues
+      q = []
+      Magent.database.collections.each do |collection|
+        if collection.name =~ /^magent\./ && collection.name !~ /errors$/
+          q << collection
+        end
+      end
+
+      q
+    end
+
+    def query_options
+      options = {}
+      options[:limit] = params[:limit].nil? ? 25 : params[:limit].to_i
+      options[:sort] = [["_id", (params[:descending] == "true" ? 'descending' : 'ascending')]]
+      options[:skip] = params[:skip].nil? ? 0 : params[:skip].to_i
+
+      options
+    end
+
+    def document_list(collection, query = {})
+      collection.find(query, query_options)
+    end
+
+    def file_size(database)
+      "%0.2f MB" % (database.stats["fileSize"] / 1024**2)
+    end
+
+    def normalize_stats(stats)
+      r={}
+      stats.each do |k,v|
+        if k =~ /size/i
+          if v.kind_of?(Hash)
+            v.each do |ki,vi|
+              v[ki]="%0.2f MB" % (vi.to_f/1024**2)
+            end
+            r[k] = v
+          else
+            r[k]="%0.2f MB" % (v.to_f/1024**2)
+          end
+        else
+          r[k]=v
+        end
+      end
+      r
+    end
+
+    def server_status
+      Magent.database.command(:serverStatus => 1)
+    end
+
+    def humanize(v, quote = true)
+      if v.nil? && quote
+        "null"
+      elsif v.kind_of?(Hash)
+        JSON.pretty_generate(v)
+      elsif v.kind_of?(Array)
+        JSON.pretty_generate(v)
+      elsif v.kind_of?(Time)
+        v.strftime("%d %B %Y %H:%M:%S").inspect
+      elsif quote
+        v.inspect
+      else
+        v
+      end
+    end
+
+    def humanize_messages(messages)
+      return "" if !messages.kind_of?(Array)
+
+      messages.map do |e|
+        name = e.first
+        args = e.last.join(", ")
+        "#{name}(#{args})"
+      end.join(" -> ")
+    end
+
+    def channel_name_for(queue_id)
+      queue_id.to_s.match("magent\.([^\.]+)")[1]
+    end
+  end
+end

data/lib/magent_web/views/index.haml

@@ -0,0 +1,14 @@
+%div(data-role="page" data-add-back-btn="true")
+  %div(data-role="header")
+    %h1
+      = @title = "All Magent Queues"
+  %div(data-role="content")
+    %ul(data-role="listview" data-filter="true")
+      -@queues.each do |queue|
+        %li
+          %a(href="/queues/#{queue.name}")
+            &=queue.name
+          %span{:class=>"ui-li-count"}
+            =0
+
+  %div(data-role="footer")

data/lib/magent_web/views/layout.haml

@@ -0,0 +1,15 @@
+!!! 5
+%html
+  %head
+    %title
+      =@title||"Magent Web"
+    %link(rel="stylesheet" href="/stylesheets/jquery.mobile-1.0b1pre.min.css")
+    %link(rel="stylesheet" href="/stylesheets/highlight.css")
+
+    %script(src="http://code.jquery.com/jquery-1.5.2.min.js")
+    %script(src="/javascripts/jquery.mobile-1.0b1pre.min.js")
+
+    %script(src="/javascripts/application.js")
+  %body
+    = yield
+

data/lib/magent_web/views/queues/failed.haml

@@ -0,0 +1,36 @@
+-count=0
+-skip = params[:skip].to_i
+%div(data-role="page" data-add-back-btn="true")
+  %div(data-role="header")
+    %h1
+      Failed Jobs for
+      &= channel_name_for(@queue.name)
+  %div(data-role="content")
+    %h1
+      Failed Jobs
+      ==(#{@errors.count})
+    -@errors.each do |document|
+      -message = document["message"]
+      -count+=1
+      %div(data-role="collapsible" data-collapsed=true)
+        %h1
+          &= "#{humanize_messages(document["message"].last)}: #{document["error"]}"
+        =haml :"shared/values", :layout => false, :locals => {:hash => document}
+        %div(data-role="navbar" data-iconpos="bottom")
+          %ul
+            %li
+              %a(href="/queues/#{@queue.name}/retry/#{document["_id"]}" data-icon="refresh")
+                Retry
+            %li
+              %a(href="/queues/#{@queue.name}/delete/#{document["_id"]}" data-icon="delete" data-theme="e")
+                Delete
+
+
+  %div(data-role="footer")
+    -if skip > 0
+      %a(href="/queues/#{@queue.name}/failed?skip=#{params[:skip].to_i-25}")
+        Previous
+    -if count >= 25
+      %a(href="/queues/#{@queue.name}/failed?skip=#{params[:skip].to_i+25}")
+        Next
+

data/lib/magent_web/views/queues/show.haml

@@ -0,0 +1,44 @@
+-count=0
+-skip = params[:skip].to_i
+%div(data-role="page" data-add-back-btn="true")
+  %div(data-role="header")
+    %h1
+      Queue:
+      &=@title=@database.name+" / #{channel_name_for(@queue.name)}"
+    %div{:"data-role" => "navbar"}
+      %ul
+        %li
+          %a(href="/queues/#{@queue.name}/failed" data-icon="alert")
+            Failed
+        %li
+          %a(href="/queues/#{@queue.name}/stats" data-icon="grid")
+            Stats
+
+
+  %div(data-role="content")
+    -if skip == 0
+      %h1
+        Information
+      -normalize_stats(@queue.stats).each do |k,v|
+        %b
+          &="#{k}="
+        &=humanize v
+        %br
+    %h1
+      Pending Jobs
+      ==(#{@messages.count})
+    -@messages.each do |document|
+      -message = document["message"]
+      -count+=1
+      %div(data-role="collapsible" data-collapsed=true)
+        %h1
+          &= "[#{document["priority"]}] #{message.first}: #{humanize_messages(message.last)}"
+        =haml :"shared/values", :layout => false, :locals => {:hash => document}
+
+  %div(data-role="footer")
+    -if skip > 0
+      %a(href="/queues/#{@queue.name}?skip=#{params[:skip].to_i-25}")
+        Previous
+    -if count >= 25
+      %a(href="/queues/#{@queue.name}?skip=#{params[:skip].to_i+25}")
+        Next

data/lib/magent_web/views/queues/stats.haml

@@ -0,0 +1,6 @@
+%div(data-role="page" data-add-back-btn="true")
+  %div(data-role="header")
+    %h1
+      =@title ="Stats"
+  %div(data-role="content")
+  %div(data-role="footer")

data/lib/magent_web/views/shared/values.haml

@@ -0,0 +1,74 @@
+-hash.each do |key,value|
+  -k = humanize(key)
+  -v = humanize(value)
+
+  -if value.kind_of?(Hash) && value.size > 5
+    %div(data-role="collapsible" data-collapsed=true data-theme="b")
+      %h2
+        &=k
+      = haml :"shared/values", :layout => false, :locals => {:hash => value}
+  -elsif value.kind_of?(Array) && value.size > 5
+    %div(data-role="collapsible" data-collapsed=true data-theme="b")
+      %h2
+        &=k
+      -tmp = []
+      -value.each_with_index do |element, index|
+        -if element.kind_of?(Hash)
+          %div(data-role="collapsible" data-collapsed=true data-theme="e")
+            %h2
+              &=index
+            = haml :"shared/values", :layout => false, :locals => {:hash => element}
+        -else
+          -tmp << element
+      - if !tmp.empty?
+        =humanize(tmp)
+        %br
+
+  -else
+    %b
+      -if key == "_type"
+        %span.type_key
+          &=k
+      -elsif key == "_id"
+        %span.pk_key
+          &=k
+      -elsif key =~ /_id$/
+        %span.fk_key
+          &=k
+      -elsif key =~ /_at$/
+        %span.date_key
+          &=k
+      -elsif ["name", "title", "error", "message"].include?(key)
+        %span.name_key
+          &=k
+      -elsif key =~ /^_/
+        %span.private_key
+          &=k
+      -else
+        %span.normal_key
+        &=k
+      \=
+
+    -if key == "_type"
+      %span.type_value
+        &=v
+    -elsif key == "_id"
+      %span.pk_value
+        &=v
+    -elsif key =~ /_id$/
+      %span.fk_value
+        &=v
+    -elsif key =~ /_at$/
+      %span.date_value
+        &=v
+    -elsif ["name", "title", "error", "message"].include?(key)
+      %span.name_value
+        &=v
+    -elsif key =~ /^_/
+      %span.private_value
+        &=v
+    -else
+      %span.normal_value
+      &=v
+    %br<
+