sqs_web 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 402d02606f9f97d50fa51892ee6b372b34d13cea
+  data.tar.gz: 85af922a688dbbe657cd445030827184203fb5eb
+SHA512:
+  metadata.gz: fbe157d2c9d2342154a0eb92f043c50664e27afc1dabe04133c93d60ac70cd5371cabb021a880c2b66c2ab77abdd91734e4952d1a7dacc3cd90c58da59a2bb04
+  data.tar.gz: 34d78f607cfaae8ab4d007d2aa698eff188ad41829007b38c319b9c99cb0f34f8a3f64b1255dbce512d8287ffb10f7b8000993b49c9136585cbd3a350a68754f

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gemspec
+# To be removed once fake_sqs merges my PR changes
+gem 'fake_sqs', git: 'https://github.com/nritholtz/fake_sqs.git', ref: '1b0d283'

data/README.markdown

@@ -0,0 +1,121 @@
+sqs_web
+===============
+[![Build Status](https://travis-ci.org/nritholtz/sqs_web.svg?branch=master)](https://travis-ci.org/nritholtz/sqs_web)
+[![Code Climate](https://codeclimate.com/github/nritholtz/sqs_web/badges/gpa.svg)](https://codeclimate.com/github/nritholtz/sqs_web)
+[![Test Coverage](https://codeclimate.com/github/nritholtz/sqs_web/badges/coverage.svg)](https://codeclimate.com/github/nritholtz/sqs_web/coverage)
+
+A [delayed_job_web](https://github.com/ejschmitt/delayed_job_web) inspired (read: stolen) interface for SQS.
+This gem was written to work anchored within rails 3 and 4 applications.
+
+Some features:
+
+* Easily view messages visible and in-flight in your configured SQS queues
+* Move any single enqueued message, or all enqueued messages, from a DLQ to the corresponding active queue
+* Remove an enqueued message, or easily remove all enqueued messages from a DLQ
+* View overview stats for the queues
+
+The interface:
+
+![Screen shot](https://www.dropbox.com/s/j14s0aqthj8w3d2/sqs_web_dashboard.png?dl=1)
+
+
+Quick Start For Rails 3 and 4 Applications
+------------------------------------
+
+Add the dependency to your Gemfile
+
+```ruby
+gem "sqs_web"
+```
+
+Install it...
+
+```ruby
+bundle
+```
+
+Add the following route to your application for accessing the interface,
+and actions related to the messages.
+
+```ruby
+match "/sqs" => SqsWeb, :anchor => false, via: [:get, :post]
+```
+
+You probably want to password protect the interface, an easy way is to add something like this your config.ru file
+
+```ruby
+if Rails.env.production?
+  SqsWeb.use Rack::Auth::Basic do |username, password|
+    username == 'username' && password == 'password'
+  end
+end
+```
+
+`sqs_web` runs as a Sinatra application within the rails application. Visit it at `/sqs`.
+
+## Supported SqsWeb configuration options
+`SqsWeb.options[:aws]` supports the following hash key/value pairs:
+- `access_key_id` : AWS Access Key. If not set will default to environment variable `AWS_ACCESS_KEY_ID` or instance profile credentials
+- `secret_access_key` : AWS Secret Access Key. If not set will default to environment variable `AWS_SECRET_ACCESS_KEY` or instance profile credentials.
+- `region`: AWS region for the SQS queue. If not set will default to environment variable `AWS_REGION` or else to `us-east-1`.
+
+`SqsWeb.options[:queues]` supports an array of strings for the SQS queue names.
+```ruby
+SqsWeb.options[:queues] =  ["TestSourceQueue", "TestSourceQueueDLQ"]
+```
+
+## Notes
+Currently, this was written in mind for being only used for DLQ management. The [AWS SQS Management Console](https://aws.amazon.com/blogs/aws/aws-management-console-now-supports-the-simple-queue-service-sqs/) should have most of the functionality that you would want out of the box, this plugin is **not meant as a replacement for the AWS SQS Management Console**, but rather as a supplement. There are some features that are not implemented yet (e.g. moving a message from a DLQ back to the source queue) in the AWS Console, and there are some additional benefits for the management screen to live within the application.
+
+This is not to say there are some features that may be duplicated or added to this plugin as it advances. In addition, our internal applications are using [Shoryuken](https://github.com/phstc/shoryuken) which uses `message attributes` (e.g. *ApproximateReceiveCount*) that become "invalid" once you pick up the message from an active queue. There is greater freedom when managing a DLQ, since this plugin is assuming that the management console (or the AWS SQS Management Console) are the only consumers of the DLQ, which solves the complexity of these `message attributes`.
+
+## Serving static assets
+
+If you mount the app on another route, you may encounter the CSS not working anymore. To work around this you can leverage a special HTTP header. Install it, activate it and configure it -- see below.
+
+### Apache
+
+    XSendFile On
+    XSendFilePath /path/to/shared/bundle
+
+[`XSendFilePath`](https://tn123.org/mod_xsendfile/) white-lists a directory from which static files are allowed to be served. This should be at least the path to where delayed_job_web is installed.
+
+Using Rails you'll have to set `config.action_dispatch.x_sendfile_header = "X-Sendfile"`.
+
+### Nginx
+
+Nginx uses an equivalent that's called `X-Accel-Redirect`, further instructions can be found [in their wiki](http://wiki.nginx.org/XSendfile).
+
+Rails' will need to be configured to `config.action_dispatch.x_sendfile_header = "X-Accel-Redirect"`.
+
+### Lighttpd
+
+Lighty is more `X-Sendfile`, like [outlined](http://redmine.lighttpd.net/projects/1/wiki/X-LIGHTTPD-send-file) in their wiki.
+
+
+Contributing
+------------
+
+* To bootstrap a `fake_sqs` environment for development purposes, run the following:
+```ruby
+bundle exec ruby scripts/bootstrap_queues.rb
+```
+
+1. Fork
+2. Hack
+3. `rake test`
+4. Send a pull request
+
+
+Releasing a new version
+-----------------------
+
+1. Update the version in `sqs_web.gemspec`
+2. `git commit sqs_web.gemspec` with the following message format:
+
+        Version x.x.x
+
+        Changelog:
+        * Some new feature
+        * Some new bug fix
+3. `rake release`

data/Rakefile

@@ -0,0 +1,27 @@
+require 'bundler'
+require 'rspec/core/rake_task'
+require 'rdoc/task'
+
+namespace :spec do
+  desc "Run specs with in-memory database"
+  RSpec::Core::RakeTask.new(:memory) do |t|
+    ENV["SQS_DATABASE"] = ":memory:"
+    t.pattern = "spec/integration/*"
+  end
+end
+
+desc "Run spec suite with in-memory"
+task :spec => ["spec:memory"]
+
+task :default => :spec
+
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "sqs_web #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+Bundler::GemHelper.install_tasks

data/bin/sqs_web

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+#
+begin
+  require 'sqs_web/application/app.rb'
+rescue LoadError => e
+  require 'rubygems'
+  path = File.expand_path '../../lib', __FILE__
+  $:.unshift(path) if File.directory?(path) && !$:.include?(path)
+  require 'sqs_web/application/app.rb'
+end

data/lib/sqs_web/application/app.rb

@@ -0,0 +1,89 @@
+require 'sinatra/base'
+require 'active_support'
+require 'aws-sdk'
+
+class SqsWeb < Sinatra::Base
+  include Navigation
+  include ControllerAction
+
+  class << self
+    def options
+      @@options ||= {queues: [], aws: {region: ENV['AWS_REGION'] || 'us-east-1'}}
+    end
+  end
+
+  helpers do
+    def flash_message
+      @flash_message ||= FlashMessage.new(session)
+    end
+  end
+
+  set :root, File.dirname(__FILE__)
+  set :static, true
+  set :public_folder, File.expand_path('../public', __FILE__)
+  set :views, File.expand_path('../views', __FILE__)
+  set :show_exceptions, false
+
+  error Exception do |exception|
+    @error = ExceptionHandle.new(exception)
+    erb :error
+  end
+
+  # Enable sessions so we can use CSRF protection
+  enable :sessions
+
+  set :protection,
+    # Various session protections
+    :session => true,
+    # Various non-default Rack::Protection options
+    :use => [
+      # Prevent destructive actions without a valid CSRF auth token
+      :authenticity_token,
+      # Prevent destructive actions with remote referrers
+      :remote_referrer
+    ],
+    # Deny the request, don't clear the session
+    :reaction => :deny
+
+  get '/overview' do
+    @stats = queue_stats
+    erb :overview
+  end
+
+  get "/dlq_console" do
+    @messages = messages
+    erb :dlq_console
+  end
+
+  %w(remove requeue).each do |action|
+    post "/#{action}/:queue_name/:message_id" do
+      process_page_single_request(action: action, params: params)
+      redirect back
+    end
+  end
+
+  %w(bulk_remove bulk_requeue).each do |action|
+    post "/#{action}" do
+      process_page_bulk_request(action: action.split('bulk_')[1], params: params) if params["message_collection"]
+      redirect back
+    end
+  end
+
+  get "/?" do
+    redirect u(:overview)
+  end
+
+  private
+  def queue_stats
+    @queues ||= SqsAction.load_queue_urls
+    SqsAction.get_queue_stats(@queues)
+  end
+
+  def messages(options={})
+    @queues ||= SqsAction.load_queue_urls
+    @queues.select{|queue| queue[:source_url]}.each_with_object([]) do |queue, messages_result|
+      PollerAction.poll_by_queue_and_result_and_options(queue, messages_result, options.merge(flash_message: flash_message))
+      SqsAction.expire_messages(messages_result.reject{|c| c[:deleted]})
+    end
+  end
+end

data/lib/sqs_web/application/controller_action.rb

@@ -0,0 +1,16 @@
+module ControllerAction
+  def process_page_single_request(options={})
+    result = messages({action: options[:action].to_sym, message_id: options[:params][:message_id], queue_name: options[:params][:queue_name]})
+    flash_message.message = "Message ID: #{options[:params][:message_id]} in Queue #{options[:params][:queue_name]} has already been deleted or is not visible." if result.empty?
+  end
+
+  def process_page_bulk_request(options={})
+    options[:params]["message_collection"].map!{|c| {message_id: c.split('/', 2)[0], queue_name: c.split('/', 2)[1]}}
+    result = messages({action: options[:action].to_sym, messages: options[:params]["message_collection"], bulk_action: true})
+    flash_message.message = if result.select{|c| c[:deleted]}.size != options[:params]["message_collection"].size
+      "One or more messages may have already been #{options[:action]}d or is not visible."
+    else
+      "Selected messages have been #{options[:action]}d successfully."
+    end
+  end
+end

data/lib/sqs_web/application/exception_handle.rb

@@ -0,0 +1,9 @@
+class ExceptionHandle
+  attr_reader :error_class, :error_message, :error_backtrace
+
+  def initialize(error)
+    @error_class = error.inspect.to_s
+    @error_message = error.message.to_s
+    @error_backtrace = error.backtrace.to_s
+  end
+end

data/lib/sqs_web/application/flash_message.rb

@@ -0,0 +1,15 @@
+class FlashMessage
+  def initialize(session)
+    @session ||= session
+  end
+
+  def message=(message)
+    @session[:flash_message] = message
+  end
+
+  def message
+    message = @session[:flash_message] #tmp get the value
+    @session[:flash_message] = nil # unset the value
+    message # display the value
+  end
+end

data/lib/sqs_web/application/navigation.rb

@@ -0,0 +1,42 @@
+module Navigation
+  def url_path(*path_parts)
+    [ path_prefix, path_parts ].join("/").squeeze('/')
+  end
+
+  alias_method :u, :url_path
+
+  def h(text)
+    Rack::Utils.escape_html(text)
+  end
+
+  def path_prefix
+    request.env['SCRIPT_NAME']
+  end
+
+  def tabs
+    [
+      {:name => 'Overview', :path => '/overview'},
+      {:name => 'DLQ Console', :path => '/dlq_console'}
+    ]
+  end
+
+  def csrf_token
+    # Set up by Rack::Protection
+    session[:csrf]
+  end
+
+  def csrf_token_tag
+    # If csrf_token is nil, and we submit a blank string authenticity_token
+    # param, Rack::Protection will fail.
+    if csrf_token
+      "<input type='hidden' name='authenticity_token' value='#{h csrf_token}'>"
+    end
+  end
+
+  def partial(template, local_vars = {})
+    @partial = true
+    erb(template.to_sym, {:layout => false}, local_vars)
+  ensure
+    @partial = false
+  end
+end

data/lib/sqs_web/application/poller_action.rb

@@ -0,0 +1,41 @@
+class PollerAction
+  
+  protected
+  def self.poll_by_queue_and_result_and_options(queue, result, options)
+    poller = Aws::SQS::QueuePoller.new(queue[:url], {client: SqsAction.sqs, skip_delete: true, idle_timeout: 0.2, 
+     wait_time_seconds: 0, visibility_timeout: options[:visibility_timeout] || 5 })
+    Timeout::timeout(30) do
+      poller.poll do |message|
+        process_message_by_result_and_options(result, 
+          options.merge({ poller: poller, message: message, queue: queue})
+        )
+      end
+    end
+  end
+
+  def self.process_message_by_result_and_options(result, options)
+    if message_match(options)
+      SqsAction.process_action_by_message(options[:message], options)
+      signal_deleted_message(result, options)
+    else
+      result << {message: options[:message], queue: options[:queue]}
+    end
+  end
+
+  def self.message_match(options)
+    if options[:bulk_action]
+      options[:messages].find{|message| match_message_by_options(message[:message_id], message[:queue_name], options)}
+    else
+      options[:action] && match_message_by_options(options[:message_id], options[:queue_name], options)
+    end
+  end
+
+  def self.match_message_by_options(message_id, queue_name, options)
+    message_id == options[:message].message_id && queue_name == options[:queue][:name]
+  end
+
+  def self.signal_deleted_message(result, options)
+    result << {message: options[:message], queue: options[:queue], deleted: true}
+    throw :stop_polling unless (options[:messages] && result.select{|c| c[:deleted]}.size < options[:messages].size)
+  end
+end

data/lib/sqs_web/application/public/javascripts/application.js

@@ -0,0 +1,58 @@
+$(function() {
+  var poll_interval = 3;
+
+  var relatizer = function(){
+    var dt = $(this).text(), relatized = $.relatizeDate(this)
+    if ($(this).parents("a").length > 0 || $(this).is("a")) {
+      $(this).relatizeDate()
+      if (!$(this).attr('title')) {
+        $(this).attr('title', dt)
+      }
+    } else {
+      $(this)
+      .text('')
+      .append( $('<a href="#" class="toggle_format" title="' + dt + '" />')
+              .append('<span class="date_time">' + dt +
+                      '</span><span class="relatized_time">' +
+                        relatized + '</span>') )
+    }
+  };
+
+  $('.time').each(relatizer);
+
+  $('.time a.toggle_format .date_time').hide();
+
+  var format_toggler = function(){
+    $('.time a.toggle_format span').toggle();
+    $(this).attr('title', $('span:hidden',this).text());
+    return false;
+  };
+
+  $('.time a.toggle_format').click(format_toggler);
+
+  $('ul li.job').hover(function() {
+    $(this).addClass('hover');
+  }, function() {
+    $(this).removeClass('hover');
+  })
+
+  $('a.backtrace').click(function (e) {
+    e.preventDefault();
+    if($(this).prev('div.backtrace:visible').length > 0) {
+      $(this).next('div.backtrace').show();
+      $(this).prev('div.backtrace').hide();
+    } else {
+      $(this).next('div.backtrace').hide();
+      $(this).prev('div.backtrace').show();
+    }
+  });
+
+  $("#select_all").click(function (e) {
+    var checked = $(this).is(':checked');
+    $('.bulk_check').attr('checked', checked);
+  });
+
+  $("#bulk_action_submit input").click(function() {
+    $('#bulk_action_form').attr("action", $(this).attr("action"));
+  });
+})