emque-consuming 1.0.0.beta4

data/lib/emque/consuming/runner.rb

@@ -0,0 +1,168 @@
+require "emque/consuming/application"
+require "emque/consuming/control"
+require "emque/consuming/pidfile"
+require "emque/consuming/status"
+require "emque/consuming/command_receivers/http_server"
+require "emque/consuming/command_receivers/unix_socket"
+require "emque/consuming/transmitter"
+
+module Emque
+  module Consuming
+    class Runner
+      include Emque::Consuming::Helpers
+
+      class << self
+        attr_accessor :instance
+      end
+
+      attr_reader :control, :pidfile, :status
+
+      def initialize(options = {})
+        self.control = Emque::Consuming::Control.new
+        self.options = options
+        self.receivers = []
+        self.status = Emque::Consuming::Status.new
+        apply_options
+        Emque::Consuming.application.initialize_logger
+        self.class.instance = self
+        self.pidfile = options.fetch(:pidfile, default_pidfile)
+        self.pid = Emque::Consuming::Pidfile.new(pidfile)
+      end
+
+      def app
+        super
+      end
+
+      def console
+        require "pry"
+        Pry.start
+      end
+
+      def http?
+        config.status == :on
+      end
+
+      def phased_restart
+        receivers.each { |r| r.stop && r.start }
+      end
+
+      def restart
+        stop && start
+      end
+
+      def restart_application
+        receivers.first.restart
+      end
+
+      def sock?
+        true
+      end
+
+      def start
+        exit_if_already_running!
+        daemonize! if daemonize?
+        write_pidfile!
+        @persist = Thread.new { loop { sleep 1 } }
+        set_process_title
+        setup_receivers
+        receivers.each(&:start)
+        persist.join
+      rescue Interrupt
+        stop
+      end
+
+      def stop(timeout: 5)
+        if persist
+          Thread.new do
+            sleep timeout
+            logger.error("Timeout Exceeded. Forcing Shutdown.")
+            persist.exit if persist.alive?
+          end
+          receivers.each(&:stop)
+          logger.info("Graceful shutdown successful.")
+          logger.info("#{config.app_name.capitalize} stopped.")
+          persist.exit if persist.alive?
+        else
+          Emque::Consuming::Transmitter.send(
+            :command => :stop,
+            :socket_path => config.socket_path
+          )
+        end
+      end
+
+      private
+
+      attr_accessor :options, :persist, :pid, :receivers
+      attr_writer :control, :pidfile, :status
+
+      def apply_options
+        options.each do |attr, val|
+          config.send("#{attr}=", val) if config.respond_to?(attr)
+        end
+      end
+
+      def config
+        Emque::Consuming.application.config
+      end
+
+      def daemonize?
+        options[:daemon]
+      end
+
+      def daemonize!
+        Process.daemon(true, true)
+
+        [$stdout, $stderr].each do |io|
+          File.open(Emque::Consuming.application.logfile, "ab") do |f|
+            io.reopen(f)
+          end
+          io.sync = true
+        end
+
+        $stdin.reopen("/dev/null")
+      end
+
+      def default_pidfile
+        File.join(
+          Emque::Consuming.application.root,
+          "tmp",
+          "pids",
+          "#{config.app_name}.pid"
+        )
+      end
+
+      def exit_if_already_running!
+        if pid.running?
+          [
+            "Pid file exists. Process #{pid} active.",
+            "Please ensure app is not running."
+          ].each do |msg|
+            logger.error(msg)
+            $stdout.puts(msg)
+          end
+
+          exit
+        end
+      end
+
+      def set_process_title
+        title = "#{config.app_name} [pidfile: #{pidfile}"
+        title << " | unix socket: #{config.socket_path}" if sock?
+        title << " | http://#{config.status_host}:#{config.status_port}" if http?
+        title << "]"
+        $0 = title
+      end
+
+      def setup_receivers
+        receivers << app
+        receivers << Emque::Consuming::CommandReceivers::UnixSocket.new if sock?
+        receivers << Emque::Consuming::CommandReceivers::HttpServer.new if http?
+      end
+
+      def write_pidfile!
+        pid.write
+        at_exit { FileUtils.rm_f(pidfile) }
+      end
+    end
+  end
+end

data/lib/emque/consuming/status.rb

@@ -0,0 +1,26 @@
+module Emque
+  module Consuming
+    class Status
+      include Emque::Consuming::Helpers
+
+      def to_hsh
+        {
+          :app => config.app_name,
+          :errors => {
+            :count => app.error_tracker.count,
+            :expire_after => app.error_tracker.expiration,
+            :limit => app.error_tracker.limit
+          },
+          :workers => {}.tap { |worker_stats|
+            app.manager.workers.each { |topic, workers|
+              worker_stats[topic] = {
+                :count => workers.size
+              }
+            }
+          }
+        }
+      end
+      alias :to_h :to_hsh
+    end
+  end
+end

data/lib/emque/consuming/tasks.rb

@@ -0,0 +1,121 @@
+module Emque
+  module Consuming
+    class Tasks
+      include Rake::DSL if defined? Rake::DSL
+      include Emque::Consuming::Helpers
+
+      def install_tasks
+        namespace :emque do
+          desc "Show the current configuration of a running instance " +
+               "(accepts SOCKET)"
+          task :configuration do
+            puts with_transmitter(:send, :configuration)
+          end
+
+          desc "Start a pry console"
+          task :console do
+            Emque::Consuming::Runner.new.console
+          end
+
+          namespace :errors do
+            desc "Clear all outstanding errors (accepts SOCKET)"
+            task :clear do
+              puts with_transmitter(:send, :errors, :clear)
+            end
+
+            desc "Change the number of seconds to SECONDS before future " +
+                 "errors expire (accepts SOCKET)"
+            task :expire_after do
+              seconds = ENV.fetch("SECONDS", 3600)
+              puts with_transmitter(:send, :errors, :expire_after, seconds)
+            end
+
+            namespace :limit do
+              desc "Decrease the error limit (accepts SOCKET)"
+              task :down do
+                puts with_transmitter(:send, :errors, :down)
+              end
+
+              desc "Increase the error limit (accepts SOCKET)"
+              task :up do
+                puts with_transmitter(:send, :errors, :up)
+              end
+            end
+          end
+
+          desc "Show the available routes"
+          task :routes do
+            require "table_print"
+            tp(
+              [].tap { |routes|
+                mappings = router.instance_eval { @mappings }
+
+                mappings.each { |topic, maps|
+                  maps.each { |mapping|
+                    mapping.instance_eval { @mapping }.each { |route, method|
+                      routes << {
+                        :route => route,
+                        :topic => topic,
+                        :consumer => mapping.consumer,
+                        :method => method,
+                        :workers => router.workers(topic)
+                      }
+                    }
+                  }
+                }
+              },
+              {:route => {:width => 50}},
+              :topic,
+              :consumer,
+              :method,
+              :workers
+            )
+          end
+
+          desc "Restart the workers inside a running instance " +
+               "(does not reload code; accepts SOCKET)"
+          task :restart do
+            with_transmitter(:send, :restart)
+          end
+
+          desc "Show the current status of a running instance " +
+               "(accepts SOCKET)"
+          task :status do
+            puts with_transmitter(:send, :status)
+          end
+
+          desc "Start a new instance (accepts PIDFILE, DAEMON)"
+          task :start do
+            daemon = ENV.fetch("DAEMON", false)
+            pidfile = ENV.fetch("PIDFILE", "tmp/pids/#{config.app_name}.pid")
+
+            Emque::Consuming::Runner.new({
+              :daemon => daemon,
+              :pidfile => pidfile
+            }).start
+          end
+
+          desc "Stop a running instance (accepts SOCKET)"
+          task :stop do
+            resp = with_transmitter(:send, :stop)
+            puts resp.length > 0 ? resp : "stopped"
+          end
+        end
+      end
+
+      private
+
+      def with_transmitter(method, command, *args)
+        socket_path = ENV.fetch("SOCKET", config.socket_path)
+        require "emque/consuming/transmitter"
+        Emque::Consuming::Transmitter.send(
+          :command => command,
+          :socket_path => socket_path,
+          :args => args
+        )
+      end
+    end
+  end
+end
+
+Emque::Consuming::Tasks.new.install_tasks

data/lib/emque/consuming/transmitter.rb

@@ -0,0 +1,31 @@
+require "fileutils"
+require "socket"
+require "oj"
+
+module Emque
+  module Consuming
+    module Transmitter
+      def self.send(command:, socket_path: "tmp/emque.sock", args: [])
+        if File.exists?(socket_path)
+          socket = UNIXSocket.new(socket_path)
+          socket.send(Oj.dump({
+            :command => command,
+            :args => args
+          }, :mode => :compat), 0)
+          response = socket.recv(10000000)
+          socket.close
+          response
+        else
+          "Socket not found at #{socket_path}"
+        end
+      rescue Errno::ECONNREFUSED
+        FileUtils.rm_f(socket_path) if File.exists?(socket_path)
+        "The UNIX Socket found at #{socket_path} was dead"
+      end
+
+      def self.method_missing(method, *args)
+        send(command: method, args: args)
+      end
+    end
+  end
+end

data/lib/emque/consuming/version.rb

@@ -0,0 +1,5 @@
+module Emque
+  module Consuming
+    VERSION = "1.0.0.beta4"
+  end
+end

data/lib/emque/consuming.rb

@@ -0,0 +1,9 @@
+require "virtus"
+
+module Emque
+  module Consuming; end
+end
+
+require "emque/consuming/version"
+require "emque/consuming/consuming"
+require "emque/consuming/cli"

data/lib/emque-consuming.rb

@@ -0,0 +1,3 @@
+module Emque; end
+
+require "emque/consuming"

data/lib/templates/.gitignore.tt

@@ -0,0 +1,25 @@
+*.rbc
+*.sassc
+.sass-cache
+capybara-*.html
+.rspec
+.rvmrc
+/.bundle
+/vendor/bundle
+/log/*
+/logs/*
+/tmp/*
+/db/*.sqlite3
+/public/system/*
+/coverage/
+/spec/tmp/*
+**.orig
+rerun.txt
+pickle-email-*.html
+.project
+config/initializers/secret_token.rb
+config/*.yml
+.ruby-version
+.rbenv-version
+/bin/stubs
+spec/fixtures/vcr_cassettes/*

data/lib/templates/Gemfile.tt

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+ruby "2.1.2"
+
+gem "rake"
+gem "emque-consuming", :git => "https://github.com/teamsnap/emque-consuming.git"

data/lib/templates/Rakefile.tt

@@ -0,0 +1,7 @@
+require_relative "config/application"
+require "emque/consuming/tasks"
+
+__DIR__ = File.dirname(__FILE__)
+Dir.glob("#{__DIR__}/tasks/*.rake").each do |rake_file|
+  import rake_file
+end

data/lib/templates/config/application.rb.tt

@@ -0,0 +1,42 @@
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", __FILE__)
+require "bundler/setup" if File.exists?(ENV["BUNDLE_GEMFILE"])
+
+require "emque/consuming"
+
+$LOAD_PATH.unshift("#{File.dirname(__FILE__)}/../lib")
+
+module <%= @name %>
+  class Application
+    include Emque::Consuming::Application
+
+    initialize_core!
+
+    <%=
+[].tap { |options_out|
+  if @options.has_key?(:app_name)
+    options_out << "config.app_name = \"#{@options[:app_name]}\""
+  end 
+  if @options.has_key?(:error_limit)
+    options_out << "config.error_limit = #{@options[:error_limit]}"
+  end
+  if @options.has_key?(:error_expiration)
+    options_out << "config.error_expiration = #{@options[:error_expiration]}"
+  end
+  options_out << "config.set_adapter(:rabbit_mq)"
+  if @options.has_key?(:status)
+    options_out << "config.status = :on"
+  end
+  if @options.has_key?(:status_host)
+    options_out << "config.status_host = \"#{@options[:status_host]}\""
+  end
+  if @options.has_key?(:status_port)
+    options_out << "config.status_port = #{@options[:status_port]}"
+  end
+  if @options.has_key?(:socket_path)
+    options_out << "config.socket_path = \"#{@options[:socket_path]}\""
+  end
+}.join("\n    ")
+    %>
+  end
+end
+

data/lib/templates/config/environments/development.rb.tt

@@ -0,0 +1,2 @@
+<%= @name %>::Application.configure do
+end

data/lib/templates/config/environments/production.rb.tt

@@ -0,0 +1,2 @@
+<%= @name %>::Application.configure do
+end

data/lib/templates/config/environments/staging.rb.tt

@@ -0,0 +1,2 @@
+<%= @name %>::Application.configure do
+end

data/lib/templates/config/environments/test.rb.tt

@@ -0,0 +1,2 @@
+<%= @name %>::Application.configure do
+end

data/lib/templates/config/routes.rb.tt

@@ -0,0 +1,8 @@
+<%= @name %>::Application.router.map do
+  # topic "samples" => SamplesConsumer[, :workers => 2] do
+    # map "samples.new" => "new_sample"
+    # ...
+  # end
+
+  # ...
+end

data/spec/application_spec.rb

@@ -0,0 +1,28 @@
+require "spec_helper"
+
+describe Emque::Consuming::Application do
+  describe "#notice_error" do
+    it "does not trigger a shutdown if the limit has not been reached" do
+      Dummy::Application.config.error_limit = 2
+      app = Dummy::Application.new
+      Emque::Consuming::Runner.instance = double(:runner)
+
+      expect(Emque::Consuming::Runner.instance).to_not receive(:stop)
+
+      app.notice_error({ :test => "failure" })
+    end
+
+    it "triggers a shutdown one the error_limit is reached" do
+      Dummy::Application.config.error_limit = 2
+      app = Dummy::Application.new
+      Emque::Consuming::Runner.instance =
+        double(:runner, :status => double(:status, :to_h => {}))
+
+      expect(Emque::Consuming::Runner.instance)
+        .to receive(:stop).exactly(1).times
+
+      app.notice_error({ :test => "failure" })
+      app.notice_error({ :test => "another failure" })
+    end
+  end
+end

data/spec/cli_spec.rb

@@ -0,0 +1,136 @@
+require "spec_helper"
+
+describe Emque::Consuming::Cli do
+  describe "starting a console" do
+    it "passes to console command to a new runner instance" do
+      runner = double(:runner, :send => true)
+
+      expect(Emque::Consuming::Runner).to receive(:new).and_return(runner)
+      expect(runner).to receive(:send).with(:console)
+
+      Emque::Consuming::Cli.new(["console"])
+    end
+  end
+
+  describe "starting an instance" do
+    it "passes the start command to a new runner instance" do
+      runner = double(:runner, :send => true)
+
+      expect(Emque::Consuming::Runner).to receive(:new).and_return(runner)
+      expect(runner).to receive(:send).with(:start)
+
+      Emque::Consuming::Cli.new(["start"])
+    end
+
+    it "passes valid arguments along to the new runner" do
+      command = ["start"]
+      valid_args = [
+        "-P", "tmp/pidfile.pid",
+        "-d",
+        "-S", "tmp/socket.sock",
+        "-e", "20",
+        "-x", "1000",
+        "--app-name", "testing",
+        "--env", "test"
+      ]
+      expected_options = {
+        :daemon => true,
+        :pidfile => "tmp/pidfile.pid",
+        :socket_path => "tmp/socket.sock",
+        :error_limit => 20,
+        :error_expiration => 1000,
+        :app_name => "testing",
+        :env => "test"
+      }
+      argv = valid_args + command
+      runner = double(:runner, :send => true)
+
+      expect(Emque::Consuming::Runner).to receive(:new)
+        .with(expected_options)
+        .and_return(runner)
+
+      Emque::Consuming::Cli.new(argv)
+    end
+
+    describe "with no options" do
+      it "passes the default options to the runner" do
+        expected_options = {
+          :daemon => false
+        }
+        runner = double(:runner, :send => true)
+
+        expect(Emque::Consuming::Runner).to receive(:new)
+          .with(expected_options)
+          .and_return(runner)
+
+        Emque::Consuming::Cli.new(["start"])
+      end
+    end
+  end
+
+  describe "stopping an instance" do
+    it "passes the stop command to a new runner instance" do
+      runner = double(:runner, :send => true)
+
+      expect(Emque::Consuming::Runner).to receive(:new).and_return(runner)
+      expect(runner).to receive(:send).with(:stop)
+
+      Emque::Consuming::Cli.new(["stop"])
+    end
+  end
+
+  describe "creating a new application" do
+    it "passes the new command to a new application generator instance" do
+      generator = double(:generator, :generate => true)
+
+      expect(Emque::Consuming::Generators::Application).to receive(:new)
+        .and_return(generator)
+      expect(generator).to receive(:generate)
+
+      Emque::Consuming::Cli.new(["new", "testapplication"])
+    end
+
+    it "passes valid arguments along to the new application generator" do
+      application_name = "testapplication"
+      command = ["new", application_name]
+      valid_args = [
+        "-P", "tmp/pidfile.pid",
+        "-d",
+        "-S", "tmp/socket.sock",
+        "-e", "20",
+        "-x", "1000",
+        "--app-name", "testing",
+        "--env", "test"
+      ]
+      expected_options = {
+        :daemon => true,
+        :pidfile => "tmp/pidfile.pid",
+        :socket_path => "tmp/socket.sock",
+        :error_limit => 20,
+        :error_expiration => 1000,
+        :app_name => "testing",
+        :env => "test"
+      }
+      argv = valid_args + command
+      generator = double(:generator, :generate => true)
+
+      expect(Emque::Consuming::Generators::Application).to receive(:new)
+        .with(expected_options, application_name)
+        .and_return(generator)
+
+      Emque::Consuming::Cli.new(argv)
+    end
+
+    it "exits if no application name is passed" do
+      expect { Emque::Consuming::Cli.new(["new"]) }.to raise_error(SystemExit)
+    end
+  end
+
+  it "exits if an invalid command is passed" do
+    expect { Emque::Consuming::Cli.new(["invalid"]) }.to raise_error(SystemExit)
+  end
+
+  it "exits if help is passed as a command" do
+    expect { Emque::Consuming::Cli.new(["help"]) }.to raise_error(SystemExit)
+  end
+end

data/spec/configuration_spec.rb

@@ -0,0 +1,47 @@
+require "spec_helper"
+
+describe Emque::Consuming::Configuration do
+  describe "#log_level" do
+    it "defaults to Logger::INFO" do
+      config = Emque::Consuming::Configuration.new
+      expect(config.log_level).to eq(Logger::INFO)
+    end
+
+    it "prefers the assigned value to the default" do
+      config = Emque::Consuming::Configuration.new
+      config.log_level = Logger::DEBUG
+      expect(config.log_level).to eq(Logger::DEBUG)
+    end
+  end
+
+  describe "#to_hsh" do
+    it "returns a hash" do
+      config = Emque::Consuming::Configuration.new
+      expect(config.to_hsh).to be_a Hash
+    end
+
+    it "returns the value of all the accessors" do
+      accessors = [
+        :app_name, :adapter, :env, :error_handlers, :error_limit,
+        :error_expiration, :log_level, :status_port, :status_host, :status,
+        :socket_path, :shutdown_handlers
+      ]
+      config = Emque::Consuming::Configuration.new
+
+      hsh = config.to_hsh
+
+      expect(hsh.keys).to eq(accessors)
+      accessors.each do |key|
+        expect(hsh.fetch(key)).to eq(config.send(key))
+      end
+    end
+  end
+
+  describe "#to_h" do
+    it "is an alias of to_hsh" do
+      config = Emque::Consuming::Configuration.new
+
+      expect(config.to_h).to eq(config.to_hsh)
+    end
+  end
+end