phantom-manager 0.0.2

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MmU1OGFhZGQzODQ2NGJkNjkwN2FkNmM3ZWI0NTE3ZWI4NWY5NzEzNA==
+  data.tar.gz: !binary |-
+    OGJjMTI5Yzk1NjVkOTczM2E4MWEyMWM2NGMxMmQ2NmUzMTdmYjM4MQ==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    MjI3NTQwNDNjMWVmZGRkYzk3NTYzZmE5MDE4YmFiNjkwNGFjMmJmMTkxOWE0
+    ODQ1MTRlYzY3NjFmMzg0MjZiZGM5ODIyOWY2NWY0NjgyNWQ0M2UyZTI1NDcx
+    MDMxZmUzZTM0OTVlMzFmYWVkNGI5MGNiZDVjNzEyYTU2Yjk3MmE=
+  data.tar.gz: !binary |-
+    MzdkODY4MjYyNDc4MzFmYjlhNTc2NGE2MDg5MmMwOWI2ZTA4ZjI4ZTkwYTA2
+    YTU2OTdlMjI4MmJlNmUyZmEyYzAxOWE5ZWQyMzQzZDdiNWRmYWFmNWM5ODRj
+    ZDJiYjUzNTU4YzcyYmYyZTlhZTdiZmEzZDhmOGUzYTQwZjcyMGQ=

data/.gitignore

@@ -0,0 +1,3 @@
+.rspec
+tags
+log/*

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in phantom-manager.gemspec
+gemspec
+
+gem 'rspec'
+gem 'debugger'

data/Gemfile.lock

@@ -0,0 +1,36 @@
+PATH
+  remote: .
+  specs:
+    phantom-manager (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    columnize (0.3.6)
+    debugger (1.4.0)
+      columnize (>= 0.3.1)
+      debugger-linecache (~> 1.1.1)
+      debugger-ruby_core_source (~> 1.2.0)
+    debugger-linecache (1.1.2)
+      debugger-ruby_core_source (>= 1.1.1)
+    debugger-ruby_core_source (1.2.0)
+    diff-lcs (1.2.4)
+    rake (10.1.0)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.3)
+    rspec-expectations (2.14.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.3)
+  debugger
+  phantom-manager!
+  rake
+  rspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Erez Rabih
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,32 @@
+# Phantom::Manager
+
+phantom-manager allows you to use multiple phantom-js processes behind an Nginx
+server. It will manage both presence and memory consumption of those processes
+and kill them when appropriate, all this in sync with the Nginx configuration
+so that all requests will get answered.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'phantom-manager'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install phantom-manager
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/phantom_monitor

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+
+$options = {}
+
+optparse = OptionParser.new do |opts|
+
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts opts
+    exit
+  end
+
+  opts.on( '-c', '--config FILE', "Configuration file path" ) do |f|
+    $options[:config] = f
+  end
+
+  opts.on( '-e', '--env ENVIRONMENT', "Environment to run on" ) do |env|
+    $options[:env] = env
+  end
+  
+end
+
+optparse.parse!
+
+mandatory = [:env, :config]
+missing = mandatory.select{ |param| $options[param].nil? }
+
+if not missing.empty?
+  puts "Missing options: #{missing.join(', ')}"
+  puts optparse
+  exit
+end
+
+lib = File.expand_path('../../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'phantom/manager/version'
+require 'monitors/memory.rb'
+require 'monitors/processes.rb'
+require 'monitors/restart_listener.rb'
+
+
+$logger.info "Starting for environment #{$options[:env]} with config file #{$options[:config]}... "
+
+Monitors::RestartListener.run
+
+$logger.info "Running memory monitor for phantom processes"
+
+Thread.new {Monitors::Memory.run}
+
+$logger.info "Running processes monitor for phantom processes"
+
+Thread.new {Monitors::Processes.run}.join
+

data/config/config.yml

@@ -0,0 +1,27 @@
+development:
+  #The time to wait between removing a phantomjs from the nginx configuration
+  #and actually killing the process.
+  phantom_termination_grace: 10
+
+  #This will be the inital port of the phantomjs processes. For example if
+  #phantom_processes_number is set to 4 and phantom_base_port to 8000 then 4
+  #phantoms, on ports 8000 8001 8002 8003 will be managed.
+  phantom_base_port: 8002
+  #Rails root path
+  rails_root: "path"
+  #Path to your nginx configuration
+  nginx_conf: "path"
+  #Path to where you want to keep your phantomjs logs
+  phantom_log_path: "path"
+  #If a phantom process memory consumption is over memory_limit for
+  #memory_retries times with memory_check_interval between each check it will
+  #be killed
+  memory_limit: 150000
+  memory_retries: 5
+  memory_check_interval: 5
+  #Number of phantom processes to keep up. If a process is missing for
+  #processes_check_retries times in a row with processes_check_interval between
+  #checks it will be brought up byt the monitor
+  phantom_processes_number: 10
+  processes_check_retries: 3
+  processes_check_interval: 15

data/lib/monitors/base.rb

@@ -0,0 +1,39 @@
+require 'utils/cfg'
+require 'utils/logger'
+require 'phantom/manager'
+require 'phantom/collector'
+
+module Monitors
+  class Base
+
+    class << self
+      def run(custom_logger = $logger)
+        @logger = custom_logger
+
+        loop do
+          perform_check
+          sleep check_interval
+        end
+      end
+
+      protected
+
+      def perform_check
+        raise NotImplementedError.new
+      end
+
+      def check_interval
+        raise NotImplementedError.new
+      end
+      
+      def running_instances
+        Phantom::Collector.get_running_instances
+      end
+
+      def logger
+        @logger ||= $logger
+      end
+    end
+
+  end
+end

data/lib/monitors/memory.rb

@@ -0,0 +1,27 @@
+require 'monitors/base'
+require 'monitors/violations_recorders/memory'
+
+module Monitors
+  class Memory < Base
+
+    class << self
+
+      def perform_check
+        logger.info "Perfoming memory check..."
+
+        running_instances.each do |p|
+          if ViolationsRecorders::Memory.is_violating?(p)
+            logger.info "process #{p.pid} was found bad!"
+            Phantom::Manager.restart(p)
+          end
+        end
+
+      end
+
+      def check_interval
+        $cfg.memory_check_interval
+      end
+    end
+
+  end
+end

data/lib/monitors/processes.rb

@@ -0,0 +1,33 @@
+require 'monitors/base'
+require 'monitors/violations_recorders/processes'
+
+module Monitors
+  class Processes < Base
+
+    class << self
+      def perform_check
+        logger.info "Perfoming processes check..."
+
+        missing_processes = Phantom::Collector.missing_ports.map do |port|
+          p = Phantom::Process.new
+          p.port = port
+          p
+        end
+
+        missing_processes.each do |p|
+          if ViolationsRecorders::Processes.is_violating?(p)
+            logger.info "found missing phantom on port #{p.port}"
+            Phantom::Manager.start(p)
+          end
+        end
+
+        logger.info "All processes are running OK" if missing_processes.empty?
+      end
+
+      def check_interval
+        $cfg.processes_check_interval
+      end
+    end
+
+  end
+end

data/lib/monitors/restart_listener.rb

@@ -0,0 +1,39 @@
+require 'phantom/manager'
+require 'utils/logger'
+require 'utils/cfg'
+require 'utils/lock'
+
+
+module Monitors
+  class RestartListener
+    class << self
+
+      def run
+        Signal.trap("USR2") do
+          respond_to_signal
+        end
+      end
+
+      def respond_to_signal
+        lock.acquire do
+          $logger.info "Initiating restart sequence"
+
+          Phantom::Collector.get_running_instances.each do |p|
+            $logger.info "Restarting process on port #{p.port}"
+            Phantom::Manager.restart(p)
+
+            sleep $cfg.phantom_termination_grace
+          end
+
+        end
+      end
+
+      private
+
+      def lock
+        @lock ||= Utils::Lock.new
+      end
+
+    end
+  end
+end

data/lib/monitors/violations_recorders/base.rb

@@ -0,0 +1,56 @@
+require 'utils/logger'
+require 'utils/cfg'
+require 'phantom/process'
+
+module Monitors
+  module ViolationsRecorders
+    class Base
+
+      class << self
+
+        def inherited(base)
+          base.instance_variable_set(:@violations, {})
+        end
+
+        @violations = {}
+      
+        def reset
+          @violations = {}
+        end
+
+        def is_violating?(process)
+          $logger.debug "checking #{process}"
+          update_violations_count(process)
+          $logger.debug "#{@violations}"
+          violating = @violations[process.send(process_attr)] == retries_limit
+          @violations[process.send(process_attr)] = 0 if violating
+          violating
+        end
+
+        protected
+
+        def update_violations_count(process)
+          @violations[process.send(process_attr)] ||= 0
+          if process_is_violating?(process)
+            @violations[process.send(process_attr)]+= 1 
+          else
+            @violations[process.send(process_attr)] = 0
+          end
+        end
+
+        def retries_limit
+          raise NotImplementedError
+        end
+
+        def process_attr
+          raise NotImplementedError
+        end
+
+        def process_is_violating?(process)
+          raise NotImplementedError
+        end
+
+      end
+    end
+  end
+end

data/lib/monitors/violations_recorders/memory.rb

@@ -0,0 +1,23 @@
+require 'monitors/violations_recorders/base'
+
+module Monitors
+  module ViolationsRecorders
+    class Memory < Base
+      class << self
+
+        def retries_limit
+          $cfg.memory_retries
+        end
+
+        def process_attr
+          :pid
+        end
+
+        def process_is_violating?(process)
+          process.memory_usage > $cfg.memory_limit
+        end
+
+      end
+    end
+  end
+end

data/lib/monitors/violations_recorders/processes.rb

@@ -0,0 +1,21 @@
+require 'monitors/violations_recorders/base'
+
+module Monitors
+  module ViolationsRecorders
+    class Processes < Base
+      class << self
+        def retries_limit
+          $cfg.processes_check_retries
+        end
+
+        def process_attr
+          :port
+        end
+
+        def process_is_violating?(process)
+          Phantom::Collector.missing_ports.include?(process.port)
+        end
+      end
+    end
+  end
+end

data/lib/nginx/manager.rb

@@ -0,0 +1,59 @@
+require 'utils/logger'
+require 'utils/cfg'
+
+module Nginx
+  class Manager
+    class << self
+
+      def remove(port)
+        $logger.info "removing #{port} from nginx"
+        modify_nginx do |ofile, iline|
+          ofile.puts(iline) unless iline =~ /#{port}/
+        end
+      end
+
+      def add(port)
+        $logger.info "adding #{port} to nginx"
+        if !port_defined?(port)
+          modify_nginx do |ofile, iline|
+            ofile.puts(iline)
+            ofile.puts(phantom_upstream(port)) if iline =~ /upstream phantomjs/
+          end
+        else
+          $logger.info "port #{port} already defined"
+        end
+      end
+
+      private
+
+      def phantom_upstream(port)
+        "  server 127.0.0.1:#{port} fail_timeout=0; # #{Time.now}"
+      end
+
+      def port_defined?(port)
+        File.readlines($cfg.nginx_conf).grep(/#{port}/).size > 0
+      end
+
+      def switch_nginx_configs
+        $logger.info "switching nginx configurations"
+        `mv #{$cfg.new_nginx_conf} #{$cfg.nginx_conf}`
+      end
+
+      def reload_nginx
+        $logger.info "reloading nginx"
+        `nginx -s reload`
+      end
+
+      def modify_nginx
+        File.open($cfg.new_nginx_conf, "w") do |ofile|
+          File.foreach($cfg.nginx_conf) do |iline|
+            yield ofile, iline
+          end
+        end
+        switch_nginx_configs
+        reload_nginx
+      end
+
+    end
+  end
+end

data/lib/phantom/collector.rb

@@ -0,0 +1,36 @@
+require 'phantom/process'
+
+module Phantom
+  class Collector
+    class << self
+
+      def on_port(port)
+        get_running_instances.find {|p| p.port == port}
+      end
+
+      def get_running_instances
+        lines = running_phantoms_shell_output.split("\n")
+        lines.map {|l| Phantom::Process.from_string(l) }
+      end
+
+      def missing_ports
+        required_ports - get_running_instances.map(&:port)
+      end
+
+      def required_ports
+        ($cfg.phantom_base_port..($cfg.phantom_base_port+$cfg.phantom_processes_number-1)).to_a
+      end
+
+      private
+
+      def running_phantoms_shell_output
+        `#{running_phantoms_shell_command}`
+      end
+
+      def running_phantoms_shell_command
+        "ps -e -www -o pid,rss,command | grep 'phantomjs' | grep -v sh | grep -v grep"
+      end
+
+    end
+  end
+end

data/lib/phantom/manager/version.rb

@@ -0,0 +1,5 @@
+module Phantom
+  module Manager
+    VERSION = "0.0.2"
+  end
+end

data/lib/phantom/manager.rb

@@ -0,0 +1,34 @@
+require "phantom/manager/version"
+require 'utils/logger'
+require 'utils/cfg'
+require 'nginx/manager'
+
+module Phantom
+  module Manager
+    class << self
+
+      def restart(process)
+        $logger.info "restarting process #{process}"
+        stop process
+        start process
+      end
+
+      def start(process)
+        $logger.info "starting process #{process.port}"
+        process.start
+        Nginx::Manager.add(process.port)
+      end
+
+      private 
+
+      def stop(process)
+        $logger.info "stopping process #{process}"
+        Nginx::Manager.remove(process.port)
+        sleep $cfg.phantom_termination_grace
+        process.kill
+      end
+
+    end
+  end
+end
+

data/lib/phantom/process.rb

@@ -0,0 +1,54 @@
+require 'utils/logger'
+require 'utils/cfg'
+require 'utils/shell'
+
+module Phantom
+  class Process
+    attr_accessor :pid, :memory_usage, :command, :port
+
+    class << self
+
+      def from_string(str)
+        args = parse_string(str)
+        new(*args)
+      end
+
+      private
+      
+      def parse_string(str)
+        parts = str.split
+        pid = parts[0].to_i
+        memory_usage = parts[1].to_i
+        command = parts[2..-1].join(" ")
+        port = parts.last.to_i
+
+        [pid, memory_usage, command, port]
+      end
+
+    end
+
+    def initialize(pid = nil, memory_usage = nil, command = nil, port = nil)
+      @pid = pid
+      @memory_usage = memory_usage
+      @command = command
+      @port = port
+    end
+
+    def kill
+      $logger.info "killing #{self}"
+      Utils::Shell.execute "kill #{pid}"
+    end
+
+    def start
+      $logger.info "starting phantomjs on port #{port}"
+      Utils::Shell.execute start_command
+    end
+
+    private
+
+    def start_command
+      "cd #{$cfg.rails_root} && phantomjs rndrme.js #{port} >>#{$cfg.phantom_log_path} 2>&1 &"
+    end
+
+  end
+end

data/lib/utils/cfg.rb

@@ -0,0 +1,18 @@
+require 'ostruct'
+require 'yaml'
+
+class Cfg
+  def self.load
+    cfg =  File.expand_path($options[:config])
+    env = $options[:env]
+    hash = YAML.load(File.open(cfg))[env]
+    hash.each do |k,v|
+      hash[k] = v.to_i if v =~ /^\d+$/
+    end
+    obj = OpenStruct.new hash
+    obj.new_nginx_conf = "#{obj.nginx_conf}.new"
+
+    $cfg = obj
+  end
+end
+Cfg.load