phantom-manager 0.0.7 → 0.0.8

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YjQ1YmY5NmYzMDgwMjAxN2MzMGU3OTcyMDZkM2NiNmQzNjNlMzRiZA==
+    MjUzNTIwOGY1MmNjYjRlNTAxYmZjZjJhNDllMjYxNjVkN2JiOGZmZg==
   data.tar.gz: !binary |-
-    NzNiNWE1MTgxYTI1N2RlMDJhZjc1YmE2YWM5ZWQ2M2RmNGRjYjg2NA==
+    MzRjOWYwZTBmN2JmMWY4NTc0M2UxY2UwMGQwYmJmOWQyZmRjNmQwOA==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    NzA2YzFlMTM1NGUxZGVlMDdiNmEwYjMxYjVkNWE0MzdiZTAyMzMyMmI2NGI2
-    ODNlMzljM2IxN2RhMzBmMGY1NzNjMmViMTVlZDZlMWMzZmU4ZDUyOTA0OTA0
-    N2MwMjFkNDE0ZjhlYjdmYWMzN2M3MDkxYjVjMTdiMjQyMDllMzM=
+    MjEzMTA0NjNkOTIxMGMxY2YzM2E1NDE0OWI4YzZkYzRlYzY2ZGYxYTYzMmIz
+    MWVlNjIxOTA4MzZhNWQ3ZGUwMGM5MDMxYzg3NDQ5MThiMDk5YWViMWEwY2M4
+    NWFlMmQ2YTFjNWQ2MDlhNTQzYzUxYTBiMTJlZDM3ODQ2YzcwNmI=
   data.tar.gz: !binary |-
-    ODg4OGQ5YzNlZmJhNjMxZTA0NmQxNGNkM2E2MTUzYjhjZWI4NDVlMmFkYmYz
-    MjliOWM3YWI2MjA3MTI1ZTg3OTlkMDdiNzVjYWIyYWFlOTVmODQ4NzJjNjFl
-    Y2E1MGJhNmQ1NDAzYjAyNWQ0NWVmNDYyZDVjYTJhYWMxZDdjZmI=
+    MzEzNDM5NGE4YjZiNzJjYmQxY2IyN2FmNTAxYTkwZWNjNTc4MzdhYWY0ZWM0
+    MWQ0ZjhkMjZiYmJiNTJhYTUyZjJjOGYxY2FlNWY1NjU0OWVkNzhiNzQ0MTA5
+    ZWNkNjA1ODZkMjBmZjUzMmI3M2U2NDhkZmI5NGVkNjMzY2NiNDk=

data/Gemfile

@@ -5,3 +5,4 @@ gemspec
 
 gem 'rspec'
 gem 'debugger'
+

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    phantom-manager (0.0.5)
+    phantom-manager (0.0.7)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -1,10 +1,25 @@
 # Phantom::Manager
 
+The architecture behind phantom-manager is:
+
+![Phantom-Architecture](http://i39.tinypic.com/2gxnz3d.png)
+
 phantom-manager allows you to use multiple phantom-js processes behind an Nginx
 server. It will manage both presence and memory consumption of those processes
 and kill them when appropriate, all this in sync with the Nginx configuration
 so that all requests will get answered.
 
+If you've got a singlepage application and you want to:
+* Render full page for GoogleBot or other web crawlers.
+* Render full page to be cached by your CDN.
+
+While:
+
+* Keeping your phantom-js processes running.
+* Preventing your phantom-js processes from memory-bloat.
+
+This is a good way to achieve it.
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -21,7 +36,88 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+1. You will need Nginx which will load balance requests between the phantom-js
+processes.
+Its conf must include a "upstream phantomjs" directive with the corresponding
+settings. For example:
+
+```
+upstream phantomjs {
+  server 127.0.0.1:8002;
+  server 127.0.0.1:8003;
+  server 127.0.0.1:8004;
+  server 127.0.0.1:8005;
+}
+```
+
+2. A customized [rndr.me js](https://github.com/jed/rndr.me) file that will fit your configuration. There is an
+   example rndrme.js [here](lib/utils/rndrme.js).
+
+   The host configuration is where phantom-js requests the page from, so be
+   sure to point it to your backend server.
+   Also, set the readyEvent to be the event you'r raising so that phantom-js
+   identify it should start rendering.
+
+3. Create a config.yml file to set the variables for phantom-manager. There's
+   an [example config](config/config.yml) with a documentation of each attribute and its meaning.
+
+4. You're ready to run phantom-manager:
+   Just run `phantom_monitor` from anywhere in your system to get the usage
+   instructions for the command line tool.
+
+   Usually, you would just `phantom_monitor -c YOUR_CONF_FILE -e YOUR_ENV`
+   The env option is there to allow your config.yml to have multiple
+   environments settings.
+
+5. The phantom_monitor process listens for USR2 signals. Once such a signal is
+   sent it will restart all processes one by one.
+
+## How Does It Work?
+
+Phantom manager will check for both presence and memory consumption of your
+phantom-js processes under the configuration you have defined.
+
+### Presence
+
+Assuming configuration:
+
+```
+phantom_base_port: 8002
+phantom_processes_number: 3
+```
+The monitor will keep phantom-js processes up on ports 8002, 8003, 8004
+If a phantom-js crashes the monitor will bring it back up.
+
+### Memory Consumption
+
+Assuming configuration:
+```
+memory_limit: 100_000
+memory_retries: 3
+memory_check_interval: 5
+```
+The monitor sample all phantom-js processes each 5 seconds and restart those
+which their memory exceeded 100MB for 3 straight samples.
+
+### Restarting Processes
+
+To restart a phantom-js process the monitor performs the following actions:
+
+1. Remove the process from nginx upstream config.
+
+2. Reload Nginx.
+
+3. Sleeps for phantom_termination_grace seconds to allow this phantom to
+   respond to all requests in its request queue.
+
+4. Kill the phantom-js process.
+
+5. Start the phantom-js process on the same port.
+
+6. Add it to the Nginx upstream configuration.
+
+7. Reload Nginx.
+
 
 ## Contributing
 

data/bin/phantom_monitor

@@ -42,9 +42,10 @@ lib = File.expand_path('../../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'phantom/manager/version'
-require 'monitors/memory.rb'
-require 'monitors/processes.rb'
-require 'monitors/restart_listener.rb'
+require 'monitors/memory'
+require 'monitors/processes'
+require 'monitors/response_time'
+require 'monitors/restart_listener'
 
 
 $logger.info "Starting for environment #{$options[:env]} with config file #{$options[:config]}... "
@@ -57,5 +58,8 @@ Thread.new {Monitors::Memory.run}
 
 $logger.info "Running processes monitor for phantom processes"
 
-Thread.new {Monitors::Processes.run}.join
+Thread.new {Monitors::Processes.run}
 
+$logger.info "Running processes monitor for response time"
+
+Thread.new {Monitors::ResponseTime.run}.join

data/config/config.yml

@@ -28,3 +28,11 @@ development:
 
   #Command to issue when launching phantomjs
   phantom_command: 'phantomjs rndrme.js'
+
+
+  #Response Time Monitor
+  #
+  # If a phantomjs process excceeds this time in response it will be killed
+  response_time_threshold: 2
+  response_time_check_retries: 2
+  response_time_check_interval: 40

data/lib/monitors/base.rb

@@ -5,12 +5,14 @@ require 'phantom/collector'
 
 module Monitors
   class Base
+    extend Logging
 
     class << self
       def run(custom_logger = $logger)
         @logger = custom_logger
 
         loop do
+          log "Performing Check..."
           perform_check
           sleep check_interval
         end
@@ -30,9 +32,6 @@ module Monitors
         Phantom::Collector.get_running_instances
       end
 
-      def logger
-        @logger ||= $logger
-      end
     end
 
   end

data/lib/monitors/memory.rb

@@ -7,11 +7,9 @@ module Monitors
     class << self
 
       def perform_check
-        logger.info "Perfoming memory check..."
-
         running_instances.each do |p|
           if ViolationsRecorders::Memory.is_violating?(p)
-            logger.info "process #{p.pid} was found bad!"
+            log "process #{p.pid} had a memory violation"
             Phantom::Manager.restart(p)
           end
         end

data/lib/monitors/processes.rb

@@ -6,8 +6,6 @@ module Monitors
 
     class << self
       def perform_check
-        logger.info "Perfoming processes check..."
-
         missing_processes = Phantom::Collector.missing_ports.map do |port|
           p = Phantom::Process.new
           p.port = port
@@ -16,12 +14,12 @@ module Monitors
 
         missing_processes.each do |p|
           if ViolationsRecorders::Processes.is_violating?(p)
-            logger.info "found missing phantom on port #{p.port}"
+            log "found missing phantom on port #{p.port}"
             Phantom::Manager.start(p)
           end
         end
 
-        logger.info "All processes are running OK" if missing_processes.empty?
+        log "All processes are running OK" if missing_processes.empty?
       end
 
       def check_interval

data/lib/monitors/response_time.rb

@@ -0,0 +1,27 @@
+require 'monitors/base'
+require 'monitors/violations_recorders/response_time'
+
+module Monitors
+  class ResponseTime < Base
+
+    class << self
+      def perform_check
+        all_processes_ok = true
+        running_instances.each do |p|
+          if ViolationsRecorders::ResponseTime.is_violating?(p)
+            all_processes_ok = false
+            log "process #{p} had a response time violation"
+            Phantom::Manager.restart(p)
+          end
+        end
+
+        log "All response time are ok" if all_processes_ok
+      end
+
+      def check_interval
+        Cfg.response_time_check_interval
+      end
+    end
+
+  end
+end

data/lib/monitors/violations_recorders/base.rb

@@ -5,6 +5,7 @@ require 'phantom/process'
 module Monitors
   module ViolationsRecorders
     class Base
+      extend Logging
 
       class << self
 
@@ -19,9 +20,9 @@ module Monitors
         end
 
         def is_violating?(process)
-          $logger.debug "checking #{process}"
+          log "checking #{process}", :debug
           update_violations_count(process)
-          $logger.debug "#{@violations}"
+          log "#{@violations}", :debug
           violating = @violations[process.send(process_attr)] == retries_limit
           @violations[process.send(process_attr)] = 0 if violating
           violating

data/lib/monitors/violations_recorders/response_time.rb

@@ -0,0 +1,41 @@
+require 'monitors/violations_recorders/base'
+require 'timeout'
+
+module Monitors
+  module ViolationsRecorders
+    class ResponseTime < Base
+      class << self
+        def retries_limit
+          Cfg.response_time_check_retries
+        end
+
+        def process_attr
+          :pid
+        end
+
+        def process_is_violating?(process)
+          time = Cfg.response_time_threshold 
+          begin 
+            Timeout.timeout(Cfg.response_time_threshold) do 
+              time = check_response_time(process)
+            end
+          rescue Timeout::Error
+            return true
+          end
+          time > Cfg.response_time_threshold
+        end
+
+        private
+
+        def check_response_time(process)
+          res = `curl -o /dev/null -s -w %{time_total}@%{http_code} #{process_url(process)}`
+          res.split("@").first.to_f
+        end
+
+        def process_url(process)
+          "http://localhost:#{process.port}"
+        end
+      end
+    end
+  end
+end

data/lib/phantom/manager/version.rb

@@ -1,5 +1,5 @@
 module Phantom
   module Manager
-    VERSION = "0.0.7"
+    VERSION = "0.0.8"
   end
 end

data/lib/utils/logger.rb

@@ -1,3 +1,13 @@
 require 'logger'
 $logger = Logger.new(STDOUT)
 $logger.level = Logger::INFO
+
+module Logging
+  def logger
+    @logger ||= $logger
+  end
+
+  def log(str, severity = :info)
+    logger.send(severity,"[#{self.name}] #{str}")
+  end
+end

data/lib/utils/rndrme.js

@@ -0,0 +1,111 @@
+var config = {
+  host: "http://localhost:8001",
+  maxTime: 30000,
+  maxBytes: 0x100000,
+  readyEvent: "renderReady",
+  loadImages: false
+}
+
+
+var system    = require("system")
+var webserver = require("webserver")
+var webpage   = require("webpage")
+
+var port = system.args[1];
+
+if (!port) {
+  console.error("No port specified in " + configPath)
+  phantom.exit(1)
+}
+
+var server    = webserver.create()
+var listening = server.listen(port, onRequest)
+
+if (!listening) {
+  console.error("Could not bind to port " + port)
+  phantom.exit(1)
+}
+
+function onRequest(req, res) {
+  var page          = webpage.create()
+  var bytesConsumed = 0
+
+  if (req.method != "GET") {
+    return send(405, toHTML("Method not accepted."))
+  }
+
+  var url = parse(req.url)
+
+  var query = url.query
+  var href = decodeURIComponent(config.host + req.url)
+
+  if (!href) {
+    return send(400, toHTML("`href` parameter is missing."))
+  }
+
+  var maxTime    = Number(query.max_time)  || config.maxTime
+  var maxBytes   = Number(query.max_bytes) || config.maxBytes
+  var readyEvent = query.ready_event       || config.readyEvent
+  var loadImages = "load_images" in query  || config.loadImages
+
+  page.settings.loadImages = loadImages
+
+  page.onInitialized = function() {
+    page.evaluate(onInit, readyEvent)
+
+    function onInit(readyEvent) {
+      window.rndrme = true;
+      window.addEventListener(readyEvent, function() {
+        setTimeout(window.callPhantom, 0)
+      })
+    }
+  }
+
+  page.onCallback = function() {
+    send(200, page.content)
+  }
+
+  var timeout = setTimeout(page.onCallback, maxTime)
+
+  console.log("(" + port + ") opening page " + href);
+
+  page.open(href)
+
+  function send(statusCode, data) {
+    clearTimeout(timeout)
+
+    res.statusCode = statusCode
+
+    res.setHeader("Content-Type", "text/html")
+    res.setHeader("Content-Length", byteLength(data))
+    res.setHeader("X-Rndrme-Bytes-Consumed", bytesConsumed.toString())
+
+    res.write(data)
+    res.close()
+
+    page.close()
+  }
+}
+
+function byteLength(str) {
+  return encodeURIComponent(str).match(/%..|./g).length
+}
+
+function toHTML(message) {
+  return "<!DOCTYPE html><body>" + message + "</body>\n"
+}
+
+function parse(url) {
+  var anchor = document.createElement("a")
+
+  anchor.href = url
+  anchor.query = {}
+
+  anchor.search.slice(1).split("&").forEach(function(pair) {
+    pair = pair.split("=").map(decodeURIComponent)
+    anchor.query[pair[0]] = pair[1]
+  })
+
+  return anchor
+}
+

data/spec/files/nginx.conf

@@ -3,7 +3,10 @@ upstream unicorn {
 }
 
 upstream phantomjs {
-  server 127.0.0.1:8020 fail_timeout=0; # 2013-08-01 16:06:02 +0300
+  server 127.0.0.1:8000 fail_timeout=0; # 2013-10-21 10:44:21 +0200
+  server 127.0.0.1:8000 fail_timeout=0; # 2013-10-21 10:44:21 +0200
+  server 127.0.0.1:8012 fail_timeout=0; # 2013-10-21 10:44:21 +0200
+  server 127.0.0.1:8013 fail_timeout=0; # 2013-10-21 10:44:21 +0200
   server 127.0.0.1:8002;
   server 127.0.0.1:8003;
   server 127.0.0.1:8004;

data/spec/lib/monitors/response_time_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+require 'monitors/response_time'
+
+module Monitors
+
+  describe ResponseTime do
+
+    subject {ResponseTime}
+
+    before do
+      @process1 = Phantom::Process.new(1, 1, "", 1)
+      @process2 = Phantom::Process.new(3, 3, "", 3)
+      subject.stub running_instances: [@process1, @process2]
+      Cfg.response_time_threshold = 2
+      Cfg.response_time_check_retries = 2
+    end
+
+    describe :perform_check do
+      context "violating process" do
+        before do
+          ViolationsRecorders::ResponseTime.stub(:is_violating?).with(@process1).and_return(true)
+          ViolationsRecorders::ResponseTime.stub(:is_violating?).with(@process2).and_return(false)
+        end
+
+        it "should restart violating process" do
+          Phantom::Manager.should_receive(:restart).with(@process1).once
+          subject.perform_check
+        end
+
+        it "should not restart non violating process" do
+          Phantom::Manager.should_not_receive(:restart).with(@process2)
+          subject.perform_check
+        end
+      end
+    end
+
+  end
+end

data/spec/lib/monitors/violations_recorders/response_time_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+require 'monitors/violations_recorders/response_time'
+
+module Monitors
+  module ViolationsRecorders
+
+    RESPONSE_TIME_THRESHOLD = 2
+
+    describe ResponseTime do
+
+      subject {ResponseTime}
+
+      describe :process_is_violating? do
+
+        before do
+          Cfg.stub(:response_time_threshold).and_return(RESPONSE_TIME_THRESHOLD)
+        end
+
+        context "timeout" do
+          before do
+            subject.stub(:check_response_time).and_raise(Timeout::Error)
+          end
+
+          it "should return true" do
+            subject.process_is_violating?(stub).should be_true
+          end
+        end
+
+        context "fast response time" do
+          before do
+            subject.stub(:check_response_time).and_return(RESPONSE_TIME_THRESHOLD - 1)
+          end
+
+          it "should return false" do
+            subject.process_is_violating?(stub).should be_false
+          end
+        end
+
+        context "slow response time" do
+          before do
+            subject.stub(:check_response_time).and_return(RESPONSE_TIME_THRESHOLD + 1)
+          end
+
+          it "should return true" do
+            subject.process_is_violating?(stub).should be_true
+          end
+        end
+      end
+
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: phantom-manager
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Erez Rabih
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-01 00:00:00.000000000 Z
+date: 2013-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -73,10 +73,12 @@ files:
 - lib/monitors/base.rb
 - lib/monitors/memory.rb
 - lib/monitors/processes.rb
+- lib/monitors/response_time.rb
 - lib/monitors/restart_listener.rb
 - lib/monitors/violations_recorders/base.rb
 - lib/monitors/violations_recorders/memory.rb
 - lib/monitors/violations_recorders/processes.rb
+- lib/monitors/violations_recorders/response_time.rb
 - lib/nginx/manager.rb
 - lib/phantom/.DS_Store
 - lib/phantom/collector.rb
@@ -87,6 +89,7 @@ files:
 - lib/utils/limited_array.rb
 - lib/utils/lock.rb
 - lib/utils/logger.rb
+- lib/utils/rndrme.js
 - lib/utils/shell.rb
 - phantom-manager.gemspec
 - spec/files/config.yml
@@ -94,10 +97,12 @@ files:
 - spec/lib/monitors/base_spec.rb
 - spec/lib/monitors/memory_spec.rb
 - spec/lib/monitors/processes_spec.rb
+- spec/lib/monitors/response_time_spec.rb
 - spec/lib/monitors/restart_listener_spec.rb
 - spec/lib/monitors/violations_recorders/base_spec.rb
 - spec/lib/monitors/violations_recorders/memory_spec.rb
 - spec/lib/monitors/violations_recorders/processes_spec.rb
+- spec/lib/monitors/violations_recorders/response_time_spec.rb
 - spec/lib/nginx/manager_spec.rb
 - spec/lib/phantom/collector_spec.rb
 - spec/lib/phantom/manager_spec.rb
@@ -138,10 +143,12 @@ test_files:
 - spec/lib/monitors/base_spec.rb
 - spec/lib/monitors/memory_spec.rb
 - spec/lib/monitors/processes_spec.rb
+- spec/lib/monitors/response_time_spec.rb
 - spec/lib/monitors/restart_listener_spec.rb
 - spec/lib/monitors/violations_recorders/base_spec.rb
 - spec/lib/monitors/violations_recorders/memory_spec.rb
 - spec/lib/monitors/violations_recorders/processes_spec.rb
+- spec/lib/monitors/violations_recorders/response_time_spec.rb
 - spec/lib/nginx/manager_spec.rb
 - spec/lib/phantom/collector_spec.rb
 - spec/lib/phantom/manager_spec.rb