indexable 0.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ea666f584199ac33afce60449c7e2ca89a58fe09
+  data.tar.gz: c6cd4101fa9c6ae24f67e8fc1fe58d0fe923cf16
+SHA512:
+  metadata.gz: 3d99e57df1a36bd2330f1f1b118c1433581a5411cd6427efd8642b2815003763a7128f51796e9d5bdcb79ef0c824cd6243f03f0380ea517d6ce8250dc3a423c9
+  data.tar.gz: 91554eb4a3c2ffa54239a0ceff1c766b9ff7d5b74641f5be6329a7293c9597df310babf217be4ca35f3f043151439100d3b15a0ed8b096c99e28fd80c0aeda13

data/lib/indexable.rb

@@ -0,0 +1,53 @@
+require 'phantomjs'
+require 'rack/request'
+
+module Rack
+  class Indexable
+    CRAWLER_USER_AGENTS = [
+      /^Twitterbot/, /^curl/, /Googlebot/, /Mediapartners/, /Adsbot-Google/,
+      /\(.*http(s|\(s\))?:\/\/.*\)/
+    ]
+
+    def initialize(app)
+      @app = app
+    end
+
+    # Detect whether the current request comes from a bot. Based on the logic used
+    # by Bustle.com (https://www.dropbox.com/s/s4oibqsxqpo3hll/bustle%20slizzle.pdf)
+    def request_from_crawler?(env)
+      user_agent  = env["HTTP_USER_AGENT"]
+      params      = Rack::Request.new(env).params
+      return false  unless user_agent
+      return true   if CRAWLER_USER_AGENTS.any? {|s| user_agent.match(s) }
+      return true   if params.has_key?('_escaped_fragment_')
+      params['nojs'].eql?('true')
+    end
+
+    def call(env)
+      status, headers, content = *@app.call(env)
+
+      if status == 200 and headers['Content-Type'].match(/^text\/html/) and request_from_crawler?(env)
+        script = ::File.dirname(__FILE__) + "/render_page.js"
+        file = Tempfile.new(['indexable', '.html'])
+
+        if content.respond_to? :body
+          html = content.body
+        else
+          html = content.join('')
+        end
+
+        file.write html
+        file.close
+        begin
+          url = Rack::Request.new(env).url
+          content = [Phantomjs.new(script, file.path, url).run]
+          status = 500 if content[0] == "Couldn't render page... orz."
+        ensure
+          file.unlink
+        end
+      end
+
+      [status, headers, content]
+    end
+  end
+end

data/lib/phantomjs.rb

@@ -0,0 +1,26 @@
+require 'timeout'
+
+class Phantomjs
+  attr_accessor :timeout
+
+  def initialize(script, *args)
+    @script = script
+    @args = args
+    @timeout = 20
+  end
+
+  def run
+    pipe = nil
+    begin
+      Timeout.timeout(@timeout) do
+        pipe = IO.popen(["phantomjs", @script] + @args)
+        Process.wait pipe.pid
+        return pipe.read
+      end
+    rescue Timeout::Error
+      Process.kill 9, pipe.pid
+      Process.wait pipe.pid
+      return "Couldn't render page... orz."
+    end
+  end
+end

data/lib/render_page.js

@@ -0,0 +1,82 @@
+// TODO Add header.
+//
+// Based off https://gist.github.com/pieterjongsma/4515412
+
+var fs      = require('fs');
+var system  = require('system');
+var page    = require('webpage').create();
+var path    = system.args[1];
+var url     = system.args[2];
+var content = fs.read(path);
+
+// Keep track of whether the page has already been exported, because the '
+// 'setTimeout's we use might cause it to be exported multiple times.
+var pageHasBeenExported = false;
+function exportPage(content) {
+  if (!pageHasBeenExported) {
+    pageHasBeenExported = true;
+    console.log(content);
+    phantom.exit();
+  }
+}
+
+// Since Ember has no method that indicates when everything has loaded, we keep
+// keep track of resource requests. The claim is that the page is ready when there
+// are no outstanding requests for resources.
+//
+// This is not strictly true, because sometimes a resource still needs to be
+// rendered after loading, or the application might still be building a request.
+// To work around this, we wait an addition 2 seconds to make sure rendering etc.
+// can take place.
+//
+// We need to keep track of request/response IDs, not just the count because larger
+// resources may be returned in chunks.
+
+var activeRequests = [];
+var cumulativeRequests = 0;
+
+page.onResourceRequested = function(requestData, networkRequest) {
+  if ((/ga\.js/gi).test(requestData.url) || (/dc\.js/gi).test(requestData.url)) {
+    // Don't load ga.js and dc.js to avoid polluting Google Analytics data
+    // regardless of whether external resources are allowed or not.
+    networkRequest.abort();
+  }
+  else {
+    activeRequests.push(requestData.id);
+    cumulativeRequests += 1;
+  }
+}
+
+page.onResourceReceived = function(response) {
+  // Remove the ID from the array.
+  activeRequests.splice(activeRequests.indexOf(response.id), 1);
+
+  // Should we output the HTML?
+  if (activeRequests.length == 0) {
+    window.setTimeout(function() {
+      if (activeRequests.length == 0) {
+        exportPage(page.content);
+      }
+    }, 2000);
+  }
+}
+
+page.onError = function(message, trace) {
+  // Catch and ignore errors.
+}
+
+page.setContent(content, url);
+
+// If the page hasn't been exported in 8 seconds something is probably wrong,
+// go ahead and export it.
+setTimeout(function() {
+  exportPage(page.content);
+}, 8000);
+
+// If there are no requests in 2 seconds, assume the page does not depend on any
+// external resources and has finished rendering.
+setTimeout(function() {
+  if (cumulativeRequests == 0) {
+    exportPage(page.content);
+  }
+}, 2000);

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification
+name: indexable
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+platform: ruby
+authors:
+- Vikhyat Korrapati
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-12-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: c@vikhyat.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/indexable.rb
+- lib/render_page.js
+- lib/phantomjs.rb
+homepage: https://github.com/vikhyat/indexable
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Rack middleware that executes javascript before serving pages to crawlers.
+test_files: []