indexable 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 64a1e1b305b64588744f3df8c892511cb49936e0
-  data.tar.gz: 13a2987bcff19059f7f1bed603b6ca48219917f9
+  metadata.gz: 0a6d2c79c0eb8d268fba3a60b0300c8ac464f32b
+  data.tar.gz: 795a31d66d9acfe8cc83e9dcf43d269886ae5833
 SHA512:
-  metadata.gz: 4cc15e9a4ccff9f84f43bdf21b6bba08ca59c6b04463560f03da641305505fcb0d72e255e8824803fed0c7b0ba1838f3e589941a0a3f2242387d7e482c2488ca
-  data.tar.gz: 18f3d230291c6bb07f628ebf7fd0b7f48811a7abda1aca2c026c87e9e3f6a20ef9c212af5335c89a5cb2b09772733f68b23868b2c76c054b827beb4ee95abccf
+  metadata.gz: 5bf720013d32fff089bdd6cece00eccc345592eb6def5c5d31670c7eb29af7d7398e8a1c51f898ee2427ccb1f2091bdd629938a006a2ac816a36a40b9dbd67f0
+  data.tar.gz: c76f9a4f1065c82c01a93d7c80c8ce8a71caf283b4f844d3b50b42d6041304c4bc8e1dd7358df0199cda77e98adaebf80837d4dbc59154c486365ef7accb1b3d

data/lib/indexable/indexable.rb

@@ -0,0 +1,53 @@
+require 'rack/request'
+require 'indexable/phantomjs'
+
+module Indexable
+  class Middleware
+    CRAWLER_USER_AGENTS = [
+      /^Twitterbot/, /^curl/, /Googlebot/, /Mediapartners/, /Adsbot-Google/,
+      /\(.*http(s|\(s\))?:\/\/.*\)/
+    ]
+
+    def initialize(app)
+      @app = app
+    end
+
+    # Detect whether the current request comes from a bot. Based on the logic used
+    # by Bustle.com (https://www.dropbox.com/s/s4oibqsxqpo3hll/bustle%20slizzle.pdf)
+    def request_from_crawler?(env)
+      user_agent  = env["HTTP_USER_AGENT"]
+      params      = Rack::Request.new(env).params
+      return false  unless user_agent
+      return true   if CRAWLER_USER_AGENTS.any? {|s| user_agent.match(s) }
+      return true   if params.has_key?('_escaped_fragment_')
+      params['nojs'].eql?('true')
+    end
+
+    def call(env)
+      status, headers, content = *@app.call(env)
+
+      if status == 200 and headers['Content-Type'].match(/^text\/html/) and request_from_crawler?(env)
+        script = ::File.dirname(__FILE__) + "/render_page.js"
+        file = Tempfile.new(['indexable', '.html'])
+
+        if content.respond_to? :body
+          html = content.body
+        else
+          html = content.join('')
+        end
+
+        file.write html
+        file.close
+        begin
+          url = Rack::Request.new(env).url
+          content = [Phantomjs.new(script, file.path, url).run]
+          status = 500 if content[0] == "Couldn't render page... orz."
+        ensure
+          file.unlink
+        end
+      end
+
+      [status, headers, content]
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indexable
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Vikhyat Korrapati
@@ -31,6 +31,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/indexable.rb
+- lib/indexable/indexable.rb
 - lib/indexable/render_page.js
 - lib/indexable/phantomjs.rb
 - lib/indexable/railtie.rb