hashbang 0.0.1.alpha

data/README.md

@@ -0,0 +1,34 @@
+# Bang. Hashbang.
+
+Hashbang will automatically enable serving HTML dumps for your AJAX web-applications according to 
+[Google conventions](http://code.google.com/web/ajaxcrawling/). Make your Rails AJAX applications indexable in no time.
+
+Hashbang will make all incoming requests containing magic `_escaped_fragment_` parameter to be served in a special way. This case will cause a subrequest to a real AJAX URL using virtual browser. This hidden browser will wait for some condition you define using well-known [Watir](http://watirwebdriver.com/) API. And then return an HTML dump.
+
+Let's say for example you've got a request to `test.com/?_escaped_fragment_=/my_hidden_page`.
+
+Hashbang will convert this URL to `test.com/#!/my_hidden_page` and open it in a virtual browser.
+
+Virtual browser will call your lambda with `browser` object as parameter. With help of this lambda you can setup the wait behavior. Here is a great introduction to [Watir wait API](http://watirwebdriver.com/waiting/). Note that your lambda will act as a block to `Watir::Wait.until`.
+
+## Installation & Example
+
+Start from your Gemfile:
+
+```
+  gem 'hashbang'
+```
+  
+Waiter can be defined inside your environments:
+
+```ruby
+  config.hashbang.waiter = -> b { b.execute_script("return Joosy.Application.loading") == false }
+```
+
+This code will wait for javascript Joosy.Application.loading variable to be set to false.
+
+## Development environment
+
+By its nature Hashbang gem requires your server to accept at least 2 concurrent connections. However WEBRick or Thin (which are mostly used as development servers) will only serve one Rails instance (and connection). That's why if you want to debug your AJAX crawling, you'll have to use `unicorn` with at least 2 workers.
+
+Note either that browser load behavior differs between development and production environments. Hashbang will use lazy load for development and therefore it may take some time for crawler to respond. While running in production environment it will start and cache browser instance at rails startup to provide the best possible response speed.

data/Rakefile

@@ -0,0 +1,8 @@
+require 'bundler/setup'
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+desc "Default: run the unit tests."
+task :default => [:spec]

data/hashbang.gemspec

@@ -0,0 +1,16 @@
+Gem::Specification.new do |s|
+  s.name        = "hashbang"
+  s.version     = "0.0.1.alpha"
+  s.platform    = Gem::Platform::RUBY
+  s.summary     = "Magic support of Google/Bing/... AJAX search indexing for your Rails apps"
+  s.email       = "boris@roundlake.ru"
+  s.homepage    = "http://roundlake.github.com/hashbang/"
+  s.description = "Hashbang will automatically enable serving HTML dumps for your AJAX web-applications according to Google conventions. Make your Rails AJAX applications indexable in no time."
+  s.authors     = ['Boris Staal']
+
+  s.files         = `git ls-files`.split("\n")
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'watir-webdriver'
+  s.add_dependency 'headless'
+end

data/init.rb

@@ -0,0 +1 @@
+require 'hashbang'

data/lib/hashbang.rb

@@ -0,0 +1,7 @@
+require 'headless'
+require 'watir-webdriver'
+
+require 'hashbang/pool'
+require 'hashbang/crawler'
+require 'hashbang/middleware'
+require 'hashbang/railtie' if defined? Rails

data/lib/hashbang/crawler.rb

@@ -0,0 +1,20 @@
+module Hashbang
+  class Crawler
+    def self.gimme(url, &block)
+      browser = Pool.grab
+
+      begin
+        browser.goto url
+
+        if block_given?
+          Watir::Wait.until { block.call browser }
+        end
+
+        browser.html
+      rescue
+      ensure
+        Hashbang::Pool.release(browser)
+      end
+    end
+  end
+end

data/lib/hashbang/middleware.rb

@@ -0,0 +1,35 @@
+module Hashbang
+  class Middleware
+    def initialize(application, config)
+      @application = application
+      @config      = config
+
+      Headless.new.start
+      Pool.setup config.pool_size
+    end
+
+    def call(environment)
+      if environment['QUERY_STRING'].include? "_escaped_fragment_"
+        url = []
+        url << environment['rack.url_scheme'] + '://'
+        url << environment['HTTP_HOST']
+        url << environment['REQUEST_PATH']
+        url << '?' unless environment['QUERY_STRING'].starts_with? '_escaped_fragment_'
+        url << environment['QUERY_STRING'].gsub(/(\&)?_escaped_fragment_=/, '#!')
+        url = url.join ''
+
+        if @config.waiter
+          html = Crawler.gimme(url) do |browser|
+            @config.waiter.call(browser)
+          end
+        else
+          html = Crawler.gimme url
+        end
+
+        [200, {"Content-Type" => "text/html"}, [html]]
+      else
+        @application.call(environment)
+      end
+    end
+  end
+end

data/lib/hashbang/pool.rb

@@ -0,0 +1,40 @@
+module Hashbang
+  class Pool
+    cattr_accessor :inited
+    cattr_accessor :pool_size
+    cattr_accessor :browsers
+    cattr_accessor :taken
+
+    def self.setup(quantity=1)
+      @@pool_size = quantity
+
+      self.init if !Rails || Rails.env != 'development'
+    end
+
+    def self.init
+      @@browsers = []
+      @@taken    = []
+
+      @@pool_size.times do
+        @@browsers << browser = Watir::Browser.new
+        at_exit { browser.close if browser.exists? }
+      end
+
+      @@inited = true
+    end
+
+    def self.grab
+      self.init unless @@inited
+
+      raise "Pool is empty" if @@browsers.length == 0
+
+      @@taken << browser = @@browsers.pop
+      browser
+    end
+
+    def self.release(browser)
+      @@taken.delete browser
+      @@browsers << browser
+    end
+  end
+end

data/lib/hashbang/railtie.rb

@@ -0,0 +1,20 @@
+module Hashbang
+  class Railtie < Rails::Railtie
+    config.hashbang = ActiveSupport::OrderedOptions.new
+
+    config.hashbang.pool_size = 1
+    config.hashbang.waiter    = false
+
+    waiters = {
+      :joosy => -> b { b.execute_script("return Joosy.Application.loading") == false }
+    }
+
+    initializer "application_controller.initialize_hashbang" do |app|
+      if app.config.hashbang.waiter.is_a? Symbol
+        app.config.hashbang.waiter = waiters[app.config.hashbang.waiter]
+      end
+
+      app.config.middleware.use "Hashbang::Middleware", app.config.hashbang
+    end
+  end
+end

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification
+name: hashbang
+version: !ruby/object:Gem::Version
+  version: 0.0.1.alpha
+  prerelease: 6
+platform: ruby
+authors:
+- Boris Staal
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: watir-webdriver
+  requirement: &70179137315820 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70179137315820
+- !ruby/object:Gem::Dependency
+  name: headless
+  requirement: &70179137315360 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70179137315360
+description: Hashbang will automatically enable serving HTML dumps for your AJAX web-applications
+  according to Google conventions. Make your Rails AJAX applications indexable in
+  no time.
+email: boris@roundlake.ru
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- hashbang.gemspec
+- init.rb
+- lib/hashbang.rb
+- lib/hashbang/crawler.rb
+- lib/hashbang/middleware.rb
+- lib/hashbang/pool.rb
+- lib/hashbang/railtie.rb
+homepage: http://roundlake.github.com/hashbang/
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: Magic support of Google/Bing/... AJAX search indexing for your Rails apps
+test_files: []