asap 0.0.5

data/.gitignore

@@ -0,0 +1,6 @@
+.bundle
+.rvmrc
+Gemfile.lock
+*.gem
+.bundle
+pkg/*

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+
+gemspec

data/LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2011 Greg Spurrier, Avik Das
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,82 @@
+== ASAP - a Parallel Fetch Library =====================================
+
+by Greg Spurrier, Avik Das
+
+== DESCRIPTION =========================================================
+
+ASAP is a JRuby library built on top of Netty and Java's NIO classes. It
+provides an embedded domain specific language for specifying a list of
+resources to fetch, all in parallel, as well specify a dependency tree
+in order to use previous results in calculating subsequent ones. The
+results of all the requests are then automatically collected into a
+simple tree-like data structure that has a one-to-one mapping to the
+tree structure specified by the code, despite the fact that results may
+arrive in an unspecified order.
+
+== EXAMPLE =============================================================
+
+Assume that a server is running on http://0.0.0.0:1234, which maps the
+paths /user/followers and /user/followers/N to a list of five numbers and
+data related to the Nth follower respectively.
+
+  require 'asap'
+
+  data = Asap do
+    get 'http://0.0.0.0:1234/user/followers' do |followers|
+      followers = followers.split("\n").map(&:to_i)
+
+      # get the first 3 followers
+      get "http://0.0.0.0:1234/user/followers/#{followers[0]}"
+      get "http://0.0.0.0:1234/user/followers/#{followers[1]}"
+      get "http://0.0.0.0:1234/user/followers/#{followers[2]}"
+
+      # or you can use a map
+      followers[3,2].each do |fi|
+        get "http://0.0.0.0:1234/user/followers/#{fi}"
+      end
+    end
+  end
+
+  p data
+
+  # => [["93\n5\n64\n74\n11",
+          ["Follower #93",
+           "Follower #5" ,
+           "Follower #64",
+           "Follower #74",
+           "Follower #11"]]]
+
+== QUICK START =========================================================
+  
+  gem install asap
+
+== DEVELOPMENT QUICK START =============================================
+
+  # Check out repository
+  git clone git://github.com/avik-das/asap.git
+  cd asap
+
+  # Make sure you're using JRuby
+
+  # Install the dependencies
+  gem install bundler
+  bundle install
+  rake install
+
+  # start the server (run in a separate window)
+  asap-mongrel-test-server
+
+  # run the tests
+  rake spec
+
+  # compare the times required to fetch the same data if it is retrieved
+  # serially versus if it is retrieved with ASAP.
+  time examples/followers-example-serial.rb
+  time examples/followers-example.rb
+  
+  # There is an alternate, EventMachine-based server, but it does not
+  # implement the /user/followers/ routes. EventMachine does not run
+  # well with JRuby, while the main library requires JRuby. However,
+  # the EventMachine-based server runs well on MRI, assuming the
+  # correct gems are installed (see the gemspec).
+  script/em_test_server.rb

data/Rakefile

@@ -0,0 +1,5 @@
+require 'bundler/setup'
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new

data/asap.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "asap/version"
+
+Gem::Specification.new do |s|
+  s.name        = "asap"
+  s.version     = Asap::Gem::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Greg Spurrier", "Avik Das"]
+  s.email       = ["gspurrier@linkedin.com", "adas@linkedin.com"]
+  s.homepage    = "http://rubygems.org/gems/didactic_clock"
+  s.summary     = %q{A JRuby library for parallel fetches.}
+  s.description = %q{A JRuby library for parallel fetches. Provides an embedded-domain specific language for declaring dependencies between resources.}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency("mongrel")
+
+  s.add_development_dependency("rspec")
+  s.add_development_dependency("rake")
+  
+  # Event machine does not run well on JRuby, but the main library requires
+  # JRuby.  The following gems should be installed separately on an MRI
+  # instance if you wish to run bin/asap-em-test-server
+  # s.add_runtime_dependency('eventmachine')
+  # s.add_runtime_dependency('eventmachine_httpserver')
+end

data/bin/asap-em-test-server

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'eventmachine'
+require 'evma_httpserver'
+require 'cgi'
+
+class MyHttpServer < EM::Connection
+  include EM::HttpServer
+
+   def post_init
+     super
+     no_environment_strings
+   end
+
+  def process_http_request
+    # the http request details are available via the following instance variables:
+    #   @http_protocol
+    #   @http_request_method
+    #   @http_cookie
+    #   @http_if_none_match
+    #   @http_content_type
+    #   @http_path_info
+    #   @http_request_uri
+    #   @http_query_string
+    #   @http_post_content
+    #   @http_headers
+
+    empty, time, message = @http_path_info.split('/')
+
+    response = EM::DelegatedHttpResponse.new(self)
+    response.status = 200
+    response.content_type 'text/html'
+    response.content = CGI.unescape(message)
+
+    EventMachine::Timer.new(time.to_i) do
+      response.send_response
+    end
+  end
+end
+
+EventMachine.run{
+  EventMachine.start_server '0.0.0.0', 1234, MyHttpServer
+}

data/bin/asap-mongrel-test-server

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'mongrel'
+
+class TimeMessageHandler < Mongrel::HttpHandler
+    def process request, response
+        response.start 200  do |head,out|
+            params = get_params request
+
+            wait_time = params[:wait_time].to_i
+            puts "Sleeping for #{wait_time} seconds"
+            sleep wait_time
+
+            head["Content-Type"] = "text/plain"
+            out.write params[:message] # no newline
+            out.flush
+        end
+    end
+
+private
+
+    def get_params request
+        uri = request.params['REQUEST_URI']
+        _, wait_time, message = *uri.split("/")
+        message = Mongrel::HttpRequest.unescape message
+        {:wait_time => wait_time, :message => message}
+    end
+end
+
+class FollowersHandler < Mongrel::HttpHandler
+    def process request, response
+        sleep 5
+
+        id = get_follower_id(request)
+        message = id ? get_one_follower(id.to_i) : get_all_followers
+
+        response.start 200  do |head,out|
+            head["Content-Type"] = "text/plain"
+            out.write message # no newline
+            out.flush
+        end
+    end
+
+private
+
+    MAX_FOLLOW_ID = 100
+
+    def get_follower_id request
+        uri = request.params['REQUEST_URI']
+        _, _, _, id = *uri.split("/")
+        id
+    end
+
+    def get_all_followers
+        (1..5).map {|i| rand(MAX_FOLLOW_ID)}.join("\n")
+    end
+
+    def get_one_follower i
+        "Follower ##{i}"
+    end
+end
+
+h = Mongrel::HttpServer.new "0.0.0.0", "1234"
+h.register "/user/followers", FollowersHandler.new
+h.register "/", TimeMessageHandler.new
+
+puts "Starting server"
+h.run.join
+puts "Shutting down server"

data/examples/followers-example-serial.rb

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+require 'open-uri'
+
+data = []
+open('http://0.0.0.0:1234/user/followers') do |fresp|
+  data << fresp.read
+  followers = data[0].split("\n").map(&:to_i)
+
+  # get the first 3 followers
+  open("http://0.0.0.0:1234/user/followers/#{followers[0]}") do |resp|
+    data << resp.read
+  end
+  open("http://0.0.0.0:1234/user/followers/#{followers[1]}") do |resp|
+    data << resp.read
+  end
+  open("http://0.0.0.0:1234/user/followers/#{followers[2]}") do |resp|
+    data << resp.read
+  end
+
+  # or you can use a map
+  followers[3,2].each do |fi|
+    open("http://0.0.0.0:1234/user/followers/#{fi}") do |resp|
+      data << resp.read
+    end
+  end
+end
+
+p data

data/examples/followers-example.out

@@ -0,0 +1,21 @@
+
+
+
+data =
+    [
+        ["44\n64\n6\n21\n87", # /user/followers
+            
+           
+           
+            ["Follower #44",  # /user/followers/44
+             "Follower #64",  # /user/followers/64
+             "Follower #6" ,  # /user/followers/6
+              
+              
+              
+             "Follower #21", "Follower #87"]]] # etc.
+
+
+
+
+

data/examples/followers-example.rb

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'asap'
+
+data = Asap do
+  get 'http://0.0.0.0:1234/user/followers' do |followers|
+    followers = followers.split("\n").map(&:to_i)
+
+    # get the first 3 followers
+    get "http://0.0.0.0:1234/user/followers/#{followers[0]}"
+    get "http://0.0.0.0:1234/user/followers/#{followers[1]}"
+    get "http://0.0.0.0:1234/user/followers/#{followers[2]}"
+
+    # or you can use a map
+    followers[3,2].each do |fi|
+      get "http://0.0.0.0:1234/user/followers/#{fi}"
+    end
+  end
+end
+
+p data

data/lib/asap.rb

@@ -0,0 +1,8 @@
+require 'asap/fetch_context'
+
+def Asap *args, &blk
+  context = Asap::FetchContext.new
+  context.instance_exec *args, &blk
+  context.join
+  context.result
+end

data/lib/asap/fetch_context.rb

@@ -0,0 +1,33 @@
+require 'asap/netty'
+
+module Asap
+  class FetchContext
+    def initialize
+      @result = []
+      @semaphore = java.util.concurrent.Semaphore.new(0)
+    end
+
+    def get(url, &blk)
+      target_index = @result.size
+      @result << nil
+      Asap::Netty.get(url) do |result|
+        if blk
+          Thread.new do
+            nested = Asap(result, &blk)
+            @result[target_index] = [result, nested]
+            @semaphore.release
+          end
+        else
+          @result[target_index] = result
+          @semaphore.release
+        end
+      end
+    end
+
+    def join
+      @semaphore.acquire(@result.size)
+    end
+
+    attr_reader :result
+  end
+end

data/lib/asap/netty.rb

@@ -0,0 +1,51 @@
+require 'uri'
+require 'jruby'
+require 'java'
+$CLASSPATH << File.expand_path('../../java/netty-3.2.4.Final.jar', File.dirname(__FILE__))
+
+require 'asap/netty/http_response_handler'
+require 'asap/netty/pipeline_factory'
+
+module Asap
+  module Netty
+    java_import java.net.InetSocketAddress
+    java_import java.util.concurrent.Executors
+    java_import org.jboss.netty.bootstrap.ClientBootstrap
+    java_import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory
+    java_import org.jboss.netty.handler.codec.http.DefaultHttpRequest
+    java_import org.jboss.netty.handler.codec.http.HttpVersion
+    java_import org.jboss.netty.handler.codec.http.HttpMethod
+    java_import org.jboss.netty.handler.codec.http.HttpHeaders
+
+
+    def self.get(url, &callback)
+      uri = URI.parse(url)
+
+      bootstrap.set_pipeline_factory(PipelineFactory.new(callback))
+
+      # Open a connection
+      future = bootstrap.connect(InetSocketAddress.new(uri.host, uri.port))
+      channel = future.awaitUninterruptibly.get_channel
+      raise 'connection failed' unless future.is_success
+
+      # Send the request
+      request = DefaultHttpRequest.new(HttpVersion::HTTP_1_0, HttpMethod::GET, uri.path)
+      request.set_header(HttpHeaders::Names::HOST, uri.host)
+      channel.write(request)
+    end
+
+    private
+
+    def self.bootstrap
+      if not @bootstrap
+        @bootstrap = ClientBootstrap.new(
+          NioClientSocketChannelFactory.new(
+            Executors.newCachedThreadPool,
+            Executors.newCachedThreadPool))
+
+        at_exit { @bootstrap.release_external_resources }
+      end
+      @bootstrap
+    end
+  end
+end

data/lib/asap/netty/http_response_handler.rb

@@ -0,0 +1,21 @@
+module Asap
+  module Netty
+    class HttpResponseHandler < org.jboss.netty.channel.SimpleChannelUpstreamHandler
+      attr_reader :callback
+
+      def initialize(callback)
+        super()
+        @callback = callback
+      end
+
+      def messageReceived(ctxt, e)
+        response = e.get_message
+        if response.get_status.get_code == 200
+          callback.call(response.get_content.to_string(org.jboss.netty.util.CharsetUtil::UTF_8))
+        else
+          raise "Request failed with #{response.get_status.get_code}"
+        end
+      end
+    end
+  end
+end

data/lib/asap/netty/pipeline_factory.rb

@@ -0,0 +1,20 @@
+module Asap
+  module Netty
+    class PipelineFactory
+      include org.jboss.netty.channel.ChannelPipelineFactory
+
+      attr_reader :callback
+
+      def initialize(callback)
+        @callback = callback
+      end
+
+      def get_pipeline
+        org.jboss.netty.channel.Channels.pipeline.tap do |pipeline|
+          pipeline.add_last("codec", org.jboss.netty.handler.codec.http.HttpClientCodec.new)
+          pipeline.add_last("handler", Asap::Netty::HttpResponseHandler.new(callback))
+        end
+      end
+    end
+  end
+end

data/lib/asap/version.rb

@@ -0,0 +1,5 @@
+module Asap
+  module Gem
+    VERSION = "0.0.5"
+  end
+end

data/spec/asap/netty_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe Asap::Netty, '.get' do
+  it 'fetches the specified resource and invokes the callback with it' do
+    semaphore = java.util.concurrent.Semaphore.new(0)
+    result = nil
+    Asap::Netty.get("http://localhost:1234/0/hello") do |data|
+      result = data
+      semaphore.release
+    end
+    semaphore.acquire
+    result.should == 'hello'
+  end
+end

data/spec/asap_spec.rb

@@ -0,0 +1,149 @@
+require 'spec_helper'
+
+def url path
+    "http://localhost:1234" + path
+end
+
+describe Asap do
+  it 'should be a module' do
+    Asap.class.should == Module
+  end
+
+  it 'should be callable as a function with a block' do
+    lambda { Asap do; end }.should_not raise_error
+  end
+
+  it 'should require a block' do
+    lambda { Asap() }.should raise_error
+  end
+
+  context 'given no gets' do
+    it 'should have an empty result' do
+      Asap do
+      end.should == []
+    end
+  end
+
+  context 'given one get' do
+    it 'should return one result' do
+      Asap do
+        get url("/0/hello")
+      end.should == ["hello"]
+    end
+  end
+
+  context 'given two gets' do
+    it 'should return two results' do
+      Asap do
+        get url("/0/hello")
+        get url("/0/world")
+      end.should == ["hello","world"]
+    end
+  end
+
+  context 'given three gets' do
+    it 'should return three results' do
+      Asap do
+        get url("/1/goodbye")
+        get url("/1/cruel")
+        get url("/1/world")
+      end.should == ["goodbye","cruel","world"]
+    end
+
+    it 'should run in parallel' do
+      start = Time.now
+      Asap do
+        get url("/1/goodbye")
+        get url("/1/cruel")
+        get url("/1/world")
+      end
+      (Time.now - start).should < 2
+    end
+  end
+
+  context 'given one nested get' do
+    it 'should return one nested result' do
+      Asap do
+        get url("/0/%2F0%2Fhello") do |resp|
+          get url(resp)
+        end
+      end.should == [["/0/hello", ["hello"]]]
+    end
+  end
+
+  context 'given two single-nested gets' do
+    it 'should return two single-nested results' do
+      Asap do
+        get url("/0/%2F0%2Fhello") do |resp|
+          get url(resp)
+        end
+        get url("/0/%2F0%2Fworld") do |resp|
+          get url(resp)
+        end
+      end.should == [["/0/hello", ["hello"]], ["/0/world", ["world"]]]
+    end
+  end
+
+  context 'given a combination of nested and flat gets' do
+    it 'should return the same combination of nested and flat results' do
+      Asap do
+        get url("/0/hello0")
+        get url("/0/%2F0%2Fhello1") do |resp|
+          get url(resp)
+        end
+        get url("/0/world0")
+        get url("/0/%2F0%2Fworld1") do |resp|
+          get url(resp)
+        end
+      end.should == ["hello0",
+        ["/0/hello1", ["hello1"]],
+        "world0",
+        ["/0/world1", ["world1"]]]
+    end
+  end
+
+  context 'given deep nesting' do
+    it 'should return a deeply-nested result' do
+      Asap do
+        get url("/0/%2F0%2F%252F0%252F%25252F0%25252Fhello") do |r1|
+          get url(r1) do |r2|
+            get url(r2) do |r3|
+              get url(r3)
+            end
+          end
+        end
+      end.should ==
+        [["/0/%2F0%2F%252F0%252Fhello", [
+          ["/0/%2F0%2Fhello", [
+            ["/0/hello", ["hello"]]]]]]]
+    end
+  end
+
+  context 'given additional arguments' do
+    it 'should pass the arguments to its block' do
+      Asap("/0/hello") do |path|
+        get url(path)
+      end.should == ["hello"]
+    end
+  end
+
+  context 'given a map over the gets' do
+    it 'should behave like the gets are listed out' do
+      Asap do
+        (1..10).each do |i|
+          get url("/1/#{i}")
+        end
+      end.should == (1..10).to_a.map(&:to_s)
+    end
+
+    it 'should behave like the gets are listed out' do
+      start = Time.now
+      Asap do
+        (1..10).each do |i|
+          get url("/1/#{i}")
+        end
+      end
+      (Time.now - start).should < 2
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require 'asap'

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification 
+name: asap
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.5
+platform: ruby
+authors: 
+  - Greg Spurrier
+  - Avik Das
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-07-18 00:00:00 -07:00
+default_executable: 
+dependencies: 
+  - !ruby/object:Gem::Dependency 
+    name: mongrel
+    prerelease: false
+    requirement: &id001 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: "0"
+    type: :runtime
+    version_requirements: *id001
+  - !ruby/object:Gem::Dependency 
+    name: rspec
+    prerelease: false
+    requirement: &id002 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: "0"
+    type: :development
+    version_requirements: *id002
+  - !ruby/object:Gem::Dependency 
+    name: rake
+    prerelease: false
+    requirement: &id003 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: "0"
+    type: :development
+    version_requirements: *id003
+description: A JRuby library for parallel fetches. Provides an embedded-domain specific language for declaring dependencies between resources.
+email: 
+  - gspurrier@linkedin.com
+  - adas@linkedin.com
+executables: 
+  - asap-em-test-server
+  - asap-mongrel-test-server
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+  - .gitignore
+  - Gemfile
+  - LICENSE
+  - README
+  - Rakefile
+  - asap.gemspec
+  - bin/asap-em-test-server
+  - bin/asap-mongrel-test-server
+  - examples/followers-example-serial.rb
+  - examples/followers-example.out
+  - examples/followers-example.rb
+  - java/netty-3.2.4.Final.jar
+  - lib/asap.rb
+  - lib/asap/fetch_context.rb
+  - lib/asap/netty.rb
+  - lib/asap/netty/http_response_handler.rb
+  - lib/asap/netty/pipeline_factory.rb
+  - lib/asap/version.rb
+  - spec/asap/netty_spec.rb
+  - spec/asap_spec.rb
+  - spec/spec_helper.rb
+has_rdoc: true
+homepage: http://rubygems.org/gems/didactic_clock
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.5.1
+signing_key: 
+specification_version: 3
+summary: A JRuby library for parallel fetches.
+test_files: 
+  - spec/asap/netty_spec.rb
+  - spec/asap_spec.rb
+  - spec/spec_helper.rb