goliath 0.9.1 → 0.9.2

data/.gitignore

@@ -16,3 +16,4 @@ _site
 examples/log
 examples/goliath.log*
 examples/goliath.pid
+examples/rasterize/thumb

data/HISTORY

@@ -0,0 +1,50 @@
+HISTORY
+=======
+
+v0.9.1 (Apr 12, 2011)
+---------------------
+  - Added extra messaging around the class not matching the file name (Carlos Brando)
+  
+  - Fix issue with POST parameters not being parsed by Goliath::Rack::Params
+  - Added support for multipart encoded POST bodies
+  - Added support for parsing nested query string parameters (Nolan Evans)
+  - Added support for parsing application/json POST bodies
+  - Content-Types outside of multipart, urlencoded and application/json will not be parsed automatically.
+  
+  - added 'run as user' option
+  - SERVER_NAME and SERVER_PORT are set to values in HOST header
+  
+  - Cleaned up spec examples (Justin Ko)
+  
+  - moved logger into 'rack.logger' key to be more Rack compliant (Env#logger added to
+    keep original API consistent)
+  - add command line option for specifying config file
+  - HTTP_CONTENT_LENGTH and HTTP_CONTENT_TYPE were changed to CONTENT_TYPE and CONTENT_LENGTH
+    to be more Rack compliant
+  - fix issue with loading config file in development mode
+  
+  - Rack::Reloader will be loaded automatically by the framework in development mode.
+
+
+v0.9.0 (Mar 9, 2011)
+--------------------
+(Initial Public Release)
+
+Goliath is an open source version of the non-blocking (asynchronous) Ruby web server framework
+powering PostRank. It is a lightweight framework designed to meet the following goals: bare
+metal performance, Rack API and middleware support, simple configuration, fully asynchronous
+processing, and readable and maintainable code (read: no callbacks).
+
+The framework is powered by an EventMachine reactor, a high-performance HTTP parser and Ruby 1.9
+runtime. One major advantage Goliath has over other asynchronous frameworks is the fact that by
+leveraging Ruby fibers, it can untangle the complicated callback-based code into a format we are
+all familiar and comfortable with: linear execution, which leads to more maintainable and readable code.
+
+While MRI is the recommend platform, Goliath has been tested to run on JRuby and Rubinius.
+
+Goliath has been in production at PostRank for over a year, serving a sustained 500 requests/s for
+internal and external applications. Many of the Goliath processes have been running for months at
+a time (read: no memory leaks) and have served hundreds of gigabytes of data without restarts. To
+scale up and provide failover and redundancy, our individual Goliath servers at PostRank are usually
+deployed behind a reverse proxy (such as HAProxy).
+

data/README.md

@@ -73,6 +73,8 @@ Goliath has been in production at PostRank for over a year, serving a sustained
 * [Stage left: Enter Goliath - HTTP Proxy + MongoDB](http://everburning.com/news/stage-left-enter-goliath/)
 * [InfoQ: Meet the Goliath of Ruby Application Servers](http://www.infoq.com/articles/meet-goliath)
 * [Node.jsはコールバック・スパゲティを招くか](http://el.jibun.atmarkit.co.jp/rails/2011/03/nodejs-d123.html)
+* [Goliath on LinuxFr.org (french)](http://linuxfr.org/news/en-vrac-spécial-ruby-jruby-sinatra-et-goliath)
+* [Goliath et ses amis (slides in french)](http://nono.github.com/Presentations/20110416_Goliath/)
 
 ## Discussion and Support
 

data/examples/activerecord/srv.rb

@@ -21,9 +21,7 @@ end
 class Srv < Goliath::API
   use Goliath::Rack::Params
   use Goliath::Rack::DefaultMimeType
-  use Goliath::Rack::Formatters::JSON
-  use Goliath::Rack::Render
-  use Goliath::Rack::ValidationError
+  use Goliath::Rack::Render, 'json'
 
   use Goliath::Rack::Validation::RequiredParam, {:key => 'id', :type => 'ID'}
   use Goliath::Rack::Validation::NumericRange, {:key => 'id', :min => 1}

data/examples/async_aroundware_demo.rb

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../lib'
+
+require 'goliath'
+require 'em-synchrony/em-http'
+require 'yajl/json_gem'
+
+#
+# Here's a way to make an asynchronous request in the middleware, and only
+# proceed with the response when both the endpoint and our middleware's
+# responses have completed.
+#
+# To run this, start the 'test_rig.rb' server on port 9002:
+#
+#   ./examples/test_rig.rb -sv -p 9002
+#
+# And then start this server on port 9000:
+#
+#   ./async_aroundware_demo.rb -sv -p 9000
+#
+# Now curl the async_aroundware_demo_multi:
+#
+#    $ time curl  'http://127.0.0.1:9000/?delay_1=1.0&delay_2=1.5'
+#         { "results": {
+#             "sleep_2": { "delay": 1.5, "actual": 1.5085558891296387 },
+#             "sleep_1": { "delay": 1.0, "actual": 1.0098700523376465 }
+#           } }
+#
+# The requests are run concurrently:
+#
+#   $ ./examples/async_aroundware_demo.rb -sv -p 9000 -e prod &
+#   [68463:INFO] 2011-05-03 23:13:03 :: Starting server on 0.0.0.0:9000 in production mode. Watch out for stones.
+#   $ ab -n10 -c10 'http://127.0.0.1:9000/?delay_1=1.5&delay_2=2.0'
+#   Connection Times (ms)
+#                 min  mean[+/-sd] median   max
+#   Connect:        0    0   0.1      0       0
+#   Processing:  2027 2111  61.6   2112    2204
+#   Waiting:     2027 2111  61.5   2112    2204
+#   Total:       2027 2112  61.5   2113    2204
+#
+#
+
+BASE_URL     = 'http://localhost:9002/'
+
+class MyResponseReceiver < Goliath::Synchrony::MultiReceiver
+  def pre_process
+    # Request with delay_1 and drop_1 -- note: 'aget', because we want execution to continue
+    req = EM::HttpRequest.new(BASE_URL).aget(:query => { :delay => env.params['delay_1'], :drop => env.params['drop_1'] })
+    add :sleep_1, req
+  end
+
+  def post_process
+    # unify the results with the results of the API call
+    responses[:callback].each{|name, resp| body[:results][name] = JSON.parse(resp.response) }
+    responses[:errback ].each{|name, err|  body[:errors][name]  = err.error     }
+    [status, headers, JSON.generate(body)]
+  end
+end
+
+class AsyncAroundwareDemo < Goliath::API
+  use Goliath::Rack::Params
+  use Goliath::Rack::Validation::NumericRange, {:key => 'delay_1', :default => 1.0, :max => 5.0, :min => 0.0, :as => Float}
+  use Goliath::Rack::Validation::NumericRange, {:key => 'delay_2', :default => 0.5, :max => 5.0, :min => 0.0, :as => Float}
+  #
+  use Goliath::Rack::AsyncAroundware, MyResponseReceiver
+
+  def response(env)
+    # Request with delay_2 and drop_2 -- note: 'get', because we want execution to proceed linearly
+    resp = EM::HttpRequest.new(BASE_URL).get(:query => { :delay => env.params['delay_2'], :drop => env.params['drop_2'] })
+
+    body = { :results => {}, :errors => {} }
+
+    if resp.response_header.status.to_i != 0
+      body[:results][:sleep_2] = JSON.parse(resp.response)
+    else
+      body[:errors ][:sleep_2] = resp.error
+    end
+
+    [200, { }, body]
+  end
+end

data/examples/async_upload.rb

@@ -7,8 +7,7 @@ require 'yajl'
 class AsyncUpload < Goliath::API
   use Goliath::Rack::Params             # parse & merge query and body parameters
   use Goliath::Rack::DefaultMimeType    # cleanup accepted media types
-  use Goliath::Rack::Formatters::JSON   # JSON output formatter
-  use Goliath::Rack::Render             # auto-negotiate response format
+  use Goliath::Rack::Render, 'json'     # auto-negotiate response format
 
   def on_headers(env, headers)
     env.logger.info 'received headers: ' + headers.inspect

data/examples/auth_and_rate_limit.rb

@@ -0,0 +1,143 @@
+#!/usr/bin/env ruby
+$: << File.join(File.dirname(__FILE__), '../lib')
+require 'goliath'
+require 'em-mongo'
+require 'em-http'
+require 'em-synchrony/em-http'
+require 'yajl/json_gem'
+
+require 'goliath/synchrony/mongo_receiver'            # has the aroundware logic for talking to mongodb
+require File.join(File.dirname(__FILE__), 'http_log') # Use the HttpLog as our actual endpoint, but include this in the middleware
+
+# Usage:
+#
+# First launch a dummy responder, like hello_world.rb or test_rig.rb:
+# ruby ./examples/hello_world.rb -sv -p 8080 -e prod &
+#
+# Then launch this script
+# ruby ./examples/auth_and_rate_limit.rb -sv -p 9000 --config $PWD/examples/config/auth_and_rate_limit.rb
+#
+
+# Tracks and enforces account and rate limit policies.
+#
+# Before the request:
+#
+# * validates the apikey exists
+# * launches requests for the account and current usage (hourly rate limit, etc)
+#
+# It then passes the request down the middleware chain; execution resumes only
+# when both the remote request and the auth info have returned.
+#
+# After remote request and auth info return:
+#
+# * Check the account exists and is valid
+# * Check the rate limit is OK
+#
+# If it passes all those checks, the request goes through; otherwise we raise an
+# error that Goliath::Rack::Validator turns into a 4xx response
+#
+# WARNING: Since this passes ALL requests through to the responder, it's only
+# suitable for idempotent requests (GET, typically).  You may need to handle
+# POST/PUT/DELETE requests differently.
+#
+#
+class AuthReceiver < Goliath::Synchrony::MongoReceiver
+  include Goliath::Validation
+  include Goliath::Rack::Validator
+  attr_accessor :account_info, :usage_info
+
+  # time period to aggregate stats over, in seconds
+  TIMEBIN_SIZE = 60 * 60
+
+  class MissingApikeyError     < BadRequestError   ; end
+  class RateLimitExceededError < ForbiddenError    ; end
+  class InvalidApikeyError     < UnauthorizedError ; end
+
+  def pre_process
+    validate_apikey!
+    first('AccountInfo', { :_id => apikey   }){|res| self.account_info = res }
+    first('UsageInfo',   { :_id => usage_id }){|res| self.usage_info   = res }
+    env.trace('pre_process_end')
+  end
+
+  def post_process
+    env.trace('post_process_beg')
+    env.logger.info [account_info, usage_info].inspect
+    self.account_info ||= {}
+    self.usage_info   ||= {}
+
+    inject_headers
+
+    EM.next_tick do
+      safely(env){ charge_usage }
+    end
+
+    safely(env, headers) do
+      check_apikey!
+      check_rate_limit!
+
+      env.trace('post_process_end')
+      [status, headers, body]
+    end
+  end
+
+  # ===========================================================================
+
+  def validate_apikey!
+    if apikey.to_s.empty?
+      raise MissingApikeyError
+    end
+  end
+
+  def check_apikey!
+    unless account_info['valid'] == true
+      raise InvalidApikeyError
+    end
+  end
+
+  def check_rate_limit!
+    return true if usage_info['calls'].to_f <= account_info['max_call_rate'].to_f
+    raise RateLimitExceededError
+  end
+
+  def charge_usage
+    update('UsageInfo', { :_id => usage_id },
+      { '$inc' => { :calls   => 1 } }, :upsert => true)
+  end
+
+  def inject_headers
+    headers.merge!({
+        'X-RateLimit-MaxRequests' => account_info['max_call_rate'].to_s,
+        'X-RateLimit-Requests'    => usage_info['calls'].to_s,
+        'X-RateLimit-Reset'       => timebin_end.to_s,
+      })
+  end
+
+  # ===========================================================================
+
+  def apikey
+    env.params['_apikey']
+  end
+
+  def usage_id
+    "#{apikey}-#{timebin}"
+  end
+
+  def timebin
+    @timebin ||= timebin_beg
+  end
+
+  def timebin_beg
+    ((Time.now.to_i / TIMEBIN_SIZE).floor * TIMEBIN_SIZE)
+  end
+
+  def timebin_end
+    timebin_beg + TIMEBIN_SIZE
+  end
+end
+
+class AuthAndRateLimit < HttpLog
+  use Goliath::Rack::Tracer, 'X-Tracer'
+  use Goliath::Rack::Params             # parse & merge query and body parameters
+  use Goliath::Rack::AsyncAroundware, AuthReceiver, 'api_auth_db'
+end

data/examples/chunked_streaming.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+$:<< '../lib' << 'lib'
+
+#
+# A simple HTTP streaming API which returns a 200 response for any GET request
+# and then emits numbers 1 through 10 in 1 second intervals using Chunked
+# transfer encoding, and finally closes the connection.
+#
+# Chunked transfer streaming works transparently with both browsers and
+# streaming consumers.
+#
+
+require 'goliath'
+
+class ChunkedStreaming < Goliath::API
+  def on_close(env)
+    env.logger.info "Connection closed."
+  end
+
+  def response(env)
+    i = 0
+    pt = EM.add_periodic_timer(1) do
+      env.chunked_stream_send("#{i}\n")
+      i += 1
+    end
+
+    EM.add_timer(10) do
+      pt.cancel
+
+      env.chunked_stream_send("!! BOOM !!\n")
+      env.chunked_stream_close
+    end
+
+    headers = { 'Content-Type' => 'text/plain', 'X-Stream' => 'Goliath' }
+    chunked_streaming_response(200, headers)
+  end
+end

data/examples/conf_test.rb

@@ -11,9 +11,7 @@ require 'goliath'
 
 class ConfTest < Goliath::API
   use Goliath::Rack::Params
-  use Goliath::Rack::DefaultMimeType
-  use Goliath::Rack::Formatters::JSON
-  use Goliath::Rack::Render
+  use Goliath::Rack::Render, 'json'
 
   def options_parser(opts, options)
     options[:test] = 0

data/examples/config/auth_and_rate_limit.rb

@@ -0,0 +1,30 @@
+import 'http_log'
+
+environment(:development) do
+
+  config['api_auth_db'] = EventMachine::Synchrony::ConnectionPool.new(:size => 20) do
+    conn = EM::Mongo::Connection.new('localhost', 27017, 1, {:reconnect_in => 1})
+    conn.db('buzzkill_test')
+  end
+
+  # for demo purposes, some dummy accounts
+  timebin = ((Time.now.to_i / 3600).floor * 3600)
+
+  # This user's calls should all go through
+  config['api_auth_db'].collection('AccountInfo').save({
+      :_id => 'i_am_awesome', 'valid' => true,  'max_call_rate' => 1_000_000 })
+
+  # this user's account is disabled
+  config['api_auth_db'].collection('AccountInfo').save({
+      :_id => 'i_am_lame',    'valid' => false, 'max_call_rate' => 1_000 })
+
+  # this user has not been seen, but will very quickly hit their limit
+  config['api_auth_db'].collection('AccountInfo').save({
+      :_id => 'i_am_limited', 'valid' => true, 'max_call_rate' =>     10 })
+
+  # fakes a user with a bunch of calls already made this hour -- two more = no yuo
+  config['api_auth_db'].collection('AccountInfo').save({
+      :_id => 'i_am_busy',    'valid' => true, 'max_call_rate' =>  1_000 })
+  config['api_auth_db'].collection('UsageInfo').save({
+      :_id => "i_am_busy-#{timebin}", 'calls' =>  999 })
+end

data/examples/config/template.rb

@@ -0,0 +1,8 @@
+config[:template] = {
+  :layout_engine => :haml,
+}
+config[:template_engines] = {
+  :haml => {
+    :escape_html   => true
+  }
+}

data/examples/content_stream.rb

@@ -12,12 +12,10 @@ require 'goliath'
 # rabbitmqctl set_permissions -p /test test ".*" ".*" ".*"
 
 class ContentStream < Goliath::API
-  use Goliath::Rack::Formatters::JSON
   use Goliath::Rack::Params
 
-  use Goliath::Rack::Render
+  use Goliath::Rack::Render, 'json'
   use Goliath::Rack::Heartbeat
-  use Goliath::Rack::ValidationError
   use Goliath::Rack::Validation::RequestMethod, %w(GET)
 
   def on_close(env)

data/examples/echo.rb

@@ -4,18 +4,20 @@ $:<< '../lib' << 'lib'
 require 'goliath'
 require 'goliath/plugins/latency'
 
-# Goliath uses multi-jon, so pick your favorite JSON serializer
+# Goliath uses multi-json, so pick your favorite JSON serializer
 # require 'json'
 require 'yajl'
 
 class Echo < Goliath::API
-  use Goliath::Rack::Params             # parse & merge query and body parameters
+  use Goliath::Rack::Tracer             # log trace statistics
   use Goliath::Rack::DefaultMimeType    # cleanup accepted media types
-  use Goliath::Rack::Formatters::JSON   # JSON output formatter
-  use Goliath::Rack::Render             # auto-negotiate response format
+  use Goliath::Rack::Render, 'json'     # auto-negotiate response format
+  use Goliath::Rack::Params             # parse & merge query and body parameters
   use Goliath::Rack::Heartbeat          # respond to /status with 200, OK (monitoring, etc)
-  use Goliath::Rack::ValidationError    # catch and render validation errors
 
+  # If you are using Golaith version <=0.9.1 you need to Goliath::Rack::ValidationError
+  # to prevent the request from remaining open after an error occurs
+  #use Goliath::Rack::ValidationError
   use Goliath::Rack::Validation::RequestMethod, %w(GET POST)           # allow GET and POST requests only
   use Goliath::Rack::Validation::RequiredParam, {:key => 'echo'}  # must provide ?echo= query or body param
 
@@ -30,4 +32,4 @@ class Echo < Goliath::API
   def response(env)
     [200, {}, process_request]
   end
-end
+end