schleyfox-ernie 2.2.1

data/.document

@@ -0,0 +1,5 @@
+README.md
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,2 @@
+*.beam
+pkg

data/History.txt

@@ -0,0 +1,40 @@
+= 2.2.0 / 2010-03-12
+  * Minor Additions
+    * Set procline for external Ruby handlers
+
+= 2.1.0 / 2010-02-20
+  * Major Additions
+    * Add access logging
+
+= 2.0.0 / 2010-02-16
+  * Major Changes
+    * Use configuration file for defining handlers
+    * Add Native Erlang modules
+    * Abstract handler logic to support handlers in any language
+    * Add High/Low connection queues
+    * Remove Ruby DSL (must use Ernie.expose now)
+
+= 1.3.0 / 2009-11-30
+  * API Additions
+    * Add loglevel for setting log level
+    * Add Ernie.auto_start bool
+  * Major changes
+    * Better logging granularity
+
+= 1.2.0 / 2009-11-23
+  * API Additions
+    * Add Ernie.expose
+  * Internal Changes
+    * Remove 15s internal timeout
+
+= 1.1.0 / 2009-10-28
+  * Major changes
+    * Remove dependency on Erlectricity
+    * Simplify processing loop
+
+= 1.0.0 / 2009-10-19
+  * No Changes. Production ready!
+
+= 0.4.0 / 2009-10-08
+  * Major changes
+    * Convert to use BERT gem.

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Tom Preston-Werner
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,306 @@
+Ernie
+=====
+
+By Tom Preston-Werner (tom@mojombo.com)
+
+Ernie is a BERT-RPC server implementation that uses an Erlang server to accept
+incoming connections, and then delegates the request to custom modules that
+you can write in any language (currently only Ruby and Erlang support is
+included).
+
+Modules that are written in Ruby or any non-Erlang language are known as
+"external" modules and you must specify how many workers of each module should
+be spawned. Requests against these modules are balanced between the workers.
+Modules that are written in Erlang are known as "native" modules and run
+within the Erlang server's runtime. Since these are spawned as lightweight
+processes, there is no balancing necessary and much less communication
+overhead when compared to external modules.
+
+Ernie supports multiple heterogenous modules. For instance, you can have an
+external Ruby module running 10 workers *and* a native Erlang module running
+simultaneously. Ernie keeps track of sending requests to the proper module.
+Using a technique called "shadowing," you can selectively optimize certain
+external module functions with native code and Ernie will handle selecting the
+correct function.
+
+See the full BERT-RPC specification at [bert-rpc.org](http://bert-rpc.org).
+
+Ernie currently supports the following BERT-RPC features:
+
+* `call` requests
+* `cast` requests
+
+Ernie was developed for GitHub and is currently in production use serving
+millions of RPC requests every day. The stability and performance have been
+exemplary.
+
+Ernie follows [Semantic Versioning](http://semver.org/) for release
+versioning.
+
+Installation
+------------
+
+Step 1: Install Erlang (R13B or higher).
+
+    http://www.erlang.org/download.html
+
+Step 2: Install Ernie:
+
+    $ [sudo] gem install ernie
+
+
+Running
+-------
+
+    Usage: ernie [command] [options]
+        -c, --config CONFIG              Config file.
+        -p, --port PORT                  Port.
+        -l, --log-level                  Log level (0-4).
+        -a, --access-log LOGFILE         Access log file
+        -d, --detached                   Run as a daemon.
+        -P, --pidfile PIDFILE            Location to write pid file.
+
+    Commands:
+      <none>                Start an Ernie server.
+      reload-handlers       Gracefully reload all of the external handlers
+                            and use the new code for all subsequent requests.
+      stats                 Print a list of connection and handler statistics.
+
+    Examples:
+      ernie -d -p 9999 -c example.cfg
+        Start the ernie server in the background on port 9999 using the
+        example.cfg configuration file.
+
+      ernie reload-handlers -p 9999
+        Reload the handlers for the ernie server currently running on
+        port 9999.
+
+
+Configuration File
+------------------
+
+Ernie configuration files are written as a series of dotted Erlang terms. Each
+term is a list of 2-tuples that specify options for a module.
+
+### Native Modules
+
+The form for native modules is:
+
+    [{module, Module},
+     {type, native},
+     {codepaths, CodePaths}].
+
+Where Module is an atom corresponding to the module name and CodePaths is a
+list of strings representing the file paths that should be added to the
+runtime's code path. These paths will be prepended to the code path and must
+include the native module's directory and the directories of any dependencies.
+
+### External Modules
+
+The form for external modules is:
+
+    [{module, Module},
+     {type, external},
+     {command, Command},
+     {count, Count}].
+
+Where Module is an atom corresponding to the module name, Command is a string
+specifying the command to be executed in order to start a worker, and Count is
+the number of workers to spawn.
+
+### Shadowing
+
+If you specify a native module and an external module of the same name (and in
+that order), Ernie will inspect the native module to see if it has the
+requested function exported and use that if it does. If it does not, then it
+will fall back on the external module. This can be used to selectively
+optimize certain functions in a module without any modifications to your
+client code.
+
+### Predicate Shadowing
+
+In some circumstances it can be nice to conditionally shadow a function in an
+external module based on the nature of the arguments. For example, you might
+want requests for `math:fib(X)` to be routed to the external module when X is
+less than 10, but to be handled by the native module when X is 10 or greater.
+This can be accomplished by implementing a function `math:fib_pred(X)` in the
+native module. Notice the `_pred` appended to the normal function name (pred
+is short for predicate). If a function like this is present, Ernie will call
+it with the requested arguments and if the return value is `true` the native
+module will be used. If the return value is `false` the external module will
+be used.
+
+
+Example Configuration File
+--------------------------
+
+The following example config file informs Ernie of two modules. The first term
+identifies a native module 'nat' that resides in the nat.beam file under the
+'/path/to/app/ebin' directory. The second term specifies an external module
+'ext' that will have 2 workers started with the command 'ruby
+/path/to/app/ernie/ext.rb'.
+
+    [{module, nat},
+     {type, native},
+     {codepaths, ["/path/to/app/ebin"]}].
+
+    [{module, ext},
+     {type, external},
+     {command, "ruby /path/to/app/ernie/ext.rb"},
+     {count, 2}].
+
+
+Access Log
+----------
+
+If you have requested that an access log be written (using the -a or
+--access-log option) then all requests will be logged to that file. Each
+request is printed on a single line. The elements of the log line are as
+follows (with comments on the right side):
+
+    ACC                                  type of message [ ACC | ERR ]
+    [2010-02-20T11:42:25.259750]         time the connection was accepted
+    0.000053                             seconds from connection to processing start
+    0.000237                             seconds from processing start to finish
+    -                                    delimiter
+    0                                    size of high queue at connect time
+    0                                    size of low queue at connect time
+    nat                                  type of handler [ nat | ext ]
+    high                                 priority [ high | low ]
+    -                                    delimiter
+    {call,nat,add,[1,2]}                 message
+
+
+Log lines are written when the request completes so they may appear out of
+order with respect to connection time. To facilitate log rotation, Ernie will
+create a new access log file if the current log file is moved or deleted.
+
+
+Native (Erlang) Handler
+-----------------------
+
+Native handlers are written as normal Erlang modules. The exported functions
+will become available to BERT-RPC clients.
+
+### Example
+
+    -module(nat).
+    -export([add/2]).
+
+    add(A, B) ->
+      A + B.
+
+### BERT-RPC Sequence Example
+
+    -> {call, nat, add, [1, 2]}
+    <- {reply, 3}
+
+
+External (Ruby) Handler
+-----------------------
+
+Included in this gem is a library called `ernie` that makes it easy to write
+Ernie handlers in Ruby. All you have to do is write a standard Ruby module and
+expose it to Ernie and the functions of that module will become available to
+BERT-RPC clients.
+
+### Example
+
+Using a Ruby module and Ernie.expose:
+
+    require 'rubygems'
+    require 'ernie'
+    
+    module Ext
+      def add(a, b)
+        a + b
+      end
+    end
+    
+    Ernie.expose(:ext, Ext)
+
+### BERT-RPC Sequence Example
+
+    -> {call, nat, add, [1, 2]}
+    <- {reply, 3}
+
+### Logging
+
+You can have logging sent to a file by adding these lines to your handler:
+
+    logfile('/var/log/ernie.log')
+    loglevel(Logger::INFO)
+
+This will log startup info, requests, and error messages to the log. Choosing
+Logger::DEBUG will include the response (be careful, doing this can generate
+very large log files).
+
+### Autostart
+
+Normally Ruby Ernie handlers will become active after the file has been loaded
+in. you can disable this behavior by setting:
+
+    Ernie.auto_start = false
+
+
+Selecting Queue Priority
+------------------------
+
+Ernie maintains High and Low priority queues for incoming connections. If
+there are any connections in the High priority queue, they will always be
+processed first. If the High priority queue is empty, connections will be
+processed from the Low priority queue. By default, connections go into the
+High priority queue. To select a queue, an info BERP of the following form
+must be sent preceding the call.
+
+    -- {info, priority, Priority}
+
+Where `Priority` is either the `high` or `low` atom. An example sequence where
+the low priority queue is being selected would look like the following.
+
+    -> {info, priority, low}
+    -> {call, nat, add, [1, 2]}
+    <- {reply, 3}
+
+
+Using the BERTRPC gem to make calls to Ernie
+--------------------------------------------
+
+You can make BERT-RPC calls from Ruby with the [BERTRPC gem](http://github.com/mojombo/bertrpc):
+
+    require 'bertrpc'
+
+    svc = BERTRPC::Service.new('localhost', 8000)
+    svc.call.ext.add(1, 2)
+    # => 3
+
+
+Contribute
+----------
+
+If you'd like to hack on Ernie, start by forking my repo on GitHub:
+
+    http://github.com/mojombo/ernie
+
+To get all of the dependencies, install the gem first. To run ernie from
+source, you must first build the Erlang code:
+
+    rake ebuild
+
+The best way to get your changes merged back into core is as follows:
+
+1. Clone down your fork
+1. Create a topic branch to contain your change
+1. Hack away
+1. Add tests and make sure everything still passes by running `rake`
+1. If you are adding new functionality, document it in the README.md
+1. Do not change the version number, I will do that on my end
+1. If necessary, rebase your commits into logical chunks, without errors
+1. Push the branch up to GitHub
+1. Send me (mojombo) a pull request for your branch
+
+
+Copyright
+---------
+
+Copyright (c) 2009 Tom Preston-Werner. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,67 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "ernie"
+    gem.rubyforge_project = "ernie"
+    gem.summary = %Q{Ernie is a BERT-RPC server implementation.}
+    gem.description = %Q{Ernie is an Erlang/Ruby hybrid BERT-RPC server implementation packaged as a gem.}
+    gem.email = "tom@mojombo.com"
+    gem.homepage = "http://github.com/mojombo/ernie"
+    gem.authors = ["Tom Preston-Werner"]
+    gem.files.include(["ext"])
+    gem.extensions << 'ext/extconf.rb'
+    gem.add_dependency('bert', '>= 1.1.0')
+    gem.add_dependency('bertrpc', '>= 1.0.0')
+
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :default => :test
+
+# require 'rake/rdoctask'
+# Rake::RDocTask.new do |rdoc|
+#   if File.exist?('VERSION.yml')
+#     config = YAML.load(File.read('VERSION.yml'))
+#     version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
+#   else
+#     version = ""
+#   end
+# 
+#   rdoc.rdoc_dir = 'rdoc'
+#   rdoc.title = "ernie #{version}"
+#   rdoc.rdoc_files.include('README*')
+#   rdoc.rdoc_files.include('lib/**/*.rb')
+# end
+
+task :ebuild do
+  ERLC_TEST_FLAGS = ""
+  ERLC_FLAGS = "-o ../ebin"
+  cd "elib"
+  sh "erlc  #{ERLC_FLAGS} #{ERLC_TEST_FLAGS} #{Dir["**/*.erl"].join(" ")}"
+end

data/VERSION.yml

@@ -0,0 +1,5 @@
+--- 
+:major: 2
+:minor: 2
+:patch: 0
+:build: 

data/bin/ernie

@@ -0,0 +1,116 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.join(File.dirname(__FILE__), *%w[.. lib]))
+ERNIE_ROOT = File.join(File.dirname(__FILE__), *%w[..])
+
+DEFAULT_ERLANG_CODEPATHS = %w[ebin]
+DEFAULT_PORT = 8000
+
+def rel(path)
+  File.join(ERNIE_ROOT, path)
+end
+
+def code_paths
+  DEFAULT_ERLANG_CODEPATHS.map {|n| "-pz #{rel(n)}" }.join(" ") + " \\"
+end
+
+def version
+  yml = YAML.load(File.read(File.join(File.dirname(__FILE__), *%w[.. VERSION.yml])))
+  "#{yml[:major]}.#{yml[:minor]}.#{yml[:patch]}"
+end
+
+require 'optparse'
+require 'pp'
+require 'yaml'
+
+help = <<HELP
+Ernie is an Erlang/Ruby BERT-RPC Server.
+
+Basic Command Line Usage:
+  ernie [command] [options]
+
+Commands:
+  <none>                Start an Ernie server.
+  reload-handlers       Gracefully reload all of the the ruby handlers
+                        and use the new code for all subsequent requests.
+
+Options:
+HELP
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = help
+  opts.version = version
+
+  opts.on("-c CONFIG", "--config CONFIG", "Config file") do |x|
+    options[:config] = x
+  end
+
+  opts.on("-p PORT", "--port PORT", "Port") do |x|
+    options[:port] = x
+  end
+
+  opts.on("-l LOGLEVEL", "--log-level LOGLEVEL", "Log level (0-4)") do |x|
+    options[:log_level] = x
+  end
+
+  opts.on("-a LOGFILE", "--access-log LOGFILE", "Access log file") do |x|
+    options[:access_log] = x
+  end
+
+  opts.on("-d", "--detached", "Run as a daemon") do
+    options[:detached] = true
+  end
+
+  opts.on("-P", "--pidfile PIDFILE", "Location to write pid file.") do |x|
+    options[:pidfile] = x
+  end
+
+  opts.on("--name NAME", "Erlang node name") do |x|
+    options[:name] = x
+  end
+end.parse!
+
+if command = ARGV[0]
+  if !%w{reload-handlers stats}.include?(command)
+    puts "Invlalid command. Valid commands are:"
+    puts "  reload-handlers"
+    puts "  stats"
+    exit(1)
+  end
+
+  require 'rubygems'
+  require 'bertrpc'
+  port = options[:port] || DEFAULT_PORT
+  svc = BERTRPC::Service.new('localhost', port)
+  puts svc.call.__admin__.send(command.gsub(/-/, '_'))
+else
+  if !options[:config]
+    puts "A config file must be specified: ernie -c /path/to/config.yml"
+    exit(1)
+  end
+
+  config = options[:config]
+  port = options[:port] || DEFAULT_PORT
+  log_level = options[:log_level] || 2
+  pidfile = options[:pidfile] ? "-ernie_server_app pidfile \"'#{options[:pidfile]}'\"" : ''
+  detached = options[:detached] ? '-detached' : ''
+  access_log = options[:access_log] ? "-ernie_server_app access_log '\"#{options[:access_log]}\"'" : ''
+  name = options[:name] ? "-name '#{options[:name]}'" : ''
+
+  cmd = %Q{erl -boot start_sasl \
+               #{detached} \
+               +Bc \
+               +K true \
+               -smp enable \
+               #{code_paths}
+               #{pidfile} \
+               #{access_log} \
+               #{name} \
+               -ernie_server_app port #{port} \
+               -ernie_server_app config '"#{config}"' \
+               -ernie_server_app log_level #{log_level} \
+               -run ernie_server_app boot}.squeeze(' ')
+  puts cmd
+  exec(cmd)
+end

data/contrib/ebench.erl

@@ -0,0 +1,76 @@
+% erlc *.erl && erl ebench.beam -run ebench start 10000 20 ext add
+
+-module(ebench).
+-export([start/1]).
+
+start([Ni, Ci, Modi, Funi]) ->
+  Nt = list_to_integer(Ni),
+  C = list_to_integer(Ci),
+  Mod = list_to_atom(Modi),
+  Fun = list_to_atom(Funi),
+  N = round(Nt / C),
+  T0 = erlang:now(),
+  Waiter = spawn(fun() -> wait(T0, N * C) end),
+  spawner(Waiter, N, C, Mod, Fun).
+
+spawner(_Waiter, _N, 0, _Mod, _Fun) ->
+  ok;
+spawner(Waiter, N, C, Mod, Fun) ->
+  spawn(fun() -> loop(Waiter, N, Mod, Fun) end),
+  spawner(Waiter, N, C - 1, Mod, Fun).
+
+% X is the total number of responses to wait for
+wait(T0, XTotal, 0) ->
+  T1 = erlang:now(),
+  Diff = timer:now_diff(T1, T0),
+  Mean = Diff / XTotal,
+  io:format("~p requests completed in ~.2fs~n", [XTotal, Diff / 1000000]),
+  io:format("Mean request time: ~.2fms (~.2f r/s)~n", [Mean / 1000, XTotal / (Diff / 1000000)]),
+  init:stop();
+wait(T0, XTotal, X) ->
+  receive
+    done -> wait(T0, XTotal, X - 1)
+  end.
+
+wait(T0, X) ->
+  wait(T0, X, X).
+
+loop(_Waiter, 0, _Mod, _Fun) ->
+  ok;
+loop(Waiter, N, Mod, Fun) ->
+  hit(Waiter, Mod, Fun),
+  loop(Waiter, N - 1, Mod, Fun).
+
+hit(Waiter, Mod, Fun) ->
+  % io:format("outgoing!~n", []),
+  Host = "localhost",
+  case gen_tcp:connect(Host, 8000, [binary, {packet, 4}]) of
+    {ok, Sock} -> process(Waiter, Mod, Fun, Sock);
+    Any ->
+      io:format("Unable to establish connection: ~p~n", [Any]),
+      Waiter ! done
+  end.
+
+process(Waiter, Mod, Fun, Sock) ->
+  % Info = term_to_binary({info, priority, [low]}),
+  % ok = gen_tcp:send(Sock, Info),
+  Request = term_to_binary({call, Mod, Fun, args(Fun)}),
+  ok = gen_tcp:send(Sock, Request),
+  receive
+    {tcp, _Port, Reply} ->
+      % io:format("~p~n", [Reply]),
+      Res = res(Fun),
+      {reply, Res} = binary_to_term(Reply);
+    {tcp_closed, Port} ->
+      io:format("Connection closed after sending data: ~p~n", [Port]);
+    Any ->
+      io:format("Unexpected message: ~p~n", [Any])
+  end,
+  Waiter ! done,
+  ok = gen_tcp:close(Sock).
+
+args(add) -> [1, 2];
+args(fib) -> [20].
+
+res(add) -> 3;
+res(fib) -> 10946.

data/ebin/ernie_server_app.app

@@ -0,0 +1,2 @@
+{application, ernie_server_app,
+  [{mod, {ernie_server_app, []}}]}.