ernie 2.0.0 → 2.1.0

data/History.txt

@@ -1,3 +1,7 @@
+= 2.1.0 / 2010-02-20
+  * Major Additions
+    * Add access logging
+
 = 2.0.0 / 2010-02-16
   * Major Changes
     * Use configuration file for defining handlers

data/README.md

@@ -40,13 +40,13 @@ versioning.
 Installation
 ------------
 
-Step 1: Install Erlang.
+Step 1: Install Erlang (R13B or higher).
 
     http://www.erlang.org/download.html
 
 Step 2: Install Ernie:
 
-    $ gem install ernie
+    $ [sudo] gem install ernie
 
 
 Running
@@ -56,6 +56,7 @@ Running
         -c, --config CONFIG              Config file.
         -p, --port PORT                  Port.
         -l, --log-level                  Log level (0-4).
+        -a, --access-log LOGFILE         Access log file
         -d, --detached                   Run as a daemon.
         -P, --pidfile PIDFILE            Location to write pid file.
 
@@ -149,6 +150,32 @@ identifies a native module 'nat' that resides in the nat.beam file under the
      {count, 2}].
 
 
+Access Log
+----------
+
+If you have requested that an access log be written (using the -a or
+--access-log option) then all requests will be logged to that file. Each
+request is printed on a single line. The elements of the log line are as
+follows (with comments on the right side):
+
+    ACC                                  type of message [ ACC | ERR ]
+    [2010-02-20T11:42:25.259750]         time the connection was accepted
+    0.000053                             seconds from connection to processing start
+    0.000237                             seconds from processing start to finish
+    -                                    delimiter
+    0                                    size of high queue at connect time
+    0                                    size of low queue at connect time
+    nat                                  type of handler [ nat | ext ]
+    high                                 priority [ high | low ]
+    -                                    delimiter
+    {call,nat,add,[1,2]}                 message
+
+
+Log lines are written when the request completes so they may appear out of
+order with respect to connection time. To facilitate log rotation, Ernie will
+create a new access log file if the current log file is moved or deleted.
+
+
 Native (Erlang) Handler
 -----------------------
 
@@ -181,6 +208,7 @@ BERT-RPC clients.
 
 Using a Ruby module and Ernie.expose:
 
+    require 'rubygems'
     require 'ernie'
     
     module Ext

data/VERSION.yml

@@ -1,5 +1,5 @@
 --- 
 :major: 2
-:minor: 0
+:minor: 1
 :patch: 0
 :build: 

data/bin/ernie

@@ -54,6 +54,10 @@ OptionParser.new do |opts|
     options[:log_level] = x
   end
 
+  opts.on("-a LOGFILE", "--access-log LOGFILE", "Access log file") do |x|
+    options[:access_log] = x
+  end
+
   opts.on("-d", "--detached", "Run as a daemon") do
     options[:detached] = true
   end
@@ -87,6 +91,7 @@ else
   log_level = options[:log_level] || 2
   pidfile = options[:pidfile] ? "-ernie_server_app pidfile \"'#{options[:pidfile]}'\"" : ''
   detached = options[:detached] ? '-detached' : ''
+  access_log = options[:access_log] ? "-ernie_server_app access_log '\"#{options[:access_log]}\"'" : ''
 
   cmd = %Q{erl -boot start_sasl \
                #{detached} \
@@ -95,6 +100,7 @@ else
                -smp enable \
                #{code_paths}
                #{pidfile} \
+               #{access_log} \
                -ernie_server_app port #{port} \
                -ernie_server_app config '"#{config}"' \
                -ernie_server_app log_level #{log_level} \

data/elib/ernie.hrl

@@ -5,6 +5,14 @@
                 map = undefined}).      % module map. tuples of {Mod, Id}
 
 -record(request, {sock = undefined,     % connection socket
+                  log = undefined,      % log information
                   infos = [],           % list of info binaries
                   action = undefined,   % action binary
-                  priority = high}).    % priority [ high | low ]
+                  priority = high}).    % priority [ high | low ]
+
+-record(log, {taccept = erlang:now(),   % time that connection was accepted
+              tprocess = erlang:now(),  % time that processing started
+              tdone = erlang:now(),     % time that processing and response is done
+              hq = 0,                   % size of high queue at acceptance
+              lq = 0,                   % size of low queue at acceptance
+              type = unk}).             % type [ unk | nat | ext ]

data/elib/ernie_access_logger.erl

@@ -0,0 +1,170 @@
+-module(ernie_access_logger).
+-behaviour(gen_server).
+
+%% api
+-export([start_link/1, start/1, acc/1, err/3, reopen/0]).
+
+%% gen_server callbacks
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
+         terminate/2, code_change/3]).
+
+-include_lib("ernie.hrl").
+
+-record(lstate, {access_file_name = undefined,
+                 access_file = undefined}).
+
+%%====================================================================
+%% API
+%%====================================================================
+
+start_link(Args) ->
+  gen_server:start_link({global, ?MODULE}, ?MODULE, Args, []).
+
+start(Args) ->
+  gen_server:start({global, ?MODULE}, ?MODULE, Args, []).
+
+acc(Request) ->
+  gen_server:cast({global, ?MODULE}, {acc, Request}).
+
+err(Request, Msg, Args) ->
+  gen_server:cast({global, ?MODULE}, {err, Request, Msg, Args}).
+
+reopen() ->
+  gen_server:cast({global, ?MODULE}, reopen).
+
+%%====================================================================
+%% gen_server callbacks
+%%====================================================================
+
+%%--------------------------------------------------------------------
+%% Function: init(Args) -> {ok, State} |
+%%                         {ok, State, Timeout} |
+%%                         ignore               |
+%%                         {stop, Reason}
+%% Description: Initiates the server
+%%--------------------------------------------------------------------
+init([undefined]) ->
+  error_logger:info_msg("~p starting~n", [?MODULE]),
+  {ok, #lstate{}};
+init([AccessFileName]) ->
+  error_logger:info_msg("~p starting~n", [?MODULE]),
+  case file:open(AccessFileName, [append]) of
+    {ok, AccessFile} ->
+      {ok, _T} = timer:apply_interval(10000, ernie_access_logger, reopen, []),
+      {ok, #lstate{access_file_name = AccessFileName,
+                   access_file = AccessFile}};
+    {error, Error} ->
+      error_logger:error_msg("Error opening access log ~p: ~p.~n", [AccessFileName, Error]),
+      {ok, #lstate{}}
+  end.
+
+%%--------------------------------------------------------------------
+%% Function: %% handle_call(Request, From, State) -> {reply, Reply, State} |
+%%                                      {reply, Reply, State, Timeout} |
+%%                                      {noreply, State} |
+%%                                      {noreply, State, Timeout} |
+%%                                      {stop, Reason, Reply, State} |
+%%                                      {stop, Reason, State}
+%% Description: Handling call messages
+%%--------------------------------------------------------------------
+handle_call(_Request, _From, State) ->
+  {reply, ok, State}.
+
+%%--------------------------------------------------------------------
+%% Function: handle_cast(Msg, State) -> {noreply, State} |
+%%                                      {noreply, State, Timeout} |
+%%                                      {stop, Reason, State}
+%% Description: Handling cast messages
+%%--------------------------------------------------------------------
+handle_cast({acc, Request}, State) ->
+  case State#lstate.access_file_name of
+    undefined -> ok;
+    _AccessFilename -> acc(Request, State)
+  end,
+  {noreply, State};
+handle_cast({err, Request, Msg, Args}, State) ->
+  case State#lstate.access_file_name of
+    undefined -> ok;
+    _AccessFilename -> err(Request, Msg, Args, State)
+  end,
+  {noreply, State};
+handle_cast(reopen, State) ->
+  case State#lstate.access_file_name of
+    undefined ->
+      {noreply, State};
+    AccessFileName ->
+      case file:read_file_info(AccessFileName) of
+        {ok, _FileInfo} ->
+          {noreply, State};
+        {error, enoent} ->
+          ok = file:close(State#lstate.access_file),
+          {ok, AccessFile} = file:open(AccessFileName, [append]),
+          {noreply, State#lstate{access_file = AccessFile}};
+        _OtherError ->
+          {noreply, #lstate{}}
+      end
+  end;
+handle_cast(_Msg, State) ->
+  {noreply, State}.
+
+handle_info(Msg, State) ->
+  error_logger:error_msg("Unexpected message: ~p~n", [Msg]),
+  {noreply, State}.
+
+terminate(_Reason, _State) -> ok.
+code_change(_OldVersion, State, _Extra) -> {ok, State}.
+
+%%====================================================================
+%% Internal
+%%====================================================================
+
+acc(Request, State) ->
+  StatString = stat_string(Request),
+  ActionString = action_string(Request),
+  Line = io_lib:fwrite("ACC ~s - ~s~n", [StatString, ActionString]),
+  file:write(State#lstate.access_file, Line).
+
+err(Request, Msg, Args, State) ->
+  StatString = stat_string(Request),
+  ActionString = action_string(Request),
+  ErrString = io_lib:fwrite(Msg, Args),
+  Line = io_lib:fwrite("ERR ~s - ~s : ~s~n", [StatString, ErrString, ActionString]),
+  file:write(State#lstate.access_file, Line).
+
+stat_string(Request) ->
+  Log = Request#request.log,
+  TAccept = time_tuple_to_iso_8601_date(Log#log.taccept),
+  D1 = time_difference_in_seconds(Log#log.taccept, Log#log.tprocess),
+  D2 = time_difference_in_seconds(Log#log.tprocess, Log#log.tdone),
+  Type = Log#log.type,
+  HQ = Log#log.hq,
+  LQ = Log#log.lq,
+  Prio = Request#request.priority,
+  Args = [TAccept, D1, D2, HQ, LQ, Type, Prio],
+  io_lib:fwrite("[~s] ~f ~f - ~B ~B ~3s ~p", Args).
+
+action_string(Request) ->
+  TermAction = binary_to_term(Request#request.action),
+  RawAction = lists:flatten(io_lib:fwrite("~1000000000.0.0p", [TermAction])),
+  case string:len(RawAction) > 150 of
+    true ->
+      Action = re:replace(RawAction, "\n", "", [global, {return, list}]),
+      [string:sub_string(Action, 1, 150), "..."];
+    false ->
+      RawAction
+  end.
+
+time_tuple_to_iso_8601_date(TimeTuple) ->
+  {{YY, MM, DD}, {H, M, S}} = calendar:now_to_local_time(TimeTuple),
+  {_MegaSecs, _Secs, MicroSecs} = TimeTuple,
+  Args = [YY, MM, DD, H, M, S, MicroSecs],
+  io_lib:fwrite("~4B-~2.10.0B-~2.10.0BT~2.10.0B:~2.10.0B:~2.10.0B.~-6.10.0B", Args).
+
+time_difference_in_seconds(T1, T2) ->
+  {_, _, MS1} = T1,
+  {_, _, MS2} = T2,
+  S1 = calendar:datetime_to_gregorian_seconds(calendar:now_to_local_time(T1)),
+  S2 = calendar:datetime_to_gregorian_seconds(calendar:now_to_local_time(T2)),
+  F1 = S1 + (MS1 / 1000000),
+  F2 = S2 + (MS2 / 1000000),
+  F2 - F1.

data/elib/ernie_access_logger_sup.erl

@@ -0,0 +1,15 @@
+-module(ernie_access_logger_sup).
+-behaviour(supervisor).
+-export([start_link/1, init/1]).
+
+start_link(AccessLog) ->
+  supervisor:start_link({local, ?MODULE}, ?MODULE, [AccessLog]).
+
+init([AccessLog]) ->
+  case AccessLog of
+    undefined -> io:format("No access log~n", []);
+    Any -> io:format("Using access log ~p~n", [Any])
+  end,
+  {ok, {{one_for_one, 1, 60},
+    [{ernie_access_logger, {ernie_access_logger, start_link, [[AccessLog]]},
+    permanent, brutal_kill, worker, [ernie_access_logger]}]}}.

data/elib/ernie_native.erl

@@ -19,4 +19,8 @@ process(ActionTerm, Request) ->
       Data = term_to_binary({error, [user, 0, <<"RuntimeError">>, BError, BTrace]}),
       gen_tcp:send(Sock, Data)
   end,
-  ok = gen_tcp:close(Sock).
+  ok = gen_tcp:close(Sock),
+  Log = Request#request.log,
+  Log2 = Log#log{tdone = erlang:now()},
+  Request2 = Request#request{log = Log2},
+  ernie_access_logger:acc(Request2).

data/elib/ernie_server.erl

@@ -64,7 +64,10 @@ handle_call(_Request, _From, State) ->
 %% Description: Handling cast messages
 %%--------------------------------------------------------------------
 handle_cast({process, Sock}, State) ->
-  Request = #request{sock = Sock},
+  Log = #log{hq = queue:len(State#state.hq),
+             lq = queue:len(State#state.lq),
+             taccept = erlang:now()},
+  Request = #request{sock = Sock, log = Log},
   State2 = receive_term(Request, State),
   {noreply, State2};
 handle_cast(kick, State) ->
@@ -244,7 +247,10 @@ process_native_request(ActionTerm, Request, Priority, Q2, State) ->
   Count = State#state.count,
   State2 = State#state{count = Count + 1},
   logger:debug("Count = ~p~n", [Count + 1]),
-  spawn(fun() -> ernie_native:process(ActionTerm, Request) end),
+  Log = Request#request.log,
+  Log2 = Log#log{type = native, tprocess = erlang:now()},
+  Request2 = Request#request{log = Log2},
+  spawn(fun() -> ernie_native:process(ActionTerm, Request2) end),
   finish(Priority, Q2, State2).
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -257,7 +263,10 @@ process_external_request(Pid, Request, Priority, Q2, State) ->
   case asset_pool:lease(Pid) of
     {ok, Asset} ->
       logger:debug("Leased asset for pool ~p~n", [Pid]),
-      spawn(fun() -> process_now(Pid, Request, Asset) end),
+      Log = Request#request.log,
+      Log2 = Log#log{type = external, tprocess = erlang:now()},
+      Request2 = Request#request{log = Log2},
+      spawn(fun() -> process_now(Pid, Request2, Asset) end),
       finish(Priority, Q2, State2);
     empty ->
       State
@@ -265,13 +274,23 @@ process_external_request(Pid, Request, Priority, Q2, State) ->
 
 process_now(Pid, Request, Asset) ->
   try unsafe_process_now(Request, Asset) of
-    _AnyResponse -> ok
+    _AnyResponse ->
+      Log = Request#request.log,
+      Log2 = Log#log{tdone = erlang:now()},
+      Request2 = Request#request{log = Log2},
+      ernie_access_logger:acc(Request2)
   catch
-    _AnyClass:_AnyError -> ok
+    AnyClass:AnyError ->
+      Log = Request#request.log,
+      Log2 = Log#log{tdone = erlang:now()},
+      Request2 = Request#request{log = Log2},
+      ernie_access_logger:err(Request2, "External process error ~w: ~w", [AnyClass, AnyError])
   after
     asset_pool:return(Pid, Asset),
     ernie_server:kick(),
-    gen_tcp:close(Request#request.sock)
+    logger:debug("Returned asset ~p~n", [Asset]),
+    gen_tcp:close(Request#request.sock),
+    logger:debug("Closed socket ~p~n", [Request#request.sock])
   end.
 
 unsafe_process_now(Request, Asset) ->
@@ -284,8 +303,7 @@ unsafe_process_now(Request, Asset) ->
       {asset, Port, Token} = Asset,
       logger:debug("Asset: ~p ~p~n", [Port, Token]),
       {ok, Data} = port_wrapper:rpc(Port, BinaryTerm),
-      gen_tcp:send(Sock, Data),
-      ok = gen_tcp:close(Sock);
+      ok = gen_tcp:send(Sock, Data);
     {cast, Mod, Fun, Args} ->
       logger:debug("Casting ~p:~p(~p)~n", [Mod, Fun, Args]),
       {asset, Port, Token} = Asset,

data/elib/ernie_server_app.erl

@@ -7,6 +7,12 @@ boot() ->
   application:start(ernie_server_app).
 
 start(_Type, _Args) ->
+  case application:get_env(ernie_server_app, access_log) of
+    {ok, AccessFile} ->
+      ernie_access_logger_sup:start_link(AccessFile);
+    undefined ->
+      ernie_access_logger_sup:start_link(undefined)
+  end,
   logger_sup:start_link(),
   ernie_server_sup:start_link().
 

data/ernie.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{ernie}
-  s.version = "2.0.0"
+  s.version = "2.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Tom Preston-Werner"]
-  s.date = %q{2010-02-16}
+  s.date = %q{2010-02-24}
   s.default_executable = %q{ernie}
   s.email = %q{tom@mojombo.com}
   s.executables = ["ernie"]
@@ -33,6 +33,8 @@ Gem::Specification.new do |s|
      "elib/asset_pool_sup.erl",
      "elib/bert.erl",
      "elib/ernie.hrl",
+     "elib/ernie_access_logger.erl",
+     "elib/ernie_access_logger_sup.erl",
      "elib/ernie_admin.erl",
      "elib/ernie_config.erl",
      "elib/ernie_native.erl",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: ernie
 version: !ruby/object:Gem::Version 
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors: 
 - Tom Preston-Werner
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-02-16 00:00:00 -08:00
+date: 2010-02-24 00:00:00 -08:00
 default_executable: ernie
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -57,6 +57,8 @@ files:
 - elib/asset_pool_sup.erl
 - elib/bert.erl
 - elib/ernie.hrl
+- elib/ernie_access_logger.erl
+- elib/ernie_access_logger_sup.erl
 - elib/ernie_admin.erl
 - elib/ernie_config.erl
 - elib/ernie_native.erl