schleyfox-ernie 2.2.1

data/elib/asset_pool.erl

@@ -0,0 +1,155 @@
+-module(asset_pool).
+-behaviour(gen_server).
+
+%% api
+-export([start_link/2, lease/1, return/2, reload_assets/1, idle_worker_count/1]).
+
+%% gen_server callbacks
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
+         terminate/2, code_change/3]).
+
+-record(state, {assets = undefined,
+                handler = undefined,
+                token = undefined}).
+
+%%====================================================================
+%% API
+%%====================================================================
+
+start_link(Handler, Count) ->
+  gen_server:start_link(?MODULE, [Handler, Count], []).
+
+lease(Pid) ->
+  gen_server:call(Pid, lease).
+
+return(Pid, Asset) ->
+  gen_server:call(Pid, {return, Asset}).
+
+reload_assets(Pid) ->
+  gen_server:call(Pid, reload_assets).
+
+idle_worker_count(Pid) ->
+  gen_server:call(Pid, idle_worker_count).
+
+%%====================================================================
+%% gen_server callbacks
+%%====================================================================
+
+%%--------------------------------------------------------------------
+%% Function: init(Args) -> {ok, State} |
+%%                         {ok, State, Timeout} |
+%%                         ignore               |
+%%                         {stop, Reason}
+%% Description: Initiates the server
+%%--------------------------------------------------------------------
+init([Handler, Count]) ->
+  process_flag(trap_exit, true),
+  error_logger:info_msg("~p starting~n", [?MODULE]),
+  Token = make_ref(),
+  Assets = start_handlers(Count, Handler, Token),
+  logger:debug("Assets = ~p~n", [Assets]),
+  {ok, #state{assets = Assets, handler = Handler, token = Token}}.
+
+%%--------------------------------------------------------------------
+%% Function: %% handle_call(Request, From, State) -> {reply, Reply, State} |
+%%                                      {reply, Reply, State, Timeout} |
+%%                                      {noreply, State} |
+%%                                      {noreply, State, Timeout} |
+%%                                      {stop, Reason, Reply, State} |
+%%                                      {stop, Reason, State}
+%% Description: Handling call messages
+%%--------------------------------------------------------------------
+handle_call(lease, _From, State) ->
+  logger:debug("Leasing...~n", []),
+  Token = State#state.token,
+  case queue:out(State#state.assets) of
+    {{value, Asset}, Assets2} ->
+      {asset, Port, AssetToken} = Asset,
+      case AssetToken =:= Token of
+        false ->
+          port_wrapper:close(Port),
+          Handler = State#state.handler,
+          NewAsset = create_asset(Handler, Token);
+        true ->
+          NewAsset = Asset
+      end,
+      {reply, {ok, NewAsset}, State#state{assets = Assets2}};
+    {empty, _Assets2} ->
+      {reply, empty, State}
+  end;
+handle_call({return, Asset}, _From, State) ->
+  Token = State#state.token,
+  {asset, Port, AssetToken} = Asset,
+  case AssetToken =:= Token of
+    false ->
+      port_wrapper:close(Port),
+      Handler = State#state.handler,
+      NewAsset = create_asset(Handler, Token);
+    true ->
+      NewAsset = Asset
+  end,
+  Assets2 = queue:in(NewAsset, State#state.assets),
+  {reply, ok, State#state{assets = Assets2}};
+handle_call(reload_assets, _From, State) ->
+  Token = make_ref(),
+  {reply, ok, State#state{token = Token}};
+handle_call(idle_worker_count, _From, State) ->
+  WorkerCount = queue:len(State#state.assets),
+  {reply, WorkerCount, State};
+handle_call(_Request, _From, State) ->
+  {reply, ok, State}.
+
+%%--------------------------------------------------------------------
+%% Function: handle_cast(Msg, State) -> {noreply, State} |
+%%                                      {noreply, State, Timeout} |
+%%                                      {stop, Reason, State}
+%% Description: Handling cast messages
+%%--------------------------------------------------------------------
+handle_cast(_Msg, State) -> {noreply, State}.
+
+handle_info({'EXIT', _Pid, normal}, State) ->
+  {noreply, State};
+handle_info({'EXIT', Pid, Error}, State) ->
+  error_logger:error_msg("Port ~p closed with ~p, restarting port...~n", [Pid, Error]),
+  ValidAssets = queue:filter(fun(Item) -> {asset, A, _T} = Item, A =/= Pid end, State#state.assets),
+  Handler = State#state.handler,
+  Token = State#state.token,
+  NewAsset = create_asset(Handler, Token),
+  Assets = queue:in(NewAsset, ValidAssets),
+  {noreply, State#state{assets = Assets}};
+handle_info(Msg, State) ->
+  error_logger:error_msg("Unexpected message: ~p~n", [Msg]),
+  {noreply, State}.
+
+terminate(_Reason, _State) -> ok.
+code_change(_OldVersion, State, _Extra) -> {ok, State}.
+
+%%====================================================================
+%% Internal
+%%====================================================================
+
+start_handlers(Count, Handler, Token) ->
+  start_handlers(queue:new(), Count, Handler, Token).
+
+start_handlers(Assets, 0, _Handler, _Token) ->
+  Assets;
+start_handlers(Assets, Count, Handler, Token) ->
+  Asset = create_asset(Handler, Token),
+  Assets2 = queue:in(Asset, Assets),
+  start_handlers(Assets2, Count - 1, Handler, Token).
+
+create_asset(Handler, Token) ->
+  Len = length(Handler),
+  case Len > 150 of
+    true -> Cmd = Handler;
+    false -> Cmd = lists:flatten(Handler ++ " --procline " ++ pad(150 - Len - 12))
+  end,
+  io:format("~p~n", [Cmd]),
+  {asset, port_wrapper:wrap_link(Cmd), Token}.
+
+pad(Size) ->
+  pad(Size, []).
+pad(0, Acc) ->
+  Acc;
+pad(Size, Acc) ->
+  pad(Size - 1, ["x" | Acc]).

data/elib/asset_pool_sup.erl

@@ -0,0 +1,13 @@
+-module(asset_pool_sup).
+-behaviour(supervisor).
+-export([start_link/2, init/1]).
+
+start_link(Handler, Number) ->
+  supervisor:start_link(?MODULE, [Handler, Number]).
+
+init([Handler, Number]) ->
+  io:format("Using handler ~p~n", [Handler]),
+  io:format("Using ~p handler instances~n", [Number]),
+  {ok, {{one_for_one, 1, 60},
+    [{asset_pool, {asset_pool, start_link, [Handler, Number]},
+    permanent, brutal_kill, worker, [asset_pool]}]}}.

data/elib/bert.erl

@@ -0,0 +1,69 @@
+%%% See http://github.com/mojombo/bert.erl for documentation.
+%%% MIT License - Copyright (c) 2009 Tom Preston-Werner <tom@mojombo.com>
+
+-module(bert).
+-version('1.1.0').
+-author("Tom Preston-Werner").
+
+-export([encode/1, decode/1]).
+
+-ifdef(TEST).
+-include("test/bert_test.erl").
+-endif.
+
+%%---------------------------------------------------------------------------
+%% Public API
+
+-spec encode(term()) -> binary().
+
+encode(Term) ->
+  term_to_binary(encode_term(Term)).
+
+-spec decode(binary()) -> term().
+
+decode(Bin) ->
+  decode_term(binary_to_term(Bin)).
+
+%%---------------------------------------------------------------------------
+%% Encode
+
+-spec encode_term(term()) -> term().
+
+encode_term(Term) ->
+  case Term of
+    [] -> {bert, nil};
+    true -> {bert, true};
+    false -> {bert, false};
+    Dict when is_record(Term, dict, 8) ->
+      {bert, dict, dict:to_list(Dict)};
+    List when is_list(Term) ->
+      lists:map((fun encode_term/1), List);
+    Tuple when is_tuple(Term) ->
+      TList = tuple_to_list(Tuple),
+      TList2 = lists:map((fun encode_term/1), TList),
+      list_to_tuple(TList2);
+    _Else -> Term
+  end.
+
+%%---------------------------------------------------------------------------
+%% Decode
+
+-spec decode_term(term()) -> term().
+
+decode_term(Term) ->
+  case Term of
+    {bert, nil} -> [];
+    {bert, true} -> true;
+    {bert, false} -> false;
+    {bert, dict, Dict} ->
+      dict:from_list(Dict);
+    {bert, Other} ->
+      {bert, Other};
+    List when is_list(Term) ->
+      lists:map((fun decode_term/1), List);
+    Tuple when is_tuple(Term) ->
+      TList = tuple_to_list(Tuple),
+      TList2 = lists:map((fun decode_term/1), TList),
+      list_to_tuple(TList2);
+    _Else -> Term
+  end.

data/elib/ernie.hrl

@@ -0,0 +1,18 @@
+-record(state, {lsock = undefined,      % the listen socket
+                hq = queue:new(),       % high priority queue
+                lq = queue:new(),       % low priority queue
+                count = 0,              % total request count
+                map = undefined}).      % module map. tuples of {Mod, Id}
+
+-record(request, {sock = undefined,     % connection socket
+                  log = undefined,      % log information
+                  infos = [],           % list of info binaries
+                  action = undefined,   % action binary
+                  priority = high}).    % priority [ high | low ]
+
+-record(log, {taccept = erlang:now(),   % time that connection was accepted
+              tprocess = erlang:now(),  % time that processing started
+              tdone = erlang:now(),     % time that processing and response is done
+              hq = 0,                   % size of high queue at acceptance
+              lq = 0,                   % size of low queue at acceptance
+              type = unk}).             % type [ unk | nat | ext ]

data/elib/ernie_access_logger.erl

@@ -0,0 +1,170 @@
+-module(ernie_access_logger).
+-behaviour(gen_server).
+
+%% api
+-export([start_link/1, start/1, acc/1, err/3, reopen/0]).
+
+%% gen_server callbacks
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
+         terminate/2, code_change/3]).
+
+-include_lib("ernie.hrl").
+
+-record(lstate, {access_file_name = undefined,
+                 access_file = undefined}).
+
+%%====================================================================
+%% API
+%%====================================================================
+
+start_link(Args) ->
+  gen_server:start_link({global, ?MODULE}, ?MODULE, Args, []).
+
+start(Args) ->
+  gen_server:start({global, ?MODULE}, ?MODULE, Args, []).
+
+acc(Request) ->
+  gen_server:cast({global, ?MODULE}, {acc, Request}).
+
+err(Request, Msg, Args) ->
+  gen_server:cast({global, ?MODULE}, {err, Request, Msg, Args}).
+
+reopen() ->
+  gen_server:cast({global, ?MODULE}, reopen).
+
+%%====================================================================
+%% gen_server callbacks
+%%====================================================================
+
+%%--------------------------------------------------------------------
+%% Function: init(Args) -> {ok, State} |
+%%                         {ok, State, Timeout} |
+%%                         ignore               |
+%%                         {stop, Reason}
+%% Description: Initiates the server
+%%--------------------------------------------------------------------
+init([undefined]) ->
+  error_logger:info_msg("~p starting~n", [?MODULE]),
+  {ok, #lstate{}};
+init([AccessFileName]) ->
+  error_logger:info_msg("~p starting~n", [?MODULE]),
+  case file:open(AccessFileName, [append]) of
+    {ok, AccessFile} ->
+      {ok, _T} = timer:apply_interval(10000, ernie_access_logger, reopen, []),
+      {ok, #lstate{access_file_name = AccessFileName,
+                   access_file = AccessFile}};
+    {error, Error} ->
+      error_logger:error_msg("Error opening access log ~p: ~p.~n", [AccessFileName, Error]),
+      {ok, #lstate{}}
+  end.
+
+%%--------------------------------------------------------------------
+%% Function: %% handle_call(Request, From, State) -> {reply, Reply, State} |
+%%                                      {reply, Reply, State, Timeout} |
+%%                                      {noreply, State} |
+%%                                      {noreply, State, Timeout} |
+%%                                      {stop, Reason, Reply, State} |
+%%                                      {stop, Reason, State}
+%% Description: Handling call messages
+%%--------------------------------------------------------------------
+handle_call(_Request, _From, State) ->
+  {reply, ok, State}.
+
+%%--------------------------------------------------------------------
+%% Function: handle_cast(Msg, State) -> {noreply, State} |
+%%                                      {noreply, State, Timeout} |
+%%                                      {stop, Reason, State}
+%% Description: Handling cast messages
+%%--------------------------------------------------------------------
+handle_cast({acc, Request}, State) ->
+  case State#lstate.access_file_name of
+    undefined -> ok;
+    _AccessFilename -> acc(Request, State)
+  end,
+  {noreply, State};
+handle_cast({err, Request, Msg, Args}, State) ->
+  case State#lstate.access_file_name of
+    undefined -> ok;
+    _AccessFilename -> err(Request, Msg, Args, State)
+  end,
+  {noreply, State};
+handle_cast(reopen, State) ->
+  case State#lstate.access_file_name of
+    undefined ->
+      {noreply, State};
+    AccessFileName ->
+      case file:read_file_info(AccessFileName) of
+        {ok, _FileInfo} ->
+          {noreply, State};
+        {error, enoent} ->
+          ok = file:close(State#lstate.access_file),
+          {ok, AccessFile} = file:open(AccessFileName, [append]),
+          {noreply, State#lstate{access_file = AccessFile}};
+        _OtherError ->
+          {noreply, #lstate{}}
+      end
+  end;
+handle_cast(_Msg, State) ->
+  {noreply, State}.
+
+handle_info(Msg, State) ->
+  error_logger:error_msg("Unexpected message: ~p~n", [Msg]),
+  {noreply, State}.
+
+terminate(_Reason, _State) -> ok.
+code_change(_OldVersion, State, _Extra) -> {ok, State}.
+
+%%====================================================================
+%% Internal
+%%====================================================================
+
+acc(Request, State) ->
+  StatString = stat_string(Request),
+  ActionString = action_string(Request),
+  Line = io_lib:fwrite("ACC ~s - ~s~n", [StatString, ActionString]),
+  file:write(State#lstate.access_file, Line).
+
+err(Request, Msg, Args, State) ->
+  StatString = stat_string(Request),
+  ActionString = action_string(Request),
+  ErrString = io_lib:fwrite(Msg, Args),
+  Line = io_lib:fwrite("ERR ~s - ~s : ~s~n", [StatString, ErrString, ActionString]),
+  file:write(State#lstate.access_file, Line).
+
+stat_string(Request) ->
+  Log = Request#request.log,
+  TAccept = time_tuple_to_iso_8601_date(Log#log.taccept),
+  D1 = time_difference_in_seconds(Log#log.taccept, Log#log.tprocess),
+  D2 = time_difference_in_seconds(Log#log.tprocess, Log#log.tdone),
+  Type = Log#log.type,
+  HQ = Log#log.hq,
+  LQ = Log#log.lq,
+  Prio = Request#request.priority,
+  Args = [TAccept, D1, D2, HQ, LQ, Type, Prio],
+  io_lib:fwrite("[~s] ~f ~f - ~B ~B ~3s ~p", Args).
+
+action_string(Request) ->
+  TermAction = binary_to_term(Request#request.action),
+  RawAction = lists:flatten(io_lib:fwrite("~1000000000.0.0p", [TermAction])),
+  case string:len(RawAction) > 150 of
+    true ->
+      Action = re:replace(RawAction, "\n", "", [global, {return, list}]),
+      [string:sub_string(Action, 1, 150), "..."];
+    false ->
+      RawAction
+  end.
+
+time_tuple_to_iso_8601_date(TimeTuple) ->
+  {{YY, MM, DD}, {H, M, S}} = calendar:now_to_local_time(TimeTuple),
+  {_MegaSecs, _Secs, MicroSecs} = TimeTuple,
+  Args = [YY, MM, DD, H, M, S, MicroSecs],
+  io_lib:fwrite("~4B-~2.10.0B-~2.10.0BT~2.10.0B:~2.10.0B:~2.10.0B.~-6.10.0B", Args).
+
+time_difference_in_seconds(T1, T2) ->
+  {_, _, MS1} = T1,
+  {_, _, MS2} = T2,
+  S1 = calendar:datetime_to_gregorian_seconds(calendar:now_to_local_time(T1)),
+  S2 = calendar:datetime_to_gregorian_seconds(calendar:now_to_local_time(T2)),
+  F1 = S1 + (MS1 / 1000000),
+  F2 = S2 + (MS2 / 1000000),
+  F2 - F1.

data/elib/ernie_access_logger_sup.erl

@@ -0,0 +1,15 @@
+-module(ernie_access_logger_sup).
+-behaviour(supervisor).
+-export([start_link/1, init/1]).
+
+start_link(AccessLog) ->
+  supervisor:start_link({local, ?MODULE}, ?MODULE, [AccessLog]).
+
+init([AccessLog]) ->
+  case AccessLog of
+    undefined -> io:format("No access log~n", []);
+    Any -> io:format("Using access log ~p~n", [Any])
+  end,
+  {ok, {{one_for_one, 1, 60},
+    [{ernie_access_logger, {ernie_access_logger, start_link, [[AccessLog]]},
+    permanent, brutal_kill, worker, [ernie_access_logger]}]}}.

data/elib/ernie_admin.erl

@@ -0,0 +1,60 @@
+-module(ernie_admin).
+-export([process/4]).
+-include_lib("ernie.hrl").
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% Process entry point
+
+process(Sock, reload_handlers, _Args, State) ->
+  spawn(fun() -> process_reload_assets(Sock, State) end),
+  State;
+process(Sock, stats, _Args, State) ->
+  spawn(fun() -> process_stats(Sock, State) end),
+  State;
+process(Sock, _Fun, _Args, State) ->
+  gen_tcp:send(Sock, term_to_binary({reply, <<"Admin function not supported.">>})),
+  ok = gen_tcp:close(Sock),
+  State.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% Reload handlers
+
+process_reload_assets(Sock, State) ->
+  lists:map((fun reload/1), State#state.map),
+  gen_tcp:send(Sock, term_to_binary({reply, <<"Handlers reloaded.">>})),
+  ok = gen_tcp:close(Sock).
+
+reload({_Mod, native}) ->
+  ok;
+reload({_Mod, Pid}) ->
+  asset_pool:reload_assets(Pid).
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% Stats
+
+process_stats(Sock, State) ->
+  CountString = stat(count, State),
+  IdleWorkersString = stat(idle, State),
+  QueueLengthString = stat(queue, State),
+  StatString = list_to_binary([CountString, IdleWorkersString, QueueLengthString]),
+  Data = term_to_binary({reply, StatString}),
+  gen_tcp:send(Sock, Data),
+  ok = gen_tcp:close(Sock).
+
+stat(count, State) ->
+  Count = State#state.count,
+  list_to_binary([<<"connections.total=">>, integer_to_list(Count), <<"\n">>]);
+stat(idle, State) ->
+  IdleMap = lists:map((fun idle/1), State#state.map),
+  list_to_binary(IdleMap);
+stat(queue, State) ->
+  HighQueueLength = queue:len(State#state.hq),
+  LowQueueLength = queue:len(State#state.lq),
+  list_to_binary([<<"queue.high=">>, integer_to_list(HighQueueLength), <<"\n">>,
+                  <<"queue.low=">>, integer_to_list(LowQueueLength), <<"\n">>]).
+
+idle({Mod, native}) ->
+  list_to_binary([<<"workers.idle.">>, atom_to_list(Mod), <<"=native\n">>]);
+idle({Mod, Pid}) ->
+  IdleCount = integer_to_list(asset_pool:idle_worker_count(Pid)),
+  list_to_binary([<<"workers.idle.">>, atom_to_list(Mod), <<"=">>, IdleCount, <<"\n">>]).

data/elib/ernie_config.erl

@@ -0,0 +1,30 @@
+-module(ernie_config).
+-export([load/1]).
+
+load(ConfigFile) ->
+  {ok, Configs} = file:consult(ConfigFile),
+  Configs2 = lists:map((fun load_single/1), Configs),
+  {ok, Configs2}.
+
+load_single(Config) ->
+  case proplists:get_value(type, Config) of
+    native ->
+      verify(native, Config),
+      CodePaths = proplists:get_value(codepaths, Config),
+      lists:map((fun code:add_patha/1), CodePaths),
+      Mod = proplists:get_value(module, Config),
+      code:load_file(Mod),
+      [{id, native} | Config];
+    external ->
+      verify(external, Config),
+      Handler = proplists:get_value(command, Config),
+      Number = proplists:get_value(count, Config),
+      {ok, SupPid} = asset_pool_sup:start_link(Handler, Number),
+      [{_Id, ChildPid, _Type, _Modules}] = supervisor:which_children(SupPid),
+      [{id, ChildPid} | Config]
+  end.
+
+verify(native, _Config) ->
+  ok;
+verify(external, _Config) ->
+  ok.

data/elib/ernie_native.erl

@@ -0,0 +1,26 @@
+-module(ernie_native).
+-export([process/2]).
+-include_lib("ernie.hrl").
+
+process(ActionTerm, Request) ->
+  {_Type, Mod, Fun, Args} = ActionTerm,
+  Sock = Request#request.sock,
+  logger:debug("Calling ~p:~p(~p)~n", [Mod, Fun, Args]),
+  try apply(Mod, Fun, Args) of
+    Result ->
+      logger:debug("Result was ~p~n", [Result]),
+      Data = bert:encode({reply, Result}),
+      gen_tcp:send(Sock, Data)
+  catch
+    error:Error ->
+      BError = list_to_binary(io_lib:format("~p", [Error])),
+      Trace = erlang:get_stacktrace(),
+      BTrace = lists:map(fun(X) -> list_to_binary(io_lib:format("~p", [X])) end, Trace),
+      Data = term_to_binary({error, [user, 0, <<"RuntimeError">>, BError, BTrace]}),
+      gen_tcp:send(Sock, Data)
+  end,
+  ok = gen_tcp:close(Sock),
+  Log = Request#request.log,
+  Log2 = Log#log{tdone = erlang:now()},
+  Request2 = Request#request{log = Log2},
+  ernie_access_logger:acc(Request2).