seomoz-riak-client 1.0.0.pre

data/Gemfile

@@ -0,0 +1,27 @@
+source :rubygems
+
+gemspec
+gem 'bundler'
+gem 'guard-rspec'
+gem 'rb-fsevent'
+gem 'growl'
+
+platforms :mri do
+  gem 'yajl-ruby'
+end
+
+platforms :jruby do
+  gem 'jruby-openssl'
+end
+
+group :integration do
+  gem 'activesupport', '~>3.0'
+end
+
+platforms :mri_18, :jruby do
+  gem 'ruby-debug'
+end
+
+platforms :mri_19 do
+  gem 'ruby-debug19'
+end

data/Guardfile

@@ -0,0 +1,14 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+gemset = ENV['RVM_GEMSET'] || 'ripple'
+gemset = "@#{gemset}" unless gemset.to_s == ''
+
+rvms = %w[ 1.8.7 1.9.2 jruby ].map do |version|
+  "#{version}@#{gemset}"
+end
+
+guard 'rspec', :cli => '--tag ~integration', :rvm => rvms do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec/riak" }
+end

data/Rakefile

@@ -0,0 +1,76 @@
+require 'rubygems'
+require 'rake/gempackagetask'
+
+gemspec = Gem::Specification.new do |gem|
+  gem.name = "seomoz-riak-client"
+  gem.summary = %Q{riak-client is a rich client for Riak, the distributed database by Basho.}
+  gem.description = %Q{riak-client is a rich client for Riak, the distributed database by Basho. It supports the full HTTP interface including storage operations, bucket configuration, link-walking and map-reduce.}
+  gem.version = File.read('../VERSION').strip
+  gem.email = "sean@basho.com"
+  gem.homepage = "http://seancribbs.github.com/ripple"
+  gem.authors = ["Sean Cribbs"]
+  gem.add_development_dependency "rspec", "~>2.6.0"
+  gem.add_development_dependency "fakeweb", ">=1.2"
+  gem.add_development_dependency "rack", ">=1.0"
+  gem.add_development_dependency "excon", "~>0.6.1"
+  gem.add_development_dependency 'rake', '~> 0.8.7'
+  gem.add_dependency "i18n", ">=0.4.0"
+  gem.add_dependency "builder", ">= 2.1.2"
+  gem.add_dependency "beefcake", "0.3.2"
+  gem.add_dependency "multi_json", "~>1.0.0"
+
+  files = FileList["**/*"]
+  # Editor and O/S files
+  files.exclude ".DS_Store", "*~", "\#*", ".\#*", "*.swp", "*.tmproj", "tmtags"
+  # Generated artifacts
+  files.exclude "coverage", "rdoc", "pkg", "doc", ".bundle", "*.rbc", ".rvmrc", ".watchr", ".rspec"
+  # Project-specific
+  files.exclude "Gemfile.lock", %r{spec/support/test_server.yml$}, "bin"
+  # Remove directories
+  files.exclude {|d| File.directory?(d) }
+
+  gem.files = files.to_a
+
+  gem.test_files = gem.files.grep(/_spec\.rb$/)
+end
+
+# Gem packaging tasks
+Rake::GemPackageTask.new(gemspec) do |pkg|
+  pkg.need_zip = false
+  pkg.need_tar = false
+end
+
+task :gem => :gemspec
+
+desc %{Build the gemspec file.}
+task :gemspec do
+  gemspec.validate
+  File.open("#{gemspec.name}.gemspec", 'w'){|f| f.write gemspec.to_ruby }
+end
+
+desc %{Release the gem to RubyGems.org}
+task :release => :gem do
+  system "gem push pkg/#{gemspec.name}-#{gemspec.version}.gem"
+end
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+
+desc "Run Unit Specs Only"
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = "spec/riak/**/*_spec.rb"
+  spec.rspec_opts = %w[--profile]
+end
+
+namespace :spec do
+  desc "Run Integration Specs Only"
+  RSpec::Core::RakeTask.new(:integration) do |spec|
+    spec.pattern = "spec/integration/**/*_spec.rb"
+    spec.rspec_opts = %w[--profile]
+  end
+
+  desc "Run All Specs"
+  task :all => %w[ spec spec:integration ]
+end
+
+task :default => "spec:all"

data/erl_src/riak_kv_test_backend.erl

@@ -0,0 +1,174 @@
+%% -------------------------------------------------------------------
+%%
+%% riak_kv_test_backend: storage engine based on ETS tables
+%%
+%%
+%% -------------------------------------------------------------------
+
+% @doc riak_kv_test_backend is a Riak storage backend using ets that
+%      exposes a reset function for efficiently clearing stored data.
+
+-module(riak_kv_test_backend).
+-behavior(riak_kv_backend).
+-behavior(gen_server).
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+-endif.
+-export([start/2,stop/1,get/2,put/3,list/1,list_bucket/2,delete/2,
+         is_empty/1, drop/1, fold/3, callback/3, reset/0]).
+
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
+         terminate/2, code_change/3]).
+
+
+% @type state() = term().
+-record(state, {t, p}).
+
+% @spec start(Partition :: integer(), Config :: proplist()) ->
+%                        {ok, state()} | {{error, Reason :: term()}, state()}
+start(Partition, _Config) ->
+    gen_server:start_link(?MODULE, [Partition], []).
+
+% @spec reset() -> ok | {error, timeout}
+reset() ->
+    Pids = lists:foldl(fun(Item, Acc) ->
+                               case lists:prefix("test_backend", atom_to_list(Item)) of
+                                   true -> [whereis(Item)|Acc];
+                                   _ -> Acc
+                               end
+                       end, [], registered()),
+    [gen_server:cast(Pid,{reset, self()})|| Pid <- Pids],
+    receive_reset(Pids).
+
+receive_reset([]) -> ok;
+receive_reset(Pids) ->
+    receive
+        {reset, Pid} ->
+            receive_reset(lists:delete(Pid, Pids))
+    after 1000 ->
+            {error, timeout}
+    end.
+
+%% @private
+init([Partition]) ->
+    PName = list_to_atom("test_backend" ++ integer_to_list(Partition)),
+    P = list_to_atom(integer_to_list(Partition)),
+    register(PName, self()),
+    {ok, #state{t=ets:new(P,[]), p=P}}.
+
+%% @private
+handle_cast({reset,From}, State) ->
+    ets:delete_all_objects(State#state.t),
+    From ! {reset, self()},
+    {noreply, State};
+handle_cast(_, State) -> {noreply, State}.
+
+%% @private
+handle_call(stop,_From,State) -> {reply, srv_stop(State), State};
+handle_call({get,BKey},_From,State) -> {reply, srv_get(State,BKey), State};
+handle_call({put,BKey,Val},_From,State) ->
+    {reply, srv_put(State,BKey,Val),State};
+handle_call({delete,BKey},_From,State) -> {reply, srv_delete(State,BKey),State};
+handle_call(list,_From,State) -> {reply, srv_list(State), State};
+handle_call({list_bucket,Bucket},_From,State) ->
+    {reply, srv_list_bucket(State, Bucket), State};
+handle_call(is_empty, _From, State) ->
+    {reply, ets:info(State#state.t, size) =:= 0, State};
+handle_call(drop, _From, State) ->
+    ets:delete(State#state.t),
+    {reply, ok, State};
+handle_call({fold, Fun0, Acc}, _From, State) ->
+    Fun = fun({{B,K}, V}, AccIn) -> Fun0({B,K}, V, AccIn) end,
+    Reply = ets:foldl(Fun, Acc, State#state.t),
+    {reply, Reply, State}.
+
+% @spec stop(state()) -> ok | {error, Reason :: term()}
+stop(SrvRef) -> gen_server:call(SrvRef,stop).
+srv_stop(State) ->
+    true = ets:delete(State#state.t),
+    ok.
+
+% get(state(), riak_object:bkey()) ->
+%   {ok, Val :: binary()} | {error, Reason :: term()}
+% key must be 160b
+get(SrvRef, BKey) -> gen_server:call(SrvRef,{get,BKey}).
+srv_get(State, BKey) ->
+    case ets:lookup(State#state.t,BKey) of
+        [] -> {error, notfound};
+        [{BKey,Val}] -> {ok, Val};
+        Err -> {error, Err}
+    end.
+
+% put(state(), riak_object:bkey(), Val :: binary()) ->
+%   ok | {error, Reason :: term()}
+% key must be 160b
+put(SrvRef, BKey, Val) -> gen_server:call(SrvRef,{put,BKey,Val}).
+srv_put(State,BKey,Val) ->
+    true = ets:insert(State#state.t, {BKey,Val}),
+    ok.
+
+% delete(state(), riak_object:bkey()) ->
+%   ok | {error, Reason :: term()}
+% key must be 160b
+delete(SrvRef, BKey) -> gen_server:call(SrvRef,{delete,BKey}).
+srv_delete(State, BKey) ->
+    true = ets:delete(State#state.t, BKey),
+    ok.
+
+% list(state()) -> [riak_object:bkey()]
+list(SrvRef) -> gen_server:call(SrvRef,list).
+srv_list(State) ->
+    MList = ets:match(State#state.t,{'$1','_'}),
+    list(MList,[]).
+list([],Acc) -> Acc;
+list([[K]|Rest],Acc) -> list(Rest,[K|Acc]).
+
+% list_bucket(term(), Bucket :: riak_object:bucket()) -> [Key :: binary()]
+list_bucket(SrvRef, Bucket) ->
+    gen_server:call(SrvRef,{list_bucket, Bucket}).
+srv_list_bucket(State, {filter, Bucket, Fun}) ->
+    MList = lists:filter(Fun, ets:match(State#state.t,{{Bucket,'$1'},'_'})),
+    list(MList,[]);
+srv_list_bucket(State, Bucket) ->
+    case Bucket of
+        '_' -> MatchSpec = {{'$1','_'},'_'};
+        _ -> MatchSpec = {{Bucket,'$1'},'_'}
+    end,
+    MList = ets:match(State#state.t,MatchSpec),
+    list(MList,[]).
+
+is_empty(SrvRef) -> gen_server:call(SrvRef, is_empty).
+
+drop(SrvRef) -> gen_server:call(SrvRef, drop).
+
+fold(SrvRef, Fun, Acc0) -> gen_server:call(SrvRef, {fold, Fun, Acc0}, infinity).
+
+%% Ignore callbacks for other backends so multi backend works
+callback(_State, _Ref, _Msg) ->
+    ok.
+
+%% @private
+handle_info(_Msg, State) -> {noreply, State}.
+
+%% @private
+terminate(_Reason, _State) -> ok.
+
+%% @private
+code_change(_OldVsn, State, _Extra) -> {ok, State}.
+
+%%
+%% Test
+%%
+-ifdef(TEST).
+
+% @private
+simple_test() ->
+    riak_kv_backend:standard_test(?MODULE, []).
+
+-ifdef(EQC).
+%% @private
+eqc_test() ->
+    ?assertEqual(true, backend_eqc:test(?MODULE, true)).
+
+-endif. % EQC
+-endif. % TEST

data/erl_src/riak_search_test_backend.erl

@@ -0,0 +1,175 @@
+%% -------------------------------------------------------------------
+%%
+%% Copyright (c) 2007-2010 Basho Technologies, Inc.  All Rights Reserved.
+%%
+%% -------------------------------------------------------------------
+
+-module(riak_search_test_backend).
+-behavior(riak_search_backend).
+
+-export([
+         reset/0,
+         start/2,
+         stop/1,
+         index/2,
+         delete/2,
+         stream/6,
+         range/8,
+         info/5,
+         fold/3,
+         is_empty/1,
+         drop/1
+        ]).
+-export([
+         stream_results/3
+        ]).
+
+-include("riak_search.hrl").
+
+-record(state, {partition, table}).
+
+reset() ->
+    {ok, Ring} = riak_core_ring_manager:get_my_ring(),
+    [ ets:delete_all_objects(list_to_atom(integer_to_list(P))) ||
+        P <- riak_core_ring:my_indices(Ring) ],
+    riak_search_config:clear(),
+    ok.
+
+start(Partition, _Config) ->
+    Table = ets:new(list_to_atom(integer_to_list(Partition)),
+                    [named_table, public, ordered_set]),
+    {ok, #state{partition=Partition, table=Table}}.
+
+stop(State) ->
+    maybe_delete(State).
+
+index(IFTVPKList, #state{table=Table}=State) ->
+    lists:foreach(
+      fun({I, F, T, V, P, K}) ->
+              Key = {b(I), b(F), b(T), b(V)},
+              case ets:lookup(Table, Key) of
+                  [{_, _, ExistingKeyClock}] ->
+                      if ExistingKeyClock > K ->
+                              %% stored data is newer
+                              ok;
+                         true ->
+                              %% stored data is older
+                              ets:update_element(Table, Key,
+                                                 [{2, P},{3, K}])
+                      end;
+                  [] ->
+                      ets:insert(Table, {Key, P, K})
+              end
+      end,
+      IFTVPKList),
+    {reply, {indexed, node()}, State}.
+
+delete(IFTVKList, State) ->
+    Table = State#state.table,
+    lists:foreach(fun(IFTVK) -> delete_fun(IFTVK, Table) end, IFTVKList),
+    {reply, {deleted, node()}, State}.
+
+delete_fun({I, F, T, V, K}, Table) ->
+    Key = {b(I), b(F), b(T), b(V)},
+    case ets:lookup(Table, Key) of
+        [{Key, _Props, ExistingKeyClock}] ->
+            if ExistingKeyClock > K ->
+                    %% stored data is newer
+                    ok;
+               true ->
+                    %% stored data is older
+                    ets:delete(Table, Key)
+            end;
+        [] ->
+            ok
+    end;
+delete_fun({I, F, T, V, _P, K}, Table) ->
+    %% copied idea from merge_index_backend
+    %% other operations include Props, though delete shouldn't
+    delete_fun({I, F, T, V, K}, Table).
+
+info(Index, Field, Term, Sender, State) ->
+    Count = ets:select_count(State#state.table,
+                             [{{{b(Index), b(Field), b(Term), '_'},
+                                '_', '_'},
+                               [],[true]}]),
+    riak_search_backend:info_response(Sender, [{Term, node(), Count}]),
+    noreply.
+
+-define(STREAM_SIZE, 100).
+
+range(Index, Field, StartTerm, EndTerm, _Size, FilterFun, Sender, State) ->
+    ST = b(StartTerm),
+    ET = b(EndTerm),
+    spawn(riak_search_ets_backend, stream_results,
+          [Sender,
+           FilterFun,
+           ets:select(State#state.table,
+                      [{{{b(Index), b(Field), '$1', '$2'}, '$3', '_'},
+                        [{'>=', '$1', ST}, {'=<', '$1', ET}],
+                        [{{'$2', '$3'}}]}],
+                      ?STREAM_SIZE)]),
+    noreply.
+
+stream(Index, Field, Term, FilterFun, Sender, State) ->
+    spawn(riak_search_ets_backend, stream_results,
+          [Sender,
+           FilterFun,
+           ets:select(State#state.table,
+                      [{{{b(Index), b(Field), b(Term), '$1'}, '$2', '_'},
+                        [], [{{'$1', '$2'}}]}],
+                      ?STREAM_SIZE)]),
+    noreply.
+
+stream_results(Sender, FilterFun, {Results0, Continuation}) ->
+    case lists:filter(fun({V,P}) -> FilterFun(V, P) end, Results0) of
+        [] ->
+            ok;
+        Results ->
+            riak_search_backend:response_results(Sender, Results)
+    end,
+    stream_results(Sender, FilterFun, ets:select(Continuation));
+stream_results(Sender, _, '$end_of_table') ->
+    riak_search_backend:response_done(Sender).
+
+fold(FoldFun, Acc, State) ->
+    Fun = fun({{I,F,T,V},P,K}, {OuterAcc, {{I,{F,T}},InnerAcc}}) ->
+                  %% same IFT, just accumulate doc/props/clock
+                  {OuterAcc, {{I,{F,T}},[{V,P,K}|InnerAcc]}};
+             ({{I,F,T,V},P,K}, {OuterAcc, {FoldKey, VPKList}}) ->
+                  %% finished a string of IFT, send it off
+                  %% (sorted order is assumed)
+                  NewOuterAcc = FoldFun(FoldKey, VPKList, OuterAcc),
+                  {NewOuterAcc, {{I,{F,T}},[{V,P,K}]}};
+             ({{I,F,T,V},P,K}, {OuterAcc, undefined}) ->
+                  %% first round through the fold - just start building
+                  {OuterAcc, {{I,{F,T}},[{V,P,K}]}}
+          end,
+    {OuterAcc0, Final} = ets:foldl(Fun, {Acc, undefined}, State#state.table),
+    OuterAcc = case Final of
+                   {FoldKey, VPKList} ->
+                       %% one last IFT to send off
+                       FoldFun(FoldKey, VPKList, OuterAcc0);
+                   undefined ->
+                       %% this partition was empty
+                       OuterAcc0
+               end,
+    {reply, OuterAcc, State}.
+
+is_empty(State) ->
+    0 == ets:info(State#state.table, size).
+
+drop(State) ->
+    maybe_delete(State).
+
+maybe_delete(State) ->
+    case lists:member(State#state.table, ets:all()) of
+        true ->
+            ets:delete(State#state.table),
+            ok;
+        false ->
+            ok
+    end.
+
+b(Binary) when is_binary(Binary) -> Binary;
+b(List) when is_list(List) -> iolist_to_binary(List).