em-riak 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +4 -0
- data/LICENSE.md +1 -0
- data/README.md +67 -0
- data/Rakefile +0 -0
- data/em-riak.gemspec +47 -0
- data/lib/em-riak/basic.rb +325 -0
- data/lib/em-riak/configurations.rb +8 -0
- data/lib/em-riak/eventmachine.rb +0 -0
- data/lib/em-riak/full_text_search.rb +0 -0
- data/lib/em-riak/grapher.rb +58 -0
- data/lib/em-riak/luwak.rb +0 -0
- data/lib/em-riak/map_reduce.rb +39 -0
- data/lib/em-riak/secondary_index.rb +68 -0
- data/lib/em-riak/utils.rb +22 -0
- data/lib/em-riak/version.rb +3 -0
- data/lib/em-riak.rb +5 -0
- metadata +116 -0
data/.gitignore
ADDED
data/LICENSE.md
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
MIT
|
data/README.md
ADDED
@@ -0,0 +1,67 @@
|
|
1
|
+
###What is this?
|
2
|
+
It is a Riak ruby client design for eventmachine.
|
3
|
+
|
4
|
+
### Concept
|
5
|
+
I tried to implement em-riak with goals : Easy to use, ORM, Fast, Flexible, Hybrid.
|
6
|
+
|
7
|
+
###### Easy To use
|
8
|
+
I tried to use the same behaviors as active-record or sequel.
|
9
|
+
So when you use em-riak, you don't tackling the "Key Design".
|
10
|
+
Instead, just add an extension into your model, and everygthings goes fine.
|
11
|
+
|
12
|
+
###### ORM (ToDo)
|
13
|
+
The same idea as upon, just add the plugins and you gain the ORM ability with NoSQL.
|
14
|
+
It only support Sequel gem currently.
|
15
|
+
|
16
|
+
###### Flexible
|
17
|
+
Concurrency is one reasons to use nosql, we may face huge read / write in the same time but our sql hard to scale.
|
18
|
+
Riak provide easy to use script that you can install, add nodes easily.
|
19
|
+
Also, you can read/write data directly, or deferrable-aware.
|
20
|
+
With Deferrable-aware, you don't block & wait for your I/O.
|
21
|
+
|
22
|
+
|
23
|
+
<br/>
|
24
|
+
### Getting Start
|
25
|
+
EmRiak::Connection.new({:bucket=>"member",:async=>false,:hosts=>["http://127.0.0.1:8091"]})
|
26
|
+
member=EmRiak.create("member1",{:name=>"Von",:age=>18})
|
27
|
+
|
28
|
+
member.handsome=true # or EmRaik.udpate("member1",{:handsome=>true})
|
29
|
+
member.save # or EmRiak.save("member1")
|
30
|
+
member.destory
|
31
|
+
|
32
|
+
member_two=EmRiak.create("member2",{:name=>"Jack",:age=>22})
|
33
|
+
member_two.add_tag(:interests,"programmer","backpacker","trader")
|
34
|
+
|
35
|
+
member_three=EmRiak.create("member3",{:name=>"Lisa",:age=>17})
|
36
|
+
member_three.add_tag(:interests,"backpacker","delicious","cocktail","movie")
|
37
|
+
member_three.remove_tag(:interest,"delicious")
|
38
|
+
|
39
|
+
EmRiak.search(:interests, "delicious") #=> {}
|
40
|
+
EmRiak.search(:interests, "movie") #=> {:member=>["member3"]}
|
41
|
+
EmRiak.search(:secondary_index, :interests, "backpacker") #=> same above
|
42
|
+
EmRiak.search(:interests, "backpacker","movie") #=> {:member=>["member2","member3"]}
|
43
|
+
|
44
|
+
EmRiak::Util.clean_bucket("member") #=> Clean all the data inside bucket
|
45
|
+
|
46
|
+
<br/>
|
47
|
+
How to work with **Asynchornize** ?
|
48
|
+
With asynchornize, you just set following at anywhere :
|
49
|
+
|
50
|
+
EmRiak.async=true
|
51
|
+
|
52
|
+
And the code behind will become asynchornize.
|
53
|
+
Because callback still under development, so get object, and search will still reamin direct http.
|
54
|
+
|
55
|
+
|
56
|
+
### Write some map-reduce
|
57
|
+
You can call **EmRiak::MapReduce.submit()** directly.
|
58
|
+
Since I just started to work on it, better interface & docs will coming soon…
|
59
|
+
|
60
|
+
|
61
|
+
### TO-DO
|
62
|
+
Async with callback block support
|
63
|
+
Map/Reduce
|
64
|
+
Full Text Seacrh
|
65
|
+
Deployment & Management
|
66
|
+
Protocal Buffer Interface
|
67
|
+
ORM with Model
|
data/Rakefile
ADDED
File without changes
|
data/em-riak.gemspec
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = 'em-riak'
|
3
|
+
s.version = '0.2.0'
|
4
|
+
s.date = '2012-12-01'
|
5
|
+
s.summary = "Riak client for eventmachine"
|
6
|
+
s.description = "An extremely fast and convenient riak client for eventmachine."
|
7
|
+
s.authors = ["Von"]
|
8
|
+
s.email = 'von@vonstark.co'
|
9
|
+
s.files = ["lib/em-riak.rb"]
|
10
|
+
s.homepage = 'http://tech.vonstark.co'
|
11
|
+
|
12
|
+
includes = %W{
|
13
|
+
lib/**/*
|
14
|
+
spec/**/*
|
15
|
+
Gemfile
|
16
|
+
Rakefile
|
17
|
+
LICENSE*
|
18
|
+
RELEASE_NOTES*
|
19
|
+
README*
|
20
|
+
.gitignore
|
21
|
+
.document
|
22
|
+
.rspec
|
23
|
+
em-riak.gemspec
|
24
|
+
}
|
25
|
+
|
26
|
+
excludes = %W{
|
27
|
+
**/*.swp
|
28
|
+
**/#*
|
29
|
+
**/.#*
|
30
|
+
**/*~
|
31
|
+
**/*.rbc
|
32
|
+
**/.DS_Store
|
33
|
+
spec/support/test_server.yml
|
34
|
+
}
|
35
|
+
|
36
|
+
s.add_runtime_dependency "escape_utils"
|
37
|
+
s.add_runtime_dependency "yajl-ruby", "~>1.1.0"
|
38
|
+
s.add_runtime_dependency "httpi", "~>1.1.1"
|
39
|
+
s.add_runtime_dependency "eventmachine", "~>1.0.0.beta.4"
|
40
|
+
s.add_runtime_dependency "em-http-request", "~>1.0.2"
|
41
|
+
|
42
|
+
files = includes.map {|glob| Dir[glob] }.flatten.select {|f| File.file?(f) }.sort
|
43
|
+
files.reject! {|f| excludes.any? {|e| File.fnmatch?(e, f) } }
|
44
|
+
|
45
|
+
s.files = files
|
46
|
+
s.require_paths = ['lib']
|
47
|
+
end
|
@@ -0,0 +1,325 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
require 'eventmachine'
|
3
|
+
require 'em-http-request'
|
4
|
+
require 'escape_utils'
|
5
|
+
require 'escape_utils/html/rack'
|
6
|
+
require 'httpi'
|
7
|
+
require 'yajl'
|
8
|
+
|
9
|
+
module EmRiak
|
10
|
+
attr_accessor :bucket, :interface, :adapter, :hosts, :cluster, :async, :http_client, :vector_lock, :replication, :debug
|
11
|
+
extend self
|
12
|
+
|
13
|
+
# A easy interface to create connections
|
14
|
+
class Connection
|
15
|
+
def initialize(*opts)
|
16
|
+
if opts && opts.length>0
|
17
|
+
opts=opts.first
|
18
|
+
methods=[:bucket, :interface, :adapter, :hosts, :cluster, :async, :http_client, :vector_lock, :repliaction, :debug]
|
19
|
+
opts.each{|key,opt| EmRiak.send "#{key}=".to_sym, opt if opt && methods.index(key.to_sym) }
|
20
|
+
else
|
21
|
+
EmRiak.bucket=RIAKBUCKET
|
22
|
+
end
|
23
|
+
EmRiak.debug=false if EmRiak.debug.nil?
|
24
|
+
end
|
25
|
+
def find; EmRiak.find(self); end
|
26
|
+
end
|
27
|
+
|
28
|
+
# -- CRUD
|
29
|
+
class StorageObject < Hash
|
30
|
+
def find; EmRiak.find(self); end
|
31
|
+
def save; EmRiak.save(self); end
|
32
|
+
def destroy; EmRiak.destroy(self); end
|
33
|
+
def add_tag(*args); EmRiak::SecondaryIndex.add_tag(self,args); end
|
34
|
+
def remove_tag(*args); EmRiak::SecondaryIndex.remove_tag(self,args); end
|
35
|
+
def save_as_key(); EmRiak::Grapher.save_object_as_key_name(self) ; end
|
36
|
+
def method_missing(name,*args)
|
37
|
+
return self[name] if key? name
|
38
|
+
if name.to_s.index("=")
|
39
|
+
sym_name=name.to_s.gsub("=","").to_sym
|
40
|
+
self[sym_name]=args.first
|
41
|
+
else
|
42
|
+
self.each { |k,v| return v if k.to_s.to_sym == name }
|
43
|
+
super.method_missing name
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
def create(key,*opts)
|
48
|
+
data={:body=>{}}
|
49
|
+
data[:head], data[:query] = handle_options("save",opts)
|
50
|
+
opts.first.each{|key,value| data[:body][key]=value} if opts && opts.count>0
|
51
|
+
|
52
|
+
url="/buckets/#{bucket}/keys/#{key_handler(key)}"
|
53
|
+
http("put",key,url,data)
|
54
|
+
end
|
55
|
+
def find(key,*opts)
|
56
|
+
data={}; link=""
|
57
|
+
data[:head], data[:query] = handle_options("get",opts)
|
58
|
+
if data[:head][:link]
|
59
|
+
link=data[:head][:link]
|
60
|
+
data[:head].reject!{|o| o==:link}
|
61
|
+
end
|
62
|
+
|
63
|
+
url="/buckets/#{bucket}/keys/#{key_handler(key)}#{link}"
|
64
|
+
http("get",key,url,data)
|
65
|
+
end
|
66
|
+
def save(obj,*opts)
|
67
|
+
data={:body=>{}}
|
68
|
+
obj.each{|key,value| data[:body][key]=value }
|
69
|
+
data[:head], data[:query] = handle_options("save",opts)
|
70
|
+
|
71
|
+
url="/buckets/#{bucket}/keys/#{obj[:riak_key]}"
|
72
|
+
http("put",obj[:riak_key],url,data)
|
73
|
+
end
|
74
|
+
def destroy(key,*opts)
|
75
|
+
data={}
|
76
|
+
obj_key=defined?(obj) ? obj[:riak_key] : (key.class==String ? key : key[:riak_key])
|
77
|
+
data[:head], data[:query] = handle_options("delete",opts)
|
78
|
+
|
79
|
+
url="/buckets/#{bucket}/keys/#{key_handler(obj_key)}"
|
80
|
+
http("put",obj_key,url,data)
|
81
|
+
key=nil
|
82
|
+
end
|
83
|
+
|
84
|
+
# -- Search
|
85
|
+
def search(*args)
|
86
|
+
return "you must provide search mode" if !SEARCH_SUPPORT.index(args[0])
|
87
|
+
|
88
|
+
method=args[0]
|
89
|
+
collect_data=if args[1].class==Hash
|
90
|
+
opts=args[1]
|
91
|
+
args[2..args.count]
|
92
|
+
else
|
93
|
+
args[1..args.count]
|
94
|
+
end
|
95
|
+
|
96
|
+
return "you must provide some vars to do search, for example, bucket name, bin name..." if !defined?(opts) && method!=:map_reduce || opts[:bin].nil? && method==:secondary_index
|
97
|
+
|
98
|
+
case method
|
99
|
+
when :secondary_index
|
100
|
+
bin=opts[:bin].to_s.index("_bin") ? opts[:bin] : "#{opts[:bin]}_bin"
|
101
|
+
mapper_key=collect_data.count>1 ? :secondary_index_multiple_query : :secondary_index_single_query
|
102
|
+
|
103
|
+
members={}
|
104
|
+
EmRiak::MapReduce.submit(mapper_key,{'bucket'=>opts[:bucket],'index'=>bin,'value'=>collect_data.join(',')}).each do |object|
|
105
|
+
if object[1]["body"]
|
106
|
+
members[object[0].to_sym]=object[1]["body"]
|
107
|
+
else
|
108
|
+
members[object[0].to_sym]=[] if !members[object[0].to_sym]
|
109
|
+
members[object[0].to_sym] << object[1]
|
110
|
+
end if object[0]!=false
|
111
|
+
end
|
112
|
+
members
|
113
|
+
when :full_text # TODO : full-text search is still under implement...
|
114
|
+
[]
|
115
|
+
when :map_reduce
|
116
|
+
EmRiak::MapReduce.submit(collect_data[0])
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def http(method,key,url,data,res=nil)
|
121
|
+
self.async && ["delete","put","post"].index(method) && key!="mapred" ? em_http(method,key,url,data) : open_http(method,key,url,data)
|
122
|
+
end
|
123
|
+
def return_body_handler(string,hash_container,handle_type,handle_type_split_mapper={"hash"=>":","string"=>"=","common"=>","})
|
124
|
+
hash_container={} if !hash_container
|
125
|
+
|
126
|
+
string=string.gsub("{","").gsub("}","") if handle_type!="hash"
|
127
|
+
split_by= hash_container.class==EmRiak::StorageObject ? "&" : ","
|
128
|
+
string.split(split_by).each do |vars|
|
129
|
+
key,value=vars.split(handle_type_split_mapper[handle_type])
|
130
|
+
value=return_body_handler(value,nil,"hash") if value && value.index("{")
|
131
|
+
value=json_decode("{#{value}}") if value && value.index(":") && value.index(",")
|
132
|
+
hash_container[key.to_sym]=value if value
|
133
|
+
end
|
134
|
+
hash_container
|
135
|
+
end
|
136
|
+
|
137
|
+
def open_http(method,key,url,data,res={})
|
138
|
+
has_http_instance?
|
139
|
+
data={:body=>{},:head=>{},:query=>{}} if !data
|
140
|
+
|
141
|
+
http_client.url = generate_path(url)
|
142
|
+
http_client.headers = key!="mapred" ? convert_params_layers(data[:head],:to_s) : data[:head]
|
143
|
+
http_client.body = key!="mapred" ? convert_params_layers(data[:body],:to_s) : data[:body]
|
144
|
+
|
145
|
+
res=HTTPI.request(method.to_sym,http_client)
|
146
|
+
res=res.body if res.class==HTTPI::Response
|
147
|
+
|
148
|
+
res=if !["2i","mapred"].index(key) && res && !["not found\n"].index(res) && ["get","put","post"].index(method)
|
149
|
+
obj=EmRiak::StorageObject.new()
|
150
|
+
if res && res!="not found\n"
|
151
|
+
if res.class==String
|
152
|
+
obj=return_body_handler(URI.unescape(res.force_encoding("UTF-8")).gsub("+"," ").gsub("=>",":"),obj,"string")
|
153
|
+
elsif res.class==Hash
|
154
|
+
res.each{|varkey,value| obj[varkey.to_sym]=value }
|
155
|
+
end
|
156
|
+
|
157
|
+
data.each{|k,v| obj[k]=v } if res.length<1
|
158
|
+
obj[:riak_key]=key if obj.length>0
|
159
|
+
end
|
160
|
+
obj
|
161
|
+
elsif key=="mapred"
|
162
|
+
res
|
163
|
+
else
|
164
|
+
nil
|
165
|
+
end
|
166
|
+
res=nil if res && res.class!=String && res.count<1
|
167
|
+
|
168
|
+
return res
|
169
|
+
end
|
170
|
+
|
171
|
+
def em_http(method,key,url,data)
|
172
|
+
data[:retry]=0 if !data[:retry]
|
173
|
+
begin
|
174
|
+
raise "Retry too much time" if data[:retry]>RETRY_TIMES
|
175
|
+
|
176
|
+
#conn options
|
177
|
+
conn_options = {
|
178
|
+
:connect_timeout => EM_REQUEST_TIMEOUT, # default connection setup timeout
|
179
|
+
:inactivity_timeout => EM_INACTIVITY_TIMEOUT, # default connection inactivity (post-setup) timeout
|
180
|
+
}
|
181
|
+
|
182
|
+
#data[:body]=data[:body] if data[:body].class==Hash && data[:head]['Content-Type'] && data[:head]['Content-Type']=="application/json"
|
183
|
+
#request options
|
184
|
+
data[:query]="?returnbody=true"
|
185
|
+
request_options=data.reject{|k,v| k==:retry}
|
186
|
+
|
187
|
+
#do job
|
188
|
+
case method
|
189
|
+
when "post"
|
190
|
+
riakHttp = EventMachine::HttpRequest.new(hosts[rand(hosts.count)]+url).post request_options
|
191
|
+
when "delete"
|
192
|
+
riakHttp = EventMachine::HttpRequest.new(hosts[rand(hosts.count)]+url).delete request_options
|
193
|
+
when "get"
|
194
|
+
riakHttp = EventMachine::HttpRequest.new(hosts[rand(hosts.count)]+url).get request_options
|
195
|
+
when "put"
|
196
|
+
riakHttp = EventMachine::HttpRequest.new(hosts[rand(hosts.count)]+url).put request_options
|
197
|
+
end
|
198
|
+
|
199
|
+
riakHttp.errback{ #Error Callback
|
200
|
+
puts "#{method} - #{key} - riak callback error #{riakHttp.response}" if EmRiak.debug
|
201
|
+
data[:retry]+=1
|
202
|
+
http(method,key,url,data)
|
203
|
+
}
|
204
|
+
|
205
|
+
#Success Callback
|
206
|
+
riakHttp.callback{ puts "#{method} - #{url} - #{key} - riak response #{riakHttp.response}" if EmRiak.debug }
|
207
|
+
rescue Exception => e
|
208
|
+
puts "em_http - #{method} - #{url} - #{key} error #{e}" if EmRiak.debug
|
209
|
+
end
|
210
|
+
|
211
|
+
# Return Response
|
212
|
+
res=if ["put","post"].index(method)
|
213
|
+
obj=EmRiak::StorageObject.new()
|
214
|
+
data.each{|k,v| obj[k]=v }
|
215
|
+
obj[:riak_key]=key if obj.length>0
|
216
|
+
obj
|
217
|
+
else
|
218
|
+
nil
|
219
|
+
end
|
220
|
+
res
|
221
|
+
end
|
222
|
+
|
223
|
+
def handle_options(action,opts,header={},query="")
|
224
|
+
#Load default header setting
|
225
|
+
case action
|
226
|
+
when "save"
|
227
|
+
header["Content-Type"]="application/json"
|
228
|
+
when "get"
|
229
|
+
header["Content-Type"]="application/json"
|
230
|
+
when "destroy"
|
231
|
+
|
232
|
+
when "add_link"
|
233
|
+
header["Content-Type"]="application/json"
|
234
|
+
when "remove_link"
|
235
|
+
# else
|
236
|
+
# header["Content-Type"]="application/json"
|
237
|
+
end
|
238
|
+
|
239
|
+
if opts && opts.count>0
|
240
|
+
#Map & overwrite if header specisfic
|
241
|
+
opts=opts.first if opts.class==Array
|
242
|
+
opts.each{|k,v|
|
243
|
+
meta=METAMAPPER[k]
|
244
|
+
if meta
|
245
|
+
header[meta[:key]] = meta[:value]
|
246
|
+
opts.reject!{true}
|
247
|
+
end
|
248
|
+
}
|
249
|
+
|
250
|
+
# --- Handle the rest headers
|
251
|
+
# Link Walk
|
252
|
+
if opts[:links] && (["save"].index(action))
|
253
|
+
links=[]
|
254
|
+
opts[:links].each{|link| links << %Q[</buckets/#{bucket}/keys/#{link[:target]}>; riaktag="#{link[:tag]}"] }
|
255
|
+
header['Link']= links.join(",")
|
256
|
+
elsif opts[:links] && (["add_link","remove_link"].index(action))
|
257
|
+
links=[]
|
258
|
+
opts[:links].each{|link| links << %Q[</buckets/#{bucket}/keys/#{link[:target]}>; riaktag="#{link[:tag]}"] }
|
259
|
+
header['Link']= links
|
260
|
+
elsif opts[:links] && action=="get"
|
261
|
+
header["Content-Type"]="multipart/mixed"
|
262
|
+
header["Accept"]="multipart/mixed"
|
263
|
+
header[:link]=""
|
264
|
+
keep=opts[:links][:keep] ? '1' : '0' #keep the result
|
265
|
+
opts[:links][:tag].each{|tag| header[:link]+= "/#{bucket},#{tag},#{keep}" }
|
266
|
+
end
|
267
|
+
# 2i
|
268
|
+
header["x-riak-index-#{opts[:bin].to_s}_bin"]=opts[:secondary_index].join(", ") if opts[:secondary_index]
|
269
|
+
|
270
|
+
# Replication, available configs : w, dw, pw
|
271
|
+
# read more at http://docs.basho.com/riak/latest/references/apis/http/HTTP-Store-Object/
|
272
|
+
replication.each{|key,value| hreader[key.to_s]=value} if replication
|
273
|
+
|
274
|
+
header["Content-Type"]=opts[:content_type] if opts[:content_type]
|
275
|
+
header["X-Riak-Vclock"]=opts[:vlock] if opts[:vlock]
|
276
|
+
header["ETag"]=opts[:etag] if opts[:etag]
|
277
|
+
header["Last-Modified"]=opts[:last_modified] if opts[:last_modified]
|
278
|
+
header["returnbody"]= "true" if !async
|
279
|
+
#header["unique"] = true if async=false
|
280
|
+
query="?returnbody=true"
|
281
|
+
opts[:query].each{|k,v| query+="&#{k}={v}" } if opts[:query]
|
282
|
+
end
|
283
|
+
[header,query]
|
284
|
+
end
|
285
|
+
|
286
|
+
def key_handler(key,map={"["=>"%5B","]"=>"%5D","+"=>"%2B","/"=>"%2F","("=>"%28",")"=>"%29",":"=>"%3A"})
|
287
|
+
# To match the key rule from Basho : If your field contains special characters, such as (‘+’,‘/’,‘[’,‘]’,‘(’,‘)’,‘:’ or space), then either surround the phrase in single quotes, or escape each special character with a backslash.
|
288
|
+
# map.each{|origin,value| key.gsub!(origin,value) }
|
289
|
+
EscapeUtils.escape_url(key.to_s.encode('UTF-8'))
|
290
|
+
end
|
291
|
+
|
292
|
+
def json_encode(obj,times=0)
|
293
|
+
y=Yajl::Encoder.new()
|
294
|
+
y.encode(obj)
|
295
|
+
rescue Exception => e
|
296
|
+
puts "json_encode error : #{e}" if EmRiak.debug
|
297
|
+
nil
|
298
|
+
end
|
299
|
+
def json_decode(obj,times=0)
|
300
|
+
y=Yajl::Parser.new()
|
301
|
+
y.parse(obj)
|
302
|
+
rescue Exception => e
|
303
|
+
puts "json_decode error : #{e}" if EmRiak.debug
|
304
|
+
nil
|
305
|
+
end
|
306
|
+
|
307
|
+
def convert_params_layers(origin,to_type,params={})
|
308
|
+
origin.each{|key,value|
|
309
|
+
converted=convert_params_layers(value,to_type) if value.class==Hash
|
310
|
+
key=key.send(to_type)
|
311
|
+
params[key]= converted ? converted : value
|
312
|
+
} if origin
|
313
|
+
params
|
314
|
+
end
|
315
|
+
|
316
|
+
private
|
317
|
+
def generate_path(url); EmRiak.hosts[rand(EmRiak.hosts.length)]+url; end
|
318
|
+
def has_http_instance?
|
319
|
+
if !EmRiak.http_client
|
320
|
+
HTTPI.log=false
|
321
|
+
HTTPI.adapter=:net_http
|
322
|
+
EmRiak.http_client=HTTPI::Request.new
|
323
|
+
end
|
324
|
+
end
|
325
|
+
end
|
File without changes
|
File without changes
|
@@ -0,0 +1,58 @@
|
|
1
|
+
module EmRiak
|
2
|
+
module Grapher
|
3
|
+
extend self
|
4
|
+
|
5
|
+
class LinkWalk
|
6
|
+
def add_link(key,*opts)
|
7
|
+
data={}
|
8
|
+
data[:head], data[:query] = EmRiak.handle_options("add_link",opts)
|
9
|
+
data[:query]= data[:query] && data[:query].length>0 ? data[:query]+"&returnbody=true" : "?returnbody=true"
|
10
|
+
|
11
|
+
url="/buckets/#{EmRiak.bucket}/keys/#{key}"
|
12
|
+
|
13
|
+
response=open("#{RIAKHOST[rand(RIAKHOST.count)]}/buckets/#{EmRiak.bucket}/keys/#{key}?returnbody=true")
|
14
|
+
body,links = rehandle_link_and_params(response)
|
15
|
+
|
16
|
+
data[:head]['Link'].each{|link| links=links.gsub(link+", ","") }
|
17
|
+
data[:head]['Link']=data[:head]['Link'].join(", ")+", "+links if data[:head]['Link'].count>0
|
18
|
+
data[:body]=body
|
19
|
+
|
20
|
+
EmRiak.http("put",key,url,data)
|
21
|
+
end
|
22
|
+
|
23
|
+
def remove_link(key,*opts)
|
24
|
+
data={}
|
25
|
+
data[:head], data[:query] = EmRiak.handle_options("add_link",opts)
|
26
|
+
data[:query]= data[:query] && data[:query].length>0 ? data[:query]+"&returnbody=true" : "?returnbody=true"
|
27
|
+
|
28
|
+
url="/buckets/#{EmRiak.bucket}/keys/#{key}"
|
29
|
+
|
30
|
+
response=open("#{RIAKHOST[rand(RIAKHOST.count)]}/buckets/#{EmRiak.bucket}/keys/#{key}?returnbody=true")
|
31
|
+
body,links = rehandle_link_and_params(response)
|
32
|
+
|
33
|
+
|
34
|
+
data[:body]=body
|
35
|
+
|
36
|
+
data[:head]['Link'].each{|link| links=links.gsub(link+", ","") }
|
37
|
+
data[:head]['Link']=links
|
38
|
+
EmRiak.http("put",key,url,data)
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
def rehandle_link_and_params(response,body={},links=[])
|
43
|
+
response.read.split("&").each{|params|
|
44
|
+
key=params.split("=")[0]
|
45
|
+
value=params.split("=")[1]
|
46
|
+
body[key.to_sym]=value
|
47
|
+
}
|
48
|
+
links=response.meta["link"]#.split(",")
|
49
|
+
#.each{|link|
|
50
|
+
#key=link.split(";")[0].split("/").last.gsub(">","")
|
51
|
+
#tag=link.split(";")[1].split("").gsub("\"","").gsub("\"","").gsub(" ","")
|
52
|
+
#links << {:target=>"key",:tag=>tag}
|
53
|
+
#}
|
54
|
+
[body,links]
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
File without changes
|
@@ -0,0 +1,39 @@
|
|
1
|
+
MRMAPPER={
|
2
|
+
:secondary_index_single_query=>"{\"inputs\" => {\"bucket\"=> \"@!bucket!@\",\"index\"=> \"@!index!@\", \"key\"=>\"@!value!@\"},
|
3
|
+
\"query\" => [{\"map\"=> {\"language\"=> \"javascript\",\"source\"=>\"function(riakObject){ var body={}; var body_arrays=riakObject.values[0].data.split('&head')[0].split('&'); for(i=0;i<body_arrays.length;i++){ body[body_arrays[i].split('=')[0]]=body_arrays[i].split('=')[1];}; return [[riakObject.key, body]]; }\"} },
|
4
|
+
{\"reduce\"=> {\"language\"=> \"erlang\", \"module\"=>\"riak_kv_mapreduce\", \"function\"=>\"reduce_identity\", \"keep\"=>true} }
|
5
|
+
]}",
|
6
|
+
:secondary_index_range_query=>"",
|
7
|
+
:secondary_index_multiple_query=>"{\"inputs\"=> {\"bucket\"=>\"@!bucket!@\",\"index\"=> \"@!index!@\",\"key_filters\"=>[[\"greater_than\",0]]},
|
8
|
+
\"query\" => [{\"map\"=> {\"language\"=>\"javascript\", \"source\"=>\"function(riakObject){var search_string='@!value!@';var search_values=search_string.split(',');var metadata=riakObject.values[0].metadata; var indexes=null; var match=false; if(metadata){ if(metadata.index){ indexes=metadata.index.@!index!@; var temp_index=indexes; if(typeof(indexes)!='object') indexes=[indexes];} if(indexes){ for(i=0;i<search_values.length; i++){if(indexes.indexOf(search_values[i])>0){match=true;break;};};};}; return (indexes && match) ? [[riakObject.bucket,riakObject.key]] : [[false,false]];}\" }},
|
9
|
+
{\"reduce\"=> {\"language\"=> \"erlang\", \"module\"=>\"riak_kv_mapreduce\", \"function\"=>\"reduce_set_union\", \"keep\"=>true} }
|
10
|
+
]}",
|
11
|
+
:full_text_query=>"",
|
12
|
+
:clean_bucket=>"{\"inputs\"=> {\"bucket\"=>\"@!bucket!@\",\"key_filters\"=>[[\"greater_than\",0]]},
|
13
|
+
\"query\" => [{\"map\"=> {\"language\"=>\"javascript\", \"source\"=>\"function(riakObject){return [[riakObject.key]];}\" }}
|
14
|
+
]}"
|
15
|
+
}
|
16
|
+
|
17
|
+
module EmRiak
|
18
|
+
module MapReduce
|
19
|
+
extend self
|
20
|
+
def submit(mode_or_string,params)
|
21
|
+
if mode_or_string.class==Symbol
|
22
|
+
map_reduce_work=MRMAPPER[mode_or_string]
|
23
|
+
map_reduce_work.gsub!(/@!(.{1,}?)!@/){|m| params[$1] }
|
24
|
+
map_reduce_work=eval(map_reduce_work)
|
25
|
+
end
|
26
|
+
url="/mapred"
|
27
|
+
map_reduce_json=EmRiak.json_encode(map_reduce_work)
|
28
|
+
data={:head=>{"Content-Type"=>"application/json"},:body=>map_reduce_json}
|
29
|
+
res=EmRiak.http("post","mapred",url,data)
|
30
|
+
|
31
|
+
if res.class==String
|
32
|
+
res=EscapeUtils.unescape_url(res.encode('UTF-8').gsub("+","")).gsub("=>",":").gsub("\"{","{").gsub("}\"","}").gsub("%20"," ")
|
33
|
+
res=EmRiak.json_decode(res)
|
34
|
+
end
|
35
|
+
res=EmRiak.convert_params_layers(res,:to_sym) if res.class==Hash
|
36
|
+
res
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
module EmRiak
|
2
|
+
module SecondaryIndex
|
3
|
+
extend self
|
4
|
+
def add_tag(data_self,args)
|
5
|
+
data,opts=handle_data_and_options(data_self,args)
|
6
|
+
submit_work("add",data_self,data,opts)
|
7
|
+
end
|
8
|
+
|
9
|
+
def remove_tag(data_self,args)
|
10
|
+
data,opts=handle_data_and_options(data_self,args)
|
11
|
+
submit_work("remove",data_self,data,opts)
|
12
|
+
end
|
13
|
+
|
14
|
+
protected
|
15
|
+
def submit_work(submit_mode,data_self,data,opts,res={:status=>"success"})
|
16
|
+
self.send "#{submit_mode}_as_index",data_self,data,opts
|
17
|
+
end
|
18
|
+
|
19
|
+
def handle_data_and_options(data_self,args)
|
20
|
+
data={:head=>data_self[:head],:body=>data_self,:query=>{}}
|
21
|
+
data[:body].delete(:head)
|
22
|
+
args=args[0] if args[0].class==Array
|
23
|
+
opts={:bin=>args[0]}
|
24
|
+
opts[:secondary_index]= if args[1] && args[1].class==Hash
|
25
|
+
args[1].each{|key,value| opts[key]=value }
|
26
|
+
args[2..args.count]
|
27
|
+
else
|
28
|
+
args[1..args.count]
|
29
|
+
end
|
30
|
+
[data, opts]
|
31
|
+
end
|
32
|
+
def merge_tag(new_heads,old_heads,res={})
|
33
|
+
return new_heads if !old_heads
|
34
|
+
new_heads.each do |key,value|
|
35
|
+
res[key]= key.match(/x-riak-index-/) && old_heads[key] ? value.split(", ").concat(old_heads[key].split(", ")).uniq.join(", ") : value
|
36
|
+
end
|
37
|
+
res
|
38
|
+
end
|
39
|
+
|
40
|
+
def add_as_index(data_self,data,opts)
|
41
|
+
temp_head, data[:query] = EmRiak.handle_options("post",opts)
|
42
|
+
data[:head]=merge_tag(temp_head,data[:head])
|
43
|
+
data_self[:head]=data[:head]
|
44
|
+
|
45
|
+
url="/buckets/#{EmRiak.bucket}/keys/#{data_self[:riak_key]}"
|
46
|
+
EmRiak.http("post","2i",url,data)
|
47
|
+
end
|
48
|
+
def destroy_tag(new_heads,old_heads)
|
49
|
+
return new_heads if !old_heads
|
50
|
+
new_heads.each do |key,value|
|
51
|
+
if key.match(/x-riak-index-/) && old_heads[key]
|
52
|
+
temp_head=old_heads[key].split(", ")
|
53
|
+
temp_head.delete(value)
|
54
|
+
old_heads[key]=temp_head.uniq.join(", ")
|
55
|
+
end
|
56
|
+
end
|
57
|
+
old_heads
|
58
|
+
end
|
59
|
+
def remove_as_index(data_self,data,opts)
|
60
|
+
temp_head, data[:query] = EmRiak.handle_options("post",opts)
|
61
|
+
data[:head]=destroy_tag(temp_head,data[:head])
|
62
|
+
data_self[:head]=data[:head]
|
63
|
+
|
64
|
+
url="/buckets/#{EmRiak.bucket}/keys/#{data_self[:riak_key]}"
|
65
|
+
EmRiak.http("post","2i",url,data)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module EmRiak
|
2
|
+
module Util
|
3
|
+
extend self
|
4
|
+
def clean_bucket(bucket_name)
|
5
|
+
keys=EmRiak::MapReduce.submit(:clean_bucket,{"bucket"=>bucket_name})
|
6
|
+
if keys
|
7
|
+
if !(keys.class==Hash && keys[:error])
|
8
|
+
keys=keys[0] if keys.class==Array && keys.count==1
|
9
|
+
keys.each{|key|
|
10
|
+
key=key[0].gsub("\\","\/")
|
11
|
+
EmRiak.destroy(URI.escape(key[0]).to_s)
|
12
|
+
}
|
13
|
+
puts "clean done"
|
14
|
+
else
|
15
|
+
puts "Something went wrong." if EmRiak.debug
|
16
|
+
end
|
17
|
+
else
|
18
|
+
puts "key empty"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/em-riak.rb
ADDED
metadata
ADDED
@@ -0,0 +1,116 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: em-riak
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.2.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Von
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-12-01 00:00:00.000000000Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: escape_utils
|
16
|
+
requirement: &70187287663420 !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: *70187287663420
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: yajl-ruby
|
27
|
+
requirement: &70187287662160 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ~>
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 1.1.0
|
33
|
+
type: :runtime
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *70187287662160
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: httpi
|
38
|
+
requirement: &70187287655840 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ~>
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: 1.1.1
|
44
|
+
type: :runtime
|
45
|
+
prerelease: false
|
46
|
+
version_requirements: *70187287655840
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: eventmachine
|
49
|
+
requirement: &70187287655260 !ruby/object:Gem::Requirement
|
50
|
+
none: false
|
51
|
+
requirements:
|
52
|
+
- - ~>
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 1.0.0.beta.4
|
55
|
+
type: :runtime
|
56
|
+
prerelease: false
|
57
|
+
version_requirements: *70187287655260
|
58
|
+
- !ruby/object:Gem::Dependency
|
59
|
+
name: em-http-request
|
60
|
+
requirement: &70187287654660 !ruby/object:Gem::Requirement
|
61
|
+
none: false
|
62
|
+
requirements:
|
63
|
+
- - ~>
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
version: 1.0.2
|
66
|
+
type: :runtime
|
67
|
+
prerelease: false
|
68
|
+
version_requirements: *70187287654660
|
69
|
+
description: An extremely fast and convenient riak client for eventmachine.
|
70
|
+
email: von@vonstark.co
|
71
|
+
executables: []
|
72
|
+
extensions: []
|
73
|
+
extra_rdoc_files: []
|
74
|
+
files:
|
75
|
+
- .gitignore
|
76
|
+
- LICENSE.md
|
77
|
+
- README.md
|
78
|
+
- Rakefile
|
79
|
+
- em-riak.gemspec
|
80
|
+
- lib/em-riak.rb
|
81
|
+
- lib/em-riak/basic.rb
|
82
|
+
- lib/em-riak/configurations.rb
|
83
|
+
- lib/em-riak/eventmachine.rb
|
84
|
+
- lib/em-riak/full_text_search.rb
|
85
|
+
- lib/em-riak/grapher.rb
|
86
|
+
- lib/em-riak/luwak.rb
|
87
|
+
- lib/em-riak/map_reduce.rb
|
88
|
+
- lib/em-riak/secondary_index.rb
|
89
|
+
- lib/em-riak/utils.rb
|
90
|
+
- lib/em-riak/version.rb
|
91
|
+
homepage: http://tech.vonstark.co
|
92
|
+
licenses: []
|
93
|
+
post_install_message:
|
94
|
+
rdoc_options: []
|
95
|
+
require_paths:
|
96
|
+
- lib
|
97
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
98
|
+
none: false
|
99
|
+
requirements:
|
100
|
+
- - ! '>='
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
104
|
+
none: false
|
105
|
+
requirements:
|
106
|
+
- - ! '>='
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: '0'
|
109
|
+
requirements: []
|
110
|
+
rubyforge_project:
|
111
|
+
rubygems_version: 1.8.6
|
112
|
+
signing_key:
|
113
|
+
specification_version: 3
|
114
|
+
summary: Riak client for eventmachine
|
115
|
+
test_files: []
|
116
|
+
has_rdoc:
|