riakpb 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +1 -2
- data/Rakefile +46 -13
- data/goog.csv +1439 -0
- data/lib/riak/map_reduce.rb +218 -0
- data/lib/riak.rb +1 -1
- data/load_stocks.rb +12 -0
- data/riakpb.gemspec +37 -0
- metadata +62 -39
@@ -0,0 +1,218 @@
|
|
1
|
+
require 'riak'
|
2
|
+
|
3
|
+
module Riak
|
4
|
+
# Class for invoking map-reduce jobs using the HTTP interface.
|
5
|
+
class MapReduce
|
6
|
+
include Util::Translation
|
7
|
+
# @return [Array<[bucket,key]>,String] The bucket/keys for input to the job, or the bucket (all keys).
|
8
|
+
# @see #add
|
9
|
+
attr_accessor :inputs
|
10
|
+
|
11
|
+
# @return [Array<Phase>] The map and reduce phases that will be executed
|
12
|
+
# @see #map
|
13
|
+
# @see #reduce
|
14
|
+
# @see #link
|
15
|
+
attr_accessor :query
|
16
|
+
|
17
|
+
# Creates a new map-reduce job.
|
18
|
+
# @param [Client] client the Riak::Client interface
|
19
|
+
# @yield [self] helpful for initializing the job
|
20
|
+
def initialize(client)
|
21
|
+
@client, @inputs, @query = client, [], []
|
22
|
+
yield self if block_given?
|
23
|
+
end
|
24
|
+
|
25
|
+
# Add or replace inputs for the job.
|
26
|
+
# @overload add(bucket)
|
27
|
+
# Run the job across all keys in the bucket. This will replace any other inputs previously added.
|
28
|
+
# @param [String, Bucket] bucket the bucket to run the job on
|
29
|
+
# @overload add(bucket,key)
|
30
|
+
# Add a bucket/key pair to the job.
|
31
|
+
# @param [String,Bucket] bucket the bucket of the object
|
32
|
+
# @param [String] key the key of the object
|
33
|
+
# @overload add(object)
|
34
|
+
# Add an object to the job (by its bucket/key)
|
35
|
+
# @param [Key] object the object to add to the inputs
|
36
|
+
# @overload add(bucket, key, keydata)
|
37
|
+
# @param [String,Bucket] bucket the bucket of the object
|
38
|
+
# @param [String] key the key of the object
|
39
|
+
# @param [String] keydata extra data to pass along with the object to the job
|
40
|
+
# @return [MapReduce] self
|
41
|
+
def add(*params)
|
42
|
+
params = params.dup.flatten
|
43
|
+
case params.size
|
44
|
+
when 1
|
45
|
+
p = params.first
|
46
|
+
case p
|
47
|
+
when Riak::Bucket
|
48
|
+
@inputs = p.name
|
49
|
+
when Riak::Key
|
50
|
+
@inputs << p.to_input
|
51
|
+
when String
|
52
|
+
@inputs = p
|
53
|
+
end
|
54
|
+
when 2..3
|
55
|
+
bucket = params.shift
|
56
|
+
bucket = bucket.name if Riak::Bucket === bucket
|
57
|
+
@inputs << params.unshift(bucket)
|
58
|
+
end
|
59
|
+
self
|
60
|
+
end
|
61
|
+
alias :<< :add
|
62
|
+
alias :include :add
|
63
|
+
|
64
|
+
# Add a map phase to the job.
|
65
|
+
# @overload map(function)
|
66
|
+
# @param [String, Array] function a Javascript function that represents the phase, or an Erlang [module,function] pair
|
67
|
+
# @overload map(function?, options)
|
68
|
+
# @param [String, Array] function a Javascript function that represents the phase, or an Erlang [module, function] pair
|
69
|
+
# @param [Hash] options extra options for the phase (see {Phase#initialize})
|
70
|
+
# @return [MapReduce] self
|
71
|
+
# @see Phase#initialize
|
72
|
+
def map(*params)
|
73
|
+
options = params.extract_options!
|
74
|
+
@query << Phase.new({:type => :map, :function => params.shift}.merge(options))
|
75
|
+
self
|
76
|
+
end
|
77
|
+
|
78
|
+
# Add a reduce phase to the job.
|
79
|
+
# @overload reduce(function)
|
80
|
+
# @param [String, Array] function a Javascript function that represents the phase, or an Erlang [module,function] pair
|
81
|
+
# @overload reduce(function?, options)
|
82
|
+
# @param [String, Array] function a Javascript function that represents the phase, or an Erlang [module, function] pair
|
83
|
+
# @param [Hash] options extra options for the phase (see {Phase#initialize})
|
84
|
+
# @return [MapReduce] self
|
85
|
+
# @see Phase#initialize
|
86
|
+
def reduce(*params)
|
87
|
+
options = params.extract_options!
|
88
|
+
@query << Phase.new({:type => :reduce, :function => params.shift}.merge(options))
|
89
|
+
self
|
90
|
+
end
|
91
|
+
|
92
|
+
# Add a link phase to the job. Link phases follow links attached to objects automatically (a special case of map).
|
93
|
+
# @param [Hash] params represents the types of links to follow
|
94
|
+
# @return [MapReduce] self
|
95
|
+
def link(params={})
|
96
|
+
bucket ||= params[:bucket]
|
97
|
+
tag ||= params[:tag]
|
98
|
+
keep = params[:keep] || false
|
99
|
+
function = {}
|
100
|
+
|
101
|
+
function[:bucket] = bucket unless bucket.nil?
|
102
|
+
function[:tag] = tag unless tag.nil?
|
103
|
+
|
104
|
+
@query << Phase.new({:type => :link, :function => function, :keep => keep})
|
105
|
+
|
106
|
+
return(self)
|
107
|
+
end
|
108
|
+
alias :walk :link
|
109
|
+
|
110
|
+
# Sets the timeout for the map-reduce job.
|
111
|
+
# @param [Fixnum] value the job timeout, in milliseconds
|
112
|
+
def timeout(value)
|
113
|
+
@timeout = value
|
114
|
+
end
|
115
|
+
|
116
|
+
# Convert the job to JSON for submission over the HTTP interface.
|
117
|
+
# @return [String] the JSON representation
|
118
|
+
def to_json(options={})
|
119
|
+
hash = {"inputs" => inputs, "query" => query.map(&:as_json)}
|
120
|
+
hash['timeout'] = @timeout.to_i if @timeout
|
121
|
+
ActiveSupport::JSON.encode(hash, options)
|
122
|
+
end
|
123
|
+
|
124
|
+
# Executes this map-reduce job.
|
125
|
+
# @return [Array<Array>] similar to link-walking, each element is an array of results from a phase where "keep" is true. If there is only one "keep" phase, only the results from that phase will be returned.
|
126
|
+
def run
|
127
|
+
response = @client.map_reduce_request(to_json, "application/json")
|
128
|
+
# ActiveSupport::JSON.decode(response[:body])
|
129
|
+
end
|
130
|
+
|
131
|
+
# Represents an individual phase in a map-reduce pipeline. Generally you'll want to call
|
132
|
+
# methods of {MapReduce} instead of using this directly.
|
133
|
+
class Phase
|
134
|
+
include Util::Translation
|
135
|
+
# @return [Symbol] the type of phase - :map, :reduce, or :link
|
136
|
+
attr_accessor :type
|
137
|
+
|
138
|
+
# @return [String, Array<String, String>, Hash, WalkSpec] For :map and :reduce types, the Javascript function to run (as a string or hash with bucket/key), or the module + function in Erlang to run. For a :link type, a {Riak::WalkSpec} or an equivalent hash.
|
139
|
+
attr_accessor :function
|
140
|
+
|
141
|
+
# @return [String] the language of the phase's function - "javascript" or "erlang". Meaningless for :link type phases.
|
142
|
+
attr_accessor :language
|
143
|
+
|
144
|
+
# @return [Boolean] whether results of this phase will be returned
|
145
|
+
attr_accessor :keep
|
146
|
+
|
147
|
+
# @return [Array] any extra static arguments to pass to the phase
|
148
|
+
attr_accessor :arg
|
149
|
+
|
150
|
+
# Creates a phase in the map-reduce pipeline
|
151
|
+
# @param [Hash] options options for the phase
|
152
|
+
# @option options [Symbol] :type one of :map, :reduce, :link
|
153
|
+
# @option options [String] :language ("javascript") "erlang" or "javascript"
|
154
|
+
# @option options [String, Array, Hash] :function In the case of Javascript, a literal function in a string, or a hash with :bucket and :key. In the case of Erlang, an Array of [module, function]. For a :link phase, a hash including any of :bucket, :tag or a WalkSpec.
|
155
|
+
# @option options [Boolean] :keep (false) whether to return the results of this phase
|
156
|
+
# @option options [Array] :arg (nil) any extra static arguments to pass to the phase
|
157
|
+
def initialize(options={})
|
158
|
+
self.type = options[:type]
|
159
|
+
self.language = options[:language] || "javascript"
|
160
|
+
self.function = options[:function]
|
161
|
+
self.keep = options[:keep] || false
|
162
|
+
self.arg = options[:arg]
|
163
|
+
end
|
164
|
+
|
165
|
+
def type=(value)
|
166
|
+
raise ArgumentError, t("invalid_phase_type") unless value.to_s =~ /^(map|reduce|link)$/i
|
167
|
+
@type = value.to_s.downcase.to_sym
|
168
|
+
end
|
169
|
+
|
170
|
+
def function=(value)
|
171
|
+
case value
|
172
|
+
when Array
|
173
|
+
raise ArgumentError, t("module_function_pair_required") unless value.size == 2
|
174
|
+
@language = "erlang"
|
175
|
+
when Hash
|
176
|
+
raise ArgumentError, t("stored_function_invalid") unless type == :link || value.has_key?(:bucket) && value.has_key?(:key)
|
177
|
+
@language = "javascript"
|
178
|
+
when String
|
179
|
+
@language = "javascript"
|
180
|
+
else
|
181
|
+
raise ArgumentError, t("invalid_function_value", :value => value.inspect)
|
182
|
+
end
|
183
|
+
@function = value
|
184
|
+
end
|
185
|
+
|
186
|
+
# Converts the phase to JSON for use while invoking a job.
|
187
|
+
# @return [String] a JSON representation of the phase
|
188
|
+
def to_json(options=nil)
|
189
|
+
ActiveSupport::JSON.encode(as_json, options)
|
190
|
+
end
|
191
|
+
|
192
|
+
# Converts the phase to its JSON-compatible representation for job invocation.
|
193
|
+
# @return [Hash] a Hash-equivalent of the phase
|
194
|
+
def as_json(options=nil)
|
195
|
+
obj = case type
|
196
|
+
when :map, :reduce
|
197
|
+
defaults = {:language => language, :keep => keep}
|
198
|
+
case function
|
199
|
+
when Hash
|
200
|
+
defaults.merge(function)
|
201
|
+
when String
|
202
|
+
if function =~ /\s*function/
|
203
|
+
defaults.merge("source" => function)
|
204
|
+
else
|
205
|
+
defaults.merge("name" => function)
|
206
|
+
end
|
207
|
+
when Array
|
208
|
+
defaults.merge("module" => function[0], "function" => function[1])
|
209
|
+
end
|
210
|
+
when :link
|
211
|
+
function.merge({:keep => keep})
|
212
|
+
end
|
213
|
+
obj["arg"] = arg if arg
|
214
|
+
{ type => obj }
|
215
|
+
end
|
216
|
+
end
|
217
|
+
end
|
218
|
+
end
|
data/lib/riak.rb
CHANGED
data/load_stocks.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'riak'
|
2
|
+
require 'csv'
|
3
|
+
|
4
|
+
client = Riak::Client.new
|
5
|
+
bucket = client["goog"]
|
6
|
+
|
7
|
+
CSV.foreach('goog.csv', :headers => true) do |row|
|
8
|
+
puts row.first[1].to_s
|
9
|
+
key = bucket[row.first[1].to_s]
|
10
|
+
key.content.value = Hash[row.to_a]
|
11
|
+
key.save
|
12
|
+
end
|
data/riakpb.gemspec
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = %q{riakpb}
|
5
|
+
s.version = "0.1.5"
|
6
|
+
|
7
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
|
+
s.authors = ["Scott Gonyea"]
|
9
|
+
s.date = %q{2010-06-16}
|
10
|
+
s.description = %q{riakpb is a protocol buffer client for Riak--the distributed database by Basho. Connects via Protocol Buffers instead of REST.}
|
11
|
+
s.email = %q{me@inherentlylame.com}
|
12
|
+
s.files = ["goog.csv", "History.txt", "lib/riak/bucket.rb", "lib/riak/client/rpc.rb", "lib/riak/client.rb", "lib/riak/client_pb.rb", "lib/riak/failed_exchange.rb", "lib/riak/failed_request.rb", "lib/riak/i18n.rb", "lib/riak/key.rb", "lib/riak/locale/en.yml", "lib/riak/map_reduce.rb", "lib/riak/riak_content.rb", "lib/riak/sibling_error.rb", "lib/riak/util/decode.rb", "lib/riak/util/encode.rb", "lib/riak/util/message_code.rb", "lib/riak/util/translation.rb", "lib/riak.rb", "load_stocks.rb", "Manifest.txt", "Rakefile", "README.rdoc", "riakpb.gemspec", "script/console", "script/destroy", "script/generate", "spec/riak/bucket_spec.rb", "spec/riak/client_spec.rb", "spec/riak/key_spec.rb", "spec/riak/map_reduce_spec.rb", "spec/riak/riak_content_spec.rb", "spec/riak/rpc_spec.rb", "spec/spec_helper.rb"]
|
13
|
+
s.homepage = %q{http://github.com/aitrus/riak-pbclient}
|
14
|
+
s.require_paths = ["lib"]
|
15
|
+
s.rubygems_version = %q{1.3.6}
|
16
|
+
s.summary = %q{riakpb is a protocol buffer client for Riak--the distributed database by Basho.}
|
17
|
+
s.test_files = ["spec/riak/bucket_spec.rb", "spec/riak/client_spec.rb", "spec/riak/key_spec.rb", "spec/riak/map_reduce_spec.rb", "spec/riak/riak_content_spec.rb", "spec/riak/rpc_spec.rb", "spec/spec_helper.rb"]
|
18
|
+
|
19
|
+
if s.respond_to? :specification_version then
|
20
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
21
|
+
s.specification_version = 3
|
22
|
+
|
23
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
24
|
+
s.add_development_dependency(%q<rspec>, ["~> 2.0.0.beta.9"])
|
25
|
+
s.add_runtime_dependency(%q<activesupport>, [">= 2.3.5"])
|
26
|
+
s.add_runtime_dependency(%q<protobuf>, [">= 0.4.4"])
|
27
|
+
else
|
28
|
+
s.add_dependency(%q<rspec>, ["~> 2.0.0.beta.9"])
|
29
|
+
s.add_dependency(%q<activesupport>, [">= 2.3.5"])
|
30
|
+
s.add_dependency(%q<protobuf>, [">= 0.4.4"])
|
31
|
+
end
|
32
|
+
else
|
33
|
+
s.add_dependency(%q<rspec>, ["~> 2.0.0.beta.9"])
|
34
|
+
s.add_dependency(%q<activesupport>, [">= 2.3.5"])
|
35
|
+
s.add_dependency(%q<protobuf>, [">= 0.4.4"])
|
36
|
+
end
|
37
|
+
end
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 1
|
8
|
-
-
|
9
|
-
version: 0.1.
|
8
|
+
- 5
|
9
|
+
version: 0.1.5
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Scott Gonyea
|
@@ -18,21 +18,23 @@ date: 2010-06-16 00:00:00 -07:00
|
|
18
18
|
default_executable:
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
|
-
name:
|
21
|
+
name: rspec
|
22
22
|
prerelease: false
|
23
23
|
requirement: &id001 !ruby/object:Gem::Requirement
|
24
24
|
requirements:
|
25
|
-
- -
|
25
|
+
- - ~>
|
26
26
|
- !ruby/object:Gem::Version
|
27
27
|
segments:
|
28
28
|
- 2
|
29
29
|
- 0
|
30
|
-
-
|
31
|
-
|
30
|
+
- 0
|
31
|
+
- beta
|
32
|
+
- 9
|
33
|
+
version: 2.0.0.beta.9
|
32
34
|
type: :development
|
33
35
|
version_requirements: *id001
|
34
36
|
- !ruby/object:Gem::Dependency
|
35
|
-
name:
|
37
|
+
name: activesupport
|
36
38
|
prerelease: false
|
37
39
|
requirement: &id002 !ruby/object:Gem::Requirement
|
38
40
|
requirements:
|
@@ -40,60 +42,75 @@ dependencies:
|
|
40
42
|
- !ruby/object:Gem::Version
|
41
43
|
segments:
|
42
44
|
- 2
|
43
|
-
-
|
44
|
-
-
|
45
|
-
version: 2.
|
46
|
-
type: :
|
45
|
+
- 3
|
46
|
+
- 5
|
47
|
+
version: 2.3.5
|
48
|
+
type: :runtime
|
47
49
|
version_requirements: *id002
|
48
|
-
|
49
|
-
|
50
|
-
|
50
|
+
- !ruby/object:Gem::Dependency
|
51
|
+
name: ruby_protobuf
|
52
|
+
prerelease: false
|
53
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
54
|
+
requirements:
|
55
|
+
- - ">="
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
segments:
|
58
|
+
- 0
|
59
|
+
- 4
|
60
|
+
- 4
|
61
|
+
version: 0.4.4
|
62
|
+
type: :runtime
|
63
|
+
version_requirements: *id003
|
64
|
+
description: riakpb is a protocol buffer client for Riak--the distributed database by Basho. Connects via Protocol Buffers instead of REST.
|
65
|
+
email: me@inherentlylame.com
|
51
66
|
executables: []
|
52
67
|
|
53
68
|
extensions: []
|
54
69
|
|
55
|
-
extra_rdoc_files:
|
56
|
-
|
57
|
-
- Manifest.txt
|
70
|
+
extra_rdoc_files: []
|
71
|
+
|
58
72
|
files:
|
73
|
+
- goog.csv
|
59
74
|
- History.txt
|
60
|
-
-
|
61
|
-
- README.rdoc
|
62
|
-
- Rakefile
|
63
|
-
- lib/riak.rb
|
64
|
-
- lib/riak/client.rb
|
75
|
+
- lib/riak/bucket.rb
|
65
76
|
- lib/riak/client/rpc.rb
|
77
|
+
- lib/riak/client.rb
|
66
78
|
- lib/riak/client_pb.rb
|
67
|
-
- lib/riak/
|
68
|
-
- lib/riak/
|
69
|
-
- lib/riak/riak_content.rb
|
79
|
+
- lib/riak/failed_exchange.rb
|
80
|
+
- lib/riak/failed_request.rb
|
70
81
|
- lib/riak/i18n.rb
|
82
|
+
- lib/riak/key.rb
|
71
83
|
- lib/riak/locale/en.yml
|
84
|
+
- lib/riak/map_reduce.rb
|
85
|
+
- lib/riak/riak_content.rb
|
86
|
+
- lib/riak/sibling_error.rb
|
72
87
|
- lib/riak/util/decode.rb
|
73
88
|
- lib/riak/util/encode.rb
|
74
89
|
- lib/riak/util/message_code.rb
|
75
90
|
- lib/riak/util/translation.rb
|
76
|
-
- lib/riak
|
77
|
-
-
|
78
|
-
-
|
91
|
+
- lib/riak.rb
|
92
|
+
- load_stocks.rb
|
93
|
+
- Manifest.txt
|
94
|
+
- Rakefile
|
95
|
+
- README.rdoc
|
96
|
+
- riakpb.gemspec
|
79
97
|
- script/console
|
80
98
|
- script/destroy
|
81
99
|
- script/generate
|
82
|
-
- spec/spec_helper.rb
|
83
|
-
- spec/riak/client_spec.rb
|
84
|
-
- spec/riak/rpc_spec.rb
|
85
100
|
- spec/riak/bucket_spec.rb
|
101
|
+
- spec/riak/client_spec.rb
|
86
102
|
- spec/riak/key_spec.rb
|
87
|
-
- spec/riak/riak_content_spec.rb
|
88
103
|
- spec/riak/map_reduce_spec.rb
|
104
|
+
- spec/riak/riak_content_spec.rb
|
105
|
+
- spec/riak/rpc_spec.rb
|
106
|
+
- spec/spec_helper.rb
|
89
107
|
has_rdoc: true
|
90
108
|
homepage: http://github.com/aitrus/riak-pbclient
|
91
109
|
licenses: []
|
92
110
|
|
93
111
|
post_install_message:
|
94
|
-
rdoc_options:
|
95
|
-
|
96
|
-
- README.rdoc
|
112
|
+
rdoc_options: []
|
113
|
+
|
97
114
|
require_paths:
|
98
115
|
- lib
|
99
116
|
required_ruby_version: !ruby/object:Gem::Requirement
|
@@ -112,10 +129,16 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
112
129
|
version: "0"
|
113
130
|
requirements: []
|
114
131
|
|
115
|
-
rubyforge_project:
|
132
|
+
rubyforge_project:
|
116
133
|
rubygems_version: 1.3.6
|
117
134
|
signing_key:
|
118
135
|
specification_version: 3
|
119
|
-
summary:
|
120
|
-
test_files:
|
121
|
-
|
136
|
+
summary: riakpb is a protocol buffer client for Riak--the distributed database by Basho.
|
137
|
+
test_files:
|
138
|
+
- spec/riak/bucket_spec.rb
|
139
|
+
- spec/riak/client_spec.rb
|
140
|
+
- spec/riak/key_spec.rb
|
141
|
+
- spec/riak/map_reduce_spec.rb
|
142
|
+
- spec/riak/riak_content_spec.rb
|
143
|
+
- spec/riak/rpc_spec.rb
|
144
|
+
- spec/spec_helper.rb
|