better-riak-client 1.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. data/LICENSE +16 -0
  2. data/README.markdown +198 -0
  3. data/RELEASE_NOTES.md +211 -0
  4. data/better-riak-client.gemspec +61 -0
  5. data/erl_src/riak_kv_test014_backend.beam +0 -0
  6. data/erl_src/riak_kv_test014_backend.erl +189 -0
  7. data/erl_src/riak_kv_test_backend.beam +0 -0
  8. data/erl_src/riak_kv_test_backend.erl +697 -0
  9. data/erl_src/riak_search_test_backend.beam +0 -0
  10. data/erl_src/riak_search_test_backend.erl +175 -0
  11. data/lib/riak/bucket.rb +221 -0
  12. data/lib/riak/client/beefcake/messages.rb +213 -0
  13. data/lib/riak/client/beefcake/object_methods.rb +111 -0
  14. data/lib/riak/client/beefcake_protobuffs_backend.rb +226 -0
  15. data/lib/riak/client/decaying.rb +36 -0
  16. data/lib/riak/client/excon_backend.rb +162 -0
  17. data/lib/riak/client/feature_detection.rb +88 -0
  18. data/lib/riak/client/http_backend/configuration.rb +211 -0
  19. data/lib/riak/client/http_backend/key_streamer.rb +43 -0
  20. data/lib/riak/client/http_backend/object_methods.rb +106 -0
  21. data/lib/riak/client/http_backend/request_headers.rb +34 -0
  22. data/lib/riak/client/http_backend/transport_methods.rb +201 -0
  23. data/lib/riak/client/http_backend.rb +340 -0
  24. data/lib/riak/client/net_http_backend.rb +82 -0
  25. data/lib/riak/client/node.rb +115 -0
  26. data/lib/riak/client/protobuffs_backend.rb +173 -0
  27. data/lib/riak/client/search.rb +91 -0
  28. data/lib/riak/client.rb +540 -0
  29. data/lib/riak/cluster.rb +151 -0
  30. data/lib/riak/core_ext/blank.rb +53 -0
  31. data/lib/riak/core_ext/deep_dup.rb +13 -0
  32. data/lib/riak/core_ext/extract_options.rb +7 -0
  33. data/lib/riak/core_ext/json.rb +15 -0
  34. data/lib/riak/core_ext/slice.rb +18 -0
  35. data/lib/riak/core_ext/stringify_keys.rb +10 -0
  36. data/lib/riak/core_ext/symbolize_keys.rb +10 -0
  37. data/lib/riak/core_ext/to_param.rb +31 -0
  38. data/lib/riak/core_ext.rb +7 -0
  39. data/lib/riak/encoding.rb +6 -0
  40. data/lib/riak/failed_request.rb +81 -0
  41. data/lib/riak/i18n.rb +5 -0
  42. data/lib/riak/json.rb +52 -0
  43. data/lib/riak/link.rb +94 -0
  44. data/lib/riak/locale/en.yml +53 -0
  45. data/lib/riak/locale/fr.yml +52 -0
  46. data/lib/riak/map_reduce/filter_builder.rb +103 -0
  47. data/lib/riak/map_reduce/phase.rb +98 -0
  48. data/lib/riak/map_reduce.rb +225 -0
  49. data/lib/riak/map_reduce_error.rb +7 -0
  50. data/lib/riak/node/configuration.rb +293 -0
  51. data/lib/riak/node/console.rb +133 -0
  52. data/lib/riak/node/control.rb +207 -0
  53. data/lib/riak/node/defaults.rb +83 -0
  54. data/lib/riak/node/generation.rb +106 -0
  55. data/lib/riak/node/log.rb +34 -0
  56. data/lib/riak/node/version.rb +43 -0
  57. data/lib/riak/node.rb +38 -0
  58. data/lib/riak/robject.rb +318 -0
  59. data/lib/riak/search.rb +3 -0
  60. data/lib/riak/serializers.rb +74 -0
  61. data/lib/riak/stamp.rb +77 -0
  62. data/lib/riak/test_server.rb +89 -0
  63. data/lib/riak/util/escape.rb +76 -0
  64. data/lib/riak/util/headers.rb +53 -0
  65. data/lib/riak/util/multipart/stream_parser.rb +62 -0
  66. data/lib/riak/util/multipart.rb +52 -0
  67. data/lib/riak/util/tcp_socket_extensions.rb +58 -0
  68. data/lib/riak/util/translation.rb +19 -0
  69. data/lib/riak/version.rb +3 -0
  70. data/lib/riak/walk_spec.rb +105 -0
  71. data/lib/riak.rb +21 -0
  72. metadata +348 -0
@@ -0,0 +1,540 @@
1
+ require 'tempfile'
2
+ require 'delegate'
3
+ require 'innertube'
4
+ require 'riak'
5
+ require 'riak/util/translation'
6
+ require 'riak/util/escape'
7
+ require 'riak/failed_request'
8
+ require 'riak/client/decaying'
9
+ require 'riak/client/node'
10
+ require 'riak/client/search'
11
+ require 'riak/client/http_backend'
12
+ require 'riak/client/net_http_backend'
13
+ require 'riak/client/excon_backend'
14
+ require 'riak/client/protobuffs_backend'
15
+ require 'riak/client/beefcake_protobuffs_backend'
16
+ require 'riak/bucket'
17
+ require 'riak/stamp'
18
+
19
+ module Riak
20
+ # A client connection to Riak.
21
+ class Client
22
+ include Util::Translation
23
+ include Util::Escape
24
+
25
+ # When using integer client IDs, the exclusive upper-bound of valid values.
26
+ MAX_CLIENT_ID = 4294967296
27
+
28
+ # Array of valid protocols
29
+ PROTOCOLS = %w[http https pbc]
30
+
31
+ # Regexp for validating hostnames, lifted from uri.rb in Ruby 1.8.6
32
+ HOST_REGEX = /^(?:(?:(?:[a-zA-Z\d](?:[-a-zA-Z\d]*[a-zA-Z\d])?)\.)*(?:[a-zA-Z](?:[-a-zA-Z\d]*[a-zA-Z\d])?)\.?|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|\[(?:(?:[a-fA-F\d]{1,4}:)*(?:[a-fA-F\d]{1,4}|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?:(?:[a-fA-F\d]{1,4}:)*[a-fA-F\d]{1,4})?::(?:(?:[a-fA-F\d]{1,4}:)*(?:[a-fA-F\d]{1,4}|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}))?)\])$/n
33
+
34
+ # Valid constructor options.
35
+ VALID_OPTIONS = [:protocol, :nodes, :client_id, :http_backend, :protobuffs_backend] | Node::VALID_OPTIONS
36
+
37
+ # Network errors.
38
+ NETWORK_ERRORS = [
39
+ EOFError,
40
+ Errno::ECONNABORTED,
41
+ Errno::ECONNREFUSED,
42
+ Errno::ECONNRESET,
43
+ Errno::ENETDOWN,
44
+ Errno::ENETRESET,
45
+ Errno::ENETUNREACH,
46
+ SocketError,
47
+ SystemCallError,
48
+ ]
49
+
50
+ Pool = ::Innertube::Pool
51
+
52
+ # @return [String] The protocol to use for the Riak endpoint
53
+ attr_reader :protocol
54
+
55
+ # @return [Array] The set of Nodes this client can communicate with.
56
+ attr_accessor :nodes
57
+
58
+ # @return [String] The internal client ID used by Riak to route responses
59
+ attr_reader :client_id
60
+
61
+ # @return [Symbol] The HTTP backend/client to use
62
+ attr_accessor :http_backend
63
+
64
+ # @return [Client::Pool] A pool of HTTP connections
65
+ attr_reader :http_pool
66
+
67
+ # @return [Symbol] The Protocol Buffers backend/client to use
68
+ attr_accessor :protobuffs_backend
69
+
70
+ # @return [Client::Pool] A pool of protobuffs connections
71
+ attr_reader :protobuffs_pool
72
+
73
+ # Creates a client connection to Riak
74
+ # @param [Hash] options configuration options for the client
75
+ # @option options [Array] :nodes A list of nodes this client connects to.
76
+ # Each element of the list is a hash which is passed to Node.new, e.g.
77
+ # {host: '127.0.0.1', pb_port: 1234, ...}.
78
+ # If no nodes are given, a single node is constructed from the remaining
79
+ # options given to Client.new.
80
+ # @option options [String] :host ('127.0.0.1') The host or IP address for the Riak endpoint
81
+ # @option options [String] :protocol ('http') The protocol to use for connecting to a node backend
82
+ # @option options [Fixnum] :http_port (8098) The port of the Riak HTTP endpoint
83
+ # @option options [Fixnum] :pb_port (8087) The port of the Riak Protocol Buffers endpoint
84
+ # @option options [String] :prefix ('/riak/') The URL path prefix to the main HTTP endpoint
85
+ # @option options [String] :mapred ('/mapred') The path to the map-reduce HTTP endpoint
86
+ # @option options [Fixnum, String] :client_id (rand(MAX_CLIENT_ID)) The internal client ID used by Riak to route responses
87
+ # @option options [String, Symbol] :http_backend (:NetHTTP) which HTTP backend to use
88
+ # @option options [String, Symbol] :protobuffs_backend (:Beefcake) which Protocol Buffers backend to use
89
+ # @option options [Boolean, Hash] :ssl (nil) The SSL options to pass to each node or true for default options
90
+ # @raise [ArgumentError] raised if any invalid options are given
91
+ def initialize(options={})
92
+ if options.include? :port
93
+ warn(t('deprecated.port', :backtrace => caller[0..2].join("\n ")))
94
+ end
95
+
96
+ unless (evil = options.keys - VALID_OPTIONS).empty?
97
+ raise ArgumentError, "#{evil.inspect} are not valid options for Client.new"
98
+ end
99
+
100
+ @nodes = (options[:nodes] || []).map do |n|
101
+ Client::Node.new self, n
102
+ end
103
+ if @nodes.empty? or options[:host] or options[:http_port] or options[:pb_port]
104
+ @nodes |= [Client::Node.new(self, options)]
105
+ end
106
+
107
+ @protobuffs_pool = Pool.new(
108
+ method(:new_protobuffs_backend),
109
+ lambda { |b| b.teardown }
110
+ )
111
+
112
+ @http_pool = Pool.new(
113
+ method(:new_http_backend),
114
+ lambda { |b| b.teardown }
115
+ )
116
+
117
+ self.protocol = options[:protocol] || "http"
118
+ self.http_backend = options[:http_backend] || :NetHTTP
119
+ self.protobuffs_backend = options[:protobuffs_backend] || :Beefcake
120
+ self.client_id = options[:client_id] if options[:client_id]
121
+ self.ssl = options[:ssl] if options[:ssl]
122
+ end
123
+
124
+ # Yields a backend for operations that are protocol-independent.
125
+ # You can change which type of backend is used by setting the
126
+ # {#protocol}.
127
+ # @yield [HTTPBackend,ProtobuffsBackend] an appropriate client backend
128
+ def backend(&block)
129
+ case @protocol.to_s
130
+ when /https?/i
131
+ http &block
132
+ when /pbc/i
133
+ protobuffs &block
134
+ end
135
+ end
136
+
137
+ # Sets basic HTTP auth on all nodes.
138
+ def basic_auth=(auth)
139
+ @nodes.each do |node|
140
+ node.basic_auth = auth
141
+ end
142
+ auth
143
+ end
144
+
145
+ # Retrieves a bucket from Riak.
146
+ # @param [String] bucket the bucket to retrieve
147
+ # @param [Hash] options options for retrieving the bucket
148
+ # @option options [Boolean] :props (false) whether to retreive the bucket properties
149
+ # @return [Bucket] the requested bucket
150
+ def bucket(name, options={})
151
+ unless (options.keys - [:props]).empty?
152
+ raise ArgumentError, "invalid options"
153
+ end
154
+ @bucket_cache ||= {}
155
+ (@bucket_cache[name] ||= Bucket.new(self, name)).tap do |b|
156
+ b.props if options[:props]
157
+ end
158
+ end
159
+ alias :[] :bucket
160
+
161
+ # Lists buckets which have keys stored in them.
162
+ # @note This is an expensive operation and should be used only
163
+ # in development.
164
+ # @return [Array<Bucket>] a list of buckets
165
+ def buckets
166
+ warn(t('list_buckets', :backtrace => caller.join("\n "))) unless Riak.disable_list_keys_warnings
167
+ backend do |b|
168
+ b.list_buckets.map {|name| Bucket.new(self, name) }
169
+ end
170
+ end
171
+ alias :list_buckets :buckets
172
+
173
+ # Choose a node from a set.
174
+ def choose_node(nodes = self.nodes)
175
+ # Prefer nodes which have gone a reasonable time without errors.
176
+ s = nodes.select do |node|
177
+ node.error_rate.value < 0.1
178
+ end
179
+
180
+ if s.empty?
181
+ # Fall back to minimally broken node.
182
+ nodes.min_by do |node|
183
+ node.error_rate.value
184
+ end
185
+ else
186
+ s[rand(s.size)]
187
+ end
188
+ end
189
+
190
+ # Set the client ID for this client. Must be a string or Fixnum value 0 =<
191
+ # value < MAX_CLIENT_ID.
192
+ # @param [String, Fixnum] value The internal client ID used by Riak to route responses
193
+ # @raise [ArgumentError] when an invalid client ID is given
194
+ # @return [String] the assigned client ID
195
+ def client_id=(value)
196
+ value = case value
197
+ when 0...MAX_CLIENT_ID, String
198
+ value
199
+ else
200
+ raise ArgumentError, t("invalid_client_id", :max_id => MAX_CLIENT_ID)
201
+ end
202
+
203
+ # Change all existing backend client IDs.
204
+ @protobuffs_pool.each do |pb|
205
+ pb.set_client_id value if pb.respond_to?(:set_client_id)
206
+ end
207
+ @client_id = value
208
+ end
209
+
210
+ def client_id
211
+ @client_id ||= backend do |b|
212
+ if b.respond_to?(:get_client_id)
213
+ b.get_client_id
214
+ else
215
+ make_client_id
216
+ end
217
+ end
218
+ end
219
+
220
+ # Deletes a file stored via the "Luwak" interface
221
+ # @param [String] filename the key/filename to delete
222
+ def delete_file(filename)
223
+ http do |h|
224
+ h.delete_file(filename)
225
+ end
226
+ true
227
+ end
228
+
229
+ # Delete an object. See Bucket#delete
230
+ def delete_object(bucket, key, options = {})
231
+ backend do |b|
232
+ b.delete_object(bucket, key, options)
233
+ end
234
+ end
235
+
236
+ # Checks whether a file exists in "Luwak".
237
+ # @param [String] key the key to check
238
+ # @return [true, false] whether the key exists in "Luwak"
239
+ def file_exists?(key)
240
+ http do |h|
241
+ h.file_exists?(key)
242
+ end
243
+ end
244
+ alias :file_exist? :file_exists?
245
+
246
+ # Bucket properties. See Bucket#props
247
+ def get_bucket_props(bucket)
248
+ backend do |b|
249
+ b.get_bucket_props bucket
250
+ end
251
+ end
252
+
253
+ # Retrieves a large file/IO object from Riak via the "Luwak"
254
+ # interface. Streams the data to a temporary file unless a block
255
+ # is given.
256
+ # @param [String] filename the key/filename for the object
257
+ # @return [IO, nil] the file (also having content_type and
258
+ # original_filename accessors). The file will need to be
259
+ # reopened to be read. nil will be returned if a block is given.
260
+ # @yield [chunk] stream contents of the file through the
261
+ # block. Passing the block will result in nil being returned
262
+ # from the method.
263
+ # @yieldparam [String] chunk a single chunk of the object's data
264
+ def get_file(filename, &block)
265
+ http do |h|
266
+ h.get_file(filename, &block)
267
+ end
268
+ end
269
+
270
+ # Queries a secondary index on a bucket. See Bucket#get_index
271
+ def get_index(bucket, index, query)
272
+ backend do |b|
273
+ b.get_index bucket, index, query
274
+ end
275
+ end
276
+
277
+ # Get an object. See Bucket#get
278
+ def get_object(bucket, key, options = {})
279
+ backend do |b|
280
+ b.fetch_object(bucket, key, options)
281
+ end
282
+ end
283
+
284
+ # Yields an HTTPBackend.
285
+ def http(&block)
286
+ recover_from @http_pool, &block
287
+ end
288
+
289
+ # Sets the desired HTTP backend
290
+ def http_backend=(value)
291
+ @http_backend = value
292
+ # Shut down existing connections using the old backend
293
+ @http_pool.clear
294
+ @http_backend
295
+ end
296
+
297
+ # @return [String] A representation suitable for IRB and debugging output.
298
+ def inspect
299
+ "#<Riak::Client #{nodes.inspect}>"
300
+ end
301
+
302
+ # Link-walk.
303
+ def link_walk(object, specs)
304
+ http do |h|
305
+ h.link_walk object, specs
306
+ end
307
+ end
308
+
309
+ # Retrieves a list of keys in the given bucket. See Bucket#keys
310
+ def list_keys(bucket, &block)
311
+ if block_given?
312
+ backend do |b|
313
+ b.list_keys bucket, &block
314
+ end
315
+ else
316
+ backend do |b|
317
+ b.list_keys bucket
318
+ end
319
+ end
320
+ end
321
+
322
+ # Executes a mapreduce request. See MapReduce#run
323
+ def mapred(mr, &block)
324
+ backend do |b|
325
+ b.mapred(mr, &block)
326
+ end
327
+ end
328
+
329
+ # Creates a new HTTP backend.
330
+ # @return [HTTPBackend] An HTTP backend for a given node.
331
+ def new_http_backend
332
+ klass = self.class.const_get("#{@http_backend}Backend")
333
+ if klass.configured?
334
+ node = choose_node(
335
+ @nodes.select do |n|
336
+ n.http?
337
+ end
338
+ )
339
+
340
+ klass.new(self, node)
341
+ else
342
+ raise t('http_configuration', :backend => @http_backend)
343
+ end
344
+ end
345
+
346
+ # Creates a new protocol buffers backend.
347
+ # @return [ProtobuffsBackend] the Protocol Buffers backend for
348
+ # a given node.
349
+ def new_protobuffs_backend
350
+ klass = self.class.const_get("#{@protobuffs_backend}ProtobuffsBackend")
351
+ if klass.configured?
352
+ node = choose_node(
353
+ @nodes.select do |n|
354
+ n.protobuffs?
355
+ end
356
+ )
357
+
358
+ klass.new(self, node)
359
+ else
360
+ raise t('protobuffs_configuration', :backend => @protobuffs_backend)
361
+ end
362
+ end
363
+
364
+ # @return [Node] An arbitrary Node.
365
+ def node
366
+ nodes[rand nodes.size]
367
+ end
368
+
369
+ # Pings the Riak cluster to check for liveness.
370
+ # @return [true,false] whether the Riak cluster is alive and reachable
371
+ def ping
372
+ backend do |b|
373
+ b.ping
374
+ end
375
+ end
376
+
377
+ # Yields a protocol buffers backend.
378
+ def protobuffs(&block)
379
+ recover_from @protobuffs_pool, &block
380
+ end
381
+
382
+ # Sets the desired Protocol Buffers backend
383
+ def protobuffs_backend=(value)
384
+ # Shutdown any connections using the old backend
385
+ @protobuffs_backend = value
386
+ @protobuffs_pool.clear
387
+ @protobuffs_backend
388
+ end
389
+
390
+ # Set the protocol of the Riak endpoint. Value must be in the
391
+ # Riak::Client::PROTOCOLS array.
392
+ # @raise [ArgumentError] if the protocol is not in PROTOCOLS
393
+ # @return [String] the protocol being assigned
394
+ def protocol=(value)
395
+ unless PROTOCOLS.include?(value.to_s)
396
+ raise ArgumentError, t("protocol_invalid", :invalid => value, :valid => PROTOCOLS.join(', '))
397
+ end
398
+
399
+ #TODO
400
+ @backend = nil
401
+ @protocol = value
402
+
403
+ case value
404
+ when 'https'
405
+ nodes.each do |node|
406
+ node.ssl = true unless node.ssl_enabled?
407
+ end
408
+ when 'http'
409
+ nodes.each do |node|
410
+ node.ssl = false
411
+ end
412
+ end
413
+
414
+ @protocol
415
+ end
416
+
417
+ # Takes a pool. Acquires a backend from the pool and yields it with
418
+ # node-specific error recovery.
419
+ def recover_from(pool)
420
+ skip_nodes = []
421
+ take_opts = {}
422
+ tries = 3
423
+
424
+ begin
425
+ # Only select nodes which we haven't used before.
426
+ unless skip_nodes.empty?
427
+ take_opts[:filter] = lambda do |backend|
428
+ not skip_nodes.include? backend.node
429
+ end
430
+ end
431
+
432
+ # Acquire a backend
433
+ pool.take(take_opts) do |backend|
434
+ begin
435
+ yield backend
436
+ rescue *NETWORK_ERRORS => e
437
+ # Network error.
438
+ tries -= 1
439
+
440
+ # Notify the node that a request against it failed.
441
+ backend.node.error_rate << 1
442
+
443
+ # Skip this node next time.
444
+ skip_nodes << backend.node
445
+
446
+ # And delete this connection.
447
+ raise Pool::BadResource, e
448
+ end
449
+ end
450
+ rescue Pool::BadResource => e
451
+ retry if tries > 0
452
+ raise e.message
453
+ end
454
+ end
455
+
456
+ # Reloads the object from Riak.
457
+ def reload_object(object, options = {})
458
+ backend do |b|
459
+ b.reload_object(object, options)
460
+ end
461
+ end
462
+
463
+ # Sets the properties on a bucket. See Bucket#props=
464
+ def set_bucket_props(bucket, properties)
465
+ # A bug in Beefcake is still giving us trouble with default booleans.
466
+ # Until it is resolved, we'll use the HTTP backend.
467
+ http do |b|
468
+ b.set_bucket_props(bucket, properties)
469
+ end
470
+ end
471
+
472
+ # Enables or disables SSL on all nodes, for HTTP backends.
473
+ def ssl=(value)
474
+ @nodes.each do |node|
475
+ node.ssl = value
476
+ end
477
+ value
478
+ end
479
+
480
+ # Exposes a {Stamp} object for use in generating unique
481
+ # identifiers.
482
+ # @return [Stamp] an ID generator
483
+ # @see Stamp#next
484
+ def stamp
485
+ @stamp ||= Riak::Stamp.new(self)
486
+ end
487
+
488
+ # Stores a large file/IO-like object in Riak via the "Luwak" interface.
489
+ # @overload store_file(filename, content_type, data)
490
+ # Stores the file at the given key/filename
491
+ # @param [String] filename the key/filename for the object
492
+ # @param [String] content_type the MIME Content-Type for the data
493
+ # @param [IO, String] data the contents of the file
494
+ # @overload store_file(content_type, data)
495
+ # Stores the file with a server-determined key/filename
496
+ # @param [String] content_type the MIME Content-Type for the data
497
+ # @param [String, #read] data the contents of the file
498
+ # @return [String] the key/filename where the object was stored
499
+ def store_file(*args)
500
+ http do |h|
501
+ h.store_file(*args)
502
+ end
503
+ end
504
+
505
+ # Stores an object in Riak.
506
+ def store_object(object, options = {})
507
+ params = {:returnbody => true}.merge(options)
508
+ backend do |b|
509
+ b.store_object(object, params)
510
+ end
511
+ end
512
+
513
+ private
514
+ def make_client_id
515
+ rand(MAX_CLIENT_ID)
516
+ end
517
+
518
+ def ssl_enable
519
+ @nodes.each do |n|
520
+ n.ssl_enable
521
+ end
522
+ end
523
+
524
+ def ssl_disable
525
+ @nodes.each do |n|
526
+ n.ssl_disable
527
+ end
528
+ end
529
+
530
+ # @private
531
+ class LuwakFile < DelegateClass(Tempfile)
532
+ attr_accessor :original_filename, :content_type
533
+ alias :key :original_filename
534
+ def initialize(fn)
535
+ super(Tempfile.new(fn))
536
+ @original_filename = fn
537
+ end
538
+ end
539
+ end
540
+ end