better-riak-client 1.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. data/LICENSE +16 -0
  2. data/README.markdown +198 -0
  3. data/RELEASE_NOTES.md +211 -0
  4. data/better-riak-client.gemspec +61 -0
  5. data/erl_src/riak_kv_test014_backend.beam +0 -0
  6. data/erl_src/riak_kv_test014_backend.erl +189 -0
  7. data/erl_src/riak_kv_test_backend.beam +0 -0
  8. data/erl_src/riak_kv_test_backend.erl +697 -0
  9. data/erl_src/riak_search_test_backend.beam +0 -0
  10. data/erl_src/riak_search_test_backend.erl +175 -0
  11. data/lib/riak/bucket.rb +221 -0
  12. data/lib/riak/client/beefcake/messages.rb +213 -0
  13. data/lib/riak/client/beefcake/object_methods.rb +111 -0
  14. data/lib/riak/client/beefcake_protobuffs_backend.rb +226 -0
  15. data/lib/riak/client/decaying.rb +36 -0
  16. data/lib/riak/client/excon_backend.rb +162 -0
  17. data/lib/riak/client/feature_detection.rb +88 -0
  18. data/lib/riak/client/http_backend/configuration.rb +211 -0
  19. data/lib/riak/client/http_backend/key_streamer.rb +43 -0
  20. data/lib/riak/client/http_backend/object_methods.rb +106 -0
  21. data/lib/riak/client/http_backend/request_headers.rb +34 -0
  22. data/lib/riak/client/http_backend/transport_methods.rb +201 -0
  23. data/lib/riak/client/http_backend.rb +340 -0
  24. data/lib/riak/client/net_http_backend.rb +82 -0
  25. data/lib/riak/client/node.rb +115 -0
  26. data/lib/riak/client/protobuffs_backend.rb +173 -0
  27. data/lib/riak/client/search.rb +91 -0
  28. data/lib/riak/client.rb +540 -0
  29. data/lib/riak/cluster.rb +151 -0
  30. data/lib/riak/core_ext/blank.rb +53 -0
  31. data/lib/riak/core_ext/deep_dup.rb +13 -0
  32. data/lib/riak/core_ext/extract_options.rb +7 -0
  33. data/lib/riak/core_ext/json.rb +15 -0
  34. data/lib/riak/core_ext/slice.rb +18 -0
  35. data/lib/riak/core_ext/stringify_keys.rb +10 -0
  36. data/lib/riak/core_ext/symbolize_keys.rb +10 -0
  37. data/lib/riak/core_ext/to_param.rb +31 -0
  38. data/lib/riak/core_ext.rb +7 -0
  39. data/lib/riak/encoding.rb +6 -0
  40. data/lib/riak/failed_request.rb +81 -0
  41. data/lib/riak/i18n.rb +5 -0
  42. data/lib/riak/json.rb +52 -0
  43. data/lib/riak/link.rb +94 -0
  44. data/lib/riak/locale/en.yml +53 -0
  45. data/lib/riak/locale/fr.yml +52 -0
  46. data/lib/riak/map_reduce/filter_builder.rb +103 -0
  47. data/lib/riak/map_reduce/phase.rb +98 -0
  48. data/lib/riak/map_reduce.rb +225 -0
  49. data/lib/riak/map_reduce_error.rb +7 -0
  50. data/lib/riak/node/configuration.rb +293 -0
  51. data/lib/riak/node/console.rb +133 -0
  52. data/lib/riak/node/control.rb +207 -0
  53. data/lib/riak/node/defaults.rb +83 -0
  54. data/lib/riak/node/generation.rb +106 -0
  55. data/lib/riak/node/log.rb +34 -0
  56. data/lib/riak/node/version.rb +43 -0
  57. data/lib/riak/node.rb +38 -0
  58. data/lib/riak/robject.rb +318 -0
  59. data/lib/riak/search.rb +3 -0
  60. data/lib/riak/serializers.rb +74 -0
  61. data/lib/riak/stamp.rb +77 -0
  62. data/lib/riak/test_server.rb +89 -0
  63. data/lib/riak/util/escape.rb +76 -0
  64. data/lib/riak/util/headers.rb +53 -0
  65. data/lib/riak/util/multipart/stream_parser.rb +62 -0
  66. data/lib/riak/util/multipart.rb +52 -0
  67. data/lib/riak/util/tcp_socket_extensions.rb +58 -0
  68. data/lib/riak/util/translation.rb +19 -0
  69. data/lib/riak/version.rb +3 -0
  70. data/lib/riak/walk_spec.rb +105 -0
  71. data/lib/riak.rb +21 -0
  72. metadata +348 -0
@@ -0,0 +1,82 @@
1
+
2
+ require 'riak/client/http_backend'
3
+ require 'riak/failed_request'
4
+
5
+ module Riak
6
+ class Client
7
+ # Uses the Ruby standard library Net::HTTP to connect to Riak.
8
+ # Conforms to the Riak::Client::HTTPBackend interface.
9
+ class NetHTTPBackend < HTTPBackend
10
+ def self.configured?
11
+ begin
12
+ require 'net/http'
13
+ require 'openssl'
14
+ true
15
+ rescue LoadError, NameError
16
+ false
17
+ end
18
+ end
19
+
20
+ # Sets the read_timeout applied to Net::HTTP connections
21
+ # Increase this if you have very long request times.
22
+ def self.read_timeout=(timeout)
23
+ @read_timeout = timeout
24
+ end
25
+
26
+ def self.read_timeout
27
+ @read_timeout ||= 4096
28
+ end
29
+
30
+ # Net::HTTP doesn't use persistent connections, so there's no
31
+ # work to do here.
32
+ def teardown; end
33
+
34
+ private
35
+ def perform(method, uri, headers, expect, data=nil) #:nodoc:
36
+ http = Net::HTTP.new(uri.host, uri.port)
37
+ http.read_timeout = self.class.read_timeout
38
+ configure_ssl(http) if @node.ssl_enabled?
39
+
40
+ request = Net::HTTP.const_get(method.to_s.capitalize).new(uri.request_uri, headers)
41
+ if String === data
42
+ request.body = data
43
+ elsif data.respond_to?(:read)
44
+ case
45
+ when data.respond_to?(:stat) # IO#stat
46
+ request.content_length = data.stat.size
47
+ when data.respond_to?(:size) # Some IO-like objects
48
+ request.content_length = data.size
49
+ else
50
+ request['Transfer-Encoding'] = 'chunked'
51
+ end
52
+ request.body_stream = data
53
+ end
54
+
55
+ {}.tap do |result|
56
+ http.request(request) do |response|
57
+ unless valid_response?(expect, response.code)
58
+ raise Riak::HTTPFailedRequest.new(method, expect, response.code.to_i, response.to_hash, response.body)
59
+ end
60
+
61
+ result.merge!({:headers => response.to_hash, :code => response.code.to_i})
62
+ response.read_body {|chunk| yield chunk } if block_given?
63
+ if return_body?(method, response.code, block_given?)
64
+ result[:body] = response.body
65
+ end
66
+ end
67
+ end
68
+ end
69
+
70
+ def configure_ssl(http)
71
+ http.use_ssl = true
72
+ http.verify_mode = OpenSSL::SSL.const_get("VERIFY_#{@node.ssl_options[:verify_mode].upcase}")
73
+ if @node.ssl_options[:pem]
74
+ http.cert = OpenSSL::X509::Certificate.new(@node.ssl_options[:pem])
75
+ http.key = OpenSSL::PKey::RSA.new(@node.ssl_options[:pem], @node.ssl_options[:pem_password])
76
+ end
77
+ http.ca_file = @node.ssl_options[:ca_file] if @node.ssl_options[:ca_file]
78
+ http.ca_path = @node.ssl_options[:ca_path] if @node.ssl_options[:ca_path]
79
+ end
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,115 @@
1
+ module Riak
2
+ class Client
3
+ class Node
4
+ # Represents a single riak node in a cluster.
5
+
6
+ include Util::Translation
7
+ include Util::Escape
8
+
9
+ VALID_OPTIONS = [:host, :http_port, :pb_port, :http_paths, :prefix,
10
+ :mapred, :luwak, :solr, :port, :basic_auth, :ssl_options, :ssl]
11
+
12
+ # For a score which halves in 10 seconds, choose
13
+ # ln(1/2)/10
14
+ ERRORS_DECAY_RATE = Math.log(0.5)/10
15
+
16
+ # What IP address or hostname does this node listen on?
17
+ attr_accessor :host
18
+ # Which port does the HTTP interface listen on?
19
+ attr_accessor :http_port
20
+ # Which port does the protocol buffers interface listen on?
21
+ attr_accessor :pb_port
22
+ # A hash of HTTP paths used on this node.
23
+ attr_accessor :http_paths
24
+ # A "user:password" string.
25
+ attr_reader :basic_auth
26
+ attr_accessor :ssl_options
27
+ # A Decaying rate of errors.
28
+ attr_reader :error_rate
29
+
30
+ def initialize(client, opts = {})
31
+ @client = client
32
+ @ssl = opts[:ssl]
33
+ @ssl_options = opts[:ssl_options]
34
+ @host = opts[:host] || "127.0.0.1"
35
+ @http_port = opts[:http_port] || opts[:port] || 8098
36
+ @pb_port = opts[:pb_port] || 8087
37
+ @http_paths = {
38
+ :prefix => opts[:prefix] || "/riak/",
39
+ :mapred => opts[:mapred] || "/mapred",
40
+ :luwak => opts[:luwak] || "/luwak",
41
+ :solr => opts[:solr] || "/solr" # Unused?
42
+ }.merge(opts[:http_paths] || {})
43
+ self.basic_auth = opts[:basic_auth]
44
+
45
+ @error_rate = Decaying.new
46
+ end
47
+
48
+ def ==(o)
49
+ o.kind_of? Node and
50
+ @host == o.host and
51
+ @http_port == o.http_port and
52
+ @pb_port == o.pb_port
53
+ end
54
+
55
+ # Sets the HTTP Basic Authentication credentials.
56
+ # @param [String] value an auth string in the form "user:password"
57
+ def basic_auth=(value)
58
+ case value
59
+ when nil
60
+ @basic_auth = nil
61
+ when String
62
+ raise ArgumentError, t("invalid_basic_auth") unless value.to_s.split(':').length === 2
63
+ @basic_auth = value
64
+ end
65
+ end
66
+
67
+ # Can this node be used for HTTP requests?
68
+ def http?
69
+ # TODO: Need to sort out capabilities
70
+ true
71
+ end
72
+
73
+ # Can this node be used for protocol buffers requests?
74
+ def protobuffs?
75
+ # TODO: Need to sort out capabilities
76
+ true
77
+ end
78
+
79
+ # Enables or disables SSL on this node to be utilized by the HTTP
80
+ # Backends
81
+ def ssl=(value)
82
+ @ssl_options ||= Hash === value ? value : {}
83
+ value ? ssl_enable : ssl_disable
84
+ end
85
+
86
+ # Checks if SSL is enabled for HTTP
87
+ def ssl_enabled?
88
+ @client.protocol == 'https' && @ssl_options.present?
89
+ end
90
+
91
+ def inspect
92
+ "#<Node #{@host}:#{@http_port}:#{@pb_port}>"
93
+ end
94
+
95
+ protected
96
+
97
+
98
+ def ssl_enable
99
+ @client.protocol = 'https' unless @client.protocol == 'https'
100
+ @ssl_options[:pem] = File.read(@ssl_options[:pem_file]) if @ssl_options[:pem_file]
101
+ @ssl_options[:verify_mode] ||= "peer" if @ssl_options.stringify_keys.any? {|k,v| %w[pem ca_file ca_path].include?(k)}
102
+ @ssl_options[:verify_mode] ||= "none"
103
+ raise ArgumentError.new(t('invalid_ssl_verify_mode', :invalid => @ssl_options[:verify_mode])) unless %w[none peer].include?(@ssl_options[:verify_mode])
104
+
105
+ @ssl_options
106
+ end
107
+
108
+ def ssl_disable
109
+ @client.protocol = 'http' unless @client.protocol == 'http'
110
+ @ssl_options = nil
111
+ end
112
+
113
+ end
114
+ end
115
+ end
@@ -0,0 +1,173 @@
1
+ require 'riak'
2
+ require 'socket'
3
+ require 'base64'
4
+ require 'digest/sha1'
5
+ require 'riak/util/translation'
6
+ require 'riak/client/feature_detection'
7
+
8
+ module Riak
9
+ class Client
10
+ class ProtobuffsBackend
11
+ include Util::Translation
12
+ include Util::Escape
13
+ include FeatureDetection
14
+
15
+ # Message Codes Enum
16
+ MESSAGE_CODES = %W[
17
+ ErrorResp
18
+ PingReq
19
+ PingResp
20
+ GetClientIdReq
21
+ GetClientIdResp
22
+ SetClientIdReq
23
+ SetClientIdResp
24
+ GetServerInfoReq
25
+ GetServerInfoResp
26
+ GetReq
27
+ GetResp
28
+ PutReq
29
+ PutResp
30
+ DelReq
31
+ DelResp
32
+ ListBucketsReq
33
+ ListBucketsResp
34
+ ListKeysReq
35
+ ListKeysResp
36
+ GetBucketReq
37
+ GetBucketResp
38
+ SetBucketReq
39
+ SetBucketResp
40
+ MapRedReq
41
+ MapRedResp
42
+ IndexReq
43
+ IndexResp
44
+ SearchQueryReq
45
+ SearchQueryResp
46
+ ].map {|s| s.intern }.freeze
47
+
48
+ def self.simple(method, code)
49
+ define_method method do
50
+ socket.write([1, MESSAGE_CODES.index(code)].pack('NC'))
51
+ decode_response
52
+ end
53
+ end
54
+
55
+ attr_accessor :client
56
+ attr_accessor :node
57
+ def initialize(client, node)
58
+ @client = client
59
+ @node = node
60
+ end
61
+
62
+ simple :ping, :PingReq
63
+ simple :get_client_id, :GetClientIdReq
64
+ simple :server_info, :GetServerInfoReq
65
+ simple :list_buckets, :ListBucketsReq
66
+
67
+ # Performs a secondary-index query via emulation through MapReduce.
68
+ # @param [String, Bucket] bucket the bucket to query
69
+ # @param [String] index the index to query
70
+ # @param [String, Integer, Range] query the equality query or
71
+ # range query to perform
72
+ # @return [Array<String>] a list of keys matching the query
73
+ def get_index(bucket, index, query)
74
+ mr = Riak::MapReduce.new(client).index(bucket, index, query)
75
+ unless mapred_phaseless?
76
+ mr.reduce(%w[riak_kv_mapreduce reduce_identity], :arg => {:reduce_phase_only_1 => true}, :keep => true)
77
+ end
78
+ mapred(mr).map {|p| p.last }
79
+ end
80
+
81
+ # Performs search query via emulation through MapReduce. This
82
+ # has more limited capabilites than native queries. Essentially,
83
+ # only the 'id' field of matched documents will ever be
84
+ # returned, the 'fl' and other options have no effect.
85
+ # @param [String] index the index to query
86
+ # @param [String] query the Lucene-style search query
87
+ # @param [Hash] options ignored in MapReduce emulation
88
+ # @return [Hash] the search results
89
+ def search(index, query, options={})
90
+ mr = Riak::MapReduce.new(client).search(index || 'search', query)
91
+ unless mapred_phaseless?
92
+ mr.reduce(%w[riak_kv_mapreduce reduce_identity], :arg => {:reduce_phase_only_1 => true}, :keep => true)
93
+ end
94
+ docs = mapred(mr).map {|d| {'id' => d[1] } }
95
+ # Since we don't get this information back from the MapReduce,
96
+ # we have to fake the max_score and num_found.
97
+ { 'docs' => docs,
98
+ 'num_found' => docs.size,
99
+ 'max_score' => 0.0 }
100
+ end
101
+
102
+ # Gracefully shuts down this connection.
103
+ def teardown
104
+ reset_socket
105
+ end
106
+
107
+ private
108
+ def get_server_version
109
+ server_info[:server_version]
110
+ end
111
+
112
+ # Implemented by subclasses
113
+ def decode_response
114
+ raise NotImplementedError
115
+ end
116
+
117
+ def socket
118
+ @socket ||= new_socket
119
+ end
120
+
121
+ def new_socket
122
+ socket = TCPSocket.new(@node.host, @node.pb_port)
123
+ socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, true)
124
+ #TODO: Should we set the client ID here?
125
+ # set_client_id @client.client_id
126
+ socket
127
+ end
128
+
129
+ def reset_socket
130
+ reset_server_version
131
+ @socket.close if @socket && !@socket.closed?
132
+ @socket = nil
133
+ end
134
+
135
+ UINTMAX = 0xffffffff
136
+ QUORUMS = {
137
+ "one" => UINTMAX - 1,
138
+ "quorum" => UINTMAX - 2,
139
+ "all" => UINTMAX - 3,
140
+ "default" => UINTMAX - 4
141
+ }.freeze
142
+
143
+ def prune_unsupported_options(req,options={})
144
+ unless quorum_controls?
145
+ [:notfound_ok, :basic_quorum, :pr, :pw].each {|k| options.delete k }
146
+ end
147
+ unless pb_head?
148
+ [:head, :return_head].each {|k| options.delete k }
149
+ end
150
+ unless tombstone_vclocks?
151
+ options.delete :deletedvclock
152
+ options.delete :vclock if req == :DelReq
153
+ end
154
+ unless pb_conditionals?
155
+ [:if_not_modified, :if_none_match, :if_modified].each {|k| options.delete k }
156
+ end
157
+ options
158
+ end
159
+
160
+ def normalize_quorums(options={})
161
+ options.dup.tap do |o|
162
+ [:r, :pr, :w, :pw, :dw, :rw].each do |k|
163
+ o[k] = normalize_quorum_value(o[k]) if o[k]
164
+ end
165
+ end
166
+ end
167
+
168
+ def normalize_quorum_value(q)
169
+ QUORUMS[q.to_s] || q.to_i
170
+ end
171
+ end
172
+ end
173
+ end
@@ -0,0 +1,91 @@
1
+ require 'builder'
2
+
3
+ module Riak
4
+ class Client
5
+ # (Riak Search) Performs a search via the Solr interface.
6
+ # @overload search(index, query, options={})
7
+ # @param [String] index the index to query on
8
+ # @param [String] query a Lucene query string
9
+ # @overload search(query, options={})
10
+ # Queries the default index
11
+ # @param [String] query a Lucene query string
12
+ # @param [Hash] options extra options for the Solr query
13
+ # @option options [String] :df the default field to search in
14
+ # @option options [String] :'q.op' the default operator between terms ("or", "and")
15
+ # @option options [String] :wt ("json") the response type - "json" and "xml" are valid
16
+ # @option options [String] :sort ('none') the field and direction to sort, e.g. "name asc"
17
+ # @option options [Fixnum] :start (0) the offset into the query to start from, e.g. for pagination
18
+ # @option options [Fixnum] :rows (10) the number of results to return
19
+ # @return [Hash] the query result, containing the 'responseHeaders' and 'response' keys
20
+ def search(*args)
21
+ options = args.extract_options!
22
+ index, query = args[-2], args[-1] # Allows nil index, while keeping it as firstargument
23
+ backend do |b|
24
+ b.search(index, query, options)
25
+ end
26
+ end
27
+ alias :select :search
28
+
29
+ # (Riak Search) Adds documents to a search index via the Solr interface.
30
+ # @overload index(index, *docs)
31
+ # Adds documents to the specified search index
32
+ # @param [String] index the index in which to add/update the given documents
33
+ # @param [Array<Hash>] docs unnested document hashes, with one key per field
34
+ # @overload index(*docs)
35
+ # Adds documents to the default search index
36
+ # @param [Array<Hash>] docs unnested document hashes, with one key per field
37
+ # @raise [ArgumentError] if any documents don't include 'id' key
38
+ def index(*args)
39
+ index = args.shift if String === args.first # Documents must be hashes of fields
40
+ raise ArgumentError.new(t("search_docs_require_id")) unless args.all? {|d| d.key?("id") || d.key?(:id) }
41
+ xml = Builder::XmlMarkup.new
42
+ xml.add do
43
+ args.each do |doc|
44
+ xml.doc do
45
+ doc.each do |k,v|
46
+ xml.field('name' => k.to_s) { xml.text!(v.to_s) }
47
+ end
48
+ end
49
+ end
50
+ end
51
+ http do |h|
52
+ h.update_search_index(index, xml.target!)
53
+ end
54
+ true
55
+ end
56
+ alias :add_doc :index
57
+
58
+ # (Riak Search) Removes documents from a search index via the Solr interface.
59
+ # @overload remove(index, specs)
60
+ # Removes documents from the specified index
61
+ # @param [String] index the index from which to remove documents
62
+ # @param [Array<Hash>] specs the specificaiton of documents to remove (must contain 'id' or 'query' keys)
63
+ # @overload remove(specs)
64
+ # Removes documents from the default index
65
+ # @param [Array<Hash>] specs the specification of documents to remove (must contain 'id' or 'query' keys)
66
+ # @raise [ArgumentError] if any document specs don't include 'id' or 'query' keys
67
+ def remove(*args)
68
+ index = args.shift if String === args.first
69
+ raise ArgumentError.new(t("search_remove_requires_id_or_query")) unless args.all? { |s|
70
+ s.include? :id or
71
+ s.include? 'id' or
72
+ s.include? :query or
73
+ s.include? 'query'
74
+ }
75
+ xml = Builder::XmlMarkup.new
76
+ xml.delete do
77
+ args.each do |spec|
78
+ spec.each do |k,v|
79
+ xml.tag!(k.to_sym, v)
80
+ end
81
+ end
82
+ end
83
+ http do |h|
84
+ h.update_search_index(index, xml.target!)
85
+ end
86
+ true
87
+ end
88
+ alias :delete_doc :remove
89
+ alias :deindex :remove
90
+ end
91
+ end