dalli 2.7.8 → 3.2.2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of dalli might be problematic. Click here for more details.

Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +5 -1
  3. data/History.md +159 -0
  4. data/README.md +27 -223
  5. data/lib/dalli/cas/client.rb +1 -57
  6. data/lib/dalli/client.rb +227 -254
  7. data/lib/dalli/compressor.rb +12 -2
  8. data/lib/dalli/key_manager.rb +113 -0
  9. data/lib/dalli/options.rb +6 -7
  10. data/lib/dalli/pipelined_getter.rb +177 -0
  11. data/lib/dalli/protocol/base.rb +241 -0
  12. data/lib/dalli/protocol/binary/request_formatter.rb +117 -0
  13. data/lib/dalli/protocol/binary/response_header.rb +36 -0
  14. data/lib/dalli/protocol/binary/response_processor.rb +239 -0
  15. data/lib/dalli/protocol/binary/sasl_authentication.rb +60 -0
  16. data/lib/dalli/protocol/binary.rb +173 -0
  17. data/lib/dalli/protocol/connection_manager.rb +252 -0
  18. data/lib/dalli/protocol/meta/key_regularizer.rb +31 -0
  19. data/lib/dalli/protocol/meta/request_formatter.rb +108 -0
  20. data/lib/dalli/protocol/meta/response_processor.rb +211 -0
  21. data/lib/dalli/protocol/meta.rb +177 -0
  22. data/lib/dalli/protocol/response_buffer.rb +54 -0
  23. data/lib/dalli/protocol/server_config_parser.rb +84 -0
  24. data/lib/dalli/protocol/ttl_sanitizer.rb +45 -0
  25. data/lib/dalli/protocol/value_compressor.rb +85 -0
  26. data/lib/dalli/protocol/value_marshaller.rb +59 -0
  27. data/lib/dalli/protocol/value_serializer.rb +91 -0
  28. data/lib/dalli/protocol.rb +8 -0
  29. data/lib/dalli/ring.rb +94 -83
  30. data/lib/dalli/server.rb +3 -746
  31. data/lib/dalli/servers_arg_normalizer.rb +54 -0
  32. data/lib/dalli/socket.rb +117 -137
  33. data/lib/dalli/version.rb +4 -1
  34. data/lib/dalli.rb +43 -15
  35. data/lib/rack/session/dalli.rb +103 -94
  36. metadata +64 -27
  37. data/lib/action_dispatch/middleware/session/dalli_store.rb +0 -82
  38. data/lib/active_support/cache/dalli_store.rb +0 -429
  39. data/lib/dalli/railtie.rb +0 -8
data/lib/dalli/ring.rb CHANGED
@@ -1,60 +1,103 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  require 'digest/sha1'
3
4
  require 'zlib'
4
5
 
5
6
  module Dalli
7
+ ##
8
+ # An implementation of a consistent hash ring, designed to minimize
9
+ # the cache miss impact of adding or removing servers from the ring.
10
+ # That is, adding or removing a server from the ring should impact
11
+ # the key -> server mapping of ~ 1/N of the stored keys where N is the
12
+ # number of servers in the ring. This is done by creating a large
13
+ # number of "points" per server, distributed over the space
14
+ # 0x00000000 - 0xFFFFFFFF. For a given key, we calculate the CRC32
15
+ # hash, and find the nearest "point" that is less than or equal to the
16
+ # the key's hash. In this implemetation, each "point" is represented
17
+ # by a Dalli::Ring::Entry.
18
+ ##
6
19
  class Ring
20
+ # The number of entries on the continuum created per server
21
+ # in an equally weighted scenario.
7
22
  POINTS_PER_SERVER = 160 # this is the default in libmemcached
8
23
 
9
24
  attr_accessor :servers, :continuum
10
25
 
11
- def initialize(servers, options)
12
- @servers = servers
13
- @continuum = nil
14
- if servers.size > 1
15
- total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
16
- continuum = []
17
- servers.each do |server|
18
- entry_count_for(server, servers.size, total_weight).times do |idx|
19
- hash = Digest::SHA1.hexdigest("#{server.name}:#{idx}")
20
- value = Integer("0x#{hash[0..7]}")
21
- continuum << Dalli::Ring::Entry.new(value, server)
22
- end
23
- end
24
- @continuum = continuum.sort_by(&:value)
26
+ def initialize(servers_arg, protocol_implementation, options)
27
+ @servers = servers_arg.map do |s|
28
+ protocol_implementation.new(s, options)
25
29
  end
30
+ @continuum = nil
31
+ @continuum = build_continuum(servers) if servers.size > 1
26
32
 
27
33
  threadsafe! unless options[:threadsafe] == false
28
34
  @failover = options[:failover] != false
29
35
  end
30
36
 
31
37
  def server_for_key(key)
32
- if @continuum
33
- hkey = hash_for(key)
34
- 20.times do |try|
35
- entryidx = binary_search(@continuum, hkey)
36
- server = @continuum[entryidx].server
37
- return server if server.alive?
38
- break unless @failover
39
- hkey = hash_for("#{try}#{key}")
40
- end
41
- else
42
- server = @servers.first
43
- return server if server && server.alive?
38
+ server = if @continuum
39
+ server_from_continuum(key)
40
+ else
41
+ @servers.first
42
+ end
43
+
44
+ # Note that the call to alive? has the side effect of initializing
45
+ # the socket
46
+ return server if server&.alive?
47
+
48
+ raise Dalli::RingError, 'No server available'
49
+ end
50
+
51
+ def server_from_continuum(key)
52
+ hkey = hash_for(key)
53
+ 20.times do |try|
54
+ server = server_for_hash_key(hkey)
55
+
56
+ # Note that the call to alive? has the side effect of initializing
57
+ # the socket
58
+ return server if server.alive?
59
+ break unless @failover
60
+
61
+ hkey = hash_for("#{try}#{key}")
44
62
  end
63
+ nil
64
+ end
45
65
 
46
- raise Dalli::RingError, "No server available"
66
+ def keys_grouped_by_server(key_arr)
67
+ key_arr.group_by do |key|
68
+ server_for_key(key)
69
+ rescue Dalli::RingError
70
+ Dalli.logger.debug { "unable to get key #{key}" }
71
+ nil
72
+ end
47
73
  end
48
74
 
49
75
  def lock
50
76
  @servers.each(&:lock!)
51
77
  begin
52
- return yield
78
+ yield
53
79
  ensure
54
80
  @servers.each(&:unlock!)
55
81
  end
56
82
  end
57
83
 
84
+ def pipeline_consume_and_ignore_responses
85
+ @servers.each do |s|
86
+ s.request(:noop)
87
+ rescue Dalli::NetworkError
88
+ # Ignore this error, as it indicates the socket is unavailable
89
+ # and there's no need to flush
90
+ end
91
+ end
92
+
93
+ def socket_timeout
94
+ @servers.first.socket_timeout
95
+ end
96
+
97
+ def close
98
+ @servers.each(&:close)
99
+ end
100
+
58
101
  private
59
102
 
60
103
  def threadsafe!
@@ -71,72 +114,40 @@ module Dalli
71
114
  ((total_servers * POINTS_PER_SERVER * server.weight) / Float(total_weight)).floor
72
115
  end
73
116
 
74
- # Native extension to perform the binary search within the continuum
75
- # space. Fallback to a pure Ruby version if the compilation doesn't work.
76
- # optional for performance and only necessary if you are using multiple
77
- # memcached servers.
78
- begin
79
- require 'inline'
80
- inline do |builder|
81
- builder.c <<-EOM
82
- int binary_search(VALUE ary, unsigned int r) {
83
- long upper = RARRAY_LEN(ary) - 1;
84
- long lower = 0;
85
- long idx = 0;
86
- ID value = rb_intern("value");
87
- VALUE continuumValue;
88
- unsigned int l;
89
-
90
- while (lower <= upper) {
91
- idx = (lower + upper) / 2;
92
-
93
- continuumValue = rb_funcall(RARRAY_PTR(ary)[idx], value, 0);
94
- l = NUM2UINT(continuumValue);
95
- if (l == r) {
96
- return idx;
97
- }
98
- else if (l > r) {
99
- upper = idx - 1;
100
- }
101
- else {
102
- lower = idx + 1;
103
- }
104
- }
105
- return upper;
106
- }
107
- EOM
108
- end
109
- rescue LoadError
117
+ def server_for_hash_key(hash_key)
110
118
  # Find the closest index in the Ring with value <= the given value
111
- def binary_search(ary, value)
112
- upper = ary.size - 1
113
- lower = 0
114
-
115
- while (lower <= upper) do
116
- idx = (lower + upper) / 2
117
- comp = ary[idx].value <=> value
118
-
119
- if comp == 0
120
- return idx
121
- elsif comp > 0
122
- upper = idx - 1
123
- else
124
- lower = idx + 1
125
- end
119
+ entryidx = @continuum.bsearch_index { |entry| entry.value > hash_key }
120
+ if entryidx.nil?
121
+ entryidx = @continuum.size - 1
122
+ else
123
+ entryidx -= 1
124
+ end
125
+ @continuum[entryidx].server
126
+ end
127
+
128
+ def build_continuum(servers)
129
+ continuum = []
130
+ total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
131
+ servers.each do |server|
132
+ entry_count_for(server, servers.size, total_weight).times do |idx|
133
+ hash = Digest::SHA1.hexdigest("#{server.name}:#{idx}")
134
+ value = Integer("0x#{hash[0..7]}")
135
+ continuum << Dalli::Ring::Entry.new(value, server)
126
136
  end
127
- upper
128
137
  end
138
+ continuum.sort_by(&:value)
129
139
  end
130
140
 
141
+ ##
142
+ # Represents a point in the consistent hash ring implementation.
143
+ ##
131
144
  class Entry
132
- attr_reader :value
133
- attr_reader :server
145
+ attr_reader :value, :server
134
146
 
135
147
  def initialize(val, srv)
136
148
  @value = val
137
149
  @server = srv
138
150
  end
139
151
  end
140
-
141
152
  end
142
153
  end