zk_recipes 0.1.0 → 0.2.0.pre1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +12 -2
- data/CHANGELOG.md +21 -0
- data/README.md +95 -7
- data/lib/zk_recipes/cache.rb +51 -36
- data/lib/zk_recipes/version.rb +1 -1
- data/lib/zk_recipes.rb +0 -1
- data/zk_recipes.gemspec +1 -1
- metadata +5 -6
- data/bin/console +0 -14
- data/bin/setup +0 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7750ecfcae437bf16df6dfbd4d23b769907f3209
|
4
|
+
data.tar.gz: ed8c8aef2d4cbe07cb669dd461230441c2537c8f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0c990d65e803782d5465e7bca85a877b3070dbff66086e70dbf6d7d6cbdf5fb6f356ee776f1d714d00d0ea07bc028a9c8f1a2ea54331090e46ea40f868101277
|
7
|
+
data.tar.gz: f25b27483c6a0d9f354694452d8304b20b54e928d9ad435904e829dbcd21907113d3122049a14c1f3266de5f048cbfec86969047bd8f8894d96ba266f1cc145f
|
data/.travis.yml
CHANGED
@@ -1,5 +1,15 @@
|
|
1
|
-
sudo: false
|
2
1
|
language: ruby
|
2
|
+
before_install:
|
3
|
+
- sudo apt-get -qq update
|
4
|
+
- sudo apt-get install -y socat pv
|
5
|
+
- gem install bundler
|
3
6
|
rvm:
|
7
|
+
- 2.3.4
|
4
8
|
- 2.4.1
|
5
|
-
|
9
|
+
- jruby-9.1.12.0
|
10
|
+
- ruby-head
|
11
|
+
matrix:
|
12
|
+
allow_failures:
|
13
|
+
- rvm: ruby-head
|
14
|
+
fast_finish: true
|
15
|
+
sudo: required
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
# Changelog
|
2
|
+
|
3
|
+
## 0.2.0
|
4
|
+
|
5
|
+
BREAKING CHANGES
|
6
|
+
|
7
|
+
- Make `on_connected` lighter: `on_connected` gets called for every watch when
|
8
|
+
a connection flaps. Make the happy path `on_connected` faster.
|
9
|
+
- Add `ZkRecipes::Cache#reopen` for resetting the cache after a `fork`
|
10
|
+
- BREAKING CHANGE: Don't use `KeyError` as part of the API; use
|
11
|
+
`ZkRecipes::Cache::PathError` instead.
|
12
|
+
- BREAKING CHANGE: Overhaul `ActiveSupport::Notifications`.
|
13
|
+
- Only one notifation now: `cache.zk_recipes`. Removed
|
14
|
+
`zk_recipes.cache.update` and `zk_recipes.cache.error`.
|
15
|
+
- All notifications have a `path`.
|
16
|
+
- Don't use notifications for unhandled exceptions.
|
17
|
+
- Development only: use `ZK_RECIPES_DEBUG=1 rspec` for debug logging.
|
18
|
+
|
19
|
+
## 0.1.0
|
20
|
+
|
21
|
+
- Initial release
|
data/README.md
CHANGED
@@ -18,7 +18,7 @@ Or install it yourself as:
|
|
18
18
|
|
19
19
|
## Usage
|
20
20
|
|
21
|
-
A cache that creates its own ZK client:
|
21
|
+
### A cache that creates its own ZK client:
|
22
22
|
|
23
23
|
```ruby
|
24
24
|
logger = Logger.new(STDERR)
|
@@ -31,20 +31,70 @@ puts cache["/test/boom"]
|
|
31
31
|
cache.close!
|
32
32
|
```
|
33
33
|
|
34
|
-
A cache that uses an existing ZK client:
|
34
|
+
### A cache that uses an existing ZK client:
|
35
35
|
|
36
36
|
```ruby
|
37
37
|
logger = Logger.new(STDERR)
|
38
|
-
zk = ZK.new("my-host:1234", connect: false
|
38
|
+
zk = ZK.new("my-host:1234", connect: false)
|
39
|
+
cache = ZkRecipes::Cache.new(logger: logger)
|
40
|
+
cache.register("/test/boom", "goat") { |string| "Hello, #{string}" }
|
41
|
+
cache.setup_callbacks(zk) # no more paths can be registered after this
|
42
|
+
|
43
|
+
puts cache["/test/boom"] # => "Hello, goat"
|
44
|
+
|
45
|
+
zk.connect
|
46
|
+
cache.wait_for_warm_cache(10) # wait up to 10s for the cache to warm
|
47
|
+
|
48
|
+
zk.create("/test/boom")
|
49
|
+
zk.set("/test/boom", "cat")
|
50
|
+
|
51
|
+
sleep 1
|
52
|
+
|
53
|
+
puts cache["/test/boom"] # => "Hello, cat"
|
54
|
+
cache.close!
|
55
|
+
zk.close!
|
56
|
+
```
|
57
|
+
|
58
|
+
### Handling forks with a cache that creates it's own ZK client:
|
59
|
+
|
60
|
+
```ruby
|
61
|
+
zk = ZK.new("my-host:1234") # zk client for writing only
|
62
|
+
|
63
|
+
logger = Logger.new(STDERR)
|
64
|
+
cache = ZkRecipes::Cache.new(host: "my-host:1234", logger: logger, timeout: 10) do |z|
|
65
|
+
z.register("/test/boom", "goat")
|
66
|
+
end
|
67
|
+
|
68
|
+
|
69
|
+
puts cache["/test/boom"] # => "goat"
|
70
|
+
|
71
|
+
if fork
|
72
|
+
# parent
|
73
|
+
zk.set("/test/boom", "mouse")
|
74
|
+
cache.close!
|
75
|
+
zk.close!
|
76
|
+
else
|
77
|
+
# child
|
78
|
+
cache.reopen # wait up to 10s for ZK to reconnect and the cache to warm
|
79
|
+
puts cache["/test/boom"] # => "mouse"
|
80
|
+
cache.close!
|
81
|
+
zk.close!
|
82
|
+
end
|
83
|
+
```
|
84
|
+
|
85
|
+
### Handling forks with an existing ZK client:
|
86
|
+
|
87
|
+
```ruby
|
88
|
+
logger = Logger.new(STDERR)
|
89
|
+
zk = ZK.new("my-host:1234", connect: false)
|
39
90
|
cache = ZkRecipes::Cache.new(logger: logger)
|
40
91
|
cache.register("/test/boom", "goat")
|
41
|
-
cache.register("/test/foo", 1) { |raw_value| raw_value.to_i * 2 }
|
42
92
|
cache.setup_callbacks(zk) # no more paths can be registered after this
|
43
93
|
|
44
94
|
puts cache["/test/boom"] # => "goat"
|
45
95
|
|
46
96
|
zk.connect
|
47
|
-
cache.wait_for_warm_cache(10) # wait 10s for the cache to warm
|
97
|
+
cache.wait_for_warm_cache(10) # wait up to 10s for the cache to warm
|
48
98
|
|
49
99
|
zk.create("/test/boom")
|
50
100
|
zk.set("/test/boom", "cat")
|
@@ -52,8 +102,46 @@ zk.set("/test/boom", "cat")
|
|
52
102
|
sleep 1
|
53
103
|
|
54
104
|
puts cache["/test/boom"] # => "cat"
|
55
|
-
|
56
|
-
|
105
|
+
|
106
|
+
if fork
|
107
|
+
# parent
|
108
|
+
zk.set("/test/boom", "mouse")
|
109
|
+
cache.close!
|
110
|
+
zk.close!
|
111
|
+
else
|
112
|
+
# child
|
113
|
+
cache.reopen
|
114
|
+
zk.reopen
|
115
|
+
cache.wait_for_warm_cache(10) # wait up to 10s for the cache to warm again
|
116
|
+
puts cache["/test/boom"] # => "mouse"
|
117
|
+
cache.close!
|
118
|
+
zk.close!
|
119
|
+
end
|
120
|
+
```
|
121
|
+
|
122
|
+
### ActiveSupport::Notifications
|
123
|
+
|
124
|
+
`ZkRecipes` use `ActiveSupport::Notifications` for callbacks.
|
125
|
+
|
126
|
+
WARNING: exceptions raised in `ActiveSupport::Notifications.subscribe` will
|
127
|
+
bubble up to the line where the notification is instrumented. MAKE SURE YOU
|
128
|
+
CATCH EXCEPTIONS IN YOUR SUBSCRIBE BLOCKS!
|
129
|
+
|
130
|
+
Example:
|
131
|
+
|
132
|
+
```ruby
|
133
|
+
ActiveSupport::Notifications.subscribe("cache.zk_recipes") do |_name, _start, _finish, _id, payload|
|
134
|
+
if payload[:error]
|
135
|
+
puts "There was a ZkRecipes::Cache error #{payload[:error]}"
|
136
|
+
puts "The error occurred trying to deserialize a value from ZK "\
|
137
|
+
"path=#{payload[:path] raw_zk_value=#{payload[:raw_value]}"
|
138
|
+
else
|
139
|
+
puts "Received and update for path=#{payload[:path}."
|
140
|
+
puts "The update in ZK happened #{payload[:latency_seconds]} seconds ago."
|
141
|
+
puts "The path's data is #{payload[:data_length]} bytes."
|
142
|
+
puts "The path's version is #{payload[:version]}."
|
143
|
+
end
|
144
|
+
end
|
57
145
|
```
|
58
146
|
|
59
147
|
## Development
|
data/lib/zk_recipes/cache.rb
CHANGED
@@ -3,17 +3,15 @@
|
|
3
3
|
module ZkRecipes
|
4
4
|
class Cache
|
5
5
|
class Error < StandardError; end
|
6
|
+
class PathError < Error; end
|
6
7
|
|
7
|
-
|
8
|
+
AS_NOTIFICATION = "cache.zk_recipes"
|
8
9
|
|
9
|
-
|
10
|
-
AS_NOTIFICATION_ERROR = "zk_recipes.cache.error"
|
11
|
-
|
12
|
-
def initialize(host: nil, logger: nil, timeout: nil, zk_opts: {})
|
10
|
+
def initialize(logger: nil, host: nil, timeout: nil, zk_opts: {})
|
13
11
|
@cache = Concurrent::Map.new
|
14
12
|
@latch = Concurrent::CountDownLatch.new
|
15
13
|
@logger = logger
|
16
|
-
@pending_updates = Concurrent::Hash.new
|
14
|
+
@pending_updates = Concurrent::Hash.new # Concurrent::Map does not implement #reject!
|
17
15
|
@registerable = true
|
18
16
|
@registered_values = Concurrent::Map.new
|
19
17
|
@session_id = nil
|
@@ -22,9 +20,10 @@ module ZkRecipes
|
|
22
20
|
|
23
21
|
if block_given?
|
24
22
|
@owned_zk = true
|
23
|
+
@warm_cache_timeout = timeout || 30
|
25
24
|
yield(self)
|
26
25
|
|
27
|
-
expiration = Time.now +
|
26
|
+
expiration = Time.now + @warm_cache_timeout
|
28
27
|
connect(host, zk_opts)
|
29
28
|
|
30
29
|
wait_for_warm_cache(expiration - Time.now)
|
@@ -41,10 +40,11 @@ module ZkRecipes
|
|
41
40
|
debug("added path=#{path} default_value=#{default_value.inspect}")
|
42
41
|
@cache[path] = CachedPath.new(default_value)
|
43
42
|
@registered_values[path] = RegisteredPath.new(default_value, block)
|
44
|
-
ActiveSupport::Notifications.instrument(
|
43
|
+
ActiveSupport::Notifications.instrument(AS_NOTIFICATION, path: path, value: default_value)
|
45
44
|
end
|
46
45
|
|
47
46
|
def setup_callbacks(zk)
|
47
|
+
raise Error, "setup_callbacks can only be called once" unless @registerable
|
48
48
|
@zk = zk
|
49
49
|
@registerable = false
|
50
50
|
|
@@ -52,47 +52,46 @@ module ZkRecipes
|
|
52
52
|
raise Error, "the ZK::Client is already connected, the cached values must be set before connecting"
|
53
53
|
end
|
54
54
|
|
55
|
-
@
|
56
|
-
|
55
|
+
@registered_values.each do |path, _value|
|
56
|
+
@watches[path] = @zk.register(path) do |event|
|
57
|
+
if event.node_event?
|
58
|
+
debug("node event=#{event.inspect} #{event.event_name} #{event.state_name}")
|
59
|
+
unless update_cache(event.path)
|
60
|
+
@pending_updates[path] = nil
|
61
|
+
@zk.defer { process_pending_updates }
|
62
|
+
end
|
63
|
+
else
|
64
|
+
warn("session event=#{event.inspect}")
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
@watches["on_connected"] = @zk.on_connected do
|
57
70
|
if @session_id == @zk.session_id
|
58
71
|
process_pending_updates
|
59
72
|
next
|
60
73
|
end
|
61
74
|
|
75
|
+
debug("on_connected new session")
|
62
76
|
@pending_updates.clear
|
63
77
|
@registered_values.each do |path, _value|
|
64
|
-
@watches[path] ||= @zk.register(path) do |event|
|
65
|
-
if event.node_event?
|
66
|
-
debug("node event=#{event.inspect} #{event.event_name} #{event.state_name}")
|
67
|
-
unless update_cache(event.path)
|
68
|
-
@pending_updates[path] = nil
|
69
|
-
@zk.defer { process_pending_updates }
|
70
|
-
end
|
71
|
-
else
|
72
|
-
warn("session event=#{event.inspect}")
|
73
|
-
end
|
74
|
-
end
|
75
78
|
@pending_updates[path] = nil unless update_cache(path)
|
76
79
|
end
|
77
80
|
@session_id = @zk.session_id
|
78
81
|
@latch.count_down
|
79
82
|
end
|
80
83
|
|
81
|
-
@zk.on_expired_session do |e|
|
82
|
-
info("on_expired_session #{e.event_name} #{e.state_name}")
|
83
|
-
end
|
84
|
-
|
85
84
|
@zk.on_exception do |e|
|
86
85
|
error("on_exception exception=#{e.inspect} backtrace=#{e.backtrace.inspect}")
|
87
|
-
ActiveSupport::Notifications.instrument(AS_NOTIFICATION_ERROR, error: e)
|
88
86
|
end
|
89
87
|
end
|
90
88
|
|
91
89
|
def wait_for_warm_cache(timeout = 30)
|
90
|
+
debug("waiting for cache to warm timeout=#{timeout}")
|
92
91
|
if @latch.wait(timeout)
|
93
92
|
true
|
94
93
|
else
|
95
|
-
warn("didn't
|
94
|
+
warn("didn't warm cache before timeout connected=#{@zk.connected?} timeout=#{timeout}")
|
96
95
|
false
|
97
96
|
end
|
98
97
|
end
|
@@ -101,16 +100,34 @@ module ZkRecipes
|
|
101
100
|
@watches.each_value(&:unsubscribe)
|
102
101
|
@watches.clear
|
103
102
|
@zk.close! if @owned_zk
|
103
|
+
@zk = nil
|
104
|
+
@cache = nil
|
105
|
+
@registered_values = nil
|
106
|
+
end
|
107
|
+
|
108
|
+
def reopen
|
109
|
+
@latch = Concurrent::CountDownLatch.new
|
110
|
+
@session_id = nil
|
111
|
+
@pending_updates.clear
|
112
|
+
if @owned_zk
|
113
|
+
expiration = Time.now + @warm_cache_timeout
|
114
|
+
@zk.reopen
|
115
|
+
wait_for_warm_cache(expiration - Time.now)
|
116
|
+
end
|
104
117
|
end
|
105
118
|
|
106
119
|
def fetch(path)
|
107
120
|
@cache.fetch(path).value
|
121
|
+
rescue KeyError
|
122
|
+
raise PathError, "no registered path for #{path.inspect}"
|
108
123
|
end
|
109
124
|
alias_method :[], :fetch
|
110
125
|
|
111
126
|
def fetch_existing(path)
|
112
127
|
cached = @cache.fetch(path)
|
113
128
|
cached.value if cached.stat&.exists?
|
129
|
+
rescue KeyError
|
130
|
+
raise PathError, "no registered path=#{path.inspect}"
|
114
131
|
end
|
115
132
|
|
116
133
|
private
|
@@ -130,7 +147,6 @@ module ZkRecipes
|
|
130
147
|
|
131
148
|
stat = @zk.stat(path, watch: true)
|
132
149
|
|
133
|
-
instrument_name = AS_NOTIFICATION_UPDATE
|
134
150
|
instrument_params = { path: path }
|
135
151
|
|
136
152
|
unless stat.exists?
|
@@ -138,17 +154,15 @@ module ZkRecipes
|
|
138
154
|
@cache[path] = CachedPath.new(value, stat)
|
139
155
|
debug("no node, setting watch path=#{path}")
|
140
156
|
instrument_params[:value] = value
|
141
|
-
ActiveSupport::Notifications.instrument(
|
157
|
+
ActiveSupport::Notifications.instrument(AS_NOTIFICATION, instrument_params)
|
142
158
|
return true
|
143
159
|
end
|
144
160
|
|
145
161
|
raw_value, stat = @zk.get(path, watch: true)
|
146
162
|
|
147
|
-
instrument_params.
|
148
|
-
|
149
|
-
|
150
|
-
data_length: stat.data_length,
|
151
|
-
)
|
163
|
+
instrument_params[:latency_seconds] = Time.now - stat.mtime_t
|
164
|
+
instrument_params[:version] = stat.version
|
165
|
+
instrument_params[:data_length] = stat.data_length
|
152
166
|
|
153
167
|
value = begin
|
154
168
|
registered_value = @registered_values.fetch(path)
|
@@ -158,19 +172,19 @@ module ZkRecipes
|
|
158
172
|
"deserialization error raw_zookeeper_value=#{raw_value.inspect} zookeeper_stat=#{stat.inspect} "\
|
159
173
|
"exception=#{e.inspect} #{e.backtrace.inspect}"
|
160
174
|
)
|
161
|
-
instrument_name = AS_NOTIFICATION_ERROR
|
162
175
|
instrument_params[:error] = e
|
163
176
|
instrument_params[:raw_value] = raw_value
|
164
177
|
registered_value.default_value
|
165
178
|
end
|
166
179
|
|
180
|
+
# TODO if there is a deserialization error, do we want to indicate that on the CachedPath?
|
167
181
|
@cache[path] = CachedPath.new(value, stat)
|
168
182
|
|
169
183
|
debug(
|
170
184
|
"updated cache path=#{path} raw_value=#{raw_value.inspect} "\
|
171
185
|
"value=#{value.inspect} stat=#{stat.inspect}"
|
172
186
|
)
|
173
|
-
ActiveSupport::Notifications.instrument(
|
187
|
+
ActiveSupport::Notifications.instrument(AS_NOTIFICATION, instrument_params)
|
174
188
|
true
|
175
189
|
rescue ::ZK::Exceptions::ZKError => e
|
176
190
|
warn("update_cache path=#{path} exception=#{e.inspect}, retrying")
|
@@ -181,6 +195,7 @@ module ZkRecipes
|
|
181
195
|
end
|
182
196
|
|
183
197
|
def process_pending_updates
|
198
|
+
return if @pending_updates.empty?
|
184
199
|
info("processing pending updates=#{@pending_updates.size}")
|
185
200
|
@pending_updates.reject! do |missed_path, _|
|
186
201
|
debug("update_cache with previously missed update path=#{missed_path}")
|
data/lib/zk_recipes/version.rb
CHANGED
data/lib/zk_recipes.rb
CHANGED
data/zk_recipes.gemspec
CHANGED
@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
|
|
15
15
|
spec.license = "MIT"
|
16
16
|
|
17
17
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
18
|
-
f.match(%r{^(test|spec|features)/})
|
18
|
+
f.match(%r{^(bin|test|spec|features)/})
|
19
19
|
end
|
20
20
|
spec.bindir = "exe"
|
21
21
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zk_recipes
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0.pre1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Lazarus
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-07-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -63,13 +63,12 @@ files:
|
|
63
63
|
- ".rspec"
|
64
64
|
- ".rubocop.yml"
|
65
65
|
- ".travis.yml"
|
66
|
+
- CHANGELOG.md
|
66
67
|
- CODE_OF_CONDUCT.md
|
67
68
|
- Gemfile
|
68
69
|
- LICENSE.txt
|
69
70
|
- README.md
|
70
71
|
- Rakefile
|
71
|
-
- bin/console
|
72
|
-
- bin/setup
|
73
72
|
- lib/zk_recipes.rb
|
74
73
|
- lib/zk_recipes/cache.rb
|
75
74
|
- lib/zk_recipes/version.rb
|
@@ -89,9 +88,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
89
88
|
version: '0'
|
90
89
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
91
90
|
requirements:
|
92
|
-
- - "
|
91
|
+
- - ">"
|
93
92
|
- !ruby/object:Gem::Version
|
94
|
-
version:
|
93
|
+
version: 1.3.1
|
95
94
|
requirements: []
|
96
95
|
rubyforge_project:
|
97
96
|
rubygems_version: 2.6.11
|
data/bin/console
DELETED
@@ -1,14 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
require "bundler/setup"
|
4
|
-
require "zk_recipes"
|
5
|
-
|
6
|
-
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
-
# with your gem easier. You can also use a different console, if you like.
|
8
|
-
|
9
|
-
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
-
# require "pry"
|
11
|
-
# Pry.start
|
12
|
-
|
13
|
-
require "irb"
|
14
|
-
IRB.start(__FILE__)
|