zk_recipes 0.1.0 → 0.2.0.pre1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0d7e707bc2f43f78a94c97736934209f049c19fb
4
- data.tar.gz: 8b93f73571a1464ee7c231d95e85b5373f0c9312
3
+ metadata.gz: 7750ecfcae437bf16df6dfbd4d23b769907f3209
4
+ data.tar.gz: ed8c8aef2d4cbe07cb669dd461230441c2537c8f
5
5
  SHA512:
6
- metadata.gz: af9c84ae0d6797865d787353221429ec7031559b1b948133511127fdf54868e0852d087ffcaf7ec1fa8c67354e2c4077ff6468338788822ad38d8a882f6afd5b
7
- data.tar.gz: 8ffdb1eb278394d79bf93ff84ce0a82e5b5ab270f640aa872c32e1561fd846a1aafe626eacc62bd68ff6ca240a4e5865712296ae3ef19523dc7aedc8515162b6
6
+ metadata.gz: 0c990d65e803782d5465e7bca85a877b3070dbff66086e70dbf6d7d6cbdf5fb6f356ee776f1d714d00d0ea07bc028a9c8f1a2ea54331090e46ea40f868101277
7
+ data.tar.gz: f25b27483c6a0d9f354694452d8304b20b54e928d9ad435904e829dbcd21907113d3122049a14c1f3266de5f048cbfec86969047bd8f8894d96ba266f1cc145f
data/.travis.yml CHANGED
@@ -1,5 +1,15 @@
1
- sudo: false
2
1
  language: ruby
2
+ before_install:
3
+ - sudo apt-get -qq update
4
+ - sudo apt-get install -y socat pv
5
+ - gem install bundler
3
6
  rvm:
7
+ - 2.3.4
4
8
  - 2.4.1
5
- before_install: gem install bundler -v 1.14.6
9
+ - jruby-9.1.12.0
10
+ - ruby-head
11
+ matrix:
12
+ allow_failures:
13
+ - rvm: ruby-head
14
+ fast_finish: true
15
+ sudo: required
data/CHANGELOG.md ADDED
@@ -0,0 +1,21 @@
1
+ # Changelog
2
+
3
+ ## 0.2.0
4
+
5
+ BREAKING CHANGES
6
+
7
+ - Make `on_connected` lighter: `on_connected` gets called for every watch when
8
+ a connection flaps. Make the happy path `on_connected` faster.
9
+ - Add `ZkRecipes::Cache#reopen` for resetting the cache after a `fork`
10
+ - BREAKING CHANGE: Don't use `KeyError` as part of the API; use
11
+ `ZkRecipes::Cache::PathError` instead.
12
+ - BREAKING CHANGE: Overhaul `ActiveSupport::Notifications`.
13
+ - Only one notifation now: `cache.zk_recipes`. Removed
14
+ `zk_recipes.cache.update` and `zk_recipes.cache.error`.
15
+ - All notifications have a `path`.
16
+ - Don't use notifications for unhandled exceptions.
17
+ - Development only: use `ZK_RECIPES_DEBUG=1 rspec` for debug logging.
18
+
19
+ ## 0.1.0
20
+
21
+ - Initial release
data/README.md CHANGED
@@ -18,7 +18,7 @@ Or install it yourself as:
18
18
 
19
19
  ## Usage
20
20
 
21
- A cache that creates its own ZK client:
21
+ ### A cache that creates its own ZK client:
22
22
 
23
23
  ```ruby
24
24
  logger = Logger.new(STDERR)
@@ -31,20 +31,70 @@ puts cache["/test/boom"]
31
31
  cache.close!
32
32
  ```
33
33
 
34
- A cache that uses an existing ZK client:
34
+ ### A cache that uses an existing ZK client:
35
35
 
36
36
  ```ruby
37
37
  logger = Logger.new(STDERR)
38
- zk = ZK.new("my-host:1234", connect: false, timeout: 5) # ZK timeout = 5s
38
+ zk = ZK.new("my-host:1234", connect: false)
39
+ cache = ZkRecipes::Cache.new(logger: logger)
40
+ cache.register("/test/boom", "goat") { |string| "Hello, #{string}" }
41
+ cache.setup_callbacks(zk) # no more paths can be registered after this
42
+
43
+ puts cache["/test/boom"] # => "Hello, goat"
44
+
45
+ zk.connect
46
+ cache.wait_for_warm_cache(10) # wait up to 10s for the cache to warm
47
+
48
+ zk.create("/test/boom")
49
+ zk.set("/test/boom", "cat")
50
+
51
+ sleep 1
52
+
53
+ puts cache["/test/boom"] # => "Hello, cat"
54
+ cache.close!
55
+ zk.close!
56
+ ```
57
+
58
+ ### Handling forks with a cache that creates it's own ZK client:
59
+
60
+ ```ruby
61
+ zk = ZK.new("my-host:1234") # zk client for writing only
62
+
63
+ logger = Logger.new(STDERR)
64
+ cache = ZkRecipes::Cache.new(host: "my-host:1234", logger: logger, timeout: 10) do |z|
65
+ z.register("/test/boom", "goat")
66
+ end
67
+
68
+
69
+ puts cache["/test/boom"] # => "goat"
70
+
71
+ if fork
72
+ # parent
73
+ zk.set("/test/boom", "mouse")
74
+ cache.close!
75
+ zk.close!
76
+ else
77
+ # child
78
+ cache.reopen # wait up to 10s for ZK to reconnect and the cache to warm
79
+ puts cache["/test/boom"] # => "mouse"
80
+ cache.close!
81
+ zk.close!
82
+ end
83
+ ```
84
+
85
+ ### Handling forks with an existing ZK client:
86
+
87
+ ```ruby
88
+ logger = Logger.new(STDERR)
89
+ zk = ZK.new("my-host:1234", connect: false)
39
90
  cache = ZkRecipes::Cache.new(logger: logger)
40
91
  cache.register("/test/boom", "goat")
41
- cache.register("/test/foo", 1) { |raw_value| raw_value.to_i * 2 }
42
92
  cache.setup_callbacks(zk) # no more paths can be registered after this
43
93
 
44
94
  puts cache["/test/boom"] # => "goat"
45
95
 
46
96
  zk.connect
47
- cache.wait_for_warm_cache(10) # wait 10s for the cache to warm
97
+ cache.wait_for_warm_cache(10) # wait up to 10s for the cache to warm
48
98
 
49
99
  zk.create("/test/boom")
50
100
  zk.set("/test/boom", "cat")
@@ -52,8 +102,46 @@ zk.set("/test/boom", "cat")
52
102
  sleep 1
53
103
 
54
104
  puts cache["/test/boom"] # => "cat"
55
- cache.close!
56
- zk.close!
105
+
106
+ if fork
107
+ # parent
108
+ zk.set("/test/boom", "mouse")
109
+ cache.close!
110
+ zk.close!
111
+ else
112
+ # child
113
+ cache.reopen
114
+ zk.reopen
115
+ cache.wait_for_warm_cache(10) # wait up to 10s for the cache to warm again
116
+ puts cache["/test/boom"] # => "mouse"
117
+ cache.close!
118
+ zk.close!
119
+ end
120
+ ```
121
+
122
+ ### ActiveSupport::Notifications
123
+
124
+ `ZkRecipes` use `ActiveSupport::Notifications` for callbacks.
125
+
126
+ WARNING: exceptions raised in `ActiveSupport::Notifications.subscribe` will
127
+ bubble up to the line where the notification is instrumented. MAKE SURE YOU
128
+ CATCH EXCEPTIONS IN YOUR SUBSCRIBE BLOCKS!
129
+
130
+ Example:
131
+
132
+ ```ruby
133
+ ActiveSupport::Notifications.subscribe("cache.zk_recipes") do |_name, _start, _finish, _id, payload|
134
+ if payload[:error]
135
+ puts "There was a ZkRecipes::Cache error #{payload[:error]}"
136
+ puts "The error occurred trying to deserialize a value from ZK "\
137
+ "path=#{payload[:path] raw_zk_value=#{payload[:raw_value]}"
138
+ else
139
+ puts "Received and update for path=#{payload[:path}."
140
+ puts "The update in ZK happened #{payload[:latency_seconds]} seconds ago."
141
+ puts "The path's data is #{payload[:data_length]} bytes."
142
+ puts "The path's version is #{payload[:version]}."
143
+ end
144
+ end
57
145
  ```
58
146
 
59
147
  ## Development
@@ -3,17 +3,15 @@
3
3
  module ZkRecipes
4
4
  class Cache
5
5
  class Error < StandardError; end
6
+ class PathError < Error; end
6
7
 
7
- extend Forwardable
8
+ AS_NOTIFICATION = "cache.zk_recipes"
8
9
 
9
- AS_NOTIFICATION_UPDATE = "zk_recipes.cache.update"
10
- AS_NOTIFICATION_ERROR = "zk_recipes.cache.error"
11
-
12
- def initialize(host: nil, logger: nil, timeout: nil, zk_opts: {})
10
+ def initialize(logger: nil, host: nil, timeout: nil, zk_opts: {})
13
11
  @cache = Concurrent::Map.new
14
12
  @latch = Concurrent::CountDownLatch.new
15
13
  @logger = logger
16
- @pending_updates = Concurrent::Hash.new
14
+ @pending_updates = Concurrent::Hash.new # Concurrent::Map does not implement #reject!
17
15
  @registerable = true
18
16
  @registered_values = Concurrent::Map.new
19
17
  @session_id = nil
@@ -22,9 +20,10 @@ module ZkRecipes
22
20
 
23
21
  if block_given?
24
22
  @owned_zk = true
23
+ @warm_cache_timeout = timeout || 30
25
24
  yield(self)
26
25
 
27
- expiration = Time.now + (timeout || 30)
26
+ expiration = Time.now + @warm_cache_timeout
28
27
  connect(host, zk_opts)
29
28
 
30
29
  wait_for_warm_cache(expiration - Time.now)
@@ -41,10 +40,11 @@ module ZkRecipes
41
40
  debug("added path=#{path} default_value=#{default_value.inspect}")
42
41
  @cache[path] = CachedPath.new(default_value)
43
42
  @registered_values[path] = RegisteredPath.new(default_value, block)
44
- ActiveSupport::Notifications.instrument(AS_NOTIFICATION_UPDATE, path: path, value: default_value)
43
+ ActiveSupport::Notifications.instrument(AS_NOTIFICATION, path: path, value: default_value)
45
44
  end
46
45
 
47
46
  def setup_callbacks(zk)
47
+ raise Error, "setup_callbacks can only be called once" unless @registerable
48
48
  @zk = zk
49
49
  @registerable = false
50
50
 
@@ -52,47 +52,46 @@ module ZkRecipes
52
52
  raise Error, "the ZK::Client is already connected, the cached values must be set before connecting"
53
53
  end
54
54
 
55
- @zk.on_connected do |e|
56
- info("on_connected session_id old=#{@session_id} new=#{@zk.session_id} #{e.event_name} #{e.state_name}")
55
+ @registered_values.each do |path, _value|
56
+ @watches[path] = @zk.register(path) do |event|
57
+ if event.node_event?
58
+ debug("node event=#{event.inspect} #{event.event_name} #{event.state_name}")
59
+ unless update_cache(event.path)
60
+ @pending_updates[path] = nil
61
+ @zk.defer { process_pending_updates }
62
+ end
63
+ else
64
+ warn("session event=#{event.inspect}")
65
+ end
66
+ end
67
+ end
68
+
69
+ @watches["on_connected"] = @zk.on_connected do
57
70
  if @session_id == @zk.session_id
58
71
  process_pending_updates
59
72
  next
60
73
  end
61
74
 
75
+ debug("on_connected new session")
62
76
  @pending_updates.clear
63
77
  @registered_values.each do |path, _value|
64
- @watches[path] ||= @zk.register(path) do |event|
65
- if event.node_event?
66
- debug("node event=#{event.inspect} #{event.event_name} #{event.state_name}")
67
- unless update_cache(event.path)
68
- @pending_updates[path] = nil
69
- @zk.defer { process_pending_updates }
70
- end
71
- else
72
- warn("session event=#{event.inspect}")
73
- end
74
- end
75
78
  @pending_updates[path] = nil unless update_cache(path)
76
79
  end
77
80
  @session_id = @zk.session_id
78
81
  @latch.count_down
79
82
  end
80
83
 
81
- @zk.on_expired_session do |e|
82
- info("on_expired_session #{e.event_name} #{e.state_name}")
83
- end
84
-
85
84
  @zk.on_exception do |e|
86
85
  error("on_exception exception=#{e.inspect} backtrace=#{e.backtrace.inspect}")
87
- ActiveSupport::Notifications.instrument(AS_NOTIFICATION_ERROR, error: e)
88
86
  end
89
87
  end
90
88
 
91
89
  def wait_for_warm_cache(timeout = 30)
90
+ debug("waiting for cache to warm timeout=#{timeout}")
92
91
  if @latch.wait(timeout)
93
92
  true
94
93
  else
95
- warn("didn't connect before timeout connected=#{@zk.connected?} timeout=#{timeout}")
94
+ warn("didn't warm cache before timeout connected=#{@zk.connected?} timeout=#{timeout}")
96
95
  false
97
96
  end
98
97
  end
@@ -101,16 +100,34 @@ module ZkRecipes
101
100
  @watches.each_value(&:unsubscribe)
102
101
  @watches.clear
103
102
  @zk.close! if @owned_zk
103
+ @zk = nil
104
+ @cache = nil
105
+ @registered_values = nil
106
+ end
107
+
108
+ def reopen
109
+ @latch = Concurrent::CountDownLatch.new
110
+ @session_id = nil
111
+ @pending_updates.clear
112
+ if @owned_zk
113
+ expiration = Time.now + @warm_cache_timeout
114
+ @zk.reopen
115
+ wait_for_warm_cache(expiration - Time.now)
116
+ end
104
117
  end
105
118
 
106
119
  def fetch(path)
107
120
  @cache.fetch(path).value
121
+ rescue KeyError
122
+ raise PathError, "no registered path for #{path.inspect}"
108
123
  end
109
124
  alias_method :[], :fetch
110
125
 
111
126
  def fetch_existing(path)
112
127
  cached = @cache.fetch(path)
113
128
  cached.value if cached.stat&.exists?
129
+ rescue KeyError
130
+ raise PathError, "no registered path=#{path.inspect}"
114
131
  end
115
132
 
116
133
  private
@@ -130,7 +147,6 @@ module ZkRecipes
130
147
 
131
148
  stat = @zk.stat(path, watch: true)
132
149
 
133
- instrument_name = AS_NOTIFICATION_UPDATE
134
150
  instrument_params = { path: path }
135
151
 
136
152
  unless stat.exists?
@@ -138,17 +154,15 @@ module ZkRecipes
138
154
  @cache[path] = CachedPath.new(value, stat)
139
155
  debug("no node, setting watch path=#{path}")
140
156
  instrument_params[:value] = value
141
- ActiveSupport::Notifications.instrument(instrument_name, instrument_params)
157
+ ActiveSupport::Notifications.instrument(AS_NOTIFICATION, instrument_params)
142
158
  return true
143
159
  end
144
160
 
145
161
  raw_value, stat = @zk.get(path, watch: true)
146
162
 
147
- instrument_params.merge!(
148
- latency_seconds: Time.now - stat.mtime_t,
149
- version: stat.version,
150
- data_length: stat.data_length,
151
- )
163
+ instrument_params[:latency_seconds] = Time.now - stat.mtime_t
164
+ instrument_params[:version] = stat.version
165
+ instrument_params[:data_length] = stat.data_length
152
166
 
153
167
  value = begin
154
168
  registered_value = @registered_values.fetch(path)
@@ -158,19 +172,19 @@ module ZkRecipes
158
172
  "deserialization error raw_zookeeper_value=#{raw_value.inspect} zookeeper_stat=#{stat.inspect} "\
159
173
  "exception=#{e.inspect} #{e.backtrace.inspect}"
160
174
  )
161
- instrument_name = AS_NOTIFICATION_ERROR
162
175
  instrument_params[:error] = e
163
176
  instrument_params[:raw_value] = raw_value
164
177
  registered_value.default_value
165
178
  end
166
179
 
180
+ # TODO if there is a deserialization error, do we want to indicate that on the CachedPath?
167
181
  @cache[path] = CachedPath.new(value, stat)
168
182
 
169
183
  debug(
170
184
  "updated cache path=#{path} raw_value=#{raw_value.inspect} "\
171
185
  "value=#{value.inspect} stat=#{stat.inspect}"
172
186
  )
173
- ActiveSupport::Notifications.instrument(instrument_name, instrument_params)
187
+ ActiveSupport::Notifications.instrument(AS_NOTIFICATION, instrument_params)
174
188
  true
175
189
  rescue ::ZK::Exceptions::ZKError => e
176
190
  warn("update_cache path=#{path} exception=#{e.inspect}, retrying")
@@ -181,6 +195,7 @@ module ZkRecipes
181
195
  end
182
196
 
183
197
  def process_pending_updates
198
+ return if @pending_updates.empty?
184
199
  info("processing pending updates=#{@pending_updates.size}")
185
200
  @pending_updates.reject! do |missed_path, _|
186
201
  debug("update_cache with previously missed update path=#{missed_path}")
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module ZkRecipes
4
- VERSION = "0.1.0"
4
+ VERSION = "0.2.0.pre1"
5
5
  end
data/lib/zk_recipes.rb CHANGED
@@ -2,7 +2,6 @@
2
2
 
3
3
  require "active_support/notifications"
4
4
  require "concurrent"
5
- require "forwardable"
6
5
  require "zk"
7
6
 
8
7
  require "zk_recipes/version"
data/zk_recipes.gemspec CHANGED
@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
15
15
  spec.license = "MIT"
16
16
 
17
17
  spec.files = `git ls-files -z`.split("\x0").reject do |f|
18
- f.match(%r{^(test|spec|features)/})
18
+ f.match(%r{^(bin|test|spec|features)/})
19
19
  end
20
20
  spec.bindir = "exe"
21
21
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: zk_recipes
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0.pre1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Lazarus
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-06-21 00:00:00.000000000 Z
11
+ date: 2017-07-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -63,13 +63,12 @@ files:
63
63
  - ".rspec"
64
64
  - ".rubocop.yml"
65
65
  - ".travis.yml"
66
+ - CHANGELOG.md
66
67
  - CODE_OF_CONDUCT.md
67
68
  - Gemfile
68
69
  - LICENSE.txt
69
70
  - README.md
70
71
  - Rakefile
71
- - bin/console
72
- - bin/setup
73
72
  - lib/zk_recipes.rb
74
73
  - lib/zk_recipes/cache.rb
75
74
  - lib/zk_recipes/version.rb
@@ -89,9 +88,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
89
88
  version: '0'
90
89
  required_rubygems_version: !ruby/object:Gem::Requirement
91
90
  requirements:
92
- - - ">="
91
+ - - ">"
93
92
  - !ruby/object:Gem::Version
94
- version: '0'
93
+ version: 1.3.1
95
94
  requirements: []
96
95
  rubyforge_project:
97
96
  rubygems_version: 2.6.11
data/bin/console DELETED
@@ -1,14 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- require "bundler/setup"
4
- require "zk_recipes"
5
-
6
- # You can add fixtures and/or initialization code here to make experimenting
7
- # with your gem easier. You can also use a different console, if you like.
8
-
9
- # (If you use this, don't forget to add pry to your Gemfile!)
10
- # require "pry"
11
- # Pry.start
12
-
13
- require "irb"
14
- IRB.start(__FILE__)
data/bin/setup DELETED
@@ -1,8 +0,0 @@
1
- #!/usr/bin/env bash
2
- set -euo pipefail
3
- IFS=$'\n\t'
4
- set -vx
5
-
6
- bundle install
7
-
8
- # Do any other automated setup that you need to do here