prefab-cloud-ruby 0.0.22 → 0.0.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -0
- data/Gemfile.lock +15 -0
- data/VERSION +1 -1
- data/lib/prefab/client.rb +2 -1
- data/lib/prefab/config_client.rb +12 -117
- data/lib/prefab-cloud-ruby.rb +1 -1
- data/prefab-cloud-ruby.gemspec +6 -3
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1174d61400436bdc42b40cf7141729065ffb7c35
|
4
|
+
data.tar.gz: f275ed0da1795f276f950bc956a105d28ed27886
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 68b0c8ee0dfac57878dd13ab41bb31d653b131a2923821f39e1165036b89c31101863aaa872f34443f502d021b69fb54575e6369cb3fde83fc362bfee8dd8b45
|
7
|
+
data.tar.gz: 92eabb20b31054ed1bce771631d893e79f86e1493aaf3375e0a1426aa64449b729a1b84daf1b7ac982a7bc0d545ff8fc98d81ae17e8638f5978ef13f2d750598
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -8,6 +8,19 @@ GEM
|
|
8
8
|
tzinfo (~> 1.1)
|
9
9
|
addressable (2.5.2)
|
10
10
|
public_suffix (>= 2.0.2, < 4.0)
|
11
|
+
aws-partitions (1.86.0)
|
12
|
+
aws-sdk-core (3.20.2)
|
13
|
+
aws-partitions (~> 1.0)
|
14
|
+
aws-sigv4 (~> 1.0)
|
15
|
+
jmespath (~> 1.0)
|
16
|
+
aws-sdk-kms (1.5.0)
|
17
|
+
aws-sdk-core (~> 3)
|
18
|
+
aws-sigv4 (~> 1.0)
|
19
|
+
aws-sdk-s3 (1.10.0)
|
20
|
+
aws-sdk-core (~> 3)
|
21
|
+
aws-sdk-kms (~> 1)
|
22
|
+
aws-sigv4 (~> 1.0)
|
23
|
+
aws-sigv4 (1.0.2)
|
11
24
|
builder (3.2.3)
|
12
25
|
concurrent-ruby (1.0.5)
|
13
26
|
descendants_tracker (0.0.4)
|
@@ -42,6 +55,7 @@ GEM
|
|
42
55
|
highline (1.7.10)
|
43
56
|
i18n (1.0.1)
|
44
57
|
concurrent-ruby (~> 1.0)
|
58
|
+
jmespath (1.4.0)
|
45
59
|
json (1.8.6)
|
46
60
|
juwelier (2.4.9)
|
47
61
|
builder
|
@@ -108,6 +122,7 @@ PLATFORMS
|
|
108
122
|
ruby
|
109
123
|
|
110
124
|
DEPENDENCIES
|
125
|
+
aws-sdk-s3 (~> 1)
|
111
126
|
bundler (~> 1.0)
|
112
127
|
concurrent-ruby (~> 1.0, >= 1.0.5)
|
113
128
|
grpc (~> 1.11.0)
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.23
|
data/lib/prefab/client.rb
CHANGED
@@ -4,7 +4,7 @@ module Prefab
|
|
4
4
|
MAX_SLEEP_SEC = 10
|
5
5
|
BASE_SLEEP_SEC = 0.5
|
6
6
|
|
7
|
-
attr_reader :account_id, :shared_cache, :stats, :namespace, :creds, :interceptor
|
7
|
+
attr_reader :account_id, :shared_cache, :stats, :namespace, :creds, :interceptor, :api_key
|
8
8
|
|
9
9
|
def initialize(api_key: ENV['PREFAB_API_KEY'],
|
10
10
|
logdev: nil,
|
@@ -20,6 +20,7 @@ module Prefab
|
|
20
20
|
@local = local
|
21
21
|
@stats = (stats || NoopStats.new)
|
22
22
|
@shared_cache = (shared_cache || NoopCache.new)
|
23
|
+
@api_key = api_key
|
23
24
|
@account_id = api_key.split("|")[0].to_i
|
24
25
|
@namespace = namespace
|
25
26
|
|
data/lib/prefab/config_client.rb
CHANGED
@@ -1,26 +1,24 @@
|
|
1
1
|
module Prefab
|
2
2
|
class ConfigClient
|
3
3
|
RECONNECT_WAIT = 5
|
4
|
-
|
5
|
-
DEFAULT_CHECKPOINT_FREQ_SEC = 10
|
6
|
-
DEFAULT_MAX_SUSPENDERS_AGE_SEC = 60
|
4
|
+
DEFAULT_CHECKPOINT_FREQ_SEC = 60
|
7
5
|
|
8
6
|
def initialize(base_client, timeout)
|
9
7
|
@base_client = base_client
|
10
8
|
@timeout = timeout
|
11
9
|
@initialization_lock = Concurrent::ReadWriteLock.new
|
12
10
|
|
13
|
-
@suspenders_max_age_secs = (ENV["PREFAB_SUSPENDERS_MAX_AGE_SEC"] || DEFAULT_MAX_SUSPENDERS_AGE_SEC)
|
14
11
|
@checkpoint_freq_secs = (ENV["PREFAB_DEFAULT_CHECKPOINT_FREQ_SEC"] || DEFAULT_CHECKPOINT_FREQ_SEC)
|
15
12
|
|
16
13
|
@config_loader = Prefab::ConfigLoader.new(@base_client)
|
17
14
|
@config_resolver = Prefab::ConfigResolver.new(@base_client, @config_loader)
|
18
15
|
|
19
16
|
@initialization_lock.acquire_write_lock
|
17
|
+
@s3 = Aws::S3::Resource.new(region: 'us-east-1')
|
20
18
|
|
21
|
-
|
22
|
-
|
23
|
-
start_checkpointing_thread
|
19
|
+
load_checkpoint
|
20
|
+
start_api_connection_thread(@config_loader.highwater_mark)
|
21
|
+
start_checkpointing_thread
|
24
22
|
end
|
25
23
|
|
26
24
|
def get(prop)
|
@@ -69,10 +67,9 @@ module Prefab
|
|
69
67
|
# Bootstrap out of the cache
|
70
68
|
# returns the high-watermark of what was in the cache
|
71
69
|
def load_checkpoint
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
deltas = Prefab::ConfigDeltas.decode(checkpoint)
|
70
|
+
obj = @s3.bucket('prefab-cloud-checkpoints-prod').object(@base_client.api_key.gsub("|", "/"))
|
71
|
+
obj.get do |f|
|
72
|
+
deltas = Prefab::ConfigDeltas.decode(f)
|
76
73
|
deltas.deltas.each do |delta|
|
77
74
|
@config_loader.set(delta)
|
78
75
|
end
|
@@ -80,33 +77,18 @@ module Prefab
|
|
80
77
|
@base_client.stats.increment("prefab.config.checkpoint.load")
|
81
78
|
@config_resolver.update
|
82
79
|
finish_init!
|
83
|
-
else
|
84
|
-
@base_client.log_internal Logger::INFO, "No checkpoint"
|
85
80
|
end
|
86
|
-
end
|
87
81
|
|
88
|
-
|
89
|
-
|
90
|
-
def save_checkpoint
|
91
|
-
begin
|
92
|
-
deltas = @config_resolver.export_api_deltas
|
93
|
-
@base_client.log_internal Logger::DEBUG, "Save Checkpoint #{@config_loader.highwater_mark} Thread #{Thread.current.object_id}"
|
94
|
-
@base_client.shared_cache.write(checkpoint_cache_key, Prefab::ConfigDeltas.encode(deltas))
|
95
|
-
@base_client.shared_cache.write(checkpoint_highwater_cache_key, @config_loader.highwater_mark)
|
96
|
-
@base_client.stats.increment("prefab.config.checkpoint.save")
|
97
|
-
rescue StandardError => exn
|
98
|
-
@base_client.log_internal Logger::INFO, "Issue Saving Checkpoint #{exn.message}"
|
99
|
-
end
|
82
|
+
rescue Aws::S3::Errors::NoSuchKey
|
83
|
+
@base_client.log_internal Logger::INFO, "No checkpoint"
|
100
84
|
end
|
101
85
|
|
102
|
-
# A thread that
|
86
|
+
# A thread that checks for a checkpoint
|
103
87
|
def start_checkpointing_thread
|
104
88
|
Thread.new do
|
105
89
|
loop do
|
106
90
|
begin
|
107
|
-
|
108
|
-
|
109
|
-
suspenders_if_needed
|
91
|
+
load_checkpoint
|
110
92
|
|
111
93
|
started_at = Time.now
|
112
94
|
delta = @checkpoint_freq_secs - (Time.now - started_at)
|
@@ -120,72 +102,6 @@ module Prefab
|
|
120
102
|
end
|
121
103
|
end
|
122
104
|
|
123
|
-
# check what our shared highwater mark is
|
124
|
-
# if it is higher than our own highwater mark, we must have missed something, so load the checkpoint
|
125
|
-
# if it is lower than our own highwater mark, save a checkpoint
|
126
|
-
# if everything is up to date, but the shared highwater mark is old, coordinate amongst other processes to have
|
127
|
-
# one process "double check" by restarting the API thread
|
128
|
-
def load_or_save_checkpoint
|
129
|
-
shared_highwater_mark = get_shared_highwater_mark
|
130
|
-
@base_client.log_internal Logger::DEBUG, "Checkpoint_if_needed highwater apx ahead/behind #{(@config_loader.highwater_mark - shared_highwater_mark) / (1000 * 10000)}"
|
131
|
-
|
132
|
-
if shared_highwater_mark > @config_loader.highwater_mark
|
133
|
-
@base_client.log_internal Logger::DEBUG, "We were behind, loading checkpoint"
|
134
|
-
load_checkpoint
|
135
|
-
elsif shared_highwater_mark < @config_loader.highwater_mark
|
136
|
-
@base_client.log_internal Logger::DEBUG, "Saving off checkpoint"
|
137
|
-
save_checkpoint
|
138
|
-
end
|
139
|
-
end
|
140
|
-
|
141
|
-
def suspenders_if_needed
|
142
|
-
if suspenders_age_is_old?
|
143
|
-
if get_shared_lock?
|
144
|
-
@base_client.log_internal Logger::DEBUG, "Suspenders > PREFAB_SUSPENDERS_MAX_AGE_SEC #{@suspenders_max_age_secs}. We have been chosen to run suspenders"
|
145
|
-
reset_api_connection
|
146
|
-
else
|
147
|
-
@base_client.log_internal Logger::DEBUG, "Suspenders > PREFAB_SUSPENDERS_MAX_AGE_SEC #{@suspenders_max_age_secs}. Other process is running suspenders"
|
148
|
-
end
|
149
|
-
end
|
150
|
-
end
|
151
|
-
|
152
|
-
def suspenders_age_is_old?
|
153
|
-
age = Time.now.to_i - get_checkpoint_suspenders
|
154
|
-
@base_client.log_internal Logger::DEBUG, "checkpoint_suspenders_is_old? apx #{age}" if age > @suspenders_max_age_secs
|
155
|
-
age > @suspenders_max_age_secs
|
156
|
-
end
|
157
|
-
|
158
|
-
def get_shared_highwater_mark
|
159
|
-
(@base_client.shared_cache.read(checkpoint_highwater_cache_key) || 0).to_i
|
160
|
-
end
|
161
|
-
|
162
|
-
def get_checkpoint_suspenders
|
163
|
-
(@base_client.shared_cache.read(suspenders_last_run_cache_key) || 0).to_i
|
164
|
-
end
|
165
|
-
|
166
|
-
def set_suspenders
|
167
|
-
@base_client.shared_cache.write(suspenders_last_run_cache_key, Time.now.to_i)
|
168
|
-
end
|
169
|
-
|
170
|
-
def get_shared_lock?
|
171
|
-
in_progess = @base_client.shared_cache.read(suspenders_reset_in_progress_cache_key)
|
172
|
-
if in_progess.nil?
|
173
|
-
@base_client.shared_cache.write(suspenders_reset_in_progress_cache_key, "true", { expires_in: SUSPENDERS_RESET_LOCK_SEC })
|
174
|
-
true
|
175
|
-
else
|
176
|
-
false
|
177
|
-
end
|
178
|
-
end
|
179
|
-
|
180
|
-
def ensure_api_connection_started
|
181
|
-
reset_api_connection if @api_connection_thread.nil?
|
182
|
-
end
|
183
|
-
|
184
|
-
def reset_api_connection
|
185
|
-
@api_connection_thread&.exit
|
186
|
-
start_api_connection_thread(@config_loader.highwater_mark)
|
187
|
-
end
|
188
|
-
|
189
105
|
def finish_init!
|
190
106
|
if @initialization_lock.write_locked?
|
191
107
|
@initialization_lock.release_write_lock
|
@@ -200,7 +116,6 @@ module Prefab
|
|
200
116
|
start_at_id: start_at_id)
|
201
117
|
@base_client.log_internal Logger::DEBUG, "start api connection thread #{start_at_id}"
|
202
118
|
@base_client.stats.increment("prefab.config.api.start")
|
203
|
-
set_suspenders
|
204
119
|
@api_connection_thread = Thread.new do
|
205
120
|
while true do
|
206
121
|
begin
|
@@ -220,26 +135,6 @@ module Prefab
|
|
220
135
|
end
|
221
136
|
end
|
222
137
|
end
|
223
|
-
|
224
|
-
def has_real_cache?
|
225
|
-
@base_client.shared_cache.class != NoopCache
|
226
|
-
end
|
227
|
-
|
228
|
-
def checkpoint_cache_key
|
229
|
-
@base_client.cache_key "config:checkpoint"
|
230
|
-
end
|
231
|
-
|
232
|
-
def suspenders_reset_in_progress_cache_key
|
233
|
-
@base_client.cache_key "config:checkpoint:updating"
|
234
|
-
end
|
235
|
-
|
236
|
-
def checkpoint_highwater_cache_key
|
237
|
-
@base_client.cache_key "config:checkpoint:highwater"
|
238
|
-
end
|
239
|
-
|
240
|
-
def suspenders_last_run_cache_key
|
241
|
-
@base_client.cache_key "config:checkpoint:suspenders"
|
242
|
-
end
|
243
138
|
end
|
244
139
|
end
|
245
140
|
|
data/lib/prefab-cloud-ruby.rb
CHANGED
data/prefab-cloud-ruby.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: prefab-cloud-ruby 0.0.
|
5
|
+
# stub: prefab-cloud-ruby 0.0.23 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "prefab-cloud-ruby".freeze
|
9
|
-
s.version = "0.0.
|
9
|
+
s.version = "0.0.23"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib".freeze]
|
13
13
|
s.authors = ["Jeff Dwyer".freeze]
|
14
|
-
s.date = "2018-05-
|
14
|
+
s.date = "2018-05-15"
|
15
15
|
s.description = "RateLimits & Config as a service".freeze
|
16
16
|
s.email = "jdwyer@prefab.cloud".freeze
|
17
17
|
s.extra_rdoc_files = [
|
@@ -60,6 +60,7 @@ Gem::Specification.new do |s|
|
|
60
60
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
61
61
|
s.add_runtime_dependency(%q<grpc>.freeze, ["~> 1.11.0"])
|
62
62
|
s.add_runtime_dependency(%q<concurrent-ruby>.freeze, [">= 1.0.5", "~> 1.0"])
|
63
|
+
s.add_runtime_dependency(%q<aws-sdk-s3>.freeze, ["~> 1"])
|
63
64
|
s.add_development_dependency(%q<grpc-tools>.freeze, ["~> 1.11.0"])
|
64
65
|
s.add_development_dependency(%q<shoulda>.freeze, [">= 0"])
|
65
66
|
s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12"])
|
@@ -69,6 +70,7 @@ Gem::Specification.new do |s|
|
|
69
70
|
else
|
70
71
|
s.add_dependency(%q<grpc>.freeze, ["~> 1.11.0"])
|
71
72
|
s.add_dependency(%q<concurrent-ruby>.freeze, [">= 1.0.5", "~> 1.0"])
|
73
|
+
s.add_dependency(%q<aws-sdk-s3>.freeze, ["~> 1"])
|
72
74
|
s.add_dependency(%q<grpc-tools>.freeze, ["~> 1.11.0"])
|
73
75
|
s.add_dependency(%q<shoulda>.freeze, [">= 0"])
|
74
76
|
s.add_dependency(%q<rdoc>.freeze, ["~> 3.12"])
|
@@ -79,6 +81,7 @@ Gem::Specification.new do |s|
|
|
79
81
|
else
|
80
82
|
s.add_dependency(%q<grpc>.freeze, ["~> 1.11.0"])
|
81
83
|
s.add_dependency(%q<concurrent-ruby>.freeze, [">= 1.0.5", "~> 1.0"])
|
84
|
+
s.add_dependency(%q<aws-sdk-s3>.freeze, ["~> 1"])
|
82
85
|
s.add_dependency(%q<grpc-tools>.freeze, ["~> 1.11.0"])
|
83
86
|
s.add_dependency(%q<shoulda>.freeze, [">= 0"])
|
84
87
|
s.add_dependency(%q<rdoc>.freeze, ["~> 3.12"])
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: prefab-cloud-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.23
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jeff Dwyer
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-05-
|
11
|
+
date: 2018-05-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: grpc
|
@@ -44,6 +44,20 @@ dependencies:
|
|
44
44
|
- - "~>"
|
45
45
|
- !ruby/object:Gem::Version
|
46
46
|
version: '1.0'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: aws-sdk-s3
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - "~>"
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '1'
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - "~>"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '1'
|
47
61
|
- !ruby/object:Gem::Dependency
|
48
62
|
name: grpc-tools
|
49
63
|
requirement: !ruby/object:Gem::Requirement
|