prefab-cloud-ruby 0.0.22 → 0.0.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 46899736646bad8d02c9ef4eca66a38fb58a10b3
4
- data.tar.gz: faa44f377c1299e333bc1860b65897fd09c9288b
3
+ metadata.gz: 1174d61400436bdc42b40cf7141729065ffb7c35
4
+ data.tar.gz: f275ed0da1795f276f950bc956a105d28ed27886
5
5
  SHA512:
6
- metadata.gz: 2b58f9f0ecfd29dd5a1531058a0dc129d7846585e7b95b36bb70c96b9bcbb57f4489432347c725527b2992318857aef7a6a91a6676d6efeb2a8610dea82d4025
7
- data.tar.gz: cc30b8afe888be463eed8035b75b6d737b9071f761380123471ca096b7ee26344d96a850c8db3117073baf770b176e15a117579e75e7cd670afdd5c054fd512d
6
+ metadata.gz: 68b0c8ee0dfac57878dd13ab41bb31d653b131a2923821f39e1165036b89c31101863aaa872f34443f502d021b69fb54575e6369cb3fde83fc362bfee8dd8b45
7
+ data.tar.gz: 92eabb20b31054ed1bce771631d893e79f86e1493aaf3375e0a1426aa64449b729a1b84daf1b7ac982a7bc0d545ff8fc98d81ae17e8638f5978ef13f2d750598
data/Gemfile CHANGED
@@ -2,6 +2,7 @@ source "https://rubygems.org"
2
2
 
3
3
  gem 'grpc', '~> 1.11.0'
4
4
  gem 'concurrent-ruby', '~> 1.0', '>= 1.0.5'
5
+ gem 'aws-sdk-s3', '~> 1'
5
6
 
6
7
  group :development do
7
8
  gem 'grpc-tools', '~> 1.11.0'
data/Gemfile.lock CHANGED
@@ -8,6 +8,19 @@ GEM
8
8
  tzinfo (~> 1.1)
9
9
  addressable (2.5.2)
10
10
  public_suffix (>= 2.0.2, < 4.0)
11
+ aws-partitions (1.86.0)
12
+ aws-sdk-core (3.20.2)
13
+ aws-partitions (~> 1.0)
14
+ aws-sigv4 (~> 1.0)
15
+ jmespath (~> 1.0)
16
+ aws-sdk-kms (1.5.0)
17
+ aws-sdk-core (~> 3)
18
+ aws-sigv4 (~> 1.0)
19
+ aws-sdk-s3 (1.10.0)
20
+ aws-sdk-core (~> 3)
21
+ aws-sdk-kms (~> 1)
22
+ aws-sigv4 (~> 1.0)
23
+ aws-sigv4 (1.0.2)
11
24
  builder (3.2.3)
12
25
  concurrent-ruby (1.0.5)
13
26
  descendants_tracker (0.0.4)
@@ -42,6 +55,7 @@ GEM
42
55
  highline (1.7.10)
43
56
  i18n (1.0.1)
44
57
  concurrent-ruby (~> 1.0)
58
+ jmespath (1.4.0)
45
59
  json (1.8.6)
46
60
  juwelier (2.4.9)
47
61
  builder
@@ -108,6 +122,7 @@ PLATFORMS
108
122
  ruby
109
123
 
110
124
  DEPENDENCIES
125
+ aws-sdk-s3 (~> 1)
111
126
  bundler (~> 1.0)
112
127
  concurrent-ruby (~> 1.0, >= 1.0.5)
113
128
  grpc (~> 1.11.0)
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.0.22
1
+ 0.0.23
data/lib/prefab/client.rb CHANGED
@@ -4,7 +4,7 @@ module Prefab
4
4
  MAX_SLEEP_SEC = 10
5
5
  BASE_SLEEP_SEC = 0.5
6
6
 
7
- attr_reader :account_id, :shared_cache, :stats, :namespace, :creds, :interceptor
7
+ attr_reader :account_id, :shared_cache, :stats, :namespace, :creds, :interceptor, :api_key
8
8
 
9
9
  def initialize(api_key: ENV['PREFAB_API_KEY'],
10
10
  logdev: nil,
@@ -20,6 +20,7 @@ module Prefab
20
20
  @local = local
21
21
  @stats = (stats || NoopStats.new)
22
22
  @shared_cache = (shared_cache || NoopCache.new)
23
+ @api_key = api_key
23
24
  @account_id = api_key.split("|")[0].to_i
24
25
  @namespace = namespace
25
26
 
@@ -1,26 +1,24 @@
1
1
  module Prefab
2
2
  class ConfigClient
3
3
  RECONNECT_WAIT = 5
4
- SUSPENDERS_RESET_LOCK_SEC = 20
5
- DEFAULT_CHECKPOINT_FREQ_SEC = 10
6
- DEFAULT_MAX_SUSPENDERS_AGE_SEC = 60
4
+ DEFAULT_CHECKPOINT_FREQ_SEC = 60
7
5
 
8
6
  def initialize(base_client, timeout)
9
7
  @base_client = base_client
10
8
  @timeout = timeout
11
9
  @initialization_lock = Concurrent::ReadWriteLock.new
12
10
 
13
- @suspenders_max_age_secs = (ENV["PREFAB_SUSPENDERS_MAX_AGE_SEC"] || DEFAULT_MAX_SUSPENDERS_AGE_SEC)
14
11
  @checkpoint_freq_secs = (ENV["PREFAB_DEFAULT_CHECKPOINT_FREQ_SEC"] || DEFAULT_CHECKPOINT_FREQ_SEC)
15
12
 
16
13
  @config_loader = Prefab::ConfigLoader.new(@base_client)
17
14
  @config_resolver = Prefab::ConfigResolver.new(@base_client, @config_loader)
18
15
 
19
16
  @initialization_lock.acquire_write_lock
17
+ @s3 = Aws::S3::Resource.new(region: 'us-east-1')
20
18
 
21
- load_or_save_checkpoint
22
- ensure_api_connection_started
23
- start_checkpointing_thread if has_real_cache?
19
+ load_checkpoint
20
+ start_api_connection_thread(@config_loader.highwater_mark)
21
+ start_checkpointing_thread
24
22
  end
25
23
 
26
24
  def get(prop)
@@ -69,10 +67,9 @@ module Prefab
69
67
  # Bootstrap out of the cache
70
68
  # returns the high-watermark of what was in the cache
71
69
  def load_checkpoint
72
- checkpoint = @base_client.shared_cache.read(checkpoint_cache_key)
73
-
74
- if checkpoint
75
- deltas = Prefab::ConfigDeltas.decode(checkpoint)
70
+ obj = @s3.bucket('prefab-cloud-checkpoints-prod').object(@base_client.api_key.gsub("|", "/"))
71
+ obj.get do |f|
72
+ deltas = Prefab::ConfigDeltas.decode(f)
76
73
  deltas.deltas.each do |delta|
77
74
  @config_loader.set(delta)
78
75
  end
@@ -80,33 +77,18 @@ module Prefab
80
77
  @base_client.stats.increment("prefab.config.checkpoint.load")
81
78
  @config_resolver.update
82
79
  finish_init!
83
- else
84
- @base_client.log_internal Logger::INFO, "No checkpoint"
85
80
  end
86
- end
87
81
 
88
- # Save off the config to a local cache as a backup
89
- #
90
- def save_checkpoint
91
- begin
92
- deltas = @config_resolver.export_api_deltas
93
- @base_client.log_internal Logger::DEBUG, "Save Checkpoint #{@config_loader.highwater_mark} Thread #{Thread.current.object_id}"
94
- @base_client.shared_cache.write(checkpoint_cache_key, Prefab::ConfigDeltas.encode(deltas))
95
- @base_client.shared_cache.write(checkpoint_highwater_cache_key, @config_loader.highwater_mark)
96
- @base_client.stats.increment("prefab.config.checkpoint.save")
97
- rescue StandardError => exn
98
- @base_client.log_internal Logger::INFO, "Issue Saving Checkpoint #{exn.message}"
99
- end
82
+ rescue Aws::S3::Errors::NoSuchKey
83
+ @base_client.log_internal Logger::INFO, "No checkpoint"
100
84
  end
101
85
 
102
- # A thread that saves current state to the cache, "checkpointing" it
86
+ # A thread that checks for a checkpoint
103
87
  def start_checkpointing_thread
104
88
  Thread.new do
105
89
  loop do
106
90
  begin
107
- load_or_save_checkpoint
108
-
109
- suspenders_if_needed
91
+ load_checkpoint
110
92
 
111
93
  started_at = Time.now
112
94
  delta = @checkpoint_freq_secs - (Time.now - started_at)
@@ -120,72 +102,6 @@ module Prefab
120
102
  end
121
103
  end
122
104
 
123
- # check what our shared highwater mark is
124
- # if it is higher than our own highwater mark, we must have missed something, so load the checkpoint
125
- # if it is lower than our own highwater mark, save a checkpoint
126
- # if everything is up to date, but the shared highwater mark is old, coordinate amongst other processes to have
127
- # one process "double check" by restarting the API thread
128
- def load_or_save_checkpoint
129
- shared_highwater_mark = get_shared_highwater_mark
130
- @base_client.log_internal Logger::DEBUG, "Checkpoint_if_needed highwater apx ahead/behind #{(@config_loader.highwater_mark - shared_highwater_mark) / (1000 * 10000)}"
131
-
132
- if shared_highwater_mark > @config_loader.highwater_mark
133
- @base_client.log_internal Logger::DEBUG, "We were behind, loading checkpoint"
134
- load_checkpoint
135
- elsif shared_highwater_mark < @config_loader.highwater_mark
136
- @base_client.log_internal Logger::DEBUG, "Saving off checkpoint"
137
- save_checkpoint
138
- end
139
- end
140
-
141
- def suspenders_if_needed
142
- if suspenders_age_is_old?
143
- if get_shared_lock?
144
- @base_client.log_internal Logger::DEBUG, "Suspenders > PREFAB_SUSPENDERS_MAX_AGE_SEC #{@suspenders_max_age_secs}. We have been chosen to run suspenders"
145
- reset_api_connection
146
- else
147
- @base_client.log_internal Logger::DEBUG, "Suspenders > PREFAB_SUSPENDERS_MAX_AGE_SEC #{@suspenders_max_age_secs}. Other process is running suspenders"
148
- end
149
- end
150
- end
151
-
152
- def suspenders_age_is_old?
153
- age = Time.now.to_i - get_checkpoint_suspenders
154
- @base_client.log_internal Logger::DEBUG, "checkpoint_suspenders_is_old? apx #{age}" if age > @suspenders_max_age_secs
155
- age > @suspenders_max_age_secs
156
- end
157
-
158
- def get_shared_highwater_mark
159
- (@base_client.shared_cache.read(checkpoint_highwater_cache_key) || 0).to_i
160
- end
161
-
162
- def get_checkpoint_suspenders
163
- (@base_client.shared_cache.read(suspenders_last_run_cache_key) || 0).to_i
164
- end
165
-
166
- def set_suspenders
167
- @base_client.shared_cache.write(suspenders_last_run_cache_key, Time.now.to_i)
168
- end
169
-
170
- def get_shared_lock?
171
- in_progess = @base_client.shared_cache.read(suspenders_reset_in_progress_cache_key)
172
- if in_progess.nil?
173
- @base_client.shared_cache.write(suspenders_reset_in_progress_cache_key, "true", { expires_in: SUSPENDERS_RESET_LOCK_SEC })
174
- true
175
- else
176
- false
177
- end
178
- end
179
-
180
- def ensure_api_connection_started
181
- reset_api_connection if @api_connection_thread.nil?
182
- end
183
-
184
- def reset_api_connection
185
- @api_connection_thread&.exit
186
- start_api_connection_thread(@config_loader.highwater_mark)
187
- end
188
-
189
105
  def finish_init!
190
106
  if @initialization_lock.write_locked?
191
107
  @initialization_lock.release_write_lock
@@ -200,7 +116,6 @@ module Prefab
200
116
  start_at_id: start_at_id)
201
117
  @base_client.log_internal Logger::DEBUG, "start api connection thread #{start_at_id}"
202
118
  @base_client.stats.increment("prefab.config.api.start")
203
- set_suspenders
204
119
  @api_connection_thread = Thread.new do
205
120
  while true do
206
121
  begin
@@ -220,26 +135,6 @@ module Prefab
220
135
  end
221
136
  end
222
137
  end
223
-
224
- def has_real_cache?
225
- @base_client.shared_cache.class != NoopCache
226
- end
227
-
228
- def checkpoint_cache_key
229
- @base_client.cache_key "config:checkpoint"
230
- end
231
-
232
- def suspenders_reset_in_progress_cache_key
233
- @base_client.cache_key "config:checkpoint:updating"
234
- end
235
-
236
- def checkpoint_highwater_cache_key
237
- @base_client.cache_key "config:checkpoint:highwater"
238
- end
239
-
240
- def suspenders_last_run_cache_key
241
- @base_client.cache_key "config:checkpoint:suspenders"
242
- end
243
138
  end
244
139
  end
245
140
 
@@ -14,4 +14,4 @@ require 'prefab/auth_interceptor'
14
14
  require 'prefab/noop_cache'
15
15
  require 'prefab/noop_stats'
16
16
  require 'prefab/murmer3'
17
-
17
+ require 'aws-sdk-s3'
@@ -2,16 +2,16 @@
2
2
  # DO NOT EDIT THIS FILE DIRECTLY
3
3
  # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
4
4
  # -*- encoding: utf-8 -*-
5
- # stub: prefab-cloud-ruby 0.0.22 ruby lib
5
+ # stub: prefab-cloud-ruby 0.0.23 ruby lib
6
6
 
7
7
  Gem::Specification.new do |s|
8
8
  s.name = "prefab-cloud-ruby".freeze
9
- s.version = "0.0.22"
9
+ s.version = "0.0.23"
10
10
 
11
11
  s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
12
12
  s.require_paths = ["lib".freeze]
13
13
  s.authors = ["Jeff Dwyer".freeze]
14
- s.date = "2018-05-03"
14
+ s.date = "2018-05-15"
15
15
  s.description = "RateLimits & Config as a service".freeze
16
16
  s.email = "jdwyer@prefab.cloud".freeze
17
17
  s.extra_rdoc_files = [
@@ -60,6 +60,7 @@ Gem::Specification.new do |s|
60
60
  if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
61
61
  s.add_runtime_dependency(%q<grpc>.freeze, ["~> 1.11.0"])
62
62
  s.add_runtime_dependency(%q<concurrent-ruby>.freeze, [">= 1.0.5", "~> 1.0"])
63
+ s.add_runtime_dependency(%q<aws-sdk-s3>.freeze, ["~> 1"])
63
64
  s.add_development_dependency(%q<grpc-tools>.freeze, ["~> 1.11.0"])
64
65
  s.add_development_dependency(%q<shoulda>.freeze, [">= 0"])
65
66
  s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12"])
@@ -69,6 +70,7 @@ Gem::Specification.new do |s|
69
70
  else
70
71
  s.add_dependency(%q<grpc>.freeze, ["~> 1.11.0"])
71
72
  s.add_dependency(%q<concurrent-ruby>.freeze, [">= 1.0.5", "~> 1.0"])
73
+ s.add_dependency(%q<aws-sdk-s3>.freeze, ["~> 1"])
72
74
  s.add_dependency(%q<grpc-tools>.freeze, ["~> 1.11.0"])
73
75
  s.add_dependency(%q<shoulda>.freeze, [">= 0"])
74
76
  s.add_dependency(%q<rdoc>.freeze, ["~> 3.12"])
@@ -79,6 +81,7 @@ Gem::Specification.new do |s|
79
81
  else
80
82
  s.add_dependency(%q<grpc>.freeze, ["~> 1.11.0"])
81
83
  s.add_dependency(%q<concurrent-ruby>.freeze, [">= 1.0.5", "~> 1.0"])
84
+ s.add_dependency(%q<aws-sdk-s3>.freeze, ["~> 1"])
82
85
  s.add_dependency(%q<grpc-tools>.freeze, ["~> 1.11.0"])
83
86
  s.add_dependency(%q<shoulda>.freeze, [">= 0"])
84
87
  s.add_dependency(%q<rdoc>.freeze, ["~> 3.12"])
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: prefab-cloud-ruby
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.22
4
+ version: 0.0.23
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeff Dwyer
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-05-03 00:00:00.000000000 Z
11
+ date: 2018-05-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: grpc
@@ -44,6 +44,20 @@ dependencies:
44
44
  - - "~>"
45
45
  - !ruby/object:Gem::Version
46
46
  version: '1.0'
47
+ - !ruby/object:Gem::Dependency
48
+ name: aws-sdk-s3
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - "~>"
52
+ - !ruby/object:Gem::Version
53
+ version: '1'
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: '1'
47
61
  - !ruby/object:Gem::Dependency
48
62
  name: grpc-tools
49
63
  requirement: !ruby/object:Gem::Requirement