prefab-cloud-ruby 0.0.13 → 0.0.14
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/lib/prefab-cloud-ruby.rb +0 -1
- data/lib/prefab/client.rb +48 -3
- data/lib/prefab/config_client.rb +26 -25
- data/lib/prefab/config_loader.rb +2 -1
- data/lib/prefab/logger_client.rb +30 -12
- data/lib/prefab/ratelimit_client.rb +1 -14
- data/prefab-cloud-ruby.gemspec +3 -4
- data/test/.prefab.test.config.yaml +1 -0
- metadata +2 -3
- data/lib/prefab/retry.rb +0 -28
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 70b7e6905adede604544ce942c02d4b77c8e368b
|
4
|
+
data.tar.gz: 9b5091d9df705f1c8dd01ab7bc218ac0bcdada04
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 75fb25229c3523976b4048c4bbc220705486062620c8f51951ff41cb80c0b575cb4bbab1cf792366c108c553b94a00a0f89b594e328d725eeb7a58bce8990865
|
7
|
+
data.tar.gz: a8cc43a7a3e102bd4bfe131f93e3ff0eb0a389cc3e8ba10357d426aefd5608ebcee9269cc6ca228ae059ee0b8ec590e09fa7a68cb5f383c176254563df4e0817
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.14
|
data/lib/prefab-cloud-ruby.rb
CHANGED
data/lib/prefab/client.rb
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
module Prefab
|
2
2
|
class Client
|
3
|
+
|
4
|
+
MAX_SLEEP_SEC = 10
|
5
|
+
BASE_SLEEP_SEC = 0.5
|
6
|
+
|
3
7
|
attr_reader :account_id, :shared_cache, :stats, :namespace, :creds, :interceptor
|
4
8
|
|
5
9
|
def initialize(api_key: ENV['PREFAB_API_KEY'],
|
@@ -21,8 +25,8 @@ module Prefab
|
|
21
25
|
@namespace = namespace
|
22
26
|
|
23
27
|
@interceptor = AuthInterceptor.new(api_key)
|
24
|
-
|
25
28
|
@creds = GRPC::Core::ChannelCredentials.new(ssl_certs)
|
29
|
+
@stubs = {}
|
26
30
|
end
|
27
31
|
|
28
32
|
def channel
|
@@ -38,7 +42,7 @@ module Prefab
|
|
38
42
|
|
39
43
|
def config_client(timeout: 5.0)
|
40
44
|
@config_client ||= Prefab::ConfigClient.new(self, timeout)
|
41
|
-
@
|
45
|
+
@config_client.init
|
42
46
|
@config_client
|
43
47
|
end
|
44
48
|
|
@@ -51,12 +55,53 @@ module Prefab
|
|
51
55
|
end
|
52
56
|
|
53
57
|
def log(base_logger = @logger)
|
54
|
-
return @logger if !@config_init
|
55
58
|
@logger_client || Prefab::LoggerClient.new(self, base_logger)
|
56
59
|
end
|
57
60
|
|
61
|
+
def log_internal(level, msg)
|
62
|
+
log.log_for level, msg, "prefab"
|
63
|
+
end
|
64
|
+
|
65
|
+
def request(service, method, req_options: {}, params: {})
|
66
|
+
opts = { timeout: 10 }.merge(req_options)
|
67
|
+
|
68
|
+
attempts = 0
|
69
|
+
start_time = Time.now
|
70
|
+
|
71
|
+
begin
|
72
|
+
attempts += 1
|
73
|
+
return stub_for(service).send(method, *params)
|
74
|
+
rescue => exception
|
75
|
+
|
76
|
+
log_internal :warn, exception
|
77
|
+
|
78
|
+
if Time.now - start_time > opts[:timeout]
|
79
|
+
raise exception
|
80
|
+
end
|
81
|
+
sleep_seconds = [BASE_SLEEP_SEC * (2 ** (attempts - 1)), MAX_SLEEP_SEC].min
|
82
|
+
sleep_seconds = sleep_seconds * (0.5 * (1 + rand()))
|
83
|
+
sleep_seconds = [BASE_SLEEP_SEC, sleep_seconds].max
|
84
|
+
log_internal :info, "Sleep #{sleep_seconds} and Reset #{service} #{method}"
|
85
|
+
sleep sleep_seconds
|
86
|
+
reset!
|
87
|
+
retry
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
58
91
|
private
|
59
92
|
|
93
|
+
def reset!
|
94
|
+
@stubs.clear
|
95
|
+
reset_channel!
|
96
|
+
end
|
97
|
+
|
98
|
+
def stub_for(service)
|
99
|
+
@stubs[service] ||= service::Stub.new(nil,
|
100
|
+
nil,
|
101
|
+
channel_override: channel,
|
102
|
+
interceptors: [@interceptor])
|
103
|
+
end
|
104
|
+
|
60
105
|
def ssl_certs
|
61
106
|
ssl_certs = ""
|
62
107
|
Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*.pem"].each do |cert|
|
data/lib/prefab/config_client.rb
CHANGED
@@ -7,16 +7,23 @@ module Prefab
|
|
7
7
|
def initialize(base_client, timeout)
|
8
8
|
@base_client = base_client
|
9
9
|
@timeout = timeout
|
10
|
-
@
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
10
|
+
@initialized = false
|
11
|
+
end
|
12
|
+
|
13
|
+
def init
|
14
|
+
if !@initialized
|
15
|
+
@initialized = true
|
16
|
+
@config_loader = Prefab::ConfigLoader.new(@base_client)
|
17
|
+
@config_resolver = Prefab::ConfigResolver.new(@base_client, @config_loader)
|
18
|
+
start_at_id = load_checkpoint
|
19
|
+
start_api_connection_thread(start_at_id)
|
20
|
+
start_checkpointing_thread
|
21
|
+
start_suspenders_thread
|
22
|
+
end
|
16
23
|
end
|
17
24
|
|
18
25
|
def get(prop)
|
19
|
-
@config_resolver.get(prop)
|
26
|
+
@config_resolver.get(prop) if @config_resolver
|
20
27
|
end
|
21
28
|
|
22
29
|
def upsert(key, config_value, namespace = nil, previous_key = nil)
|
@@ -26,7 +33,8 @@ module Prefab
|
|
26
33
|
upsert_req = Prefab::UpsertRequest.new(config_delta: config_delta)
|
27
34
|
upsert_req.previous_key = previous_key if previous_key&.present?
|
28
35
|
|
29
|
-
|
36
|
+
@base_client.request Prefab::ConfigService, :upsert, req_options: {timeout: @timeout}, params: upsert_req
|
37
|
+
|
30
38
|
@config_loader.set(config_delta)
|
31
39
|
@config_loader.rm(previous_key) if previous_key&.present?
|
32
40
|
@config_resolver.update
|
@@ -35,7 +43,6 @@ module Prefab
|
|
35
43
|
def reset
|
36
44
|
@base_client.reset_channel!
|
37
45
|
@_stub = nil
|
38
|
-
@_stub_with_timeout = nil
|
39
46
|
end
|
40
47
|
|
41
48
|
def to_s
|
@@ -56,14 +63,6 @@ module Prefab
|
|
56
63
|
interceptors: [@base_client.interceptor])
|
57
64
|
end
|
58
65
|
|
59
|
-
def stub_with_timeout
|
60
|
-
@_stub_with_timeout = Prefab::ConfigService::Stub.new(nil,
|
61
|
-
nil,
|
62
|
-
channel_override: @base_client.channel,
|
63
|
-
timeout: @timeout,
|
64
|
-
interceptors: [@base_client.interceptor])
|
65
|
-
end
|
66
|
-
|
67
66
|
def cache_key
|
68
67
|
"prefab:config:checkpoint"
|
69
68
|
end
|
@@ -77,14 +76,14 @@ module Prefab
|
|
77
76
|
if checkpoint
|
78
77
|
deltas = Prefab::ConfigDeltas.decode(checkpoint)
|
79
78
|
deltas.deltas.each do |delta|
|
80
|
-
@base_client.
|
79
|
+
@base_client.log_internal :debug, "checkpoint set #{delta.key} #{delta.value.int} #{delta.value.string} #{delta.id} "
|
81
80
|
@config_loader.set(delta)
|
82
81
|
start_at_id = [delta.id, start_at_id].max
|
83
82
|
end
|
84
|
-
@base_client.
|
83
|
+
@base_client.log_internal :info, "Found checkpoint with highwater id #{start_at_id}"
|
85
84
|
@config_resolver.update
|
86
85
|
else
|
87
|
-
@base_client.
|
86
|
+
@base_client.log_internal :info, "No checkpoint"
|
88
87
|
end
|
89
88
|
|
90
89
|
start_at_id
|
@@ -101,10 +100,10 @@ module Prefab
|
|
101
100
|
sleep(delta)
|
102
101
|
end
|
103
102
|
deltas = @config_resolver.export_api_deltas
|
104
|
-
@base_client.
|
103
|
+
@base_client.log_internal :debug, "Save Checkpoint #{deltas.deltas.map {|d| d.id}.max} Thread #{Thread.current.object_id}"
|
105
104
|
@base_client.shared_cache.write(cache_key, Prefab::ConfigDeltas.encode(deltas))
|
106
105
|
rescue StandardError => exn
|
107
|
-
@base_client.
|
106
|
+
@base_client.log_internal :info, "Issue Checkpointing #{exn.message}"
|
108
107
|
end
|
109
108
|
end
|
110
109
|
end
|
@@ -126,7 +125,7 @@ module Prefab
|
|
126
125
|
@config_resolver.update
|
127
126
|
end
|
128
127
|
rescue => e
|
129
|
-
@base_client.
|
128
|
+
@base_client.log_internal :info, ("config client encountered #{e.message} pausing #{RECONNECT_WAIT}")
|
130
129
|
reset
|
131
130
|
sleep(RECONNECT_WAIT)
|
132
131
|
end
|
@@ -145,7 +144,9 @@ module Prefab
|
|
145
144
|
started_at = Time.now
|
146
145
|
config_req = Prefab::ConfigServicePointer.new(account_id: @base_client.account_id,
|
147
146
|
start_at_id: start_at_suspenders)
|
148
|
-
|
147
|
+
|
148
|
+
resp = @base_client.request Prefab::ConfigService, :get_config, req_options: {timeout: @timeout}, params: config_req
|
149
|
+
|
149
150
|
resp.each do |r|
|
150
151
|
r.deltas.each do |delta|
|
151
152
|
@config_loader.set(delta)
|
@@ -155,7 +156,7 @@ module Prefab
|
|
155
156
|
rescue GRPC::DeadlineExceeded
|
156
157
|
# Ignore. This is a streaming endpoint, but we only need a single response
|
157
158
|
rescue => e
|
158
|
-
@base_client.
|
159
|
+
@base_client.log_internal :info, "Suspenders encountered an issue #{e.message}"
|
159
160
|
end
|
160
161
|
|
161
162
|
delta = SUSPENDERS_FREQ_SEC - (Time.now - started_at)
|
data/lib/prefab/config_loader.rb
CHANGED
@@ -62,9 +62,10 @@ module Prefab
|
|
62
62
|
|
63
63
|
def load(filename)
|
64
64
|
if File.exist? filename
|
65
|
+
@base_client.log_internal :warn, "Load #{filename}"
|
65
66
|
YAML.load_file(filename)
|
66
67
|
else
|
67
|
-
@base_client.
|
68
|
+
@base_client.log_internal :warn, "No file #{filename}"
|
68
69
|
{}
|
69
70
|
end
|
70
71
|
end
|
data/lib/prefab/logger_client.rb
CHANGED
@@ -1,40 +1,58 @@
|
|
1
1
|
module Prefab
|
2
2
|
class LoggerClient
|
3
3
|
|
4
|
+
SEP = ".".freeze
|
5
|
+
BASE = "log_level".freeze
|
6
|
+
|
4
7
|
def initialize(base_client, base_logger)
|
5
8
|
@base_client = base_client
|
6
9
|
@base_logger = base_logger
|
7
10
|
end
|
8
11
|
|
9
|
-
def info msg
|
10
|
-
pf_log :info, msg, caller_locations(1, 1)[0]
|
11
|
-
end
|
12
|
-
|
13
12
|
def debug msg
|
14
13
|
pf_log :debug, msg, caller_locations(1, 1)[0]
|
15
14
|
end
|
16
15
|
|
16
|
+
def info msg
|
17
|
+
pf_log :info, msg, caller_locations(1, 1)[0]
|
18
|
+
end
|
19
|
+
|
17
20
|
def warn msg
|
18
21
|
pf_log :warn, msg, caller_locations(1, 1)[0]
|
19
22
|
end
|
20
23
|
|
21
|
-
def
|
22
|
-
|
24
|
+
def error msg
|
25
|
+
pf_log :error, msg, caller_locations(1, 1)[0]
|
26
|
+
end
|
27
|
+
|
28
|
+
def log_for level, msg, loc
|
29
|
+
pf_log_internal level, msg, "", loc
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def pf_log(level, msg, loc)
|
35
|
+
pf_log_internal level, msg, loc.absolute_path, loc.base_label
|
36
|
+
end
|
37
|
+
|
38
|
+
def pf_log_internal(level, msg, absolute_path, base_label)
|
39
|
+
|
40
|
+
path = absolute_path + ""
|
23
41
|
path.slice! Dir.pwd
|
24
42
|
|
25
|
-
path = "#{path.gsub("/",
|
26
|
-
path.slice!
|
43
|
+
path = "#{path.gsub("/", SEP).gsub(".rb", "")}#{SEP}#{base_label}"
|
44
|
+
path.slice! SEP
|
27
45
|
|
28
|
-
|
29
|
-
|
30
|
-
path.split(":").inject([base]) do |memo, n|
|
46
|
+
closest_log_level_match = @base_client.config_client.get(BASE) || :warn
|
47
|
+
path.split(SEP).inject([BASE]) do |memo, n|
|
31
48
|
memo << n
|
32
|
-
val = @base_client.config_client.get memo.join(
|
49
|
+
val = @base_client.config_client.get memo.join(SEP)
|
33
50
|
unless val.nil?
|
34
51
|
closest_log_level_match = val
|
35
52
|
end
|
36
53
|
memo
|
37
54
|
end
|
55
|
+
|
38
56
|
if val(closest_log_level_match) <= val(level)
|
39
57
|
@base_logger.unknown "#{level.to_s.upcase.ljust(5)} #{path} #{msg}"
|
40
58
|
end
|
@@ -26,7 +26,7 @@ module Prefab
|
|
26
26
|
allow_partial_response: allow_partial_response
|
27
27
|
)
|
28
28
|
|
29
|
-
result =
|
29
|
+
result = @base_client.request Prefab::RateLimitService, :limit_check, req_options: {timeout: @timeout}, params: req
|
30
30
|
|
31
31
|
reset = result.limit_reset_at
|
32
32
|
@base_client.shared_cache.write(expiry_cache_key, reset) unless reset < 1 # protobuf default int to 0
|
@@ -41,19 +41,6 @@ module Prefab
|
|
41
41
|
|
42
42
|
private
|
43
43
|
|
44
|
-
def reset
|
45
|
-
@base_client.reset_channel!
|
46
|
-
@_stub = nil
|
47
|
-
end
|
48
|
-
|
49
|
-
def stub
|
50
|
-
@_stub ||= Prefab::RateLimitService::Stub.new(nil,
|
51
|
-
nil,
|
52
|
-
channel_override: @base_client.channel,
|
53
|
-
timeout: @timeout,
|
54
|
-
interceptors: [@base_client.interceptor])
|
55
|
-
end
|
56
|
-
|
57
44
|
def handle_error(e, on_error, groups)
|
58
45
|
@base_client.stats.increment("prefab.ratelimit.error", tags: ["type:limit"])
|
59
46
|
|
data/prefab-cloud-ruby.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: prefab-cloud-ruby 0.0.
|
5
|
+
# stub: prefab-cloud-ruby 0.0.14 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "prefab-cloud-ruby".freeze
|
9
|
-
s.version = "0.0.
|
9
|
+
s.version = "0.0.14"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib".freeze]
|
13
13
|
s.authors = ["Jeff Dwyer".freeze]
|
14
|
-
s.date = "2018-02-
|
14
|
+
s.date = "2018-02-20"
|
15
15
|
s.description = "RateLimits & Config as a service".freeze
|
16
16
|
s.email = "jdwyer@prefab.cloud".freeze
|
17
17
|
s.extra_rdoc_files = [
|
@@ -37,7 +37,6 @@ Gem::Specification.new do |s|
|
|
37
37
|
"lib/prefab/noop_cache.rb",
|
38
38
|
"lib/prefab/noop_stats.rb",
|
39
39
|
"lib/prefab/ratelimit_client.rb",
|
40
|
-
"lib/prefab/retry.rb",
|
41
40
|
"lib/prefab_pb.rb",
|
42
41
|
"lib/prefab_services_pb.rb",
|
43
42
|
"prefab-cloud-ruby.gemspec",
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: prefab-cloud-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.14
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jeff Dwyer
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-02-
|
11
|
+
date: 2018-02-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: grpc
|
@@ -154,7 +154,6 @@ files:
|
|
154
154
|
- lib/prefab/noop_cache.rb
|
155
155
|
- lib/prefab/noop_stats.rb
|
156
156
|
- lib/prefab/ratelimit_client.rb
|
157
|
-
- lib/prefab/retry.rb
|
158
157
|
- lib/prefab_pb.rb
|
159
158
|
- lib/prefab_services_pb.rb
|
160
159
|
- prefab-cloud-ruby.gemspec
|
data/lib/prefab/retry.rb
DELETED
@@ -1,28 +0,0 @@
|
|
1
|
-
class Retry
|
2
|
-
MAX_SLEEP_SEC = 10
|
3
|
-
BASE_SLEEP_SEC = 0.5
|
4
|
-
|
5
|
-
# GRPC Generally handles timeouts for us
|
6
|
-
# but if the connection is broken we want to retry up until the timeout
|
7
|
-
def self.it(stub_factory, rpc, req, timeout, reset)
|
8
|
-
attempts = 0
|
9
|
-
start_time = Time.now
|
10
|
-
|
11
|
-
begin
|
12
|
-
attempts += 1
|
13
|
-
return stub_factory.call.send(rpc, req)
|
14
|
-
rescue => exception
|
15
|
-
|
16
|
-
if Time.now - start_time > timeout
|
17
|
-
raise exception
|
18
|
-
end
|
19
|
-
sleep_seconds = [BASE_SLEEP_SEC * (2 ** (attempts - 1)), MAX_SLEEP_SEC].min
|
20
|
-
sleep_seconds = sleep_seconds * (0.5 * (1 + rand()))
|
21
|
-
sleep_seconds = [BASE_SLEEP_SEC, sleep_seconds].max
|
22
|
-
puts "Sleep #{sleep_seconds} and Reset"
|
23
|
-
sleep sleep_seconds
|
24
|
-
reset.call
|
25
|
-
retry
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|