xcflushd 1.0.0.rc2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,263 @@
1
+ module Xcflushd
2
+ # The error handling could be improved to try to avoid losing reports
3
+ # However, there are trade-offs to be made. Complex error handling can
4
+ # complicate a lot the code. Also, there are no guarantees that the code in
5
+ # the rescue clauses will be executed correctly. For example, if an smembers
6
+ # operations fails because Redis is not accessible, and the error handling
7
+ # consists of performing other operations to Redis, the error handling could
8
+ # fail too.
9
+ # Some characteristics of Redis, like the absence of rollbacks limit the
10
+ # kind of things we can do in case of error.
11
+ # In the future, we might explore other options like lua scripts or keeping a
12
+ # journal (in Redis or disk).
13
+ class Storage
14
+
15
+ # Some Redis operations might block the server for a long time if they need
16
+ # to operate on big collections of keys or values.
17
+ # For that reason, when using pipelines, instead of sending all the keys in
18
+ # a single pipeline, we send them in batches.
19
+ # If the batch is too big, we might block the server for a long time. If it
20
+ # is too little, we will waste time in network round-trips to the server.
21
+ REDIS_BATCH_KEYS = 500
22
+ private_constant :REDIS_BATCH_KEYS
23
+
24
+ RETRIEVING_REPORTS_ERROR = 'Reports cannot be retrieved.'.freeze
25
+ private_constant :RETRIEVING_REPORTS_ERROR
26
+
27
+ SOME_REPORTS_MISSING_ERROR = 'Some reports could not be retrieved'.freeze
28
+ private_constant :SOME_REPORTS_MISSING_ERROR
29
+
30
+ CLEANUP_ERROR = 'Failed to delete some keys that are no longer needed.'.freeze
31
+ private_constant :CLEANUP_ERROR
32
+
33
+ class RenewAuthError < Flusher::XcflushdError
34
+ def initialize(service_id, credentials)
35
+ super("Error while renewing the auth for service ID: #{service_id} "\
36
+ "and credentials: #{credentials}")
37
+ end
38
+ end
39
+
40
+ def initialize(storage, logger, storage_keys)
41
+ @storage = storage
42
+ @logger = logger
43
+ @storage_keys = storage_keys
44
+ end
45
+
46
+ # This performs a cleanup of the reports to be flushed.
47
+ # We can decide later whether it is better to leave this responsibility
48
+ # to the caller of the method.
49
+ #
50
+ # Returns an array of hashes where each of them has a service_id,
51
+ # credentials, and a usage. The usage is another hash where the keys are
52
+ # the metrics and the values are guaranteed to respond to to_i and to_s.
53
+ def reports_to_flush
54
+ # The Redis rename command overwrites the key with the new name if it
55
+ # exists. This means that if the rename operation fails in a flush cycle,
56
+ # and succeeds in a next one, the data that the key had in the first
57
+ # flush cycle will be lost.
58
+ # For that reason, every time we need to rename a key, we will use a
59
+ # unique suffix. This way, when the rename operation fails, the key
60
+ # will not be overwritten later, and we will be able to recover its
61
+ # content.
62
+ suffix = suffix_for_unique_naming
63
+
64
+ report_keys = report_keys_to_flush(suffix)
65
+ if report_keys.empty?
66
+ logger.warn "No reports available to flush"
67
+ report_keys
68
+ else
69
+ reports(report_keys, suffix)
70
+ end
71
+ end
72
+
73
+ def renew_auths(service_id, credentials, authorizations, auth_ttl)
74
+ hash_key = hash_key(:auth, service_id, credentials)
75
+
76
+ authorizations.each_slice(REDIS_BATCH_KEYS) do |authorizations_slice|
77
+ authorizations_slice.each do |metric, auth|
78
+ storage.hset(hash_key, metric, auth_value(auth))
79
+ end
80
+ end
81
+
82
+ set_auth_validity(service_id, credentials, auth_ttl)
83
+
84
+ rescue Redis::BaseError
85
+ raise RenewAuthError.new(service_id, credentials)
86
+ end
87
+
88
+ def report(reports)
89
+ reports.each do |report|
90
+ increase_usage(report)
91
+ add_to_set_keys_cached_reports(report)
92
+ end
93
+ end
94
+
95
+ private
96
+
97
+ attr_reader :storage, :logger, :storage_keys
98
+
99
+ def report_keys_to_flush(suffix)
100
+ begin
101
+ return [] if storage.scard(set_keys_cached_reports) == 0
102
+ storage.rename(set_keys_cached_reports,
103
+ set_keys_flushing_reports(suffix))
104
+ rescue Redis::BaseError
105
+ # We could not even start the process of getting the reports, so just
106
+ # log an error and return [].
107
+ logger.error(RETRIEVING_REPORTS_ERROR)
108
+ return []
109
+ end
110
+
111
+ flushing_reports = flushing_report_keys(suffix)
112
+
113
+ keys_with_flushing_prefix = flushing_reports.map do |key|
114
+ storage_keys.name_key_to_flush(key, suffix)
115
+ end
116
+
117
+ # Hash with old names as keys and new ones as values
118
+ key_names = Hash[flushing_reports.zip(keys_with_flushing_prefix)]
119
+ rename(key_names)
120
+
121
+ key_names.values
122
+ end
123
+
124
+ def flushing_report_keys(suffix)
125
+ res = storage.smembers(set_keys_flushing_reports(suffix))
126
+ rescue Redis::BaseError
127
+ logger.error(RETRIEVING_REPORTS_ERROR)
128
+ []
129
+ else
130
+ # We only delete the set if there is not an error. If there is an error,
131
+ # it's not deleted, so it can be recovered later.
132
+ delete([set_keys_flushing_reports(suffix)])
133
+ res
134
+ end
135
+
136
+ # Returns a report (hash with service_id, credentials, and usage) for each of
137
+ # the keys received.
138
+ def reports(keys_to_flush, suffix)
139
+ result = []
140
+
141
+ keys_to_flush.each_slice(REDIS_BATCH_KEYS) do |keys|
142
+ begin
143
+ usages = storage.pipelined { keys.each { |k| storage.hgetall(k) } }
144
+ rescue Redis::BaseError
145
+ # The reports in a batch where hgetall failed will not be reported
146
+ # now, but they will not be lost. They keys will not be deleted, so
147
+ # we will be able to retrieve them later and retry.
148
+ # We cannot know which ones failed because we are using a pipeline.
149
+ logger.error(SOME_REPORTS_MISSING_ERROR)
150
+ else
151
+ keys.each_with_index do |key, i|
152
+ # The usage could be empty if we failed to rename the key in the
153
+ # previous step. hgetall returns {} for keys that do not exist.
154
+ unless usages[i].empty?
155
+ service_id, creds = storage_keys.service_and_creds(key, suffix)
156
+ result << { service_id: service_id,
157
+ credentials: creds,
158
+ usage: usages[i] }
159
+ end
160
+ end
161
+ delete(keys)
162
+ end
163
+ end
164
+
165
+ result
166
+ end
167
+
168
+ def rename(keys)
169
+ keys.each_slice(REDIS_BATCH_KEYS) do |keys_slice|
170
+ begin
171
+ storage.pipelined do
172
+ keys_slice.each do |old_name, new_name|
173
+ storage.rename(old_name, new_name)
174
+ end
175
+ end
176
+ rescue Redis::BaseError
177
+ # The cached reports will not be reported now, but they will not be
178
+ # lost. They will be reported next time there are hits for that
179
+ # specific metric.
180
+ # We cannot know which ones failed because we are using a pipeline.
181
+ logger.warn(SOME_REPORTS_MISSING_ERROR)
182
+ end
183
+ end
184
+ end
185
+
186
+ def delete(keys)
187
+ tries ||= 3
188
+ storage.del(keys)
189
+ rescue Redis::BaseError
190
+ # Failing to delete certain keys could be problematic. That's why we
191
+ # retry in case the error is temporary, like a network hiccup.
192
+ #
193
+ # When we rename keys, we give them a unique suffix so they are not
194
+ # overwritten in the next cycle and we can retrieve their content
195
+ # later. On the other hand, when we can retrieve their content
196
+ # successfully, we delete them. The problem is that the delete operation
197
+ # can fail. When trying to recover contents of keys that failed to be
198
+ # renamed we'll not be able to distinguish these 2 cases:
199
+ # 1) The key is there because we decided not to delete it to retrieve
200
+ # its content later.
201
+ # 2) The key is there because the delete operation failed.
202
+ # We could take a look at the logs to figure out what happened, but of
203
+ # course that is not an ideal solution.
204
+ if tries > 0
205
+ tries -= 1
206
+ sleep(0.1)
207
+ retry
208
+ else
209
+ logger.error("#{CLEANUP_ERROR} Keys: #{keys}")
210
+ end
211
+ end
212
+
213
+ def set_auth_validity(service_id, credentials, auth_ttl)
214
+ # Redis does not allow us to set a TTL for hash key fields. TTLs can only
215
+ # be applied to the key containing the hash. This is not a problem
216
+ # because we always renew all the metrics of an application at the same
217
+ # time.
218
+ storage.expire(hash_key(:auth, service_id, credentials), auth_ttl)
219
+ end
220
+
221
+ def increase_usage(report)
222
+ hash_key = hash_key(:report, report[:service_id], report[:credentials])
223
+
224
+ report[:usage].each_slice(REDIS_BATCH_KEYS) do |usages|
225
+ usages.each do |usage|
226
+ metric, value = usage
227
+ storage.hincrby(hash_key, metric, value)
228
+ end
229
+ end
230
+ end
231
+
232
+ def add_to_set_keys_cached_reports(report)
233
+ hash_key = hash_key(:report, report[:service_id], report[:credentials])
234
+ storage.sadd(set_keys_cached_reports, hash_key)
235
+ end
236
+
237
+ def auth_value(auth)
238
+ if auth.authorized?
239
+ '1'.freeze
240
+ else
241
+ auth.reason ? "0:#{auth.reason}" : '0'.freeze
242
+ end
243
+ end
244
+
245
+ def suffix_for_unique_naming
246
+ "_#{Time.now.utc.strftime('%Y%m%d%H%M%S'.freeze)}"
247
+ end
248
+
249
+ def set_keys_flushing_reports(suffix)
250
+ "#{storage_keys::SET_KEYS_FLUSHING_REPORTS}#{suffix}"
251
+ end
252
+
253
+ def hash_key(type, service_id, credentials)
254
+ storage_keys.send("#{type}_hash_key", service_id, credentials)
255
+ end
256
+
257
+ def set_keys_cached_reports
258
+ storage_keys::SET_KEYS_CACHED_REPORTS
259
+ end
260
+
261
+ end
262
+
263
+ end
@@ -0,0 +1,113 @@
1
+ module Xcflushd
2
+
3
+ # This class defines the interface of the flusher with Redis. It defines how
4
+ # to build all the keys that contain cached reports and authorizations, and
5
+ # also, all the keys used by the pubsub mechanism.
6
+ class StorageKeys
7
+
8
+ # Note: Some of the keys and messages in this class contain the credentials
9
+ # needed to authenticate an application. Credentials always appear in
10
+ # sorted in alphabetical order. They need to be, otherwise, we could have
11
+ # several keys or messages that refer to the same credentials.
12
+
13
+ # Pubsub channel in which a client publishes for asking about the
14
+ # authorization status of an application.
15
+ AUTH_REQUESTS_CHANNEL = 'xc_channel_auth_requests'.freeze
16
+
17
+ # Set that contains the keys of the cached reports
18
+ SET_KEYS_CACHED_REPORTS = 'report_keys'.freeze
19
+
20
+ # Set that contains the keys of the cached reports to be flushed
21
+ SET_KEYS_FLUSHING_REPORTS = 'flushing_report_keys'.freeze
22
+
23
+ # Prefix of pubsub channels where the authorization statuses are published.
24
+ AUTH_RESPONSES_CHANNEL_PREFIX = 'xc_channel_auth_response:'.freeze
25
+ private_constant :AUTH_RESPONSES_CHANNEL_PREFIX
26
+
27
+ # Prefix to identify cached reports.
28
+ REPORT_KEY_PREFIX = 'report,'.freeze
29
+ private_constant :REPORT_KEY_PREFIX
30
+
31
+ # Prefix to identify cached reports that are ready to be flushed
32
+ KEY_TO_FLUSH_PREFIX = 'to_flush:'.freeze
33
+ private_constant :KEY_TO_FLUSH_PREFIX
34
+
35
+ class << self
36
+
37
+ # Returns the storage key that contains the cached authorizations for the
38
+ # given { service_id, credentials } pair.
39
+ def auth_hash_key(service_id, credentials)
40
+ hash_key(:auth, service_id, credentials)
41
+ end
42
+
43
+ # Returns the storage key that contains the cached reports for the given
44
+ # { service_id, credentials } pair.
45
+ def report_hash_key(service_id, credentials)
46
+ hash_key(:report, service_id, credentials)
47
+ end
48
+
49
+ # Pubsub channel to which the client subscribes to receive a response
50
+ # after asking for an authorization.
51
+ def pubsub_auths_resp_channel(service_id, credentials, metric)
52
+ AUTH_RESPONSES_CHANNEL_PREFIX +
53
+ "service_id:#{service_id}," +
54
+ "#{credentials.to_sorted_escaped_s}," +
55
+ "metric:#{metric}"
56
+ end
57
+
58
+ # Returns a hash that contains service_id, credentials, and metric from
59
+ # a message published in the pubsub channel for auth requests.
60
+ # Expected format of the message:
61
+ # service_id:<service_id>,<credentials>,metric:<metric>.
62
+ # With all the ',' and ':' in the values escaped.
63
+ # <credentials> contains the credentials needed for authentication
64
+ # separated by ','. For example: app_id:my_app_id,user_key:my_user_key.
65
+ def pubsub_auth_msg_2_auth_info(msg)
66
+ msg_split = msg.split(/(?<!\\),/)
67
+ service_id = msg_split.first.sub('service_id:'.freeze, ''.freeze)
68
+ creds = Credentials.from(
69
+ msg_split[1..-2].join(',').sub('credentials:'.freeze, ''.freeze))
70
+ metric = msg_split.last.sub('metric:'.freeze, ''.freeze)
71
+
72
+ res = { service_id: service_id, credentials: creds, metric: metric }
73
+ res.map do |k, v|
74
+ # Credentials are already unescaped
75
+ [k, v.is_a?(Credentials) ? v : v.gsub("\\,", ','.freeze)
76
+ .gsub("\\:", ':'.freeze)]
77
+ end.to_h
78
+ end
79
+
80
+ # Returns an array of size 2 with a service and the credentials encoded
81
+ # given a key marked as 'to be flushed' and its suffix.
82
+ def service_and_creds(key_to_flush, suffix)
83
+ escaped_service, escaped_creds =
84
+ key_to_flush.sub("#{KEY_TO_FLUSH_PREFIX}#{REPORT_KEY_PREFIX}", '')
85
+ .sub(suffix, '')
86
+ .split(/(?<!\\),/)
87
+
88
+ # escaped_service is a string with 'service_id:' followed by the escaped
89
+ # service ID. escaped_creds starts with 'credentials:' and is followed
90
+ # by the escaped credentials.
91
+ service = escaped_service
92
+ .sub('service_id:'.freeze, ''.freeze)
93
+ .gsub("\\,", ','.freeze).gsub("\\:", ':'.freeze)
94
+
95
+ creds = Credentials.from(escaped_creds.sub(
96
+ 'credentials:'.freeze, ''.freeze))
97
+
98
+ [service, creds]
99
+ end
100
+
101
+ def name_key_to_flush(report_key, suffix)
102
+ "#{KEY_TO_FLUSH_PREFIX}#{report_key}#{suffix}"
103
+ end
104
+
105
+ private
106
+
107
+ def hash_key(type, service_id, creds)
108
+ "#{type},service_id:#{service_id},#{creds.to_sorted_escaped_s}"
109
+ end
110
+
111
+ end
112
+ end
113
+ end
@@ -0,0 +1,12 @@
1
+ # Helper for default threading values.
2
+ require 'concurrent'
3
+
4
+ module Xcflushd
5
+ module Threading
6
+ def self.default_threads_value
7
+ cpus = Concurrent.processor_count
8
+ # default thread pool minimum is zero
9
+ return 0, cpus * 4
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,3 @@
1
+ module Xcflushd
2
+ VERSION = "1.0.0.rc2"
3
+ end
data/lib/xcflushd.rb ADDED
@@ -0,0 +1,11 @@
1
+ require 'xcflushd/logger'
2
+ require 'xcflushd/flusher'
3
+ require 'xcflushd/authorization'
4
+ require 'xcflushd/storage_keys'
5
+ require 'xcflushd/credentials'
6
+ require 'xcflushd/authorizer'
7
+ require 'xcflushd/reporter'
8
+ require 'xcflushd/storage'
9
+ require 'xcflushd/flusher_error_handler'
10
+ require 'xcflushd/priority_auth_renewer'
11
+ require 'xcflushd/version'
data/script/test ADDED
@@ -0,0 +1,10 @@
1
+ #!/bin/bash
2
+
3
+ SCRIPT_DIR=$(dirname "$(readlink -f $0)")
4
+
5
+ pushd ${SCRIPT_DIR} > /dev/null
6
+ export TEST_COVERAGE=1
7
+ bundle exec rake spec
8
+ STATUS=$?
9
+ popd > /dev/null
10
+ exit ${STATUS}
data/xcflushd.gemspec ADDED
@@ -0,0 +1,39 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'xcflushd/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "xcflushd"
8
+ spec.version = Xcflushd::VERSION
9
+ spec.authors = ["Alejandro Martinez Ruiz", "David Ortiz Lopez"]
10
+ spec.email = ["support@3scale.net"]
11
+
12
+ spec.summary = %q{Daemon for flushing XC reports to 3scale.}
13
+ spec.description = %q{Daemon for flushing XC reports to 3scale.}
14
+ spec.homepage = "https://github.com/3scale/xcflushd"
15
+
16
+ spec.license = "Apache-2.0"
17
+
18
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
19
+ spec.bindir = "exe"
20
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
21
+ spec.require_paths = ["lib"]
22
+
23
+ spec.required_ruby_version = '>= 2.1.0'
24
+
25
+ spec.add_runtime_dependency "3scale_client", "~> 2.10.0"
26
+ spec.add_runtime_dependency "gli", "= 2.14.0"
27
+ spec.add_runtime_dependency "redis", "= 3.3.1"
28
+ spec.add_runtime_dependency "hiredis", "= 0.6.1"
29
+ spec.add_runtime_dependency "concurrent-ruby", "1.0.2"
30
+ spec.add_runtime_dependency "net-http-persistent", "2.9.4"
31
+ spec.add_runtime_dependency "daemons", "= 1.2.4"
32
+
33
+ spec.add_development_dependency "bundler", "~> 1.12"
34
+ spec.add_development_dependency "rake", "~> 11.0"
35
+ spec.add_development_dependency "rspec", "~> 3.0"
36
+ spec.add_development_dependency "fakeredis", "~> 0.6.0"
37
+ spec.add_development_dependency "simplecov", "~> 0.12.0"
38
+ spec.add_development_dependency "rubocop", "~> 0.46.0"
39
+ end