apisonator 3.4.0 → 3.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/Gemfile.lock +1 -1
- data/Gemfile.on_prem.lock +1 -1
- data/lib/3scale/backend.rb +1 -0
- data/lib/3scale/backend/alert_limit.rb +5 -1
- data/lib/3scale/backend/alerts.rb +76 -36
- data/lib/3scale/backend/constants.rb +2 -0
- data/lib/3scale/backend/stats/aggregator.rb +30 -34
- data/lib/3scale/backend/stats/cleaner.rb +0 -6
- data/lib/3scale/backend/transactor/usage_report.rb +1 -1
- data/lib/3scale/backend/usage.rb +10 -3
- data/lib/3scale/backend/utilization.rb +83 -0
- data/lib/3scale/backend/version.rb +1 -1
- data/licenses.xml +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 78b328c64420fa58f4480eb39b555856dd9a032ba78eed32f084767988cea59d
|
4
|
+
data.tar.gz: de454b45677bc1c133eeda34c8fb4ebfe56253c4b4de851f1d192652c5baf999
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bf7613f456c9810217adfbdd5f345b86aa59d9abdccd44d308d846e7f271849d484601900348a4dcd4a16437d9e0376b7927e3675374a9be7a2af723c2820626
|
7
|
+
data.tar.gz: deb0823a0648f55bf24c24da6727afa4e62d2d5d92eeeee5545b9f6c8af38b2105bc8bb8caf3f7b28c187e020080b9b9f0b43721a42dfece2d6963d698a23e9a
|
data/CHANGELOG.md
CHANGED
@@ -2,6 +2,13 @@
|
|
2
2
|
|
3
3
|
Notable changes to Apisonator will be tracked in this document.
|
4
4
|
|
5
|
+
## 3.4.1 - 2021-06-16
|
6
|
+
|
7
|
+
### Changed
|
8
|
+
|
9
|
+
- Introduced performance optimizations around alerts detection
|
10
|
+
([#287](https://github.com/3scale/apisonator/pull/287)).
|
11
|
+
|
5
12
|
## 3.4.0 - 2021-06-14
|
6
13
|
|
7
14
|
### Added
|
data/Gemfile.lock
CHANGED
data/Gemfile.on_prem.lock
CHANGED
data/lib/3scale/backend.rb
CHANGED
@@ -44,6 +44,7 @@ require '3scale/backend/queue_storage'
|
|
44
44
|
require '3scale/backend/errors'
|
45
45
|
require '3scale/backend/stats'
|
46
46
|
require '3scale/backend/usage_limit'
|
47
|
+
require '3scale/backend/utilization'
|
47
48
|
require '3scale/backend/alerts'
|
48
49
|
require '3scale/backend/event_storage'
|
49
50
|
require '3scale/backend/worker'
|
@@ -9,7 +9,11 @@ module ThreeScale
|
|
9
9
|
attr_accessor :service_id, :value
|
10
10
|
|
11
11
|
def save
|
12
|
-
|
12
|
+
if valid?
|
13
|
+
res = storage.sadd(key_allowed_set(service_id), value.to_i)
|
14
|
+
Alerts::UsagesChecked.invalidate_for_service(service_id)
|
15
|
+
res
|
16
|
+
end
|
13
17
|
end
|
14
18
|
|
15
19
|
def to_hash
|
@@ -33,6 +33,11 @@ module ThreeScale
|
|
33
33
|
def key_current_id
|
34
34
|
'alerts/current_id'.freeze
|
35
35
|
end
|
36
|
+
|
37
|
+
def key_usage_already_checked(service_id, app_id)
|
38
|
+
prefix = key_prefix(service_id, app_id)
|
39
|
+
"#{prefix}usage_already_checked"
|
40
|
+
end
|
36
41
|
end
|
37
42
|
|
38
43
|
extend self
|
@@ -45,43 +50,82 @@ module ThreeScale
|
|
45
50
|
FIRST_ALERT_BIN = ALERT_BINS.first
|
46
51
|
RALERT_BINS = ALERT_BINS.reverse.freeze
|
47
52
|
|
48
|
-
|
49
|
-
|
53
|
+
# This class is useful to reduce the amount of information that we need to
|
54
|
+
# fetch from Redis to determine whether an alert should be raised.
|
55
|
+
# In summary, alerts are raised at the application level and we need to
|
56
|
+
# wait for 24h before raising a new one for the same level (ALERTS_BIN
|
57
|
+
# above).
|
58
|
+
#
|
59
|
+
# This class allows us to check all the usage limits once and then not
|
60
|
+
# check all of them again (just the ones in the report job) until:
|
61
|
+
# 1) A specific alert has expired (24h passed since it was triggered).
|
62
|
+
# 2) A new alert bin is enabled for the service.
|
63
|
+
class UsagesChecked
|
64
|
+
extend KeyHelpers
|
65
|
+
extend StorageHelpers
|
66
|
+
include Memoizer::Decorator
|
67
|
+
|
68
|
+
def self.need_to_check_all?(service_id, app_id)
|
69
|
+
!storage.exists(key_usage_already_checked(service_id, app_id))
|
70
|
+
end
|
71
|
+
memoize :need_to_check_all?
|
50
72
|
|
51
|
-
|
73
|
+
def self.mark_all_checked(service_id, app_id)
|
74
|
+
ttl = ALERT_BINS.map do |bin|
|
75
|
+
ttl = storage.ttl(key_already_notified(service_id, app_id, bin))
|
52
76
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
77
|
+
# Redis returns -2 if key does not exist, and -1 if it exists without
|
78
|
+
# a TTL (we know this should not happen for the alert bins).
|
79
|
+
# In those cases we should just set the TTL to the max (ALERT_TTL).
|
80
|
+
ttl >= 0 ? ttl : ALERT_TTL
|
81
|
+
end.min
|
57
82
|
|
58
|
-
|
59
|
-
|
60
|
-
max_record = nil
|
61
|
-
max = proc do |item|
|
62
|
-
if item.max_value > 0
|
63
|
-
utilization = item.current_value / item.max_value.to_f
|
64
|
-
|
65
|
-
if utilization > max_utilization
|
66
|
-
max_record = item
|
67
|
-
max_utilization = utilization
|
68
|
-
end
|
69
|
-
end
|
83
|
+
storage.setex(key_usage_already_checked(service_id, app_id), ttl, '1'.freeze)
|
84
|
+
Memoizer.clear(Memoizer.build_key(self, :need_to_check_all?, service_id, app_id))
|
70
85
|
end
|
71
86
|
|
72
|
-
|
87
|
+
def self.invalidate(service_id, app_id)
|
88
|
+
storage.del(key_usage_already_checked(service_id, app_id))
|
89
|
+
end
|
90
|
+
|
91
|
+
def self.invalidate_for_service(service_id)
|
92
|
+
app_ids = []
|
93
|
+
cursor = 0
|
94
|
+
|
95
|
+
loop do
|
96
|
+
cursor, ids = storage.sscan(
|
97
|
+
Application.applications_set_key(service_id), cursor, count: SCAN_SLICE
|
98
|
+
)
|
99
|
+
|
100
|
+
app_ids += ids
|
73
101
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
102
|
+
break if cursor.to_i == 0
|
103
|
+
end
|
104
|
+
|
105
|
+
invalidate_batch(service_id, app_ids)
|
106
|
+
end
|
107
|
+
|
108
|
+
def self.invalidate_batch(service_id, app_ids)
|
109
|
+
app_ids.each_slice(PIPELINED_SLICE_SIZE) do |ids|
|
110
|
+
keys = ids.map { |app_id| key_usage_already_checked(service_id, app_id) }
|
111
|
+
storage.del(keys)
|
112
|
+
end
|
78
113
|
end
|
114
|
+
private_class_method :invalidate_batch
|
115
|
+
end
|
116
|
+
|
117
|
+
def can_raise_more_alerts?(service_id, app_id)
|
118
|
+
allowed_bins = allowed_set_for_service(service_id).sort
|
119
|
+
|
120
|
+
return false if allowed_bins.empty?
|
79
121
|
|
80
|
-
|
122
|
+
# If the bin with the highest value has already been notified, there's
|
123
|
+
# no need to notify anything else.
|
124
|
+
not notified?(service_id, app_id, allowed_bins.last)
|
81
125
|
end
|
82
126
|
|
83
|
-
def update_utilization(service_id, app_id,
|
84
|
-
discrete = utilization_discrete(
|
127
|
+
def update_utilization(service_id, app_id, utilization)
|
128
|
+
discrete = utilization_discrete(utilization.ratio)
|
85
129
|
|
86
130
|
keys = alert_keys(service_id, app_id, discrete)
|
87
131
|
|
@@ -91,18 +135,19 @@ module ThreeScale
|
|
91
135
|
end
|
92
136
|
|
93
137
|
if already_alerted.nil? && allowed && discrete.to_i > 0
|
94
|
-
next_id, _ = storage.pipelined do
|
138
|
+
next_id, _, _ = storage.pipelined do
|
95
139
|
storage.incr(keys[:current_id])
|
96
140
|
storage.setex(keys[:already_notified], ALERT_TTL, "1")
|
141
|
+
UsagesChecked.invalidate(service_id, app_id)
|
97
142
|
end
|
98
143
|
|
99
144
|
alert = { :id => next_id,
|
100
145
|
:utilization => discrete,
|
101
|
-
:max_utilization =>
|
146
|
+
:max_utilization => utilization.ratio,
|
102
147
|
:application_id => app_id,
|
103
148
|
:service_id => service_id,
|
104
|
-
:timestamp =>
|
105
|
-
:limit =>
|
149
|
+
:timestamp => Time.now.utc,
|
150
|
+
:limit => utilization.to_s }
|
106
151
|
|
107
152
|
Backend::EventStorage::store(:alert, alert)
|
108
153
|
end
|
@@ -116,11 +161,6 @@ module ThreeScale
|
|
116
161
|
end || FIRST_ALERT_BIN
|
117
162
|
end
|
118
163
|
|
119
|
-
def formatted_limit(record)
|
120
|
-
"#{record.metric_name} per #{record.period}: "\
|
121
|
-
"#{record.current_value}/#{record.max_value}"
|
122
|
-
end
|
123
|
-
|
124
164
|
def allowed_set_for_service(service_id)
|
125
165
|
storage.smembers(key_allowed_set(service_id)).map(&:to_i) # Redis returns strings always
|
126
166
|
end
|
@@ -59,7 +59,7 @@ module ThreeScale
|
|
59
59
|
touched_apps = aggregate(transactions, current_bucket)
|
60
60
|
|
61
61
|
ApplicationEvents.generate(touched_apps.values)
|
62
|
-
update_alerts(
|
62
|
+
update_alerts(transactions)
|
63
63
|
begin
|
64
64
|
ApplicationEvents.ping
|
65
65
|
rescue ApplicationEvents::PingFailed => e
|
@@ -137,39 +137,35 @@ module ThreeScale
|
|
137
137
|
logger.info(MAX_BUCKETS_CREATED_MSG)
|
138
138
|
end
|
139
139
|
|
140
|
-
def update_alerts(
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
if max_utilization
|
168
|
-
Alerts.update_utilization(service_id,
|
169
|
-
values[:application_id],
|
170
|
-
max_utilization,
|
171
|
-
max_record,
|
172
|
-
current_timestamp)
|
140
|
+
def update_alerts(transactions)
|
141
|
+
transactions.group_by { |tx| tx.application_id }.each do |app_id, txs|
|
142
|
+
service_id = txs.first.service_id # All the txs of an app belong to the same service
|
143
|
+
|
144
|
+
# Finding the max utilization can be costly because it involves
|
145
|
+
# loading usage limits and current usages. That's why before that,
|
146
|
+
# we check if there are any alerts that can be raised.
|
147
|
+
next unless Alerts.can_raise_more_alerts?(service_id, app_id)
|
148
|
+
|
149
|
+
begin
|
150
|
+
max_utilization = if Alerts::UsagesChecked.need_to_check_all?(service_id, app_id)
|
151
|
+
Utilization.max_in_all_metrics(service_id, app_id).tap do
|
152
|
+
Alerts::UsagesChecked.mark_all_checked(service_id, app_id)
|
153
|
+
end
|
154
|
+
else
|
155
|
+
# metrics_ids here includes the metrics
|
156
|
+
# explicitly reported plus their parents in
|
157
|
+
# the hierarchy.
|
158
|
+
metric_ids = txs.map { |tx| tx.usage.keys }.flatten.uniq
|
159
|
+
Utilization.max_in_metrics(service_id, app_id, metric_ids)
|
160
|
+
end
|
161
|
+
rescue ApplicationNotFound
|
162
|
+
# The app could have been deleted at some point since the job
|
163
|
+
# was enqueued. No need to update alerts in that case.
|
164
|
+
next
|
165
|
+
end
|
166
|
+
|
167
|
+
if max_utilization && max_utilization.ratio > 0
|
168
|
+
Alerts.update_utilization(service_id, app_id, max_utilization)
|
173
169
|
end
|
174
170
|
end
|
175
171
|
end
|
@@ -36,12 +36,6 @@ module ThreeScale
|
|
36
36
|
KEY_SERVICES_TO_DELETE = 'set_with_services_marked_for_deletion'.freeze
|
37
37
|
private_constant :KEY_SERVICES_TO_DELETE
|
38
38
|
|
39
|
-
SLEEP_BETWEEN_SCANS = 0.01 # In seconds
|
40
|
-
private_constant :SLEEP_BETWEEN_SCANS
|
41
|
-
|
42
|
-
SCAN_SLICE = 500
|
43
|
-
private_constant :SCAN_SLICE
|
44
|
-
|
45
39
|
STATS_KEY_PREFIX = 'stats/'.freeze
|
46
40
|
private_constant :STATS_KEY_PREFIX
|
47
41
|
|
data/lib/3scale/backend/usage.rb
CHANGED
@@ -3,12 +3,19 @@ module ThreeScale
|
|
3
3
|
class Usage
|
4
4
|
class << self
|
5
5
|
def application_usage(application, timestamp)
|
6
|
-
usage(application, timestamp) do |metric_id, instance_period|
|
6
|
+
usage(application.usage_limits, timestamp) do |metric_id, instance_period|
|
7
7
|
Stats::Keys.application_usage_value_key(
|
8
8
|
application.service_id, application.id, metric_id, instance_period)
|
9
9
|
end
|
10
10
|
end
|
11
11
|
|
12
|
+
def application_usage_for_limits(application, timestamp, usage_limits)
|
13
|
+
usage(usage_limits, timestamp) do |metric_id, instance_period|
|
14
|
+
Stats::Keys.application_usage_value_key(
|
15
|
+
application.service_id, application.id, metric_id, instance_period)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
12
19
|
def is_set?(usage_str)
|
13
20
|
usage_str && usage_str[0] == '#'.freeze
|
14
21
|
end
|
@@ -25,7 +32,7 @@ module ThreeScale
|
|
25
32
|
|
26
33
|
private
|
27
34
|
|
28
|
-
def usage(
|
35
|
+
def usage(usage_limits, timestamp)
|
29
36
|
# The timestamp does not change, so we can generate all the
|
30
37
|
# instantiated periods just once.
|
31
38
|
# This is important. Without this, the code can generate many instance
|
@@ -33,7 +40,7 @@ module ThreeScale
|
|
33
40
|
# time.
|
34
41
|
instance_periods = Period::instance_periods_for_ts(timestamp)
|
35
42
|
|
36
|
-
pairs = metric_period_pairs
|
43
|
+
pairs = metric_period_pairs usage_limits
|
37
44
|
return {} if pairs.empty?
|
38
45
|
|
39
46
|
keys = pairs.map do |(metric_id, period)|
|
@@ -0,0 +1,83 @@
|
|
1
|
+
module ThreeScale
|
2
|
+
module Backend
|
3
|
+
class Utilization
|
4
|
+
include Comparable
|
5
|
+
|
6
|
+
attr_reader :metric_id, :period, :max_value, :current_value
|
7
|
+
|
8
|
+
def initialize(limit, current_value)
|
9
|
+
@metric_id = limit.metric_id
|
10
|
+
@period = limit.period
|
11
|
+
@max_value = limit.value
|
12
|
+
@current_value = current_value
|
13
|
+
@encoded = encoded(limit, current_value)
|
14
|
+
end
|
15
|
+
|
16
|
+
def ratio
|
17
|
+
return 0 if max_value == 0 # Disabled metric
|
18
|
+
current_value/max_value.to_f
|
19
|
+
end
|
20
|
+
|
21
|
+
# Returns in the format needed by the Alerts class.
|
22
|
+
def to_s
|
23
|
+
@encoded
|
24
|
+
end
|
25
|
+
|
26
|
+
def <=>(other)
|
27
|
+
# Consider "disabled" the lowest ones
|
28
|
+
if ratio == 0 && other.ratio == 0
|
29
|
+
return max_value <=> other.max_value
|
30
|
+
end
|
31
|
+
|
32
|
+
ratio <=> other.ratio
|
33
|
+
end
|
34
|
+
|
35
|
+
# Note: this can return nil
|
36
|
+
def self.max_in_all_metrics(service_id, app_id)
|
37
|
+
application = Backend::Application.load!(service_id, app_id)
|
38
|
+
|
39
|
+
usage = Usage.application_usage(application, Time.now.getutc)
|
40
|
+
|
41
|
+
status = Transactor::Status.new(service_id: service_id,
|
42
|
+
application: application,
|
43
|
+
values: usage)
|
44
|
+
|
45
|
+
# Preloads all the metric names to avoid fetching them one by one when
|
46
|
+
# generating the usage reports
|
47
|
+
application.load_metric_names
|
48
|
+
|
49
|
+
max = status.application_usage_reports.map do |usage_report|
|
50
|
+
Utilization.new(usage_report.usage_limit, usage_report.current_value)
|
51
|
+
end.max
|
52
|
+
|
53
|
+
# Avoid returning a utilization for disabled metrics
|
54
|
+
max && max.max_value > 0 ? max : nil
|
55
|
+
end
|
56
|
+
|
57
|
+
# Note: this can return nil
|
58
|
+
def self.max_in_metrics(service_id, app_id, metric_ids)
|
59
|
+
application = Backend::Application.load!(service_id, app_id)
|
60
|
+
|
61
|
+
limits = UsageLimit.load_for_affecting_metrics(
|
62
|
+
service_id, application.plan_id, metric_ids
|
63
|
+
)
|
64
|
+
|
65
|
+
usage = Usage.application_usage_for_limits(application, Time.now.getutc, limits)
|
66
|
+
|
67
|
+
max = limits.map do |limit|
|
68
|
+
Utilization.new(limit, usage[limit.period][limit.metric_id])
|
69
|
+
end.max
|
70
|
+
|
71
|
+
# Avoid returning a utilization for disabled metrics
|
72
|
+
max && max.max_value > 0 ? max : nil
|
73
|
+
end
|
74
|
+
|
75
|
+
private
|
76
|
+
|
77
|
+
def encoded(limit, current_value)
|
78
|
+
metric_name = Metric.load_name(limit.service_id, limit.metric_id)
|
79
|
+
"#{metric_name} per #{limit.period}: #{current_value}/#{limit.value}"
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
data/licenses.xml
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: apisonator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.4.
|
4
|
+
version: 3.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Ciganek
|
@@ -16,7 +16,7 @@ authors:
|
|
16
16
|
autorequire:
|
17
17
|
bindir: bin
|
18
18
|
cert_chain: []
|
19
|
-
date: 2021-06-
|
19
|
+
date: 2021-06-17 00:00:00.000000000 Z
|
20
20
|
dependencies: []
|
21
21
|
description: This gem provides a daemon that handles authorization and reporting of
|
22
22
|
web services managed by 3scale.
|
@@ -164,6 +164,7 @@ files:
|
|
164
164
|
- lib/3scale/backend/usage_limit.rb
|
165
165
|
- lib/3scale/backend/use_cases/provider_key_change_use_case.rb
|
166
166
|
- lib/3scale/backend/util.rb
|
167
|
+
- lib/3scale/backend/utilization.rb
|
167
168
|
- lib/3scale/backend/validators.rb
|
168
169
|
- lib/3scale/backend/validators/base.rb
|
169
170
|
- lib/3scale/backend/validators/key.rb
|