apisonator 3.0.1.1 → 3.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +20 -0
- data/Gemfile.lock +1 -1
- data/Gemfile.on_prem.lock +1 -1
- data/lib/3scale/backend/configuration.rb +1 -3
- data/lib/3scale/backend/listener_metrics.rb +65 -8
- data/lib/3scale/backend/transactor/notify_batcher.rb +6 -2
- data/lib/3scale/backend/transactor/notify_job.rb +37 -17
- data/lib/3scale/backend/version.rb +1 -1
- data/licenses.xml +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: edd1887f6aa71c17fb5279f98988c9a3b5e631e80bc1f4142b6619b5f5e24f48
|
4
|
+
data.tar.gz: b6478d14e88a177480396a7d46d5f4e33cea683aea17303175ad18ae646e546f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 944b75385817abc4d2f8828c61b2301d5cdfbb6c1bb681b7a3165590f23ad1b6c80451fcf579b8f3cf0010fe5a95e65605998f5d8ec94f1ff4ce690556b3011a
|
7
|
+
data.tar.gz: f3f44b1b2ebfe6c8fcbbf36b90dd44e5a7875d997a36b97288417472fa4156689508bfc95afdf196a7a168f768aa6e38521502b93078f128994959d184c1cd37
|
data/CHANGELOG.md
CHANGED
@@ -2,6 +2,26 @@
|
|
2
2
|
|
3
3
|
Notable changes to Apisonator will be tracked in this document.
|
4
4
|
|
5
|
+
## 3.1.0 - 2020-10-14
|
6
|
+
|
7
|
+
### Added
|
8
|
+
|
9
|
+
- Prometheus metrics for the internal API
|
10
|
+
([#236](https://github.com/3scale/apisonator/pull/236)).
|
11
|
+
- Docs with a detailed explanation about how counter updates are performed
|
12
|
+
([#239](https://github.com/3scale/apisonator/pull/239)).
|
13
|
+
|
14
|
+
### Changed
|
15
|
+
|
16
|
+
- NotifyJobs are run only when the service ID is explicitly defined
|
17
|
+
([#238](https://github.com/3scale/apisonator/pull/238)).
|
18
|
+
|
19
|
+
### Fixed
|
20
|
+
|
21
|
+
- Fixed corner case that raised "TransactionTimestampNotWithinRange" in notify
|
22
|
+
jobs ([#235](https://github.com/3scale/apisonator/pull/235)).
|
23
|
+
|
24
|
+
|
5
25
|
## 3.0.1.1 - 2020-07-28
|
6
26
|
|
7
27
|
### Changed
|
data/Gemfile.lock
CHANGED
data/Gemfile.on_prem.lock
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require '3scale/backend/configuration/loader'
|
2
2
|
require '3scale/backend/environment'
|
3
3
|
require '3scale/backend/configurable'
|
4
|
+
require '3scale/backend/errors'
|
4
5
|
|
5
6
|
module ThreeScale
|
6
7
|
module Backend
|
@@ -77,9 +78,6 @@ module ThreeScale
|
|
77
78
|
master_metrics = [:transactions, :transactions_authorize]
|
78
79
|
config.master.metrics = Struct.new(*master_metrics).new
|
79
80
|
|
80
|
-
# Default config
|
81
|
-
config.master_service_id = 1
|
82
|
-
|
83
81
|
# This setting controls whether the listener can create event buckets in
|
84
82
|
# Redis. We do not want all the listeners creating buckets yet, as we do
|
85
83
|
# not know exactly the rate at which we can send events to Kinesis
|
@@ -4,14 +4,34 @@ require 'rack'
|
|
4
4
|
module ThreeScale
|
5
5
|
module Backend
|
6
6
|
class ListenerMetrics
|
7
|
-
|
7
|
+
AUTH_AND_REPORT_REQUEST_TYPES = {
|
8
8
|
'/transactions/authorize.xml' => 'authorize',
|
9
9
|
'/transactions/oauth_authorize.xml' => 'authorize_oauth',
|
10
10
|
'/transactions/authrep.xml' => 'authrep',
|
11
11
|
'/transactions/oauth_authrep.xml' => 'authrep_oauth',
|
12
12
|
'/transactions.xml' => 'report'
|
13
13
|
}
|
14
|
-
private_constant :
|
14
|
+
private_constant :AUTH_AND_REPORT_REQUEST_TYPES
|
15
|
+
|
16
|
+
INTERNAL_API_PATHS = [
|
17
|
+
[/\/services\/.*\/alert_limits/, 'alerts'],
|
18
|
+
[/\/services\/.*\/applications\/.*\/keys/, 'application_keys'],
|
19
|
+
[/\/services\/.*\/applications\/.*\/referrer_filters/, 'application_referrer_filters'],
|
20
|
+
[/\/services\/.*\/applications/, 'applications'],
|
21
|
+
[/\/services\/.*\/errors/, 'errors'],
|
22
|
+
[/\/events/, 'events'],
|
23
|
+
[/\/services\/.*\/metrics/, 'metrics'],
|
24
|
+
[/\/service_tokens/, 'service_tokens'],
|
25
|
+
[/\/services/, 'services'],
|
26
|
+
[/\/services\/.*\/stats/, 'stats'],
|
27
|
+
[/\/services\/.*\/plans\/.*\/usagelimits/, 'usage_limits'],
|
28
|
+
[/\/services\/.*\/applications\/.*\/utilization/, 'utilization'],
|
29
|
+
].freeze
|
30
|
+
private_constant :INTERNAL_API_PATHS
|
31
|
+
|
32
|
+
# Most requests will be under 100ms, so use a higher granularity from there
|
33
|
+
TIME_BUCKETS = [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.25, 0.5, 0.75, 1]
|
34
|
+
private_constant :TIME_BUCKETS
|
15
35
|
|
16
36
|
class << self
|
17
37
|
ERRORS_4XX_TO_TRACK = Set[403, 404, 409].freeze
|
@@ -27,9 +47,12 @@ module ThreeScale
|
|
27
47
|
end
|
28
48
|
|
29
49
|
def report_resp_code(path, resp_code)
|
30
|
-
|
50
|
+
req_type = req_type(path)
|
51
|
+
prometheus_group = prometheus_group(req_type)
|
52
|
+
|
53
|
+
Yabeda.send(prometheus_group).response_codes.increment(
|
31
54
|
{
|
32
|
-
request_type:
|
55
|
+
request_type: req_type,
|
33
56
|
resp_code: code_group(resp_code)
|
34
57
|
},
|
35
58
|
by: 1
|
@@ -37,8 +60,11 @@ module ThreeScale
|
|
37
60
|
end
|
38
61
|
|
39
62
|
def report_response_time(path, request_time)
|
40
|
-
|
41
|
-
|
63
|
+
req_type = req_type(path)
|
64
|
+
prometheus_group = prometheus_group(req_type)
|
65
|
+
|
66
|
+
Yabeda.send(prometheus_group).response_times.measure(
|
67
|
+
{ request_type: req_type },
|
42
68
|
request_time
|
43
69
|
)
|
44
70
|
end
|
@@ -69,8 +95,21 @@ module ThreeScale
|
|
69
95
|
comment 'Response times'
|
70
96
|
unit :seconds
|
71
97
|
tags %i[request_type]
|
72
|
-
|
73
|
-
|
98
|
+
buckets TIME_BUCKETS
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
group :apisonator_listener_internal_api do
|
103
|
+
counter :response_codes do
|
104
|
+
comment 'Response codes'
|
105
|
+
tags %i[request_type resp_code]
|
106
|
+
end
|
107
|
+
|
108
|
+
histogram :response_times do
|
109
|
+
comment 'Response times'
|
110
|
+
unit :seconds
|
111
|
+
tags %i[request_type]
|
112
|
+
buckets TIME_BUCKETS
|
74
113
|
end
|
75
114
|
end
|
76
115
|
end
|
@@ -93,6 +132,24 @@ module ThreeScale
|
|
93
132
|
'unknown'.freeze
|
94
133
|
end
|
95
134
|
end
|
135
|
+
|
136
|
+
def req_type(path)
|
137
|
+
AUTH_AND_REPORT_REQUEST_TYPES[path] || internal_api_req_type(path)
|
138
|
+
end
|
139
|
+
|
140
|
+
def internal_api_req_type(path)
|
141
|
+
(_regex, type) = INTERNAL_API_PATHS.find { |(regex, _)| regex.match path }
|
142
|
+
type
|
143
|
+
end
|
144
|
+
|
145
|
+
# Returns the group as defined in .define_metrics
|
146
|
+
def prometheus_group(request_type)
|
147
|
+
if AUTH_AND_REPORT_REQUEST_TYPES.values.include? request_type
|
148
|
+
:apisonator_listener
|
149
|
+
else
|
150
|
+
:apisonator_listener_internal_api
|
151
|
+
end
|
152
|
+
end
|
96
153
|
end
|
97
154
|
end
|
98
155
|
end
|
@@ -30,9 +30,13 @@ module ThreeScale
|
|
30
30
|
end
|
31
31
|
|
32
32
|
def notify(provider_key, usage)
|
33
|
-
#
|
33
|
+
# We need the master service ID to report its metrics. If it's not
|
34
|
+
# set, we don't need to notify anything.
|
35
|
+
# Batch several notifications together so that we can process just one
|
34
36
|
# job for a group of them.
|
35
|
-
|
37
|
+
unless configuration.master_service_id.to_s.empty?
|
38
|
+
notify_batch(provider_key, usage)
|
39
|
+
end
|
36
40
|
end
|
37
41
|
|
38
42
|
def notify_batch(provider_key, usage)
|
@@ -7,8 +7,6 @@ module ThreeScale
|
|
7
7
|
extend Configurable
|
8
8
|
@queue = :main
|
9
9
|
|
10
|
-
InvalidMasterServiceId = Class.new(ThreeScale::Backend::Error)
|
11
|
-
|
12
10
|
class << self
|
13
11
|
def perform_logged(provider_key, usage, timestamp, _enqueue_time)
|
14
12
|
application_id = Application.load_id_by_key(master_service_id, provider_key)
|
@@ -16,12 +14,42 @@ module ThreeScale
|
|
16
14
|
if application_id && Application.exists?(master_service_id, application_id)
|
17
15
|
master_metrics = Metric.load_all(master_service_id)
|
18
16
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
17
|
+
begin
|
18
|
+
ProcessJob.perform([{
|
19
|
+
service_id: master_service_id,
|
20
|
+
application_id: application_id,
|
21
|
+
timestamp: timestamp,
|
22
|
+
usage: master_metrics.process_usage(usage)
|
23
|
+
}])
|
24
|
+
rescue MetricInvalid => e
|
25
|
+
# This happens when the master account in Porta does not have
|
26
|
+
# the notify metrics defined (by default "transactions" and
|
27
|
+
# "transactions/authorize"). These metrics need to be created in
|
28
|
+
# Porta, Apisonator does not have a way to guarantee that
|
29
|
+
# they're defined.
|
30
|
+
# Notice that this rescue prevents the job from being retried.
|
31
|
+
# Apisonator can't know when the metrics will be created (if
|
32
|
+
# ever) so it's better to log the error rather than retrying
|
33
|
+
# these jobs for an undefined period of time.
|
34
|
+
Worker.logger.notify(e)
|
35
|
+
return [false, "#{e}"]
|
36
|
+
rescue TransactionTimestampNotWithinRange => e
|
37
|
+
# This is very unlikely to happen. The timestamps in a notify
|
38
|
+
# job are not set by users, they are set by the listeners. If
|
39
|
+
# this error happens it might mean that:
|
40
|
+
# a) The worker started processing this job way after the
|
41
|
+
# listener produced it. This can happen for example if we make
|
42
|
+
# some requests to a listener with no workers. The listeners
|
43
|
+
# will enqueue some notify jobs. If we start a worker hours
|
44
|
+
# later, we might see this error.
|
45
|
+
# b) There's some kind of clock skew issue.
|
46
|
+
# c) There's a bug.
|
47
|
+
#
|
48
|
+
# We can't raise here, because then, the job will be retried,
|
49
|
+
# but it's going to fail always if it has an old timestamp.
|
50
|
+
Worker.logger.notify(e)
|
51
|
+
return [false, "#{provider_key} #{application_id} #{e}"]
|
52
|
+
end
|
25
53
|
end
|
26
54
|
[true, "#{provider_key} #{application_id || '--'}"]
|
27
55
|
end
|
@@ -29,15 +57,7 @@ module ThreeScale
|
|
29
57
|
private
|
30
58
|
|
31
59
|
def master_service_id
|
32
|
-
|
33
|
-
|
34
|
-
unless value
|
35
|
-
raise InvalidMasterServiceId,
|
36
|
-
"Can't find master service id. Make sure the \"master_service_id\" "\
|
37
|
-
'configuration value is set correctly'
|
38
|
-
end
|
39
|
-
|
40
|
-
value.to_s
|
60
|
+
configuration.master_service_id.to_s
|
41
61
|
end
|
42
62
|
end
|
43
63
|
end
|
data/licenses.xml
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: apisonator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Ciganek
|
@@ -16,7 +16,7 @@ authors:
|
|
16
16
|
autorequire:
|
17
17
|
bindir: bin
|
18
18
|
cert_chain: []
|
19
|
-
date: 2020-
|
19
|
+
date: 2020-10-14 00:00:00.000000000 Z
|
20
20
|
dependencies: []
|
21
21
|
description: This gem provides a daemon that handles authorization and reporting of
|
22
22
|
web services managed by 3scale.
|