apisonator 3.0.1.1 → 3.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +74 -0
- data/Gemfile.base +7 -2
- data/Gemfile.lock +8 -7
- data/Gemfile.on_prem.lock +8 -7
- data/Rakefile +33 -11
- data/app/api/internal/service_tokens.rb +8 -0
- data/app/api/internal/stats.rb +6 -25
- data/lib/3scale/backend/application_events.rb +2 -4
- data/lib/3scale/backend/configuration.rb +2 -9
- data/lib/3scale/backend/errors.rb +0 -6
- data/lib/3scale/backend/job_fetcher.rb +28 -22
- data/lib/3scale/backend/listener_metrics.rb +67 -8
- data/lib/3scale/backend/stats.rb +0 -4
- data/lib/3scale/backend/stats/aggregators/base.rb +8 -1
- data/lib/3scale/backend/stats/cleaner.rb +109 -28
- data/lib/3scale/backend/stats/keys.rb +6 -0
- data/lib/3scale/backend/stats/period_commons.rb +0 -3
- data/lib/3scale/backend/transactor.rb +31 -4
- data/lib/3scale/backend/transactor/notify_batcher.rb +6 -2
- data/lib/3scale/backend/transactor/notify_job.rb +37 -17
- data/lib/3scale/backend/version.rb +1 -1
- data/lib/3scale/backend/worker_async.rb +22 -1
- data/licenses.xml +5 -5
- metadata +2 -6
- data/lib/3scale/backend/stats/delete_job_def.rb +0 -60
- data/lib/3scale/backend/stats/key_generator.rb +0 -73
- data/lib/3scale/backend/stats/partition_eraser_job.rb +0 -58
- data/lib/3scale/backend/stats/partition_generator_job.rb +0 -46
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'async'
|
2
|
+
require 'redis-namespace'
|
2
3
|
require '3scale/backend/job_fetcher'
|
3
4
|
|
4
5
|
module ThreeScale
|
@@ -10,6 +11,9 @@ module ThreeScale
|
|
10
11
|
DEFAULT_MAX_CONCURRENT_JOBS = 20
|
11
12
|
private_constant :DEFAULT_MAX_CONCURRENT_JOBS
|
12
13
|
|
14
|
+
RESQUE_REDIS_NAMESPACE = :resque
|
15
|
+
private_constant :RESQUE_REDIS_NAMESPACE
|
16
|
+
|
13
17
|
def initialize(options = {})
|
14
18
|
trap('TERM') { shutdown }
|
15
19
|
trap('INT') { shutdown }
|
@@ -17,7 +21,7 @@ module ThreeScale
|
|
17
21
|
@one_off = options[:one_off]
|
18
22
|
@jobs = Queue.new # Thread-safe queue
|
19
23
|
|
20
|
-
@job_fetcher = options[:job_fetcher] || JobFetcher.new
|
24
|
+
@job_fetcher = options[:job_fetcher] || JobFetcher.new(redis_client: redis_client)
|
21
25
|
|
22
26
|
@max_concurrent_jobs = configuration.async_worker.max_concurrent_jobs ||
|
23
27
|
DEFAULT_MAX_CONCURRENT_JOBS
|
@@ -64,6 +68,10 @@ module ThreeScale
|
|
64
68
|
# unblocks when there are new jobs or when .close() is called
|
65
69
|
job = @jobs.pop
|
66
70
|
|
71
|
+
# If job is nil, it means that the queue is closed. No more jobs are
|
72
|
+
# going to be pushed, so shutdown.
|
73
|
+
shutdown unless job
|
74
|
+
|
67
75
|
break if @shutdown
|
68
76
|
|
69
77
|
@reactor.async { perform(job) }
|
@@ -83,6 +91,19 @@ module ThreeScale
|
|
83
91
|
Async { @job_fetcher.start(@jobs) }
|
84
92
|
end
|
85
93
|
end
|
94
|
+
|
95
|
+
# Returns a new Redis client with namespace "resque".
|
96
|
+
# In the async worker, the job fetcher runs in a separate thread, and we
|
97
|
+
# need to avoid sharing an already instantiated client like the one in
|
98
|
+
# Resque::Helpers initialized in lib/3scale/backend.rb (Resque.redis).
|
99
|
+
# Failing to do so, will raise errors because of fibers shared across
|
100
|
+
# threads.
|
101
|
+
def redis_client
|
102
|
+
Redis::Namespace.new(
|
103
|
+
RESQUE_REDIS_NAMESPACE,
|
104
|
+
redis: QueueStorage.connection(Backend.environment, Backend.configuration)
|
105
|
+
)
|
106
|
+
end
|
86
107
|
end
|
87
108
|
end
|
88
109
|
end
|
data/licenses.xml
CHANGED
@@ -23,7 +23,7 @@
|
|
23
23
|
</dependency>
|
24
24
|
<dependency>
|
25
25
|
<packageName>apisonator</packageName>
|
26
|
-
<version>3.
|
26
|
+
<version>3.3.1</version>
|
27
27
|
<licenses>
|
28
28
|
<license>
|
29
29
|
<name>Apache 2.0</name>
|
@@ -93,7 +93,7 @@
|
|
93
93
|
</dependency>
|
94
94
|
<dependency>
|
95
95
|
<packageName>async-redis</packageName>
|
96
|
-
<version>0.5.
|
96
|
+
<version>0.5.1</version>
|
97
97
|
<licenses>
|
98
98
|
<license>
|
99
99
|
<name>MIT</name>
|
@@ -525,7 +525,7 @@
|
|
525
525
|
</dependency>
|
526
526
|
<dependency>
|
527
527
|
<packageName>nio4r</packageName>
|
528
|
-
<version>2.5.
|
528
|
+
<version>2.5.4</version>
|
529
529
|
<licenses>
|
530
530
|
<license>
|
531
531
|
<name>MIT</name>
|
@@ -769,7 +769,7 @@
|
|
769
769
|
</dependency>
|
770
770
|
<dependency>
|
771
771
|
<packageName>redis-namespace</packageName>
|
772
|
-
<version>1.
|
772
|
+
<version>1.8.0</version>
|
773
773
|
<licenses>
|
774
774
|
<license>
|
775
775
|
<name>MIT</name>
|
@@ -1043,7 +1043,7 @@
|
|
1043
1043
|
</dependency>
|
1044
1044
|
<dependency>
|
1045
1045
|
<packageName>timers</packageName>
|
1046
|
-
<version>4.3.
|
1046
|
+
<version>4.3.2</version>
|
1047
1047
|
<licenses>
|
1048
1048
|
<license>
|
1049
1049
|
<name>MIT</name>
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: apisonator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Ciganek
|
@@ -16,7 +16,7 @@ authors:
|
|
16
16
|
autorequire:
|
17
17
|
bindir: bin
|
18
18
|
cert_chain: []
|
19
|
-
date:
|
19
|
+
date: 2021-02-11 00:00:00.000000000 Z
|
20
20
|
dependencies: []
|
21
21
|
description: This gem provides a daemon that handles authorization and reporting of
|
22
22
|
web services managed by 3scale.
|
@@ -136,11 +136,7 @@ files:
|
|
136
136
|
- lib/3scale/backend/stats/bucket_storage.rb
|
137
137
|
- lib/3scale/backend/stats/cleaner.rb
|
138
138
|
- lib/3scale/backend/stats/codes_commons.rb
|
139
|
-
- lib/3scale/backend/stats/delete_job_def.rb
|
140
|
-
- lib/3scale/backend/stats/key_generator.rb
|
141
139
|
- lib/3scale/backend/stats/keys.rb
|
142
|
-
- lib/3scale/backend/stats/partition_eraser_job.rb
|
143
|
-
- lib/3scale/backend/stats/partition_generator_job.rb
|
144
140
|
- lib/3scale/backend/stats/period_commons.rb
|
145
141
|
- lib/3scale/backend/stats/stats_parser.rb
|
146
142
|
- lib/3scale/backend/stats/storage.rb
|
@@ -1,60 +0,0 @@
|
|
1
|
-
module ThreeScale
|
2
|
-
module Backend
|
3
|
-
module Stats
|
4
|
-
class DeleteJobDef
|
5
|
-
ATTRIBUTES = %i[service_id applications metrics from to context_info].freeze
|
6
|
-
private_constant :ATTRIBUTES
|
7
|
-
attr_reader(*ATTRIBUTES)
|
8
|
-
|
9
|
-
def self.attribute_names
|
10
|
-
ATTRIBUTES
|
11
|
-
end
|
12
|
-
|
13
|
-
def initialize(params = {})
|
14
|
-
ATTRIBUTES.each do |key|
|
15
|
-
instance_variable_set("@#{key}".to_sym, params[key]) unless params[key].nil?
|
16
|
-
end
|
17
|
-
validate
|
18
|
-
end
|
19
|
-
|
20
|
-
def run_async
|
21
|
-
Resque.enqueue(PartitionGeneratorJob, Time.now.getutc.to_f, service_id, applications,
|
22
|
-
metrics, from, to, context_info)
|
23
|
-
end
|
24
|
-
|
25
|
-
def to_json
|
26
|
-
to_hash.to_json
|
27
|
-
end
|
28
|
-
|
29
|
-
def to_hash
|
30
|
-
Hash[ATTRIBUTES.collect { |key| [key, send(key)] }]
|
31
|
-
end
|
32
|
-
|
33
|
-
private
|
34
|
-
|
35
|
-
def validate
|
36
|
-
# from and to valid epoch times
|
37
|
-
raise_validation_error('from field not integer') unless from.is_a? Integer
|
38
|
-
raise_validation_error('from field is zero') if from.zero?
|
39
|
-
raise_validation_error('to field not integer') unless to.is_a? Integer
|
40
|
-
raise_validation_error('to field is zero') if to.zero?
|
41
|
-
raise_validation_error('from < to fields') if Time.at(to) < Time.at(from)
|
42
|
-
# application is array
|
43
|
-
raise_validation_error('applications field') unless applications.is_a? Array
|
44
|
-
raise_validation_error('applications values') unless applications.all? do |x|
|
45
|
-
x.is_a?(String) || x.is_a?(Integer)
|
46
|
-
end
|
47
|
-
# metrics is array
|
48
|
-
raise_validation_error('metrics field') unless metrics.is_a? Array
|
49
|
-
raise_validation_error('metrics values') unless metrics.all? do |x|
|
50
|
-
x.is_a?(String) || x.is_a?(Integer)
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
def raise_validation_error(msg)
|
55
|
-
raise DeleteServiceStatsValidationError.new(service_id, msg)
|
56
|
-
end
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|
@@ -1,73 +0,0 @@
|
|
1
|
-
module ThreeScale
|
2
|
-
module Backend
|
3
|
-
module Stats
|
4
|
-
class KeyGenerator
|
5
|
-
attr_reader :service_id, :applications, :metrics, :from, :to
|
6
|
-
|
7
|
-
def initialize(service_id:, applications: [], metrics: [], from:, to:, **)
|
8
|
-
@service_id = service_id
|
9
|
-
@applications = applications
|
10
|
-
@metrics = metrics
|
11
|
-
@from = from
|
12
|
-
@to = to
|
13
|
-
end
|
14
|
-
|
15
|
-
def keys
|
16
|
-
response_code_service_keys +
|
17
|
-
response_code_application_keys +
|
18
|
-
usage_service_keys +
|
19
|
-
usage_application_keys
|
20
|
-
end
|
21
|
-
|
22
|
-
private
|
23
|
-
|
24
|
-
def periods(granularities)
|
25
|
-
granularities.flat_map do |granularity|
|
26
|
-
(Period[granularity].new(Time.at(from))..Period[granularity].new(Time.at(to))).to_a
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
def response_codes
|
31
|
-
CodesCommons::TRACKED_CODES + CodesCommons::TRACKED_CODE_GROUPS
|
32
|
-
end
|
33
|
-
|
34
|
-
def response_code_service_keys
|
35
|
-
periods(PeriodCommons::PERMANENT_SERVICE_GRANULARITIES).flat_map do |period|
|
36
|
-
response_codes.flat_map do |response_code|
|
37
|
-
Keys.service_response_code_value_key(service_id, response_code, period)
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
def response_code_application_keys
|
43
|
-
periods(PeriodCommons::PERMANENT_EXPANDED_GRANULARITIES).flat_map do |period|
|
44
|
-
response_codes.flat_map do |response_code|
|
45
|
-
applications.flat_map do |application|
|
46
|
-
Keys.application_response_code_value_key(service_id, application,
|
47
|
-
response_code, period)
|
48
|
-
end
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
def usage_service_keys
|
54
|
-
periods(PeriodCommons::PERMANENT_SERVICE_GRANULARITIES).flat_map do |period|
|
55
|
-
metrics.flat_map do |metric|
|
56
|
-
Keys.service_usage_value_key(service_id, metric, period)
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
def usage_application_keys
|
62
|
-
periods(PeriodCommons::PERMANENT_EXPANDED_GRANULARITIES).flat_map do |period|
|
63
|
-
metrics.flat_map do |metric|
|
64
|
-
applications.flat_map do |application|
|
65
|
-
Keys.application_usage_value_key(service_id, application, metric, period)
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
@@ -1,58 +0,0 @@
|
|
1
|
-
module ThreeScale
|
2
|
-
module Backend
|
3
|
-
module Stats
|
4
|
-
# Job for deleting service stats
|
5
|
-
# Perform actual key deletion from a key partition definition
|
6
|
-
class PartitionEraserJob < BackgroundJob
|
7
|
-
# low priority queue
|
8
|
-
@queue = :stats
|
9
|
-
|
10
|
-
class << self
|
11
|
-
include StorageHelpers
|
12
|
-
include Configurable
|
13
|
-
|
14
|
-
def perform_logged(_enqueue_time, service_id, applications, metrics,
|
15
|
-
from, to, offset, length, context_info = {})
|
16
|
-
job = DeleteJobDef.new(
|
17
|
-
service_id: service_id,
|
18
|
-
applications: applications,
|
19
|
-
metrics: metrics,
|
20
|
-
from: from,
|
21
|
-
to: to
|
22
|
-
)
|
23
|
-
|
24
|
-
validate_job(job, offset, length)
|
25
|
-
|
26
|
-
stats_key_gen = KeyGenerator.new(job.to_hash)
|
27
|
-
|
28
|
-
stats_key_gen.keys.drop(offset).take(length).each_slice(configuration.stats.delete_batch_size) do |slice|
|
29
|
-
storage.del(slice)
|
30
|
-
end
|
31
|
-
|
32
|
-
[true, { job: job.to_hash, offset: offset, lenght: length }.to_json]
|
33
|
-
rescue Backend::Error => error
|
34
|
-
[false, "#{service_id} #{error}"]
|
35
|
-
end
|
36
|
-
|
37
|
-
private
|
38
|
-
|
39
|
-
def validate_job(job, offset, length)
|
40
|
-
unless offset.is_a? Integer
|
41
|
-
raise DeleteServiceStatsValidationError.new(job.service_id, 'offset field value ' \
|
42
|
-
"[#{offset}] validation error")
|
43
|
-
end
|
44
|
-
|
45
|
-
unless length.is_a? Integer
|
46
|
-
raise DeleteServiceStatsValidationError.new(job.service_id, 'length field value ' \
|
47
|
-
"[#{length}] validation error")
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
def enqueue_time(args)
|
52
|
-
args[0]
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|
58
|
-
end
|
@@ -1,46 +0,0 @@
|
|
1
|
-
module ThreeScale
|
2
|
-
module Backend
|
3
|
-
module Stats
|
4
|
-
# Job for deleting service stats
|
5
|
-
# Maps delete job definition to a set of non overlapping key set partitions
|
6
|
-
class PartitionGeneratorJob < BackgroundJob
|
7
|
-
# low priority queue
|
8
|
-
@queue = :stats
|
9
|
-
|
10
|
-
class << self
|
11
|
-
include Configurable
|
12
|
-
|
13
|
-
def perform_logged(_enqueue_time, service_id, applications, metrics,
|
14
|
-
from, to, context_info = {})
|
15
|
-
job = DeleteJobDef.new(
|
16
|
-
service_id: service_id,
|
17
|
-
applications: applications,
|
18
|
-
metrics: metrics,
|
19
|
-
from: from,
|
20
|
-
to: to
|
21
|
-
)
|
22
|
-
|
23
|
-
stats_key_gen = KeyGenerator.new(job.to_hash)
|
24
|
-
|
25
|
-
# Generate partitions
|
26
|
-
0.step(stats_key_gen.keys.count, configuration.stats.delete_partition_batch_size).each do |idx|
|
27
|
-
Resque.enqueue(PartitionEraserJob, Time.now.getutc.to_f, service_id, applications,
|
28
|
-
metrics, from, to, idx,
|
29
|
-
configuration.stats.delete_partition_batch_size, context_info)
|
30
|
-
end
|
31
|
-
|
32
|
-
[true, job.to_json]
|
33
|
-
rescue Backend::Error => error
|
34
|
-
[false, "#{service_id} #{error}"]
|
35
|
-
end
|
36
|
-
|
37
|
-
private
|
38
|
-
|
39
|
-
def enqueue_time(args)
|
40
|
-
args[0]
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|