apisonator 3.3.0 → 3.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a8e132216edb5cd3b44967acab917d4ca4f50c692c22fb5b81c7ffff9c5d598b
4
- data.tar.gz: e2e31c92b450398dbcd3a3dcc3d31ed39b2d530048de23aa06b95c4930809249
3
+ metadata.gz: a812d330bdcb770421ed926c478c8a8b48087ba33ad28ab80317a48db935f432
4
+ data.tar.gz: 5bc8b40c69fbf5a6a680b71692dc66433746b053c0dbf3dcfe6e3d4aacd656af
5
5
  SHA512:
6
- metadata.gz: 288b9c506a32caf81e2b3bcbf06b347e1f39f32a295a61d146ebc9b7f5953458f27114c3eceb08ef00da6293487d02f49b5c7688af16f87c0aa0ff4c70cfa48f
7
- data.tar.gz: 025f8fcf1edb087de5e97ad43ecca61e25c97f061f0002b75d210a1052fa76355c5ea8a34515018d41b21428791d255341e2f36d2c178872ae549d3f97f2e52e
6
+ metadata.gz: 807c4d8cfc932e600780f4ba97fa5cfa3afcaca04ad0e27395e92fca31407d96e360412a290566c78955e5cf12895a5f4a70e589474de045bf3f0f7cf21311af
7
+ data.tar.gz: 17bf0d7c783ee36b78a885a441846edeaf8906185ab183c91ffc7f0c7f7c13e45b4237c0c7adb1a228794b3f64d4a0729f6eb8360a59910021d65a0b99c1edc1
data/CHANGELOG.md CHANGED
@@ -2,6 +2,13 @@
2
2
 
3
3
  Notable changes to Apisonator will be tracked in this document.
4
4
 
5
+ ## 3.3.1 - 2021-02-11
6
+
7
+ ### Fixed
8
+
9
+ - Usages with `#0` (set to 0) no longer generate unnecessary stats keys in Redis
10
+ ([#258](https://github.com/3scale/apisonator/pull/258)).
11
+
5
12
  ## 3.3.0 - 2021-02-09
6
13
 
7
14
  ### Added
data/Gemfile.base CHANGED
@@ -15,11 +15,15 @@ platform :ruby do
15
15
  end
16
16
 
17
17
  group :test do
18
+ # Newer versions of rack-test don't work well with rspec-api-documentation.
19
+ # See https://github.com/rack/rack-test/pull/223 &
20
+ # https://github.com/zipmark/rspec_api_documentation/issues/342
21
+ gem 'rack-test', '= 0.8.2'
22
+
18
23
  gem 'benchmark-ips', '~> 2.7.2'
19
24
  gem 'mocha', '~> 1.3'
20
25
  gem 'nokogiri', '~> 1.10.8'
21
26
  gem 'pkg-config', '~> 1.1.7'
22
- gem 'rack-test', '~> 0.8.2'
23
27
  gem 'resque_unit', '~> 0.4.4', source: 'https://rubygems.org'
24
28
  gem 'test-unit', '~> 3.2.6'
25
29
  gem 'resque_spec', '~> 0.17.0'
data/Gemfile.lock CHANGED
@@ -35,7 +35,7 @@ GIT
35
35
  PATH
36
36
  remote: .
37
37
  specs:
38
- apisonator (3.3.0)
38
+ apisonator (3.3.1)
39
39
 
40
40
  GEM
41
41
  remote: https://rubygems.org/
@@ -288,7 +288,7 @@ DEPENDENCIES
288
288
  pry-doc (~> 0.11.1)
289
289
  puma!
290
290
  rack (~> 2.1.4)
291
- rack-test (~> 0.8.2)
291
+ rack-test (= 0.8.2)
292
292
  rake (~> 13.0)
293
293
  redis!
294
294
  redis-namespace (~> 1.8.0)
data/Gemfile.on_prem.lock CHANGED
@@ -35,7 +35,7 @@ GIT
35
35
  PATH
36
36
  remote: .
37
37
  specs:
38
- apisonator (3.3.0)
38
+ apisonator (3.3.1)
39
39
 
40
40
  GEM
41
41
  remote: https://rubygems.org/
@@ -269,7 +269,7 @@ DEPENDENCIES
269
269
  pry-doc (~> 0.11.1)
270
270
  puma!
271
271
  rack (~> 2.1.4)
272
- rack-test (~> 0.8.2)
272
+ rack-test (= 0.8.2)
273
273
  rake (~> 13.0)
274
274
  redis!
275
275
  redis-namespace (~> 1.8.0)
@@ -6,32 +6,13 @@ module ThreeScale
6
6
  respond_with_404('service not found') unless Service.exists?(params[:service_id])
7
7
  end
8
8
 
9
- # This is very slow and needs to be disabled until the performance
10
- # issues are solved. In the meanwhile, the job will just return OK.
11
- =begin
12
- delete '' do |service_id|
13
- delete_stats_job_attrs = api_params Stats::DeleteJobDef
14
- delete_stats_job_attrs[:service_id] = service_id
15
- delete_stats_job_attrs[:from] = delete_stats_job_attrs[:from].to_i
16
- delete_stats_job_attrs[:to] = delete_stats_job_attrs[:to].to_i
17
- begin
18
- Stats::DeleteJobDef.new(delete_stats_job_attrs).run_async
19
- rescue DeleteServiceStatsValidationError => e
20
- [400, headers, { status: :error, error: e.message }.to_json]
21
- else
22
- { status: :to_be_deleted }.to_json
23
- end
24
- =end
25
-
26
- # This is an alternative to the above. It just adds the service to a
27
- # Redis set to marked is as "to be deleted".
28
- # Later a script can read that set and actually delete the keys.
29
- # Read the docs of the Stats::Cleaner class for more details.
9
+ # This adds the service to a Redis set to mark is as "to be deleted".
10
+ # Later a script can read that set and actually delete the keys. Read
11
+ # the docs of the Stats::Cleaner class for more details.
30
12
  #
31
- # Notice that this method ignores the "from" and "to" parameters. When
32
- # system calls this method, they're always interested in deleting all
33
- # the keys. They were just passing "from" and "to" to make the
34
- # implementation of the option above easier.
13
+ # Notice that this method ignores the "from" and "to" parameters used in
14
+ # previous versions. When system calls this method, they're always
15
+ # interested in deleting all the keys.
35
16
  delete '' do |service_id|
36
17
  Stats::Cleaner.mark_service_to_be_deleted(service_id)
37
18
  { status: :to_be_deleted }.to_json
@@ -32,8 +32,6 @@ module ThreeScale
32
32
 
33
33
  CONFIG_DELETE_STATS_BATCH_SIZE = 50
34
34
  private_constant :CONFIG_DELETE_STATS_BATCH_SIZE
35
- CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE = 1000
36
- private_constant :CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE
37
35
 
38
36
  @configuration = Configuration::Loader.new
39
37
 
@@ -54,7 +52,7 @@ module ThreeScale
54
52
  config.add_section(:analytics_redis, :server,
55
53
  :connect_timeout, :read_timeout, :write_timeout)
56
54
  config.add_section(:hoptoad, :service, :api_key)
57
- config.add_section(:stats, :bucket_size, :delete_batch_size, :delete_partition_batch_size)
55
+ config.add_section(:stats, :bucket_size, :delete_batch_size)
58
56
  config.add_section(:redshift, :host, :port, :dbname, :user, :password)
59
57
  config.add_section(:statsd, :host, :port)
60
58
  config.add_section(:internal_api, :user, :password)
@@ -125,9 +123,6 @@ module ThreeScale
125
123
  config.stats.delete_batch_size = parse_int(config.stats.delete_batch_size,
126
124
  CONFIG_DELETE_STATS_BATCH_SIZE)
127
125
 
128
- config.stats.delete_partition_batch_size = parse_int(config.stats.delete_partition_batch_size,
129
- CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE)
130
-
131
126
  # often we don't have a log_file setting - generate it here from
132
127
  # the log_path setting.
133
128
  log_file = config.log_file
@@ -292,12 +292,6 @@ module ThreeScale
292
292
  end
293
293
  end
294
294
 
295
- class DeleteServiceStatsValidationError < Error
296
- def initialize(service_id, msg)
297
- super "Delete stats job context validation error. Service: #{service_id}. Error: #{msg}"
298
- end
299
- end
300
-
301
295
  class EndUsersNoLongerSupported < BadRequest
302
296
  def initialize
303
297
  super 'End-users are no longer supported, do not specify the user_id parameter'.freeze
@@ -1,8 +1,4 @@
1
1
  require '3scale/backend/stats/codes_commons'
2
2
  require '3scale/backend/stats/period_commons'
3
3
  require '3scale/backend/stats/aggregator'
4
- require '3scale/backend/stats/delete_job_def'
5
- require '3scale/backend/stats/key_generator'
6
- require '3scale/backend/stats/partition_generator_job'
7
- require '3scale/backend/stats/partition_eraser_job'
8
4
  require '3scale/backend/stats/cleaner'
@@ -20,7 +20,14 @@ module ThreeScale
20
20
  key = counter_key(prefix_key, granularity.new(timestamp))
21
21
  expire_time = Stats::PeriodCommons.expire_time_for_granularity(granularity)
22
22
 
23
- store_key(cmd, key, value, expire_time)
23
+ # We don't need to store stats keys set to 0. It wastes Redis
24
+ # memory because for rate-limiting and stats, a key of set to 0
25
+ # is equivalent to a key that does not exist.
26
+ if cmd == :set && value == 0
27
+ storage.del(key)
28
+ else
29
+ store_key(cmd, key, value, expire_time)
30
+ end
24
31
 
25
32
  unless Stats::PeriodCommons::EXCLUDED_FOR_BUCKETS.include?(granularity)
26
33
  keys_for_bucket << key
@@ -12,9 +12,6 @@ module ThreeScale
12
12
  GRANULARITY_EXPIRATION_TIME = { Period[:minute] => 180 }.freeze
13
13
  private_constant :GRANULARITY_EXPIRATION_TIME
14
14
 
15
- PERMANENT_SERVICE_GRANULARITIES = (SERVICE_GRANULARITIES - GRANULARITY_EXPIRATION_TIME.keys).freeze
16
- PERMANENT_EXPANDED_GRANULARITIES = (EXPANDED_GRANULARITIES - GRANULARITY_EXPIRATION_TIME.keys).freeze
17
-
18
15
  # We are not going to send metrics with granularity 'eternity' or
19
16
  # 'week' to Kinesis, so there is no point in storing them in Redis
20
17
  # buckets.
@@ -1,5 +1,5 @@
1
1
  module ThreeScale
2
2
  module Backend
3
- VERSION = '3.3.0'
3
+ VERSION = '3.3.1'
4
4
  end
5
5
  end
data/licenses.xml CHANGED
@@ -23,7 +23,7 @@
23
23
  </dependency>
24
24
  <dependency>
25
25
  <packageName>apisonator</packageName>
26
- <version>3.3.0</version>
26
+ <version>3.3.1</version>
27
27
  <licenses>
28
28
  <license>
29
29
  <name>Apache 2.0</name>
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: apisonator
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.3.0
4
+ version: 3.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Adam Ciganek
@@ -16,7 +16,7 @@ authors:
16
16
  autorequire:
17
17
  bindir: bin
18
18
  cert_chain: []
19
- date: 2021-02-09 00:00:00.000000000 Z
19
+ date: 2021-02-11 00:00:00.000000000 Z
20
20
  dependencies: []
21
21
  description: This gem provides a daemon that handles authorization and reporting of
22
22
  web services managed by 3scale.
@@ -136,11 +136,7 @@ files:
136
136
  - lib/3scale/backend/stats/bucket_storage.rb
137
137
  - lib/3scale/backend/stats/cleaner.rb
138
138
  - lib/3scale/backend/stats/codes_commons.rb
139
- - lib/3scale/backend/stats/delete_job_def.rb
140
- - lib/3scale/backend/stats/key_generator.rb
141
139
  - lib/3scale/backend/stats/keys.rb
142
- - lib/3scale/backend/stats/partition_eraser_job.rb
143
- - lib/3scale/backend/stats/partition_generator_job.rb
144
140
  - lib/3scale/backend/stats/period_commons.rb
145
141
  - lib/3scale/backend/stats/stats_parser.rb
146
142
  - lib/3scale/backend/stats/storage.rb
@@ -1,60 +0,0 @@
1
- module ThreeScale
2
- module Backend
3
- module Stats
4
- class DeleteJobDef
5
- ATTRIBUTES = %i[service_id applications metrics from to context_info].freeze
6
- private_constant :ATTRIBUTES
7
- attr_reader(*ATTRIBUTES)
8
-
9
- def self.attribute_names
10
- ATTRIBUTES
11
- end
12
-
13
- def initialize(params = {})
14
- ATTRIBUTES.each do |key|
15
- instance_variable_set("@#{key}".to_sym, params[key]) unless params[key].nil?
16
- end
17
- validate
18
- end
19
-
20
- def run_async
21
- Resque.enqueue(PartitionGeneratorJob, Time.now.getutc.to_f, service_id, applications,
22
- metrics, from, to, context_info)
23
- end
24
-
25
- def to_json
26
- to_hash.to_json
27
- end
28
-
29
- def to_hash
30
- Hash[ATTRIBUTES.collect { |key| [key, send(key)] }]
31
- end
32
-
33
- private
34
-
35
- def validate
36
- # from and to valid epoch times
37
- raise_validation_error('from field not integer') unless from.is_a? Integer
38
- raise_validation_error('from field is zero') if from.zero?
39
- raise_validation_error('to field not integer') unless to.is_a? Integer
40
- raise_validation_error('to field is zero') if to.zero?
41
- raise_validation_error('from < to fields') if Time.at(to) < Time.at(from)
42
- # application is array
43
- raise_validation_error('applications field') unless applications.is_a? Array
44
- raise_validation_error('applications values') unless applications.all? do |x|
45
- x.is_a?(String) || x.is_a?(Integer)
46
- end
47
- # metrics is array
48
- raise_validation_error('metrics field') unless metrics.is_a? Array
49
- raise_validation_error('metrics values') unless metrics.all? do |x|
50
- x.is_a?(String) || x.is_a?(Integer)
51
- end
52
- end
53
-
54
- def raise_validation_error(msg)
55
- raise DeleteServiceStatsValidationError.new(service_id, msg)
56
- end
57
- end
58
- end
59
- end
60
- end
@@ -1,73 +0,0 @@
1
- module ThreeScale
2
- module Backend
3
- module Stats
4
- class KeyGenerator
5
- attr_reader :service_id, :applications, :metrics, :from, :to
6
-
7
- def initialize(service_id:, applications: [], metrics: [], from:, to:, **)
8
- @service_id = service_id
9
- @applications = applications
10
- @metrics = metrics
11
- @from = from
12
- @to = to
13
- end
14
-
15
- def keys
16
- response_code_service_keys +
17
- response_code_application_keys +
18
- usage_service_keys +
19
- usage_application_keys
20
- end
21
-
22
- private
23
-
24
- def periods(granularities)
25
- granularities.flat_map do |granularity|
26
- (Period[granularity].new(Time.at(from))..Period[granularity].new(Time.at(to))).to_a
27
- end
28
- end
29
-
30
- def response_codes
31
- CodesCommons::TRACKED_CODES + CodesCommons::TRACKED_CODE_GROUPS
32
- end
33
-
34
- def response_code_service_keys
35
- periods(PeriodCommons::PERMANENT_SERVICE_GRANULARITIES).flat_map do |period|
36
- response_codes.flat_map do |response_code|
37
- Keys.service_response_code_value_key(service_id, response_code, period)
38
- end
39
- end
40
- end
41
-
42
- def response_code_application_keys
43
- periods(PeriodCommons::PERMANENT_EXPANDED_GRANULARITIES).flat_map do |period|
44
- response_codes.flat_map do |response_code|
45
- applications.flat_map do |application|
46
- Keys.application_response_code_value_key(service_id, application,
47
- response_code, period)
48
- end
49
- end
50
- end
51
- end
52
-
53
- def usage_service_keys
54
- periods(PeriodCommons::PERMANENT_SERVICE_GRANULARITIES).flat_map do |period|
55
- metrics.flat_map do |metric|
56
- Keys.service_usage_value_key(service_id, metric, period)
57
- end
58
- end
59
- end
60
-
61
- def usage_application_keys
62
- periods(PeriodCommons::PERMANENT_EXPANDED_GRANULARITIES).flat_map do |period|
63
- metrics.flat_map do |metric|
64
- applications.flat_map do |application|
65
- Keys.application_usage_value_key(service_id, application, metric, period)
66
- end
67
- end
68
- end
69
- end
70
- end
71
- end
72
- end
73
- end
@@ -1,58 +0,0 @@
1
- module ThreeScale
2
- module Backend
3
- module Stats
4
- # Job for deleting service stats
5
- # Perform actual key deletion from a key partition definition
6
- class PartitionEraserJob < BackgroundJob
7
- # low priority queue
8
- @queue = :stats
9
-
10
- class << self
11
- include StorageHelpers
12
- include Configurable
13
-
14
- def perform_logged(_enqueue_time, service_id, applications, metrics,
15
- from, to, offset, length, context_info = {})
16
- job = DeleteJobDef.new(
17
- service_id: service_id,
18
- applications: applications,
19
- metrics: metrics,
20
- from: from,
21
- to: to
22
- )
23
-
24
- validate_job(job, offset, length)
25
-
26
- stats_key_gen = KeyGenerator.new(job.to_hash)
27
-
28
- stats_key_gen.keys.drop(offset).take(length).each_slice(configuration.stats.delete_batch_size) do |slice|
29
- storage.del(slice)
30
- end
31
-
32
- [true, { job: job.to_hash, offset: offset, lenght: length }.to_json]
33
- rescue Backend::Error => error
34
- [false, "#{service_id} #{error}"]
35
- end
36
-
37
- private
38
-
39
- def validate_job(job, offset, length)
40
- unless offset.is_a? Integer
41
- raise DeleteServiceStatsValidationError.new(job.service_id, 'offset field value ' \
42
- "[#{offset}] validation error")
43
- end
44
-
45
- unless length.is_a? Integer
46
- raise DeleteServiceStatsValidationError.new(job.service_id, 'length field value ' \
47
- "[#{length}] validation error")
48
- end
49
- end
50
-
51
- def enqueue_time(args)
52
- args[0]
53
- end
54
- end
55
- end
56
- end
57
- end
58
- end
@@ -1,46 +0,0 @@
1
- module ThreeScale
2
- module Backend
3
- module Stats
4
- # Job for deleting service stats
5
- # Maps delete job definition to a set of non overlapping key set partitions
6
- class PartitionGeneratorJob < BackgroundJob
7
- # low priority queue
8
- @queue = :stats
9
-
10
- class << self
11
- include Configurable
12
-
13
- def perform_logged(_enqueue_time, service_id, applications, metrics,
14
- from, to, context_info = {})
15
- job = DeleteJobDef.new(
16
- service_id: service_id,
17
- applications: applications,
18
- metrics: metrics,
19
- from: from,
20
- to: to
21
- )
22
-
23
- stats_key_gen = KeyGenerator.new(job.to_hash)
24
-
25
- # Generate partitions
26
- 0.step(stats_key_gen.keys.count, configuration.stats.delete_partition_batch_size).each do |idx|
27
- Resque.enqueue(PartitionEraserJob, Time.now.getutc.to_f, service_id, applications,
28
- metrics, from, to, idx,
29
- configuration.stats.delete_partition_batch_size, context_info)
30
- end
31
-
32
- [true, job.to_json]
33
- rescue Backend::Error => error
34
- [false, "#{service_id} #{error}"]
35
- end
36
-
37
- private
38
-
39
- def enqueue_time(args)
40
- args[0]
41
- end
42
- end
43
- end
44
- end
45
- end
46
- end