apisonator 3.3.0 → 3.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a8e132216edb5cd3b44967acab917d4ca4f50c692c22fb5b81c7ffff9c5d598b
-  data.tar.gz: e2e31c92b450398dbcd3a3dcc3d31ed39b2d530048de23aa06b95c4930809249
+  metadata.gz: a812d330bdcb770421ed926c478c8a8b48087ba33ad28ab80317a48db935f432
+  data.tar.gz: 5bc8b40c69fbf5a6a680b71692dc66433746b053c0dbf3dcfe6e3d4aacd656af
 SHA512:
-  metadata.gz: 288b9c506a32caf81e2b3bcbf06b347e1f39f32a295a61d146ebc9b7f5953458f27114c3eceb08ef00da6293487d02f49b5c7688af16f87c0aa0ff4c70cfa48f
-  data.tar.gz: 025f8fcf1edb087de5e97ad43ecca61e25c97f061f0002b75d210a1052fa76355c5ea8a34515018d41b21428791d255341e2f36d2c178872ae549d3f97f2e52e
+  metadata.gz: 807c4d8cfc932e600780f4ba97fa5cfa3afcaca04ad0e27395e92fca31407d96e360412a290566c78955e5cf12895a5f4a70e589474de045bf3f0f7cf21311af
+  data.tar.gz: 17bf0d7c783ee36b78a885a441846edeaf8906185ab183c91ffc7f0c7f7c13e45b4237c0c7adb1a228794b3f64d4a0729f6eb8360a59910021d65a0b99c1edc1

data/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 Notable changes to Apisonator will be tracked in this document.
 
+## 3.3.1 - 2021-02-11
+
+### Fixed
+
+- Usages with `#0` (set to 0) no longer generate unnecessary stats keys in Redis
+([#258](https://github.com/3scale/apisonator/pull/258)).
+
 ## 3.3.0 - 2021-02-09
 
 ### Added

data/Gemfile.base

@@ -15,11 +15,15 @@ platform :ruby do
 end
 
 group :test do
+  # Newer versions of rack-test don't work well with rspec-api-documentation.
+  # See https://github.com/rack/rack-test/pull/223 &
+  # https://github.com/zipmark/rspec_api_documentation/issues/342
+  gem 'rack-test',     '= 0.8.2'
+
   gem 'benchmark-ips', '~> 2.7.2'
   gem 'mocha',         '~> 1.3'
   gem 'nokogiri',      '~> 1.10.8'
   gem 'pkg-config',    '~> 1.1.7'
-  gem 'rack-test',     '~> 0.8.2'
   gem 'resque_unit',   '~> 0.4.4', source: 'https://rubygems.org'
   gem 'test-unit',     '~> 3.2.6'
   gem 'resque_spec',   '~> 0.17.0'

data/Gemfile.lock

@@ -35,7 +35,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.3.0)
+    apisonator (3.3.1)
 
 GEM
   remote: https://rubygems.org/
@@ -288,7 +288,7 @@ DEPENDENCIES
   pry-doc (~> 0.11.1)
   puma!
   rack (~> 2.1.4)
-  rack-test (~> 0.8.2)
+  rack-test (= 0.8.2)
   rake (~> 13.0)
   redis!
   redis-namespace (~> 1.8.0)

data/Gemfile.on_prem.lock

@@ -35,7 +35,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.3.0)
+    apisonator (3.3.1)
 
 GEM
   remote: https://rubygems.org/
@@ -269,7 +269,7 @@ DEPENDENCIES
   pry-doc (~> 0.11.1)
   puma!
   rack (~> 2.1.4)
-  rack-test (~> 0.8.2)
+  rack-test (= 0.8.2)
   rake (~> 13.0)
   redis!
   redis-namespace (~> 1.8.0)

data/app/api/internal/stats.rb

@@ -6,32 +6,13 @@ module ThreeScale
           respond_with_404('service not found') unless Service.exists?(params[:service_id])
         end
 
-        # This is very slow and needs to be disabled until the performance
-        # issues are solved. In the meanwhile, the job will just return OK.
-=begin
-        delete '' do |service_id|
-          delete_stats_job_attrs = api_params Stats::DeleteJobDef
-          delete_stats_job_attrs[:service_id] = service_id
-          delete_stats_job_attrs[:from] = delete_stats_job_attrs[:from].to_i
-          delete_stats_job_attrs[:to] = delete_stats_job_attrs[:to].to_i
-          begin
-            Stats::DeleteJobDef.new(delete_stats_job_attrs).run_async
-          rescue DeleteServiceStatsValidationError => e
-            [400, headers, { status: :error, error: e.message }.to_json]
-          else
-            { status: :to_be_deleted }.to_json
-          end
-=end
-
-        # This is an alternative to the above. It just adds the service to a
-        # Redis set to marked is as "to be deleted".
-        # Later a script can read that set and actually delete the keys.
-        # Read the docs of the Stats::Cleaner class for more details.
+        # This adds the service to a Redis set to mark is as "to be deleted".
+        # Later a script can read that set and actually delete the keys. Read
+        # the docs of the Stats::Cleaner class for more details.
         #
-        # Notice that this method ignores the "from" and "to" parameters. When
-        # system calls this method, they're always interested in deleting all
-        # the keys. They were just passing "from" and "to" to make the
-        # implementation of the option above easier.
+        # Notice that this method ignores the "from" and "to" parameters used in
+        # previous versions. When system calls this method, they're always
+        # interested in deleting all the keys.
         delete '' do |service_id|
           Stats::Cleaner.mark_service_to_be_deleted(service_id)
           { status: :to_be_deleted }.to_json

data/lib/3scale/backend/configuration.rb

@@ -32,8 +32,6 @@ module ThreeScale
 
     CONFIG_DELETE_STATS_BATCH_SIZE = 50
     private_constant :CONFIG_DELETE_STATS_BATCH_SIZE
-    CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE = 1000
-    private_constant :CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE
 
     @configuration = Configuration::Loader.new
 
@@ -54,7 +52,7 @@ module ThreeScale
       config.add_section(:analytics_redis, :server,
                          :connect_timeout, :read_timeout, :write_timeout)
       config.add_section(:hoptoad, :service, :api_key)
-      config.add_section(:stats, :bucket_size, :delete_batch_size, :delete_partition_batch_size)
+      config.add_section(:stats, :bucket_size, :delete_batch_size)
       config.add_section(:redshift, :host, :port, :dbname, :user, :password)
       config.add_section(:statsd, :host, :port)
       config.add_section(:internal_api, :user, :password)
@@ -125,9 +123,6 @@ module ThreeScale
       config.stats.delete_batch_size = parse_int(config.stats.delete_batch_size,
                                                  CONFIG_DELETE_STATS_BATCH_SIZE)
 
-      config.stats.delete_partition_batch_size = parse_int(config.stats.delete_partition_batch_size,
-                                                           CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE)
-
       # often we don't have a log_file setting - generate it here from
       # the log_path setting.
       log_file = config.log_file

data/lib/3scale/backend/errors.rb

@@ -292,12 +292,6 @@ module ThreeScale
       end
     end
 
-    class DeleteServiceStatsValidationError < Error
-      def initialize(service_id, msg)
-        super "Delete stats job context validation error. Service: #{service_id}. Error: #{msg}"
-      end
-    end
-
     class EndUsersNoLongerSupported < BadRequest
       def initialize
         super 'End-users are no longer supported, do not specify the user_id parameter'.freeze

data/lib/3scale/backend/stats.rb

@@ -1,8 +1,4 @@
 require '3scale/backend/stats/codes_commons'
 require '3scale/backend/stats/period_commons'
 require '3scale/backend/stats/aggregator'
-require '3scale/backend/stats/delete_job_def'
-require '3scale/backend/stats/key_generator'
-require '3scale/backend/stats/partition_generator_job'
-require '3scale/backend/stats/partition_eraser_job'
 require '3scale/backend/stats/cleaner'

data/lib/3scale/backend/stats/aggregators/base.rb

@@ -20,7 +20,14 @@ module ThreeScale
                 key = counter_key(prefix_key, granularity.new(timestamp))
                 expire_time = Stats::PeriodCommons.expire_time_for_granularity(granularity)
 
-                store_key(cmd, key, value, expire_time)
+                # We don't need to store stats keys set to 0. It wastes Redis
+                # memory because for rate-limiting and stats, a key of set to 0
+                # is equivalent to a key that does not exist.
+                if cmd == :set && value == 0
+                  storage.del(key)
+                else
+                  store_key(cmd, key, value, expire_time)
+                end
 
                 unless Stats::PeriodCommons::EXCLUDED_FOR_BUCKETS.include?(granularity)
                   keys_for_bucket << key

data/lib/3scale/backend/stats/period_commons.rb

@@ -12,9 +12,6 @@ module ThreeScale
         GRANULARITY_EXPIRATION_TIME = { Period[:minute] => 180 }.freeze
         private_constant :GRANULARITY_EXPIRATION_TIME
 
-        PERMANENT_SERVICE_GRANULARITIES = (SERVICE_GRANULARITIES - GRANULARITY_EXPIRATION_TIME.keys).freeze
-        PERMANENT_EXPANDED_GRANULARITIES = (EXPANDED_GRANULARITIES - GRANULARITY_EXPIRATION_TIME.keys).freeze
-
         # We are not going to send metrics with granularity 'eternity' or
         # 'week' to Kinesis, so there is no point in storing them in Redis
         # buckets.

data/lib/3scale/backend/version.rb

@@ -1,5 +1,5 @@
 module ThreeScale
   module Backend
-    VERSION = '3.3.0'
+    VERSION = '3.3.1'
   end
 end

data/licenses.xml

@@ -23,7 +23,7 @@
       </dependency>
           <dependency>
         <packageName>apisonator</packageName>
-        <version>3.3.0</version>
+        <version>3.3.1</version>
         <licenses>
                       <license>
               <name>Apache 2.0</name>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apisonator
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.3.1
 platform: ruby
 authors:
 - Adam Ciganek
@@ -16,7 +16,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-09 00:00:00.000000000 Z
+date: 2021-02-11 00:00:00.000000000 Z
 dependencies: []
 description: This gem provides a daemon that handles authorization and reporting of
   web services managed by 3scale.
@@ -136,11 +136,7 @@ files:
 - lib/3scale/backend/stats/bucket_storage.rb
 - lib/3scale/backend/stats/cleaner.rb
 - lib/3scale/backend/stats/codes_commons.rb
-- lib/3scale/backend/stats/delete_job_def.rb
-- lib/3scale/backend/stats/key_generator.rb
 - lib/3scale/backend/stats/keys.rb
-- lib/3scale/backend/stats/partition_eraser_job.rb
-- lib/3scale/backend/stats/partition_generator_job.rb
 - lib/3scale/backend/stats/period_commons.rb
 - lib/3scale/backend/stats/stats_parser.rb
 - lib/3scale/backend/stats/storage.rb

data/lib/3scale/backend/stats/delete_job_def.rb

@@ -1,60 +0,0 @@
-module ThreeScale
-  module Backend
-    module Stats
-      class DeleteJobDef
-        ATTRIBUTES = %i[service_id applications metrics from to context_info].freeze
-        private_constant :ATTRIBUTES
-        attr_reader(*ATTRIBUTES)
-
-        def self.attribute_names
-          ATTRIBUTES
-        end
-
-        def initialize(params = {})
-          ATTRIBUTES.each do |key|
-            instance_variable_set("@#{key}".to_sym, params[key]) unless params[key].nil?
-          end
-          validate
-        end
-
-        def run_async
-          Resque.enqueue(PartitionGeneratorJob, Time.now.getutc.to_f, service_id, applications,
-                         metrics, from, to, context_info)
-        end
-
-        def to_json
-          to_hash.to_json
-        end
-
-        def to_hash
-          Hash[ATTRIBUTES.collect { |key| [key, send(key)] }]
-        end
-
-        private
-
-        def validate
-          # from and to valid epoch times
-          raise_validation_error('from field not integer') unless from.is_a? Integer
-          raise_validation_error('from field is zero') if from.zero?
-          raise_validation_error('to field not integer') unless to.is_a? Integer
-          raise_validation_error('to field is zero') if to.zero?
-          raise_validation_error('from < to fields') if Time.at(to) < Time.at(from)
-          # application is array
-          raise_validation_error('applications field') unless applications.is_a? Array
-          raise_validation_error('applications values') unless applications.all? do |x|
-            x.is_a?(String) || x.is_a?(Integer)
-          end
-          # metrics is array
-          raise_validation_error('metrics field') unless metrics.is_a? Array
-          raise_validation_error('metrics values') unless metrics.all? do |x|
-            x.is_a?(String) || x.is_a?(Integer)
-          end
-        end
-
-        def raise_validation_error(msg)
-          raise DeleteServiceStatsValidationError.new(service_id, msg)
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/stats/key_generator.rb

@@ -1,73 +0,0 @@
-module ThreeScale
-  module Backend
-    module Stats
-      class KeyGenerator
-        attr_reader :service_id, :applications, :metrics, :from, :to
-
-        def initialize(service_id:, applications: [], metrics: [], from:, to:, **)
-          @service_id = service_id
-          @applications = applications
-          @metrics = metrics
-          @from = from
-          @to = to
-        end
-
-        def keys
-          response_code_service_keys +
-            response_code_application_keys +
-            usage_service_keys +
-            usage_application_keys
-        end
-
-        private
-
-        def periods(granularities)
-          granularities.flat_map do |granularity|
-            (Period[granularity].new(Time.at(from))..Period[granularity].new(Time.at(to))).to_a
-          end
-        end
-
-        def response_codes
-          CodesCommons::TRACKED_CODES + CodesCommons::TRACKED_CODE_GROUPS
-        end
-
-        def response_code_service_keys
-          periods(PeriodCommons::PERMANENT_SERVICE_GRANULARITIES).flat_map do |period|
-            response_codes.flat_map do |response_code|
-              Keys.service_response_code_value_key(service_id, response_code, period)
-            end
-          end
-        end
-
-        def response_code_application_keys
-          periods(PeriodCommons::PERMANENT_EXPANDED_GRANULARITIES).flat_map do |period|
-            response_codes.flat_map do |response_code|
-              applications.flat_map do |application|
-                Keys.application_response_code_value_key(service_id, application,
-                                                         response_code, period)
-              end
-            end
-          end
-        end
-
-        def usage_service_keys
-          periods(PeriodCommons::PERMANENT_SERVICE_GRANULARITIES).flat_map do |period|
-            metrics.flat_map do |metric|
-              Keys.service_usage_value_key(service_id, metric, period)
-            end
-          end
-        end
-
-        def usage_application_keys
-          periods(PeriodCommons::PERMANENT_EXPANDED_GRANULARITIES).flat_map do |period|
-            metrics.flat_map do |metric|
-              applications.flat_map do |application|
-                Keys.application_usage_value_key(service_id, application, metric, period)
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/stats/partition_eraser_job.rb

@@ -1,58 +0,0 @@
-module ThreeScale
-  module Backend
-    module Stats
-      # Job for deleting service stats
-      # Perform actual key deletion from a key partition definition
-      class PartitionEraserJob < BackgroundJob
-        # low priority queue
-        @queue = :stats
-
-        class << self
-          include StorageHelpers
-          include Configurable
-
-          def perform_logged(_enqueue_time, service_id, applications, metrics,
-                             from, to, offset, length, context_info = {})
-            job = DeleteJobDef.new(
-              service_id: service_id,
-              applications: applications,
-              metrics: metrics,
-              from: from,
-              to: to
-            )
-
-            validate_job(job, offset, length)
-
-            stats_key_gen = KeyGenerator.new(job.to_hash)
-
-            stats_key_gen.keys.drop(offset).take(length).each_slice(configuration.stats.delete_batch_size) do |slice|
-              storage.del(slice)
-            end
-
-            [true, { job: job.to_hash, offset: offset, lenght: length }.to_json]
-          rescue Backend::Error => error
-            [false, "#{service_id} #{error}"]
-          end
-
-          private
-
-          def validate_job(job, offset, length)
-            unless offset.is_a? Integer
-              raise DeleteServiceStatsValidationError.new(job.service_id, 'offset field value ' \
-                                                          "[#{offset}] validation error")
-            end
-
-            unless length.is_a? Integer
-              raise DeleteServiceStatsValidationError.new(job.service_id, 'length field value ' \
-                                                          "[#{length}] validation error")
-            end
-          end
-
-          def enqueue_time(args)
-            args[0]
-          end
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/stats/partition_generator_job.rb

@@ -1,46 +0,0 @@
-module ThreeScale
-  module Backend
-    module Stats
-      # Job for deleting service stats
-      # Maps delete job definition to a set of non overlapping key set partitions
-      class PartitionGeneratorJob < BackgroundJob
-        # low priority queue
-        @queue = :stats
-
-        class << self
-          include Configurable
-
-          def perform_logged(_enqueue_time, service_id, applications, metrics,
-                             from, to, context_info = {})
-            job = DeleteJobDef.new(
-              service_id: service_id,
-              applications: applications,
-              metrics: metrics,
-              from: from,
-              to: to
-            )
-
-            stats_key_gen = KeyGenerator.new(job.to_hash)
-
-            # Generate partitions
-            0.step(stats_key_gen.keys.count, configuration.stats.delete_partition_batch_size).each do |idx|
-              Resque.enqueue(PartitionEraserJob, Time.now.getutc.to_f, service_id, applications,
-                             metrics, from, to, idx,
-                             configuration.stats.delete_partition_batch_size, context_info)
-            end
-
-            [true, job.to_json]
-          rescue Backend::Error => error
-            [false, "#{service_id} #{error}"]
-          end
-
-          private
-
-          def enqueue_time(args)
-            args[0]
-          end
-        end
-      end
-    end
-  end
-end