apisonator 3.1.0 → 3.3.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: edd1887f6aa71c17fb5279f98988c9a3b5e631e80bc1f4142b6619b5f5e24f48
-  data.tar.gz: b6478d14e88a177480396a7d46d5f4e33cea683aea17303175ad18ae646e546f
+  metadata.gz: '09ccae8a2719bc3717d22dcdd78c5c996cbd4774f2528b91d77c74fbbe2e51b7'
+  data.tar.gz: 4a9f237a6d1b25f9cc763817030cbe562536a6d5ed89374326db2f99ea82971c
 SHA512:
-  metadata.gz: 944b75385817abc4d2f8828c61b2301d5cdfbb6c1bb681b7a3165590f23ad1b6c80451fcf579b8f3cf0010fe5a95e65605998f5d8ec94f1ff4ce690556b3011a
-  data.tar.gz: f3f44b1b2ebfe6c8fcbbf36b90dd44e5a7875d997a36b97288417472fa4156689508bfc95afdf196a7a168f768aa6e38521502b93078f128994959d184c1cd37
+  metadata.gz: 2818f8b3a97b701218c28ae93339650db57a0229181ee3573a24ffe215611747c1381e9f0e643eb9dc85b81d13d43336a08252a88cd63cd7fc78ed9a182c5c83
+  data.tar.gz: 7a9c2e389c239094e2c6f303749056d5a8ea2d216f5097e98e9a87ef5c3bcd4ea2d190c4b8277d659608f7c80f57d5d727bf65eb9330f4c08fb8777ab5a1ae81

data/CHANGELOG.md

@@ -2,6 +2,67 @@
 
 Notable changes to Apisonator will be tracked in this document.
 
+## 3.3.1.1 - 2021-02-12
+
+### Changed
+
+- Updated our Puma fork to v4.3.7
+([#261](https://github.com/3scale/apisonator/pull/261)).
+
+## 3.3.1 - 2021-02-11
+
+### Fixed
+
+- Usages with `#0` (set to 0) no longer generate unnecessary stats keys in Redis
+([#258](https://github.com/3scale/apisonator/pull/258)).
+
+## 3.3.0 - 2021-02-09
+
+### Added
+
+- Rake task to delete stats keys set to 0 in the DB left there because of [this
+issue](https://github.com/3scale/apisonator/pull/247)
+([#250](https://github.com/3scale/apisonator/pull/250)).
+
+### Fixed
+
+- Made the worker more reliable when configured in async mode. Now it handles
+connection errors better
+([#253](https://github.com/3scale/apisonator/pull/253)),
+([#254](https://github.com/3scale/apisonator/pull/254)), and
+([#255](https://github.com/3scale/apisonator/pull/255)).
+
+### Changed
+
+- Updated async-redis to v0.5.1
+([#251](https://github.com/3scale/apisonator/pull/251)).
+
+## 3.2.1 - 2021-01-22
+
+### Fixed
+
+- Reports of 0 hits no longer generate unnecessary stats keys in Redis
+([#247](https://github.com/3scale/apisonator/pull/247)).
+
+## 3.2.0 - 2021-01-19
+
+### Added
+
+- New endpoint in the internal API to get the provider key for a given (token,
+service_id) pair ([#243](https://github.com/3scale/apisonator/pull/243)).
+
+### Changed
+
+- The config file used when running in a Docker image now parses "1" and "true"
+(case-insensitive) as true
+([#245](https://github.com/3scale/apisonator/pull/245)).
+
+### Fixed
+
+- Fixed some metrics of the internal API that were not being counted
+correctly([#244](https://github.com/3scale/apisonator/pull/244)).
+
+
 ## 3.1.0 - 2020-10-14
 
 ### Added

data/Gemfile.base

@@ -15,11 +15,15 @@ platform :ruby do
 end
 
 group :test do
+  # Newer versions of rack-test don't work well with rspec-api-documentation.
+  # See https://github.com/rack/rack-test/pull/223 &
+  # https://github.com/zipmark/rspec_api_documentation/issues/342
+  gem 'rack-test',     '= 0.8.2'
+
   gem 'benchmark-ips', '~> 2.7.2'
   gem 'mocha',         '~> 1.3'
   gem 'nokogiri',      '~> 1.10.8'
   gem 'pkg-config',    '~> 1.1.7'
-  gem 'rack-test',     '~> 0.8.2'
   gem 'resque_unit',   '~> 0.4.4', source: 'https://rubygems.org'
   gem 'test-unit',     '~> 3.2.6'
   gem 'resque_spec',   '~> 0.17.0'
@@ -42,7 +46,7 @@ group :development, :test do
 end
 
 # Default server by platform
-gem 'puma', git: 'https://github.com/3scale/puma', ref: 'b034371406690d3e6c2a9301c4a48bd721f3efc3'
+gem 'puma', git: 'https://github.com/3scale/puma', branch: '3scale-4.3.7'
 # gems required by the runner
 gem 'gli', '~> 2.16.1', require: nil
 # Workers
@@ -53,13 +57,14 @@ gem 'rake', '~> 13.0'
 gem 'builder', '= 3.2.3'
 # Use a patched resque to allow reusing their Airbrake Failure class
 gem 'resque', git: 'https://github.com/3scale/resque', branch: '3scale'
+gem 'redis-namespace', '~>1.8.0'
 gem 'rack', '~> 2.1.4'
 gem 'sinatra', '~> 2.0.3'
 gem 'sinatra-contrib', '~> 2.0.3'
 # Optional external error logging services
 gem 'bugsnag', '~> 6', require: nil
 gem 'yabeda-prometheus', '~> 0.5.0'
-gem 'async-redis', '~> 0.5'
+gem 'async-redis', '~> 0.5.1'
 gem 'falcon', '~> 0.35'
 
 # Use a patched redis-rb that fixes an issue when trying to connect with

data/Gemfile.lock

@@ -1,9 +1,10 @@
 GIT
   remote: https://github.com/3scale/puma
-  revision: b034371406690d3e6c2a9301c4a48bd721f3efc3
-  ref: b034371406690d3e6c2a9301c4a48bd721f3efc3
+  revision: c0601d08695839b8ffd0f380e91c3b91c1e8b754
+  branch: 3scale-4.3.7
   specs:
-    puma (2.15.3)
+    puma (4.3.7)
+      nio4r (~> 2.0)
 
 GIT
   remote: https://github.com/3scale/redis-rb
@@ -35,7 +36,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.1.0)
+    apisonator (3.3.1.1)
 
 GEM
   remote: https://rubygems.org/
@@ -70,7 +71,7 @@ GEM
       async (~> 1.14)
     async-pool (0.2.0)
       async (~> 1.8)
-    async-redis (0.5.0)
+    async-redis (0.5.1)
       async (~> 1.8)
       async-io (~> 1.10)
       async-pool (~> 0.2)
@@ -142,7 +143,7 @@ GEM
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
     net-ssh (4.2.0)
-    nio4r (2.5.2)
+    nio4r (2.5.4)
     nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
     parslet (1.8.2)
@@ -178,7 +179,7 @@ GEM
     rack-test (0.8.2)
       rack (>= 1.0, < 3)
     rake (13.0.1)
-    redis-namespace (1.6.0)
+    redis-namespace (1.8.0)
       redis (>= 3.0.4)
     resque_spec (0.17.0)
       resque (>= 1.19.0)
@@ -241,7 +242,7 @@ GEM
     thread_safe (0.3.6)
     tilt (2.0.8)
     timecop (0.9.1)
-    timers (4.3.0)
+    timers (4.3.2)
     toml (0.2.0)
       parslet (~> 1.8.0)
     tzinfo (1.2.7)
@@ -267,7 +268,7 @@ PLATFORMS
 DEPENDENCIES
   airbrake (= 4.3.1)
   apisonator!
-  async-redis (~> 0.5)
+  async-redis (~> 0.5.1)
   async-rspec
   aws-sdk (= 2.4.2)
   benchmark-ips (~> 2.7.2)
@@ -288,9 +289,10 @@ DEPENDENCIES
   pry-doc (~> 0.11.1)
   puma!
   rack (~> 2.1.4)
-  rack-test (~> 0.8.2)
+  rack-test (= 0.8.2)
   rake (~> 13.0)
   redis!
+  redis-namespace (~> 1.8.0)
   resque!
   resque_spec (~> 0.17.0)
   resque_unit (~> 0.4.4)!

data/Gemfile.on_prem.lock

@@ -1,9 +1,10 @@
 GIT
   remote: https://github.com/3scale/puma
-  revision: b034371406690d3e6c2a9301c4a48bd721f3efc3
-  ref: b034371406690d3e6c2a9301c4a48bd721f3efc3
+  revision: c0601d08695839b8ffd0f380e91c3b91c1e8b754
+  branch: 3scale-4.3.7
   specs:
-    puma (2.15.3)
+    puma (4.3.7)
+      nio4r (~> 2.0)
 
 GIT
   remote: https://github.com/3scale/redis-rb
@@ -35,7 +36,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.1.0)
+    apisonator (3.3.1.1)
 
 GEM
   remote: https://rubygems.org/
@@ -67,7 +68,7 @@ GEM
       async (~> 1.14)
     async-pool (0.2.0)
       async (~> 1.8)
-    async-redis (0.5.0)
+    async-redis (0.5.1)
       async (~> 1.8)
       async-io (~> 1.10)
       async-pool (~> 0.2)
@@ -131,7 +132,7 @@ GEM
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
     net-ssh (4.2.0)
-    nio4r (2.5.2)
+    nio4r (2.5.4)
     nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
     parslet (1.8.2)
@@ -166,7 +167,7 @@ GEM
     rack-test (0.8.2)
       rack (>= 1.0, < 3)
     rake (13.0.1)
-    redis-namespace (1.6.0)
+    redis-namespace (1.8.0)
       redis (>= 3.0.4)
     resque_spec (0.17.0)
       resque (>= 1.19.0)
@@ -227,7 +228,7 @@ GEM
     thread_safe (0.3.6)
     tilt (2.0.8)
     timecop (0.9.1)
-    timers (4.3.0)
+    timers (4.3.2)
     toml (0.2.0)
       parslet (~> 1.8.0)
     tzinfo (1.2.7)
@@ -250,7 +251,7 @@ PLATFORMS
 
 DEPENDENCIES
   apisonator!
-  async-redis (~> 0.5)
+  async-redis (~> 0.5.1)
   async-rspec
   benchmark-ips (~> 2.7.2)
   bugsnag (~> 6)
@@ -269,9 +270,10 @@ DEPENDENCIES
   pry-doc (~> 0.11.1)
   puma!
   rack (~> 2.1.4)
-  rack-test (~> 0.8.2)
+  rack-test (= 0.8.2)
   rake (~> 13.0)
   redis!
+  redis-namespace (~> 1.8.0)
   resque!
   resque_spec (~> 0.17.0)
   resque_unit (~> 0.4.4)!

data/Rakefile

@@ -261,27 +261,49 @@ task :reschedule_failed_jobs do
        "Pending failed jobs: #{result[:failed_current]}."
 end
 
-desc 'Delete stats of services marked for deletion'
 namespace :stats do
+  desc 'Delete stats of services marked for deletion'
   task :cleanup, [:redis_urls, :log_deleted_keys] do |_, args|
-    redis_urls = args[:redis_urls] && args[:redis_urls].split(' ')
+    redis_conns = redis_conns(args[:redis_urls])
 
-    if redis_urls.nil? || redis_urls.empty?
+    if redis_conns.empty?
       puts 'No Redis URLs specified'
       exit(false)
     end
 
-    redis_clients = redis_urls.map do |redis_url|
-      parsed_uri = URI.parse(ThreeScale::Backend::Storage::Helpers.send(
-        :to_redis_uri, redis_url)
-      )
-      Redis.new(host: parsed_uri.host, port: parsed_uri.port)
+    ThreeScale::Backend::Stats::Cleaner.delete!(
+      redis_conns, log_deleted_keys: logger_for_deleted_keys(args[:log_deleted_keys])
+    )
+  end
+
+  desc 'Delete stats keys set to 0'
+  task :delete_stats_keys_set_to_0, [:redis_urls, :log_deleted_keys] do |_, args|
+    redis_conns = redis_conns(args[:redis_urls])
+
+    if redis_conns.empty?
+      puts 'No Redis URLs specified'
+      exit(false)
     end
 
-    log_deleted = args[:log_deleted_keys] == 'true' ? STDOUT : nil
+    ThreeScale::Backend::Stats::Cleaner.delete_stats_keys_set_to_0(
+      redis_conns, log_deleted_keys: logger_for_deleted_keys(args[:log_deleted_keys])
+    )
+  end
+end
 
-    ThreeScale::Backend::Stats::Cleaner.delete!(
-      redis_clients, log_deleted_keys: log_deleted
+def redis_conns(urls)
+  redis_urls = urls && urls.split(' ')
+
+  return [] if redis_urls.nil? || redis_urls.empty?
+
+  redis_urls.map do |redis_url|
+    parsed_uri = URI.parse(ThreeScale::Backend::Storage::Helpers.send(
+      :to_redis_uri, redis_url)
     )
+    Redis.new(host: parsed_uri.host, port: parsed_uri.port)
   end
 end
+
+def logger_for_deleted_keys(arg_log_deleted_keys)
+  arg_log_deleted_keys == 'true' ? STDOUT : nil
+end

data/app/api/internal/service_tokens.rb

@@ -7,6 +7,14 @@ module ThreeScale
           ServiceToken.exists?(token, service_id) ? 200 : 404
         end
 
+        get '/:token/:service_id/provider_key' do |token, service_id|
+          if ServiceToken.exists?(token, service_id)
+            { status: :found, provider_key: Service.provider_key_for(service_id) }.to_json
+          else
+            respond_with_404('token/service combination not found'.freeze)
+          end
+        end
+
         post '/' do
           check_tokens_param!
 

data/app/api/internal/stats.rb

@@ -6,32 +6,13 @@ module ThreeScale
           respond_with_404('service not found') unless Service.exists?(params[:service_id])
         end
 
-        # This is very slow and needs to be disabled until the performance
-        # issues are solved. In the meanwhile, the job will just return OK.
-=begin
-        delete '' do |service_id|
-          delete_stats_job_attrs = api_params Stats::DeleteJobDef
-          delete_stats_job_attrs[:service_id] = service_id
-          delete_stats_job_attrs[:from] = delete_stats_job_attrs[:from].to_i
-          delete_stats_job_attrs[:to] = delete_stats_job_attrs[:to].to_i
-          begin
-            Stats::DeleteJobDef.new(delete_stats_job_attrs).run_async
-          rescue DeleteServiceStatsValidationError => e
-            [400, headers, { status: :error, error: e.message }.to_json]
-          else
-            { status: :to_be_deleted }.to_json
-          end
-=end
-
-        # This is an alternative to the above. It just adds the service to a
-        # Redis set to marked is as "to be deleted".
-        # Later a script can read that set and actually delete the keys.
-        # Read the docs of the Stats::Cleaner class for more details.
+        # This adds the service to a Redis set to mark is as "to be deleted".
+        # Later a script can read that set and actually delete the keys. Read
+        # the docs of the Stats::Cleaner class for more details.
         #
-        # Notice that this method ignores the "from" and "to" parameters. When
-        # system calls this method, they're always interested in deleting all
-        # the keys. They were just passing "from" and "to" to make the
-        # implementation of the option above easier.
+        # Notice that this method ignores the "from" and "to" parameters used in
+        # previous versions. When system calls this method, they're always
+        # interested in deleting all the keys.
         delete '' do |service_id|
           Stats::Cleaner.mark_service_to_be_deleted(service_id)
           { status: :to_be_deleted }.to_json

data/lib/3scale/backend/application_events.rb

@@ -40,10 +40,8 @@ module ThreeScale
       private
 
       def self.first_traffic(service_id, application_id)
-        key = Stats::Keys.applications_key_prefix(
-          Stats::Keys.service_key_prefix(service_id)
-        )
-        if storage.sadd(key, encode_key(application_id))
+        if storage.sadd(Stats::Keys.set_of_apps_with_traffic(service_id),
+                        encode_key(application_id))
           EventStorage.store(:first_traffic,
                              { service_id:     service_id,
                                application_id: application_id,

data/lib/3scale/backend/configuration.rb

@@ -32,8 +32,6 @@ module ThreeScale
 
     CONFIG_DELETE_STATS_BATCH_SIZE = 50
     private_constant :CONFIG_DELETE_STATS_BATCH_SIZE
-    CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE = 1000
-    private_constant :CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE
 
     @configuration = Configuration::Loader.new
 
@@ -54,7 +52,7 @@ module ThreeScale
       config.add_section(:analytics_redis, :server,
                          :connect_timeout, :read_timeout, :write_timeout)
       config.add_section(:hoptoad, :service, :api_key)
-      config.add_section(:stats, :bucket_size, :delete_batch_size, :delete_partition_batch_size)
+      config.add_section(:stats, :bucket_size, :delete_batch_size)
       config.add_section(:redshift, :host, :port, :dbname, :user, :password)
       config.add_section(:statsd, :host, :port)
       config.add_section(:internal_api, :user, :password)
@@ -125,9 +123,6 @@ module ThreeScale
       config.stats.delete_batch_size = parse_int(config.stats.delete_batch_size,
                                                  CONFIG_DELETE_STATS_BATCH_SIZE)
 
-      config.stats.delete_partition_batch_size = parse_int(config.stats.delete_partition_batch_size,
-                                                           CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE)
-
       # often we don't have a log_file setting - generate it here from
       # the log_path setting.
       log_file = config.log_file

data/lib/3scale/backend/errors.rb

@@ -292,12 +292,6 @@ module ThreeScale
       end
     end
 
-    class DeleteServiceStatsValidationError < Error
-      def initialize(service_id, msg)
-        super "Delete stats job context validation error. Service: #{service_id}. Error: #{msg}"
-      end
-    end
-
     class EndUsersNoLongerSupported < BadRequest
       def initialize
         super 'End-users are no longer supported, do not specify the user_id parameter'.freeze

data/lib/3scale/backend/job_fetcher.rb

@@ -32,25 +32,6 @@ module ThreeScale
             DEFAULT_WAIT_BEFORE_FETCHING_MORE_JOBS
       end
 
-      def pop_from_queue
-        begin
-          encoded_job = @redis.blpop(*@queues, timeout: @fetch_timeout)
-        rescue Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE => e
-          raise RedisConnectionError.new(e.message)
-        rescue Redis::CommandError => e
-          # Redis::CommandError from redis-rb can be raised for multiple
-          # reasons, so we need to check the error message to distinguish
-          # connection errors from the rest.
-          if e.message == 'ERR Connection timed out'.freeze
-            raise RedisConnectionError.new(e.message)
-          else
-            raise e
-          end
-        end
-
-        encoded_job
-      end
-
       def fetch
         encoded_job = pop_from_queue
         return nil if encoded_job.nil? || encoded_job.empty?
@@ -99,10 +80,11 @@ module ThreeScale
 
               # Re-instantiate Redis instance. This is needed to recover from
               # Errno::EPIPE, not sure if there are others.
-              @redis = ThreeScale::Backend::QueueStorage.connection(
-                  ThreeScale::Backend.environment,
-                  ThreeScale::Backend.configuration
+              @redis = Redis::Namespace.new(
+                WorkerAsync.const_get(:RESQUE_REDIS_NAMESPACE),
+                redis: QueueStorage.connection(Backend.environment, Backend.configuration)
               )
+
              # If there is a different kind of error, it's probably a
              # programming error. Like sending an invalid blpop command to
              # Redis. In that case, let the worker crash.
@@ -111,12 +93,36 @@ module ThreeScale
           end
         end
 
+      rescue Exception => e
+        Worker.logger.notify(e)
+      ensure
         job_queue.close
       end
 
       def shutdown
         @shutdown = true
       end
+
+      private
+
+      def pop_from_queue
+        begin
+          encoded_job = @redis.blpop(*@queues, timeout: @fetch_timeout)
+        rescue Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE => e
+          raise RedisConnectionError.new(e.message)
+        rescue Redis::CommandError => e
+          # Redis::CommandError from redis-rb can be raised for multiple
+          # reasons, so we need to check the error message to distinguish
+          # connection errors from the rest.
+          if e.message == 'ERR Connection timed out'.freeze
+            raise RedisConnectionError.new(e.message)
+          else
+            raise e
+          end
+        end
+
+        encoded_job
+      end
     end
   end
 end

data/lib/3scale/backend/listener_metrics.rb

@@ -13,19 +13,21 @@ module ThreeScale
       }
       private_constant :AUTH_AND_REPORT_REQUEST_TYPES
 
+      # Only the first match is taken into account, that's why for example,
+      # "/\/services\/.*\/stats/" needs to appear before "/\/services/"
       INTERNAL_API_PATHS = [
-        [/\/services\/.*\/alert_limits/, 'alerts'],
-        [/\/services\/.*\/applications\/.*\/keys/, 'application_keys'],
-        [/\/services\/.*\/applications\/.*\/referrer_filters/, 'application_referrer_filters'],
-        [/\/services\/.*\/applications/, 'applications'],
-        [/\/services\/.*\/errors/, 'errors'],
-        [/\/events/, 'events'],
-        [/\/services\/.*\/metrics/, 'metrics'],
-        [/\/service_tokens/, 'service_tokens'],
-        [/\/services/, 'services'],
-        [/\/services\/.*\/stats/, 'stats'],
-        [/\/services\/.*\/plans\/.*\/usagelimits/, 'usage_limits'],
-        [/\/services\/.*\/applications\/.*\/utilization/, 'utilization'],
+        [/\/services\/.*\/alert_limits/, 'alerts'.freeze],
+        [/\/services\/.*\/applications\/.*\/keys/, 'application_keys'.freeze],
+        [/\/services\/.*\/applications\/.*\/referrer_filters/, 'application_referrer_filters'.freeze],
+        [/\/services\/.*\/applications\/.*\/utilization/, 'utilization'.freeze],
+        [/\/services\/.*\/applications/, 'applications'.freeze],
+        [/\/services\/.*\/errors/, 'errors'.freeze],
+        [/\/events/, 'events'.freeze],
+        [/\/services\/.*\/metrics/, 'metrics'.freeze],
+        [/\/service_tokens/, 'service_tokens'.freeze],
+        [/\/services\/.*\/stats/, 'stats'.freeze],
+        [/\/services\/.*\/plans\/.*\/usagelimits/, 'usage_limits'.freeze],
+        [/\/services/, 'services'.freeze],
       ].freeze
       private_constant :INTERNAL_API_PATHS
 

data/lib/3scale/backend/stats.rb

@@ -1,8 +1,4 @@
 require '3scale/backend/stats/codes_commons'
 require '3scale/backend/stats/period_commons'
 require '3scale/backend/stats/aggregator'
-require '3scale/backend/stats/delete_job_def'
-require '3scale/backend/stats/key_generator'
-require '3scale/backend/stats/partition_generator_job'
-require '3scale/backend/stats/partition_eraser_job'
 require '3scale/backend/stats/cleaner'

data/lib/3scale/backend/stats/aggregators/base.rb

@@ -20,7 +20,14 @@ module ThreeScale
                 key = counter_key(prefix_key, granularity.new(timestamp))
                 expire_time = Stats::PeriodCommons.expire_time_for_granularity(granularity)
 
-                store_key(cmd, key, value, expire_time)
+                # We don't need to store stats keys set to 0. It wastes Redis
+                # memory because for rate-limiting and stats, a key of set to 0
+                # is equivalent to a key that does not exist.
+                if cmd == :set && value == 0
+                  storage.del(key)
+                else
+                  store_key(cmd, key, value, expire_time)
+                end
 
                 unless Stats::PeriodCommons::EXCLUDED_FOR_BUCKETS.include?(granularity)
                   keys_for_bucket << key

data/lib/3scale/backend/stats/cleaner.rb

@@ -45,6 +45,12 @@ module ThreeScale
         STATS_KEY_PREFIX = 'stats/'.freeze
         private_constant :STATS_KEY_PREFIX
 
+        REDIS_CONN_ERRORS = [Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE].freeze
+        private_constant :REDIS_CONN_ERRORS
+
+        MAX_RETRIES_REDIS_ERRORS = 3
+        private_constant :MAX_RETRIES_REDIS_ERRORS
+
         class << self
           include Logging
           def mark_service_to_be_deleted(service_id)
@@ -77,37 +83,73 @@ module ThreeScale
             logger.info("Going to delete the stats keys for these services: #{services.to_a}")
 
             unless services.empty?
-              delete_successful = true
-              redis_conns.each do |redis_conn|
+              _ok, failed = redis_conns.partition do |redis_conn|
                 begin
                   delete_keys(redis_conn, services, log_deleted_keys)
-                # If it's a connection error, mark as failed and continue
-                # cleaning other shards. If it's another kind of error, it
-                # could be a bug, so better re-raise.
-                rescue Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE => e
-                  logger.error("Error while deleting stats of server #{redis_conn}: #{e}")
-                  delete_successful = false
-                rescue Redis::CommandError => e
-                  # Redis::CommandError from redis-rb can be raised for multiple
-                  # reasons, so we need to check the error message to distinguish
-                  # connection errors from the rest.
-                  if e.message == 'ERR Connection timed out'.freeze
-                    logger.error("Error while deleting stats of server #{redis_conn}: #{e}")
-                    delete_successful = false
-                  else
-                    raise e
-                  end
+                  true
+                rescue => e
+                  handle_redis_exception(e, redis_conn)
+                  false
                 end
               end
 
-              remove_services_from_delete_set(services) if delete_successful
+              with_retries { remove_services_from_delete_set(services) } if failed.empty?
+
+              failed.each do |failed_conn|
+                logger.error("Error while deleting stats of server #{failed_conn}")
+              end
             end
 
             logger.info("Finished deleting the stats keys for these services: #{services.to_a}")
           end
 
+          # Deletes all the stats keys set to 0.
+          #
+          # Stats keys set to 0 are useless and occupy Redis memory
+          # unnecessarily. They were generated due to a bug in previous versions
+          # of Apisonator.
+          # Ref: https://github.com/3scale/apisonator/pull/247
+          #
+          # As the .delete function, this one also receives a collection of
+          # instantiated Redis clients and those need to connect to Redis
+          # servers directly.
+          #
+          # @param [Array] redis_conns Instantiated Redis clients.
+          # @param [IO] log_deleted_keys IO where to write the logs. Defaults to
+          #             nil (logs nothing).
+          def delete_stats_keys_set_to_0(redis_conns, log_deleted_keys: nil)
+            _ok, failed = redis_conns.partition do |redis_conn|
+              begin
+                delete_stats_keys_with_val_0(redis_conn, log_deleted_keys)
+                true
+              rescue => e
+                handle_redis_exception(e, redis_conn)
+                false
+              end
+            end
+
+            failed.each do |failed_conn|
+              logger.error("Error while deleting stats of server #{failed_conn}")
+            end
+          end
+
           private
 
+          def handle_redis_exception(exception, redis_conn)
+            # If it's a connection error, do nothing so we can continue with
+            # other shards. If it's another kind of error, it could be caused by
+            # a bug, so better re-raise.
+
+            case exception
+            when *REDIS_CONN_ERRORS
+              # Do nothing.
+            when Redis::CommandError
+              raise exception if exception.message != 'ERR Connection timed out'.freeze
+            else
+              raise exception
+            end
+          end
+
           # Returns a set with the services included in the
           # SET_WITH_SERVICES_MARKED_FOR_DELETION Redis set.
           def services_to_delete
@@ -133,19 +175,21 @@ module ThreeScale
             cursor = 0
 
             loop do
-              cursor, keys = redis_conn.scan(cursor, count: SCAN_SLICE)
+              with_retries do
+                cursor, keys = redis_conn.scan(cursor, count: SCAN_SLICE)
 
-              to_delete = keys.select { |key| delete_key?(key, services) }
+                to_delete = keys.select { |key| delete_key?(key, services) }
 
-              unless to_delete.empty?
-                if log_deleted_keys
-                  values = redis_conn.mget(*(to_delete.to_a))
-                  to_delete.each_with_index do |k, i|
-                    log_deleted_keys.puts "#{k} #{values[i]}"
+                unless to_delete.empty?
+                  if log_deleted_keys
+                    values = redis_conn.mget(*(to_delete.to_a))
+                    to_delete.each_with_index do |k, i|
+                      log_deleted_keys.puts "#{k} #{values[i]}"
+                    end
                   end
-                end
 
-                redis_conn.del(to_delete)
+                  redis_conn.del(to_delete)
+                end
               end
 
               break if cursor.to_i == 0
@@ -188,6 +232,43 @@ module ThreeScale
             # simply ignore those keys.
             nil
           end
+
+          def delete_stats_keys_with_val_0(redis_conn, log_deleted_keys)
+            cursor = 0
+
+            loop do
+              with_retries do
+                cursor, keys = redis_conn.scan(cursor, count: SCAN_SLICE)
+
+                stats_keys = keys.select { |k| is_stats_key?(k) }
+
+                unless stats_keys.empty?
+                  values = redis_conn.mget(*stats_keys)
+                  to_delete = stats_keys.zip(values).select { |_, v| v == '0'.freeze }.map(&:first)
+
+                  unless to_delete.empty?
+                    redis_conn.del(to_delete)
+                    to_delete.each { |k| log_deleted_keys.puts k } if log_deleted_keys
+                  end
+                end
+              end
+
+              break if cursor.to_i == 0
+
+              sleep(SLEEP_BETWEEN_SCANS)
+            end
+          end
+
+          def with_retries(max = MAX_RETRIES_REDIS_ERRORS)
+            retries = 0
+            begin
+              yield
+            rescue Exception => e
+              retries += 1
+              retry if retries < max
+              raise e
+            end
+          end
         end
       end
     end

data/lib/3scale/backend/stats/keys.rb

@@ -70,6 +70,12 @@ module ThreeScale
           key
         end
 
+        def set_of_apps_with_traffic(service_id)
+          Stats::Keys.applications_key_prefix(
+            Stats::Keys.service_key_prefix(service_id)
+          )
+        end
+
         # We want all the buckets to go to the same Redis shard.
         # The reason is that SUNION support in Twemproxy requires that the
         # supplied keys hash to the same server.

data/lib/3scale/backend/stats/period_commons.rb

@@ -12,9 +12,6 @@ module ThreeScale
         GRANULARITY_EXPIRATION_TIME = { Period[:minute] => 180 }.freeze
         private_constant :GRANULARITY_EXPIRATION_TIME
 
-        PERMANENT_SERVICE_GRANULARITIES = (SERVICE_GRANULARITIES - GRANULARITY_EXPIRATION_TIME.keys).freeze
-        PERMANENT_EXPANDED_GRANULARITIES = (EXPANDED_GRANULARITIES - GRANULARITY_EXPIRATION_TIME.keys).freeze
-
         # We are not going to send metrics with granularity 'eternity' or
         # 'week' to Kinesis, so there is no point in storing them in Redis
         # buckets.

data/lib/3scale/backend/transactor.rb

@@ -20,8 +20,14 @@ module ThreeScale
       def report(provider_key, service_id, transactions, context_info = {})
         service = Service.load_with_provider_key!(service_id, provider_key)
 
-        report_enqueue(service.id, transactions, context_info)
-        notify_report(provider_key, transactions.size)
+        # A usage of 0 does not affect rate-limits or stats, so we do not need
+        # to report it.
+        filtered_transactions = filter_usages_with_0(transactions.clone)
+
+        return if filtered_transactions.empty?
+
+        report_enqueue(service.id, filtered_transactions, context_info)
+        notify_report(provider_key, filtered_transactions.size)
       end
 
       def authorize(provider_key, params, context_info = {})
@@ -137,9 +143,17 @@ module ThreeScale
 
         usage = params[:usage]
 
-        if (usage || params[:log]) && status.authorized?
+        filtered_usage = filter_metrics_without_inc(usage.clone) if usage
+
+        if ((filtered_usage && !filtered_usage.empty?) || params[:log]) && status.authorized?
           application_id = status.application.id
-          report_enqueue(status.service_id, { 0 => {"app_id" => application_id, "usage" => usage, "log" => params[:log] } }, request: { extensions: request_info[:extensions] })
+
+          report_enqueue(
+            status.service_id,
+            { 0 => {"app_id" => application_id, "usage" => filtered_usage, "log" => params[:log] } },
+            request: { extensions: request_info[:extensions] }
+          )
+
           notify_authrep(provider_key, usage ? 1 : 0)
         else
           notify_authorize(provider_key)
@@ -182,6 +196,19 @@ module ThreeScale
         end
       end
 
+      def filter_usages_with_0(transactions)
+        # There are plenty of existing tests using both a string and a symbol
+        # when accessing the usage.
+        transactions.delete_if do |_idx, tx|
+          (usage = tx['usage'.freeze] || tx[:usage]) or next
+          filter_metrics_without_inc(usage).empty?
+        end
+      end
+
+      def filter_metrics_without_inc(usage)
+        usage.delete_if { |_metric, delta| delta.to_s == '0'.freeze }
+      end
+
       def storage
         Storage.instance
       end

data/lib/3scale/backend/version.rb

@@ -1,5 +1,5 @@
 module ThreeScale
   module Backend
-    VERSION = '3.1.0'
+    VERSION = '3.3.1.1'
   end
 end

data/lib/3scale/backend/worker_async.rb

@@ -1,4 +1,5 @@
 require 'async'
+require 'redis-namespace'
 require '3scale/backend/job_fetcher'
 
 module ThreeScale
@@ -10,6 +11,9 @@ module ThreeScale
       DEFAULT_MAX_CONCURRENT_JOBS = 20
       private_constant :DEFAULT_MAX_CONCURRENT_JOBS
 
+      RESQUE_REDIS_NAMESPACE = :resque
+      private_constant :RESQUE_REDIS_NAMESPACE
+
       def initialize(options = {})
         trap('TERM') { shutdown }
         trap('INT')  { shutdown }
@@ -17,7 +21,7 @@ module ThreeScale
         @one_off = options[:one_off]
         @jobs = Queue.new # Thread-safe queue
 
-        @job_fetcher = options[:job_fetcher] || JobFetcher.new
+        @job_fetcher = options[:job_fetcher] || JobFetcher.new(redis_client: redis_client)
 
         @max_concurrent_jobs = configuration.async_worker.max_concurrent_jobs ||
             DEFAULT_MAX_CONCURRENT_JOBS
@@ -64,6 +68,10 @@ module ThreeScale
           # unblocks when there are new jobs or when .close() is called
           job = @jobs.pop
 
+          # If job is nil, it means that the queue is closed. No more jobs are
+          # going to be pushed, so shutdown.
+          shutdown unless job
+
           break if @shutdown
 
           @reactor.async { perform(job) }
@@ -83,6 +91,19 @@ module ThreeScale
           Async { @job_fetcher.start(@jobs) }
         end
       end
+
+      # Returns a new Redis client with namespace "resque".
+      # In the async worker, the job fetcher runs in a separate thread, and we
+      # need to avoid sharing an already instantiated client like the one in
+      # Resque::Helpers initialized in lib/3scale/backend.rb (Resque.redis).
+      # Failing to do so, will raise errors because of fibers shared across
+      # threads.
+      def redis_client
+        Redis::Namespace.new(
+          RESQUE_REDIS_NAMESPACE,
+          redis: QueueStorage.connection(Backend.environment, Backend.configuration)
+        )
+      end
     end
   end
 end

data/licenses.xml

@@ -23,7 +23,7 @@
       </dependency>
           <dependency>
         <packageName>apisonator</packageName>
-        <version>3.1.0</version>
+        <version>3.3.1.1</version>
         <licenses>
                       <license>
               <name>Apache 2.0</name>
@@ -93,7 +93,7 @@
       </dependency>
           <dependency>
         <packageName>async-redis</packageName>
-        <version>0.5.0</version>
+        <version>0.5.1</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -525,7 +525,7 @@
       </dependency>
           <dependency>
         <packageName>nio4r</packageName>
-        <version>2.5.2</version>
+        <version>2.5.4</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -709,7 +709,7 @@
       </dependency>
           <dependency>
         <packageName>puma</packageName>
-        <version>2.15.3</version>
+        <version>4.3.7</version>
         <licenses>
                       <license>
               <name>New BSD</name>
@@ -769,7 +769,7 @@
       </dependency>
           <dependency>
         <packageName>redis-namespace</packageName>
-        <version>1.6.0</version>
+        <version>1.8.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -1043,7 +1043,7 @@
       </dependency>
           <dependency>
         <packageName>timers</packageName>
-        <version>4.3.0</version>
+        <version>4.3.2</version>
         <licenses>
                       <license>
               <name>MIT</name>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apisonator
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.3.1.1
 platform: ruby
 authors:
 - Adam Ciganek
@@ -16,7 +16,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-14 00:00:00.000000000 Z
+date: 2021-02-12 00:00:00.000000000 Z
 dependencies: []
 description: This gem provides a daemon that handles authorization and reporting of
   web services managed by 3scale.
@@ -136,11 +136,7 @@ files:
 - lib/3scale/backend/stats/bucket_storage.rb
 - lib/3scale/backend/stats/cleaner.rb
 - lib/3scale/backend/stats/codes_commons.rb
-- lib/3scale/backend/stats/delete_job_def.rb
-- lib/3scale/backend/stats/key_generator.rb
 - lib/3scale/backend/stats/keys.rb
-- lib/3scale/backend/stats/partition_eraser_job.rb
-- lib/3scale/backend/stats/partition_generator_job.rb
 - lib/3scale/backend/stats/period_commons.rb
 - lib/3scale/backend/stats/stats_parser.rb
 - lib/3scale/backend/stats/storage.rb

data/lib/3scale/backend/stats/delete_job_def.rb

@@ -1,60 +0,0 @@
-module ThreeScale
-  module Backend
-    module Stats
-      class DeleteJobDef
-        ATTRIBUTES = %i[service_id applications metrics from to context_info].freeze
-        private_constant :ATTRIBUTES
-        attr_reader(*ATTRIBUTES)
-
-        def self.attribute_names
-          ATTRIBUTES
-        end
-
-        def initialize(params = {})
-          ATTRIBUTES.each do |key|
-            instance_variable_set("@#{key}".to_sym, params[key]) unless params[key].nil?
-          end
-          validate
-        end
-
-        def run_async
-          Resque.enqueue(PartitionGeneratorJob, Time.now.getutc.to_f, service_id, applications,
-                         metrics, from, to, context_info)
-        end
-
-        def to_json
-          to_hash.to_json
-        end
-
-        def to_hash
-          Hash[ATTRIBUTES.collect { |key| [key, send(key)] }]
-        end
-
-        private
-
-        def validate
-          # from and to valid epoch times
-          raise_validation_error('from field not integer') unless from.is_a? Integer
-          raise_validation_error('from field is zero') if from.zero?
-          raise_validation_error('to field not integer') unless to.is_a? Integer
-          raise_validation_error('to field is zero') if to.zero?
-          raise_validation_error('from < to fields') if Time.at(to) < Time.at(from)
-          # application is array
-          raise_validation_error('applications field') unless applications.is_a? Array
-          raise_validation_error('applications values') unless applications.all? do |x|
-            x.is_a?(String) || x.is_a?(Integer)
-          end
-          # metrics is array
-          raise_validation_error('metrics field') unless metrics.is_a? Array
-          raise_validation_error('metrics values') unless metrics.all? do |x|
-            x.is_a?(String) || x.is_a?(Integer)
-          end
-        end
-
-        def raise_validation_error(msg)
-          raise DeleteServiceStatsValidationError.new(service_id, msg)
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/stats/key_generator.rb

@@ -1,73 +0,0 @@
-module ThreeScale
-  module Backend
-    module Stats
-      class KeyGenerator
-        attr_reader :service_id, :applications, :metrics, :from, :to
-
-        def initialize(service_id:, applications: [], metrics: [], from:, to:, **)
-          @service_id = service_id
-          @applications = applications
-          @metrics = metrics
-          @from = from
-          @to = to
-        end
-
-        def keys
-          response_code_service_keys +
-            response_code_application_keys +
-            usage_service_keys +
-            usage_application_keys
-        end
-
-        private
-
-        def periods(granularities)
-          granularities.flat_map do |granularity|
-            (Period[granularity].new(Time.at(from))..Period[granularity].new(Time.at(to))).to_a
-          end
-        end
-
-        def response_codes
-          CodesCommons::TRACKED_CODES + CodesCommons::TRACKED_CODE_GROUPS
-        end
-
-        def response_code_service_keys
-          periods(PeriodCommons::PERMANENT_SERVICE_GRANULARITIES).flat_map do |period|
-            response_codes.flat_map do |response_code|
-              Keys.service_response_code_value_key(service_id, response_code, period)
-            end
-          end
-        end
-
-        def response_code_application_keys
-          periods(PeriodCommons::PERMANENT_EXPANDED_GRANULARITIES).flat_map do |period|
-            response_codes.flat_map do |response_code|
-              applications.flat_map do |application|
-                Keys.application_response_code_value_key(service_id, application,
-                                                         response_code, period)
-              end
-            end
-          end
-        end
-
-        def usage_service_keys
-          periods(PeriodCommons::PERMANENT_SERVICE_GRANULARITIES).flat_map do |period|
-            metrics.flat_map do |metric|
-              Keys.service_usage_value_key(service_id, metric, period)
-            end
-          end
-        end
-
-        def usage_application_keys
-          periods(PeriodCommons::PERMANENT_EXPANDED_GRANULARITIES).flat_map do |period|
-            metrics.flat_map do |metric|
-              applications.flat_map do |application|
-                Keys.application_usage_value_key(service_id, application, metric, period)
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/stats/partition_eraser_job.rb

@@ -1,58 +0,0 @@
-module ThreeScale
-  module Backend
-    module Stats
-      # Job for deleting service stats
-      # Perform actual key deletion from a key partition definition
-      class PartitionEraserJob < BackgroundJob
-        # low priority queue
-        @queue = :stats
-
-        class << self
-          include StorageHelpers
-          include Configurable
-
-          def perform_logged(_enqueue_time, service_id, applications, metrics,
-                             from, to, offset, length, context_info = {})
-            job = DeleteJobDef.new(
-              service_id: service_id,
-              applications: applications,
-              metrics: metrics,
-              from: from,
-              to: to
-            )
-
-            validate_job(job, offset, length)
-
-            stats_key_gen = KeyGenerator.new(job.to_hash)
-
-            stats_key_gen.keys.drop(offset).take(length).each_slice(configuration.stats.delete_batch_size) do |slice|
-              storage.del(slice)
-            end
-
-            [true, { job: job.to_hash, offset: offset, lenght: length }.to_json]
-          rescue Backend::Error => error
-            [false, "#{service_id} #{error}"]
-          end
-
-          private
-
-          def validate_job(job, offset, length)
-            unless offset.is_a? Integer
-              raise DeleteServiceStatsValidationError.new(job.service_id, 'offset field value ' \
-                                                          "[#{offset}] validation error")
-            end
-
-            unless length.is_a? Integer
-              raise DeleteServiceStatsValidationError.new(job.service_id, 'length field value ' \
-                                                          "[#{length}] validation error")
-            end
-          end
-
-          def enqueue_time(args)
-            args[0]
-          end
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/stats/partition_generator_job.rb

@@ -1,46 +0,0 @@
-module ThreeScale
-  module Backend
-    module Stats
-      # Job for deleting service stats
-      # Maps delete job definition to a set of non overlapping key set partitions
-      class PartitionGeneratorJob < BackgroundJob
-        # low priority queue
-        @queue = :stats
-
-        class << self
-          include Configurable
-
-          def perform_logged(_enqueue_time, service_id, applications, metrics,
-                             from, to, context_info = {})
-            job = DeleteJobDef.new(
-              service_id: service_id,
-              applications: applications,
-              metrics: metrics,
-              from: from,
-              to: to
-            )
-
-            stats_key_gen = KeyGenerator.new(job.to_hash)
-
-            # Generate partitions
-            0.step(stats_key_gen.keys.count, configuration.stats.delete_partition_batch_size).each do |idx|
-              Resque.enqueue(PartitionEraserJob, Time.now.getutc.to_f, service_id, applications,
-                             metrics, from, to, idx,
-                             configuration.stats.delete_partition_batch_size, context_info)
-            end
-
-            [true, job.to_json]
-          rescue Backend::Error => error
-            [false, "#{service_id} #{error}"]
-          end
-
-          private
-
-          def enqueue_time(args)
-            args[0]
-          end
-        end
-      end
-    end
-  end
-end