apisonator 3.0.1.1 → 3.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bd4ac963d174350e0e93513a0bfe572110c9e90659f7907277996358c9761902
-  data.tar.gz: 3c327a805079b402cc65330cb4ff3e84a247ef9c57c781a9e4e103be283cc8a5
+  metadata.gz: a812d330bdcb770421ed926c478c8a8b48087ba33ad28ab80317a48db935f432
+  data.tar.gz: 5bc8b40c69fbf5a6a680b71692dc66433746b053c0dbf3dcfe6e3d4aacd656af
 SHA512:
-  metadata.gz: 52f2e4795a62a1ed314b21d1b3839dc78c2e6f8ee5bcb6fce02d9b5cd1ced3004915a78927b9fd667d210a2bb547fe34f27844dd267cdb405ff36eb629d7e1c8
-  data.tar.gz: df81a8aceb61a99c1151f3418c156ce70e1c18a9e2ab28db80d9f35424bf73fe82cfe46bed02c6da5e229cb212344432dd603520cd1dea8940e4448c14e6c3e8
+  metadata.gz: 807c4d8cfc932e600780f4ba97fa5cfa3afcaca04ad0e27395e92fca31407d96e360412a290566c78955e5cf12895a5f4a70e589474de045bf3f0f7cf21311af
+  data.tar.gz: 17bf0d7c783ee36b78a885a441846edeaf8906185ab183c91ffc7f0c7f7c13e45b4237c0c7adb1a228794b3f64d4a0729f6eb8360a59910021d65a0b99c1edc1

data/CHANGELOG.md

@@ -2,6 +2,80 @@
 
 Notable changes to Apisonator will be tracked in this document.
 
+## 3.3.1 - 2021-02-11
+
+### Fixed
+
+- Usages with `#0` (set to 0) no longer generate unnecessary stats keys in Redis
+([#258](https://github.com/3scale/apisonator/pull/258)).
+
+## 3.3.0 - 2021-02-09
+
+### Added
+
+- Rake task to delete stats keys set to 0 in the DB left there because of [this
+issue](https://github.com/3scale/apisonator/pull/247)
+([#250](https://github.com/3scale/apisonator/pull/250)).
+
+### Fixed
+
+- Made the worker more reliable when configured in async mode. Now it handles
+connection errors better
+([#253](https://github.com/3scale/apisonator/pull/253)),
+([#254](https://github.com/3scale/apisonator/pull/254)), and
+([#255](https://github.com/3scale/apisonator/pull/255)).
+
+### Changed
+
+- Updated async-redis to v0.5.1
+([#251](https://github.com/3scale/apisonator/pull/251)).
+
+## 3.2.1 - 2021-01-22
+
+### Fixed
+
+- Reports of 0 hits no longer generate unnecessary stats keys in Redis
+([#247](https://github.com/3scale/apisonator/pull/247)).
+
+## 3.2.0 - 2021-01-19
+
+### Added
+
+- New endpoint in the internal API to get the provider key for a given (token,
+service_id) pair ([#243](https://github.com/3scale/apisonator/pull/243)).
+
+### Changed
+
+- The config file used when running in a Docker image now parses "1" and "true"
+(case-insensitive) as true
+([#245](https://github.com/3scale/apisonator/pull/245)).
+
+### Fixed
+
+- Fixed some metrics of the internal API that were not being counted
+correctly([#244](https://github.com/3scale/apisonator/pull/244)).
+
+
+## 3.1.0 - 2020-10-14
+
+### Added
+
+- Prometheus metrics for the internal API
+([#236](https://github.com/3scale/apisonator/pull/236)).
+- Docs with a detailed explanation about how counter updates are performed
+([#239](https://github.com/3scale/apisonator/pull/239)).
+
+### Changed
+
+- NotifyJobs are run only when the service ID is explicitly defined
+([#238](https://github.com/3scale/apisonator/pull/238)).
+
+### Fixed
+
+- Fixed corner case that raised "TransactionTimestampNotWithinRange" in notify
+jobs ([#235](https://github.com/3scale/apisonator/pull/235)).
+
+
 ## 3.0.1.1 - 2020-07-28
 
 ### Changed

data/Gemfile.base

@@ -15,11 +15,15 @@ platform :ruby do
 end
 
 group :test do
+  # Newer versions of rack-test don't work well with rspec-api-documentation.
+  # See https://github.com/rack/rack-test/pull/223 &
+  # https://github.com/zipmark/rspec_api_documentation/issues/342
+  gem 'rack-test',     '= 0.8.2'
+
   gem 'benchmark-ips', '~> 2.7.2'
   gem 'mocha',         '~> 1.3'
   gem 'nokogiri',      '~> 1.10.8'
   gem 'pkg-config',    '~> 1.1.7'
-  gem 'rack-test',     '~> 0.8.2'
   gem 'resque_unit',   '~> 0.4.4', source: 'https://rubygems.org'
   gem 'test-unit',     '~> 3.2.6'
   gem 'resque_spec',   '~> 0.17.0'
@@ -53,13 +57,14 @@ gem 'rake', '~> 13.0'
 gem 'builder', '= 3.2.3'
 # Use a patched resque to allow reusing their Airbrake Failure class
 gem 'resque', git: 'https://github.com/3scale/resque', branch: '3scale'
+gem 'redis-namespace', '~>1.8.0'
 gem 'rack', '~> 2.1.4'
 gem 'sinatra', '~> 2.0.3'
 gem 'sinatra-contrib', '~> 2.0.3'
 # Optional external error logging services
 gem 'bugsnag', '~> 6', require: nil
 gem 'yabeda-prometheus', '~> 0.5.0'
-gem 'async-redis', '~> 0.5'
+gem 'async-redis', '~> 0.5.1'
 gem 'falcon', '~> 0.35'
 
 # Use a patched redis-rb that fixes an issue when trying to connect with

data/Gemfile.lock

@@ -35,7 +35,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.0.1.1)
+    apisonator (3.3.1)
 
 GEM
   remote: https://rubygems.org/
@@ -70,7 +70,7 @@ GEM
       async (~> 1.14)
     async-pool (0.2.0)
       async (~> 1.8)
-    async-redis (0.5.0)
+    async-redis (0.5.1)
       async (~> 1.8)
       async-io (~> 1.10)
       async-pool (~> 0.2)
@@ -142,7 +142,7 @@ GEM
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
     net-ssh (4.2.0)
-    nio4r (2.5.2)
+    nio4r (2.5.4)
     nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
     parslet (1.8.2)
@@ -178,7 +178,7 @@ GEM
     rack-test (0.8.2)
       rack (>= 1.0, < 3)
     rake (13.0.1)
-    redis-namespace (1.6.0)
+    redis-namespace (1.8.0)
       redis (>= 3.0.4)
     resque_spec (0.17.0)
       resque (>= 1.19.0)
@@ -241,7 +241,7 @@ GEM
     thread_safe (0.3.6)
     tilt (2.0.8)
     timecop (0.9.1)
-    timers (4.3.0)
+    timers (4.3.2)
     toml (0.2.0)
       parslet (~> 1.8.0)
     tzinfo (1.2.7)
@@ -267,7 +267,7 @@ PLATFORMS
 DEPENDENCIES
   airbrake (= 4.3.1)
   apisonator!
-  async-redis (~> 0.5)
+  async-redis (~> 0.5.1)
   async-rspec
   aws-sdk (= 2.4.2)
   benchmark-ips (~> 2.7.2)
@@ -288,9 +288,10 @@ DEPENDENCIES
   pry-doc (~> 0.11.1)
   puma!
   rack (~> 2.1.4)
-  rack-test (~> 0.8.2)
+  rack-test (= 0.8.2)
   rake (~> 13.0)
   redis!
+  redis-namespace (~> 1.8.0)
   resque!
   resque_spec (~> 0.17.0)
   resque_unit (~> 0.4.4)!

data/Gemfile.on_prem.lock

@@ -35,7 +35,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.0.1.1)
+    apisonator (3.3.1)
 
 GEM
   remote: https://rubygems.org/
@@ -67,7 +67,7 @@ GEM
       async (~> 1.14)
     async-pool (0.2.0)
       async (~> 1.8)
-    async-redis (0.5.0)
+    async-redis (0.5.1)
       async (~> 1.8)
       async-io (~> 1.10)
       async-pool (~> 0.2)
@@ -131,7 +131,7 @@ GEM
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
     net-ssh (4.2.0)
-    nio4r (2.5.2)
+    nio4r (2.5.4)
     nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
     parslet (1.8.2)
@@ -166,7 +166,7 @@ GEM
     rack-test (0.8.2)
       rack (>= 1.0, < 3)
     rake (13.0.1)
-    redis-namespace (1.6.0)
+    redis-namespace (1.8.0)
       redis (>= 3.0.4)
     resque_spec (0.17.0)
       resque (>= 1.19.0)
@@ -227,7 +227,7 @@ GEM
     thread_safe (0.3.6)
     tilt (2.0.8)
     timecop (0.9.1)
-    timers (4.3.0)
+    timers (4.3.2)
     toml (0.2.0)
       parslet (~> 1.8.0)
     tzinfo (1.2.7)
@@ -250,7 +250,7 @@ PLATFORMS
 
 DEPENDENCIES
   apisonator!
-  async-redis (~> 0.5)
+  async-redis (~> 0.5.1)
   async-rspec
   benchmark-ips (~> 2.7.2)
   bugsnag (~> 6)
@@ -269,9 +269,10 @@ DEPENDENCIES
   pry-doc (~> 0.11.1)
   puma!
   rack (~> 2.1.4)
-  rack-test (~> 0.8.2)
+  rack-test (= 0.8.2)
   rake (~> 13.0)
   redis!
+  redis-namespace (~> 1.8.0)
   resque!
   resque_spec (~> 0.17.0)
   resque_unit (~> 0.4.4)!

data/Rakefile

@@ -261,27 +261,49 @@ task :reschedule_failed_jobs do
        "Pending failed jobs: #{result[:failed_current]}."
 end
 
-desc 'Delete stats of services marked for deletion'
 namespace :stats do
+  desc 'Delete stats of services marked for deletion'
   task :cleanup, [:redis_urls, :log_deleted_keys] do |_, args|
-    redis_urls = args[:redis_urls] && args[:redis_urls].split(' ')
+    redis_conns = redis_conns(args[:redis_urls])
 
-    if redis_urls.nil? || redis_urls.empty?
+    if redis_conns.empty?
       puts 'No Redis URLs specified'
       exit(false)
     end
 
-    redis_clients = redis_urls.map do |redis_url|
-      parsed_uri = URI.parse(ThreeScale::Backend::Storage::Helpers.send(
-        :to_redis_uri, redis_url)
-      )
-      Redis.new(host: parsed_uri.host, port: parsed_uri.port)
+    ThreeScale::Backend::Stats::Cleaner.delete!(
+      redis_conns, log_deleted_keys: logger_for_deleted_keys(args[:log_deleted_keys])
+    )
+  end
+
+  desc 'Delete stats keys set to 0'
+  task :delete_stats_keys_set_to_0, [:redis_urls, :log_deleted_keys] do |_, args|
+    redis_conns = redis_conns(args[:redis_urls])
+
+    if redis_conns.empty?
+      puts 'No Redis URLs specified'
+      exit(false)
     end
 
-    log_deleted = args[:log_deleted_keys] == 'true' ? STDOUT : nil
+    ThreeScale::Backend::Stats::Cleaner.delete_stats_keys_set_to_0(
+      redis_conns, log_deleted_keys: logger_for_deleted_keys(args[:log_deleted_keys])
+    )
+  end
+end
 
-    ThreeScale::Backend::Stats::Cleaner.delete!(
-      redis_clients, log_deleted_keys: log_deleted
+def redis_conns(urls)
+  redis_urls = urls && urls.split(' ')
+
+  return [] if redis_urls.nil? || redis_urls.empty?
+
+  redis_urls.map do |redis_url|
+    parsed_uri = URI.parse(ThreeScale::Backend::Storage::Helpers.send(
+      :to_redis_uri, redis_url)
     )
+    Redis.new(host: parsed_uri.host, port: parsed_uri.port)
   end
 end
+
+def logger_for_deleted_keys(arg_log_deleted_keys)
+  arg_log_deleted_keys == 'true' ? STDOUT : nil
+end

data/app/api/internal/service_tokens.rb

@@ -7,6 +7,14 @@ module ThreeScale
           ServiceToken.exists?(token, service_id) ? 200 : 404
         end
 
+        get '/:token/:service_id/provider_key' do |token, service_id|
+          if ServiceToken.exists?(token, service_id)
+            { status: :found, provider_key: Service.provider_key_for(service_id) }.to_json
+          else
+            respond_with_404('token/service combination not found'.freeze)
+          end
+        end
+
         post '/' do
           check_tokens_param!
 

data/app/api/internal/stats.rb

@@ -6,32 +6,13 @@ module ThreeScale
           respond_with_404('service not found') unless Service.exists?(params[:service_id])
         end
 
-        # This is very slow and needs to be disabled until the performance
-        # issues are solved. In the meanwhile, the job will just return OK.
-=begin
-        delete '' do |service_id|
-          delete_stats_job_attrs = api_params Stats::DeleteJobDef
-          delete_stats_job_attrs[:service_id] = service_id
-          delete_stats_job_attrs[:from] = delete_stats_job_attrs[:from].to_i
-          delete_stats_job_attrs[:to] = delete_stats_job_attrs[:to].to_i
-          begin
-            Stats::DeleteJobDef.new(delete_stats_job_attrs).run_async
-          rescue DeleteServiceStatsValidationError => e
-            [400, headers, { status: :error, error: e.message }.to_json]
-          else
-            { status: :to_be_deleted }.to_json
-          end
-=end
-
-        # This is an alternative to the above. It just adds the service to a
-        # Redis set to marked is as "to be deleted".
-        # Later a script can read that set and actually delete the keys.
-        # Read the docs of the Stats::Cleaner class for more details.
+        # This adds the service to a Redis set to mark is as "to be deleted".
+        # Later a script can read that set and actually delete the keys. Read
+        # the docs of the Stats::Cleaner class for more details.
         #
-        # Notice that this method ignores the "from" and "to" parameters. When
-        # system calls this method, they're always interested in deleting all
-        # the keys. They were just passing "from" and "to" to make the
-        # implementation of the option above easier.
+        # Notice that this method ignores the "from" and "to" parameters used in
+        # previous versions. When system calls this method, they're always
+        # interested in deleting all the keys.
         delete '' do |service_id|
           Stats::Cleaner.mark_service_to_be_deleted(service_id)
           { status: :to_be_deleted }.to_json

data/lib/3scale/backend/application_events.rb

@@ -40,10 +40,8 @@ module ThreeScale
       private
 
       def self.first_traffic(service_id, application_id)
-        key = Stats::Keys.applications_key_prefix(
-          Stats::Keys.service_key_prefix(service_id)
-        )
-        if storage.sadd(key, encode_key(application_id))
+        if storage.sadd(Stats::Keys.set_of_apps_with_traffic(service_id),
+                        encode_key(application_id))
           EventStorage.store(:first_traffic,
                              { service_id:     service_id,
                                application_id: application_id,

data/lib/3scale/backend/configuration.rb

@@ -1,6 +1,7 @@
 require '3scale/backend/configuration/loader'
 require '3scale/backend/environment'
 require '3scale/backend/configurable'
+require '3scale/backend/errors'
 
 module ThreeScale
   module Backend
@@ -31,8 +32,6 @@ module ThreeScale
 
     CONFIG_DELETE_STATS_BATCH_SIZE = 50
     private_constant :CONFIG_DELETE_STATS_BATCH_SIZE
-    CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE = 1000
-    private_constant :CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE
 
     @configuration = Configuration::Loader.new
 
@@ -53,7 +52,7 @@ module ThreeScale
       config.add_section(:analytics_redis, :server,
                          :connect_timeout, :read_timeout, :write_timeout)
       config.add_section(:hoptoad, :service, :api_key)
-      config.add_section(:stats, :bucket_size, :delete_batch_size, :delete_partition_batch_size)
+      config.add_section(:stats, :bucket_size, :delete_batch_size)
       config.add_section(:redshift, :host, :port, :dbname, :user, :password)
       config.add_section(:statsd, :host, :port)
       config.add_section(:internal_api, :user, :password)
@@ -77,9 +76,6 @@ module ThreeScale
       master_metrics = [:transactions, :transactions_authorize]
       config.master.metrics = Struct.new(*master_metrics).new
 
-      # Default config
-      config.master_service_id  = 1
-
       # This setting controls whether the listener can create event buckets in
       # Redis. We do not want all the listeners creating buckets yet, as we do
       # not know exactly the rate at which we can send events to Kinesis
@@ -127,9 +123,6 @@ module ThreeScale
       config.stats.delete_batch_size = parse_int(config.stats.delete_batch_size,
                                                  CONFIG_DELETE_STATS_BATCH_SIZE)
 
-      config.stats.delete_partition_batch_size = parse_int(config.stats.delete_partition_batch_size,
-                                                           CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE)
-
       # often we don't have a log_file setting - generate it here from
       # the log_path setting.
       log_file = config.log_file

data/lib/3scale/backend/errors.rb

@@ -292,12 +292,6 @@ module ThreeScale
       end
     end
 
-    class DeleteServiceStatsValidationError < Error
-      def initialize(service_id, msg)
-        super "Delete stats job context validation error. Service: #{service_id}. Error: #{msg}"
-      end
-    end
-
     class EndUsersNoLongerSupported < BadRequest
       def initialize
         super 'End-users are no longer supported, do not specify the user_id parameter'.freeze

data/lib/3scale/backend/job_fetcher.rb

@@ -32,25 +32,6 @@ module ThreeScale
             DEFAULT_WAIT_BEFORE_FETCHING_MORE_JOBS
       end
 
-      def pop_from_queue
-        begin
-          encoded_job = @redis.blpop(*@queues, timeout: @fetch_timeout)
-        rescue Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE => e
-          raise RedisConnectionError.new(e.message)
-        rescue Redis::CommandError => e
-          # Redis::CommandError from redis-rb can be raised for multiple
-          # reasons, so we need to check the error message to distinguish
-          # connection errors from the rest.
-          if e.message == 'ERR Connection timed out'.freeze
-            raise RedisConnectionError.new(e.message)
-          else
-            raise e
-          end
-        end
-
-        encoded_job
-      end
-
       def fetch
         encoded_job = pop_from_queue
         return nil if encoded_job.nil? || encoded_job.empty?
@@ -99,10 +80,11 @@ module ThreeScale
 
               # Re-instantiate Redis instance. This is needed to recover from
               # Errno::EPIPE, not sure if there are others.
-              @redis = ThreeScale::Backend::QueueStorage.connection(
-                  ThreeScale::Backend.environment,
-                  ThreeScale::Backend.configuration
+              @redis = Redis::Namespace.new(
+                WorkerAsync.const_get(:RESQUE_REDIS_NAMESPACE),
+                redis: QueueStorage.connection(Backend.environment, Backend.configuration)
               )
+
              # If there is a different kind of error, it's probably a
              # programming error. Like sending an invalid blpop command to
              # Redis. In that case, let the worker crash.
@@ -111,12 +93,36 @@ module ThreeScale
           end
         end
 
+      rescue Exception => e
+        Worker.logger.notify(e)
+      ensure
         job_queue.close
       end
 
       def shutdown
         @shutdown = true
       end
+
+      private
+
+      def pop_from_queue
+        begin
+          encoded_job = @redis.blpop(*@queues, timeout: @fetch_timeout)
+        rescue Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE => e
+          raise RedisConnectionError.new(e.message)
+        rescue Redis::CommandError => e
+          # Redis::CommandError from redis-rb can be raised for multiple
+          # reasons, so we need to check the error message to distinguish
+          # connection errors from the rest.
+          if e.message == 'ERR Connection timed out'.freeze
+            raise RedisConnectionError.new(e.message)
+          else
+            raise e
+          end
+        end
+
+        encoded_job
+      end
     end
   end
 end