apisonator 3.2.1 → 3.3.3

data/lib/3scale/backend/stats/keys.rb

@@ -70,6 +70,12 @@ module ThreeScale
           key
         end
 
+        def set_of_apps_with_traffic(service_id)
+          Stats::Keys.applications_key_prefix(
+            Stats::Keys.service_key_prefix(service_id)
+          )
+        end
+
         # We want all the buckets to go to the same Redis shard.
         # The reason is that SUNION support in Twemproxy requires that the
         # supplied keys hash to the same server.

data/lib/3scale/backend/stats/period_commons.rb

@@ -12,9 +12,6 @@ module ThreeScale
         GRANULARITY_EXPIRATION_TIME = { Period[:minute] => 180 }.freeze
         private_constant :GRANULARITY_EXPIRATION_TIME
 
-        PERMANENT_SERVICE_GRANULARITIES = (SERVICE_GRANULARITIES - GRANULARITY_EXPIRATION_TIME.keys).freeze
-        PERMANENT_EXPANDED_GRANULARITIES = (EXPANDED_GRANULARITIES - GRANULARITY_EXPIRATION_TIME.keys).freeze
-
         # We are not going to send metrics with granularity 'eternity' or
         # 'week' to Kinesis, so there is no point in storing them in Redis
         # buckets.

data/lib/3scale/backend/version.rb

@@ -1,5 +1,5 @@
 module ThreeScale
   module Backend
-    VERSION = '3.2.1'
+    VERSION = '3.3.3'
   end
 end

data/lib/3scale/backend/worker_async.rb

@@ -1,4 +1,5 @@
 require 'async'
+require 'redis-namespace'
 require '3scale/backend/job_fetcher'
 
 module ThreeScale
@@ -10,6 +11,9 @@ module ThreeScale
       DEFAULT_MAX_CONCURRENT_JOBS = 20
       private_constant :DEFAULT_MAX_CONCURRENT_JOBS
 
+      RESQUE_REDIS_NAMESPACE = :resque
+      private_constant :RESQUE_REDIS_NAMESPACE
+
       def initialize(options = {})
         trap('TERM') { shutdown }
         trap('INT')  { shutdown }
@@ -17,7 +21,7 @@ module ThreeScale
         @one_off = options[:one_off]
         @jobs = Queue.new # Thread-safe queue
 
-        @job_fetcher = options[:job_fetcher] || JobFetcher.new
+        @job_fetcher = options[:job_fetcher] || JobFetcher.new(redis_client: redis_client)
 
         @max_concurrent_jobs = configuration.async_worker.max_concurrent_jobs ||
             DEFAULT_MAX_CONCURRENT_JOBS
@@ -64,6 +68,10 @@ module ThreeScale
           # unblocks when there are new jobs or when .close() is called
           job = @jobs.pop
 
+          # If job is nil, it means that the queue is closed. No more jobs are
+          # going to be pushed, so shutdown.
+          shutdown unless job
+
           break if @shutdown
 
           @reactor.async { perform(job) }
@@ -83,6 +91,19 @@ module ThreeScale
           Async { @job_fetcher.start(@jobs) }
         end
       end
+
+      # Returns a new Redis client with namespace "resque".
+      # In the async worker, the job fetcher runs in a separate thread, and we
+      # need to avoid sharing an already instantiated client like the one in
+      # Resque::Helpers initialized in lib/3scale/backend.rb (Resque.redis).
+      # Failing to do so, will raise errors because of fibers shared across
+      # threads.
+      def redis_client
+        Redis::Namespace.new(
+          RESQUE_REDIS_NAMESPACE,
+          redis: QueueStorage.connection(Backend.environment, Backend.configuration)
+        )
+      end
     end
   end
 end

data/lib/3scale/prometheus_server.rb

@@ -4,7 +4,7 @@
 require_relative '../3scale/backend/listener_metrics'
 
 # Config is not loaded at this point, so read ENV instead.
-if ENV['CONFIG_LISTENER_PROMETHEUS_METRICS_ENABLED'].to_s == 'true'
+if ENV['CONFIG_LISTENER_PROMETHEUS_METRICS_ENABLED'].to_s.downcase.freeze == 'true'.freeze
   prometheus_port = ENV['CONFIG_LISTENER_PROMETHEUS_METRICS_PORT']
   ThreeScale::Backend::ListenerMetrics.start_metrics_server(prometheus_port)
 end

data/licenses.xml

@@ -23,7 +23,7 @@
       </dependency>
           <dependency>
         <packageName>apisonator</packageName>
-        <version>3.2.1</version>
+        <version>3.3.3</version>
         <licenses>
                       <license>
               <name>Apache 2.0</name>
@@ -93,7 +93,7 @@
       </dependency>
           <dependency>
         <packageName>async-redis</packageName>
-        <version>0.5.0</version>
+        <version>0.5.1</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -233,7 +233,7 @@
       </dependency>
           <dependency>
         <packageName>coderay</packageName>
-        <version>1.1.2</version>
+        <version>1.1.3</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -421,7 +421,7 @@
       </dependency>
           <dependency>
         <packageName>method_source</packageName>
-        <version>0.9.0</version>
+        <version>1.0.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -525,7 +525,7 @@
       </dependency>
           <dependency>
         <packageName>nio4r</packageName>
-        <version>2.5.2</version>
+        <version>2.5.4</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -679,7 +679,7 @@
       </dependency>
           <dependency>
         <packageName>pry</packageName>
-        <version>0.11.3</version>
+        <version>0.14.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -699,7 +699,7 @@
       </dependency>
           <dependency>
         <packageName>pry-doc</packageName>
-        <version>0.11.1</version>
+        <version>1.1.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -709,7 +709,7 @@
       </dependency>
           <dependency>
         <packageName>puma</packageName>
-        <version>2.15.3</version>
+        <version>4.3.7</version>
         <licenses>
                       <license>
               <name>New BSD</name>
@@ -769,7 +769,7 @@
       </dependency>
           <dependency>
         <packageName>redis-namespace</packageName>
-        <version>1.6.0</version>
+        <version>1.8.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -1043,7 +1043,7 @@
       </dependency>
           <dependency>
         <packageName>timers</packageName>
-        <version>4.3.0</version>
+        <version>4.3.2</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -1143,7 +1143,7 @@
       </dependency>
           <dependency>
         <packageName>yard</packageName>
-        <version>0.9.20</version>
+        <version>0.9.26</version>
         <licenses>
                       <license>
               <name>MIT</name>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apisonator
 version: !ruby/object:Gem::Version
-  version: 3.2.1
+  version: 3.3.3
 platform: ruby
 authors:
 - Adam Ciganek
@@ -16,7 +16,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-22 00:00:00.000000000 Z
+date: 2021-03-09 00:00:00.000000000 Z
 dependencies: []
 description: This gem provides a daemon that handles authorization and reporting of
   web services managed by 3scale.
@@ -136,11 +136,7 @@ files:
 - lib/3scale/backend/stats/bucket_storage.rb
 - lib/3scale/backend/stats/cleaner.rb
 - lib/3scale/backend/stats/codes_commons.rb
-- lib/3scale/backend/stats/delete_job_def.rb
-- lib/3scale/backend/stats/key_generator.rb
 - lib/3scale/backend/stats/keys.rb
-- lib/3scale/backend/stats/partition_eraser_job.rb
-- lib/3scale/backend/stats/partition_generator_job.rb
 - lib/3scale/backend/stats/period_commons.rb
 - lib/3scale/backend/stats/stats_parser.rb
 - lib/3scale/backend/stats/storage.rb
@@ -211,8 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.7
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.8
+rubygems_version: 3.2.10
 signing_key: 
 specification_version: 4
 summary: 3scale web service management system backend

data/lib/3scale/backend/stats/delete_job_def.rb

@@ -1,60 +0,0 @@
-module ThreeScale
-  module Backend
-    module Stats
-      class DeleteJobDef
-        ATTRIBUTES = %i[service_id applications metrics from to context_info].freeze
-        private_constant :ATTRIBUTES
-        attr_reader(*ATTRIBUTES)
-
-        def self.attribute_names
-          ATTRIBUTES
-        end
-
-        def initialize(params = {})
-          ATTRIBUTES.each do |key|
-            instance_variable_set("@#{key}".to_sym, params[key]) unless params[key].nil?
-          end
-          validate
-        end
-
-        def run_async
-          Resque.enqueue(PartitionGeneratorJob, Time.now.getutc.to_f, service_id, applications,
-                         metrics, from, to, context_info)
-        end
-
-        def to_json
-          to_hash.to_json
-        end
-
-        def to_hash
-          Hash[ATTRIBUTES.collect { |key| [key, send(key)] }]
-        end
-
-        private
-
-        def validate
-          # from and to valid epoch times
-          raise_validation_error('from field not integer') unless from.is_a? Integer
-          raise_validation_error('from field is zero') if from.zero?
-          raise_validation_error('to field not integer') unless to.is_a? Integer
-          raise_validation_error('to field is zero') if to.zero?
-          raise_validation_error('from < to fields') if Time.at(to) < Time.at(from)
-          # application is array
-          raise_validation_error('applications field') unless applications.is_a? Array
-          raise_validation_error('applications values') unless applications.all? do |x|
-            x.is_a?(String) || x.is_a?(Integer)
-          end
-          # metrics is array
-          raise_validation_error('metrics field') unless metrics.is_a? Array
-          raise_validation_error('metrics values') unless metrics.all? do |x|
-            x.is_a?(String) || x.is_a?(Integer)
-          end
-        end
-
-        def raise_validation_error(msg)
-          raise DeleteServiceStatsValidationError.new(service_id, msg)
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/stats/key_generator.rb

@@ -1,73 +0,0 @@
-module ThreeScale
-  module Backend
-    module Stats
-      class KeyGenerator
-        attr_reader :service_id, :applications, :metrics, :from, :to
-
-        def initialize(service_id:, applications: [], metrics: [], from:, to:, **)
-          @service_id = service_id
-          @applications = applications
-          @metrics = metrics
-          @from = from
-          @to = to
-        end
-
-        def keys
-          response_code_service_keys +
-            response_code_application_keys +
-            usage_service_keys +
-            usage_application_keys
-        end
-
-        private
-
-        def periods(granularities)
-          granularities.flat_map do |granularity|
-            (Period[granularity].new(Time.at(from))..Period[granularity].new(Time.at(to))).to_a
-          end
-        end
-
-        def response_codes
-          CodesCommons::TRACKED_CODES + CodesCommons::TRACKED_CODE_GROUPS
-        end
-
-        def response_code_service_keys
-          periods(PeriodCommons::PERMANENT_SERVICE_GRANULARITIES).flat_map do |period|
-            response_codes.flat_map do |response_code|
-              Keys.service_response_code_value_key(service_id, response_code, period)
-            end
-          end
-        end
-
-        def response_code_application_keys
-          periods(PeriodCommons::PERMANENT_EXPANDED_GRANULARITIES).flat_map do |period|
-            response_codes.flat_map do |response_code|
-              applications.flat_map do |application|
-                Keys.application_response_code_value_key(service_id, application,
-                                                         response_code, period)
-              end
-            end
-          end
-        end
-
-        def usage_service_keys
-          periods(PeriodCommons::PERMANENT_SERVICE_GRANULARITIES).flat_map do |period|
-            metrics.flat_map do |metric|
-              Keys.service_usage_value_key(service_id, metric, period)
-            end
-          end
-        end
-
-        def usage_application_keys
-          periods(PeriodCommons::PERMANENT_EXPANDED_GRANULARITIES).flat_map do |period|
-            metrics.flat_map do |metric|
-              applications.flat_map do |application|
-                Keys.application_usage_value_key(service_id, application, metric, period)
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/stats/partition_eraser_job.rb

@@ -1,58 +0,0 @@
-module ThreeScale
-  module Backend
-    module Stats
-      # Job for deleting service stats
-      # Perform actual key deletion from a key partition definition
-      class PartitionEraserJob < BackgroundJob
-        # low priority queue
-        @queue = :stats
-
-        class << self
-          include StorageHelpers
-          include Configurable
-
-          def perform_logged(_enqueue_time, service_id, applications, metrics,
-                             from, to, offset, length, context_info = {})
-            job = DeleteJobDef.new(
-              service_id: service_id,
-              applications: applications,
-              metrics: metrics,
-              from: from,
-              to: to
-            )
-
-            validate_job(job, offset, length)
-
-            stats_key_gen = KeyGenerator.new(job.to_hash)
-
-            stats_key_gen.keys.drop(offset).take(length).each_slice(configuration.stats.delete_batch_size) do |slice|
-              storage.del(slice)
-            end
-
-            [true, { job: job.to_hash, offset: offset, lenght: length }.to_json]
-          rescue Backend::Error => error
-            [false, "#{service_id} #{error}"]
-          end
-
-          private
-
-          def validate_job(job, offset, length)
-            unless offset.is_a? Integer
-              raise DeleteServiceStatsValidationError.new(job.service_id, 'offset field value ' \
-                                                          "[#{offset}] validation error")
-            end
-
-            unless length.is_a? Integer
-              raise DeleteServiceStatsValidationError.new(job.service_id, 'length field value ' \
-                                                          "[#{length}] validation error")
-            end
-          end
-
-          def enqueue_time(args)
-            args[0]
-          end
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/stats/partition_generator_job.rb

@@ -1,46 +0,0 @@
-module ThreeScale
-  module Backend
-    module Stats
-      # Job for deleting service stats
-      # Maps delete job definition to a set of non overlapping key set partitions
-      class PartitionGeneratorJob < BackgroundJob
-        # low priority queue
-        @queue = :stats
-
-        class << self
-          include Configurable
-
-          def perform_logged(_enqueue_time, service_id, applications, metrics,
-                             from, to, context_info = {})
-            job = DeleteJobDef.new(
-              service_id: service_id,
-              applications: applications,
-              metrics: metrics,
-              from: from,
-              to: to
-            )
-
-            stats_key_gen = KeyGenerator.new(job.to_hash)
-
-            # Generate partitions
-            0.step(stats_key_gen.keys.count, configuration.stats.delete_partition_batch_size).each do |idx|
-              Resque.enqueue(PartitionEraserJob, Time.now.getutc.to_f, service_id, applications,
-                             metrics, from, to, idx,
-                             configuration.stats.delete_partition_batch_size, context_info)
-            end
-
-            [true, job.to_json]
-          rescue Backend::Error => error
-            [false, "#{service_id} #{error}"]
-          end
-
-          private
-
-          def enqueue_time(args)
-            args[0]
-          end
-        end
-      end
-    end
-  end
-end