apisonator 2.100.0

data/lib/3scale/backend/stats/keys.rb

@@ -0,0 +1,104 @@
+module ThreeScale
+  module Backend
+    module Stats
+      module Keys
+        module_function
+
+        extend Backend::StorageKeyHelpers
+
+        # @note The { ... } is the key tag. See redis docs for more info
+        # about key tags.
+        def service_key_prefix(service_id)
+          "stats/{service:#{service_id}}"
+        end
+
+        # @note For backwards compatibility, the key is called cinstance.
+        # It will be eventually renamed to application.
+        def application_key_prefix(prefix, application_id)
+          "#{prefix}/cinstance:#{application_id}"
+        end
+
+        def applications_key_prefix(prefix)
+          "#{prefix}/cinstances"
+        end
+
+        def metric_key_prefix(prefix, metric_id)
+          "#{prefix}/metric:#{metric_id}"
+        end
+
+        def response_code_key_prefix(prefix, response_code)
+          "#{prefix}/response_code:#{response_code}"
+        end
+
+        def service_usage_value_key(service_id, metric_id, period)
+          service_key = service_key_prefix(service_id)
+          metric_key  = metric_key_prefix(service_key, metric_id)
+
+          encode_key(counter_key(metric_key, period))
+        end
+
+        def application_usage_value_key(service_id, app_id, metric_id, period)
+          service_key = service_key_prefix(service_id)
+          app_key     = application_key_prefix(service_key, app_id)
+          metric_key  = metric_key_prefix(app_key, metric_id)
+
+          encode_key(counter_key(metric_key, period))
+        end
+
+        def service_response_code_value_key(service_id, response_code, period)
+          service_key        = service_key_prefix(service_id)
+          response_code_key  = response_code_key_prefix(service_key, response_code)
+
+          encode_key(counter_key(response_code_key, period))
+        end
+
+        def application_response_code_value_key(service_id, app_id, response_code, period)
+          service_key        = service_key_prefix(service_id)
+          app_key            = application_key_prefix(service_key, app_id)
+          response_code_key  = response_code_key_prefix(app_key, response_code)
+
+          encode_key(counter_key(response_code_key, period))
+        end
+
+        def counter_key(prefix, period)
+          granularity = period.granularity
+          key = "#{prefix}/#{granularity}"
+          if granularity.to_sym != :eternity
+            key += ":#{period.start.to_compact_s}"
+          end
+
+          key
+        end
+
+        # We want all the buckets to go to the same Redis shard.
+        # The reason is that SUNION support in Twemproxy requires that the
+        # supplied keys hash to the same server.
+        # We are already using a hash tag in the Twemproxy config file: "{}".
+        # For that reason, if we specify a key that contains something like
+        # "{stats_bucket}", we can be sure that all of them will be in the same
+        # shard.
+        def changed_keys_bucket_key(bucket)
+          "{stats_bucket}:#{bucket}"
+        end
+
+        def changed_keys_key
+          "keys_changed_set"
+        end
+
+        def transaction_keys(transaction, item, value)
+          service_key     = service_key_prefix(transaction.service_id)
+          application_key = application_key_prefix(service_key,
+                                                   transaction.application_id)
+
+          method = "#{item}_key_prefix".to_sym
+
+          {
+            service:     public_send(method, service_key, value),
+            application: public_send(method, application_key, value),
+          }
+        end
+
+      end
+    end
+  end
+end

data/lib/3scale/backend/stats/partition_eraser_job.rb

@@ -0,0 +1,58 @@
+module ThreeScale
+  module Backend
+    module Stats
+      # Job for deleting service stats
+      # Perform actual key deletion from a key partition definition
+      class PartitionEraserJob < BackgroundJob
+        # low priority queue
+        @queue = :stats
+
+        class << self
+          include StorageHelpers
+          include Configurable
+
+          def perform_logged(_enqueue_time, service_id, applications, metrics,
+                             from, to, offset, length, context_info = {})
+            job = DeleteJobDef.new(
+              service_id: service_id,
+              applications: applications,
+              metrics: metrics,
+              from: from,
+              to: to
+            )
+
+            validate_job(job, offset, length)
+
+            stats_key_gen = KeyGenerator.new(job.to_hash)
+
+            stats_key_gen.keys.drop(offset).take(length).each_slice(configuration.stats.delete_batch_size) do |slice|
+              storage.del(slice)
+            end
+
+            [true, { job: job.to_hash, offset: offset, lenght: length }.to_json]
+          rescue Backend::Error => error
+            [false, "#{service_id} #{error}"]
+          end
+
+          private
+
+          def validate_job(job, offset, length)
+            unless offset.is_a? Integer
+              raise DeleteServiceStatsValidationError.new(job.service_id, 'offset field value ' \
+                                                          "[#{offset}] validation error")
+            end
+
+            unless length.is_a? Integer
+              raise DeleteServiceStatsValidationError.new(job.service_id, 'length field value ' \
+                                                          "[#{length}] validation error")
+            end
+          end
+
+          def enqueue_time(args)
+            args[0]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/stats/partition_generator_job.rb

@@ -0,0 +1,46 @@
+module ThreeScale
+  module Backend
+    module Stats
+      # Job for deleting service stats
+      # Maps delete job definition to a set of non overlapping key set partitions
+      class PartitionGeneratorJob < BackgroundJob
+        # low priority queue
+        @queue = :stats
+
+        class << self
+          include Configurable
+
+          def perform_logged(_enqueue_time, service_id, applications, metrics,
+                             from, to, context_info = {})
+            job = DeleteJobDef.new(
+              service_id: service_id,
+              applications: applications,
+              metrics: metrics,
+              from: from,
+              to: to
+            )
+
+            stats_key_gen = KeyGenerator.new(job.to_hash)
+
+            # Generate partitions
+            0.step(stats_key_gen.keys.count, configuration.stats.delete_partition_batch_size).each do |idx|
+              Resque.enqueue(PartitionEraserJob, Time.now.getutc.to_f, service_id, applications,
+                             metrics, from, to, idx,
+                             configuration.stats.delete_partition_batch_size, context_info)
+            end
+
+            [true, job.to_json]
+          rescue Backend::Error => error
+            [false, "#{service_id} #{error}"]
+          end
+
+          private
+
+          def enqueue_time(args)
+            args[0]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/stats/period_commons.rb

@@ -0,0 +1,34 @@
+module ThreeScale
+  module Backend
+    module Stats
+      module PeriodCommons
+        SERVICE_GRANULARITIES = %i[eternity month week day hour].map do |g|
+          Period[g]
+        end.freeze
+
+        # For applications and users
+        EXPANDED_GRANULARITIES = (SERVICE_GRANULARITIES + [Period[:year], Period[:minute]]).freeze
+
+        GRANULARITY_EXPIRATION_TIME = { Period[:minute] => 180 }.freeze
+        private_constant :GRANULARITY_EXPIRATION_TIME
+
+        PERMANENT_SERVICE_GRANULARITIES = (SERVICE_GRANULARITIES - GRANULARITY_EXPIRATION_TIME.keys).freeze
+        PERMANENT_EXPANDED_GRANULARITIES = (EXPANDED_GRANULARITIES - GRANULARITY_EXPIRATION_TIME.keys).freeze
+
+        # We are not going to send metrics with granularity 'eternity' or
+        # 'week' to Kinesis, so there is no point in storing them in Redis
+        # buckets.
+        EXCLUDED_FOR_BUCKETS = [Period[:eternity], Period[:week]].freeze
+
+        # Return an array of granularities given a metric_type
+        def self.granularities(metric_type)
+          metric_type == :service ? SERVICE_GRANULARITIES : EXPANDED_GRANULARITIES
+        end
+
+        def self.expire_time_for_granularity(granularity)
+          GRANULARITY_EXPIRATION_TIME[granularity]
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/stats/stats_parser.rb

@@ -0,0 +1,141 @@
+module ThreeScale
+  module Backend
+    module Stats
+      class StatsParser
+        # This parser converts a stats key of Redis into a Hash
+        # This is an example of stats key that this parser converts:
+        # "stats/{service:1006}/cinstance:a60/metric:255/week:20151130"
+
+        # This class contains code stolen from Alex's script:
+        # /script/redis/stats_keys_2_csv
+        # We can think about unifying code later.
+
+        DATE_COLS = [
+            'year'.freeze,
+            'month'.freeze,
+            'day'.freeze,
+            'hour'.freeze,
+            'minute'.freeze,
+        ].freeze
+        private_constant :DATE_COLS
+
+        PERIODS = [
+            *DATE_COLS,
+            'week'.freeze,
+            'eternity'.freeze,
+        ].freeze
+        private_constant :PERIODS
+
+        NON_DATE_PERIODS = (PERIODS - DATE_COLS).freeze
+        private_constant :NON_DATE_PERIODS
+
+        ALL_COLUMNS = [
+            *DATE_COLS,
+            'period'.freeze,
+            'service'.freeze,
+            'cinstance'.freeze,
+            'uinstance'.freeze,
+            'metric'.freeze,
+            'response_code'.freeze,
+            'value'.freeze,
+        ].freeze
+        private_constant :ALL_COLUMNS
+
+        REQUIRED_COLS = [
+            *DATE_COLS,
+            'period'.freeze,
+            'service'.freeze,
+            'value'.freeze,
+        ].freeze
+        private_constant :REQUIRED_COLS
+
+        StatsKeyValueInvalid = Class.new(ThreeScale::Backend::Error)
+
+        class << self
+
+          def parse(stats_key, value)
+            key_value_to_hash(stats_key, value)
+          end
+
+          private
+
+          def key_value_to_hash(key,value)
+            key_value = ("\"#{key}\":\"#{value}\"")
+
+            # some keys have things like "field1:xxx/uinstance:N/A/field3:yyy" WTF.
+            key_value.gsub!(/:N\/A/, ':'.freeze)
+
+            h = Hash[str2ary(prepare_str_from(key_value))]
+
+            result = fix_dates_and_periods(h)
+
+            all_required_columns = REQUIRED_COLS.all? { |col| h.has_key?(col) }
+            no_extra_columns = (h.keys - ALL_COLUMNS).empty?
+
+            unless all_required_columns && no_extra_columns
+              raise StatsKeyValueInvalid, "Error parsing #{key_value}"
+            end
+
+            date_cols_to_timestamp(h)
+
+            Hash[result.map{ |k, v| [k.to_sym, v] }]
+          end
+
+          def prepare_str_from(line)
+            _, key, _, val, *_ = line.split('"')
+            "#{key.gsub(/[\{\}]/, '')}/value:#{val}"
+          end
+
+          def str2ary(str)
+            str.split('/')[1..-1].map do |kv|
+              kv.split(':')
+            end
+          end
+
+          def fix_dates_and_periods(hash)
+            period = hash.keys.find { |k| PERIODS.include? k }
+            if period
+              hash['period'.freeze] = period
+              period_val = (period == 'eternity' ? '' : hash[period].dup)
+              fix_date_cols(hash, period_val)
+              NON_DATE_PERIODS.each { |ndp| hash.delete ndp }
+            end
+            hash
+          end
+
+          def fix_date_cols(hash, period_val)
+            DATE_COLS.each do |date_col|
+              hash[date_col] = if date_col == 'year'
+                                 period_val.slice! 0, 4
+                               else
+                                 period_val.slice! 0, 2
+                               end
+
+              if hash[date_col].empty?
+                hash[date_col] = nil
+              elsif hash[date_col].length == 1 # because of 'compacted' times
+                hash[date_col] = hash[date_col] + '0'
+              end
+            end
+          end
+
+          # Adds the 'timestamp' key to the hash. The value follows the format:
+          # YYYYMMDD HH:mm. This function also deletes all the date columns
+          # from the given hash
+          def date_cols_to_timestamp(hash)
+            hash['timestamp'.freeze] = timestamp(hash)
+            DATE_COLS.each { |date_col| hash.delete(date_col) }
+            hash
+          end
+
+          def timestamp(hash)
+            return '' if hash['period'] == 'eternity'
+            timestamp = hash['year'] + hash['month'] + hash['day'] + ' '
+            timestamp << (hash['hour'] ? hash['hour'] : '00')
+            timestamp << (hash['minute'] ? (':' + hash['minute']) : ':00')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/stats/storage.rb

@@ -0,0 +1,113 @@
+require_relative '../storage'
+require_relative 'keys'
+require '3scale/backend/analytics/kinesis/adapter'
+require '3scale/backend/stats/bucket_reader'
+require '3scale/backend/stats/bucket_storage'
+require '3scale/backend/stats/stats_parser'
+
+module ThreeScale
+  module Backend
+    module Stats
+      class Storage
+
+        STATS_ENABLED_KEY = 'stats:enabled'.freeze
+        private_constant :STATS_ENABLED_KEY
+
+        DISABLED_BECAUSE_EMERGENCY_KEY = 'stats:disabled_emergency'.freeze
+        private_constant :DISABLED_BECAUSE_EMERGENCY_KEY
+
+        class << self
+          include Memoizer::Decorator
+
+          def enabled?
+            storage.get(STATS_ENABLED_KEY).to_i == 1
+          end
+          memoize :enabled?
+
+          def enable!
+            storage.set(STATS_ENABLED_KEY, '1')
+          end
+
+          # Bucket storage can be disabled because an 'emergency' happened.
+          # If too many buckets accumulate, we disable the feature because
+          # the memory occupied by Redis can grow very quickly.
+          # Check the code in the Aggregator class to check the conditions
+          # that trigger this 'emergency'.
+          def disable!(emergency = false)
+            storage.del(STATS_ENABLED_KEY)
+
+            if emergency
+              storage.set(DISABLED_BECAUSE_EMERGENCY_KEY, '1')
+            else
+              storage.del(DISABLED_BECAUSE_EMERGENCY_KEY)
+            end
+          end
+
+          # Returns whether the last time that bucket storage was disabled was
+          # because of an emergency. Notice that this method can return 'true'
+          # even when enabled? is true.
+          def last_disable_was_emergency?
+            storage.get(DISABLED_BECAUSE_EMERGENCY_KEY).to_i == 1
+          end
+          memoize :last_disable_was_emergency?
+
+          def bucket_storage
+            @bucket_storage ||= BucketStorage.new(stats_storage)
+          end
+
+          def bucket_reader
+            @bucket_reader ||= BucketReader.new(config.stats.bucket_size,
+                                                bucket_storage,
+                                                storage)
+          end
+
+          def kinesis_adapter
+            @kinesis_adapter ||= Analytics::Kinesis::Adapter.new(
+              config.kinesis_stream_name,
+              kinesis_client,
+              stats_storage
+            )
+          end
+
+          private
+
+          def storage
+            Backend::Storage.instance
+          end
+
+          # This is a separate storage used only for the analytics system. More
+          # specifically, the only things saved in this storage are:
+          #   - The buckets handled by the BucketStorage class.
+          #   - The batches of events created by the Kinesis::Adapter class.
+          def stats_storage
+            @stats_storage ||= if config.analytics_redis && config.analytics_redis[:server]
+                                 stats_storage_from_config
+                               else
+                                 # A stats storage has not been specified, so
+                                 # use the same one for everything.
+                                 storage
+                               end
+          end
+
+          def stats_storage_from_config
+            options = Backend::Storage::Helpers.config_with(config.analytics_redis)
+
+            Backend::Storage.new(options)
+          end
+
+          def config
+            Backend.configuration
+          end
+
+          def kinesis_client
+            @kinesis_client ||= Aws::Firehose::Client.new(
+                region: config.kinesis_region,
+                access_key_id: config.aws_access_key_id,
+                secret_access_key: config.aws_secret_access_key)
+          end
+        end
+
+      end
+    end
+  end
+end