apisonator 2.100.0

data/lib/3scale/backend/validators/referrer.rb

@@ -0,0 +1,60 @@
+module ThreeScale
+  module Backend
+    module Validators
+      class Referrer < Base
+
+        # There are some users that depend on a previous version of the
+        # pattern_matches? method. We make this configurable to avoid breaking
+        # changes for those users.
+
+        class << self
+          def define_pattern_match(legacy)
+            class_eval do
+              if legacy
+                def pattern_matches?(pattern, value)
+                  /#{pattern}/ =~ value
+                end
+              else
+                def pattern_matches?(pattern, value)
+                  /\A#{pattern}\z/ =~ value
+                end
+              end
+            end
+          end
+          private :define_pattern_match
+
+          def behave_as_legacy(legacy)
+            # don't blow up if we still didn't define it
+            remove_method :pattern_matches? rescue NameError
+            define_pattern_match legacy
+          end
+          private :behave_as_legacy unless ThreeScale::Backend.test? # leave it public for tests
+        end
+
+        behave_as_legacy ThreeScale::Backend.configuration.legacy_referrer_filters
+
+        BYPASS = '*'.freeze
+        private_constant :BYPASS
+
+        def apply
+          if service.referrer_filters_required? && application.has_referrer_filters?
+            if application.referrer_filters.any? { |filter| matches?(filter, params[:referrer]) }
+              succeed!
+            else
+              fail!(ReferrerNotAllowed.new(params[:referrer]))
+            end
+          else
+            succeed!
+          end
+        end
+
+        private
+
+        def matches?(pattern, value)
+          return true if value == BYPASS
+          pattern_matches?(Regexp.escape(pattern).gsub('\*', '.*'), value)
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/validators/service_state.rb

@@ -0,0 +1,15 @@
+module ThreeScale
+  module Backend
+    module Validators
+      class ServiceState < Base
+        def apply
+          if service.active?
+            succeed!
+          else
+            fail!(ServiceNotActive.new)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/validators/state.rb

@@ -0,0 +1,15 @@
+module ThreeScale
+  module Backend
+    module Validators
+      class State < Base
+        def apply
+          if application.active?
+            succeed!
+          else
+            fail!(ApplicationNotActive.new)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/version.rb

@@ -0,0 +1,5 @@
+module ThreeScale
+  module Backend
+    VERSION = '2.100.0'
+  end
+end

data/lib/3scale/backend/views/oauth_access_tokens.builder

@@ -0,0 +1,14 @@
+xml.instruct!
+xml.oauth_access_tokens do
+  @tokens.each do |t|
+    # Some bright head leaked the -1 that Redis uses to indicate there is no TTL
+    # associated with a key. The behaviour is inconsistent because somehow we
+    # BOTH expect that a token with no TTL does not have a "ttl" attribute in
+    # the generated XML and at the same time we expect (in some tests) that such
+    # tokens have this field with a -1 value.
+    #
+    # Just enforce the ttl to be -1 when there is none.
+    attrs = { :ttl => t.ttl || -1 }
+    xml.oauth_access_token t.token, attrs
+  end
+end

data/lib/3scale/backend/views/oauth_app_id_by_token.builder

@@ -0,0 +1,4 @@
+xml.instruct!
+xml.application do
+  xml.app_id @token_to_app_id
+end

data/lib/3scale/backend/worker.rb

@@ -0,0 +1,87 @@
+require '3scale/backend/configuration'
+require '3scale/backend/logging/worker'
+require '3scale/backend/logging/external'
+
+module ThreeScale
+  module Backend
+    # Worker for working off background jobs. This is very stripped down version of the
+    # default resque worker with parts of code shamelessly stolen from it. The reason for
+    # this stripping is that the resque one does fork before processing each job, and that
+    # is too slow.
+
+    # This is a module that's meant to be included from the different workers.
+    # Now we have WorkerSync and WorkerAsync. Those classes need to implement
+    # #work, which is responsible for fetching jobs from the queue and running
+    # them by calling perform(job).
+    module Worker
+      include Resque::Helpers
+      include Configurable
+
+      def self.new(options = {})
+        Logging::Worker.configure_logging(self, options[:log_file])
+        Logging::External.setup_worker
+
+        if configuration.worker_prometheus_metrics.enabled
+          require '3scale/backend/worker_metrics'
+          WorkerMetrics.start_metrics_server
+        end
+
+        if options[:async]
+          # Conditional require is done to require async-* libs only when
+          # needed and avoid possible side-effects.
+          require '3scale/backend/worker_async'
+          WorkerAsync.new(options)
+        else
+          require '3scale/backend/worker_sync'
+          WorkerSync.new(options)
+        end
+      end
+
+      # == Options
+      #
+      # - :one_off           - if true, will process one job, then quit
+      #
+      def self.work(options = {})
+        Process.setproctitle("3scale_backend_worker #{Backend::VERSION}")
+        options[:async] = configuration.redis.async
+        new(options).work
+      end
+
+      def work
+        raise 'Missing implementation of #work'
+      end
+
+      def shutdown
+        @shutdown = true
+      end
+
+      def to_s
+        @to_s ||= "#{hostname}:#{Process.pid}"
+      end
+
+      def one_off?
+        @one_off
+      end
+
+      private
+
+      def perform(job)
+        job.perform
+      rescue Exception => e
+        job.fail(e)
+      end
+
+      def register_worker
+        redis.sadd(:workers, self)
+      end
+
+      def unregister_worker
+        redis.srem(:workers, self)
+      end
+
+      def hostname
+        @hostname ||= (ENV['HOSTNAME'] || `hostname`.chomp)
+      end
+    end
+  end
+end

data/lib/3scale/backend/worker_async.rb

@@ -0,0 +1,88 @@
+require 'async'
+require '3scale/backend/job_fetcher'
+
+module ThreeScale
+  module Backend
+    class WorkerAsync
+      include Backend::Worker
+      include Configurable
+
+      DEFAULT_MAX_CONCURRENT_JOBS = 20
+      private_constant :DEFAULT_MAX_CONCURRENT_JOBS
+
+      def initialize(options = {})
+        trap('TERM') { shutdown }
+        trap('INT')  { shutdown }
+
+        @one_off = options[:one_off]
+        @jobs = Queue.new # Thread-safe queue
+
+        @job_fetcher = options[:job_fetcher] || JobFetcher.new
+
+        @max_concurrent_jobs = configuration.async_worker.max_concurrent_jobs ||
+            DEFAULT_MAX_CONCURRENT_JOBS
+
+        @reactor = Async::Reactor.new
+      end
+
+      def work
+        if one_off?
+          Async { process_one }
+          return
+        end
+
+        Async { register_worker }
+
+        fetch_jobs_thread = start_thread_to_fetch_jobs
+
+        loop do
+          break if @shutdown
+          schedule_jobs
+          @reactor.run
+        end
+
+        fetch_jobs_thread.join
+
+        # Ensure that we do not leave any jobs in memory
+        @reactor.async { perform(@jobs.pop) } until @jobs.empty?
+        @reactor.run
+
+        Async { unregister_worker }
+      end
+
+      def shutdown
+        @job_fetcher.shutdown
+        @shutdown = true
+      end
+
+      private
+
+      def schedule_jobs
+        scheduled = 0
+
+        loop do
+          # unblocks when there are new jobs or when .close() is called
+          job = @jobs.pop
+
+          break if @shutdown
+
+          @reactor.async { perform(job) }
+          scheduled += 1
+
+          break if @jobs.empty? or scheduled >= @max_concurrent_jobs
+        end
+      end
+
+      def process_one
+        job = @job_fetcher.fetch
+        perform(job) if job
+      end
+
+      def start_thread_to_fetch_jobs
+        Thread.new do
+          Async { @job_fetcher.start(@jobs) }
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/worker_metrics.rb

@@ -0,0 +1,44 @@
+require 'yabeda/prometheus'
+
+Yabeda.configure do
+  group :apisonator_worker do
+    counter :job_count do
+      comment "Total number of jobs processed"
+      tags %i[type]
+    end
+
+    histogram :job_runtime do
+      comment "How long jobs take to run"
+      unit :seconds
+      tags %i[type]
+      buckets [0.005, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]
+    end
+  end
+end
+Yabeda.configure!
+
+module ThreeScale
+  module Backend
+    class WorkerMetrics
+      include Configurable
+
+      def self.start_metrics_server
+        # Yabeda does not accept the port as a param
+        port = configuration.worker_prometheus_metrics.port
+        ENV['PROMETHEUS_EXPORTER_PORT'] = port.to_s if port
+
+        Yabeda::Prometheus::Exporter.start_metrics_server!
+      end
+
+      def self.increase_job_count(job_class_name)
+        Yabeda.apisonator_worker.job_count.increment(
+            { type: job_class_name }, by: 1
+        )
+      end
+
+      def self.report_runtime(job_class_name, runtime)
+        Yabeda.apisonator_worker.job_runtime.measure({ type: job_class_name }, runtime)
+      end
+    end
+  end
+end

data/lib/3scale/backend/worker_sync.rb

@@ -0,0 +1,32 @@
+require '3scale/backend/job_fetcher'
+
+module ThreeScale
+  module Backend
+    class WorkerSync
+      include Backend::Worker
+
+      def initialize(options = {})
+        trap('TERM') { shutdown }
+        trap('INT')  { shutdown }
+
+        @one_off = options[:one_off]
+        @job_fetcher = options[:job_fetcher] || JobFetcher.new
+      end
+
+      def work
+        register_worker
+
+        loop do
+          break if @shutdown
+
+          job = @job_fetcher.fetch
+          perform(job) if job
+
+          break if one_off?
+        end
+
+        unregister_worker
+      end
+    end
+  end
+end

data/lib/3scale/bundler_shim.rb

@@ -0,0 +1,17 @@
+# Setup bundler if present before anything else.
+#
+# We are not guaranteed to be running in a Bundler context, so make sure we get
+# the correct environment set up to require the correct code.
+begin
+  require 'bundler/setup'
+  if !Bundler::SharedHelpers.in_bundle?
+    # Gemfile not found, try with relative Gemfile from us
+    require 'pathname'
+    ENV['BUNDLE_GEMFILE'] = File.expand_path(File.join('..', '..', '..', 'Gemfile'),
+                                             Pathname.new(__FILE__).realpath)
+    require 'bundler'
+    Bundler.setup
+  end
+rescue LoadError, Bundler::BundlerError => e
+  STDERR.puts "CRITICAL: Bundler could not be loaded properly - #{e.message}"
+end

data/lib/3scale/prometheus_server.rb

@@ -0,0 +1,10 @@
+# When required, starts the prometheus metrics server for the listener if the
+# CONFIG_LISTENER_PROMETHEUS_METRICS_ENABLED env is set to true.
+
+require_relative '../3scale/backend/listener_metrics'
+
+# Config is not loaded at this point, so read ENV instead.
+if ENV['CONFIG_LISTENER_PROMETHEUS_METRICS_ENABLED'].to_s == 'true'
+  prometheus_port = ENV['CONFIG_LISTENER_PROMETHEUS_METRICS_PORT']
+  ThreeScale::Backend::ListenerMetrics.start_metrics_server(prometheus_port)
+end

data/lib/3scale/tasks/connectivity.rake

@@ -0,0 +1,41 @@
+namespace :connectivity do
+  desc 'Check connectivity of Redis Storage'
+  task :redis_storage_check do
+      redis_instance = ThreeScale::Backend::Storage.instance
+
+      if Environment.using_async_redis?
+        async_ping(redis_instance, 'storage')
+      else
+        ping(redis_instance, 'storage')
+      end
+  end
+
+  desc 'Check connectivity of Redis Queue Storage'
+  task :redis_storage_queue_check do
+      redis_instance = ThreeScale::Backend::QueueStorage.connection(
+        ThreeScale::Backend.environment,
+        ThreeScale::Backend.configuration,
+      )
+
+      if Environment.using_async_redis?
+        async_ping(redis_instance, 'queue storage')
+      else
+        ping(redis_instance, 'queue storage')
+      end
+  end
+
+  private
+
+  def async_ping(redis_instance, storage_type)
+    Async { ping(redis_instance, storage_type) }
+  end
+
+  def ping(redis_instance, storage_type)
+    redis_instance.ping
+  rescue => e
+    warn "Error connecting to Redis #{storage_type}: #{e}"
+    exit(false)
+  else
+    puts "Connection to Redis #{storage_type} performed successfully"
+  end
+end