apisonator 2.100.0

data/bin/3scale_backend_worker

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+require '3scale/backend'
+require 'daemons'
+
+if ARGV.delete '--version'
+  STDOUT.puts "3scale_backend_worker version #{ThreeScale::Backend::VERSION}"
+  exit 0
+end
+
+options = {
+  multiple: true,
+  dir_mode: :normal,
+  dir: '/var/run/3scale'
+}
+
+# make --no-daemonize an alias of --ontop
+options[:ontop] = true if ARGV.delete '--no-daemonize'
+
+if !File.writable? options[:dir]
+  require 'fileutils'
+  FileUtils.mkdir_p options[:dir]
+end
+
+Daemons.run_proc('3scale_backend_worker', options) do
+  ThreeScale::Backend::Worker.work
+end

data/config.ru

@@ -0,0 +1,4 @@
+require_relative 'lib/3scale/bundler_shim'
+require '3scale/backend/rack'
+
+ThreeScale::Backend::Rack.run self

data/config/puma.rb

@@ -0,0 +1,192 @@
+#!/usr/bin/env puma
+
+# The directory to operate out of.
+#
+# The default is the current directory.
+#
+# directory '/u/apps/lolcat'
+
+# Use an object or block as the rack application. This allows the
+# config file to be the application itself.
+#
+# app do |env|
+#   puts env
+#
+#   body = 'Hello, World!'
+#
+#   [200, { 'Content-Type' => 'text/plain', 'Content-Length' => body.length.to_s }, [body]]
+# end
+
+# Load "path" as a rackup file.
+#
+# The default is "config.ru".
+#
+# rackup '/u/apps/lolcat/config.ru'
+rackup 'config.ru'
+
+# Set the environment in which the rack's app will run. The value must be a string.
+#
+# The default is "development".
+#
+# environment 'production'
+
+# Daemonize the server into the background. Highly suggest that
+# this be combined with "pidfile" and "stdout_redirect".
+#
+# The default is "false".
+#
+# daemonize
+# daemonize false
+
+# Store the pid of the server in the file at "path".
+#
+# pidfile '/u/apps/lolcat/tmp/pids/puma.pid'
+
+# Use "path" as the file to store the server info state. This is
+# used by "pumactl" to query and control the server.
+#
+# state_path '/u/apps/lolcat/tmp/pids/puma.state'
+
+# Redirect STDOUT and STDERR to files specified. The 3rd parameter
+# ("append") specifies whether the output is appended, the default is
+# "false".
+#
+# stdout_redirect '/u/apps/lolcat/log/stdout', '/u/apps/lolcat/log/stderr'
+# stdout_redirect '/u/apps/lolcat/log/stdout', '/u/apps/lolcat/log/stderr', true
+
+# Disable request logging.
+#
+# The default is "false".
+#
+quiet
+
+# Configure "min" to be the minimum number of threads to use to answer
+# requests and "max" the maximum.
+#
+# The default is "0, 16".
+#
+# threads 0, 16
+
+# Bind the server to "url". "tcp://", "unix://" and "ssl://" are the only
+# accepted protocols.
+#
+# The default is "tcp://0.0.0.0:9292".
+#
+# bind 'tcp://0.0.0.0:9292'
+# bind 'unix:///var/run/puma.sock'
+# bind 'unix:///var/run/puma.sock?umask=0111'
+# bind 'ssl://127.0.0.1:9292?key=path_to_key&cert=path_to_cert'
+
+# Instead of "bind 'ssl://127.0.0.1:9292?key=path_to_key&cert=path_to_cert'" you
+# can also use the "ssl_bind" option.
+#
+# ssl_bind '127.0.0.1', '9292', { key: path_to_key, cert: path_to_cert }
+
+# Code to run before doing a restart. This code should
+# close log files, database connections, etc.
+#
+# This can be called multiple times to add code each time.
+#
+# on_restart do
+#   puts 'On restart...'
+# end
+
+# Command to use to restart puma. This should be just how to
+# load puma itself (ie. 'ruby -Ilib bin/puma'), not the arguments
+# to puma, as those are the same as the original process.
+#
+# restart_command '/u/app/lolcat/bin/restart_puma'
+
+# === Cluster mode ===
+
+# How many worker processes to run.
+#
+# The default is "0".
+#
+# workers 2
+
+# Code to run when a worker boots to setup the process before booting
+# the app.
+#
+# This can be called multiple times to add hooks.
+#
+# on_worker_boot do
+#   puts 'On worker boot...'
+# end
+on_worker_boot do |_idx|
+  require_relative '../lib/3scale/backend/version'
+  title = $0.dup
+  title << " [3scale_backend #{ThreeScale::Backend::VERSION}]"
+  if Process.respond_to? :setproctitle
+    Process.setproctitle title
+  else
+    $0 = title
+  end
+end
+
+# Code to run when a worker boots to setup the process after booting
+# the app.
+#
+# This can be called multiple times to add hooks.
+#
+# after_worker_boot do
+#   puts 'After worker boot...'
+# end
+
+# Code to run when a worker shutdown.
+#
+#
+# on_worker_shutdown do
+#   puts 'On worker shutdown...'
+# end
+
+# Allow workers to reload bundler context when master process is issued
+# a USR1 signal. This allows proper reloading of gems while the master
+# is preserved across a phased-restart. (incompatible with preload_app)
+# (off by default)
+
+prune_bundler
+
+# Preload the application before starting the workers; this conflicts with
+# phased restart feature. (off by default)
+
+# preload_app!
+
+# Additional text to display in process listing
+#
+# tag 'app name'
+#
+# If you do not specify a tag, Puma will infer it. If you do not want Puma
+# to add a tag, use an empty string.
+tag ''
+
+# Verifies that all workers have checked in to the master process within
+# the given timeout. If not the worker process will be restarted. Default
+# value is 60 seconds.
+#
+# worker_timeout 60
+
+# Change the default worker timeout for booting
+#
+# If unspecified, this defaults to the value of worker_timeout.
+#
+# worker_boot_timeout 60
+
+# === Puma control rack application ===
+
+# Start the puma control rack application on "url". This application can
+# be communicated with to control the main server. Additionally, you can
+# provide an authentication token, so all requests to the control server
+# will need to include that token as a query parameter. This allows for
+# simple authentication.
+#
+# Check out https://github.com/puma/puma/blob/master/lib/puma/app/status.rb
+# to see what the app has available.
+#
+# activate_control_app 'unix:///var/run/pumactl.sock'
+# activate_control_app 'unix:///var/run/pumactl.sock', { auth_token: '12345' }
+# activate_control_app 'unix:///var/run/pumactl.sock', { no_token: true }
+
+before_fork do
+  require_relative '../lib/3scale/prometheus_server'
+end

data/config/schedule.rb

@@ -0,0 +1,9 @@
+job_type :backend_rake, 'cd :path && /usr/local/bin/bundle exec rake2.4 :task --silent'
+
+every 2.minutes do
+  backend_rake 'stats:kinesis:send', path: '/home/bender/backend'
+end
+
+every 1.hour do
+  backend_rake 'stats:redshift:import', path: '/home/bender/backend'
+end

data/ext/mkrf_conf.rb

@@ -0,0 +1,64 @@
+# This extension is a hack to allow gem install to take Gemfile into account.
+#
+require 'rubygems'
+require 'pathname'
+
+BUNDLER_GEM = 'bundler'
+
+def bundled_with(lockfile)
+  File.read(lockfile).
+    lines.each_cons(2).find do |f, _|
+      f == "BUNDLED WITH\n".freeze
+    end.last.strip
+end
+
+def bundler_requirements(version, reqtype = '=')
+  ["#{reqtype} #{version}"]
+end
+
+def gem_install_bundler(requirements)
+  require 'rubygems/command'
+  require 'rubygems/commands/install_command'
+  begin
+    Gem::Command.build_args = ARGV
+  rescue NoMethodError
+  end
+
+  instcmd = Gem::Commands::InstallCommand.new
+  instcmd.handle_options ['-N', BUNDLER_GEM, '--version', *requirements]
+  begin
+    instcmd.execute
+  rescue Gem::SystemExitException => e
+    raise e unless e.exit_code.zero?
+  end
+end
+
+def get_bundler(requirements)
+  Gem::Specification.find_by_name(BUNDLER_GEM, *requirements)
+end
+
+begin
+  extdir = Pathname.new(__FILE__).dirname.realpath
+  rootdir = File.expand_path('..', extdir)
+  gemfile = File.join rootdir, 'Gemfile'
+  bundled_with_version = bundled_with(gemfile + '.lock')
+  bundler_reqs = bundler_requirements bundled_with_version
+  gem_install_bundler bundler_reqs
+  bundler = get_bundler bundler_reqs
+  bundler_version = bundler.version
+  raise "installed bundler version #{bundler_version} must match required " \
+    "version #{bundled_with_version}" if bundled_with_version != bundler_version.version
+  bundle_bin = bundler.bin_file 'bundle'
+  IO.popen(%W{#{Gem.ruby} #{bundle_bin} install --without development test --gemfile=#{gemfile}}) do |io|
+    STDOUT.puts "Running bundler #{bundler_version} at #{bundle_bin} from #{Dir.pwd} with pid #{io.pid}"
+    io.each_line { |l| puts l }
+  end
+  raise "bundle install failed: #{$?.inspect}" unless $?.success?
+  # create dummy rakefile to indicate success
+  File.open(File.join(extdir, 'Rakefile'), 'w') do |f|
+    f.write("task :default\n")
+  end
+rescue => e
+  STDERR.puts "Error: #{e.message} - #{e.class}"
+  exit 1
+end

data/lib/3scale/backend.rb

@@ -0,0 +1,67 @@
+# load the bundler shim
+require_relative 'bundler_shim'
+
+require 'builder'
+require 'hiredis'
+
+require 'redis'
+require '3scale/backend/extensions/redis'
+
+require 'resque'
+require 'securerandom'
+require 'sinatra/base'
+require 'time'
+require 'yajl'
+require 'yaml'
+require 'digest/md5'
+
+# Require here the classes needed for configuring Backend
+require '3scale/backend/environment'
+require '3scale/backend/version'
+require '3scale/backend/constants'
+require '3scale/backend/configuration'
+require '3scale/backend/util'
+require '3scale/backend/manifest'
+require '3scale/backend/logging'
+
+# A lot of classes depend on the required modules above, so don't place them
+# above this point.
+require '3scale/backend/logging/middleware'
+require '3scale/backend/period'
+require '3scale/backend/storage_helpers'
+require '3scale/backend/storage_key_helpers'
+require '3scale/backend/storable'
+require '3scale/backend/usage'
+require '3scale/backend/rack'
+require '3scale/backend/extensions'
+require '3scale/backend/background_job'
+require '3scale/backend/storage'
+require '3scale/backend/oauth'
+require '3scale/backend/memoizer'
+require '3scale/backend/application'
+require '3scale/backend/error_storage'
+require '3scale/backend/metric'
+require '3scale/backend/service'
+require '3scale/backend/queue_storage'
+require '3scale/backend/errors'
+require '3scale/backend/stats'
+require '3scale/backend/usage_limit'
+require '3scale/backend/alerts'
+require '3scale/backend/event_storage'
+require '3scale/backend/worker'
+require '3scale/backend/service_token'
+require '3scale/backend/distributed_lock'
+require '3scale/backend/failed_jobs_scheduler'
+require '3scale/backend/transactor'
+require '3scale/backend/saas'
+require '3scale/backend/listener'
+
+Resque.redis = ThreeScale::Backend::QueueStorage.connection(
+  ThreeScale::Backend.environment,
+  ThreeScale::Backend.configuration,
+)
+
+# Need to monkey-patch Resque when using async
+if ThreeScale::Backend.configuration.redis.async
+  Resque.extend ThreeScale::Backend::StorageAsync::ResqueExtensions
+end

data/lib/3scale/backend/alert_limit.rb

@@ -0,0 +1,56 @@
+module ThreeScale
+  module Backend
+    class AlertLimit
+      module KeyHelpers
+        def key(service_id)
+          "alerts/service_id:#{service_id}/allowed_set"
+        end
+      end
+
+      include KeyHelpers
+      extend KeyHelpers
+
+      include Storable
+
+      attr_accessor :service_id, :value
+
+      def save
+        storage.sadd(key(service_id), value.to_i) if valid?
+      end
+
+      def to_hash
+        {
+          service_id: service_id,
+          value:      value.to_i,
+        }
+      end
+
+      def self.load_all(service_id)
+        values = storage.smembers(key(service_id))
+        values.map do |value|
+          new(service_id: service_id, value: value.to_i)
+        end
+      end
+
+      def self.save(service_id, value)
+        alert_limit = new(service_id: service_id, value: value)
+        alert_limit if alert_limit.save
+      end
+
+      def self.delete(service_id, value)
+        storage.srem(key(service_id), value.to_i) if valid_value?(value)
+      end
+
+      def self.valid_value?(value)
+        val = value.to_i
+        Alerts::ALERT_BINS.include?(val) && val.to_s == value.to_s
+      end
+
+      private
+
+      def valid?
+        self.class.valid_value?(value)
+      end
+    end
+  end
+end

data/lib/3scale/backend/alerts.rb

@@ -0,0 +1,137 @@
+module ThreeScale
+  module Backend
+    module Alerts
+      module KeyHelpers
+        private
+
+        # The compacted hour in the params refers to the
+        # TimeHacks.to_compact_s method.
+        def alert_keys(service_id, app_id, discrete_utilization, compacted_hour_start)
+          {
+            already_notified: key_already_notified(service_id, app_id, discrete_utilization),
+            allowed: key_allowed_set(service_id),
+            current_max: key_current_max(service_id, app_id, compacted_hour_start),
+            current_id: key_current_id
+          }
+        end
+
+        def key_prefix(service_id, app_id = nil)
+          prefix = "alerts/service_id:#{service_id}/"
+          prefix << "app_id:#{app_id}/" if app_id
+          prefix
+        end
+
+        def key_already_notified(service_id, app_id, discrete_utilization)
+          prefix = key_prefix(service_id, app_id)
+          "#{prefix}#{discrete_utilization}/already_notified"
+        end
+
+        def key_allowed_set(service_id)
+          prefix = key_prefix(service_id)
+          "#{prefix}allowed_set"
+        end
+
+        def key_current_max(service_id, app_id, compacted_hour_start)
+          prefix = key_prefix(service_id, app_id)
+          "#{prefix}#{compacted_hour_start}/current_max"
+        end
+
+        def key_current_id
+          'alerts/current_id'.freeze
+        end
+      end
+
+      extend self
+      extend KeyHelpers
+
+      ALERT_TTL       = 24*3600 # 1 day (only one message per day)
+      ## zero must be here and sorted, yes or yes
+      ALERT_BINS      = [0, 50, 80, 90, 100, 120, 150, 200, 300].freeze
+      FIRST_ALERT_BIN = ALERT_BINS.first
+      RALERT_BINS     = ALERT_BINS.reverse.freeze
+
+      def utilization(app_usage_reports)
+        max_utilization = -1.0
+        max_record = nil
+        max = proc do |item|
+          if item.max_value > 0
+            utilization = item.current_value / item.max_value.to_f
+
+            if utilization > max_utilization
+              max_record = item
+              max_utilization = utilization
+            end
+          end
+        end
+
+        app_usage_reports.each(&max)
+
+        if max_utilization == -1
+          ## case that all the limits have max_value==0
+          max_utilization = 0
+          max_record = app_usage_reports.first
+        end
+
+        [max_utilization, max_record]
+      end
+
+      def update_utilization(service_id, app_id, max_utilization, max_record, timestamp)
+        discrete = utilization_discrete(max_utilization)
+        max_utilization_i = (max_utilization * 100.0).round
+
+        beginning_of_day = Period::Boundary.day_start(timestamp)
+        period_hour = Period::Boundary.hour_start(timestamp).to_compact_s
+        # UNIX timestamp for key expiration - add 1 day + 5 mins
+        expire_at = (beginning_of_day + 86700).to_i
+
+        keys = alert_keys(service_id, app_id, discrete, period_hour)
+
+        already_alerted, allowed, current_max, _ = storage.pipelined do
+          storage.get(keys[:already_notified])
+          storage.sismember(keys[:allowed], discrete)
+          storage.get(keys[:current_max])
+          storage.expireat(keys[:current_max], expire_at)
+        end
+
+        ## update the status of utilization
+        if max_utilization_i > current_max.to_i
+          storage.set(keys[:current_max], max_utilization_i)
+        end
+
+        if already_alerted.nil? && allowed && discrete.to_i > 0
+          next_id, _ = storage.pipelined do
+            storage.incr(keys[:current_id])
+            storage.setex(keys[:already_notified], ALERT_TTL, "1")
+          end
+
+          alert = { :id => next_id,
+                    :utilization => discrete,
+                    :max_utilization => max_utilization,
+                    :application_id => app_id,
+                    :service_id => service_id,
+                    :timestamp => timestamp,
+                    :limit => formatted_limit(max_record) }
+
+          Backend::EventStorage::store(:alert, alert)
+        end
+      end
+
+      def utilization_discrete(utilization)
+        u = utilization * 100.0
+        # reverse search
+        RALERT_BINS.find do |b|
+          u >= b
+        end || FIRST_ALERT_BIN
+      end
+
+      def formatted_limit(record)
+        "#{record.metric_name} per #{record.period}: "\
+        "#{record.current_value}/#{record.max_value}"
+      end
+
+      def storage
+        Storage.instance
+      end
+    end
+  end
+end