rails_autoscale_agent 0.8.0 → 0.9.0.beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vscode/tasks.json +84 -0
- data/README.md +1 -1
- data/lib/rails_autoscale_agent/autoscale_api.rb +12 -3
- data/lib/rails_autoscale_agent/config.rb +7 -3
- data/lib/rails_autoscale_agent/logger.rb +1 -1
- data/lib/rails_autoscale_agent/measurement.rb +5 -3
- data/lib/rails_autoscale_agent/registration.rb +3 -1
- data/lib/rails_autoscale_agent/report.rb +6 -8
- data/lib/rails_autoscale_agent/reporter.rb +8 -8
- data/lib/rails_autoscale_agent/request.rb +16 -11
- data/lib/rails_autoscale_agent/store.rb +2 -2
- data/lib/rails_autoscale_agent/version.rb +1 -1
- data/lib/rails_autoscale_agent/worker_adapters/delayed_job.rb +45 -41
- data/lib/rails_autoscale_agent/worker_adapters/que.rb +34 -30
- data/lib/rails_autoscale_agent/worker_adapters/resque.rb +32 -0
- data/lib/rails_autoscale_agent/worker_adapters/sidekiq.rb +22 -18
- data/rails_autoscale_agent.gemspec +2 -2
- metadata +24 -8
    
        checksums.yaml
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            ---
         | 
| 2 2 | 
             
            SHA256:
         | 
| 3 | 
            -
              metadata.gz:  | 
| 4 | 
            -
              data.tar.gz:  | 
| 3 | 
            +
              metadata.gz: 998351ab2e804fc443d63825d3a71255d6539f225ef791a4819391d735d29e51
         | 
| 4 | 
            +
              data.tar.gz: 83df02c312d3fd2a1504c782b699c6ac5e26444145ff885ecff808ebf2a9cfc3
         | 
| 5 5 | 
             
            SHA512:
         | 
| 6 | 
            -
              metadata.gz:  | 
| 7 | 
            -
              data.tar.gz:  | 
| 6 | 
            +
              metadata.gz: 6a34893873a4fc6b926e93d89002b3527f8adce615f97d1694753995df89666cfea3c37f55d435011edbe3e6c39928f835d7594c79cd310feb2267bfc1bbb1fa
         | 
| 7 | 
            +
              data.tar.gz: 00b6943d706c93e1d93dc01259d6982618083b0d7359b212eb0187c26edac6f979a11c0c7a387671d60f80a6eb1ef13e74f5182b9f92b814dc01cd9d0f8c5b6b
         | 
    
        data/.vscode/tasks.json
    ADDED
    
    | @@ -0,0 +1,84 @@ | |
| 1 | 
            +
            {
         | 
| 2 | 
            +
              // See https://go.microsoft.com/fwlink/?LinkId=733558
         | 
| 3 | 
            +
              // for the documentation about the tasks.json format
         | 
| 4 | 
            +
              "version": "2.0.0",
         | 
| 5 | 
            +
              "tasks": [
         | 
| 6 | 
            +
                {
         | 
| 7 | 
            +
                  "label": "test: all",
         | 
| 8 | 
            +
                  "group": "test",
         | 
| 9 | 
            +
                  "command": "rspec",
         | 
| 10 | 
            +
                  "runOptions": {
         | 
| 11 | 
            +
                    "reevaluateOnRerun": false
         | 
| 12 | 
            +
                  },
         | 
| 13 | 
            +
                  "problemMatcher": {
         | 
| 14 | 
            +
                    "owner": "ruby",
         | 
| 15 | 
            +
                    "fileLocation": ["relative", "${workspaceRoot}"],
         | 
| 16 | 
            +
                    "severity": "error",
         | 
| 17 | 
            +
                    "pattern": [
         | 
| 18 | 
            +
                      {
         | 
| 19 | 
            +
                        "regexp": "^\\s{7}(.*)$",
         | 
| 20 | 
            +
                        "message": 1
         | 
| 21 | 
            +
                      },
         | 
| 22 | 
            +
                      {
         | 
| 23 | 
            +
                        "regexp": "^\\s+#\\s(\\./.*):(\\d+):(in.*)$",
         | 
| 24 | 
            +
                        "file": 1,
         | 
| 25 | 
            +
                        "line": 2,
         | 
| 26 | 
            +
                        "code": 3
         | 
| 27 | 
            +
                      }
         | 
| 28 | 
            +
                    ]
         | 
| 29 | 
            +
                  }
         | 
| 30 | 
            +
                },
         | 
| 31 | 
            +
                {
         | 
| 32 | 
            +
                  "label": "test: file",
         | 
| 33 | 
            +
                  "group": "test",
         | 
| 34 | 
            +
                  "command": "rspec",
         | 
| 35 | 
            +
                  "args": ["${relativeFile}"],
         | 
| 36 | 
            +
                  "runOptions": {
         | 
| 37 | 
            +
                    "reevaluateOnRerun": false
         | 
| 38 | 
            +
                  },
         | 
| 39 | 
            +
                  "problemMatcher": {
         | 
| 40 | 
            +
                    "owner": "ruby",
         | 
| 41 | 
            +
                    "fileLocation": ["relative", "${workspaceRoot}"],
         | 
| 42 | 
            +
                    "severity": "error",
         | 
| 43 | 
            +
                    "pattern": [
         | 
| 44 | 
            +
                      {
         | 
| 45 | 
            +
                        "regexp": "^\\s{7}(.*)$",
         | 
| 46 | 
            +
                        "message": 1
         | 
| 47 | 
            +
                      },
         | 
| 48 | 
            +
                      {
         | 
| 49 | 
            +
                        "regexp": "^\\s+#\\s(\\./.*):(\\d+):(in.*)$",
         | 
| 50 | 
            +
                        "file": 1,
         | 
| 51 | 
            +
                        "line": 2,
         | 
| 52 | 
            +
                        "code": 3
         | 
| 53 | 
            +
                      }
         | 
| 54 | 
            +
                    ]
         | 
| 55 | 
            +
                  }
         | 
| 56 | 
            +
                },
         | 
| 57 | 
            +
                {
         | 
| 58 | 
            +
                  "label": "test: line",
         | 
| 59 | 
            +
                  "group": "test",
         | 
| 60 | 
            +
                  "command": "rspec",
         | 
| 61 | 
            +
                  "args": ["${relativeFile}:${lineNumber}"],
         | 
| 62 | 
            +
                  "runOptions": {
         | 
| 63 | 
            +
                    "reevaluateOnRerun": false
         | 
| 64 | 
            +
                  },
         | 
| 65 | 
            +
                  "problemMatcher": {
         | 
| 66 | 
            +
                    "owner": "ruby",
         | 
| 67 | 
            +
                    "fileLocation": ["relative", "${workspaceRoot}"],
         | 
| 68 | 
            +
                    "severity": "error",
         | 
| 69 | 
            +
                    "pattern": [
         | 
| 70 | 
            +
                      {
         | 
| 71 | 
            +
                        "regexp": "^\\s{7}(.*)$",
         | 
| 72 | 
            +
                        "message": 1
         | 
| 73 | 
            +
                      },
         | 
| 74 | 
            +
                      {
         | 
| 75 | 
            +
                        "regexp": "^\\s+#\\s(\\./.*):(\\d+):(in.*)$",
         | 
| 76 | 
            +
                        "file": 1,
         | 
| 77 | 
            +
                        "line": 2,
         | 
| 78 | 
            +
                        "code": 3
         | 
| 79 | 
            +
                      }
         | 
| 80 | 
            +
                    ]
         | 
| 81 | 
            +
                  }
         | 
| 82 | 
            +
                }
         | 
| 83 | 
            +
              ]
         | 
| 84 | 
            +
            }
         | 
    
        data/README.md
    CHANGED
    
    | @@ -74,7 +74,7 @@ Reach out to help@railsautoscale.com if you run into any other problems. | |
| 74 74 |  | 
| 75 75 | 
             
            After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
         | 
| 76 76 |  | 
| 77 | 
            -
            To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
         | 
| 77 | 
            +
            To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, commit it, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
         | 
| 78 78 |  | 
| 79 79 | 
             
            ## Contributing
         | 
| 80 80 |  | 
| @@ -11,8 +11,8 @@ module RailsAutoscaleAgent | |
| 11 11 |  | 
| 12 12 | 
             
                SUCCESS = 'success'
         | 
| 13 13 |  | 
| 14 | 
            -
                def initialize( | 
| 15 | 
            -
                  @ | 
| 14 | 
            +
                def initialize(config)
         | 
| 15 | 
            +
                  @config = config
         | 
| 16 16 | 
             
                end
         | 
| 17 17 |  | 
| 18 18 | 
             
                def report_metrics!(report_params, timings_csv)
         | 
| @@ -24,6 +24,10 @@ module RailsAutoscaleAgent | |
| 24 24 | 
             
                  post_json '/registrations', registration: registration_params
         | 
| 25 25 | 
             
                end
         | 
| 26 26 |  | 
| 27 | 
            +
                def report_exception!(ex)
         | 
| 28 | 
            +
                  post_json '/exceptions', message: ex.inspect, backtrace: ex.backtrace.join("\n")
         | 
| 29 | 
            +
                end
         | 
| 30 | 
            +
             | 
| 27 31 | 
             
                private
         | 
| 28 32 |  | 
| 29 33 | 
             
                def post_json(path, data)
         | 
| @@ -37,9 +41,14 @@ module RailsAutoscaleAgent | |
| 37 41 | 
             
                end
         | 
| 38 42 |  | 
| 39 43 | 
             
                def post_raw(options)
         | 
| 40 | 
            -
                  uri = URI.parse("#{@ | 
| 44 | 
            +
                  uri = URI.parse("#{@config.api_base_url}#{options.fetch(:path)}")
         | 
| 41 45 | 
             
                  ssl = uri.scheme == 'https'
         | 
| 42 46 |  | 
| 47 | 
            +
                  if @config.dev_mode
         | 
| 48 | 
            +
                    logger.debug "[DEV_MODE] Skipping request to #{uri}"
         | 
| 49 | 
            +
                    return SuccessResponse.new('{}')
         | 
| 50 | 
            +
                  end
         | 
| 51 | 
            +
             | 
| 43 52 | 
             
                  response = Net::HTTP.start(uri.host, uri.port, use_ssl: ssl) do |http|
         | 
| 44 53 | 
             
                    request = Net::HTTP::Post.new(uri.request_uri, options[:headers] || {})
         | 
| 45 54 | 
             
                    request.body = options.fetch(:body)
         | 
| @@ -7,26 +7,29 @@ module RailsAutoscaleAgent | |
| 7 7 | 
             
                include Singleton
         | 
| 8 8 |  | 
| 9 9 | 
             
                attr_accessor :report_interval, :logger, :api_base_url, :max_request_size,
         | 
| 10 | 
            -
                              :dyno, :pid, :addon_name, :worker_adapters
         | 
| 10 | 
            +
                              :dyno, :pid, :addon_name, :worker_adapters, :dev_mode
         | 
| 11 11 |  | 
| 12 12 | 
             
                def initialize
         | 
| 13 13 | 
             
                  require 'rails_autoscale_agent/worker_adapters/sidekiq'
         | 
| 14 14 | 
             
                  require 'rails_autoscale_agent/worker_adapters/delayed_job'
         | 
| 15 15 | 
             
                  require 'rails_autoscale_agent/worker_adapters/que'
         | 
| 16 | 
            +
                  require 'rails_autoscale_agent/worker_adapters/resque'
         | 
| 16 17 | 
             
                  @worker_adapters = [
         | 
| 17 18 | 
             
                    WorkerAdapters::Sidekiq.instance,
         | 
| 18 19 | 
             
                    WorkerAdapters::DelayedJob.instance,
         | 
| 19 20 | 
             
                    WorkerAdapters::Que.instance,
         | 
| 21 | 
            +
                    WorkerAdapters::Resque.instance,
         | 
| 20 22 | 
             
                  ]
         | 
| 21 23 |  | 
| 22 24 | 
             
                  # Allow the add-on name to be configured - needed for testing
         | 
| 23 25 | 
             
                  @addon_name = ENV['RAILS_AUTOSCALE_ADDON'] || 'RAILS_AUTOSCALE'
         | 
| 24 26 | 
             
                  @api_base_url = ENV["#{@addon_name}_URL"]
         | 
| 27 | 
            +
                  @dev_mode = ENV['RAILS_AUTOSCALE_DEV'] == 'true'
         | 
| 25 28 | 
             
                  @pid = Process.pid
         | 
| 26 29 | 
             
                  @max_request_size = 100_000 # ignore request payloads over 100k since they skew the queue times
         | 
| 27 | 
            -
                  @report_interval =  | 
| 30 | 
            +
                  @report_interval = 10 # this default will be overwritten during Reporter#register!
         | 
| 28 31 | 
             
                  @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new(STDOUT)
         | 
| 29 | 
            -
                  @dyno = ENV['DYNO']
         | 
| 32 | 
            +
                  @dyno = @dev_mode ? 'dev.1' : ENV['DYNO']
         | 
| 30 33 | 
             
                end
         | 
| 31 34 |  | 
| 32 35 | 
             
                def to_s
         | 
| @@ -37,5 +40,6 @@ module RailsAutoscaleAgent | |
| 37 40 | 
             
                  @max_request_size
         | 
| 38 41 | 
             
                end
         | 
| 39 42 |  | 
| 43 | 
            +
                alias_method :dev_mode?, :dev_mode
         | 
| 40 44 | 
             
              end
         | 
| 41 45 | 
             
            end
         | 
| @@ -24,7 +24,7 @@ module RailsAutoscaleAgent | |
| 24 24 | 
             
                  # to DEBUG level in production).
         | 
| 25 25 | 
             
                  # This uses a separate logger so that RAILS_AUTOSCALE_DEBUG
         | 
| 26 26 | 
             
                  # shows debug logs regardless of Rails log level.
         | 
| 27 | 
            -
                  debug_logger.debug tag(msg) if ENV['RAILS_AUTOSCALE_DEBUG'] == 'true'
         | 
| 27 | 
            +
                  debug_logger.debug tag(msg) if ENV['RAILS_AUTOSCALE_DEBUG'] == 'true' || Config.instance.dev_mode?
         | 
| 28 28 | 
             
                end
         | 
| 29 29 |  | 
| 30 30 | 
             
                private
         | 
| @@ -1,9 +1,11 @@ | |
| 1 1 | 
             
            # frozen_string_literal: true
         | 
| 2 2 |  | 
| 3 3 | 
             
            module RailsAutoscaleAgent
         | 
| 4 | 
            -
              class Measurement < Struct.new(:time, :value, :queue_name)
         | 
| 5 | 
            -
                 | 
| 6 | 
            -
             | 
| 4 | 
            +
              class Measurement < Struct.new(:time, :value, :queue_name, :metric)
         | 
| 5 | 
            +
                # No queue_name is assumed to be a web request measurement
         | 
| 6 | 
            +
                # Metrics: qt = queue time (default), qd = queue depth (needed for Resque support)
         | 
| 7 | 
            +
                def initialize(time, value, queue_name = nil, metric = nil)
         | 
| 8 | 
            +
                  super time.utc, value.to_i, queue_name, metric
         | 
| 7 9 | 
             
                end
         | 
| 8 10 | 
             
              end
         | 
| 9 11 | 
             
            end
         | 
| @@ -3,7 +3,7 @@ | |
| 3 3 | 
             
            require 'rails_autoscale_agent/version'
         | 
| 4 4 |  | 
| 5 5 | 
             
            module RailsAutoscaleAgent
         | 
| 6 | 
            -
              class Registration < Struct.new(:config)
         | 
| 6 | 
            +
              class Registration < Struct.new(:config, :worker_adapters)
         | 
| 7 7 |  | 
| 8 8 | 
             
                def to_params
         | 
| 9 9 | 
             
                  {
         | 
| @@ -12,6 +12,8 @@ module RailsAutoscaleAgent | |
| 12 12 | 
             
                    ruby_version: RUBY_VERSION,
         | 
| 13 13 | 
             
                    rails_version: defined?(Rails) && Rails.version,
         | 
| 14 14 | 
             
                    gem_version: VERSION,
         | 
| 15 | 
            +
                    # example: { worker_adapters: 'Sidekiq,Que' }
         | 
| 16 | 
            +
                    worker_adapters: worker_adapters.map { |o| o.class.name.split('::').last }.join(','),
         | 
| 15 17 | 
             
                  }
         | 
| 16 18 | 
             
                end
         | 
| 17 19 | 
             
              end
         | 
| @@ -19,14 +19,12 @@ module RailsAutoscaleAgent | |
| 19 19 | 
             
                def to_csv
         | 
| 20 20 | 
             
                  String.new.tap do |result|
         | 
| 21 21 | 
             
                    @measurements.each do |measurement|
         | 
| 22 | 
            -
                      result <<  | 
| 23 | 
            -
             | 
| 24 | 
            -
             | 
| 25 | 
            -
             | 
| 26 | 
            -
             | 
| 27 | 
            -
             | 
| 28 | 
            -
                        result << measurement.queue_name
         | 
| 29 | 
            -
                      end
         | 
| 22 | 
            +
                      result << [
         | 
| 23 | 
            +
                        measurement.time.to_i,
         | 
| 24 | 
            +
                        measurement.value,
         | 
| 25 | 
            +
                        measurement.queue_name,
         | 
| 26 | 
            +
                        measurement.metric,
         | 
| 27 | 
            +
                      ].join(',')
         | 
| 30 28 |  | 
| 31 29 | 
             
                      result << "\n"
         | 
| 32 30 | 
             
                    end
         | 
| @@ -21,14 +21,14 @@ module RailsAutoscaleAgent | |
| 21 21 | 
             
                  @started = true
         | 
| 22 22 | 
             
                  @worker_adapters = config.worker_adapters.select(&:enabled?)
         | 
| 23 23 |  | 
| 24 | 
            -
                  if !config.api_base_url
         | 
| 24 | 
            +
                  if !config.api_base_url && !config.dev_mode?
         | 
| 25 25 | 
             
                    logger.info "Reporter not started: #{config.addon_name}_URL is not set"
         | 
| 26 26 | 
             
                    return
         | 
| 27 27 | 
             
                  end
         | 
| 28 28 |  | 
| 29 29 | 
             
                  Thread.new do
         | 
| 30 30 | 
             
                    loop do
         | 
| 31 | 
            -
                      register!(config) unless @registered
         | 
| 31 | 
            +
                      register!(config, @worker_adapters) unless @registered
         | 
| 32 32 |  | 
| 33 33 | 
             
                      # Stagger reporting to spread out reports from many processes
         | 
| 34 34 | 
             
                      multiplier = 1 - (rand / 4) # between 0.75 and 1.0
         | 
| @@ -41,7 +41,7 @@ module RailsAutoscaleAgent | |
| 41 41 | 
             
                        # Exceptions in threads other than the main thread will fail silently
         | 
| 42 42 | 
             
                        # https://ruby-doc.org/core-2.2.0/Thread.html#class-Thread-label-Exception+handling
         | 
| 43 43 | 
             
                        logger.error "Reporter error: #{ex.inspect}"
         | 
| 44 | 
            -
                         | 
| 44 | 
            +
                        AutoscaleApi.new(config.api_base_url).report_exception!(ex)
         | 
| 45 45 | 
             
                      end
         | 
| 46 46 | 
             
                    end
         | 
| 47 47 | 
             
                  end
         | 
| @@ -58,7 +58,7 @@ module RailsAutoscaleAgent | |
| 58 58 | 
             
                    logger.info "Reporting #{report.measurements.size} measurements"
         | 
| 59 59 |  | 
| 60 60 | 
             
                    params = report.to_params(config)
         | 
| 61 | 
            -
                    result = AutoscaleApi.new(config | 
| 61 | 
            +
                    result = AutoscaleApi.new(config).report_metrics!(params, report.to_csv)
         | 
| 62 62 |  | 
| 63 63 | 
             
                    case result
         | 
| 64 64 | 
             
                    when AutoscaleApi::SuccessResponse
         | 
| @@ -71,16 +71,16 @@ module RailsAutoscaleAgent | |
| 71 71 | 
             
                  end
         | 
| 72 72 | 
             
                end
         | 
| 73 73 |  | 
| 74 | 
            -
                def register!(config)
         | 
| 75 | 
            -
                  params = Registration.new(config).to_params
         | 
| 76 | 
            -
                  result = AutoscaleApi.new(config | 
| 74 | 
            +
                def register!(config, worker_adapters)
         | 
| 75 | 
            +
                  params = Registration.new(config, worker_adapters).to_params
         | 
| 76 | 
            +
                  result = AutoscaleApi.new(config).register_reporter!(params)
         | 
| 77 77 |  | 
| 78 78 | 
             
                  case result
         | 
| 79 79 | 
             
                  when AutoscaleApi::SuccessResponse
         | 
| 80 80 | 
             
                    @registered = true
         | 
| 81 81 | 
             
                    config.report_interval = result.data['report_interval'] if result.data['report_interval']
         | 
| 82 82 | 
             
                    config.max_request_size = result.data['max_request_size'] if result.data['max_request_size']
         | 
| 83 | 
            -
                    worker_adapters_msg =  | 
| 83 | 
            +
                    worker_adapters_msg = worker_adapters.map { |a| a.class.name }.join(', ')
         | 
| 84 84 | 
             
                    logger.info "Reporter starting, will report every #{config.report_interval} seconds or so. Worker adapters: [#{worker_adapters_msg}]"
         | 
| 85 85 | 
             
                  when AutoscaleApi::FailureResponse
         | 
| 86 86 | 
             
                    logger.error "Reporter failed to register: #{result.failure_message}"
         | 
| @@ -4,17 +4,17 @@ module RailsAutoscaleAgent | |
| 4 4 | 
             
              class Request
         | 
| 5 5 | 
             
                include Logger
         | 
| 6 6 |  | 
| 7 | 
            -
                attr_reader :id, :entered_queue_at, :path, :method, :size
         | 
| 8 | 
            -
             | 
| 9 7 | 
             
                def initialize(env, config)
         | 
| 10 8 | 
             
                  @config = config
         | 
| 11 9 | 
             
                  @id = env['HTTP_X_REQUEST_ID']
         | 
| 12 | 
            -
                  @path = env['PATH_INFO']
         | 
| 13 | 
            -
                  @method = env['REQUEST_METHOD'].downcase
         | 
| 14 10 | 
             
                  @size = env['rack.input'].respond_to?(:size) ? env['rack.input'].size : 0
         | 
| 11 | 
            +
                  @request_body_wait = env['puma.request_body_wait'].to_i
         | 
| 15 12 |  | 
| 16 | 
            -
                  if unix_millis = env['HTTP_X_REQUEST_START']
         | 
| 17 | 
            -
                     | 
| 13 | 
            +
                  @entered_queue_at = if unix_millis = env['HTTP_X_REQUEST_START']
         | 
| 14 | 
            +
                    Time.at(unix_millis.to_f / 1000)
         | 
| 15 | 
            +
                  elsif config.dev_mode?
         | 
| 16 | 
            +
                    # In dev mode, fake a queue time of 0-1000ms
         | 
| 17 | 
            +
                    Time.now - rand + @request_body_wait
         | 
| 18 18 | 
             
                  end
         | 
| 19 19 | 
             
                end
         | 
| 20 20 |  | 
| @@ -23,12 +23,17 @@ module RailsAutoscaleAgent | |
| 23 23 | 
             
                end
         | 
| 24 24 |  | 
| 25 25 | 
             
                def queue_time
         | 
| 26 | 
            -
                  if entered_queue_at
         | 
| 27 | 
            -
                    queue_time = ((Time.now - entered_queue_at) * 1000).to_i
         | 
| 28 | 
            -
             | 
| 29 | 
            -
                     | 
| 26 | 
            +
                  if @entered_queue_at
         | 
| 27 | 
            +
                    queue_time = ((Time.now - @entered_queue_at) * 1000).to_i
         | 
| 28 | 
            +
             | 
| 29 | 
            +
                    # Subtract the time Puma spent waiting on the request body. It's irrelevant to capacity-related queue time.
         | 
| 30 | 
            +
                    # Without this, slow clients and large request payloads will skew queue time.
         | 
| 31 | 
            +
                    queue_time -= @request_body_wait
         | 
| 32 | 
            +
             | 
| 33 | 
            +
                    logger.debug "Request queue_time=#{queue_time}ms body_wait=#{@request_body_wait}ms request_id=#{@id} size=#{@size}"
         | 
| 30 34 |  | 
| 31 | 
            -
                     | 
| 35 | 
            +
                    # Safeguard against negative queue times (should not happen in practice)
         | 
| 36 | 
            +
                    queue_time > 0 ? queue_time : 0
         | 
| 32 37 | 
             
                  end
         | 
| 33 38 | 
             
                end
         | 
| 34 39 | 
             
              end
         | 
| @@ -15,8 +15,8 @@ module RailsAutoscaleAgent | |
| 15 15 | 
             
                  @measurements = []
         | 
| 16 16 | 
             
                end
         | 
| 17 17 |  | 
| 18 | 
            -
                def push(value, time = Time.now, queue_name = nil)
         | 
| 19 | 
            -
                  @measurements << Measurement.new(time, value, queue_name)
         | 
| 18 | 
            +
                def push(value, time = Time.now, queue_name = nil, metric = nil)
         | 
| 19 | 
            +
                  @measurements << Measurement.new(time, value, queue_name, metric)
         | 
| 20 20 | 
             
                end
         | 
| 21 21 |  | 
| 22 22 | 
             
                def pop_report
         | 
| @@ -2,61 +2,65 @@ | |
| 2 2 |  | 
| 3 3 | 
             
            require 'rails_autoscale_agent/logger'
         | 
| 4 4 |  | 
| 5 | 
            -
            module  | 
| 6 | 
            -
               | 
| 7 | 
            -
                 | 
| 8 | 
            -
             | 
| 5 | 
            +
            module RailsAutoscaleAgent
         | 
| 6 | 
            +
              module WorkerAdapters
         | 
| 7 | 
            +
                class DelayedJob
         | 
| 8 | 
            +
                  include RailsAutoscaleAgent::Logger
         | 
| 9 | 
            +
                  include Singleton
         | 
| 9 10 |  | 
| 10 | 
            -
             | 
| 11 | 
            -
             | 
| 12 | 
            -
             | 
| 11 | 
            +
                  class << self
         | 
| 12 | 
            +
                    attr_accessor :queues
         | 
| 13 | 
            +
                  end
         | 
| 13 14 |  | 
| 14 | 
            -
             | 
| 15 | 
            -
             | 
| 16 | 
            -
             | 
| 17 | 
            -
             | 
| 15 | 
            +
                  def initialize
         | 
| 16 | 
            +
                    # Track the known queues so we can continue reporting on queues that don't
         | 
| 17 | 
            +
                    # currently have enqueued jobs.
         | 
| 18 | 
            +
                    self.class.queues = Set.new
         | 
| 18 19 |  | 
| 19 | 
            -
             | 
| 20 | 
            -
             | 
| 20 | 
            +
                    install if enabled?
         | 
| 21 | 
            +
                  end
         | 
| 21 22 |  | 
| 22 | 
            -
             | 
| 23 | 
            -
             | 
| 24 | 
            -
             | 
| 23 | 
            +
                  def enabled?
         | 
| 24 | 
            +
                    defined? ::Delayed
         | 
| 25 | 
            +
                  end
         | 
| 25 26 |  | 
| 26 | 
            -
             | 
| 27 | 
            -
             | 
| 28 | 
            -
             | 
| 27 | 
            +
                  def collect!(store)
         | 
| 28 | 
            +
                    log_msg = String.new
         | 
| 29 | 
            +
                    t = Time.now
         | 
| 29 30 |  | 
| 30 | 
            -
             | 
| 31 | 
            -
             | 
| 32 | 
            -
             | 
| 31 | 
            +
                    # Ignore failed jobs (they skew latency measurement due to the original run_at)
         | 
| 32 | 
            +
                    sql = 'SELECT queue, min(run_at) FROM delayed_jobs WHERE attempts = 0 GROUP BY queue'
         | 
| 33 | 
            +
                    run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
         | 
| 34 | 
            +
                    queues = self.class.queues | run_at_by_queue.keys
         | 
| 33 35 |  | 
| 34 | 
            -
             | 
| 35 | 
            -
             | 
| 36 | 
            -
             | 
| 37 | 
            -
             | 
| 38 | 
            -
             | 
| 39 | 
            -
             | 
| 40 | 
            -
             | 
| 36 | 
            +
                    queues.each do |queue|
         | 
| 37 | 
            +
                      next if queue.nil? || queue.empty?
         | 
| 38 | 
            +
                      run_at = run_at_by_queue[queue]
         | 
| 39 | 
            +
                      run_at = Time.parse(run_at) if run_at.is_a?(String)
         | 
| 40 | 
            +
                      latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
         | 
| 41 | 
            +
                      store.push latency_ms, t, queue
         | 
| 42 | 
            +
                      log_msg << "dj.#{queue}=#{latency_ms} "
         | 
| 43 | 
            +
                    end
         | 
| 41 44 |  | 
| 42 | 
            -
             | 
| 43 | 
            -
             | 
| 45 | 
            +
                    logger.debug log_msg unless log_msg.empty?
         | 
| 46 | 
            +
                  end
         | 
| 44 47 |  | 
| 45 | 
            -
             | 
| 48 | 
            +
                  private
         | 
| 46 49 |  | 
| 47 | 
            -
             | 
| 48 | 
            -
             | 
| 49 | 
            -
             | 
| 50 | 
            +
                  def install
         | 
| 51 | 
            +
                    plugin = Class.new(Delayed::Plugin) do
         | 
| 52 | 
            +
                      require 'delayed_job'
         | 
| 50 53 |  | 
| 51 | 
            -
             | 
| 52 | 
            -
             | 
| 53 | 
            -
             | 
| 54 | 
            -
             | 
| 54 | 
            +
                      callbacks do |lifecycle|
         | 
| 55 | 
            +
                        lifecycle.before(:enqueue) do |job, &block|
         | 
| 56 | 
            +
                          queue = job.queue || 'default'
         | 
| 57 | 
            +
                          WorkerAdapters::DelayedJob.queues.add queue
         | 
| 58 | 
            +
                        end
         | 
| 55 59 | 
             
                      end
         | 
| 56 60 | 
             
                    end
         | 
| 57 | 
            -
                  end
         | 
| 58 61 |  | 
| 59 | 
            -
             | 
| 62 | 
            +
                    Delayed::Worker.plugins << plugin
         | 
| 63 | 
            +
                  end
         | 
| 60 64 | 
             
                end
         | 
| 61 65 | 
             
              end
         | 
| 62 66 | 
             
            end
         | 
| @@ -3,42 +3,46 @@ | |
| 3 3 | 
             
            require 'rails_autoscale_agent/logger'
         | 
| 4 4 | 
             
            require 'time'
         | 
| 5 5 |  | 
| 6 | 
            -
            module  | 
| 7 | 
            -
               | 
| 8 | 
            -
                 | 
| 9 | 
            -
             | 
| 6 | 
            +
            module RailsAutoscaleAgent
         | 
| 7 | 
            +
              module WorkerAdapters
         | 
| 8 | 
            +
                class Que
         | 
| 9 | 
            +
                  include RailsAutoscaleAgent::Logger
         | 
| 10 | 
            +
                  include Singleton
         | 
| 10 11 |  | 
| 11 | 
            -
             | 
| 12 | 
            +
                  DEFAULT_QUEUES = ['default']
         | 
| 12 13 |  | 
| 13 | 
            -
             | 
| 14 | 
            -
             | 
| 15 | 
            -
             | 
| 16 | 
            -
             | 
| 17 | 
            -
                def initialize
         | 
| 18 | 
            -
                  self.class.queues = DEFAULT_QUEUES
         | 
| 19 | 
            -
                end
         | 
| 20 | 
            -
             | 
| 21 | 
            -
                def enabled?
         | 
| 22 | 
            -
                  defined? ::Que
         | 
| 23 | 
            -
                end
         | 
| 24 | 
            -
             | 
| 25 | 
            -
                def collect!(store)
         | 
| 26 | 
            -
                  log_msg = String.new('Que latency ')
         | 
| 27 | 
            -
                  t = Time.now
         | 
| 14 | 
            +
                  class << self
         | 
| 15 | 
            +
                    attr_accessor :queues
         | 
| 16 | 
            +
                  end
         | 
| 28 17 |  | 
| 29 | 
            -
                   | 
| 30 | 
            -
             | 
| 31 | 
            -
                   | 
| 18 | 
            +
                  def initialize
         | 
| 19 | 
            +
                    self.class.queues = DEFAULT_QUEUES
         | 
| 20 | 
            +
                  end
         | 
| 32 21 |  | 
| 33 | 
            -
                   | 
| 34 | 
            -
                     | 
| 35 | 
            -
                    run_at = Time.parse(run_at) if run_at.is_a?(String)
         | 
| 36 | 
            -
                    latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
         | 
| 37 | 
            -
                    store.push latency_ms, t, queue
         | 
| 38 | 
            -
                    log_msg << "#{queue}=#{latency_ms} "
         | 
| 22 | 
            +
                  def enabled?
         | 
| 23 | 
            +
                    defined? ::Que
         | 
| 39 24 | 
             
                  end
         | 
| 40 25 |  | 
| 41 | 
            -
                   | 
| 26 | 
            +
                  def collect!(store)
         | 
| 27 | 
            +
                    log_msg = String.new
         | 
| 28 | 
            +
                    t = Time.now
         | 
| 29 | 
            +
             | 
| 30 | 
            +
                    # Ignore failed jobs (they skew latency measurement due to the original run_at)
         | 
| 31 | 
            +
                    sql = 'SELECT queue, min(run_at) FROM que_jobs WHERE error_count = 0 GROUP BY queue'
         | 
| 32 | 
            +
                    run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
         | 
| 33 | 
            +
                    self.class.queues |= run_at_by_queue.keys
         | 
| 34 | 
            +
             | 
| 35 | 
            +
                    self.class.queues.each do |queue|
         | 
| 36 | 
            +
                      next if queue.nil? || queue.empty?
         | 
| 37 | 
            +
                      run_at = run_at_by_queue[queue]
         | 
| 38 | 
            +
                      run_at = Time.parse(run_at) if run_at.is_a?(String)
         | 
| 39 | 
            +
                      latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
         | 
| 40 | 
            +
                      store.push latency_ms, t, queue
         | 
| 41 | 
            +
                      log_msg << "que.#{queue}=#{latency_ms} "
         | 
| 42 | 
            +
                    end
         | 
| 43 | 
            +
             | 
| 44 | 
            +
                    logger.debug log_msg unless log_msg.empty?
         | 
| 45 | 
            +
                  end
         | 
| 42 46 | 
             
                end
         | 
| 43 47 | 
             
              end
         | 
| 44 48 | 
             
            end
         | 
| @@ -0,0 +1,32 @@ | |
| 1 | 
            +
            # frozen_string_literal: true
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            require 'rails_autoscale_agent/logger'
         | 
| 4 | 
            +
             | 
| 5 | 
            +
            module RailsAutoscaleAgent
         | 
| 6 | 
            +
              module WorkerAdapters
         | 
| 7 | 
            +
                class Resque
         | 
| 8 | 
            +
                  include RailsAutoscaleAgent::Logger
         | 
| 9 | 
            +
                  include Singleton
         | 
| 10 | 
            +
             | 
| 11 | 
            +
                  def enabled?
         | 
| 12 | 
            +
                    require 'resque'
         | 
| 13 | 
            +
                    true
         | 
| 14 | 
            +
                  rescue LoadError
         | 
| 15 | 
            +
                    false
         | 
| 16 | 
            +
                  end
         | 
| 17 | 
            +
             | 
| 18 | 
            +
                  def collect!(store)
         | 
| 19 | 
            +
                    log_msg = String.new
         | 
| 20 | 
            +
             | 
| 21 | 
            +
                    ::Resque.queues.each do |queue|
         | 
| 22 | 
            +
                      next if queue.nil? || queue.empty?
         | 
| 23 | 
            +
                      depth = ::Resque.size(queue)
         | 
| 24 | 
            +
                      store.push depth, Time.now, queue, :qd
         | 
| 25 | 
            +
                      log_msg << "resque-qd.#{queue}=#{depth} "
         | 
| 26 | 
            +
                    end
         | 
| 27 | 
            +
             | 
| 28 | 
            +
                    logger.debug log_msg
         | 
| 29 | 
            +
                  end
         | 
| 30 | 
            +
                end
         | 
| 31 | 
            +
              end
         | 
| 32 | 
            +
            end
         | 
| @@ -2,28 +2,32 @@ | |
| 2 2 |  | 
| 3 3 | 
             
            require 'rails_autoscale_agent/logger'
         | 
| 4 4 |  | 
| 5 | 
            -
            module  | 
| 6 | 
            -
               | 
| 7 | 
            -
                 | 
| 8 | 
            -
             | 
| 5 | 
            +
            module RailsAutoscaleAgent
         | 
| 6 | 
            +
              module WorkerAdapters
         | 
| 7 | 
            +
                class Sidekiq
         | 
| 8 | 
            +
                  include RailsAutoscaleAgent::Logger
         | 
| 9 | 
            +
                  include Singleton
         | 
| 9 10 |  | 
| 10 | 
            -
             | 
| 11 | 
            -
             | 
| 12 | 
            -
             | 
| 13 | 
            -
             | 
| 14 | 
            -
             | 
| 15 | 
            -
             | 
| 11 | 
            +
                  def enabled?
         | 
| 12 | 
            +
                    require 'sidekiq/api'
         | 
| 13 | 
            +
                    true
         | 
| 14 | 
            +
                  rescue LoadError
         | 
| 15 | 
            +
                    false
         | 
| 16 | 
            +
                  end
         | 
| 16 17 |  | 
| 17 | 
            -
             | 
| 18 | 
            -
             | 
| 18 | 
            +
                  def collect!(store)
         | 
| 19 | 
            +
                    log_msg = String.new
         | 
| 19 20 |  | 
| 20 | 
            -
             | 
| 21 | 
            -
             | 
| 22 | 
            -
             | 
| 23 | 
            -
             | 
| 24 | 
            -
             | 
| 21 | 
            +
                    ::Sidekiq::Queue.all.each do |queue|
         | 
| 22 | 
            +
                      latency_ms = (queue.latency * 1000).ceil
         | 
| 23 | 
            +
                      depth = queue.size
         | 
| 24 | 
            +
                      store.push latency_ms, Time.now, queue.name, :qt
         | 
| 25 | 
            +
                      store.push depth, Time.now, queue.name, :qd
         | 
| 26 | 
            +
                      log_msg << "sidekiq-qt.#{queue.name}=#{latency_ms} sidekiq-qd.#{queue.name}=#{depth} "
         | 
| 27 | 
            +
                    end
         | 
| 25 28 |  | 
| 26 | 
            -
             | 
| 29 | 
            +
                    logger.debug log_msg
         | 
| 30 | 
            +
                  end
         | 
| 27 31 | 
             
                end
         | 
| 28 32 | 
             
              end
         | 
| 29 33 | 
             
            end
         | 
| @@ -21,10 +21,10 @@ Gem::Specification.new do |spec| | |
| 21 21 | 
             
              spec.add_development_dependency "rspec", "~> 3.0"
         | 
| 22 22 | 
             
              spec.add_development_dependency "vcr", "~> 3.0"
         | 
| 23 23 | 
             
              spec.add_development_dependency "webmock"
         | 
| 24 | 
            -
               | 
| 25 | 
            -
              spec.add_development_dependency "pry", "0.12.2"
         | 
| 24 | 
            +
              spec.add_development_dependency "pry"
         | 
| 26 25 | 
             
              spec.add_development_dependency "pry-byebug"
         | 
| 27 26 | 
             
              spec.add_development_dependency "sidekiq", "~> 5.0"
         | 
| 28 27 | 
             
              spec.add_development_dependency "delayed_job"
         | 
| 29 28 | 
             
              spec.add_development_dependency "que"
         | 
| 29 | 
            +
              spec.add_development_dependency "activesupport"
         | 
| 30 30 | 
             
            end
         | 
    
        metadata
    CHANGED
    
    | @@ -1,14 +1,14 @@ | |
| 1 1 | 
             
            --- !ruby/object:Gem::Specification
         | 
| 2 2 | 
             
            name: rails_autoscale_agent
         | 
| 3 3 | 
             
            version: !ruby/object:Gem::Version
         | 
| 4 | 
            -
              version: 0. | 
| 4 | 
            +
              version: 0.9.0.beta.2
         | 
| 5 5 | 
             
            platform: ruby
         | 
| 6 6 | 
             
            authors:
         | 
| 7 7 | 
             
            - Adam McCrea
         | 
| 8 8 | 
             
            autorequire: 
         | 
| 9 9 | 
             
            bindir: bin
         | 
| 10 10 | 
             
            cert_chain: []
         | 
| 11 | 
            -
            date: 2020- | 
| 11 | 
            +
            date: 2020-06-10 00:00:00.000000000 Z
         | 
| 12 12 | 
             
            dependencies:
         | 
| 13 13 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 14 14 | 
             
              name: bundler
         | 
| @@ -84,16 +84,16 @@ dependencies: | |
| 84 84 | 
             
              name: pry
         | 
| 85 85 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| 86 86 | 
             
                requirements:
         | 
| 87 | 
            -
                - -  | 
| 87 | 
            +
                - - ">="
         | 
| 88 88 | 
             
                  - !ruby/object:Gem::Version
         | 
| 89 | 
            -
                    version: 0 | 
| 89 | 
            +
                    version: '0'
         | 
| 90 90 | 
             
              type: :development
         | 
| 91 91 | 
             
              prerelease: false
         | 
| 92 92 | 
             
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 93 93 | 
             
                requirements:
         | 
| 94 | 
            -
                - -  | 
| 94 | 
            +
                - - ">="
         | 
| 95 95 | 
             
                  - !ruby/object:Gem::Version
         | 
| 96 | 
            -
                    version: 0 | 
| 96 | 
            +
                    version: '0'
         | 
| 97 97 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 98 98 | 
             
              name: pry-byebug
         | 
| 99 99 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| @@ -150,6 +150,20 @@ dependencies: | |
| 150 150 | 
             
                - - ">="
         | 
| 151 151 | 
             
                  - !ruby/object:Gem::Version
         | 
| 152 152 | 
             
                    version: '0'
         | 
| 153 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 154 | 
            +
              name: activesupport
         | 
| 155 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 156 | 
            +
                requirements:
         | 
| 157 | 
            +
                - - ">="
         | 
| 158 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 159 | 
            +
                    version: '0'
         | 
| 160 | 
            +
              type: :development
         | 
| 161 | 
            +
              prerelease: false
         | 
| 162 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 163 | 
            +
                requirements:
         | 
| 164 | 
            +
                - - ">="
         | 
| 165 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 166 | 
            +
                    version: '0'
         | 
| 153 167 | 
             
            description: 
         | 
| 154 168 | 
             
            email:
         | 
| 155 169 | 
             
            - adam@adamlogic.com
         | 
| @@ -161,6 +175,7 @@ files: | |
| 161 175 | 
             
            - ".rspec"
         | 
| 162 176 | 
             
            - ".ruby-version"
         | 
| 163 177 | 
             
            - ".travis.yml"
         | 
| 178 | 
            +
            - ".vscode/tasks.json"
         | 
| 164 179 | 
             
            - Gemfile
         | 
| 165 180 | 
             
            - LICENSE.txt
         | 
| 166 181 | 
             
            - README.md
         | 
| @@ -183,6 +198,7 @@ files: | |
| 183 198 | 
             
            - lib/rails_autoscale_agent/version.rb
         | 
| 184 199 | 
             
            - lib/rails_autoscale_agent/worker_adapters/delayed_job.rb
         | 
| 185 200 | 
             
            - lib/rails_autoscale_agent/worker_adapters/que.rb
         | 
| 201 | 
            +
            - lib/rails_autoscale_agent/worker_adapters/resque.rb
         | 
| 186 202 | 
             
            - lib/rails_autoscale_agent/worker_adapters/sidekiq.rb
         | 
| 187 203 | 
             
            - log/.gitkeep
         | 
| 188 204 | 
             
            - rails_autoscale_agent.gemspec
         | 
| @@ -201,9 +217,9 @@ required_ruby_version: !ruby/object:Gem::Requirement | |
| 201 217 | 
             
                  version: '0'
         | 
| 202 218 | 
             
            required_rubygems_version: !ruby/object:Gem::Requirement
         | 
| 203 219 | 
             
              requirements:
         | 
| 204 | 
            -
              - - " | 
| 220 | 
            +
              - - ">"
         | 
| 205 221 | 
             
                - !ruby/object:Gem::Version
         | 
| 206 | 
            -
                  version:  | 
| 222 | 
            +
                  version: 1.3.1
         | 
| 207 223 | 
             
            requirements: []
         | 
| 208 224 | 
             
            rubygems_version: 3.0.3
         | 
| 209 225 | 
             
            signing_key: 
         |