zuora_connectD 1.7.09

data/lib/middleware/metrics_middleware.rb

@@ -0,0 +1,110 @@
+module Middleware
+  require 'uri'
+
+  # Object of this class is passed to the ActiveSupport::Notification hook
+  class PageRequest
+
+    # This method is triggered when a non error page is loaded (not 404)
+    def call(name, started, finished, unique_id, payload)
+      # If the url contains any css or JavaScript files then do not collect metrics for them
+      return nil if ["css", "assets", "jpg", "png", "jpeg", "ico"].any? { |word| payload[:path].include?(word) }
+
+      # Getting the endpoint and the content_type
+      content_hash = {:html => "text/html", :js => "application/javascript", :json => "application/json"}
+      content_type = content_hash.key?(payload[:format]) ? content_hash[payload[:format]] : payload[:format]
+      content_type = content_type.gsub('text/javascript', 'application/javascript')
+
+      # payloads with 500 requests do not have status as it is not set by the controller
+      # https://github.com/rails/rails/issues/33335
+      #status_code = payload[:status] ? payload[:status] : payload[:exception_object].present? ? 500 : ""
+      if payload[:exception].present? 
+        status_code, exception = [500, payload[:exception].first]
+      else
+        status_code, exception = [payload[:status], nil]
+      end
+
+      tags = {method: payload[:method], status: status_code, error_type: exception, content_type: content_type, controller: payload[:controller], action: payload[:action]}.compact
+
+      values = {view_time: payload[:view_runtime], db_time: payload[:db_runtime], response_time: ((finished-started)*1000)}.compact
+      values = values.map{ |k,v| [k,v.round(2)]}.to_h
+
+      ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
+    end
+  end
+
+  class MetricsMiddleware
+
+    require "zuora_connectD/version"
+    require "zuora_apiD/version"
+
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      start_time = Time.now
+      @status, @headers, @response = @app.call(env)
+
+      # If the url contains any CSS or JavaScript files then do not collect metrics for them
+      if ["css", "assets", "jpg", "png", "jpeg", "ico"].any? { |word| env['PATH_INFO'].include?(word) } || /.*\.js$/.match(env['PATH_INFO'])
+        tags = {status: @status, controller: 'ActionController', action: 'Assets', app_instance: 0}
+        values = {response_time: ((Time.now - start_time)*1000).round(2) }
+        ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: 'request-inbound-assets', tags: tags, values: values)
+      end
+
+      if defined? Prometheus
+        #Prometheus Stuff
+        if env['PATH_INFO'] == '/connect/internal/metrics'
+
+          #Do something before each scrape
+          if defined? Resque.redis
+
+            app_name = ENV['DEIS_APP'].present? ? "#{ENV['DEIS_APP']}" : "#{Rails.application.class.parent_name}"
+            begin
+
+              Resque.redis.ping
+
+              Prometheus::REDIS_CONNECTION.set({connection:'redis',name:app_name},1)
+              Prometheus::FINISHED_JOBS.set({type:'resque',name:app_name},Resque.info[:processed])
+              Prometheus::PENDING_JOBS.set({type:'resque',name:app_name},Resque.info[:pending])
+              Prometheus::ACTIVE_WORKERS.set({type:'resque',name:app_name},Resque.info[:working])
+              Prometheus::WORKERS.set({type:'resque',name:app_name},Resque.info[:workers])
+              Prometheus::FAILED_JOBS.set({type:'resque',name:app_name},Resque.info[:failed])
+
+            rescue Redis::CannotConnectError
+                Prometheus::REDIS_CONNECTION.set({connection:'redis',name:app_name},0)
+            end
+
+            if ZuoraConnect.configuration.custom_prometheus_update_block != nil
+              ZuoraConnect.configuration.custom_prometheus_update_block.call()
+            end
+          end
+
+        end
+      end
+
+      # Uncomment following block of code for handling engine requests/requests without controller
+      # else
+      #   # Handling requests which do not have controllers (engines)
+      if env["SCRIPT_NAME"].present?
+        controller_path = "#{env['SCRIPT_NAME'][1..-1]}"
+        controller_path = controller_path.sub("/", "::")
+        request_path = "#{controller_path}#UnknownAction"
+      else
+        # Writing to telegraf: Handle 404
+        if [404].include?(@status)
+          content_type = @headers['Content-Type'].split(';')[0] if @headers['Content-Type']
+          content_type = content_type.gsub('text/javascript', 'application/javascript')
+          tags = {status: @status, content_type: content_type}
+       
+          tags = tags.merge({controller: 'ActionController', action: 'RoutingError' })
+          
+          values = {response_time: ((Time.now - start_time)*1000).round(2) }
+
+          ZuoraConnect::AppInstanceBase.write_to_telegraf(direction: :inbound, tags: tags, values: values)
+        end
+      end
+      [@status, @headers, @response]
+    end
+  end
+end

data/lib/resque/additions.rb

@@ -0,0 +1,53 @@
+module Resque
+  module Additions
+    def dequeue_from(queue, klass, *args)
+      ####### ------ Resque Job --------
+      # Perform before_dequeue hooks. Don't perform dequeue if any hook returns false
+      before_hooks = Plugin.before_dequeue_hooks(klass).collect do |hook|
+        klass.send(hook, *args)
+      end
+      return if before_hooks.any? { |result| result == false }
+
+      destroyed = Job.destroy(queue, klass, *args)
+
+      Plugin.after_dequeue_hooks(klass).each do |hook|
+        klass.send(hook, *args)
+      end
+
+      destroyed
+    end
+    
+    ####### ------ Resque Delayed Job --------
+    # Returns delayed jobs schedule timestamp for +klass+, +args+.
+    def scheduled_at_with_queue(queue, klass, *args)
+      search = encode(job_to_hash_with_queue(queue,klass, args))
+      redis.smembers("timestamps:#{search}").map do |key|
+        key.tr('delayed:', '').to_i
+      end
+    end
+
+    # Given an encoded item, remove it from the delayed_queue
+    def remove_delayed_with_queue(queue, klass, *args)
+      search = encode(job_to_hash_with_queue(queue,klass, args))
+      remove_delayed_job(search)
+    end
+
+    #Given a timestamp and job (klass + args) it removes all instances and
+    # returns the count of jobs removed.
+    #
+    # O(N) where N is the number of jobs scheduled to fire at the given
+    # timestamp
+    def remove_delayed_job_with_queue_from_timestamp(timestamp, queue, klass, *args)
+      return 0 if Resque.inline?
+
+      key = "delayed:#{timestamp.to_i}"
+      encoded_job = encode(job_to_hash_with_queue(queue, klass, args))
+
+      redis.srem("timestamps:#{encoded_job}", key)
+      count = redis.lrem(key, 0, encoded_job)
+      clean_up_timestamp(key, timestamp)
+
+      count
+    end
+  end
+end

data/lib/resque/dynamic_queues.rb

@@ -0,0 +1,142 @@
+module Resque
+  module DynamicQueues
+    def filter_busy_queues qs
+      busy_queues = Resque::Worker.working.map { |worker| worker.job["queue"] }.compact
+      Array(qs.dup).compact - busy_queues
+    end
+
+    def rotated_queues
+      @n ||= 0
+      @n += 1
+      rot_queues = queues # since we rely on the resque-dynamic-queues plugin, this is all the queues, expanded out
+      if rot_queues.size > 0
+        @n = @n % rot_queues.size
+        rot_queues.rotate(@n)
+      else
+        rot_queues
+      end
+    end
+
+    def queue_depth queuename
+      busy_queues = Resque::Worker.working.map { |worker| worker.job["queue"] }.compact
+      # find the queuename, count it.
+      busy_queues.select {|q| q == queuename }.size
+    end
+
+    DEFAULT_QUEUE_DEPTH = 0
+    def should_work_on_queue? queuename
+      return true if @queues.include? '*'  # workers with QUEUES=* are special and are not subject to queue depth setting
+      max = DEFAULT_QUEUE_DEPTH
+      unless ENV["RESQUE_QUEUE_DEPTH"].nil? || ENV["RESQUE_QUEUE_DEPTH"] == ""
+        max = ENV["RESQUE_QUEUE_DEPTH"].to_i
+      end
+      return true if max == 0 # 0 means no limiting
+      cur_depth = queue_depth(queuename)
+      log! "queue #{queuename} depth = #{cur_depth} max = #{max}"
+      return true if cur_depth < max
+      false
+    end
+
+    def reserve_with_round_robin
+      grouped_queues = queues.sort.group_by{|u| /(\d{1,20})_.*/.match(u) ? /(\d{1,20})_.*/.match(u).captures.first : nil}
+
+      #Instance queue grouping
+      if !grouped_queues.keys.include?(nil) && grouped_queues.keys.size > 0
+        @n ||= 0
+        @n += 1
+        @n = @n % grouped_queues.keys.size
+
+        grouped_queues.keys.rotate(@n).each do |key|
+          grouped_queues[key].each do |queue|
+            log! "Checking #{queue}"
+            if should_work_on_queue?(queue) && @job_in_progress = Resque::Job.reserve(queue)
+              log! "Found job on #{queue}"
+              return @job_in_progress
+            end
+          end
+          @n += 1 # Start the next search at the queue after the one from which we pick a job.
+        end
+        nil
+      else 
+        return reserve_without_round_robin
+      end
+      
+    rescue Exception => e
+      log "Error reserving job: #{e.inspect}"
+      log e.backtrace.join("\n")
+      raise e
+    end
+
+    # Returns a list of queues to use when searching for a job.
+    #
+    # A splat ("*") means you want every queue (in alpha order) - this
+    # can be useful for dynamically adding new queues.
+    #
+    # The splat can also be used as a wildcard within a queue name,
+    # e.g. "*high*", and negation can be indicated with a prefix of "!"
+    #
+    # An @key can be used to dynamically look up the queue list for key from redis.
+    # If no key is supplied, it defaults to the worker's hostname, and wildcards
+    # and negations can be used inside this dynamic queue list.   Set the queue
+    # list for a key with Resque.set_dynamic_queue(key, ["q1", "q2"]
+    #
+    def queues_with_dynamic
+      queue_names = @queues.dup
+      
+      return queues_without_dynamic if queue_names.grep(/(^!)|(^@)|(\*)/).size == 0
+
+      real_queues = Resque.queues
+      matched_queues = []
+
+      #Remove Queues under Api Limits
+      api_limit_instances = Redis.current.keys('APILimits:*').map {|key| key.split('APILimits:').last.to_i}
+      real_queues = real_queues.select {|key| key if !api_limit_instances.include?((key.match(/^(\d*)_.*/) || [])[1].to_i)} ## 2
+      
+      #Queue Pausing 
+      paused_instances = Redis.current.keys('resque:PauseQueue:*').map {|key| key.split('resque:PauseQueue:').last.to_i}
+      real_queues = real_queues.select {|key| key if !paused_instances.include?((key.match(/^(\d*)_.*/) || [])[1].to_i)}
+
+      while q = queue_names.shift
+        q = q.to_s
+
+        if q =~ /^(!)?@(.*)/
+          key = $2.strip
+          key = hostname if key.size == 0
+
+          add_queues = Resque.get_dynamic_queue(key)
+          add_queues.map! { |q| q.gsub!(/^!/, '') || q.gsub!(/^/, '!') } if $1
+
+          queue_names.concat(add_queues)
+          next
+        end
+
+        if q =~ /^!/
+          negated = true
+          q = q[1..-1]
+        end
+
+        patstr = q.gsub(/\*/, '.*')
+        pattern = /^#{patstr}$/
+        if negated
+          matched_queues -= matched_queues.grep(pattern)
+        else
+          matches = real_queues.grep(/^#{pattern}$/)
+          matches = [q] if matches.size == 0 && q == patstr
+          matched_queues.concat(matches.sort)
+        end
+      end
+     
+      return matched_queues.uniq
+    end
+
+
+    def self.included(receiver)
+      receiver.class_eval do
+        alias queues_without_dynamic queues
+        alias queues queues_with_dynamic
+        alias reserve_without_round_robin reserve
+        alias reserve reserve_with_round_robin
+      end
+    end
+  end
+end

data/lib/resque/self_lookup.rb

@@ -0,0 +1,19 @@
+module Resque
+  module SelfLookup
+    def payload_class_enhanced
+      @payload_class ||= constantize(@payload['class'])
+      @payload_class.instance_eval { class << self; self end }.send(:attr_accessor, :worker)
+      @payload_class.instance_eval { class << self; self end }.send(:attr_accessor, :job)
+      @payload_class.worker =  self.worker
+      @payload_class.job =  self
+      return @payload_class
+    end
+
+    def self.included(receiver)
+      receiver.class_eval do
+        alias payload_class_old payload_class
+        alias payload_class payload_class_enhanced
+      end
+    end
+  end
+end

data/lib/resque/silence_done.rb

@@ -0,0 +1,71 @@
+module Resque  
+  module SilenceDone
+    def perform_no_log(job)
+      begin
+        if fork_per_job?
+          reconnect
+          run_hook :after_fork, job
+        end
+        job.perform
+      rescue Object => e
+        report_failed_job(job,e)
+      else
+        # log_with_severity :info, "done: #{job.inspect}"
+      ensure
+        yield job if block_given?
+      end
+    end
+
+    # def work(interval = 5.0, &block)
+    #   interval = Float(interval)
+    #   startup
+
+    #   loop do
+    #     break if shutdown?
+
+    #     unless work_one_job(&block)
+    #       break if interval.zero?
+    #       log_with_severity :debug, "Sleeping for #{interval} seconds"
+    #       procline paused? ? "Paused" : "Waiting for #{queues.join(',')}"
+    #       sleep interval
+    #     end
+    #   end
+
+    #   unregister_worker
+    # rescue Exception => exception
+    #   return if exception.class == SystemExit && !@child && run_at_exit_hooks
+    #   log_with_severity :error, "Failed to start worker : #{exception.inspect}"
+    #   unregister_worker(exception)
+    # end
+
+    def work_one_job_no_log(job = nil, &block)
+      return false if paused?
+      return false unless job ||= reserve
+
+      working_on job
+      procline "Processing #{job.queue} since #{Time.now.to_i} [#{job.payload_class_name}]"
+
+      #log_with_severity :info, "got: #{job.inspect}"
+      job.worker = self
+
+      if fork_per_job?
+        perform_with_fork(job, &block)
+      else
+        perform(job, &block)
+      end
+
+      done_working
+      true
+    end
+
+    def self.included(receiver)
+      receiver.class_eval do
+        alias work_one_job_with_log work_one_job
+        alias work_one_job work_one_job_no_log
+
+        alias perform_with_log perform
+        alias perform perform_no_log
+      end
+    end
+  end
+end

data/lib/tasks/zuora_connect_tasks.rake

@@ -0,0 +1,24 @@
+# desc "Explaining what the task does"
+# task :connect do
+#   # Task goes here
+# end
+
+namespace :db do
+  desc 'Also create shared_extensions Schema'
+  task :extensions => :environment  do
+    # Create Schema
+    ActiveRecord::Base.connection.execute 'CREATE SCHEMA IF NOT EXISTS shared_extensions;'
+    # Enable Hstore
+    ActiveRecord::Base.connection.execute 'CREATE EXTENSION IF NOT EXISTS HSTORE SCHEMA shared_extensions;'
+    # Enable UUID-OSSP
+    ActiveRecord::Base.connection.execute 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp" SCHEMA shared_extensions;'
+  end
+end
+
+Rake::Task["db:create"].enhance do
+  Rake::Task["db:extensions"].invoke
+end
+
+Rake::Task["db:test:purge"].enhance do
+  Rake::Task["db:extensions"].invoke
+end

data/lib/zuora_connectD.rb

@@ -0,0 +1,41 @@
+require 'zuora_connectD/configuration'
+require "zuora_connectD/engine"
+require 'zuora_connectD/exceptions'
+require 'zuora_connectD/controllers/helpers'
+require 'zuora_connectD/views/helpers'
+require 'zuora_connectD/railtie'
+require 'resque/additions'
+require 'resque/dynamic_queues'
+require 'resque/silence_done'
+require 'resque/self_lookup'
+require 'metrics/influx/point_value'
+require 'metrics/net'
+
+module ZuoraConnect
+  class << self
+    attr_accessor :configuration
+  end
+  module Controllers
+    autoload :Helpers,        'zuora_connect/controllers/helpers'
+  end
+
+  module Views
+    ActionView::Base.send(:include, Helpers)
+  end
+
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  def self.reset
+    @configuration = Configuration.new
+  end
+
+  def self.configure
+    yield(configuration)
+    ::Apartment.excluded_models << "Delayed::Job" if configuration.delayed_job
+    ::Apartment.excluded_models.concat(configuration.additional_apartment_models) if configuration.additional_apartment_models.class == Array
+
+    return configuration
+  end
+end