bearcat 1.0.0 → 1.5.24

data/lib/bearcat/client_module.rb

@@ -0,0 +1,103 @@
+module Bearcat
+  class Client < Footrest::Client
+    module ClientModule
+      ARG_REGEX = /:(\w+)/
+      attr_reader :_registered_endpoints
+
+      def prefix(prefix)
+        past_prefix = @current_prefix
+        @current_prefix = (past_prefix || '') + prefix
+        yield
+      ensure
+        @current_prefix = past_prefix
+      end
+
+      def endpoint(method, identifier, url = "", defaults: {}, &blk)
+        if @current_prefix && !url.start_with?('/')
+          url = url[2..-1] if url.start_with?('./')
+          url = @current_prefix + url
+        end
+
+        args = url.to_enum(:scan, ARG_REGEX).map { Regexp.last_match }
+        arg_names = args.map{|m| m[1]}
+
+        @_registered_endpoints ||= {}
+        @_registered_endpoints[identifier] = { symbol: identifier, method: method, url: url }
+
+        # TODO: Consider generating the method using class_eval and a template - this will improve runtime performance
+        # signature_bits = []
+        # logical_bits = []
+        # args.each do |m|
+        #   name = [1]
+        # end
+
+        # interpolated_url = url.gsub(ARG_REGEX) do |m|
+        #   '#{' + m[1] + '}'
+        # end
+
+        # class_eval <<~RUBY
+        #   def #{identifier}(*args, **kwargs)
+        #     parameters = {
+
+        #     }
+
+        #     #{method}(#{interpolated_url})
+        #   end
+        # RUBY
+
+        define_method(identifier) do |*args, **kwargs|
+          url_arguments = {}
+          parameters = { }.with_indifferent_access
+          parameters.merge!(defaults)
+
+          args.each_with_index do |v, i|
+            if arg_names[i]
+              url_arguments[arg_names[i]] = v
+            elsif i == arg_names.count && v.is_a?(Hash)
+              parameters.merge!(v)
+            else
+              raise ArgumentError, "Too many arguments passed to #{identifier}" unless arg_names[i]
+            end
+          end
+
+          parameters.merge!(kwargs)
+
+          preq = BearcatRequest.new(url, url_arguments, parameters)
+          yield preq if block_given?
+
+          arg_names.each do |an|
+            preq.arguments[an] = preq.parameters.delete(an) unless preq.arguments.key?(an)
+            raise ArgumentError, "Missing argument #{an}" unless preq.arguments.key?(an)
+          end
+
+          send(method, preq.interpoated_url, preq.parameters)
+        end
+      end
+
+      def context_types(types, &blk)
+        types.each(&blk)
+      end
+
+      %i[get post delete put patch head].each do |mthd|
+        define_method(mthd) do |*args, **kwargs, &blk|
+          endpoint(mthd, *args, **kwargs, &blk)
+        end
+      end
+
+      BearcatRequest = Struct.new(:url, :arguments, :parameters) do
+        def interpoated_url
+          url.gsub(ARG_REGEX) do |_|
+            m = Regexp.last_match
+            val = arguments[m[1]]
+            val = val.canvas_id if val.respond_to?(:canvas_id)
+            val = val.id if val.respond_to?(:id)
+            val = val['id'].presence || val[:id].presence || val if val.is_a?(Hash)
+            val = val.to_s if val.present?
+            val = URI.encode_www_form_component(val) if val.is_a?(String)
+            val
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bearcat/rate_limiting/increment_bucket.lua

@@ -0,0 +1,33 @@
+local cache_key = KEYS[1]
+
+local amount = tonumber(ARGV[1])
+local current_time = tonumber(ARGV[2])
+local outflow = tonumber(ARGV[3])
+local maximum = tonumber(ARGV[4])
+
+local leak = function(count, last_touched, current_time, outflow)
+  if count > 0 then
+    local timespan = current_time - last_touched
+    local loss = outflow * timespan
+    if loss > 0 then
+      count = count - loss
+    end
+  end
+  return count, last_touched
+end
+
+local count, last_touched = unpack(redis.call('HMGET', cache_key, 'count', 'timestamp'))
+count = tonumber(count or 0)
+last_touched = tonumber(last_touched or current_time)
+
+count, last_touched = leak(count, last_touched, current_time, outflow)
+if count < 0 then count = 0 end
+
+count = count + amount
+if count < 0 then count = 0 end
+if count > maximum then count = maximum end
+
+redis.call('HMSET', cache_key, 'count', count, 'timestamp', current_time)
+redis.call('EXPIRE', cache_key, 600)
+
+return { tostring(count), tostring(current_time) }

data/lib/bearcat/rate_limiting/redis_script.rb

@@ -0,0 +1,164 @@
+require 'pathname'
+require 'digest/sha1'
+require 'erb'
+
+# Modified from https://github.com/Shopify/wolverine/blob/master/lib/wolverine/script.rb
+
+module Bearcat
+  module RateLimiting
+    # {RedisScript} represents a lua script in the filesystem. It loads the script
+    # from disk and handles talking to redis to execute it. Error handling
+    # is handled by {LuaError}.
+    class RedisScript
+
+      # Loads the script file from disk and calculates its +SHA1+ sum.
+      #
+      # @param file [Pathname] the full path to the indicated file
+      def initialize(file)
+        @file = Pathname.new(file)
+      end
+
+      # Passes the script and supplied arguments to redis for evaulation.
+      # It first attempts to use a script redis has already cached by using
+      # the +EVALSHA+ command, but falls back to providing the full script
+      # text via +EVAL+ if redis has not seen this script before. Future
+      # invocations will then use +EVALSHA+ without erroring.
+      #
+      # @param redis [Redis] the redis connection to run against
+      # @param args [*Objects] the arguments to the script
+      # @return [Object] the value passed back by redis after script execution
+      # @raise [LuaError] if the script failed to compile of encountered a
+      #   runtime error
+      def call(redis, *args)
+        t = Time.now
+        begin
+          redis.evalsha(digest, *args)
+        rescue => e
+          e.message =~ /NOSCRIPT/ ? redis.eval(content, *args) : raise
+        end
+      rescue => e
+        if LuaError.intercepts?(e)
+          raise LuaError.new(e, @file, content)
+        else
+          raise
+        end
+      end
+
+      def content
+        @content ||= load_lua(@file)
+      end
+
+      def digest
+        @digest ||= Digest::SHA1.hexdigest content
+      end
+
+      private
+
+      def script_path
+        @file.basename
+        # Rails.root + 'app/redis_lua'
+      end
+
+      def relative_path
+        @path ||= @file.relative_path_from(script_path)
+      end
+
+      def load_lua(file)
+        TemplateContext.new(script_path).template(script_path + file)
+      end
+
+      class TemplateContext
+        def initialize(script_path)
+          @script_path = script_path
+        end
+
+        def template(pathname)
+          @partial_templates ||= {}
+          ERB.new(File.read(pathname)).result binding
+        end
+
+        # helper method to include a lua partial within another lua script
+        #
+        # @param relative_path [String] the relative path to the script from
+        #     `script_path`
+        def include_partial(relative_path)
+          unless @partial_templates.has_key? relative_path
+            @partial_templates[relative_path] = nil
+            template( Pathname.new("#{@script_path}/#{relative_path}") )
+          end
+        end
+      end
+
+      # Reformats errors raised by redis representing failures while executing
+      # a lua script. The default errors have confusing messages and backtraces,
+      # and a type of +RuntimeError+. This class improves the message and
+      # modifies the backtrace to include the lua script itself in a reasonable
+      # way.
+      class LuaError < StandardError
+        PATTERN = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/
+        WOLVERINE_LIB_PATH = File.expand_path('../../', __FILE__)
+        CONTEXT_LINE_NUMBER = 2
+
+        attr_reader :error, :file, :content
+
+        # Is this error one that should be reformatted?
+        #
+        # @param error [StandardError] the original error raised by redis
+        # @return [Boolean] is this an error that should be reformatted?
+        def self.intercepts? error
+          error.message =~ PATTERN
+        end
+
+        # Initialize a new {LuaError} from an existing redis error, adjusting
+        # the message and backtrace in the process.
+        #
+        # @param error [StandardError] the original error raised by redis
+        # @param file [Pathname] full path to the lua file the error ocurred in
+        # @param content [String] lua file content the error ocurred in
+        def initialize error, file, content
+          @error = error
+          @file = file
+          @content = content
+
+          @error.message =~ PATTERN
+          _stage, line_number, message = $1, $2, $3
+          error_context = generate_error_context(content, line_number.to_i)
+
+          super "#{message}\n\n#{error_context}\n\n"
+          set_backtrace generate_backtrace file, line_number
+        end
+
+        private
+
+        def generate_error_context(content, line_number)
+          lines = content.lines.to_a
+          beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
+          ending_line_number = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
+          line_number_width = ending_line_number.to_s.length
+
+          (beginning_line_number..ending_line_number).map do |number|
+            indicator = number == line_number ? '=>' : '  '
+            formatted_number = "%#{line_number_width}d" % number
+            " #{indicator} #{formatted_number}: #{lines[number - 1]}"
+          end.join.chomp
+        end
+
+        def generate_backtrace(file, line_number)
+          pre_wolverine = backtrace_before_entering_wolverine(@error.backtrace)
+          index_of_first_wolverine_line = (@error.backtrace.size - pre_wolverine.size - 1)
+          pre_wolverine.unshift(@error.backtrace[index_of_first_wolverine_line])
+          pre_wolverine.unshift("#{file}:#{line_number}")
+          pre_wolverine
+        end
+
+        def backtrace_before_entering_wolverine(backtrace)
+          backtrace.reverse.take_while { |line| ! line_from_wolverine(line) }.reverse
+        end
+
+        def line_from_wolverine(line)
+          line.split(':').first.include?(WOLVERINE_LIB_PATH)
+        end
+      end
+    end
+  end
+end

data/lib/bearcat/rate_limiting.rb

@@ -0,0 +1,69 @@
+require 'digest/sha1'
+require_relative 'rate_limiting/redis_script'
+
+module Bearcat
+  module RateLimiting
+    LEAK_RATE = 10
+    GUESS_COST = 60
+    MAXIMUM = 600
+
+    class RateLimiter
+      attr_reader :namespace
+
+      def initialize(namespace: "bearcat", **kwargs)
+        @namespace = namespace
+        @params = kwargs
+      end
+
+      def apply(key, guess: GUESS_COST, on_sleep: nil, max_sleep: 15)
+        current = increment("#{key}", 0, guess)
+        cost = guess
+        remaining = MAXIMUM - current[:count]
+
+        if remaining < 0
+          tts = -remaining / LEAK_RATE
+          tts = [tts, max_sleep].min if max_sleep
+          on_sleep.call(tts) if on_sleep
+          sleep tts
+        end
+
+        cost = yield
+      ensure
+        increment(key, (cost || 0), -guess)
+      end
+    end
+
+    class RedisLimiter < RateLimiter
+      INCR = RedisScript.new(Pathname.new(__FILE__) + "../rate_limiting/increment_bucket.lua")
+
+      def increment(key, amount, pending_amount = 0)
+        ts = Time.now.to_f
+
+        redis do |r|
+          actual, timestamp = INCR.call(r, ["#{key}|reported"], [amount, ts, LEAK_RATE, MAXIMUM + 300])
+          {
+            count: actual.to_f,
+            timestamp: timestamp.to_f,
+          }
+        end
+      end
+
+      def checkin_known(key, remaining_time)
+        redis do |r|
+          r.mapped_hmset("#{key}|reported", {
+            count: MAXIMUM + 200 - remaining_time,
+            timestamp: Time.now.to_f,
+          })
+        end
+      end
+
+      def redis(&blk)
+        if @params[:redis]
+          @params[:redis].call(blk)
+        else
+          ::Bearcat.redis(&blk)
+        end
+      end
+    end
+  end
+end

data/lib/bearcat/redis_connection.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "connection_pool"
+require "redis"
+require "uri"
+
+# Adapted from Sidekiq 6.x Implementation
+
+module Bearcat
+  module RedisConnection
+    KNOWN_REDIS_URL_ENVS = %w[REDIS_URL REDISTOGO_URL REDISCLOUD_URL].freeze
+
+    class << self
+      def configured?(prefix, explicit: false)
+        determine_redis_provider(prefix: prefix, explicit: true)
+      end
+
+      def create(options = {})
+        symbolized_options = options.transform_keys(&:to_sym)
+
+        if !symbolized_options[:url] && (u = determine_redis_provider(prefix: options[:env_prefix]))
+          symbolized_options[:url] = u
+        end
+
+        size = if symbolized_options[:size]
+          symbolized_options[:size]
+        elsif symbolized_options[:env_prefix] && (v = ENV["#{symbolized_options[:env_prefix]}_REDIS_POOL_SIZE"])
+          Integer(v)
+        elsif ENV["RAILS_MAX_THREADS"]
+          Integer(ENV["RAILS_MAX_THREADS"])
+        else
+          5
+        end
+
+        pool_timeout = symbolized_options[:pool_timeout] || 1
+
+        ConnectionPool.new(timeout: pool_timeout, size: size) do
+          namespace = symbolized_options[:namespace]
+          client = Redis.new client_opts(symbolized_options)
+
+          if namespace
+            begin
+              require "redis/namespace"
+              Redis::Namespace.new(namespace, redis: client)
+            rescue LoadError
+              Rails.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
+                                   "Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
+              exit(-127)
+            end
+          else
+            client
+          end
+        end
+      end
+
+      private
+
+      def client_opts(options)
+        opts = options.dup
+        if opts[:namespace]
+          opts.delete(:namespace)
+        end
+
+        opts.delete(:size)
+        opts.delete(:env_prefix)
+
+        if opts[:network_timeout]
+          opts[:timeout] = opts[:network_timeout]
+          opts.delete(:network_timeout)
+        end
+
+        opts[:reconnect_attempts] ||= 1
+
+        opts
+      end
+
+      def determine_redis_provider(prefix: nil, explicit: false)
+        vars = []
+
+        if prefix.present?
+          if (ptr = ENV["#{prefix}_REDIS_PROVIDER"]).present?
+            return ENV[ptr]
+          else
+            vars.push(*KNOWN_REDIS_URL_ENVS.map { |e| ENV["#{prefix}_#{e}"] })
+          end
+        end
+
+        if !explicit || !prefix.present?
+          if (ptr = ENV["REDIS_PROVIDER"]).present?
+            vars << ptr # Intentionally not a return
+          else
+            vars.push(*KNOWN_REDIS_URL_ENVS)
+          end
+        end
+
+        vars.select!(&:present?)
+
+        vars.each do |e|
+          return ENV[e] if ENV[e].present?
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/bearcat/spec_helpers.rb

@@ -0,0 +1,125 @@
+require 'bearcat'
+
+# Bearcat RSpec Helpers
+# Can be included in your spec_helper.rb file by using:
+# - `require 'bearcat/spec_helpers'`
+# And
+# - `config.include Bearcat::SpecHelpers`
+# This helper requires `gem 'method_source'` in your test group
+module Bearcat::SpecHelpers
+  extend ActiveSupport::Concern
+
+  SOURCE_REGEX = /(?<method>get|post|delete|put)\((?<quote>\\?('|"))(?<url>.*)\k<quote>/
+
+  # Helper method to Stub Bearcat requests.
+  # Automagically parses the Bearcat method source to determine the correct URL to stub.
+  # Accepts optional keyword parameters to interpolate specific values into the URL.
+  # Returns a mostly-normal Webmock stub (:to_return has been overridden to allow :body to be set to a Hash)
+  def stub_bearcat(endpoint, prefix: nil, method: :auto, **kwargs)
+    cfg = _bearcat_resolve_config(endpoint, kwargs, method: method)
+
+    url = Regexp.escape(_bearcat_resolve_prefix(prefix)) + cfg[:url]
+    url += "?" if url.end_with?('/')
+    stub = stub_request(cfg[:method], Regexp.new(url))
+
+    # Override the to_return method to accept a Hash as body:
+    stub.define_singleton_method(:to_return, ->(*resps, &blk) {
+      if blk
+        super do |*args|
+          resp = blk.call(*args)
+          resp[:headers] ||= {}
+          resp[:headers]["Content-Type"] ||= "application/json"
+          resp[:body] = resp[:body].to_json
+          resp
+        end
+      else
+        resps.map do |resp|
+          resp[:headers] ||= {}
+          resp[:headers]["Content-Type"] ||= "application/json"
+          resp[:body] = resp[:body].to_json
+        end
+        super(*resps)
+      end
+    })
+
+    stub
+  end
+
+  private
+
+  def _bearcat_resolve_config(endpoint, url_context, method: :auto)
+    case endpoint
+    when Symbol
+      if (cfg = Bearcat::Client.registered_endpoints[endpoint]).present?
+        bits = []
+        url = cfg[:url]
+        lend = 0
+        url.scan(/:(?<key>\w+)/) do |key|
+          m = Regexp.last_match
+          between = url[lend..m.begin(0)-1]
+          bits << between if between.present? && (m.begin(0) > lend)
+          lend = m.end(0)
+          bits << (url_context[m[:key].to_sym] || /[\w:]+/)
+        end
+        between = url[lend..-1]
+        bits << between if between.present?
+
+        url = bits.map do |bit|
+          next bit.source if bit.is_a?(Regexp)
+          bit = bit.canvas_id if bit.respond_to?(:canvas_id)
+          Regexp.escape(bit.to_s)
+        end.join
+
+        { method: method == :auto ? cfg[:method] : method, url: url }
+      else
+        ruby_method = Bearcat::Client.instance_method(endpoint)
+        match = SOURCE_REGEX.match(ruby_method.source)
+        bits = []
+        url = match[:url].gsub(/\/$/, '')
+        lend = 0
+        url.scan(/#\{(?<key>.*?)\}/) do |key|
+          m = Regexp.last_match
+          between = url[lend..m.begin(0)-1]
+          bits << between if between.present? && (m.begin(0) > lend)
+          lend = m.end(0)
+          bits << (url_context[m[:key].to_sym] || /[\w:]+/)
+        end
+        between = url[lend..-1]
+        bits << between if between.present?
+
+        url = bits.map do |bit|
+          next bit.source if bit.is_a?(Regexp)
+          bit = bit.canvas_id if bit.respond_to?(:canvas_id)
+          Regexp.escape(bit.to_s)
+        end.join
+
+        { method: method == :auto ? (match ? match[:method].to_sym : :get) : method, url: url }
+      end
+    when String
+      raise "Cannot use method :auto when passing string endpoint" if method == :auto
+      { method: method, url: Regexp.escape(endpoint) }
+    when Regexp
+      raise "Cannot use method :auto when passing regex endpoint" if method == :auto
+      { method: method, url: Regexp.escape(endpoint) }
+    end
+  end
+
+  def _bearcat_resolve_prefix(prefix)
+    if prefix == true
+      prefix = canvas_api_client if defined? canvas_api_client
+      prefix = canvas_sync_client if defined? canvas_sync_client
+    end
+    prefix = case prefix
+    when nil
+      ''
+    when false
+      ''
+    when true
+      raise "stub_bearcat() prefix: set to true, but neither canvas_(sync|api)_client are defined"
+    when Bearcat::Client
+      prefix.prefix
+    when String
+      prefix
+    end
+  end
+end

data/lib/bearcat/version.rb

@@ -1,3 +1,3 @@
 module Bearcat
-  VERSION = '1.0.0' unless defined?(Bearcat::VERSION)
+  VERSION = '1.5.24' unless defined?(Bearcat::VERSION)
 end

data/lib/bearcat.rb

@@ -1,2 +1,51 @@
 require 'bearcat/version'
 require 'bearcat/client'
+require 'bearcat/redis_connection'
+
+module Bearcat
+  class << self
+    require 'logger'
+    attr_accessor :master_rate_limit
+    attr_writer :rate_limit_min, :rate_limits, :max_sleep_seconds,
+                :min_sleep_seconds, :logger, :rate_limiter
+
+    def configure
+      yield self if block_given?
+    end
+
+    def rate_limiter
+      @rate_limiter
+    end
+
+    def rate_limit_min
+      @rate_limit_min ||= 175
+    end
+
+    def min_sleep_seconds
+      @min_sleep_seconds ||= 5
+    end
+
+    def max_sleep_seconds
+      @max_sleep_seconds ||= 60
+    end
+
+    def logger
+      return @logger if defined? @logger
+      @logger = Logger.new(STDOUT)
+      @logger.level = Logger::DEBUG
+      @logger
+    end
+
+    def redis_pool
+      require 'bearcat/redis_connection'
+      @redis_pool ||= Bearcat::RedisConnection.create(env_prefix: "BEARCAT")
+    end
+
+    def redis
+      raise ArgumentError, "requires a block" unless block_given?
+      redis_pool.with do |conn|
+        yield conn
+      end
+    end
+  end
+end

data/lib/catalogcat/api_array.rb

@@ -0,0 +1,22 @@
+require 'bearcat/api_array'
+
+module Catalogcat
+  class ApiArray < Bearcat::ApiArray
+    def self.array_key(response)
+      key = nil
+      if response.env[:method] == :get
+        path = response.env[:url].path
+        key = 'courses' if path =~ %r{.*/courses}
+        key = 'course' if path =~ %r{.*/courses/[0-9]*}
+        key = 'catalogs' if path =~ %r{.*/catalogs}
+        key = 'enrollments' if path =~ %r{.*/enrollments}
+        key = 'order' if path =~ %r{.*/order/[0-9]*}
+        key = 'orders' if path =~ %r{.*/orders}
+        key = 'completed_certificates' if path =~ %r{.*/completed_certificates}
+        key = 'user_registrations' if path =~ %r{.*/user_registrations}
+        key = 'email_domain_sets' if path =~ %r{.*/email_domain_sets}
+      end
+      key.present? ? key : super(response)
+    end
+  end
+end