bearcat 1.4.13 → 1.5.0.beta1

data/lib/bearcat/client/search.rb

@@ -1,11 +1,9 @@
 module Bearcat
   class Client < Footrest::Client
     module Search
+      extend ClientModule
 
-      def find_recipients(params={})
-        get('/api/v1/conversations/find_recipients', params)
-      end
-
+      get :find_recipients, '/api/v1/conversations/find_recipients'
     end
   end
 end

data/lib/bearcat/client/sections.rb

@@ -1,35 +1,20 @@
 module Bearcat
   class Client < Footrest::Client
     module Sections
-
-      def course_sections(course, params={})
-        get("/api/v1/courses/#{course.to_s}/sections", params)
-      end
-
-      def section(section)
-        get("/api/v1/sections/#{section.to_s}")
-      end
-
-      def create_section(course, params)
-        post("/api/v1/courses/#{course.to_s}/sections", params)
-      end
-
-      def update_section(section, params)
-        put("/api/v1/sections/#{section.to_s}", params)
-      end
-
-      def delete_section(section)
-        delete("/api/v1/sections/#{section.to_s}")
+      extend ClientModule
+
+      prefix "/api/v1/sections/:section/" do
+        get :section
+        put :update_section
+        delete :delete_section
+        post :crosslist_section, "crosslist/:new_course_id"
+        delete :decrosslist_section, "crosslist"
       end
 
-      def crosslist_section(section_id, new_course_id)
-        post("/api/v1/sections/#{section_id.to_s}/crosslist/#{new_course_id.to_s}")
+      prefix "/api/v1/courses/:course/sections/" do
+        get :course_sections
+        post :create_section
       end
-
-      def decrosslist_section(section_id)
-        delete("/api/v1/sections/#{section_id.to_s}/crosslist")
-      end
-
     end
   end
 end

data/lib/bearcat/client/submissions.rb

@@ -1,37 +1,23 @@
 module Bearcat
   class Client < Footrest::Client
     module Submissions
+      extend ClientModule
 
-      def course_submissions(course, params={})
-        get("/api/v1/courses/#{course.to_s}/students/submissions", params)
+      prefix "/api/v1/courses/:course/assignments/:assignment/submissions/" do
+        get :get_course_submissions
+        get :user_course_assignment_submission, ":user"
       end
 
-      def section_submissions(section, params={})
-        get("/api/v1/sections/#{section.to_s}/students/submissions", params)
-      end
-
-      def get_course_submissions(course, assignment, params={})
-        get("/api/v1/courses/#{course.to_s}/assignments/#{assignment}/submissions", params)
-      end
-
-      def user_course_assignment_submission(course, assignment, user, params={})
-        get("/api/v1/courses/#{course.to_s}/assignments/#{assignment.to_s}/submissions/#{user.to_s}", params)
-      end
-
-      def course_submission(course, assignment, params)
-        post("/api/v1/courses/#{course}/assignments/#{assignment}/submissions", params)
-      end
-
-      def section_submission(section, assignment, params)
-        post("/api/v1/sections/#{section}/assignments/#{assignment}/submissions", params)
-      end
-
-      def grade_course_submission(course, assignment, user, params)
-        put("/api/v1/courses/#{course}/assignments/#{assignment}/submissions/#{user}", params)
-      end
+      context_types %i[course section] do |ct|
+        prefix "/api/v1/#{ct}s/:#{ct}/" do
+          get :"#{ct}_submissions", "students/submissions"
 
-      def grade_section_submission(section, assignment, user, params)
-        put("/api/v1/sections/#{section}/assignments/#{assignment}/submissions/#{user}", params)
+          prefix "assignments/:assignment/submissions/" do
+            post :"#{ct}_submission"
+            put :"grade_#{ct}_submission", ":user"
+            post :"#{ct}_update_grades", "update_grades"
+          end
+        end
       end
 
       def course_file_upload_submission(course, assignment, user, file_data, params={})
@@ -42,14 +28,6 @@ module Bearcat
         file_upload_submission(section, assignment, user, file_data, params, type: :section)
       end
 
-      def course_update_grades(course, assignment, params={})
-        post("/api/v1/courses/#{course}/assignments/#{assignment}/submissions/update_grades", params)
-      end
-
-      def section_update_grades(section, assignment, params={})
-        post("/api/v1/sections/#{section}/assignments/#{assignment}/submissions/update_grades", params)
-      end
-
       protected
 
       # @param file_data One of an array of file_path strings or an array of Hashes, each being upload file params plus the file's path
@@ -86,7 +64,6 @@ module Bearcat
         params[:submission] = sub_params
         send("#{type}_submission".to_sym, type_id, assignment, params)
       end
-
     end
   end
 end

data/lib/bearcat/client/tabs.rb

@@ -1,15 +1,12 @@
 module Bearcat
   class Client < Footrest::Client
     module Tabs
+      extend ClientModule
 
-      def get_tabs(course, params={})
-        get("/api/v1/courses/#{course}/tabs", params)
+      prefix "/api/v1/courses/:course/tabs/" do
+        get :get_tabs
+        put :update_tab, ":tab"
       end
-
-      def update_tab(id, course, params={})
-        put("/api/v1/courses/#{course}/tabs/#{id}", params)
-      end
-
     end
   end
 end

data/lib/bearcat/client/users.rb

@@ -1,33 +1,26 @@
 module Bearcat
   class Client < Footrest::Client
     module Users
+      extend ClientModule
 
-      def list_users(account, params={})
-        get("/api/v1/accounts/#{account.to_s}/users", params)
+      prefix "/api/v1/accounts/:account/users/" do
+        get :list_users
+        post :add_user
       end
 
-      def user_avatars(user, params={})
-        get("/api/v1/users/#{user.to_s}/avatars", params)
-      end
-
-      def add_user(account, params={})
-        post("/api/v1/accounts/#{account.to_s}/users", params)
-      end
-
-      def user_detail(user, params={})
-        get("/api/v1/users/#{user.to_s}", params)
-      end
-
-      def user_profile(user, params={})
-        get("/api/v1/users/#{user.to_s}/profile", params)
-      end
-
-      def user_logins(user, params={})
-        get("/api/v1/users/#{user.to_s}/logins", params)
-      end
-
-      def communication_channels(user, params={})
-        get("/api/v1/users/#{user.to_s}/communication_channels", params)
+      prefix "/api/v1/users/" do
+        prefix ":user/" do
+          get :user_detail
+          get :user_avatars, "avatars"
+          get :user_profile, "profile"
+          get :user_logins, "logins"
+          get :communication_channels, "communication_channels"
+          get :page_views, "page_views"
+          put :user_merge, "merge_into/:merge_into_user"
+          get :user_assignments, "courses/:course/assignments"
+          get :dashboard_positions, "dashboard_positions"
+          put :update_dashboard_positions, "dashboard_positions"
+        end
       end
 
       # scope: food
@@ -52,26 +45,6 @@ module Bearcat
         delete("/api/v1/users/#{user}/custom_data/#{scope}", params)
       end
 
-      def page_views(user, params = {})
-        get("/api/v1/users/#{user}/page_views", params)
-      end
-
-      def user_merge(user, merge_into_user)
-        put("/api/v1/users/#{user}/merge_into/#{merge_into_user}")
-      end
-
-      def user_assignments(user, course, params = {})
-        get("/api/v1/users/#{user}/courses/#{course}/assignments", params)
-      end
-
-      def dashboard_positions(user, params = {})
-        get("/api/v1/users/#{user}/dashboard_positions", params)
-      end
-
-      def update_dashboard_positions(user, params = {})
-        put("/api/v1/users/#{user}/dashboard_positions", params)
-      end
-
       def favorite_courses(user, params = {})
         params.merge!({as_user_id: user})
         get("/api/v1/users/self/favorites/courses", params)

data/lib/bearcat/client.rb

@@ -1,76 +1,104 @@
 require 'active_support/core_ext/hash'
 require 'footrest/client'
-require 'paul_walker'
+require_relative 'rate_limiting'
+require_relative 'client_module'
 
 module Bearcat
   class Client < Footrest::Client
     require 'bearcat/api_array'
 
+    @added_modules = []
+
     Dir[File.join(__dir__, 'client', '*.rb')].each do |mod|
       mname = File.basename(mod, '.*').camelize
       mname = 'GraphQL' if mname == 'GraphQl'
       require mod
-      include "Bearcat::Client::#{mname}".constantize
+      lmod = "Bearcat::Client::#{mname}".constantize
+      include lmod
+      @added_modules << lmod
+    end
+
+    def self.registered_endpoints
+      @registered_endpoints ||= @added_modules.reduce({}) do |h, m|
+        h.merge!(m._registered_endpoints) rescue h
+      end
+      @registered_endpoints
     end
 
-    # Override Footrest request for ApiArray support
     def request(method, &block)
-      enforce_rate_limits
-      response = connection.send(method, &block)
-      apply_rate_limits(response.headers['x-rate-limit-remaining'])
+      response = rate_limited_request do
+        connection.send(method, &block)
+      end
       ApiArray.process_response(response, self)
     end
 
-    def enforce_rate_limits
-      return unless limit_remaining.present?
-      return unless limit_remaining < Bearcat.rate_limit_min
+    protected
 
-      tts = ((Bearcat.rate_limit_min - limit_remaining) / 5).ceil
-      tts = Bearcat.min_sleep_seconds if tts < Bearcat.min_sleep_seconds
-      tts = Bearcat.max_sleep_seconds if tts > Bearcat.max_sleep_seconds
+    def rate_limited_request
+      return yield unless rate_limiter
 
+      canvas_rate_limits= 0
+      response = nil
 
-      message = "Canvas API rate limit minimum #{Bearcat.rate_limit_min} reached. "\
-        "Sleeping for #{tts} second(s) to catch up ~zzZZ~. "\
-        "Limit Remaining: #{limit_remaining}"
-      Bearcat.logger.debug(message)
+      begin
+        rate_limiter.apply(
+          rate_limit_key,
+          max_sleep: Bearcat.max_sleep_seconds,
+          on_sleep: ->(tts) {
+            message = "Canvas API rate limit reached; sleeping for #{tts.to_i} second(s) to catch up."
+            Bearcat.logger.debug(message)
+          },
+        ) do
+          response = yield
+          0
+        end
+      rescue Footrest::HttpError::Forbidden => err
+        # Somehow our rate-limiting didn't limit enough and Canvas stopped us.
+        response = err.response
+        if canvas_rate_limits < 2 && err.message.include?("(Rate Limit Exceeded)")
+          canvas_rate_limits += 1
+          rate_limiter.checkin_known(rate_limit_key, 0)
 
-      sleep(tts)
-    end
+          message = "Canvas API applied rate limit; upticking Bearcat rate-limit avoidance and retrying (Retry #{canvas_rate_limits})."
+          Bearcat.logger.debug(message)
 
-    def apply_rate_limits(limit)
-      return if limit.nil?
-      self.limit_remaining = limit.to_i
-    end
+          retry
+        end
+        raise
+      ensure
+        headers = response.try(:response_headers) || response.try(:headers) || {}
+        # -50 to provide a little extra leeway and hopefully be more proactive, making sure we don't even get close to Canvas throwing a 403, even if an out-of-band process is involved
+        rate_limiter.checkin_known(rate_limit_key, headers['x-rate-limit-remaining'].to_f - 100) if response
+      end
 
-    def using_master_rate_limit?
-      config[:master_rate_limit].present? || Bearcat.master_rate_limit.present?
+      response
     end
 
-    def limit_remaining
-      if using_master_rate_limit?
-        Bearcat.master_mutex.synchronize do
-          limit = PaulWalker::RateLimit.get(config[:token], config[:token])
-          if limit.nil?
-            PaulWalker::RateLimit.add(config[:token], config[:token], 0, Bearcat::rate_limit_min)
-            limit = { current: 0 }.with_indifferent_access
-          end
-          Bearcat.logger.debug limit['current'].to_s
-          limit['current']
+    def rate_limiter
+      @rate_limiter ||= begin
+        rl = config[:rate_limiter] || Bearcat.rate_limiter
+        master_rate_limit = config[:master_rate_limit].present? ? config[:master_rate_limit] : Bearcat.master_rate_limit
+
+        if rl.nil? && master_rate_limit.nil? && defined?(Rails) && Rails.env.production?
+          master_rate_limit = true if defined?(::Sidekiq)
         end
-      else
-        Bearcat.rate_limits[config[:token]]
-      end
-    end
 
-    def limit_remaining=(value)
-      if using_master_rate_limit?
-        Bearcat.master_mutex.synchronize do
-          PaulWalker::RateLimit.add(config[:token], config[:token], value, Bearcat::rate_limit_min)
+        if rl.nil? && master_rate_limit
+          rl = RateLimiting::RedisLimiter
+        end
+
+        if rl.is_a?(Class)
+          rl.new()
+        elsif rl.present?
+          rl
         end
-      else
-        Bearcat.rate_limits[config[:token]] = value
       end
     end
+
+    private
+
+    def rate_limit_key
+      Digest::SHA1.hexdigest(config[:token])
+    end
   end
 end

data/lib/bearcat/client_module.rb

@@ -0,0 +1,101 @@
+module Bearcat
+  class Client < Footrest::Client
+    module ClientModule
+      ARG_REGEX = /:(\w+)/
+      attr_reader :_registered_endpoints
+
+      def prefix(prefix)
+        past_prefix = @current_prefix
+        @current_prefix = (past_prefix || '') + prefix
+        yield
+      ensure
+        @current_prefix = past_prefix
+      end
+
+      def endpoint(method, identifier, url = "", defaults: {}, &blk)
+        if @current_prefix && !url.start_with?('/')
+          url = url[2..] if url.start_with?('./')
+          url = @current_prefix + url
+        end
+
+        args = url.to_enum(:scan, ARG_REGEX).map { Regexp.last_match }
+        arg_names = args.map{|m| m[1]}
+
+        @_registered_endpoints ||= {}
+        @_registered_endpoints[identifier] = { symbol: identifier, method: method, url: url }
+
+        # TODO: Consider generating the method using class_eval and a template - this will improve runtime performance
+        # signature_bits = []
+        # logical_bits = []
+        # args.each do |m|
+        #   name = [1]
+        # end
+
+        # interpolated_url = url.gsub(ARG_REGEX) do |m|
+        #   '#{' + m[1] + '}'
+        # end
+
+        # class_eval <<~RUBY
+        #   def #{identifier}(*args, **kwargs)
+        #     parameters = {
+
+        #     }
+
+        #     #{method}(#{interpolated_url})
+        #   end
+        # RUBY
+
+        define_method(identifier) do |*args, **kwargs|
+          url_arguments = {}
+          parameters = { }.with_indifferent_access
+          parameters.merge!(defaults)
+
+          args.each_with_index do |v, i|
+            if arg_names[i]
+              url_arguments[arg_names[i]] = v
+            elsif i == arg_names.count && v.is_a?(Hash)
+              parameters.merge!(v)
+            else
+              raise ArgumentError, "Too many arguments passed to #{identifier}" unless arg_names[i]
+            end
+          end
+
+          parameters.merge!(kwargs)
+
+          preq = BearcatRequest.new(url, url_arguments, parameters)
+          yield preq if block_given?
+
+          arg_names.each do |an|
+            preq.arguments[an] = preq.parameters.delete(an) unless preq.arguments.key?(an)
+            raise ArgumentError, "Missing argument #{an}" unless preq.arguments.key?(an)
+          end
+
+          send(method, preq.interpoated_url, preq.parameters)
+        end
+      end
+
+      def context_types(types, &blk)
+        types.each(&blk)
+      end
+
+      %i[get post delete put patch head].each do |mthd|
+        define_method(mthd) do |*args, **kwargs, &blk|
+          endpoint(mthd, *args, **kwargs, &blk)
+        end
+      end
+
+      BearcatRequest = Struct.new(:url, :arguments, :parameters) do
+        def interpoated_url
+          url.gsub(ARG_REGEX) do |_|
+            m = Regexp.last_match
+            val = arguments[m[1]]
+            val = val.canvas_id if val.respond_to?(:canvas_id)
+            val = val.id if val.respond_to?(:id)
+            val = val['id'].presence || val[:id].presence || val if val.is_a?(Hash)
+            val
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bearcat/rate_limiting/redis_script.rb

@@ -0,0 +1,164 @@
+require 'pathname'
+require 'digest/sha1'
+require 'erb'
+
+# Modified from https://github.com/Shopify/wolverine/blob/master/lib/wolverine/script.rb
+
+module Bearcat
+  module RateLimiting
+    # {RedisScript} represents a lua script in the filesystem. It loads the script
+    # from disk and handles talking to redis to execute it. Error handling
+    # is handled by {LuaError}.
+    class RedisScript
+
+      # Loads the script file from disk and calculates its +SHA1+ sum.
+      #
+      # @param file [Pathname] the full path to the indicated file
+      def initialize(file)
+        @file = Pathname.new(file)
+      end
+
+      # Passes the script and supplied arguments to redis for evaulation.
+      # It first attempts to use a script redis has already cached by using
+      # the +EVALSHA+ command, but falls back to providing the full script
+      # text via +EVAL+ if redis has not seen this script before. Future
+      # invocations will then use +EVALSHA+ without erroring.
+      #
+      # @param redis [Redis] the redis connection to run against
+      # @param args [*Objects] the arguments to the script
+      # @return [Object] the value passed back by redis after script execution
+      # @raise [LuaError] if the script failed to compile of encountered a
+      #   runtime error
+      def call(redis, *args)
+        t = Time.now
+        begin
+          redis.evalsha(digest, *args)
+        rescue => e
+          e.message =~ /NOSCRIPT/ ? redis.eval(content, *args) : raise
+        end
+      rescue => e
+        if LuaError.intercepts?(e)
+          raise LuaError.new(e, @file, content)
+        else
+          raise
+        end
+      end
+
+      def content
+        @content ||= load_lua(@file)
+      end
+
+      def digest
+        @digest ||= Digest::SHA1.hexdigest content
+      end
+
+      private
+
+      def script_path
+        @file.basename
+        # Rails.root + 'app/redis_lua'
+      end
+
+      def relative_path
+        @path ||= @file.relative_path_from(script_path)
+      end
+
+      def load_lua(file)
+        TemplateContext.new(script_path).template(script_path + file)
+      end
+
+      class TemplateContext
+        def initialize(script_path)
+          @script_path = script_path
+        end
+
+        def template(pathname)
+          @partial_templates ||= {}
+          ERB.new(File.read(pathname)).result binding
+        end
+
+        # helper method to include a lua partial within another lua script
+        #
+        # @param relative_path [String] the relative path to the script from
+        #     `script_path`
+        def include_partial(relative_path)
+          unless @partial_templates.has_key? relative_path
+            @partial_templates[relative_path] = nil
+            template( Pathname.new("#{@script_path}/#{relative_path}") )
+          end
+        end
+      end
+
+      # Reformats errors raised by redis representing failures while executing
+      # a lua script. The default errors have confusing messages and backtraces,
+      # and a type of +RuntimeError+. This class improves the message and
+      # modifies the backtrace to include the lua script itself in a reasonable
+      # way.
+      class LuaError < StandardError
+        PATTERN = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/
+        WOLVERINE_LIB_PATH = File.expand_path('../../', __FILE__)
+        CONTEXT_LINE_NUMBER = 2
+
+        attr_reader :error, :file, :content
+
+        # Is this error one that should be reformatted?
+        #
+        # @param error [StandardError] the original error raised by redis
+        # @return [Boolean] is this an error that should be reformatted?
+        def self.intercepts? error
+          error.message =~ PATTERN
+        end
+
+        # Initialize a new {LuaError} from an existing redis error, adjusting
+        # the message and backtrace in the process.
+        #
+        # @param error [StandardError] the original error raised by redis
+        # @param file [Pathname] full path to the lua file the error ocurred in
+        # @param content [String] lua file content the error ocurred in
+        def initialize error, file, content
+          @error = error
+          @file = file
+          @content = content
+
+          @error.message =~ PATTERN
+          _stage, line_number, message = $1, $2, $3
+          error_context = generate_error_context(content, line_number.to_i)
+
+          super "#{message}\n\n#{error_context}\n\n"
+          set_backtrace generate_backtrace file, line_number
+        end
+
+        private
+
+        def generate_error_context(content, line_number)
+          lines = content.lines.to_a
+          beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
+          ending_line_number = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
+          line_number_width = ending_line_number.to_s.length
+
+          (beginning_line_number..ending_line_number).map do |number|
+            indicator = number == line_number ? '=>' : '  '
+            formatted_number = "%#{line_number_width}d" % number
+            " #{indicator} #{formatted_number}: #{lines[number - 1]}"
+          end.join.chomp
+        end
+
+        def generate_backtrace(file, line_number)
+          pre_wolverine = backtrace_before_entering_wolverine(@error.backtrace)
+          index_of_first_wolverine_line = (@error.backtrace.size - pre_wolverine.size - 1)
+          pre_wolverine.unshift(@error.backtrace[index_of_first_wolverine_line])
+          pre_wolverine.unshift("#{file}:#{line_number}")
+          pre_wolverine
+        end
+
+        def backtrace_before_entering_wolverine(backtrace)
+          backtrace.reverse.take_while { |line| ! line_from_wolverine(line) }.reverse
+        end
+
+        def line_from_wolverine(line)
+          line.split(':').first.include?(WOLVERINE_LIB_PATH)
+        end
+      end
+    end
+  end
+end