bearcat 1.4.12 → 1.5.0.beta1

data/lib/bearcat/rate_limiting/redis_script.rb

@@ -0,0 +1,164 @@
+require 'pathname'
+require 'digest/sha1'
+require 'erb'
+
+# Modified from https://github.com/Shopify/wolverine/blob/master/lib/wolverine/script.rb
+
+module Bearcat
+  module RateLimiting
+    # {RedisScript} represents a lua script in the filesystem. It loads the script
+    # from disk and handles talking to redis to execute it. Error handling
+    # is handled by {LuaError}.
+    class RedisScript
+
+      # Loads the script file from disk and calculates its +SHA1+ sum.
+      #
+      # @param file [Pathname] the full path to the indicated file
+      def initialize(file)
+        @file = Pathname.new(file)
+      end
+
+      # Passes the script and supplied arguments to redis for evaulation.
+      # It first attempts to use a script redis has already cached by using
+      # the +EVALSHA+ command, but falls back to providing the full script
+      # text via +EVAL+ if redis has not seen this script before. Future
+      # invocations will then use +EVALSHA+ without erroring.
+      #
+      # @param redis [Redis] the redis connection to run against
+      # @param args [*Objects] the arguments to the script
+      # @return [Object] the value passed back by redis after script execution
+      # @raise [LuaError] if the script failed to compile of encountered a
+      #   runtime error
+      def call(redis, *args)
+        t = Time.now
+        begin
+          redis.evalsha(digest, *args)
+        rescue => e
+          e.message =~ /NOSCRIPT/ ? redis.eval(content, *args) : raise
+        end
+      rescue => e
+        if LuaError.intercepts?(e)
+          raise LuaError.new(e, @file, content)
+        else
+          raise
+        end
+      end
+
+      def content
+        @content ||= load_lua(@file)
+      end
+
+      def digest
+        @digest ||= Digest::SHA1.hexdigest content
+      end
+
+      private
+
+      def script_path
+        @file.basename
+        # Rails.root + 'app/redis_lua'
+      end
+
+      def relative_path
+        @path ||= @file.relative_path_from(script_path)
+      end
+
+      def load_lua(file)
+        TemplateContext.new(script_path).template(script_path + file)
+      end
+
+      class TemplateContext
+        def initialize(script_path)
+          @script_path = script_path
+        end
+
+        def template(pathname)
+          @partial_templates ||= {}
+          ERB.new(File.read(pathname)).result binding
+        end
+
+        # helper method to include a lua partial within another lua script
+        #
+        # @param relative_path [String] the relative path to the script from
+        #     `script_path`
+        def include_partial(relative_path)
+          unless @partial_templates.has_key? relative_path
+            @partial_templates[relative_path] = nil
+            template( Pathname.new("#{@script_path}/#{relative_path}") )
+          end
+        end
+      end
+
+      # Reformats errors raised by redis representing failures while executing
+      # a lua script. The default errors have confusing messages and backtraces,
+      # and a type of +RuntimeError+. This class improves the message and
+      # modifies the backtrace to include the lua script itself in a reasonable
+      # way.
+      class LuaError < StandardError
+        PATTERN = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/
+        WOLVERINE_LIB_PATH = File.expand_path('../../', __FILE__)
+        CONTEXT_LINE_NUMBER = 2
+
+        attr_reader :error, :file, :content
+
+        # Is this error one that should be reformatted?
+        #
+        # @param error [StandardError] the original error raised by redis
+        # @return [Boolean] is this an error that should be reformatted?
+        def self.intercepts? error
+          error.message =~ PATTERN
+        end
+
+        # Initialize a new {LuaError} from an existing redis error, adjusting
+        # the message and backtrace in the process.
+        #
+        # @param error [StandardError] the original error raised by redis
+        # @param file [Pathname] full path to the lua file the error ocurred in
+        # @param content [String] lua file content the error ocurred in
+        def initialize error, file, content
+          @error = error
+          @file = file
+          @content = content
+
+          @error.message =~ PATTERN
+          _stage, line_number, message = $1, $2, $3
+          error_context = generate_error_context(content, line_number.to_i)
+
+          super "#{message}\n\n#{error_context}\n\n"
+          set_backtrace generate_backtrace file, line_number
+        end
+
+        private
+
+        def generate_error_context(content, line_number)
+          lines = content.lines.to_a
+          beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
+          ending_line_number = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
+          line_number_width = ending_line_number.to_s.length
+
+          (beginning_line_number..ending_line_number).map do |number|
+            indicator = number == line_number ? '=>' : '  '
+            formatted_number = "%#{line_number_width}d" % number
+            " #{indicator} #{formatted_number}: #{lines[number - 1]}"
+          end.join.chomp
+        end
+
+        def generate_backtrace(file, line_number)
+          pre_wolverine = backtrace_before_entering_wolverine(@error.backtrace)
+          index_of_first_wolverine_line = (@error.backtrace.size - pre_wolverine.size - 1)
+          pre_wolverine.unshift(@error.backtrace[index_of_first_wolverine_line])
+          pre_wolverine.unshift("#{file}:#{line_number}")
+          pre_wolverine
+        end
+
+        def backtrace_before_entering_wolverine(backtrace)
+          backtrace.reverse.take_while { |line| ! line_from_wolverine(line) }.reverse
+        end
+
+        def line_from_wolverine(line)
+          line.split(':').first.include?(WOLVERINE_LIB_PATH)
+        end
+      end
+    end
+  end
+end

data/lib/bearcat/rate_limiting.rb

@@ -0,0 +1,70 @@
+require 'digest/sha1'
+require_relative 'rate_limiting/redis_script'
+
+module Bearcat
+  module RateLimiting
+    LEAK_RATE = 10
+    GUESS_COST = 60
+    MAXIMUM = 600
+
+    class RateLimiter
+      attr_reader :namespace
+
+      def initialize(namespace: "bearcat", **kwargs)
+        @namespace = namespace
+        @params = kwargs
+      end
+
+      def apply(key, guess: GUESS_COST, on_sleep: nil, max_sleep: 15)
+        current = increment("#{key}", 0, guess)
+        cost = guess
+        remaining = MAXIMUM - current[:count]
+
+        if remaining < 0
+          tts = -remaining / LEAK_RATE
+          tts = [tts, max_sleep].min if max_sleep
+          on_sleep.call(tts) if on_sleep
+          sleep tts
+        end
+
+        cost = yield
+      ensure
+        increment(key, (cost || 0), -guess)
+      end
+    end
+
+    class RedisLimiter < RateLimiter
+      INCR = RedisScript.new(Pathname.new(__FILE__) + "../rate_limiting/increment_bucket.lua")
+
+      def increment(key, amount, pending_amount = 0)
+        ts = Time.now.to_f
+
+        redis do |r|
+          actual, timestamp = INCR.call(r, ["#{key}|reported"], [amount, ts, LEAK_RATE, MAXIMUM + 300])
+          {
+            count: actual.to_f,
+            timestamp: timestamp.to_f,
+          }
+        end
+      end
+
+      def checkin_known(key, remaining_time)
+        redis do |r|
+          r.mapped_hmset("#{key}|reported", {
+            count: MAXIMUM + 200 - remaining_time,
+            timestamp: Time.now.to_f,
+          })
+        end
+      end
+
+      def redis(&blk)
+        # TODO Add bearcat-native ConnectionPool instead of relying on application or Sidekiq to do so
+        if @params[:redis]
+          @params[:redis].call(blk)
+        else
+          ::Sidekiq.redis(&blk)
+        end
+      end
+    end
+  end
+end

data/lib/bearcat/spec_helpers.rb

@@ -16,36 +16,10 @@ module Bearcat::SpecHelpers
   # Accepts optional keyword parameters to interpolate specific values into the URL.
   # Returns a mostly-normal Webmock stub (:to_return has been overridden to allow :body to be set to a Hash)
   def stub_bearcat(endpoint, prefix: nil, method: :auto, **kwargs)
-    url = case endpoint
-    when Symbol
-      ruby_method = Bearcat::Client.instance_method(endpoint)
-      match = SOURCE_REGEX.match(ruby_method.source)
-      bits = []
-      url = match[:url].gsub(/\/$/, '')
-      lend = 0
-      url.scan(/#\{(?<key>.*?)\}/) do |key|
-        m = Regexp.last_match
-        between = url[lend..m.begin(0)-1]
-        bits << between if between.present? && (m.begin(0) > lend)
-        lend = m.end(0)
-        bits << (kwargs[m[:key].to_sym] || /\w+/)
-      end
-      between = url[lend..-1]
-      bits << between if between.present?
-
-      bits.map do |bit|
-        next bit.source if bit.is_a?(Regexp)
-        bit = bit.canvas_id if bit.respond_to?(:canvas_id)
-        Regexp.escape(bit.to_s)
-      end.join
-    when String
-      Regexp.escape(endpoint)
-    when Regexp
-      endpoint.source
-    end
+    cfg = _bearcat_resolve_config(endpoint, method: method)
 
-    url = Regexp.escape(resolve_prefix(prefix)) + url
-    stub = stub_request(method == :auto ? (match ? match[:method].to_sym : :get) : method, Regexp.new(url))
+    url = Regexp.escape(_bearcat_resolve_prefix(prefix)) + cfg[:url]
+    stub = stub_request(cfg[:method], Regexp.new(url))
 
     # Override the to_return method to accept a Hash as body:
     stub.define_singleton_method(:to_return, ->(*resps, &blk) {
@@ -72,7 +46,64 @@ module Bearcat::SpecHelpers
 
   private
 
-  def resolve_prefix(prefix)
+  def _bearcat_resolve_config(endpoint, url_context, method: :auto)
+    case endpoint
+    when Symbol
+      if (cfg = Bearcat::Client.registered_endpoints[endpoint]).present?
+        bits = []
+        url = cfg[:url]
+        lend = 0
+        url.scan(/:(?<key>\w+)/) do |key|
+          m = Regexp.last_match
+          between = url[lend..m.begin(0)-1]
+          bits << between if between.present? && (m.begin(0) > lend)
+          lend = m.end(0)
+          bits << (url_context[m[:key].to_sym] || /\w+/)
+        end
+        between = url[lend..-1]
+        bits << between if between.present?
+
+        url = bits.map do |bit|
+          next bit.source if bit.is_a?(Regexp)
+          bit = bit.canvas_id if bit.respond_to?(:canvas_id)
+          Regexp.escape(bit.to_s)
+        end.join
+
+        { method: method == :auto ? cfg[:method] : method, url: url }
+      else
+        ruby_method = Bearcat::Client.instance_method(endpoint)
+        match = SOURCE_REGEX.match(ruby_method.source)
+        bits = []
+        url = match[:url].gsub(/\/$/, '')
+        lend = 0
+        url.scan(/#\{(?<key>.*?)\}/) do |key|
+          m = Regexp.last_match
+          between = url[lend..m.begin(0)-1]
+          bits << between if between.present? && (m.begin(0) > lend)
+          lend = m.end(0)
+          bits << (url_context[m[:key].to_sym] || /\w+/)
+        end
+        between = url[lend..-1]
+        bits << between if between.present?
+
+        url = bits.map do |bit|
+          next bit.source if bit.is_a?(Regexp)
+          bit = bit.canvas_id if bit.respond_to?(:canvas_id)
+          Regexp.escape(bit.to_s)
+        end.join
+
+        { method: method == :auto ? (match ? match[:method].to_sym : :get) : method, url: url }
+      end
+    when String
+      raise "Cannot use method :auto when passing string endpoint" if method == :auto
+      { method: method, url: Regexp.escape(endpoint) }
+    when Regexp
+      raise "Cannot use method :auto when passing regex endpoint" if method == :auto
+      { method: method, url: Regexp.escape(endpoint) }
+    end
+  end
+
+  def _bearcat_resolve_prefix(prefix)
     if prefix == true
       prefix = canvas_api_client if defined? canvas_api_client
       prefix = canvas_sync_client if defined? canvas_sync_client
@@ -90,4 +121,4 @@ module Bearcat::SpecHelpers
       prefix
     end
   end
-end
+end

data/lib/bearcat/version.rb

@@ -1,3 +1,3 @@
 module Bearcat
-  VERSION = '1.4.12' unless defined?(Bearcat::VERSION)
+  VERSION = '1.5.0.beta1' unless defined?(Bearcat::VERSION)
 end

data/lib/bearcat.rb

@@ -4,24 +4,23 @@ require 'bearcat/client'
 module Bearcat
   class << self
     require 'logger'
-    attr_writer :enforce_rate_limits, :rate_limit_min, :rate_limits, :max_sleep_seconds,
-                :min_sleep_seconds, :logger, :master_rate_limit, :master_mutex,
-                :rate_limit_threshold
+    attr_writer :rate_limit_min, :rate_limits, :max_sleep_seconds,
+                :min_sleep_seconds, :logger, :master_rate_limit, :rate_limiter
 
     def configure
       yield self if block_given?
     end
 
-    def rate_limit_min
-      @rate_limit_min ||= 175
+    def rate_limiter
+      @rate_limiter
     end
 
-    def enforce_rate_limits
-      @enforce_rate_limits ||= false
+    def rate_limit_min
+      @rate_limit_min ||= 175
     end
 
-    def rate_limits
-      @rate_limits ||= {}
+    def min_sleep_seconds
+      @min_sleep_seconds ||= 5
     end
 
     def max_sleep_seconds
@@ -32,23 +31,11 @@ module Bearcat
       @master_rate_limit ||= false
     end
 
-    def master_mutex
-      @master_mutex ||= Mutex.new
-    end
-
-    def rate_limit_threshold
-      @rate_limit_threshold ||= 125
-    end
-
     def logger
       return @logger if defined? @logger
       @logger = Logger.new(STDOUT)
       @logger.level = Logger::DEBUG
       @logger
     end
-
-    def min_sleep_seconds
-      @min_sleep_seconds ||= 5
-    end
   end
 end

data/lib/catalogcat/api_array.rb

@@ -0,0 +1,18 @@
+require 'bearcat/api_array'
+
+module Catalogcat
+  class ApiArray < Bearcat::ApiArray
+    def self.array_key(response)
+      key = nil
+      if response.env[:method] == :get
+        path = response.env[:url].path
+        key = 'courses' if path =~ %r{.*/courses}
+        key = 'course' if path =~ %r{.*/courses/[0-9]*}
+        key = 'catalogs' if path =~ %r{.*/catalogs}
+        key = 'enrollments' if path =~ %r{.*/enrollments}
+        key = 'order' if path =~ %r{.*/order/[0-9]*}
+      end
+      key.present? ? key : super(response)
+    end
+  end
+end

data/lib/catalogcat/client/catalogs.rb

@@ -0,0 +1,21 @@
+module Catalogcat
+  class Client < Footrest::Client
+    module Catalogs
+      def list_catalogs(params = {})
+        get('/api/v1/catalogs', params)
+      end
+
+      def applicants(params = {})
+        get('/api/v1/applicants', params)
+      end
+
+      def completed_certificates(params = {})
+        get('/api/v1/completed_certificates', params)
+      end
+
+      def get_order(id, params = {})
+        get("/api/v1/orders/#{id}", params)
+      end
+    end
+  end
+end

data/lib/catalogcat/client/certificates.rb

@@ -0,0 +1,9 @@
+module Catalogcat
+  class Client < Footrest::Client
+    module Courses
+      def create_certificate(params = {})
+        post('/api/v1/certificates', params)
+      end
+    end
+  end
+end

data/lib/catalogcat/client/courses.rb

@@ -0,0 +1,25 @@
+module Catalogcat
+  class Client < Footrest::Client
+    module Courses
+      def list_courses(page = 1, params = {})
+        get("/api/v1/courses?page=#{page}&per_page=100", params)
+      end
+
+      def create_course(params = {})
+        post('/api/v1/courses', params)
+      end
+
+      def update_course(id, params = {})
+        put("/api/v1/courses/#{id}", params)
+      end
+
+      def get_course(id, params = {})
+        get("/api/v1/courses/#{id}", params)
+      end
+
+      def delete_course(course)
+        delete("/api/v1/courses/#{course}")
+      end
+    end
+  end
+end

data/lib/catalogcat/client/orders.rb

@@ -0,0 +1,9 @@
+module Catalogcat
+  class Client < Footrest::Client
+    module Orders
+      def get_order(id, params = {})
+        get("api/v1/orders/#{id}", params)
+      end
+    end
+  end
+end

data/lib/catalogcat/client.rb

@@ -0,0 +1,39 @@
+require 'footrest/client'
+
+module Catalogcat
+  class Client < Footrest::Client
+    require 'catalogcat/api_array' # monkey patch
+
+    Dir[File.join(__dir__, 'client', '*.rb')].each do |mod|
+      mname = File.basename(mod, '.*').camelize
+      require mod
+      include "Catalogcat::Client::#{mname}".constantize
+    end
+
+    # Override Footrest connection to use Token authorization
+    # rubocop:disable Naming/AccessorMethodName
+    def set_connection(config)
+      super
+      connection.builder.insert(Footrest::RaiseFootrestErrors, ExtendedRaiseFootrestErrors)
+      connection.builder.delete(Footrest::RaiseFootrestErrors)
+      connection.headers[:authorization].sub! 'Bearer', 'Token'
+    end
+    # rubocop:enable Naming/AccessorMethodName
+
+    # Override Footrest request for ApiArray support
+    def request(method, &block)
+      response = connection.send(method, &block)
+      raise Footrest::HttpError::ErrorBase, response if response.status >= 400
+
+      Catalogcat::ApiArray.process_response(response, self)
+    end
+  end
+
+  class ExtendedRaiseFootrestErrors < Footrest::RaiseFootrestErrors
+    def on_complete(response)
+      super
+      key = response[:status].to_i
+      raise ERROR_MAP[key.floor(-2)], response if key >= 400
+    end
+  end
+end

data/lib/catalogcat/version.rb

@@ -0,0 +1,3 @@
+module Catalogcat
+  VERSION = 'bearcat' unless defined?(Catalogcat::VERSION)
+end

data/lib/catalogcat.rb

@@ -0,0 +1,14 @@
+require 'catalogcat/client'
+
+module Catalogcat
+  class << self
+    require 'logger'
+
+    def logger
+      return @logger if defined? @logger
+      @logger = Logger.new(STDOUT)
+      @logger.level = Logger::DEBUG
+      @logger
+    end
+  end
+end

data/spec/bearcat/client/canvas_files_spec.rb

@@ -13,8 +13,7 @@ describe Bearcat::Client::CanvasFiles do
     stub_request(:post, "https://upload-url.invalid/").
       to_return(status: 302, headers: {'Location' => 'https://confirm-upload.invalid/confirm?param=true'})
 
-    stub_post(@client, "/confirm").
-      with(:body => {"param" => ["true"]}).to_return(json_response('canvas_files', 'upload_success.json'))
+    stub_get(@client, '/confirm').with(query: { param: true }).to_return(json_response('canvas_files', 'upload_success.json'))
 
     response = @client.upload_file('my/upload/path', fixture('bearcat.jpg'))
     expect(response['id']).to eq 123

data/spec/bearcat/client/content_migrations_spec.rb

@@ -7,12 +7,12 @@ describe Bearcat::Client::ContentMigrations do
 
   it 'uploads a file' do
     stub_post(@client, "/my/upload/path").
-    to_return(json_response('content_migration_files/response.json'))
+      to_return(json_response('content_migration_files/response.json'))
 
     stub_request(:post, "http://host/files_api").
       to_return(status: 302, headers: {'Location' => 'https://confirm-upload.invalid/confirm?param=true'})
 
-    stub_post(@client, '/confirm').to_return(json_response('content_migration_files/upload_success.json'))
+    stub_get(@client, '/confirm').with(query: { param: true }).to_return(json_response('content_migration_files/upload_success.json'))
 
     opts = {migration_type: 'canvas_cartridge_importer', pre_attachment: {name: 'cc.imscc', size: '2034'}}
     response = @client.upload_content_package('my/upload/path', fixture('cc.imscc'), opts)

data/spec/bearcat/client/courses_spec.rb

@@ -49,8 +49,7 @@ describe Bearcat::Client::Sections do
     stub_request(:post, "http://host/files_api").
       to_return(status: 302, headers: {'Location' => 'https://confirm-upload.invalid/confirm?param=true'})
 
-    stub_post(@client, "/confirm").
-      with(:body => {"param" => ["true"]}).to_return(json_response('content_migration_files', 'upload_success.json'))
+    stub_get(@client, '/confirm').with(query: { param: true }).to_return(json_response('content_migration_files', 'upload_success.json'))
 
     opts = {'migration_type' => 'canvas_cartridge_importer', 'pre_attachment[name]' => 'cc.imscc', 'pre_attachment[size]' => '2034'}
     response = @client.create_content_migration('659', fixture('cc.imscc'), opts)
@@ -66,8 +65,7 @@ describe Bearcat::Client::Sections do
       stub_request(:post, "http://host/files_api").
         to_return(status: 302, headers: {'Location' => 'https://confirm-upload.invalid/confirm?param=true'})
 
-      stub_post(@client, "/confirm").
-        with(:body => {"param" => ["true"]}).to_return(json_response('content_migration_files', 'upload_success.json'))
+      stub_get(@client, '/confirm').with(query: { param: true }).to_return(json_response('content_migration_files', 'upload_success.json'))
 
       opts = {'migration_type' => 'canvas_cartridge_importer', 'pre_attachment[name]' => 'cc.imscc', 'pre_attachment[size]' => '2034'}
       content_migration_response, file_upload_response = @client.create_content_migration_with_both_responses('659', fixture('cc.imscc'), opts)

data/spec/bearcat/client/learning_outcomes_spec.rb

@@ -8,7 +8,7 @@ describe Bearcat::Client::LearningOutcomes do
   end
 
   it "returns an individual learning outcome" do
-    stub_get(@client, "/api/v1/outcomes/1").to_return(json_response("learning_outcomes.json"))
+    stub_get(@client, "/api/v1/outcomes/1").to_return(json_response("learning_outcome.json"))
     outcome = @client.learning_outcome(1)
     outcome["id"].should == 1
     outcome["context_type"].should == "Course"
@@ -16,7 +16,7 @@ describe Bearcat::Client::LearningOutcomes do
   end
 
   it "updates an individual learning outcome" do
-    stub_put(@client, "/api/v1/outcomes/1").to_return(json_response("learning_outcomes.json"))
+    stub_put(@client, "/api/v1/outcomes/1").to_return(json_response("learning_outcome.json"))
     outcome = @client.update_learning_outcome(1)
     outcome["id"].should == 1
     outcome["context_type"].should == "Course"

data/spec/bearcat/client/submissions_spec.rb

@@ -45,6 +45,13 @@ describe Bearcat::Client::Submissions do
       response['id'].should == 8444510
     end
 
+    it "submits multiple files" do
+      @client.stub(:upload_file).and_return({})
+      stub_post(@client, '/api/v1/courses/1/assignments/2/submissions').to_return(json_response('submissions', 'submission.json'))
+      response = @client.course_file_upload_submission(1, 2, 3, [fixture('bearcat.jpg'), fixture('file.csv')])
+      response['id'].should == 8444510
+    end
+
     it "updates grades" do
       params = {"grade_data[123]" => "19"}
       stub_post(@client, "/api/v1/courses/1/assignments/1/submissions/update_grades").with(body: {"grade_data"=>["19"]}).to_return(json_response("progress.json"))
@@ -56,6 +63,13 @@ describe Bearcat::Client::Submissions do
 
   context 'section' do
     it "submits a file" do
+      @client.stub(:upload_file).and_return({})
+      stub_post(@client, '/api/v1/sections/1/assignments/2/submissions').to_return(json_response('submissions', 'submission.json'))
+      response = @client.section_file_upload_submission(1, 2, 3, [fixture('bearcat.jpg'), fixture('file.csv')])
+      response['id'].should == 8444510
+    end
+
+    it "submits multiple files" do
       @client.stub(:upload_file).and_return({})
       stub_post(@client, '/api/v1/sections/1/assignments/2/submissions').to_return(json_response('submissions', 'submission.json'))
       response = @client.section_file_upload_submission(1, 2, 3, fixture('bearcat.jpg'))

data/spec/bearcat/client_spec.rb

@@ -1,16 +1,13 @@
 require 'helper'
 
 describe Bearcat::Client do
-
   it "sets the domain" do
     client = Bearcat::Client.new(domain: "http://canvas.instructure.com")
     client.config[:domain].should == "http://canvas.instructure.com"
-
   end
 
   it "sets the authtoken" do
     client = Bearcat::Client.new(token: "test_token")
     client.config[:token].should == "test_token"
   end
-
-end
+end