bearcat 1.4.13 → 1.5.0.beta1

data/lib/bearcat/rate_limiting.rb

@@ -0,0 +1,70 @@
+require 'digest/sha1'
+require_relative 'rate_limiting/redis_script'
+
+module Bearcat
+  module RateLimiting
+    LEAK_RATE = 10
+    GUESS_COST = 60
+    MAXIMUM = 600
+
+    class RateLimiter
+      attr_reader :namespace
+
+      def initialize(namespace: "bearcat", **kwargs)
+        @namespace = namespace
+        @params = kwargs
+      end
+
+      def apply(key, guess: GUESS_COST, on_sleep: nil, max_sleep: 15)
+        current = increment("#{key}", 0, guess)
+        cost = guess
+        remaining = MAXIMUM - current[:count]
+
+        if remaining < 0
+          tts = -remaining / LEAK_RATE
+          tts = [tts, max_sleep].min if max_sleep
+          on_sleep.call(tts) if on_sleep
+          sleep tts
+        end
+
+        cost = yield
+      ensure
+        increment(key, (cost || 0), -guess)
+      end
+    end
+
+    class RedisLimiter < RateLimiter
+      INCR = RedisScript.new(Pathname.new(__FILE__) + "../rate_limiting/increment_bucket.lua")
+
+      def increment(key, amount, pending_amount = 0)
+        ts = Time.now.to_f
+
+        redis do |r|
+          actual, timestamp = INCR.call(r, ["#{key}|reported"], [amount, ts, LEAK_RATE, MAXIMUM + 300])
+          {
+            count: actual.to_f,
+            timestamp: timestamp.to_f,
+          }
+        end
+      end
+
+      def checkin_known(key, remaining_time)
+        redis do |r|
+          r.mapped_hmset("#{key}|reported", {
+            count: MAXIMUM + 200 - remaining_time,
+            timestamp: Time.now.to_f,
+          })
+        end
+      end
+
+      def redis(&blk)
+        # TODO Add bearcat-native ConnectionPool instead of relying on application or Sidekiq to do so
+        if @params[:redis]
+          @params[:redis].call(blk)
+        else
+          ::Sidekiq.redis(&blk)
+        end
+      end
+    end
+  end
+end

data/lib/bearcat/spec_helpers.rb

@@ -16,36 +16,10 @@ module Bearcat::SpecHelpers
   # Accepts optional keyword parameters to interpolate specific values into the URL.
   # Returns a mostly-normal Webmock stub (:to_return has been overridden to allow :body to be set to a Hash)
   def stub_bearcat(endpoint, prefix: nil, method: :auto, **kwargs)
-    url = case endpoint
-    when Symbol
-      ruby_method = Bearcat::Client.instance_method(endpoint)
-      match = SOURCE_REGEX.match(ruby_method.source)
-      bits = []
-      url = match[:url].gsub(/\/$/, '')
-      lend = 0
-      url.scan(/#\{(?<key>.*?)\}/) do |key|
-        m = Regexp.last_match
-        between = url[lend..m.begin(0)-1]
-        bits << between if between.present? && (m.begin(0) > lend)
-        lend = m.end(0)
-        bits << (kwargs[m[:key].to_sym] || /\w+/)
-      end
-      between = url[lend..-1]
-      bits << between if between.present?
-
-      bits.map do |bit|
-        next bit.source if bit.is_a?(Regexp)
-        bit = bit.canvas_id if bit.respond_to?(:canvas_id)
-        Regexp.escape(bit.to_s)
-      end.join
-    when String
-      Regexp.escape(endpoint)
-    when Regexp
-      endpoint.source
-    end
+    cfg = _bearcat_resolve_config(endpoint, method: method)
 
-    url = Regexp.escape(resolve_prefix(prefix)) + url
-    stub = stub_request(method == :auto ? (match ? match[:method].to_sym : :get) : method, Regexp.new(url))
+    url = Regexp.escape(_bearcat_resolve_prefix(prefix)) + cfg[:url]
+    stub = stub_request(cfg[:method], Regexp.new(url))
 
     # Override the to_return method to accept a Hash as body:
     stub.define_singleton_method(:to_return, ->(*resps, &blk) {
@@ -72,7 +46,64 @@ module Bearcat::SpecHelpers
 
   private
 
-  def resolve_prefix(prefix)
+  def _bearcat_resolve_config(endpoint, url_context, method: :auto)
+    case endpoint
+    when Symbol
+      if (cfg = Bearcat::Client.registered_endpoints[endpoint]).present?
+        bits = []
+        url = cfg[:url]
+        lend = 0
+        url.scan(/:(?<key>\w+)/) do |key|
+          m = Regexp.last_match
+          between = url[lend..m.begin(0)-1]
+          bits << between if between.present? && (m.begin(0) > lend)
+          lend = m.end(0)
+          bits << (url_context[m[:key].to_sym] || /\w+/)
+        end
+        between = url[lend..-1]
+        bits << between if between.present?
+
+        url = bits.map do |bit|
+          next bit.source if bit.is_a?(Regexp)
+          bit = bit.canvas_id if bit.respond_to?(:canvas_id)
+          Regexp.escape(bit.to_s)
+        end.join
+
+        { method: method == :auto ? cfg[:method] : method, url: url }
+      else
+        ruby_method = Bearcat::Client.instance_method(endpoint)
+        match = SOURCE_REGEX.match(ruby_method.source)
+        bits = []
+        url = match[:url].gsub(/\/$/, '')
+        lend = 0
+        url.scan(/#\{(?<key>.*?)\}/) do |key|
+          m = Regexp.last_match
+          between = url[lend..m.begin(0)-1]
+          bits << between if between.present? && (m.begin(0) > lend)
+          lend = m.end(0)
+          bits << (url_context[m[:key].to_sym] || /\w+/)
+        end
+        between = url[lend..-1]
+        bits << between if between.present?
+
+        url = bits.map do |bit|
+          next bit.source if bit.is_a?(Regexp)
+          bit = bit.canvas_id if bit.respond_to?(:canvas_id)
+          Regexp.escape(bit.to_s)
+        end.join
+
+        { method: method == :auto ? (match ? match[:method].to_sym : :get) : method, url: url }
+      end
+    when String
+      raise "Cannot use method :auto when passing string endpoint" if method == :auto
+      { method: method, url: Regexp.escape(endpoint) }
+    when Regexp
+      raise "Cannot use method :auto when passing regex endpoint" if method == :auto
+      { method: method, url: Regexp.escape(endpoint) }
+    end
+  end
+
+  def _bearcat_resolve_prefix(prefix)
     if prefix == true
       prefix = canvas_api_client if defined? canvas_api_client
       prefix = canvas_sync_client if defined? canvas_sync_client
@@ -90,4 +121,4 @@ module Bearcat::SpecHelpers
       prefix
     end
   end
-end
+end

data/lib/bearcat/version.rb

@@ -1,3 +1,3 @@
 module Bearcat
-  VERSION = '1.4.13' unless defined?(Bearcat::VERSION)
+  VERSION = '1.5.0.beta1' unless defined?(Bearcat::VERSION)
 end

data/lib/bearcat.rb

@@ -4,24 +4,23 @@ require 'bearcat/client'
 module Bearcat
   class << self
     require 'logger'
-    attr_writer :enforce_rate_limits, :rate_limit_min, :rate_limits, :max_sleep_seconds,
-                :min_sleep_seconds, :logger, :master_rate_limit, :master_mutex,
-                :rate_limit_threshold
+    attr_writer :rate_limit_min, :rate_limits, :max_sleep_seconds,
+                :min_sleep_seconds, :logger, :master_rate_limit, :rate_limiter
 
     def configure
       yield self if block_given?
     end
 
-    def rate_limit_min
-      @rate_limit_min ||= 175
+    def rate_limiter
+      @rate_limiter
     end
 
-    def enforce_rate_limits
-      @enforce_rate_limits ||= false
+    def rate_limit_min
+      @rate_limit_min ||= 175
     end
 
-    def rate_limits
-      @rate_limits ||= {}
+    def min_sleep_seconds
+      @min_sleep_seconds ||= 5
     end
 
     def max_sleep_seconds
@@ -32,23 +31,11 @@ module Bearcat
       @master_rate_limit ||= false
     end
 
-    def master_mutex
-      @master_mutex ||= Mutex.new
-    end
-
-    def rate_limit_threshold
-      @rate_limit_threshold ||= 125
-    end
-
     def logger
       return @logger if defined? @logger
       @logger = Logger.new(STDOUT)
       @logger.level = Logger::DEBUG
       @logger
     end
-
-    def min_sleep_seconds
-      @min_sleep_seconds ||= 5
-    end
   end
 end

data/lib/catalogcat/api_array.rb

@@ -0,0 +1,18 @@
+require 'bearcat/api_array'
+
+module Catalogcat
+  class ApiArray < Bearcat::ApiArray
+    def self.array_key(response)
+      key = nil
+      if response.env[:method] == :get
+        path = response.env[:url].path
+        key = 'courses' if path =~ %r{.*/courses}
+        key = 'course' if path =~ %r{.*/courses/[0-9]*}
+        key = 'catalogs' if path =~ %r{.*/catalogs}
+        key = 'enrollments' if path =~ %r{.*/enrollments}
+        key = 'order' if path =~ %r{.*/order/[0-9]*}
+      end
+      key.present? ? key : super(response)
+    end
+  end
+end

data/lib/catalogcat/client/catalogs.rb

@@ -0,0 +1,21 @@
+module Catalogcat
+  class Client < Footrest::Client
+    module Catalogs
+      def list_catalogs(params = {})
+        get('/api/v1/catalogs', params)
+      end
+
+      def applicants(params = {})
+        get('/api/v1/applicants', params)
+      end
+
+      def completed_certificates(params = {})
+        get('/api/v1/completed_certificates', params)
+      end
+
+      def get_order(id, params = {})
+        get("/api/v1/orders/#{id}", params)
+      end
+    end
+  end
+end

data/lib/catalogcat/client/certificates.rb

@@ -0,0 +1,9 @@
+module Catalogcat
+  class Client < Footrest::Client
+    module Courses
+      def create_certificate(params = {})
+        post('/api/v1/certificates', params)
+      end
+    end
+  end
+end

data/lib/catalogcat/client/courses.rb

@@ -0,0 +1,25 @@
+module Catalogcat
+  class Client < Footrest::Client
+    module Courses
+      def list_courses(page = 1, params = {})
+        get("/api/v1/courses?page=#{page}&per_page=100", params)
+      end
+
+      def create_course(params = {})
+        post('/api/v1/courses', params)
+      end
+
+      def update_course(id, params = {})
+        put("/api/v1/courses/#{id}", params)
+      end
+
+      def get_course(id, params = {})
+        get("/api/v1/courses/#{id}", params)
+      end
+
+      def delete_course(course)
+        delete("/api/v1/courses/#{course}")
+      end
+    end
+  end
+end

data/lib/catalogcat/client/orders.rb

@@ -0,0 +1,9 @@
+module Catalogcat
+  class Client < Footrest::Client
+    module Orders
+      def get_order(id, params = {})
+        get("api/v1/orders/#{id}", params)
+      end
+    end
+  end
+end

data/lib/catalogcat/client.rb

@@ -0,0 +1,39 @@
+require 'footrest/client'
+
+module Catalogcat
+  class Client < Footrest::Client
+    require 'catalogcat/api_array' # monkey patch
+
+    Dir[File.join(__dir__, 'client', '*.rb')].each do |mod|
+      mname = File.basename(mod, '.*').camelize
+      require mod
+      include "Catalogcat::Client::#{mname}".constantize
+    end
+
+    # Override Footrest connection to use Token authorization
+    # rubocop:disable Naming/AccessorMethodName
+    def set_connection(config)
+      super
+      connection.builder.insert(Footrest::RaiseFootrestErrors, ExtendedRaiseFootrestErrors)
+      connection.builder.delete(Footrest::RaiseFootrestErrors)
+      connection.headers[:authorization].sub! 'Bearer', 'Token'
+    end
+    # rubocop:enable Naming/AccessorMethodName
+
+    # Override Footrest request for ApiArray support
+    def request(method, &block)
+      response = connection.send(method, &block)
+      raise Footrest::HttpError::ErrorBase, response if response.status >= 400
+
+      Catalogcat::ApiArray.process_response(response, self)
+    end
+  end
+
+  class ExtendedRaiseFootrestErrors < Footrest::RaiseFootrestErrors
+    def on_complete(response)
+      super
+      key = response[:status].to_i
+      raise ERROR_MAP[key.floor(-2)], response if key >= 400
+    end
+  end
+end

data/lib/catalogcat/version.rb

@@ -0,0 +1,3 @@
+module Catalogcat
+  VERSION = 'bearcat' unless defined?(Catalogcat::VERSION)
+end

data/lib/catalogcat.rb

@@ -0,0 +1,14 @@
+require 'catalogcat/client'
+
+module Catalogcat
+  class << self
+    require 'logger'
+
+    def logger
+      return @logger if defined? @logger
+      @logger = Logger.new(STDOUT)
+      @logger.level = Logger::DEBUG
+      @logger
+    end
+  end
+end

data/spec/bearcat/client/canvas_files_spec.rb

@@ -13,8 +13,7 @@ describe Bearcat::Client::CanvasFiles do
     stub_request(:post, "https://upload-url.invalid/").
       to_return(status: 302, headers: {'Location' => 'https://confirm-upload.invalid/confirm?param=true'})
 
-    stub_get(@client, "/confirm").
-      with(:query => {"param" => ["true"]}).to_return(json_response('canvas_files', 'upload_success.json'))
+    stub_get(@client, '/confirm').with(query: { param: true }).to_return(json_response('canvas_files', 'upload_success.json'))
 
     response = @client.upload_file('my/upload/path', fixture('bearcat.jpg'))
     expect(response['id']).to eq 123

data/spec/bearcat/client/content_migrations_spec.rb

@@ -12,7 +12,7 @@ describe Bearcat::Client::ContentMigrations do
     stub_request(:post, "http://host/files_api").
       to_return(status: 302, headers: {'Location' => 'https://confirm-upload.invalid/confirm?param=true'})
 
-    stub_get(@client, '/confirm').with(:query => {"param" => ["true"]}).to_return(json_response('content_migration_files/upload_success.json'))
+    stub_get(@client, '/confirm').with(query: { param: true }).to_return(json_response('content_migration_files/upload_success.json'))
 
     opts = {migration_type: 'canvas_cartridge_importer', pre_attachment: {name: 'cc.imscc', size: '2034'}}
     response = @client.upload_content_package('my/upload/path', fixture('cc.imscc'), opts)

data/spec/bearcat/client/courses_spec.rb

@@ -49,8 +49,7 @@ describe Bearcat::Client::Sections do
     stub_request(:post, "http://host/files_api").
       to_return(status: 302, headers: {'Location' => 'https://confirm-upload.invalid/confirm?param=true'})
 
-    stub_get(@client, "/confirm").
-      with(:query => {"param" => ["true"]}).to_return(json_response('content_migration_files', 'upload_success.json'))
+    stub_get(@client, '/confirm').with(query: { param: true }).to_return(json_response('content_migration_files', 'upload_success.json'))
 
     opts = {'migration_type' => 'canvas_cartridge_importer', 'pre_attachment[name]' => 'cc.imscc', 'pre_attachment[size]' => '2034'}
     response = @client.create_content_migration('659', fixture('cc.imscc'), opts)
@@ -66,8 +65,7 @@ describe Bearcat::Client::Sections do
       stub_request(:post, "http://host/files_api").
         to_return(status: 302, headers: {'Location' => 'https://confirm-upload.invalid/confirm?param=true'})
 
-      stub_get(@client, "/confirm").
-        with(:query => {"param" => ["true"]}).to_return(json_response('content_migration_files', 'upload_success.json'))
+      stub_get(@client, '/confirm').with(query: { param: true }).to_return(json_response('content_migration_files', 'upload_success.json'))
 
       opts = {'migration_type' => 'canvas_cartridge_importer', 'pre_attachment[name]' => 'cc.imscc', 'pre_attachment[size]' => '2034'}
       content_migration_response, file_upload_response = @client.create_content_migration_with_both_responses('659', fixture('cc.imscc'), opts)

data/spec/bearcat/client_spec.rb

@@ -1,16 +1,13 @@
 require 'helper'
 
 describe Bearcat::Client do
-
   it "sets the domain" do
     client = Bearcat::Client.new(domain: "http://canvas.instructure.com")
     client.config[:domain].should == "http://canvas.instructure.com"
-
   end
 
   it "sets the authtoken" do
     client = Bearcat::Client.new(token: "test_token")
     client.config[:token].should == "test_token"
   end
-
-end
+end

data/spec/bearcat/rate_limiting_spec.rb

@@ -0,0 +1,62 @@
+require 'helper'
+
+require 'sidekiq'
+
+describe Bearcat::RateLimiting do
+  let!(:subject) { Bearcat::RateLimiting::RedisLimiter.new }
+  let!(:client) { Bearcat::Client.new(rate_limiter: subject, token: SecureRandom.hex) }
+  let!(:token) { client.send(:rate_limit_key) }
+
+  def fillup(count, rate: 100)
+    count.times do
+      subject.apply(token) do
+        rate
+      end
+    end
+  end
+
+  it "fills the bucket with each request" do
+    allow(Time).to receive(:now).and_return(Time.at(5000))
+    expect(subject).to_not receive(:sleep)
+    fillup(5)
+    expect(subject.increment(token, 0, 0)).to eql({ count: 500.0, timestamp: 5000.0 })
+  end
+
+  it "drains the bucket over time" do
+    allow(Time).to receive(:now).and_return(Time.at(5000))
+    expect(subject).to_not receive(:sleep)
+    fillup(5)
+
+    allow(Time).to receive(:now).and_return(Time.at(5009))
+    expect(subject.increment(token, 0, 0)).to eql({ count: 410.0, timestamp: 5009.0 })
+  end
+
+  it "sleeps requests when full" do
+    allow(Time).to receive(:now).and_return(Time.at(5000))
+    expect(subject).to receive(:sleep).at_least(:once)
+    fillup(10)
+  end
+
+  it "rate limits requests" do
+    allow(Time).to receive(:now).and_return(Time.at(5000))
+    allow(subject).to receive(:sleep)
+    fillup(10)
+
+    stub_request(:get, /.*/).to_return(body: "{}")
+    expect(subject).to receive(:sleep).once
+    client.course(1)
+  end
+
+  it "retries after a Canvas 403 - Rate Limited" do
+    stub_request(:get, /.*/).to_return(status: 403, body: "(Rate Limit Exceeded)")
+    expect(subject).to receive(:apply).exactly(3).times.and_call_original
+    expect(subject).to receive(:sleep).twice
+    expect { client.course(1) }.to raise_error(Footrest::HttpError::Forbidden)
+  end
+
+  it "pushes x-rate-limit-remaining to the bucket" do
+    stub_request(:get, /.*/).to_return(headers: { 'X-Rate-Limit-Remaining' => 120 }, body: "{}")
+    expect(subject).to receive(:checkin_known).with(token, 20)
+    client.course(1)
+  end
+end

data/spec/bearcat/stub_bearcat_spec.rb

@@ -0,0 +1,15 @@
+require 'helper'
+
+require 'bearcat/spec_helpers'
+
+describe Bearcat::SpecHelpers do
+  include Bearcat::SpecHelpers
+  let!(:client) { Bearcat::Client.new(token: SecureRandom.hex) }
+
+  describe "#stub_bearcat" do
+    it "works as expected" do
+      stub_bearcat(:course).to_return(body: { id: 10 })
+      expect(client.course(1)[:id]).to eql 10
+    end
+  end
+end

data/spec/helper.rb

@@ -1,3 +1,4 @@
+require 'active_support/core_ext/module'
 require 'bearcat'
 require 'rspec'
 require 'webmock/rspec'