bridge_api 0.1.54 → 0.1.63

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d39d608d1b30d380fb6fdb79ad8284d5c2e2e1fd132623b89717cf94a9f9071
-  data.tar.gz: a8a72c682e3868563aa290f61b1dabb3295e34aff8ef6d9c61cff9f927a32bba
+  metadata.gz: f7d6ea6a260943cb99e83ffdd26cf7c0e68d0d383ca31e7cd96a16065261903a
+  data.tar.gz: a8e36b9a696de1d2245dcb8bea4844a21fac2b6da185032b07101d424e66143c
 SHA512:
-  metadata.gz: 8b93008180e1bdb8af16f4fbc88f4fa642cbe20e655fcac04dc34ea0c5ba20b1f38ef3b5c32e098224f48da463a7e7288fc7726c57232fc6e5c666d741c505b4
-  data.tar.gz: 10af8a2e42a4ca6db28abdeba4258bf6559a4b6f4ffc3907a0fe3d0f8820015a82b92a2001e08ae25cc3af2bd0653f4de58526f0301db6f0320db96d65a11d0a
+  metadata.gz: c9fb10e6dc27d66c2ba3273ebfd7b0770245590256d1782be1cf9398b3a61c4ef36e2af7677da48322cabc35d0ca932a2ab3828f650b523e0ba85f71e6b3db14
+  data.tar.gz: a828d6bef4d11150bd2b4968fdeb81908c06e44de3ab8fb044fff5bc09c61b586d15c466b12ca41496bb5e3d1d5990ab12464691d6b6876327b7094451fd58e8

data/Gemfile.lock

@@ -1,17 +1,16 @@
 PATH
   remote: .
   specs:
-    bridge_api (0.1.48)
+    bridge_api (0.1.60)
       faraday (~> 0.9.0)
       faraday_middleware (>= 0.12.2)
       footrest (>= 0.5.1)
-      logging (>= 2.2.2)
       paul_walker (>= 0.1.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.2.2)
+    activesupport (5.2.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -20,29 +19,24 @@ GEM
       public_suffix (>= 2.0.2, < 4.0)
     byebug (8.2.5)
     coderay (1.1.2)
-    concurrent-ruby (1.1.3)
+    concurrent-ruby (1.1.5)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     diff-lcs (1.3)
     faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
-    faraday_middleware (0.12.2)
+    faraday_middleware (0.13.1)
       faraday (>= 0.7.4, < 1.0)
     footrest (0.5.3)
       activesupport (>= 3.0.0)
       faraday (>= 0.9.0, < 1)
       link_header (>= 0.0.7)
     hashdiff (0.3.7)
-    i18n (1.1.1)
+    i18n (1.6.0)
       concurrent-ruby (~> 1.0)
     link_header (0.0.8)
-    little-plugger (1.1.4)
-    logging (2.2.2)
-      little-plugger (~> 1.1)
-      multi_json (~> 1.10)
     method_source (0.9.0)
     minitest (5.11.3)
-    multi_json (1.13.1)
     multipart-post (2.0.0)
     paul_walker (0.1.1)
       redis (>= 4.0)
@@ -54,7 +48,7 @@ GEM
     rack-protection (1.5.5)
       rack
     rake (0.9.6)
-    redis (4.0.3)
+    redis (4.1.0)
     rspec (2.99.0)
       rspec-core (~> 2.99.0)
       rspec-expectations (~> 2.99.0)
@@ -92,4 +86,4 @@ DEPENDENCIES
   webmock (~> 1.22.6)
 
 BUNDLED WITH
-   1.17.1
+   1.17.2

data/README.md

@@ -25,3 +25,20 @@ instantiating several instances of `client`, you can also supply your client
 initializer with `master_rate_limit: true`. This will use a redis
 keystore to track your rate limits across all instances of `client` and
 properly throttle your application as needed.
+
+## Token Pools
+
+This supports token pooling to get around rate limiting issues in bridge (but should never be used without Core approval)
+To use with Tokens:
+```
+client = BridgeAPI::Client.new(tokens: ['token1', 'token2'], prefix: "https://yourdomain.bridgeapp.com")
+```
+To use with API Keys:
+```
+client = BridgeAPI::Client.new(api_keys: {'key1' => 'secret1', 'key2' => 'secret2'}, prefix: "https://yourdomain.bridgeapp.com")
+```
+
+The logic will start with the first token provided, and check against the redis cache to see what its current limit
+is. If its already under the limit threshold, it will rotate to the next in the pool, and perform the same check. It
+will do this till it finds one that is not under the threshold, and use it. If none are found, it will sleep for the 
+configured time, and then try to make a call.

data/bridge_api.gemspec

@@ -30,6 +30,5 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'faraday', '~> 0.9.0'
   gem.add_dependency 'faraday_middleware', '>= 0.12.2'
   gem.add_dependency 'footrest', '>= 0.5.1'
-  gem.add_dependency 'logging', '>= 2.2.2'
   gem.add_dependency 'paul_walker', '>= 0.1.1'
 end

data/lib/bridge_api.rb

@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 require 'bridge_api/version'
 require 'bridge_api/client'
 
 module BridgeAPI
   class << self
-    require 'logging'
-    attr_writer :enforce_rate_limits, :rate_limit_min, :rate_limits, :max_sleep_seconds, :min_sleep_seconds,
+    require 'logger'
+    attr_writer :enforce_rate_limits, :beginning_rate_limit, :rate_limits, :max_sleep_seconds, :min_sleep_seconds,
                 :logger, :master_rate_limit, :master_mutex, :rate_limit_threshold
 
     def configure
@@ -15,8 +17,8 @@ module BridgeAPI
       @enforce_rate_limits ||= false
     end
 
-    def rate_limit_min
-      @rate_limit_min ||= 30
+    def beginning_rate_limit
+      @beginning_rate_limit ||= 30
     end
 
     def rate_limits
@@ -41,7 +43,7 @@ module BridgeAPI
 
     def logger
       return @logger if defined? @logger
-      @logger = Logging.logger(STDOUT)
+      @logger = Logger.new(STDOUT)
       @logger.level = :debug
       @logger
     end

data/lib/bridge_api/api_array.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module BridgeAPI
   class ApiArray
@@ -18,7 +19,7 @@ module BridgeAPI
       @linked = {}
       @meta = {}
       @extra_meta_fields = []
-      pattern = /.*(\/api\/.*)/
+      pattern = %r{.*(/api/.*)}
       path = response.env.url
       matches = pattern.match(path.to_s)
       mapping = nil
@@ -67,7 +68,6 @@ module BridgeAPI
       !@next_page.nil?
     end
 
-
     def next_page
       load_page(@next_page)
     end
@@ -119,8 +119,10 @@ module BridgeAPI
 
     def get_response_content(response)
       return [] unless response.body.is_a?(Hash)
+
       content = response.body.reject { |k, _v| @meta_fields.include?(k) || @extra_meta_fields.include?(k) }
       return content.values[0] unless content.empty?
+
       []
     end
 

data/lib/bridge_api/client.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'footrest/client'
 require 'faraday'
 require 'footrest'
@@ -12,60 +14,126 @@ module BridgeAPI
   class Client < Footrest::Client
     require 'bridge_api/api_array'
 
-    DATA_DUMP_DOWNLOAD_PATH = '/data_dumps/download'.freeze
-    DATA_DUMP_PATH = '/data_dumps'.freeze
-    COURSE_TEMPLATE_PATH = '/course_templates'.freeze
-    ENROLLMENT_PATH = '/enrollments'.freeze
-    LTI_TOOLS_PATH = '/lti_tools'.freeze
-    PROGRAM_PATH = '/programs'.freeze
-    PROGRAM_ENROLLMENT_PATH = '/learners'.freeze
-    USER_PATH = '/users'.freeze
-    GROUPS_PATH = '/groups'.freeze
-    MANAGER_PATH = '/managers'.freeze
-    ADMIN_PATH = '/admin'.freeze
-    AUTHOR_PATH = '/author'.freeze
-    LEARNER_PATH = '/learner'.freeze
-    LEARNERS_PATH  = '/learners'.freeze
-    LEARNER_ITEMS_PATH = '/learner_items'.freeze
-    CUSTOM_FIELD_PATH = '/custom_fields'.freeze
-    SUB_ACCOUNT_PATH = '/sub_accounts'.freeze
-    SUPPORT_PATH = '/support'.freeze
-    ACCOUNT_PATH = '/accounts'.freeze
-    CLONE_OBJECTS_PATH = '/clone_objects'.freeze
-    API_VERSION = 1
-    API_PATH = '/api'.freeze
-    ROLE_PATH = '/roles'.freeze
-    AFFILIATED_SUBACCOUNTS = '/affiliated_sub_accounts'.freeze
-    BATCH_PATH = '/batch'.freeze
-    LIVE_COURSES_PATH = '/live_courses'.freeze
-    SESSIONS_PATH = '/sessions'.freeze
-    PUBLISH_PATH = '/publish'.freeze
-    WEB_CONFERENCE_PATH = '/web_conference'.freeze
-    RESTORE_PATH = '/restore'.freeze
-    DUE_DATE_PATH = '/due_date'.freeze
-    RESET_PATH = '/reset'.freeze
-    RESULT_MAPPING = {}
+    DATA_DUMP_DOWNLOAD_PATH = '/data_dumps/download'
+    DATA_DUMP_PATH          = '/data_dumps'
+    COURSE_TEMPLATE_PATH    = '/course_templates'
+    ENROLLMENT_PATH         = '/enrollments'
+    LTI_TOOLS_PATH          = '/lti_tools'
+    PROGRAM_PATH            = '/programs'
+    PROGRAM_ENROLLMENT_PATH = '/learners'
+    USER_PATH               = '/users'
+    GROUPS_PATH             = '/groups'
+    MANAGER_PATH            = '/managers'
+    ADMIN_PATH              = '/admin'
+    AUTHOR_PATH             = '/author'
+    LEARNER_PATH            = '/learner'
+    LEARNERS_PATH           = '/learners'
+    LEARNER_ITEMS_PATH      = '/learner_items'
+    CUSTOM_FIELD_PATH       = '/custom_fields'
+    SUB_ACCOUNT_PATH        = '/sub_accounts'
+    SUPPORT_PATH            = '/support'
+    ACCOUNT_PATH            = '/accounts'
+    CLONE_OBJECTS_PATH      = '/clone_objects'
+    API_VERSION             = 1
+    API_PATH                = '/api'
+    ROLE_PATH               = '/roles'
+    AFFILIATED_SUBACCOUNTS  = '/affiliated_sub_accounts'
+    BATCH_PATH              = '/batch'
+    LIVE_COURSES_PATH       = '/live_courses'
+    SESSIONS_PATH           = '/sessions'
+    PUBLISH_PATH            = '/publish'
+    WEB_CONFERENCE_PATH     = '/web_conference'
+    RESTORE_PATH            = '/restore'
+    DUE_DATE_PATH           = '/due_date'
+    RESET_PATH              = '/reset'
+    RESULT_MAPPING          = {}
 
     Dir[File.dirname(__FILE__) + '/client/*.rb'].each do |file|
       require file
       include const_get(File.basename(file).gsub('.rb', '').split('_').map(&:capitalize).join('').to_s)
     end
 
+    def initialize(options = {}, &block)
+      if BridgeAPI.enforce_rate_limits && has_token_pool?(options)
+        options = initialize_from_token_pool(options)
+      end
+      super
+    end
+
     # Override Footrest request for ApiArray support
     def request(method, &block)
-      enforce_rate_limits
+      if has_token_pool?(config)
+        (config[:api_tokens] || config[:api_keys].keys).size.times do
+          break unless rate_limit_reached?
+
+          rotate_token!
+        end
+      end
+      enforce_rate_limits if rate_limit_reached?
       response = connection.send(method, &block)
       apply_rate_limits(response)
       ApiArray.process_response(response, self, RESULT_MAPPING)
     end
 
+    def has_token_pool?(config)
+      config[:api_keys].is_a?(Hash) && config[:api_keys].keys.count >= 1 ||
+        config[:api_tokens].is_a?(Array) && config[:api_tokens].count >= 1
+    end
+
+    # Since a pool is passed in, initialize first token with the first token passed in
+    def initialize_from_token_pool(config)
+      if config[:api_keys].is_a?(Hash)
+        creds = config[:api_keys].first
+        config[:api_key]    ||= creds[0]
+        config[:api_secret] ||= creds[1]
+      elsif config[:api_tokens].is_a?(Array)
+        config[:token] ||= config[:api_tokens].first
+      end
+      config
+    end
+
+    # rotates to the next token in the pool (by order in which they were provided)
+    def rotate_token!
+      BridgeAPI.master_mutex.synchronize do
+        old_api_key = config[:api_key]
+        if config[:api_keys].is_a?(Hash)
+          keys = config[:api_keys].keys
+          return if keys.count <= 1
+
+          key                 = get_next_key(keys, config[:api_key])
+          config[:api_key]    = key
+          config[:api_secret] = config[:api_keys][key]
+        elsif config[:api_tokens].is_a?(Array)
+          keys = config[:api_tokens]
+          return if keys.count <= 1
+
+          token = get_next_key(keys, config[:token])
+          config[:token] = token
+        end
+        BridgeAPI.logger.debug('rotating API Keys')
+        set_connection(config)
+      end
+    end
+
+    def get_next_key(keys, current_key)
+      i = keys.index(current_key) || -1
+      i = (i + 2) > keys.count ? 0 : (i + 1)
+      keys[i]
+    end
+
+    def rate_limit_reached?
+      return false unless BridgeAPI.enforce_rate_limits && limit_remaining.present?
+
+      limit_remaining < BridgeAPI.rate_limit_threshold
+    end
+
     def enforce_rate_limits
-      return unless BridgeAPI.enforce_rate_limits && limit_remaining.present?
-      return unless limit_remaining < BridgeAPI.rate_limit_threshold
-      tts = ((BridgeAPI.rate_limit_min - limit_remaining) / 5).ceil
-      tts = BridgeAPI.min_sleep_seconds if tts < BridgeAPI.min_sleep_seconds
-      tts = BridgeAPI.max_sleep_seconds if tts > BridgeAPI.max_sleep_seconds
-      message = "Bridge API rate limit minimum #{BridgeAPI.rate_limit_min} reached for key: '#{config[:api_key]}'. "\
+      return unless rate_limit_reached?
+
+      tts     = ((BridgeAPI.beginning_rate_limit - limit_remaining) / 5).ceil
+      tts     = BridgeAPI.min_sleep_seconds if tts < BridgeAPI.min_sleep_seconds
+      tts     = BridgeAPI.max_sleep_seconds if tts > BridgeAPI.max_sleep_seconds
+      message = "Bridge API rate limit minimum #{BridgeAPI.rate_limit_threshold} reached for key: '#{config[:api_key]}'. "\
         "Sleeping for #{tts} second(s) to catch up ~zzZZ~. "\
         "Limit Remaining: #{limit_remaining}"
       BridgeAPI.logger.debug(message)
@@ -79,6 +147,7 @@ module BridgeAPI
     def apply_rate_limits(response)
       limit = response.headers['x-rate-limit-remaining']
       return if limit.nil?
+
       BridgeAPI.logger.debug("BRIDGE RATE LIMIT REMAINING: #{limit}")
       self.limit_remaining = limit.to_i
     end
@@ -88,8 +157,8 @@ module BridgeAPI
         BridgeAPI.master_mutex.synchronize do
           limit = PaulWalker::RateLimit.get(config[:api_key], config[:api_key])
           if limit.nil?
-            PaulWalker::RateLimit.add(config[:api_key], config[:api_key], 0, BridgeAPI::rate_limit_min)
-            limit = {current: 0}.with_indifferent_access
+            PaulWalker::RateLimit.add(config[:api_key], config[:api_key], 0, BridgeAPI.beginning_rate_limit)
+            limit = { current: 0 }.with_indifferent_access
           end
           limit['current']
         end
@@ -101,7 +170,7 @@ module BridgeAPI
     def limit_remaining=(value)
       if using_master_rate_limit?
         BridgeAPI.master_mutex.synchronize do
-          PaulWalker::RateLimit.add(config[:api_key], config[:api_key], value, BridgeAPI::rate_limit_min)
+          PaulWalker::RateLimit.add(config[:api_key], config[:api_key], value, BridgeAPI.beginning_rate_limit)
         end
       else
         BridgeAPI.rate_limits[config[:api_key]] = value
@@ -110,26 +179,25 @@ module BridgeAPI
 
     def set_connection(config)
       config[:logger] = config[:logging] if config[:logging]
-      @connection = Faraday.new(url: config[:prefix]) do |faraday|
-        faraday.request                     :multipart
-        faraday.request                     :url_encoded
+      @connection     = Faraday.new(url: config[:prefix]) do |faraday|
+        faraday.request :multipart
+        faraday.request :url_encoded
         if config[:logger] == true
           faraday.response :logger
         elsif config[:logger]
           faraday.use Faraday::Response::Logger, config[:logger]
         end
-        faraday.use                         Footrest::FollowRedirects, limit: 5 unless config[:follow_redirects] == false
-        faraday.adapter                     Faraday.default_adapter
-        faraday.use                         Footrest::ParseJson, content_type: /\bjson$/
-        faraday.use                         Footrest::RaiseFootrestErrors
-        faraday.use                         Footrest::Pagination
-        faraday.headers[:accept]          = 'application/json'
-        faraday.headers[:authorization]   = "Bearer #{config[:token]}" if config[:token]
-        faraday.headers[:user_agent]      = 'Footrest'
+        faraday.use Footrest::FollowRedirects, limit: 5 unless config[:follow_redirects] == false
+        faraday.adapter Faraday.default_adapter
+        faraday.use Footrest::ParseJson, content_type: /\bjson$/
+        faraday.use Footrest::RaiseFootrestErrors
+        faraday.use Footrest::Pagination
+        faraday.headers[:accept]     = 'application/json'
+        faraday.headers[:user_agent] = 'Footrest'
         if config[:api_key] && config[:api_secret]
-          faraday.headers[:authorization] =   'Basic ' + Base64.strict_encode64("#{config[:api_key]}:#{config[:api_secret]}")
+          faraday.headers[:authorization] = 'Basic ' + Base64.strict_encode64("#{config[:api_key]}:#{config[:api_secret]}")
         elsif config[:token]
-          faraday.headers[:authorization] =   "Bearer #{config[:token]}"
+          faraday.headers[:authorization] = "Bearer #{config[:token]}"
         else
           raise 'No api authorization provided'
         end

data/lib/bridge_api/client/account.rb

@@ -1,7 +1,8 @@
+# frozen_string_literal: true
+
 module BridgeAPI
   class Client
     module Account
-
       def create_account(params = {})
         post("#{API_PATH}#{ADMIN_PATH}/sub_accounts", params)
       end

data/lib/bridge_api/client/affiliation.rb

@@ -1,17 +1,19 @@
+# frozen_string_literal: true
+
 module BridgeAPI
-    class Client
-      module Affiliation
-        def list_affiliations(params = {})
-          get("#{API_PATH}#{AUTHOR_PATH}#{AFFILIATED_SUBACCOUNTS}", params)
-        end
+  class Client
+    module Affiliation
+      def list_affiliations(params = {})
+        get("#{API_PATH}#{AUTHOR_PATH}#{AFFILIATED_SUBACCOUNTS}", params)
+      end
 
-        def revoke_affiliations_batch(params = {})
-          put("#{API_PATH}#{AUTHOR_PATH}#{AFFILIATED_SUBACCOUNTS}/revoke_batch", params)
-        end
+      def revoke_affiliations_batch(params = {})
+        put("#{API_PATH}#{AUTHOR_PATH}#{AFFILIATED_SUBACCOUNTS}/revoke_batch", params)
+      end
 
-        def share_affiliations_batch(params = {})
-          put("#{API_PATH}#{AUTHOR_PATH}#{AFFILIATED_SUBACCOUNTS}/share_batch", params)
-        end
+      def share_affiliations_batch(params = {})
+        put("#{API_PATH}#{AUTHOR_PATH}#{AFFILIATED_SUBACCOUNTS}/share_batch", params)
       end
     end
   end
+  end