lex-microsoft_teams 0.6.50 → 0.6.52

data/lib/legion/extensions/microsoft_teams/faraday/retry_after.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'time'
+require 'legion/extensions/microsoft_teams/errors'
+
+module Legion
+  module Extensions
+    module MicrosoftTeams
+      module Faraday
+        # Faraday middleware that retries throttled responses honoring the
+        # upstream Retry-After header per RFC 9110 §10.2.3 (originally
+        # specified in RFC 7231 §7.1.3). Retries HTTP 429 by default; 503
+        # and 504 are opt-in via `retry_statuses:`.
+        #
+        # Retry-After is parsed in two forms:
+        #
+        # * delta-seconds  (e.g. "120")               — used as-is
+        # * HTTP-date      (e.g. "Wed, 27 May 2026 12:00:00 GMT")
+        #                                              — converted to delta
+        #                                                from current UTC,
+        #                                                clamped to >= 0
+        #
+        # The advertised wait is jittered by ±(jitter * wait) to avoid
+        # thundering-herd behavior across instances sharing one Entra app
+        # registration's Graph quota.
+        #
+        # When `max_retries` is reached, or cumulative wait would exceed
+        # `max_wait`, the middleware raises `Errors::Throttled` carrying
+        # the last advertised Retry-After (nil if the header was missing or
+        # unparseable), the final HTTP status, attempt count, and request
+        # path. Raising centrally — rather than returning a raw 429 and
+        # trusting every caller to detect it — is the difference between
+        # one typed event the fleet can defer on, and 60+ runner callsites
+        # that silently treat throttle envelopes as data.
+        class RetryAfter < ::Faraday::Middleware
+          DEFAULT_MAX_RETRIES    = 3
+          DEFAULT_MAX_WAIT       = 60.0
+          DEFAULT_JITTER         = 0.2
+          DEFAULT_FALLBACK_WAIT  = 1.0
+          DEFAULT_RETRY_STATUSES = [429].freeze
+
+          # Parse an HTTP Retry-After header value.
+          #
+          # @param raw [String, nil] the raw header value
+          # @param clock [#call] a callable returning current UTC Time,
+          #   injectable for deterministic tests
+          # @return [Float, nil] seconds to wait, or nil if `raw` is absent,
+          #   empty, or neither a numeric delta nor a valid HTTP-date
+          def self.parse_header(raw, clock: -> { Time.now.utc })
+            return nil if raw.nil?
+
+            value = raw.to_s.strip
+            return nil if value.empty?
+            return value.to_f if value.match?(/\A\d+(\.\d+)?\z/)
+
+            begin
+              target = Time.httpdate(value).utc
+              [(target - clock.call), 0.0].max
+              # rubocop:disable Legion/RescueLogging/NoCapture
+              # Pure parser — no logger access. The instance method
+              # `parse_advertised` warns on the same condition with full
+              # context; logging twice would just be noise.
+            rescue ArgumentError
+              nil
+              # rubocop:enable Legion/RescueLogging/NoCapture
+            end
+          end
+
+          def initialize(app, # rubocop:disable Metrics/ParameterLists
+                         max_retries: DEFAULT_MAX_RETRIES,
+                         max_wait: DEFAULT_MAX_WAIT,
+                         jitter: DEFAULT_JITTER,
+                         fallback_wait: DEFAULT_FALLBACK_WAIT,
+                         retry_statuses: DEFAULT_RETRY_STATUSES,
+                         sleeper: ->(seconds) { sleep(seconds) },
+                         logger: nil,
+                         clock: -> { Time.now.utc })
+            super(app)
+            @max_retries    = Integer(max_retries)
+            @max_wait       = Float(max_wait)
+            @jitter         = Float(jitter)
+            @fallback_wait  = Float(fallback_wait)
+            @retry_statuses = Array(retry_statuses).map(&:to_i).freeze
+            @sleeper        = sleeper
+            @logger         = logger
+            @clock          = clock
+          end
+
+          def call(env)
+            attempts          = 0
+            total_wait        = 0.0
+            last_advertised   = nil
+
+            loop do
+              response = @app.call(env.dup)
+              return response unless retryable?(response.status)
+
+              last_advertised = parse_advertised(response)
+              wait            = compute_wait(last_advertised)
+
+              if attempts >= @max_retries || (total_wait + wait) > @max_wait
+                log_giveup(env, response, attempts, total_wait)
+                raise Errors::Throttled.new(
+                  status:      response.status,
+                  retry_after: last_advertised,
+                  request:     request_path(env),
+                  attempts:    attempts
+                )
+              end
+
+              attempts   += 1
+              total_wait += wait
+              log_retry(env, response, wait, attempts)
+              @sleeper.call(wait)
+            end
+          end
+
+          private
+
+          def retryable?(status)
+            @retry_statuses.include?(status.to_i)
+          end
+
+          # Parse the advertised Retry-After value from a response. Returns
+          # nil if the header is missing or unparseable — callers branch on
+          # nil to distinguish "no server guidance" from "retry now".
+          def parse_advertised(response)
+            raw = retry_after_header(response)
+            parsed = self.class.parse_header(raw, clock: @clock)
+            @logger&.warn("[microsoft_teams][retry_after] unparseable Retry-After=#{raw.inspect}") if raw && !raw.to_s.strip.empty? && parsed.nil?
+            parsed
+          end
+
+          # Computes the actual wait the middleware will sleep before the
+          # next attempt. Falls back to `@fallback_wait` only when the
+          # server gave no usable guidance; jitter is always applied so
+          # concurrent instances don't synchronize their retries.
+          def compute_wait(advertised)
+            seconds = advertised || @fallback_wait
+            apply_jitter(seconds)
+          end
+
+          def retry_after_header(response)
+            headers = response_headers(response)
+            return nil unless headers
+
+            headers['Retry-After'] ||
+              headers['retry-after'] ||
+              headers['RETRY-AFTER']
+          end
+
+          # Faraday's Response exposes headers via #headers; some test
+          # doubles may not, so look in both common places.
+          def response_headers(response)
+            if response.respond_to?(:headers) && response.headers
+              response.headers
+            elsif response.respond_to?(:response_headers)
+              response.response_headers
+            end
+          end
+
+          def apply_jitter(seconds)
+            return seconds if @jitter.zero?
+
+            spread = seconds * @jitter
+            offset = ((rand * 2.0) - 1.0) * spread
+            wait   = seconds + offset
+            wait.negative? ? 0.0 : wait
+          end
+
+          def request_path(env)
+            env.url.respond_to?(:path) ? env.url.path : env.url.to_s
+            # rubocop:disable Legion/RescueLogging/NoCapture
+            # Defensive fallback for malformed env.url; the path is only used
+            # for log lines and error messages, never for control flow.
+          rescue StandardError
+            nil
+            # rubocop:enable Legion/RescueLogging/NoCapture
+          end
+
+          def log_retry(env, response, wait, attempts)
+            return unless @logger
+
+            @logger.warn(
+              "[microsoft_teams][retry_after] status=#{response.status} " \
+              "method=#{env.method.to_s.upcase} path=#{request_path(env)} " \
+              "wait=#{format('%.2f', wait)}s attempt=#{attempts}"
+            )
+          rescue StandardError => e
+            warn("[microsoft_teams][retry_after] log_retry suppressed #{e.class}: #{e.message}")
+          end
+
+          def log_giveup(env, response, attempts, total_wait)
+            return unless @logger
+
+            @logger.error(
+              "[microsoft_teams][retry_after] giving up; status=#{response.status} " \
+              "method=#{env.method.to_s.upcase} path=#{request_path(env)} " \
+              "attempts=#{attempts} total_wait=#{format('%.2f', total_wait)}s"
+            )
+          rescue StandardError => e
+            warn("[microsoft_teams][retry_after] log_giveup suppressed #{e.class}: #{e.message}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/microsoft_teams/faraday/throttle_circuit.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'legion/extensions/microsoft_teams/errors'
+
+module Legion
+  module Extensions
+    module MicrosoftTeams
+      module Faraday
+        class ThrottleCircuit < ::Faraday::Middleware
+          CIRCUIT_KEY = 'microsoft_teams:graph:circuit:v1:global'
+          DEFAULT_SOFT_PERCENTAGE = 0.8
+          DEFAULT_SOFT_TTL = 60
+          DEFAULT_FALLBACK_TTL = 60
+          DEFAULT_INSIGHTS_TTL = 600
+
+          def initialize(app,
+                         soft_percentage: DEFAULT_SOFT_PERCENTAGE,
+                         soft_ttl: DEFAULT_SOFT_TTL,
+                         fallback_ttl: DEFAULT_FALLBACK_TTL,
+                         insights_ttl: DEFAULT_INSIGHTS_TTL,
+                         logger: nil)
+            super(app)
+            @soft_percentage = Float(soft_percentage)
+            @soft_ttl = Integer(soft_ttl)
+            @fallback_ttl = Integer(fallback_ttl)
+            @insights_ttl = Integer(insights_ttl)
+            @logger = logger
+          end
+
+          def call(env)
+            remaining = circuit_remaining
+            if remaining&.positive?
+              path = request_path(env)
+              @logger&.debug("[throttle_circuit] circuit open, #{remaining}s remaining, blocking #{path}")
+              raise Errors::Throttled.new(
+                status:      429,
+                retry_after: remaining,
+                request:     path,
+                attempts:    0
+              )
+            end
+
+            response = @app.call(env)
+            check_throttle_percentage(env, response)
+            response
+          rescue Errors::Throttled => e
+            set_hard_circuit(path: request_path(env), retry_after: e.retry_after)
+            raise
+          end
+
+          private
+
+          def circuit_remaining
+            return nil unless cache_available?
+
+            raw = Legion::Cache.get(CIRCUIT_KEY) # rubocop:disable Legion/HelperMigration/DirectCache
+            return nil unless raw
+
+            expires_at = raw.to_f
+            remaining = expires_at - Time.now.to_f
+            remaining.positive? ? remaining.ceil : nil
+          rescue StandardError => e
+            @logger&.debug("[throttle_circuit] circuit_remaining error: #{e.message}")
+            nil
+          end
+
+          def check_throttle_percentage(env, response)
+            headers = response_headers(response)
+            return unless headers
+
+            raw = headers['x-ms-throttle-limit-percentage'] ||
+                  headers['X-Ms-Throttle-Limit-Percentage']
+            return unless raw
+
+            percentage = raw.to_f
+            return unless percentage >= @soft_percentage
+
+            path = request_path(env)
+            log_throttle_diagnostics(headers, path: path, percentage: percentage)
+            set_circuit(ttl: @soft_ttl)
+          end
+
+          def log_throttle_diagnostics(headers, path:, percentage:)
+            scope = headers['x-ms-throttle-scope'] || headers['X-Ms-Throttle-Scope']
+            info = headers['x-ms-throttle-information'] || headers['X-Ms-Throttle-Information']
+            ru = headers['x-ms-resource-unit'] || headers['X-Ms-Resource-Unit']
+            req_id = headers['request-id'] || headers['Request-Id']
+            client_req_id = headers['client-request-id'] || headers['Client-Request-Id']
+
+            @logger&.warn(
+              "[throttle_circuit] soft circuit: percentage=#{percentage} " \
+              "path=#{path} ttl=#{@soft_ttl}s scope=#{scope} " \
+              "info=#{info} resource_unit=#{ru} " \
+              "request_id=#{req_id} client_request_id=#{client_req_id}"
+            )
+          end
+
+          def set_hard_circuit(path:, retry_after:)
+            ttl = if retry_after&.positive?
+                    [retry_after.ceil, 600].min
+                  else
+                    classified_ttl(path)
+                  end
+            @logger&.error(
+              "[throttle_circuit] hard circuit: path=#{path} " \
+              "retry_after=#{retry_after} ttl=#{ttl}s"
+            )
+            set_circuit(ttl: ttl)
+          end
+
+          def classified_ttl(path)
+            return @insights_ttl if path&.match?(%r{/me/people|/me/insights|aiInsights})
+            return @fallback_ttl if path&.match?(/chats|channels|messages|presence|meetings/)
+
+            @fallback_ttl
+          end
+
+          def set_circuit(ttl:)
+            return unless cache_available?
+
+            expires_at = Time.now.to_f + ttl
+            Legion::Cache.set(CIRCUIT_KEY, expires_at.to_s, ttl: ttl, async: false) # rubocop:disable Legion/HelperMigration/DirectCache
+          rescue StandardError => e
+            @logger&.debug("[throttle_circuit] set_circuit error: #{e.message}")
+          end
+
+          def cache_available?
+            defined?(Legion::Cache) && Legion::Cache.respond_to?(:get)
+          end
+
+          def request_path(env)
+            env.url.respond_to?(:path) ? env.url.path : env.url.to_s
+          rescue StandardError => e
+            @logger&.debug("[throttle_circuit] request_path error: #{e.message}")
+            nil
+          end
+
+          def response_headers(response)
+            if response.respond_to?(:headers) && response.headers
+              response.headers
+            elsif response.respond_to?(:response_headers)
+              response.response_headers
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/microsoft_teams/helpers/client.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require 'faraday'
+require 'legion/extensions/microsoft_teams/faraday/throttle_circuit'
+require 'legion/extensions/microsoft_teams/faraday/retry_after'
 
 module Legion
   module Extensions
@@ -12,8 +14,10 @@ module Legion
 
           def graph_connection(token: nil, api_url: 'https://graph.microsoft.com/v1.0', **_opts)
             token ||= entra_delegated_token
-            Faraday.new(url: api_url) do |conn|
+            ::Faraday.new(url: api_url) do |conn|
               conn.request :json
+              conn.use Legion::Extensions::MicrosoftTeams::Faraday::ThrottleCircuit, **throttle_circuit_options
+              conn.use Legion::Extensions::MicrosoftTeams::Faraday::RetryAfter, **retry_after_options
               conn.response :json, content_type: /\bjson$/
               conn.headers['Authorization'] = "Bearer #{token}" if token
               conn.headers['Content-Type'] = 'application/json'
@@ -21,8 +25,10 @@ module Legion
           end
 
           def bot_connection(token: nil, service_url: 'https://smba.trafficmanager.net/teams/', **_opts)
-            Faraday.new(url: service_url) do |conn|
+            ::Faraday.new(url: service_url) do |conn|
               conn.request :json
+              conn.use Legion::Extensions::MicrosoftTeams::Faraday::ThrottleCircuit, **throttle_circuit_options
+              conn.use Legion::Extensions::MicrosoftTeams::Faraday::RetryAfter, **retry_after_options
               conn.response :json, content_type: /\bjson$/
               conn.headers['Authorization'] = "Bearer #{token}" if token
               conn.headers['Content-Type'] = 'application/json'
@@ -34,7 +40,7 @@ module Legion
           end
 
           def oauth_connection(tenant_id: 'common', **_opts)
-            Faraday.new(url: "https://login.microsoftonline.com/#{tenant_id}") do |conn|
+            ::Faraday.new(url: "https://login.microsoftonline.com/#{tenant_id}") do |conn|
               conn.request :url_encoded
               conn.response :json, content_type: /\bjson$/
             end
@@ -48,6 +54,77 @@ module Legion
             handle_exception(e, level: :debug, operation: 'Client#entra_delegated_token')
             nil
           end
+
+          def throttle_circuit_options
+            cfg = throttle_circuit_settings || {}
+            {
+              soft_percentage: fetch_setting(cfg, :soft_percentage, 0.8),
+              soft_ttl:        fetch_setting(cfg, :soft_ttl,        60),
+              fallback_ttl:    fetch_setting(cfg, :fallback_ttl,    60),
+              insights_ttl:    fetch_setting(cfg, :insights_ttl,    600),
+              logger:          retry_after_logger
+            }
+          end
+
+          def throttle_circuit_settings
+            return nil unless respond_to?(:settings, true)
+
+            section = settings
+            return nil unless section.respond_to?(:dig)
+
+            section.dig(:client, :throttle_circuit) || section.dig('client', 'throttle_circuit')
+          end
+
+          # Tunable knobs for the Retry-After middleware. Reads from the lex
+          # settings under `client.retry.*` (rooted at the gem's settings
+          # namespace via `Helpers::Lex`, so the full key is
+          # `microsoft_teams.client.retry.*`). Uses `fetch` rather than `||`
+          # so an operator who explicitly sets `max_retries: 0` to disable
+          # retries gets exactly that — not the default. Wires the Lex
+          # logger so retry / giveup events get structured logging.
+          def retry_after_options
+            cfg = retry_after_settings || {}
+            {
+              max_retries:    fetch_setting(cfg, :max_retries,    3),
+              max_wait:       fetch_setting(cfg, :max_wait,       60.0),
+              jitter:         fetch_setting(cfg, :jitter,         0.2),
+              fallback_wait:  fetch_setting(cfg, :fallback_wait,  1.0),
+              retry_statuses: fetch_setting(cfg, :retry_statuses,
+                                            Legion::Extensions::MicrosoftTeams::Faraday::RetryAfter::DEFAULT_RETRY_STATUSES),
+              logger:         retry_after_logger
+            }
+          end
+
+          def retry_after_settings
+            return nil unless respond_to?(:settings, true)
+
+            section = settings
+            return nil unless section.respond_to?(:dig)
+
+            section.dig(:client, :retry) || section.dig('client', 'retry')
+          end
+
+          # Read a setting under both symbol and string keys, falling back
+          # to `default` only when neither is present. `fetch` is required
+          # so falsey values (0, false, nil) survive — `||` would silently
+          # promote them to the default.
+          def fetch_setting(cfg, key, default)
+            cfg.fetch(key) { cfg.fetch(key.to_s, default) }
+          end
+
+          # Bind the Lex logger so the middleware emits structured retry /
+          # giveup events. If `log` raises before the include chain is
+          # ready, fall back to `Legion::Logging` (the underlying logger
+          # the helper wraps) so we never silently lose telemetry — losing
+          # those signals is exactly how the original fleet outage went
+          # undiagnosed for days.
+          def retry_after_logger
+            log
+          rescue StandardError => e
+            fallback = defined?(::Legion::Logging) ? ::Legion::Logging : nil
+            fallback&.error("[microsoft_teams][client] retry_after_logger fallback: #{e.class}: #{e.message}")
+            fallback
+          end
         end
       end
     end

data/lib/legion/extensions/microsoft_teams/helpers/graph_cache.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module MicrosoftTeams
+      module Helpers
+        module GraphCache
+          include Legion::Cache::Helper if defined?(Legion::Cache::Helper)
+
+          def graph_cache_ttl
+            settings = teams_extension_settings
+            settings.dig(:cache, :graph_ttl) || 300
+          end
+
+          def cached_graph_get(conn:, path:, params: {}, ttl: nil, shared: false)
+            effective_ttl = ttl || graph_cache_ttl
+            key = graph_cache_key(path: path, params: params, shared: shared)
+
+            cache_fetch(key, ttl: effective_ttl) do
+              resp = conn.get(path, params)
+              resp.body
+            end
+          rescue StandardError => e
+            handle_exception(e, level: :debug, operation: 'GraphCache#cached_graph_get', path: path)
+            conn.get(path, params).body
+          end
+
+          def graph_user_key
+            return @graph_user_key if defined?(@graph_user_key)
+
+            @graph_user_key = (Legion::Identity::Process.id if defined?(Legion::Identity::Process) && Legion::Identity::Process.resolved?)
+          end
+
+          def invalidate_graph_cache(path:, params: {}, shared: false)
+            key = graph_cache_key(path: path, params: params, shared: shared)
+            cache_delete(key)
+          rescue StandardError => e
+            handle_exception(e, level: :debug, operation: 'GraphCache#invalidate_graph_cache')
+          end
+
+          private
+
+          def graph_cache_key(path:, params: {}, shared: false)
+            scope = if shared
+                      'shared'
+                    else
+                      graph_user_key || 'anon'
+                    end
+            param_str = params.empty? ? '' : ":#{params.sort.map { |k, v| "#{k}=#{v}" }.join('&')}"
+            "graph:#{scope}:#{path}#{param_str}"
+          end
+
+          def teams_extension_settings
+            return {} unless defined?(Legion::Settings)
+
+            Legion::Settings[:microsoft_teams] || {}
+          rescue StandardError => e
+            handle_exception(e, level: :debug, operation: 'GraphCache#teams_extension_settings')
+            {}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/microsoft_teams/helpers/graph_client.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'legion/extensions/microsoft_teams/errors'
+require 'legion/extensions/microsoft_teams/faraday/retry_after'
+
 module Legion
   module Extensions
     module MicrosoftTeams
@@ -62,12 +65,29 @@ module Legion
             when 403
               detail = error_message || 'Caller does not have sufficient permissions to perform this action.'
               raise GraphError, "Graph API 403 Forbidden on #{path}: #{detail}"
+            when 429, 503, 504
+              # Defensive: the RetryAfter middleware raises Throttled itself
+              # when it exhausts retries, so this branch is rarely hit. It
+              # still fires when a caller uses a Faraday connection without
+              # the middleware installed (custom tests, ad-hoc tooling).
+              raise Errors::Throttled.new(
+                status:      response.status,
+                retry_after: Faraday::RetryAfter.parse_header(retry_after_value(response)),
+                request:     path
+              )
             else
               base_message = "Graph API #{response.status} on #{path}"
               base_message = "#{base_message}: #{error_message}" if error_message
               raise GraphError, base_message
             end
           end
+
+          def retry_after_value(response)
+            headers = response.respond_to?(:headers) ? response.headers : nil
+            return nil unless headers
+
+            headers['Retry-After'] || headers['retry-after'] || headers['RETRY-AFTER']
+          end
         end
       end
     end