lex-microsoft_teams 0.6.50 → 0.6.52

data/lib/legion/extensions/microsoft_teams/helpers/throttle_aware.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+require 'monitor'
+require 'legion/extensions/microsoft_teams/errors'
+
+module Legion
+  module Extensions
+    module MicrosoftTeams
+      module Helpers
+        # Mixin that lets an `Actors::Every`-style poller honour a Graph
+        # throttle by *deferring its own next run* instead of re-firing on
+        # its fixed timer interval.
+        #
+        # Background: `Faraday::RetryAfter` raises `Errors::Throttled`
+        # centrally when it exhausts retries (or when the advertised wait
+        # alone exceeds the retry budget), and `Faraday::ThrottleCircuit`
+        # opens a shared circuit so the whole fleet stops hammering a quota
+        # that Graph has already flagged. But the actors that *drive* those
+        # calls schedule themselves with a `Concurrent::TimerTask` whose
+        # `execution_interval` is fixed at boot. Catching `Throttled` in a
+        # `rescue` block stops the current tick, but the next tick still
+        # fires on the original cadence — so a poller on a 30–300s interval
+        # keeps walking straight back into an open circuit, emitting an
+        # ERROR every interval and contributing nothing but log noise (and,
+        # via the shared circuit, refreshing the block for cheaper callers).
+        #
+        # This mixin closes that gap. An actor wraps its Graph work in
+        # {#with_throttle_deferral}. On `Errors::Throttled` it records a
+        # "suppress until" instant of `now + retry_after` (falling back to a
+        # bounded default when the server gave no usable `Retry-After`).
+        # Subsequent ticks short-circuit via {#throttle_suppressed?} until
+        # that instant passes, so the actor effectively reschedules itself
+        # at `now + retry_after` rather than `now + interval` — exactly the
+        # behaviour the throttle integration spec and the 0.6.51 CHANGELOG
+        # flagged as the outstanding follow-up.
+        #
+        # State is per-actor-instance and guarded by a monitor because an
+        # actor's `manual` runs on a `Concurrent` thread-pool worker. The
+        # mixin reads the carried `retry_after` only when
+        # `retry_after_known?` is true; otherwise it applies
+        # {DEFAULT_DEFERRAL} so a throttle with a missing/garbage header
+        # still backs the poller off rather than letting it spin.
+        module ThrottleAware
+          # Fallback deferral (seconds) used when a `Throttled` carries no
+          # usable `retry_after` (header absent or unparseable). One minute
+          # matches Microsoft Graph's documented typical Retry-After and the
+          # circuit middleware's `DEFAULT_FALLBACK_TTL`.
+          DEFAULT_DEFERRAL = 60.0
+
+          # Hard ceiling (seconds) on a single deferral so a pathological
+          # advertised `Retry-After` can't park a poller for an unbounded
+          # time. Mirrors the circuit middleware's 600s cap in
+          # `ThrottleCircuit#set_hard_circuit`.
+          MAX_DEFERRAL = 600.0
+
+          # Run `block` unless the actor is currently deferring after a
+          # recent throttle. If a `Throttled` escapes the block, record the
+          # deferral window and swallow it (the throttle has already been
+          # logged at the middleware/circuit layer; re-raising would just
+          # trip the actor's generic `rescue StandardError` and double-log).
+          #
+          # @param now [Time] injectable clock for deterministic tests
+          # @return [Object, nil] the block's value, or nil when suppressed
+          #   or when a throttle was caught
+          def with_throttle_deferral(now: Time.now)
+            if throttle_suppressed?(now: now)
+              log_throttle_skip(now: now)
+              return nil
+            end
+
+            yield
+          rescue Legion::Extensions::MicrosoftTeams::Errors::Throttled => e
+            defer_after_throttle(e, now: now)
+            nil
+          end
+
+          # @return [Boolean] true while the actor is inside a deferral
+          #   window opened by a previous throttle.
+          def throttle_suppressed?(now: Time.now)
+            until_at = throttled_until
+            !until_at.nil? && now < until_at
+          end
+
+          # The instant before which the actor should not poll again, or nil
+          # if no deferral is active. Thread-safe read.
+          def throttled_until
+            throttle_monitor.synchronize { @throttled_until }
+          end
+
+          # Seconds remaining in the current deferral window (0.0 if none /
+          # elapsed). Useful for logging and for tests.
+          def throttle_remaining(now: Time.now)
+            until_at = throttled_until
+            return 0.0 if until_at.nil?
+
+            remaining = until_at - now
+            remaining.positive? ? remaining : 0.0
+          end
+
+          # Open (or extend) a deferral window of `seconds` from `now`. A
+          # later throttle never *shortens* an existing window — we keep the
+          # furthest-out instant so overlapping throttles compose safely.
+          def defer_for(seconds, now: Time.now)
+            window = clamp_deferral(seconds)
+            target = now + window
+            throttle_monitor.synchronize do
+              @throttled_until = if @throttled_until.nil? || target > @throttled_until
+                                   target
+                                 else
+                                   @throttled_until
+                                 end
+            end
+            window
+          end
+
+          # Clear any active deferral. Called implicitly is unnecessary —
+          # {#throttle_suppressed?} expires on its own once the clock passes
+          # `throttled_until` — but exposed for tests and explicit resets.
+          def reset_throttle_deferral
+            throttle_monitor.synchronize { @throttled_until = nil }
+          end
+
+          private
+
+          def defer_after_throttle(error, now: Time.now)
+            seconds = deferral_seconds_for(error)
+            window  = defer_for(seconds, now: now)
+            log.warn(
+              "[microsoft_teams][throttle_defer] #{actor_label}: " \
+              "deferring next run #{format('%.1f', window)}s " \
+              "(retry_after=#{error.retry_after_known? ? format('%.1f', error.retry_after) : 'unknown'} " \
+              "path=#{error.request})"
+            )
+          end
+
+          # Short, log-friendly name for the including actor. Falls back to
+          # the full class string for anonymous classes (whose `name` is
+          # nil), so the log line never raises on `name.split`.
+          def actor_label
+            (self.class.name || self.class.to_s).split('::').last
+          end
+
+          # Prefer the server-advised wait; fall back to {DEFAULT_DEFERRAL}
+          # only when the header was absent or unparseable.
+          def deferral_seconds_for(error)
+            if error.respond_to?(:retry_after_known?) && error.retry_after_known?
+              error.retry_after
+            else
+              DEFAULT_DEFERRAL
+            end
+          end
+
+          def clamp_deferral(seconds)
+            value = begin
+              Float(seconds)
+              # rubocop:disable Legion/RescueLogging/NoCapture
+              # Pure coercion fallback for a non-numeric deferral; the value
+              # is bounded immediately below, so a bad input degrades to the
+              # default rather than warranting its own log line.
+            rescue ArgumentError, TypeError
+              DEFAULT_DEFERRAL
+              # rubocop:enable Legion/RescueLogging/NoCapture
+            end
+            value = DEFAULT_DEFERRAL if value <= 0
+            [value, MAX_DEFERRAL].min
+          end
+
+          def log_throttle_skip(now: Time.now)
+            log.debug(
+              "[microsoft_teams][throttle_defer] #{actor_label}: " \
+              "skipping run, #{format('%.1f', throttle_remaining(now: now))}s deferral remaining"
+            )
+          end
+
+          def throttle_monitor
+            # `||=` is safe here: the first tick of an actor runs before any
+            # concurrent re-entry (the `Every` base guards re-entry with its
+            # own AtomicBoolean), so the monitor is initialised single-threaded.
+            @throttle_monitor ||= Monitor.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/microsoft_teams/runners/api_ingest.rb

@@ -2,7 +2,9 @@
 
 require 'json'
 require 'digest'
+require 'legion/extensions/microsoft_teams/errors'
 require 'legion/extensions/microsoft_teams/helpers/client'
+require 'legion/extensions/microsoft_teams/helpers/graph_cache'
 require 'legion/extensions/microsoft_teams/helpers/permission_guard'
 require 'legion/extensions/microsoft_teams/helpers/high_water_mark'
 
@@ -15,6 +17,7 @@ module Legion
           include Helpers::Client
           include Helpers::PermissionGuard
           include Helpers::HighWaterMark
+          include Helpers::GraphCache
           extend self
 
           definition :ingest_api, mcp_exposed: false
@@ -41,6 +44,7 @@ module Legion
 
             existing_hashes = load_existing_hashes
             conn = graph_connection(token: token)
+            chat_index = build_chat_member_index(conn: conn, chats: chats)
             stored = 0
             skipped = 0
             people_ingested = 0
@@ -48,7 +52,7 @@ module Legion
             person_texts = Hash.new { |h, k| h[k] = [] }
 
             people.each do |person|
-              chat = match_chat_to_person(chats: chats, person: person, conn: conn)
+              chat = find_chat_for_person_indexed(person: person, chat_index: chat_index)
               unless chat
                 log.debug("ApiIngest: no chat match for #{person['displayName']} " \
                           "(email=#{person.dig('scoredEmailAddresses', 0, 'address')}, id=#{person['id']})")
@@ -101,6 +105,16 @@ module Legion
             { result: { stored: stored, skipped: skipped, people_ingested: people_ingested,
                         people_found: people.length, chats_found: chats.length,
                         apollo: apollo_results } }
+          # rubocop:disable Legion/RescueLogging/NoCapture
+          # Re-raise unlogged: surface the typed throttle to the caller (the
+          # ApiIngest actor) so it can defer its next scheduled run by the
+          # advertised retry_after. The throttle is already logged at the
+          # middleware/circuit layer; folding it into an error result here
+          # would hide the one signal the actor needs to stop re-charging
+          # the shared Graph circuit on its fixed interval.
+          rescue Errors::Throttled
+            raise
+            # rubocop:enable Legion/RescueLogging/NoCapture
           rescue StandardError => e
             handle_exception(e, level: :error, operation: 'ApiIngest#ingest_api')
             { result: { stored: stored || 0, skipped: skipped || 0, error: e.message } }
@@ -164,33 +178,42 @@ module Legion
             []
           end
 
-          def match_chat_to_person(chats:, person:, conn:)
-            email = person.dig('scoredEmailAddresses', 0, 'address')&.downcase
-            display_name = person['displayName']&.downcase
-            user_id = person['id']
-            return nil unless email || user_id || display_name
-
-            chats.find do |chat|
-              members_resp = conn.get("chats/#{chat['id']}/members")
-              members = (members_resp.body || {}).fetch('value', [])
-              members.any? do |m|
-                match_member?(m, email: email, user_id: user_id, display_name: display_name)
+          def build_chat_member_index(conn:, chats:)
+            by_email = {}
+            by_user_id = {}
+            by_name = {}
+
+            chats.each do |chat|
+              members = cached_graph_get(conn: conn, path: "chats/#{chat['id']}/members",
+                                         shared: true)
+                        .then { |body| (body || {}).fetch('value', []) }
+              members.each do |m|
+                email = m['email']&.downcase
+                alt_email = m.dig('additionalData', 'email')&.downcase
+                uid = m['userId']
+                name = m['displayName']&.downcase
+
+                by_email[email] ||= chat if email
+                by_email[alt_email] ||= chat if alt_email
+                by_user_id[uid] ||= chat if uid
+                by_name[name] ||= chat if name
               end
             end
+
+            { email: by_email, user_id: by_user_id, name: by_name }
           rescue StandardError => e
-            handle_exception(e, level: :debug, operation: 'ApiIngest#match_chat_to_person')
-            nil
+            handle_exception(e, level: :warn, operation: 'ApiIngest#build_chat_member_index')
+            { email: {}, user_id: {}, name: {} }
           end
 
-          def match_member?(member, email:, user_id:, display_name:)
-            return true if email && member['email']&.downcase == email
-            return true if user_id && member['userId'] == user_id
-            return true if email && member.dig('additionalData', 'email')&.downcase == email
-
-            member_name = member['displayName']&.downcase
-            return true if display_name && member_name && member_name == display_name
+          def find_chat_for_person_indexed(person:, chat_index:)
+            email = person.dig('scoredEmailAddresses', 0, 'address')&.downcase
+            user_id = person['id']
+            display_name = person['displayName']&.downcase
 
-            false
+            chat_index[:email][email] ||
+              chat_index[:user_id][user_id] ||
+              chat_index[:name][display_name]
           end
 
           def fetch_chat_messages(conn:, chat_id:, depth: 50)

data/lib/legion/extensions/microsoft_teams/runners/profile_ingest.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 require 'json'
+require 'legion/extensions/microsoft_teams/errors'
 require 'legion/extensions/microsoft_teams/helpers/client'
+require 'legion/extensions/microsoft_teams/helpers/graph_cache'
 require 'legion/extensions/microsoft_teams/helpers/permission_guard'
 require 'legion/extensions/microsoft_teams/helpers/high_water_mark'
 require 'legion/extensions/microsoft_teams/helpers/transform_definitions'
@@ -15,6 +17,7 @@ module Legion
           include Helpers::Client
           include Helpers::PermissionGuard
           include Helpers::HighWaterMark
+          include Helpers::GraphCache
           extend self
 
           definition :full_ingest, mcp_exposed: false
@@ -34,6 +37,15 @@ module Legion
             teams_result = ingest_teams_and_meetings(token: token)
 
             { self: self_result, people: people_result, conversations: conv_result, teams: teams_result }
+          rescue Errors::Throttled => e
+            # Once Graph throttles us, the shared circuit is open and every
+            # remaining stage would just raise Throttled(attempts: 0)
+            # instantly — a burst of identical errors that ingests nothing.
+            # Stop the fan-out at the first throttled stage and report it.
+            log.warn('[microsoft_teams][profile_ingest] throttled mid-ingest; ' \
+                     'aborting remaining stages (retry_after=' \
+                     "#{e.retry_after_known? ? format('%.1f', e.retry_after) : 'unknown'} path=#{e.request})")
+            { throttled: true, retry_after: e.retry_after, request: e.request }
           end
 
           def ingest_self(token:, **)
@@ -65,6 +77,13 @@ module Legion
             end
 
             { profile: profile, presence: presence }
+          # rubocop:disable Legion/RescueLogging/NoCapture
+          # Re-raise unlogged: full_ingest logs the throttle once and aborts
+          # the fan-out. A throttle is a fleet signal, not a per-stage error;
+          # logging it here too would just duplicate the line per stage.
+          rescue Errors::Throttled
+            raise
+            # rubocop:enable Legion/RescueLogging/NoCapture
           rescue StandardError => e
             handle_exception(e, level: :error, operation: 'ProfileIngest#ingest_self')
             { error: e.message }
@@ -97,6 +116,13 @@ module Legion
             end
 
             { people: people, count: people.length }
+          # rubocop:disable Legion/RescueLogging/NoCapture
+          # Re-raise unlogged: full_ingest logs the throttle once and aborts
+          # the fan-out. A throttle is a fleet signal, not a per-stage error;
+          # logging it here too would just duplicate the line per stage.
+          rescue Errors::Throttled
+            raise
+            # rubocop:enable Legion/RescueLogging/NoCapture
           rescue StandardError => e
             handle_exception(e, level: :error, operation: 'ProfileIngest#ingest_people')
             { error: e.message, skipped: false }
@@ -106,15 +132,19 @@ module Legion
             return { ingested: 0 } if people.empty?
 
             conn = graph_connection(token: token)
-            chats_resp = conn.get('me/chats', { '$top' => 50 })
-            chats = (chats_resp.body || {}).fetch('value', [])
+            chats = cached_graph_get(conn: conn, path: 'me/chats',
+                                     params: { '$top' => 50 })
+                    .then { |body| (body || {}).fetch('value', []) }
+            one_on_ones = chats.select { |c| c['chatType'] == 'oneOnOne' }
+
+            email_to_chat = build_chat_member_index(conn: conn, chats: one_on_ones)
             ingested = 0
 
             people.first(top_people).each do |person|
-              email = person.dig('scoredEmailAddresses', 0, 'address')
+              email = person.dig('scoredEmailAddresses', 0, 'address')&.downcase
               next unless email
 
-              chat = find_chat_for_person(chats: chats, email: email, conn: conn)
+              chat = email_to_chat[email]
               next unless chat
 
               messages = fetch_new_messages(conn: conn, chat_id: chat['id'], depth: message_depth)
@@ -142,6 +172,13 @@ module Legion
             end
 
             { ingested: ingested }
+          # rubocop:disable Legion/RescueLogging/NoCapture
+          # Re-raise unlogged: full_ingest logs the throttle once and aborts
+          # the fan-out. A throttle is a fleet signal, not a per-stage error;
+          # logging it here too would just duplicate the line per stage.
+          rescue Errors::Throttled
+            raise
+            # rubocop:enable Legion/RescueLogging/NoCapture
           rescue StandardError => e
             handle_exception(e, level: :error, operation: 'ProfileIngest#ingest_conversations')
             { error: e.message, ingested: ingested || 0 }
@@ -188,6 +225,13 @@ module Legion
             end
 
             { teams: teams_count, meetings: meetings_count }
+          # rubocop:disable Legion/RescueLogging/NoCapture
+          # Re-raise unlogged: full_ingest logs the throttle once and aborts
+          # the fan-out. A throttle is a fleet signal, not a per-stage error;
+          # logging it here too would just duplicate the line per stage.
+          rescue Errors::Throttled
+            raise
+            # rubocop:enable Legion/RescueLogging/NoCapture
           rescue StandardError => e
             handle_exception(e, level: :error, operation: 'ProfileIngest#ingest_teams_and_meetings')
             { error: e.message }
@@ -206,15 +250,27 @@ module Legion
 
           private
 
-          def find_chat_for_person(chats:, email:, conn:)
-            chats.select { |c| c['chatType'] == 'oneOnOne' }.find do |chat|
-              members_resp = conn.get("chats/#{chat['id']}/members")
-              members = (members_resp.body || {}).fetch('value', [])
-              members.any? { |m| m['email']&.downcase == email.downcase }
+          def build_chat_member_index(conn:, chats:)
+            index = {}
+            chats.each do |chat|
+              members = cached_graph_get(conn: conn, path: "chats/#{chat['id']}/members",
+                                         shared: true)
+                        .then { |body| (body || {}).fetch('value', []) }
+              members.each do |m|
+                email = m['email']&.downcase
+                index[email] = chat if email && !index.key?(email)
+              end
             end
+            index
+          # rubocop:disable Legion/RescueLogging/NoCapture
+          # Re-raise unlogged: propagate to full_ingest so the per-chat
+          # fan-out stops on the first throttle; full_ingest logs it once.
+          rescue Errors::Throttled
+            raise
+            # rubocop:enable Legion/RescueLogging/NoCapture
           rescue StandardError => e
-            handle_exception(e, level: :debug, operation: 'ProfileIngest#find_chat_for_person')
-            nil
+            handle_exception(e, level: :warn, operation: 'ProfileIngest#build_chat_member_index')
+            {}
           end
 
           def fetch_new_messages(conn:, chat_id:, depth: 50)
@@ -224,6 +280,12 @@ module Legion
 
             resp = conn.get("chats/#{chat_id}/messages", params)
             (resp.body || {}).fetch('value', [])
+          # rubocop:disable Legion/RescueLogging/NoCapture
+          # Re-raise unlogged: propagate to full_ingest so the per-chat
+          # fan-out stops on the first throttle; full_ingest logs it once.
+          rescue Errors::Throttled
+            raise
+            # rubocop:enable Legion/RescueLogging/NoCapture
           rescue StandardError => e
             handle_exception(e, level: :warn, operation: 'ProfileIngest#fetch_new_messages',
                              chat_id: chat_id)

data/lib/legion/extensions/microsoft_teams/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module MicrosoftTeams
-      VERSION = '0.6.50'
+      VERSION = '0.6.52'
     end
   end
 end

data/lib/legion/extensions/microsoft_teams.rb

@@ -2,6 +2,9 @@
 
 require 'legion/extensions/identity/entra/helpers/token_manager'
 require 'legion/extensions/microsoft_teams/version'
+require 'legion/extensions/microsoft_teams/errors'
+require 'legion/extensions/microsoft_teams/faraday/retry_after'
+require 'legion/extensions/microsoft_teams/helpers/throttle_aware'
 require 'legion/extensions/microsoft_teams/helpers/client'
 require 'legion/extensions/microsoft_teams/runners/auth'
 require 'legion/extensions/microsoft_teams/runners/teams'
@@ -29,6 +32,7 @@ require 'legion/extensions/microsoft_teams/runners/app_installations'
 require 'legion/extensions/microsoft_teams/runners/files'
 
 # Helpers (bot)
+require 'legion/extensions/microsoft_teams/helpers/graph_cache'
 require 'legion/extensions/microsoft_teams/helpers/high_water_mark'
 require 'legion/extensions/microsoft_teams/helpers/prompt_resolver'
 require 'legion/extensions/microsoft_teams/helpers/trace_retriever'
@@ -60,6 +64,62 @@ module Legion
     module MicrosoftTeams
       extend Legion::Extensions::Core if Legion::Extensions.const_defined? :Core, false
 
+      def self.default_settings # rubocop:disable Metrics/MethodLength
+        {
+          api_ingest:           {
+            enabled:       true,
+            interval:      3600,
+            top_people:    15,
+            message_depth: 50,
+            skip_bots:     true
+          },
+          incremental_sync:     {
+            enabled:       true,
+            interval:      900,
+            top_people:    10,
+            message_depth: 50
+          },
+          profile_ingest:       {
+            enabled:       true,
+            top_people:    10,
+            message_depth: 50
+          },
+          presence_poller:      {
+            enabled:  false,
+            interval: 300
+          },
+          meeting_ingest:       {
+            enabled:  true,
+            interval: 900
+          },
+          channel_poller:       {
+            enabled:               false,
+            interval:              120,
+            max_teams:             10,
+            max_channels_per_team: 5
+          },
+          direct_chat_poller:   {
+            enabled:  false,
+            interval: 30
+          },
+          observed_chat_poller: {
+            enabled:  false,
+            interval: 60
+          },
+          cache:                {
+            graph_ttl: 300
+          },
+          client:               {
+            throttle_circuit: {
+              soft_percentage: 0.8,
+              soft_ttl:        60,
+              fallback_ttl:    60,
+              insights_ttl:    600
+            }
+          }
+        }
+      end
+
       def self.trigger_words
         %w[teams microsoft_teams microsoftteams microsoft-teams msteams ms-teams]
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-microsoft_teams
 version: !ruby/object:Gem::Version
-  version: 0.6.50
+  version: 0.6.52
 platform: ruby
 authors:
 - Esity
@@ -207,13 +207,18 @@ files:
 - lib/legion/extensions/microsoft_teams/actors/presence_poller.rb
 - lib/legion/extensions/microsoft_teams/actors/profile_ingest.rb
 - lib/legion/extensions/microsoft_teams/client.rb
+- lib/legion/extensions/microsoft_teams/errors.rb
+- lib/legion/extensions/microsoft_teams/faraday/retry_after.rb
+- lib/legion/extensions/microsoft_teams/faraday/throttle_circuit.rb
 - lib/legion/extensions/microsoft_teams/helpers/client.rb
+- lib/legion/extensions/microsoft_teams/helpers/graph_cache.rb
 - lib/legion/extensions/microsoft_teams/helpers/graph_client.rb
 - lib/legion/extensions/microsoft_teams/helpers/high_water_mark.rb
 - lib/legion/extensions/microsoft_teams/helpers/permission_guard.rb
 - lib/legion/extensions/microsoft_teams/helpers/prompt_resolver.rb
 - lib/legion/extensions/microsoft_teams/helpers/session_manager.rb
 - lib/legion/extensions/microsoft_teams/helpers/subscription_registry.rb
+- lib/legion/extensions/microsoft_teams/helpers/throttle_aware.rb
 - lib/legion/extensions/microsoft_teams/helpers/trace_retriever.rb
 - lib/legion/extensions/microsoft_teams/helpers/transform_definitions.rb
 - lib/legion/extensions/microsoft_teams/local_cache/extractor.rb