lex-microsoft_teams 0.6.51 → 0.6.53

data/lib/legion/extensions/microsoft_teams/helpers/throttle_aware.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+require 'monitor'
+require 'legion/extensions/microsoft_teams/errors'
+
+module Legion
+  module Extensions
+    module MicrosoftTeams
+      module Helpers
+        # Mixin that lets an `Actors::Every`-style poller honour a Graph
+        # throttle by *deferring its own next run* instead of re-firing on
+        # its fixed timer interval.
+        #
+        # Background: `Faraday::RetryAfter` raises `Errors::Throttled`
+        # centrally when it exhausts retries (or when the advertised wait
+        # alone exceeds the retry budget), and `Faraday::ThrottleCircuit`
+        # opens a shared circuit so the whole fleet stops hammering a quota
+        # that Graph has already flagged. But the actors that *drive* those
+        # calls schedule themselves with a `Concurrent::TimerTask` whose
+        # `execution_interval` is fixed at boot. Catching `Throttled` in a
+        # `rescue` block stops the current tick, but the next tick still
+        # fires on the original cadence — so a poller on a 30–300s interval
+        # keeps walking straight back into an open circuit, emitting an
+        # ERROR every interval and contributing nothing but log noise (and,
+        # via the shared circuit, refreshing the block for cheaper callers).
+        #
+        # This mixin closes that gap. An actor wraps its Graph work in
+        # {#with_throttle_deferral}. On `Errors::Throttled` it records a
+        # "suppress until" instant of `now + retry_after` (falling back to a
+        # bounded default when the server gave no usable `Retry-After`).
+        # Subsequent ticks short-circuit via {#throttle_suppressed?} until
+        # that instant passes, so the actor effectively reschedules itself
+        # at `now + retry_after` rather than `now + interval` — exactly the
+        # behaviour the throttle integration spec and the 0.6.51 CHANGELOG
+        # flagged as the outstanding follow-up.
+        #
+        # State is per-actor-instance and guarded by a monitor because an
+        # actor's `manual` runs on a `Concurrent` thread-pool worker. The
+        # mixin reads the carried `retry_after` only when
+        # `retry_after_known?` is true; otherwise it applies
+        # {DEFAULT_DEFERRAL} so a throttle with a missing/garbage header
+        # still backs the poller off rather than letting it spin.
+        module ThrottleAware
+          # Fallback deferral (seconds) used when a `Throttled` carries no
+          # usable `retry_after` (header absent or unparseable). One minute
+          # matches Microsoft Graph's documented typical Retry-After and the
+          # circuit middleware's `DEFAULT_FALLBACK_TTL`.
+          DEFAULT_DEFERRAL = 60.0
+
+          # Hard ceiling (seconds) on a single deferral so a pathological
+          # advertised `Retry-After` can't park a poller for an unbounded
+          # time. Mirrors the circuit middleware's 600s cap in
+          # `ThrottleCircuit#set_hard_circuit`.
+          MAX_DEFERRAL = 600.0
+
+          # Run `block` unless the actor is currently deferring after a
+          # recent throttle. If a `Throttled` escapes the block, record the
+          # deferral window and swallow it (the throttle has already been
+          # logged at the middleware/circuit layer; re-raising would just
+          # trip the actor's generic `rescue StandardError` and double-log).
+          #
+          # @param now [Time] injectable clock for deterministic tests
+          # @return [Object, nil] the block's value, or nil when suppressed
+          #   or when a throttle was caught
+          def with_throttle_deferral(now: Time.now)
+            if throttle_suppressed?(now: now)
+              log_throttle_skip(now: now)
+              return nil
+            end
+
+            yield
+          rescue Legion::Extensions::MicrosoftTeams::Errors::Throttled => e
+            defer_after_throttle(e, now: now)
+            nil
+          end
+
+          # @return [Boolean] true while the actor is inside a deferral
+          #   window opened by a previous throttle.
+          def throttle_suppressed?(now: Time.now)
+            until_at = throttled_until
+            !until_at.nil? && now < until_at
+          end
+
+          # The instant before which the actor should not poll again, or nil
+          # if no deferral is active. Thread-safe read.
+          def throttled_until
+            throttle_monitor.synchronize { @throttled_until }
+          end
+
+          # Seconds remaining in the current deferral window (0.0 if none /
+          # elapsed). Useful for logging and for tests.
+          def throttle_remaining(now: Time.now)
+            until_at = throttled_until
+            return 0.0 if until_at.nil?
+
+            remaining = until_at - now
+            remaining.positive? ? remaining : 0.0
+          end
+
+          # Open (or extend) a deferral window of `seconds` from `now`. A
+          # later throttle never *shortens* an existing window — we keep the
+          # furthest-out instant so overlapping throttles compose safely.
+          def defer_for(seconds, now: Time.now)
+            window = clamp_deferral(seconds)
+            target = now + window
+            throttle_monitor.synchronize do
+              @throttled_until = if @throttled_until.nil? || target > @throttled_until
+                                   target
+                                 else
+                                   @throttled_until
+                                 end
+            end
+            window
+          end
+
+          # Clear any active deferral. Called implicitly is unnecessary —
+          # {#throttle_suppressed?} expires on its own once the clock passes
+          # `throttled_until` — but exposed for tests and explicit resets.
+          def reset_throttle_deferral
+            throttle_monitor.synchronize { @throttled_until = nil }
+          end
+
+          private
+
+          def defer_after_throttle(error, now: Time.now)
+            seconds = deferral_seconds_for(error)
+            window  = defer_for(seconds, now: now)
+            log.warn(
+              "[microsoft_teams][throttle_defer] #{actor_label}: " \
+              "deferring next run #{format('%.1f', window)}s " \
+              "(retry_after=#{error.retry_after_known? ? format('%.1f', error.retry_after) : 'unknown'} " \
+              "path=#{error.request})"
+            )
+          end
+
+          # Short, log-friendly name for the including actor. Falls back to
+          # the full class string for anonymous classes (whose `name` is
+          # nil), so the log line never raises on `name.split`.
+          def actor_label
+            (self.class.name || self.class.to_s).split('::').last
+          end
+
+          # Prefer the server-advised wait; fall back to {DEFAULT_DEFERRAL}
+          # only when the header was absent or unparseable.
+          def deferral_seconds_for(error)
+            if error.respond_to?(:retry_after_known?) && error.retry_after_known?
+              error.retry_after
+            else
+              DEFAULT_DEFERRAL
+            end
+          end
+
+          def clamp_deferral(seconds)
+            value = begin
+              Float(seconds)
+              # rubocop:disable Legion/RescueLogging/NoCapture
+              # Pure coercion fallback for a non-numeric deferral; the value
+              # is bounded immediately below, so a bad input degrades to the
+              # default rather than warranting its own log line.
+            rescue ArgumentError, TypeError
+              DEFAULT_DEFERRAL
+              # rubocop:enable Legion/RescueLogging/NoCapture
+            end
+            value = DEFAULT_DEFERRAL if value <= 0
+            [value, MAX_DEFERRAL].min
+          end
+
+          def log_throttle_skip(now: Time.now)
+            log.debug(
+              "[microsoft_teams][throttle_defer] #{actor_label}: " \
+              "skipping run, #{format('%.1f', throttle_remaining(now: now))}s deferral remaining"
+            )
+          end
+
+          def throttle_monitor
+            # `||=` is safe here: the first tick of an actor runs before any
+            # concurrent re-entry (the `Every` base guards re-entry with its
+            # own AtomicBoolean), so the monitor is initialised single-threaded.
+            @throttle_monitor ||= Monitor.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/microsoft_teams/helpers/trace_retriever.rb

@@ -131,7 +131,7 @@ module Legion
             return unless defined?(log)
 
             if defined?(handle_exception)
-              handle_exception(error, level: :debug, operation: "TraceRetriever##{method}")
+              handle_exception(error, level: :warn, operation: "TraceRetriever##{method}")
             else
               log.debug("TraceRetriever##{method} failed: #{error.message}")
             end

data/lib/legion/extensions/microsoft_teams/local_cache/sstable_reader.rb

@@ -102,7 +102,7 @@ module Legion
           def decode_varint(data, pos)
             result = 0
             shift = 0
-            loop do
+            10.times do # varint max 10 bytes (64-bit)
               return [nil, pos] if pos >= data.bytesize
 
               byte = data.getbyte(pos)

data/lib/legion/extensions/microsoft_teams/runners/api_ingest.rb

@@ -2,6 +2,7 @@
 
 require 'json'
 require 'digest'
+require 'legion/extensions/microsoft_teams/errors'
 require 'legion/extensions/microsoft_teams/helpers/client'
 require 'legion/extensions/microsoft_teams/helpers/graph_cache'
 require 'legion/extensions/microsoft_teams/helpers/permission_guard'
@@ -21,16 +22,12 @@ module Legion
 
           definition :ingest_api, mcp_exposed: false
 
-          # Fetch top contacts via /me/people, then pull recent messages from
-          # their 1:1 chats. Stores each message as an individual memory trace
-          # (same format as CacheIngest) with dedup by content hash.
-          #
-          # Requires a delegated token with Chat.Read and People.Read scopes.
           def ingest_api(token:, top_people: 15, message_depth: 50, skip_bots: true, imprint_active: false, **) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
             log.debug("ApiIngest#ingest_api top_people=#{top_people} message_depth=#{message_depth}")
             return error_result('lex-memory not loaded') unless memory_available?
             return error_result('no token provided') unless token && !token.empty?
 
+            @fetch_failures = 0
             restore_hwm_from_traces
 
             people = fetch_top_people(token: token, top: top_people)
@@ -103,16 +100,29 @@ module Legion
 
             { result: { stored: stored, skipped: skipped, people_ingested: people_ingested,
                         people_found: people.length, chats_found: chats.length,
+                        fetch_failures: @fetch_failures,
                         apollo: apollo_results } }
+          # rubocop:disable Legion/RescueLogging/NoCapture
+          # Re-raise unlogged: surface the typed throttle to the caller (the
+          # ApiIngest actor) so it can defer its next scheduled run by the
+          # advertised retry_after. The throttle is already logged at the
+          # middleware/circuit layer; folding it into an error result here
+          # would hide the one signal the actor needs to stop re-charging
+          # the shared Graph circuit on its fixed interval.
+          rescue Errors::Throttled
+            raise
+            # rubocop:enable Legion/RescueLogging/NoCapture
           rescue StandardError => e
             handle_exception(e, level: :error, operation: 'ApiIngest#ingest_api')
-            { result: { stored: stored || 0, skipped: skipped || 0, error: e.message } }
+            { result: { stored: stored || 0, skipped: skipped || 0,
+                        fetch_failures: @fetch_failures || 0, error: e.message } }
           end
 
           include Legion::Extensions::Helpers::Lex if Legion::Extensions.const_defined?(:Helpers, false) &&
                                                       Legion::Extensions::Helpers.const_defined?(:Lex, false)
 
           MAX_CHAT_PAGES = 10
+          CHAT_TYPE_PRIORITY = { 'oneOnOne' => 0, 'group' => 1, 'meeting' => 2 }.freeze
 
           private
 
@@ -124,15 +134,24 @@ module Legion
             resp = conn.get('me/people', { '$top' => top })
 
             log.debug("ApiIngest: fetch_top_people status=#{resp.status} count=#{(resp.body || {}).fetch('value', []).size}")
-            if resp.status == 403
-              record_denial('/me/people', resp.body.dig('error', 'message') || 'Forbidden')
+            unless (200..299).cover?(resp.status)
+              error_code = resp.body&.dig('error', 'code')
+              log.warn('[microsoft_teams][api_ingest] fetch_top_people non-2xx: ' \
+                       "status=#{resp.status} error_code=#{error_code}")
+              record_denial('/me/people', resp.body&.dig('error', 'message') || 'Forbidden') if resp.status == 403
+              @fetch_failures = (@fetch_failures || 0) + 1
               return []
             end
 
             people = (resp.body || {}).fetch('value', [])
             people.sort_by { |p| -(p.dig('scoredEmailAddresses', 0, 'relevanceScore') || 0) }
+          # rubocop:disable Legion/RescueLogging/NoCapture
+          rescue Errors::Throttled
+            raise
+            # rubocop:enable Legion/RescueLogging/NoCapture
           rescue StandardError => e
             handle_exception(e, level: :warn, operation: 'ApiIngest#fetch_top_people')
+            @fetch_failures = (@fetch_failures || 0) + 1
             []
           end
 
@@ -143,8 +162,17 @@ module Legion
             params = { '$top' => 50 }
             pages = 0
 
-            loop do
+            MAX_CHAT_PAGES.times do
               resp = conn.get(url, params)
+
+              unless (200..299).cover?(resp.status)
+                error_code = resp.body&.dig('error', 'code')
+                log.warn('[microsoft_teams][api_ingest] fetch_one_on_one_chats non-2xx: ' \
+                         "status=#{resp.status} error_code=#{error_code}")
+                @fetch_failures = (@fetch_failures || 0) + 1
+                break
+              end
+
               body = resp.body || {}
               chats = body.fetch('value', [])
               all_chats.concat(chats)
@@ -152,7 +180,6 @@ module Legion
 
               next_link = body['@odata.nextLink']
               break unless next_link
-              break if pages >= MAX_CHAT_PAGES
 
               url = next_link
               params = {}
@@ -162,8 +189,13 @@ module Legion
             filtered = all_chats.select { |c| allowed_types.include?(c['chatType']) }
             log.info("ApiIngest: fetched #{all_chats.size} chats (#{pages} pages), #{filtered.size} eligible (1:1/group/meeting)")
             filtered
+          # rubocop:disable Legion/RescueLogging/NoCapture
+          rescue Errors::Throttled
+            raise
+            # rubocop:enable Legion/RescueLogging/NoCapture
           rescue StandardError => e
             handle_exception(e, level: :warn, operation: 'ApiIngest#fetch_one_on_one_chats')
+            @fetch_failures = (@fetch_failures || 0) + 1
             []
           end
 
@@ -172,9 +204,10 @@ module Legion
             by_user_id = {}
             by_name = {}
 
-            chats.each do |chat|
+            sorted = chats.sort_by { |c| CHAT_TYPE_PRIORITY[c['chatType']] || 99 }
+            sorted.each do |chat|
               members = cached_graph_get(conn: conn, path: "chats/#{chat['id']}/members",
-                                         shared: true)
+                                         shared: true, ttl: members_cache_ttl)
                         .then { |body| (body || {}).fetch('value', []) }
               members.each do |m|
                 email = m['email']&.downcase
@@ -190,8 +223,13 @@ module Legion
             end
 
             { email: by_email, user_id: by_user_id, name: by_name }
+          # rubocop:disable Legion/RescueLogging/NoCapture
+          rescue Errors::Throttled
+            raise
+            # rubocop:enable Legion/RescueLogging/NoCapture
           rescue StandardError => e
             handle_exception(e, level: :warn, operation: 'ApiIngest#build_chat_member_index')
+            @fetch_failures = (@fetch_failures || 0) + 1
             { email: {}, user_id: {}, name: {} }
           end
 
@@ -205,17 +243,41 @@ module Legion
               chat_index[:name][display_name]
           end
 
+          def hwm_client_side_cut(messages:, hwm:)
+            return messages unless hwm&.dig(:last_message_at)
+
+            cutoff = hwm[:last_message_at]
+            seen = Set.new
+            messages.take_while { |m| m['createdDateTime'] && m['createdDateTime'] > cutoff }
+                    .select { |m| seen.add?(m['id'] || Digest::SHA256.hexdigest(m.dig('body', 'content').to_s)[0, 16]) }
+          end
+
           def fetch_chat_messages(conn:, chat_id:, depth: 50)
             hwm = get_extended_hwm(chat_id: chat_id)
             params = { '$top' => depth, '$orderby' => 'createdDateTime desc' }
-            params['$filter'] = "createdDateTime gt #{hwm[:last_message_at]}" if hwm&.dig(:last_message_at)
 
             resp = conn.get("chats/#{chat_id}/messages", params)
-            log.debug("ApiIngest: fetch_messages chat=#{chat_id} count=#{(resp.body || {}).fetch('value', []).size}")
-            (resp.body || {}).fetch('value', [])
+
+            unless (200..299).cover?(resp.status)
+              error_code = resp.body&.dig('error', 'code')
+              log.warn('[microsoft_teams][api_ingest] fetch_chat_messages non-2xx: ' \
+                       "chat_id=#{chat_id} status=#{resp.status} error_code=#{error_code}")
+              @fetch_failures = (@fetch_failures || 0) + 1
+              return []
+            end
+
+            messages = (resp.body || {}).fetch('value', [])
+            messages = hwm_client_side_cut(messages: messages, hwm: hwm)
+            log.debug("ApiIngest: fetch_messages chat=#{chat_id} count=#{messages.size}")
+            messages
+          # rubocop:disable Legion/RescueLogging/NoCapture
+          rescue Errors::Throttled
+            raise
+            # rubocop:enable Legion/RescueLogging/NoCapture
           rescue StandardError => e
             handle_exception(e, level: :warn, operation: 'ApiIngest#fetch_chat_messages',
                              chat_id: chat_id)
+            @fetch_failures = (@fetch_failures || 0) + 1
             []
           end
 
@@ -284,7 +346,7 @@ module Legion
             end
             hashes
           rescue StandardError => e
-            handle_exception(e, level: :debug, operation: 'ApiIngest#load_existing_hashes')
+            handle_exception(e, level: :warn, operation: 'ApiIngest#load_existing_hashes')
             Set.new
           end
 
@@ -313,12 +375,12 @@ module Legion
               trace_ids.each_cons(2) do |id_a, id_b|
                 store.record_coactivation(id_a, id_b)
               rescue StandardError => e
-                handle_exception(e, level: :debug, operation: 'ApiIngest#coactivate_thread_traces',
+                handle_exception(e, level: :warn, operation: 'ApiIngest#coactivate_thread_traces',
                                  id_a: id_a, id_b: id_b)
               end
             end
           rescue StandardError => e
-            handle_exception(e, level: :debug, operation: 'ApiIngest#coactivate_thread_traces')
+            handle_exception(e, level: :warn, operation: 'ApiIngest#coactivate_thread_traces')
           end
 
           def publish_to_apollo(person_texts)
@@ -377,7 +439,7 @@ module Legion
 
             { success: true, count: result[:entities].length }
           rescue StandardError => e
-            handle_exception(e, level: :debug, operation: 'ApiIngest#extract_and_ingest_entities',
+            handle_exception(e, level: :warn, operation: 'ApiIngest#extract_and_ingest_entities',
                              person_name: person_name)
             { success: false, count: 0 }
           end
@@ -396,6 +458,16 @@ module Legion
             @apollo_knowledge_runner ||= Object.new.extend(Legion::Extensions::Apollo::Runners::Knowledge)
           end
 
+          def members_cache_ttl
+            return @members_cache_ttl if defined?(@members_cache_ttl)
+
+            @members_cache_ttl = if respond_to?(:settings, true) && settings.respond_to?(:dig)
+                                   settings.dig(:cache, :members_ttl) || 86_400
+                                 else
+                                   86_400
+                                 end
+          end
+
           def error_result(message)
             { result: { stored: 0, skipped: 0, error: message } }
           end

data/lib/legion/extensions/microsoft_teams/runners/bot.rb

@@ -200,7 +200,7 @@ module Legion
 
             retrieve_context(message: message, owner_id: owner_id, chat_id: chat_id)
           rescue StandardError => e
-            handle_exception(e, level: :debug, operation: 'Bot#retrieve_trace_context') if defined?(handle_exception)
+            handle_exception(e, level: :warn, operation: 'Bot#retrieve_trace_context') if defined?(handle_exception)
             nil
           end
 
@@ -231,7 +231,7 @@ module Legion
             Legion::Settings.dig(:microsoft_teams, :bot, :observe, :enabled) == true
           end
 
-          def notify_enabled?(**_kwargs)
+          def notify_enabled?(**)
             return false unless defined?(Legion::Settings)
 
             Legion::Settings.dig(:microsoft_teams, :bot, :observe, :notify) == true

data/lib/legion/extensions/microsoft_teams/runners/cache_ingest.rb

@@ -123,12 +123,12 @@ module Legion
               trace_ids.each_cons(2) do |id_a, id_b|
                 store.record_coactivation(id_a, id_b)
               rescue StandardError => e
-                handle_exception(e, level: :debug, operation: 'CacheIngest#coactivate_thread_traces',
+                handle_exception(e, level: :warn, operation: 'CacheIngest#coactivate_thread_traces',
                                  id_a: id_a, id_b: id_b)
               end
             end
           rescue StandardError => e
-            handle_exception(e, level: :debug, operation: 'CacheIngest#coactivate_thread_traces')
+            handle_exception(e, level: :warn, operation: 'CacheIngest#coactivate_thread_traces')
           end
         end
       end