openc3 7.4.0 → 7.5.0

data/lib/openc3/utilities/questdb_client.rb

@@ -632,6 +632,20 @@ module OpenC3
       false
     end
 
+    # Return the names of all tables in a single query. Used to avoid querying
+    # (and failing on) tables that were never created, e.g. commands that have
+    # never been sent, which is the common case for targets with thousands of packets.
+    # QuestDB table names are case insensitive so the names are upcased.
+    #
+    # @return [Set<String>, nil] Upcased table names, or nil if they cannot be queried
+    def self.table_names(db_shard: 0)
+      result = query_with_retry("SELECT table_name FROM tables()", max_retries: 1, label: "table names", db_shard: db_shard)
+      return nil unless result
+      result.values.each_with_object(Set.new) { |row, set| set << row[0].to_s.upcase }
+    rescue StandardError
+      nil
+    end
+
     # QuestDB column types that can be aggregated with min/max/avg/stddev.
     # Used by reduced queries to decide whether a converted (__C) column is
     # numeric or must fall back to the raw column (e.g. states stored as VARCHAR).
@@ -928,6 +942,10 @@ module OpenC3
     # @return [Array, Hash] Array of [value, limits_state] pairs per row, or {} if no results.
     #   Single-row results return a flat array; multi-row results return array of arrays.
     def self.tsdb_lookup(items, start_time:, end_time: nil, scope: "DEFAULT")
+      # Every item is a placeholder for an item which doesn't exist, so there's
+      # nothing to query. Return a single row of nil values, one per item.
+      return Array.new(items.length) { [nil, nil] } if items.all? { |item| item[2].nil? }
+
       # Group items by db_shard number while preserving their original positions
       db_shard_groups = {} # db_shard => { positions: [], items: [] }
       items.each_with_index do |item, pos|
@@ -944,62 +962,36 @@ module OpenC3
         return tsdb_lookup_single_db_shard(group[:items], start_time: start_time, end_time: end_time, scope: scope, db_shard: db_shard)
       end
 
-      # Cross-db_shard: execute per-db_shard queries and merge results
-      db_shard_results = {} # db_shard => data
+      # Cross-db_shard: execute per-db_shard queries (as arrays of rows) and merge results
+      db_shard_rows = {} # db_shard => rows
       db_shard_groups.each do |db_shard, group|
-        result = tsdb_lookup_single_db_shard(group[:items], start_time: start_time, end_time: end_time, scope: scope, db_shard: db_shard)
-        db_shard_results[db_shard] = result
+        result = tsdb_lookup_single_db_shard(group[:items], start_time: start_time, end_time: end_time, scope: scope, db_shard: db_shard, flatten: false)
+        db_shard_rows[db_shard] = result.is_a?(Array) ? result : []
       end
 
-      # If all db_shards returned empty, return empty
-      return {} if db_shard_results.values.all? { |r| r == {} }
-
-      # Merge results positionally back into the original item order.
-      # For single-row results (no end_time), merge flat arrays.
-      # For multi-row results, each db_shard may have different row counts;
-      # use the maximum row count and fill missing positions with [nil, nil].
-      if !end_time
-        # Single-row mode: each db_shard returns a flat array of [value, limits] pairs.
-        # Merge them into the original item order.
-        merged = Array.new(items.length) { [nil, nil] }
-        db_shard_groups.each do |db_shard, group|
-          result = db_shard_results[db_shard]
-          next if result == {} || !result.is_a?(Array)
+      # Merge results positionally back into the original item order. Each db_shard
+      # may have different row counts so use the maximum row count and fill missing
+      # positions with [nil, nil]. If all db_shards returned empty, return empty.
+      max_rows = db_shard_rows.values.map(&:length).max
+      return {} if max_rows == 0
+
+      merged = Array.new(max_rows) { Array.new(items.length) { [nil, nil] } }
+      db_shard_groups.each do |db_shard, group|
+        db_shard_rows[db_shard].each_with_index do |row, row_num|
           group[:positions].each_with_index do |orig_pos, db_shard_idx|
-            merged[orig_pos] = result[db_shard_idx] if result[db_shard_idx]
+            merged[row_num][orig_pos] = row[db_shard_idx] if row[db_shard_idx]
           end
         end
-        merged
-      else
-        # Multi-row mode: find max row count across db_shards
-        max_rows = 0
-        db_shard_groups.each do |db_shard, _group|
-          result = db_shard_results[db_shard]
-          next if result == {}
-          count = result.is_a?(Array) ? result.length : 0
-          max_rows = count if count > max_rows
-        end
-        return {} if max_rows == 0
-
-        merged = Array.new(max_rows) { Array.new(items.length) { [nil, nil] } }
-        db_shard_groups.each do |db_shard, group|
-          result = db_shard_results[db_shard]
-          next if result == {}
-          rows = result.is_a?(Array) ? result : []
-          rows.each_with_index do |row, row_num|
-            next unless row.is_a?(Array)
-            group[:positions].each_with_index do |orig_pos, db_shard_idx|
-              merged[row_num][orig_pos] = row[db_shard_idx] if row[db_shard_idx]
-            end
-          end
-        end
-        merged
       end
+      # Match the single db_shard behavior of returning a flat array for a single row
+      return merged[0] if max_rows == 1
+      merged
     end
 
     # Execute a tsdb_lookup query against a single db_shard.
     # This contains the original ASOF JOIN logic for items all on the same QuestDB instance.
-    def self.tsdb_lookup_single_db_shard(items, start_time:, end_time: nil, scope: "DEFAULT", db_shard: 0)
+    # When flatten is true a single row result is returned as a flat array rather than an array of rows.
+    def self.tsdb_lookup_single_db_shard(items, start_time:, end_time: nil, scope: "DEFAULT", db_shard: 0, flatten: true)
       tables = {}
       names = []
       nil_count = 0
@@ -1060,6 +1052,11 @@ module OpenC3
         end
       end
 
+      # Every item in this db_shard is a placeholder so there's no table to query.
+      # Return no results and let tsdb_lookup fill these positions with [nil, nil]
+      # when it merges the db_shards (an all placeholder lookup returns before this)
+      return {} if tables.empty?
+
       # Add needed timestamp columns to the SELECT for calculated items
       needed_timestamps.each do |table_index, ts_columns|
         ts_columns.each do |ts_col|
@@ -1147,7 +1144,7 @@ module OpenC3
           data[row_num].insert(position, [calculated_value, nil])
         end
       end
-      if result.ntuples == 1
+      if flatten and result.ntuples == 1
         data = data[0]
       end
       data

data/lib/openc3/utilities/running_script.rb

@@ -112,16 +112,7 @@ module OpenC3
               RunningScript.instance.perform_pause
             else
               if method.to_s == 'open_bucket_dialog'
-                bucket = input['bucket']
-                path = input['path']
-                filename = input['filename']
-                # Path from bucket dialog already includes scope prefix (e.g. DEFAULT/targets/...)
-                # _get_storage_file prepends scope, so strip it from the path to avoid doubling
-                scope = RunningScript.instance.scope
-                path = path.sub(/\A#{Regexp.escape(scope)}\//, '')
-                file = _get_storage_file(path, bucket: bucket, scope: scope)
-                file.filename = filename
-                return file
+                return _get_bucket_dialog_file(input, scope: RunningScript.instance.scope)
               elsif (method.to_s.include?('open_file'))
                 files = input.map do |filename|
                   file = _get_storage_file("tmp/#{filename}", scope: RunningScript.instance.scope)
@@ -275,10 +266,19 @@ module OpenC3
 
       def goto(line_no_or_procedure_name, line_no = nil)
         if line_no.nil?
-          start(RunningScript.instance.current_filename, line_no: line_no_or_procedure_name, bind_variables: true, complete: true)
+          procedure_name = RunningScript.instance.current_filename
+          line_no = line_no_or_procedure_name
         else
-          start(line_no_or_procedure_name, line_no: line_no, bind_variables: true, complete: true)
+          procedure_name = line_no_or_procedure_name
         end
+        # goto typed at the debug prompt is evaluated on the thread that processes
+        # go/step/stop messages. Running the script there deadlocks at the next pause,
+        # so hand it to the run thread the same way the editor's goto does.
+        unless RunningScript.run_thread?
+          RunningScript.instance.execute_while_paused(procedure_name, line_no, nil, true)
+          return
+        end
+        start(procedure_name, line_no: line_no, bind_variables: true, complete: true)
       end
 
       # Require an additional ruby file
@@ -797,6 +797,10 @@ class RunningScript
     @@instance = value
   end
 
+  def self.run_thread?
+    Thread.current == @@run_thread
+  end
+
   def self.line_delay
     @@line_delay
   end
@@ -1103,9 +1107,10 @@ class RunningScript
     trace
   end
 
-  def execute_while_paused(filename, line_no = 1, end_line_no = nil)
+  # bind_variables of nil binds only when filename is the currently running file
+  def execute_while_paused(filename, line_no = 1, end_line_no = nil, bind_variables = nil)
     if @script_status.state == 'paused' or @script_status.state == 'error' or @script_status.state == 'breakpoint'
-      @execute_while_paused_info = { filename: filename, line_no: line_no, end_line_no: end_line_no }
+      @execute_while_paused_info = { filename: filename, line_no: line_no, end_line_no: end_line_no, bind_variables: bind_variables }
     else
       scriptrunner_puts("Cannot execute selection or goto unless script is paused, breakpoint, or in error state")
     end
@@ -1217,10 +1222,9 @@ class RunningScript
 
   def check_execute_while_paused
     if @execute_while_paused_info
-      if @script_status.current_filename == @execute_while_paused_info[:filename]
-        bind_variables = true
-      else
-        bind_variables = false
+      bind_variables = @execute_while_paused_info[:bind_variables]
+      if bind_variables.nil?
+        bind_variables = (@script_status.current_filename == @execute_while_paused_info[:filename])
       end
       if @execute_while_paused_info[:end_line_no]
         # Execute Selection While Paused

data/lib/openc3/utilities/store_autoload.rb

@@ -54,6 +54,14 @@ module OpenC3
     @@db_shard_cache_mutex = Mutex.new
     DB_SHARD_CACHE_TIMEOUT = 60 # seconds
 
+    # Maximum number of entries read_topics returns PER STREAM when the caller
+    # does not pass a count. XREAD without COUNT returns everything from the
+    # offset to the tail of the stream, which for a consumer that has fallen
+    # behind on a high-rate stream can be an enormous reply materialized in
+    # memory all at once. Callers loop on their offsets so a capped read simply
+    # continues on the next call.
+    READ_TOPICS_DEFAULT_COUNT = 1000
+
     # Mutex used to ensure that only one instance is created
     @@instance_mutex = Mutex.new
 
@@ -225,8 +233,10 @@ module OpenC3
       return offsets
     end
 
+    # @param count [Integer] Maximum entries to return per stream. nil uses READ_TOPICS_DEFAULT_COUNT.
     def read_topics(topics, offsets = nil, timeout_ms = 1000, count = nil)
       return {} if topics.empty?
+      count ||= READ_TOPICS_DEFAULT_COUNT
       Thread.current[:topic_offsets] ||= {}
       topic_offsets = Thread.current[:topic_offsets]
       begin
@@ -264,13 +274,20 @@ module OpenC3
     # @option opts [String]  :id          the entry id, default value is `*`, it means auto generation,
     #   if `nil` id is passed it will be changed to `*`
     # @option opts [Integer] :maxlen      max length of entries, default value is `nil`, it means will grow forever
-    # @option opts [String] :approximate whether to add `~` modifier of maxlen or not, default value is 'true'
+    # @option opts [String] :approximate whether to add `~` modifier of maxlen or minid or not, default value is 'true'
+    # @option opts [String] :minid       trim entries with ids lower than this as part of the add (XADD MINID).
+    #   Cannot be combined with maxlen. See Topic.stream_safety_minid for the safety cap applied to target streams.
     #
     # @return [String] the entry id
-    def write_topic(topic, msg_hash, id = '*', maxlen = nil, approximate = 'true')
+    def write_topic(topic, msg_hash, id = '*', maxlen = nil, approximate = 'true', minid: nil)
       id = '*' if id.nil?
+      raise ArgumentError, "write_topic can't supply both maxlen and minid" if maxlen and minid
       @redis_pool.with do |redis|
-        return redis.xadd(topic, msg_hash, id: id, maxlen: maxlen, approximate: approximate)
+        if minid
+          return redis.xadd(topic, msg_hash, id: id, minid: minid, approximate: approximate)
+        else
+          return redis.xadd(topic, msg_hash, id: id, maxlen: maxlen, approximate: approximate)
+        end
       end
     end
 

data/lib/openc3/utilities/store_queued.rb

@@ -24,20 +24,32 @@ module OpenC3
     # Mutex used to ensure that only one instance is created
     @@instance_mutex = Mutex.new
 
+    DEFAULT_UPDATE_INTERVAL = 1 # seconds
+
     # Get the singleton instance for the given db_shard
-    # Sets the update interval to 1 second by default
-    def self.instance(update_interval = 1, db_shard: 0) # seconds
+    # Uses the interval from set_update_interval_all_shards if not given
+    def self.instance(update_interval = nil, db_shard: 0)
       @instances ||= []
       the_instance = @instances[db_shard]
       return the_instance if the_instance
 
       @@instance_mutex.synchronize do
         @instances ||= []
-        @instances[db_shard] ||= self.new(update_interval, db_shard: db_shard)
+        @instances[db_shard] ||= self.new(update_interval || @update_interval || DEFAULT_UPDATE_INTERVAL, db_shard: db_shard)
         return @instances[db_shard]
       end
     end
 
+    # Set the update interval on every db_shard instance, including ones created later
+    def self.set_update_interval_all_shards(interval)
+      return unless interval > 0.0
+
+      @@instance_mutex.synchronize do
+        @update_interval = interval
+        (@instances || []).compact.each { |the_instance| the_instance.set_update_interval(interval) }
+      end
+    end
+
     # Delegate all unknown class methods to delegate to the instance (db_shard 0)
     def self.method_missing(message, *args, **kwargs, &)
       self.instance.public_send(message, *args, **kwargs, &)
@@ -49,6 +61,8 @@ module OpenC3
       @store = store_instance()
       # Queue to hold the store requests
       @store_queue = Queue.new
+      # Orders queueing against the final drain in shutdown
+      @shutdown_mutex = Mutex.new
       # Sleeper used to delay update thread
       @update_sleeper = Sleeper.new
 
@@ -102,11 +116,18 @@ module OpenC3
       @update_sleeper.cancel if @update_sleeper
       OpenC3.kill_thread(self, @update_thread) if @update_thread
       @update_thread = nil
-      # Drain the queue before shutdown
-      begin
-        process_queue()
-      rescue
-        # Best effort - Redis may already be unavailable during shutdown
+      # Nothing drains the queue once the thread is gone, so from here on write
+      # straight through rather than silently dropping whatever gets queued.
+      # Drain first and flip the flag last, under the mutex, so a direct write
+      # can't be overwritten by an older queued one and nothing is queued after
+      # the drain.
+      @shutdown_mutex.synchronize do
+        begin
+          process_queue()
+        rescue
+          # Best effort - Redis may already be unavailable during shutdown
+        end
+        @shutdown = true
       end
     end
 
@@ -114,7 +135,14 @@ module OpenC3
 
     # Record the message for pipelining by the thread
     def method_missing(message, *args, **kwargs, &block)
-      @store_queue.push(MessageStruct.new(message, args, kwargs, block))
+      # @shutdown never goes back to false, so once set the lock can be skipped
+      unless @shutdown
+        queued = @shutdown_mutex.synchronize do
+          @store_queue.push(MessageStruct.new(message, args, kwargs, block)) unless @shutdown
+        end
+        return queued if queued
+      end
+      @store.public_send(message, *args, **kwargs, &block)
     end
 
     # Returns the store we're working with

data/lib/openc3/utilities/target_file.rb

@@ -31,6 +31,15 @@ module OpenC3
       name.sub(/\*$/, '')
     end
 
+    # Reject names that could resolve outside the scope's targets_modified
+    # directory. The bucket normalizes '..' segments in keys just like the
+    # filesystem does, so this must be checked before any bucket call.
+    def self.validate_name(name)
+      unless OpenC3::LocalMode.safe_key?(name)
+        raise ArgumentError, "Invalid target file name: #{name.inspect}"
+      end
+    end
+
     def self.all(scope, path_matchers, target: nil)
       target = target.upcase if target
 
@@ -46,14 +55,26 @@ module OpenC3
         prefix = "#{scope}/targets/"
         modified_prefix = "#{scope}/targets_modified/"
       end
-      result, _ = remote_target_files(bucket_client: bucket, prefix: prefix, include_temp: false, path_matchers: path_matchers)
-      modified, temp = remote_target_files(bucket_client: bucket, prefix: modified_prefix, include_temp: include_temp, path_matchers: path_matchers)
+      if target and OpenC3::LocalMode.local_only_target?(target)
+        # Local only targets are never listed from the bucket
+        result = Set.new
+        modified = Set.new
+        temp = Set.new
+      else
+        result, _ = remote_target_files(bucket_client: bucket, prefix: prefix, include_temp: false, path_matchers: path_matchers)
+        modified, temp = remote_target_files(bucket_client: bucket, prefix: modified_prefix, include_temp: include_temp, path_matchers: path_matchers)
+      end
 
       # Add in local targets_modified if present
-      if ENV['OPENC3_LOCAL_MODE']
+      if ENV['OPENC3_LOCAL_MODE'] or !OpenC3::LocalMode.local_only_targets.empty?
         local_modified = OpenC3::LocalMode.local_target_files(scope: scope, target: target, path_matchers: path_matchers, include_temp: include_temp)
         local_modified.each do |filename|
-          if include_temp and filename.include?(TEMP_FOLDER)
+          if OpenC3::LocalMode.local_only_name?(filename)
+            # The local file is the only copy so it isn't marked as modified
+            result << filename
+          elsif !ENV['OPENC3_LOCAL_MODE']
+            next
+          elsif include_temp and filename.include?(TEMP_FOLDER)
             temp << filename
           else
             modified << filename unless modified.include?(filename)
@@ -100,16 +121,14 @@ module OpenC3
 
     def self.body(scope, name)
       name = strip_modified(name) # Remove '*' that indicates modified
+      # Local only targets are read from the local mode volume and never the bucket
+      if OpenC3::LocalMode.local_only_name?(name)
+        return local_body(scope, name)
+      end
       # First try opening a potentially modified version by looking for the modified target
       if ENV['OPENC3_LOCAL_MODE']
-        local_file = OpenC3::LocalMode.open_local_file(name, scope: scope)
-        if local_file
-          if File.extname(name) == ".bin"
-            return local_file.read
-          else
-            return local_file.read.force_encoding('UTF-8')
-          end
-        end
+        text = local_body(scope, name)
+        return text if text
       end
 
       bucket = Bucket.getClient()
@@ -136,6 +155,11 @@ module OpenC3
       # file listing, not a name any caller means to write. Remove it to avoid 
       # creating a file with a '*' in the name.
       name = strip_modified(name)
+      validate_name(name)
+      if OpenC3::LocalMode.local_only_name?(name)
+        OpenC3::LocalMode.put_target_file("#{scope}/targets_modified/#{name}", text, scope: scope)
+        return true
+      end
       if ENV['OPENC3_LOCAL_MODE']
         OpenC3::LocalMode.put_target_file("#{scope}/targets_modified/#{name}", text, scope: scope)
       end
@@ -163,6 +187,11 @@ module OpenC3
       # Match body/create so deleting a name taken straight from a listing
       # removes the file the user actually picked
       name = strip_modified(name)
+      validate_name(name)
+      if OpenC3::LocalMode.local_only_name?(name)
+        OpenC3::LocalMode.delete_local("#{scope}/targets_modified/#{name}")
+        return true
+      end
       if ENV['OPENC3_LOCAL_MODE']
         OpenC3::LocalMode.delete_local("#{scope}/targets_modified/#{name}")
       end
@@ -177,6 +206,20 @@ module OpenC3
 
     # protected
 
+    def self.local_body(scope, name)
+      local_file = OpenC3::LocalMode.open_local_file(name, scope: scope)
+      return nil unless local_file
+      begin
+        if File.extname(name) == ".bin"
+          local_file.read
+        else
+          local_file.read.force_encoding('UTF-8')
+        end
+      ensure
+        local_file.close
+      end
+    end
+
     def self.remote_target_files(bucket_client:, prefix:, include_temp: false, path_matchers: nil)
       result = Set.new
       temp = Set.new
@@ -191,6 +234,8 @@ module OpenC3
           temp << split_key[2..-1].join('/') if include_temp
           next
         end
+        # Local only targets are never read from the bucket
+        next if OpenC3::LocalMode.local_only_target?(split_key[2])
 
         if path_matchers
           found = false

data/lib/openc3/utilities/topic_lag_monitor.rb

@@ -0,0 +1,193 @@
+# encoding: ascii-8bit
+
+# Copyright 2026 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE.md for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'openc3/utilities/logger'
+
+module OpenC3
+  # Tracks how far a microservice is behind the Redis streams it consumes.
+  #
+  # Every processed message updates the lag gauge metric with the maximum lag
+  # seen over the last metric update interval. When the lag crosses the warning
+  # or critical threshold a user notification is published, and a recovery
+  # notification is published once the microservice has caught back up (with
+  # hysteresis so a lag hovering around a threshold doesn't flap).
+  #
+  # Streams are trimmed by time once their data is logged, so a consumer can
+  # only skip trimmed data after lagging far past the critical threshold.
+  #
+  # Recovery assumes each monitor is fed by a single reader that reads every
+  # stream with pending entries on each XREAD (COUNT limits how many entries
+  # but not which streams). A backlogged topic then reports its high lag every
+  # read cycle, so fresh messages on other topics can't clear the alert early.
+  class TopicLagMonitor
+    OK = 0
+    WARN = 1
+    CRITICAL = 2
+
+    # Lag must drop below threshold * CLEAR_RATIO to count as recovering
+    CLEAR_RATIO = 0.5
+    # Matches Metric::UPDATE_INTERVAL so every lag spike is published
+    GAUGE_WINDOW_SECONDS = 5.0
+
+    attr_reader :state
+
+    # Lag in seconds that triggers the warning and critical notifications.
+    # A warning <= 0 disables notifications. Also used by Data Flows so its
+    # indicators match the notifications.
+    def self.thresholds
+      {
+        'warning' => env_float('OPENC3_LAG_WARN_SECONDS', 5.0),
+        'critical' => env_float('OPENC3_LAG_CRITICAL_SECONDS', 30.0),
+      }
+    end
+
+    def self.env_float(name, default)
+      value = ENV.fetch(name, '').strip
+      return default if value.empty?
+      Float(value)
+    rescue ArgumentError
+      default
+    end
+
+    def initialize(name:, logger:, metric: nil)
+      @name = name
+      @logger = logger
+      @metric = metric
+      thresholds = self.class.thresholds
+      @warn_seconds = thresholds['warning']
+      @critical_seconds = thresholds['critical']
+      @clear_seconds = self.class.env_float('OPENC3_LAG_CLEAR_SECONDS', 10.0)
+      @renotify_seconds = self.class.env_float('OPENC3_LAG_RENOTIFY_SECONDS', 300.0)
+      @state = OK
+      @max_lag = 0.0
+      @max_lag_topic = nil
+      @below_since = nil
+      @below_ok_since = nil
+      @last_notify = nil
+      @last_notify_level = OK
+      @notified = false
+      @gauges = {}
+      @mutex = Mutex.new
+    end
+
+    # Record that a message was read from a topic
+    #
+    # @param topic [String] Redis stream the message was read from
+    # @param msg_id [String] Redis stream id of the message
+    # @param metric_name [String] Gauge metric to update with the lag
+    # @param help [String] Help text for the gauge metric
+    # @return [Float] The lag in seconds
+    def record(topic, msg_id, metric_name: nil, help: nil, now: Time.now.to_f)
+      lag = now - (id_ms(msg_id) / 1000.0)
+      update_gauge(metric_name, help, lag, now) if @metric and metric_name
+      return lag if @warn_seconds <= 0 # Notifications disabled
+
+      # Fast path for the common case of keeping up: skip the mutex and state machine
+      return lag if @state == OK and lag < @warn_seconds
+
+      @mutex.synchronize { update_state(topic, lag, now) }
+      return lag
+    end
+
+    protected
+
+    # Topics read by one microservice can lag by very different amounts, so
+    # publish the maximum lag rather than whichever topic was read last. The
+    # previous window's maximum is held through the current window so a spike
+    # just before a window boundary is still published.
+    def update_gauge(metric_name, help, lag, now)
+      window = (now / GAUGE_WINDOW_SECONDS).to_i
+      gauge = @gauges[metric_name]
+      if gauge.nil?
+        gauge = @gauges[metric_name] = { window: window, max: lag, previous_max: 0.0 }
+      elsif window != gauge[:window]
+        gauge[:previous_max] = (window == gauge[:window] + 1) ? gauge[:max] : 0.0
+        gauge[:window] = window
+        gauge[:max] = lag
+      elsif lag > gauge[:max]
+        gauge[:max] = lag
+      else
+        return # Published value is unchanged
+      end
+      value = (gauge[:max] > gauge[:previous_max]) ? gauge[:max] : gauge[:previous_max]
+      @metric.set(name: metric_name, value: value, type: 'gauge', unit: 'seconds', help: help)
+    end
+
+    def update_state(topic, lag, now)
+      if lag < @warn_seconds * CLEAR_RATIO
+        @below_ok_since ||= now
+      else
+        @below_ok_since = nil
+      end
+      if lag > @max_lag
+        @max_lag = lag
+        @max_lag_topic = topic
+      end
+      level = if lag >= @critical_seconds
+        CRITICAL
+      elsif lag >= @warn_seconds
+        WARN
+      else
+        OK
+      end
+
+      if level > @state
+        @state = level
+        @below_since = nil
+        # Bursty load can make a microservice recover and fall behind over and
+        # over, so falling back to an already announced level soon after the
+        # last notification continues that episode rather than notifying again
+        unless @last_notify and level <= @last_notify_level and (now - @last_notify) < @renotify_seconds
+          notify_lag(topic, lag, now)
+        end
+      elsif @state != OK
+        threshold = (@state == CRITICAL) ? @critical_seconds : @warn_seconds
+        if lag < threshold * CLEAR_RATIO
+          @below_since ||= now
+          if (now - @below_since) >= @clear_seconds
+            @below_since = nil
+            if @state == CRITICAL and (@below_ok_since.nil? or (now - @below_ok_since) < @clear_seconds)
+              @state = WARN
+              @below_since = @below_ok_since
+            else
+              @state = OK
+              @logger.info("#{@name} has caught up processing (max lag #{@max_lag.round(1)}s on #{@max_lag_topic})", type: Logger::NOTIFICATION) if @notified
+              @notified = false
+              @max_lag = 0.0
+              @max_lag_topic = nil
+            end
+          end
+        else
+          @below_since = nil
+          notify_lag(topic, lag, now) if (now - @last_notify) >= @renotify_seconds
+        end
+      end
+    end
+
+    def notify_lag(topic, lag, now)
+      message = "#{@name} is falling behind processing: #{lag.round(1)}s behind on #{topic} (max #{@max_lag.round(1)}s on #{@max_lag_topic})"
+      if @state == CRITICAL
+        @logger.error(message, type: Logger::NOTIFICATION)
+      else
+        @logger.warn(message, type: Logger::NOTIFICATION)
+      end
+      @last_notify = now
+      @last_notify_level = @state
+      @notified = true
+    end
+
+    def id_ms(id)
+      id.to_s.split('-')[0].to_i
+    end
+  end
+end