openc3 7.4.1 → 7.5.0

data/lib/openc3/utilities/local_mode.rb

@@ -17,39 +17,11 @@ require 'openc3/core_ext/file'
 # require 'openc3/models/gem_model' # These are used but also create circular dependency
 # require 'openc3/models/plugin_model' # These are used but also create circular dependency
 require 'openc3/utilities/bucket'
+require 'openc3/utilities/default_plugins'
 
 module OpenC3
   module LocalMode
     OPENC3_LOCAL_MODE_PATH = ENV['OPENC3_LOCAL_MODE_PATH'] || "/plugins"
-    # When updating update local_mode.py, PluginsTab.vue, plugins.p.spec.ts
-    DEFAULT_PLUGINS = [
-      'openc3-cosmos-tool-admin',
-      'openc3-cosmos-tool-bucketexplorer',
-      'openc3-cosmos-tool-cmdsender',
-      'openc3-cosmos-tool-cmdqueue', # Enterprise only
-      'openc3-cosmos-tool-cmdhistory', # Enterprise only
-      'openc3-cosmos-tool-cmdtlmserver',
-      'openc3-cosmos-tool-dataextractor',
-      'openc3-cosmos-tool-dataviewer',
-      'openc3-cosmos-tool-docs',
-      'openc3-cosmos-tool-handbooks',
-      'openc3-cosmos-tool-iframe',
-      'openc3-cosmos-tool-limitsmonitor',
-      'openc3-cosmos-tool-packetviewer',
-      'openc3-cosmos-tool-scriptrunner',
-      'openc3-cosmos-tool-tablemanager',
-      'openc3-cosmos-tool-tlmgrapher',
-      'openc3-cosmos-tool-tlmviewer',
-      'openc3-cosmos-enterprise-tool-admin', # Enterprise only
-      'openc3-cosmos-tool-autonomic', # Enterprise only
-      'openc3-cosmos-tool-calendar', # Enterprise only
-      'openc3-cosmos-tool-grafana', # Enterprise only
-      'openc3-cosmos-tool-logexplorer', # Enterprise only
-      'openc3-cosmos-tool-notebooks', # Enterprise only
-      'openc3-cosmos-tool-systemhealth', # Enterprise only
-      'openc3-enterprise-tool-base', # Enterprise only
-      'openc3-tool-base',
-    ]
 
     # Install plugins from local plugins folder
     # Can only be used from openc3cli because calls top_level load_plugin
@@ -280,7 +252,7 @@ module OpenC3
     end
 
     def self.update_local_plugin_files(full_folder_path, plugin_file_path, plugin_hash, gem_name)
-      return if DEFAULT_PLUGINS.include?(gem_name)
+      return if DefaultPlugins::NAMES.include?(gem_name)
       puts "Updating local plugin files: #{full_folder_path}"
       FileUtils.mkdir_p(full_folder_path)
       gems = scan_plugin_dir(full_folder_path).first
@@ -378,8 +350,8 @@ module OpenC3
     end
 
     def self.put_target_file(path, io_or_string, scope:) # NOSONAR - scope: is part of the caller-facing signature
-      full_folder_path = "#{OPENC3_LOCAL_MODE_PATH}/#{path}"
-      return unless File.expand_path(full_folder_path).start_with?(OPENC3_LOCAL_MODE_PATH)
+      full_folder_path = key_path(path)
+      return unless full_folder_path and path.start_with?("#{scope}/")
       FileUtils.mkdir_p(File.dirname(full_folder_path))
       File.open(full_folder_path, 'wb') do |file|
         if String === io_or_string
@@ -392,8 +364,8 @@ module OpenC3
     end
 
     def self.open_local_file(path, scope:)
-      full_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/targets_modified/#{path}"
-      if File.expand_path(full_path).start_with?(OPENC3_LOCAL_MODE_PATH)
+      full_path = scope_path(scope, "targets_modified/#{path}")
+      if full_path and safe_key?(path)
         return File.open(full_path, 'rb')
       end
       nil
@@ -452,8 +424,8 @@ module OpenC3
     def self.save_tool_config(scope, tool, name, data)
       return unless ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
       json = JSON.parse(data, allow_nan: true, create_additions: true)
-      config_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/tool_config/#{tool}/#{name}.json"
-      return unless File.expand_path(config_path).start_with?(OPENC3_LOCAL_MODE_PATH)
+      config_path = scope_path(scope, "tool_config/#{tool}/#{name}.json")
+      return unless config_path and safe_key?("#{tool}/#{name}")
       FileUtils.mkdir_p(File.dirname(config_path))
       File.open(config_path, 'w') do |file|
         file.write(JSON.pretty_generate(json, allow_nan: true))
@@ -462,8 +434,8 @@ module OpenC3
 
     def self.delete_tool_config(scope, tool, name)
       return unless ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
-      config_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/tool_config/#{tool}/#{name}.json"
-      return unless File.expand_path(config_path).start_with?(OPENC3_LOCAL_MODE_PATH)
+      config_path = scope_path(scope, "tool_config/#{tool}/#{name}.json")
+      return unless config_path and safe_key?("#{tool}/#{name}")
       FileUtils.rm_f(config_path)
     end
 
@@ -508,8 +480,8 @@ module OpenC3
 
     def self.save_setting(scope, name, data)
       return unless ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
-      config_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/settings/#{name}.json"
-      return unless File.expand_path(config_path).start_with?(OPENC3_LOCAL_MODE_PATH)
+      config_path = scope_path(scope, "settings/#{name}.json")
+      return unless config_path and safe_key?(name)
       FileUtils.mkdir_p(File.dirname(config_path))
       # Anything can be stored as a setting. Strings are written directly since
       # they're already the serialized form. Anything else is written as JSON so
@@ -518,26 +490,86 @@ module OpenC3
       File.write(config_path, data)
     end
 
+    # Target names from OPENC3_LOCAL_ONLY_TARGETS (comma separated). These
+    # targets keep their target files only in the local mode volume and never
+    # read or write them in the bucket. Read on every call so it can't go stale.
+    def self.local_only_targets
+      ENV['OPENC3_LOCAL_ONLY_TARGETS'].to_s.split(',').map { |name| name.strip.upcase }.reject(&:empty?)
+    end
+
+    def self.local_only_target?(target_name)
+      return false if target_name.nil? or target_name.to_s.empty?
+      local_only_targets.include?(target_name.to_s.upcase)
+    end
+
+    # name is relative to targets / targets_modified, e.g. "INST/procedures/test.rb"
+    def self.local_only_name?(name)
+      local_only_target?(name.to_s.split('/')[0])
+    end
+
+    # key is a config bucket key, e.g. "DEFAULT/targets_modified/INST/procedures/test.rb"
+    def self.local_only_key?(key)
+      split_key = key.to_s.split('/')
+      return false unless ['targets', 'targets_modified'].include?(split_key[1])
+      local_only_target?(split_key[2])
+    end
+
     # Helper methods
 
+    # True if the key has no '.' / '..' segments, no empty segments, no leading
+    # slash and no backslash. Such a key always resolves under its first
+    # segment (the scope), both on disk and in the bucket (the bucket
+    # normalizes '..' segments, so a traversal key reaches other scopes there too).
+    def self.safe_key?(key)
+      return false unless String === key
+      return false if key.empty? or key.start_with?('/') or key.include?('\\') or key.include?("\0")
+      key.split('/', -1).none? { |segment| segment.empty? or segment == '.' or segment == '..' }
+    end
+
+    # True if path resolves to root or somewhere beneath it. Compares against
+    # root plus a separator so a sibling such as /plugins_old never matches.
+    def self.path_within?(path, root)
+      expanded = File.expand_path(path)
+      root = File.expand_path(root)
+      expanded == root or expanded.start_with?(root + File::SEPARATOR)
+    end
+
+    # Local file path for a scope relative path, or nil if the path would
+    # leave the scope directory
+    def self.scope_path(scope, relative_path)
+      return nil unless safe_key?(scope) and !scope.include?('/')
+      scope_root = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}"
+      full_path = "#{scope_root}/#{relative_path}"
+      return nil unless path_within?(full_path, scope_root)
+      full_path
+    end
+
+    # Local file path for a bucket key, or nil if the key is not a safe key
+    def self.key_path(key)
+      return nil unless safe_key?(key)
+      full_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
+      return nil unless path_within?(full_path, OPENC3_LOCAL_MODE_PATH)
+      full_path
+    end
+
     def self.sync_remote_to_local(bucket, key)
-      local_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
-      return unless File.expand_path(local_path).start_with?(OPENC3_LOCAL_MODE_PATH)
+      local_path = key_path(key)
+      return unless local_path
       FileUtils.mkdir_p(File.dirname(local_path))
       bucket.get_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: key, path: local_path)
     end
 
     def self.sync_local_to_remote(bucket, key)
-      local_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
-      return unless File.expand_path(local_path).start_with?(OPENC3_LOCAL_MODE_PATH)
+      local_path = key_path(key)
+      return unless local_path
       File.open(local_path, 'rb') do |read_file|
         bucket.put_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: key, body: read_file)
       end
     end
 
     def self.delete_local(key)
-      local_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
-      return unless File.expand_path(local_path).start_with?(OPENC3_LOCAL_MODE_PATH)
+      local_path = key_path(key)
+      return unless local_path
       File.delete(local_path) if File.exist?(local_path)
       nil
     end
@@ -580,6 +612,10 @@ module OpenC3
       # Build catalogs
       local_catalog = build_local_catalog(scope: scope)
       remote_catalog = build_remote_catalog(bucket, scope: scope)
+      # Local only targets never sync in either direction. Any stale copies
+      # already in the bucket are left alone rather than pulled down or deleted.
+      local_catalog.reject! { |key, _size| local_only_key?(key) }
+      remote_catalog.reject! { |key, _size| local_only_key?(key) }
 
       # Find and Handle Differences
       local_catalog.each do |key, size|

data/lib/openc3/utilities/metric.rb

@@ -133,7 +133,23 @@ module OpenC3
     def graceful_kill
     end
 
+    # Environment variable the operator sets on every microservice it spawns to
+    # say "I am already reporting process metrics for you". Without it an
+    # in-process generator and the operator would both be writing cpu/memory for
+    # the same microservice.
+    SUPERVISED_ENV_VAR = 'OPENC3_OPERATOR_PROCESS_METRICS'.freeze
+
+    # Whether something outside this process is already reporting its process
+    # level metrics.
+    def self.supervised?
+      !ENV[SUPERVISED_ENV_VAR].nil?
+    end
+
     def self.add_update_generator(object)
+      # Generators produce process level metrics (cpu, memory). Skip them
+      # entirely when the operator is already reporting those for us.
+      return if supervised?
+
       @@update_generators << object
     end
   end

data/lib/openc3/utilities/process_stats.rb

@@ -0,0 +1,160 @@
+# encoding: ascii-8bit
+
+# Copyright 2026 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE.md for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'etc'
+
+# See: https://man7.org/linux/man-pages/man5/proc.5.html
+
+module OpenC3
+  # Samples CPU and memory for an arbitrary pid out of /proc/<pid>/stat.
+  #
+  # Every microservice is bootstrapped by plugin_microservice.rb, which exec()s
+  # the configured cmd. exec replaces the process image, so everything the
+  # bootstrap set up - including the Metric update thread - is gone. A
+  # microservice whose cmd is itself an OpenC3::Microservice builds a new Metric
+  # and keeps reporting, but a plugin that runs a Rails app, a Python script or
+  # any other binary never loads the OpenC3 libraries and so can never report on
+  # itself. Sampling by pid lets the operator report on behalf of those
+  # processes without requiring anything of the plugin.
+  class ProcessStats
+    # /proc/<pid>/stat field offsets, 0 based, after the comm field has been
+    # collapsed to a single token. proc(5) numbers these fields from 1.
+    UTIME_INDEX = 13
+    STIME_INDEX = 14
+    NUM_THREADS_INDEX = 19
+    STARTTIME_INDEX = 21
+    VSIZE_INDEX = 22
+    RSS_INDEX = 23
+
+    # Near universal Linux default. Only used if sysconf can't answer - a zero
+    # here would turn every CPU number into Infinity or NaN.
+    DEFAULT_USER_HZ = 100.0
+    DEFAULT_PAGESIZE = 4096
+
+    attr_reader :pid
+
+    class << self
+      # Whether this kernel exposes the procfs data we need at all. macOS and
+      # Windows don't, and neither do some hardened container runtimes.
+      def supported?
+        return @supported unless @supported.nil?
+        @supported = File.exist?('/proc/self/stat')
+      rescue Exception
+        @supported = false
+      end
+
+      def pagesize
+        @pagesize ||= sysconf(:SC_PAGESIZE, DEFAULT_PAGESIZE)
+      end
+
+      def user_hz
+        @user_hz ||= sysconf(:SC_CLK_TCK, DEFAULT_USER_HZ).to_f
+      end
+
+      # Wall time the machine booted. /proc/<pid>/stat reports starttime
+      # relative to boot, so this is what turns it into an absolute time.
+      def boot_time
+        @boot_time ||= begin
+          Time.now - File.read('/proc/uptime').split[0].to_f
+        rescue Exception
+          Time.now
+        end
+      end
+
+      private
+
+      def sysconf(name, default)
+        return default unless Etc.const_defined?(name)
+
+        value = Etc.sysconf(Etc.const_get(name))
+        (value and value > 0) ? value : default
+      rescue Exception
+        default
+      end
+    end
+
+    def initialize(pid)
+      @pid = pid
+      @start_time = nil
+      @last_sample_time = nil
+      # nil rather than 0 so a process that has genuinely burned no CPU yet
+      # still starts producing sample deltas on the next pass
+      @cpu_time = nil
+      @average_cpu_utilization = 0.0
+      @sample_cpu_utilization = 0.0
+      @virtual_memory_bytes = 0.0
+      @resident_memory_bytes = 0.0
+      @num_threads = 0
+    end
+
+    # @return [Boolean] whether a usable sample was taken
+    def sample
+      stat = read_stat()
+      return false unless stat
+
+      fields = split_stat(stat)
+      return false if fields.length <= RSS_INDEX
+
+      sample_time = Time.now
+      user_hz = self.class.user_hz
+      cpu_time = (fields[UTIME_INDEX].to_f + fields[STIME_INDEX].to_f) / user_hz
+      @start_time ||= self.class.boot_time + (fields[STARTTIME_INDEX].to_f / user_hz)
+
+      if @cpu_time and @last_sample_time
+        time_delta = sample_time - @last_sample_time
+        @sample_cpu_utilization = (cpu_time - @cpu_time) / time_delta if time_delta > 0
+      end
+      @cpu_time = cpu_time
+      @last_sample_time = sample_time
+
+      elapsed = sample_time - @start_time
+      @average_cpu_utilization = cpu_time / elapsed if elapsed > 0
+
+      @num_threads = fields[NUM_THREADS_INDEX].to_i
+      @virtual_memory_bytes = fields[VSIZE_INDEX].to_f
+      @resident_memory_bytes = fields[RSS_INDEX].to_f * self.class.pagesize
+      true
+    end
+
+    # Shaped like OpenC3::Metric's internal data hash so it can be handed
+    # straight to MetricModel.
+    def values
+      {
+        'average_cpu_utilization' => { 'value' => @average_cpu_utilization, 'type' => 'gauge', 'unit' => 'ratio' },
+        'sample_cpu_utilization' => { 'value' => @sample_cpu_utilization, 'type' => 'gauge', 'unit' => 'ratio' },
+        'virtual_memory_bytes' => { 'value' => @virtual_memory_bytes, 'type' => 'gauge', 'unit' => 'bytes' },
+        'resident_memory_bytes' => { 'value' => @resident_memory_bytes, 'type' => 'gauge', 'unit' => 'bytes' },
+        'num_threads' => { 'value' => @num_threads, 'type' => 'gauge' },
+      }
+    end
+
+    private
+
+    def read_stat
+      File.read("/proc/#{@pid}/stat")
+    rescue SystemCallError, IOError
+      # The process can exit between being listed and being read
+      nil
+    end
+
+    # The comm field is wrapped in parens and may itself contain spaces and
+    # parens, so split from the last close paren rather than trying to match it.
+    # Two placeholders stand in for pid and comm to keep the proc(5) offsets.
+    def split_stat(stat)
+      close = stat.rindex(')')
+      return [] unless close
+
+      ['0', '0'].concat(stat[(close + 1)..-1].to_s.split)
+    end
+  end
+end

data/lib/openc3/utilities/pypi_url.rb

@@ -18,6 +18,18 @@ module OpenC3
   class PypiUrl
     DEFAULT = 'https://pypi.org/simple'
 
+    # Turn a configured PyPI base URL (the pypi_url setting or PYPI_URL) into
+    # its simple index URL. Trailing slashes are dropped first, matching the
+    # ${PYPI_URL%/}/simple the image builds use, so https://mirror/ does not
+    # give https://mirror//simple and https://pypi.org/ still equals DEFAULT.
+    #
+    # @param base_url [String, nil] the configured base url
+    # @return [String, nil] the simple index url, or nil when base_url is nil
+    def self.index_url(base_url)
+      return nil if base_url.nil?
+      "#{base_url.sub(%r{/+\z}, '')}/simple"
+    end
+
     # Validate that a resolved pypi_url is an http(s) URL before it is handed to
     # pip. The value can come from a user-writable setting or ENV, so a malformed
     # or non-http value is rejected and replaced with the default rather than
@@ -35,5 +47,58 @@ module OpenC3
       Logger.error("Invalid pypi_url '#{pypi_url}' (#{e.message}); falling back to #{DEFAULT}")
       DEFAULT
     end
+
+    # Environment variable that opts a deployment into insecure (unverified TLS)
+    # connections to the pypi index host, for operators running a private index
+    # with a self-signed certificate. OPENC3_ prefixed like our other settings,
+    # and kept out of uv's own UV_* namespace (uv reads UV_INSECURE_HOST itself).
+    INSECURE_HOST_ENV = 'OPENC3_PYPI_ALLOW_INSECURE'
+
+    # Previous name for INSECURE_HOST_ENV, still honored so existing helm values
+    # and compose files keep working. It was named for pip's --trusted-host, but
+    # every install path goes through uv (openc3/bin/pipinstall is a uv pip
+    # install wrapper), so the flag it produces is uv's --allow-insecure-host.
+    DEPRECATED_INSECURE_HOST_ENV = 'PIP_ENABLE_TRUSTED_HOST'
+
+    # Build the argv array of index arguments shared by every uv invocation.
+    #
+    # --default-index rather than -i/--index-url: the latter is deprecated by uv
+    # on both `uv sync` and `uv pip install`. Note this only affects the index uv
+    # resolves against, so it has no effect on `uv sync --frozen`, where uv.lock
+    # already pins each package's registry and wheel URL.
+    #
+    # --allow-insecure-host rather than --trusted-host: --trusted-host is an
+    # undocumented uv alias, so use the real uv option.
+    #
+    # @param pypi_url [String] the resolved and validated pypi index url
+    # @return [Array<String>] arguments to append to a uv command line
+    def self.build_args(pypi_url)
+      args = ["--default-index", pypi_url]
+      # --default-index does not win on its own: uv searches a named index
+      # declared in the plugin's own [tool.uv].index table first, so a plugin
+      # can quietly resolve against its author's index instead of the one the
+      # operator configured. --no-config makes uv ignore that table. Gated on
+      # the operator having actually designated an index, because at the public
+      # default there is no operator policy to enforce and the plugin author's
+      # own index configuration is the only signal there is.
+      #
+      # This is half the story: a package pinned with [tool.uv].sources is
+      # project metadata that survives --no-config. openc3/bin/uvinstall and
+      # openc3/bin/pipinstall add --no-sources for that, on the commands where
+      # uv actually resolves.
+      args << "--no-config" if pypi_url != DEFAULT
+      if allow_insecure_host?
+        args += ["--allow-insecure-host", URI.parse(pypi_url).host]
+      end
+      args
+    end
+
+    # @return [Boolean] whether the deployment opted into insecure connections
+    #   to the pypi index host
+    def self.allow_insecure_host?
+      # Empty counts as unset: compose passes both names through as
+      # ${VAR:-}, so an unset .env entry arrives as an empty string.
+      !ENV[INSECURE_HOST_ENV].to_s.empty? || !ENV[DEPRECATED_INSECURE_HOST_ENV].to_s.empty?
+    end
   end
 end

data/lib/openc3/utilities/questdb_client.rb

@@ -632,6 +632,20 @@ module OpenC3
       false
     end
 
+    # Return the names of all tables in a single query. Used to avoid querying
+    # (and failing on) tables that were never created, e.g. commands that have
+    # never been sent, which is the common case for targets with thousands of packets.
+    # QuestDB table names are case insensitive so the names are upcased.
+    #
+    # @return [Set<String>, nil] Upcased table names, or nil if they cannot be queried
+    def self.table_names(db_shard: 0)
+      result = query_with_retry("SELECT table_name FROM tables()", max_retries: 1, label: "table names", db_shard: db_shard)
+      return nil unless result
+      result.values.each_with_object(Set.new) { |row, set| set << row[0].to_s.upcase }
+    rescue StandardError
+      nil
+    end
+
     # QuestDB column types that can be aggregated with min/max/avg/stddev.
     # Used by reduced queries to decide whether a converted (__C) column is
     # numeric or must fall back to the raw column (e.g. states stored as VARCHAR).

data/lib/openc3/utilities/running_script.rb

@@ -112,16 +112,7 @@ module OpenC3
               RunningScript.instance.perform_pause
             else
               if method.to_s == 'open_bucket_dialog'
-                bucket = input['bucket']
-                path = input['path']
-                filename = input['filename']
-                # Path from bucket dialog already includes scope prefix (e.g. DEFAULT/targets/...)
-                # _get_storage_file prepends scope, so strip it from the path to avoid doubling
-                scope = RunningScript.instance.scope
-                path = path.sub(/\A#{Regexp.escape(scope)}\//, '')
-                file = _get_storage_file(path, bucket: bucket, scope: scope)
-                file.filename = filename
-                return file
+                return _get_bucket_dialog_file(input, scope: RunningScript.instance.scope)
               elsif (method.to_s.include?('open_file'))
                 files = input.map do |filename|
                   file = _get_storage_file("tmp/#{filename}", scope: RunningScript.instance.scope)
@@ -275,10 +266,19 @@ module OpenC3
 
       def goto(line_no_or_procedure_name, line_no = nil)
         if line_no.nil?
-          start(RunningScript.instance.current_filename, line_no: line_no_or_procedure_name, bind_variables: true, complete: true)
+          procedure_name = RunningScript.instance.current_filename
+          line_no = line_no_or_procedure_name
         else
-          start(line_no_or_procedure_name, line_no: line_no, bind_variables: true, complete: true)
+          procedure_name = line_no_or_procedure_name
         end
+        # goto typed at the debug prompt is evaluated on the thread that processes
+        # go/step/stop messages. Running the script there deadlocks at the next pause,
+        # so hand it to the run thread the same way the editor's goto does.
+        unless RunningScript.run_thread?
+          RunningScript.instance.execute_while_paused(procedure_name, line_no, nil, true)
+          return
+        end
+        start(procedure_name, line_no: line_no, bind_variables: true, complete: true)
       end
 
       # Require an additional ruby file
@@ -797,6 +797,10 @@ class RunningScript
     @@instance = value
   end
 
+  def self.run_thread?
+    Thread.current == @@run_thread
+  end
+
   def self.line_delay
     @@line_delay
   end
@@ -1103,9 +1107,10 @@ class RunningScript
     trace
   end
 
-  def execute_while_paused(filename, line_no = 1, end_line_no = nil)
+  # bind_variables of nil binds only when filename is the currently running file
+  def execute_while_paused(filename, line_no = 1, end_line_no = nil, bind_variables = nil)
     if @script_status.state == 'paused' or @script_status.state == 'error' or @script_status.state == 'breakpoint'
-      @execute_while_paused_info = { filename: filename, line_no: line_no, end_line_no: end_line_no }
+      @execute_while_paused_info = { filename: filename, line_no: line_no, end_line_no: end_line_no, bind_variables: bind_variables }
     else
       scriptrunner_puts("Cannot execute selection or goto unless script is paused, breakpoint, or in error state")
     end
@@ -1217,10 +1222,9 @@ class RunningScript
 
   def check_execute_while_paused
     if @execute_while_paused_info
-      if @script_status.current_filename == @execute_while_paused_info[:filename]
-        bind_variables = true
-      else
-        bind_variables = false
+      bind_variables = @execute_while_paused_info[:bind_variables]
+      if bind_variables.nil?
+        bind_variables = (@script_status.current_filename == @execute_while_paused_info[:filename])
       end
       if @execute_while_paused_info[:end_line_no]
         # Execute Selection While Paused

data/lib/openc3/utilities/store_autoload.rb

@@ -54,6 +54,14 @@ module OpenC3
     @@db_shard_cache_mutex = Mutex.new
     DB_SHARD_CACHE_TIMEOUT = 60 # seconds
 
+    # Maximum number of entries read_topics returns PER STREAM when the caller
+    # does not pass a count. XREAD without COUNT returns everything from the
+    # offset to the tail of the stream, which for a consumer that has fallen
+    # behind on a high-rate stream can be an enormous reply materialized in
+    # memory all at once. Callers loop on their offsets so a capped read simply
+    # continues on the next call.
+    READ_TOPICS_DEFAULT_COUNT = 1000
+
     # Mutex used to ensure that only one instance is created
     @@instance_mutex = Mutex.new
 
@@ -225,8 +233,10 @@ module OpenC3
       return offsets
     end
 
+    # @param count [Integer] Maximum entries to return per stream. nil uses READ_TOPICS_DEFAULT_COUNT.
     def read_topics(topics, offsets = nil, timeout_ms = 1000, count = nil)
       return {} if topics.empty?
+      count ||= READ_TOPICS_DEFAULT_COUNT
       Thread.current[:topic_offsets] ||= {}
       topic_offsets = Thread.current[:topic_offsets]
       begin
@@ -264,13 +274,20 @@ module OpenC3
     # @option opts [String]  :id          the entry id, default value is `*`, it means auto generation,
     #   if `nil` id is passed it will be changed to `*`
     # @option opts [Integer] :maxlen      max length of entries, default value is `nil`, it means will grow forever
-    # @option opts [String] :approximate whether to add `~` modifier of maxlen or not, default value is 'true'
+    # @option opts [String] :approximate whether to add `~` modifier of maxlen or minid or not, default value is 'true'
+    # @option opts [String] :minid       trim entries with ids lower than this as part of the add (XADD MINID).
+    #   Cannot be combined with maxlen. See Topic.stream_safety_minid for the safety cap applied to target streams.
     #
     # @return [String] the entry id
-    def write_topic(topic, msg_hash, id = '*', maxlen = nil, approximate = 'true')
+    def write_topic(topic, msg_hash, id = '*', maxlen = nil, approximate = 'true', minid: nil)
       id = '*' if id.nil?
+      raise ArgumentError, "write_topic can't supply both maxlen and minid" if maxlen and minid
       @redis_pool.with do |redis|
-        return redis.xadd(topic, msg_hash, id: id, maxlen: maxlen, approximate: approximate)
+        if minid
+          return redis.xadd(topic, msg_hash, id: id, minid: minid, approximate: approximate)
+        else
+          return redis.xadd(topic, msg_hash, id: id, maxlen: maxlen, approximate: approximate)
+        end
       end
     end