openc3 7.4.1 → 7.5.0

data/lib/openc3/utilities/store_queued.rb

@@ -24,20 +24,32 @@ module OpenC3
     # Mutex used to ensure that only one instance is created
     @@instance_mutex = Mutex.new
 
+    DEFAULT_UPDATE_INTERVAL = 1 # seconds
+
     # Get the singleton instance for the given db_shard
-    # Sets the update interval to 1 second by default
-    def self.instance(update_interval = 1, db_shard: 0) # seconds
+    # Uses the interval from set_update_interval_all_shards if not given
+    def self.instance(update_interval = nil, db_shard: 0)
       @instances ||= []
       the_instance = @instances[db_shard]
       return the_instance if the_instance
 
       @@instance_mutex.synchronize do
         @instances ||= []
-        @instances[db_shard] ||= self.new(update_interval, db_shard: db_shard)
+        @instances[db_shard] ||= self.new(update_interval || @update_interval || DEFAULT_UPDATE_INTERVAL, db_shard: db_shard)
         return @instances[db_shard]
       end
     end
 
+    # Set the update interval on every db_shard instance, including ones created later
+    def self.set_update_interval_all_shards(interval)
+      return unless interval > 0.0
+
+      @@instance_mutex.synchronize do
+        @update_interval = interval
+        (@instances || []).compact.each { |the_instance| the_instance.set_update_interval(interval) }
+      end
+    end
+
     # Delegate all unknown class methods to delegate to the instance (db_shard 0)
     def self.method_missing(message, *args, **kwargs, &)
       self.instance.public_send(message, *args, **kwargs, &)
@@ -49,6 +61,8 @@ module OpenC3
       @store = store_instance()
       # Queue to hold the store requests
       @store_queue = Queue.new
+      # Orders queueing against the final drain in shutdown
+      @shutdown_mutex = Mutex.new
       # Sleeper used to delay update thread
       @update_sleeper = Sleeper.new
 
@@ -102,11 +116,18 @@ module OpenC3
       @update_sleeper.cancel if @update_sleeper
       OpenC3.kill_thread(self, @update_thread) if @update_thread
       @update_thread = nil
-      # Drain the queue before shutdown
-      begin
-        process_queue()
-      rescue
-        # Best effort - Redis may already be unavailable during shutdown
+      # Nothing drains the queue once the thread is gone, so from here on write
+      # straight through rather than silently dropping whatever gets queued.
+      # Drain first and flip the flag last, under the mutex, so a direct write
+      # can't be overwritten by an older queued one and nothing is queued after
+      # the drain.
+      @shutdown_mutex.synchronize do
+        begin
+          process_queue()
+        rescue
+          # Best effort - Redis may already be unavailable during shutdown
+        end
+        @shutdown = true
       end
     end
 
@@ -114,7 +135,14 @@ module OpenC3
 
     # Record the message for pipelining by the thread
     def method_missing(message, *args, **kwargs, &block)
-      @store_queue.push(MessageStruct.new(message, args, kwargs, block))
+      # @shutdown never goes back to false, so once set the lock can be skipped
+      unless @shutdown
+        queued = @shutdown_mutex.synchronize do
+          @store_queue.push(MessageStruct.new(message, args, kwargs, block)) unless @shutdown
+        end
+        return queued if queued
+      end
+      @store.public_send(message, *args, **kwargs, &block)
     end
 
     # Returns the store we're working with

data/lib/openc3/utilities/target_file.rb

@@ -31,6 +31,15 @@ module OpenC3
       name.sub(/\*$/, '')
     end
 
+    # Reject names that could resolve outside the scope's targets_modified
+    # directory. The bucket normalizes '..' segments in keys just like the
+    # filesystem does, so this must be checked before any bucket call.
+    def self.validate_name(name)
+      unless OpenC3::LocalMode.safe_key?(name)
+        raise ArgumentError, "Invalid target file name: #{name.inspect}"
+      end
+    end
+
     def self.all(scope, path_matchers, target: nil)
       target = target.upcase if target
 
@@ -46,14 +55,26 @@ module OpenC3
         prefix = "#{scope}/targets/"
         modified_prefix = "#{scope}/targets_modified/"
       end
-      result, _ = remote_target_files(bucket_client: bucket, prefix: prefix, include_temp: false, path_matchers: path_matchers)
-      modified, temp = remote_target_files(bucket_client: bucket, prefix: modified_prefix, include_temp: include_temp, path_matchers: path_matchers)
+      if target and OpenC3::LocalMode.local_only_target?(target)
+        # Local only targets are never listed from the bucket
+        result = Set.new
+        modified = Set.new
+        temp = Set.new
+      else
+        result, _ = remote_target_files(bucket_client: bucket, prefix: prefix, include_temp: false, path_matchers: path_matchers)
+        modified, temp = remote_target_files(bucket_client: bucket, prefix: modified_prefix, include_temp: include_temp, path_matchers: path_matchers)
+      end
 
       # Add in local targets_modified if present
-      if ENV['OPENC3_LOCAL_MODE']
+      if ENV['OPENC3_LOCAL_MODE'] or !OpenC3::LocalMode.local_only_targets.empty?
         local_modified = OpenC3::LocalMode.local_target_files(scope: scope, target: target, path_matchers: path_matchers, include_temp: include_temp)
         local_modified.each do |filename|
-          if include_temp and filename.include?(TEMP_FOLDER)
+          if OpenC3::LocalMode.local_only_name?(filename)
+            # The local file is the only copy so it isn't marked as modified
+            result << filename
+          elsif !ENV['OPENC3_LOCAL_MODE']
+            next
+          elsif include_temp and filename.include?(TEMP_FOLDER)
             temp << filename
           else
             modified << filename unless modified.include?(filename)
@@ -100,16 +121,14 @@ module OpenC3
 
     def self.body(scope, name)
       name = strip_modified(name) # Remove '*' that indicates modified
+      # Local only targets are read from the local mode volume and never the bucket
+      if OpenC3::LocalMode.local_only_name?(name)
+        return local_body(scope, name)
+      end
       # First try opening a potentially modified version by looking for the modified target
       if ENV['OPENC3_LOCAL_MODE']
-        local_file = OpenC3::LocalMode.open_local_file(name, scope: scope)
-        if local_file
-          if File.extname(name) == ".bin"
-            return local_file.read
-          else
-            return local_file.read.force_encoding('UTF-8')
-          end
-        end
+        text = local_body(scope, name)
+        return text if text
       end
 
       bucket = Bucket.getClient()
@@ -136,6 +155,11 @@ module OpenC3
       # file listing, not a name any caller means to write. Remove it to avoid 
       # creating a file with a '*' in the name.
       name = strip_modified(name)
+      validate_name(name)
+      if OpenC3::LocalMode.local_only_name?(name)
+        OpenC3::LocalMode.put_target_file("#{scope}/targets_modified/#{name}", text, scope: scope)
+        return true
+      end
       if ENV['OPENC3_LOCAL_MODE']
         OpenC3::LocalMode.put_target_file("#{scope}/targets_modified/#{name}", text, scope: scope)
       end
@@ -163,6 +187,11 @@ module OpenC3
       # Match body/create so deleting a name taken straight from a listing
       # removes the file the user actually picked
       name = strip_modified(name)
+      validate_name(name)
+      if OpenC3::LocalMode.local_only_name?(name)
+        OpenC3::LocalMode.delete_local("#{scope}/targets_modified/#{name}")
+        return true
+      end
       if ENV['OPENC3_LOCAL_MODE']
         OpenC3::LocalMode.delete_local("#{scope}/targets_modified/#{name}")
       end
@@ -177,6 +206,20 @@ module OpenC3
 
     # protected
 
+    def self.local_body(scope, name)
+      local_file = OpenC3::LocalMode.open_local_file(name, scope: scope)
+      return nil unless local_file
+      begin
+        if File.extname(name) == ".bin"
+          local_file.read
+        else
+          local_file.read.force_encoding('UTF-8')
+        end
+      ensure
+        local_file.close
+      end
+    end
+
     def self.remote_target_files(bucket_client:, prefix:, include_temp: false, path_matchers: nil)
       result = Set.new
       temp = Set.new
@@ -191,6 +234,8 @@ module OpenC3
           temp << split_key[2..-1].join('/') if include_temp
           next
         end
+        # Local only targets are never read from the bucket
+        next if OpenC3::LocalMode.local_only_target?(split_key[2])
 
         if path_matchers
           found = false

data/lib/openc3/utilities/topic_lag_monitor.rb

@@ -0,0 +1,193 @@
+# encoding: ascii-8bit
+
+# Copyright 2026 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE.md for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'openc3/utilities/logger'
+
+module OpenC3
+  # Tracks how far a microservice is behind the Redis streams it consumes.
+  #
+  # Every processed message updates the lag gauge metric with the maximum lag
+  # seen over the last metric update interval. When the lag crosses the warning
+  # or critical threshold a user notification is published, and a recovery
+  # notification is published once the microservice has caught back up (with
+  # hysteresis so a lag hovering around a threshold doesn't flap).
+  #
+  # Streams are trimmed by time once their data is logged, so a consumer can
+  # only skip trimmed data after lagging far past the critical threshold.
+  #
+  # Recovery assumes each monitor is fed by a single reader that reads every
+  # stream with pending entries on each XREAD (COUNT limits how many entries
+  # but not which streams). A backlogged topic then reports its high lag every
+  # read cycle, so fresh messages on other topics can't clear the alert early.
+  class TopicLagMonitor
+    OK = 0
+    WARN = 1
+    CRITICAL = 2
+
+    # Lag must drop below threshold * CLEAR_RATIO to count as recovering
+    CLEAR_RATIO = 0.5
+    # Matches Metric::UPDATE_INTERVAL so every lag spike is published
+    GAUGE_WINDOW_SECONDS = 5.0
+
+    attr_reader :state
+
+    # Lag in seconds that triggers the warning and critical notifications.
+    # A warning <= 0 disables notifications. Also used by Data Flows so its
+    # indicators match the notifications.
+    def self.thresholds
+      {
+        'warning' => env_float('OPENC3_LAG_WARN_SECONDS', 5.0),
+        'critical' => env_float('OPENC3_LAG_CRITICAL_SECONDS', 30.0),
+      }
+    end
+
+    def self.env_float(name, default)
+      value = ENV.fetch(name, '').strip
+      return default if value.empty?
+      Float(value)
+    rescue ArgumentError
+      default
+    end
+
+    def initialize(name:, logger:, metric: nil)
+      @name = name
+      @logger = logger
+      @metric = metric
+      thresholds = self.class.thresholds
+      @warn_seconds = thresholds['warning']
+      @critical_seconds = thresholds['critical']
+      @clear_seconds = self.class.env_float('OPENC3_LAG_CLEAR_SECONDS', 10.0)
+      @renotify_seconds = self.class.env_float('OPENC3_LAG_RENOTIFY_SECONDS', 300.0)
+      @state = OK
+      @max_lag = 0.0
+      @max_lag_topic = nil
+      @below_since = nil
+      @below_ok_since = nil
+      @last_notify = nil
+      @last_notify_level = OK
+      @notified = false
+      @gauges = {}
+      @mutex = Mutex.new
+    end
+
+    # Record that a message was read from a topic
+    #
+    # @param topic [String] Redis stream the message was read from
+    # @param msg_id [String] Redis stream id of the message
+    # @param metric_name [String] Gauge metric to update with the lag
+    # @param help [String] Help text for the gauge metric
+    # @return [Float] The lag in seconds
+    def record(topic, msg_id, metric_name: nil, help: nil, now: Time.now.to_f)
+      lag = now - (id_ms(msg_id) / 1000.0)
+      update_gauge(metric_name, help, lag, now) if @metric and metric_name
+      return lag if @warn_seconds <= 0 # Notifications disabled
+
+      # Fast path for the common case of keeping up: skip the mutex and state machine
+      return lag if @state == OK and lag < @warn_seconds
+
+      @mutex.synchronize { update_state(topic, lag, now) }
+      return lag
+    end
+
+    protected
+
+    # Topics read by one microservice can lag by very different amounts, so
+    # publish the maximum lag rather than whichever topic was read last. The
+    # previous window's maximum is held through the current window so a spike
+    # just before a window boundary is still published.
+    def update_gauge(metric_name, help, lag, now)
+      window = (now / GAUGE_WINDOW_SECONDS).to_i
+      gauge = @gauges[metric_name]
+      if gauge.nil?
+        gauge = @gauges[metric_name] = { window: window, max: lag, previous_max: 0.0 }
+      elsif window != gauge[:window]
+        gauge[:previous_max] = (window == gauge[:window] + 1) ? gauge[:max] : 0.0
+        gauge[:window] = window
+        gauge[:max] = lag
+      elsif lag > gauge[:max]
+        gauge[:max] = lag
+      else
+        return # Published value is unchanged
+      end
+      value = (gauge[:max] > gauge[:previous_max]) ? gauge[:max] : gauge[:previous_max]
+      @metric.set(name: metric_name, value: value, type: 'gauge', unit: 'seconds', help: help)
+    end
+
+    def update_state(topic, lag, now)
+      if lag < @warn_seconds * CLEAR_RATIO
+        @below_ok_since ||= now
+      else
+        @below_ok_since = nil
+      end
+      if lag > @max_lag
+        @max_lag = lag
+        @max_lag_topic = topic
+      end
+      level = if lag >= @critical_seconds
+        CRITICAL
+      elsif lag >= @warn_seconds
+        WARN
+      else
+        OK
+      end
+
+      if level > @state
+        @state = level
+        @below_since = nil
+        # Bursty load can make a microservice recover and fall behind over and
+        # over, so falling back to an already announced level soon after the
+        # last notification continues that episode rather than notifying again
+        unless @last_notify and level <= @last_notify_level and (now - @last_notify) < @renotify_seconds
+          notify_lag(topic, lag, now)
+        end
+      elsif @state != OK
+        threshold = (@state == CRITICAL) ? @critical_seconds : @warn_seconds
+        if lag < threshold * CLEAR_RATIO
+          @below_since ||= now
+          if (now - @below_since) >= @clear_seconds
+            @below_since = nil
+            if @state == CRITICAL and (@below_ok_since.nil? or (now - @below_ok_since) < @clear_seconds)
+              @state = WARN
+              @below_since = @below_ok_since
+            else
+              @state = OK
+              @logger.info("#{@name} has caught up processing (max lag #{@max_lag.round(1)}s on #{@max_lag_topic})", type: Logger::NOTIFICATION) if @notified
+              @notified = false
+              @max_lag = 0.0
+              @max_lag_topic = nil
+            end
+          end
+        else
+          @below_since = nil
+          notify_lag(topic, lag, now) if (now - @last_notify) >= @renotify_seconds
+        end
+      end
+    end
+
+    def notify_lag(topic, lag, now)
+      message = "#{@name} is falling behind processing: #{lag.round(1)}s behind on #{topic} (max #{@max_lag.round(1)}s on #{@max_lag_topic})"
+      if @state == CRITICAL
+        @logger.error(message, type: Logger::NOTIFICATION)
+      else
+        @logger.warn(message, type: Logger::NOTIFICATION)
+      end
+      @last_notify = now
+      @last_notify_level = @state
+      @notified = true
+    end
+
+    def id_ms(id)
+      id.to_s.split('-')[0].to_i
+    end
+  end
+end

data/lib/openc3/version.rb

@@ -1,14 +1,14 @@
 # encoding: ascii-8bit
 
-OPENC3_VERSION = '7.4.1'
+OPENC3_VERSION = '7.5.0'
 module OpenC3
   module Version
     MAJOR = '7'
-    MINOR = '4'
-    PATCH = '1'
+    MINOR = '5'
+    PATCH = '0'
     OTHER = ''
-    BUILD = '2d4089e0687654512c2eb84722a5a4fa699dd5c3'
+    BUILD = 'f77108e1b9d90505ae1ba0a893150c837f879288'
   end
-  VERSION = '7.4.1'
-  GEM_VERSION = '7.4.1'
+  VERSION = '7.5.0'
+  GEM_VERSION = '7.5.0'
 end

data/templates/conversion/conversion.py

@@ -1,6 +1,6 @@
 from openc3.conversions.conversion import Conversion
-# Using tlm() requires the following:
-# from openc3.api.tlm_api import tlm
+# Using System.telemetry.packet() requires the following:
+# from openc3.system.system import System
 
 # Custom conversion class
 # See https://docs.openc3.com/docs/configuration/conversions
@@ -20,4 +20,8 @@ class <%= conversion_class %>(Conversion):
         # Used when conversion is applied to a regular (not DERIVED) item
         # NOTE: You can also use packet.read("ITEM") to get additional values
         # return value / 2 * packet.read("OTHER_ITEM")
+        #
+        # In a READ_CONVERSION, read the latest value from a different packet in the same target
+        # other = System.telemetry.packet(packet.target_name, "OTHER_PACKET")
+        # return value * other.read("OTHER_ITEM")
         return value

data/templates/conversion/conversion.rb

@@ -20,6 +20,11 @@ module OpenC3
       # Used when conversion is applied to a regular (not DERIVED) item
       # NOTE: You can also use packet.read("ITEM") to get additional values
       # return value / 2 * packet.read("OTHER_ITEM")
+      #
+      # In a READ_CONVERSION, read the latest value from a different packet in the same target
+      # other = System.telemetry.packet(packet.target_name, "OTHER_PACKET")
+      # return value * other.read("OTHER_ITEM")
+      return value
     end
   end
 end

data/templates/plugin/.github/workflows/playwright.yml

@@ -0,0 +1,62 @@
+# Builds this plugin and installs it into a real COSMOS with Playwright.
+#
+# Runs the most recent COSMOS Core release in Docker, builds the plugin gem, and
+# drives the Admin tool the way an operator would: Install From File, submit the
+# variables dialog, and wait for the install to report Complete. That catches
+# what unit tests can't - a plugin.txt that doesn't parse, a target that doesn't
+# build, a microservice that won't start.
+#
+# Nothing here needs configuring as the plugin grows. Tool pages are discovered
+# from the COSMOS API after install and checked for browser console errors, and
+# a frontend is built with pnpm whenever the plugin has a package.json, so
+# `cli generate tool` and `cli generate widget` are covered with no edit here.
+#
+# Runs on every commit and once a week. The weekly run is what catches a new
+# COSMOS release breaking this plugin. Note that GitHub disables scheduled
+# workflows in a repo with no activity for 60 days.
+
+name: Playwright
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: ["**"]
+  # Off the hour because scheduled runs at :00 get queued behind everyone else's
+  schedule:
+    - cron: '27 7 * * 1'
+  workflow_dispatch:
+    inputs:
+      cosmos_version:
+        description: 'COSMOS version to run, or latest'
+        required: false
+        type: string
+        default: 'latest'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  # Superseding a pull request run is fine, but a scheduled run and a push to
+  # main share a ref, so cancelling on those would have them kill each other
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  playwright:
+    uses: OpenC3/.github/.github/workflows/plugin-playwright-reusable.yml@main
+    with:
+      plugin_directory: .
+      cosmos_version: ${{ inputs.cosmos_version || 'latest' }}
+      # Once you have added targets with `cli generate target`, list them here
+      # and the install is checked for them:
+      # expected_targets: MY_TARGET ANOTHER_TARGET
+      #
+      # install_demo: true  # Uncomment if this plugin references INST/INST2
+      # is_tool: true  # Uncomment to require that this plugin registers a tool
+      #
+      # Tool pages are checked for browser console errors automatically. Add
+      # regexes here for errors to allow:
+      # console_ignore: |
+      #   some third party gripe
+      #   another known message

data/templates/plugin/.github/workflows/release.yml

@@ -0,0 +1,45 @@
+# Builds the gem, tags the commit, and creates a GitHub release. Run it by hand
+# from the Actions tab and give it a version number.
+#
+# Both publish steps are off by default so a first release works with no setup
+# at all. Tick the box in the run dialog when you are ready to publish, after
+# adding the matching secret to the repo:
+#
+#   Deploy to OpenC3 App Store  ->  OPENC3_API_TOKEN  (your store account token)
+#   Publish the gem to RubyGems ->  RUBYGEMS_API_KEY  (your RubyGems API key)
+#
+# A tool or widget frontend needs nothing set here: once the plugin has a
+# package.json it is built with pnpm before the gem is packaged.
+
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version number'
+        required: true
+        type: string
+      deploy_to_store:
+        description: 'Deploy to OpenC3 App Store'
+        required: false
+        type: boolean
+        default: false
+      deploy_to_rubygems:
+        description: 'Publish the gem to RubyGems'
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  release:
+    uses: OpenC3/.github/.github/workflows/release-cosmos-plugin-reusable.yml@main
+    with:
+      version: ${{ inputs.version }}
+      deploy_to_store: ${{ inputs.deploy_to_store }}
+      deploy_to_rubygems: ${{ inputs.deploy_to_rubygems }}
+      plugin_directory: .
+      # is_tool: true  # Uncomment to fail the release if package.json is missing
+    secrets:
+      OPENC3_API_TOKEN: ${{ secrets.OPENC3_API_TOKEN }}
+      RUBYGEMS_API_KEY: ${{ secrets.RUBYGEMS_API_KEY }}

data/templates/plugin/.github/workflows/unit-tests.yml

@@ -0,0 +1,35 @@
+# Runs this plugin's unit tests on every push and pull request.
+#
+# Python tests are discovered in tests/ or test/, Ruby specs in spec/ or specs/.
+# Whichever this plugin doesn't have is skipped, so there is nothing to set up:
+# add a test file and it starts running. A plugin with no tests at all passes
+# with a warning until you set require_tests below.
+#
+# Dependencies are picked up automatically - requirements.txt for the code under
+# test, <test dir>/requirements.txt for test only ones, and a Gemfile for Ruby
+# when there is one.
+
+name: Unit Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: ["**"]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  unit-tests:
+    uses: OpenC3/.github/.github/workflows/plugin-unit-tests-reusable.yml@main
+    with:
+      plugin_directory: .
+      # python_version: '3.12'  # Uncomment to pin a different Python
+      # ruby_version: '3.4'  # Uncomment to pin a different Ruby
+      # require_tests: true  # Uncomment to fail the build when there are no tests
+      # gemfile: Gemfile.dev  # Uncomment if the specs need a different Gemfile than the plugin's own

data/templates/plugin/.gitignore

@@ -1 +1,7 @@
 node_modules
+<% if @@language == 'py' -%>
+.venv/
+__pycache__/
+.pytest_cache/
+.ruff_cache/
+<% end -%>