openc3 7.2.1 → 7.4.0

data/lib/openc3/topics/decom_interface_topic.rb

@@ -46,7 +46,7 @@ module OpenC3
       raise "Timeout of #{timeout}s waiting for cmd ack. Does target '#{target_name}' exist?"
     end
 
-    def self.inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, stored: false, timeout: 5, scope:)
+    def self.inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, stored: false, timeout: 5, received_time: nil, scope:)
       data = {}
       data['target_name'] = target_name.to_s.upcase
       data['packet_name'] = packet_name.to_s.upcase
@@ -55,6 +55,7 @@ module OpenC3
 
       db_shard = Store.db_shard_for_target(target_name, scope: scope)
       data['stored'] = stored
+      data['received_time'] = received_time unless received_time.nil?
       ack_topic = "{#{scope}__ACKCMD}TARGET__#{target_name}"
       Topic.update_topic_offsets([ack_topic], db_shard: db_shard)
       decom_id = Topic.write_topic("#{scope}__DECOMINTERFACE__{#{target_name}}",

data/lib/openc3/topics/interface_topic.rb

@@ -148,7 +148,7 @@ module OpenC3
       Topic.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'protocol_cmd' => JSON.generate(data, allow_nan: true) }, '*', 100, db_shard: db_shard)
     end
 
-    def self.inject_tlm(interface_name, target_name, packet_name, item_hash = nil, type: :CONVERTED, stored: false, timeout: nil, scope:)
+    def self.inject_tlm(interface_name, target_name, packet_name, item_hash = nil, type: :CONVERTED, stored: false, timeout: nil, received_time: nil, scope:)
       interface_name = interface_name.upcase
       db_shard = _db_shard_for_interface(interface_name, scope: scope)
 
@@ -162,6 +162,7 @@ module OpenC3
       data['item_hash'] = item_hash
       data['type'] = type
       data['stored'] = stored
+      data['received_time'] = received_time unless received_time.nil?
       cmd_id = Topic.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'inject_tlm' => JSON.generate(data, allow_nan: true) }, '*', 100, db_shard: db_shard)
       time = Time.now
       while (Time.now - time) < timeout
@@ -211,7 +212,12 @@ module OpenC3
       while (Time.now - time) < timeout
         Topic.read_topics([ack_topic], db_shard: db_shard) do |_topic, _msg_id, msg_hash, _redis|
           if msg_hash["id"] == cmd_id
-            return JSON.parse(msg_hash["result"], :allow_nan => true, :create_additions => true)
+            begin
+              return JSON.parse(msg_hash["result"], :allow_nan => true, :create_additions => true)
+            rescue JSON::ParserError
+              # The microservice returns a plain error message rather than JSON if details raises
+              raise "interface_details failed: #{msg_hash['result']}"
+            end
           end
         end
       end

data/lib/openc3/topics/limits_event_topic.rb

@@ -100,7 +100,7 @@ module OpenC3
 
       when :LIMITS_SET
         sets = sets(scope: scope)
-        raise "Set '#{event[:set]}' does not exist!" unless sets.key?(event[:set])
+        raise "Limits set '#{event[:set]}' does not exist" unless sets.key?(event[:set])
 
         # Set all existing sets to "false"
         sets = sets.transform_values! { |_key, _value| "false" }

data/lib/openc3/topics/router_topic.rb

@@ -170,7 +170,12 @@ module OpenC3
       while (Time.now - time) < timeout
         Topic.read_topics([ack_topic], db_shard: db_shard) do |_topic, _msg_id, msg_hash, _redis|
           if msg_hash["id"] == cmd_id
-            return JSON.parse(msg_hash["result"], :allow_nan => true, :create_additions => true)
+            begin
+              return JSON.parse(msg_hash["result"], :allow_nan => true, :create_additions => true)
+            rescue JSON::ParserError
+              # The microservice returns a plain error message rather than JSON if details raises
+              raise "router_details failed: #{msg_hash['result']}"
+            end
           end
         end
       end

data/lib/openc3/utilities/authentication.rb

@@ -28,7 +28,7 @@ module OpenC3
   # OpenC3 COSMOS Core authentication code
   class OpenC3Authentication
     def initialize()
-      password = ENV['OPENC3_API_PASSWORD']
+      password = ENV.fetch('OPENC3_API_PASSWORD', nil)
       if password.nil?
         raise OpenC3AuthenticationError, "Authentication requires environment variable OPENC3_API_PASSWORD"
       end
@@ -172,8 +172,8 @@ module OpenC3
       if ENV['OPENC3_API_USER'] and ENV['OPENC3_API_PASSWORD']
         # Username and password
         data = {
-          'username' => ENV['OPENC3_API_USER'],
-          'password' => ENV['OPENC3_API_PASSWORD'],
+          'username' => ENV.fetch('OPENC3_API_USER', nil),
+          'password' => ENV.fetch('OPENC3_API_PASSWORD', nil),
           'client_id' => client_id,
           'grant_type' => 'password',
           'scope' => openid_scope
@@ -189,7 +189,7 @@ module OpenC3
         @refresh_expires_at = current_time + oath['refresh_expires_in'] - REFRESH_OFFSET_SECONDS
       else
         # Offline Access Token
-        @refresh_token ||= ENV['OPENC3_API_TOKEN']
+        @refresh_token ||= ENV.fetch('OPENC3_API_TOKEN', nil)
         _refresh_token(current_time)
       end
     end

data/lib/openc3/utilities/aws_bucket.rb

@@ -99,9 +99,9 @@ module OpenC3
     # Apply bucket policy to grant ScriptRunner user access to config and logs buckets
     # This provides reduced permissions compared to the root user for security
     def ensure_scriptrunner_policy(config_bucket, logs_bucket)
-      sr_username = ENV['OPENC3_SR_BUCKET_USERNAME']
+      sr_username = ENV.fetch('OPENC3_SR_BUCKET_USERNAME', nil)
       return unless sr_username
-      return if sr_username == ENV['OPENC3_BUCKET_USERNAME'] # Same as root, no policy needed
+      return if sr_username == ENV.fetch('OPENC3_BUCKET_USERNAME', nil) # Same as root, no policy needed
 
       # Policy for config bucket - read targets, read/write targets_modified
       # Note: versitygw expects Principal as comma-separated raw usernames
@@ -213,6 +213,14 @@ module OpenC3
     # If the key is not found return nil
     rescue Aws::S3::Errors::NoSuchKey
       nil
+    # AWS S3 returns 403 AccessDenied rather than 404 NoSuchKey when the caller
+    # does not have s3:ListBucket on the bucket. Callers which probe for an
+    # optional key (i.e. TargetFile.body checking targets_modified first) would
+    # otherwise raise instead of falling through, so treat it as not found.
+    # Warn since a genuine permission problem looks identical from here.
+    rescue Aws::S3::Errors::AccessDenied
+      Logger.warn("Access denied reading #{bucket}/#{key}. Treating as not found. Verify bucket permissions if this key should exist.")
+      nil
     end
 
     def list_objects(bucket:, prefix: nil, max_request: 1000, max_total: 100_000)
@@ -233,7 +241,7 @@ module OpenC3
       # Array of objects with key and size methods
       result
     rescue Aws::S3::Errors::NoSuchBucket
-      raise NotFound, "Bucket '#{bucket}' does not exist."
+      raise NotFound, "Bucket '#{bucket}' does not exist (list_objects)"
     end
 
     # Lists the files under a specified path
@@ -283,7 +291,7 @@ module OpenC3
       end
       result
     rescue Aws::S3::Errors::NoSuchBucket
-      raise NotFound, "Bucket '#{bucket}' does not exist."
+      raise NotFound, "Bucket '#{bucket}' does not exist (list_files)"
     end
 
     # get metadata for a specific object

data/lib/openc3/utilities/bucket.rb

@@ -23,7 +23,7 @@ module OpenC3
     def self.getClient
       raise 'OPENC3_CLOUD environment variable is required' unless ENV['OPENC3_CLOUD']
       # Base is AwsBucket which works with S3-compatible storage (versitygw), Enterprise implements additional
-      bucket_class = ENV['OPENC3_CLOUD'].capitalize + 'Bucket'
+      bucket_class = ENV.fetch('OPENC3_CLOUD', 'local').capitalize + 'Bucket'
       klass = OpenC3.require_class('openc3/utilities/'+bucket_class.class_name_to_filename)
       klass.new
     end

data/lib/openc3/utilities/bucket_file_cache.rb

@@ -163,16 +163,24 @@ class BucketFileCache
 
     @current_disk_usage = 0
     @queued_bucket_files = []
+    # Mirrors @queued_bucket_files as bucket_path => true. Membership is checked
+    # once per file close in unreserve() and once per entry in age_out_files(),
+    # both of which are O(cache size) linear scans against the bare Array.
+    @queued_path_hash = {}
     @bucket_file_hash = {}
     bucket_file = nil
 
+    # NOTE: check_time must be initialized outside the loop or it is pushed
+    # forward on every iteration and the age out check below never fires
+    check_time = Time.now + CHECK_TIME_SECONDS # Check for aged out files periodically
+
     @thread = Thread.new do
       client = OpenC3::Bucket.getClient()
       while true
-        check_time = Time.now + CHECK_TIME_SECONDS # Check for aged out files periodically
         if @queued_bucket_files.length > 0 and @current_disk_usage < MAX_DISK_USAGE
           @@mutex.synchronize do
             bucket_file = @queued_bucket_files.shift
+            @queued_path_hash.delete(bucket_file.bucket_path) if bucket_file
           end
           begin
             retrieved = bucket_file.retrieve(client)
@@ -187,18 +195,7 @@ class BucketFileCache
           # Nothing to do or disk full
           if Time.now > check_time
             check_time = Time.now + CHECK_TIME_SECONDS
-            # Delete any files that aren't reserved and are old
-            removed_files = []
-            @bucket_file_hash.each do |bucket_path, bucket_file|
-              deleted = bucket_file.age_check
-              if deleted
-                removed_files << bucket_path
-                @current_disk_usage -= bucket_file.size
-              end
-            end
-            removed_files.each do |bucket_path|
-              @bucket_file_hash.delete(bucket_path)
-            end
+            age_out_files()
           end
           sleep(1)
         end
@@ -216,8 +213,9 @@ class BucketFileCache
     return instance().reserve(bucket_path)
   end
 
-  def self.unreserve(bucket_path)
-    return instance().unreserve(bucket_path)
+  # @param bucket_file_or_path [BucketFile|String] BucketFile or its bucket path
+  def self.unreserve(bucket_file_or_path)
+    return instance().unreserve(bucket_file_or_path)
   end
 
   def hint(bucket_paths)
@@ -236,30 +234,65 @@ class BucketFileCache
       retrieved = bucket_file.reserve
       @current_disk_usage += bucket_file.size if retrieved
       @queued_bucket_files.delete(bucket_file)
+      @queued_path_hash.delete(bucket_path)
       return bucket_file
     end
   end
 
-  def unreserve(bucket_path)
+  # Callers hold the BucketFile returned by reserve() but the cache is keyed by
+  # bucket path, so accept either and normalize to the path.
+  # @param bucket_file_or_path [BucketFile|String] BucketFile or its bucket path
+  def unreserve(bucket_file_or_path)
+    if bucket_file_or_path.is_a?(BucketFile)
+      released = bucket_file_or_path
+      bucket_path = released.bucket_path
+    else
+      released = nil
+      bucket_path = bucket_file_or_path
+    end
     @@mutex.synchronize do
       bucket_file = @bucket_file_hash[bucket_path]
-      if bucket_file
-        bucket_file.unreserve
-        if bucket_file.reservation_count <= 0 and !@queued_bucket_files.include?(bucket_file)
-          @current_disk_usage -= bucket_file.size
-          @bucket_file_hash.delete(bucket_file)
-        end
+      return if bucket_file.nil?
+      # Releasing to zero drops the entry, so the next reserve of the same path
+      # builds a new BucketFile. A BucketFile that is no longer the entry for
+      # its path was therefore already released, and releasing it again would
+      # decrement its replacement and delete a local file another reader still
+      # has open. Only the current owner of the path may release it.
+      return unless released.nil? or released.equal?(bucket_file)
+
+      bucket_file.unreserve
+      if bucket_file.reservation_count <= 0 and !@queued_path_hash[bucket_path]
+        @current_disk_usage -= bucket_file.size
+        @bucket_file_hash.delete(bucket_path)
       end
     end
   end
 
   # Private
 
+  # Delete the local copy of any file that is old and no longer reserved.
+  # Files still waiting in the download queue are skipped because the download
+  # thread would re-add their size to @current_disk_usage after removal.
+  def age_out_files
+    @@mutex.synchronize do
+      @bucket_file_hash.delete_if do |bucket_path, bucket_file|
+        next false if @queued_path_hash[bucket_path]
+        if bucket_file.age_check
+          @current_disk_usage -= bucket_file.size
+          true
+        else
+          false
+        end
+      end
+    end
+  end
+
   def create_bucket_file(bucket_path)
     bucket_file = @bucket_file_hash[bucket_path]
     unless bucket_file
       bucket_file = BucketFile.new(bucket_path)
       @queued_bucket_files << bucket_file
+      @queued_path_hash[bucket_path] = true
       @bucket_file_hash[bucket_path] = bucket_file
     end
     return bucket_file

data/lib/openc3/utilities/bucket_utilities.rb

@@ -25,8 +25,7 @@ module OpenC3
     DIRECTORY_TIMESTAMP_FORMAT = "%Y%m%d"
 
     def self.bucket_load(*args, scope: $openc3_scope)
-      scope = ENV['OPENC3_SCOPE'] unless scope
-      scope = 'DEFAULT' unless scope
+      scope = ENV.fetch('OPENC3_SCOPE', 'DEFAULT') unless scope
       path = args[0]
 
       # Only support TARGET files
@@ -93,7 +92,7 @@ module OpenC3
           # this to work with really large files. Otherwise the entire file has
           # to be held in memory!
           File.open(filename, 'rb') do |file|
-            client.put_object(bucket: ENV['OPENC3_LOGS_BUCKET'], key: bucket_key, body: file, metadata: metadata)
+            client.put_object(bucket: ENV.fetch('OPENC3_LOGS_BUCKET', nil), key: bucket_key, body: file, metadata: metadata)
           end
         rescue => err
           # Try to upload file three times
@@ -104,7 +103,7 @@ module OpenC3
           retry
         end
 
-        Logger.debug "wrote #{ENV['OPENC3_LOGS_BUCKET']}/#{bucket_key}"
+        Logger.debug "wrote #{ENV.fetch('OPENC3_LOGS_BUCKET', nil)}/#{bucket_key}"
 
         File.delete(orig_filename) if orig_filename
         File.delete(filename)

data/lib/openc3/utilities/config_overlay.rb

@@ -0,0 +1,75 @@
+# encoding: ascii-8bit
+
+# Copyright 2026 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE.md for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+module OpenC3
+  # Shared predicates describing which parts of the user-writable config overlay
+  # (targets_modified) a non-admin may write.
+  #
+  # The cmd_tlm overlay (targets_modified/<TARGET>/cmd_tlm/...) is loaded by
+  # System.setup_targets and processed by PacketConfig, which evaluates
+  # GENERIC_READ_CONVERSION / GENERIC_WRITE_CONVERSION blocks as code in the
+  # decom microservices. Writing it is therefore an admin operation, matching the
+  # tier COSMOS requires everywhere else code is introduced (plugin install).
+  # Only the server-side dynamic-packet mechanism (TargetModel#dynamic_update)
+  # writes that area without going through an API request.
+  #
+  # Every API writer that can reach the overlay must consult these predicates:
+  #   - storage_controller#get_upload_presigned_request / delete (presigned S3 upload)
+  #   - tables_controller#save / save_as / generate / destroy (Table -> TargetFile)
+  #   - scripts_controller#create / destroy (Script -> TargetFile)
+  module ConfigOverlay
+    # Config bucket areas a non-admin is allowed to write at all
+    NON_ADMIN_AREAS = ['targets_modified', 'tmp'].freeze
+
+    # Target subdirectory whose contents are executed as code
+    CODE_AREA = 'cmd_tlm'
+
+    # Split a path into segments, or nil if the path is not canonical.
+    #
+    # Positional segment checks (parts[1], parts[3]) can be bypassed by a path
+    # the object store normalizes differently, so empty '//' segments, '.'/'..'
+    # segments, and leading/trailing slashes yield nil and every caller fails
+    # closed (requires admin) on nil.
+    def self.canonical_parts(path)
+      return nil if path.nil? || path.empty?
+      return nil if path.start_with?('/') || path.end_with?('/')
+      parts = path.split('/')
+      return nil if parts.any? { |part| part.empty? || part == '.' || part == '..' }
+      parts
+    end
+
+    # True if the overlay-relative name targets the cmd_tlm subtree, i.e. writing
+    # it requires admin. Name is relative to targets_modified/, e.g.
+    # "<TARGET>/cmd_tlm/tlm.txt". Fails closed (true) on non-canonical names.
+    def self.cmd_tlm_overlay?(name)
+      parts = canonical_parts(name)
+      return true if parts.nil?
+      # parts: <TARGET> / <area> / ...
+      parts[1] == CODE_AREA
+    end
+
+    # True if the given config bucket key is an overlay path a non-admin may
+    # write. Key is the full bucket key, e.g.
+    # "<SCOPE>/targets_modified/<TARGET>/screens/x.txt". Fails closed (false) on
+    # non-canonical keys.
+    def self.non_admin_writable_key?(key)
+      parts = canonical_parts(key)
+      return false if parts.nil?
+      # parts: <SCOPE> / <area> / <TARGET> / <subdir> / ...
+      area = parts[1]
+      return false unless NON_ADMIN_AREAS.include?(area)
+      return false if area == 'targets_modified' && parts[3] == CODE_AREA
+      true
+    end
+  end
+end

data/lib/openc3/utilities/crc.rb

@@ -75,18 +75,22 @@ module OpenC3
         @bit_size = 8
         pack = 'C'
         filter_mask = 0xFF
+        right_shift = 0
       when 'OpenC3::Crc16'
         @bit_size = 16
         pack = 'S'
         filter_mask = 0xFFFF
+        right_shift = 8
       when 'OpenC3::Crc32'
         @bit_size = 32
         pack = 'I'
         filter_mask = 0xFFFFFFFF
+        right_shift = 24
       when 'OpenC3::Crc64'
         @bit_size = 64
         pack = 'Q'
         filter_mask = 0xFFFFFFFFFFFFFFFF
+        right_shift = 56
       end
       if RUBY_ENGINE == 'ruby' and !ENV['OPENC3_NO_EXT']
         (0..255).each do |index|
@@ -96,6 +100,29 @@ module OpenC3
         (0..255).each do |index|
           @table << (compute_table_entry(index, @bit_size) & filter_mask)
         end
+
+        @filter_mask = filter_mask
+        @right_shift = right_shift
+        @bit_reverse = method("bit_reverse_#{@bit_size}")
+
+        # A reflected (least-significant-bit first) table avoids reversing each
+        # input byte in calc. CRC32 and CRC64 get three additional tables so
+        # their common reflected forms can process four bytes per Ruby loop.
+        if @reflect
+          reflected_table = Array.new(256) do |index|
+            @bit_reverse.call(@table[BIT_REVERSE_TABLE[index]])
+          end
+          @reflected_tables = [reflected_table]
+          if @bit_size >= 32
+            3.times do
+              previous_table = @reflected_tables[-1]
+              @reflected_tables << Array.new(256) do |index|
+                value = previous_table[index]
+                (value >> 8) ^ reflected_table[value & 0xFF]
+              end
+            end
+          end
+        end
       end
     end
 
@@ -140,47 +167,47 @@ module OpenC3
 
     if RUBY_ENGINE != 'ruby' or ENV['OPENC3_NO_EXT']
       def calc(data, seed = @seed)
-        crc = seed
-
-        case @bit_size
-        when 8
-          right_shift = 0
-          filter_mask = 0xFF
-          final_bit_reverse = method(:bit_reverse_8)
-        when 16
-          right_shift = 8
-          filter_mask = 0xFFFF
-          final_bit_reverse = method(:bit_reverse_16)
-        when 32
-          right_shift = 24
-          filter_mask = 0xFFFFFFFF
-          final_bit_reverse = method(:bit_reverse_32)
-        when 64
-          right_shift = 56
-          filter_mask = 0xFFFFFFFFFFFFFFFF
-          final_bit_reverse = method(:bit_reverse_64)
-        end
+        seed = @seed if seed.nil?
 
         if @reflect
-          data.each_byte do |byte|
-            crc = ((crc << 8) & filter_mask) ^ @table[(crc >> right_shift) ^ bit_reverse_8(byte)]
+          crc = @bit_reverse.call(seed)
+          reflected_table = @reflected_tables[0]
+          index = 0
+
+          if @bit_size >= 32
+            table1 = @reflected_tables[1]
+            table2 = @reflected_tables[2]
+            table3 = @reflected_tables[3]
+            word_count = data.bytesize / 4
+            unless word_count.zero?
+              data.unpack("V#{word_count}").each do |word|
+                value = crc ^ word
+                crc = (value >> 32) ^
+                      table3[value & 0xFF] ^
+                      table2[(value >> 8) & 0xFF] ^
+                      table1[(value >> 16) & 0xFF] ^
+                      reflected_table[(value >> 24) & 0xFF]
+              end
+              index = word_count * 4
+            end
           end
 
-          if @xor
-            return final_bit_reverse.call(crc ^ filter_mask)
-          else
-            return final_bit_reverse.call(crc)
+          while index < data.bytesize
+            crc = (crc >> 8) ^ reflected_table[(crc ^ data.getbyte(index)) & 0xFF]
+            index += 1
           end
+
+          return @xor ? crc ^ @filter_mask : crc
         else
+          crc = seed
+          table = @table
+          filter_mask = @filter_mask
+          right_shift = @right_shift
           data.each_byte do |byte|
-            crc = ((crc << 8) & filter_mask) ^ @table[(crc >> right_shift) ^ byte]
+            crc = ((crc << 8) & filter_mask) ^ table[(crc >> right_shift) ^ byte]
           end
 
-          if @xor
-            return crc ^ filter_mask
-          else
-            return crc
-          end
+          return @xor ? crc ^ filter_mask : crc
         end
       end
     end

data/lib/openc3/utilities/csv.rb

@@ -59,7 +59,7 @@ module OpenC3
     # @param index [Integer] Which value to return
     # @return [Boolean] Single value converted to a boolean (true or false)
     def bool(item, index = 0)
-      raise "#{item} not found" unless keys.include?(item)
+      raise "CSV key '#{item}' not found for bool conversion" unless keys.include?(item)
 
       if Range === index
         @hash[item][index].map do |x|
@@ -91,7 +91,7 @@ module OpenC3
     # @param index [Integer] Which value to return
     # @return [Integer] Single value converted to an integer
     def int(item, index = 0)
-      raise "#{item} not found" unless keys.include?(item)
+      raise "CSV key '#{item}' not found for int conversion" unless keys.include?(item)
 
       if Range === index
         @hash[item][index].map { |x| x.to_i }
@@ -107,7 +107,7 @@ module OpenC3
     # @param index [Integer] Which value to return
     # @return [Float] Single value converted to a float
     def float(item, index = 0)
-      raise "#{item} not found" unless keys.include?(item)
+      raise "CSV key '#{item}' not found for float conversion" unless keys.include?(item)
 
       if Range === index
         @hash[item][index].map { |x| x.to_f }
@@ -122,7 +122,7 @@ module OpenC3
     # @param index [Integer] Which value to return
     # @return [String] Single value converted to a string
     def string(item, index = 0)
-      raise "#{item} not found" unless keys.include?(item)
+      raise "CSV key '#{item}' not found for string lookup" unless keys.include?(item)
 
       if Range === index
         @hash[item][index].map { |x| x.to_s }
@@ -138,7 +138,7 @@ module OpenC3
     # @param index [Integer] Which value to return
     # @return [Symbol] Single value converted to a symbol
     def symbol(item, index = 0)
-      raise "#{item} not found" unless keys.include?(item)
+      raise "CSV key '#{item}' not found for symbol lookup" unless keys.include?(item)
 
       if Range === index
         @hash[item][index].map { |x| x.intern }