openc3 7.0.0.pre.rc2 → 7.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a7c545191fe3248b560961c422b5906474a068d06df3d480a35a61dba621e469
-  data.tar.gz: 77b9201a38591fad76787947918fd6910a8599e09368cdfdf9d0870520a9f073
+  metadata.gz: 0a06901f01bc9d8e7b6f0b123296da67475c08ea9cf50ef788a86b6da1308fcb
+  data.tar.gz: 34732b912cea1b4677f3e94ac0ba9f437e99217ccf54f9250816176ef5b63db1
 SHA512:
-  metadata.gz: 164c14d7fd49d30bcaf8f8838cd2e3d63c3fbbccb514e7b5a40348868e68317100d8a48beda0a3015959d0f49d3093b40b1184e517093ae7a4df4aafb8710be0
-  data.tar.gz: cb6276f37e41717565abbbb24d7df1006cf4dfbfdeff92c76b34638dff29279dcfed536aced5a488d1aacf63541568d3a0c8da4b7edd5ecaed9f792b3a12a7ae
+  metadata.gz: 5cda606ccb010606ad4b432685f53428cbf10b216f1cd06f2f55baac28fb723514dc3f8dfeadafc0958734237d3a7faf3f42439695481145914434e2a4bb1a58
+  data.tar.gz: 98532ea424c421554d45a6d155fadfc27e67ddf969a96be7653daebb30d5530dbb7601e13b5d3a3afa7dfd56718c161c060e81d67a5ee967a9248225554e1a03

data/bin/openc3cli

@@ -1348,12 +1348,21 @@ if not ARGV[0].nil? # argument(s) given
       exit 0
     end
     client = OpenC3::Bucket.getClient()
-    ENV.map do |key, value|
-      if key.match(/^OPENC3_(.+)_BUCKET$/) && !value.empty?
-        client.create(value)
+    if ENV.fetch('OPENC3_CLOUD', 'local') == 'local'
+      # In local mode we want to create all buckets to ensure they exist
+      # Cloud deployments will have created buckets during provisioning
+      # so we only need to ensure the correct policies and permissions are in place
+      ENV.map do |key, value|
+        if key.match(/^OPENC3_(.+)_BUCKET$/) && !value.empty?
+          client.create(value)
+        end
       end
     end
-    client.ensure_public(ENV['OPENC3_TOOLS_BUCKET'])
+    # Unless explicitly disabled, ensure the tools bucket is public
+    unless ENV.fetch("OPENC3_NO_BUCKET_POLICY", false)
+      client.ensure_public(ENV['OPENC3_TOOLS_BUCKET'])
+    end
+    # Always ensure the scriptrunner policy is in place since it is required for script execution
     client.ensure_scriptrunner_policy(ENV['OPENC3_CONFIG_BUCKET'], ENV['OPENC3_LOGS_BUCKET'])
 
   when 'runmigrations'

data/bin/pipinstall

@@ -1,14 +1,13 @@
 #!/bin/sh
-python3 -m venv $PYTHONUSERBASE
-source $PYTHONUSERBASE/bin/activate
-echo "pip3 install $@"
-pip3 install "$@"
+uv venv "$PYTHONUSERBASE"
+echo "uv pip install $@"
+uv pip install --python "$PYTHONUSERBASE" "$@"
 if [ $? -eq 0 ]; then
     echo "Command succeeded"
 else
     echo "Command failed - retrying with --no-index"
-    pip3 install --no-index "$@"
-    if [ $? -eq 0 ]; then
-        echo "ERROR: pip3 install failed"
+    uv pip install --python "$PYTHONUSERBASE" --no-index "$@"
+    if [ $? -ne 0 ]; then
+        echo "ERROR: uv pip install failed"
     fi
 fi

data/bin/pipuninstall

@@ -1,10 +1,8 @@
 #!/bin/sh
-python3 -m venv $PYTHONUSERBASE
-source $PYTHONUSERBASE/bin/activate
-echo "pip3 uninstall $@"
-pip3 uninstall "$@"
+echo "uv pip uninstall $@"
+uv pip uninstall --python "$PYTHONUSERBASE" "$@"
 if [ $? -eq 0 ]; then
     echo "Command succeeded"
 else
-    echo "ERROR: pip3 uninstall failed"
+    echo "ERROR: uv pip uninstall failed"
 fi

data/data/config/interface_modifiers.yaml

@@ -53,7 +53,7 @@ RECONNECT_DELAY:
   parameters:
     - name: Delay
       required: true
-      description: Delay in seconds between reconnect attempts. The default is 15 seconds.
+      description: Delay in seconds between reconnect attempts. The default is 5 seconds.
       values: ([0-9]*[.])?[0-9]+
 DISABLE_DISCONNECT:
   summary: Disable the Disconnect button on the Interfaces tab in the Server

data/data/config/item_modifiers.yaml

@@ -72,7 +72,8 @@ GENERIC_READ_CONVERSION_START:
     class (Note, referencing the packet as 'myself' is still supported for backwards
     compatibility). The last line of code should return the converted
     value. The GENERIC_READ_CONVERSION_END keyword specifies that all lines of
-    code for the conversion have been given.
+    code for the conversion have been given. To specify the bit size, type, and array size of the converted data,
+    use the CONVERTED_DATA keyword.
   warning: Generic conversions are not a good long term solution. Consider creating
     a conversion class and using READ_CONVERSION instead. READ_CONVERSION is easier
     to debug and has higher performance.
@@ -81,22 +82,33 @@ GENERIC_READ_CONVERSION_START:
       GENERIC_READ_CONVERSION_START
         (value * 1.5).to_i # Convert the value by a scale factor
       GENERIC_READ_CONVERSION_END
+      CONVERTED_DATA 32 UINT
   python_example: |
     APPEND_ITEM ITEM1 32 UINT
       GENERIC_READ_CONVERSION_START
         int(value * 1.5) # Convert the value by a scale factor
       GENERIC_READ_CONVERSION_END
+      CONVERTED_DATA 32 UINT
+GENERIC_READ_CONVERSION_END:
+  summary: Complete a generic read conversion
+CONVERTED_DATA:
+  summary: Defines the bit size, type, and array size of the converted data for a read conversion
+  description: This keyword is used in conjunction with DERIVED items to specify the bit size, type, and array size of the converted data.
+    If this keyword is not used, DERIVED items are stored as strings in the decommutated data.
+  since: 7.0.0
   parameters:
+    - name: Converted Bit Size
+      required: true
+      description: Bit size of converted value
+      values: \d+
     - name: Converted Type
-      required: false
+      required: true
       description: Type of the converted value
       values: <%= %w(INT UINT FLOAT STRING BLOCK) %>
-    - name: Converted Bit Size
+    - name: Converted Array Size
       required: false
-      description: Bit size of converted value
+      description: Bit size of the total array if the converted value is an array. Only specified if the converted type is an array.
       values: \d+
-GENERIC_READ_CONVERSION_END:
-  summary: Complete a generic read conversion
 LIMITS:
   summary: Defines a set of limits for a telemetry item
   description: If limits are violated a message is printed in the Command and Telemetry Server

data/data/config/telemetry.yaml

@@ -10,7 +10,7 @@ TELEMETRY:
       required: true
       description: Name of the target this telemetry packet is associated with
       values: .+
-    - name: Command
+    - name: Packet
       required: true
       description:
         Name of this telemetry packet. Also referred to as its mnemonic.

data/data/config/widgets.yaml

@@ -894,6 +894,16 @@ Telemetry Widgets:
         END
         LIMITSCOLOR INST HEALTH_STATUS TEMP2 # Default is label with just item name
         LIMITSCOLOR INST HEALTH_STATUS TEMP3 CONVERTED 20 TRUE # Full TGT/PKT/ITEM label
+        LIMITSCOLOR INST HEALTH_STATUS TEMP4
+          SETTING ASTRO TRUE
+      settings:
+        ASTRO:
+          summary: Display Astro status icons instead of a colored circle
+          description:
+            When set, the LIMITSCOLOR renders an Astro (rux-status) icon whose shape reflects
+            the severity level, improving accessibility for colorblind users.
+            Limits colors are automatically mapped to Astro statuses
+            (GREEN to normal, RED to critical, YELLOW to caution, BLUE to standby).
     VALUELIMITSBAR:
       summary: Displays an item VALUE followed by LIMITSBAR
       parameters:

data/lib/openc3/accessors/json_accessor.rb

@@ -20,7 +20,7 @@ require 'openc3/accessors/accessor'
 OpenC3.disable_warnings do
   class JsonPath
     def self.process_object(obj_or_str, opts = {})
-      obj_or_str.is_a?(String) ? MultiJson.load(obj_or_str, max_nesting: opts[:max_nesting], create_additions: true, allow_nan: true) : obj_or_str
+      obj_or_str.is_a?(String) ? JSON.parse(obj_or_str, max_nesting: opts[:max_nesting], create_additions: true, allow_nan: true) : obj_or_str
     end
   end
 end

data/lib/openc3/api/cmd_api.rb

@@ -545,6 +545,8 @@ module OpenC3
           target_name: target_name,
           cmd_name: cmd_name,
           cmd_params: cmd_params,
+          validate: validate,
+          timeout: timeout,
           username: username,
           scope: scope)
       else

data/lib/openc3/api/settings_api.rb

@@ -73,6 +73,8 @@ module OpenC3
       authorize(permission: 'admin', manual: manual, scope: scope, token: token)
       SettingModel.set({ name: name, data: data }, scope: scope)
       LocalMode.save_setting(scope, name, data)
+      username = user_info(token)['username'] || 'Anonymous'
+      Logger.info("User #{username} saved setting '#{name}': #{data}", scope: scope, user: username)
     end
     # save_setting is DEPRECATED
     alias save_setting set_setting

data/lib/openc3/api/tlm_api.rb

@@ -23,7 +23,7 @@
 
 require 'openc3/models/target_model'
 require 'openc3/models/cvt_model'
-require 'openc3/packets/packet'
+# require 'openc3/packets/packet' # Circular require
 require 'openc3/topics/telemetry_topic'
 require 'openc3/topics/interface_topic'
 require 'openc3/topics/decom_interface_topic'
@@ -281,7 +281,7 @@ module OpenC3
 
           case value_type
           when 'FORMATTED', 'WITH_UNITS'
-            if item['format_string']
+            if item['format_string'] or item['units']
               results << [target_name, orig_packet_name, item_name, 'FORMATTED'].join('__')
             # This logic must match the logic in Packet#decom
             elsif item['states'] or (item['read_conversion'] and item['data_type'] != 'DERIVED')
@@ -304,7 +304,7 @@ module OpenC3
           if item['limits']['DEFAULT']
             results[-1] += '__LIMITS'
           end
-        rescue RuntimeError => e
+        rescue RuntimeError
           results << nil
         end
       end

data/lib/openc3/config/config_parser.rb

@@ -15,10 +15,11 @@
 # This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
-require 'openc3/top_level'
+# require 'openc3/top_level' # Circular require
 require 'openc3/ext/config_parser' if RUBY_ENGINE == 'ruby' and !ENV['OPENC3_NO_EXT']
 require 'erb'
 require 'fileutils'
+require 'tempfile'
 
 module OpenC3
   # Reads OpenC3 style configuration data which consists of keywords followed
@@ -218,6 +219,7 @@ module OpenC3
                    &)
       ensure
         file.close unless file.closed?
+        file.unlink
       end
     end
 
@@ -417,9 +419,7 @@ module OpenC3
       elsif copy.include?(':') # Check for Windows drive letter
         copy = copy.split(':')[1]
       end
-      parsed_filename = File.join(Dir.tmpdir, 'openc3', 'tmp', copy)
-      FileUtils.mkdir_p(File.dirname(parsed_filename)) # Create the path
-      file = File.open(parsed_filename, 'w+')
+      file = Tempfile.new(copy)
       file.puts output
       file.rewind # Rewind so the file is ready to read
       file

data/lib/openc3/conversions/conversion.rb

@@ -20,11 +20,11 @@ module OpenC3
   class Conversion
     # @return [Symbol] The converted data type. Must be one of
     #   {OpenC3::StructureItem#data_type}
-    attr_reader :converted_type
+    attr_accessor :converted_type
     # @return [Integer] The size in bits of the converted value
-    attr_reader :converted_bit_size
+    attr_accessor :converted_bit_size
     # @return [Integer] The size in bits of the converted array value
-    attr_reader :converted_array_size
+    attr_accessor :converted_array_size
     # @return [Array] The arguments passed to the conversion
     attr_reader :params
 

data/lib/openc3/core_ext/faraday.rb

@@ -1,4 +1,8 @@
+# Remove warnings in CGI
+saved_verbose = $VERBOSE
+$VERBOSE = false
 require 'faraday'
+$VERBOSE = saved_verbose
 
 module Faraday
   class Response

data/lib/openc3/logs/log_writer.rb

@@ -126,6 +126,7 @@ module OpenC3
       @cleanup_times = []
       @previous_time_nsec_since_epoch = nil
       @tmp_dir = Dir.mktmpdir
+      @wait_threads = []
 
       # This is an optimization to avoid creating a new entry object
       # each time we create an entry which we do a LOT!
@@ -154,9 +155,8 @@ module OpenC3
 
     # Stops all logging and closes the current log file.
     def stop
-      threads = nil
-      @mutex.synchronize { threads = close_file(false); @logging_enabled = false; }
-      return threads
+      @mutex.synchronize { close_file(false); @logging_enabled = false; }
+      return @wait_threads
     end
 
     # Stop all logging, close the current log file, and kill the logging threads.
@@ -173,6 +173,13 @@ module OpenC3
       return threads
     end
 
+    def cleanup
+      if @tmp_dir
+        FileUtils.remove_entry_secure(@tmp_dir, true)
+        @tmp_dir = nil
+      end
+    end
+
     def graceful_kill
       @cancel_threads = true
     end
@@ -307,8 +314,19 @@ module OpenC3
     # to keep a full file's worth of data in the stream. This is what prevents continuous stream growth.
     # Returns thread that moves log to bucket
     def close_file(take_mutex = true)
-      threads = []
       @mutex.lock if take_mutex
+
+      # Remove old wait_threads
+      to_remove = []
+      @wait_threads.each do |thread|
+        unless thread.alive?
+          to_remove << thread
+        end
+      end
+      to_remove.each do |thread|
+        @wait_threads.delete(thread)
+      end
+
       begin
         if @file
           begin
@@ -322,7 +340,7 @@ module OpenC3
               # Cleanup timestamps here so they are unset for the next file
               @first_time = nil
               @last_time = nil
-              threads << BucketUtilities.move_log_file_to_bucket(@filename, bucket_key)
+              @wait_threads << BucketUtilities.move_log_file_to_bucket(@filename, bucket_key)
               # Now that the file is in storage, trim the Redis stream after a delay
               @cleanup_offsets << {}
               @last_offsets.each do |redis_topic, last_offset|
@@ -342,7 +360,7 @@ module OpenC3
       ensure
         @mutex.unlock if take_mutex
       end
-      return threads
+      return @wait_threads
     end
 
     def bucket_filename

data/lib/openc3/logs/packet_log_writer.rb

@@ -137,7 +137,6 @@ module OpenC3
     # Closing a log file isn't critical so we just log an error
     # Returns threads that moves log to bucket
     def close_file(take_mutex = true)
-      threads = []
       @mutex.lock if take_mutex
       begin
         # Need to write the OFFSET_MARKER for each packet
@@ -145,12 +144,10 @@ module OpenC3
           write_entry(:OFFSET_MARKER, nil, nil, nil, nil, nil, last_offset + ',' + redis_topic, nil) if @file
         end
 
-        threads.concat(super(false))
-
+        return super(false)
       ensure
         @mutex.unlock if take_mutex
       end
-      return threads
     end
 
     def get_packet_index(cmd_or_tlm, target_name, packet_name, entry_type, data)

data/lib/openc3/logs/stream_log_pair.rb

@@ -43,13 +43,20 @@ module OpenC3
 
     # Close any open stream log files
     def stop
-      @read_log.stop
-      @write_log.stop
+      threads = @read_log.stop
+      threads.concat(@write_log.stop)
+      return threads
     end
 
     def shutdown
-      @read_log.shutdown
-      @write_log.shutdown
+      threads = @read_log.shutdown
+      threads.concat(@write_log.shutdown)
+      return threads
+    end
+
+    def cleanup
+      @read_log.cleanup
+      @write_log.cleanup
     end
 
     # Clone the stream log pair

data/lib/openc3/logs/text_log_writer.rb

@@ -68,7 +68,6 @@ module OpenC3
     # Closing a log file isn't critical so we just log an error
     # Returns threads that moves log to bucket
     def close_file(take_mutex = true)
-      threads = []
       @mutex.lock if take_mutex
       begin
         # Need to write the OFFSET_MARKER for each packet
@@ -79,12 +78,10 @@ module OpenC3
           write_entry(time.to_nsec_from_epoch, data.as_json(allow_nan: true).to_json(allow_nan: true)) if @file
         end
 
-        threads.concat(super(false))
-
+        return super(false)
       ensure
         @mutex.unlock if take_mutex
       end
-      return threads
     end
 
     def extension

data/lib/openc3/microservices/interface_microservice.rb

@@ -779,7 +779,6 @@ module OpenC3
         else
           @logger.error "#{@interface.name}: #{connect_error.formatted}"
           unless @connection_failed_messages.include?(connect_error.message)
-            OpenC3.write_exception_file(connect_error)
             @connection_failed_messages << connect_error.message
           end
         end
@@ -800,7 +799,6 @@ module OpenC3
         else
           @logger.error "#{@interface.name}: #{err.formatted}"
           unless @connection_lost_messages.include?(err.message)
-            OpenC3.write_exception_file(err)
             @connection_lost_messages << err.message
           end
         end
@@ -888,6 +886,14 @@ module OpenC3
     def shutdown(_sig = nil)
       @logger.info "#{@interface ? @interface.name : @name}: shutdown requested"
       stop()
+      if @interface and @interface.stream_log_pair
+        threads = @interface.stream_log_pair.shutdown
+        # Wait for all the logging threads to move files to buckets
+        threads.flatten.compact.each do |thread|
+          thread.join
+        end
+        @interface.stream_log_pair.cleanup
+      end
       super()
     end
 

data/lib/openc3/microservices/log_microservice.rb

@@ -125,8 +125,8 @@ module OpenC3
     def shutdown
       # Make sure all the existing logs are properly closed down
       threads = []
-      @plws.each do |target_name, plw_hash|
-        plw_hash.each do |type, plw|
+      @plws.each do |_target_name, plw_hash|
+        plw_hash.each do |_type, plw|
           threads.concat(plw.shutdown)
         end
       end
@@ -134,6 +134,11 @@ module OpenC3
       threads.flatten.compact.each do |thread|
         thread.join
       end
+      @plws.each do |_target_name, plw_hash|
+        plw_hash.each do |_type, plw|
+          plw.cleanup
+        end
+      end
       super()
     end
   end

data/lib/openc3/microservices/microservice.rb

@@ -88,6 +88,7 @@ module OpenC3
       @name = name
       split_name = name.split("__")
       raise "Name #{name} doesn't match convention of SCOPE__TYPE__NAME" if split_name.length != 3
+      microservice_type = split_name[1].to_s.upcase
 
       @scope = split_name[0]
       $openc3_scope = @scope
@@ -102,8 +103,14 @@ module OpenC3
 
       OpenC3.setup_open_telemetry(@name, false)
 
+      @temp_dir = OpenC3.sanitize_path(File.join(Dir.tmpdir, @name))
+
       # Create temp folder for this microservice
-      @temp_dir = Dir.mktmpdir
+      # This will already have been setup by plugin_microservice.rb if USER
+      if is_plugin or microservice_type != 'USER'
+        FileUtils.remove_entry_secure(@temp_dir, true)
+        Dir.mkdir(@temp_dir)
+      end
 
       # Get microservice configuration from Redis
       @config = MicroserviceModel.get(name: @name, scope: @scope)
@@ -142,14 +149,13 @@ module OpenC3
         cmd_array = @config["cmd"]
 
         # Get Microservice files from bucket storage
-        temp_dir = Dir.mktmpdir
         bucket = ENV['OPENC3_CONFIG_BUCKET']
         client = Bucket.getClient()
 
         prefix = "#{@scope}/microservices/#{@name}/"
         file_count = 0
         client.list_objects(bucket: bucket, prefix: prefix).each do |object|
-          response_target = File.join(temp_dir, object.key.split(prefix)[-1])
+          response_target = OpenC3.sanitize_path(File.join(@temp_dir, object.key.split(prefix)[-1]))
           FileUtils.mkdir_p(File.dirname(response_target))
           client.get_object(bucket: bucket, key: object.key, path: response_target)
           file_count += 1
@@ -157,7 +163,7 @@ module OpenC3
 
         # Adjust @work_dir to microservice files downloaded if files and a relative path
         if file_count > 0 and @work_dir[0] != '/'
-          @work_dir = File.join(temp_dir, @work_dir)
+          @work_dir = OpenC3.sanitize_path(File.join(@temp_dir, @work_dir))
         end
 
         # Check Syntax on any ruby files

data/lib/openc3/microservices/queue_microservice.rb

@@ -18,6 +18,8 @@ require 'openc3/utilities/authentication'
 require 'openc3/api/api'
 
 module OpenC3
+  saved_verbose = $VERBOSE
+  $VERBOSE = false
   module Script
     private
     # Override the prompt_for_hazardous method to always return true since there is no user to prompt
@@ -25,6 +27,7 @@ module OpenC3
       return true
     end
   end
+  $VERBOSE = saved_verbose
 
   # The queue processor runs in a single thread and processes commands via cmd_api.
   class QueueProcessor
@@ -71,10 +74,14 @@ module OpenC3
               else
                 cmd_params = {}
               end
-              cmd(command['target_name'], command['cmd_name'], cmd_params, queue: false, scope: @scope)
+              validate = command.key?('validate') ? command['validate'] : true
+              timeout = command['timeout']
+              cmd(command['target_name'], command['cmd_name'], cmd_params, queue: false, validate: validate, timeout: timeout, scope: @scope)
             elsif command['value']
               # Legacy format: use single string parameter for backwards compatibility
-              cmd(command['value'], queue: false, scope: @scope)
+              validate = command.key?('validate') ? command['validate'] : true
+              timeout = command['timeout']
+              cmd(command['value'], queue: false, validate: validate, timeout: timeout, scope: @scope)
             else
               @logger.error "QueueProcessor: Invalid command format, missing required fields"
             end