openc3 6.1.0 → 6.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be3485e674a1fe4a236e3a71b57aff4d7d7398a0638a21d8069994739a53edab
-  data.tar.gz: 911ee7331cc9100dc1dfbb7abe3e7f38baaa5b201be2974d882f0ef01701e9da
+  metadata.gz: 7afa541be45293ddc3e884af4bc3a3bdff2fb910a8b3fb1fba16f4ba5c825741
+  data.tar.gz: dd6e9ad3e292dfcaaa8c6b1db9807e398f46d162f8b382cca176589be39bbdf3
 SHA512:
-  metadata.gz: eafafaac4ae90e0bb594446fefdfc7b9e6eb96be4b08c5798d827d60a2377fad9252bca9d2e4495b8d4073283aa957d89fab0214d998bd9c4bbbb5d9174000cc
-  data.tar.gz: ea90586ba1347a74964c3cbb181810463caf507bb22c88bed8253b2fc9164b7f1648300d35163aba95f9e96f29f31ccb6f4406063dfefa7d33fd8057acddc416
+  metadata.gz: fb2b9b43fd05e466b804648ff255a17687f7620728d4bc646817ab04c36ecb87a2b58a8e5094d8e0488186c8142fc877f4c80ae2335b3dad0173b01bb74a02a7
+  data.tar.gz: 5cd84c02078c957ea4dc1df6c305f60f0f240fce674d4328114b855f70393b4c27fa6cb710aa4375449421b6fe4a0100ee925fefecf8691fbe71af37a1d85245

data/data/config/microservice.yaml

@@ -173,3 +173,8 @@ MICROSERVICE:
           values: \d+
       example: |
         SHARD 0
+    STOPPED:
+      summary: Initially creates the microservice in a stopped state (not enabled)
+      since: 6.2.0
+      example: |
+        STOPPED

data/data/config/target.yaml

@@ -154,7 +154,7 @@ TARGET:
       parameters:
         - name: Time
           required: true
-          description: Number of seconds between runs of the cleanup process (default = 900 = 15 minutes)
+          description: Number of seconds between runs of the cleanup process (default = 600 = 10 minutes)
           values: \d+
     REDUCER_DISABLE:
       summary: Disables the data reduction microservice for the target

data/data/config/widgets.yaml

@@ -850,13 +850,17 @@ Telemetry Widgets:
           required: false
           description: Radius of the circle (default is 10)
           values: .*
-        - name: Full Item Name
+        - name: Item Name Display
           required: false
-          description: Show the full item name (default is false)
+          description: Show the full item name, e.g. TGT PKT ITEM (true), no item name (nil or none) or just the item name (false). Default is false.
           values: .*
       example: |
-        LIMITSCOLOR INST HEALTH_STATUS TEMP1 CONVERTED 30 TRUE
-        LIMITSCOLOR INST HEALTH_STATUS TEMP1
+        HORIZONTAL
+          LIMITSCOLOR INST HEALTH_STATUS TEMP1 CONVERTED 10 NIL # No label
+          LABEL '1st Temp'
+        END
+        LIMITSCOLOR INST HEALTH_STATUS TEMP2 # Default is label with just item name
+        LIMITSCOLOR INST HEALTH_STATUS TEMP3 CONVERTED 20 TRUE # Full TGT/PKT/ITEM label
     VALUELIMITSBAR:
       summary: Displays an item VALUE followed by LIMITSBAR
       parameters:

data/lib/openc3/api/cmd_api.rb

@@ -108,7 +108,7 @@ module OpenC3
     # Build a command binary
     #
     # @since 5.8.0
-    def build_cmd(*args, range_check: true, raw: false, manual: false, scope: $openc3_scope, token: $openc3_token, **kwargs)
+    def build_cmd(*args, range_check: true, raw: false, manual: false, timeout: 5, scope: $openc3_scope, token: $openc3_token, **kwargs)
       extract_string_kwargs_to_args(args, kwargs)
       case args.length
       when 1
@@ -129,7 +129,7 @@ module OpenC3
       cmd_name = cmd_name.upcase
       cmd_params = cmd_params.transform_keys(&:upcase)
       authorize(permission: 'cmd_info', target_name: target_name, manual: manual, scope: scope, token: token)
-      DecomInterfaceTopic.build_cmd(target_name, cmd_name, cmd_params, range_check, raw, scope: scope)
+      DecomInterfaceTopic.build_cmd(target_name, cmd_name, cmd_params, range_check, raw, timeout: timeout, scope: scope)
     end
     # build_command is DEPRECATED
     alias build_command build_cmd

data/lib/openc3/api/settings_api.rb

@@ -91,7 +91,7 @@ module OpenC3
       if response.success?
         NewsModel.set(response.body)
       else
-        NewsModel.news_error(response)
+        NewsModel.news_error("Error contacting OpenC3 news feed (status: #{response.status})")
       end
 
       # Test code to update the news feed with a dummy message
@@ -100,6 +100,8 @@ module OpenC3
       # json.unshift( { date: Time.now.utc.iso8601, title: "News at #{Time.now}", body: "The news feed has been updated at #{Time.now}." })
       # json.pop if json.length > 5
       # NewsModel.set(json.to_json)
+    rescue Exception => e
+      NewsModel.news_error("Error contacting OpenC3 news feed. #{e.message})")
     end
   end
 end

data/lib/openc3/conversions/generic_conversion.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2024, OpenC3, Inc.
+# All changes Copyright 2025, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -55,7 +55,7 @@ module OpenC3
     # (see OpenC3::Conversion#call)
     def call(value, packet, buffer)
       myself = packet # For backwards compatibility
-      if true or myself # Remove unused variable warning for myself
+      if myself # Remove unused variable warning for myself
         return eval(@code_to_eval)
       end
     end

data/lib/openc3/interfaces/file_interface.rb

@@ -1,6 +1,6 @@
 # encoding: ascii-8bit
 
-# Copyright 2024 OpenC3, Inc.
+# Copyright 2025 OpenC3, Inc.
 # All Rights Reserved.
 #
 # This program is free software; you can modify and/or redistribute it
@@ -72,15 +72,14 @@ module OpenC3
     end
 
     def connect
-      super()
+      super() # Reset the protocols
 
       if @telemetry_read_folder
         @listener = Listen.to(@telemetry_read_folder, force_polling: @polling) do |modified, added, removed|
           @queue << added if added
         end
-        @listener.start # starts a listener thread--does not block
+        @listener.start # starts a listener thread - does not block
       end
-
       @connected = true
     end
 

data/lib/openc3/interfaces/interface.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2024, OpenC3, Inc.
+# All changes Copyright 2025, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -223,6 +223,21 @@ module OpenC3
       end
     end
 
+    # Called immediately after the interface is connected.
+    # By default this method will run any commands specified by the CONNECT_CMD option
+    def post_connect
+      connect_cmds = @options['CONNECT_CMD']
+      if connect_cmds
+        connect_cmds.each do |log_dont_log, cmd_string|
+          if log_dont_log.upcase == 'DONT_LOG'
+            cmd(cmd_string, log_message: false)
+          else
+            cmd(cmd_string)
+          end
+        end
+      end
+    end
+
     # Indicates if the interface is connected to its target(s) or not. Must be
     # implemented by a subclass.
     def connected?
@@ -499,9 +514,9 @@ module OpenC3
     def set_option(option_name, option_values)
       option_name_upcase = option_name.upcase
 
-      # PERIODIC_CMD is special because there could be more than 1 periodic command
+      # CONNECT_CMD and PERIODIC_CMD are special because there could be more than 1
       # so we store them in an array for processing during connect()
-      if option_name_upcase == 'PERIODIC_CMD'
+      if option_name_upcase == 'PERIODIC_CMD' or option_name_upcase == 'CONNECT_CMD'
         # OPTION PERIODIC_CMD LOG/DONT_LOG 1.0 "INST COLLECT with TYPE NORMAL"
         @options[option_name_upcase] ||= []
         @options[option_name_upcase] << option_values.clone

data/lib/openc3/interfaces/stream_interface.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
+# All changes Copyright 2025, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -40,7 +40,7 @@ module OpenC3
     end
 
     def connect
-      super()
+      super() # Reset the protocols
       @stream.connect if @stream
     end
 

data/lib/openc3/microservices/cleanup_microservice.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
+# All changes Copyright 2025, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -35,44 +35,33 @@ module OpenC3
       @sleeper = Sleeper.new
     end
 
-    def cleanup(areas, cleanup_poll_time)
-      bucket = Bucket.getClient()
-      while true
-        break if @cancel_thread
-
-        @state = 'GETTING_OBJECTS'
-        start_time = Time.now
-        areas.each do |prefix, retain_time|
-          next unless retain_time
-          time = start_time - retain_time
-          oldest_list = BucketUtilities.files_between_time(ENV['OPENC3_LOGS_BUCKET'], prefix, nil, time)
-          if oldest_list.length > 0
-            @state = 'DELETING_OBJECTS'
-            oldest_list.each_slice(1000) do |slice|
-              # The delete_objects function utilizes an MD5 hash when verifying the checksums, which is not
-              # FIPS compliant (https://github.com/aws/aws-sdk-ruby/issues/2645).
-              # delete_object does NOT require an MD5 hash and will work on FIPS compliant systems. It is
-              # probably less performant, but we can instead delete each item one at a time.
-              # bucket.delete_objects(bucket: ENV['OPENC3_LOGS_BUCKET'], keys: slice)
-              slice.each do |item|
-                bucket.delete_object(bucket: ENV['OPENC3_LOGS_BUCKET'], key: item)
-              end
-              @logger.debug("Cleanup deleted #{slice.length} log files")
-              @delete_count += slice.length
-              @metric.set(name: 'cleanup_delete_total', value: @delete_count, type: 'counter')
+    def cleanup(areas, bucket)
+      @state = 'GETTING_OBJECTS'
+      start_time = Time.now
+      areas.each do |prefix, retain_time|
+        next unless retain_time
+        time = start_time - retain_time
+        oldest_list = BucketUtilities.files_between_time(ENV['OPENC3_LOGS_BUCKET'], prefix, nil, time)
+        if oldest_list.length > 0
+          @state = 'DELETING_OBJECTS'
+          oldest_list.each_slice(1000) do |slice|
+            # The delete_objects function utilizes an MD5 hash when verifying the checksums, which is not
+            # FIPS compliant (https://github.com/aws/aws-sdk-ruby/issues/2645).
+            # delete_object does NOT require an MD5 hash and will work on FIPS compliant systems. It is
+            # probably less performant, but we can instead delete each item one at a time.
+            # bucket.delete_objects(bucket: ENV['OPENC3_LOGS_BUCKET'], keys: slice)
+            slice.each do |item|
+              bucket.delete_object(bucket: ENV['OPENC3_LOGS_BUCKET'], key: item)
             end
+            @logger.debug("Cleanup deleted #{slice.length} log files")
+            @delete_count += slice.length
+            @metric.set(name: 'cleanup_delete_total', value: @delete_count, type: 'counter')
           end
         end
-
-        @count += 1
-        @metric.set(name: 'cleanup_total', value: @count, type: 'counter')
-
-        @state = 'SLEEPING'
-        break if @sleeper.sleep(cleanup_poll_time)
       end
     end
 
-    def run
+    def get_areas_and_poll_time
       split_name = @name.split("__")
       target_name = split_name[-1]
       target = TargetModel.get_model(name: target_name, scope: @scope)
@@ -86,8 +75,22 @@ module OpenC3
         ["#{@scope}/reduced_hour_logs/tlm/#{target_name}", target.reduced_hour_log_retain_time],
         ["#{@scope}/reduced_day_logs/tlm/#{target_name}", target.reduced_day_log_retain_time],
       ]
+      return areas, target.cleanup_poll_time
+    end
 
-      cleanup(areas, target.cleanup_poll_time)
+    def run
+      bucket = Bucket.getClient()
+      while true
+        break if @cancel_thread
+        areas, poll_time = get_areas_and_poll_time()
+        cleanup(areas, bucket)
+
+        @count += 1
+        @metric.set(name: 'cleanup_total', value: @count, type: 'counter')
+
+        @state = 'SLEEPING'
+        break if @sleeper.sleep(poll_time)
+      end
     end
 
     def shutdown
@@ -97,4 +100,8 @@ module OpenC3
   end
 end
 
-OpenC3::CleanupMicroservice.run if __FILE__ == $0
+if __FILE__ == $0
+  OpenC3::CleanupMicroservice.run
+  ThreadManager.instance.shutdown
+  ThreadManager.instance.join
+end

data/lib/openc3/microservices/decom_microservice.rb

@@ -14,19 +14,75 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2024, OpenC3, Inc.
+# All changes Copyright 2025, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 require 'time'
+require 'thread'
 require 'openc3/microservices/microservice'
 require 'openc3/microservices/interface_decom_common'
 require 'openc3/topics/telemetry_decom_topic'
 require 'openc3/topics/limits_event_topic'
 
 module OpenC3
+  class LimitsResponseThread
+    def initialize(microservice_name:, queue:, logger:, metric:, scope:)
+      @microservice_name = microservice_name
+      @queue = queue
+      @logger = logger
+      @metric = metric
+      @scope = scope
+      @count = 0
+      @error_count = 0
+      @metric.set(name: 'limits_response_total', value: @count, type: 'counter')
+      @metric.set(name: 'limits_response_error_total', value: @error_count, type: 'counter')
+    end
+
+    def start
+      @thread = Thread.new do
+        run()
+      rescue Exception => e
+        @logger.error "#{@microservice_name}: Limits Response thread died: #{e.formatted}"
+        raise e
+      end
+      ThreadManager.instance.register(@thread, stop_object: self)
+    end
+
+    def stop
+      if @thread
+        OpenC3.kill_thread(self, @thread)
+        @thread = nil
+      end
+    end
+
+    def graceful_kill
+      @queue << [nil, nil, nil]
+    end
+
+    def run
+      while true
+        packet, item, old_limits_state = @queue.pop()
+        break if packet.nil?
+
+        begin
+          item.limits.response.call(packet, item, old_limits_state)
+        rescue Exception => e
+          @error_count += 1
+          @metric.set(name: 'limits_response_error_total', value: @error_count, type: 'counter')
+          @logger.error "#{packet.target_name} #{packet.packet_name} #{item.name} Limits Response Exception!"
+          @logger.error "Called with old_state = #{old_limits_state}, new_state = #{item.limits.state}"
+          @logger.error e.filtered
+        end
+
+        @count += 1
+        @metric.set(name: 'limits_response_total', value: @count, type: 'counter')
+      end
+    end
+  end
+
   class DecomMicroservice < Microservice
     include InterfaceDecomCommon
     LIMITS_STATE_INDEX = { RED_LOW: 0, YELLOW_LOW: 1, YELLOW_HIGH: 2, RED_HIGH: 3, GREEN_LOW: 4, GREEN_HIGH: 5 }
@@ -44,9 +100,14 @@ module OpenC3
       @error_count = 0
       @metric.set(name: 'decom_total', value: @count, type: 'counter')
       @metric.set(name: 'decom_error_total', value: @error_count, type: 'counter')
+      @limits_response_queue = Queue.new
+      @limits_response_thread = nil
     end
 
     def run
+      @limits_response_thread = LimitsResponseThread.new(microservice_name: @name, queue: @limits_response_queue, logger: @logger, metric: @metric, scope: @scope)
+      @limits_response_thread.start()
+
       setup_microservice_topic()
       while true
         break if @cancel_thread
@@ -81,6 +142,9 @@ module OpenC3
           @logger.error("Decom error: #{e.formatted}")
         end
       end
+
+      @limits_response_thread.stop()
+      @limits_response_thread = nil
     end
 
     def decom_packet(_topic, msg_id, msg_hash, _redis)
@@ -119,8 +183,9 @@ module OpenC3
         # but that is rescued separately in the limits_change_callback
         packet.check_limits(System.limits_set)
 
-        # This is what updates the CVT
+        # This is what actually decommutates the packet and updates the CVT
         TelemetryDecomTopic.write_packet(packet, scope: @scope)
+
         diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
         @metric.set(name: 'decom_duration_seconds', value: diff, type: 'gauge', unit: 'seconds')
       end
@@ -179,19 +244,16 @@ module OpenC3
       LimitsEventTopic.write(event, scope: @scope)
 
       if item.limits.response
-        begin
-          # TODO: The limits response is user code and should be run as a separate thread / process
-          # If this code blocks it will delay TelemetryDecomTopic.write_packet
-          item.limits.response.call(packet, item, old_limits_state)
-        rescue Exception => e
-          @error = e
-          @logger.error "#{packet.target_name} #{packet.packet_name} #{item.name} Limits Response Exception!"
-          @logger.error "Called with old_state = #{old_limits_state}, new_state = #{item.limits.state}"
-          @logger.error e.filtered
-        end
+        copied_packet = packet.deep_copy
+        copied_item = packet.items[item.name]
+        @limits_response_queue << [copied_packet, copied_item, old_limits_state]
       end
     end
   end
 end
 
-OpenC3::DecomMicroservice.run if __FILE__ == $0
+if __FILE__ == $0
+  OpenC3::DecomMicroservice.run
+  ThreadManager.instance.shutdown
+  ThreadManager.instance.join
+end

data/lib/openc3/microservices/interface_microservice.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2024, OpenC3, Inc.
+# All changes Copyright 2025, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -70,14 +70,16 @@ module OpenC3
         @logger.error "#{@interface.name}: Command handler thread died: #{e.formatted}"
         raise e
       end
+      ThreadManager.instance.register(@thread, stop_object: self)
     end
 
-    def stop
+    def stop()
       OpenC3.kill_thread(self, @thread)
     end
 
     def graceful_kill
       InterfaceTopic.shutdown(@interface, scope: @scope)
+      sleep(0.001) # Allow other threads to run
     end
 
     def run
@@ -104,6 +106,7 @@ module OpenC3
             @metric.set(name: 'interface_directive_total', value: @directive_count, type: 'counter') if @metric
             if msg_hash['shutdown']
               @logger.info "#{@interface.name}: Shutdown requested"
+              InterfaceTopic.clear_topics(InterfaceTopic.topics(@interface, scope: @scope))
               return
             end
             if msg_hash['connect']
@@ -344,6 +347,7 @@ module OpenC3
         @logger.error "#{@router.name}: Telemetry handler thread died: #{e.formatted}"
         raise e
       end
+      ThreadManager.instance.register(@thread, stop_object: self)
     end
 
     def stop
@@ -352,6 +356,7 @@ module OpenC3
 
     def graceful_kill
       RouterTopic.shutdown(@router, scope: @scope)
+      sleep(0.001) # Allow other threads to run
     end
 
     def run
@@ -367,6 +372,7 @@ module OpenC3
 
           if msg_hash['shutdown']
             @logger.info "#{@router.name}: Shutdown requested"
+            RouterTopic.clear_topics(RouterTopic.topics(@router, scope: @scope))
             return
           end
           if msg_hash['connect']
@@ -726,6 +732,7 @@ module OpenC3
       @logger.info "#{@interface.name}: Connect #{@interface.connection_string}"
       begin
         @interface.connect
+        @interface.post_connect
       rescue Exception => e
         begin
           @interface.disconnect # Ensure disconnect is called at least once on a partial connect
@@ -808,4 +815,8 @@ module OpenC3
   end
 end
 
-OpenC3::InterfaceMicroservice.run if __FILE__ == $0
+if __FILE__ == $0
+  OpenC3::InterfaceMicroservice.run
+  ThreadManager.instance.shutdown
+  ThreadManager.instance.join
+end

data/lib/openc3/microservices/log_microservice.rb

@@ -142,4 +142,8 @@ module OpenC3
   end
 end
 
-OpenC3::LogMicroservice.run if __FILE__ == $0
+if __FILE__ == $0
+  OpenC3::LogMicroservice.run
+  ThreadManager.instance.shutdown
+  ThreadManager.instance.join
+end

data/lib/openc3/microservices/microservice.rb

@@ -30,6 +30,7 @@ OpenC3.require_file 'openc3/utilities/bucket'
 OpenC3.require_file 'openc3/utilities/secrets'
 OpenC3.require_file 'openc3/utilities/sleeper'
 OpenC3.require_file 'openc3/utilities/open_telemetry'
+OpenC3.require_file 'openc3/utilities/thread_manager'
 OpenC3.require_file 'openc3/models/microservice_model'
 OpenC3.require_file 'openc3/models/microservice_status_model'
 OpenC3.require_file 'tmpdir'
@@ -49,22 +50,27 @@ module OpenC3
     def self.run(name = nil)
       name = ENV['OPENC3_MICROSERVICE_NAME'] unless name
       microservice = self.new(name)
-      begin
-        MicroserviceStatusModel.set(microservice.as_json(:allow_nan => true), scope: microservice.scope)
-        microservice.state = 'RUNNING'
-        microservice.run
-        microservice.state = 'FINISHED'
-      rescue Exception => e
-        if SystemExit === e or SignalException === e
-          microservice.state = 'KILLED'
-        else
-          microservice.error = e
-          microservice.state = 'DIED_ERROR'
-          Logger.fatal("Microservice #{name} dying from exception\n#{e.formatted}")
+      thread = Thread.new do
+        begin
+          MicroserviceStatusModel.set(microservice.as_json(:allow_nan => true), scope: microservice.scope)
+          microservice.state = 'RUNNING'
+          microservice.run
+          microservice.state = 'FINISHED'
+        rescue Exception => e
+          if SystemExit === e or SignalException === e
+            microservice.state = 'KILLED'
+          else
+            microservice.error = e
+            microservice.state = 'DIED_ERROR'
+            Logger.fatal("Microservice #{name} dying from exception\n#{e.formatted}")
+          end
+        ensure
+          MicroserviceStatusModel.set(microservice.as_json(:allow_nan => true), scope: microservice.scope)
         end
-      ensure
-        MicroserviceStatusModel.set(microservice.as_json(:allow_nan => true), scope: microservice.scope)
       end
+      ThreadManager.instance.register(thread, shutdown_object: microservice)
+      ThreadManager.instance.monitor
+      ThreadManager.instance.shutdown
     end
 
     def as_json(*a)
@@ -192,6 +198,7 @@ module OpenC3
           @logger.error "#{@name} status thread died: #{e.formatted}"
           raise e
         end
+        ThreadManager.instance.register(@microservice_status_thread)
       end
     end
 
@@ -206,7 +213,7 @@ module OpenC3
       @cancel_thread = true
       @microservice_status_sleeper.cancel if @microservice_status_sleeper
       MicroserviceStatusModel.set(as_json(:allow_nan => true), scope: @scope)
-      FileUtils.remove_entry(@temp_dir) if File.exist?(@temp_dir)
+      FileUtils.remove_entry_secure(@temp_dir, true)
       @metric.shutdown
       @logger.debug("Shutting down microservice complete: #{@name}")
       @shutdown_complete = true

data/lib/openc3/microservices/multi_microservice.rb

@@ -18,45 +18,41 @@
 
 require 'openc3/microservices/microservice'
 require 'openc3/topics/topic'
+require 'openc3/utilities/thread_manager'
 
 module OpenC3
   class MultiMicroservice < Microservice
     def run
-      @threads = []
       ARGV.each do |microservice_name|
         microservice_model = MicroserviceModel.get_model(name: microservice_name, scope: @scope)
-        @threads << Thread.new do
-          cmd_line = microservice_model.cmd.join(' ')
-          split_cmd_line = cmd_line.split(' ')
-          filename = nil
-          split_cmd_line.each do |item|
-            if File.extname(item) == '.rb'
-              filename = item
-              break
+        if microservice_model.enabled
+          thread = Thread.new do
+            cmd_line = microservice_model.cmd.join(' ')
+            split_cmd_line = cmd_line.split(' ')
+            filename = nil
+            split_cmd_line.each do |item|
+              if File.extname(item) == '.rb'
+                filename = item
+                break
+              end
             end
+            raise "Could not determine class filename from '#{cmd_line}'" unless filename
+            OpenC3.set_working_dir(microservice_model.work_dir) do
+              require File.join(microservice_model.work_dir, filename)
+            end
+            klass = filename.filename_to_class_name.to_class
+            klass.run(microservice_model.name)
           end
-          raise "Could not determine class filename from '#{cmd_line}'" unless filename
-          OpenC3.set_working_dir(microservice_model.work_dir) do
-            require File.join(microservice_model.work_dir, filename)
-          end
-          klass = filename.filename_to_class_name.to_class
-          klass.run(microservice_model.name)
-        end
-      end
-      @threads.each do |thread|
-        thread.join
-      end
-    end
-
-    def shutdown
-      super()
-      if @threads
-        @threads.each do |thread|
-          thread.join
+          ThreadManager.instance.register(thread)
         end
       end
+      ThreadManager.instance.monitor
+      ThreadManager.instance.shutdown
     end
   end
 end
-
-OpenC3::MultiMicroservice.run if __FILE__ == $0
+if __FILE__ == $0
+  OpenC3::MultiMicroservice.run
+  ThreadManager.instance.shutdown
+  ThreadManager.instance.join
+end