openc3 6.10.4 → 7.0.0.pre.rc2

data/lib/openc3/models/target_model.rb

@@ -3,18 +3,13 @@
 # Copyright 2022 Ball Aerospace & Technologies Corp.
 # All Rights Reserved.
 #
-# This program is free software; you can modify and/or redistribute it
-# under the terms of the GNU Affero General Public License
-# as published by the Free Software Foundation; version 3 with
-# attribution addendums as found in the LICENSE.txt
-#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE.md for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2025, OpenC3, Inc.
+# All changes Copyright 2026, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -53,7 +48,10 @@ module OpenC3
     VALID_TYPES = %i(CMD TLM)
     ERB_EXTENSIONS = %w(.txt .rb .py .json .yaml .yml)
     ITEM_MAP_CACHE_TIMEOUT = 10.0
+    PACKET_CACHE_TIMEOUT = 10.0
     @@item_map_cache = {}
+    @@packet_cache = {}
+    @@packet_cache_mutex = Mutex.new
     @@sync_packet_count_data = {}
     @@sync_packet_count_time = nil
     @@sync_packet_count_delay_seconds = 1.0 # Sync packet counts every second
@@ -69,19 +67,12 @@ module OpenC3
     attr_accessor :cmd_log_cycle_time
     attr_accessor :cmd_log_cycle_size
     attr_accessor :cmd_log_retain_time
-    attr_accessor :cmd_decom_log_cycle_time
-    attr_accessor :cmd_decom_log_cycle_size
-    attr_accessor :cmd_decom_log_retain_time
     attr_accessor :tlm_buffer_depth
     attr_accessor :tlm_log_cycle_time
     attr_accessor :tlm_log_cycle_size
     attr_accessor :tlm_log_retain_time
-    attr_accessor :tlm_decom_log_cycle_time
-    attr_accessor :tlm_decom_log_cycle_size
-    attr_accessor :tlm_decom_log_retain_time
-    attr_accessor :reduced_minute_log_retain_time
-    attr_accessor :reduced_hour_log_retain_time
-    attr_accessor :reduced_day_log_retain_time
+    attr_accessor :cmd_decom_retain_time
+    attr_accessor :tlm_decom_retain_time
     attr_accessor :cleanup_poll_time
     attr_accessor :needs_dependencies
     attr_accessor :target_microservices
@@ -208,11 +199,33 @@ module OpenC3
     def self.packet(target_name, packet_name, type: :TLM, scope:)
       raise "Unknown type #{type} for #{target_name} #{packet_name}" unless VALID_TYPES.include?(type)
 
+      # Check cache first
+      cache_key = "#{scope}__#{type}__#{target_name}__#{packet_name}"
+      @@packet_cache_mutex.synchronize do
+        cached = @@packet_cache[cache_key]
+        if cached && (Time.now - cached[:time]) < PACKET_CACHE_TIMEOUT
+          return cached[:packet]
+        end
+      end
+
       # Assume it exists and just try to get it to avoid an extra call to Store.exist?
       json = Store.hget("#{scope}__openc3#{type.to_s.downcase}__#{target_name}", packet_name)
       raise "Packet '#{target_name} #{packet_name}' does not exist" if json.nil?
 
-      JSON.parse(json, allow_nan: true, create_additions: true)
+      packet = JSON.parse(json, allow_nan: true, create_additions: true)
+
+      # Store in cache
+      @@packet_cache_mutex.synchronize do
+        @@packet_cache[cache_key] = { packet: packet, time: Time.now }
+      end
+
+      packet
+    end
+
+    def self.clear_packet_cache
+      @@packet_cache_mutex.synchronize do
+        @@packet_cache.clear
+      end
     end
 
     # @return [Array<Hash>] All packet hashes under the target_name
@@ -236,6 +249,12 @@ module OpenC3
     def self.set_packet(target_name, packet_name, packet, type: :TLM, scope:)
       raise "Unknown type #{type} for #{target_name} #{packet_name}" unless VALID_TYPES.include?(type)
 
+      # Invalidate cache entry
+      cache_key = "#{scope}__#{type}__#{target_name}__#{packet_name}"
+      @@packet_cache_mutex.synchronize do
+        @@packet_cache.delete(cache_key)
+      end
+
       begin
         Store.hset("#{scope}__openc3#{type.to_s.downcase}__#{target_name}", packet_name, JSON.generate(packet.as_json, allow_nan: true))
       rescue JSON::GeneratorError => e
@@ -359,24 +378,15 @@ module OpenC3
       cmd_log_cycle_time: 600,
       cmd_log_cycle_size: 50_000_000,
       cmd_log_retain_time: nil,
-      cmd_decom_log_cycle_time: 600,
-      cmd_decom_log_cycle_size: 50_000_000,
-      cmd_decom_log_retain_time: nil,
       tlm_buffer_depth: 60,
       tlm_log_cycle_time: 600,
       tlm_log_cycle_size: 50_000_000,
       tlm_log_retain_time: nil,
-      tlm_decom_log_cycle_time: 600,
-      tlm_decom_log_cycle_size: 50_000_000,
-      tlm_decom_log_retain_time: nil,
-      reduced_minute_log_retain_time: nil,
-      reduced_hour_log_retain_time: nil,
-      reduced_day_log_retain_time: nil,
-      cleanup_poll_time: 600,
+      cmd_decom_retain_time: nil,
+      tlm_decom_retain_time: nil,
+      cleanup_poll_time: 3600,
       needs_dependencies: false,
-      target_microservices: {'REDUCER' => [[]]},
-      reducer_disable: false,
-      reducer_max_cpu_utilization: 30.0,
+      target_microservices: {},
       disable_erb: nil,
       shard: 0,
       scope:
@@ -384,16 +394,10 @@ module OpenC3
       super("#{scope}__#{PRIMARY_KEY}", name: name, plugin: plugin, updated_at: updated_at,
         cmd_buffer_depth: cmd_buffer_depth, cmd_log_cycle_time: cmd_log_cycle_time, cmd_log_cycle_size: cmd_log_cycle_size,
         cmd_log_retain_time: cmd_log_retain_time,
-        cmd_decom_log_cycle_time: cmd_decom_log_cycle_time, cmd_decom_log_cycle_size: cmd_decom_log_cycle_size,
-        cmd_decom_log_retain_time: cmd_decom_log_retain_time,
         tlm_buffer_depth: tlm_buffer_depth, tlm_log_cycle_time: tlm_log_cycle_time, tlm_log_cycle_size: tlm_log_cycle_size,
         tlm_log_retain_time: tlm_log_retain_time,
-        tlm_decom_log_cycle_time: tlm_decom_log_cycle_time, tlm_decom_log_cycle_size: tlm_decom_log_cycle_size,
-        tlm_decom_log_retain_time: tlm_decom_log_retain_time,
-        reduced_minute_log_retain_time: reduced_minute_log_retain_time,
-        reduced_hour_log_retain_time: reduced_hour_log_retain_time, reduced_day_log_retain_time: reduced_day_log_retain_time,
+        cmd_decom_retain_time: cmd_decom_retain_time, tlm_decom_retain_time: tlm_decom_retain_time,
         cleanup_poll_time: cleanup_poll_time, needs_dependencies: needs_dependencies, target_microservices: target_microservices,
-        reducer_disable: reducer_disable, reducer_max_cpu_utilization: reducer_max_cpu_utilization,
         scope: scope)
       @folder_name = folder_name
       @requires = requires
@@ -406,24 +410,15 @@ module OpenC3
       @cmd_log_cycle_time = cmd_log_cycle_time
       @cmd_log_cycle_size = cmd_log_cycle_size
       @cmd_log_retain_time = cmd_log_retain_time
-      @cmd_decom_log_cycle_time = cmd_decom_log_cycle_time
-      @cmd_decom_log_cycle_size = cmd_decom_log_cycle_size
-      @cmd_decom_log_retain_time = cmd_decom_log_retain_time
       @tlm_buffer_depth = tlm_buffer_depth
       @tlm_log_cycle_time = tlm_log_cycle_time
       @tlm_log_cycle_size = tlm_log_cycle_size
       @tlm_log_retain_time = tlm_log_retain_time
-      @tlm_decom_log_cycle_time = tlm_decom_log_cycle_time
-      @tlm_decom_log_cycle_size = tlm_decom_log_cycle_size
-      @tlm_decom_log_retain_time = tlm_decom_log_retain_time
-      @reduced_minute_log_retain_time = reduced_minute_log_retain_time
-      @reduced_hour_log_retain_time = reduced_hour_log_retain_time
-      @reduced_day_log_retain_time = reduced_day_log_retain_time
+      @cmd_decom_retain_time = cmd_decom_retain_time
+      @tlm_decom_retain_time = tlm_decom_retain_time
       @cleanup_poll_time = cleanup_poll_time
       @needs_dependencies = needs_dependencies
       @target_microservices = target_microservices
-      @reducer_disable = reducer_disable
-      @reducer_max_cpu_utilization = reducer_max_cpu_utilization
       @disable_erb = disable_erb
       @shard = shard.to_i # to_i to handle nil
       @bucket = Bucket.getClient()
@@ -446,24 +441,15 @@ module OpenC3
         'cmd_log_cycle_time' => @cmd_log_cycle_time,
         'cmd_log_cycle_size' => @cmd_log_cycle_size,
         'cmd_log_retain_time' => @cmd_log_retain_time,
-        'cmd_decom_log_cycle_time' => @cmd_decom_log_cycle_time,
-        'cmd_decom_log_cycle_size' => @cmd_decom_log_cycle_size,
-        'cmd_decom_log_retain_time' => @cmd_decom_log_retain_time,
         'tlm_buffer_depth' => @tlm_buffer_depth,
         'tlm_log_cycle_time' => @tlm_log_cycle_time,
         'tlm_log_cycle_size' => @tlm_log_cycle_size,
         'tlm_log_retain_time' => @tlm_log_retain_time,
-        'tlm_decom_log_cycle_time' => @tlm_decom_log_cycle_time,
-        'tlm_decom_log_cycle_size' => @tlm_decom_log_cycle_size,
-        'tlm_decom_log_retain_time' => @tlm_decom_log_retain_time,
-        'reduced_minute_log_retain_time' => @reduced_minute_log_retain_time,
-        'reduced_hour_log_retain_time' => @reduced_hour_log_retain_time,
-        'reduced_day_log_retain_time' => @reduced_day_log_retain_time,
+        'cmd_decom_retain_time' => @cmd_decom_retain_time,
+        'tlm_decom_retain_time' => @tlm_decom_retain_time,
         'cleanup_poll_time' => @cleanup_poll_time,
         'needs_dependencies' => @needs_dependencies,
         'target_microservices' => @target_microservices.as_json(),
-        'reducer_disable' => @reducer_disable,
-        'reducer_max_cpu_utilization' => @reducer_max_cpu_utilization,
         'disable_erb' => @disable_erb,
         'shard' => @shard,
       }
@@ -482,19 +468,11 @@ module OpenC3
         parser.verify_num_parameters(1, 1, "#{keyword} <Maximum file size in bytes>")
         @cmd_log_cycle_size = parameters[0].to_i
       when 'CMD_LOG_RETAIN_TIME'
-        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for cmd log files in seconds - nil = Forever>")
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for raw cmd log files in seconds - nil = Forever>")
         @cmd_log_retain_time = ConfigParser.handle_nil(parameters[0])
         @cmd_log_retain_time = @cmd_log_retain_time.to_i if @cmd_log_retain_time
-      when 'CMD_DECOM_LOG_CYCLE_TIME'
-        parser.verify_num_parameters(1, 1, "#{keyword} <Maximum time between files in seconds>")
-        @cmd_decom_log_cycle_time = parameters[0].to_i
-      when 'CMD_DECOM_LOG_CYCLE_SIZE'
-        parser.verify_num_parameters(1, 1, "#{keyword} <Maximum file size in bytes>")
-        @cmd_decom_log_cycle_size = parameters[0].to_i
-      when 'CMD_DECOM_LOG_RETAIN_TIME'
-        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for cmd decom log files in seconds - nil = Forever>")
-        @cmd_decom_log_retain_time = ConfigParser.handle_nil(parameters[0])
-        @cmd_decom_log_retain_time = @cmd_decom_log_retain_time.to_i if @cmd_decom_log_retain_time
+      when 'CMD_DECOM_LOG_CYCLE_TIME', 'CMD_DECOM_LOG_CYCLE_SIZE', 'CMD_DECOM_LOG_RETAIN_TIME'
+        # DEPRECATED keywords - do nothing
       when 'TLM_BUFFER_DEPTH'
         parser.verify_num_parameters(1, 1, "#{keyword} <Number of telemetry packets to buffer to ensure logged in order>")
         @tlm_buffer_depth = parameters[0].to_i
@@ -505,60 +483,41 @@ module OpenC3
         parser.verify_num_parameters(1, 1, "#{keyword} <Maximum file size in bytes>")
         @tlm_log_cycle_size = parameters[0].to_i
       when 'TLM_LOG_RETAIN_TIME'
-        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for tlm log files in seconds - nil = Forever>")
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for raw tlm log files in seconds - nil = Forever>")
         @tlm_log_retain_time = ConfigParser.handle_nil(parameters[0])
         @tlm_log_retain_time = @tlm_log_retain_time.to_i if @tlm_log_retain_time
-      when 'TLM_DECOM_LOG_CYCLE_TIME'
-        parser.verify_num_parameters(1, 1, "#{keyword} <Maximum time between files in seconds>")
-        @tlm_decom_log_cycle_time = parameters[0].to_i
-      when 'TLM_DECOM_LOG_CYCLE_SIZE'
-        parser.verify_num_parameters(1, 1, "#{keyword} <Maximum file size in bytes>")
-        @tlm_decom_log_cycle_size = parameters[0].to_i
-      when 'TLM_DECOM_LOG_RETAIN_TIME'
-        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for tlm decom log files in seconds - nil = Forever>")
-        @tlm_decom_log_retain_time = ConfigParser.handle_nil(parameters[0])
-        @tlm_decom_log_retain_time = @tlm_decom_log_retain_time.to_i if @tlm_decom_log_retain_time
-      when 'REDUCED_MINUTE_LOG_RETAIN_TIME'
-        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for reduced minute log files in seconds - nil = Forever>")
-        @reduced_minute_log_retain_time = ConfigParser.handle_nil(parameters[0])
-        @reduced_minute_log_retain_time = @reduced_minute_log_retain_time.to_i if @reduced_minute_log_retain_time
-      when 'REDUCED_HOUR_LOG_RETAIN_TIME'
-        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for reduced hour log files in seconds - nil = Forever>")
-        @reduced_hour_log_retain_time = ConfigParser.handle_nil(parameters[0])
-        @reduced_hour_log_retain_time = @reduced_hour_log_retain_time.to_i if @reduced_hour_log_retain_time
-      when 'REDUCED_DAY_LOG_RETAIN_TIME'
-        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for reduced day log files in seconds - nil = Forever>")
-        @reduced_day_log_retain_time = ConfigParser.handle_nil(parameters[0])
-        @reduced_day_log_retain_time = @reduced_day_log_retain_time.to_i if @reduced_day_log_retain_time
+      when 'TLM_DECOM_LOG_CYCLE_TIME', 'TLM_DECOM_LOG_CYCLE_SIZE', 'TLM_DECOM_LOG_RETAIN_TIME'
+        # DEPRECATED keywords - do nothing
+      when 'CMD_DECOM_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time with unit (e.g., 30d, 1y) - nil = Forever>")
+        @cmd_decom_retain_time = ConfigParser.handle_nil(parameters[0])
+        if @cmd_decom_retain_time and !@cmd_decom_retain_time.match?(/^\d+[hdwMy]$/)
+          raise ConfigParser::Error.new(parser, "CMD_DECOM_RETAIN_TIME must be a number followed by h, d, w, M, or y (e.g., 24h, 7d, 1y)")
+        end
+      when 'TLM_DECOM_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time with unit (e.g., 30d, 1y) - nil = Forever>")
+        @tlm_decom_retain_time = ConfigParser.handle_nil(parameters[0])
+        if @tlm_decom_retain_time and !@tlm_decom_retain_time.match?(/^\d+[hdwMy]$/)
+          raise ConfigParser::Error.new(parser, "TLM_DECOM_RETAIN_TIME must be a number followed by h, d, w, M, or y (e.g., 24h, 7d, 1y)")
+        end
+      when 'REDUCED_MINUTE_LOG_RETAIN_TIME', 'REDUCED_HOUR_LOG_RETAIN_TIME', 'REDUCED_DAY_LOG_RETAIN_TIME', 'REDUCED_LOG_RETAIN_TIME'
+        # DEPRECATED
+      when 'REDUCER_DISABLE', 'REDUCER_DISABLED', 'REDUCER_MAX_CPU_UTILIZATION', 'REDUCED_MAX_CPU_UTILIZATION'
+        # DEPRECATED
       when 'LOG_RETAIN_TIME'
-        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for all log files in seconds - nil = Forever>")
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for all raw log files in seconds - nil = Forever>")
         log_retain_time = ConfigParser.handle_nil(parameters[0])
         if log_retain_time
           @cmd_log_retain_time = log_retain_time.to_i
-          @cmd_decom_log_retain_time = log_retain_time.to_i
           @tlm_log_retain_time = log_retain_time.to_i
-          @tlm_decom_log_retain_time = log_retain_time.to_i
         end
-      when 'REDUCED_LOG_RETAIN_TIME'
-        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for all reduced log files in seconds - nil = Forever>")
-        reduced_log_retain_time = ConfigParser.handle_nil(parameters[0])
-        if reduced_log_retain_time
-          @reduced_minute_log_retain_time = reduced_log_retain_time.to_i
-          @reduced_hour_log_retain_time = reduced_log_retain_time.to_i
-          @reduced_day_log_retain_time = reduced_log_retain_time.to_i
-        end
-      when 'REDUCER_DISABLE', 'REDUCER_DISABLED' # Handle typos
-        @reducer_disable = true
-      when 'REDUCER_MAX_CPU_UTILIZATION', 'REDUCED_MAX_CPU_UTILIZATION' # Handle typos
-        parser.verify_num_parameters(1, 1, "#{keyword} <Max cpu utilization to allocate to the reducer microservice - 0.0 to 100.0>")
-        @reducer_max_cpu_utilization = Float(parameters[0])
       when 'CLEANUP_POLL_TIME'
         parser.verify_num_parameters(1, 1, "#{keyword} <Cleanup polling period in seconds>")
         @cleanup_poll_time = parameters[0].to_i
       when 'TARGET_MICROSERVICE'
-        parser.verify_num_parameters(1, 1, "#{keyword} <Type: DECOM COMMANDLOG DECOMCMDLOG PACKETLOG DECOMLOG REDUCER CLEANUP>")
+        parser.verify_num_parameters(1, 1, "#{keyword} <Type: DECOM COMMANDLOG PACKETLOG CLEANUP>")
         type = parameters[0].to_s.upcase
-        unless %w(DECOM COMMANDLOG DECOMCMDLOG PACKETLOG DECOMLOG REDUCER CLEANUP).include?(type)
+        unless %w(DECOM COMMANDLOG PACKETLOG CLEANUP).include?(type)
           raise "Unknown TARGET_MICROSERVICE #{type}"
         end
         @target_microservices[type] ||= []
@@ -567,7 +526,7 @@ module OpenC3
       when 'PACKET'
         if @current_target_microservice
           parser.verify_num_parameters(1, 1, "#{keyword} <Packet Name>")
-          if @current_target_microservice == 'REDUCER' or @current_target_microservice == 'CLEANUP'
+          if @current_target_microservice == 'CLEANUP'
             raise ConfigParser::Error.new(parser, "PACKET cannot be used with target microservice #{@current_target_microservice}")
           end
           @target_microservices[@current_target_microservice][-1] << parameters[0].to_s.upcase
@@ -654,8 +613,13 @@ module OpenC3
 
     def undeploy
       prefix = "#{@scope}/targets/#{@name}/"
-      @bucket.list_objects(bucket: ENV['OPENC3_CONFIG_BUCKET'], prefix: prefix).each do |object|
-        @bucket.delete_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: object.key)
+      objects = @bucket.list_objects(bucket: ENV['OPENC3_CONFIG_BUCKET'], prefix: prefix)
+      keys = objects.map(&:key)
+      if keys.length > 0
+        # Batch delete in chunks of 1000 (S3 limit)
+        keys.each_slice(1000) do |key_batch|
+          @bucket.delete_objects(bucket: ENV['OPENC3_CONFIG_BUCKET'], keys: key_batch)
+        end
       end
 
       self.class.get_model(name: @name, scope: @scope).limits_groups.each do |group|
@@ -668,9 +632,6 @@ module OpenC3
       self.class.packets(@name, scope: @scope).each do |packet|
         Topic.del("#{@scope}__TELEMETRY__{#{@name}}__#{packet['packet_name']}")
         Topic.del("#{@scope}__DECOM__{#{@name}}__#{packet['packet_name']}")
-        Topic.del("#{@scope}__REDUCED_MINUTE__{#{@name}}__#{packet['packet_name']}")
-        Topic.del("#{@scope}__REDUCED_HOUR__{#{@name}}__#{packet['packet_name']}")
-        Topic.del("#{@scope}__REDUCED_DAY__{#{@name}}__#{packet['packet_name']}")
         CvtModel.del(target_name: @name, packet_name: packet['packet_name'], scope: @scope)
       end
       LimitsEventTopic.delete(@name, scope: @scope)
@@ -680,7 +641,7 @@ module OpenC3
       Store.del("#{@scope}__COMMANDCNTS__{#{@name}}")
 
       # Note: these match the names of the services in deploy_microservices
-      %w(MULTI DECOM COMMANDLOG DECOMCMDLOG PACKETLOG DECOMLOG REDUCER CLEANUP).each do |type|
+      %w(MULTI DECOM COMMANDLOG PACKETLOG CLEANUP).each do |type|
         target_microservices = @target_microservices[type]
         if target_microservices
           max_instances = target_microservices.length + 1
@@ -929,16 +890,14 @@ module OpenC3
       if cmd_or_tlm == :TELEMETRY
         Topic.write_topic("MICROSERVICE__#{@scope}__PACKETLOG__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json})
         add_topics_to_microservice("#{@scope}__PACKETLOG__#{@name}", raw_topics)
-        Topic.write_topic("MICROSERVICE__#{@scope}__DECOMLOG__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => decom_topics.as_json.to_json})
-        add_topics_to_microservice("#{@scope}__DECOMLOG__#{@name}", decom_topics)
         Topic.write_topic("MICROSERVICE__#{@scope}__DECOM__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json})
         add_topics_to_microservice("#{@scope}__DECOM__#{@name}", raw_topics)
       else
         Topic.write_topic("MICROSERVICE__#{@scope}__COMMANDLOG__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json})
         add_topics_to_microservice("#{@scope}__COMMANDLOG__#{@name}", raw_topics)
-        Topic.write_topic("MICROSERVICE__#{@scope}__DECOMCMDLOG__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => decom_topics.as_json.to_json})
-        add_topics_to_microservice("#{@scope}__DECOMCMDLOG__#{@name}", decom_topics)
       end
+      Topic.write_topic("MICROSERVICE__#{@scope}__TSDB__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => decom_topics.as_json.to_json})
+      add_topics_to_microservice("#{@scope}__TSDB__#{@name}", decom_topics)
     end
 
     def add_topics_to_microservice(microservice_name, topics)
@@ -976,33 +935,6 @@ module OpenC3
       Logger.info "Configured microservice #{microservice_name}"
     end
 
-    def deploy_decomcmdlog_microservice(gem_path, variables, topics, instance = nil, parent = nil)
-      microservice_name = "#{@scope}__DECOMCMDLOG#{instance}__#{@name}"
-      microservice = MicroserviceModel.new(
-        name: microservice_name,
-        folder_name: @folder_name,
-        cmd: ["ruby", "log_microservice.rb", microservice_name],
-        work_dir: '/openc3/lib/openc3/microservices',
-        options: [
-          ["RAW_OR_DECOM", "DECOM"],
-          ["CMD_OR_TLM", "CMD"],
-          ["CYCLE_TIME", @cmd_decom_log_cycle_time],
-          ["CYCLE_SIZE", @cmd_decom_log_cycle_size],
-          ["BUFFER_DEPTH", @cmd_buffer_depth]
-        ],
-        topics: topics,
-        plugin: @plugin,
-        parent: parent,
-        needs_dependencies: @needs_dependencies,
-        shard: @shard,
-        scope: @scope
-      )
-      microservice.create
-      microservice.deploy(gem_path, variables)
-      @children << microservice_name if parent
-      Logger.info "Configured microservice #{microservice_name}"
-    end
-
     def deploy_packetlog_microservice(gem_path, variables, topics, instance = nil, parent = nil)
       microservice_name = "#{@scope}__PACKETLOG#{instance}__#{@name}"
       microservice = MicroserviceModel.new(
@@ -1030,47 +962,23 @@ module OpenC3
       Logger.info "Configured microservice #{microservice_name}"
     end
 
-    def deploy_decomlog_microservice(gem_path, variables, topics, instance = nil, parent = nil)
-      microservice_name = "#{@scope}__DECOMLOG#{instance}__#{@name}"
-      microservice = MicroserviceModel.new(
-        name: microservice_name,
-        folder_name: @folder_name,
-        cmd: ["ruby", "log_microservice.rb", microservice_name],
-        work_dir: '/openc3/lib/openc3/microservices',
-        options: [
-          ["RAW_OR_DECOM", "DECOM"],
-          ["CMD_OR_TLM", "TLM"],
-          ["CYCLE_TIME", @tlm_decom_log_cycle_time],
-          ["CYCLE_SIZE", @tlm_decom_log_cycle_size],
-          ["BUFFER_DEPTH", @tlm_buffer_depth]
-        ],
-        topics: topics,
-        plugin: @plugin,
-        parent: parent,
-        needs_dependencies: @needs_dependencies,
-        shard: @shard,
-        scope: @scope
-      )
-      microservice.create
-      microservice.deploy(gem_path, variables)
-      @children << microservice_name if parent
-      Logger.info "Configured microservice #{microservice_name}"
-    end
-
     def deploy_decom_microservice(target, gem_path, variables, topics, instance = nil, parent = nil)
       microservice_name = "#{@scope}__DECOM#{instance}__#{@name}"
       # Assume Ruby initially
       filename = 'decom_microservice.rb'
       work_dir = '/openc3/lib/openc3/microservices'
+      language_cmd = target.language
       if target.language == 'python'
         filename = 'decom_microservice.py'
         work_dir.sub!('openc3/lib', 'openc3/python')
         parent = nil
+        # Use venv Python to ensure editable packages are found
+        language_cmd = ENV['OPENC3_PYTHON_BIN'] || '/openc3/python/.venv/bin/python'
       end
       microservice = MicroserviceModel.new(
         name: microservice_name,
         folder_name: @folder_name,
-        cmd: [target.language, filename, microservice_name],
+        cmd: [language_cmd, filename, microservice_name],
         work_dir: work_dir,
         topics: topics,
         target_names: [@name],
@@ -1088,11 +996,15 @@ module OpenC3
 
     def deploy_tsdb_microservice(gem_path, variables, topics, instance = nil, parent = nil)
       microservice_name = "#{@scope}__TSDB#{instance}__#{@name}"
+      options = []
+      options << ["CMD_DECOM_RETAIN_TIME", @cmd_decom_retain_time] if @cmd_decom_retain_time
+      options << ["TLM_DECOM_RETAIN_TIME", @tlm_decom_retain_time] if @tlm_decom_retain_time
       microservice = MicroserviceModel.new(
         name: microservice_name,
         folder_name: @folder_name,
-        cmd: ["python", "quest_microservice.py", microservice_name],
+        cmd: [ENV['OPENC3_PYTHON_BIN'] || '/openc3/python/.venv/bin/python', "tsdb_microservice.py", microservice_name],
         work_dir: "/openc3/python/openc3/microservices",
+        options: options,
         topics: topics,
         plugin: @plugin,
         parent: nil,
@@ -1105,30 +1017,6 @@ module OpenC3
       Logger.info "Configured microservice #{microservice_name}"
     end
 
-    def deploy_reducer_microservice(gem_path, variables, topics, instance = nil, parent = nil)
-      microservice_name = "#{@scope}__REDUCER#{instance}__#{@name}"
-      microservice = MicroserviceModel.new(
-        name: microservice_name,
-        folder_name: @folder_name,
-        cmd: ["ruby", "reducer_microservice.rb", microservice_name],
-        work_dir: '/openc3/lib/openc3/microservices',
-        options: [
-          ["MAX_CPU_UTILIZATION", @reducer_max_cpu_utilization],
-          ["BUFFER_DEPTH", @tlm_buffer_depth]
-        ],
-        topics: topics,
-        plugin: @plugin,
-        parent: parent,
-        needs_dependencies: @needs_dependencies,
-        shard: @shard,
-        scope: @scope
-      )
-      microservice.create
-      microservice.deploy(gem_path, variables)
-      @children << microservice_name if parent
-      Logger.info "Configured microservice #{microservice_name}"
-    end
-
     def deploy_cleanup_microservice(gem_path, variables, instance = nil, parent = nil)
       microservice_name = "#{@scope}__CLEANUP#{instance}__#{@name}"
       microservice = MicroserviceModel.new(
@@ -1232,7 +1120,7 @@ module OpenC3
       end
 
       @parent = nil
-      %w(DECOM COMMANDLOG DECOMCMDLOG PACKETLOG DECOMLOG REDUCER CLEANUP).each do |type|
+      %w(DECOM COMMANDLOG PACKETLOG CLEANUP).each do |type|
         unless @target_microservices[type]
           @parent = "#{@scope}__MULTI__#{@name}"
           break
@@ -1244,11 +1132,6 @@ module OpenC3
         deploy_target_microservices('COMMANDLOG', command_topic_list, "#{@scope}__COMMAND__{#{@name}}") do |topics, instance, parent|
           deploy_commmandlog_microservice(gem_path, variables, topics, instance, parent)
         end
-
-        # DecomCmdLog Microservice
-        deploy_target_microservices('DECOMCMDLOG', decom_command_topic_list, "#{@scope}__DECOMCMD__{#{@name}}") do |topics, instance, parent|
-          deploy_decomcmdlog_microservice(gem_path, variables, topics, instance, parent)
-        end
       end
 
       unless packet_topic_list.empty?
@@ -1257,34 +1140,21 @@ module OpenC3
           deploy_packetlog_microservice(gem_path, variables, topics, instance, parent)
         end
 
-        # DecomLog Microservice
-        deploy_target_microservices('DECOMLOG', decom_topic_list, "#{@scope}__DECOM__{#{@name}}") do |topics, instance, parent|
-          deploy_decomlog_microservice(gem_path, variables, topics, instance, parent)
-        end
-
         # Decommutation Microservice
         deploy_target_microservices('DECOM', packet_topic_list, "#{@scope}__TELEMETRY__{#{@name}}") do |topics, instance, parent|
           deploy_decom_microservice(system.targets[@name], gem_path, variables, topics, instance, parent)
         end
+      end
 
-        # TSDB Microservice
-        if ENV['OPENC3_TSDB_HOSTNAME'] and ENV['OPENC3_TSDB_QUERY_PORT'] and ENV['OPENC3_TSDB_INGEST_PORT'] and ENV['OPENC3_TSDB_USERNAME'] and ENV['OPENC3_TSDB_PASSWORD']
-          deploy_target_microservices('TSDB', decom_topic_list, "#{@scope}__DECOM__{#{@name}}") do |topics, instance, parent|
-            deploy_tsdb_microservice(gem_path, variables, topics, instance, parent)
-          end
-        end
-
-        # Reducer Microservice
-        unless @reducer_disable
-          # TODO: Does Reducer even need a topic list?
-          deploy_target_microservices('REDUCER', decom_topic_list, "#{@scope}__DECOM__{#{@name}}") do |topics, instance, parent|
-            deploy_reducer_microservice(gem_path, variables, topics, instance, parent)
-          end
+      # TSDB Microservice - subscribes to both decommutated telemetry and commands
+      tsdb_topic_list = decom_topic_list + decom_command_topic_list
+      if !tsdb_topic_list.empty? and ENV['OPENC3_TSDB_HOSTNAME'] and ENV['OPENC3_TSDB_QUERY_PORT'] and ENV['OPENC3_TSDB_INGEST_PORT'] and ENV['OPENC3_TSDB_USERNAME'] and ENV['OPENC3_TSDB_PASSWORD']
+        deploy_target_microservices('TSDB', tsdb_topic_list, "#{@scope}__DECOM") do |topics, instance, parent|
+          deploy_tsdb_microservice(gem_path, variables, topics, instance, parent)
         end
       end
 
-      if @cmd_log_retain_time or @cmd_decom_log_retain_time or @tlm_log_retain_time or @tlm_decom_log_retain_time or
-         @reduced_minute_log_retain_time or @reduced_hour_log_retain_time or @reduced_day_log_retain_time
+      if @cmd_log_retain_time or @tlm_log_retain_time
         # Cleanup Microservice
         deploy_target_microservices('CLEANUP', nil, nil) do |_, instance, parent|
           deploy_cleanup_microservice(gem_path, variables, instance, parent)

data/lib/openc3/models/timeline_model.rb

@@ -3,18 +3,13 @@
 # Copyright 2022 Ball Aerospace & Technologies Corp.
 # All Rights Reserved.
 #
-# This program is free software; you can modify and/or redistribute it
-# under the terms of the GNU Affero General Public License
-# as published by the Free Software Foundation; version 3 with
-# attribution addendums as found in the LICENSE.txt
-#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE.md for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2024, OpenC3, Inc.
+# All changes Copyright 2026, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license

data/lib/openc3/models/tool_config_model.rb

@@ -3,18 +3,13 @@
 # Copyright 2022 Ball Aerospace & Technologies Corp.
 # All Rights Reserved.
 #
-# This program is free software; you can modify and/or redistribute it
-# under the terms of the GNU Affero General Public License
-# as published by the Free Software Foundation; version 3 with
-# attribution addendums as found in the LICENSE.txt
-#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE.md for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
+# All changes Copyright 2026, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license

data/lib/openc3/models/tool_model.rb

@@ -3,18 +3,13 @@
 # Copyright 2022 Ball Aerospace & Technologies Corp.
 # All Rights Reserved.
 #
-# This program is free software; you can modify and/or redistribute it
-# under the terms of the GNU Affero General Public License
-# as published by the Free Software Foundation; version 3 with
-# attribution addendums as found in the LICENSE.txt
-#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE.md for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
+# All changes Copyright 2026, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -281,8 +276,13 @@ module OpenC3
       if @folder_name and @folder_name.to_s.length > 0
         bucket = Bucket.getClient
         prefix = "#{@folder_name}/"
-        bucket.list_objects(bucket: ENV['OPENC3_TOOLS_BUCKET'], prefix: prefix).each do |object|
-          bucket.delete_object(bucket: ENV['OPENC3_TOOLS_BUCKET'], key: object.key)
+        objects = bucket.list_objects(bucket: ENV['OPENC3_TOOLS_BUCKET'], prefix: prefix)
+        keys = objects.map(&:key)
+        if keys.length > 0
+          # Batch delete in chunks of 1000 (S3 limit)
+          keys.each_slice(1000) do |key_batch|
+            bucket.delete_objects(bucket: ENV['OPENC3_TOOLS_BUCKET'], keys: key_batch)
+          end
           ConfigTopic.write({ kind: 'deleted', type: 'tool', name: @folder_name, plugin: @plugin }, scope: @scope)
         end
       end