cosmos 5.0.2 → 5.0.5

data/lib/cosmos/models/sorted_model.rb

@@ -0,0 +1,167 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This program may also be used under the terms of a commercial or
+# enterprise edition license of COSMOS if purchased from the
+# copyright holder
+
+# https://www.rubydoc.info/gems/redis/Redis/Commands/SortedSets
+# https://redis.io/docs/manual/data-types/data-types-tutorial/#sorted-sets
+
+require 'cosmos/models/model'
+require 'cosmos/topics/calendar_topic'
+
+module Cosmos
+  # Put these under the Cosmos module so they are easily accessed in the controller as
+  # Cosmos::SortedError vs Cosmos::SortedModel::Error
+  class SortedError < StandardError; end
+  class SortedInputError < SortedError; end
+  class SortedOverlapError < SortedError; end
+
+  class SortedModel < Model
+    SORTED_TYPE = 'sorted'.freeze # To be overriden by base class
+    PRIMARY_KEY = '__SORTED'.freeze # To be overriden by base class
+
+    # MUST be overriden by any subclasses
+    def self.pk(scope)
+      "#{scope}#{PRIMARY_KEY}"
+    end
+
+    # @return [String|nil] String of the saved json or nil if start not found
+    def self.get(start:, scope:)
+      result = Store.zrangebyscore(self.pk(scope), start, start)
+      return JSON.parse(result[0]) unless result.empty?
+      nil
+    end
+
+    # @return [Array<Hash>] Array up to the limit of the models (as Hash objects) stored under the primary key
+    def self.all(scope:, limit: 100)
+      result = Store.zrevrangebyscore(self.pk(scope), '+inf', '-inf', limit: [0, limit])
+      result.map { |item| JSON.parse(item) }
+    end
+
+    # @return [String|nil] json or nil if metadata empty
+    def self.get_current_value(scope:)
+      start = Time.now.to_i
+      array = Store.zrevrangebyscore(self.pk(scope), start, '-inf', limit: [0, 1])
+      return nil if array.empty?
+      return array[0]
+    end
+
+    # @param start [Integer] Start time to return values (inclusive)
+    # @param stop [Integer] Stop time to return values (inclusive)
+    # @return [Array|nil] Array up to 100 of this model or empty array
+    def self.range(start:, stop:, scope:, limit: 100)
+      if start > stop
+        raise SortedInputError.new "start: #{start} must be before stop: #{stop}"
+      end
+      result = Store.zrangebyscore(self.pk(scope), start, stop, limit: [0, limit])
+      result.map { |item| JSON.parse(item) }
+    end
+
+    # @return [Integer] count of the members stored under the primary key
+    def self.count(scope:)
+      Store.zcard(self.pk(scope))
+    end
+
+    # Remove member from a sorted set
+    # @return [Integer] count of the members removed, 0 if not found
+    def self.destroy(scope:, start:)
+      Store.zremrangebyscore(self.pk(scope), start, start)
+    end
+
+    # Remove members from min to max of the sorted set.
+    # @return [Integer] count of the members removed
+    def self.range_destroy(scope:, start:, stop:)
+      Store.zremrangebyscore(self.pk(scope), start, stop)
+    end
+
+    attr_reader :start
+
+    # @param [Integer] start - start used to store data
+    # @param [String] scope - Cosmos scope to track event to
+    # @param [Anything] kwargs - Any kwargs to store in the JSON
+    def initialize(start:, scope:, type: SORTED_TYPE, **kwargs)
+      # Name becomes the start in the base class
+      super(self.class.pk(scope), name: start.to_s, scope: scope, **kwargs)
+      @type = type # For the as_json, from_json round trip
+      @start = start
+    end
+
+    # start MUST be a positive integer
+    def validate_start(update: false)
+      unless @start.is_a?(Integer)
+        raise SortedInputError.new "start must be integer: #{@start}"
+      end
+      if @start.to_i < 0
+        raise SortedInputError.new "start must be positive: #{@start}"
+      end
+      if !update and self.class.get(start: @start, scope: @scope)
+        raise SortedOverlapError.new "duplicate, existing data at #{@start}"
+      end
+      @start = @start.to_i
+    end
+
+    # Update the Redis hash at primary_key based on the initial passed start
+    # The member is set to the JSON generated via calling as_json
+    def create(update: false)
+      validate_start(update: update)
+      @updated_at = Time.now.to_nsec_from_epoch
+      SortedModel.destroy(scope: @scope, start: update) if update
+      Store.zadd(@primary_key, @start, JSON.generate(as_json()))
+      if update
+        notify(kind: 'updated')
+      else
+        notify(kind: 'created')
+      end
+    end
+
+    # Update the Redis hash at primary_key
+    def update(start:)
+      orig_start = @start
+      @start = start
+      create(update: orig_start)
+    end
+
+    # destroy the activity from the redis database
+    def destroy
+      self.class.destroy(scope: @scope, start: @start)
+      notify(kind: 'deleted')
+    end
+
+    # @return [] update the redis stream / timeline topic that something has changed
+    def notify(kind:, extra: nil)
+      notification = {
+        'data' => JSON.generate(as_json()),
+        'kind' => kind,
+        'type' => 'calendar',
+      }
+      notification['extra'] = extra unless extra.nil?
+      begin
+        CalendarTopic.write_entry(notification, scope: @scope)
+      rescue StandardError => e
+        raise SortedError.new "Failed to write to stream: #{notification}, #{e}"
+      end
+    end
+
+    # @return [Hash] JSON encoding of this model
+    def as_json
+      { **super(),
+        'start' => @start,
+        'type' => SORTED_TYPE,
+      }
+    end
+  end
+end

data/lib/cosmos/models/target_model.rb

@@ -22,6 +22,7 @@ require 'cosmos/models/model'
 require 'cosmos/models/cvt_model'
 require 'cosmos/models/microservice_model'
 require 'cosmos/topics/limits_event_topic'
+require 'cosmos/topics/config_topic'
 require 'cosmos/system'
 require 'cosmos/utilities/s3'
 require 'cosmos/utilities/zip'
@@ -51,12 +52,20 @@ module Cosmos
     attr_accessor :id
     attr_accessor :cmd_log_cycle_time
     attr_accessor :cmd_log_cycle_size
+    attr_accessor :cmd_log_retain_time
     attr_accessor :cmd_decom_log_cycle_time
     attr_accessor :cmd_decom_log_cycle_size
+    attr_accessor :cmd_decom_log_retain_time
     attr_accessor :tlm_log_cycle_time
     attr_accessor :tlm_log_cycle_size
+    attr_accessor :tlm_log_retain_time
     attr_accessor :tlm_decom_log_cycle_time
     attr_accessor :tlm_decom_log_cycle_size
+    attr_accessor :tlm_decom_log_retain_time
+    attr_accessor :reduced_minute_log_retain_time
+    attr_accessor :reduced_hour_log_retain_time
+    attr_accessor :reduced_day_log_retain_time
+    attr_accessor :cleanup_poll_time
     attr_accessor :needs_dependencies
 
     # NOTE: The following three class methods are used by the ModelController
@@ -97,6 +106,11 @@ module Cosmos
       result
     end
 
+    # @return [Array>Hash>] All packet hashes under the target_name
+    def self.all_packet_name_descriptions(target_name, type: :TLM, scope:)
+      self.packets(target_name, type: type, scope: scope).map! { |hash| hash.slice("packet_name", "description") }
+    end
+
     def self.set_packet(target_name, packet_name, packet, type: :TLM, scope:)
       raise "Unknown type #{type} for #{target_name} #{packet_name}" unless VALID_TYPES.include?(type)
 
@@ -113,7 +127,6 @@ module Cosmos
       packet = packet(target_name, packet_name, type: type, scope: scope)
       item = packet['items'].find { |item| item['name'] == item_name.to_s }
       raise "Item '#{packet['target_name']} #{packet['packet_name']} #{item_name}' does not exist" unless item
-
       item
     end
 
@@ -149,7 +162,7 @@ module Cosmos
       when 'TARGET'
         usage = "#{keyword} <TARGET FOLDER NAME> <TARGET NAME>"
         parser.verify_num_parameters(2, 2, usage)
-        parser.verify_parameters_underscores(2) # Target name is the 2nd parameter
+        parser.verify_parameter_naming(2) # Target name is the 2nd parameter
         return self.new(name: parameters[1].to_s.upcase, folder_name: parameters[0].to_s.upcase, plugin: plugin, scope: scope)
       else
         raise ConfigParser::Error.new(parser, "Unknown keyword and parameters for Target: #{keyword} #{parameters.join(" ")}")
@@ -171,20 +184,35 @@ module Cosmos
       plugin: nil,
       cmd_log_cycle_time: 600,
       cmd_log_cycle_size: 50_000_000,
+      cmd_log_retain_time: nil,
       cmd_decom_log_cycle_time: 600,
       cmd_decom_log_cycle_size: 50_000_000,
+      cmd_decom_log_retain_time: nil,
       tlm_log_cycle_time: 600,
       tlm_log_cycle_size: 50_000_000,
+      tlm_log_retain_time: nil,
       tlm_decom_log_cycle_time: 600,
       tlm_decom_log_cycle_size: 50_000_000,
+      tlm_decom_log_retain_time: nil,
+      reduced_minute_log_retain_time: nil,
+      reduced_hour_log_retain_time: nil,
+      reduced_day_log_retain_time: nil,
+      cleanup_poll_time: 900,
       needs_dependencies: false,
       scope:
     )
       super("#{scope}__#{PRIMARY_KEY}", name: name, plugin: plugin, updated_at: updated_at,
         cmd_log_cycle_time: cmd_log_cycle_time, cmd_log_cycle_size: cmd_log_cycle_size,
+        cmd_log_retain_time: cmd_log_retain_time,
         cmd_decom_log_cycle_time: cmd_decom_log_cycle_time, cmd_decom_log_cycle_size: cmd_decom_log_cycle_size,
+        cmd_decom_log_retain_time: cmd_decom_log_retain_time,
         tlm_log_cycle_time: tlm_log_cycle_time, tlm_log_cycle_size: tlm_log_cycle_size,
+        tlm_log_retain_time: tlm_log_retain_time,
         tlm_decom_log_cycle_time: tlm_decom_log_cycle_time, tlm_decom_log_cycle_size: tlm_decom_log_cycle_size,
+        tlm_decom_log_retain_time: tlm_decom_log_retain_time,
+        reduced_minute_log_retain_time: reduced_minute_log_retain_time,
+        reduced_hour_log_retain_time: reduced_hour_log_retain_time, reduced_day_log_retain_time: reduced_day_log_retain_time,
+        cleanup_poll_time: cleanup_poll_time, needs_dependencies: needs_dependencies,
         scope: scope)
       @folder_name = folder_name
       @requires = requires
@@ -197,12 +225,20 @@ module Cosmos
       @id = id
       @cmd_log_cycle_time = cmd_log_cycle_time
       @cmd_log_cycle_size = cmd_log_cycle_size
+      @cmd_log_retain_time = cmd_log_retain_time
       @cmd_decom_log_cycle_time = cmd_decom_log_cycle_time
       @cmd_decom_log_cycle_size = cmd_decom_log_cycle_size
+      @cmd_decom_log_retain_time = cmd_decom_log_retain_time
       @tlm_log_cycle_time = tlm_log_cycle_time
       @tlm_log_cycle_size = tlm_log_cycle_size
+      @tlm_log_retain_time = tlm_log_retain_time
       @tlm_decom_log_cycle_time = tlm_decom_log_cycle_time
       @tlm_decom_log_cycle_size = tlm_decom_log_cycle_size
+      @tlm_decom_log_retain_time = tlm_decom_log_retain_time
+      @reduced_minute_log_retain_time = reduced_minute_log_retain_time
+      @reduced_hour_log_retain_time = reduced_hour_log_retain_time
+      @reduced_day_log_retain_time = reduced_day_log_retain_time
+      @cleanup_poll_time = cleanup_poll_time
       @needs_dependencies = needs_dependencies
     end
 
@@ -222,12 +258,20 @@ module Cosmos
         'plugin' => @plugin,
         'cmd_log_cycle_time' => @cmd_log_cycle_time,
         'cmd_log_cycle_size' => @cmd_log_cycle_size,
+        'cmd_log_retain_time' => @cmd_log_retain_time,
         'cmd_decom_log_cycle_time' => @cmd_decom_log_cycle_time,
         'cmd_decom_log_cycle_size' => @cmd_decom_log_cycle_size,
+        'cmd_decom_log_retain_time' => @cmd_decom_log_retain_time,
         'tlm_log_cycle_time' => @tlm_log_cycle_time,
         'tlm_log_cycle_size' => @tlm_log_cycle_size,
+        'tlm_log_retain_time' => @tlm_log_retain_time,
         'tlm_decom_log_cycle_time' => @tlm_decom_log_cycle_time,
         'tlm_decom_log_cycle_size' => @tlm_decom_log_cycle_size,
+        'tlm_decom_log_retain_time' => @tlm_decom_log_retain_time,
+        'reduced_minute_log_retain_time' => @reduced_minute_log_retain_time,
+        'reduced_hour_log_retain_time' => @reduced_hour_log_retain_time,
+        'reduced_day_log_retain_time' => @reduced_day_log_retain_time,
+        'cleanup_poll_time' => @cleanup_poll_time,
         'needs_dependencies' => @needs_dependencies,
       }
     end
@@ -245,31 +289,79 @@ module Cosmos
       when 'CMD_LOG_CYCLE_SIZE'
         parser.verify_num_parameters(1, 1, "#{keyword} <Maximum file size in bytes>")
         @cmd_log_cycle_size = parameters[0].to_i
+      when 'CMD_LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for cmd log files in seconds - nil = Forever>")
+        @cmd_log_retain_time = ConfigParser.handle_nil(parameters[0])
+        @cmd_log_retain_time = @cmd_log_retain_time.to_i if @cmd_log_retain_time
       when 'CMD_DECOM_LOG_CYCLE_TIME'
         parser.verify_num_parameters(1, 1, "#{keyword} <Maximum time between files in seconds>")
         @cmd_decom_log_cycle_time = parameters[0].to_i
       when 'CMD_DECOM_LOG_CYCLE_SIZE'
         parser.verify_num_parameters(1, 1, "#{keyword} <Maximum file size in bytes>")
         @cmd_decom_log_cycle_size = parameters[0].to_i
+      when 'CMD_DECOM_LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for cmd decom log files in seconds - nil = Forever>")
+        @cmd_decom_log_retain_time = ConfigParser.handle_nil(parameters[0])
+        @cmd_decom_log_retain_time = @cmd_decom_log_retain_time.to_i if @cmd_decom_log_retain_time
       when 'TLM_LOG_CYCLE_TIME'
         parser.verify_num_parameters(1, 1, "#{keyword} <Maximum time between files in seconds>")
         @tlm_log_cycle_time = parameters[0].to_i
       when 'TLM_LOG_CYCLE_SIZE'
         parser.verify_num_parameters(1, 1, "#{keyword} <Maximum file size in bytes>")
         @tlm_log_cycle_size = parameters[0].to_i
+      when 'TLM_LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for tlm log files in seconds - nil = Forever>")
+        @tlm_log_retain_time = ConfigParser.handle_nil(parameters[0])
+        @tlm_log_retain_time = @tlm_log_retain_time.to_i if @tlm_log_retain_time
       when 'TLM_DECOM_LOG_CYCLE_TIME'
         parser.verify_num_parameters(1, 1, "#{keyword} <Maximum time between files in seconds>")
         @tlm_decom_log_cycle_time = parameters[0].to_i
       when 'TLM_DECOM_LOG_CYCLE_SIZE'
         parser.verify_num_parameters(1, 1, "#{keyword} <Maximum file size in bytes>")
         @tlm_decom_log_cycle_size = parameters[0].to_i
+      when 'TLM_DECOM_LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for tlm decom log files in seconds - nil = Forever>")
+        @tlm_decom_log_retain_time = ConfigParser.handle_nil(parameters[0])
+        @tlm_decom_log_retain_time = @tlm_decom_log_retain_time.to_i if @tlm_decom_log_retain_time
+      when 'REDUCED_MINUTE_LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for reduced minute log files in seconds - nil = Forever>")
+        @reduced_minute_log_retain_time = ConfigParser.handle_nil(parameters[0])
+        @reduced_minute_log_retain_time = @reduced_minute_log_retain_time.to_i if @reduced_minute_log_retain_time
+      when 'REDUCED_HOUR_LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for reduced hour log files in seconds - nil = Forever>")
+        @reduced_hour_log_retain_time = ConfigParser.handle_nil(parameters[0])
+        @reduced_hour_log_retain_time = @reduced_hour_log_retain_time.to_i if @reduced_hour_log_retain_time
+      when 'REDUCED_DAY_LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for reduced day log files in seconds - nil = Forever>")
+        @reduced_day_log_retain_time = ConfigParser.handle_nil(parameters[0])
+        @reduced_day_log_retain_time = @reduced_day_log_retain_time.to_i if @reduced_day_log_retain_time
+      when 'LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for all log files in seconds - nil = Forever>")
+        log_retain_time = ConfigParser.handle_nil(parameters[0])
+        if log_retain_time
+          @cmd_log_retain_time = log_retain_time.to_i
+          @cmd_decom_log_retain_time = log_retain_time.to_i
+          @tlm_log_retain_time = log_retain_time.to_i
+          @tlm_decom_log_retain_time = log_retain_time.to_i
+        end
+      when 'REDUCED_LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for all reduced log files in seconds - nil = Forever>")
+        reduced_log_retain_time = ConfigParser.handle_nil(parameters[0])
+        if reduced_log_retain_time
+          @reduced_minute_log_retain_time = reduced_log_retain_time.to_i
+          @reduced_hour_log_retain_time = reduced_log_retain_time.to_i
+          @reduced_day_log_retain_time = reduced_log_retain_time.to_i
+        end
+      when 'CLEANUP_POLL_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Cleanup polling period in seconds>")
+        @cleanup_poll_time = parameters[0].to_i
       else
         raise ConfigParser::Error.new(parser, "Unknown keyword and parameters for Target: #{keyword} #{parameters.join(" ")}")
       end
       return nil
     end
 
-    def deploy(gem_path, variables)
+    def deploy(gem_path, variables, validate_only: false)
       rubys3_client = Aws::S3::Client.new
       variables["target_name"] = @name
       start_path = "/targets/#{@folder_name}/"
@@ -289,7 +381,7 @@ module Cosmos
           data = File.read(filename, mode: "rb")
           begin
             Cosmos.set_working_dir(File.dirname(filename)) do
-              data = ERB.new(data).result(binding.set_variables(variables)) if data.is_printable? and File.basename(filename)[0] != '_'
+              data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable? and File.basename(filename)[0] != '_'
             end
           rescue => error
             raise "ERB error parsing: #{filename}: #{error.formatted}"
@@ -298,14 +390,23 @@ module Cosmos
           FileUtils.mkdir_p(File.dirname(local_path))
           File.open(local_path, 'wb') { |file| file.write(data) }
           found = true
-          rubys3_client.put_object(bucket: 'config', key: key, body: data)
+          rubys3_client.put_object(bucket: 'config', key: key, body: data) unless validate_only
         end
         raise "No target files found at #{target_path}" unless found
 
         target_folder = File.join(temp_dir, @name)
-        build_target_archive(rubys3_client, temp_dir, target_folder)
-        system = update_store(temp_dir)
-        deploy_microservices(gem_path, variables, system)
+        # Build a System for just this target
+        system = System.new([@name], temp_dir)
+        if variables["xtce_output"]
+          puts "Converting target #{@name} to .xtce files in #{variables["xtce_output"]}/#{@name}"
+          system.packet_config.to_xtce(variables["xtce_output"])
+        end
+        unless validate_only
+          build_target_archive(rubys3_client, temp_dir, target_folder)
+          system = update_store(system)
+          deploy_microservices(gem_path, variables, system)
+          ConfigTopic.write({ kind: 'created', type: 'target', name: @name, plugin: @plugin }, scope: @scope)
+        end
       ensure
         FileUtils.remove_entry(temp_dir) if temp_dir and File.exist?(temp_dir)
       end
@@ -322,12 +423,12 @@ module Cosmos
         Store.hdel("#{@scope}__limits_groups", group)
       end
       self.class.packets(@name, type: :CMD, scope: @scope).each do |packet|
-        Store.del("#{@scope}__COMMAND__{#{@name}}__#{packet['packet_name']}")
-        Store.del("#{@scope}__DECOMCMD__{#{@name}}__#{packet['packet_name']}")
+        Topic.del("#{@scope}__COMMAND__{#{@name}}__#{packet['packet_name']}")
+        Topic.del("#{@scope}__DECOMCMD__{#{@name}}__#{packet['packet_name']}")
       end
       self.class.packets(@name, scope: @scope).each do |packet|
-        Store.del("#{@scope}__TELEMETRY__{#{@name}}__#{packet['packet_name']}")
-        Store.del("#{@scope}__DECOM__{#{@name}}__#{packet['packet_name']}")
+        Topic.del("#{@scope}__TELEMETRY__{#{@name}}__#{packet['packet_name']}")
+        Topic.del("#{@scope}__DECOM__{#{@name}}__#{packet['packet_name']}")
         CvtModel.del(target_name: @name, packet_name: packet['packet_name'], scope: @scope)
         LimitsEventTopic.delete(@name, packet['packet_name'], scope: @scope)
       end
@@ -335,10 +436,11 @@ module Cosmos
       Store.del("#{@scope}__cosmoscmd__#{@name}")
 
       # Note: these match the names of the services in deploy_microservices
-      %w(DECOM COMMANDLOG DECOMCMDLOG PACKETLOG DECOMLOG REDUCER).each do |type|
+      %w(DECOM COMMANDLOG DECOMCMDLOG PACKETLOG DECOMLOG REDUCER CLEANUP).each do |type|
         model = MicroserviceModel.get_model(name: "#{@scope}__#{type}__#{@name}", scope: @scope)
         model.destroy if model
       end
+      ConfigTopic.write({ kind: 'deleted', type: 'target', name: @name, plugin: @plugin }, scope: @scope)
     end
 
     ##################################################
@@ -364,7 +466,7 @@ module Cosmos
 
       begin
         Cosmos.set_working_dir(File.dirname(path)) do
-          return ERB.new(File.read(path)).result(b)
+          return ERB.new(File.read(path), trim_mode: "-").result(b)
         end
       rescue => error
         raise "ERB error parsing: #{path}: #{error.formatted}"
@@ -406,9 +508,7 @@ module Cosmos
       end
     end
 
-    def update_store(temp_dir)
-      # Build a System for just this target
-      system = System.new([@name], temp_dir)
+    def update_store(system)
       target = system.targets[@name]
 
       # Add in the information from the target and update
@@ -494,10 +594,10 @@ module Cosmos
         # No telemetry packets for this target
       end
       # It's ok to call initialize_streams with an empty array
-      Store.initialize_streams(command_topic_list)
-      Store.initialize_streams(decom_command_topic_list)
-      Store.initialize_streams(packet_topic_list)
-      Store.initialize_streams(decom_topic_list)
+      Topic.initialize_streams(command_topic_list)
+      Topic.initialize_streams(decom_command_topic_list)
+      Topic.initialize_streams(packet_topic_list)
+      Topic.initialize_streams(decom_topic_list)
 
       unless command_topic_list.empty?
         # CommandLog Microservice
@@ -515,7 +615,7 @@ module Cosmos
           ],
           topics: command_topic_list,
           target_names: [@name],
-          plugin: plugin,
+          plugin: @plugin,
           needs_dependencies: @needs_dependencies,
           scope: @scope
         )
@@ -538,7 +638,7 @@ module Cosmos
           ],
           topics: decom_command_topic_list,
           target_names: [@name],
-          plugin: plugin,
+          plugin: @plugin,
           needs_dependencies: @needs_dependencies,
           scope: @scope
         )
@@ -563,7 +663,7 @@ module Cosmos
           ],
           topics: packet_topic_list,
           target_names: [@name],
-          plugin: plugin,
+          plugin: @plugin,
           needs_dependencies: @needs_dependencies,
           scope: @scope
         )
@@ -586,7 +686,7 @@ module Cosmos
           ],
           topics: decom_topic_list,
           target_names: [@name],
-          plugin: plugin,
+          plugin: @plugin,
           needs_dependencies: @needs_dependencies,
           scope: @scope
         )
@@ -603,7 +703,7 @@ module Cosmos
           work_dir: '/cosmos/lib/cosmos/microservices',
           topics: packet_topic_list,
           target_names: [@name],
-          plugin: plugin,
+          plugin: @plugin,
           needs_dependencies: @needs_dependencies,
           scope: @scope
         )
@@ -619,7 +719,7 @@ module Cosmos
           cmd: ["ruby", "reducer_microservice.rb", microservice_name],
           work_dir: '/cosmos/lib/cosmos/microservices',
           topics: decom_topic_list,
-          plugin: plugin,
+          plugin: @plugin,
           needs_dependencies: @needs_dependencies,
           scope: @scope
         )
@@ -627,6 +727,22 @@ module Cosmos
         microservice.deploy(gem_path, variables)
         Logger.info "Configured microservice #{microservice_name}"
       end
+
+      if @cmd_log_retain_time or @cmd_decom_log_retain_time or @tlm_log_retain_time or @tlm_decom_log_retain_time or
+         @reduced_minute_log_retain_time or @reduced_hour_log_retain_time or @reduced_day_log_retain_time
+        # Cleanup Microservice
+        microservice_name = "#{@scope}__CLEANUP__#{@name}"
+        microservice = MicroserviceModel.new(
+          name: microservice_name,
+          cmd: ["ruby", "cleanup_microservice.rb", microservice_name],
+          work_dir: '/cosmos/lib/cosmos/microservices',
+          plugin: @plugin,
+          scope: @scope
+        )
+        microservice.create
+        microservice.deploy(gem_path, variables)
+        Logger.info "Configured microservice #{microservice_name}"
+      end
     end
   end
 end

data/lib/cosmos/models/tool_config_model.rb

@@ -0,0 +1,38 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 Ball Aerospace & Technologies Corp.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This program may also be used under the terms of a commercial or
+# enterprise edition license of COSMOS if purchased from the
+# copyright holder
+
+module Cosmos
+  class ToolConfigModel
+    def self.list_configs(tool, scope: $cosmos_scope)
+      Store.hkeys("#{scope}__config__#{tool}")
+    end
+
+    def self.load_config(tool, name, scope: $cosmos_scope)
+      Store.hget("#{scope}__config__#{tool}", name)
+    end
+
+    def self.save_config(tool, name, data, scope: $cosmos_scope)
+      Store.hset("#{scope}__config__#{tool}", name, data)
+    end
+
+    def self.delete_config(tool, name, scope: $cosmos_scope)
+      Store.hdel("#{scope}__config__#{tool}", name)
+    end
+  end
+end

data/lib/cosmos/models/tool_model.rb

@@ -209,13 +209,11 @@ module Cosmos
       return nil
     end
 
-    def deploy(gem_path, variables)
+    def deploy(gem_path, variables, validate_only: false)
       return unless @folder_name
 
-      rubys3_client = Aws::S3::Client.new
-
       # Ensure tools bucket exists
-      Cosmos::S3Utilities.ensure_public_bucket('tools')
+      Cosmos::S3Utilities.ensure_public_bucket('tools') unless validate_only
 
       variables["tool_name"] = @name
       start_path = "/tools/#{@folder_name}/"
@@ -223,16 +221,17 @@ module Cosmos
         next if filename == '.' or filename == '..' or File.directory?(filename)
 
         key = filename.split(gem_path + '/tools/')[-1]
-
         extension = filename.split('.')[-1]
         content_type = Rack::Mime.mime_type(".#{extension}")
 
-        cache_control = Cosmos::S3Utilities.get_cache_control(filename)
-
         # Load tool files
         data = File.read(filename, mode: "rb")
-        data = ERB.new(data).result(binding.set_variables(variables)) if data.is_printable?
-        rubys3_client.put_object(bucket: 'tools', content_type: content_type, cache_control: cache_control, key: key, body: data)
+        data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable?
+        unless validate_only
+          cache_control = Cosmos::S3Utilities.get_cache_control(filename)
+          Aws::S3::Client.new.put_object(bucket: 'tools', content_type: content_type, cache_control: cache_control, key: key, body: data)
+          ConfigTopic.write({ kind: 'created', type: 'tool', name: @folder_name, plugin: @plugin }, scope: @scope)
+        end
       end
     end
 
@@ -242,6 +241,7 @@ module Cosmos
         prefix = "#{@folder_name}/"
         rubys3_client.list_objects(bucket: 'tools', prefix: prefix).contents.each do |object|
           rubys3_client.delete_object(bucket: 'tools', key: object.key)
+          ConfigTopic.write({ kind: 'deleted', type: 'tool', name: @folder_name, plugin: @plugin }, scope: @scope)
         end
       end
     end

data/lib/cosmos/models/widget_model.rb

@@ -108,25 +108,25 @@ module Cosmos
       raise ConfigParser::Error.new(parser, "Unknown keyword and parameters for Widget: #{keyword} #{parameters.join(" ")}")
     end
 
-    def deploy(gem_path, variables)
-      rubys3_client = Aws::S3::Client.new
-
+    def deploy(gem_path, variables, validate_only: false)
       # Ensure tools bucket exists
-      Cosmos::S3Utilities.ensure_public_bucket('tools')
+      Cosmos::S3Utilities.ensure_public_bucket('tools') unless validate_only
 
       filename = gem_path + "/tools/widgets/" + @full_name + '/' + @filename
 
-      cache_control = Cosmos::S3Utilities.get_cache_control(@filename)
-
       # Load widget file
       data = File.read(filename, mode: "rb")
       Cosmos.set_working_dir(File.dirname(filename)) do
-        data = ERB.new(data).result(binding.set_variables(variables)) if data.is_printable?
+        data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable?
+      end
+      unless validate_only
+        cache_control = Cosmos::S3Utilities.get_cache_control(@filename)
+        # TODO: support widgets that aren't just a single js file (and its associated map file)
+        rubys3_client = Aws::S3::Client.new
+        rubys3_client.put_object(bucket: 'tools', content_type: 'application/javascript', cache_control: cache_control, key: @s3_key, body: data)
+        data = File.read(filename + '.map', mode: "rb")
+        rubys3_client.put_object(bucket: 'tools', content_type: 'application/json', cache_control: cache_control, key: @s3_key + '.map', body: data)
       end
-      # TODO: support widgets that aren't just a single js file (and its associated map file)
-      rubys3_client.put_object(bucket: 'tools', content_type: 'application/javascript', cache_control: cache_control, key: @s3_key, body: data)
-      data = File.read(filename + '.map', mode: "rb")
-      rubys3_client.put_object(bucket: 'tools', content_type: 'application/json', cache_control: cache_control, key: @s3_key + '.map', body: data)
     end
 
     def undeploy