cosmos 5.0.2 → 5.0.3

data/lib/cosmos/config/config_parser.rb

@@ -238,27 +238,24 @@ module Cosmos
       end
     end
 
-    # Verifies the indicated parameter(s) in the config don't start or end
-    # with an underscore and don't contain a double underscore. Raises an
-    # Error if they do.
+    # Verifies the indicated parameter in the config doesn't start or end
+    # with an underscore, doesn't contain a double underscore, doesn't contain
+    # spaces and doesn't start with a close bracket.
     #
-    # @param [Integer|Range<Integer>|Array<Integer>] indeces_to_check The
-    #   indeces of the parameters that must follow underscore rules
-    def verify_parameters_underscores(indeces_to_check, usage = "")
-      indeces_to_check = (1..@parameters.length) if indeces_to_check.nil?
-      indeces_to_check = [indeces_to_check] unless indeces_to_check.respond_to? 'each'
-
-      indeces_to_check.each do |index|
-        param = @parameters[index - 1]
-        if param.end_with? '_'
-          raise Error.new(self, "Parameter #{index} (#{@parameters[index - 1]}) for #{@keyword} cannot end with an underscore ('_').", usage, @url)
-        end
-        if param.include? '__'
-          raise Error.new(self, "Parameter #{index} (#{@parameters[index - 1]}) for #{@keyword} cannot contain a double underscore ('__').", usage, @url)
-        end
-        if param.include? ' '
-          raise Error.new(self, "Parameter #{index} (#{@parameters[index - 1]}) for #{@keyword} cannot contain a space (' ').", usage, @url)
-        end
+    # @param [Integer] index The index of the parameter to check
+    def verify_parameter_naming(index, usage = "")
+      param = @parameters[index - 1]
+      if param.end_with? '_'
+        raise Error.new(self, "Parameter #{index} (#{param}) for #{@keyword} cannot end with an underscore ('_').", usage, @url)
+      end
+      if param.include? '__'
+        raise Error.new(self, "Parameter #{index} (#{param}) for #{@keyword} cannot contain a double underscore ('__').", usage, @url)
+      end
+      if param.include? ' '
+        raise Error.new(self, "Parameter #{index} (#{param}) for #{@keyword} cannot contain a space (' ').", usage, @url)
+      end
+      if param.start_with?('}')
+        raise Error.new(self, "Parameter #{index} (#{param}) for #{@keyword} cannot start with a close bracket ('}').", usage, @url)
       end
     end
 

data/lib/cosmos/conversions/generic_conversion.rb

@@ -73,5 +73,5 @@ module Cosmos
     def as_json
       { 'class' => self.class.name.to_s, 'params' => [@code_to_eval, @converted_type, @converted_bit_size] }
     end
-  end # class GenericConversion
-end # module Cosmos
+  end
+end

data/lib/cosmos/conversions/polynomial_conversion.rb

@@ -29,13 +29,10 @@ module Cosmos
     # Initializes the conversion with the given polynomial coefficients. Sets
     # the converted_type to :FLOAT and the converted_bit_size to 64.
     #
-    # @param coeff_array [Array<Float>] The polynomial coefficients
-    def initialize(coeff_array)
+    # @param coeffs [Array<Float>] The polynomial coefficients
+    def initialize(*coeffs)
       super()
-      @coeffs = []
-      coeff_array.each do |coeff|
-        @coeffs << coeff.to_f
-      end
+      @coeffs = coeffs.map { |coeff| coeff.to_f }
       @converted_type = :FLOAT
       @converted_bit_size = 64
     end
@@ -84,5 +81,5 @@ module Cosmos
     def as_json
       { 'class' => self.class.name.to_s, 'params' => @coeffs }
     end
-  end # class PolynomialConversion
-end # module Cosmos
+  end
+end

data/lib/cosmos/conversions/segmented_polynomial_conversion.rb

@@ -23,6 +23,9 @@ module Cosmos
   # Segmented polynomial conversions consist of polynomial conversions that are
   # applied for a range of values.
   class SegmentedPolynomialConversion < Conversion
+    # @return [Array<Segment>] Segments which make up this conversion
+    attr_reader :segments
+
     # A polynomial conversion segment which applies the conversion from the
     # lower bound (inclusive) until another segment's lower bound is
     # encountered.
@@ -31,6 +34,7 @@ module Cosmos
       #   should apply. All values >= to this value will be converted using the
       #   given coefficients.
       attr_reader :lower_bound
+
       # @return [Array<Integer>] The polynomial coefficients
       attr_reader :coeffs
 
@@ -57,6 +61,14 @@ module Cosmos
         return other_segment.lower_bound <=> @lower_bound
       end
 
+      # Implement equality operator primarily for ease of testing
+      #
+      # @param segment [Segment] Other segment
+      def ==(other_segment)
+        @lower_bound == other_segment.lower_bound &&
+          @coeffs == other_segment.coeffs
+      end
+
       # Perform the polynomial conversion
       #
       # @param value [Numeric] The value to convert
@@ -71,9 +83,15 @@ module Cosmos
     end
 
     # Initialize the converted_type to :FLOAT and converted_bit_size to 64.
-    def initialize
+    #
+    # @param segments [Array] Array of segments typically generated by as_json
+    #   Format similar to the following: [[15, [3, 2]], [10, [1, 2]]]
+    #   Where each entry is an array with the first value as the lower_bound
+    #   and the other entry is an array of the coefficients for that segment.
+    def initialize(segments = [])
       super()
       @segments = []
+      segments.each { |lower_bound, coeffs| add_segment(lower_bound, *coeffs) }
       @converted_type = :FLOAT
       @converted_bit_size = 64
     end
@@ -95,9 +113,7 @@ module Cosmos
     def call(value, packet, buffer)
       # Try to find correct segment
       @segments.each do |segment|
-        if value >= segment.lower_bound
-          return segment.calculate(value)
-        end
+        return segment.calculate(value) if value >= segment.lower_bound
       end
 
       # Default to using segment with smallest lower_bound
@@ -112,7 +128,7 @@ module Cosmos
     # @return [String] The name of the class followed by a description of all
     #   the polynomial segments.
     def to_s
-      result = ""
+      result = ''
       count = 0
       @segments.each do |segment|
         result << "\n" if count > 0
@@ -136,7 +152,8 @@ module Cosmos
     def to_config(read_or_write)
       config = ''
       @segments.each do |segment|
-        config << "    SEG_POLY_#{read_or_write}_CONVERSION #{segment.lower_bound} #{segment.coeffs.join(' ')}\n"
+        config <<
+          "    SEG_POLY_#{read_or_write}_CONVERSION #{segment.lower_bound} #{segment.coeffs.join(' ')}\n"
       end
       config
     end
@@ -146,7 +163,7 @@ module Cosmos
       @segments.each do |segment|
         params << [segment.lower_bound, segment.coeffs]
       end
-      { 'class' => self.class.name.to_s, 'params' => params }
+      { 'class' => self.class.name.to_s, 'params' => [params] }
     end
-  end # class SegmentedPolynomialConversion
-end # module Cosmos
+  end
+end

data/lib/cosmos/io/json_drb.rb

@@ -269,7 +269,11 @@ module Cosmos
               response = JsonRpcSuccessResponse.new(result, request.id)
             end
           rescue Exception => error
-            Logger.error error.formatted
+            # Filter out the framework stack trace (rails, rack, puma etc)
+            lines = error.formatted.split("\n")
+            i = lines.find_index { |row| row.include?('actionpack') || row.include?('activesupport') }
+            Logger.error lines[0...i].join("\n")
+
             if request.id
               if NoMethodError === error
                 error_code = JsonRpcError::ErrorCode::METHOD_NOT_FOUND

data/lib/cosmos/microservices/cleanup_microservice.rb

@@ -17,50 +17,49 @@
 # enterprise edition license of COSMOS if purchased from the
 # copyright holder
 
+require 'cosmos/models/target_model'
 require 'cosmos/microservices/microservice'
 require 'cosmos/utilities/s3'
 
 module Cosmos
   class CleanupMicroservice < Microservice
     def run
-      # Update settings from config
-      @config['options'].each do |option|
-        case option[0].upcase
-        when 'SIZE' # Max size to use in S3 in bytes
-          @size = option[1].to_i
-        when 'DELAY' # Delay between size checks
-          @delay = option[1].to_i
-        when 'BUCKET' # Which bucket to monitor
-          @bucket = option[1]
-        when 'PREFIX' # Path into bucket to monitor
-          @prefix = option[1]
-        else
-          Logger.error("Unknown option passed to microservice #{@name}: #{option}")
-        end
-      end
-
-      raise "Microservice #{@name} not fully configured" unless @size and @delay and @bucket and @prefix
+      split_name = @name.split("__")
+      target_name = split_name[-1]
+      target = TargetModel.get_model(name: target_name, scope: @scope)
 
       rubys3_client = Aws::S3::Client.new
       while true
         break if @cancel_thread
 
         @state = 'GETTING_OBJECTS'
-        total_size, oldest_list = S3Utilities.get_total_size_and_oldest_list(@bucket, @prefix)
-        delete_items = []
-        oldest_list.each do |item|
-          break if total_size <= @size
-
-          delete_items << { :key => item.key }
-          total_size -= item.size
-        end
-        if delete_items.length > 0
-          @state = 'DELETING_OBJECTS'
-          rubys3_client.delete_objects({ bucket: @bucket, delete: { objects: delete_items } })
+        start_time = Time.now
+        [
+         ["#{@scope}/raw_logs/cmd/#{target_name}/", target.cmd_log_retain_time], 
+         ["#{@scope}/decom_logs/cmd/#{target_name}/", target.cmd_decom_log_retain_time], 
+         ["#{@scope}/raw_logs/tlm/#{target_name}/", target.tlm_log_retain_time], 
+         ["#{@scope}/decom_logs/tlm/#{target_name}/", target.tlm_decom_log_retain_time],
+         ["#{@scope}/reduced_minute_logs/tlm/#{target_name}/", target.reduced_minute_log_retain_time],
+         ["#{@scope}/reduced_hour_logs/tlm/#{target_name}/", target.reduced_hour_log_retain_time],
+         ["#{@scope}/reduced_day_logs/tlm/#{target_name}/", target.reduced_day_log_retain_time],
+        ].each do |prefix, retain_time|
+          next unless retain_time
+          time = start_time - retain_time
+          total_size, oldest_list = S3Utilities.list_files_before_time('logs', prefix, time)
+          delete_items = []
+          oldest_list.each do |item|
+            delete_items << { :key => item.key }
+          end
+          if delete_items.length > 0
+            @state = 'DELETING_OBJECTS'
+            rubys3_client.delete_objects({ bucket: 'logs', delete: { objects: delete_items } })
+            Logger.info("Deleted #{delete_items.length} #{target_name} log files")
+          end
         end
+
         @count += 1
         @state = 'SLEEPING'
-        break if @microservice_sleeper.sleep(@delay)
+        break if @microservice_sleeper.sleep(target.cleanup_poll_time)
       end
     end
   end

data/lib/cosmos/microservices/microservice.rb

@@ -75,7 +75,7 @@ module Cosmos
 
       @name = name
       split_name = name.split("__")
-      raise "Microservice names should be scope, type, and then name" if split_name.length != 3
+      raise "Name #{name} doesn't match convention of SCOPE__TYPE__NAME" if split_name.length != 3
 
       @scope = split_name[0]
       $cosmos_scope = @scope

data/lib/cosmos/models/gem_model.rb

@@ -84,7 +84,7 @@ module Cosmos
         rubygems_url = get_setting('rubygems_url')
         Gem.sources = [rubygems_url] if rubygems_url
         Gem.done_installing_hooks.clear
-        Gem.install(gem_file_path, {:build_args => '--no-document'})
+        Gem.install(gem_file_path, "> 0.pre", :build_args => ['--no-document'], :prerelease => true)
       rescue => err
         message = "Gem file #{gem_file_path} error installing to /gems\n#{err.formatted}"
         Logger.error message

data/lib/cosmos/models/scope_model.rb

@@ -51,24 +51,6 @@ module Cosmos
     def deploy(gem_path, variables)
       seed_database()
 
-      # Cleanup Microservice
-      microservice_name = "#{@scope}__CLEANUP__S3"
-      microservice = MicroserviceModel.new(
-        name: microservice_name,
-        cmd: ["ruby", "cleanup_microservice.rb", microservice_name],
-        work_dir: '/cosmos/lib/cosmos/microservices',
-        options: [
-          ["SIZE", "20_000_000_000"], # Max Size to keep in S3
-          ["DELAY", "300"], # Delay between size checks
-          ["BUCKET", "logs"], # Bucket to monitor
-          ["PREFIX", @scope + "/"], # Path into bucket to monitor
-        ],
-        scope: @scope
-      )
-      microservice.create
-      microservice.deploy(gem_path, variables)
-      Logger.info "Configured microservice #{microservice_name}"
-
       # COSMOS Log Microservice
       microservice_name = "#{@scope}__COSMOS__LOG"
       microservice = MicroserviceModel.new(
@@ -146,8 +128,6 @@ module Cosmos
     end
 
     def undeploy
-      model = MicroserviceModel.get_model(name: "#{@scope}__CLEANUP__S3", scope: @scope)
-      model.destroy if model
       model = MicroserviceModel.get_model(name: "#{@scope}__COSMOS__LOG", scope: @scope)
       model.destroy if model
       model = MicroserviceModel.get_model(name: "#{@scope}__NOTIFICATION__LOG", scope: @scope)

data/lib/cosmos/models/target_model.rb

@@ -51,12 +51,20 @@ module Cosmos
     attr_accessor :id
     attr_accessor :cmd_log_cycle_time
     attr_accessor :cmd_log_cycle_size
+    attr_accessor :cmd_log_retain_time
     attr_accessor :cmd_decom_log_cycle_time
     attr_accessor :cmd_decom_log_cycle_size
+    attr_accessor :cmd_decom_log_retain_time
     attr_accessor :tlm_log_cycle_time
     attr_accessor :tlm_log_cycle_size
+    attr_accessor :tlm_log_retain_time
     attr_accessor :tlm_decom_log_cycle_time
     attr_accessor :tlm_decom_log_cycle_size
+    attr_accessor :tlm_decom_log_retain_time
+    attr_accessor :reduced_minute_log_retain_time
+    attr_accessor :reduced_hour_log_retain_time
+    attr_accessor :reduced_day_log_retain_time
+    attr_accessor :cleanup_poll_time
     attr_accessor :needs_dependencies
 
     # NOTE: The following three class methods are used by the ModelController
@@ -97,6 +105,11 @@ module Cosmos
       result
     end
 
+    # @return [Array>Hash>] All packet hashes under the target_name
+    def self.all_packet_name_descriptions(target_name, type: :TLM, scope:)
+      self.packets(target_name, type: type, scope: scope).map! { |hash| hash.slice("packet_name", "description") }
+    end
+
     def self.set_packet(target_name, packet_name, packet, type: :TLM, scope:)
       raise "Unknown type #{type} for #{target_name} #{packet_name}" unless VALID_TYPES.include?(type)
 
@@ -113,7 +126,6 @@ module Cosmos
       packet = packet(target_name, packet_name, type: type, scope: scope)
       item = packet['items'].find { |item| item['name'] == item_name.to_s }
       raise "Item '#{packet['target_name']} #{packet['packet_name']} #{item_name}' does not exist" unless item
-
       item
     end
 
@@ -149,7 +161,7 @@ module Cosmos
       when 'TARGET'
         usage = "#{keyword} <TARGET FOLDER NAME> <TARGET NAME>"
         parser.verify_num_parameters(2, 2, usage)
-        parser.verify_parameters_underscores(2) # Target name is the 2nd parameter
+        parser.verify_parameter_naming(2) # Target name is the 2nd parameter
         return self.new(name: parameters[1].to_s.upcase, folder_name: parameters[0].to_s.upcase, plugin: plugin, scope: scope)
       else
         raise ConfigParser::Error.new(parser, "Unknown keyword and parameters for Target: #{keyword} #{parameters.join(" ")}")
@@ -171,20 +183,35 @@ module Cosmos
       plugin: nil,
       cmd_log_cycle_time: 600,
       cmd_log_cycle_size: 50_000_000,
+      cmd_log_retain_time: nil,
       cmd_decom_log_cycle_time: 600,
       cmd_decom_log_cycle_size: 50_000_000,
+      cmd_decom_log_retain_time: nil,
       tlm_log_cycle_time: 600,
       tlm_log_cycle_size: 50_000_000,
+      tlm_log_retain_time: nil,
       tlm_decom_log_cycle_time: 600,
       tlm_decom_log_cycle_size: 50_000_000,
+      tlm_decom_log_retain_time: nil,
+      reduced_minute_log_retain_time: nil,
+      reduced_hour_log_retain_time: nil,
+      reduced_day_log_retain_time: nil,
+      cleanup_poll_time: 900,
       needs_dependencies: false,
       scope:
     )
       super("#{scope}__#{PRIMARY_KEY}", name: name, plugin: plugin, updated_at: updated_at,
         cmd_log_cycle_time: cmd_log_cycle_time, cmd_log_cycle_size: cmd_log_cycle_size,
+        cmd_log_retain_time: cmd_log_retain_time,
         cmd_decom_log_cycle_time: cmd_decom_log_cycle_time, cmd_decom_log_cycle_size: cmd_decom_log_cycle_size,
+        cmd_decom_log_retain_time: cmd_decom_log_retain_time,
         tlm_log_cycle_time: tlm_log_cycle_time, tlm_log_cycle_size: tlm_log_cycle_size,
+        tlm_log_retain_time: tlm_log_retain_time,
         tlm_decom_log_cycle_time: tlm_decom_log_cycle_time, tlm_decom_log_cycle_size: tlm_decom_log_cycle_size,
+        tlm_decom_log_retain_time: tlm_decom_log_retain_time, 
+        reduced_minute_log_retain_time: reduced_minute_log_retain_time,
+        reduced_hour_log_retain_time: reduced_hour_log_retain_time, reduced_day_log_retain_time: reduced_day_log_retain_time,
+        cleanup_poll_time: cleanup_poll_time, needs_dependencies: needs_dependencies,
         scope: scope)
       @folder_name = folder_name
       @requires = requires
@@ -197,12 +224,20 @@ module Cosmos
       @id = id
       @cmd_log_cycle_time = cmd_log_cycle_time
       @cmd_log_cycle_size = cmd_log_cycle_size
+      @cmd_log_retain_time = cmd_log_retain_time
       @cmd_decom_log_cycle_time = cmd_decom_log_cycle_time
       @cmd_decom_log_cycle_size = cmd_decom_log_cycle_size
+      @cmd_decom_log_retain_time = cmd_decom_log_retain_time
       @tlm_log_cycle_time = tlm_log_cycle_time
       @tlm_log_cycle_size = tlm_log_cycle_size
+      @tlm_log_retain_time = tlm_log_retain_time
       @tlm_decom_log_cycle_time = tlm_decom_log_cycle_time
       @tlm_decom_log_cycle_size = tlm_decom_log_cycle_size
+      @tlm_decom_log_retain_time = tlm_decom_log_retain_time
+      @reduced_minute_log_retain_time = reduced_minute_log_retain_time
+      @reduced_hour_log_retain_time = reduced_hour_log_retain_time
+      @reduced_day_log_retain_time = reduced_day_log_retain_time
+      @cleanup_poll_time = cleanup_poll_time
       @needs_dependencies = needs_dependencies
     end
 
@@ -222,12 +257,20 @@ module Cosmos
         'plugin' => @plugin,
         'cmd_log_cycle_time' => @cmd_log_cycle_time,
         'cmd_log_cycle_size' => @cmd_log_cycle_size,
+        'cmd_log_retain_time' => @cmd_log_retain_time,
         'cmd_decom_log_cycle_time' => @cmd_decom_log_cycle_time,
         'cmd_decom_log_cycle_size' => @cmd_decom_log_cycle_size,
+        'cmd_decom_log_retain_time' => @cmd_decom_log_retain_time,
         'tlm_log_cycle_time' => @tlm_log_cycle_time,
         'tlm_log_cycle_size' => @tlm_log_cycle_size,
+        'tlm_log_retain_time' => @tlm_log_retain_time,
         'tlm_decom_log_cycle_time' => @tlm_decom_log_cycle_time,
         'tlm_decom_log_cycle_size' => @tlm_decom_log_cycle_size,
+        'tlm_decom_log_retain_time' => @tlm_decom_log_retain_time,
+        'reduced_minute_log_retain_time' => @reduced_minute_log_retain_time,
+        'reduced_hour_log_retain_time' => @reduced_hour_log_retain_time,
+        'reduced_day_log_retain_time' => @reduced_day_log_retain_time,        
+        'cleanup_poll_time' => @cleanup_poll_time,
         'needs_dependencies' => @needs_dependencies,
       }
     end
@@ -245,24 +288,72 @@ module Cosmos
       when 'CMD_LOG_CYCLE_SIZE'
         parser.verify_num_parameters(1, 1, "#{keyword} <Maximum file size in bytes>")
         @cmd_log_cycle_size = parameters[0].to_i
+      when 'CMD_LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for cmd log files in seconds - nil = Forever>")
+        @cmd_log_retain_time = ConfigParser.handle_nil(parameters[0])
+        @cmd_log_retain_time = @cmd_log_retain_time.to_i if @cmd_log_retain_time
       when 'CMD_DECOM_LOG_CYCLE_TIME'
         parser.verify_num_parameters(1, 1, "#{keyword} <Maximum time between files in seconds>")
         @cmd_decom_log_cycle_time = parameters[0].to_i
       when 'CMD_DECOM_LOG_CYCLE_SIZE'
         parser.verify_num_parameters(1, 1, "#{keyword} <Maximum file size in bytes>")
         @cmd_decom_log_cycle_size = parameters[0].to_i
+      when 'CMD_DECOM_LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for cmd decom log files in seconds - nil = Forever>")
+        @cmd_decom_log_retain_time = ConfigParser.handle_nil(parameters[0])
+        @cmd_decom_log_retain_time = @cmd_decom_log_retain_time.to_i if @cmd_decom_log_retain_time        
       when 'TLM_LOG_CYCLE_TIME'
         parser.verify_num_parameters(1, 1, "#{keyword} <Maximum time between files in seconds>")
         @tlm_log_cycle_time = parameters[0].to_i
       when 'TLM_LOG_CYCLE_SIZE'
         parser.verify_num_parameters(1, 1, "#{keyword} <Maximum file size in bytes>")
         @tlm_log_cycle_size = parameters[0].to_i
+      when 'TLM_LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for tlm log files in seconds - nil = Forever>")
+        @tlm_log_retain_time = ConfigParser.handle_nil(parameters[0])
+        @tlm_log_retain_time = @tlm_log_retain_time.to_i if @tlm_log_retain_time        
       when 'TLM_DECOM_LOG_CYCLE_TIME'
         parser.verify_num_parameters(1, 1, "#{keyword} <Maximum time between files in seconds>")
         @tlm_decom_log_cycle_time = parameters[0].to_i
       when 'TLM_DECOM_LOG_CYCLE_SIZE'
         parser.verify_num_parameters(1, 1, "#{keyword} <Maximum file size in bytes>")
         @tlm_decom_log_cycle_size = parameters[0].to_i
+      when 'TLM_DECOM_LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for tlm decom log files in seconds - nil = Forever>")
+        @tlm_decom_log_retain_time = ConfigParser.handle_nil(parameters[0])
+        @tlm_decom_log_retain_time = @tlm_decom_log_retain_time.to_i if @tlm_decom_log_retain_time
+      when 'REDUCED_MINUTE_LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for reduced minute log files in seconds - nil = Forever>")
+        @reduced_minute_log_retain_time = ConfigParser.handle_nil(parameters[0])
+        @reduced_minute_log_retain_time = @reduced_minute_log_retain_time.to_i if @reduced_minute_log_retain_time
+      when 'REDUCED_HOUR_LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for reduced hour log files in seconds - nil = Forever>")
+        @reduced_hour_log_retain_time = ConfigParser.handle_nil(parameters[0])
+        @reduced_hour_log_retain_time = @reduced_hour_log_retain_time.to_i if @reduced_hour_log_retain_time
+      when 'REDUCED_DAY_LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for reduced day log files in seconds - nil = Forever>")
+        @reduced_day_log_retain_time = ConfigParser.handle_nil(parameters[0])
+        @reduced_day_log_retain_time = @reduced_day_log_retain_time.to_i if @reduced_day_log_retain_time                
+      when 'LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for all log files in seconds - nil = Forever>")
+        log_retain_time = ConfigParser.handle_nil(parameters[0])
+        if log_retain_time 
+          @cmd_log_retain_time = log_retain_time.to_i
+          @cmd_decom_log_retain_time = log_retain_time.to_i   
+          @tlm_log_retain_time = log_retain_time.to_i
+          @tlm_decom_log_retain_time = log_retain_time.to_i
+        end     
+      when 'REDUCED_LOG_RETAIN_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Retention time for all reduced log files in seconds - nil = Forever>")
+        reduced_log_retain_time = ConfigParser.handle_nil(parameters[0])
+        if reduced_log_retain_time 
+          @reduced_minute_log_retain_time = reduced_log_retain_time.to_i
+          @reduced_hour_log_retain_time = reduced_log_retain_time.to_i   
+          @reduced_day_log_retain_time = reduced_log_retain_time.to_i
+        end
+      when 'CLEANUP_POLL_TIME'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Cleanup polling period in seconds>")
+        @cleanup_poll_time = parameters[0].to_i
       else
         raise ConfigParser::Error.new(parser, "Unknown keyword and parameters for Target: #{keyword} #{parameters.join(" ")}")
       end
@@ -335,7 +426,7 @@ module Cosmos
       Store.del("#{@scope}__cosmoscmd__#{@name}")
 
       # Note: these match the names of the services in deploy_microservices
-      %w(DECOM COMMANDLOG DECOMCMDLOG PACKETLOG DECOMLOG REDUCER).each do |type|
+      %w(DECOM COMMANDLOG DECOMCMDLOG PACKETLOG DECOMLOG REDUCER CLEANUP).each do |type|
         model = MicroserviceModel.get_model(name: "#{@scope}__#{type}__#{@name}", scope: @scope)
         model.destroy if model
       end
@@ -627,6 +718,22 @@ module Cosmos
         microservice.deploy(gem_path, variables)
         Logger.info "Configured microservice #{microservice_name}"
       end
+
+      if @cmd_log_retain_time or @cmd_decom_log_retain_time or @tlm_log_retain_time or @tlm_decom_log_retain_time or
+         @reduced_minute_log_retain_time or @reduced_hour_log_retain_time or @reduced_day_log_retain_time
+        # Cleanup Microservice
+        microservice_name = "#{@scope}__CLEANUP__#{@name}"
+        microservice = MicroserviceModel.new(
+          name: microservice_name,
+          cmd: ["ruby", "cleanup_microservice.rb", microservice_name],
+          work_dir: '/cosmos/lib/cosmos/microservices',
+          plugin: plugin,
+          scope: @scope
+        )
+        microservice.create
+        microservice.deploy(gem_path, variables)
+        Logger.info "Configured microservice #{microservice_name}"
+      end
     end
   end
 end

data/lib/cosmos/packets/packet.rb

@@ -445,6 +445,11 @@ module Cosmos
       @meta ||= {}
     end
 
+    # Sets packet specific metadata
+    def meta=(meta)
+      @meta = meta
+    end
+
     # Indicates if the packet has been identified
     # @return [TrueClass or FalseClass]
     def identified?
@@ -1009,6 +1014,24 @@ module Cosmos
       config
     end
 
+    def self.from_json(hash)
+      endianness = hash['endianness'] ? hash['endianness'].intern : nil # Convert to symbol
+      packet = Packet.new(hash['target_name'], hash['packet_name'], endianness, hash['description'])
+      packet.short_buffer_allowed = hash['short_buffer_allowed']
+      packet.hazardous = hash['hazardous']
+      packet.hazardous_description = hash['hazardous_description']
+      packet.messages_disabled = hash['messages_disabled']
+      packet.disabled = hash['disabled']
+      packet.hidden = hash['hidden']
+      # packet.stale is read only
+      packet.meta = hash['meta']
+      # Can't convert processors
+      hash['items'].each do |item|
+        packet.define(PacketItem.from_json(item))
+      end
+      packet
+    end
+
     protected
 
     # Performs packet specific processing on the packet.

data/lib/cosmos/packets/packet_config.rb

@@ -463,8 +463,8 @@ module Cosmos
       when 'POLY_READ_CONVERSION', 'POLY_WRITE_CONVERSION'
         usage = "#{keyword} <C0> <C1> <C2> ..."
         parser.verify_num_parameters(1, nil, usage)
-        @current_item.read_conversion = PolynomialConversion.new(params) if keyword.include? "READ"
-        @current_item.write_conversion = PolynomialConversion.new(params) if keyword.include? "WRITE"
+        @current_item.read_conversion = PolynomialConversion.new(*params) if keyword.include? "READ"
+        @current_item.write_conversion = PolynomialConversion.new(*params) if keyword.include? "WRITE"
 
       # Apply a segmented polynomial conversion to the current item
       # after it is read from the telemetry packet