cosmos 5.0.2 → 5.0.5

data/lib/cosmos/api/tlm_api.rb

@@ -19,6 +19,8 @@
 
 require 'cosmos/models/target_model'
 require 'cosmos/models/cvt_model'
+require 'cosmos/packets/packet'
+require 'cosmos/topics/telemetry_topic'
 require 'cosmos/utilities/s3'
 
 module Cosmos
@@ -38,10 +40,11 @@ module Cosmos
                        'get_tlm_packet',
                        'get_tlm_values',
                        'get_all_telemetry',
+                       'get_all_telemetry_list',
                        'get_telemetry',
                        'get_item',
                        'subscribe_packets',
-                       'get_packet',
+                       'get_packets',
                        'get_all_tlm_info',
                        'get_tlm_cnt',
                        'get_packet_derived_items',
@@ -108,11 +111,11 @@ module Cosmos
 
     # Injects a packet into the system as if it was received from an interface
     #
-    # @param target_name[String] Target name of the packet
-    # @param packet_name[String] Packet name of the packet
-    # @param item_hash[Hash] Hash of item_name and value for each item you want to change from the current value table
+    # @param target_name [String] Target name of the packet
+    # @param packet_name [String] Packet name of the packet
+    # @param item_hash [Hash] Hash of item_name and value for each item you want to change from the current value table
     # @param type [Symbol] Telemetry type, :RAW, :CONVERTED (default), :FORMATTED, or :WITH_UNITS
-    def inject_tlm(target_name, packet_name, item_hash = nil, log: true, type: :CONVERTED, scope: $cosmos_scope, token: $cosmos_token)
+    def inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, scope: $cosmos_scope, token: $cosmos_token)
       authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, scope: scope, token: token)
       unless CvtModel::VALUE_TYPES.include?(type.intern)
         raise "Unknown type '#{type}' for #{target_name} #{packet_name}"
@@ -125,19 +128,18 @@ module Cosmos
         # Check that the packet exists ... exceptions are raised if not
         TargetModel.packet(target_name, packet_name, scope: scope)
       end
-      inject = {}
-      inject['inject_tlm'] = true
-      inject['log'] = log
-      inject['target_name'] = target_name
-      inject['packet_name'] = packet_name
-      inject['item_hash'] = JSON.generate(item_hash) if item_hash
-      inject['type'] = type
-
-      InterfaceModel.all(scope: scope).each do |name, interface|
-        if interface['target_names'].include? target_name
-          Store.write_topic("{#{scope}__CMD}INTERFACE__#{interface['name']}", inject, '*', 100)
+
+      packet_hash = get_telemetry(target_name, packet_name, scope: scope, token: token)
+      packet = Packet.from_json(packet_hash)
+      if item_hash
+        item_hash.each do |name, value|
+          packet.write(name.to_s, value, type)
         end
       end
+      packet.received_time = Time.now.sys
+      # TODO: New packet so received_count is not correct
+      packet.received_count += 1
+      TelemetryTopic.write_packet(packet, scope: scope)
     end
 
     # Override the current value table such that a particular item always
@@ -188,7 +190,7 @@ module Cosmos
       authorize(permission: 'tlm', target_name: target_name, packet_name: packet_name, scope: scope, token: token)
       TargetModel.packet(target_name, packet_name, scope: scope)
       topic = "#{scope}__TELEMETRY__{#{target_name}}__#{packet_name}"
-      msg_id, msg_hash = Store.instance.read_topic_last(topic)
+      msg_id, msg_hash = Topic.get_newest_message(topic)
       if msg_id
         msg_hash['buffer'] = msg_hash['buffer'].b
         return msg_hash
@@ -245,7 +247,17 @@ module Cosmos
     # @return [Array<Hash>] Array of all telemetry packet hashes
     def get_all_telemetry(target_name, scope: $cosmos_scope, token: $cosmos_token)
       authorize(permission: 'tlm', target_name: target_name, scope: scope, token: token)
-      TargetModel.packets(target_name, scope: scope)
+      TargetModel.packets(target_name, type: :TLM, scope: scope)
+    end
+
+    # Returns an array of all telemetry packet's name and descriptions
+    #
+    # @since 5.0.3
+    # @param target_name [String] Name of the target
+    # @return [Array<Hash>] Array of all telemetry packet name and descriptions
+    def get_all_telemetry_list(target_name, scope: $cosmos_scope, token: $cosmos_token)
+      authorize(permission: 'tlm', target_name: target_name, scope: scope, token: token)
+      TargetModel.all_packet_name_descriptions(target_name, type: :TLM, scope: scope)
     end
 
     # Returns a telemetry packet hash
@@ -271,41 +283,53 @@ module Cosmos
       TargetModel.packet_item(target_name, packet_name, item_name, scope: scope)
     end
 
+    # 2x double underscore since __ is reserved
+    SUBSCRIPTION_DELIMITER = '____'
+
     # Subscribe to a list of packets. An ID is returned which is passed to
-    # get_packet(id) to yield packets back to a block.
+    # get_packets(id) to return packets.
     #
     # @param packets [Array<Array<String, String>>] Array of arrays consisting of target name, packet name
-    # @return [String] ID which should be passed to get_packet
+    # @return [String] ID which should be passed to get_packets
     def subscribe_packets(packets, scope: $cosmos_scope, token: $cosmos_token)
       if !packets.is_a?(Array) || !packets[0].is_a?(Array)
         raise ArgumentError, "packets must be nested array: [['TGT','PKT'],...]"
       end
 
-      result = [Time.now.to_nsec_from_epoch]
+      result = {}
       packets.each do |target_name, packet_name|
         authorize(permission: 'tlm', target_name: target_name, packet_name: packet_name, scope: scope, token: token)
-        result << "#{scope}__DECOM__{#{target_name}}__#{packet_name}"
+        topic = "#{scope}__DECOM__{#{target_name}}__#{packet_name}"
+        id, _ = Topic.get_newest_message(topic)
+        result[topic] = id ? id : '0-0'
       end
-      result.join("\n")
+      result.to_a.join(SUBSCRIPTION_DELIMITER)
     end
     # Alias the singular as well since that matches COSMOS 4
     alias subscribe_packet subscribe_packets
 
-    # Get a packet which was previously subscribed to by subscribe_packet.
-    # This method takes a block and yields back packet hashes.
-    def get_packet(id, scope: $cosmos_scope, token: $cosmos_token)
+    # Get packets based on ID returned from subscribe_packet.
+    # @param id [String] ID returned from subscribe_packets or last call to get_packets
+    # @param block [Integer] Number of milliseconds to block when requesting packets
+    # @param count [Integer] Maximum number of packets to return from EACH packet stream
+    # @return [Array<String, Array<Hash>] Array of the ID and array of all packets found
+    def get_packets(id, block: nil, count: 1000, scope: $cosmos_scope, token: $cosmos_token)
       authorize(permission: 'tlm', scope: scope, token: token)
-      offset, *topics = id.split("\n")
-      offsets = []
-      # Create a common array of offsets for each of the topics
-      topics.length.times do
-        offsets << (offset.to_i / 1_000_000).to_s + '-0'
-      end
-      Topic.read_topics(topics, offsets) do |topic, msg_id, msg_hash, redis|
-        json_hash = JSON.parse(msg_hash['json_data'])
-        msg_hash.delete('json_data')
-        yield msg_hash.merge(json_hash)
+      # Split the list of topic, ID values and turn it into a hash for easy updates
+      lookup = Hash[*id.split(SUBSCRIPTION_DELIMITER)]
+      xread = Topic.read_topics(lookup.keys, lookup.values, block, count)
+      # Return the original ID and nil if we didn't get anything
+      return [id, nil] if xread.empty?
+      packets = []
+      xread.each do |topic, data|
+        data.each do |id, msg_hash|
+          lookup[topic] = id # save the new ID
+          json_hash = JSON.parse(msg_hash['json_data'])
+          msg_hash.delete('json_data')
+          packets << msg_hash.merge(json_hash)
+        end
       end
+      return [lookup.to_a.join(SUBSCRIPTION_DELIMITER), packets]
     end
 
     # Get the receive count for a telemetry packet
@@ -316,7 +340,7 @@ module Cosmos
     def get_tlm_cnt(target_name, packet_name, scope: $cosmos_scope, token: $cosmos_token)
       authorize(permission: 'system', target_name: target_name, packet_name: packet_name, scope: scope, token: token)
       TargetModel.packet(target_name, packet_name, scope: scope)
-      _get_cnt("#{scope}__TELEMETRY__{#{target_name}}__#{packet_name}")
+      Topic.get_cnt("#{scope}__TELEMETRY__{#{target_name}}__#{packet_name}")
     end
 
     # Get information on all telemetry packets
@@ -329,14 +353,14 @@ module Cosmos
         TargetModel.packets(target_name, scope: scope).each do | packet |
           packet_name = packet['packet_name']
           key = "#{scope}__TELEMETRY__{#{target_name}}__#{packet_name}"
-          result << [target_name, packet_name, _get_cnt(key)]
+          result << [target_name, packet_name, Topic.get_cnt(key)]
         end
       end
       ['UNKNOWN'].each do | x |
         key = "#{scope}__TELEMETRY__{#{x}}__#{x}"
-        result << [x, x, _get_cnt(key)]
+        result << [x, x, Topic.get_cnt(key)]
       end
-      # Return the results sorted by target, packet
+      # Return the result sorted by target, packet
       result.sort_by { |a| [a[0], a[1]] }
     end
 
@@ -401,7 +425,7 @@ module Cosmos
         TargetModel.packets(target_name, scope: scope).each do |packet|
           item = packet['items'].find { |item| item['name'] == item_name }
           if item
-            _, msg_hash = Store.instance.get_oldest_message("#{scope}__DECOM__{#{target_name}}__#{packet['packet_name']}")
+            _, msg_hash = Topic.get_oldest_message("#{scope}__DECOM__{#{target_name}}__#{packet['packet_name']}")
             if msg_hash && msg_hash['time'] && msg_hash['time'].to_i > latest
               packet_name = packet['packet_name']
               latest = msg_hash['time'].to_i

data/lib/cosmos/config/config_parser.rb

@@ -168,7 +168,7 @@ module Cosmos
         path = File.join(File.dirname(@filename), template_name)
       end
       Cosmos.set_working_dir(File.dirname(path)) do
-        return ERB.new(File.read(path)).result(b)
+        return ERB.new(File.read(path), trim_mode: "-").result(b)
       end
     end
 
@@ -238,27 +238,24 @@ module Cosmos
       end
     end
 
-    # Verifies the indicated parameter(s) in the config don't start or end
-    # with an underscore and don't contain a double underscore. Raises an
-    # Error if they do.
+    # Verifies the indicated parameter in the config doesn't start or end
+    # with an underscore, doesn't contain a double underscore, doesn't contain
+    # spaces and doesn't start with a close bracket.
     #
-    # @param [Integer|Range<Integer>|Array<Integer>] indeces_to_check The
-    #   indeces of the parameters that must follow underscore rules
-    def verify_parameters_underscores(indeces_to_check, usage = "")
-      indeces_to_check = (1..@parameters.length) if indeces_to_check.nil?
-      indeces_to_check = [indeces_to_check] unless indeces_to_check.respond_to? 'each'
-
-      indeces_to_check.each do |index|
-        param = @parameters[index - 1]
-        if param.end_with? '_'
-          raise Error.new(self, "Parameter #{index} (#{@parameters[index - 1]}) for #{@keyword} cannot end with an underscore ('_').", usage, @url)
-        end
-        if param.include? '__'
-          raise Error.new(self, "Parameter #{index} (#{@parameters[index - 1]}) for #{@keyword} cannot contain a double underscore ('__').", usage, @url)
-        end
-        if param.include? ' '
-          raise Error.new(self, "Parameter #{index} (#{@parameters[index - 1]}) for #{@keyword} cannot contain a space (' ').", usage, @url)
-        end
+    # @param [Integer] index The index of the parameter to check
+    def verify_parameter_naming(index, usage = "")
+      param = @parameters[index - 1]
+      if param.end_with? '_'
+        raise Error.new(self, "Parameter #{index} (#{param}) for #{@keyword} cannot end with an underscore ('_').", usage, @url)
+      end
+      if param.include? '__'
+        raise Error.new(self, "Parameter #{index} (#{param}) for #{@keyword} cannot contain a double underscore ('__').", usage, @url)
+      end
+      if param.include? ' '
+        raise Error.new(self, "Parameter #{index} (#{param}) for #{@keyword} cannot contain a space (' ').", usage, @url)
+      end
+      if param.start_with?('}')
+        raise Error.new(self, "Parameter #{index} (#{param}) for #{@keyword} cannot start with a close bracket ('}').", usage, @url)
       end
     end
 
@@ -380,7 +377,7 @@ module Cosmos
         output = nil
         if run_erb
           Cosmos.set_working_dir(File.dirname(filename)) do
-            output = ERB.new(File.read(filename)).result(binding.set_variables(variables))
+            output = ERB.new(File.read(filename), trim_mode: "-").result(binding.set_variables(variables))
           end
         else
           output = File.read(filename)

data/lib/cosmos/config/meta_config_parser.rb

@@ -50,7 +50,7 @@ module Cosmos
 
       output = nil
       Cosmos.set_working_dir(File.dirname(path)) do
-        output = ERB.new(File.read(path)).result(binding)
+        output = ERB.new(File.read(path), trim_mode: "-").result(binding)
       end
       tf.write(output)
       tf.close

data/lib/cosmos/conversions/generic_conversion.rb

@@ -73,5 +73,5 @@ module Cosmos
     def as_json
       { 'class' => self.class.name.to_s, 'params' => [@code_to_eval, @converted_type, @converted_bit_size] }
     end
-  end # class GenericConversion
-end # module Cosmos
+  end
+end

data/lib/cosmos/conversions/polynomial_conversion.rb

@@ -29,13 +29,10 @@ module Cosmos
     # Initializes the conversion with the given polynomial coefficients. Sets
     # the converted_type to :FLOAT and the converted_bit_size to 64.
     #
-    # @param coeff_array [Array<Float>] The polynomial coefficients
-    def initialize(coeff_array)
+    # @param coeffs [Array<Float>] The polynomial coefficients
+    def initialize(*coeffs)
       super()
-      @coeffs = []
-      coeff_array.each do |coeff|
-        @coeffs << coeff.to_f
-      end
+      @coeffs = coeffs.map { |coeff| coeff.to_f }
       @converted_type = :FLOAT
       @converted_bit_size = 64
     end
@@ -84,5 +81,5 @@ module Cosmos
     def as_json
       { 'class' => self.class.name.to_s, 'params' => @coeffs }
     end
-  end # class PolynomialConversion
-end # module Cosmos
+  end
+end

data/lib/cosmos/conversions/segmented_polynomial_conversion.rb

@@ -23,6 +23,9 @@ module Cosmos
   # Segmented polynomial conversions consist of polynomial conversions that are
   # applied for a range of values.
   class SegmentedPolynomialConversion < Conversion
+    # @return [Array<Segment>] Segments which make up this conversion
+    attr_reader :segments
+
     # A polynomial conversion segment which applies the conversion from the
     # lower bound (inclusive) until another segment's lower bound is
     # encountered.
@@ -31,6 +34,7 @@ module Cosmos
       #   should apply. All values >= to this value will be converted using the
       #   given coefficients.
       attr_reader :lower_bound
+
       # @return [Array<Integer>] The polynomial coefficients
       attr_reader :coeffs
 
@@ -57,6 +61,14 @@ module Cosmos
         return other_segment.lower_bound <=> @lower_bound
       end
 
+      # Implement equality operator primarily for ease of testing
+      #
+      # @param segment [Segment] Other segment
+      def ==(other_segment)
+        @lower_bound == other_segment.lower_bound &&
+          @coeffs == other_segment.coeffs
+      end
+
       # Perform the polynomial conversion
       #
       # @param value [Numeric] The value to convert
@@ -71,9 +83,15 @@ module Cosmos
     end
 
     # Initialize the converted_type to :FLOAT and converted_bit_size to 64.
-    def initialize
+    #
+    # @param segments [Array] Array of segments typically generated by as_json
+    #   Format similar to the following: [[15, [3, 2]], [10, [1, 2]]]
+    #   Where each entry is an array with the first value as the lower_bound
+    #   and the other entry is an array of the coefficients for that segment.
+    def initialize(segments = [])
       super()
       @segments = []
+      segments.each { |lower_bound, coeffs| add_segment(lower_bound, *coeffs) }
       @converted_type = :FLOAT
       @converted_bit_size = 64
     end
@@ -95,9 +113,7 @@ module Cosmos
     def call(value, packet, buffer)
       # Try to find correct segment
       @segments.each do |segment|
-        if value >= segment.lower_bound
-          return segment.calculate(value)
-        end
+        return segment.calculate(value) if value >= segment.lower_bound
       end
 
       # Default to using segment with smallest lower_bound
@@ -112,7 +128,7 @@ module Cosmos
     # @return [String] The name of the class followed by a description of all
     #   the polynomial segments.
     def to_s
-      result = ""
+      result = ''
       count = 0
       @segments.each do |segment|
         result << "\n" if count > 0
@@ -136,7 +152,8 @@ module Cosmos
     def to_config(read_or_write)
       config = ''
       @segments.each do |segment|
-        config << "    SEG_POLY_#{read_or_write}_CONVERSION #{segment.lower_bound} #{segment.coeffs.join(' ')}\n"
+        config <<
+          "    SEG_POLY_#{read_or_write}_CONVERSION #{segment.lower_bound} #{segment.coeffs.join(' ')}\n"
       end
       config
     end
@@ -146,7 +163,7 @@ module Cosmos
       @segments.each do |segment|
         params << [segment.lower_bound, segment.coeffs]
       end
-      { 'class' => self.class.name.to_s, 'params' => params }
+      { 'class' => self.class.name.to_s, 'params' => [params] }
     end
-  end # class SegmentedPolynomialConversion
-end # module Cosmos
+  end
+end

data/lib/cosmos/io/json_drb.rb

@@ -269,7 +269,11 @@ module Cosmos
               response = JsonRpcSuccessResponse.new(result, request.id)
             end
           rescue Exception => error
-            Logger.error error.formatted
+            # Filter out the framework stack trace (rails, rack, puma etc)
+            lines = error.formatted.split("\n")
+            i = lines.find_index { |row| row.include?('actionpack') || row.include?('activesupport') }
+            Logger.error lines[0...i].join("\n")
+
             if request.id
               if NoMethodError === error
                 error_code = JsonRpcError::ErrorCode::METHOD_NOT_FOUND

data/lib/cosmos/logs/log_writer.rb

@@ -19,7 +19,7 @@
 
 require 'thread'
 require 'cosmos/config/config_parser'
-require 'cosmos/utilities/store'
+require 'cosmos/topics/topic'
 require 'cosmos/utilities/s3'
 
 module Cosmos
@@ -32,9 +32,41 @@ module Cosmos
     # @return [true/false] Whether logging is enabled
     attr_reader :logging_enabled
 
+    # @return cycle_time [Integer] The amount of time in seconds before creating
+    #   a new log file. This can be combined with cycle_size but is better used
+    #   independently.
+    attr_reader :cycle_time
+
+    # @return cycle_hour [Integer] The time at which to cycle the log. Combined with
+    #   cycle_minute to cycle the log daily at the specified time. If nil, the log
+    #   will be cycled hourly at the specified cycle_minute.
+    attr_reader :cycle_hour
+
+    # @return cycle_minute [Integer] The time at which to cycle the log. See cycle_hour
+    #   for more information.
+    attr_reader :cycle_minute
+
+    # @return [Time] Time that the current log file started
+    attr_reader :start_time
+
+    # @return [Mutex] Instance mutex protecting file
+    attr_reader :mutex
+
     # The cycle time interval. Cycle times are only checked at this level of
     # granularity.
-    CYCLE_TIME_INTERVAL = 2
+    CYCLE_TIME_INTERVAL = 10
+
+    # Mutex protecting class variables
+    @@mutex = Mutex.new
+
+    # Array of instances used to keep track of cycling logs
+    @@instances = []
+
+    # Thread used to cycle logs across all log writers
+    @@cycle_thread = nil
+
+    # Sleeper used to delay cycle thread
+    @@cycle_sleeper = nil
 
     # @param remote_log_directory [String] The s3 path to store the log files
     # @param logging_enabled [Boolean] Whether to start with logging enabled
@@ -89,11 +121,15 @@ module Cosmos
       # each time we create an entry which we do a LOT!
       @entry = String.new
 
-      @cycle_thread = nil
       if @cycle_time or @cycle_hour or @cycle_minute
-        @cycle_sleeper = Sleeper.new
-        @cycle_thread = Cosmos.safe_thread("Log cycle") do
-          cycle_thread_body()
+        @@mutex.synchronize do
+          @@instances << self
+
+          unless @@cycle_thread
+            @@cycle_thread = Cosmos.safe_thread("Log cycle") do
+              cycle_thread_body()
+            end
+          end
         end
       end
     end
@@ -113,10 +149,13 @@ module Cosmos
     # Stop all logging, close the current log file, and kill the logging threads.
     def shutdown
       stop()
-      if @cycle_thread
-        @cycle_sleeper.cancel
-        Cosmos.kill_thread(self, @cycle_thread)
-        @cycle_thread = nil
+      @@mutex.synchronize do
+        @@instances.delete(self)
+        if @@instances.length <= 0
+          @@cycle_sleeper.cancel if @@cycle_sleeper
+          Cosmos.kill_thread(self, @@cycle_thread) if @@cycle_thread
+          @@cycle_thread = nil
+        end
       end
     end
 
@@ -143,28 +182,38 @@ module Cosmos
     end
 
     def cycle_thread_body
+      @@cycle_sleeper = Sleeper.new
       while true
-        # The check against start_time needs to be mutex protected to prevent a packet coming in between the check
-        # and closing the file
-        @mutex.synchronize do
-          utc_now = Time.now.utc
-          # Logger.debug("start:#{@start_time.to_f} now:#{utc_now.to_f} cycle:#{@cycle_time} new:#{(utc_now - @start_time) > @cycle_time}")
-          if @logging_enabled and
-             (
-               # Cycle based on total time logging
-               (@cycle_time and (utc_now - @start_time) > @cycle_time) or
-
-               # Cycle daily at a specific time
-               (@cycle_hour and @cycle_minute and utc_now.hour == @cycle_hour and utc_now.min == @cycle_minute and @start_time.yday != utc_now.yday) or
-
-               # Cycle hourly at a specific time
-               (@cycle_minute and not @cycle_hour and utc_now.min == @cycle_minute and @start_time.hour != utc_now.hour)
-             )
-            close_file(false)
+        start_time = Time.now
+        @@mutex.synchronize do
+          @@instances.each do |instance|
+            # The check against start_time needs to be mutex protected to prevent a packet coming in between the check
+            # and closing the file
+            instance.mutex.synchronize do
+              utc_now = Time.now.utc
+              # Logger.debug("start:#{@start_time.to_f} now:#{utc_now.to_f} cycle:#{@cycle_time} new:#{(utc_now - @start_time) > @cycle_time}")
+              if instance.logging_enabled and
+                (
+                  # Cycle based on total time logging
+                  (instance.cycle_time and (utc_now - instance.start_time) > instance.cycle_time) or
+
+                  # Cycle daily at a specific time
+                  (instance.cycle_hour and instance.cycle_minute and utc_now.hour == instance.cycle_hour and utc_now.min == instance.cycle_minute and instance.start_time.yday != utc_now.yday) or
+
+                  # Cycle hourly at a specific time
+                  (instance.cycle_minute and not instance.cycle_hour and utc_now.min == instance.cycle_minute and instance.start_time.hour != utc_now.hour)
+                )
+                instance.close_file(false)
+              end
+            end
           end
         end
+
         # Only check whether to cycle at a set interval
-        break if @cycle_sleeper.sleep(CYCLE_TIME_INTERVAL)
+        run_time = Time.now - start_time
+        sleep_time = CYCLE_TIME_INTERVAL - run_time
+        sleep_time = 0 if sleep_time < 0
+        break if @@cycle_sleeper.sleep(sleep_time)
       end
     end
 
@@ -211,7 +260,7 @@ module Cosmos
             S3Utilities.move_log_file_to_s3(@filename, s3_key)
             # Now that the file is in S3, trim the Redis stream up until the previous file.
             # This keeps one file worth of data in Redis as a safety buffer
-            Cosmos::Store.trim_topic(@redis_topic, @previous_file_redis_offset) if @redis_topic and @previous_file_redis_offset
+            Topic.trim_topic(@redis_topic, @previous_file_redis_offset) if @redis_topic and @previous_file_redis_offset
             @previous_file_redis_offset = @last_offset
           rescue Exception => err
             Logger.instance.error "Error closing #{@filename} : #{err.formatted}"

data/lib/cosmos/microservices/cleanup_microservice.rb

@@ -17,50 +17,49 @@
 # enterprise edition license of COSMOS if purchased from the
 # copyright holder
 
+require 'cosmos/models/target_model'
 require 'cosmos/microservices/microservice'
 require 'cosmos/utilities/s3'
 
 module Cosmos
   class CleanupMicroservice < Microservice
     def run
-      # Update settings from config
-      @config['options'].each do |option|
-        case option[0].upcase
-        when 'SIZE' # Max size to use in S3 in bytes
-          @size = option[1].to_i
-        when 'DELAY' # Delay between size checks
-          @delay = option[1].to_i
-        when 'BUCKET' # Which bucket to monitor
-          @bucket = option[1]
-        when 'PREFIX' # Path into bucket to monitor
-          @prefix = option[1]
-        else
-          Logger.error("Unknown option passed to microservice #{@name}: #{option}")
-        end
-      end
-
-      raise "Microservice #{@name} not fully configured" unless @size and @delay and @bucket and @prefix
+      split_name = @name.split("__")
+      target_name = split_name[-1]
+      target = TargetModel.get_model(name: target_name, scope: @scope)
 
       rubys3_client = Aws::S3::Client.new
       while true
         break if @cancel_thread
 
         @state = 'GETTING_OBJECTS'
-        total_size, oldest_list = S3Utilities.get_total_size_and_oldest_list(@bucket, @prefix)
-        delete_items = []
-        oldest_list.each do |item|
-          break if total_size <= @size
-
-          delete_items << { :key => item.key }
-          total_size -= item.size
-        end
-        if delete_items.length > 0
-          @state = 'DELETING_OBJECTS'
-          rubys3_client.delete_objects({ bucket: @bucket, delete: { objects: delete_items } })
+        start_time = Time.now
+        [
+         ["#{@scope}/raw_logs/cmd/#{target_name}/", target.cmd_log_retain_time], 
+         ["#{@scope}/decom_logs/cmd/#{target_name}/", target.cmd_decom_log_retain_time], 
+         ["#{@scope}/raw_logs/tlm/#{target_name}/", target.tlm_log_retain_time], 
+         ["#{@scope}/decom_logs/tlm/#{target_name}/", target.tlm_decom_log_retain_time],
+         ["#{@scope}/reduced_minute_logs/tlm/#{target_name}/", target.reduced_minute_log_retain_time],
+         ["#{@scope}/reduced_hour_logs/tlm/#{target_name}/", target.reduced_hour_log_retain_time],
+         ["#{@scope}/reduced_day_logs/tlm/#{target_name}/", target.reduced_day_log_retain_time],
+        ].each do |prefix, retain_time|
+          next unless retain_time
+          time = start_time - retain_time
+          total_size, oldest_list = S3Utilities.list_files_before_time('logs', prefix, time)
+          delete_items = []
+          oldest_list.each do |item|
+            delete_items << { :key => item.key }
+          end
+          if delete_items.length > 0
+            @state = 'DELETING_OBJECTS'
+            rubys3_client.delete_objects({ bucket: 'logs', delete: { objects: delete_items } })
+            Logger.info("Deleted #{delete_items.length} #{target_name} log files")
+          end
         end
+
         @count += 1
         @state = 'SLEEPING'
-        break if @microservice_sleeper.sleep(@delay)
+        break if @microservice_sleeper.sleep(target.cleanup_poll_time)
       end
     end
   end

data/lib/cosmos/microservices/decom_microservice.rb

@@ -31,7 +31,7 @@ module Cosmos
 
     def initialize(*args)
       super(*args)
-      Store.update_topic_offsets(@topics)
+      Topic.update_topic_offsets(@topics)
       System.telemetry.limits_change_callback = method(:limits_change_callback)
     end