openc3 5.10.1 → 5.11.0

data/lib/openc3/microservices/cleanup_microservice.rb

@@ -35,26 +35,14 @@ module OpenC3
       @sleeper = Sleeper.new
     end
 
-    def run
-      split_name = @name.split("__")
-      target_name = split_name[-1]
-      target = TargetModel.get_model(name: target_name, scope: @scope)
-
+    def cleanup(areas, cleanup_poll_time)
       bucket = Bucket.getClient()
       while true
         break if @cancel_thread
 
         @state = 'GETTING_OBJECTS'
         start_time = Time.now
-        [
-         ["#{@scope}/raw_logs/cmd/#{target_name}", target.cmd_log_retain_time],
-         ["#{@scope}/decom_logs/cmd/#{target_name}", target.cmd_decom_log_retain_time],
-         ["#{@scope}/raw_logs/tlm/#{target_name}", target.tlm_log_retain_time],
-         ["#{@scope}/decom_logs/tlm/#{target_name}", target.tlm_decom_log_retain_time],
-         ["#{@scope}/reduced_minute_logs/tlm/#{target_name}", target.reduced_minute_log_retain_time],
-         ["#{@scope}/reduced_hour_logs/tlm/#{target_name}", target.reduced_hour_log_retain_time],
-         ["#{@scope}/reduced_day_logs/tlm/#{target_name}", target.reduced_day_log_retain_time],
-        ].each do |prefix, retain_time|
+        areas.each do |prefix, retain_time|
           next unless retain_time
           time = start_time - retain_time
           oldest_list = BucketUtilities.files_between_time(ENV['OPENC3_LOGS_BUCKET'], prefix, nil, time)
@@ -62,7 +50,7 @@ module OpenC3
             @state = 'DELETING_OBJECTS'
             oldest_list.each_slice(1000) do |slice|
               bucket.delete_objects(bucket: ENV['OPENC3_LOGS_BUCKET'], keys: slice)
-              @logger.debug("Deleted #{slice.length} #{target_name} log files")
+              @logger.debug("Cleanup deleted #{slice.length} log files")
               @delete_count += slice.length
               @metric.set(name: 'cleanup_delete_total', value: @delete_count, type: 'counter')
             end
@@ -71,11 +59,30 @@ module OpenC3
 
         @count += 1
         @metric.set(name: 'cleanup_total', value: @count, type: 'counter')
+
         @state = 'SLEEPING'
-        break if @sleeper.sleep(target.cleanup_poll_time)
+        break if @sleeper.sleep(cleanup_poll_time)
       end
     end
 
+    def run
+      split_name = @name.split("__")
+      target_name = split_name[-1]
+      target = TargetModel.get_model(name: target_name, scope: @scope)
+
+      areas = [
+        ["#{@scope}/raw_logs/cmd/#{target_name}", target.cmd_log_retain_time],
+        ["#{@scope}/decom_logs/cmd/#{target_name}", target.cmd_decom_log_retain_time],
+        ["#{@scope}/raw_logs/tlm/#{target_name}", target.tlm_log_retain_time],
+        ["#{@scope}/decom_logs/tlm/#{target_name}", target.tlm_decom_log_retain_time],
+        ["#{@scope}/reduced_minute_logs/tlm/#{target_name}", target.reduced_minute_log_retain_time],
+        ["#{@scope}/reduced_hour_logs/tlm/#{target_name}", target.reduced_hour_log_retain_time],
+        ["#{@scope}/reduced_day_logs/tlm/#{target_name}", target.reduced_day_log_retain_time],
+      ]
+
+      cleanup(areas, target.cleanup_poll_time)
+    end
+
     def shutdown
       @sleeper.cancel
       super()

data/lib/openc3/microservices/decom_microservice.rb

@@ -24,8 +24,6 @@ require 'openc3/microservices/microservice'
 require 'openc3/microservices/interface_decom_common'
 require 'openc3/topics/telemetry_decom_topic'
 require 'openc3/topics/limits_event_topic'
-require 'openc3/topics/notifications_topic'
-require 'openc3/models/notification_model'
 
 module OpenC3
   class DecomMicroservice < Microservice
@@ -93,8 +91,14 @@ module OpenC3
 
         packet = System.telemetry.packet(target_name, packet_name)
         packet.stored = ConfigParser.handle_true_false(msg_hash["stored"])
+        # Note: Packet time will be recalculated as part of decom so not setting
         packet.received_time = Time.from_nsec_from_epoch(msg_hash["received_time"].to_i)
         packet.received_count = msg_hash["received_count"].to_i
+        extra = msg_hash["extra"]
+        if extra and extra.length > 0
+          extra = JSON.parse(extra, allow_nan: true, create_additions: true)
+          packet.extra = extra
+        end
         packet.buffer = msg_hash["buffer"]
         packet.check_limits(System.limits_set) # Process all the limits and call the limits_change_callback (as necessary)
 
@@ -128,25 +132,9 @@ module OpenC3
         when :BLUE, :GREEN, :GREEN_LOW, :GREEN_HIGH
           @logger.info message
         when :YELLOW, :YELLOW_LOW, :YELLOW_HIGH
-          notification = NotificationModel.new(
-            time: time_nsec,
-            severity: "caution",
-            url: "/tools/limitsmonitor",
-            title: "#{packet.target_name} #{packet.packet_name} #{item.name} #{item.limits.state}",
-            body: "#{item.name} is #{item.limits.state}"
-          )
-          NotificationsTopic.write_notification(notification.as_json(:allow_nan => true), scope: @scope)
-          @logger.warn message
+          @logger.warn(message, type: Logger::NOTIFICATION)
         when :RED, :RED_LOW, :RED_HIGH
-          notification = NotificationModel.new(
-            time: time_nsec,
-            severity: "critical",
-            url: "/tools/limitsmonitor",
-            title: "#{packet.target_name} #{packet.packet_name} #{item.name} #{item.limits.state}",
-            body: "#{item.name} is #{item.limits.state}"
-          )
-          NotificationsTopic.write_notification(notification.as_json(:allow_nan => true), scope: @scope)
-          @logger.error message
+          @logger.error(message, type: Logger::ALERT)
         end
       end
 

data/lib/openc3/microservices/log_microservice.rb

@@ -110,7 +110,9 @@ module OpenC3
         packet_type = :JSON_PACKET
         data_key = "json_data"
       end
-      @plws[target_name][rt_or_stored].buffered_write(packet_type, @cmd_or_tlm, target_name, packet_name, msg_hash["time"].to_i, rt_or_stored == :STORED, msg_hash[data_key], nil, topic, msg_id)
+      received_time_nsec_since_epoch = msg_hash["received_time"]
+      received_time_nsec_since_epoch = received_time_nsec_since_epoch.to_i if received_time_nsec_since_epoch
+      @plws[target_name][rt_or_stored].buffered_write(packet_type, @cmd_or_tlm, target_name, packet_name, msg_hash["time"].to_i, rt_or_stored == :STORED, msg_hash[data_key], nil, topic, msg_id, received_time_nsec_since_epoch: received_time_nsec_since_epoch, extra: msg_hash['extra'])
     rescue => err
       @error = err
       @logger.error("#{@name} error: #{err.formatted}")

data/lib/openc3/microservices/reaction_microservice.rb

@@ -22,8 +22,6 @@
 
 require 'openc3/microservices/microservice'
 require 'openc3/models/reaction_model'
-require 'openc3/models/notification_model'
-require 'openc3/topics/notifications_topic'
 require 'openc3/models/trigger_model'
 require 'openc3/topics/autonomic_topic'
 require 'openc3/utilities/authentication'
@@ -305,15 +303,28 @@ module OpenC3
     end
 
     def run_notify(reaction:, action:)
-      notification = NotificationModel.new(
-        time: Time.now.to_nsec_from_epoch,
-        severity: action['severity'],
-        url: "/tools/autonomic/reactions",
-        title: "#{reaction.name} run",
-        body: action['value']
-      )
-      NotificationsTopic.write_notification(notification.as_json(:allow_nan => true), scope: @scope)
-      @logger.info "ReactionWorker-#{@ident} #{reaction.name} notify action complete, body: #{action['value']}, severity: #{action['severity']}"
+      message = "ReactionWorker-#{@ident} #{reaction.name} notify action complete, body: #{action['value']}"
+      url = "/tools/autonomic/reactions"
+      case action['severity'].to_s.upcase
+      when 'FATAL'
+        @logger.fatal(message, url: url, type: Logger::ALERT)
+      when 'ERROR', 'CRITICAL'
+        @logger.error(message, url: url, type: Logger::ALERT)
+      when 'WARN', 'CAUTION', 'SERIOUS'
+        @logger.warn(message, url: url, type: Logger::NOTIFICATION)
+      when 'INFO', 'NORMAL', 'STANDBY', 'OFF'
+        @logger.info(message, url: url, type: Logger::NOTIFICATION)
+      when 'DEBUG'
+        level = @logger.level
+        begin
+          @logger.level = Logger::DEBUG
+          @logger.debug(message, url: url, type: Logger::NOTIFICATION)
+        ensure
+          @logger.level = level
+        end
+      else
+        raise "Unknown severity: #{action['severity']}"
+      end
     end
 
     def run_command(reaction:, action:)

data/lib/openc3/microservices/reducer_microservice.rb

@@ -25,8 +25,10 @@ require 'openc3/topics/topic'
 require 'openc3/topics/telemetry_reduced_topics'
 require 'openc3/packets/json_packet'
 require 'openc3/utilities/bucket_file_cache'
+require 'openc3/utilities/throttle'
 require 'openc3/models/reducer_model'
 require 'openc3/logs/buffered_packet_log_writer'
+require 'openc3/ext/reducer_microservice' if RUBY_ENGINE == 'ruby' and !ENV['OPENC3_NO_EXT']
 require 'rufus-scheduler'
 
 module OpenC3
@@ -96,6 +98,8 @@ module OpenC3
           case option[0].upcase
           when 'BUFFER_DEPTH' # Buffer depth to write in time order
             @buffer_depth = option[1].to_i
+          when 'MAX_CPU_UTILIZATION'
+            @max_cpu_utilization = Float(option[1])
           else
             @logger.error("Unknown option passed to microservice #{@name}: #{option}")
           end
@@ -103,6 +107,7 @@ module OpenC3
       end
 
       @buffer_depth = 10 unless @buffer_depth
+      @max_cpu_utilization = 30.0 unless @max_cpu_utilization
       @target_name = name.split('__')[-1]
       @packet_logs = {}
 
@@ -189,12 +194,14 @@ module OpenC3
     end
 
     def process_file(filename, type, entry_nanoseconds, file_nanoseconds)
+      throttle = OpenC3::Throttle.new(@max_cpu_utilization)
       file = BucketFile.new(filename)
       file.retrieve
       unless file.local_path
         @logger.warn("Reducer Warning: #{filename}: Could not be retrieved")
         return
       end
+      throttle.throttle_sleep
 
       # Determine if we already have a PacketLogWriter created
       _, _, scope, target_name, _, rt_or_stored, _ = File.basename(filename).split('__')
@@ -217,6 +224,7 @@ module OpenC3
       # The lifetime of all these variables is a single file - single target / multiple packets
       reducer_state = {}
       plr = OpenC3::PacketLogReader.new
+      throttle.throttle_sleep
       plr.each(file.local_path) do |packet|
         # Check to see if we should start a new log file before processing this packet
         current_time = packet.packet_time.to_nsec_from_epoch
@@ -239,131 +247,27 @@ module OpenC3
         end
 
         if type == 'minute'
-          state.entry_samples ||= packet.json_hash.dup # Grab all the samples from the first packet
-          if state.first
-            state.raw_values = packet.read_all(:RAW, nil, packet.read_all_names(:RAW)).select { |key, value| value.is_a?(Numeric) }
-            state.raw_keys ||= state.raw_values.keys
-            state.converted_values = packet.read_all(:CONVERTED, nil, packet.read_all_names(:CONVERTED)).select { |key, value| value.is_a?(Numeric) }
-            state.converted_keys ||= state.converted_values.keys
-          else
-            state.raw_values = packet.read_all(:RAW, nil, state.raw_keys).select { |key, value| value.is_a?(Numeric) }
-            state.converted_values = packet.read_all(:CONVERTED, nil, state.converted_keys).select { |key, value| value.is_a?(Numeric) }
-          end
+          get_min_samples(packet, state)
         else
-          # Hour or Day
-          state.entry_samples ||= extract_entry_samples(packet)
-          if state.first
-            state.raw_max_values = packet.read_all(:RAW, :MAX, packet.read_all_names(:RAW, :MAX))
-            state.raw_keys = state.raw_max_values.keys
-            state.converted_max_values = packet.read_all(:CONVERTED, :MAX, packet.read_all_names(:CONVERTED, :MAX))
-            state.converted_keys = state.converted_max_values.keys
-          else
-            state.raw_max_values = packet.read_all(:RAW, :MAX, state.raw_keys)
-            state.converted_max_values = packet.read_all(:CONVERTED, :MAX, state.converted_keys)
-          end
-          state.raw_min_values = packet.read_all(:RAW, :MIN, state.raw_keys)
-          state.raw_avg_values = packet.read_all(:RAW, :AVG, state.raw_keys)
-          state.raw_stddev_values = packet.read_all(:RAW, :STDDEV, state.raw_keys)
-          state.converted_min_values = packet.read_all(:CONVERTED, :MIN, state.converted_keys)
-          state.converted_avg_values = packet.read_all(:CONVERTED, :AVG, state.converted_keys)
-          state.converted_stddev_values = packet.read_all(:CONVERTED, :STDDEV, state.converted_keys)
+          get_hour_day_samples(packet, state)
         end
 
         reduced = state.reduced
         if type == 'minute'
-          # Update statistics for this packet's raw values
-          state.raw_values.each do |key, value|
-            if value
-              reduced["#{key}__VALS"] ||= []
-              reduced["#{key}__VALS"] << value
-              reduced["#{key}__N"] ||= value
-              reduced["#{key}__N"] = value if value < reduced["#{key}__N"]
-              reduced["#{key}__X"] ||= value
-              reduced["#{key}__X"] = value if value > reduced["#{key}__X"]
-            end
-          end
-
-          # Update statistics for this packet's converted values
-          state.converted_values.each do |key, value|
-            if value
-              reduced["#{key}__CVALS"] ||= []
-              reduced["#{key}__CVALS"] << value
-              reduced["#{key}__CN"] ||= value
-              reduced["#{key}__CN"] = value if value < reduced["#{key}__CN"]
-              reduced["#{key}__CX"] ||= value
-              reduced["#{key}__CX"] = value if value > reduced["#{key}__CX"]
-            end
-          end
+          update_min_stats(reduced, state)
         else
-          # Update statistics for this packet's raw values
-          state.raw_max_values.each do |key, value|
-            if value
-              max_key = "#{key}__X"
-              reduced[max_key] ||= value
-              reduced[max_key] = value if value > reduced[max_key]
-            end
-          end
-          state.raw_min_values.each do |key, value|
-            if value
-              min_key = "#{key}__N"
-              reduced[min_key] ||= value
-              reduced[min_key] = value if value < reduced[min_key]
-            end
-          end
-          state.raw_avg_values.each do |key, value|
-            if value
-              avg_values_key = "#{key}__AVGVALS"
-              reduced[avg_values_key] ||= []
-              reduced[avg_values_key] << value
-            end
-          end
-          state.raw_stddev_values.each do |key, value|
-            if value
-              stddev_values_key = "#{key}__STDDEVVALS"
-              reduced[stddev_values_key] ||= []
-              reduced[stddev_values_key] << value
-            end
-          end
-
-          # Update statistics for this packet's converted values
-          state.converted_max_values.each do |key, value|
-            if value
-              max_key = "#{key}__CX"
-              reduced[max_key] ||= value
-              reduced[max_key] = value if value > reduced[max_key]
-            end
-          end
-          state.converted_min_values.each do |key, value|
-            if value
-              min_key = "#{key}__CN"
-              reduced[min_key] ||= value
-              reduced[min_key] = value if value < reduced[min_key]
-            end
-          end
-          state.converted_avg_values.each do |key, value|
-            if value
-              avg_values_key = "#{key}__CAVGVALS"
-              reduced[avg_values_key] ||= []
-              reduced[avg_values_key] << value
-            end
-          end
-          state.converted_stddev_values.each do |key, value|
-            if value
-              stddev_values_key = "#{key}__CSTDDEVVALS"
-              reduced[stddev_values_key] ||= []
-              reduced[stddev_values_key] << value
-            end
-          end
-
-          reduced["_NUM_SAMPLES__VALS"] ||= []
-          reduced["_NUM_SAMPLES__VALS"] << packet.read('_NUM_SAMPLES')
+          update_raw_hour_day_stats(reduced, state)
+          update_converted_hour_day_stats(packet, reduced, state)
         end
-
         state.first = false
+
+        throttle.throttle_sleep
       end
       file.delete # Remove the local copy
 
-      write_all_entries(reducer_state, plw, type, target_name, stored)
+      write_all_entries(reducer_state, plw, type, target_name, stored, throttle)
+
+      @logger.debug("Reducer Throttle: #{filename}: total_time: #{Time.now - throttle.reset_time}, sleep_time: #{throttle.total_sleep_time}")
 
       @count += 1
       @metric.set(name: 'reducer_total', value: @count, type: 'counter')
@@ -380,6 +284,140 @@ module OpenC3
       false
     end
 
+    def get_min_samples(packet, state)
+      state.entry_samples ||= packet.json_hash.dup # Grab all the samples from the first packet
+      if state.first
+        state.raw_values = packet.read_all(:RAW, nil, packet.read_all_names(:RAW)).select { |key, value| value.is_a?(Numeric) }
+        state.raw_keys ||= state.raw_values.keys
+        state.converted_values = packet.read_all(:CONVERTED, nil, packet.read_all_names(:CONVERTED)).select { |key, value| value.is_a?(Numeric) }
+        state.converted_keys ||= state.converted_values.keys
+      else
+        state.raw_values = packet.read_all(:RAW, nil, state.raw_keys).select { |key, value| value.is_a?(Numeric) }
+        state.converted_values = packet.read_all(:CONVERTED, nil, state.converted_keys).select { |key, value| value.is_a?(Numeric) }
+      end
+    end
+
+    def get_hour_day_samples(packet, state)
+      # Hour or Day
+      state.entry_samples ||= extract_entry_samples(packet)
+      if state.first
+        state.raw_max_values = packet.read_all(:RAW, :MAX, packet.read_all_names(:RAW, :MAX))
+        state.raw_keys = state.raw_max_values.keys
+        state.converted_max_values = packet.read_all(:CONVERTED, :MAX, packet.read_all_names(:CONVERTED, :MAX))
+        state.converted_keys = state.converted_max_values.keys
+      else
+        state.raw_max_values = packet.read_all(:RAW, :MAX, state.raw_keys)
+        state.converted_max_values = packet.read_all(:CONVERTED, :MAX, state.converted_keys)
+      end
+      state.raw_min_values = packet.read_all(:RAW, :MIN, state.raw_keys)
+      state.raw_avg_values = packet.read_all(:RAW, :AVG, state.raw_keys)
+      state.raw_stddev_values = packet.read_all(:RAW, :STDDEV, state.raw_keys)
+      state.converted_min_values = packet.read_all(:CONVERTED, :MIN, state.converted_keys)
+      state.converted_avg_values = packet.read_all(:CONVERTED, :AVG, state.converted_keys)
+      state.converted_stddev_values = packet.read_all(:CONVERTED, :STDDEV, state.converted_keys)
+    end
+
+    if RUBY_ENGINE != 'ruby' or ENV['OPENC3_NO_EXT']
+      def update_min_stats(reduced, state)
+        # Update statistics for this packet's raw values
+        state.raw_values.each do |key, value|
+          if value
+            vals_key = "#{key}__VALS"
+            reduced[vals_key] ||= []
+            reduced[vals_key] << value
+            n_key = "#{key}__N"
+            reduced[n_key] ||= value
+            reduced[n_key] = value if value < reduced[n_key]
+            x_key = "#{key}__X"
+            reduced[x_key] ||= value
+            reduced[x_key] = value if value > reduced[x_key]
+          end
+        end
+
+        # Update statistics for this packet's converted values
+        state.converted_values.each do |key, value|
+          if value
+            cvals_key = "#{key}__CVALS"
+            reduced[cvals_key] ||= []
+            reduced[cvals_key] << value
+            cn_key = "#{key}__CN"
+            reduced[cn_key] ||= value
+            reduced[cn_key] = value if value < reduced[cn_key]
+            cx_key = "#{key}__CX"
+            reduced[cx_key] ||= value
+            reduced[cx_key] = value if value > reduced[cx_key]
+          end
+        end
+      end
+    end
+
+    def update_raw_hour_day_stats(reduced, state)
+      # Update statistics for this packet's raw values
+      state.raw_max_values.each do |key, value|
+        if value
+          max_key = "#{key}__X"
+          reduced[max_key] ||= value
+          reduced[max_key] = value if value > reduced[max_key]
+        end
+      end
+      state.raw_min_values.each do |key, value|
+        if value
+          min_key = "#{key}__N"
+          reduced[min_key] ||= value
+          reduced[min_key] = value if value < reduced[min_key]
+        end
+      end
+      state.raw_avg_values.each do |key, value|
+        if value
+          avg_values_key = "#{key}__AVGVALS"
+          reduced[avg_values_key] ||= []
+          reduced[avg_values_key] << value
+        end
+      end
+      state.raw_stddev_values.each do |key, value|
+        if value
+          stddev_values_key = "#{key}__STDDEVVALS"
+          reduced[stddev_values_key] ||= []
+          reduced[stddev_values_key] << value
+        end
+      end
+    end
+
+    def update_converted_hour_day_stats(packet, reduced, state)
+      # Update statistics for this packet's converted values
+      state.converted_max_values.each do |key, value|
+        if value
+          max_key = "#{key}__CX"
+          reduced[max_key] ||= value
+          reduced[max_key] = value if value > reduced[max_key]
+        end
+      end
+      state.converted_min_values.each do |key, value|
+        if value
+          min_key = "#{key}__CN"
+          reduced[min_key] ||= value
+          reduced[min_key] = value if value < reduced[min_key]
+        end
+      end
+      state.converted_avg_values.each do |key, value|
+        if value
+          avg_values_key = "#{key}__CAVGVALS"
+          reduced[avg_values_key] ||= []
+          reduced[avg_values_key] << value
+        end
+      end
+      state.converted_stddev_values.each do |key, value|
+        if value
+          stddev_values_key = "#{key}__CSTDDEVVALS"
+          reduced[stddev_values_key] ||= []
+          reduced[stddev_values_key] << value
+        end
+      end
+
+      reduced["_NUM_SAMPLES__VALS"] ||= []
+      reduced["_NUM_SAMPLES__VALS"] << packet.read('_NUM_SAMPLES')
+    end
+
     def check_new_file(reducer_state, plw, type, target_name, stored, current_time, file_nanoseconds)
       plw_first_time_nsec = plw.buffered_first_time_nsec
       if plw_first_time_nsec && ((current_time - plw_first_time_nsec) >= file_nanoseconds)
@@ -408,9 +446,10 @@ module OpenC3
       return state
     end
 
-    def write_all_entries(reducer_state, plw, type, target_name, stored)
+    def write_all_entries(reducer_state, plw, type, target_name, stored, throttle = nil)
       reducer_state.each do |packet_name, state|
         write_entry(state, plw, type, target_name, packet_name, stored)
+        throttle.throttle_sleep if throttle
       end
     end
 
@@ -484,26 +523,31 @@ module OpenC3
       # Calculate Average
       weighted_sum = 0
       avg = reduced["#{key}#{avgvals_key}"]
-      avg.each_with_index do |val, i|
-        weighted_sum += (val * samples[i])
+      if avg
+        avg.each_with_index do |val, i|
+          weighted_sum += (val * samples[i])
+        end
+        reduced["#{key}#{avg_key}"] = weighted_sum / samples_sum
       end
-      reduced["#{key}#{avg_key}"] = weighted_sum / samples_sum
 
       # Do the STDDEV calc last so we can use the previously calculated AVG
       # See https://math.stackexchange.com/questions/1547141/aggregating-standard-deviation-to-a-summary-point
       s2 = 0
-      reduced["#{key}#{stddevvals_key}"].each_with_index do |val, i|
-        # puts "i:#{i} val:#{val} samples[i]:#{samples[i]} avg[i]:#{avg[i]}"
-        s2 += (samples[i] * avg[i]**2 + val**2)
-      end
-
-      # Note: For very large numbers with very small deviations this sqrt can fail.
-      # If so then just set the stddev to 0.
-      begin
-        reduced["#{key}#{stddev_key}"] =
-          Math.sqrt(s2 / samples_sum - reduced["#{key}#{avg_key}"])
-      rescue Exception
-        reduced["#{key}#{stddev_key}"] = 0.0
+      stddev = reduced["#{key}#{stddevvals_key}"]
+      if stddev
+        stddev.each_with_index do |val, i|
+          # puts "i:#{i} val:#{val} samples[i]:#{samples[i]} avg[i]:#{avg[i]}"
+          s2 += (samples[i] * avg[i]**2 + val**2)
+        end
+
+        # Note: For very large numbers with very small deviations this sqrt can fail.
+        # If so then just set the stddev to 0.
+        begin
+          reduced["#{key}#{stddev_key}"] =
+            Math.sqrt(s2 / samples_sum - reduced["#{key}#{avg_key}"])
+        rescue Exception
+          reduced["#{key}#{stddev_key}"] = 0.0
+        end
       end
 
       reduced.delete("#{key}#{avgvals_key}")

data/lib/openc3/{models/notification_model.rb → microservices/scope_cleanup_microservice.rb}

@@ -1,6 +1,6 @@
 # encoding: ascii-8bit
 
-# Copyright 2022 Ball Aerospace & Technologies Corp.
+# Copyright 2023 OpenC3, Inc.
 # All Rights Reserved.
 #
 # This program is free software; you can modify and/or redistribute it
@@ -12,32 +12,31 @@
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU Affero General Public License for more details.
-
-# Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
-# All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
+require 'openc3/models/scope_model'
+require 'openc3/microservices/cleanup_microservice'
+
 module OpenC3
-  class NotificationModel
-    attr_reader :time, :severity, :url, :title, :body
+  class ScopeCleanupMicroservice < CleanupMicroservice
+    def run
+      scope = ScopeModel.get_model(name: @scope, scope: @scope)
 
-    def initialize(time:, severity:, url:, title:, body:)
-      @time = time
-      @severity = severity
-      @url = url
-      @title = title
-      @body = body
-    end
+      areas = [
+        ["#{@scope}/text_logs", scope.text_log_retain_time],
+        ["#{@scope}/tool_logs", scope.tool_log_retain_time],
+      ]
 
-    def as_json(*a)
-      { "time" => @time,
-        "severity" => @severity,
-        "url" => @url,
-        "title" => @title,
-        "body" => @body }
+      if @scope == 'DEFAULT'
+        areas << ["NOSCOPE/text_logs", scope.text_log_retain_time]
+        areas << ["NOSCOPE/tool_logs", scope.tool_log_retain_time]
+      end
+
+      cleanup(areas, scope.cleanup_poll_time)
     end
   end
 end
+
+OpenC3::CleanupMicroservice.run if __FILE__ == $0

data/lib/openc3/microservices/text_log_microservice.rb

@@ -22,6 +22,7 @@
 
 require 'openc3/microservices/microservice'
 require 'openc3/topics/topic'
+require 'openc3/io/json_rpc'
 
 module OpenC3
   class TextLogMicroservice < Microservice
@@ -77,11 +78,7 @@ module OpenC3
       msgid_seconds_from_epoch = msg_id.split('-')[0].to_i / 1000.0
       delta = Time.now.to_f - msgid_seconds_from_epoch
       @metric.set(name: 'text_log_topic_delta_seconds', value: delta, type: 'gauge', unit: 'seconds', help: 'Delta time between data written to stream and text log start')
-
-      keys = msg_hash.keys
-      keys.delete("time")
-      entry = keys.reduce("") { |data, key| data + "#{key}: #{msg_hash[key]}\t" }
-      @tlws[topic].write(msg_hash["time"].to_i, entry, topic, msg_id)
+      @tlws[topic].write(msg_hash["time"].to_i, msg_hash.as_json(allow_nan: true).to_json(allow_nan: true), topic, msg_id)
       @count += 1
     rescue => err
       @error = err

data/lib/openc3/microservices/timeline_microservice.rb

@@ -23,7 +23,6 @@
 require 'openc3/utilities/authentication'
 require 'openc3/microservices/microservice'
 require 'openc3/models/activity_model'
-require 'openc3/models/notification_model'
 require 'openc3/models/timeline_model'
 require 'openc3/topics/timeline_topic'
 

data/lib/openc3/microservices/trigger_group_microservice.rb

@@ -21,7 +21,6 @@
 # if purchased from OpenC3, Inc.
 
 require 'openc3/microservices/microservice'
-require 'openc3/models/notification_model'
 require 'openc3/models/trigger_model'
 require 'openc3/topics/autonomic_topic'
 require 'openc3/utilities/authentication'