openc3 5.17.1 → 5.19.0

data/lib/openc3/logs/log_writer.rb

@@ -20,13 +20,12 @@
 # This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
-require 'thread'
 require 'openc3/config/config_parser'
 require 'openc3/topics/topic'
 require 'openc3/utilities/bucket_utilities'
 
 module OpenC3
-  # Creates a log. Can automatically cycle the log based on an elasped
+  # Creates a log. Can automatically cycle the log based on an elapsed
   # time period or when the log file reaches a predefined size.
   class LogWriter
     # @return [String] The filename of the packet log
@@ -275,10 +274,10 @@ module OpenC3
       @last_time = nil
       @previous_time_nsec_since_epoch = nil
       Logger.debug "Log File Opened : #{@filename}"
-    rescue => err
-      Logger.error "Error starting new log file: #{err.formatted}"
+    rescue => e
+      Logger.error "Error starting new log file: #{e.formatted}"
       @logging_enabled = false
-      OpenC3.handle_critical_exception(err)
+      OpenC3.handle_critical_exception(e)
     end
 
     # @enforce_time_order requires the timestamps on each write to be greater than the previous
@@ -328,10 +327,10 @@ module OpenC3
             @last_offsets.each do |redis_topic, last_offset|
               @cleanup_offsets[-1][redis_topic] = last_offset
             end
-            @cleanup_times << Time.now + CLEANUP_DELAY
+            @cleanup_times << (Time.now + CLEANUP_DELAY)
             @last_offsets.clear
-          rescue Exception => err
-            Logger.error "Error closing #{@filename} : #{err.formatted}"
+          rescue Exception => e
+            Logger.error "Error closing #{@filename} : #{e.formatted}"
           end
 
           @file = nil

data/lib/openc3/logs/packet_log_writer.rb

@@ -26,7 +26,7 @@ require 'openc3/models/target_model'
 require 'cbor'
 
 module OpenC3
-  # Creates a packet log. Can automatically cycle the log based on an elasped
+  # Creates a packet log. Can automatically cycle the log based on an elapsed
   # time period or when the log file reaches a predefined size.
   class PacketLogWriter < LogWriter
     include PacketLogConstants
@@ -112,9 +112,9 @@ module OpenC3
       ensure
         @mutex.unlock if take_mutex
       end
-    rescue => err
-      Logger.instance.error "Error writing #{@filename} : #{err.formatted}"
-      OpenC3.handle_critical_exception(err)
+    rescue => e
+      Logger.instance.error "Error writing #{@filename} : #{e.formatted}"
+      OpenC3.handle_critical_exception(e)
     end
 
     # Starting a new file is a critical operation so the entire method is
@@ -133,10 +133,10 @@ module OpenC3
       @next_target_index = 0
       @target_dec_entries = []
       @packet_dec_entries = []
-    rescue => err
-      Logger.error "Error starting new log file: #{err.formatted}"
+    rescue => e
+      Logger.error "Error starting new log file: #{e.formatted}"
       @logging_enabled = false
-      OpenC3.handle_critical_exception(err)
+      OpenC3.handle_critical_exception(e)
     end
 
     # Closing a log file isn't critical so we just log an error

data/lib/openc3/logs/text_log_writer.rb

@@ -24,7 +24,7 @@ require 'openc3/logs/log_writer'
 require 'socket'
 
 module OpenC3
-  # Creates a text log. Can automatically cycle the log based on an elasped
+  # Creates a text log. Can automatically cycle the log based on an elapsed
   # time period or when the log file reaches a predefined size.
   class TextLogWriter < LogWriter
     NEWLINE = "\n".freeze
@@ -48,9 +48,9 @@ module OpenC3
         prepare_write(time_nsec_since_epoch, data.length, redis_topic, redis_offset)
         write_entry(time_nsec_since_epoch, data) if @file
       end
-    rescue => err
-      Logger.instance.error "Error writing #{@filename} : #{err.formatted}"
-      OpenC3.handle_critical_exception(err)
+    rescue => e
+      Logger.instance.error "Error writing #{@filename} : #{e.formatted}"
+      OpenC3.handle_critical_exception(e)
     end
 
     def write_entry(time_nsec_since_epoch, data)

data/lib/openc3/microservices/decom_microservice.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
+# All changes Copyright 2024, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -102,9 +102,22 @@ module OpenC3
           packet.extra = extra
         end
         packet.buffer = msg_hash["buffer"]
-        packet.process # Run processors
-        packet.check_limits(System.limits_set) # Process all the limits and call the limits_change_callback (as necessary)
+        # Processors are user code points which must be rescued
+        # so the TelemetryDecomTopic can write the packet
+        begin
+          packet.process # Run processors
+        rescue Exception => e
+          @error_count += 1
+          @metric.set(name: 'decom_error_total', value: @error_count, type: 'counter')
+          @error = e
+          @logger.error e.message
+        end
+        # Process all the limits and call the limits_change_callback (as necessary)
+        # check_limits also can call user code in the limits response
+        # but that is rescued separately in the limits_change_callback
+        packet.check_limits(System.limits_set)
 
+        # This is what updates the CVT
         TelemetryDecomTopic.write_packet(packet, scope: @scope)
         diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
         @metric.set(name: 'decom_duration_seconds', value: diff, type: 'gauge', unit: 'seconds')
@@ -150,12 +163,14 @@ module OpenC3
 
       if item.limits.response
         begin
+          # TODO: The limits response is user code and should be run as a separate thread / process
+          # If this code blocks it will delay TelemetryDecomTopic.write_packet
           item.limits.response.call(packet, item, old_limits_state)
         rescue Exception => e
           @error = e
           @logger.error "#{packet.target_name} #{packet.packet_name} #{item.name} Limits Response Exception!"
           @logger.error "Called with old_state = #{old_limits_state}, new_state = #{item.limits.state}"
-          @logger.error e.formatted
+          @logger.error e.filtered
         end
       end
     end

data/lib/openc3/microservices/interface_microservice.rb

@@ -31,6 +31,7 @@ require 'openc3/topics/command_topic'
 require 'openc3/topics/command_decom_topic'
 require 'openc3/topics/interface_topic'
 require 'openc3/topics/router_topic'
+require 'openc3/interfaces/interface'
 
 module OpenC3
   class InterfaceCmdHandlerThread
@@ -191,22 +192,59 @@ module OpenC3
               next e.message
             end
 
+            command.extra ||= {}
+            command.extra['cmd_string'] = msg_hash['cmd_string']
+            command.extra['username'] = msg_hash['username']
             if hazardous_check
               hazardous, hazardous_description = System.commands.cmd_pkt_hazardous?(command)
               # Return back the error, description, and the formatted command
               # This allows the error handler to simply re-send the command
-              next "HazardousError\n#{hazardous_description}\n#{System.commands.format(command)}" if hazardous
+              next "HazardousError\n#{hazardous_description}\n#{msg_hash['cmd_string']}" if hazardous
             end
 
+            validate = ConfigParser.handle_true_false(msg_hash['validate'])
             begin
               if @interface.connected?
+                result = true
+                reason = nil
+                if command.validator and validate
+                  begin
+                    result, reason = command.validator.pre_check(command)
+                  rescue => e
+                    result = false
+                    reason = e.message
+                  end
+                  # Explicitly check for false to allow nil to represent unknown
+                  if result == false
+                    message = "pre_check returned false for #{msg_hash['cmd_string']} due to #{reason}"
+                    raise WriteRejectError.new(message)
+                  end
+                end
+
                 @count += 1
                 @metric.set(name: 'interface_cmd_total', value: @count, type: 'counter') if @metric
-
                 @interface.write(command)
-                CommandTopic.write_packet(command, scope: @scope)
+
+                if command.validator and validate
+                  begin
+                    result, reason = command.validator.post_check(command)
+                  rescue => e
+                    result = false
+                    reason = e.message
+                  end
+                  command.extra['cmd_success'] = result
+                  command.extra['cmd_reason'] = reason if reason
+                end
+
                 CommandDecomTopic.write_packet(command, scope: @scope)
+                CommandTopic.write_packet(command, scope: @scope)
                 InterfaceStatusModel.set(@interface.as_json(:allow_nan => true), queued: true, scope: @scope)
+
+                # Explicitly check for false to allow nil to represent unknown
+                if result == false
+                  message = "post_check returned false for #{msg_hash['cmd_string']} due to #{reason}"
+                  raise WriteRejectError.new(message)
+                end
                 next 'SUCCESS'
               else
                 next "Interface not connected: #{@interface.name}"

data/lib/openc3/microservices/microservice.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2023, OpenC3, Inc.
+# All changes Copyright 2024, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -54,13 +54,13 @@ module OpenC3
         microservice.state = 'RUNNING'
         microservice.run
         microservice.state = 'FINISHED'
-      rescue Exception => err
-        if SystemExit === err or SignalException === err
+      rescue Exception => e
+        if SystemExit === e or SignalException === e
           microservice.state = 'KILLED'
         else
-          microservice.error = err
+          microservice.error = e
           microservice.state = 'DIED_ERROR'
-          Logger.fatal("Microservice #{name} dying from exception\n#{err.formatted}")
+          Logger.fatal("Microservice #{name} dying from exception\n#{e.formatted}")
         end
       ensure
         MicroserviceStatusModel.set(microservice.as_json(:allow_nan => true), scope: microservice.scope)
@@ -171,7 +171,7 @@ module OpenC3
               # Run ruby syntax so we can log those
               syntax_check, _ = Open3.capture2e("ruby -c #{ruby_filename}")
               if /Syntax OK/.match?(syntax_check)
-                @logger.info("Ruby microservice #{@name} file #{ruby_filename} passed syntax check\n", scope: @scope)
+                @logger.debug("Ruby microservice #{@name} file #{ruby_filename} passed syntax check\n", scope: @scope)
               else
                 @logger.error("Ruby microservice #{@name} file #{ruby_filename} failed syntax check\n#{syntax_check}", scope: @scope)
               end
@@ -188,9 +188,9 @@ module OpenC3
             MicroserviceStatusModel.set(as_json(:allow_nan => true), scope: @scope) unless @cancel_thread
             break if @microservice_status_sleeper.sleep(@microservice_status_period_seconds)
           end
-        rescue Exception => err
-          @logger.error "#{@name} status thread died: #{err.formatted}"
-          raise err
+        rescue Exception => e
+          @logger.error "#{@name} status thread died: #{e.formatted}"
+          raise e
         end
       end
     end
@@ -208,7 +208,7 @@ module OpenC3
       MicroserviceStatusModel.set(as_json(:allow_nan => true), scope: @scope)
       FileUtils.remove_entry(@temp_dir) if File.exist?(@temp_dir)
       @metric.shutdown
-      @logger.info("Shutting down microservice complete: #{@name}")
+      @logger.debug("Shutting down microservice complete: #{@name}")
       @shutdown_complete = true
     end
 
@@ -220,7 +220,7 @@ module OpenC3
     end
 
     # Returns if the command was handled
-    def microservice_cmd(topic, msg_id, msg_hash, redis)
+    def microservice_cmd(topic, msg_id, msg_hash, _redis)
       command = msg_hash['command']
       case command
       when 'ADD_TOPICS'

data/lib/openc3/microservices/reaction_microservice.rb

@@ -368,7 +368,7 @@ module OpenC3
   end
 
   # The reaction snooze manager starts a thread pool and keeps track of when a
-  # reaction is activated and to evalute triggers when the snooze is complete.
+  # reaction is activated and to evaluate triggers when the snooze is complete.
   class ReactionSnoozeManager
     attr_reader :name, :scope, :share, :thread_pool
 
@@ -432,7 +432,7 @@ module OpenC3
 
     def shutdown
       @cancel_thread = true
-      @worker_count.times do |i|
+      @worker_count.times do |_i|
         @share.queue_base.enqueue(kind: nil, data: nil)
       end
     end

data/lib/openc3/microservices/scope_cleanup_microservice.rb

@@ -22,7 +22,7 @@ require 'openc3/microservices/cleanup_microservice'
 module OpenC3
   class ScopeCleanupMicroservice < CleanupMicroservice
     def run
-      scope = ScopeModel.get_model(name: @scope, scope: @scope)
+      scope = ScopeModel.get_model(name: @scope)
 
       areas = [
         ["#{@scope}/text_logs", scope.text_log_retain_time],

data/lib/openc3/microservices/timeline_microservice.rb

@@ -24,6 +24,7 @@ require 'openc3/utilities/authentication'
 require 'openc3/microservices/microservice'
 require 'openc3/models/activity_model'
 require 'openc3/models/timeline_model'
+require 'openc3/models/tool_config_model'
 require 'openc3/topics/timeline_topic'
 
 require 'openc3/script'
@@ -82,45 +83,66 @@ module OpenC3
       end
     end
 
+    def get_exec_setting()
+      json = ToolConfigModel.load_config('calendar-settings', 'default', scope: @scope)
+      if json
+        settings = JSON.parse(json)
+        return settings['execEnabled']
+      else
+        # Default is execute
+        return true
+      end
+    end
+
     def run_command(activity)
       @logger.info "#{@timeline_name} run_command > #{activity.as_json(:allow_nan => true)}"
       begin
-        username = activity.data['username']
-        token = get_token(username)
-        raise "No token available for username: #{username}" unless token
-        cmd_no_hazardous_check(activity.data['command'], scope: @scope, token: token)
-        activity.commit(status: 'completed', fulfillment: true)
+        if get_exec_setting()
+          username = activity.data['username']
+          token = get_token(username)
+          raise "No token available for username: #{username}" unless token
+          cmd_no_hazardous_check(activity.data['command'], scope: @scope, token: token)
+          activity.commit(status: 'completed', fulfillment: true)
+        else
+          activity.commit(status: 'disabled', message: 'Execution is disabled')
+          @logger.warn "#{@timeline_name} run_command disabled > #{activity.as_json(:allow_nan => true)}"
+        end
       rescue StandardError => e
         activity.commit(status: 'failed', message: e.message)
-        @logger.error "#{@timeline_name} run_cmd failed > #{activity.as_json(:allow_nan => true)}, #{e.formatted}"
+        @logger.error "#{@timeline_name} run_command failed > #{activity.as_json(:allow_nan => true)}, #{e.formatted}"
       end
     end
 
     def run_script(activity)
       @logger.info "#{@timeline_name} run_script > #{activity.as_json(:allow_nan => true)}"
       begin
-        username = activity.data['username']
-        token = get_token(username)
-        raise "No token available for username: #{username}" unless token
-        request = Net::HTTP::Post.new(
-          "/script-api/scripts/#{activity.data['script']}/run?scope=#{@scope}",
-          'Content-Type' => 'application/json',
-          'Authorization' => token
-        )
-        request.body = JSON.generate({
-          'scope' => @scope,
-          'environment' => activity.data['environment'],
-          'timeline' => @timeline_name,
-          'id' => activity.start
-        })
-        hostname = ENV['OPENC3_SCRIPT_HOSTNAME'] || 'openc3-cosmos-script-runner-api'
-        response = Net::HTTP.new(hostname, 2902).request(request)
-        raise "failed to call #{hostname}, for script: #{activity.data['script']}, response code: #{response.code}" if response.code != '200'
-
-        activity.commit(status: 'completed', message: "#{activity.data['script']} => #{response.body}", fulfillment: true)
+        if get_exec_setting()
+          username = activity.data['username']
+          token = get_token(username)
+          raise "No token available for username: #{username}" unless token
+          request = Net::HTTP::Post.new(
+            "/script-api/scripts/#{activity.data['script']}/run?scope=#{@scope}",
+            'Content-Type' => 'application/json',
+            'Authorization' => token
+          )
+          request.body = JSON.generate({
+            'scope' => @scope,
+            'environment' => activity.data['environment'],
+            'timeline' => @timeline_name,
+            'id' => activity.start
+          })
+          hostname = ENV['OPENC3_SCRIPT_HOSTNAME'] || 'openc3-cosmos-script-runner-api'
+          response = Net::HTTP.new(hostname, 2902).request(request)
+          raise "failed to call #{hostname}, for script: #{activity.data['script']}, response code: #{response.code}" if response.code != '200'
+
+          activity.commit(status: 'completed', message: "#{activity.data['script']} => #{response.body}", fulfillment: true)
+        else
+          activity.commit(status: 'disabled', message: 'Execution is disabled')
+          @logger.warn "#{@timeline_name} run_script disabled > #{activity.as_json(:allow_nan => true)}"
+        end
       rescue StandardError => e
         activity.commit(status: 'failed', message: e.message)
-        @logger.error "#{@timeline_name} run_script failed > #{activity.as_json(:allow_nan => true).to_s}, #{e.message}"
+        @logger.error "#{@timeline_name} run_script failed > #{activity.as_json(:allow_nan => true)}, #{e.message}"
       end
     end
 
@@ -212,7 +234,7 @@ module OpenC3
     def run
       @logger.info "#{@timeline_name} timeline manager running"
       loop do
-        start = Time.now.to_i
+        start = Time.now.to_f
         @schedule.activities.each do |activity|
           start_difference = activity.start - start
           if start_difference <= 0 && @schedule.not_queued?(activity.start)
@@ -230,18 +252,18 @@ module OpenC3
         sleep(1)
         break if @cancel_thread
       end
-      @logger.info "#{@timeline_name} timeine manager exiting"
+      @logger.info "#{@timeline_name} timeline manager exiting"
     end
 
     # Add task to remove events older than 7 days
     def add_expire_activity
-      now = Time.now.to_i
-      @expire = now + 3_000
+      now = Time.now.to_f
+      @expire = now + 3540 # Needs to be less than 3600 which is the hour we store in memory
       activity = ActivityModel.new(
         name: @timeline_name,
         scope: @scope,
         start: 0,
-        stop: (now - 86_400 * 7),
+        stop: (now - (86_400 * 7)),
         kind: 'expire',
         data: {}
       )
@@ -283,13 +305,13 @@ module OpenC3
       super(name)
       @timeline_name = name.split('__')[2]
       @schedule = Schedule.new(@timeline_name)
-      @manager = TimelineManager.new(name: @timeline_name, logger: @logger, scope: scope, schedule: @schedule)
+      @manager = TimelineManager.new(name: @timeline_name, logger: @logger, scope: @scope, schedule: @schedule)
       @manager_thread = nil
       @read_topic = true
     end
 
     def run
-      @logger.info "#{@name} timeine running"
+      @logger.info "#{@name} timeline running"
       @manager_thread = Thread.new { @manager.run }
       loop do
         current_activities = ActivityModel.activities(name: @timeline_name, scope: @scope)
@@ -299,19 +321,19 @@ module OpenC3
         block_for_updates()
         break if @cancel_thread
       end
-      @logger.info "#{@name} timeine exitting"
+      @logger.info "#{@name} timeline exiting"
     end
 
     def topic_lookup_functions
       {
         'timeline' => {
-          'created' => :timeline_nop,
+          'created' => :timeline_noop,
           'refresh' => :schedule_refresh,
-          'updated' => :timeline_nop,
-          'deleted' => :timeline_nop
+          'updated' => :timeline_noop,
+          'deleted' => :timeline_noop
         },
         'activity' => {
-          'event' => :timeline_nop,
+          'event' => :timeline_noop,
           'created' => :create_activity_from_event,
           'updated' => :schedule_refresh,
           'deleted' => :remove_activity_from_event
@@ -335,7 +357,7 @@ module OpenC3
       end
     end
 
-    def timeline_nop(data)
+    def timeline_noop(data)
       @logger.debug "#{@name} timeline web socket event: #{data}"
     end
 
@@ -347,8 +369,8 @@ module OpenC3
     # Add the activity to the schedule. We don't need to hold the job in memory
     # if it is longer than an hour away. A refresh task will update that.
     def create_activity_from_event(data)
-      diff = data['start'] - Time.now.to_i
-      return unless (2..3600).include? diff
+      diff = data['start'] - Time.now.to_f
+      return if diff < 0 or diff > 3600
 
       activity = ActivityModel.from_json(data, name: @timeline_name, scope: @scope)
       @schedule.add_activity(activity)
@@ -357,17 +379,20 @@ module OpenC3
     # Remove the activity from the schedule. We don't need to remove the activity
     # if it is longer than an hour away. It will be removed from the data.
     def remove_activity_from_event(data)
-      diff = data['start'] - Time.now.to_i
-      return unless (2..3600).include? diff
+      diff = data['start'] - Time.now.to_f
+      return if diff < 0 or diff > 3600
 
       activity = ActivityModel.from_json(data, name: @timeline_name, scope: @scope)
       @schedule.remove_activity(activity)
     end
 
     def shutdown
-      @read_topic = false
       @manager.shutdown
-      super
+      # super also sets @cancel_thread = true but we want to set it first
+      # so when we set @read_topic = false the run loop stops
+      @cancel_thread = true
+      @read_topic = false
+      super()
     end
   end
 end

data/lib/openc3/microservices/trigger_group_microservice.rb

@@ -445,7 +445,7 @@ module OpenC3
         visited["#{trigger.name}__P"] = Hash.new
       end
       if visited["#{head.name}__P"][trigger.name]
-        # Not sure if this is posible as on create it validates that the dependents are already created
+        # Not sure if this is possible as on create it validates that the dependents are already created
         message = "loop detected from #{head.name} -> #{trigger.name} path: #{visited["#{head.name}__P"]}"
         notify(name: trigger.name, severity: 'error', message: message)
         return visited["#{trigger.name}__R"] = -1
@@ -494,7 +494,7 @@ module OpenC3
   end
 
   # The trigger manager starts a thread pool and subscribes
-  # to the telemtry decom topic. It adds the "packet" to the thread pool queue
+  # to the telemetry decom topic. It adds the "packet" to the thread pool queue
   # and the thread will evaluate the "trigger".
   class TriggerGroupManager
     attr_reader :name, :scope, :share, :group, :topics, :thread_pool
@@ -621,7 +621,7 @@ module OpenC3
       loop do
         triggers = TriggerModel.all(scope: @scope, group: @group)
         @share.trigger_base.rebuild(triggers: triggers)
-        @manager.refresh() # Everytime we do a full base update we refesh the manager
+        @manager.refresh() # Every time we do a full base update we refresh the manager
         break if @cancel_thread
         block_for_updates()
         break if @cancel_thread

data/lib/openc3/migrations/20240915000000_activity_uuid.rb

@@ -0,0 +1,28 @@
+require 'openc3/utilities/migration'
+require 'openc3/models/scope_model'
+require 'openc3/models/timeline_model'
+
+module OpenC3
+  class ActivityUuid < Migration
+    def self.run
+      ScopeModel.names.each do |scope|
+        TimelineModel.names.each do |key|
+          name = key.split('__').last
+          json = Store.zrange("#{scope}#{ActivityModel::PRIMARY_KEY}__#{name}", 0, -1)
+          parsed = json.map { |value| JSON.parse(value, :allow_nan => true, :create_additions => true) }
+          parsed.each_with_index do |activity, index|
+            if activity['uuid'].nil?
+              activity['uuid'] = SecureRandom.uuid
+              Store.zrem("#{scope}#{ActivityModel::PRIMARY_KEY}__#{name}", json[index])
+              Store.zadd("#{scope}#{ActivityModel::PRIMARY_KEY}__#{name}", activity['start'], JSON.generate(activity))
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+unless ENV['OPENC3_NO_MIGRATE']
+  OpenC3::ActivityUuid.run
+end