openc3 6.2.1 → 6.4.0

data/lib/openc3/logs/packet_log_reader.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2024, OpenC3, Inc.
+# All changes Copyright 2025, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -309,7 +309,7 @@ module OpenC3
         if cbor
           extra = CBOR.decode(extra_encoded)
         else
-          extra = JSON.parse(extra_encode, allow_nan: true, create_additions: true)
+          extra = JSON.parse(extra_encoded, allow_nan: true, create_additions: true)
         end
       end
       data = entry[next_offset..-1]

data/lib/openc3/logs/packet_log_writer.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
+# All changes Copyright 2025, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license

data/lib/openc3/microservices/interface_decom_common.rb

@@ -15,6 +15,9 @@
 #
 # This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
+#
+# A portion of this file was funded by Blue Origin Enterprises, L.P.
+# See https://github.com/OpenC3/cosmos/pull/1963
 
 require 'openc3/topics/telemetry_topic'
 require 'openc3/system/system'
@@ -33,8 +36,8 @@ module OpenC3
           packet.write(name.to_s, value, type)
         end
       end
-      packet.received_count += 1
       packet.received_time = Time.now.sys
+      packet.received_count = TargetModel.increment_telemetry_count(packet.target_name, packet.packet_name, 1, scope: @scope)
       TelemetryTopic.write_packet(packet, scope: @scope)
     # If the inject_tlm parameters are bad we rescue so
     # interface_microservice and decom_microservice can continue

data/lib/openc3/microservices/interface_microservice.rb

@@ -19,6 +19,9 @@
 #
 # This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
+#
+# A portion of this file was funded by Blue Origin Enterprises, L.P.
+# See https://github.com/OpenC3/cosmos/pull/1963
 
 require 'openc3/microservices/microservice'
 require 'openc3/microservices/interface_decom_common'
@@ -37,7 +40,7 @@ require 'openc3/interfaces/interface'
 begin
   require 'openc3-enterprise/models/critical_cmd_model'
 rescue LoadError
-  # Should never actual be used in Open Source
+  # LoadError expected in COSMOS Core
 end
 
 module OpenC3
@@ -128,7 +131,7 @@ module OpenC3
                 @logger.info "#{@interface.name}: Write raw"
                 # A raw interface write results in an UNKNOWN packet
                 command = System.commands.packet('UNKNOWN', 'UNKNOWN')
-                command.received_count += 1
+                command.received_count = TargetModel.increment_command_count('UNKNOWN', 'UNKNOWN', 1, scope: @scope)
                 command = command.clone
                 command.buffer = msg_hash['raw']
                 command.received_time = Time.now
@@ -215,14 +218,15 @@ module OpenC3
                   command = System.commands.identify(cmd_buffer, @interface.cmd_target_names)
                 end
                 unless command
-                  command = System.commands.packet('UNKNOWN', 'UNKNOWN')
-                  command.received_count += 1
-                  command = command.clone
+                  command = System.commands.packet('UNKNOWN', 'UNKNOWN').clone
                   command.buffer = cmd_buffer
                 end
               else
                 raise "Invalid command received:\n #{msg_hash}"
               end
+              orig_command = System.commands.packet(command.target_name, command.packet_name)
+              orig_command.received_count = TargetModel.increment_command_count(command.target_name, command.packet_name, 1, scope: @scope)
+              command.received_count = orig_command.received_count
               command.received_time = Time.now
             rescue => e
               @logger.error "#{@interface.name}: #{msg_hash}"
@@ -504,6 +508,9 @@ module OpenC3
       end
 
       @queued = false
+      @sync_packet_count_data = {}
+      @sync_packet_count_time = nil
+      @sync_packet_count_delay_seconds = 1.0 # Sync packet counts every second
       @interface.options.each do |option_name, option_values|
         if option_name.upcase == 'OPTIMIZE_THROUGHPUT'
           @queued = true
@@ -511,6 +518,9 @@ module OpenC3
           EphemeralStoreQueued.instance.set_update_interval(update_interval)
           StoreQueued.instance.set_update_interval(update_interval)
         end
+        if option_name.upcase == 'SYNC_PACKET_COUNT_DELAY_SECONDS'
+          @sync_packet_count_delay_seconds = option_values[0].to_f
+        end
       end
 
       @interface_thread_sleeper = Sleeper.new
@@ -678,7 +688,7 @@ module OpenC3
       end
 
       # Write to stream
-      packet.received_count += 1
+      sync_tlm_packet_counts(packet)
       TelemetryTopic.write_packet(packet, queued: @queued, scope: @scope)
     end
 
@@ -812,6 +822,59 @@ module OpenC3
     def graceful_kill
       # Just to avoid warning
     end
+
+    def sync_tlm_packet_counts(packet)
+      if @sync_packet_count_delay_seconds <= 0
+        # Perfect but slow method
+        packet.received_count = TargetModel.increment_telemetry_count(packet.target_name, packet.packet_name, 1, scope: @scope)
+      else
+        # Eventually consistent method
+        # Only sync every period (default 1 second) to avoid hammering Redis
+        # This is a trade off between speed and accuracy
+        # The packet count is eventually consistent
+        @sync_packet_count_data[packet.target_name] ||= {}
+        @sync_packet_count_data[packet.target_name][packet.packet_name] ||= 0
+        @sync_packet_count_data[packet.target_name][packet.packet_name] += 1
+
+        # Ensures counters change between syncs
+        packet.received_count += 1
+
+        # Check if we need to sync the packet counts
+        if @sync_packet_count_time.nil? or (Time.now - @sync_packet_count_time) > @sync_packet_count_delay_seconds
+          @sync_packet_count_time = Time.now
+
+          inc_count = 0
+          # Use pipeline to make this one transaction
+          result = Store.redis_pool.pipelined do
+            # Increment global counters for packets received
+            @sync_packet_count_data.each do |target_name, packet_data|
+              packet_data.each do |packet_name, count|
+                TargetModel.increment_telemetry_count(target_name, packet_name, count, scope: @scope)
+                inc_count += 1
+              end
+            end
+            @sync_packet_count_data = {}
+
+            # Get all the packet counts with the global counters
+            @interface.tlm_target_names.each do |target_name|
+              TargetModel.get_all_telemetry_counts(target_name, scope: @scope)
+            end
+            TargetModel.get_all_telemetry_counts('UNKNOWN', scope: @scope)
+          end
+          @interface.tlm_target_names.each do |target_name|
+            result[inc_count].each do |packet_name, count|
+              update_packet = System.telemetry.packet(target_name, packet_name)
+              update_packet.received_count = count.to_i
+            end
+            inc_count += 1
+          end
+          result[inc_count].each do |packet_name, count|
+            update_packet = System.telemetry.packet('UNKNOWN', packet_name)
+            update_packet.received_count = count.to_i
+          end
+        end
+      end
+    end
   end
 end
 

data/lib/openc3/microservices/microservice.rb

@@ -64,6 +64,7 @@ module OpenC3
             microservice.state = 'DIED_ERROR'
             Logger.fatal("Microservice #{name} dying from exception\n#{e.formatted}")
           end
+          microservice.shutdown # Dying in crash so should try to shutdown
         ensure
           MicroserviceStatusModel.set(microservice.as_json(:allow_nan => true), scope: microservice.scope)
         end
@@ -207,9 +208,10 @@ module OpenC3
       shutdown()
     end
 
-    def shutdown
+    def shutdown(state = 'STOPPED')
       return if @shutdown_complete
       @logger.info("Shutting down microservice: #{@name}")
+      @state = state
       @cancel_thread = true
       @microservice_status_sleeper.cancel if @microservice_status_sleeper
       MicroserviceStatusModel.set(as_json(:allow_nan => true), scope: @scope)

data/lib/openc3/microservices/multi_microservice.rb

@@ -25,7 +25,7 @@ module OpenC3
     def run
       ARGV.each do |microservice_name|
         microservice_model = MicroserviceModel.get_model(name: microservice_name, scope: @scope)
-        if microservice_model.enabled
+        if microservice_model and microservice_model.enabled
           thread = Thread.new do
             cmd_line = microservice_model.cmd.join(' ')
             split_cmd_line = cmd_line.split(' ')

data/lib/openc3/migrations/20250402000000_periodic_only_default.rb

@@ -0,0 +1,24 @@
+require 'openc3/utilities/migration'
+require 'openc3/models/scope_model'
+require 'openc3/models/microservice_model'
+
+module OpenC3
+  class PeriodicOnlyDefault < Migration
+    def self.run
+      ScopeModel.get_all_models(scope: nil).each do |scope, scope_model|
+        next if scope == 'DEFAULT'
+        model = MicroserviceModel.get_model(name: "#{scope}__SCOPEMULTI__#{scope}", scope: scope)
+        if model
+          model.cmd.delete("#{scope}__PERIODIC__#{scope}")
+          model.update
+        end
+        model = MicroserviceModel.get_model(name: "#{scope}__PERIODIC__#{scope}", scope: scope)
+        model.destroy if model
+      end
+    end
+  end
+end
+
+unless ENV['OPENC3_NO_MIGRATE']
+  OpenC3::PeriodicOnlyDefault.run
+end

data/lib/openc3/models/model.rb

@@ -139,7 +139,7 @@ module OpenC3
 
     # Update the Redis hash at primary_key and set the field "name"
     # to the JSON generated via calling as_json
-    def create(update: false, force: false, queued: false)
+    def create(update: false, force: false, queued: false, isoformat: false)
       unless force
         existing = self.class.store.hget(@primary_key, @name)
         if existing
@@ -148,7 +148,11 @@ module OpenC3
           raise "#{@primary_key}:#{@name} doesn't exist at update" if update
         end
       end
-      @updated_at = Time.now.to_nsec_from_epoch
+      if isoformat
+        @updated_at = Time.now.utc.iso8601
+      else
+        @updated_at = Time.now.utc.to_nsec_from_epoch
+      end
 
       if queued
         write_store = self.class.store_queued

data/lib/openc3/models/offline_access_model.rb

@@ -19,28 +19,29 @@
 require 'openc3/models/model'
 
 module OpenC3
+  # Note: This model is locked to the DEFAULT scope
   class OfflineAccessModel < Model
-    PRIMARY_KEY = 'openc3__offline_access'
+    PRIMARY_KEY = 'DEFAULT__openc3__offline_access'
 
     attr_accessor :offline_access_token
 
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
     def self.get(name:, scope:)
-      super("#{scope}__#{PRIMARY_KEY}", name: name)
+      super(PRIMARY_KEY, name: name)
     end
 
     def self.names(scope:)
-      super("#{scope}__#{PRIMARY_KEY}")
+      super(PRIMARY_KEY)
     end
 
     def self.all(scope:)
-      super("#{scope}__#{PRIMARY_KEY}")
+      super(PRIMARY_KEY)
     end
     # END NOTE
 
-    def initialize(name:, offline_access_token: nil, updated_at: nil, scope:)
-      super("#{scope}__#{PRIMARY_KEY}", name: name, scope: scope)
+    def initialize(name:, offline_access_token: nil, updated_at: nil, scope: 'DEFAULT')
+      super(PRIMARY_KEY, name: name, updated_at: updated_at, scope: 'DEFAULT')
       @offline_access_token = offline_access_token
     end
 

data/lib/openc3/models/scope_model.rb

@@ -350,8 +350,11 @@ module OpenC3
       # UNKNOWN PacketLog Microservice
       deploy_unknown_packetlog_microservice(gem_path, variables, @parent)
 
-      # Periodic Microservice
-      deploy_periodic_microservice(gem_path, variables, @parent)
+      # Only DEFAULT scope
+      if @scope == 'DEFAULT'
+        # Periodic Microservice
+        deploy_periodic_microservice(gem_path, variables, @parent)
+      end
 
       # Scope Cleanup Microservice
       deploy_scopecleanup_microservice(gem_path, variables, @parent)

data/lib/openc3/models/script_status_model.rb

@@ -0,0 +1,242 @@
+# encoding: ascii-8bit
+
+# Copyright 2025 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'openc3/models/model'
+
+module OpenC3
+  class ScriptStatusModel < Model
+    # Note: ScriptRunner only has permissions for keys that start with running-script
+    RUNNING_PRIMARY_KEY = 'running-script'
+    COMPLETED_PRIMARY_KEY = 'running-script-completed'
+
+    def id
+      return @name
+    end
+    attr_reader :state # spawning, init, running, paused, waiting, breakpoint, error, crashed, stopped, completed, completed_errors, killed
+    attr_accessor :shard
+    attr_accessor :filename
+    attr_accessor :current_filename
+    attr_accessor :line_no
+    attr_accessor :start_line_no
+    attr_accessor :end_line_no
+    attr_accessor :username
+    attr_accessor :user_full_name
+    attr_accessor :start_time
+    attr_accessor :end_time
+    attr_accessor :disconnect
+    attr_accessor :environment
+    attr_accessor :suite_runner
+    attr_accessor :errors
+    attr_accessor :pid
+    attr_accessor :log
+    attr_accessor :report
+
+    # NOTE: The following three class methods are used by the ModelController
+    # and are reimplemented to enable various Model class methods to work
+    def self.get(name:, scope:, type: "auto")
+      if type == "auto" or type == "running"
+        # Check for running first
+        running = super("#{RUNNING_PRIMARY_KEY}__#{scope}", name: name)
+        return running if running
+      end
+      return super("#{COMPLETED_PRIMARY_KEY}__#{scope}", name: name)
+    end
+
+    def self.names(scope:, type: "running")
+      if type == "running"
+        return super("#{RUNNING_PRIMARY_KEY}__#{scope}")
+      else
+        return super("#{COMPLETED_PRIMARY_KEY}__#{scope}")
+      end
+    end
+
+    def self.all(scope:, offset: 0, limit: 10, type: "running")
+      if type == "running"
+        keys = self.store.zrevrange("#{RUNNING_PRIMARY_KEY}__#{scope}__LIST", offset.to_i, offset.to_i + limit.to_i - 1)
+        return [] if keys.empty?
+        result = self.store.redis_pool.pipelined do
+          keys.each do |key|
+            self.store.hget("#{RUNNING_PRIMARY_KEY}__#{scope}", key)
+          end
+        end
+        result = result.map do |r|
+          if r.nil?
+            nil
+          else
+            JSON.parse(r, :allow_nan => true, :create_additions => true)
+          end
+        end
+        return result
+      else
+        keys = self.store.zrevrange("#{COMPLETED_PRIMARY_KEY}__#{scope}__LIST", offset.to_i, offset.to_i + limit.to_i - 1)
+        return [] if keys.empty?
+        result = self.store.redis_pool.pipelined do
+          keys.each do |key|
+            self.store.hget("#{COMPLETED_PRIMARY_KEY}__#{scope}", key)
+          end
+        end
+        result = result.map do |r|
+          if r.nil?
+            nil
+          else
+            JSON.parse(r, :allow_nan => true, :create_additions => true)
+          end
+        end
+        return result
+      end
+    end
+
+    def self.count(scope:, type: "running")
+      if type == "running"
+        return self.store.zcount("#{RUNNING_PRIMARY_KEY}__#{scope}__LIST", 0, Float::INFINITY)
+      else
+        return self.store.zcount("#{COMPLETED_PRIMARY_KEY}__#{scope}__LIST", 0, Float::INFINITY)
+      end
+    end
+
+    def initialize(
+      name:, # id
+      state:, # spawning, init, running, paused, waiting, error, breakpoint, crashed, stopped, completed, completed_errors, killed
+      shard: 0, # Future enhancement of script runner shards
+      filename:, # The initial filename
+      current_filename: nil, # The current filename
+      line_no: 0, # The current line number
+      start_line_no: 1, # The line number to start the script at
+      end_line_no: nil, # The line number to end the script at
+      username:, # The username of the person who started the script
+      user_full_name:, # The full name of the person who started the script
+      start_time:, # The time the script started ISO format
+      end_time: nil, # The time the script ended ISO format
+      disconnect: false,
+      environment: nil,
+      suite_runner: nil,
+      errors: nil,
+      pid: nil,
+      log: nil,
+      report: nil,
+      updated_at: nil,
+      scope:
+    )
+      @state = state
+      if is_complete?()
+        super("#{COMPLETED_PRIMARY_KEY}__#{scope}", name: name, updated_at: updated_at, plugin: nil, scope: scope)
+      else
+        super("#{RUNNING_PRIMARY_KEY}__#{scope}", name: name, updated_at: updated_at, plugin: nil, scope: scope)
+      end
+      @shard = shard.to_i
+      @filename = filename
+      @current_filename = current_filename
+      @line_no = line_no
+      @start_line_no = start_line_no
+      @end_line_no = end_line_no
+      @username = username
+      @user_full_name = user_full_name
+      @start_time = start_time
+      @end_time = end_time
+      @disconnect = disconnect
+      @environment = environment
+      @suite_runner = suite_runner
+      @errors = errors
+      @pid = pid
+      @log = log
+      @report = report
+    end
+
+    def is_complete?
+      return (@state == 'completed' or @state == 'completed_errors' or @state == 'stopped' or @state == 'crashed' or @state == 'killed')
+    end
+
+    def state=(new_state)
+      # If the state is already a flavor of complete, leave it alone (first wins)
+      if not is_complete?()
+        @state = new_state
+        # If setting to complete, check for errors
+        # and set the state to complete_errors if they exist
+        if @state == 'completed' and @errors
+          @state = 'completed_errors'
+        end
+      end
+    end
+
+    # Update the Redis hash at primary_key and set the field "name"
+    # to the JSON generated via calling as_json
+    def create(update: false, force: false, queued: false, isoformat: true)
+      @updated_at = Time.now.utc.to_nsec_from_epoch
+
+      if queued
+        write_store = self.class.store_queued
+      else
+        write_store = self.class.store
+      end
+      write_store.hset(@primary_key, @name, JSON.generate(self.as_json(:allow_nan => true), :allow_nan => true))
+
+      # Also add to ordered set on create
+      write_store.zadd(@primary_key + "__LIST", @name.to_i, @name) if not update
+    end
+
+    def update(force: false, queued: false)
+      # Magically handle the change from running to completed
+      if is_complete?() and @primary_key == "#{RUNNING_PRIMARY_KEY}__#{@scope}"
+        # Destroy the running key
+        destroy()
+        @destroyed = false
+
+        # Move to completed
+        @primary_key = "#{COMPLETED_PRIMARY_KEY}__#{@scope}"
+        create(update: false, force: force, queued: queued, isoformat: true)
+      else
+        create(update: true, force: force, queued: queued, isoformat: true)
+      end
+    end
+
+    # Delete the model from the Store
+    def destroy
+      @destroyed = true
+      undeploy()
+      self.class.store.hdel(@primary_key, @name)
+      # Also remove from ordered set
+      self.class.store.zremrangebyscore(@primary_key + "__LIST", @name.to_i, @name.to_i)
+    end
+
+    def as_json(*a)
+      {
+        'name' => @name,
+        'state' => @state,
+        'shard' => @shard,
+        'filename' => @filename,
+        'current_filename' => @current_filename,
+        'line_no' => @line_no,
+        'start_line_no' => @start_line_no,
+        'end_line_no' => @end_line_no,
+        'username' => @username,
+        'user_full_name' => @user_full_name,
+        'start_time' => @start_time,
+        'end_time' => @end_time,
+        'disconnect' => @disconnect,
+        'environment' => @environment,
+        'suite_runner' => @suite_runner,
+        'errors' => @errors,
+        'pid' => @pid,
+        'log' => @log,
+        'report' => @report,
+        'updated_at' => @updated_at,
+        'scope' => @scope
+      }
+    end
+  end
+end