openc3 7.0.0.pre.rc2 → 7.0.0

data/lib/openc3/microservices/scope_cleanup_microservice.rb

@@ -16,6 +16,120 @@ require 'openc3/microservices/cleanup_microservice'
 
 module OpenC3
   class ScopeCleanupMicroservice < CleanupMicroservice
+    TSDB_HEALTH_QUERY =
+"SELECT
+    table_name,
+    table_row_count,
+    wal_pending_row_count,
+    CASE
+        WHEN table_suspended THEN 'SUSPENDED'
+        WHEN table_memory_pressure_level = 2 THEN 'BACKOFF'
+        WHEN table_memory_pressure_level = 1 THEN 'PRESSURE'
+        ELSE 'OK'
+    END AS status,
+    wal_txn - table_txn AS lag_txns,
+    table_write_amp_p50 AS write_amp,
+    table_merge_rate_p99 AS slowest_merge
+FROM tables()
+WHERE walEnabled
+ORDER BY
+    table_suspended DESC,
+    table_memory_pressure_level DESC,
+    wal_pending_row_count DESC;"
+
+    GROWTH_NUM_SAMPLE_PERIODS = 4
+
+    def initialize(*args)
+      super(*args)
+      @run_time = nil
+      @cleanup_poll_time = nil
+      @delta_time = 0.0
+      @wal_pending_row_count = {}
+      @lag_txns = {}
+    end
+
+    def cleanup(areas, bucket)
+      current_time = Time.now
+      if @run_time
+        delta = current_time - @run_time
+        if delta > 0.0
+          @delta_time += delta
+        end
+      end
+      @run_time = current_time
+      if @delta_time > @cleanup_poll_time
+        @delta_time = 0.0
+        super(areas, bucket)
+      end
+
+      # Always check TSDB health
+      if @scope == 'DEFAULT'
+        begin
+          conn = OpenC3::QuestDBClient.connection
+          result = conn.exec(TSDB_HEALTH_QUERY)
+          columns = result.fields
+          rows = result.values
+
+          table_name_column = columns.index("table_name")
+          wal_pending_row_count_column = columns.index("wal_pending_row_count")
+          status_column = columns.index("status")
+          lag_txns_column = columns.index("lag_txns")
+
+          rows.each do |values|
+            table_name = values[table_name_column]
+            wal_pending_row_count = values[wal_pending_row_count_column].to_i
+            status = values[status_column]
+            lag_txns = values[lag_txns_column].to_i
+
+            if status != 'OK'
+              @logger.error("QuestDB: #{table_name} in bad state: #{status}")
+
+              if status == 'SUSPENDED'
+                # Try to automatically unsuspend
+                @logger.info("QuestDB: Attempting to unsuspend: #{table_name}")
+                conn.exec("ALTER TABLE #{table_name} RESUME WAL;")
+              end
+            end
+
+            @wal_pending_row_count[table_name] ||= []
+            @wal_pending_row_count[table_name] << wal_pending_row_count
+            @lag_txns[table_name] ||= []
+            @lag_txns[table_name] << lag_txns
+
+            if @wal_pending_row_count[table_name].length > GROWTH_NUM_SAMPLE_PERIODS
+              if detect_growth(@wal_pending_row_count[table_name], GROWTH_NUM_SAMPLE_PERIODS)
+                # Crossed threshold of sample periods of growth
+                @logger.error("QuestDB: #{table_name} has growing wal_pending_row_count: #{wal_pending_row_count}")
+              end
+
+              # Leave the last GROWTH_NUM_SAMPLE_PERIODS samples
+              @wal_pending_row_count[table_name] = @wal_pending_row_count[table_name][-GROWTH_NUM_SAMPLE_PERIODS..-1]
+            end
+
+            if @lag_txns[table_name].length > GROWTH_NUM_SAMPLE_PERIODS
+              if detect_growth(@lag_txns[table_name], GROWTH_NUM_SAMPLE_PERIODS)
+                # Crossed threshold of sample periods of growth
+                @logger.error("QuestDB: #{table_name} has growing lag_txns: #{lag_txns}")
+              end
+
+              # Leave the last GROWTH_NUM_SAMPLE_PERIODS samples
+              @lag_txns[table_name] = @lag_txns[table_name][-GROWTH_NUM_SAMPLE_PERIODS..-1]
+            end
+          end
+        rescue => e
+          OpenC3::QuestDBClient.disconnect
+          @logger.error("QuestDB Error: #{e.formatted}")
+        end
+      end
+    end
+
+    def detect_growth(array, num_samples)
+      num_samples.times do |index|
+        return false if array[index + 1] <= array[index]
+      end
+      return true
+    end
+
     def get_areas_and_poll_time
       scope = ScopeModel.get_model(name: @scope)
       areas = [
@@ -28,7 +142,8 @@ module OpenC3
         areas << ["NOSCOPE/tool_logs/sr", scope.tool_log_retain_time]
       end
 
-      return areas, scope.cleanup_poll_time
+      @cleanup_poll_time = scope.cleanup_poll_time
+      return areas, 60 # Run every 1 minute for TSDB checks
     end
   end
 end

data/lib/openc3/microservices/text_log_microservice.rb

@@ -98,13 +98,16 @@ module OpenC3
     def shutdown
       # Make sure all the existing logs are properly closed down
       threads = []
-      @tlws.each do |topic, tlw|
+      @tlws.each do |_topic, tlw|
         threads.concat(tlw.shutdown)
       end
       # Wait for all the logging threads to move files to buckets
       threads.flatten.compact.each do |thread|
         thread.join
       end
+      @tlws.each do |_topic, tlw|
+        tlw.cleanup
+      end
       super()
     end
   end

data/lib/openc3/migrations/20241208080000_no_critical_cmd.rb

@@ -13,7 +13,7 @@ module OpenC3
     end
 
     def self.run
-      ScopeModel.get_all_models(scope: nil).each do |scope, scope_model|
+      ScopeModel.get_all_models(scope: nil).each do |scope, _scope_model|
         model = MicroserviceModel.get_model(name: "#{scope}__CRITICALCMD__#{scope}", scope: scope)
         if BASE # Only remove the critical command model if we're not enterprise
           model.destroy if model

data/lib/openc3/migrations/20250402000000_periodic_only_default.rb

@@ -5,7 +5,7 @@ require 'openc3/models/microservice_model'
 module OpenC3
   class PeriodicOnlyDefault < Migration
     def self.run
-      ScopeModel.get_all_models(scope: nil).each do |scope, scope_model|
+      ScopeModel.get_all_models(scope: nil).each do |scope, _scope_model|
         next if scope == 'DEFAULT'
         model = MicroserviceModel.get_model(name: "#{scope}__SCOPEMULTI__#{scope}", scope: scope)
         if model

data/lib/openc3/migrations/20260203000000_remove_store_id.rb

@@ -0,0 +1,28 @@
+require 'openc3/utilities/migration'
+require 'openc3/models/scope_model'
+require 'openc3/models/plugin_model'
+
+module OpenC3
+  # Removes the store_id property from plugin models. It got renamed to
+  # store_plugin_id in PR #2858 but that also depends on having
+  # store_version_id, which didn't exist prior to this version and can't
+  # be determined without some introspection on the plugin and querying
+  # the app store (online). When store_plugin_id is unset, COSMOS treats
+  # the plugin like it was installed from a gem file.
+  class RemoveStoreId < Migration
+    def self.run
+      ScopeModel.get_all_models(scope: nil).each do |scope, _scope_model|
+        plugin_models = PluginModel.all(scope: scope)
+        plugin_models.each do |_name, plugin_model|
+          plugin_model.delete("store_id")
+          model = PluginModel.from_json(plugin_model, scope: scope)
+          model.update()
+        end
+      end
+    end
+  end
+end
+
+unless ENV['OPENC3_NO_MIGRATE']
+  OpenC3::RemoveStoreId.run
+end

data/lib/openc3/migrations/20260204000000_remove_decom_reducer.rb

@@ -10,17 +10,21 @@
 # if purchased from OpenC3, Inc.
 
 require 'openc3/utilities/migration'
+require 'openc3/utilities/bucket'
 require 'openc3/models/scope_model'
 require 'openc3/models/target_model'
 require 'openc3/models/microservice_model'
+require 'openc3/models/plugin_model'
 
 module OpenC3
   class RemoveDecomLogSettings < Migration
     def self.run
-      ScopeModel.get_all_models(scope: nil).each do |scope, scope_model|
+      ScopeModel.get_all_models(scope: nil).each do |scope, _scope_model|
         target_models = TargetModel.all(scope: scope)
         target_models.each do |name, target_model|
           # Remove deprecated decom log settings from target model
+          target_model.delete("cmd_unique_id_mode")
+          target_model.delete("tlm_unique_id_mode")
           target_model.delete("cmd_decom_log_cycle_time")
           target_model.delete("cmd_decom_log_cycle_size")
           target_model.delete("cmd_decom_log_retain_time")
@@ -51,6 +55,30 @@ module OpenC3
           end
         end
       end
+
+      # Reinstall all plugins to regenerate microservice configs with correct settings
+      # This must happen after removing deprecated keys above
+      client = Bucket.getClient()
+      unless client.exist?(ENV['OPENC3_CONFIG_BUCKET']) && client.exist?(ENV['OPENC3_TOOLS_BUCKET'])
+        Logger.info("Skipping plugin reinstall - buckets do not exist yet (fresh install or new storage backend)")
+        return
+      end
+
+      ScopeModel.get_all_models(scope: nil).each do |scope, _scope_model|
+        plugins = PluginModel.all(scope: scope)
+        plugins.each do |plugin_name, plugin_data|
+          begin
+            Logger.info("Reinstalling plugin #{plugin_name} in scope #{scope}")
+            plugin_model = PluginModel.from_json(plugin_data, scope: scope)
+            plugin_model.undeploy
+            plugin_model.restore
+            Logger.info("Successfully reinstalled plugin #{plugin_name} in scope #{scope}")
+          rescue Exception => e
+            Logger.error("Error reinstalling plugin #{plugin_name} in scope #{scope}: #{e.formatted}")
+            # Continue with other plugins even if one fails
+          end
+        end
+      end
     end
   end
 end

data/lib/openc3/models/activity_model.rb

@@ -18,6 +18,7 @@
 # https://www.rubydoc.info/gems/redis/Redis/Commands/SortedSets
 
 require 'openc3/models/model'
+require 'openc3/models/timeline_model'
 require 'openc3/topics/timeline_topic'
 require 'securerandom'
 
@@ -28,6 +29,11 @@ module OpenC3
 
   class ActivityModel < Model
     MAX_DURATION = Time::SEC_PER_DAY
+    # Grace window (in seconds) to allow creating activities slightly in the past.
+    # This handles race conditions where real-time activity notifications arrive
+    # after the start time has already passed (e.g. from external systems).
+    # This is consistent with the -15 second window in the timeline microservice.
+    START_GRACE_SECONDS = 15
     PRIMARY_KEY = '__openc3_timelines'.freeze # MUST be equal to `TimelineModel::PRIMARY_KEY` minus the leading __
     # See run_activity(activity) in openc3/lib/openc3/microservices/timeline_microservice.rb
     VALID_KINDS = %w(command script reserve expire)
@@ -212,7 +218,7 @@ module OpenC3
     end
 
     # validate the input to the rules we have created for timelines.
-    # - A task's start MUST NOT be in the past.
+    # - A task's start MUST NOT be more than START_GRACE_SECONDS in the past.
     # - A task's start MUST be before the stop.
     # - A task CAN NOT be longer than MAX_DURATION (86400) in seconds.
     # - A task MUST have a kind.
@@ -230,8 +236,8 @@ module OpenC3
       rescue NoMethodError
         raise ActivityInputError.new "start and stop must be seconds: #{start}, #{stop}"
       end
-      if now_f >= start and kind != 'expire'
-        raise ActivityInputError.new "activity must be in the future, current_time: #{now_f} vs #{start}"
+      if now_f >= start + START_GRACE_SECONDS and kind != 'expire'
+        raise ActivityInputError.new "activity must not be more than #{START_GRACE_SECONDS} seconds in the past, current_time: #{now_f} vs #{start}"
       elsif duration > MAX_DURATION and kind != 'expire'
         raise ActivityInputError.new "activity can not be longer than #{MAX_DURATION} seconds"
       elsif duration <= 0
@@ -261,7 +267,13 @@ module OpenC3
 
     # Update the Redis hash at primary_key and set the score equal to the start Epoch time
     # the member is set to the JSON generated via calling as_json
-    def create(overlap: true)
+    def create(overlap: true, username: nil)
+      # Validate that the timeline exists in this scope before creating activities.
+      # Activities must be attached to an existing timeline within the same scope.
+      unless TimelineModel.get(name: @name, scope: @scope)
+        raise ActivityError.new "timeline '#{@name}' does not exist in scope '#{@scope}'"
+      end
+
       if @recurring['end'] and @recurring['frequency'] and @recurring['span']
         # First validate the initial recurring activity ... all others are just offsets
         validate_input(start: @start, stop: @stop, kind: @kind, data: @data)
@@ -290,7 +302,7 @@ module OpenC3
 
         # Update @updated_at and add an event assuming it all completes ok
         @updated_at = Time.now.to_nsec_from_epoch
-        add_event(status: 'created')
+        add_event(status: 'created', username: username)
 
         Store.multi do |multi|
           (@start..@recurring['end']).step(recurrence).each do |start_time|
@@ -326,7 +338,7 @@ module OpenC3
           end
         end
         @updated_at = Time.now.to_nsec_from_epoch
-        add_event(status: 'created')
+        add_event(status: 'created', username: username)
         Store.zadd(@primary_key, @start, JSON.generate(self.as_json, allow_nan: true))
         notify(kind: 'created')
       end
@@ -335,7 +347,7 @@ module OpenC3
     # Update the Redis hash at primary_key and remove the current activity at the current score
     # and update the score to the new score equal to the start Epoch time this uses a multi
     # to execute both the remove and create.
-    def update(start:, stop:, kind:, data:, overlap: true)
+    def update(start:, stop:, kind:, data:, overlap: true, username: nil)
       array = Store.zrangebyscore(@primary_key, @start, @start)
       if array.length == 0
         raise ActivityError.new "failed to find activity at: #{@start}"
@@ -350,10 +362,22 @@ module OpenC3
           raise ActivityOverlapError.new "failed to update #{old_start}, no activities can overlap, collision: #{collision}"
         end
       end
+
+      # Compute changeset for audit trail before applying changes
+      changes = {}
+      diff_field(changes, 'start', @start, start)
+      diff_field(changes, 'stop', @stop, stop)
+      diff_field(changes, 'kind', @kind, kind)
+      old_data = @data.reject { |k, _| k == 'username' }
+      new_data = data.reject { |k, _| k == 'username' }
+      (old_data.keys | new_data.keys).each do |key|
+        diff_field(changes, "data.#{key}", old_data[key], new_data[key])
+      end
+
       set_input(start: start, stop: stop, kind: kind, data: data, events: @events)
       @updated_at = Time.now.to_nsec_from_epoch
 
-      add_event(status: 'updated')
+      add_event(status: 'updated', username: username, changes: changes.empty? ? nil : changes)
       json = Store.zrangebyscore(@primary_key, old_start, old_start)
       parsed = json.map { |value| JSON.parse(value, allow_nan: true, create_additions: true) }
       parsed.each_with_index do |value, index|
@@ -397,14 +421,22 @@ module OpenC3
 
     # add_event will make an event. This will NOT save the object to the redis database
     # @param [String] status - the event status such as "queued" or "updated" or "created"
-    def add_event(status:)
+    # @param [String] username - optional username of who performed the action
+    # @param [Hash] changes - optional hash describing what fields changed (for audit)
+    def add_event(status:, username: nil, changes: nil)
       event = {
         'time' => Time.now.to_i,
         'event' => status
       }
+      event['username'] = username if username
+      event['changes'] = changes if changes
       @events << event
     end
 
+    def diff_field(changes, field, old_val, new_val)
+      changes[field] = {'old' => old_val, 'new' => new_val} unless old_val == new_val
+    end
+
     # update the redis stream / timeline topic that something has changed
     def notify(kind:, extra: nil)
       notification = {

data/lib/openc3/models/auth_model.rb

@@ -41,6 +41,9 @@ module OpenC3
 
     MIN_PASSWORD_LENGTH = 8
 
+    SESSION_PREFIX = "ses_"
+    OTP_PREFIX = "otp_"
+
     def self.set?(key = PRIMARY_KEY)
       Store.exists(key) == 1
     end
@@ -58,36 +61,49 @@ module OpenC3
 
       return false if service_only
 
-      return verify_no_service(token, no_password: no_password)
+      mode = no_password ? :token : :any
+      return verify_no_service(token, mode: mode)
     end
 
     # Checks whether the provided token is a valid user password or session token.
     # @param token [String] the plaintext password or session token to check (required)
-    # @param no_password [Boolean] enforces use of a session token (default: true)
+    # @param mode [String] optionally restrict verification to just the password or token. Valid values: :password, :token, or :any (default :token)
     # @return [Boolean] whether the provided password/token is valid
-    def self.verify_no_service(token, no_password: true)
+    def self.verify_no_service(token, mode: :token)
+      modes = [:password, :token, :any]
+      raise ArgumentError, "Invalid mode '#{mode}': must be one of #{modes}" unless modes.include?(mode)
+
       return false if token.nil? or token.empty?
 
       # Check cached session tokens and password hash
       time = Time.now
-      return true if @@session_cache and (time - @@session_cache_time) < SESSION_CACHE_TIMEOUT and @@session_cache[token]
-      unless no_password
-        return true if @@pw_hash_cache and (time - @@pw_hash_cache_time) < PW_HASH_CACHE_TIMEOUT and Argon2::Password.verify_password(token, @@pw_hash_cache)
+      unless mode == :password
+        if @@session_cache and (time - @@session_cache_time) < SESSION_CACHE_TIMEOUT and @@session_cache[token]
+          terminate_otp(token)
+          return true
+        end
+
+        # Check stored session tokens
+        @@session_cache = Store.hgetall(SESSIONS_KEY)
+        @@session_cache_time = time
+        if @@session_cache[token]
+          terminate_otp(token)
+          return true
+        end
       end
 
-      # Check stored session tokens
-      @@session_cache = Store.hgetall(SESSIONS_KEY)
-      @@session_cache_time = time
-      return true if @@session_cache[token]
+      unless mode == :token
+        return true if @@pw_hash_cache and (time - @@pw_hash_cache_time) < PW_HASH_CACHE_TIMEOUT and Argon2::Password.verify_password(token, @@pw_hash_cache)
 
-      return false if no_password
+        # Check stored password hash
+        pw_hash = Store.get(PRIMARY_KEY)
+        raise "invalid password hash" if pw_hash.nil? || !pw_hash.start_with?("$argon2") # Catch users who didn't run the migration utility when upgrading to COSMOS 7
+        @@pw_hash_cache = pw_hash
+        @@pw_hash_cache_time = time
+        return true if Argon2::Password.verify_password(token, @@pw_hash_cache)
+      end
 
-      # Check stored password hash
-      pw_hash = Store.get(PRIMARY_KEY)
-      raise "invalid password hash" unless pw_hash.start_with?("$argon2") # Catch users who didn't run the migration utility when upgrading to COSMOS 7
-      @@pw_hash_cache = pw_hash
-      @@pw_hash_cache_time = time
-      return Argon2::Password.verify_password(token, @@pw_hash_cache)
+      return false
     end
 
     def self.set(password, old_password, key = PRIMARY_KEY)
@@ -96,17 +112,25 @@ module OpenC3
 
       if set?(key)
         raise "old_password must not be nil or empty" if old_password.nil? or old_password.empty?
-        raise "old_password incorrect" unless verify_no_service(old_password, no_password: false)
+        raise "old_password incorrect" unless verify_no_service(old_password, mode: :password)
       end
       pw_hash = Argon2::Password.create(password, profile: ARGON2_PROFILE)
       Store.set(key, pw_hash)
       @@pw_hash_cache = nil
       @@pw_hash_cache_time = nil
+      logout
     end
 
     # Creates a new session token. DO NOT CALL BEFORE VERIFYING.
-    def self.generate_session
+    # @param otp [Boolean] whether to create a one-time use token (default: false)
+    # @return [String] the new session token
+    def self.generate_session(otp: false)
       token = SecureRandom.urlsafe_base64(nil, false)
+      if otp
+        token = OTP_PREFIX + token
+      else
+        token = SESSION_PREFIX + token
+      end
       Store.hset(SESSIONS_KEY, token, Time.now.iso8601)
       return token
     end
@@ -117,5 +141,16 @@ module OpenC3
       @@session_cache = nil
       @@session_cache_time = nil
     end
+
+    # Terminates the given session token.
+    def self.terminate(token)
+      Store.hdel(SESSIONS_KEY, token)
+      @@session_cache.delete(token) if @@session_cache
+    end
+
+    # Terminates the session if the token is an OTP.
+    def self.terminate_otp(token)
+      terminate(token) if token.start_with?(OTP_PREFIX)
+    end
   end
 end