openc3 6.6.0 → 6.8.0

data/lib/openc3/microservices/queue_microservice.rb

@@ -0,0 +1,166 @@
+# encoding: ascii-8bit
+
+# Copyright 2025 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'openc3/microservices/microservice'
+require 'openc3/topics/queue_topic'
+require 'openc3/utilities/authentication'
+require 'openc3/script'
+
+module OpenC3
+  # The queue processor runs in a single thread and processes commands via cmd_api.
+  class QueueProcessor
+    attr_accessor :state
+    attr_reader :name, :scope
+
+    def initialize(name:, state:, logger:, scope:)
+      @name = name
+      @logger = logger
+      @scope = scope
+      @state = state
+      @cancel_thread = false
+    end
+
+    def get_token(username)
+      if ENV['OPENC3_API_CLIENT'].nil?
+        ENV['OPENC3_API_PASSWORD'] ||= ENV['OPENC3_SERVICE_PASSWORD']
+        return OpenC3Authentication.new().token
+      else
+        # Check for offline access token
+        model = nil
+        model = OpenC3::OfflineAccessModel.get_model(name: username, scope: @scope) if username and username != ''
+        if model and model.offline_access_token
+          auth = OpenC3KeycloakAuthentication.new(ENV['OPENC3_KEYCLOAK_URL'])
+          return auth.get_token_from_refresh_token(model.offline_access_token)
+        else
+          return nil
+        end
+      end
+    end
+
+    def run
+      while true
+        if @state == 'RELEASE'
+          process_queued_commands()
+        else
+          sleep 0.2
+        end
+        break if @cancel_thread
+      end
+    end
+
+    def process_queued_commands
+      while @state == 'RELEASE'
+        begin
+          _queue_name, command_data, _timestamp = Store.bzpopmin("#{@scope}:#{@name}", timeout: 0.2)
+          if command_data
+            command = JSON.parse(command_data)
+            username = command['username']
+            token = get_token(username)
+            # It's important to set queue: false here to avoid infinite recursion when
+            # OPENC3_DEFAULT_QUEUE is set because commands would be re-queued to the default queue
+            cmd_no_hazardous_check(command['value'], queue: false, scope: @scope, token: token)
+          end
+        rescue StandardError => e
+          @logger.error "QueueProcessor failed to process command from queue #{@name}\n#{e.message}"
+        end
+        break if @cancel_thread
+      end
+    end
+
+    def shutdown
+      @cancel_thread = true
+    end
+  end
+
+  # The queue microservice starts a processor then gets the queue entries from redis.
+  # It then monitors the QueueTopic for changes.
+  class QueueMicroservice < Microservice
+    attr_reader :name, :processor, :processor_thread
+
+    def initialize(*args)
+      super(*args)
+      @queue_name = @name.split('__')[2]
+
+      initial_state = 'HOLD'
+      (@config['options'] || []).each do |option|
+        case option[0].upcase
+        when 'QUEUE_STATE'
+          initial_state = option[1]
+        else
+          @logger.error("Unknown option passed to microservice #{@name}: #{option}")
+        end
+      end
+
+      @processor = QueueProcessor.new(name: @queue_name, state: initial_state, logger: @logger, scope: @scope)
+      @processor_thread = nil
+      @read_topic = true
+    end
+
+    def run
+      @logger.info "QueueMicroservice running"
+      @processor_thread = Thread.new { @processor.run }
+
+      # Let the frontend know that the microservice has been deployed and is running
+      notification = {
+        'kind' => 'deployed',
+        # name and updated_at fields are required for Event formatting
+        'data' => JSON.generate({
+          'name' => @name,
+          'updated_at' => Time.now.to_nsec_from_epoch,
+        }),
+      }
+      QueueTopic.write_notification(notification, scope: @scope)
+
+      loop do
+        break if @cancel_thread
+        block_for_updates()
+      end
+      @processor.shutdown()
+      @processor_thread.join() if @processor_thread
+      @logger.info "QueueMicroservice exiting"
+    end
+
+    def block_for_updates
+      @read_topic = true
+      while @read_topic && !@cancel_thread
+        begin
+          QueueTopic.read_topics(@topics) do |_topic, _msg_id, msg_hash, _redis|
+            data = JSON.parse(msg_hash['data']) if msg_hash['data']
+            if data['name'] == @queue_name and msg_hash['kind'] == 'updated'
+              @processor.state = data['state']
+            end
+          end
+        rescue StandardError => e
+          @logger.error "QueueMicroservice failed to read topics #{@topics}\n#{e.formatted}"
+        end
+      end
+    end
+
+    def shutdown
+      @read_topic = false
+      @processor.shutdown() if @processor
+      super
+    end
+  end
+end
+
+if __FILE__ == $0
+  OpenC3::QueueMicroservice.run
+  OpenC3::ThreadManager.instance.shutdown
+  OpenC3::ThreadManager.instance.join
+end

data/lib/openc3/models/cvt_model.rb

@@ -14,12 +14,14 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
+# All changes Copyright 2025, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
+require 'pg'
+require 'thread'
 require 'openc3/utilities/store'
 require 'openc3/utilities/store_queued'
 require 'openc3/models/target_model'
@@ -28,6 +30,8 @@ module OpenC3
   class CvtModel
     @@packet_cache = {}
     @@override_cache = {}
+    @@conn = nil
+    @@conn_mutex = Mutex.new
 
     VALUE_TYPES = [:RAW, :CONVERTED, :FORMATTED, :WITH_UNITS]
     def self.build_json_from_packet(packet)
@@ -117,22 +121,149 @@ module OpenC3
       end
     end
 
+    def self.tsdb_lookup(items, start_time:, end_time: nil)
+      tables = {}
+      names = []
+      nil_count = 0
+      items.each do |item|
+        target_name, packet_name, item_name, value_type, limits = item
+        # They will all be nil when item is a nil value
+        # A nil value indicates a value that does not exist as returned by get_tlm_available
+        if item_name.nil?
+          # We know timestamp always exists so we can use it to fill in the nil value
+          names << "timestamp as __nil#{nil_count}"
+          nil_count += 1
+          next
+        end
+        # See https://questdb.com/docs/reference/api/ilp/advanced-settings/#name-restrictions
+        table_name = "#{target_name}__#{packet_name}".gsub(/[?,'"\/:\)\(\+\*\%~]/, '_')
+        tables[table_name] = 1
+        index = tables.find_index {|k,v| k == table_name }
+        # See https://questdb.com/docs/reference/api/ilp/advanced-settings/#name-restrictions
+        # NOTE: Semicolon added as it appears invalid
+        item_name = item_name.gsub(/[?\.,'"\\\/:\)\(\+\-\*\%~;]/, '_')
+        case value_type
+        when 'WITH_UNITS'
+          names << "\"T#{index}.#{item_name}__U\""
+        when 'FORMATTED'
+          names << "\"T#{index}.#{item_name}__F\""
+        when 'CONVERTED'
+          names << "\"T#{index}.#{item_name}__C\""
+        else
+          names << "\"T#{index}.#{item_name}\""
+        end
+        if limits
+          names << "\"T#{index}.#{item_name}__L\""
+        end
+      end
+
+      # Build the SQL query
+      query = "SELECT #{names.join(", ")} FROM "
+      tables.each_with_index do |(table_name, _), index|
+        if index == 0
+          query += "#{table_name} as T#{index} "
+        else
+          query += "ASOF JOIN #{table_name} as T#{index} "
+        end
+      end
+      if start_time && !end_time
+        query += "WHERE T0.timestamp < '#{start_time}' LIMIT -1"
+      elsif start_time && end_time
+        query += "WHERE T0.timestamp >= '#{start_time}' AND T0.timestamp < '#{end_time}'"
+      end
+
+      retry_count = 0
+      begin
+        @@conn_mutex.synchronize do
+          @@conn ||= PG::Connection.new(host: ENV['OPENC3_TSDB_HOSTNAME'],
+                                        port: ENV['OPENC3_TSDB_QUERY_PORT'],
+                                        user: ENV['OPENC3_TSDB_USERNAME'],
+                                        password: ENV['OPENC3_TSDB_PASSWORD'],
+                                        dbname: 'qdb')
+          # Default connection is all strings but we want to map to the correct types
+          if @@conn.type_map_for_results.is_a? PG::TypeMapAllStrings
+            # TODO: This doesn't seem to be round tripping UINT64 correctly
+            # Try playback with P_2.2,2 and P(:6;): from the DEMO
+            @@conn.type_map_for_results = PG::BasicTypeMapForResults.new @@conn
+          end
+
+          result = @@conn.exec(query)
+          if result.nil? or result.ntuples == 0
+            return {}
+          else
+            data = []
+            # Build up a results set that is an array of arrays
+            # Each nested array is a set of 2 items: [value, limits state]
+            # If the item does not have limits the limits state is nil
+            result.each_with_index do |tuples, index|
+              data[index] ||= []
+              row_index = 0
+              tuples.each do |tuple|
+                if tuple[0].include?("__L")
+                  data[index][row_index - 1][1] = tuple[1]
+                elsif tuple[0] =~ /^__nil/
+                  data[index][row_index] = [nil, nil]
+                  row_index += 1
+                else
+                  data[index][row_index] = [tuple[1], nil]
+                  row_index += 1
+                end
+              end
+            end
+            # If we only have one row then we return a single array
+            if result.ntuples == 1
+              data = data[0]
+            end
+            return data
+          end
+        end
+      rescue IOError, PG::Error => e
+        # Retry the query because various errors can occur that are recoverable
+        retry_count += 1
+        if retry_count > 4
+          # After the 5th retry just raise the error
+          raise "Error querying QuestDB: #{e.message}"
+        end
+        Logger.warn("QuestDB: Retrying due to error: #{e.message}")
+        Logger.warn("QuestDB: Last query: #{query}") # Log the last query for debugging
+        @@conn_mutex.synchronize do
+          if @@conn and !@@conn.finished?
+            @@conn.finish()
+          end
+          @@conn = nil # Force the new connection
+        end
+        sleep 0.1
+        retry
+      end
+    end
+
     # Return all item values and limit state from the CVT
     #
     # @param items [Array<String>] Items to return. Must be formatted as TGT__PKT__ITEM__TYPE
     # @param stale_time [Integer] Time in seconds from Time.now that value will be marked stale
     # @return [Array] Array of values
-    def self.get_tlm_values(items, stale_time: 30, cache_timeout: nil, scope: $openc3_scope)
+    def self.get_tlm_values(items, stale_time: 30, cache_timeout: nil, start_time: nil, end_time: nil, scope: $openc3_scope)
       now = Time.now
       results = []
       lookups = []
       packet_lookup = {}
       overrides = {}
+
+      # If a start_time is passed we're doing a QuestDB lookup and directly return the results
+      # TODO: This currently does NOT support the override values
+      if start_time
+        return tsdb_lookup(items, start_time: start_time, end_time: end_time)
+      end
+
       # First generate a lookup hash of all the items represented so we can query the CVT
       items.each { |item| _parse_item(now, lookups, overrides, item, cache_timeout: cache_timeout, scope: scope) }
 
       now = now.to_f
       lookups.each do |target_packet_key, target_name, packet_name, value_keys|
+        if target_packet_key.nil?
+          results << [nil, nil]
+          next
+        end
         unless packet_lookup[target_packet_key]
           packet_lookup[target_packet_key] = get(target_name: target_name, packet_name: packet_name, cache_timeout: cache_timeout, scope: scope)
         end
@@ -161,7 +292,8 @@ module OpenC3
             if hash.key?(value_keys[-1])
               item_result[1] = nil
             else
-              raise "Item '#{target_name} #{packet_name} #{value_keys[-1]}' does not exist"
+              item_result[0] = nil
+              item_result[1] = nil
             end
           end
         end
@@ -335,6 +467,11 @@ module OpenC3
     # return an ordered array of hash with keys
     def self._parse_item(now, lookups, overrides, item, cache_timeout:, scope:)
       target_name, packet_name, item_name, value_type = item
+      # They will all be nil when item is a nil value
+      if item_name.nil?
+        lookups << nil
+        return
+      end
 
       # We build lookup keys by including all the less formatted types to gracefully degrade lookups
       # This allows the user to specify WITH_UNITS and if there is no conversions it will simply return the RAW value

data/lib/openc3/models/plugin_model.rb

@@ -55,6 +55,7 @@ module OpenC3
     attr_accessor :variables
     attr_accessor :plugin_txt_lines
     attr_accessor :needs_dependencies
+    attr_accessor :store_id
 
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
@@ -72,7 +73,7 @@ module OpenC3
 
     # Called by the PluginsController to parse the plugin variables
     # Doesn't actually create the plugin during the phase
-    def self.install_phase1(gem_file_path, existing_variables: nil, existing_plugin_txt_lines: nil, process_existing: false, scope:, validate_only: false)
+    def self.install_phase1(gem_file_path, existing_variables: nil, existing_plugin_txt_lines: nil, store_id: nil, process_existing: false, scope:, validate_only: false)
       gem_name = File.basename(gem_file_path).split("__")[0]
 
       temp_dir = Dir.mktmpdir
@@ -130,7 +131,8 @@ module OpenC3
           end
         end
 
-        model = PluginModel.new(name: gem_name, variables: variables, plugin_txt_lines: plugin_txt_lines, scope: scope)
+        store_id = Integer(store_id) if store_id
+        model = PluginModel.new(name: gem_name, variables: variables, plugin_txt_lines: plugin_txt_lines, store_id: store_id, scope: scope)
         result = model.as_json(:allow_nan => true)
         result['existing_plugin_txt_lines'] = existing_plugin_txt_lines if existing_plugin_txt_lines and not process_existing and existing_plugin_txt_lines != result['plugin_txt_lines']
         return result
@@ -299,6 +301,7 @@ module OpenC3
       variables: {},
       plugin_txt_lines: [],
       needs_dependencies: false,
+      store_id: nil,
       updated_at: nil,
       scope:
     )
@@ -306,6 +309,7 @@ module OpenC3
       @variables = variables
       @plugin_txt_lines = plugin_txt_lines
       @needs_dependencies = ConfigParser.handle_true_false(needs_dependencies)
+      @store_id = store_id
     end
 
     def create(update: false, force: false, queued: false)
@@ -319,6 +323,7 @@ module OpenC3
         'variables' => @variables,
         'plugin_txt_lines' => @plugin_txt_lines,
         'needs_dependencies' => @needs_dependencies,
+        'store_id' => @store_id,
         'updated_at' => @updated_at
       }
     end

data/lib/openc3/models/plugin_store_model.rb

@@ -0,0 +1,70 @@
+# encoding: ascii-8bit
+
+# Copyright 2025 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'openc3/models/model'
+require 'openc3/utilities/store'
+
+module OpenC3
+  class PluginStoreModel < Model
+    PRIMARY_KEY = 'openc3_plugin_store'
+
+    def self.set(plugin_store_data)
+      Store.set(PRIMARY_KEY, plugin_store_data)
+    end
+
+    def self.all()
+      Store.get(PRIMARY_KEY)
+    end
+
+    def self.plugin_store_error(message)
+      Store.set(PRIMARY_KEY, [{
+        date: Time.now.utc.iso8601,
+        title: 'Plugin Store Error',
+        body: message,
+        error: true,
+      }].to_json)
+    end
+
+    def self.get_by_id(id)
+      plugins = JSON.parse(all()) rescue []
+      plugins.find { |plugin| plugin["id"] == Integer(id) }
+    end
+
+    def self.update
+      setting = SettingModel.get(name: 'store_url', scope: 'DEFAULT')
+      store_url = setting['data'] if setting
+      store_url = 'https://store.openc3.com' if store_url.nil? or store_url.strip.empty?
+      conn = Faraday.new(
+        url: store_url,
+      )
+      response = conn.get('/cosmos_plugins/json')
+      if response.success?
+        self.set(response.body)
+      else
+        self.plugin_store_error("Error contacting plugin store at #{store_url} (status: #{response.status})")
+      end
+    rescue Exception => e
+      self.plugin_store_error("Error contacting plugin store at #{store_url}. #{e.message})")
+    end
+
+    def self.ensure_exists
+      plugins = self.all()
+      self.update() if plugins.nil? or plugins.length.zero? or plugins[0]['error']
+    end
+  end
+end