openc3 7.0.0 → 7.1.0

data/lib/openc3/utilities/questdb_client.rb

@@ -28,42 +28,104 @@ module OpenC3
     class QuestDBError < StandardError; end
 
     # Thread-local PG connection storage using Concurrent::ThreadLocalVar.
-    # Each thread gets its own connection to avoid thread-safety issues with PG::Connection.
+    # Each thread gets its own connections (per db_shard) to avoid thread-safety issues with PG::Connection.
     # Connections are automatically garbage collected when threads terminate.
-    @thread_conn = Concurrent::ThreadLocalVar.new(nil)
+    # Value is a Hash: { db_shard_number => PG::Connection }
+    @thread_conns = Concurrent::ThreadLocalVar.new { Hash.new } # NOSONAR
+
+    # DB_Shard cache: { "scope__target_name" => [db_shard_number, Time] }
+    @db_shard_cache = {}
+    @db_shard_cache_mutex = Mutex.new
+    DB_SHARD_CACHE_TIMEOUT = 60 # seconds
+
+    # Resolve the hostname for a given db_shard number.
+    # If OPENC3_TSDB_HOSTNAME contains "SHARDNUM", it is replaced with the db_shard number.
+    # Otherwise, all db_shards connect to the same host (backward compatible).
+    def self.hostname_for_db_shard(db_shard)
+      ENV['OPENC3_TSDB_HOSTNAME'].to_s.gsub("SHARDNUM", db_shard.to_s)
+    end
+
+    # Look up the db_shard number for a target from TargetModel with a 1-minute cache.
+    # Non-target-specific data (nil target_name) always returns db_shard 0.
+    def self.db_shard_for_target(target_name, scope: "DEFAULT")
+      return 0 unless target_name
+
+      cache_key = "#{scope}__#{target_name}"
+      now = Time.now
+
+      @db_shard_cache_mutex.synchronize do
+        cached = @db_shard_cache[cache_key]
+        if cached
+          db_shard, cached_at = cached
+          return db_shard if (now - cached_at) < DB_SHARD_CACHE_TIMEOUT
+        end
+      end
+
+      # Cache miss or expired — look up from TargetModel
+      begin
+        model = TargetModel.get(name: target_name, scope: scope)
+        db_shard = model ? model['db_shard'].to_i : 0
+      rescue
+        db_shard = 0
+      end
+
+      @db_shard_cache_mutex.synchronize do
+        @db_shard_cache[cache_key] = [db_shard, now]
+      end
+
+      db_shard
+    end
 
-    # Get or create a thread-local PG connection with type mapping configured.
+    # Get or create a thread-local PG connection for the given db_shard with type mapping configured.
     # Returns the thread-local connection - callers should not close it.
-    def self.connection
-      conn = @thread_conn.value
-      if conn.nil? || conn.finished?
-        conn = PG::Connection.new(
-          host: ENV['OPENC3_TSDB_HOSTNAME'],
-          port: ENV['OPENC3_TSDB_QUERY_PORT'],
-          user: ENV['OPENC3_TSDB_USERNAME'],
-          password: ENV['OPENC3_TSDB_PASSWORD'],
-          dbname: 'qdb'
-        )
-        conn.type_map_for_results = PG::BasicTypeMapForResults.new(conn)
-        @thread_conn.value = conn
+    def self.connection(db_shard: 0)
+      conns = @thread_conns.value
+      conn = conns[db_shard]
+      if conn and not conn.finished?
+        begin
+          conn.check_socket
+          return conn
+        rescue
+          # Will need to reconnect
+        end
       end
+      conn = PG::Connection.new(
+        host: hostname_for_db_shard(db_shard),
+        port: ENV['OPENC3_TSDB_QUERY_PORT'],
+        user: ENV['OPENC3_TSDB_USERNAME'],
+        password: ENV['OPENC3_TSDB_PASSWORD'],
+        dbname: 'qdb'
+      )
+      conn.type_map_for_results = PG::BasicTypeMapForResults.new(conn)
+      conns[db_shard] = conn
+      @thread_conns.value = conns
       conn
     end
 
-    # Reset the connection for the current thread. Used after errors.
-    def self.disconnect
-      conn = @thread_conn.value
-      if conn && !conn.finished?
-        conn.finish
+    # Reset the connection(s) for the current thread. Used after errors.
+    # If db_shard is nil, closes all db_shard connections. Otherwise closes only the specified db_shard.
+    def self.disconnect(db_shard: nil)
+      conns = @thread_conns.value
+      if db_shard.nil?
+        conns.each_value do |conn|
+          conn.finish if conn && !conn.finished?
+        end
+        @thread_conns.value = {}
+      else
+        conn = conns[db_shard]
+        if conn && !conn.finished?
+          conn.finish
+        end
+        conns.delete(db_shard)
+        @thread_conns.value = conns
       end
-      @thread_conn.value = nil
     end
 
     # Health check - attempt to connect and immediately close.
     # Returns true if successful, raises on failure.
-    def self.check_connection
+    def self.check_connection(db_shard: 0)
       conn = PG::Connection.new(
-        host: ENV['OPENC3_TSDB_HOSTNAME'],
+        host: hostname_for_db_shard(db_shard),
         port: ENV['OPENC3_TSDB_QUERY_PORT'],
         user: ENV['OPENC3_TSDB_USERNAME'],
         password: ENV['OPENC3_TSDB_PASSWORD'],
@@ -127,14 +189,15 @@ module OpenC3
     # - Arrays are JSON-encoded: "[1, 2, 3]" or '["a", "b"]'
     # - Objects/Hashes are JSON-encoded: '{"key": "value"}'
     # - Binary data (BLOCK) is base64-encoded
-    # - Large integers (64-bit) are stored as DECIMAL
+    # - Large integers (≥64-bit) are stored as VARCHAR strings
     #
     # @param value [Object] The value to decode
     # @param data_type [String] COSMOS data type (INT, UINT, FLOAT, STRING, BLOCK, DERIVED, etc.)
     # @param array_size [Integer, nil] If not nil, indicates this is an array item
     # @return [Object] The decoded value
     def self.decode_value(value, data_type: nil, array_size: nil)
-      # Handle BigDecimal values from QuestDB DECIMAL columns (used for 64-bit integers)
+      # Handle BigDecimal values from legacy QuestDB DECIMAL columns
+      # (pre-existing tables may still use DECIMAL; new tables use VARCHAR)
       if value.is_a?(BigDecimal)
         return value.to_i if data_type == 'INT' || data_type == 'UINT'
         return value
@@ -167,7 +230,7 @@ module OpenC3
         end
       end
 
-      # Integer values stored as strings (fallback path, normally DECIMAL)
+      # Integer values stored as VARCHAR strings (≥64-bit integers)
       if data_type == 'INT' || data_type == 'UINT'
         begin
           return Integer(value)
@@ -290,14 +353,14 @@ module OpenC3
     # @param label [String, nil] Optional label for log messages
     # @return [PG::Result, nil] Query result
     # @raise [RuntimeError] After exhausting retries
-    def self.query_with_retry(query, params: [], max_retries: 5, label: nil)
+    def self.query_with_retry(query, params: [], max_retries: 5, label: nil, db_shard: 0)
       retry_count = 0
       begin
-        conn = connection
+        conn = connection(db_shard: db_shard)
         if params.empty?
-          conn.exec(query)
+          return conn.exec(query)
         else
-          conn.exec_params(query, params)
+          return conn.exec_params(query, params)
         end
       rescue IOError, PG::Error => e
         retry_count += 1
@@ -306,7 +369,7 @@ module OpenC3
         end
         Logger.warn("TSDB#{label ? " #{label}" : ""}: Retrying due to error: #{e.message}")
         Logger.warn("TSDB#{label ? " #{label}" : ""}: Last query: #{query}")
-        disconnect
+        disconnect(db_shard: db_shard)
         sleep 0.1
         retry
       end
@@ -542,11 +605,11 @@ module OpenC3
     # @param start_time [Integer] Nanosecond start time
     # @param end_time [Integer, nil] Nanosecond end time
     # @return [Boolean]
-    def self.table_has_data?(table_name, start_time, end_time)
-      query = "SELECT 1 FROM #{table_name}"
+    def self.table_has_data?(table_name, start_time, end_time, db_shard: 0)
+      query = "SELECT 1 FROM \"#{table_name}\""
       query += time_where_clause(start_time, end_time)
       query += " LIMIT 1"
-      result = query_with_retry(query, max_retries: 1, label: "table_has_data")
+      result = query_with_retry(query, max_retries: 1, label: "table_has_data", db_shard: db_shard)
       result && result.ntuples > 0
     rescue RuntimeError
       false
@@ -559,13 +622,13 @@ module OpenC3
     # @param page_size [Integer] Number of rows per page
     # @param label [String] Label for log messages
     # @yield [PG::Result] Each page of results
-    def self.paginate_query(query, page_size, label:)
+    def self.paginate_query(query, page_size, label:, db_shard: 0)
       min = 0
       max = page_size
       loop do
         query_offset = "#{query} LIMIT #{min}, #{max}"
         Logger.debug("QuestDB #{label}: #{query_offset}")
-        result = query_with_retry(query_offset, label: label)
+        result = query_with_retry(query_offset, label: label, db_shard: db_shard)
         min += page_size
         max += page_size
         if result.nil? or result.ntuples == 0
@@ -589,7 +652,7 @@ module OpenC3
       names << TIMESTAMP_SELECT
       names << "RECEIVED_TIMESECONDS" if include_received_ts
       names << "COSMOS_EXTRA"
-      query = "SELECT #{names.join(', ')} FROM #{table_name}"
+      query = "SELECT #{names.join(', ')} FROM \"#{table_name}\""
       query += time_where_clause(start_time, end_time)
       query
     end
@@ -807,6 +870,8 @@ module OpenC3
 
     # Query historical telemetry data from QuestDB for a list of items.
     # Builds the SQL query, executes it, and decodes all results.
+    # Supports cross-db_shard queries by grouping items by db_shard, executing
+    # separate queries per db_shard, and merging results positionally.
     #
     # @param items [Array] Array of [target_name, packet_name, item_name, value_type, limits]
     #   item_name may be nil to indicate a placeholder (non-existent item)
@@ -816,6 +881,78 @@ module OpenC3
     # @return [Array, Hash] Array of [value, limits_state] pairs per row, or {} if no results.
     #   Single-row results return a flat array; multi-row results return array of arrays.
     def self.tsdb_lookup(items, start_time:, end_time: nil, scope: "DEFAULT")
+      # Group items by db_shard number while preserving their original positions
+      db_shard_groups = {} # db_shard => { positions: [], items: [] }
+      items.each_with_index do |item, pos|
+        target_name = item[0]
+        db_shard = db_shard_for_target(target_name, scope: scope)
+        db_shard_groups[db_shard] ||= { positions: [], items: [] }
+        db_shard_groups[db_shard][:positions] << pos
+        db_shard_groups[db_shard][:items] << item
+      end
+
+      # Single-db_shard fast path (most common case)
+      if db_shard_groups.length == 1
+        db_shard, group = db_shard_groups.first
+        return tsdb_lookup_single_db_shard(group[:items], start_time: start_time, end_time: end_time, scope: scope, db_shard: db_shard)
+      end
+
+      # Cross-db_shard: execute per-db_shard queries and merge results
+      db_shard_results = {} # db_shard => data
+      db_shard_groups.each do |db_shard, group|
+        result = tsdb_lookup_single_db_shard(group[:items], start_time: start_time, end_time: end_time, scope: scope, db_shard: db_shard)
+        db_shard_results[db_shard] = result
+      end
+
+      # If all db_shards returned empty, return empty
+      return {} if db_shard_results.values.all? { |r| r == {} }
+
+      # Merge results positionally back into the original item order.
+      # For single-row results (no end_time), merge flat arrays.
+      # For multi-row results, each db_shard may have different row counts;
+      # use the maximum row count and fill missing positions with [nil, nil].
+      if !end_time
+        # Single-row mode: each db_shard returns a flat array of [value, limits] pairs.
+        # Merge them into the original item order.
+        merged = Array.new(items.length) { [nil, nil] }
+        db_shard_groups.each do |db_shard, group|
+          result = db_shard_results[db_shard]
+          next if result == {} || !result.is_a?(Array)
+          group[:positions].each_with_index do |orig_pos, db_shard_idx|
+            merged[orig_pos] = result[db_shard_idx] if result[db_shard_idx]
+          end
+        end
+        merged
+      else
+        # Multi-row mode: find max row count across db_shards
+        max_rows = 0
+        db_shard_groups.each do |db_shard, _group|
+          result = db_shard_results[db_shard]
+          next if result == {}
+          count = result.is_a?(Array) ? result.length : 0
+          max_rows = count if count > max_rows
+        end
+        return {} if max_rows == 0
+
+        merged = Array.new(max_rows) { Array.new(items.length) { [nil, nil] } }
+        db_shard_groups.each do |db_shard, group|
+          result = db_shard_results[db_shard]
+          next if result == {}
+          rows = result.is_a?(Array) ? result : []
+          rows.each_with_index do |row, row_num|
+            next unless row.is_a?(Array)
+            group[:positions].each_with_index do |orig_pos, db_shard_idx|
+              merged[row_num][orig_pos] = row[db_shard_idx] if row[db_shard_idx]
+            end
+          end
+        end
+        merged
+      end
+    end
+
+    # Execute a tsdb_lookup query against a single db_shard.
+    # This contains the original ASOF JOIN logic for items all on the same QuestDB instance.
+    def self.tsdb_lookup_single_db_shard(items, start_time:, end_time: nil, scope: "DEFAULT", db_shard: 0)
       tables = {}
       names = []
       nil_count = 0
@@ -887,9 +1024,9 @@ module OpenC3
       query = "SELECT #{names.join(", ")} FROM "
       tables.each_with_index do |(table_name, _), index|
         if index == 0
-          query += "#{table_name} as T#{index} "
+          query += "\"#{table_name}\" as T#{index} "
         else
-          query += "ASOF JOIN #{table_name} as T#{index} "
+          query += "ASOF JOIN \"#{table_name}\" as T#{index} "
         end
       end
       query_params = []
@@ -902,7 +1039,7 @@ module OpenC3
         query_params << end_time
       end
 
-      result = query_with_retry(query, params: query_params, label: "tsdb_lookup")
+      result = query_with_retry(query, params: query_params, label: "tsdb_lookup", db_shard: db_shard)
       if result.nil? or result.ntuples == 0
         return {}
       end