openc3 7.0.1 → 7.1.0

data/lib/openc3/utilities/questdb_client.rb

@@ -28,42 +28,104 @@ module OpenC3
     class QuestDBError < StandardError; end
 
     # Thread-local PG connection storage using Concurrent::ThreadLocalVar.
-    # Each thread gets its own connection to avoid thread-safety issues with PG::Connection.
+    # Each thread gets its own connections (per db_shard) to avoid thread-safety issues with PG::Connection.
     # Connections are automatically garbage collected when threads terminate.
-    @thread_conn = Concurrent::ThreadLocalVar.new(nil)
+    # Value is a Hash: { db_shard_number => PG::Connection }
+    @thread_conns = Concurrent::ThreadLocalVar.new { Hash.new } # NOSONAR
+
+    # DB_Shard cache: { "scope__target_name" => [db_shard_number, Time] }
+    @db_shard_cache = {}
+    @db_shard_cache_mutex = Mutex.new
+    DB_SHARD_CACHE_TIMEOUT = 60 # seconds
+
+    # Resolve the hostname for a given db_shard number.
+    # If OPENC3_TSDB_HOSTNAME contains "SHARDNUM", it is replaced with the db_shard number.
+    # Otherwise, all db_shards connect to the same host (backward compatible).
+    def self.hostname_for_db_shard(db_shard)
+      ENV['OPENC3_TSDB_HOSTNAME'].to_s.gsub("SHARDNUM", db_shard.to_s)
+    end
+
+    # Look up the db_shard number for a target from TargetModel with a 1-minute cache.
+    # Non-target-specific data (nil target_name) always returns db_shard 0.
+    def self.db_shard_for_target(target_name, scope: "DEFAULT")
+      return 0 unless target_name
+
+      cache_key = "#{scope}__#{target_name}"
+      now = Time.now
+
+      @db_shard_cache_mutex.synchronize do
+        cached = @db_shard_cache[cache_key]
+        if cached
+          db_shard, cached_at = cached
+          return db_shard if (now - cached_at) < DB_SHARD_CACHE_TIMEOUT
+        end
+      end
+
+      # Cache miss or expired — look up from TargetModel
+      begin
+        model = TargetModel.get(name: target_name, scope: scope)
+        db_shard = model ? model['db_shard'].to_i : 0
+      rescue
+        db_shard = 0
+      end
+
+      @db_shard_cache_mutex.synchronize do
+        @db_shard_cache[cache_key] = [db_shard, now]
+      end
+
+      db_shard
+    end
 
-    # Get or create a thread-local PG connection with type mapping configured.
+    # Get or create a thread-local PG connection for the given db_shard with type mapping configured.
     # Returns the thread-local connection - callers should not close it.
-    def self.connection
-      conn = @thread_conn.value
-      if conn.nil? || conn.finished?
-        conn = PG::Connection.new(
-          host: ENV['OPENC3_TSDB_HOSTNAME'],
-          port: ENV['OPENC3_TSDB_QUERY_PORT'],
-          user: ENV['OPENC3_TSDB_USERNAME'],
-          password: ENV['OPENC3_TSDB_PASSWORD'],
-          dbname: 'qdb'
-        )
-        conn.type_map_for_results = PG::BasicTypeMapForResults.new(conn)
-        @thread_conn.value = conn
+    def self.connection(db_shard: 0)
+      conns = @thread_conns.value
+      conn = conns[db_shard]
+      if conn and not conn.finished?
+        begin
+          conn.check_socket
+          return conn
+        rescue
+          # Will need to reconnect
+        end
       end
+      conn = PG::Connection.new(
+        host: hostname_for_db_shard(db_shard),
+        port: ENV['OPENC3_TSDB_QUERY_PORT'],
+        user: ENV['OPENC3_TSDB_USERNAME'],
+        password: ENV['OPENC3_TSDB_PASSWORD'],
+        dbname: 'qdb'
+      )
+      conn.type_map_for_results = PG::BasicTypeMapForResults.new(conn)
+      conns[db_shard] = conn
+      @thread_conns.value = conns
       conn
     end
 
-    # Reset the connection for the current thread. Used after errors.
-    def self.disconnect
-      conn = @thread_conn.value
-      if conn && !conn.finished?
-        conn.finish
+    # Reset the connection(s) for the current thread. Used after errors.
+    # If db_shard is nil, closes all db_shard connections. Otherwise closes only the specified db_shard.
+    def self.disconnect(db_shard: nil)
+      conns = @thread_conns.value
+      if db_shard.nil?
+        conns.each_value do |conn|
+          conn.finish if conn && !conn.finished?
+        end
+        @thread_conns.value = {}
+      else
+        conn = conns[db_shard]
+        if conn && !conn.finished?
+          conn.finish
+        end
+        conns.delete(db_shard)
+        @thread_conns.value = conns
       end
-      @thread_conn.value = nil
     end
 
     # Health check - attempt to connect and immediately close.
     # Returns true if successful, raises on failure.
-    def self.check_connection
+    def self.check_connection(db_shard: 0)
       conn = PG::Connection.new(
-        host: ENV['OPENC3_TSDB_HOSTNAME'],
+        host: hostname_for_db_shard(db_shard),
         port: ENV['OPENC3_TSDB_QUERY_PORT'],
         user: ENV['OPENC3_TSDB_USERNAME'],
         password: ENV['OPENC3_TSDB_PASSWORD'],
@@ -291,14 +353,14 @@ module OpenC3
     # @param label [String, nil] Optional label for log messages
     # @return [PG::Result, nil] Query result
     # @raise [RuntimeError] After exhausting retries
-    def self.query_with_retry(query, params: [], max_retries: 5, label: nil)
+    def self.query_with_retry(query, params: [], max_retries: 5, label: nil, db_shard: 0)
       retry_count = 0
       begin
-        conn = connection
+        conn = connection(db_shard: db_shard)
         if params.empty?
-          conn.exec(query)
+          return conn.exec(query)
         else
-          conn.exec_params(query, params)
+          return conn.exec_params(query, params)
         end
       rescue IOError, PG::Error => e
         retry_count += 1
@@ -307,7 +369,7 @@ module OpenC3
         end
         Logger.warn("TSDB#{label ? " #{label}" : ""}: Retrying due to error: #{e.message}")
         Logger.warn("TSDB#{label ? " #{label}" : ""}: Last query: #{query}")
-        disconnect
+        disconnect(db_shard: db_shard)
         sleep 0.1
         retry
       end
@@ -543,11 +605,11 @@ module OpenC3
     # @param start_time [Integer] Nanosecond start time
     # @param end_time [Integer, nil] Nanosecond end time
     # @return [Boolean]
-    def self.table_has_data?(table_name, start_time, end_time)
-      query = "SELECT 1 FROM #{table_name}"
+    def self.table_has_data?(table_name, start_time, end_time, db_shard: 0)
+      query = "SELECT 1 FROM \"#{table_name}\""
       query += time_where_clause(start_time, end_time)
       query += " LIMIT 1"
-      result = query_with_retry(query, max_retries: 1, label: "table_has_data")
+      result = query_with_retry(query, max_retries: 1, label: "table_has_data", db_shard: db_shard)
       result && result.ntuples > 0
     rescue RuntimeError
       false
@@ -560,13 +622,13 @@ module OpenC3
     # @param page_size [Integer] Number of rows per page
     # @param label [String] Label for log messages
     # @yield [PG::Result] Each page of results
-    def self.paginate_query(query, page_size, label:)
+    def self.paginate_query(query, page_size, label:, db_shard: 0)
       min = 0
       max = page_size
       loop do
         query_offset = "#{query} LIMIT #{min}, #{max}"
         Logger.debug("QuestDB #{label}: #{query_offset}")
-        result = query_with_retry(query_offset, label: label)
+        result = query_with_retry(query_offset, label: label, db_shard: db_shard)
         min += page_size
         max += page_size
         if result.nil? or result.ntuples == 0
@@ -590,7 +652,7 @@ module OpenC3
       names << TIMESTAMP_SELECT
       names << "RECEIVED_TIMESECONDS" if include_received_ts
       names << "COSMOS_EXTRA"
-      query = "SELECT #{names.join(', ')} FROM #{table_name}"
+      query = "SELECT #{names.join(', ')} FROM \"#{table_name}\""
       query += time_where_clause(start_time, end_time)
       query
     end
@@ -808,6 +870,8 @@ module OpenC3
 
     # Query historical telemetry data from QuestDB for a list of items.
     # Builds the SQL query, executes it, and decodes all results.
+    # Supports cross-db_shard queries by grouping items by db_shard, executing
+    # separate queries per db_shard, and merging results positionally.
     #
     # @param items [Array] Array of [target_name, packet_name, item_name, value_type, limits]
     #   item_name may be nil to indicate a placeholder (non-existent item)
@@ -817,6 +881,78 @@ module OpenC3
     # @return [Array, Hash] Array of [value, limits_state] pairs per row, or {} if no results.
     #   Single-row results return a flat array; multi-row results return array of arrays.
     def self.tsdb_lookup(items, start_time:, end_time: nil, scope: "DEFAULT")
+      # Group items by db_shard number while preserving their original positions
+      db_shard_groups = {} # db_shard => { positions: [], items: [] }
+      items.each_with_index do |item, pos|
+        target_name = item[0]
+        db_shard = db_shard_for_target(target_name, scope: scope)
+        db_shard_groups[db_shard] ||= { positions: [], items: [] }
+        db_shard_groups[db_shard][:positions] << pos
+        db_shard_groups[db_shard][:items] << item
+      end
+
+      # Single-db_shard fast path (most common case)
+      if db_shard_groups.length == 1
+        db_shard, group = db_shard_groups.first
+        return tsdb_lookup_single_db_shard(group[:items], start_time: start_time, end_time: end_time, scope: scope, db_shard: db_shard)
+      end
+
+      # Cross-db_shard: execute per-db_shard queries and merge results
+      db_shard_results = {} # db_shard => data
+      db_shard_groups.each do |db_shard, group|
+        result = tsdb_lookup_single_db_shard(group[:items], start_time: start_time, end_time: end_time, scope: scope, db_shard: db_shard)
+        db_shard_results[db_shard] = result
+      end
+
+      # If all db_shards returned empty, return empty
+      return {} if db_shard_results.values.all? { |r| r == {} }
+
+      # Merge results positionally back into the original item order.
+      # For single-row results (no end_time), merge flat arrays.
+      # For multi-row results, each db_shard may have different row counts;
+      # use the maximum row count and fill missing positions with [nil, nil].
+      if !end_time
+        # Single-row mode: each db_shard returns a flat array of [value, limits] pairs.
+        # Merge them into the original item order.
+        merged = Array.new(items.length) { [nil, nil] }
+        db_shard_groups.each do |db_shard, group|
+          result = db_shard_results[db_shard]
+          next if result == {} || !result.is_a?(Array)
+          group[:positions].each_with_index do |orig_pos, db_shard_idx|
+            merged[orig_pos] = result[db_shard_idx] if result[db_shard_idx]
+          end
+        end
+        merged
+      else
+        # Multi-row mode: find max row count across db_shards
+        max_rows = 0
+        db_shard_groups.each do |db_shard, _group|
+          result = db_shard_results[db_shard]
+          next if result == {}
+          count = result.is_a?(Array) ? result.length : 0
+          max_rows = count if count > max_rows
+        end
+        return {} if max_rows == 0
+
+        merged = Array.new(max_rows) { Array.new(items.length) { [nil, nil] } }
+        db_shard_groups.each do |db_shard, group|
+          result = db_shard_results[db_shard]
+          next if result == {}
+          rows = result.is_a?(Array) ? result : []
+          rows.each_with_index do |row, row_num|
+            next unless row.is_a?(Array)
+            group[:positions].each_with_index do |orig_pos, db_shard_idx|
+              merged[row_num][orig_pos] = row[db_shard_idx] if row[db_shard_idx]
+            end
+          end
+        end
+        merged
+      end
+    end
+
+    # Execute a tsdb_lookup query against a single db_shard.
+    # This contains the original ASOF JOIN logic for items all on the same QuestDB instance.
+    def self.tsdb_lookup_single_db_shard(items, start_time:, end_time: nil, scope: "DEFAULT", db_shard: 0)
       tables = {}
       names = []
       nil_count = 0
@@ -888,9 +1024,9 @@ module OpenC3
       query = "SELECT #{names.join(", ")} FROM "
       tables.each_with_index do |(table_name, _), index|
         if index == 0
-          query += "#{table_name} as T#{index} "
+          query += "\"#{table_name}\" as T#{index} "
         else
-          query += "ASOF JOIN #{table_name} as T#{index} "
+          query += "ASOF JOIN \"#{table_name}\" as T#{index} "
         end
       end
       query_params = []
@@ -903,7 +1039,7 @@ module OpenC3
         query_params << end_time
       end
 
-      result = query_with_retry(query, params: query_params, label: "tsdb_lookup")
+      result = query_with_retry(query, params: query_params, label: "tsdb_lookup", db_shard: db_shard)
       if result.nil? or result.ntuples == 0
         return {}
       end

data/lib/openc3/utilities/reingest_job.rb

@@ -0,0 +1,377 @@
+# encoding: ascii-8bit
+
+# Copyright 2026 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE.md for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'fileutils'
+require 'tmpdir'
+require 'openc3/system/system'
+require 'openc3/utilities/bucket'
+require 'openc3/utilities/bucket_utilities'
+require 'openc3/utilities/logger'
+require 'openc3/utilities/questdb_client'
+require 'openc3/logs/packet_log_reader'
+require 'openc3/microservices/decom_common'
+require 'openc3/models/reingest_job_model'
+
+module OpenC3
+  class ReingestJobError < StandardError; end
+
+  # Replays raw .bin.gz log files from a bucket, decommutating each packet via
+  # DecomCommon.decom_and_publish(check_limits: false) so historical data
+  # reaches QuestDB without re-firing limits events.
+  #
+  # Runs synchronously (caller wraps in a Thread). Tracks state in a
+  # ReingestJobModel. DEDUP is enabled on affected tables during the job and
+  # disabled in an ensure block on completion (or after a cooldown window so
+  # in-flight WAL commits are covered).
+  #
+  # target_version:
+  #   - 'as_logged' (default): each file is decoded with the target config hash
+  #     that was in effect when the packets were originally logged. Files are
+  #     grouped by their embedded target_id and System is rebuilt per group.
+  #   - 'current': all files are decoded with the latest target config.
+  #   - <hash>: explicit hash, used for every file in the job.
+  class ReingestJob
+    # How often to persist progress during the ingest pass (write every N packets)
+    STATUS_UPDATE_EVERY = 500
+    # How often to tick the heartbeat during the cooldown sleep
+    HEARTBEAT_INTERVAL_SEC = 10
+
+    # Reingest rebuilds the process-global System singleton. Serialize all
+    # reingest jobs running in this process so they don't stomp each other.
+    @@run_mutex = Mutex.new
+
+    def initialize(job_id:, files:, path:, bucket:, scope:,
+                   target_version: 'as_logged',
+                   dedup_cooldown_seconds: ENV.fetch('OPENC3_REINGEST_DEDUP_COOLDOWN', 60).to_i,
+                   logger: Logger)
+      @job_id = job_id
+      @files = files
+      @path = path
+      @bucket_env = bucket
+      @scope = scope
+      @target_version = target_version
+      @dedup_cooldown_seconds = dedup_cooldown_seconds
+      @logger = logger
+    end
+
+    def run
+      tmp_dir = Dir.mktmpdir
+      job = load_job
+      dedup_enabled_by_us = []
+      db_shard = 0
+      @@run_mutex.synchronize do
+        begin
+          mark(job, state: 'Running', progress_phase: 'downloading',
+               started_at: Time.now.utc.iso8601,
+               progress_total: @files.length)
+
+          # Parse target from path, e.g. "DEFAULT/raw_logs/tlm/INST/20260421/"
+          # → "INST". Fail fast if the path doesn't encode one — otherwise
+          # ingest would run against whatever System was loaded in this process
+          # from a prior job (or raise opaquely inside PacketLogReader), and
+          # the job could mark Complete with rows written under the wrong
+          # target config.
+          path_parts = @path.to_s.split('/').reject(&:empty?)
+          unless path_parts.length >= 4 && path_parts[1] == 'raw_logs'
+            raise ReingestJobError, "Cannot determine target from path '#{@path}'; expected '{scope}/raw_logs/{tlm|cmd}/{target}/'"
+          end
+          target = path_parts[3]
+          db_shard = QuestDBClient.db_shard_for_target(target, scope: @scope)
+
+          local_files = download_and_uncompress(job, tmp_dir)
+
+          # Pass 1: read raw (no System required) to discover table names and
+          # each file's embedded target hash. File hashes are what the "as
+          # logged" mode uses to pick the right target_version per file.
+          mark(job, progress_phase: 'enabling_dedup', progress_current: 0,
+               progress_total: 0)
+          table_names, file_versions = discover_tables_and_versions(local_files)
+          mark(job, table_names: table_names, progress_total: table_names.length)
+
+          dedup_enabled_by_us, preexisting = enable_dedup(job, table_names, db_shard)
+          mark(job,
+               dedup_enabled_by_us: dedup_enabled_by_us,
+               dedup_preexisting: preexisting,
+               dedup_enabled_at: Time.now.utc.iso8601)
+
+          # Pass 2: group files by the target_version we'll load for them,
+          # then ingest each group under its own System instance.
+          groups = group_files_by_version(local_files, file_versions)
+          mark(job, versions_used: groups.keys,
+               progress_phase: 'ingesting', progress_current: 0,
+               progress_total: 0, packets_written: 0)
+          ingest_all_groups(job, groups, target)
+
+          mark(job, progress_phase: 'dedup_cooldown')
+          cooldown(job)
+
+          mark(job, progress_phase: 'disabling_dedup')
+          disabled = disable_dedup(job, dedup_enabled_by_us, db_shard)
+          mark(job, dedup_disabled_tables: disabled,
+               dedup_disabled_at: Time.now.utc.iso8601,
+               state: 'Complete',
+               finished_at: Time.now.utc.iso8601)
+        rescue Exception => e
+          @logger.error("Reingest job #{@job_id} failed: #{e.message}\n#{e.backtrace.first(10).join("\n")}")
+          # Always try to revert DEDUP even on crash so user tables are not left altered
+          disabled_on_crash = []
+          begin
+            disabled_on_crash = disable_dedup(job, dedup_enabled_by_us, db_shard)
+          rescue => de
+            @logger.error("Reingest job #{@job_id} failed to disable DEDUP during crash cleanup: #{de.message}")
+          end
+          mark(job,
+               dedup_disabled_tables: disabled_on_crash,
+               dedup_disabled_at: Time.now.utc.iso8601,
+               state: 'Crashed',
+               error: e.message,
+               finished_at: Time.now.utc.iso8601)
+        ensure
+          FileUtils.remove_entry_secure(tmp_dir, true) if tmp_dir && File.directory?(tmp_dir)
+        end
+      end
+    end
+
+    private
+
+    def load_job
+      ReingestJobModel.get_model(name: @job_id, scope: @scope) or
+        raise ReingestJobError, "ReingestJobModel #{@job_id} not found in scope #{@scope}"
+    end
+
+    # Merge attrs into the model and persist. Model#update refreshes updated_at,
+    # which doubles as the heartbeat used by the stale-check.
+    def mark(job, **attrs)
+      attrs.each { |k, v| job.send("#{k}=", v) }
+      job.update
+    end
+
+    def download_and_uncompress(job, tmp_dir)
+      bucket_name = ENV.fetch(@bucket_env) { |name| raise ReingestJobError, "Unknown bucket #{name}" }
+      bucket_client = Bucket.getClient()
+      local_files = []
+      tmp_root = File.expand_path(tmp_dir) + File::SEPARATOR
+      @files.each_with_index do |filename, i|
+        key = "#{@path}#{filename}"
+        temp_file = File.expand_path(File.join(tmp_dir, filename))
+        # Defense-in-depth: the controller validates filenames, but refuse to
+        # write outside tmp_dir if any caller (tests, future callers) bypasses it.
+        unless temp_file.start_with?(tmp_root)
+          raise ReingestJobError, "Invalid filename escapes tmp dir: #{filename}"
+        end
+        FileUtils.mkdir_p(File.dirname(temp_file))
+        bucket_client.get_object(bucket: bucket_name, key: key, path: temp_file)
+        if File.extname(filename) == '.gz'
+          decompressed = BucketUtilities.uncompress_file(temp_file)
+          File.delete(temp_file)
+          local_files << decompressed
+        else
+          local_files << temp_file
+        end
+        mark(job, progress_current: i + 1)
+      end
+      local_files
+    end
+
+    # Read each file in raw mode (identify_and_define=false) to collect
+    # {scope}__TLM__{target}__{packet} table names and the first target
+    # declaration hash embedded in each file. Returns [table_names, file_versions]
+    # where file_versions maps local_file_path → hex hash (or nil if the file
+    # has no hash, which happens for pre-6.x log files).
+    def discover_tables_and_versions(local_files)
+      table_names = Set.new
+      file_versions = {}
+      local_files.each do |local_file|
+        reader = PacketLogReader.new
+        reader.each(local_file, false) do |packet|
+          next unless packet.target_name && packet.packet_name
+          cmd_or_tlm = packet.cmd_or_tlm == :CMD ? 'CMD' : 'TLM'
+          table_names.add("#{@scope}__#{cmd_or_tlm}__#{packet.target_name}__#{packet.packet_name}")
+        end
+        ids = reader.instance_variable_get(:@target_ids) || []
+        file_versions[local_file] = ids.first ? ids.first.unpack1('H*') : nil
+      end
+      [table_names.to_a, file_versions]
+    end
+
+    # Returns a Hash of target_version → [local_file, ...]. The value at key
+    # 'current' means "use System with the latest config"; any other value is
+    # a specific hash string used as target_version in System.setup_targets.
+    def group_files_by_version(local_files, file_versions)
+      groups = Hash.new { |h, k| h[k] = [] }
+      case @target_version
+      when 'current'
+        groups['current'] = local_files.dup
+      when 'as_logged', nil
+        local_files.each do |file|
+          version = file_versions[file] || 'current'
+          groups[version] << file
+        end
+      else
+        # Caller passed an explicit hash; use it for every file.
+        groups[@target_version] = local_files.dup
+      end
+      groups
+    end
+
+    # For each version group, rebuild System under that version and ingest
+    # the group's files. `@@run_mutex` in `run` protects other threads from
+    # seeing a transient nil @@instance.
+    #
+    # If the requested target archive (a specific hash) is missing from the
+    # config bucket — which happens in dev setups where every `openc3.sh start`
+    # regenerates the target archive with a fresh timestamp-appended gem
+    # version — we fall back to 'current' and record a warning on the job so
+    # the UI can surface it. This matters because the old historical archive
+    # the log file references may no longer exist.
+    def ingest_all_groups(job, groups, target)
+      packets_written = 0
+      last_status_at = 0
+      warnings = (job.warnings || []).dup
+      groups.each do |version, files|
+        resolved = load_system_with_fallback(target, version, warnings)
+        unless resolved
+          # Even the 'current' fallback failed; skip this group rather than
+          # publish empty json_data for every packet.
+          mark(job, warnings: warnings)
+          next
+        end
+        mark(job, warnings: warnings) if warnings.any?
+        files.each do |file|
+          packets_written, last_status_at = ingest_file(job, file, packets_written, last_status_at)
+        end
+      end
+      mark(job, packets_written: packets_written, warnings: warnings)
+    end
+
+    # Returns the target_version that was actually loaded, or nil if even the
+    # 'current' fallback failed. Appends human-readable entries to `warnings`
+    # for any fallback or failure.
+    def load_system_with_fallback(target, version, warnings)
+      begin
+        load_system(target, version)
+        return version
+      rescue => e
+        if version == 'current'
+          # Caller explicitly requested 'current' and that failed; no further
+          # fallback exists — propagate so the outer rescue marks Crashed.
+          raise
+        end
+        @logger.warn("Reingest job #{@job_id}: target archive for #{target} version '#{version}' unavailable (#{e.class}: #{e.message}); falling back to 'current'")
+        warnings << "Version '#{version}' archive missing; used 'current' instead"
+      end
+
+      begin
+        load_system(target, 'current')
+        'current'
+      rescue => e
+        @logger.error("Reingest job #{@job_id}: fallback to 'current' also failed: #{e.class}: #{e.message}")
+        warnings << "Version '#{version}' archive missing and 'current' also failed (#{e.message})"
+        nil
+      end
+    end
+
+    def load_system(target, version)
+      System.reset_instance!
+      System.setup_targets([target], Dir.tmpdir, scope: @scope, target_version: version)
+    end
+
+    def ingest_file(job, local_file, packets_written, last_status_at)
+      reader = PacketLogReader.new
+      reader.each(local_file, true) do |packet|
+        next unless packet.target_name && packet.packet_name
+        packet.stored = true
+        DecomCommon.decom_and_publish(
+          packet,
+          scope: @scope,
+          target_names: [packet.target_name],
+          logger: @logger,
+          name: "REINGEST:#{@job_id}",
+          check_limits: false,
+        )
+        packets_written += 1
+        if packets_written - last_status_at >= STATUS_UPDATE_EVERY
+          mark(job, packets_written: packets_written)
+          last_status_at = packets_written
+        end
+      end
+      [packets_written, last_status_at]
+    end
+
+    # Returns [enabled_by_us, preexisting]. Only tables we enable are recorded
+    # in enabled_by_us; pre-existing DEDUP tables are left untouched on teardown.
+    def enable_dedup(job, table_names, db_shard)
+      enabled_by_us = []
+      preexisting = []
+      conn = QuestDBClient.connection(db_shard: db_shard)
+      table_names.each_with_index do |table_name, i|
+        begin
+          already = dedup_already_enabled?(conn, table_name)
+          if already
+            preexisting << table_name
+          else
+            conn.exec("ALTER TABLE '#{table_name}' DEDUP ENABLE UPSERT KEYS(PACKET_TIMESECONDS)")
+            enabled_by_us << table_name
+          end
+        rescue => e
+          @logger.warn("Failed to enable DEDUP on #{table_name}: #{e.message}")
+        end
+        mark(job, progress_current: i + 1)
+      end
+      [enabled_by_us, preexisting]
+    end
+
+    # QuestDB exposes per-table dedup status via tables() function.
+    # Falls back to false (treat as not-enabled, will issue ALTER) on any error.
+    def dedup_already_enabled?(conn, table_name)
+      result = conn.exec_params(
+        "SELECT dedup FROM tables() WHERE table_name = $1",
+        [table_name],
+      )
+      return false if result.ntuples == 0
+      value = result[0]['dedup']
+      value == true || value == 't' || value.to_s.downcase == 'true'
+    rescue => e
+      @logger.warn("Could not query DEDUP status for #{table_name}: #{e.message}")
+      false
+    end
+
+    # Sleep dedup_cooldown_seconds, ticking the heartbeat so the stale-check
+    # doesn't misfire during the wait. This gives the Python TsdbMicroservice
+    # and QuestDB WAL time to commit reingested rows while DEDUP is still on.
+    def cooldown(job)
+      remaining = @dedup_cooldown_seconds
+      while remaining > 0
+        step = [HEARTBEAT_INTERVAL_SEC, remaining].min
+        sleep(step)
+        remaining -= step
+        mark(job) # heartbeat only
+      end
+    end
+
+    def disable_dedup(job, tables, db_shard)
+      disabled = []
+      conn = QuestDBClient.connection(db_shard: db_shard)
+      tables.each_with_index do |table_name, i|
+        begin
+          conn.exec("ALTER TABLE '#{table_name}' DEDUP DISABLE")
+          disabled << table_name
+        rescue => e
+          @logger.warn("Failed to disable DEDUP on #{table_name}: #{e.message}")
+        end
+        mark(job, progress_current: i + 1, progress_total: tables.length)
+      end
+      disabled
+    end
+  end
+end

data/lib/openc3/utilities/ruby_lex_utils.rb

@@ -16,7 +16,9 @@
 # if purchased from OpenC3, Inc.
 
 require 'irb/ruby-lex'
+require 'ripper'
 require 'prism'
+require 'ripper'
 
 class RubyLexUtils
   OPENING_DELIMITER_TYPES = %i(PARENTHESIS_LEFT BRACKET_LEFT BRACE_LEFT BRACKET_LEFT_ARRAY)