openc3 7.0.1 → 7.1.1

data/lib/openc3/models/plugin_model.rb

@@ -158,12 +158,12 @@ module OpenC3
             variables[current_variable_name]['description'] = params[0]
           when 'VARIABLE_STATE'
             usage = "#{keyword} <Display Text> <Value>"
-            parser.verify_num_parameters(2, 2, usage)
+            parser.verify_num_parameters(1, 2, usage)
             unless current_variable_name
               raise "VARIABLE_STATE must follow a VARIABLE definition"
             end
             variables[current_variable_name]['options'] ||= []
-            option = { 'value' => params[1], 'text' => params[0] }
+            option = { 'value' => params[1] || params[0], 'text' => params[0] }
             variables[current_variable_name]['options'] << option
           end
         end
@@ -229,12 +229,13 @@ module OpenC3
         plugin_model.homepage = pkg.spec.homepage
         plugin_model.repository = pkg.spec.metadata['source_code_uri'] # this key because it's in the official gemspec examples
         plugin_model.keywords = pkg.spec.metadata['openc3_store_keywords']&.split(/, ?/)
-        img_path = pkg.spec.metadata['openc3_store_image']
-        unless img_path
-          default_img_path = 'public/store_img.png'
-          full_default_path = File.join(gem_path, default_img_path)
-          img_path = default_img_path if File.exist? full_default_path
-        end
+        # Resolve the plugin's store image path. The gemspec generated by
+        # `cli generate plugin` sets `openc3_store_image` to `public/store_img.png`
+        # but does not create that file, so we must verify existence regardless
+        # of whether the path came from metadata or the default — otherwise the
+        # frontend hits a 500 when fetching the missing file.
+        img_path = pkg.spec.metadata['openc3_store_image'] || 'public/store_img.png'
+        img_path = nil unless File.exist?(File.join(gem_path, img_path))
         package_name = "#{pkg.spec.name}-#{pkg.spec.version}"
         plugin_model.img_path = File.join('gems', package_name, img_path) if img_path # convert this filesystem path to volumes mount path
         plugin_model.update() unless validate_only
@@ -525,5 +526,16 @@ module OpenC3
       end
       return result.sort
     end
+
+    # Remove the backing gem for an unloaded plugin so it disappears from
+    # GemModel.names (and the admin Packages tab). Skips the removal if any
+    # other PluginModel (any scope, any counter) still references the gem,
+    # via the existing PluginModel.gem_names check inside GemModel.destroy.
+    def self.cleanup_gem(plugin_name, scope:)
+      gem_filename = plugin_name.split("__")[0]
+      OpenC3::GemModel.destroy(gem_filename, log_and_raise_needed_errors: false)
+    rescue => e
+      Logger.warn("Could not remove gem #{gem_filename}: #{e.message}", scope: scope)
+    end
   end
 end

data/lib/openc3/models/queue_model.rb

@@ -44,34 +44,37 @@ module OpenC3
       model = get_model(name: name, scope: scope)
       raise QueueError, "Queue '#{name}' not found in scope '#{scope}'" unless model
 
-      if model.state != 'DISABLE'
-        result = Store.zrevrange("#{scope}:#{name}", 0, 0, with_scores: true)
-        if result.empty?
-          id = 1.0
-        else
-          id = result[0][1].to_f + 1
-        end
-
-        # Build command data with support for both formats
-        command_data = { username: username, timestamp: Time.now.to_nsec_from_epoch, validate: validate, timeout: timeout }
-        if target_name && cmd_name
-          # New format: store target_name, cmd_name, and cmd_params separately
-          command_data[:target_name] = target_name
-          command_data[:cmd_name] = cmd_name
-          command_data[:cmd_params] = JSON.generate(cmd_params.as_json, allow_nan: true) if cmd_params
-        elsif command
-          # Legacy format: store command string for backwards compatibility
-          command_data[:value] = command
-        else
-          raise QueueError, "Must provide either command string or target_name/cmd_name parameters"
-        end
-
-        Store.zadd("#{scope}:#{name}", id, command_data.to_json)
-        model.notify(kind: 'command')
-      else
+      if model.state == 'DISABLE'
         error_msg = command || "#{target_name} #{cmd_name}"
         raise QueueError, "Queue '#{name}' is disabled. Command '#{error_msg}' not queued."
       end
+
+      result = Store.zrevrange("#{scope}:#{name}", 0, 0, with_scores: true)
+      id = result.empty? ? 1.0 : result[0][1].to_f + 1
+
+      command_data = build_command_data(username: username, command: command, target_name: target_name,
+                                        cmd_name: cmd_name, cmd_params: cmd_params, validate: validate, timeout: timeout)
+      Store.zadd("#{scope}:#{name}", id, command_data.to_json)
+      model.notify(kind: 'command')
+    end
+
+    # Build the hash that gets serialized into Redis. Always uses symbol keys so
+    # downstream code in this class can access values consistently. cmd_params is
+    # JSON-encoded as a string so binary data survives the round-trip via as_json.
+    def self.build_command_data(username:, command: nil, target_name: nil, cmd_name: nil, cmd_params: nil, validate: nil, timeout: nil)
+      command_data = { username: username, timestamp: Time.now.to_nsec_from_epoch }
+      if target_name && cmd_name
+        command_data[:target_name] = target_name
+        command_data[:cmd_name] = cmd_name
+        command_data[:cmd_params] = JSON.generate(cmd_params.as_json, allow_nan: true) if cmd_params
+      elsif command
+        command_data[:value] = command
+      else
+        raise QueueError, "Must provide either command string or target_name/cmd_name parameters"
+      end
+      command_data[:validate] = validate unless validate.nil?
+      command_data[:timeout] = timeout unless timeout.nil?
+      command_data
     end
 
     attr_accessor :name, :state
@@ -115,49 +118,36 @@ module OpenC3
       QueueTopic.write_notification(notification, scope: @scope)
     end
 
-    def insert_command(id, command_data)
+    def insert_command(id: nil, username:, command: nil, target_name: nil, cmd_name: nil, cmd_params: nil, validate: nil, timeout: nil)
       if @state == 'DISABLE'
-        if command_data['value']
-          command_name = command_data['value']
-        else
-          command_name = "#{command_data['target_name']} #{command_data['cmd_name']}"
-        end
+        command_name = command || "#{target_name} #{cmd_name}"
         raise QueueError, "Queue '#{@name}' is disabled. Command '#{command_name}' not queued."
       end
 
       unless id
         result = Store.zrevrange("#{@scope}:#{@name}", 0, 0, with_scores: true)
-        if result.empty?
-          id = 1.0
-        else
-          id = result[0][1].to_f + 1
-        end
+        id = result.empty? ? 1.0 : result[0][1].to_f + 1
       end
 
-      # Convert cmd_params values to JSON-safe format if present
-      if command_data['cmd_params']
-        command_data['cmd_params'] = JSON.generate(command_data['cmd_params'].as_json, allow_nan: true)
-      end
+      command_data = self.class.build_command_data(username: username, command: command, target_name: target_name,
+                                                   cmd_name: cmd_name, cmd_params: cmd_params, validate: validate, timeout: timeout)
       Store.zadd("#{@scope}:#{@name}", id, command_data.to_json)
       notify(kind: 'command')
     end
 
-    def update_command(id:, command:, username:, validate: nil, timeout: nil)
+    def update_command(id:, username:, command: nil, target_name: nil, cmd_name: nil, cmd_params: nil, validate: nil, timeout: nil)
       if @state == 'DISABLE'
         raise QueueError, "Queue '#{@name}' is disabled. Command at id #{id} not updated."
       end
 
-      # Check if command exists at the given id
       existing = Store.zrangebyscore("#{@scope}:#{@name}", id, id)
       if existing.empty?
         raise QueueError, "No command found at id #{id} in queue '#{@name}'"
       end
 
-      # Remove the existing command and add the new one at the same id
       Store.zremrangebyscore("#{@scope}:#{@name}", id, id)
-      command_data = { username: username, value: command, timestamp: Time.now.to_nsec_from_epoch }
-      command_data[:validate] = validate unless validate.nil?
-      command_data[:timeout] = timeout unless timeout.nil?
+      command_data = self.class.build_command_data(username: username, command: command, target_name: target_name,
+                                                   cmd_name: cmd_name, cmd_params: cmd_params, validate: validate, timeout: timeout)
       Store.zadd("#{@scope}:#{@name}", id, command_data.to_json)
       notify(kind: 'command')
     end

data/lib/openc3/models/reingest_job_model.rb

@@ -0,0 +1,153 @@
+# encoding: ascii-8bit
+
+# Copyright 2026 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE.md for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'openc3/models/model'
+
+module OpenC3
+  # Tracks one run of OpenC3::ReingestJob. The job updates this record from a
+  # background thread; the storage_controller status endpoint reads it.
+  # `updated_at` doubles as the heartbeat — if a Running record hasn't been
+  # touched in STALE_THRESHOLD_SEC, the status endpoint surfaces it as 'Stale'.
+  class ReingestJobModel < Model
+    PRIMARY_KEY = 'openc3_reingest_job'
+    STALE_THRESHOLD_SEC = 60
+
+    STATES = %w[Queued Running Complete Crashed Stale].freeze
+    PHASES = %w[downloading enabling_dedup ingesting dedup_cooldown disabling_dedup].freeze
+
+    attr_accessor :state
+    attr_accessor :files
+    attr_accessor :bucket
+    attr_accessor :path
+    attr_accessor :table_names
+    attr_accessor :target_version
+    attr_accessor :versions_used
+    attr_accessor :warnings
+    attr_accessor :progress_phase
+    attr_accessor :progress_current
+    attr_accessor :progress_total
+    attr_accessor :packets_written
+    attr_accessor :dedup_enabled_by_us
+    attr_accessor :dedup_preexisting
+    attr_accessor :dedup_disabled_tables
+    attr_accessor :dedup_cooldown_seconds
+    attr_accessor :dedup_enabled_at
+    attr_accessor :dedup_disabled_at
+    attr_accessor :error
+    attr_accessor :started_at
+    attr_accessor :finished_at
+
+    def self.get(name:, scope:)
+      super("#{scope}__#{PRIMARY_KEY}", name: name)
+    end
+
+    def self.names(scope:)
+      super("#{scope}__#{PRIMARY_KEY}")
+    end
+
+    def self.all(scope:)
+      all = super("#{scope}__#{PRIMARY_KEY}")
+      all.sort_by { |_key, value| value['updated_at'] }.reverse
+    end
+
+    def initialize(
+      name:,
+      state: 'Queued',
+      files: [],
+      bucket: nil,
+      path: nil,
+      table_names: [],
+      target_version: 'as_logged',
+      versions_used: [],
+      warnings: [],
+      progress_phase: nil,
+      progress_current: 0,
+      progress_total: 0,
+      packets_written: 0,
+      dedup_enabled_by_us: [],
+      dedup_preexisting: [],
+      dedup_disabled_tables: [],
+      dedup_cooldown_seconds: 60,
+      dedup_enabled_at: nil,
+      dedup_disabled_at: nil,
+      error: nil,
+      started_at: nil,
+      finished_at: nil,
+      updated_at: nil,
+      plugin: nil,
+      scope:
+    )
+      super("#{scope}__#{PRIMARY_KEY}", name: name, updated_at: updated_at, plugin: plugin, scope: scope)
+      @state = state
+      @files = files
+      @bucket = bucket
+      @path = path
+      @table_names = table_names
+      @target_version = target_version
+      @versions_used = versions_used
+      @warnings = warnings
+      @progress_phase = progress_phase
+      @progress_current = progress_current
+      @progress_total = progress_total
+      @packets_written = packets_written
+      @dedup_enabled_by_us = dedup_enabled_by_us
+      @dedup_preexisting = dedup_preexisting
+      @dedup_disabled_tables = dedup_disabled_tables
+      @dedup_cooldown_seconds = dedup_cooldown_seconds
+      @dedup_enabled_at = dedup_enabled_at
+      @dedup_disabled_at = dedup_disabled_at
+      @error = error
+      @started_at = started_at
+      @finished_at = finished_at
+    end
+
+    # True if state is Running but the heartbeat (updated_at) is older than
+    # STALE_THRESHOLD_SEC. Callers should surface state as 'Stale' in that case.
+    def stale?
+      return false unless @state == 'Running'
+      return false unless @updated_at
+      age_nsec = Time.now.to_nsec_from_epoch - @updated_at.to_i
+      age_nsec > STALE_THRESHOLD_SEC * 1_000_000_000
+    end
+
+    def as_json(*_a)
+      {
+        'name' => @name,
+        'state' => stale? ? 'Stale' : @state,
+        'files' => @files,
+        'bucket' => @bucket,
+        'path' => @path,
+        'table_names' => @table_names,
+        'target_version' => @target_version,
+        'versions_used' => @versions_used,
+        'warnings' => @warnings,
+        'progress_phase' => @progress_phase,
+        'progress_current' => @progress_current,
+        'progress_total' => @progress_total,
+        'packets_written' => @packets_written,
+        'dedup_enabled_by_us' => @dedup_enabled_by_us,
+        'dedup_preexisting' => @dedup_preexisting,
+        'dedup_disabled_tables' => @dedup_disabled_tables,
+        'dedup_cooldown_seconds' => @dedup_cooldown_seconds,
+        'dedup_enabled_at' => @dedup_enabled_at,
+        'dedup_disabled_at' => @dedup_disabled_at,
+        'error' => @error,
+        'started_at' => @started_at,
+        'finished_at' => @finished_at,
+        'updated_at' => @updated_at,
+        'plugin' => @plugin,
+        'scope' => @scope,
+      }
+    end
+  end
+end

data/lib/openc3/models/scope_model.rb

@@ -389,8 +389,9 @@ module OpenC3
       end
 
       # Delete the topics we created for the scope
-      Topic.del("#{@scope}__COMMAND__{UNKNOWN}__UNKNOWN")
-      Topic.del("#{@scope}__TELEMETRY__{UNKNOWN}__UNKNOWN")
+      db_shard = Store.db_shard_for_target('UNKNOWN', scope: @scope)
+      Topic.del("#{@scope}__COMMAND__{UNKNOWN}__UNKNOWN", db_shard: db_shard)
+      Topic.del("#{@scope}__TELEMETRY__{UNKNOWN}__UNKNOWN", db_shard: db_shard)
       Topic.del("#{@scope}__openc3_targets")
       Topic.del("#{@scope}__CONFIG")
     end

data/lib/openc3/models/script_status_model.rb

@@ -66,17 +66,9 @@ module OpenC3
         keys = self.store.zrevrange("#{RUNNING_PRIMARY_KEY}__#{scope}__LIST", offset.to_i, offset.to_i + limit.to_i - 1)
         return [] if keys.empty?
         result = []
-        if $openc3_redis_cluster
-          # No pipelining for cluster mode
-          # because it requires using the same shard for all keys
+        result = self.store.redis_pool.pipelined do
           keys.each do |key|
-            result << self.store.hget("#{RUNNING_PRIMARY_KEY}__#{scope}", key)
-          end
-        else
-          result = self.store.redis_pool.pipelined do
-            keys.each do |key|
-              self.store.hget("#{RUNNING_PRIMARY_KEY}__#{scope}", key)
-            end
+            self.store.hget("#{RUNNING_PRIMARY_KEY}__#{scope}", key)
           end
         end
         result = result.map do |r|
@@ -91,17 +83,9 @@ module OpenC3
         keys = self.store.zrevrange("#{COMPLETED_PRIMARY_KEY}__#{scope}__LIST", offset.to_i, offset.to_i + limit.to_i - 1)
         return [] if keys.empty?
         result = []
-        if $openc3_redis_cluster
-          # No pipelining for cluster mode
-          # because it requires using the same shard for all keys
+        result = self.store.redis_pool.pipelined do
           keys.each do |key|
-            result << self.store.hget("#{COMPLETED_PRIMARY_KEY}__#{scope}", key)
-          end
-        else
-          result = self.store.redis_pool.pipelined do
-            keys.each do |key|
-              self.store.hget("#{COMPLETED_PRIMARY_KEY}__#{scope}", key)
-            end
+            self.store.hget("#{COMPLETED_PRIMARY_KEY}__#{scope}", key)
           end
         end
         result = result.map do |r|