openc3 7.2.1 → 7.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/openc3cli +97 -6
- data/bin/pipinstall +5 -4
- data/bin/pipuninstall +5 -1
- data/bin/uvinstall +149 -0
- data/data/config/item_modifiers.yaml +2 -1
- data/data/config/microservice.yaml +5 -1
- data/data/config/screen.yaml +2 -1
- data/data/config/target.yaml +21 -0
- data/data/config/tool.yaml +3 -1
- data/data/config/widgets.yaml +16 -3
- data/data/xtce_schemas/SpaceSystem_20180204.xsd +5918 -0
- data/data/xtce_schemas/xml.xsd +116 -0
- data/ext/openc3/ext/telemetry/telemetry.c +1 -1
- data/lib/openc3/accessors/template_accessor.rb +2 -2
- data/lib/openc3/api/interface_api.rb +2 -2
- data/lib/openc3/api/limits_api.rb +27 -13
- data/lib/openc3/api/router_api.rb +3 -3
- data/lib/openc3/api/tlm_api.rb +1 -1
- data/lib/openc3/interfaces/http_client_interface.rb +1 -1
- data/lib/openc3/io/json_drb.rb +24 -8
- data/lib/openc3/io/json_rpc.rb +0 -2
- data/lib/openc3/logs/log_writer.rb +43 -7
- data/lib/openc3/logs/packet_log_writer.rb +1 -2
- data/lib/openc3/microservices/interface_microservice.rb +1 -1
- data/lib/openc3/migrations/20260731000000_system_health_json.rb +41 -0
- data/lib/openc3/models/cvt_model.rb +5 -5
- data/lib/openc3/models/gem_model.rb +1 -1
- data/lib/openc3/models/interface_model.rb +1 -1
- data/lib/openc3/models/microservice_model.rb +21 -0
- data/lib/openc3/models/plugin_model.rb +204 -50
- data/lib/openc3/models/python_package_model.rb +207 -11
- data/lib/openc3/models/scope_model.rb +34 -30
- data/lib/openc3/models/target_model.rb +180 -6
- data/lib/openc3/models/tool_model.rb +33 -20
- data/lib/openc3/operators/microservice_operator.rb +33 -4
- data/lib/openc3/packets/commands.rb +4 -4
- data/lib/openc3/packets/limits.rb +15 -7
- data/lib/openc3/packets/packet.rb +1 -1
- data/lib/openc3/packets/parsers/state_parser.rb +1 -1
- data/lib/openc3/packets/parsers/xtce_converter.rb +183 -103
- data/lib/openc3/packets/parsers/xtce_parser.rb +71 -23
- data/lib/openc3/packets/structure.rb +2 -2
- data/lib/openc3/packets/telemetry.rb +3 -3
- data/lib/openc3/script/commands.rb +1 -1
- data/lib/openc3/script/extract.rb +20 -25
- data/lib/openc3/script/web_socket_api.rb +44 -4
- data/lib/openc3/topics/command_decom_topic.rb +16 -1
- data/lib/openc3/topics/limits_event_topic.rb +1 -1
- data/lib/openc3/utilities/aws_bucket.rb +10 -2
- data/lib/openc3/utilities/csv.rb +5 -5
- data/lib/openc3/utilities/local_mode.rb +4 -1
- data/lib/openc3/utilities/ruby_lex_utils.rb +5 -1
- data/lib/openc3/utilities/running_script.rb +145 -90
- data/lib/openc3/utilities/script.rb +46 -10
- data/lib/openc3/version.rb +6 -6
- data/templates/plugin/plugin.gemspec +1 -1
- data/templates/tool_angular/package.json +2 -2
- data/templates/tool_react/package.json +1 -1
- data/templates/tool_svelte/package.json +1 -1
- data/templates/tool_vue/package.json +3 -3
- data/templates/widget/package.json +2 -2
- metadata +5 -3
- data/templates/tool_vue/.nycrc +0 -3
- data/templates/widget/.nycrc +0 -3
|
@@ -185,11 +185,24 @@ module OpenC3
|
|
|
185
185
|
# Called by the PluginsController to create the plugin
|
|
186
186
|
# Because this uses ERB it must be run in a separate process from the API to
|
|
187
187
|
# prevent corruption and single require problems in the current process
|
|
188
|
-
|
|
188
|
+
# diff_only: dry run that renders the plugin's target files and returns the
|
|
189
|
+
# list of modified files ("TARGET/path") whose live content differs from
|
|
190
|
+
# what this plugin would deploy — used to warn before an upgrade which user
|
|
191
|
+
# modifications it would supersede. Implies validate_only (no side effects).
|
|
192
|
+
def self.install_phase2(plugin_hash, scope:, gem_file_path: nil, validate_only: false, diff_only: false)
|
|
193
|
+
# diff_only implies a dry run; derive a local flag instead of mutating
|
|
194
|
+
# the validate_only parameter.
|
|
195
|
+
dry_run = validate_only || diff_only
|
|
189
196
|
# Register plugin to aid in uninstall if install fails
|
|
190
197
|
plugin_hash.delete("existing_plugin_txt_lines")
|
|
198
|
+
# Version History upgrade hints (threaded from the admin install through
|
|
199
|
+
# update_plugin). Extracted before the splat below because PluginModel
|
|
200
|
+
# has no such attributes. version_history_files lists modified files the
|
|
201
|
+
# user chose to take from the plugin; username attributes the upgrade.
|
|
202
|
+
upgrade_username = plugin_hash.delete("username")
|
|
203
|
+
upgrade_version_files = plugin_hash.delete("version_history_files")
|
|
191
204
|
plugin_model = PluginModel.new(**(plugin_hash.transform_keys(&:to_sym)), scope: scope)
|
|
192
|
-
plugin_model.create unless
|
|
205
|
+
plugin_model.create unless dry_run
|
|
193
206
|
|
|
194
207
|
temp_dir = Dir.mktmpdir
|
|
195
208
|
begin
|
|
@@ -206,10 +219,11 @@ module OpenC3
|
|
|
206
219
|
# Attempt to remove all older versions of this same plugin before install to prevent version conflicts
|
|
207
220
|
# Especially on downgrades
|
|
208
221
|
# Leave the same version if it already exists
|
|
209
|
-
|
|
222
|
+
# Skipped for dry_run/diff_only: a dry run must not mutate gems.
|
|
223
|
+
OpenC3::GemModel.destroy_all_other_versions(File.basename(gem_file_path)) unless dry_run
|
|
210
224
|
|
|
211
225
|
# Actually install the gem now (slow)
|
|
212
|
-
OpenC3::GemModel.install(gem_file_path, scope: scope) unless
|
|
226
|
+
OpenC3::GemModel.install(gem_file_path, scope: scope) unless dry_run
|
|
213
227
|
|
|
214
228
|
# Extract gem contents
|
|
215
229
|
gem_path = File.join(temp_dir, "gem")
|
|
@@ -224,7 +238,7 @@ module OpenC3
|
|
|
224
238
|
|
|
225
239
|
plugin_model.minimum_cosmos_version = pkg.spec.metadata['openc3_cosmos_minimum_version']
|
|
226
240
|
|
|
227
|
-
# Process
|
|
241
|
+
# Process OpenC3 Store metadata
|
|
228
242
|
plugin_model.title = pkg.spec.metadata['openc3_store_title'] || pkg.spec.summary.strip
|
|
229
243
|
plugin_model.description = pkg.spec.metadata['openc3_store_description'] || pkg.spec.description.strip
|
|
230
244
|
plugin_model.licenses = pkg.spec.licenses
|
|
@@ -239,8 +253,19 @@ module OpenC3
|
|
|
239
253
|
img_path = pkg.spec.metadata['openc3_store_image'] || 'public/store_img.png'
|
|
240
254
|
img_path = nil unless File.exist?(File.join(gem_path, img_path))
|
|
241
255
|
package_name = "#{pkg.spec.name}-#{pkg.spec.version}"
|
|
256
|
+
# Build the upgrade context once the gem version is known; TargetModel
|
|
257
|
+
# deploys use it either to collect a modified-file diff (diff_only) or
|
|
258
|
+
# to version modified files taken from the plugin.
|
|
259
|
+
upgrade_context = nil
|
|
260
|
+
if diff_only
|
|
261
|
+
upgrade_context = { diff_collector: [] }
|
|
262
|
+
elsif upgrade_version_files && !upgrade_version_files.empty?
|
|
263
|
+
upgrade_context = { username: upgrade_username,
|
|
264
|
+
plugin: "#{pkg.spec.name} #{pkg.spec.version}",
|
|
265
|
+
version_files: upgrade_version_files }
|
|
266
|
+
end
|
|
242
267
|
plugin_model.img_path = File.join('gems', package_name, img_path) if img_path # convert this filesystem path to volumes mount path
|
|
243
|
-
plugin_model.update() unless
|
|
268
|
+
plugin_model.update() unless dry_run
|
|
244
269
|
|
|
245
270
|
needs_dependencies = pkg.spec.runtime_dependencies.length > 0
|
|
246
271
|
needs_dependencies = true if Dir.exist?(File.join(gem_path, 'lib'))
|
|
@@ -250,46 +275,46 @@ module OpenC3
|
|
|
250
275
|
requirements_path = File.join(gem_path, 'requirements.txt')
|
|
251
276
|
|
|
252
277
|
if File.exist?(pyproject_path) || File.exist?(requirements_path)
|
|
253
|
-
|
|
254
|
-
pypi_url = get_setting('pypi_url', scope: scope)
|
|
255
|
-
if pypi_url
|
|
256
|
-
pypi_url += '/simple'
|
|
257
|
-
end
|
|
258
|
-
rescue => e
|
|
259
|
-
Logger.error("Failed to retrieve pypi_url: #{e.formatted}")
|
|
260
|
-
ensure
|
|
261
|
-
if pypi_url.nil?
|
|
262
|
-
# If Redis isn't running try the ENV, then simply pypi.org/simple
|
|
263
|
-
pypi_url = ENV['PYPI_URL']
|
|
264
|
-
if pypi_url
|
|
265
|
-
pypi_url += '/simple'
|
|
266
|
-
end
|
|
267
|
-
pypi_url ||= PypiUrl::DEFAULT
|
|
268
|
-
end
|
|
269
|
-
end
|
|
270
|
-
pypi_url = PypiUrl.validate(pypi_url)
|
|
278
|
+
pypi_url = resolve_pypi_url(scope: scope)
|
|
271
279
|
unless validate_only
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
#
|
|
275
|
-
#
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
Logger.info "Installing python packages
|
|
283
|
-
|
|
280
|
+
pypi_args = build_pypi_args(pypi_url)
|
|
281
|
+
|
|
282
|
+
# Install Python dependencies into an isolated per-plugin venv when UV
|
|
283
|
+
# is available. Each plugin gets its own venv at /gems/plugin_venvs/<name>/.venv
|
|
284
|
+
# so that plugins with conflicting Python dependency versions don't interfere.
|
|
285
|
+
# If UV is unavailable or the install fails, fall back to the shared pipinstall
|
|
286
|
+
# which installs into PYTHONUSERBASE (the legacy shared environment).
|
|
287
|
+
uv_installed = ENV['OPENC3_USE_UV'] != 'false' && system('which uv > /dev/null 2>&1')
|
|
288
|
+
if uv_installed
|
|
289
|
+
plugin_venv_name = plugin_venv_name(scope: scope, plugin_name: plugin_model.name)
|
|
290
|
+
Logger.info "Installing python packages into per-plugin venv '#{plugin_venv_name}' with pypi_url=#{pypi_url}"
|
|
291
|
+
uv_args = [plugin_venv_name, gem_path] + pypi_args
|
|
292
|
+
output, status = Open3.capture2e("/openc3/bin/uvinstall", *uv_args)
|
|
293
|
+
puts output
|
|
294
|
+
unless status.success?
|
|
295
|
+
Logger.warn "UV per-plugin install failed, falling back to shared pipinstall"
|
|
296
|
+
uv_installed = false
|
|
297
|
+
end
|
|
284
298
|
end
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
299
|
+
|
|
300
|
+
unless uv_installed
|
|
301
|
+
pip_args = pypi_args.dup
|
|
302
|
+
if File.exist?(pyproject_path)
|
|
303
|
+
Logger.info "Installing python packages from pyproject.toml with pypi_url=#{pypi_url}"
|
|
304
|
+
pip_args << gem_path
|
|
305
|
+
else
|
|
306
|
+
Logger.info "Installing python packages from requirements.txt with pypi_url=#{pypi_url}"
|
|
307
|
+
pip_args += ["-r", requirements_path]
|
|
308
|
+
end
|
|
309
|
+
# Capture output and check exit code so failures surface as a warning
|
|
310
|
+
# rather than silently succeeding. pipinstall is non-fatal: the plugin
|
|
311
|
+
# continues to install even if Python packages fail so that non-Python
|
|
312
|
+
# functionality still works.
|
|
313
|
+
output, status = Open3.capture2e("/openc3/bin/pipinstall", *pip_args)
|
|
314
|
+
puts output
|
|
315
|
+
unless status.success?
|
|
316
|
+
Logger.warn "Python package installation failed. Plugin Python microservices may not function correctly."
|
|
317
|
+
end
|
|
293
318
|
end
|
|
294
319
|
end
|
|
295
320
|
needs_dependencies = true
|
|
@@ -304,7 +329,7 @@ module OpenC3
|
|
|
304
329
|
end
|
|
305
330
|
if needs_dependencies
|
|
306
331
|
plugin_model.needs_dependencies = true
|
|
307
|
-
plugin_model.update unless
|
|
332
|
+
plugin_model.update unless dry_run
|
|
308
333
|
end
|
|
309
334
|
|
|
310
335
|
# Temporarily add all lib folders from the gem to the end of the load path
|
|
@@ -343,8 +368,12 @@ module OpenC3
|
|
|
343
368
|
when 'TARGET', 'INTERFACE', 'ROUTER', 'MICROSERVICE', 'TOOL', 'WIDGET', 'SCRIPT_ENGINE'
|
|
344
369
|
begin
|
|
345
370
|
if current_model
|
|
346
|
-
current_model.create unless
|
|
347
|
-
current_model.
|
|
371
|
+
current_model.create unless dry_run
|
|
372
|
+
if current_model.is_a?(OpenC3::TargetModel)
|
|
373
|
+
current_model.deploy(gem_path, erb_variables, validate_only: dry_run, upgrade_context: upgrade_context)
|
|
374
|
+
else
|
|
375
|
+
current_model.deploy(gem_path, erb_variables, validate_only: dry_run)
|
|
376
|
+
end
|
|
348
377
|
end
|
|
349
378
|
# If something goes wrong in create, or more likely in deploy,
|
|
350
379
|
# we want to clear the current_model and try to instantiate the next
|
|
@@ -363,8 +392,12 @@ module OpenC3
|
|
|
363
392
|
end
|
|
364
393
|
end
|
|
365
394
|
if current_model
|
|
366
|
-
current_model.create unless
|
|
367
|
-
current_model.
|
|
395
|
+
current_model.create unless dry_run
|
|
396
|
+
if current_model.is_a?(OpenC3::TargetModel)
|
|
397
|
+
current_model.deploy(gem_path, erb_variables, validate_only: dry_run, upgrade_context: upgrade_context)
|
|
398
|
+
else
|
|
399
|
+
current_model.deploy(gem_path, erb_variables, validate_only: dry_run)
|
|
400
|
+
end
|
|
368
401
|
current_model = nil
|
|
369
402
|
end
|
|
370
403
|
end
|
|
@@ -375,15 +408,26 @@ module OpenC3
|
|
|
375
408
|
end
|
|
376
409
|
rescue => e
|
|
377
410
|
# Install failed - need to cleanup
|
|
378
|
-
plugin_model.destroy unless
|
|
411
|
+
plugin_model.destroy unless dry_run
|
|
379
412
|
raise e
|
|
380
413
|
ensure
|
|
381
414
|
FileUtils.remove_entry_secure(temp_dir, true)
|
|
382
415
|
tf.unlink if tf
|
|
383
416
|
end
|
|
417
|
+
return upgrade_context[:diff_collector].uniq if diff_only
|
|
384
418
|
return plugin_model.as_json()
|
|
385
419
|
end
|
|
386
420
|
|
|
421
|
+
# Dry run: which modified files would this plugin's install supersede?
|
|
422
|
+
# Returns a list of "TARGET/path" names whose live (modified) content
|
|
423
|
+
# differs from the rendered plugin content. Read-only; no side effects.
|
|
424
|
+
def self.modified_diff(plugin_hash, scope:)
|
|
425
|
+
install_phase2(plugin_hash, scope: scope, diff_only: true)
|
|
426
|
+
rescue => e
|
|
427
|
+
Logger.warn("PluginModel.modified_diff failed: #{e.message}")
|
|
428
|
+
[]
|
|
429
|
+
end
|
|
430
|
+
|
|
387
431
|
def initialize(
|
|
388
432
|
name:,
|
|
389
433
|
variables: {},
|
|
@@ -482,6 +526,19 @@ module OpenC3
|
|
|
482
526
|
errors << e
|
|
483
527
|
end
|
|
484
528
|
end
|
|
529
|
+
# Remove the per-plugin UV virtual environment directory that was created
|
|
530
|
+
# during install_phase2. This cleans up the .venv, pyproject.toml, uv.lock,
|
|
531
|
+
# and .uv_managed marker so disk space is reclaimed on plugin uninstall.
|
|
532
|
+
begin
|
|
533
|
+
plugin_venv_name = self.class.plugin_venv_name(scope: @scope, plugin_name: @name)
|
|
534
|
+
plugin_venv_path = File.join('/gems', 'plugin_venvs', plugin_venv_name)
|
|
535
|
+
if File.directory?(plugin_venv_path)
|
|
536
|
+
Logger.info("Removing per-plugin Python venv: #{plugin_venv_path}")
|
|
537
|
+
FileUtils.rm_rf(plugin_venv_path)
|
|
538
|
+
end
|
|
539
|
+
rescue Exception => e
|
|
540
|
+
errors << e
|
|
541
|
+
end
|
|
485
542
|
# Raise all the errors at once
|
|
486
543
|
if errors.length > 0
|
|
487
544
|
message = ''
|
|
@@ -528,6 +585,103 @@ module OpenC3
|
|
|
528
585
|
return result.sort
|
|
529
586
|
end
|
|
530
587
|
|
|
588
|
+
# Resolve the PyPI URL from settings, environment, or default.
|
|
589
|
+
# Used by both install_phase2 and migrate_to_uv! to avoid duplication.
|
|
590
|
+
def self.resolve_pypi_url(scope:)
|
|
591
|
+
pypi_url = nil
|
|
592
|
+
begin
|
|
593
|
+
pypi_url = get_setting('pypi_url', scope: scope)
|
|
594
|
+
pypi_url += '/simple' if pypi_url
|
|
595
|
+
rescue => e
|
|
596
|
+
Logger.error("Failed to retrieve pypi_url: #{e.formatted}")
|
|
597
|
+
ensure
|
|
598
|
+
if pypi_url.nil?
|
|
599
|
+
pypi_url = ENV['PYPI_URL']
|
|
600
|
+
pypi_url += '/simple' if pypi_url
|
|
601
|
+
pypi_url ||= PypiUrl::DEFAULT
|
|
602
|
+
end
|
|
603
|
+
end
|
|
604
|
+
PypiUrl.validate(pypi_url)
|
|
605
|
+
end
|
|
606
|
+
|
|
607
|
+
# Build the argv array for pypi index and trusted-host arguments.
|
|
608
|
+
def self.build_pypi_args(pypi_url)
|
|
609
|
+
args = ["-i", pypi_url]
|
|
610
|
+
args += ["--trusted-host", URI.parse(pypi_url).host] unless ENV['PIP_ENABLE_TRUSTED_HOST'].nil?
|
|
611
|
+
args
|
|
612
|
+
end
|
|
613
|
+
|
|
614
|
+
# Build a sanitized venv directory name from scope and plugin name.
|
|
615
|
+
# Replaces characters that are not alphanumeric, underscore, or hyphen with underscores.
|
|
616
|
+
def self.plugin_venv_name(scope:, plugin_name:)
|
|
617
|
+
"#{scope}__#{plugin_name}".tr('^a-zA-Z0-9_-', '_')
|
|
618
|
+
end
|
|
619
|
+
|
|
620
|
+
# Check if this plugin needs migration to a per-plugin UV virtual environment.
|
|
621
|
+
# Returns true if the plugin has Python dependencies but no .uv_managed marker exists.
|
|
622
|
+
def needs_uv_migration?
|
|
623
|
+
return false unless @needs_dependencies
|
|
624
|
+
|
|
625
|
+
plugin_venv_name = self.class.plugin_venv_name(scope: @scope, plugin_name: @name)
|
|
626
|
+
marker_path = File.join('/gems', 'plugin_venvs', plugin_venv_name, '.uv_managed')
|
|
627
|
+
!File.exist?(marker_path)
|
|
628
|
+
end
|
|
629
|
+
|
|
630
|
+
# Migrate this plugin from the shared Python venv to a per-plugin UV virtual environment.
|
|
631
|
+
# Non-fatal: logs a warning on failure, plugin continues using the shared venv.
|
|
632
|
+
# Returns true on success, false on failure.
|
|
633
|
+
def migrate_to_uv!(scope:)
|
|
634
|
+
plugin_venv_name = self.class.plugin_venv_name(scope: scope, plugin_name: @name)
|
|
635
|
+
marker_path = File.join('/gems', 'plugin_venvs', plugin_venv_name, '.uv_managed')
|
|
636
|
+
if File.exist?(marker_path)
|
|
637
|
+
Logger.info("Plugin '#{@name}' is already migrated to a per-plugin UV venv")
|
|
638
|
+
return true
|
|
639
|
+
end
|
|
640
|
+
|
|
641
|
+
gem_name = @name.split("__")[0]
|
|
642
|
+
gem_file_path = OpenC3::GemModel.get(gem_name)
|
|
643
|
+
|
|
644
|
+
temp_dir = Dir.mktmpdir
|
|
645
|
+
begin
|
|
646
|
+
# Extract gem contents (same pattern as install_phase2)
|
|
647
|
+
gem_path = File.join(temp_dir, "gem")
|
|
648
|
+
FileUtils.mkdir_p(gem_path)
|
|
649
|
+
pkg = Gem::Package.new(gem_file_path)
|
|
650
|
+
pkg.extract_files(gem_path)
|
|
651
|
+
|
|
652
|
+
# Check for Python dependency files
|
|
653
|
+
pyproject_path = File.join(gem_path, 'pyproject.toml')
|
|
654
|
+
requirements_path = File.join(gem_path, 'requirements.txt')
|
|
655
|
+
|
|
656
|
+
unless File.exist?(pyproject_path) || File.exist?(requirements_path)
|
|
657
|
+
Logger.info("Plugin #{@name} has no Python dependencies to migrate")
|
|
658
|
+
return true
|
|
659
|
+
end
|
|
660
|
+
|
|
661
|
+
pypi_url = self.class.resolve_pypi_url(scope: scope)
|
|
662
|
+
pypi_args = self.class.build_pypi_args(pypi_url)
|
|
663
|
+
|
|
664
|
+
# Run uvinstall for this plugin
|
|
665
|
+
plugin_venv_name = self.class.plugin_venv_name(scope: @scope, plugin_name: @name)
|
|
666
|
+
Logger.info("Migrating plugin '#{@name}' to per-plugin UV venv '#{plugin_venv_name}'")
|
|
667
|
+
uv_args = [plugin_venv_name, gem_path] + pypi_args
|
|
668
|
+
output, status = Open3.capture2e("/openc3/bin/uvinstall", *uv_args)
|
|
669
|
+
puts output
|
|
670
|
+
if status.success?
|
|
671
|
+
Logger.info("Successfully migrated plugin '#{@name}' to per-plugin UV venv")
|
|
672
|
+
return true
|
|
673
|
+
else
|
|
674
|
+
Logger.warn("UV migration failed for plugin '#{@name}'. Plugin will continue using shared venv.")
|
|
675
|
+
return false
|
|
676
|
+
end
|
|
677
|
+
rescue => e
|
|
678
|
+
Logger.warn("UV migration failed for plugin '#{@name}': #{e.message}. Plugin will continue using shared venv.")
|
|
679
|
+
return false
|
|
680
|
+
ensure
|
|
681
|
+
FileUtils.remove_entry_secure(temp_dir, true)
|
|
682
|
+
end
|
|
683
|
+
end
|
|
684
|
+
|
|
531
685
|
# Remove the backing gem for an unloaded plugin so it disappears from
|
|
532
686
|
# GemModel.names (and the admin Packages tab). Skips the removal if any
|
|
533
687
|
# other PluginModel (any scope, any counter) still references the gem,
|
|
@@ -12,10 +12,13 @@
|
|
|
12
12
|
# if purchased from OpenC3, Inc.
|
|
13
13
|
|
|
14
14
|
require 'fileutils'
|
|
15
|
+
require 'open3'
|
|
16
|
+
require 'set'
|
|
15
17
|
require 'openc3/utilities/process_manager'
|
|
16
18
|
require 'openc3/utilities/pypi_url'
|
|
17
19
|
require 'openc3/api/api'
|
|
18
20
|
require 'pathname'
|
|
21
|
+
require 'json'
|
|
19
22
|
|
|
20
23
|
module OpenC3
|
|
21
24
|
# This class acts like a Model but doesn't inherit from Model because it doesn't
|
|
@@ -26,14 +29,162 @@ module OpenC3
|
|
|
26
29
|
extend Api
|
|
27
30
|
|
|
28
31
|
DIST_INFO = '.dist-info'
|
|
32
|
+
PLUGIN_VENVS_DIR = '/gems/plugin_venvs' # Per-plugin isolated venvs created by uvinstall
|
|
33
|
+
SYSTEM_VENV_DIR = '/openc3/python/.venv' # Core openc3 Python library venv (read-only)
|
|
34
|
+
DEFAULT_UV_CACHE_DIR = '/gems/uv' # UV wheel cache, seeded from the Docker image at init
|
|
35
|
+
UPLOADS_DIR_NAME = 'uploads' # Subdirectory under UV cache for uploaded .whl files
|
|
36
|
+
|
|
37
|
+
# Normalize a package name to lowercase with hyphens (PEP 503 canonical form)
|
|
38
|
+
def self.normalize_pkg_name(name)
|
|
39
|
+
name.tr('_', '-').downcase
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Parse a PEP 427 wheel filename into [normalized_name, version].
|
|
43
|
+
# Handles browser-appended duplicate suffixes like "(1)" on the stem.
|
|
44
|
+
# Returns nil for non-wheel files or malformed names.
|
|
45
|
+
# Example: "numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl" => ["numpy", "2.4.6"]
|
|
46
|
+
def self.parse_wheel_filename(filename)
|
|
47
|
+
# Strip browser duplicate suffixes like " (1)" before the .whl extension
|
|
48
|
+
# Two steps to avoid any space quantifier in the regex:
|
|
49
|
+
# 1. Remove "(N).whl" at end, replacing with just ".whl"
|
|
50
|
+
# 2. Clean up leftover spaces before .whl with rstrip
|
|
51
|
+
clean = filename.sub(/\(\d+\)\.whl\z/i, '.whl')
|
|
52
|
+
clean = "#{clean.chomp('.whl').rstrip}.whl" if clean.end_with?('.whl')
|
|
53
|
+
return nil unless clean.end_with?('.whl')
|
|
54
|
+
|
|
55
|
+
# PEP 427: {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl
|
|
56
|
+
# We need at least name-version-python-abi-platform (5 segments)
|
|
57
|
+
stem = clean.chomp('.whl')
|
|
58
|
+
parts = stem.split('-')
|
|
59
|
+
return nil if parts.length < 5
|
|
60
|
+
|
|
61
|
+
# The version is always the second segment; everything before it is the name
|
|
62
|
+
# (some packages have hyphens in their name that become underscores in the wheel)
|
|
63
|
+
# PEP 427 guarantees: last 3 segments are python-abi-platform,
|
|
64
|
+
# optional build tag before those, then version, then name (may have multiple segments)
|
|
65
|
+
# Simplest reliable approach: version is always at index 1
|
|
66
|
+
name = normalize_pkg_name(parts[0])
|
|
67
|
+
version = parts[1]
|
|
68
|
+
|
|
69
|
+
# Sanity check: version should start with a digit
|
|
70
|
+
return nil unless version.match?(/\A\d/)
|
|
71
|
+
|
|
72
|
+
[name, version]
|
|
73
|
+
end
|
|
29
74
|
|
|
30
75
|
def self.names
|
|
31
|
-
|
|
32
|
-
|
|
76
|
+
result = {}
|
|
77
|
+
|
|
78
|
+
# Collect all packages available in the UV download cache
|
|
79
|
+
# This includes system packages (seeded from the Docker image) plus
|
|
80
|
+
# any additional packages downloaded during plugin installs
|
|
81
|
+
cached = cached_packages
|
|
82
|
+
result['cached'] = cached.sort unless cached.empty?
|
|
83
|
+
|
|
84
|
+
# Collect packages from per-plugin venvs
|
|
85
|
+
if File.directory?(PLUGIN_VENVS_DIR)
|
|
86
|
+
Dir.glob("#{PLUGIN_VENVS_DIR}/*/").each do |plugin_dir|
|
|
87
|
+
plugin_name = File.basename(plugin_dir)
|
|
88
|
+
venv_dir = File.join(plugin_dir, '.venv')
|
|
89
|
+
next unless File.directory?(venv_dir)
|
|
90
|
+
|
|
91
|
+
# Always include plugin venvs even if empty so they remain visible
|
|
92
|
+
# in the Admin UI and selectable as install targets
|
|
93
|
+
packages = packages_in_venv(venv_dir)
|
|
94
|
+
result[plugin_name] = packages.sort
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Also collect packages from the shared venv for backwards compatibility
|
|
99
|
+
shared_packages = shared_venv_packages
|
|
100
|
+
result['shared'] = shared_packages.sort unless shared_packages.empty?
|
|
101
|
+
|
|
102
|
+
return result
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# List packages in a specific venv by scanning dist-info directories.
|
|
106
|
+
# Returns normalized names (lowercase, hyphens) for consistent display.
|
|
107
|
+
def self.packages_in_venv(venv_dir)
|
|
108
|
+
packages = []
|
|
109
|
+
# Look for site-packages in the venv's lib directory
|
|
110
|
+
Dir.glob("#{venv_dir}/lib/*/site-packages").each do |site_packages|
|
|
111
|
+
next unless File.directory?(site_packages)
|
|
112
|
+
Pathname.new(site_packages).children.each do |child|
|
|
113
|
+
if child.directory? && File.extname(child) == DIST_INFO
|
|
114
|
+
raw_name = File.basename(child, DIST_INFO)
|
|
115
|
+
# Normalize: split name from version, normalize name, rejoin
|
|
116
|
+
match = raw_name.match(/\A(.+?)-(\d.*)/)
|
|
117
|
+
if match
|
|
118
|
+
packages << "#{normalize_pkg_name(match[1])}-#{match[2]}"
|
|
119
|
+
else
|
|
120
|
+
packages << normalize_pkg_name(raw_name)
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
packages
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# List packages in the system venv (/openc3/python/.venv)
|
|
129
|
+
def self.system_venv_packages
|
|
130
|
+
packages_in_venv(SYSTEM_VENV_DIR)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# List unique packages in the UV download cache.
|
|
134
|
+
# UV cache structure: wheels-v<N>/<registry>/<package-name>/<version-pytag-abitag-platform>
|
|
135
|
+
# e.g. wheels-v6/pypi/numpy/2.4.6-cp312-cp312-musllinux_1_2_aarch64
|
|
136
|
+
# Also scans the uploads/ subdirectory for user-uploaded .whl files.
|
|
137
|
+
def self.cached_packages
|
|
138
|
+
cache_dir = ENV.fetch('UV_CACHE_DIR', DEFAULT_UV_CACHE_DIR)
|
|
139
|
+
return [] unless File.directory?(cache_dir)
|
|
140
|
+
|
|
141
|
+
packages = Set.new
|
|
142
|
+
# Glob 4 levels deep: wheels-v<N>/<registry>/<package-name>/<version-entry>
|
|
143
|
+
Dir.glob("#{cache_dir}/wheels-v*/*/*/*").each do |entry|
|
|
144
|
+
basename = File.basename(entry)
|
|
145
|
+
# Skip UV metadata sidecar files (.http, .msgpack)
|
|
146
|
+
next if basename.end_with?('.http', '.msgpack')
|
|
147
|
+
|
|
148
|
+
# Version entries start with a digit
|
|
149
|
+
match = basename.match(/\A(\d[^-]*)/)
|
|
150
|
+
next unless match
|
|
151
|
+
|
|
152
|
+
# Parent directory name is the package name
|
|
153
|
+
pkg_name = File.basename(File.dirname(entry)).tr('_', '-').downcase
|
|
154
|
+
packages.add("#{pkg_name}-#{match[1]}")
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Scan uploaded wheels stored at <cache_dir>/uploads/*.whl
|
|
158
|
+
uploads_dir = File.join(cache_dir, UPLOADS_DIR_NAME)
|
|
159
|
+
if File.directory?(uploads_dir)
|
|
160
|
+
Dir.glob("#{uploads_dir}/*.whl").each do |whl_path|
|
|
161
|
+
parsed = parse_wheel_filename(File.basename(whl_path))
|
|
162
|
+
next unless parsed
|
|
163
|
+
|
|
164
|
+
packages.add("#{parsed[0]}-#{parsed[1]}")
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
packages.to_a
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# List packages in the shared venv (backwards compatibility)
|
|
172
|
+
def self.shared_venv_packages
|
|
173
|
+
packages = []
|
|
174
|
+
pythonuserbase = ENV.fetch('PYTHONUSERBASE', nil)
|
|
175
|
+
return packages unless pythonuserbase
|
|
176
|
+
|
|
177
|
+
paths = Dir.glob("#{pythonuserbase}/lib/*")
|
|
33
178
|
paths.each do |path|
|
|
34
|
-
|
|
179
|
+
site_packages = File.join(path, 'site-packages')
|
|
180
|
+
next unless File.directory?(site_packages)
|
|
181
|
+
Pathname.new(site_packages).children.each do |child|
|
|
182
|
+
if child.directory? && File.extname(child) == DIST_INFO
|
|
183
|
+
packages << File.basename(child, DIST_INFO)
|
|
184
|
+
end
|
|
185
|
+
end
|
|
35
186
|
end
|
|
36
|
-
|
|
187
|
+
packages
|
|
37
188
|
end
|
|
38
189
|
|
|
39
190
|
def self.get(name)
|
|
@@ -43,17 +194,26 @@ module OpenC3
|
|
|
43
194
|
if result.length > 0
|
|
44
195
|
return result[0] if File.exist?(result[0])
|
|
45
196
|
end
|
|
46
|
-
raise "Package #{name} not found"
|
|
197
|
+
raise "Package '#{name}' not found"
|
|
47
198
|
end
|
|
48
199
|
|
|
49
|
-
def self.put(package_file_path, package_install: true, scope:)
|
|
200
|
+
def self.put(package_file_path, package_install: true, scope:, plugin: nil)
|
|
50
201
|
if File.file?(package_file_path)
|
|
51
202
|
package_filename = File.basename(package_file_path)
|
|
52
203
|
FileUtils.mkdir_p("#{ENV['PYTHONUSERBASE']}/cache") unless Dir.exist?("#{ENV['PYTHONUSERBASE']}/cache")
|
|
53
204
|
cache_path = "#{ENV['PYTHONUSERBASE']}/cache/#{File.basename(package_file_path)}"
|
|
54
205
|
FileUtils.cp(package_file_path, cache_path)
|
|
206
|
+
|
|
207
|
+
# Copy uploaded .whl files to the UV uploads directory so they appear
|
|
208
|
+
# in the "Cached" section of the Admin Packages UI
|
|
209
|
+
if package_filename.end_with?('.whl')
|
|
210
|
+
uploads_dir = File.join(ENV.fetch('UV_CACHE_DIR', DEFAULT_UV_CACHE_DIR), UPLOADS_DIR_NAME)
|
|
211
|
+
FileUtils.mkdir_p(uploads_dir)
|
|
212
|
+
FileUtils.cp(package_file_path, File.join(uploads_dir, package_filename))
|
|
213
|
+
end
|
|
214
|
+
|
|
55
215
|
if package_install
|
|
56
|
-
return self.install(cache_path, scope: scope)
|
|
216
|
+
return self.install(cache_path, scope: scope, plugin: plugin)
|
|
57
217
|
end
|
|
58
218
|
else
|
|
59
219
|
message = "Package file #{package_file_path} does not exist!"
|
|
@@ -63,7 +223,11 @@ module OpenC3
|
|
|
63
223
|
return nil
|
|
64
224
|
end
|
|
65
225
|
|
|
66
|
-
|
|
226
|
+
# Install a Python package via pipinstall. When +plugin+ is provided, the
|
|
227
|
+
# package is installed into that plugin's per-plugin venv instead of the
|
|
228
|
+
# shared PYTHONUSERBASE. This is used by the Admin Packages tab when a user
|
|
229
|
+
# selects a specific plugin venv as the install target.
|
|
230
|
+
def self.install(name_or_path, scope:, plugin: nil)
|
|
67
231
|
if File.exist?(name_or_path)
|
|
68
232
|
package_file_path = name_or_path
|
|
69
233
|
else
|
|
@@ -94,18 +258,50 @@ module OpenC3
|
|
|
94
258
|
else
|
|
95
259
|
pip_args = ["-i", pypi_url, "--trusted-host", URI.parse(pypi_url).host, package_file_path]
|
|
96
260
|
end
|
|
97
|
-
|
|
261
|
+
spawn_env = {}
|
|
262
|
+
if plugin
|
|
263
|
+
venv_path = "#{PLUGIN_VENVS_DIR}/#{plugin}/.venv"
|
|
264
|
+
spawn_env['PIPINSTALL_VENV'] = venv_path
|
|
265
|
+
end
|
|
266
|
+
result = OpenC3::ProcessManager.instance.spawn(["/openc3/bin/pipinstall"] + pip_args, "package_install", package_filename, Time.now + 3600.0, scope: scope, env: spawn_env)
|
|
98
267
|
return result.name
|
|
99
268
|
end
|
|
100
269
|
|
|
101
|
-
|
|
270
|
+
# Uninstall a Python package. When +plugin+ is provided, the package is
|
|
271
|
+
# removed from that plugin's per-plugin venv; otherwise from the shared venv.
|
|
272
|
+
def self.destroy(name, scope:, plugin: nil)
|
|
102
273
|
package_name, version = self.extract_name_and_version(name)
|
|
103
274
|
Logger.info "Uninstalling package: #{name}"
|
|
104
275
|
pip_args = [package_name]
|
|
105
|
-
|
|
276
|
+
spawn_env = {}
|
|
277
|
+
if plugin
|
|
278
|
+
venv_path = "#{PLUGIN_VENVS_DIR}/#{plugin}/.venv"
|
|
279
|
+
spawn_env['PIPINSTALL_VENV'] = venv_path
|
|
280
|
+
end
|
|
281
|
+
result = OpenC3::ProcessManager.instance.spawn(["/openc3/bin/pipuninstall"] + pip_args, "package_uninstall", name, Time.now + 3600.0, scope: scope, env: spawn_env)
|
|
106
282
|
return result.name
|
|
107
283
|
end
|
|
108
284
|
|
|
285
|
+
# Returns a hash of plugin_name => "uv pip list" text output for each plugin venv.
|
|
286
|
+
def self.trees
|
|
287
|
+
result = {}
|
|
288
|
+
|
|
289
|
+
if File.directory?(PLUGIN_VENVS_DIR)
|
|
290
|
+
Dir.glob("#{PLUGIN_VENVS_DIR}/*/").each do |plugin_dir|
|
|
291
|
+
plugin_name = File.basename(plugin_dir)
|
|
292
|
+
venv_dir = File.join(plugin_dir, '.venv')
|
|
293
|
+
next unless File.directory?(venv_dir)
|
|
294
|
+
|
|
295
|
+
stdout, status = Open3.capture2('uv', 'pip', 'list', '--python', venv_dir)
|
|
296
|
+
if status.success? && stdout.lines.length > 2
|
|
297
|
+
result[plugin_name] = stdout.rstrip
|
|
298
|
+
end
|
|
299
|
+
end
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
result
|
|
303
|
+
end
|
|
304
|
+
|
|
109
305
|
def self.extract_name_and_version(name)
|
|
110
306
|
split_name = name.split('-')
|
|
111
307
|
if split_name.length > 1
|