openc3 7.4.0 → 7.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. checksums.yaml +4 -4
  2. data/bin/openc3cli +4 -3
  3. data/bin/pipinstall +18 -5
  4. data/bin/pipuninstall +1 -1
  5. data/bin/uvinstall +36 -8
  6. data/data/config/default_plugins.yaml +43 -0
  7. data/data/config/interface_modifiers.yaml +22 -7
  8. data/data/config/item_modifiers.yaml +14 -4
  9. data/data/config/plugins.yaml +1 -1
  10. data/data/config/widgets.yaml +1 -1
  11. data/lib/openc3/api/cmd_api.rb +10 -13
  12. data/lib/openc3/api/interface_api.rb +1 -2
  13. data/lib/openc3/api/limits_api.rb +1 -1
  14. data/lib/openc3/api/metrics_api.rb +26 -1
  15. data/lib/openc3/api/router_api.rb +1 -2
  16. data/lib/openc3/api/tlm_api.rb +21 -21
  17. data/lib/openc3/config/config_parser.rb +3 -1
  18. data/lib/openc3/core_ext/tempfile.rb +2 -0
  19. data/lib/openc3/interfaces/file_interface.rb +6 -1
  20. data/lib/openc3/interfaces/http_client_interface.rb +14 -6
  21. data/lib/openc3/interfaces/interface.rb +27 -3
  22. data/lib/openc3/interfaces/protocols/protocol.rb +8 -0
  23. data/lib/openc3/interfaces/read_queue_interface.rb +331 -0
  24. data/lib/openc3/interfaces/stream_interface.rb +22 -3
  25. data/lib/openc3/interfaces/tcpip_server_interface.rb +13 -0
  26. data/lib/openc3/interfaces/udp_interface.rb +16 -2
  27. data/lib/openc3/interfaces.rb +1 -0
  28. data/lib/openc3/io/json_api_object.rb +24 -3
  29. data/lib/openc3/io/json_drb_object.rb +3 -2
  30. data/lib/openc3/logs/log_writer.rb +13 -4
  31. data/lib/openc3/microservices/decom_microservice.rb +7 -4
  32. data/lib/openc3/microservices/interface_microservice.rb +146 -50
  33. data/lib/openc3/microservices/log_microservice.rb +1 -3
  34. data/lib/openc3/microservices/microservice.rb +25 -1
  35. data/lib/openc3/microservices/plugin_microservice.rb +32 -0
  36. data/lib/openc3/microservices/text_log_microservice.rb +1 -3
  37. data/lib/openc3/models/interface_model.rb +1 -1
  38. data/lib/openc3/models/interface_status_model.rb +6 -0
  39. data/lib/openc3/models/metric_model.rb +52 -4
  40. data/lib/openc3/models/microservice_model.rb +7 -3
  41. data/lib/openc3/models/plugin_model.rb +12 -12
  42. data/lib/openc3/models/python_package_model.rb +3 -13
  43. data/lib/openc3/models/reaction_model.rb +6 -3
  44. data/lib/openc3/models/setting_model.rb +5 -3
  45. data/lib/openc3/models/target_model.rb +83 -9
  46. data/lib/openc3/operators/microservice_operator.rb +5 -0
  47. data/lib/openc3/operators/operator.rb +69 -0
  48. data/lib/openc3/packets/packet.rb +10 -4
  49. data/lib/openc3/packets/packet_config.rb +48 -2
  50. data/lib/openc3/packets/packet_item.rb +27 -14
  51. data/lib/openc3/packets/parsers/state_parser.rb +1 -1
  52. data/lib/openc3/packets/parsers/xtce_parser.rb +8 -1
  53. data/lib/openc3/packets/structure_item.rb +3 -2
  54. data/lib/openc3/script/api_shared.rb +58 -84
  55. data/lib/openc3/script/extract.rb +215 -19
  56. data/lib/openc3/script/plugins.rb +4 -11
  57. data/lib/openc3/script/storage.rb +81 -10
  58. data/lib/openc3/script/tables.rb +5 -1
  59. data/lib/openc3/system/system.rb +9 -6
  60. data/lib/openc3/topics/command_decom_topic.rb +1 -1
  61. data/lib/openc3/topics/command_topic.rb +2 -1
  62. data/lib/openc3/topics/router_topic.rb +2 -1
  63. data/lib/openc3/topics/telemetry_decom_topic.rb +1 -1
  64. data/lib/openc3/topics/telemetry_topic.rb +3 -2
  65. data/lib/openc3/topics/topic.rb +154 -2
  66. data/lib/openc3/utilities/cli_generator.rb +67 -15
  67. data/lib/openc3/utilities/config_overlay.rb +35 -17
  68. data/lib/openc3/utilities/default_plugins.rb +33 -0
  69. data/lib/openc3/utilities/local_mode.rb +82 -46
  70. data/lib/openc3/utilities/metric.rb +16 -0
  71. data/lib/openc3/utilities/process_stats.rb +160 -0
  72. data/lib/openc3/utilities/pypi_url.rb +65 -0
  73. data/lib/openc3/utilities/questdb_client.rb +43 -46
  74. data/lib/openc3/utilities/running_script.rb +22 -18
  75. data/lib/openc3/utilities/store_autoload.rb +20 -3
  76. data/lib/openc3/utilities/store_queued.rb +37 -9
  77. data/lib/openc3/utilities/target_file.rb +57 -12
  78. data/lib/openc3/utilities/topic_lag_monitor.rb +193 -0
  79. data/lib/openc3/version.rb +5 -5
  80. data/templates/conversion/conversion.py +6 -2
  81. data/templates/conversion/conversion.rb +5 -0
  82. data/templates/plugin/.github/workflows/playwright.yml +62 -0
  83. data/templates/plugin/.github/workflows/release.yml +45 -0
  84. data/templates/plugin/.github/workflows/unit-tests.yml +35 -0
  85. data/templates/plugin/.gitignore +6 -0
  86. data/templates/plugin/README.md +114 -2
  87. data/templates/plugin/lib/PACKAGE/__init__.py +4 -0
  88. data/templates/plugin/plugin.gemspec +10 -1
  89. data/templates/plugin/pyproject.toml +119 -0
  90. data/templates/tool_angular/package.json +2 -2
  91. data/templates/tool_angular/pnpm-workspace.yaml +16 -0
  92. data/templates/tool_react/package.json +1 -1
  93. data/templates/tool_react/pnpm-workspace.yaml +16 -0
  94. data/templates/tool_svelte/package.json +2 -1
  95. data/templates/tool_svelte/pnpm-workspace.yaml +20 -0
  96. data/templates/tool_vue/package.json +3 -3
  97. data/templates/tool_vue/pnpm-workspace.yaml +16 -0
  98. data/templates/widget/package.json +2 -2
  99. data/templates/widget/pnpm-workspace.yaml +16 -0
  100. metadata +36 -1
@@ -15,24 +15,27 @@ module OpenC3
15
15
  # Shared predicates describing which parts of the user-writable config overlay
16
16
  # (targets_modified) a non-admin may write.
17
17
  #
18
- # The cmd_tlm overlay (targets_modified/<TARGET>/cmd_tlm/...) is loaded by
19
- # System.setup_targets and processed by PacketConfig, which evaluates
20
- # GENERIC_READ_CONVERSION / GENERIC_WRITE_CONVERSION blocks as code in the
21
- # decom microservices. Writing it is therefore an admin operation, matching the
22
- # tier COSMOS requires everywhere else code is introduced (plugin install).
23
- # Only the server-side dynamic-packet mechanism (TargetModel#dynamic_update)
24
- # writes that area without going through an API request.
18
+ # Two target subdirectories hold definitions whose GENERIC_READ_CONVERSION /
19
+ # GENERIC_WRITE_CONVERSION blocks are evaluated as code:
20
+ # - <TARGET>/cmd_tlm/...: loaded by System.setup_targets and processed by
21
+ # PacketConfig in the decom microservices. Only the server-side
22
+ # dynamic-packet mechanism (TargetModel#dynamic_update) writes that area
23
+ # without going through an API request.
24
+ # - <TARGET>/tables/config/...: table definitions, processed by TableConfig
25
+ # in cmd-tlm-api for every Table Manager operation.
26
+ # Writing either is therefore an admin operation, matching the tier COSMOS
27
+ # requires everywhere else code is introduced (plugin install).
25
28
  #
26
29
  # Every API writer that can reach the overlay must consult these predicates:
27
30
  # - storage_controller#get_upload_presigned_request / delete (presigned S3 upload)
28
- # - tables_controller#save / save_as / generate / destroy (Table -> TargetFile)
31
+ # - tables_controller#save / save_as / report / destroy (Table -> TargetFile)
29
32
  # - scripts_controller#create / destroy (Script -> TargetFile)
30
33
  module ConfigOverlay
31
34
  # Config bucket areas a non-admin is allowed to write at all
32
35
  NON_ADMIN_AREAS = ['targets_modified', 'tmp'].freeze
33
36
 
34
- # Target subdirectory whose contents are executed as code
35
- CODE_AREA = 'cmd_tlm'
37
+ # Target subdirectories whose contents are executed as code
38
+ CODE_AREAS = [['cmd_tlm'], ['tables', 'config']].freeze
36
39
 
37
40
  # Split a path into segments, or nil if the path is not canonical.
38
41
  #
@@ -48,14 +51,23 @@ module OpenC3
48
51
  parts
49
52
  end
50
53
 
51
- # True if the overlay-relative name targets the cmd_tlm subtree, i.e. writing
52
- # it requires admin. Name is relative to targets_modified/, e.g.
53
- # "<TARGET>/cmd_tlm/tlm.txt". Fails closed (true) on non-canonical names.
54
- def self.cmd_tlm_overlay?(name)
54
+ # True if the overlay-relative name targets a code area (cmd_tlm or
55
+ # tables/config), i.e. writing it requires admin. Name is relative to
56
+ # targets_modified/, e.g. "<TARGET>/cmd_tlm/tlm.txt". Fails closed (true)
57
+ # on non-canonical names.
58
+ def self.code_overlay?(name)
55
59
  parts = canonical_parts(name)
56
60
  return true if parts.nil?
57
- # parts: <TARGET> / <area> / ...
58
- parts[1] == CODE_AREA
61
+ code_area?(parts)
62
+ end
63
+
64
+ # True if the target-relative name is a canonical table definition path,
65
+ # e.g. "<TARGET>/tables/config/table_def.txt". Table only parses definitions
66
+ # from here, so a definition can't be pointed at a non-admin overlay area.
67
+ def self.table_definition?(name)
68
+ parts = canonical_parts(name)
69
+ return false if parts.nil?
70
+ parts.length > 3 and parts[1, 2] == ['tables', 'config']
59
71
  end
60
72
 
61
73
  # True if the given config bucket key is an overlay path a non-admin may
@@ -68,8 +80,14 @@ module OpenC3
68
80
  # parts: <SCOPE> / <area> / <TARGET> / <subdir> / ...
69
81
  area = parts[1]
70
82
  return false unless NON_ADMIN_AREAS.include?(area)
71
- return false if area == 'targets_modified' && parts[3] == CODE_AREA
83
+ return false if area == 'targets_modified' && code_area?(parts[2..])
72
84
  true
73
85
  end
86
+
87
+ # target_parts: <TARGET> / <subdir> / ...
88
+ def self.code_area?(target_parts)
89
+ CODE_AREAS.any? { |area| target_parts[1, area.length] == area }
90
+ end
91
+ private_class_method :code_area?
74
92
  end
75
93
  end
@@ -0,0 +1,33 @@
1
+ # encoding: ascii-8bit
2
+
3
+ # Copyright 2026 OpenC3, Inc.
4
+ # All Rights Reserved.
5
+ #
6
+ # This program is distributed in the hope that it will be useful,
7
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
8
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
9
+ # See LICENSE.md for more details.
10
+ #
11
+ # This file may also be used under the terms of a commercial license
12
+ # if purchased from OpenC3, Inc.
13
+
14
+ require 'yaml'
15
+
16
+ module OpenC3
17
+ # The plugins that ship with COSMOS. The list lives in
18
+ # data/config/default_plugins.yaml and is shared with the Admin Plugins tab
19
+ # and the script API via the 'default' flag from GET /openc3-api/plugins/all
20
+ module DefaultPlugins
21
+ # Short names, i.e. without the version or the __ suffix
22
+ NAMES = YAML.safe_load_file(
23
+ File.expand_path('../../../data/config/default_plugins.yaml', __dir__)
24
+ ).freeze
25
+
26
+ # True if the plugin (e.g. "openc3-cosmos-tool-admin-6.0.0.gem__20260101000000")
27
+ # is one that ships with COSMOS
28
+ def self.default?(plugin_name)
29
+ # Strip the __ suffix and the trailing -version.gem to get the short name
30
+ NAMES.include?(plugin_name.split('__')[0].split('-')[0..-2].join('-'))
31
+ end
32
+ end
33
+ end
@@ -17,39 +17,11 @@ require 'openc3/core_ext/file'
17
17
  # require 'openc3/models/gem_model' # These are used but also create circular dependency
18
18
  # require 'openc3/models/plugin_model' # These are used but also create circular dependency
19
19
  require 'openc3/utilities/bucket'
20
+ require 'openc3/utilities/default_plugins'
20
21
 
21
22
  module OpenC3
22
23
  module LocalMode
23
24
  OPENC3_LOCAL_MODE_PATH = ENV['OPENC3_LOCAL_MODE_PATH'] || "/plugins"
24
- # When updating update local_mode.py, PluginsTab.vue, plugins.p.spec.ts
25
- DEFAULT_PLUGINS = [
26
- 'openc3-cosmos-tool-admin',
27
- 'openc3-cosmos-tool-bucketexplorer',
28
- 'openc3-cosmos-tool-cmdsender',
29
- 'openc3-cosmos-tool-cmdqueue', # Enterprise only
30
- 'openc3-cosmos-tool-cmdhistory', # Enterprise only
31
- 'openc3-cosmos-tool-cmdtlmserver',
32
- 'openc3-cosmos-tool-dataextractor',
33
- 'openc3-cosmos-tool-dataviewer',
34
- 'openc3-cosmos-tool-docs',
35
- 'openc3-cosmos-tool-handbooks',
36
- 'openc3-cosmos-tool-iframe',
37
- 'openc3-cosmos-tool-limitsmonitor',
38
- 'openc3-cosmos-tool-packetviewer',
39
- 'openc3-cosmos-tool-scriptrunner',
40
- 'openc3-cosmos-tool-tablemanager',
41
- 'openc3-cosmos-tool-tlmgrapher',
42
- 'openc3-cosmos-tool-tlmviewer',
43
- 'openc3-cosmos-enterprise-tool-admin', # Enterprise only
44
- 'openc3-cosmos-tool-autonomic', # Enterprise only
45
- 'openc3-cosmos-tool-calendar', # Enterprise only
46
- 'openc3-cosmos-tool-grafana', # Enterprise only
47
- 'openc3-cosmos-tool-logexplorer', # Enterprise only
48
- 'openc3-cosmos-tool-notebooks', # Enterprise only
49
- 'openc3-cosmos-tool-systemhealth', # Enterprise only
50
- 'openc3-enterprise-tool-base', # Enterprise only
51
- 'openc3-tool-base',
52
- ]
53
25
 
54
26
  # Install plugins from local plugins folder
55
27
  # Can only be used from openc3cli because calls top_level load_plugin
@@ -280,7 +252,7 @@ module OpenC3
280
252
  end
281
253
 
282
254
  def self.update_local_plugin_files(full_folder_path, plugin_file_path, plugin_hash, gem_name)
283
- return if DEFAULT_PLUGINS.include?(gem_name)
255
+ return if DefaultPlugins::NAMES.include?(gem_name)
284
256
  puts "Updating local plugin files: #{full_folder_path}"
285
257
  FileUtils.mkdir_p(full_folder_path)
286
258
  gems = scan_plugin_dir(full_folder_path).first
@@ -378,8 +350,8 @@ module OpenC3
378
350
  end
379
351
 
380
352
  def self.put_target_file(path, io_or_string, scope:) # NOSONAR - scope: is part of the caller-facing signature
381
- full_folder_path = "#{OPENC3_LOCAL_MODE_PATH}/#{path}"
382
- return unless File.expand_path(full_folder_path).start_with?(OPENC3_LOCAL_MODE_PATH)
353
+ full_folder_path = key_path(path)
354
+ return unless full_folder_path and path.start_with?("#{scope}/")
383
355
  FileUtils.mkdir_p(File.dirname(full_folder_path))
384
356
  File.open(full_folder_path, 'wb') do |file|
385
357
  if String === io_or_string
@@ -392,8 +364,8 @@ module OpenC3
392
364
  end
393
365
 
394
366
  def self.open_local_file(path, scope:)
395
- full_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/targets_modified/#{path}"
396
- if File.expand_path(full_path).start_with?(OPENC3_LOCAL_MODE_PATH)
367
+ full_path = scope_path(scope, "targets_modified/#{path}")
368
+ if full_path and safe_key?(path)
397
369
  return File.open(full_path, 'rb')
398
370
  end
399
371
  nil
@@ -452,8 +424,8 @@ module OpenC3
452
424
  def self.save_tool_config(scope, tool, name, data)
453
425
  return unless ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
454
426
  json = JSON.parse(data, allow_nan: true, create_additions: true)
455
- config_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/tool_config/#{tool}/#{name}.json"
456
- return unless File.expand_path(config_path).start_with?(OPENC3_LOCAL_MODE_PATH)
427
+ config_path = scope_path(scope, "tool_config/#{tool}/#{name}.json")
428
+ return unless config_path and safe_key?("#{tool}/#{name}")
457
429
  FileUtils.mkdir_p(File.dirname(config_path))
458
430
  File.open(config_path, 'w') do |file|
459
431
  file.write(JSON.pretty_generate(json, allow_nan: true))
@@ -462,8 +434,8 @@ module OpenC3
462
434
 
463
435
  def self.delete_tool_config(scope, tool, name)
464
436
  return unless ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
465
- config_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/tool_config/#{tool}/#{name}.json"
466
- return unless File.expand_path(config_path).start_with?(OPENC3_LOCAL_MODE_PATH)
437
+ config_path = scope_path(scope, "tool_config/#{tool}/#{name}.json")
438
+ return unless config_path and safe_key?("#{tool}/#{name}")
467
439
  FileUtils.rm_f(config_path)
468
440
  end
469
441
 
@@ -508,8 +480,8 @@ module OpenC3
508
480
 
509
481
  def self.save_setting(scope, name, data)
510
482
  return unless ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
511
- config_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/settings/#{name}.json"
512
- return unless File.expand_path(config_path).start_with?(OPENC3_LOCAL_MODE_PATH)
483
+ config_path = scope_path(scope, "settings/#{name}.json")
484
+ return unless config_path and safe_key?(name)
513
485
  FileUtils.mkdir_p(File.dirname(config_path))
514
486
  # Anything can be stored as a setting. Strings are written directly since
515
487
  # they're already the serialized form. Anything else is written as JSON so
@@ -518,26 +490,86 @@ module OpenC3
518
490
  File.write(config_path, data)
519
491
  end
520
492
 
493
+ # Target names from OPENC3_LOCAL_ONLY_TARGETS (comma separated). These
494
+ # targets keep their target files only in the local mode volume and never
495
+ # read or write them in the bucket. Read on every call so it can't go stale.
496
+ def self.local_only_targets
497
+ ENV['OPENC3_LOCAL_ONLY_TARGETS'].to_s.split(',').map { |name| name.strip.upcase }.reject(&:empty?)
498
+ end
499
+
500
+ def self.local_only_target?(target_name)
501
+ return false if target_name.nil? or target_name.to_s.empty?
502
+ local_only_targets.include?(target_name.to_s.upcase)
503
+ end
504
+
505
+ # name is relative to targets / targets_modified, e.g. "INST/procedures/test.rb"
506
+ def self.local_only_name?(name)
507
+ local_only_target?(name.to_s.split('/')[0])
508
+ end
509
+
510
+ # key is a config bucket key, e.g. "DEFAULT/targets_modified/INST/procedures/test.rb"
511
+ def self.local_only_key?(key)
512
+ split_key = key.to_s.split('/')
513
+ return false unless ['targets', 'targets_modified'].include?(split_key[1])
514
+ local_only_target?(split_key[2])
515
+ end
516
+
521
517
  # Helper methods
522
518
 
519
+ # True if the key has no '.' / '..' segments, no empty segments, no leading
520
+ # slash and no backslash. Such a key always resolves under its first
521
+ # segment (the scope), both on disk and in the bucket (the bucket
522
+ # normalizes '..' segments, so a traversal key reaches other scopes there too).
523
+ def self.safe_key?(key)
524
+ return false unless String === key
525
+ return false if key.empty? or key.start_with?('/') or key.include?('\\') or key.include?("\0")
526
+ key.split('/', -1).none? { |segment| segment.empty? or segment == '.' or segment == '..' }
527
+ end
528
+
529
+ # True if path resolves to root or somewhere beneath it. Compares against
530
+ # root plus a separator so a sibling such as /plugins_old never matches.
531
+ def self.path_within?(path, root)
532
+ expanded = File.expand_path(path)
533
+ root = File.expand_path(root)
534
+ expanded == root or expanded.start_with?(root + File::SEPARATOR)
535
+ end
536
+
537
+ # Local file path for a scope relative path, or nil if the path would
538
+ # leave the scope directory
539
+ def self.scope_path(scope, relative_path)
540
+ return nil unless safe_key?(scope) and !scope.include?('/')
541
+ scope_root = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}"
542
+ full_path = "#{scope_root}/#{relative_path}"
543
+ return nil unless path_within?(full_path, scope_root)
544
+ full_path
545
+ end
546
+
547
+ # Local file path for a bucket key, or nil if the key is not a safe key
548
+ def self.key_path(key)
549
+ return nil unless safe_key?(key)
550
+ full_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
551
+ return nil unless path_within?(full_path, OPENC3_LOCAL_MODE_PATH)
552
+ full_path
553
+ end
554
+
523
555
  def self.sync_remote_to_local(bucket, key)
524
- local_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
525
- return unless File.expand_path(local_path).start_with?(OPENC3_LOCAL_MODE_PATH)
556
+ local_path = key_path(key)
557
+ return unless local_path
526
558
  FileUtils.mkdir_p(File.dirname(local_path))
527
559
  bucket.get_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: key, path: local_path)
528
560
  end
529
561
 
530
562
  def self.sync_local_to_remote(bucket, key)
531
- local_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
532
- return unless File.expand_path(local_path).start_with?(OPENC3_LOCAL_MODE_PATH)
563
+ local_path = key_path(key)
564
+ return unless local_path
533
565
  File.open(local_path, 'rb') do |read_file|
534
566
  bucket.put_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: key, body: read_file)
535
567
  end
536
568
  end
537
569
 
538
570
  def self.delete_local(key)
539
- local_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
540
- return unless File.expand_path(local_path).start_with?(OPENC3_LOCAL_MODE_PATH)
571
+ local_path = key_path(key)
572
+ return unless local_path
541
573
  File.delete(local_path) if File.exist?(local_path)
542
574
  nil
543
575
  end
@@ -580,6 +612,10 @@ module OpenC3
580
612
  # Build catalogs
581
613
  local_catalog = build_local_catalog(scope: scope)
582
614
  remote_catalog = build_remote_catalog(bucket, scope: scope)
615
+ # Local only targets never sync in either direction. Any stale copies
616
+ # already in the bucket are left alone rather than pulled down or deleted.
617
+ local_catalog.reject! { |key, _size| local_only_key?(key) }
618
+ remote_catalog.reject! { |key, _size| local_only_key?(key) }
583
619
 
584
620
  # Find and Handle Differences
585
621
  local_catalog.each do |key, size|
@@ -133,7 +133,23 @@ module OpenC3
133
133
  def graceful_kill
134
134
  end
135
135
 
136
+ # Environment variable the operator sets on every microservice it spawns to
137
+ # say "I am already reporting process metrics for you". Without it an
138
+ # in-process generator and the operator would both be writing cpu/memory for
139
+ # the same microservice.
140
+ SUPERVISED_ENV_VAR = 'OPENC3_OPERATOR_PROCESS_METRICS'.freeze
141
+
142
+ # Whether something outside this process is already reporting its process
143
+ # level metrics.
144
+ def self.supervised?
145
+ !ENV[SUPERVISED_ENV_VAR].nil?
146
+ end
147
+
136
148
  def self.add_update_generator(object)
149
+ # Generators produce process level metrics (cpu, memory). Skip them
150
+ # entirely when the operator is already reporting those for us.
151
+ return if supervised?
152
+
137
153
  @@update_generators << object
138
154
  end
139
155
  end
@@ -0,0 +1,160 @@
1
+ # encoding: ascii-8bit
2
+
3
+ # Copyright 2026 OpenC3, Inc.
4
+ # All Rights Reserved.
5
+ #
6
+ # This program is distributed in the hope that it will be useful,
7
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
8
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
9
+ # See LICENSE.md for more details.
10
+ #
11
+ # This file may also be used under the terms of a commercial license
12
+ # if purchased from OpenC3, Inc.
13
+
14
+ require 'etc'
15
+
16
+ # See: https://man7.org/linux/man-pages/man5/proc.5.html
17
+
18
+ module OpenC3
19
+ # Samples CPU and memory for an arbitrary pid out of /proc/<pid>/stat.
20
+ #
21
+ # Every microservice is bootstrapped by plugin_microservice.rb, which exec()s
22
+ # the configured cmd. exec replaces the process image, so everything the
23
+ # bootstrap set up - including the Metric update thread - is gone. A
24
+ # microservice whose cmd is itself an OpenC3::Microservice builds a new Metric
25
+ # and keeps reporting, but a plugin that runs a Rails app, a Python script or
26
+ # any other binary never loads the OpenC3 libraries and so can never report on
27
+ # itself. Sampling by pid lets the operator report on behalf of those
28
+ # processes without requiring anything of the plugin.
29
+ class ProcessStats
30
+ # /proc/<pid>/stat field offsets, 0 based, after the comm field has been
31
+ # collapsed to a single token. proc(5) numbers these fields from 1.
32
+ UTIME_INDEX = 13
33
+ STIME_INDEX = 14
34
+ NUM_THREADS_INDEX = 19
35
+ STARTTIME_INDEX = 21
36
+ VSIZE_INDEX = 22
37
+ RSS_INDEX = 23
38
+
39
+ # Near universal Linux default. Only used if sysconf can't answer - a zero
40
+ # here would turn every CPU number into Infinity or NaN.
41
+ DEFAULT_USER_HZ = 100.0
42
+ DEFAULT_PAGESIZE = 4096
43
+
44
+ attr_reader :pid
45
+
46
+ class << self
47
+ # Whether this kernel exposes the procfs data we need at all. macOS and
48
+ # Windows don't, and neither do some hardened container runtimes.
49
+ def supported?
50
+ return @supported unless @supported.nil?
51
+ @supported = File.exist?('/proc/self/stat')
52
+ rescue Exception
53
+ @supported = false
54
+ end
55
+
56
+ def pagesize
57
+ @pagesize ||= sysconf(:SC_PAGESIZE, DEFAULT_PAGESIZE)
58
+ end
59
+
60
+ def user_hz
61
+ @user_hz ||= sysconf(:SC_CLK_TCK, DEFAULT_USER_HZ).to_f
62
+ end
63
+
64
+ # Wall time the machine booted. /proc/<pid>/stat reports starttime
65
+ # relative to boot, so this is what turns it into an absolute time.
66
+ def boot_time
67
+ @boot_time ||= begin
68
+ Time.now - File.read('/proc/uptime').split[0].to_f
69
+ rescue Exception
70
+ Time.now
71
+ end
72
+ end
73
+
74
+ private
75
+
76
+ def sysconf(name, default)
77
+ return default unless Etc.const_defined?(name)
78
+
79
+ value = Etc.sysconf(Etc.const_get(name))
80
+ (value and value > 0) ? value : default
81
+ rescue Exception
82
+ default
83
+ end
84
+ end
85
+
86
+ def initialize(pid)
87
+ @pid = pid
88
+ @start_time = nil
89
+ @last_sample_time = nil
90
+ # nil rather than 0 so a process that has genuinely burned no CPU yet
91
+ # still starts producing sample deltas on the next pass
92
+ @cpu_time = nil
93
+ @average_cpu_utilization = 0.0
94
+ @sample_cpu_utilization = 0.0
95
+ @virtual_memory_bytes = 0.0
96
+ @resident_memory_bytes = 0.0
97
+ @num_threads = 0
98
+ end
99
+
100
+ # @return [Boolean] whether a usable sample was taken
101
+ def sample
102
+ stat = read_stat()
103
+ return false unless stat
104
+
105
+ fields = split_stat(stat)
106
+ return false if fields.length <= RSS_INDEX
107
+
108
+ sample_time = Time.now
109
+ user_hz = self.class.user_hz
110
+ cpu_time = (fields[UTIME_INDEX].to_f + fields[STIME_INDEX].to_f) / user_hz
111
+ @start_time ||= self.class.boot_time + (fields[STARTTIME_INDEX].to_f / user_hz)
112
+
113
+ if @cpu_time and @last_sample_time
114
+ time_delta = sample_time - @last_sample_time
115
+ @sample_cpu_utilization = (cpu_time - @cpu_time) / time_delta if time_delta > 0
116
+ end
117
+ @cpu_time = cpu_time
118
+ @last_sample_time = sample_time
119
+
120
+ elapsed = sample_time - @start_time
121
+ @average_cpu_utilization = cpu_time / elapsed if elapsed > 0
122
+
123
+ @num_threads = fields[NUM_THREADS_INDEX].to_i
124
+ @virtual_memory_bytes = fields[VSIZE_INDEX].to_f
125
+ @resident_memory_bytes = fields[RSS_INDEX].to_f * self.class.pagesize
126
+ true
127
+ end
128
+
129
+ # Shaped like OpenC3::Metric's internal data hash so it can be handed
130
+ # straight to MetricModel.
131
+ def values
132
+ {
133
+ 'average_cpu_utilization' => { 'value' => @average_cpu_utilization, 'type' => 'gauge', 'unit' => 'ratio' },
134
+ 'sample_cpu_utilization' => { 'value' => @sample_cpu_utilization, 'type' => 'gauge', 'unit' => 'ratio' },
135
+ 'virtual_memory_bytes' => { 'value' => @virtual_memory_bytes, 'type' => 'gauge', 'unit' => 'bytes' },
136
+ 'resident_memory_bytes' => { 'value' => @resident_memory_bytes, 'type' => 'gauge', 'unit' => 'bytes' },
137
+ 'num_threads' => { 'value' => @num_threads, 'type' => 'gauge' },
138
+ }
139
+ end
140
+
141
+ private
142
+
143
+ def read_stat
144
+ File.read("/proc/#{@pid}/stat")
145
+ rescue SystemCallError, IOError
146
+ # The process can exit between being listed and being read
147
+ nil
148
+ end
149
+
150
+ # The comm field is wrapped in parens and may itself contain spaces and
151
+ # parens, so split from the last close paren rather than trying to match it.
152
+ # Two placeholders stand in for pid and comm to keep the proc(5) offsets.
153
+ def split_stat(stat)
154
+ close = stat.rindex(')')
155
+ return [] unless close
156
+
157
+ ['0', '0'].concat(stat[(close + 1)..-1].to_s.split)
158
+ end
159
+ end
160
+ end
@@ -18,6 +18,18 @@ module OpenC3
18
18
  class PypiUrl
19
19
  DEFAULT = 'https://pypi.org/simple'
20
20
 
21
+ # Turn a configured PyPI base URL (the pypi_url setting or PYPI_URL) into
22
+ # its simple index URL. Trailing slashes are dropped first, matching the
23
+ # ${PYPI_URL%/}/simple the image builds use, so https://mirror/ does not
24
+ # give https://mirror//simple and https://pypi.org/ still equals DEFAULT.
25
+ #
26
+ # @param base_url [String, nil] the configured base url
27
+ # @return [String, nil] the simple index url, or nil when base_url is nil
28
+ def self.index_url(base_url)
29
+ return nil if base_url.nil?
30
+ "#{base_url.sub(%r{/+\z}, '')}/simple"
31
+ end
32
+
21
33
  # Validate that a resolved pypi_url is an http(s) URL before it is handed to
22
34
  # pip. The value can come from a user-writable setting or ENV, so a malformed
23
35
  # or non-http value is rejected and replaced with the default rather than
@@ -35,5 +47,58 @@ module OpenC3
35
47
  Logger.error("Invalid pypi_url '#{pypi_url}' (#{e.message}); falling back to #{DEFAULT}")
36
48
  DEFAULT
37
49
  end
50
+
51
+ # Environment variable that opts a deployment into insecure (unverified TLS)
52
+ # connections to the pypi index host, for operators running a private index
53
+ # with a self-signed certificate. OPENC3_ prefixed like our other settings,
54
+ # and kept out of uv's own UV_* namespace (uv reads UV_INSECURE_HOST itself).
55
+ INSECURE_HOST_ENV = 'OPENC3_PYPI_ALLOW_INSECURE'
56
+
57
+ # Previous name for INSECURE_HOST_ENV, still honored so existing helm values
58
+ # and compose files keep working. It was named for pip's --trusted-host, but
59
+ # every install path goes through uv (openc3/bin/pipinstall is a uv pip
60
+ # install wrapper), so the flag it produces is uv's --allow-insecure-host.
61
+ DEPRECATED_INSECURE_HOST_ENV = 'PIP_ENABLE_TRUSTED_HOST'
62
+
63
+ # Build the argv array of index arguments shared by every uv invocation.
64
+ #
65
+ # --default-index rather than -i/--index-url: the latter is deprecated by uv
66
+ # on both `uv sync` and `uv pip install`. Note this only affects the index uv
67
+ # resolves against, so it has no effect on `uv sync --frozen`, where uv.lock
68
+ # already pins each package's registry and wheel URL.
69
+ #
70
+ # --allow-insecure-host rather than --trusted-host: --trusted-host is an
71
+ # undocumented uv alias, so use the real uv option.
72
+ #
73
+ # @param pypi_url [String] the resolved and validated pypi index url
74
+ # @return [Array<String>] arguments to append to a uv command line
75
+ def self.build_args(pypi_url)
76
+ args = ["--default-index", pypi_url]
77
+ # --default-index does not win on its own: uv searches a named index
78
+ # declared in the plugin's own [tool.uv].index table first, so a plugin
79
+ # can quietly resolve against its author's index instead of the one the
80
+ # operator configured. --no-config makes uv ignore that table. Gated on
81
+ # the operator having actually designated an index, because at the public
82
+ # default there is no operator policy to enforce and the plugin author's
83
+ # own index configuration is the only signal there is.
84
+ #
85
+ # This is half the story: a package pinned with [tool.uv].sources is
86
+ # project metadata that survives --no-config. openc3/bin/uvinstall and
87
+ # openc3/bin/pipinstall add --no-sources for that, on the commands where
88
+ # uv actually resolves.
89
+ args << "--no-config" if pypi_url != DEFAULT
90
+ if allow_insecure_host?
91
+ args += ["--allow-insecure-host", URI.parse(pypi_url).host]
92
+ end
93
+ args
94
+ end
95
+
96
+ # @return [Boolean] whether the deployment opted into insecure connections
97
+ # to the pypi index host
98
+ def self.allow_insecure_host?
99
+ # Empty counts as unset: compose passes both names through as
100
+ # ${VAR:-}, so an unset .env entry arrives as an empty string.
101
+ !ENV[INSECURE_HOST_ENV].to_s.empty? || !ENV[DEPRECATED_INSECURE_HOST_ENV].to_s.empty?
102
+ end
38
103
  end
39
104
  end