openc3 7.4.0 → 7.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/openc3cli +4 -3
- data/bin/pipinstall +18 -5
- data/bin/pipuninstall +1 -1
- data/bin/uvinstall +36 -8
- data/data/config/default_plugins.yaml +43 -0
- data/data/config/interface_modifiers.yaml +22 -7
- data/data/config/item_modifiers.yaml +14 -4
- data/data/config/plugins.yaml +1 -1
- data/data/config/widgets.yaml +1 -1
- data/lib/openc3/api/cmd_api.rb +10 -13
- data/lib/openc3/api/interface_api.rb +1 -2
- data/lib/openc3/api/limits_api.rb +1 -1
- data/lib/openc3/api/metrics_api.rb +26 -1
- data/lib/openc3/api/router_api.rb +1 -2
- data/lib/openc3/api/tlm_api.rb +21 -21
- data/lib/openc3/config/config_parser.rb +3 -1
- data/lib/openc3/core_ext/tempfile.rb +2 -0
- data/lib/openc3/interfaces/file_interface.rb +6 -1
- data/lib/openc3/interfaces/http_client_interface.rb +14 -6
- data/lib/openc3/interfaces/interface.rb +27 -3
- data/lib/openc3/interfaces/protocols/protocol.rb +8 -0
- data/lib/openc3/interfaces/read_queue_interface.rb +331 -0
- data/lib/openc3/interfaces/stream_interface.rb +22 -3
- data/lib/openc3/interfaces/tcpip_server_interface.rb +13 -0
- data/lib/openc3/interfaces/udp_interface.rb +16 -2
- data/lib/openc3/interfaces.rb +1 -0
- data/lib/openc3/io/json_api_object.rb +24 -3
- data/lib/openc3/io/json_drb_object.rb +3 -2
- data/lib/openc3/logs/log_writer.rb +13 -4
- data/lib/openc3/microservices/decom_microservice.rb +7 -4
- data/lib/openc3/microservices/interface_microservice.rb +146 -50
- data/lib/openc3/microservices/log_microservice.rb +1 -3
- data/lib/openc3/microservices/microservice.rb +25 -1
- data/lib/openc3/microservices/plugin_microservice.rb +32 -0
- data/lib/openc3/microservices/text_log_microservice.rb +1 -3
- data/lib/openc3/models/interface_model.rb +1 -1
- data/lib/openc3/models/interface_status_model.rb +6 -0
- data/lib/openc3/models/metric_model.rb +52 -4
- data/lib/openc3/models/microservice_model.rb +7 -3
- data/lib/openc3/models/plugin_model.rb +12 -12
- data/lib/openc3/models/python_package_model.rb +3 -13
- data/lib/openc3/models/reaction_model.rb +6 -3
- data/lib/openc3/models/setting_model.rb +5 -3
- data/lib/openc3/models/target_model.rb +83 -9
- data/lib/openc3/operators/microservice_operator.rb +5 -0
- data/lib/openc3/operators/operator.rb +69 -0
- data/lib/openc3/packets/packet.rb +10 -4
- data/lib/openc3/packets/packet_config.rb +48 -2
- data/lib/openc3/packets/packet_item.rb +27 -14
- data/lib/openc3/packets/parsers/state_parser.rb +1 -1
- data/lib/openc3/packets/parsers/xtce_parser.rb +8 -1
- data/lib/openc3/packets/structure_item.rb +3 -2
- data/lib/openc3/script/api_shared.rb +58 -84
- data/lib/openc3/script/extract.rb +215 -19
- data/lib/openc3/script/plugins.rb +4 -11
- data/lib/openc3/script/storage.rb +81 -10
- data/lib/openc3/script/tables.rb +5 -1
- data/lib/openc3/system/system.rb +9 -6
- data/lib/openc3/topics/command_decom_topic.rb +1 -1
- data/lib/openc3/topics/command_topic.rb +2 -1
- data/lib/openc3/topics/router_topic.rb +2 -1
- data/lib/openc3/topics/telemetry_decom_topic.rb +1 -1
- data/lib/openc3/topics/telemetry_topic.rb +3 -2
- data/lib/openc3/topics/topic.rb +154 -2
- data/lib/openc3/utilities/cli_generator.rb +67 -15
- data/lib/openc3/utilities/config_overlay.rb +35 -17
- data/lib/openc3/utilities/default_plugins.rb +33 -0
- data/lib/openc3/utilities/local_mode.rb +82 -46
- data/lib/openc3/utilities/metric.rb +16 -0
- data/lib/openc3/utilities/process_stats.rb +160 -0
- data/lib/openc3/utilities/pypi_url.rb +65 -0
- data/lib/openc3/utilities/questdb_client.rb +43 -46
- data/lib/openc3/utilities/running_script.rb +22 -18
- data/lib/openc3/utilities/store_autoload.rb +20 -3
- data/lib/openc3/utilities/store_queued.rb +37 -9
- data/lib/openc3/utilities/target_file.rb +57 -12
- data/lib/openc3/utilities/topic_lag_monitor.rb +193 -0
- data/lib/openc3/version.rb +5 -5
- data/templates/conversion/conversion.py +6 -2
- data/templates/conversion/conversion.rb +5 -0
- data/templates/plugin/.github/workflows/playwright.yml +62 -0
- data/templates/plugin/.github/workflows/release.yml +45 -0
- data/templates/plugin/.github/workflows/unit-tests.yml +35 -0
- data/templates/plugin/.gitignore +6 -0
- data/templates/plugin/README.md +114 -2
- data/templates/plugin/lib/PACKAGE/__init__.py +4 -0
- data/templates/plugin/plugin.gemspec +10 -1
- data/templates/plugin/pyproject.toml +119 -0
- data/templates/tool_angular/package.json +2 -2
- data/templates/tool_angular/pnpm-workspace.yaml +16 -0
- data/templates/tool_react/package.json +1 -1
- data/templates/tool_react/pnpm-workspace.yaml +16 -0
- data/templates/tool_svelte/package.json +2 -1
- data/templates/tool_svelte/pnpm-workspace.yaml +20 -0
- data/templates/tool_vue/package.json +3 -3
- data/templates/tool_vue/pnpm-workspace.yaml +16 -0
- data/templates/widget/package.json +2 -2
- data/templates/widget/pnpm-workspace.yaml +16 -0
- metadata +36 -1
|
@@ -15,24 +15,27 @@ module OpenC3
|
|
|
15
15
|
# Shared predicates describing which parts of the user-writable config overlay
|
|
16
16
|
# (targets_modified) a non-admin may write.
|
|
17
17
|
#
|
|
18
|
-
#
|
|
19
|
-
#
|
|
20
|
-
#
|
|
21
|
-
# decom microservices.
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
18
|
+
# Two target subdirectories hold definitions whose GENERIC_READ_CONVERSION /
|
|
19
|
+
# GENERIC_WRITE_CONVERSION blocks are evaluated as code:
|
|
20
|
+
# - <TARGET>/cmd_tlm/...: loaded by System.setup_targets and processed by
|
|
21
|
+
# PacketConfig in the decom microservices. Only the server-side
|
|
22
|
+
# dynamic-packet mechanism (TargetModel#dynamic_update) writes that area
|
|
23
|
+
# without going through an API request.
|
|
24
|
+
# - <TARGET>/tables/config/...: table definitions, processed by TableConfig
|
|
25
|
+
# in cmd-tlm-api for every Table Manager operation.
|
|
26
|
+
# Writing either is therefore an admin operation, matching the tier COSMOS
|
|
27
|
+
# requires everywhere else code is introduced (plugin install).
|
|
25
28
|
#
|
|
26
29
|
# Every API writer that can reach the overlay must consult these predicates:
|
|
27
30
|
# - storage_controller#get_upload_presigned_request / delete (presigned S3 upload)
|
|
28
|
-
# - tables_controller#save / save_as /
|
|
31
|
+
# - tables_controller#save / save_as / report / destroy (Table -> TargetFile)
|
|
29
32
|
# - scripts_controller#create / destroy (Script -> TargetFile)
|
|
30
33
|
module ConfigOverlay
|
|
31
34
|
# Config bucket areas a non-admin is allowed to write at all
|
|
32
35
|
NON_ADMIN_AREAS = ['targets_modified', 'tmp'].freeze
|
|
33
36
|
|
|
34
|
-
# Target
|
|
35
|
-
|
|
37
|
+
# Target subdirectories whose contents are executed as code
|
|
38
|
+
CODE_AREAS = [['cmd_tlm'], ['tables', 'config']].freeze
|
|
36
39
|
|
|
37
40
|
# Split a path into segments, or nil if the path is not canonical.
|
|
38
41
|
#
|
|
@@ -48,14 +51,23 @@ module OpenC3
|
|
|
48
51
|
parts
|
|
49
52
|
end
|
|
50
53
|
|
|
51
|
-
# True if the overlay-relative name targets
|
|
52
|
-
# it requires admin. Name is relative to
|
|
53
|
-
# "<TARGET>/cmd_tlm/tlm.txt". Fails closed (true)
|
|
54
|
-
|
|
54
|
+
# True if the overlay-relative name targets a code area (cmd_tlm or
|
|
55
|
+
# tables/config), i.e. writing it requires admin. Name is relative to
|
|
56
|
+
# targets_modified/, e.g. "<TARGET>/cmd_tlm/tlm.txt". Fails closed (true)
|
|
57
|
+
# on non-canonical names.
|
|
58
|
+
def self.code_overlay?(name)
|
|
55
59
|
parts = canonical_parts(name)
|
|
56
60
|
return true if parts.nil?
|
|
57
|
-
|
|
58
|
-
|
|
61
|
+
code_area?(parts)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# True if the target-relative name is a canonical table definition path,
|
|
65
|
+
# e.g. "<TARGET>/tables/config/table_def.txt". Table only parses definitions
|
|
66
|
+
# from here, so a definition can't be pointed at a non-admin overlay area.
|
|
67
|
+
def self.table_definition?(name)
|
|
68
|
+
parts = canonical_parts(name)
|
|
69
|
+
return false if parts.nil?
|
|
70
|
+
parts.length > 3 and parts[1, 2] == ['tables', 'config']
|
|
59
71
|
end
|
|
60
72
|
|
|
61
73
|
# True if the given config bucket key is an overlay path a non-admin may
|
|
@@ -68,8 +80,14 @@ module OpenC3
|
|
|
68
80
|
# parts: <SCOPE> / <area> / <TARGET> / <subdir> / ...
|
|
69
81
|
area = parts[1]
|
|
70
82
|
return false unless NON_ADMIN_AREAS.include?(area)
|
|
71
|
-
return false if area == 'targets_modified' && parts[
|
|
83
|
+
return false if area == 'targets_modified' && code_area?(parts[2..])
|
|
72
84
|
true
|
|
73
85
|
end
|
|
86
|
+
|
|
87
|
+
# target_parts: <TARGET> / <subdir> / ...
|
|
88
|
+
def self.code_area?(target_parts)
|
|
89
|
+
CODE_AREAS.any? { |area| target_parts[1, area.length] == area }
|
|
90
|
+
end
|
|
91
|
+
private_class_method :code_area?
|
|
74
92
|
end
|
|
75
93
|
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# encoding: ascii-8bit
|
|
2
|
+
|
|
3
|
+
# Copyright 2026 OpenC3, Inc.
|
|
4
|
+
# All Rights Reserved.
|
|
5
|
+
#
|
|
6
|
+
# This program is distributed in the hope that it will be useful,
|
|
7
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
8
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
9
|
+
# See LICENSE.md for more details.
|
|
10
|
+
#
|
|
11
|
+
# This file may also be used under the terms of a commercial license
|
|
12
|
+
# if purchased from OpenC3, Inc.
|
|
13
|
+
|
|
14
|
+
require 'yaml'
|
|
15
|
+
|
|
16
|
+
module OpenC3
|
|
17
|
+
# The plugins that ship with COSMOS. The list lives in
|
|
18
|
+
# data/config/default_plugins.yaml and is shared with the Admin Plugins tab
|
|
19
|
+
# and the script API via the 'default' flag from GET /openc3-api/plugins/all
|
|
20
|
+
module DefaultPlugins
|
|
21
|
+
# Short names, i.e. without the version or the __ suffix
|
|
22
|
+
NAMES = YAML.safe_load_file(
|
|
23
|
+
File.expand_path('../../../data/config/default_plugins.yaml', __dir__)
|
|
24
|
+
).freeze
|
|
25
|
+
|
|
26
|
+
# True if the plugin (e.g. "openc3-cosmos-tool-admin-6.0.0.gem__20260101000000")
|
|
27
|
+
# is one that ships with COSMOS
|
|
28
|
+
def self.default?(plugin_name)
|
|
29
|
+
# Strip the __ suffix and the trailing -version.gem to get the short name
|
|
30
|
+
NAMES.include?(plugin_name.split('__')[0].split('-')[0..-2].join('-'))
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -17,39 +17,11 @@ require 'openc3/core_ext/file'
|
|
|
17
17
|
# require 'openc3/models/gem_model' # These are used but also create circular dependency
|
|
18
18
|
# require 'openc3/models/plugin_model' # These are used but also create circular dependency
|
|
19
19
|
require 'openc3/utilities/bucket'
|
|
20
|
+
require 'openc3/utilities/default_plugins'
|
|
20
21
|
|
|
21
22
|
module OpenC3
|
|
22
23
|
module LocalMode
|
|
23
24
|
OPENC3_LOCAL_MODE_PATH = ENV['OPENC3_LOCAL_MODE_PATH'] || "/plugins"
|
|
24
|
-
# When updating update local_mode.py, PluginsTab.vue, plugins.p.spec.ts
|
|
25
|
-
DEFAULT_PLUGINS = [
|
|
26
|
-
'openc3-cosmos-tool-admin',
|
|
27
|
-
'openc3-cosmos-tool-bucketexplorer',
|
|
28
|
-
'openc3-cosmos-tool-cmdsender',
|
|
29
|
-
'openc3-cosmos-tool-cmdqueue', # Enterprise only
|
|
30
|
-
'openc3-cosmos-tool-cmdhistory', # Enterprise only
|
|
31
|
-
'openc3-cosmos-tool-cmdtlmserver',
|
|
32
|
-
'openc3-cosmos-tool-dataextractor',
|
|
33
|
-
'openc3-cosmos-tool-dataviewer',
|
|
34
|
-
'openc3-cosmos-tool-docs',
|
|
35
|
-
'openc3-cosmos-tool-handbooks',
|
|
36
|
-
'openc3-cosmos-tool-iframe',
|
|
37
|
-
'openc3-cosmos-tool-limitsmonitor',
|
|
38
|
-
'openc3-cosmos-tool-packetviewer',
|
|
39
|
-
'openc3-cosmos-tool-scriptrunner',
|
|
40
|
-
'openc3-cosmos-tool-tablemanager',
|
|
41
|
-
'openc3-cosmos-tool-tlmgrapher',
|
|
42
|
-
'openc3-cosmos-tool-tlmviewer',
|
|
43
|
-
'openc3-cosmos-enterprise-tool-admin', # Enterprise only
|
|
44
|
-
'openc3-cosmos-tool-autonomic', # Enterprise only
|
|
45
|
-
'openc3-cosmos-tool-calendar', # Enterprise only
|
|
46
|
-
'openc3-cosmos-tool-grafana', # Enterprise only
|
|
47
|
-
'openc3-cosmos-tool-logexplorer', # Enterprise only
|
|
48
|
-
'openc3-cosmos-tool-notebooks', # Enterprise only
|
|
49
|
-
'openc3-cosmos-tool-systemhealth', # Enterprise only
|
|
50
|
-
'openc3-enterprise-tool-base', # Enterprise only
|
|
51
|
-
'openc3-tool-base',
|
|
52
|
-
]
|
|
53
25
|
|
|
54
26
|
# Install plugins from local plugins folder
|
|
55
27
|
# Can only be used from openc3cli because calls top_level load_plugin
|
|
@@ -280,7 +252,7 @@ module OpenC3
|
|
|
280
252
|
end
|
|
281
253
|
|
|
282
254
|
def self.update_local_plugin_files(full_folder_path, plugin_file_path, plugin_hash, gem_name)
|
|
283
|
-
return if
|
|
255
|
+
return if DefaultPlugins::NAMES.include?(gem_name)
|
|
284
256
|
puts "Updating local plugin files: #{full_folder_path}"
|
|
285
257
|
FileUtils.mkdir_p(full_folder_path)
|
|
286
258
|
gems = scan_plugin_dir(full_folder_path).first
|
|
@@ -378,8 +350,8 @@ module OpenC3
|
|
|
378
350
|
end
|
|
379
351
|
|
|
380
352
|
def self.put_target_file(path, io_or_string, scope:) # NOSONAR - scope: is part of the caller-facing signature
|
|
381
|
-
full_folder_path =
|
|
382
|
-
return unless
|
|
353
|
+
full_folder_path = key_path(path)
|
|
354
|
+
return unless full_folder_path and path.start_with?("#{scope}/")
|
|
383
355
|
FileUtils.mkdir_p(File.dirname(full_folder_path))
|
|
384
356
|
File.open(full_folder_path, 'wb') do |file|
|
|
385
357
|
if String === io_or_string
|
|
@@ -392,8 +364,8 @@ module OpenC3
|
|
|
392
364
|
end
|
|
393
365
|
|
|
394
366
|
def self.open_local_file(path, scope:)
|
|
395
|
-
full_path = "
|
|
396
|
-
if
|
|
367
|
+
full_path = scope_path(scope, "targets_modified/#{path}")
|
|
368
|
+
if full_path and safe_key?(path)
|
|
397
369
|
return File.open(full_path, 'rb')
|
|
398
370
|
end
|
|
399
371
|
nil
|
|
@@ -452,8 +424,8 @@ module OpenC3
|
|
|
452
424
|
def self.save_tool_config(scope, tool, name, data)
|
|
453
425
|
return unless ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
|
|
454
426
|
json = JSON.parse(data, allow_nan: true, create_additions: true)
|
|
455
|
-
config_path = "
|
|
456
|
-
return unless
|
|
427
|
+
config_path = scope_path(scope, "tool_config/#{tool}/#{name}.json")
|
|
428
|
+
return unless config_path and safe_key?("#{tool}/#{name}")
|
|
457
429
|
FileUtils.mkdir_p(File.dirname(config_path))
|
|
458
430
|
File.open(config_path, 'w') do |file|
|
|
459
431
|
file.write(JSON.pretty_generate(json, allow_nan: true))
|
|
@@ -462,8 +434,8 @@ module OpenC3
|
|
|
462
434
|
|
|
463
435
|
def self.delete_tool_config(scope, tool, name)
|
|
464
436
|
return unless ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
|
|
465
|
-
config_path = "
|
|
466
|
-
return unless
|
|
437
|
+
config_path = scope_path(scope, "tool_config/#{tool}/#{name}.json")
|
|
438
|
+
return unless config_path and safe_key?("#{tool}/#{name}")
|
|
467
439
|
FileUtils.rm_f(config_path)
|
|
468
440
|
end
|
|
469
441
|
|
|
@@ -508,8 +480,8 @@ module OpenC3
|
|
|
508
480
|
|
|
509
481
|
def self.save_setting(scope, name, data)
|
|
510
482
|
return unless ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
|
|
511
|
-
config_path = "
|
|
512
|
-
return unless
|
|
483
|
+
config_path = scope_path(scope, "settings/#{name}.json")
|
|
484
|
+
return unless config_path and safe_key?(name)
|
|
513
485
|
FileUtils.mkdir_p(File.dirname(config_path))
|
|
514
486
|
# Anything can be stored as a setting. Strings are written directly since
|
|
515
487
|
# they're already the serialized form. Anything else is written as JSON so
|
|
@@ -518,26 +490,86 @@ module OpenC3
|
|
|
518
490
|
File.write(config_path, data)
|
|
519
491
|
end
|
|
520
492
|
|
|
493
|
+
# Target names from OPENC3_LOCAL_ONLY_TARGETS (comma separated). These
|
|
494
|
+
# targets keep their target files only in the local mode volume and never
|
|
495
|
+
# read or write them in the bucket. Read on every call so it can't go stale.
|
|
496
|
+
def self.local_only_targets
|
|
497
|
+
ENV['OPENC3_LOCAL_ONLY_TARGETS'].to_s.split(',').map { |name| name.strip.upcase }.reject(&:empty?)
|
|
498
|
+
end
|
|
499
|
+
|
|
500
|
+
def self.local_only_target?(target_name)
|
|
501
|
+
return false if target_name.nil? or target_name.to_s.empty?
|
|
502
|
+
local_only_targets.include?(target_name.to_s.upcase)
|
|
503
|
+
end
|
|
504
|
+
|
|
505
|
+
# name is relative to targets / targets_modified, e.g. "INST/procedures/test.rb"
|
|
506
|
+
def self.local_only_name?(name)
|
|
507
|
+
local_only_target?(name.to_s.split('/')[0])
|
|
508
|
+
end
|
|
509
|
+
|
|
510
|
+
# key is a config bucket key, e.g. "DEFAULT/targets_modified/INST/procedures/test.rb"
|
|
511
|
+
def self.local_only_key?(key)
|
|
512
|
+
split_key = key.to_s.split('/')
|
|
513
|
+
return false unless ['targets', 'targets_modified'].include?(split_key[1])
|
|
514
|
+
local_only_target?(split_key[2])
|
|
515
|
+
end
|
|
516
|
+
|
|
521
517
|
# Helper methods
|
|
522
518
|
|
|
519
|
+
# True if the key has no '.' / '..' segments, no empty segments, no leading
|
|
520
|
+
# slash and no backslash. Such a key always resolves under its first
|
|
521
|
+
# segment (the scope), both on disk and in the bucket (the bucket
|
|
522
|
+
# normalizes '..' segments, so a traversal key reaches other scopes there too).
|
|
523
|
+
def self.safe_key?(key)
|
|
524
|
+
return false unless String === key
|
|
525
|
+
return false if key.empty? or key.start_with?('/') or key.include?('\\') or key.include?("\0")
|
|
526
|
+
key.split('/', -1).none? { |segment| segment.empty? or segment == '.' or segment == '..' }
|
|
527
|
+
end
|
|
528
|
+
|
|
529
|
+
# True if path resolves to root or somewhere beneath it. Compares against
|
|
530
|
+
# root plus a separator so a sibling such as /plugins_old never matches.
|
|
531
|
+
def self.path_within?(path, root)
|
|
532
|
+
expanded = File.expand_path(path)
|
|
533
|
+
root = File.expand_path(root)
|
|
534
|
+
expanded == root or expanded.start_with?(root + File::SEPARATOR)
|
|
535
|
+
end
|
|
536
|
+
|
|
537
|
+
# Local file path for a scope relative path, or nil if the path would
|
|
538
|
+
# leave the scope directory
|
|
539
|
+
def self.scope_path(scope, relative_path)
|
|
540
|
+
return nil unless safe_key?(scope) and !scope.include?('/')
|
|
541
|
+
scope_root = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}"
|
|
542
|
+
full_path = "#{scope_root}/#{relative_path}"
|
|
543
|
+
return nil unless path_within?(full_path, scope_root)
|
|
544
|
+
full_path
|
|
545
|
+
end
|
|
546
|
+
|
|
547
|
+
# Local file path for a bucket key, or nil if the key is not a safe key
|
|
548
|
+
def self.key_path(key)
|
|
549
|
+
return nil unless safe_key?(key)
|
|
550
|
+
full_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
|
|
551
|
+
return nil unless path_within?(full_path, OPENC3_LOCAL_MODE_PATH)
|
|
552
|
+
full_path
|
|
553
|
+
end
|
|
554
|
+
|
|
523
555
|
def self.sync_remote_to_local(bucket, key)
|
|
524
|
-
local_path =
|
|
525
|
-
return unless
|
|
556
|
+
local_path = key_path(key)
|
|
557
|
+
return unless local_path
|
|
526
558
|
FileUtils.mkdir_p(File.dirname(local_path))
|
|
527
559
|
bucket.get_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: key, path: local_path)
|
|
528
560
|
end
|
|
529
561
|
|
|
530
562
|
def self.sync_local_to_remote(bucket, key)
|
|
531
|
-
local_path =
|
|
532
|
-
return unless
|
|
563
|
+
local_path = key_path(key)
|
|
564
|
+
return unless local_path
|
|
533
565
|
File.open(local_path, 'rb') do |read_file|
|
|
534
566
|
bucket.put_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: key, body: read_file)
|
|
535
567
|
end
|
|
536
568
|
end
|
|
537
569
|
|
|
538
570
|
def self.delete_local(key)
|
|
539
|
-
local_path =
|
|
540
|
-
return unless
|
|
571
|
+
local_path = key_path(key)
|
|
572
|
+
return unless local_path
|
|
541
573
|
File.delete(local_path) if File.exist?(local_path)
|
|
542
574
|
nil
|
|
543
575
|
end
|
|
@@ -580,6 +612,10 @@ module OpenC3
|
|
|
580
612
|
# Build catalogs
|
|
581
613
|
local_catalog = build_local_catalog(scope: scope)
|
|
582
614
|
remote_catalog = build_remote_catalog(bucket, scope: scope)
|
|
615
|
+
# Local only targets never sync in either direction. Any stale copies
|
|
616
|
+
# already in the bucket are left alone rather than pulled down or deleted.
|
|
617
|
+
local_catalog.reject! { |key, _size| local_only_key?(key) }
|
|
618
|
+
remote_catalog.reject! { |key, _size| local_only_key?(key) }
|
|
583
619
|
|
|
584
620
|
# Find and Handle Differences
|
|
585
621
|
local_catalog.each do |key, size|
|
|
@@ -133,7 +133,23 @@ module OpenC3
|
|
|
133
133
|
def graceful_kill
|
|
134
134
|
end
|
|
135
135
|
|
|
136
|
+
# Environment variable the operator sets on every microservice it spawns to
|
|
137
|
+
# say "I am already reporting process metrics for you". Without it an
|
|
138
|
+
# in-process generator and the operator would both be writing cpu/memory for
|
|
139
|
+
# the same microservice.
|
|
140
|
+
SUPERVISED_ENV_VAR = 'OPENC3_OPERATOR_PROCESS_METRICS'.freeze
|
|
141
|
+
|
|
142
|
+
# Whether something outside this process is already reporting its process
|
|
143
|
+
# level metrics.
|
|
144
|
+
def self.supervised?
|
|
145
|
+
!ENV[SUPERVISED_ENV_VAR].nil?
|
|
146
|
+
end
|
|
147
|
+
|
|
136
148
|
def self.add_update_generator(object)
|
|
149
|
+
# Generators produce process level metrics (cpu, memory). Skip them
|
|
150
|
+
# entirely when the operator is already reporting those for us.
|
|
151
|
+
return if supervised?
|
|
152
|
+
|
|
137
153
|
@@update_generators << object
|
|
138
154
|
end
|
|
139
155
|
end
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# encoding: ascii-8bit
|
|
2
|
+
|
|
3
|
+
# Copyright 2026 OpenC3, Inc.
|
|
4
|
+
# All Rights Reserved.
|
|
5
|
+
#
|
|
6
|
+
# This program is distributed in the hope that it will be useful,
|
|
7
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
8
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
9
|
+
# See LICENSE.md for more details.
|
|
10
|
+
#
|
|
11
|
+
# This file may also be used under the terms of a commercial license
|
|
12
|
+
# if purchased from OpenC3, Inc.
|
|
13
|
+
|
|
14
|
+
require 'etc'
|
|
15
|
+
|
|
16
|
+
# See: https://man7.org/linux/man-pages/man5/proc.5.html
|
|
17
|
+
|
|
18
|
+
module OpenC3
|
|
19
|
+
# Samples CPU and memory for an arbitrary pid out of /proc/<pid>/stat.
|
|
20
|
+
#
|
|
21
|
+
# Every microservice is bootstrapped by plugin_microservice.rb, which exec()s
|
|
22
|
+
# the configured cmd. exec replaces the process image, so everything the
|
|
23
|
+
# bootstrap set up - including the Metric update thread - is gone. A
|
|
24
|
+
# microservice whose cmd is itself an OpenC3::Microservice builds a new Metric
|
|
25
|
+
# and keeps reporting, but a plugin that runs a Rails app, a Python script or
|
|
26
|
+
# any other binary never loads the OpenC3 libraries and so can never report on
|
|
27
|
+
# itself. Sampling by pid lets the operator report on behalf of those
|
|
28
|
+
# processes without requiring anything of the plugin.
|
|
29
|
+
class ProcessStats
|
|
30
|
+
# /proc/<pid>/stat field offsets, 0 based, after the comm field has been
|
|
31
|
+
# collapsed to a single token. proc(5) numbers these fields from 1.
|
|
32
|
+
UTIME_INDEX = 13
|
|
33
|
+
STIME_INDEX = 14
|
|
34
|
+
NUM_THREADS_INDEX = 19
|
|
35
|
+
STARTTIME_INDEX = 21
|
|
36
|
+
VSIZE_INDEX = 22
|
|
37
|
+
RSS_INDEX = 23
|
|
38
|
+
|
|
39
|
+
# Near universal Linux default. Only used if sysconf can't answer - a zero
|
|
40
|
+
# here would turn every CPU number into Infinity or NaN.
|
|
41
|
+
DEFAULT_USER_HZ = 100.0
|
|
42
|
+
DEFAULT_PAGESIZE = 4096
|
|
43
|
+
|
|
44
|
+
attr_reader :pid
|
|
45
|
+
|
|
46
|
+
class << self
|
|
47
|
+
# Whether this kernel exposes the procfs data we need at all. macOS and
|
|
48
|
+
# Windows don't, and neither do some hardened container runtimes.
|
|
49
|
+
def supported?
|
|
50
|
+
return @supported unless @supported.nil?
|
|
51
|
+
@supported = File.exist?('/proc/self/stat')
|
|
52
|
+
rescue Exception
|
|
53
|
+
@supported = false
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def pagesize
|
|
57
|
+
@pagesize ||= sysconf(:SC_PAGESIZE, DEFAULT_PAGESIZE)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def user_hz
|
|
61
|
+
@user_hz ||= sysconf(:SC_CLK_TCK, DEFAULT_USER_HZ).to_f
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Wall time the machine booted. /proc/<pid>/stat reports starttime
|
|
65
|
+
# relative to boot, so this is what turns it into an absolute time.
|
|
66
|
+
def boot_time
|
|
67
|
+
@boot_time ||= begin
|
|
68
|
+
Time.now - File.read('/proc/uptime').split[0].to_f
|
|
69
|
+
rescue Exception
|
|
70
|
+
Time.now
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
private
|
|
75
|
+
|
|
76
|
+
def sysconf(name, default)
|
|
77
|
+
return default unless Etc.const_defined?(name)
|
|
78
|
+
|
|
79
|
+
value = Etc.sysconf(Etc.const_get(name))
|
|
80
|
+
(value and value > 0) ? value : default
|
|
81
|
+
rescue Exception
|
|
82
|
+
default
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def initialize(pid)
|
|
87
|
+
@pid = pid
|
|
88
|
+
@start_time = nil
|
|
89
|
+
@last_sample_time = nil
|
|
90
|
+
# nil rather than 0 so a process that has genuinely burned no CPU yet
|
|
91
|
+
# still starts producing sample deltas on the next pass
|
|
92
|
+
@cpu_time = nil
|
|
93
|
+
@average_cpu_utilization = 0.0
|
|
94
|
+
@sample_cpu_utilization = 0.0
|
|
95
|
+
@virtual_memory_bytes = 0.0
|
|
96
|
+
@resident_memory_bytes = 0.0
|
|
97
|
+
@num_threads = 0
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# @return [Boolean] whether a usable sample was taken
|
|
101
|
+
def sample
|
|
102
|
+
stat = read_stat()
|
|
103
|
+
return false unless stat
|
|
104
|
+
|
|
105
|
+
fields = split_stat(stat)
|
|
106
|
+
return false if fields.length <= RSS_INDEX
|
|
107
|
+
|
|
108
|
+
sample_time = Time.now
|
|
109
|
+
user_hz = self.class.user_hz
|
|
110
|
+
cpu_time = (fields[UTIME_INDEX].to_f + fields[STIME_INDEX].to_f) / user_hz
|
|
111
|
+
@start_time ||= self.class.boot_time + (fields[STARTTIME_INDEX].to_f / user_hz)
|
|
112
|
+
|
|
113
|
+
if @cpu_time and @last_sample_time
|
|
114
|
+
time_delta = sample_time - @last_sample_time
|
|
115
|
+
@sample_cpu_utilization = (cpu_time - @cpu_time) / time_delta if time_delta > 0
|
|
116
|
+
end
|
|
117
|
+
@cpu_time = cpu_time
|
|
118
|
+
@last_sample_time = sample_time
|
|
119
|
+
|
|
120
|
+
elapsed = sample_time - @start_time
|
|
121
|
+
@average_cpu_utilization = cpu_time / elapsed if elapsed > 0
|
|
122
|
+
|
|
123
|
+
@num_threads = fields[NUM_THREADS_INDEX].to_i
|
|
124
|
+
@virtual_memory_bytes = fields[VSIZE_INDEX].to_f
|
|
125
|
+
@resident_memory_bytes = fields[RSS_INDEX].to_f * self.class.pagesize
|
|
126
|
+
true
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Shaped like OpenC3::Metric's internal data hash so it can be handed
|
|
130
|
+
# straight to MetricModel.
|
|
131
|
+
def values
|
|
132
|
+
{
|
|
133
|
+
'average_cpu_utilization' => { 'value' => @average_cpu_utilization, 'type' => 'gauge', 'unit' => 'ratio' },
|
|
134
|
+
'sample_cpu_utilization' => { 'value' => @sample_cpu_utilization, 'type' => 'gauge', 'unit' => 'ratio' },
|
|
135
|
+
'virtual_memory_bytes' => { 'value' => @virtual_memory_bytes, 'type' => 'gauge', 'unit' => 'bytes' },
|
|
136
|
+
'resident_memory_bytes' => { 'value' => @resident_memory_bytes, 'type' => 'gauge', 'unit' => 'bytes' },
|
|
137
|
+
'num_threads' => { 'value' => @num_threads, 'type' => 'gauge' },
|
|
138
|
+
}
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
private
|
|
142
|
+
|
|
143
|
+
def read_stat
|
|
144
|
+
File.read("/proc/#{@pid}/stat")
|
|
145
|
+
rescue SystemCallError, IOError
|
|
146
|
+
# The process can exit between being listed and being read
|
|
147
|
+
nil
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# The comm field is wrapped in parens and may itself contain spaces and
|
|
151
|
+
# parens, so split from the last close paren rather than trying to match it.
|
|
152
|
+
# Two placeholders stand in for pid and comm to keep the proc(5) offsets.
|
|
153
|
+
def split_stat(stat)
|
|
154
|
+
close = stat.rindex(')')
|
|
155
|
+
return [] unless close
|
|
156
|
+
|
|
157
|
+
['0', '0'].concat(stat[(close + 1)..-1].to_s.split)
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
end
|
|
@@ -18,6 +18,18 @@ module OpenC3
|
|
|
18
18
|
class PypiUrl
|
|
19
19
|
DEFAULT = 'https://pypi.org/simple'
|
|
20
20
|
|
|
21
|
+
# Turn a configured PyPI base URL (the pypi_url setting or PYPI_URL) into
|
|
22
|
+
# its simple index URL. Trailing slashes are dropped first, matching the
|
|
23
|
+
# ${PYPI_URL%/}/simple the image builds use, so https://mirror/ does not
|
|
24
|
+
# give https://mirror//simple and https://pypi.org/ still equals DEFAULT.
|
|
25
|
+
#
|
|
26
|
+
# @param base_url [String, nil] the configured base url
|
|
27
|
+
# @return [String, nil] the simple index url, or nil when base_url is nil
|
|
28
|
+
def self.index_url(base_url)
|
|
29
|
+
return nil if base_url.nil?
|
|
30
|
+
"#{base_url.sub(%r{/+\z}, '')}/simple"
|
|
31
|
+
end
|
|
32
|
+
|
|
21
33
|
# Validate that a resolved pypi_url is an http(s) URL before it is handed to
|
|
22
34
|
# pip. The value can come from a user-writable setting or ENV, so a malformed
|
|
23
35
|
# or non-http value is rejected and replaced with the default rather than
|
|
@@ -35,5 +47,58 @@ module OpenC3
|
|
|
35
47
|
Logger.error("Invalid pypi_url '#{pypi_url}' (#{e.message}); falling back to #{DEFAULT}")
|
|
36
48
|
DEFAULT
|
|
37
49
|
end
|
|
50
|
+
|
|
51
|
+
# Environment variable that opts a deployment into insecure (unverified TLS)
|
|
52
|
+
# connections to the pypi index host, for operators running a private index
|
|
53
|
+
# with a self-signed certificate. OPENC3_ prefixed like our other settings,
|
|
54
|
+
# and kept out of uv's own UV_* namespace (uv reads UV_INSECURE_HOST itself).
|
|
55
|
+
INSECURE_HOST_ENV = 'OPENC3_PYPI_ALLOW_INSECURE'
|
|
56
|
+
|
|
57
|
+
# Previous name for INSECURE_HOST_ENV, still honored so existing helm values
|
|
58
|
+
# and compose files keep working. It was named for pip's --trusted-host, but
|
|
59
|
+
# every install path goes through uv (openc3/bin/pipinstall is a uv pip
|
|
60
|
+
# install wrapper), so the flag it produces is uv's --allow-insecure-host.
|
|
61
|
+
DEPRECATED_INSECURE_HOST_ENV = 'PIP_ENABLE_TRUSTED_HOST'
|
|
62
|
+
|
|
63
|
+
# Build the argv array of index arguments shared by every uv invocation.
|
|
64
|
+
#
|
|
65
|
+
# --default-index rather than -i/--index-url: the latter is deprecated by uv
|
|
66
|
+
# on both `uv sync` and `uv pip install`. Note this only affects the index uv
|
|
67
|
+
# resolves against, so it has no effect on `uv sync --frozen`, where uv.lock
|
|
68
|
+
# already pins each package's registry and wheel URL.
|
|
69
|
+
#
|
|
70
|
+
# --allow-insecure-host rather than --trusted-host: --trusted-host is an
|
|
71
|
+
# undocumented uv alias, so use the real uv option.
|
|
72
|
+
#
|
|
73
|
+
# @param pypi_url [String] the resolved and validated pypi index url
|
|
74
|
+
# @return [Array<String>] arguments to append to a uv command line
|
|
75
|
+
def self.build_args(pypi_url)
|
|
76
|
+
args = ["--default-index", pypi_url]
|
|
77
|
+
# --default-index does not win on its own: uv searches a named index
|
|
78
|
+
# declared in the plugin's own [tool.uv].index table first, so a plugin
|
|
79
|
+
# can quietly resolve against its author's index instead of the one the
|
|
80
|
+
# operator configured. --no-config makes uv ignore that table. Gated on
|
|
81
|
+
# the operator having actually designated an index, because at the public
|
|
82
|
+
# default there is no operator policy to enforce and the plugin author's
|
|
83
|
+
# own index configuration is the only signal there is.
|
|
84
|
+
#
|
|
85
|
+
# This is half the story: a package pinned with [tool.uv].sources is
|
|
86
|
+
# project metadata that survives --no-config. openc3/bin/uvinstall and
|
|
87
|
+
# openc3/bin/pipinstall add --no-sources for that, on the commands where
|
|
88
|
+
# uv actually resolves.
|
|
89
|
+
args << "--no-config" if pypi_url != DEFAULT
|
|
90
|
+
if allow_insecure_host?
|
|
91
|
+
args += ["--allow-insecure-host", URI.parse(pypi_url).host]
|
|
92
|
+
end
|
|
93
|
+
args
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# @return [Boolean] whether the deployment opted into insecure connections
|
|
97
|
+
# to the pypi index host
|
|
98
|
+
def self.allow_insecure_host?
|
|
99
|
+
# Empty counts as unset: compose passes both names through as
|
|
100
|
+
# ${VAR:-}, so an unset .env entry arrives as an empty string.
|
|
101
|
+
!ENV[INSECURE_HOST_ENV].to_s.empty? || !ENV[DEPRECATED_INSECURE_HOST_ENV].to_s.empty?
|
|
102
|
+
end
|
|
38
103
|
end
|
|
39
104
|
end
|