openc3 7.0.1 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/openc3cli +47 -3
- data/data/config/item_modifiers.yaml +1 -1
- data/data/config/microservice.yaml +12 -1
- data/data/config/parameter_modifiers.yaml +49 -7
- data/data/config/target.yaml +11 -0
- data/data/config/target_config.yaml +6 -2
- data/lib/openc3/api/cmd_api.rb +2 -1
- data/lib/openc3/api/metrics_api.rb +11 -1
- data/lib/openc3/api/tlm_api.rb +21 -6
- data/lib/openc3/core_ext/faraday.rb +1 -1
- data/lib/openc3/io/json_api.rb +1 -1
- data/lib/openc3/logs/log_writer.rb +3 -1
- data/lib/openc3/microservices/decom_common.rb +128 -0
- data/lib/openc3/microservices/decom_microservice.rb +26 -95
- data/lib/openc3/microservices/interface_decom_common.rb +6 -2
- data/lib/openc3/microservices/interface_microservice.rb +10 -8
- data/lib/openc3/microservices/log_microservice.rb +1 -1
- data/lib/openc3/microservices/microservice.rb +3 -2
- data/lib/openc3/microservices/queue_microservice.rb +1 -1
- data/lib/openc3/microservices/scope_cleanup_microservice.rb +60 -46
- data/lib/openc3/microservices/text_log_microservice.rb +1 -2
- data/lib/openc3/models/cvt_model.rb +24 -13
- data/lib/openc3/models/db_sharded_model.rb +110 -0
- data/lib/openc3/models/interface_model.rb +9 -0
- data/lib/openc3/models/interface_status_model.rb +33 -3
- data/lib/openc3/models/metric_model.rb +96 -37
- data/lib/openc3/models/microservice_model.rb +7 -0
- data/lib/openc3/models/microservice_status_model.rb +30 -3
- data/lib/openc3/models/reingest_job_model.rb +153 -0
- data/lib/openc3/models/scope_model.rb +3 -2
- data/lib/openc3/models/script_status_model.rb +4 -20
- data/lib/openc3/models/target_model.rb +113 -100
- data/lib/openc3/packets/packet_config.rb +4 -1
- data/lib/openc3/script/script.rb +2 -2
- data/lib/openc3/script/script_runner.rb +4 -4
- data/lib/openc3/script/telemetry.rb +3 -3
- data/lib/openc3/script/web_socket_api.rb +29 -22
- data/lib/openc3/system/system.rb +20 -3
- data/lib/openc3/topics/command_decom_topic.rb +4 -2
- data/lib/openc3/topics/command_topic.rb +8 -5
- data/lib/openc3/topics/decom_interface_topic.rb +15 -10
- data/lib/openc3/topics/interface_topic.rb +71 -29
- data/lib/openc3/topics/limits_event_topic.rb +62 -41
- data/lib/openc3/topics/router_topic.rb +61 -21
- data/lib/openc3/topics/system_events_topic.rb +18 -1
- data/lib/openc3/topics/telemetry_decom_topic.rb +2 -1
- data/lib/openc3/topics/telemetry_topic.rb +4 -2
- data/lib/openc3/topics/topic.rb +77 -5
- data/lib/openc3/utilities/aws_bucket.rb +2 -0
- data/lib/openc3/utilities/cli_generator.rb +3 -2
- data/lib/openc3/utilities/metric.rb +15 -1
- data/lib/openc3/utilities/questdb_client.rb +173 -37
- data/lib/openc3/utilities/reingest_job.rb +377 -0
- data/lib/openc3/utilities/ruby_lex_utils.rb +2 -0
- data/lib/openc3/utilities/store_autoload.rb +78 -52
- data/lib/openc3/utilities/store_queued.rb +20 -12
- data/lib/openc3/version.rb +6 -6
- data/templates/plugin/plugin.gemspec +13 -1
- data/templates/tool_angular/package.json +2 -2
- data/templates/tool_react/package.json +1 -1
- data/templates/tool_svelte/package.json +1 -1
- data/templates/tool_vue/package.json +3 -3
- data/templates/tool_vue/src/router.js +2 -2
- data/templates/widget/package.json +2 -2
- metadata +7 -3
|
@@ -81,6 +81,7 @@ module OpenC3
|
|
|
81
81
|
attr_accessor :children
|
|
82
82
|
attr_accessor :disable_erb
|
|
83
83
|
attr_accessor :shard
|
|
84
|
+
attr_accessor :db_shard
|
|
84
85
|
|
|
85
86
|
# NOTE: The following three class methods are used by the ModelController
|
|
86
87
|
# and are reimplemented to enable various Model class methods to work
|
|
@@ -202,6 +203,11 @@ module OpenC3
|
|
|
202
203
|
return result
|
|
203
204
|
end
|
|
204
205
|
|
|
206
|
+
# Get a Store instance routed to the correct db_shard for a target
|
|
207
|
+
def self.store_for_target(target_name, scope:)
|
|
208
|
+
Store.instance(db_shard: Store.db_shard_for_target(target_name, scope: scope))
|
|
209
|
+
end
|
|
210
|
+
|
|
205
211
|
# @return [Hash] Packet hash or raises an exception
|
|
206
212
|
def self.packet(target_name, packet_name, type: :TLM, scope:)
|
|
207
213
|
raise "Unknown type #{type} for #{target_name} #{packet_name}" unless VALID_TYPES.include?(type)
|
|
@@ -216,7 +222,7 @@ module OpenC3
|
|
|
216
222
|
end
|
|
217
223
|
|
|
218
224
|
# Assume it exists and just try to get it to avoid an extra call to Store.exist?
|
|
219
|
-
json =
|
|
225
|
+
json = store_for_target(target_name, scope: scope).hget("#{scope}__openc3#{type.to_s.downcase}__#{target_name}", packet_name)
|
|
220
226
|
raise "Packet '#{target_name} #{packet_name}' does not exist" if json.nil?
|
|
221
227
|
|
|
222
228
|
packet = JSON.parse(json, allow_nan: true, create_additions: true)
|
|
@@ -241,7 +247,7 @@ module OpenC3
|
|
|
241
247
|
raise "Target '#{target_name}' does not exist for scope: #{scope}" unless get(name: target_name, scope: scope)
|
|
242
248
|
|
|
243
249
|
result = []
|
|
244
|
-
packets =
|
|
250
|
+
packets = store_for_target(target_name, scope: scope).hgetall("#{scope}__openc3#{type.to_s.downcase}__#{target_name}")
|
|
245
251
|
packets.sort.each do |_packet_name, packet_json|
|
|
246
252
|
result << JSON.parse(packet_json, allow_nan: true, create_additions: true)
|
|
247
253
|
end
|
|
@@ -263,7 +269,7 @@ module OpenC3
|
|
|
263
269
|
end
|
|
264
270
|
|
|
265
271
|
begin
|
|
266
|
-
|
|
272
|
+
store_for_target(target_name, scope: scope).hset("#{scope}__openc3#{type.to_s.downcase}__#{target_name}", packet_name, JSON.generate(packet.as_json, allow_nan: true))
|
|
267
273
|
rescue JSON::GeneratorError => e
|
|
268
274
|
Logger.error("Invalid text present in #{target_name} #{packet_name} #{type.to_s.downcase} packet")
|
|
269
275
|
raise e
|
|
@@ -296,7 +302,7 @@ module OpenC3
|
|
|
296
302
|
|
|
297
303
|
# @return [Array<String>] All the item names for every packet in a target
|
|
298
304
|
def self.all_item_names(target_name, type: :TLM, scope:)
|
|
299
|
-
items =
|
|
305
|
+
items = store_for_target(target_name, scope: scope).zrange("#{scope}__openc3tlm__#{target_name}__allitems", 0, -1)
|
|
300
306
|
items = rebuild_target_allitems_list(target_name, type: type, scope: scope) if items.empty?
|
|
301
307
|
items
|
|
302
308
|
end
|
|
@@ -308,12 +314,12 @@ module OpenC3
|
|
|
308
314
|
TargetModel.add_to_target_allitems_list(target_name, item['name'], scope: scope)
|
|
309
315
|
end
|
|
310
316
|
end
|
|
311
|
-
|
|
317
|
+
store_for_target(target_name, scope: scope).zrange("#{scope}__openc3tlm__#{target_name}__allitems", 0, -1) # return the new sorted set to let redis do the sorting
|
|
312
318
|
end
|
|
313
319
|
|
|
314
320
|
def self.add_to_target_allitems_list(target_name, item_name, scope:)
|
|
315
321
|
score = 0 # https://redis.io/docs/latest/develop/data-types/sorted-sets/#lexicographical-scores
|
|
316
|
-
|
|
322
|
+
store_for_target(target_name, scope: scope).zadd("#{scope}__openc3tlm__#{target_name}__allitems", score, item_name)
|
|
317
323
|
end
|
|
318
324
|
|
|
319
325
|
# @return [Hash{String => Array<Array<String, String, String>>}]
|
|
@@ -331,12 +337,13 @@ module OpenC3
|
|
|
331
337
|
return item_map if item_map and (Time.now - cache_time) < ITEM_MAP_CACHE_TIMEOUT
|
|
332
338
|
item_map_key = "#{scope}__#{target_name}__item_to_packet_map"
|
|
333
339
|
target_name = target_name.upcase
|
|
334
|
-
|
|
340
|
+
store = store_for_target(target_name, scope: scope)
|
|
341
|
+
json_data = store.get(item_map_key)
|
|
335
342
|
if json_data
|
|
336
343
|
item_map = JSON.parse(json_data, allow_nan: true, create_additions: true)
|
|
337
344
|
else
|
|
338
345
|
item_map = build_item_to_packet_map(target_name, scope: scope)
|
|
339
|
-
|
|
346
|
+
store.set(item_map_key, JSON.generate(item_map, allow_nan: true))
|
|
340
347
|
end
|
|
341
348
|
@@item_map_cache[target_name] = [Time.now, item_map]
|
|
342
349
|
return item_map
|
|
@@ -396,6 +403,7 @@ module OpenC3
|
|
|
396
403
|
target_microservices: {},
|
|
397
404
|
disable_erb: nil,
|
|
398
405
|
shard: 0,
|
|
406
|
+
db_shard: 0,
|
|
399
407
|
scope:
|
|
400
408
|
)
|
|
401
409
|
super("#{scope}__#{PRIMARY_KEY}", name: name, plugin: plugin, updated_at: updated_at, scope: scope)
|
|
@@ -421,6 +429,7 @@ module OpenC3
|
|
|
421
429
|
@target_microservices = target_microservices
|
|
422
430
|
@disable_erb = disable_erb
|
|
423
431
|
@shard = shard.to_i # to_i to handle nil
|
|
432
|
+
@db_shard = db_shard.to_i # to_i to handle nil
|
|
424
433
|
@bucket = Bucket.getClient()
|
|
425
434
|
@children = []
|
|
426
435
|
end
|
|
@@ -452,6 +461,7 @@ module OpenC3
|
|
|
452
461
|
'target_microservices' => @target_microservices.as_json(),
|
|
453
462
|
'disable_erb' => @disable_erb,
|
|
454
463
|
'shard' => @shard,
|
|
464
|
+
'db_shard' => @db_shard,
|
|
455
465
|
}
|
|
456
466
|
end
|
|
457
467
|
|
|
@@ -543,6 +553,10 @@ module OpenC3
|
|
|
543
553
|
parser.verify_num_parameters(1, 1, "#{keyword} <Shard Number Starting from 0>")
|
|
544
554
|
@shard = Integer(parameters[0])
|
|
545
555
|
|
|
556
|
+
when 'DB_SHARD'
|
|
557
|
+
parser.verify_num_parameters(1, 1, "#{keyword} <Shard Number Starting from 0>")
|
|
558
|
+
@db_shard = Integer(parameters[0])
|
|
559
|
+
|
|
546
560
|
else
|
|
547
561
|
raise ConfigParser::Error.new(parser, "Unknown keyword and parameters for Target: #{keyword} #{parameters.join(" ")}")
|
|
548
562
|
end
|
|
@@ -626,19 +640,20 @@ module OpenC3
|
|
|
626
640
|
Store.hdel("#{@scope}__limits_groups", group)
|
|
627
641
|
end
|
|
628
642
|
self.class.packets(@name, type: :CMD, scope: @scope).each do |packet|
|
|
629
|
-
Topic.del("#{@scope}__COMMAND__{#{@name}}__#{packet['packet_name']}")
|
|
630
|
-
Topic.del("#{@scope}__DECOMCMD__{#{@name}}__#{packet['packet_name']}")
|
|
643
|
+
Topic.del("#{@scope}__COMMAND__{#{@name}}__#{packet['packet_name']}", db_shard: @db_shard)
|
|
644
|
+
Topic.del("#{@scope}__DECOMCMD__{#{@name}}__#{packet['packet_name']}", db_shard: @db_shard)
|
|
631
645
|
end
|
|
632
646
|
self.class.packets(@name, scope: @scope).each do |packet|
|
|
633
|
-
Topic.del("#{@scope}__TELEMETRY__{#{@name}}__#{packet['packet_name']}")
|
|
634
|
-
Topic.del("#{@scope}__DECOM__{#{@name}}__#{packet['packet_name']}")
|
|
647
|
+
Topic.del("#{@scope}__TELEMETRY__{#{@name}}__#{packet['packet_name']}", db_shard: @db_shard)
|
|
648
|
+
Topic.del("#{@scope}__DECOM__{#{@name}}__#{packet['packet_name']}", db_shard: @db_shard)
|
|
635
649
|
CvtModel.del(target_name: @name, packet_name: packet['packet_name'], scope: @scope)
|
|
636
650
|
end
|
|
637
651
|
LimitsEventTopic.delete(@name, scope: @scope)
|
|
638
|
-
Store.
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
652
|
+
db_shard_store = Store.instance(db_shard: @db_shard)
|
|
653
|
+
db_shard_store.del("#{@scope}__openc3tlm__#{@name}")
|
|
654
|
+
db_shard_store.del("#{@scope}__openc3cmd__#{@name}")
|
|
655
|
+
db_shard_store.del("#{@scope}__TELEMETRYCNTS__{#{@name}}")
|
|
656
|
+
db_shard_store.del("#{@scope}__COMMANDCNTS__{#{@name}}")
|
|
642
657
|
|
|
643
658
|
# Note: these match the names of the services in deploy_microservices
|
|
644
659
|
%w(MULTI DECOM COMMANDLOG PACKETLOG CLEANUP).each do |type|
|
|
@@ -657,7 +672,7 @@ module OpenC3
|
|
|
657
672
|
end
|
|
658
673
|
# Delete item_map
|
|
659
674
|
item_map_key = "#{@scope}__#{@name}__item_to_packet_map"
|
|
660
|
-
|
|
675
|
+
db_shard_store.del(item_map_key)
|
|
661
676
|
@@item_map_cache[@name] = nil
|
|
662
677
|
|
|
663
678
|
topic = { kind: 'deleted', type: 'target', name: @name }
|
|
@@ -752,16 +767,17 @@ module OpenC3
|
|
|
752
767
|
end
|
|
753
768
|
|
|
754
769
|
def update_store_telemetry(packet_hash, clear_old: true)
|
|
770
|
+
db_shard_store = Store.instance(db_shard: @db_shard)
|
|
755
771
|
packet_hash.each do |target_name, packets|
|
|
756
772
|
if clear_old
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
773
|
+
db_shard_store.del("#{@scope}__openc3tlm__#{target_name}")
|
|
774
|
+
db_shard_store.del("#{@scope}__openc3tlm__#{target_name}__allitems")
|
|
775
|
+
db_shard_store.del("#{@scope}__TELEMETRYCNTS__{#{target_name}}")
|
|
760
776
|
end
|
|
761
777
|
packets.each do |packet_name, packet|
|
|
762
778
|
Logger.debug "Configuring tlm packet: #{target_name} #{packet_name}"
|
|
763
779
|
begin
|
|
764
|
-
|
|
780
|
+
db_shard_store.hset("#{@scope}__openc3tlm__#{target_name}", packet_name, JSON.generate(packet.as_json, allow_nan: true))
|
|
765
781
|
rescue JSON::GeneratorError => e
|
|
766
782
|
Logger.error("Invalid text present in #{target_name} #{packet_name} tlm packet")
|
|
767
783
|
raise e
|
|
@@ -777,15 +793,16 @@ module OpenC3
|
|
|
777
793
|
end
|
|
778
794
|
|
|
779
795
|
def update_store_commands(packet_hash, clear_old: true)
|
|
796
|
+
db_shard_store = Store.instance(db_shard: @db_shard)
|
|
780
797
|
packet_hash.each do |target_name, packets|
|
|
781
798
|
if clear_old
|
|
782
|
-
|
|
783
|
-
|
|
799
|
+
db_shard_store.del("#{@scope}__openc3cmd__#{target_name}")
|
|
800
|
+
db_shard_store.del("#{@scope}__COMMANDCNTS__{#{target_name}}")
|
|
784
801
|
end
|
|
785
802
|
packets.each do |packet_name, packet|
|
|
786
803
|
Logger.debug "Configuring cmd packet: #{target_name} #{packet_name}"
|
|
787
804
|
begin
|
|
788
|
-
|
|
805
|
+
db_shard_store.hset("#{@scope}__openc3cmd__#{target_name}", packet_name, JSON.generate(packet.as_json, allow_nan: true))
|
|
789
806
|
rescue JSON::GeneratorError => e
|
|
790
807
|
Logger.error("Invalid text present in #{target_name} #{packet_name} cmd packet")
|
|
791
808
|
raise e
|
|
@@ -818,7 +835,7 @@ module OpenC3
|
|
|
818
835
|
# Create item_map
|
|
819
836
|
item_map_key = "#{@scope}__#{@name}__item_to_packet_map"
|
|
820
837
|
item_map = self.class.build_item_to_packet_map(@name, scope: @scope)
|
|
821
|
-
Store.set(item_map_key, JSON.generate(item_map, allow_nan: true))
|
|
838
|
+
Store.instance(db_shard: @db_shard).set(item_map_key, JSON.generate(item_map, allow_nan: true))
|
|
822
839
|
@@item_map_cache[@name] = [Time.now, item_map]
|
|
823
840
|
end
|
|
824
841
|
|
|
@@ -888,15 +905,15 @@ module OpenC3
|
|
|
888
905
|
end
|
|
889
906
|
end
|
|
890
907
|
if cmd_or_tlm == :TELEMETRY
|
|
891
|
-
Topic.write_topic("MICROSERVICE__#{@scope}__PACKETLOG__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json})
|
|
908
|
+
Topic.write_topic("MICROSERVICE__#{@scope}__PACKETLOG__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json}, db_shard: @db_shard)
|
|
892
909
|
add_topics_to_microservice("#{@scope}__PACKETLOG__#{@name}", raw_topics)
|
|
893
|
-
Topic.write_topic("MICROSERVICE__#{@scope}__DECOM__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json})
|
|
910
|
+
Topic.write_topic("MICROSERVICE__#{@scope}__DECOM__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json}, db_shard: @db_shard)
|
|
894
911
|
add_topics_to_microservice("#{@scope}__DECOM__#{@name}", raw_topics)
|
|
895
912
|
else
|
|
896
|
-
Topic.write_topic("MICROSERVICE__#{@scope}__COMMANDLOG__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json})
|
|
913
|
+
Topic.write_topic("MICROSERVICE__#{@scope}__COMMANDLOG__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json}, db_shard: @db_shard)
|
|
897
914
|
add_topics_to_microservice("#{@scope}__COMMANDLOG__#{@name}", raw_topics)
|
|
898
915
|
end
|
|
899
|
-
Topic.write_topic("MICROSERVICE__#{@scope}__TSDB__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => decom_topics.as_json.to_json})
|
|
916
|
+
Topic.write_topic("MICROSERVICE__#{@scope}__TSDB__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => decom_topics.as_json.to_json}, db_shard: @db_shard)
|
|
900
917
|
add_topics_to_microservice("#{@scope}__TSDB__#{@name}", decom_topics)
|
|
901
918
|
end
|
|
902
919
|
|
|
@@ -927,6 +944,7 @@ module OpenC3
|
|
|
927
944
|
parent: parent,
|
|
928
945
|
needs_dependencies: @needs_dependencies,
|
|
929
946
|
shard: @shard,
|
|
947
|
+
db_shard: @db_shard,
|
|
930
948
|
scope: @scope
|
|
931
949
|
)
|
|
932
950
|
microservice.create
|
|
@@ -954,6 +972,7 @@ module OpenC3
|
|
|
954
972
|
parent: parent,
|
|
955
973
|
needs_dependencies: @needs_dependencies,
|
|
956
974
|
shard: @shard,
|
|
975
|
+
db_shard: @db_shard,
|
|
957
976
|
scope: @scope
|
|
958
977
|
)
|
|
959
978
|
microservice.create
|
|
@@ -986,6 +1005,7 @@ module OpenC3
|
|
|
986
1005
|
parent: parent,
|
|
987
1006
|
needs_dependencies: @needs_dependencies,
|
|
988
1007
|
shard: @shard,
|
|
1008
|
+
db_shard: @db_shard,
|
|
989
1009
|
scope: @scope
|
|
990
1010
|
)
|
|
991
1011
|
microservice.create
|
|
@@ -1010,6 +1030,7 @@ module OpenC3
|
|
|
1010
1030
|
parent: nil,
|
|
1011
1031
|
needs_dependencies: @needs_dependencies,
|
|
1012
1032
|
shard: @shard,
|
|
1033
|
+
db_shard: @db_shard,
|
|
1013
1034
|
scope: @scope
|
|
1014
1035
|
)
|
|
1015
1036
|
microservice.create
|
|
@@ -1026,6 +1047,7 @@ module OpenC3
|
|
|
1026
1047
|
plugin: @plugin,
|
|
1027
1048
|
parent: parent,
|
|
1028
1049
|
shard: @shard,
|
|
1050
|
+
db_shard: @db_shard,
|
|
1029
1051
|
scope: @scope
|
|
1030
1052
|
)
|
|
1031
1053
|
microservice.create
|
|
@@ -1044,6 +1066,7 @@ module OpenC3
|
|
|
1044
1066
|
plugin: @plugin,
|
|
1045
1067
|
needs_dependencies: @needs_dependencies,
|
|
1046
1068
|
shard: @shard,
|
|
1069
|
+
db_shard: @db_shard,
|
|
1047
1070
|
scope: @scope
|
|
1048
1071
|
)
|
|
1049
1072
|
microservice.create
|
|
@@ -1139,11 +1162,19 @@ module OpenC3
|
|
|
1139
1162
|
deploy_target_microservices('PACKETLOG', packet_topic_list, "#{@scope}__TELEMETRY__{#{@name}}") do |topics, instance, parent|
|
|
1140
1163
|
deploy_packetlog_microservice(gem_path, variables, topics, instance, parent)
|
|
1141
1164
|
end
|
|
1165
|
+
end
|
|
1142
1166
|
|
|
1143
|
-
|
|
1167
|
+
# Decommutation Microservice - also handles build_cmd / inject_tlm /
|
|
1168
|
+
# get_tlm_buffer via the DECOMINTERFACE topic, so deploy whenever the
|
|
1169
|
+
# target has commands or telemetry, not just telemetry.
|
|
1170
|
+
if packet_topic_list.any?
|
|
1144
1171
|
deploy_target_microservices('DECOM', packet_topic_list, "#{@scope}__TELEMETRY__{#{@name}}") do |topics, instance, parent|
|
|
1145
1172
|
deploy_decom_microservice(system.targets[@name], gem_path, variables, topics, instance, parent)
|
|
1146
1173
|
end
|
|
1174
|
+
elsif command_topic_list.any?
|
|
1175
|
+
# Cmd-only target: deploy DECOM with no tlm topics so build_cmd works.
|
|
1176
|
+
# DecomMicroservice subscribes to DECOMINTERFACE in its initializer.
|
|
1177
|
+
deploy_decom_microservice(system.targets[@name], gem_path, variables, [], nil, @parent)
|
|
1147
1178
|
end
|
|
1148
1179
|
|
|
1149
1180
|
# TSDB Microservice - subscribes to both decommutated telemetry and commands
|
|
@@ -1168,7 +1199,7 @@ module OpenC3
|
|
|
1168
1199
|
end
|
|
1169
1200
|
|
|
1170
1201
|
def self.increment_telemetry_count(target_name, packet_name, count, scope:)
|
|
1171
|
-
result =
|
|
1202
|
+
result = store_for_target(target_name, scope: scope).hincrby("#{scope}__TELEMETRYCNTS__{#{target_name}}", packet_name, count)
|
|
1172
1203
|
if String === result
|
|
1173
1204
|
return result.to_i
|
|
1174
1205
|
else
|
|
@@ -1178,7 +1209,7 @@ module OpenC3
|
|
|
1178
1209
|
|
|
1179
1210
|
def self.get_all_telemetry_counts(target_name, scope:)
|
|
1180
1211
|
result = {}
|
|
1181
|
-
get_all =
|
|
1212
|
+
get_all = store_for_target(target_name, scope: scope).hgetall("#{scope}__TELEMETRYCNTS__{#{target_name}}")
|
|
1182
1213
|
if Hash === get_all
|
|
1183
1214
|
get_all.each do |key, value|
|
|
1184
1215
|
result[key] = value.to_i
|
|
@@ -1189,7 +1220,7 @@ module OpenC3
|
|
|
1189
1220
|
end
|
|
1190
1221
|
|
|
1191
1222
|
def self.get_telemetry_count(target_name, packet_name, scope:)
|
|
1192
|
-
value =
|
|
1223
|
+
value = store_for_target(target_name, scope: scope).hget("#{scope}__TELEMETRYCNTS__{#{target_name}}", packet_name)
|
|
1193
1224
|
if String === value
|
|
1194
1225
|
return value.to_i
|
|
1195
1226
|
elsif value.nil?
|
|
@@ -1200,30 +1231,26 @@ module OpenC3
|
|
|
1200
1231
|
end
|
|
1201
1232
|
|
|
1202
1233
|
def self.get_telemetry_counts(target_packets, scope:)
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1234
|
+
# Group by db_shard, preserving original index
|
|
1235
|
+
db_shard_groups = {} # db_shard => [{index:, target_name:, packet_name:}]
|
|
1236
|
+
target_packets.each_with_index do |(target_name, packet_name), idx|
|
|
1237
|
+
target_name = target_name.upcase
|
|
1238
|
+
packet_name = packet_name.upcase
|
|
1239
|
+
db_shard = Store.db_shard_for_target(target_name, scope: scope)
|
|
1240
|
+
db_shard_groups[db_shard] ||= []
|
|
1241
|
+
db_shard_groups[db_shard] << { index: idx, target_name: target_name, packet_name: packet_name }
|
|
1242
|
+
end
|
|
1243
|
+
|
|
1244
|
+
counts = Array.new(target_packets.length, 0)
|
|
1245
|
+
db_shard_groups.each do |db_shard, entries|
|
|
1246
|
+
store = Store.instance(db_shard: db_shard)
|
|
1247
|
+
result = store.redis_pool.pipelined do
|
|
1248
|
+
entries.each do |entry|
|
|
1249
|
+
store.hget("#{scope}__TELEMETRYCNTS__{#{entry[:target_name]}}", entry[:packet_name])
|
|
1218
1250
|
end
|
|
1219
1251
|
end
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
result.each do |count|
|
|
1223
|
-
if count
|
|
1224
|
-
counts << count.to_i
|
|
1225
|
-
else
|
|
1226
|
-
counts << 0
|
|
1252
|
+
entries.each_with_index do |entry, i|
|
|
1253
|
+
counts[entry[:index]] = result[i] ? result[i].to_i : 0
|
|
1227
1254
|
end
|
|
1228
1255
|
end
|
|
1229
1256
|
return counts
|
|
@@ -1267,7 +1294,7 @@ module OpenC3
|
|
|
1267
1294
|
end
|
|
1268
1295
|
|
|
1269
1296
|
def self.sync_tlm_packet_counts(packet, tlm_target_names, scope:)
|
|
1270
|
-
if @@sync_packet_count_delay_seconds <= 0
|
|
1297
|
+
if @@sync_packet_count_delay_seconds <= 0
|
|
1271
1298
|
# Perfect but slow method
|
|
1272
1299
|
packet.received_count = increment_telemetry_count(packet.target_name, packet.packet_name, 1, scope: scope)
|
|
1273
1300
|
else
|
|
@@ -1288,26 +1315,17 @@ module OpenC3
|
|
|
1288
1315
|
if @@sync_packet_count_time.nil? or (Time.now - @@sync_packet_count_time) > @@sync_packet_count_delay_seconds
|
|
1289
1316
|
@@sync_packet_count_time = Time.now
|
|
1290
1317
|
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
@@sync_packet_count_data.each do |target_name, packet_data|
|
|
1296
|
-
packet_data.each do |packet_name, count|
|
|
1297
|
-
increment_telemetry_count(target_name, packet_name, count, scope: scope)
|
|
1298
|
-
inc_count += 1
|
|
1299
|
-
end
|
|
1300
|
-
end
|
|
1301
|
-
@@sync_packet_count_data = {}
|
|
1302
|
-
|
|
1303
|
-
# Get all the packet counts with the global counters
|
|
1304
|
-
tlm_target_names.each do |target_name|
|
|
1305
|
-
get_all_telemetry_counts(target_name, scope: scope)
|
|
1318
|
+
# Increment global counters for packets received
|
|
1319
|
+
@@sync_packet_count_data.each do |target_name, packet_data|
|
|
1320
|
+
packet_data.each do |packet_name, count|
|
|
1321
|
+
increment_telemetry_count(target_name, packet_name, count, scope: scope)
|
|
1306
1322
|
end
|
|
1307
|
-
get_all_telemetry_counts('UNKNOWN', scope: scope)
|
|
1308
1323
|
end
|
|
1324
|
+
@@sync_packet_count_data = {}
|
|
1325
|
+
|
|
1326
|
+
# Get all the packet counts with the global counters
|
|
1309
1327
|
tlm_target_names.each do |target_name|
|
|
1310
|
-
|
|
1328
|
+
get_all_telemetry_counts(target_name, scope: scope).each do |packet_name, count|
|
|
1311
1329
|
begin
|
|
1312
1330
|
update_packet = System.telemetry.packet(target_name, packet_name)
|
|
1313
1331
|
update_packet.received_count = count.to_i
|
|
@@ -1319,9 +1337,8 @@ module OpenC3
|
|
|
1319
1337
|
end
|
|
1320
1338
|
end
|
|
1321
1339
|
end
|
|
1322
|
-
inc_count += 1
|
|
1323
1340
|
end
|
|
1324
|
-
|
|
1341
|
+
get_all_telemetry_counts('UNKNOWN', scope: scope).each do |packet_name, count|
|
|
1325
1342
|
begin
|
|
1326
1343
|
update_packet = System.telemetry.packet('UNKNOWN', packet_name)
|
|
1327
1344
|
update_packet.received_count = count.to_i
|
|
@@ -1338,7 +1355,7 @@ module OpenC3
|
|
|
1338
1355
|
end
|
|
1339
1356
|
|
|
1340
1357
|
def self.increment_command_count(target_name, packet_name, count, scope:)
|
|
1341
|
-
result =
|
|
1358
|
+
result = store_for_target(target_name, scope: scope).hincrby("#{scope}__COMMANDCNTS__{#{target_name}}", packet_name, count)
|
|
1342
1359
|
if String === result
|
|
1343
1360
|
return result.to_i
|
|
1344
1361
|
else
|
|
@@ -1348,7 +1365,7 @@ module OpenC3
|
|
|
1348
1365
|
|
|
1349
1366
|
def self.get_all_command_counts(target_name, scope:)
|
|
1350
1367
|
result = {}
|
|
1351
|
-
get_all =
|
|
1368
|
+
get_all = store_for_target(target_name, scope: scope).hgetall("#{scope}__COMMANDCNTS__{#{target_name}}")
|
|
1352
1369
|
if Hash === get_all
|
|
1353
1370
|
get_all.each do |key, value|
|
|
1354
1371
|
result[key] = value.to_i
|
|
@@ -1359,7 +1376,7 @@ module OpenC3
|
|
|
1359
1376
|
end
|
|
1360
1377
|
|
|
1361
1378
|
def self.get_command_count(target_name, packet_name, scope:)
|
|
1362
|
-
value =
|
|
1379
|
+
value = store_for_target(target_name, scope: scope).hget("#{scope}__COMMANDCNTS__{#{target_name}}", packet_name)
|
|
1363
1380
|
if String === value
|
|
1364
1381
|
return value.to_i
|
|
1365
1382
|
elsif value.nil?
|
|
@@ -1370,30 +1387,26 @@ module OpenC3
|
|
|
1370
1387
|
end
|
|
1371
1388
|
|
|
1372
1389
|
def self.get_command_counts(target_packets, scope:)
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1390
|
+
# Group by db_shard, preserving original index
|
|
1391
|
+
db_shard_groups = {} # db_shard => [{index:, target_name:, packet_name:}]
|
|
1392
|
+
target_packets.each_with_index do |(target_name, packet_name), idx|
|
|
1393
|
+
target_name = target_name.upcase
|
|
1394
|
+
packet_name = packet_name.upcase
|
|
1395
|
+
db_shard = Store.db_shard_for_target(target_name, scope: scope)
|
|
1396
|
+
db_shard_groups[db_shard] ||= []
|
|
1397
|
+
db_shard_groups[db_shard] << { index: idx, target_name: target_name, packet_name: packet_name }
|
|
1398
|
+
end
|
|
1399
|
+
|
|
1400
|
+
counts = Array.new(target_packets.length, 0)
|
|
1401
|
+
db_shard_groups.each do |db_shard, entries|
|
|
1402
|
+
store = Store.instance(db_shard: db_shard)
|
|
1403
|
+
result = store.redis_pool.pipelined do
|
|
1404
|
+
entries.each do |entry|
|
|
1405
|
+
store.hget("#{scope}__COMMANDCNTS__{#{entry[:target_name]}}", entry[:packet_name])
|
|
1388
1406
|
end
|
|
1389
1407
|
end
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
result.each do |count|
|
|
1393
|
-
if count
|
|
1394
|
-
counts << count.to_i
|
|
1395
|
-
else
|
|
1396
|
-
counts << 0
|
|
1408
|
+
entries.each_with_index do |entry, i|
|
|
1409
|
+
counts[entry[:index]] = result[i] ? result[i].to_i : 0
|
|
1397
1410
|
end
|
|
1398
1411
|
end
|
|
1399
1412
|
return counts
|
|
@@ -437,9 +437,12 @@ module OpenC3
|
|
|
437
437
|
if packet.id_items.length > 0
|
|
438
438
|
key = []
|
|
439
439
|
id_signature = ""
|
|
440
|
+
# Accessor class is part of the signature so packets in the same target
|
|
441
|
+
# with different accessors trigger unique_id_mode (different accessors
|
|
442
|
+
# decode the buffer differently, so the hash-lookup path is unsafe).
|
|
440
443
|
packet.id_items.each do |item|
|
|
441
444
|
key << item.id_value
|
|
442
|
-
id_signature << "__#{item.key}__#{item.bit_offset}__#{item.bit_size}__#{item.data_type}"
|
|
445
|
+
id_signature << "__#{item.key}__#{item.bit_offset}__#{item.bit_size}__#{item.data_type}__#{packet.accessor.class.to_s}"
|
|
443
446
|
end
|
|
444
447
|
target_id_value_hash[key] = packet
|
|
445
448
|
target_id_signature = id_signature_hash[packet.target_name]
|
data/lib/openc3/script/script.rb
CHANGED
|
@@ -44,11 +44,11 @@ require 'openc3/utilities/authentication'
|
|
|
44
44
|
$api_server = nil
|
|
45
45
|
$script_runner_api_server = nil
|
|
46
46
|
$disconnect = false
|
|
47
|
-
$openc3_scope = ENV
|
|
47
|
+
$openc3_scope = ENV.fetch('OPENC3_SCOPE', 'DEFAULT')
|
|
48
48
|
$openc3_in_cluster = false
|
|
49
49
|
|
|
50
50
|
saved_verbose = $VERBOSE
|
|
51
|
-
$VERBOSE =
|
|
51
|
+
$VERBOSE = nil
|
|
52
52
|
|
|
53
53
|
module OpenC3
|
|
54
54
|
module Script
|
|
@@ -37,9 +37,9 @@ module OpenC3
|
|
|
37
37
|
|
|
38
38
|
def script_syntax_check(script, scope: $openc3_scope)
|
|
39
39
|
endpoint = "/script-api/scripts/temp.rb/syntax"
|
|
40
|
-
# Explicitly set the headers to plain
|
|
40
|
+
# Explicitly set the headers to text/plain so the request.body is set correctly
|
|
41
41
|
headers = {
|
|
42
|
-
'Content-Type': 'plain
|
|
42
|
+
'Content-Type': 'text/plain',
|
|
43
43
|
}
|
|
44
44
|
response = $script_runner_api_server.request('post', endpoint, headers: headers, data: script, scope: scope)
|
|
45
45
|
if response.nil? || response.status != 200
|
|
@@ -129,9 +129,9 @@ module OpenC3
|
|
|
129
129
|
|
|
130
130
|
def script_instrumented(script, scope: $openc3_scope)
|
|
131
131
|
endpoint = "/script-api/scripts/temp.rb/instrumented"
|
|
132
|
-
# Explicitly set the headers to plain
|
|
132
|
+
# Explicitly set the headers to text/plain so the request.body is set correctly
|
|
133
133
|
headers = {
|
|
134
|
-
'Content-Type': 'plain
|
|
134
|
+
'Content-Type': 'text/plain',
|
|
135
135
|
}
|
|
136
136
|
response = $script_runner_api_server.request('post', endpoint, headers: headers, data: script, scope: scope)
|
|
137
137
|
if response.nil? || response.status != 200
|
|
@@ -40,9 +40,9 @@ module OpenC3
|
|
|
40
40
|
# inject_tlm, set_tlm, override_tlm, and normalize_tlm are implemented here simply to add a puts
|
|
41
41
|
# these methods modify the telemetry so the user should be notified in the Script Runner log messages
|
|
42
42
|
|
|
43
|
-
def inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, scope: $openc3_scope, token: $openc3_token)
|
|
44
|
-
puts "inject_tlm(\"#{target_name}\", \"#{packet_name}\", #{item_hash}, type: #{type})"
|
|
45
|
-
$api_server.method_missing(:inject_tlm, target_name, packet_name, item_hash, type: type, scope: scope, token: token)
|
|
43
|
+
def inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, stored: false, scope: $openc3_scope, token: $openc3_token)
|
|
44
|
+
puts "inject_tlm(\"#{target_name}\", \"#{packet_name}\", #{item_hash}, type: #{type}, stored: #{stored})"
|
|
45
|
+
$api_server.method_missing(:inject_tlm, target_name, packet_name, item_hash, type: type, stored: stored, scope: scope, token: token)
|
|
46
46
|
end
|
|
47
47
|
|
|
48
48
|
def set_tlm(*args, type: :ALL, scope: $openc3_scope, token: $openc3_token)
|
|
@@ -50,34 +50,41 @@ module OpenC3
|
|
|
50
50
|
start_time = Time.now
|
|
51
51
|
while true
|
|
52
52
|
message = read_message()
|
|
53
|
-
|
|
53
|
+
# Empty string is a normal end-of-stream signal when ActionCable / anycable-go
|
|
54
|
+
# closes the WS. Treat it the same as nil so consumer `while (resp = api.read)`
|
|
55
|
+
# loops exit cleanly instead of hitting JSON::ParserError on JSON.parse("").
|
|
56
|
+
return nil if message.nil? || message.empty?
|
|
57
|
+
|
|
58
|
+
begin
|
|
54
59
|
json_hash = JSON.parse(message, allow_nan: true, create_additions: true)
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
if
|
|
64
|
-
raise "
|
|
65
|
-
end
|
|
66
|
-
if timeout
|
|
67
|
-
end_time = Time.now
|
|
68
|
-
if (end_time - start_time) > timeout
|
|
69
|
-
raise Timeout::Error, "No Data Timeout"
|
|
70
|
-
end
|
|
60
|
+
rescue JSON::ParserError
|
|
61
|
+
# Defense-in-depth: treat malformed frames as end-of-stream rather than crashing.
|
|
62
|
+
return nil
|
|
63
|
+
end
|
|
64
|
+
if ignore_protocol_messages
|
|
65
|
+
type = json_hash['type']
|
|
66
|
+
if type # ping, welcome, confirm_subscription, reject_subscription, disconnect
|
|
67
|
+
if type == 'disconnect'
|
|
68
|
+
if json_hash['reason'] == 'unauthorized'
|
|
69
|
+
raise "Unauthorized"
|
|
71
70
|
end
|
|
72
|
-
|
|
73
|
-
|
|
71
|
+
end
|
|
72
|
+
if type == 'reject_subscription'
|
|
73
|
+
raise "Subscription Rejected"
|
|
74
|
+
end
|
|
75
|
+
if timeout
|
|
76
|
+
end_time = Time.now
|
|
77
|
+
if (end_time - start_time) > timeout
|
|
78
|
+
raise Timeout::Error, "No Data Timeout"
|
|
74
79
|
end
|
|
75
|
-
next
|
|
76
80
|
end
|
|
81
|
+
if defined? RunningScript and RunningScript.instance
|
|
82
|
+
raise StopScript if RunningScript.instance.stop?
|
|
83
|
+
end
|
|
84
|
+
next
|
|
77
85
|
end
|
|
78
|
-
return json_hash['message']
|
|
79
86
|
end
|
|
80
|
-
return message
|
|
87
|
+
return json_hash['message']
|
|
81
88
|
end
|
|
82
89
|
end
|
|
83
90
|
|