openc3 7.0.1 → 7.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/bin/openc3cli +47 -3
  3. data/data/config/item_modifiers.yaml +1 -1
  4. data/data/config/microservice.yaml +12 -1
  5. data/data/config/parameter_modifiers.yaml +49 -7
  6. data/data/config/target.yaml +11 -0
  7. data/data/config/target_config.yaml +6 -2
  8. data/lib/openc3/api/cmd_api.rb +2 -1
  9. data/lib/openc3/api/metrics_api.rb +11 -1
  10. data/lib/openc3/api/tlm_api.rb +21 -6
  11. data/lib/openc3/core_ext/faraday.rb +1 -1
  12. data/lib/openc3/io/json_api.rb +1 -1
  13. data/lib/openc3/logs/log_writer.rb +3 -1
  14. data/lib/openc3/microservices/decom_common.rb +128 -0
  15. data/lib/openc3/microservices/decom_microservice.rb +26 -95
  16. data/lib/openc3/microservices/interface_decom_common.rb +6 -2
  17. data/lib/openc3/microservices/interface_microservice.rb +10 -8
  18. data/lib/openc3/microservices/log_microservice.rb +1 -1
  19. data/lib/openc3/microservices/microservice.rb +3 -2
  20. data/lib/openc3/microservices/queue_microservice.rb +1 -1
  21. data/lib/openc3/microservices/scope_cleanup_microservice.rb +60 -46
  22. data/lib/openc3/microservices/text_log_microservice.rb +1 -2
  23. data/lib/openc3/models/cvt_model.rb +24 -13
  24. data/lib/openc3/models/db_sharded_model.rb +110 -0
  25. data/lib/openc3/models/interface_model.rb +9 -0
  26. data/lib/openc3/models/interface_status_model.rb +33 -3
  27. data/lib/openc3/models/metric_model.rb +96 -37
  28. data/lib/openc3/models/microservice_model.rb +7 -0
  29. data/lib/openc3/models/microservice_status_model.rb +30 -3
  30. data/lib/openc3/models/reingest_job_model.rb +153 -0
  31. data/lib/openc3/models/scope_model.rb +3 -2
  32. data/lib/openc3/models/script_status_model.rb +4 -20
  33. data/lib/openc3/models/target_model.rb +113 -100
  34. data/lib/openc3/packets/packet_config.rb +4 -1
  35. data/lib/openc3/script/script.rb +2 -2
  36. data/lib/openc3/script/script_runner.rb +4 -4
  37. data/lib/openc3/script/telemetry.rb +3 -3
  38. data/lib/openc3/script/web_socket_api.rb +29 -22
  39. data/lib/openc3/system/system.rb +20 -3
  40. data/lib/openc3/topics/command_decom_topic.rb +4 -2
  41. data/lib/openc3/topics/command_topic.rb +8 -5
  42. data/lib/openc3/topics/decom_interface_topic.rb +15 -10
  43. data/lib/openc3/topics/interface_topic.rb +71 -29
  44. data/lib/openc3/topics/limits_event_topic.rb +62 -41
  45. data/lib/openc3/topics/router_topic.rb +61 -21
  46. data/lib/openc3/topics/system_events_topic.rb +18 -1
  47. data/lib/openc3/topics/telemetry_decom_topic.rb +2 -1
  48. data/lib/openc3/topics/telemetry_topic.rb +4 -2
  49. data/lib/openc3/topics/topic.rb +77 -5
  50. data/lib/openc3/utilities/aws_bucket.rb +2 -0
  51. data/lib/openc3/utilities/cli_generator.rb +3 -2
  52. data/lib/openc3/utilities/metric.rb +15 -1
  53. data/lib/openc3/utilities/questdb_client.rb +173 -37
  54. data/lib/openc3/utilities/reingest_job.rb +377 -0
  55. data/lib/openc3/utilities/ruby_lex_utils.rb +2 -0
  56. data/lib/openc3/utilities/store_autoload.rb +78 -52
  57. data/lib/openc3/utilities/store_queued.rb +20 -12
  58. data/lib/openc3/version.rb +6 -6
  59. data/templates/plugin/plugin.gemspec +13 -1
  60. data/templates/tool_angular/package.json +2 -2
  61. data/templates/tool_react/package.json +1 -1
  62. data/templates/tool_svelte/package.json +1 -1
  63. data/templates/tool_vue/package.json +3 -3
  64. data/templates/tool_vue/src/router.js +2 -2
  65. data/templates/widget/package.json +2 -2
  66. metadata +7 -3
@@ -81,6 +81,7 @@ module OpenC3
81
81
  attr_accessor :children
82
82
  attr_accessor :disable_erb
83
83
  attr_accessor :shard
84
+ attr_accessor :db_shard
84
85
 
85
86
  # NOTE: The following three class methods are used by the ModelController
86
87
  # and are reimplemented to enable various Model class methods to work
@@ -202,6 +203,11 @@ module OpenC3
202
203
  return result
203
204
  end
204
205
 
206
+ # Get a Store instance routed to the correct db_shard for a target
207
+ def self.store_for_target(target_name, scope:)
208
+ Store.instance(db_shard: Store.db_shard_for_target(target_name, scope: scope))
209
+ end
210
+
205
211
  # @return [Hash] Packet hash or raises an exception
206
212
  def self.packet(target_name, packet_name, type: :TLM, scope:)
207
213
  raise "Unknown type #{type} for #{target_name} #{packet_name}" unless VALID_TYPES.include?(type)
@@ -216,7 +222,7 @@ module OpenC3
216
222
  end
217
223
 
218
224
  # Assume it exists and just try to get it to avoid an extra call to Store.exist?
219
- json = Store.hget("#{scope}__openc3#{type.to_s.downcase}__#{target_name}", packet_name)
225
+ json = store_for_target(target_name, scope: scope).hget("#{scope}__openc3#{type.to_s.downcase}__#{target_name}", packet_name)
220
226
  raise "Packet '#{target_name} #{packet_name}' does not exist" if json.nil?
221
227
 
222
228
  packet = JSON.parse(json, allow_nan: true, create_additions: true)
@@ -241,7 +247,7 @@ module OpenC3
241
247
  raise "Target '#{target_name}' does not exist for scope: #{scope}" unless get(name: target_name, scope: scope)
242
248
 
243
249
  result = []
244
- packets = Store.hgetall("#{scope}__openc3#{type.to_s.downcase}__#{target_name}")
250
+ packets = store_for_target(target_name, scope: scope).hgetall("#{scope}__openc3#{type.to_s.downcase}__#{target_name}")
245
251
  packets.sort.each do |_packet_name, packet_json|
246
252
  result << JSON.parse(packet_json, allow_nan: true, create_additions: true)
247
253
  end
@@ -263,7 +269,7 @@ module OpenC3
263
269
  end
264
270
 
265
271
  begin
266
- Store.hset("#{scope}__openc3#{type.to_s.downcase}__#{target_name}", packet_name, JSON.generate(packet.as_json, allow_nan: true))
272
+ store_for_target(target_name, scope: scope).hset("#{scope}__openc3#{type.to_s.downcase}__#{target_name}", packet_name, JSON.generate(packet.as_json, allow_nan: true))
267
273
  rescue JSON::GeneratorError => e
268
274
  Logger.error("Invalid text present in #{target_name} #{packet_name} #{type.to_s.downcase} packet")
269
275
  raise e
@@ -296,7 +302,7 @@ module OpenC3
296
302
 
297
303
  # @return [Array<String>] All the item names for every packet in a target
298
304
  def self.all_item_names(target_name, type: :TLM, scope:)
299
- items = Store.zrange("#{scope}__openc3tlm__#{target_name}__allitems", 0, -1)
305
+ items = store_for_target(target_name, scope: scope).zrange("#{scope}__openc3tlm__#{target_name}__allitems", 0, -1)
300
306
  items = rebuild_target_allitems_list(target_name, type: type, scope: scope) if items.empty?
301
307
  items
302
308
  end
@@ -308,12 +314,12 @@ module OpenC3
308
314
  TargetModel.add_to_target_allitems_list(target_name, item['name'], scope: scope)
309
315
  end
310
316
  end
311
- Store.zrange("#{scope}__openc3tlm__#{target_name}__allitems", 0, -1) # return the new sorted set to let redis do the sorting
317
+ store_for_target(target_name, scope: scope).zrange("#{scope}__openc3tlm__#{target_name}__allitems", 0, -1) # return the new sorted set to let redis do the sorting
312
318
  end
313
319
 
314
320
  def self.add_to_target_allitems_list(target_name, item_name, scope:)
315
321
  score = 0 # https://redis.io/docs/latest/develop/data-types/sorted-sets/#lexicographical-scores
316
- Store.zadd("#{scope}__openc3tlm__#{target_name}__allitems", score, item_name)
322
+ store_for_target(target_name, scope: scope).zadd("#{scope}__openc3tlm__#{target_name}__allitems", score, item_name)
317
323
  end
318
324
 
319
325
  # @return [Hash{String => Array<Array<String, String, String>>}]
@@ -331,12 +337,13 @@ module OpenC3
331
337
  return item_map if item_map and (Time.now - cache_time) < ITEM_MAP_CACHE_TIMEOUT
332
338
  item_map_key = "#{scope}__#{target_name}__item_to_packet_map"
333
339
  target_name = target_name.upcase
334
- json_data = Store.get(item_map_key)
340
+ store = store_for_target(target_name, scope: scope)
341
+ json_data = store.get(item_map_key)
335
342
  if json_data
336
343
  item_map = JSON.parse(json_data, allow_nan: true, create_additions: true)
337
344
  else
338
345
  item_map = build_item_to_packet_map(target_name, scope: scope)
339
- Store.set(item_map_key, JSON.generate(item_map, allow_nan: true))
346
+ store.set(item_map_key, JSON.generate(item_map, allow_nan: true))
340
347
  end
341
348
  @@item_map_cache[target_name] = [Time.now, item_map]
342
349
  return item_map
@@ -396,6 +403,7 @@ module OpenC3
396
403
  target_microservices: {},
397
404
  disable_erb: nil,
398
405
  shard: 0,
406
+ db_shard: 0,
399
407
  scope:
400
408
  )
401
409
  super("#{scope}__#{PRIMARY_KEY}", name: name, plugin: plugin, updated_at: updated_at, scope: scope)
@@ -421,6 +429,7 @@ module OpenC3
421
429
  @target_microservices = target_microservices
422
430
  @disable_erb = disable_erb
423
431
  @shard = shard.to_i # to_i to handle nil
432
+ @db_shard = db_shard.to_i # to_i to handle nil
424
433
  @bucket = Bucket.getClient()
425
434
  @children = []
426
435
  end
@@ -452,6 +461,7 @@ module OpenC3
452
461
  'target_microservices' => @target_microservices.as_json(),
453
462
  'disable_erb' => @disable_erb,
454
463
  'shard' => @shard,
464
+ 'db_shard' => @db_shard,
455
465
  }
456
466
  end
457
467
 
@@ -543,6 +553,10 @@ module OpenC3
543
553
  parser.verify_num_parameters(1, 1, "#{keyword} <Shard Number Starting from 0>")
544
554
  @shard = Integer(parameters[0])
545
555
 
556
+ when 'DB_SHARD'
557
+ parser.verify_num_parameters(1, 1, "#{keyword} <Shard Number Starting from 0>")
558
+ @db_shard = Integer(parameters[0])
559
+
546
560
  else
547
561
  raise ConfigParser::Error.new(parser, "Unknown keyword and parameters for Target: #{keyword} #{parameters.join(" ")}")
548
562
  end
@@ -626,19 +640,20 @@ module OpenC3
626
640
  Store.hdel("#{@scope}__limits_groups", group)
627
641
  end
628
642
  self.class.packets(@name, type: :CMD, scope: @scope).each do |packet|
629
- Topic.del("#{@scope}__COMMAND__{#{@name}}__#{packet['packet_name']}")
630
- Topic.del("#{@scope}__DECOMCMD__{#{@name}}__#{packet['packet_name']}")
643
+ Topic.del("#{@scope}__COMMAND__{#{@name}}__#{packet['packet_name']}", db_shard: @db_shard)
644
+ Topic.del("#{@scope}__DECOMCMD__{#{@name}}__#{packet['packet_name']}", db_shard: @db_shard)
631
645
  end
632
646
  self.class.packets(@name, scope: @scope).each do |packet|
633
- Topic.del("#{@scope}__TELEMETRY__{#{@name}}__#{packet['packet_name']}")
634
- Topic.del("#{@scope}__DECOM__{#{@name}}__#{packet['packet_name']}")
647
+ Topic.del("#{@scope}__TELEMETRY__{#{@name}}__#{packet['packet_name']}", db_shard: @db_shard)
648
+ Topic.del("#{@scope}__DECOM__{#{@name}}__#{packet['packet_name']}", db_shard: @db_shard)
635
649
  CvtModel.del(target_name: @name, packet_name: packet['packet_name'], scope: @scope)
636
650
  end
637
651
  LimitsEventTopic.delete(@name, scope: @scope)
638
- Store.del("#{@scope}__openc3tlm__#{@name}")
639
- Store.del("#{@scope}__openc3cmd__#{@name}")
640
- Store.del("#{@scope}__TELEMETRYCNTS__{#{@name}}")
641
- Store.del("#{@scope}__COMMANDCNTS__{#{@name}}")
652
+ db_shard_store = Store.instance(db_shard: @db_shard)
653
+ db_shard_store.del("#{@scope}__openc3tlm__#{@name}")
654
+ db_shard_store.del("#{@scope}__openc3cmd__#{@name}")
655
+ db_shard_store.del("#{@scope}__TELEMETRYCNTS__{#{@name}}")
656
+ db_shard_store.del("#{@scope}__COMMANDCNTS__{#{@name}}")
642
657
 
643
658
  # Note: these match the names of the services in deploy_microservices
644
659
  %w(MULTI DECOM COMMANDLOG PACKETLOG CLEANUP).each do |type|
@@ -657,7 +672,7 @@ module OpenC3
657
672
  end
658
673
  # Delete item_map
659
674
  item_map_key = "#{@scope}__#{@name}__item_to_packet_map"
660
- Store.del(item_map_key)
675
+ db_shard_store.del(item_map_key)
661
676
  @@item_map_cache[@name] = nil
662
677
 
663
678
  topic = { kind: 'deleted', type: 'target', name: @name }
@@ -752,16 +767,17 @@ module OpenC3
752
767
  end
753
768
 
754
769
  def update_store_telemetry(packet_hash, clear_old: true)
770
+ db_shard_store = Store.instance(db_shard: @db_shard)
755
771
  packet_hash.each do |target_name, packets|
756
772
  if clear_old
757
- Store.del("#{@scope}__openc3tlm__#{target_name}")
758
- Store.del("#{@scope}__openc3tlm__#{target_name}__allitems")
759
- Store.del("#{@scope}__TELEMETRYCNTS__{#{target_name}}")
773
+ db_shard_store.del("#{@scope}__openc3tlm__#{target_name}")
774
+ db_shard_store.del("#{@scope}__openc3tlm__#{target_name}__allitems")
775
+ db_shard_store.del("#{@scope}__TELEMETRYCNTS__{#{target_name}}")
760
776
  end
761
777
  packets.each do |packet_name, packet|
762
778
  Logger.debug "Configuring tlm packet: #{target_name} #{packet_name}"
763
779
  begin
764
- Store.hset("#{@scope}__openc3tlm__#{target_name}", packet_name, JSON.generate(packet.as_json, allow_nan: true))
780
+ db_shard_store.hset("#{@scope}__openc3tlm__#{target_name}", packet_name, JSON.generate(packet.as_json, allow_nan: true))
765
781
  rescue JSON::GeneratorError => e
766
782
  Logger.error("Invalid text present in #{target_name} #{packet_name} tlm packet")
767
783
  raise e
@@ -777,15 +793,16 @@ module OpenC3
777
793
  end
778
794
 
779
795
  def update_store_commands(packet_hash, clear_old: true)
796
+ db_shard_store = Store.instance(db_shard: @db_shard)
780
797
  packet_hash.each do |target_name, packets|
781
798
  if clear_old
782
- Store.del("#{@scope}__openc3cmd__#{target_name}")
783
- Store.del("#{@scope}__COMMANDCNTS__{#{target_name}}")
799
+ db_shard_store.del("#{@scope}__openc3cmd__#{target_name}")
800
+ db_shard_store.del("#{@scope}__COMMANDCNTS__{#{target_name}}")
784
801
  end
785
802
  packets.each do |packet_name, packet|
786
803
  Logger.debug "Configuring cmd packet: #{target_name} #{packet_name}"
787
804
  begin
788
- Store.hset("#{@scope}__openc3cmd__#{target_name}", packet_name, JSON.generate(packet.as_json, allow_nan: true))
805
+ db_shard_store.hset("#{@scope}__openc3cmd__#{target_name}", packet_name, JSON.generate(packet.as_json, allow_nan: true))
789
806
  rescue JSON::GeneratorError => e
790
807
  Logger.error("Invalid text present in #{target_name} #{packet_name} cmd packet")
791
808
  raise e
@@ -818,7 +835,7 @@ module OpenC3
818
835
  # Create item_map
819
836
  item_map_key = "#{@scope}__#{@name}__item_to_packet_map"
820
837
  item_map = self.class.build_item_to_packet_map(@name, scope: @scope)
821
- Store.set(item_map_key, JSON.generate(item_map, allow_nan: true))
838
+ Store.instance(db_shard: @db_shard).set(item_map_key, JSON.generate(item_map, allow_nan: true))
822
839
  @@item_map_cache[@name] = [Time.now, item_map]
823
840
  end
824
841
 
@@ -888,15 +905,15 @@ module OpenC3
888
905
  end
889
906
  end
890
907
  if cmd_or_tlm == :TELEMETRY
891
- Topic.write_topic("MICROSERVICE__#{@scope}__PACKETLOG__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json})
908
+ Topic.write_topic("MICROSERVICE__#{@scope}__PACKETLOG__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json}, db_shard: @db_shard)
892
909
  add_topics_to_microservice("#{@scope}__PACKETLOG__#{@name}", raw_topics)
893
- Topic.write_topic("MICROSERVICE__#{@scope}__DECOM__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json})
910
+ Topic.write_topic("MICROSERVICE__#{@scope}__DECOM__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json}, db_shard: @db_shard)
894
911
  add_topics_to_microservice("#{@scope}__DECOM__#{@name}", raw_topics)
895
912
  else
896
- Topic.write_topic("MICROSERVICE__#{@scope}__COMMANDLOG__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json})
913
+ Topic.write_topic("MICROSERVICE__#{@scope}__COMMANDLOG__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json}, db_shard: @db_shard)
897
914
  add_topics_to_microservice("#{@scope}__COMMANDLOG__#{@name}", raw_topics)
898
915
  end
899
- Topic.write_topic("MICROSERVICE__#{@scope}__TSDB__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => decom_topics.as_json.to_json})
916
+ Topic.write_topic("MICROSERVICE__#{@scope}__TSDB__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => decom_topics.as_json.to_json}, db_shard: @db_shard)
900
917
  add_topics_to_microservice("#{@scope}__TSDB__#{@name}", decom_topics)
901
918
  end
902
919
 
@@ -927,6 +944,7 @@ module OpenC3
927
944
  parent: parent,
928
945
  needs_dependencies: @needs_dependencies,
929
946
  shard: @shard,
947
+ db_shard: @db_shard,
930
948
  scope: @scope
931
949
  )
932
950
  microservice.create
@@ -954,6 +972,7 @@ module OpenC3
954
972
  parent: parent,
955
973
  needs_dependencies: @needs_dependencies,
956
974
  shard: @shard,
975
+ db_shard: @db_shard,
957
976
  scope: @scope
958
977
  )
959
978
  microservice.create
@@ -986,6 +1005,7 @@ module OpenC3
986
1005
  parent: parent,
987
1006
  needs_dependencies: @needs_dependencies,
988
1007
  shard: @shard,
1008
+ db_shard: @db_shard,
989
1009
  scope: @scope
990
1010
  )
991
1011
  microservice.create
@@ -1010,6 +1030,7 @@ module OpenC3
1010
1030
  parent: nil,
1011
1031
  needs_dependencies: @needs_dependencies,
1012
1032
  shard: @shard,
1033
+ db_shard: @db_shard,
1013
1034
  scope: @scope
1014
1035
  )
1015
1036
  microservice.create
@@ -1026,6 +1047,7 @@ module OpenC3
1026
1047
  plugin: @plugin,
1027
1048
  parent: parent,
1028
1049
  shard: @shard,
1050
+ db_shard: @db_shard,
1029
1051
  scope: @scope
1030
1052
  )
1031
1053
  microservice.create
@@ -1044,6 +1066,7 @@ module OpenC3
1044
1066
  plugin: @plugin,
1045
1067
  needs_dependencies: @needs_dependencies,
1046
1068
  shard: @shard,
1069
+ db_shard: @db_shard,
1047
1070
  scope: @scope
1048
1071
  )
1049
1072
  microservice.create
@@ -1139,11 +1162,19 @@ module OpenC3
1139
1162
  deploy_target_microservices('PACKETLOG', packet_topic_list, "#{@scope}__TELEMETRY__{#{@name}}") do |topics, instance, parent|
1140
1163
  deploy_packetlog_microservice(gem_path, variables, topics, instance, parent)
1141
1164
  end
1165
+ end
1142
1166
 
1143
- # Decommutation Microservice
1167
+ # Decommutation Microservice - also handles build_cmd / inject_tlm /
1168
+ # get_tlm_buffer via the DECOMINTERFACE topic, so deploy whenever the
1169
+ # target has commands or telemetry, not just telemetry.
1170
+ if packet_topic_list.any?
1144
1171
  deploy_target_microservices('DECOM', packet_topic_list, "#{@scope}__TELEMETRY__{#{@name}}") do |topics, instance, parent|
1145
1172
  deploy_decom_microservice(system.targets[@name], gem_path, variables, topics, instance, parent)
1146
1173
  end
1174
+ elsif command_topic_list.any?
1175
+ # Cmd-only target: deploy DECOM with no tlm topics so build_cmd works.
1176
+ # DecomMicroservice subscribes to DECOMINTERFACE in its initializer.
1177
+ deploy_decom_microservice(system.targets[@name], gem_path, variables, [], nil, @parent)
1147
1178
  end
1148
1179
 
1149
1180
  # TSDB Microservice - subscribes to both decommutated telemetry and commands
@@ -1168,7 +1199,7 @@ module OpenC3
1168
1199
  end
1169
1200
 
1170
1201
  def self.increment_telemetry_count(target_name, packet_name, count, scope:)
1171
- result = Store.hincrby("#{scope}__TELEMETRYCNTS__{#{target_name}}", packet_name, count)
1202
+ result = store_for_target(target_name, scope: scope).hincrby("#{scope}__TELEMETRYCNTS__{#{target_name}}", packet_name, count)
1172
1203
  if String === result
1173
1204
  return result.to_i
1174
1205
  else
@@ -1178,7 +1209,7 @@ module OpenC3
1178
1209
 
1179
1210
  def self.get_all_telemetry_counts(target_name, scope:)
1180
1211
  result = {}
1181
- get_all = Store.hgetall("#{scope}__TELEMETRYCNTS__{#{target_name}}")
1212
+ get_all = store_for_target(target_name, scope: scope).hgetall("#{scope}__TELEMETRYCNTS__{#{target_name}}")
1182
1213
  if Hash === get_all
1183
1214
  get_all.each do |key, value|
1184
1215
  result[key] = value.to_i
@@ -1189,7 +1220,7 @@ module OpenC3
1189
1220
  end
1190
1221
 
1191
1222
  def self.get_telemetry_count(target_name, packet_name, scope:)
1192
- value = Store.hget("#{scope}__TELEMETRYCNTS__{#{target_name}}", packet_name)
1223
+ value = store_for_target(target_name, scope: scope).hget("#{scope}__TELEMETRYCNTS__{#{target_name}}", packet_name)
1193
1224
  if String === value
1194
1225
  return value.to_i
1195
1226
  elsif value.nil?
@@ -1200,30 +1231,26 @@ module OpenC3
1200
1231
  end
1201
1232
 
1202
1233
  def self.get_telemetry_counts(target_packets, scope:)
1203
- result = []
1204
- if $openc3_redis_cluster
1205
- # No pipelining for cluster mode
1206
- # because it requires using the same shard for all keys
1207
- target_packets.each do |target_name, packet_name|
1208
- target_name = target_name.upcase
1209
- packet_name = packet_name.upcase
1210
- result << Store.hget("#{scope}__TELEMETRYCNTS__{#{target_name}}", packet_name)
1211
- end
1212
- else
1213
- result = Store.redis_pool.pipelined do
1214
- target_packets.each do |target_name, packet_name|
1215
- target_name = target_name.upcase
1216
- packet_name = packet_name.upcase
1217
- Store.hget("#{scope}__TELEMETRYCNTS__{#{target_name}}", packet_name)
1234
+ # Group by db_shard, preserving original index
1235
+ db_shard_groups = {} # db_shard => [{index:, target_name:, packet_name:}]
1236
+ target_packets.each_with_index do |(target_name, packet_name), idx|
1237
+ target_name = target_name.upcase
1238
+ packet_name = packet_name.upcase
1239
+ db_shard = Store.db_shard_for_target(target_name, scope: scope)
1240
+ db_shard_groups[db_shard] ||= []
1241
+ db_shard_groups[db_shard] << { index: idx, target_name: target_name, packet_name: packet_name }
1242
+ end
1243
+
1244
+ counts = Array.new(target_packets.length, 0)
1245
+ db_shard_groups.each do |db_shard, entries|
1246
+ store = Store.instance(db_shard: db_shard)
1247
+ result = store.redis_pool.pipelined do
1248
+ entries.each do |entry|
1249
+ store.hget("#{scope}__TELEMETRYCNTS__{#{entry[:target_name]}}", entry[:packet_name])
1218
1250
  end
1219
1251
  end
1220
- end
1221
- counts = []
1222
- result.each do |count|
1223
- if count
1224
- counts << count.to_i
1225
- else
1226
- counts << 0
1252
+ entries.each_with_index do |entry, i|
1253
+ counts[entry[:index]] = result[i] ? result[i].to_i : 0
1227
1254
  end
1228
1255
  end
1229
1256
  return counts
@@ -1267,7 +1294,7 @@ module OpenC3
1267
1294
  end
1268
1295
 
1269
1296
  def self.sync_tlm_packet_counts(packet, tlm_target_names, scope:)
1270
- if @@sync_packet_count_delay_seconds <= 0 or $openc3_redis_cluster
1297
+ if @@sync_packet_count_delay_seconds <= 0
1271
1298
  # Perfect but slow method
1272
1299
  packet.received_count = increment_telemetry_count(packet.target_name, packet.packet_name, 1, scope: scope)
1273
1300
  else
@@ -1288,26 +1315,17 @@ module OpenC3
1288
1315
  if @@sync_packet_count_time.nil? or (Time.now - @@sync_packet_count_time) > @@sync_packet_count_delay_seconds
1289
1316
  @@sync_packet_count_time = Time.now
1290
1317
 
1291
- inc_count = 0
1292
- # Use pipeline to make this one transaction
1293
- result = Store.redis_pool.pipelined do
1294
- # Increment global counters for packets received
1295
- @@sync_packet_count_data.each do |target_name, packet_data|
1296
- packet_data.each do |packet_name, count|
1297
- increment_telemetry_count(target_name, packet_name, count, scope: scope)
1298
- inc_count += 1
1299
- end
1300
- end
1301
- @@sync_packet_count_data = {}
1302
-
1303
- # Get all the packet counts with the global counters
1304
- tlm_target_names.each do |target_name|
1305
- get_all_telemetry_counts(target_name, scope: scope)
1318
+ # Increment global counters for packets received
1319
+ @@sync_packet_count_data.each do |target_name, packet_data|
1320
+ packet_data.each do |packet_name, count|
1321
+ increment_telemetry_count(target_name, packet_name, count, scope: scope)
1306
1322
  end
1307
- get_all_telemetry_counts('UNKNOWN', scope: scope)
1308
1323
  end
1324
+ @@sync_packet_count_data = {}
1325
+
1326
+ # Get all the packet counts with the global counters
1309
1327
  tlm_target_names.each do |target_name|
1310
- result[inc_count].each do |packet_name, count|
1328
+ get_all_telemetry_counts(target_name, scope: scope).each do |packet_name, count|
1311
1329
  begin
1312
1330
  update_packet = System.telemetry.packet(target_name, packet_name)
1313
1331
  update_packet.received_count = count.to_i
@@ -1319,9 +1337,8 @@ module OpenC3
1319
1337
  end
1320
1338
  end
1321
1339
  end
1322
- inc_count += 1
1323
1340
  end
1324
- result[inc_count].each do |packet_name, count|
1341
+ get_all_telemetry_counts('UNKNOWN', scope: scope).each do |packet_name, count|
1325
1342
  begin
1326
1343
  update_packet = System.telemetry.packet('UNKNOWN', packet_name)
1327
1344
  update_packet.received_count = count.to_i
@@ -1338,7 +1355,7 @@ module OpenC3
1338
1355
  end
1339
1356
 
1340
1357
  def self.increment_command_count(target_name, packet_name, count, scope:)
1341
- result = Store.hincrby("#{scope}__COMMANDCNTS__{#{target_name}}", packet_name, count)
1358
+ result = store_for_target(target_name, scope: scope).hincrby("#{scope}__COMMANDCNTS__{#{target_name}}", packet_name, count)
1342
1359
  if String === result
1343
1360
  return result.to_i
1344
1361
  else
@@ -1348,7 +1365,7 @@ module OpenC3
1348
1365
 
1349
1366
  def self.get_all_command_counts(target_name, scope:)
1350
1367
  result = {}
1351
- get_all = Store.hgetall("#{scope}__COMMANDCNTS__{#{target_name}}")
1368
+ get_all = store_for_target(target_name, scope: scope).hgetall("#{scope}__COMMANDCNTS__{#{target_name}}")
1352
1369
  if Hash === get_all
1353
1370
  get_all.each do |key, value|
1354
1371
  result[key] = value.to_i
@@ -1359,7 +1376,7 @@ module OpenC3
1359
1376
  end
1360
1377
 
1361
1378
  def self.get_command_count(target_name, packet_name, scope:)
1362
- value = Store.hget("#{scope}__COMMANDCNTS__{#{target_name}}", packet_name)
1379
+ value = store_for_target(target_name, scope: scope).hget("#{scope}__COMMANDCNTS__{#{target_name}}", packet_name)
1363
1380
  if String === value
1364
1381
  return value.to_i
1365
1382
  elsif value.nil?
@@ -1370,30 +1387,26 @@ module OpenC3
1370
1387
  end
1371
1388
 
1372
1389
  def self.get_command_counts(target_packets, scope:)
1373
- result = []
1374
- if $openc3_redis_cluster
1375
- # No pipelining for cluster mode
1376
- # because it requires using the same shard for all keys
1377
- target_packets.each do |target_name, packet_name|
1378
- target_name = target_name.upcase
1379
- packet_name = packet_name.upcase
1380
- result << Store.hget("#{scope}__COMMANDCNTS__{#{target_name}}", packet_name)
1381
- end
1382
- else
1383
- result = Store.redis_pool.pipelined do
1384
- target_packets.each do |target_name, packet_name|
1385
- target_name = target_name.upcase
1386
- packet_name = packet_name.upcase
1387
- Store.hget("#{scope}__COMMANDCNTS__{#{target_name}}", packet_name)
1390
+ # Group by db_shard, preserving original index
1391
+ db_shard_groups = {} # db_shard => [{index:, target_name:, packet_name:}]
1392
+ target_packets.each_with_index do |(target_name, packet_name), idx|
1393
+ target_name = target_name.upcase
1394
+ packet_name = packet_name.upcase
1395
+ db_shard = Store.db_shard_for_target(target_name, scope: scope)
1396
+ db_shard_groups[db_shard] ||= []
1397
+ db_shard_groups[db_shard] << { index: idx, target_name: target_name, packet_name: packet_name }
1398
+ end
1399
+
1400
+ counts = Array.new(target_packets.length, 0)
1401
+ db_shard_groups.each do |db_shard, entries|
1402
+ store = Store.instance(db_shard: db_shard)
1403
+ result = store.redis_pool.pipelined do
1404
+ entries.each do |entry|
1405
+ store.hget("#{scope}__COMMANDCNTS__{#{entry[:target_name]}}", entry[:packet_name])
1388
1406
  end
1389
1407
  end
1390
- end
1391
- counts = []
1392
- result.each do |count|
1393
- if count
1394
- counts << count.to_i
1395
- else
1396
- counts << 0
1408
+ entries.each_with_index do |entry, i|
1409
+ counts[entry[:index]] = result[i] ? result[i].to_i : 0
1397
1410
  end
1398
1411
  end
1399
1412
  return counts
@@ -437,9 +437,12 @@ module OpenC3
437
437
  if packet.id_items.length > 0
438
438
  key = []
439
439
  id_signature = ""
440
+ # Accessor class is part of the signature so packets in the same target
441
+ # with different accessors trigger unique_id_mode (different accessors
442
+ # decode the buffer differently, so the hash-lookup path is unsafe).
440
443
  packet.id_items.each do |item|
441
444
  key << item.id_value
442
- id_signature << "__#{item.key}__#{item.bit_offset}__#{item.bit_size}__#{item.data_type}"
445
+ id_signature << "__#{item.key}__#{item.bit_offset}__#{item.bit_size}__#{item.data_type}__#{packet.accessor.class.to_s}"
443
446
  end
444
447
  target_id_value_hash[key] = packet
445
448
  target_id_signature = id_signature_hash[packet.target_name]
@@ -44,11 +44,11 @@ require 'openc3/utilities/authentication'
44
44
  $api_server = nil
45
45
  $script_runner_api_server = nil
46
46
  $disconnect = false
47
- $openc3_scope = ENV['OPENC3_SCOPE'] || 'DEFAULT'
47
+ $openc3_scope = ENV.fetch('OPENC3_SCOPE', 'DEFAULT')
48
48
  $openc3_in_cluster = false
49
49
 
50
50
  saved_verbose = $VERBOSE
51
- $VERBOSE = false
51
+ $VERBOSE = nil
52
52
 
53
53
  module OpenC3
54
54
  module Script
@@ -37,9 +37,9 @@ module OpenC3
37
37
 
38
38
  def script_syntax_check(script, scope: $openc3_scope)
39
39
  endpoint = "/script-api/scripts/temp.rb/syntax"
40
- # Explicitly set the headers to plain/text so the request.body is set correctly
40
+ # Explicitly set the headers to text/plain so the request.body is set correctly
41
41
  headers = {
42
- 'Content-Type': 'plain/text',
42
+ 'Content-Type': 'text/plain',
43
43
  }
44
44
  response = $script_runner_api_server.request('post', endpoint, headers: headers, data: script, scope: scope)
45
45
  if response.nil? || response.status != 200
@@ -129,9 +129,9 @@ module OpenC3
129
129
 
130
130
  def script_instrumented(script, scope: $openc3_scope)
131
131
  endpoint = "/script-api/scripts/temp.rb/instrumented"
132
- # Explicitly set the headers to plain/text so the request.body is set correctly
132
+ # Explicitly set the headers to text/plain so the request.body is set correctly
133
133
  headers = {
134
- 'Content-Type': 'plain/text',
134
+ 'Content-Type': 'text/plain',
135
135
  }
136
136
  response = $script_runner_api_server.request('post', endpoint, headers: headers, data: script, scope: scope)
137
137
  if response.nil? || response.status != 200
@@ -40,9 +40,9 @@ module OpenC3
40
40
  # inject_tlm, set_tlm, override_tlm, and normalize_tlm are implemented here simply to add a puts
41
41
  # these methods modify the telemetry so the user should be notified in the Script Runner log messages
42
42
 
43
- def inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, scope: $openc3_scope, token: $openc3_token)
44
- puts "inject_tlm(\"#{target_name}\", \"#{packet_name}\", #{item_hash}, type: #{type})"
45
- $api_server.method_missing(:inject_tlm, target_name, packet_name, item_hash, type: type, scope: scope, token: token)
43
+ def inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, stored: false, scope: $openc3_scope, token: $openc3_token)
44
+ puts "inject_tlm(\"#{target_name}\", \"#{packet_name}\", #{item_hash}, type: #{type}, stored: #{stored})"
45
+ $api_server.method_missing(:inject_tlm, target_name, packet_name, item_hash, type: type, stored: stored, scope: scope, token: token)
46
46
  end
47
47
 
48
48
  def set_tlm(*args, type: :ALL, scope: $openc3_scope, token: $openc3_token)
@@ -50,34 +50,41 @@ module OpenC3
50
50
  start_time = Time.now
51
51
  while true
52
52
  message = read_message()
53
- if message
53
+ # Empty string is a normal end-of-stream signal when ActionCable / anycable-go
54
+ # closes the WS. Treat it the same as nil so consumer `while (resp = api.read)`
55
+ # loops exit cleanly instead of hitting JSON::ParserError on JSON.parse("").
56
+ return nil if message.nil? || message.empty?
57
+
58
+ begin
54
59
  json_hash = JSON.parse(message, allow_nan: true, create_additions: true)
55
- if ignore_protocol_messages
56
- type = json_hash['type']
57
- if type # ping, welcome, confirm_subscription, reject_subscription, disconnect
58
- if type == 'disconnect'
59
- if json_hash['reason'] == 'unauthorized'
60
- raise "Unauthorized"
61
- end
62
- end
63
- if type == 'reject_subscription'
64
- raise "Subscription Rejected"
65
- end
66
- if timeout
67
- end_time = Time.now
68
- if (end_time - start_time) > timeout
69
- raise Timeout::Error, "No Data Timeout"
70
- end
60
+ rescue JSON::ParserError
61
+ # Defense-in-depth: treat malformed frames as end-of-stream rather than crashing.
62
+ return nil
63
+ end
64
+ if ignore_protocol_messages
65
+ type = json_hash['type']
66
+ if type # ping, welcome, confirm_subscription, reject_subscription, disconnect
67
+ if type == 'disconnect'
68
+ if json_hash['reason'] == 'unauthorized'
69
+ raise "Unauthorized"
71
70
  end
72
- if defined? RunningScript and RunningScript.instance
73
- raise StopScript if RunningScript.instance.stop?
71
+ end
72
+ if type == 'reject_subscription'
73
+ raise "Subscription Rejected"
74
+ end
75
+ if timeout
76
+ end_time = Time.now
77
+ if (end_time - start_time) > timeout
78
+ raise Timeout::Error, "No Data Timeout"
74
79
  end
75
- next
76
80
  end
81
+ if defined? RunningScript and RunningScript.instance
82
+ raise StopScript if RunningScript.instance.stop?
83
+ end
84
+ next
77
85
  end
78
- return json_hash['message']
79
86
  end
80
- return message
87
+ return json_hash['message']
81
88
  end
82
89
  end
83
90