openc3 7.0.1 → 7.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/bin/openc3cli +47 -3
  3. data/data/config/item_modifiers.yaml +1 -1
  4. data/data/config/microservice.yaml +12 -1
  5. data/data/config/parameter_modifiers.yaml +49 -7
  6. data/data/config/target.yaml +11 -0
  7. data/data/config/target_config.yaml +6 -2
  8. data/lib/openc3/api/cmd_api.rb +2 -1
  9. data/lib/openc3/api/metrics_api.rb +11 -1
  10. data/lib/openc3/api/tlm_api.rb +21 -6
  11. data/lib/openc3/core_ext/faraday.rb +1 -1
  12. data/lib/openc3/io/json_api.rb +1 -1
  13. data/lib/openc3/logs/log_writer.rb +3 -1
  14. data/lib/openc3/microservices/decom_common.rb +128 -0
  15. data/lib/openc3/microservices/decom_microservice.rb +26 -95
  16. data/lib/openc3/microservices/interface_decom_common.rb +6 -2
  17. data/lib/openc3/microservices/interface_microservice.rb +10 -8
  18. data/lib/openc3/microservices/log_microservice.rb +1 -1
  19. data/lib/openc3/microservices/microservice.rb +3 -2
  20. data/lib/openc3/microservices/queue_microservice.rb +1 -1
  21. data/lib/openc3/microservices/scope_cleanup_microservice.rb +60 -46
  22. data/lib/openc3/microservices/text_log_microservice.rb +1 -2
  23. data/lib/openc3/models/cvt_model.rb +24 -13
  24. data/lib/openc3/models/db_sharded_model.rb +110 -0
  25. data/lib/openc3/models/interface_model.rb +9 -0
  26. data/lib/openc3/models/interface_status_model.rb +33 -3
  27. data/lib/openc3/models/metric_model.rb +96 -37
  28. data/lib/openc3/models/microservice_model.rb +7 -0
  29. data/lib/openc3/models/microservice_status_model.rb +30 -3
  30. data/lib/openc3/models/reingest_job_model.rb +153 -0
  31. data/lib/openc3/models/scope_model.rb +3 -2
  32. data/lib/openc3/models/script_status_model.rb +4 -20
  33. data/lib/openc3/models/target_model.rb +113 -100
  34. data/lib/openc3/packets/packet_config.rb +4 -1
  35. data/lib/openc3/script/script.rb +2 -2
  36. data/lib/openc3/script/script_runner.rb +4 -4
  37. data/lib/openc3/script/telemetry.rb +3 -3
  38. data/lib/openc3/script/web_socket_api.rb +29 -22
  39. data/lib/openc3/system/system.rb +20 -3
  40. data/lib/openc3/topics/command_decom_topic.rb +4 -2
  41. data/lib/openc3/topics/command_topic.rb +8 -5
  42. data/lib/openc3/topics/decom_interface_topic.rb +15 -10
  43. data/lib/openc3/topics/interface_topic.rb +71 -29
  44. data/lib/openc3/topics/limits_event_topic.rb +62 -41
  45. data/lib/openc3/topics/router_topic.rb +61 -21
  46. data/lib/openc3/topics/system_events_topic.rb +18 -1
  47. data/lib/openc3/topics/telemetry_decom_topic.rb +2 -1
  48. data/lib/openc3/topics/telemetry_topic.rb +4 -2
  49. data/lib/openc3/topics/topic.rb +77 -5
  50. data/lib/openc3/utilities/aws_bucket.rb +2 -0
  51. data/lib/openc3/utilities/cli_generator.rb +3 -2
  52. data/lib/openc3/utilities/metric.rb +15 -1
  53. data/lib/openc3/utilities/questdb_client.rb +173 -37
  54. data/lib/openc3/utilities/reingest_job.rb +377 -0
  55. data/lib/openc3/utilities/ruby_lex_utils.rb +2 -0
  56. data/lib/openc3/utilities/store_autoload.rb +78 -52
  57. data/lib/openc3/utilities/store_queued.rb +20 -12
  58. data/lib/openc3/version.rb +6 -6
  59. data/templates/plugin/plugin.gemspec +13 -1
  60. data/templates/tool_angular/package.json +2 -2
  61. data/templates/tool_react/package.json +1 -1
  62. data/templates/tool_svelte/package.json +1 -1
  63. data/templates/tool_vue/package.json +3 -3
  64. data/templates/tool_vue/src/router.js +2 -2
  65. data/templates/widget/package.json +2 -2
  66. metadata +7 -3
@@ -0,0 +1,110 @@
1
+ # encoding: ascii-8bit
2
+
3
+ # Copyright 2026 OpenC3, Inc.
4
+ # All Rights Reserved.
5
+ #
6
+ # This program is distributed in the hope that it will be useful,
7
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
8
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
9
+ # See LICENSE.md for more details.
10
+ #
11
+ # This file may also be used under the terms of a commercial license
12
+ # if purchased from OpenC3, Inc.
13
+
14
+ # Mixin that provides db_shard-aware Redis operations with hard caching.
15
+ # Including classes must define two class methods:
16
+ # _lookup_db_shard(name, scope:) -> Integer
17
+ # _collect_db_shards(scope:) -> Set
18
+ module OpenC3
19
+ module DbShardedModel
20
+ def self.included(base)
21
+ base.extend(ClassMethods)
22
+ end
23
+
24
+ module ClassMethods
25
+ # Lookup of db_shard for a given name.
26
+ # Hard-cached only when use_cache: true (intended for the set/create path
27
+ # where the db_shard won't change within the process lifetime).
28
+ def _db_shard_for_name(name, scope:, use_cache: false)
29
+ cache = nil
30
+
31
+ if use_cache
32
+ cache = (@db_shard_cache ||= {})
33
+ cache_key = "#{scope}__#{name}"
34
+ cached = cache[cache_key]
35
+ return cached unless cached.nil?
36
+ end
37
+
38
+ db_shard = _lookup_db_shard(name, scope: scope)
39
+
40
+ if use_cache
41
+ cache[cache_key] = db_shard
42
+ end
43
+
44
+ db_shard
45
+ end
46
+
47
+ # Collect all active db_shards (always fresh lookup, no cache).
48
+ def _active_db_shards(scope:)
49
+ _collect_db_shards(scope: scope)
50
+ end
51
+
52
+ # DB_Shard-aware get: looks up the db_shard for name, reads from the correct store instance.
53
+ def _db_sharded_get(key, name:, scope:)
54
+ db_shard = _db_shard_for_name(name, scope: scope)
55
+ json = store.instance(db_shard: db_shard).hget(key, name)
56
+ json ? JSON.parse(json, allow_nan: true, create_additions: true) : nil
57
+ end
58
+
59
+ # DB_Shard-aware names: iterates all active db_shards and collects keys.
60
+ def _db_sharded_names(key, scope:)
61
+ result = []
62
+ _active_db_shards(scope: scope).each do |db_shard|
63
+ result.concat(store.instance(db_shard: db_shard).hkeys(key))
64
+ end
65
+ result.uniq.sort
66
+ end
67
+
68
+ # DB_Shard-aware all: iterates all active db_shards and collects all values.
69
+ def _db_sharded_all(key, scope:)
70
+ result = {}
71
+ _active_db_shards(scope: scope).each do |db_shard|
72
+ hash = store.instance(db_shard: db_shard).hgetall(key)
73
+ hash.each do |k, value|
74
+ result[k] = JSON.parse(value, allow_nan: true, create_additions: true)
75
+ end
76
+ end
77
+ result
78
+ end
79
+ end
80
+
81
+ # DB_Shard-aware create: writes to the store instance for the given db_shard.
82
+ def _db_sharded_create(db_shard, update: false, force: false, queued: false, isoformat: false, expire_seconds: nil)
83
+ db_shard_store = self.class.store.instance(db_shard: db_shard)
84
+ unless force
85
+ existing = db_shard_store.hget(@primary_key, @name)
86
+ if existing
87
+ raise RuntimeError.new("#{@primary_key}:#{@name} already exists at create") unless update
88
+ else
89
+ raise RuntimeError.new("#{@primary_key}:#{@name} doesn't exist at update") if update
90
+ end
91
+ end
92
+ @updated_at = isoformat ? Time.now.utc.iso8601 : Time.now.utc.to_nsec_from_epoch
93
+
94
+ if queued
95
+ store = self.class.store_queued.instance(db_shard: db_shard)
96
+ store.hset(@primary_key, @name, JSON.generate(self.as_json(), allow_nan: true))
97
+ store.call(:hexpire, @primary_key, expire_seconds, 'FIELDS', 1, @name) if expire_seconds
98
+ else
99
+ db_shard_store.hset(@primary_key, @name, JSON.generate(self.as_json(), allow_nan: true))
100
+ db_shard_store.call(:hexpire, @primary_key, expire_seconds, 'FIELDS', 1, @name) if expire_seconds
101
+ end
102
+ end
103
+
104
+ # DB_Shard-aware destroy: deletes from the store instance for the given db_shard.
105
+ def _db_sharded_destroy(db_shard)
106
+ @destroyed = true
107
+ self.class.store.instance(db_shard: db_shard).hdel(@primary_key, @name)
108
+ end
109
+ end
110
+ end
@@ -48,6 +48,7 @@ module OpenC3
48
48
  attr_accessor :ports
49
49
  attr_accessor :prefix
50
50
  attr_accessor :shard
51
+ attr_accessor :db_shard
51
52
 
52
53
  # NOTE: The following three class methods are used by the ModelController
53
54
  # and are reimplemented to enable various Model class methods to work
@@ -122,6 +123,7 @@ module OpenC3
122
123
  container: nil,
123
124
  prefix: nil,
124
125
  shard: 0,
126
+ db_shard: 0,
125
127
  scope:
126
128
  )
127
129
  if self.class._get_type == 'INTERFACE'
@@ -177,6 +179,7 @@ module OpenC3
177
179
  @container = container
178
180
  @prefix = prefix
179
181
  @shard = shard.to_i # to_i to handle nil
182
+ @db_shard = db_shard.to_i # to_i to handle nil
180
183
  @secrets = secrets
181
184
  end
182
185
 
@@ -246,6 +249,7 @@ module OpenC3
246
249
  'container' => @container,
247
250
  'prefix' => @prefix,
248
251
  'shard' => @shard,
252
+ 'db_shard' => @db_shard,
249
253
  'updated_at' => @updated_at
250
254
  }
251
255
  end
@@ -398,6 +402,10 @@ module OpenC3
398
402
  when 'SHARD'
399
403
  parser.verify_num_parameters(1, 1, "#{keyword} <Shard Number Starting from 0>")
400
404
  @shard = Integer(parameters[0])
405
+
406
+ when 'DB_SHARD'
407
+ parser.verify_num_parameters(1, 1, "#{keyword} <Shard Number Starting from 0>")
408
+ @db_shard = Integer(parameters[0])
401
409
  else
402
410
  raise ConfigParser::Error.new(parser, "Unknown keyword and parameters for Interface/Router: #{keyword} #{parameters.join(" ")}")
403
411
 
@@ -423,6 +431,7 @@ module OpenC3
423
431
  secrets: @secrets,
424
432
  prefix: @prefix,
425
433
  shard: @shard,
434
+ db_shard: @db_shard,
426
435
  scope: @scope
427
436
  )
428
437
  unless validate_only
@@ -16,12 +16,15 @@
16
16
  # if purchased from OpenC3, Inc.
17
17
 
18
18
  require 'openc3/models/model'
19
+ require 'openc3/models/db_sharded_model'
19
20
 
20
21
  module OpenC3
21
22
  # Stores the status about an interface. This class also implements logic
22
23
  # to handle status for a router since the functionality is identical
23
24
  # (only difference is the Redis key used).
24
25
  class InterfaceStatusModel < Model
26
+ include DbShardedModel
27
+
25
28
  INTERFACES_PRIMARY_KEY = 'openc3_interface_status'
26
29
  ROUTERS_PRIMARY_KEY = 'openc3_router_status'
27
30
 
@@ -34,18 +37,37 @@ module OpenC3
34
37
  attr_accessor :txcnt
35
38
  attr_accessor :rxcnt
36
39
 
40
+ # Look up db_shard from the corresponding InterfaceModel or RouterModel.
41
+ def self._lookup_db_shard(name, scope:)
42
+ type = _get_type
43
+ key = type == 'INTERFACESTATUS' ? "#{scope}__openc3_interfaces" : "#{scope}__openc3_routers"
44
+ json = Store.hget(key, name)
45
+ json ? (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i : 0
46
+ end
47
+
48
+ # Collect all unique db_shard values from InterfaceModels or RouterModels.
49
+ def self._collect_db_shards(scope:)
50
+ db_shards = Set.new([0])
51
+ type = _get_type
52
+ key = type == 'INTERFACESTATUS' ? "#{scope}__openc3_interfaces" : "#{scope}__openc3_routers"
53
+ Store.hgetall(key).each do |_name, json|
54
+ db_shards << (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i
55
+ end
56
+ db_shards
57
+ end
58
+
37
59
  # NOTE: The following three class methods are used by the ModelController
38
60
  # and are reimplemented to enable various Model class methods to work
39
61
  def self.get(name:, scope:)
40
- super("#{scope}__#{_get_key}", name: name)
62
+ _db_sharded_get("#{scope}__#{_get_key}", name: name, scope: scope)
41
63
  end
42
64
 
43
65
  def self.names(scope:)
44
- super("#{scope}__#{_get_key}")
66
+ _db_sharded_names("#{scope}__#{_get_key}", scope: scope)
45
67
  end
46
68
 
47
69
  def self.all(scope:)
48
- super("#{scope}__#{_get_key}")
70
+ _db_sharded_all("#{scope}__#{_get_key}", scope: scope)
49
71
  end
50
72
  # END NOTE
51
73
 
@@ -96,6 +118,14 @@ module OpenC3
96
118
  @rxcnt = rxcnt
97
119
  end
98
120
 
121
+ def create(update: false, force: false, queued: false, isoformat: false)
122
+ _db_sharded_create(self.class._db_shard_for_name(@name, scope: @scope, use_cache: true), update: update, force: force, queued: queued, isoformat: isoformat)
123
+ end
124
+
125
+ def destroy
126
+ _db_sharded_destroy(self.class._db_shard_for_name(@name, scope: @scope))
127
+ end
128
+
99
129
  def as_json(*a)
100
130
  {
101
131
  'name' => @name,
@@ -16,25 +16,46 @@
16
16
  # if purchased from OpenC3, Inc.
17
17
 
18
18
  require 'openc3/models/model'
19
+ require 'openc3/models/db_sharded_model'
19
20
 
20
21
  module OpenC3
21
22
  class MetricModel < EphemeralModel
23
+ include DbShardedModel
24
+
22
25
  PRIMARY_KEY = '__openc3__metric'.freeze
26
+ METRIC_EXPIRE_SECONDS = 3600 # Expire metrics after 1 hour
23
27
 
24
28
  attr_accessor :values
29
+ attr_accessor :db_shard
30
+
31
+ # Look up db_shard from the corresponding MicroserviceModel.
32
+ def self._lookup_db_shard(name, scope:) # NOSONAR
33
+ json = Store.hget('openc3_microservices', name)
34
+ json ? (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i : 0
35
+ end
36
+
37
+ # Collect all unique db_shard values from MicroserviceModels.
38
+ def self._collect_db_shards(scope:)
39
+ db_shards = Set.new([0])
40
+ Store.hgetall('openc3_microservices').each do |name, json|
41
+ next if scope and name.split("__")[0] != scope
42
+ db_shards << (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i
43
+ end
44
+ db_shards
45
+ end
25
46
 
26
47
  # NOTE: The following three class methods are used by the ModelController
27
48
  # and are reimplemented to enable various Model class methods to work
28
49
  def self.get(name:, scope:)
29
- super("#{scope}#{PRIMARY_KEY}", name: name)
50
+ _db_sharded_get("#{scope}#{PRIMARY_KEY}", name: name, scope: scope)
30
51
  end
31
52
 
32
53
  def self.names(scope:)
33
- super("#{scope}#{PRIMARY_KEY}")
54
+ _db_sharded_names("#{scope}#{PRIMARY_KEY}", scope: scope)
34
55
  end
35
56
 
36
57
  def self.all(scope:)
37
- super("#{scope}#{PRIMARY_KEY}")
58
+ _db_sharded_all("#{scope}#{PRIMARY_KEY}", scope: scope)
38
59
  end
39
60
 
40
61
  # Sets (updates) the redis hash of this model
@@ -42,23 +63,34 @@ module OpenC3
42
63
  def self.set(json, scope:, queued: true)
43
64
  json[:scope] = scope
44
65
  json.transform_keys!(&:to_sym)
45
- self.new(**json).create(force: true, queued: queued)
66
+ self.new(**json).create(force: true, queued: queued, expire_seconds: METRIC_EXPIRE_SECONDS)
46
67
  end
47
68
 
48
69
  def self.destroy(scope:, name:)
49
- EphemeralStore.hdel("#{scope}#{PRIMARY_KEY}", name)
70
+ db_shard = _db_shard_for_name(name, scope: scope)
71
+ store.instance(db_shard: db_shard).hdel("#{scope}#{PRIMARY_KEY}", name)
50
72
  end
51
73
 
52
- def initialize(name:, values: {}, scope:)
74
+ def initialize(name:, values: {}, db_shard: 0, scope:)
53
75
  super("#{scope}#{PRIMARY_KEY}", name: name, scope: scope)
54
76
  @values = values
77
+ @db_shard = db_shard.to_i
78
+ end
79
+
80
+ def create(update: false, force: false, queued: false, isoformat: false, expire_seconds: nil)
81
+ _db_sharded_create(@db_shard, update: update, force: force, queued: queued, isoformat: isoformat, expire_seconds: expire_seconds)
82
+ end
83
+
84
+ def destroy
85
+ _db_sharded_destroy(@db_shard)
55
86
  end
56
87
 
57
88
  def as_json(*a)
58
89
  {
59
90
  'name' => @name,
60
91
  'updated_at' => @updated_at,
61
- 'values' => @values.as_json(*a)
92
+ 'values' => @values.as_json(*a),
93
+ 'db_shard' => @db_shard,
62
94
  }
63
95
  end
64
96
 
@@ -74,37 +106,64 @@ module OpenC3
74
106
  end
75
107
 
76
108
  def self.redis_metrics
77
- result = {}
109
+ # This prevents a circular dependency
110
+ require 'openc3/models/scope_model' # NOSONAR
111
+ require 'openc3/models/target_model' # NOSONAR
112
+
113
+ db_shards = Set.new
114
+ OpenC3::ScopeModel.names.each do |scope|
115
+ targets = OpenC3::TargetModel.all(scope: scope)
116
+ targets.each do |_target_name, target_hash|
117
+ db_shards << target_hash['db_shard'].to_i
118
+ end
119
+ end
78
120
 
79
- metrics = OpenC3::Store.info("all")
80
- result['redis_connected_clients_total'] = metrics['connected_clients']
81
- result['redis_used_memory_rss_total'] = metrics['used_memory_rss']
82
- result['redis_commands_processed_total'] = metrics['total_commands_processed']
83
- result['redis_iops'] = metrics['instantaneous_ops_per_sec']
84
- result['redis_instantaneous_input_kbps'] = metrics['instantaneous_input_kbps']
85
- result['redis_instantaneous_output_kbps'] = metrics['instantaneous_output_kbps']
86
- result['redis_hget_p50_seconds'], result['redis_hget_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hget'])
87
- result['redis_hgetall_p50_seconds'], result['redis_hgetall_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hgetall'])
88
- result['redis_hset_p50_seconds'], result['redis_hset_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hset'])
89
- result['redis_xadd_p50_seconds'], result['redis_xadd_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xadd'])
90
- result['redis_xread_p50_seconds'], result['redis_xread_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xread'])
91
- result['redis_xrevrange_p50_seconds'], result['redis_xrevrange_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xrevrange'])
92
- result['redis_xtrim_p50_seconds'], result['redis_xtrim_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xtrim'])
93
-
94
- metrics = OpenC3::EphemeralStore.info("all")
95
- result['redis_ephemeral_connected_clients_total'] = metrics['connected_clients']
96
- result['redis_ephemeral_used_memory_rss_total'] = metrics['used_memory_rss']
97
- result['redis_ephemeral_commands_processed_total'] = metrics['total_commands_processed']
98
- result['redis_ephemeral_iops'] = metrics['instantaneous_ops_per_sec']
99
- result['redis_ephemeral_instantaneous_input_kbps'] = metrics['instantaneous_input_kbps']
100
- result['redis_ephemeral_instantaneous_output_kbps'] = metrics['instantaneous_output_kbps']
101
- result['redis_ephemeral_hget_p50_seconds'], result['redis_ephemeral_hget_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hget'])
102
- result['redis_ephemeral_hgetall_p50_seconds'], result['redis_ephemeral_hgetall_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hgetall'])
103
- result['redis_ephemeral_hset_p50_seconds'], result['redis_ephemeral_hset_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hset'])
104
- result['redis_ephemeral_xadd_p50_seconds'], result['redis_ephemeral_xadd_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xadd'])
105
- result['redis_ephemeral_xread_p50_seconds'], result['redis_ephemeral_xread_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xread'])
106
- result['redis_ephemeral_xrevrange_p50_seconds'], result['redis_ephemeral_xrevrange_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xrevrange'])
107
- result['redis_ephemeral_xtrim_p50_seconds'], result['redis_ephemeral_xtrim_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xtrim'])
121
+ result = {}
122
+ db_shards.each do |index|
123
+ db_shard_result = {}
124
+ metrics = OpenC3::Store.instance(db_shard: index).info("all")
125
+ db_shard_result["redis_connected_clients_total"] = metrics['connected_clients']
126
+ db_shard_result["redis_used_memory_rss_total"] = metrics['used_memory_rss']
127
+ db_shard_result["redis_commands_processed_total"] = metrics['total_commands_processed']
128
+ db_shard_result["redis_iops"] = metrics['instantaneous_ops_per_sec']
129
+ db_shard_result["redis_instantaneous_input_kbps"] = metrics['instantaneous_input_kbps']
130
+ db_shard_result["redis_instantaneous_output_kbps"] = metrics['instantaneous_output_kbps']
131
+ db_shard_result["redis_instantaneous_eventloop_cps"] = metrics['instantaneous_eventloop_cycles_per_sec']
132
+ db_shard_result["redis_instantaneous_eventloop_duration_usec"] = metrics['instantaneous_eventloop_duration_usec']
133
+ db_shard_result["redis_cpu_sys"] = metrics['used_cpu_sys']
134
+ db_shard_result["redis_cpu_user"] = metrics['used_cpu_user']
135
+ db_shard_result["redis_error_noauth_total"] = metrics['errorstat_NOAUTH'].to_s.split("count=")[-1].to_i
136
+ db_shard_result["redis_error_noperm_total"] = metrics['errorstat_NOPERM'].to_s.split("count=")[-1].to_i
137
+ db_shard_result["redis_hget_p50_seconds"], db_shard_result["redis_hget_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hget'])
138
+ db_shard_result["redis_hgetall_p50_seconds"], db_shard_result["redis_hgetall_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hgetall'])
139
+ db_shard_result["redis_hset_p50_seconds"], db_shard_result["redis_hset_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hset'])
140
+ db_shard_result["redis_xadd_p50_seconds"], db_shard_result["redis_xadd_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xadd'])
141
+ db_shard_result["redis_xread_p50_seconds"], db_shard_result["redis_xread_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xread'])
142
+ db_shard_result["redis_xrevrange_p50_seconds"], db_shard_result["redis_xrevrange_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xrevrange'])
143
+ db_shard_result["redis_xtrim_p50_seconds"], db_shard_result["redis_xtrim_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xtrim'])
144
+
145
+ metrics = OpenC3::EphemeralStore.instance(db_shard: index).info("all")
146
+ db_shard_result["redis_ephemeral_connected_clients_total"] = metrics['connected_clients']
147
+ db_shard_result["redis_ephemeral_used_memory_rss_total"] = metrics['used_memory_rss']
148
+ db_shard_result["redis_ephemeral_commands_processed_total"] = metrics['total_commands_processed']
149
+ db_shard_result["redis_ephemeral_iops"] = metrics['instantaneous_ops_per_sec']
150
+ db_shard_result["redis_ephemeral_instantaneous_input_kbps"] = metrics['instantaneous_input_kbps']
151
+ db_shard_result["redis_ephemeral_instantaneous_output_kbps"] = metrics['instantaneous_output_kbps']
152
+ db_shard_result["redis_ephemeral_instantaneous_eventloop_cps"] = metrics['instantaneous_eventloop_cycles_per_sec']
153
+ db_shard_result["redis_ephemeral_instantaneous_eventloop_duration_usec"] = metrics['instantaneous_eventloop_duration_usec']
154
+ db_shard_result["redis_ephemeral_cpu_sys"] = metrics['used_cpu_sys']
155
+ db_shard_result["redis_ephemeral_cpu_user"] = metrics['used_cpu_user']
156
+ db_shard_result["redis_ephemeral_error_noauth_total"] = metrics['errorstat_NOAUTH'].to_s.split("count=")[-1].to_i
157
+ db_shard_result["redis_ephemeral_error_noperm_total"] = metrics['errorstat_NOPERM'].to_s.split("count=")[-1].to_i
158
+ db_shard_result["redis_ephemeral_hget_p50_seconds"], db_shard_result["redis_ephemeral_hget_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hget'])
159
+ db_shard_result["redis_ephemeral_hgetall_p50_seconds"], db_shard_result["redis_ephemeral_hgetall_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hgetall'])
160
+ db_shard_result["redis_ephemeral_hset_p50_seconds"], db_shard_result["redis_ephemeral_hset_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hset'])
161
+ db_shard_result["redis_ephemeral_xadd_p50_seconds"], db_shard_result["redis_ephemeral_xadd_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xadd'])
162
+ db_shard_result["redis_ephemeral_xread_p50_seconds"], db_shard_result["redis_ephemeral_xread_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xread'])
163
+ db_shard_result["redis_ephemeral_xrevrange_p50_seconds"], db_shard_result["redis_ephemeral_xrevrange_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xrevrange'])
164
+ db_shard_result["redis_ephemeral_xtrim_p50_seconds"], db_shard_result["redis_ephemeral_xtrim_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xtrim'])
165
+ result[index] = db_shard_result
166
+ end
108
167
 
109
168
  return result
110
169
  end
@@ -41,6 +41,7 @@ module OpenC3
41
41
  attr_accessor :disable_erb
42
42
  attr_accessor :ignore_changes
43
43
  attr_accessor :shard
44
+ attr_accessor :db_shard
44
45
  attr_accessor :enabled
45
46
 
46
47
  # NOTE: The following three class methods are used by the ModelController
@@ -104,6 +105,7 @@ module OpenC3
104
105
  disable_erb: nil,
105
106
  ignore_changes: nil,
106
107
  shard: 0,
108
+ db_shard: 0,
107
109
  enabled: true,
108
110
  scope:
109
111
  )
@@ -132,6 +134,7 @@ module OpenC3
132
134
  @disable_erb = disable_erb
133
135
  @ignore_changes = ignore_changes
134
136
  @shard = shard.to_i # to_i to handle nil
137
+ @db_shard = db_shard.to_i # to_i to handle nil
135
138
  @enabled = enabled
136
139
  @enabled = true if @enabled.nil?
137
140
  @bucket = Bucket.getClient()
@@ -158,6 +161,7 @@ module OpenC3
158
161
  'disable_erb' => @disable_erb,
159
162
  'ignore_changes' => @ignore_changes,
160
163
  'shard' => @shard,
164
+ 'db_shard' => @db_shard,
161
165
  'enabled' => @enabled,
162
166
  }
163
167
  end
@@ -236,6 +240,9 @@ module OpenC3
236
240
  when 'SHARD'
237
241
  parser.verify_num_parameters(1, 1, "#{keyword} <Shard Number Starting from 0>")
238
242
  @shard = Integer(parameters[0])
243
+ when 'DB_SHARD'
244
+ parser.verify_num_parameters(1, 1, "#{keyword} <DB_Shard Number Starting from 0>")
245
+ @db_shard = Integer(parameters[0])
239
246
  when 'STOPPED'
240
247
  parser.verify_num_parameters(0, 0, "#{keyword}")
241
248
  @enabled = false
@@ -16,9 +16,12 @@
16
16
  # if purchased from OpenC3, Inc.
17
17
 
18
18
  require 'openc3/models/model'
19
+ require 'openc3/models/db_sharded_model'
19
20
 
20
21
  module OpenC3
21
22
  class MicroserviceStatusModel < Model
23
+ include DbShardedModel
24
+
22
25
  PRIMARY_KEY = 'openc3_microservice_status'
23
26
 
24
27
  attr_accessor :state
@@ -26,18 +29,34 @@ module OpenC3
26
29
  attr_accessor :error
27
30
  attr_accessor :custom
28
31
 
32
+ # Look up db_shard from the corresponding MicroserviceModel.
33
+ def self._lookup_db_shard(name, scope:) # NOSONAR
34
+ json = Store.hget('openc3_microservices', name)
35
+ json ? (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i : 0
36
+ end
37
+
38
+ # Collect all unique db_shard values from MicroserviceModels.
39
+ def self._collect_db_shards(scope:)
40
+ db_shards = Set.new([0])
41
+ Store.hgetall('openc3_microservices').each do |name, json|
42
+ next if scope and name.split("__")[0] != scope
43
+ db_shards << (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i
44
+ end
45
+ db_shards
46
+ end
47
+
29
48
  # NOTE: The following three class methods are used by the ModelController
30
49
  # and are reimplemented to enable various Model class methods to work
31
50
  def self.get(name:, scope:)
32
- super("#{scope}__#{PRIMARY_KEY}", name: name)
51
+ _db_sharded_get("#{scope}__#{PRIMARY_KEY}", name: name, scope: scope)
33
52
  end
34
53
 
35
54
  def self.names(scope:)
36
- super("#{scope}__#{PRIMARY_KEY}")
55
+ _db_sharded_names("#{scope}__#{PRIMARY_KEY}", scope: scope)
37
56
  end
38
57
 
39
58
  def self.all(scope:)
40
- super("#{scope}__#{PRIMARY_KEY}")
59
+ _db_sharded_all("#{scope}__#{PRIMARY_KEY}", scope: scope)
41
60
  end
42
61
 
43
62
  def initialize(
@@ -57,6 +76,14 @@ module OpenC3
57
76
  @custom = custom
58
77
  end
59
78
 
79
+ def create(update: false, force: false, queued: false, isoformat: false)
80
+ _db_sharded_create(self.class._db_shard_for_name(@name, scope: @scope, use_cache: true), update: update, force: force, queued: queued, isoformat: isoformat)
81
+ end
82
+
83
+ def destroy
84
+ _db_sharded_destroy(self.class._db_shard_for_name(@name, scope: @scope))
85
+ end
86
+
60
87
  def as_json(*a)
61
88
  {
62
89
  'name' => @name,