openc3 7.0.1 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/openc3cli +47 -3
- data/data/config/item_modifiers.yaml +1 -1
- data/data/config/microservice.yaml +12 -1
- data/data/config/parameter_modifiers.yaml +49 -7
- data/data/config/target.yaml +11 -0
- data/data/config/target_config.yaml +6 -2
- data/lib/openc3/api/cmd_api.rb +2 -1
- data/lib/openc3/api/metrics_api.rb +11 -1
- data/lib/openc3/api/tlm_api.rb +21 -6
- data/lib/openc3/core_ext/faraday.rb +1 -1
- data/lib/openc3/io/json_api.rb +1 -1
- data/lib/openc3/logs/log_writer.rb +3 -1
- data/lib/openc3/microservices/decom_common.rb +128 -0
- data/lib/openc3/microservices/decom_microservice.rb +26 -95
- data/lib/openc3/microservices/interface_decom_common.rb +6 -2
- data/lib/openc3/microservices/interface_microservice.rb +10 -8
- data/lib/openc3/microservices/log_microservice.rb +1 -1
- data/lib/openc3/microservices/microservice.rb +3 -2
- data/lib/openc3/microservices/queue_microservice.rb +1 -1
- data/lib/openc3/microservices/scope_cleanup_microservice.rb +60 -46
- data/lib/openc3/microservices/text_log_microservice.rb +1 -2
- data/lib/openc3/models/cvt_model.rb +24 -13
- data/lib/openc3/models/db_sharded_model.rb +110 -0
- data/lib/openc3/models/interface_model.rb +9 -0
- data/lib/openc3/models/interface_status_model.rb +33 -3
- data/lib/openc3/models/metric_model.rb +96 -37
- data/lib/openc3/models/microservice_model.rb +7 -0
- data/lib/openc3/models/microservice_status_model.rb +30 -3
- data/lib/openc3/models/reingest_job_model.rb +153 -0
- data/lib/openc3/models/scope_model.rb +3 -2
- data/lib/openc3/models/script_status_model.rb +4 -20
- data/lib/openc3/models/target_model.rb +113 -100
- data/lib/openc3/packets/packet_config.rb +4 -1
- data/lib/openc3/script/script.rb +2 -2
- data/lib/openc3/script/script_runner.rb +4 -4
- data/lib/openc3/script/telemetry.rb +3 -3
- data/lib/openc3/script/web_socket_api.rb +29 -22
- data/lib/openc3/system/system.rb +20 -3
- data/lib/openc3/topics/command_decom_topic.rb +4 -2
- data/lib/openc3/topics/command_topic.rb +8 -5
- data/lib/openc3/topics/decom_interface_topic.rb +15 -10
- data/lib/openc3/topics/interface_topic.rb +71 -29
- data/lib/openc3/topics/limits_event_topic.rb +62 -41
- data/lib/openc3/topics/router_topic.rb +61 -21
- data/lib/openc3/topics/system_events_topic.rb +18 -1
- data/lib/openc3/topics/telemetry_decom_topic.rb +2 -1
- data/lib/openc3/topics/telemetry_topic.rb +4 -2
- data/lib/openc3/topics/topic.rb +77 -5
- data/lib/openc3/utilities/aws_bucket.rb +2 -0
- data/lib/openc3/utilities/cli_generator.rb +3 -2
- data/lib/openc3/utilities/metric.rb +15 -1
- data/lib/openc3/utilities/questdb_client.rb +173 -37
- data/lib/openc3/utilities/reingest_job.rb +377 -0
- data/lib/openc3/utilities/ruby_lex_utils.rb +2 -0
- data/lib/openc3/utilities/store_autoload.rb +78 -52
- data/lib/openc3/utilities/store_queued.rb +20 -12
- data/lib/openc3/version.rb +6 -6
- data/templates/plugin/plugin.gemspec +13 -1
- data/templates/tool_angular/package.json +2 -2
- data/templates/tool_react/package.json +1 -1
- data/templates/tool_svelte/package.json +1 -1
- data/templates/tool_vue/package.json +3 -3
- data/templates/tool_vue/src/router.js +2 -2
- data/templates/widget/package.json +2 -2
- metadata +7 -3
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# encoding: ascii-8bit
|
|
2
|
+
|
|
3
|
+
# Copyright 2026 OpenC3, Inc.
|
|
4
|
+
# All Rights Reserved.
|
|
5
|
+
#
|
|
6
|
+
# This program is distributed in the hope that it will be useful,
|
|
7
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
8
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
9
|
+
# See LICENSE.md for more details.
|
|
10
|
+
#
|
|
11
|
+
# This file may also be used under the terms of a commercial license
|
|
12
|
+
# if purchased from OpenC3, Inc.
|
|
13
|
+
|
|
14
|
+
# Mixin that provides db_shard-aware Redis operations with hard caching.
|
|
15
|
+
# Including classes must define two class methods:
|
|
16
|
+
# _lookup_db_shard(name, scope:) -> Integer
|
|
17
|
+
# _collect_db_shards(scope:) -> Set
|
|
18
|
+
module OpenC3
|
|
19
|
+
module DbShardedModel
|
|
20
|
+
def self.included(base)
|
|
21
|
+
base.extend(ClassMethods)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
module ClassMethods
|
|
25
|
+
# Lookup of db_shard for a given name.
|
|
26
|
+
# Hard-cached only when use_cache: true (intended for the set/create path
|
|
27
|
+
# where the db_shard won't change within the process lifetime).
|
|
28
|
+
def _db_shard_for_name(name, scope:, use_cache: false)
|
|
29
|
+
cache = nil
|
|
30
|
+
|
|
31
|
+
if use_cache
|
|
32
|
+
cache = (@db_shard_cache ||= {})
|
|
33
|
+
cache_key = "#{scope}__#{name}"
|
|
34
|
+
cached = cache[cache_key]
|
|
35
|
+
return cached unless cached.nil?
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
db_shard = _lookup_db_shard(name, scope: scope)
|
|
39
|
+
|
|
40
|
+
if use_cache
|
|
41
|
+
cache[cache_key] = db_shard
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
db_shard
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Collect all active db_shards (always fresh lookup, no cache).
|
|
48
|
+
def _active_db_shards(scope:)
|
|
49
|
+
_collect_db_shards(scope: scope)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# DB_Shard-aware get: looks up the db_shard for name, reads from the correct store instance.
|
|
53
|
+
def _db_sharded_get(key, name:, scope:)
|
|
54
|
+
db_shard = _db_shard_for_name(name, scope: scope)
|
|
55
|
+
json = store.instance(db_shard: db_shard).hget(key, name)
|
|
56
|
+
json ? JSON.parse(json, allow_nan: true, create_additions: true) : nil
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# DB_Shard-aware names: iterates all active db_shards and collects keys.
|
|
60
|
+
def _db_sharded_names(key, scope:)
|
|
61
|
+
result = []
|
|
62
|
+
_active_db_shards(scope: scope).each do |db_shard|
|
|
63
|
+
result.concat(store.instance(db_shard: db_shard).hkeys(key))
|
|
64
|
+
end
|
|
65
|
+
result.uniq.sort
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# DB_Shard-aware all: iterates all active db_shards and collects all values.
|
|
69
|
+
def _db_sharded_all(key, scope:)
|
|
70
|
+
result = {}
|
|
71
|
+
_active_db_shards(scope: scope).each do |db_shard|
|
|
72
|
+
hash = store.instance(db_shard: db_shard).hgetall(key)
|
|
73
|
+
hash.each do |k, value|
|
|
74
|
+
result[k] = JSON.parse(value, allow_nan: true, create_additions: true)
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
result
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# DB_Shard-aware create: writes to the store instance for the given db_shard.
|
|
82
|
+
def _db_sharded_create(db_shard, update: false, force: false, queued: false, isoformat: false, expire_seconds: nil)
|
|
83
|
+
db_shard_store = self.class.store.instance(db_shard: db_shard)
|
|
84
|
+
unless force
|
|
85
|
+
existing = db_shard_store.hget(@primary_key, @name)
|
|
86
|
+
if existing
|
|
87
|
+
raise RuntimeError.new("#{@primary_key}:#{@name} already exists at create") unless update
|
|
88
|
+
else
|
|
89
|
+
raise RuntimeError.new("#{@primary_key}:#{@name} doesn't exist at update") if update
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
@updated_at = isoformat ? Time.now.utc.iso8601 : Time.now.utc.to_nsec_from_epoch
|
|
93
|
+
|
|
94
|
+
if queued
|
|
95
|
+
store = self.class.store_queued.instance(db_shard: db_shard)
|
|
96
|
+
store.hset(@primary_key, @name, JSON.generate(self.as_json(), allow_nan: true))
|
|
97
|
+
store.call(:hexpire, @primary_key, expire_seconds, 'FIELDS', 1, @name) if expire_seconds
|
|
98
|
+
else
|
|
99
|
+
db_shard_store.hset(@primary_key, @name, JSON.generate(self.as_json(), allow_nan: true))
|
|
100
|
+
db_shard_store.call(:hexpire, @primary_key, expire_seconds, 'FIELDS', 1, @name) if expire_seconds
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# DB_Shard-aware destroy: deletes from the store instance for the given db_shard.
|
|
105
|
+
def _db_sharded_destroy(db_shard)
|
|
106
|
+
@destroyed = true
|
|
107
|
+
self.class.store.instance(db_shard: db_shard).hdel(@primary_key, @name)
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
@@ -48,6 +48,7 @@ module OpenC3
|
|
|
48
48
|
attr_accessor :ports
|
|
49
49
|
attr_accessor :prefix
|
|
50
50
|
attr_accessor :shard
|
|
51
|
+
attr_accessor :db_shard
|
|
51
52
|
|
|
52
53
|
# NOTE: The following three class methods are used by the ModelController
|
|
53
54
|
# and are reimplemented to enable various Model class methods to work
|
|
@@ -122,6 +123,7 @@ module OpenC3
|
|
|
122
123
|
container: nil,
|
|
123
124
|
prefix: nil,
|
|
124
125
|
shard: 0,
|
|
126
|
+
db_shard: 0,
|
|
125
127
|
scope:
|
|
126
128
|
)
|
|
127
129
|
if self.class._get_type == 'INTERFACE'
|
|
@@ -177,6 +179,7 @@ module OpenC3
|
|
|
177
179
|
@container = container
|
|
178
180
|
@prefix = prefix
|
|
179
181
|
@shard = shard.to_i # to_i to handle nil
|
|
182
|
+
@db_shard = db_shard.to_i # to_i to handle nil
|
|
180
183
|
@secrets = secrets
|
|
181
184
|
end
|
|
182
185
|
|
|
@@ -246,6 +249,7 @@ module OpenC3
|
|
|
246
249
|
'container' => @container,
|
|
247
250
|
'prefix' => @prefix,
|
|
248
251
|
'shard' => @shard,
|
|
252
|
+
'db_shard' => @db_shard,
|
|
249
253
|
'updated_at' => @updated_at
|
|
250
254
|
}
|
|
251
255
|
end
|
|
@@ -398,6 +402,10 @@ module OpenC3
|
|
|
398
402
|
when 'SHARD'
|
|
399
403
|
parser.verify_num_parameters(1, 1, "#{keyword} <Shard Number Starting from 0>")
|
|
400
404
|
@shard = Integer(parameters[0])
|
|
405
|
+
|
|
406
|
+
when 'DB_SHARD'
|
|
407
|
+
parser.verify_num_parameters(1, 1, "#{keyword} <Shard Number Starting from 0>")
|
|
408
|
+
@db_shard = Integer(parameters[0])
|
|
401
409
|
else
|
|
402
410
|
raise ConfigParser::Error.new(parser, "Unknown keyword and parameters for Interface/Router: #{keyword} #{parameters.join(" ")}")
|
|
403
411
|
|
|
@@ -423,6 +431,7 @@ module OpenC3
|
|
|
423
431
|
secrets: @secrets,
|
|
424
432
|
prefix: @prefix,
|
|
425
433
|
shard: @shard,
|
|
434
|
+
db_shard: @db_shard,
|
|
426
435
|
scope: @scope
|
|
427
436
|
)
|
|
428
437
|
unless validate_only
|
|
@@ -16,12 +16,15 @@
|
|
|
16
16
|
# if purchased from OpenC3, Inc.
|
|
17
17
|
|
|
18
18
|
require 'openc3/models/model'
|
|
19
|
+
require 'openc3/models/db_sharded_model'
|
|
19
20
|
|
|
20
21
|
module OpenC3
|
|
21
22
|
# Stores the status about an interface. This class also implements logic
|
|
22
23
|
# to handle status for a router since the functionality is identical
|
|
23
24
|
# (only difference is the Redis key used).
|
|
24
25
|
class InterfaceStatusModel < Model
|
|
26
|
+
include DbShardedModel
|
|
27
|
+
|
|
25
28
|
INTERFACES_PRIMARY_KEY = 'openc3_interface_status'
|
|
26
29
|
ROUTERS_PRIMARY_KEY = 'openc3_router_status'
|
|
27
30
|
|
|
@@ -34,18 +37,37 @@ module OpenC3
|
|
|
34
37
|
attr_accessor :txcnt
|
|
35
38
|
attr_accessor :rxcnt
|
|
36
39
|
|
|
40
|
+
# Look up db_shard from the corresponding InterfaceModel or RouterModel.
|
|
41
|
+
def self._lookup_db_shard(name, scope:)
|
|
42
|
+
type = _get_type
|
|
43
|
+
key = type == 'INTERFACESTATUS' ? "#{scope}__openc3_interfaces" : "#{scope}__openc3_routers"
|
|
44
|
+
json = Store.hget(key, name)
|
|
45
|
+
json ? (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i : 0
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Collect all unique db_shard values from InterfaceModels or RouterModels.
|
|
49
|
+
def self._collect_db_shards(scope:)
|
|
50
|
+
db_shards = Set.new([0])
|
|
51
|
+
type = _get_type
|
|
52
|
+
key = type == 'INTERFACESTATUS' ? "#{scope}__openc3_interfaces" : "#{scope}__openc3_routers"
|
|
53
|
+
Store.hgetall(key).each do |_name, json|
|
|
54
|
+
db_shards << (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i
|
|
55
|
+
end
|
|
56
|
+
db_shards
|
|
57
|
+
end
|
|
58
|
+
|
|
37
59
|
# NOTE: The following three class methods are used by the ModelController
|
|
38
60
|
# and are reimplemented to enable various Model class methods to work
|
|
39
61
|
def self.get(name:, scope:)
|
|
40
|
-
|
|
62
|
+
_db_sharded_get("#{scope}__#{_get_key}", name: name, scope: scope)
|
|
41
63
|
end
|
|
42
64
|
|
|
43
65
|
def self.names(scope:)
|
|
44
|
-
|
|
66
|
+
_db_sharded_names("#{scope}__#{_get_key}", scope: scope)
|
|
45
67
|
end
|
|
46
68
|
|
|
47
69
|
def self.all(scope:)
|
|
48
|
-
|
|
70
|
+
_db_sharded_all("#{scope}__#{_get_key}", scope: scope)
|
|
49
71
|
end
|
|
50
72
|
# END NOTE
|
|
51
73
|
|
|
@@ -96,6 +118,14 @@ module OpenC3
|
|
|
96
118
|
@rxcnt = rxcnt
|
|
97
119
|
end
|
|
98
120
|
|
|
121
|
+
def create(update: false, force: false, queued: false, isoformat: false)
|
|
122
|
+
_db_sharded_create(self.class._db_shard_for_name(@name, scope: @scope, use_cache: true), update: update, force: force, queued: queued, isoformat: isoformat)
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def destroy
|
|
126
|
+
_db_sharded_destroy(self.class._db_shard_for_name(@name, scope: @scope))
|
|
127
|
+
end
|
|
128
|
+
|
|
99
129
|
def as_json(*a)
|
|
100
130
|
{
|
|
101
131
|
'name' => @name,
|
|
@@ -16,25 +16,46 @@
|
|
|
16
16
|
# if purchased from OpenC3, Inc.
|
|
17
17
|
|
|
18
18
|
require 'openc3/models/model'
|
|
19
|
+
require 'openc3/models/db_sharded_model'
|
|
19
20
|
|
|
20
21
|
module OpenC3
|
|
21
22
|
class MetricModel < EphemeralModel
|
|
23
|
+
include DbShardedModel
|
|
24
|
+
|
|
22
25
|
PRIMARY_KEY = '__openc3__metric'.freeze
|
|
26
|
+
METRIC_EXPIRE_SECONDS = 3600 # Expire metrics after 1 hour
|
|
23
27
|
|
|
24
28
|
attr_accessor :values
|
|
29
|
+
attr_accessor :db_shard
|
|
30
|
+
|
|
31
|
+
# Look up db_shard from the corresponding MicroserviceModel.
|
|
32
|
+
def self._lookup_db_shard(name, scope:) # NOSONAR
|
|
33
|
+
json = Store.hget('openc3_microservices', name)
|
|
34
|
+
json ? (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i : 0
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Collect all unique db_shard values from MicroserviceModels.
|
|
38
|
+
def self._collect_db_shards(scope:)
|
|
39
|
+
db_shards = Set.new([0])
|
|
40
|
+
Store.hgetall('openc3_microservices').each do |name, json|
|
|
41
|
+
next if scope and name.split("__")[0] != scope
|
|
42
|
+
db_shards << (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i
|
|
43
|
+
end
|
|
44
|
+
db_shards
|
|
45
|
+
end
|
|
25
46
|
|
|
26
47
|
# NOTE: The following three class methods are used by the ModelController
|
|
27
48
|
# and are reimplemented to enable various Model class methods to work
|
|
28
49
|
def self.get(name:, scope:)
|
|
29
|
-
|
|
50
|
+
_db_sharded_get("#{scope}#{PRIMARY_KEY}", name: name, scope: scope)
|
|
30
51
|
end
|
|
31
52
|
|
|
32
53
|
def self.names(scope:)
|
|
33
|
-
|
|
54
|
+
_db_sharded_names("#{scope}#{PRIMARY_KEY}", scope: scope)
|
|
34
55
|
end
|
|
35
56
|
|
|
36
57
|
def self.all(scope:)
|
|
37
|
-
|
|
58
|
+
_db_sharded_all("#{scope}#{PRIMARY_KEY}", scope: scope)
|
|
38
59
|
end
|
|
39
60
|
|
|
40
61
|
# Sets (updates) the redis hash of this model
|
|
@@ -42,23 +63,34 @@ module OpenC3
|
|
|
42
63
|
def self.set(json, scope:, queued: true)
|
|
43
64
|
json[:scope] = scope
|
|
44
65
|
json.transform_keys!(&:to_sym)
|
|
45
|
-
self.new(**json).create(force: true, queued: queued)
|
|
66
|
+
self.new(**json).create(force: true, queued: queued, expire_seconds: METRIC_EXPIRE_SECONDS)
|
|
46
67
|
end
|
|
47
68
|
|
|
48
69
|
def self.destroy(scope:, name:)
|
|
49
|
-
|
|
70
|
+
db_shard = _db_shard_for_name(name, scope: scope)
|
|
71
|
+
store.instance(db_shard: db_shard).hdel("#{scope}#{PRIMARY_KEY}", name)
|
|
50
72
|
end
|
|
51
73
|
|
|
52
|
-
def initialize(name:, values: {}, scope:)
|
|
74
|
+
def initialize(name:, values: {}, db_shard: 0, scope:)
|
|
53
75
|
super("#{scope}#{PRIMARY_KEY}", name: name, scope: scope)
|
|
54
76
|
@values = values
|
|
77
|
+
@db_shard = db_shard.to_i
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def create(update: false, force: false, queued: false, isoformat: false, expire_seconds: nil)
|
|
81
|
+
_db_sharded_create(@db_shard, update: update, force: force, queued: queued, isoformat: isoformat, expire_seconds: expire_seconds)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def destroy
|
|
85
|
+
_db_sharded_destroy(@db_shard)
|
|
55
86
|
end
|
|
56
87
|
|
|
57
88
|
def as_json(*a)
|
|
58
89
|
{
|
|
59
90
|
'name' => @name,
|
|
60
91
|
'updated_at' => @updated_at,
|
|
61
|
-
'values' => @values.as_json(*a)
|
|
92
|
+
'values' => @values.as_json(*a),
|
|
93
|
+
'db_shard' => @db_shard,
|
|
62
94
|
}
|
|
63
95
|
end
|
|
64
96
|
|
|
@@ -74,37 +106,64 @@ module OpenC3
|
|
|
74
106
|
end
|
|
75
107
|
|
|
76
108
|
def self.redis_metrics
|
|
77
|
-
|
|
109
|
+
# This prevents a circular dependency
|
|
110
|
+
require 'openc3/models/scope_model' # NOSONAR
|
|
111
|
+
require 'openc3/models/target_model' # NOSONAR
|
|
112
|
+
|
|
113
|
+
db_shards = Set.new
|
|
114
|
+
OpenC3::ScopeModel.names.each do |scope|
|
|
115
|
+
targets = OpenC3::TargetModel.all(scope: scope)
|
|
116
|
+
targets.each do |_target_name, target_hash|
|
|
117
|
+
db_shards << target_hash['db_shard'].to_i
|
|
118
|
+
end
|
|
119
|
+
end
|
|
78
120
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
121
|
+
result = {}
|
|
122
|
+
db_shards.each do |index|
|
|
123
|
+
db_shard_result = {}
|
|
124
|
+
metrics = OpenC3::Store.instance(db_shard: index).info("all")
|
|
125
|
+
db_shard_result["redis_connected_clients_total"] = metrics['connected_clients']
|
|
126
|
+
db_shard_result["redis_used_memory_rss_total"] = metrics['used_memory_rss']
|
|
127
|
+
db_shard_result["redis_commands_processed_total"] = metrics['total_commands_processed']
|
|
128
|
+
db_shard_result["redis_iops"] = metrics['instantaneous_ops_per_sec']
|
|
129
|
+
db_shard_result["redis_instantaneous_input_kbps"] = metrics['instantaneous_input_kbps']
|
|
130
|
+
db_shard_result["redis_instantaneous_output_kbps"] = metrics['instantaneous_output_kbps']
|
|
131
|
+
db_shard_result["redis_instantaneous_eventloop_cps"] = metrics['instantaneous_eventloop_cycles_per_sec']
|
|
132
|
+
db_shard_result["redis_instantaneous_eventloop_duration_usec"] = metrics['instantaneous_eventloop_duration_usec']
|
|
133
|
+
db_shard_result["redis_cpu_sys"] = metrics['used_cpu_sys']
|
|
134
|
+
db_shard_result["redis_cpu_user"] = metrics['used_cpu_user']
|
|
135
|
+
db_shard_result["redis_error_noauth_total"] = metrics['errorstat_NOAUTH'].to_s.split("count=")[-1].to_i
|
|
136
|
+
db_shard_result["redis_error_noperm_total"] = metrics['errorstat_NOPERM'].to_s.split("count=")[-1].to_i
|
|
137
|
+
db_shard_result["redis_hget_p50_seconds"], db_shard_result["redis_hget_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hget'])
|
|
138
|
+
db_shard_result["redis_hgetall_p50_seconds"], db_shard_result["redis_hgetall_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hgetall'])
|
|
139
|
+
db_shard_result["redis_hset_p50_seconds"], db_shard_result["redis_hset_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hset'])
|
|
140
|
+
db_shard_result["redis_xadd_p50_seconds"], db_shard_result["redis_xadd_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xadd'])
|
|
141
|
+
db_shard_result["redis_xread_p50_seconds"], db_shard_result["redis_xread_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xread'])
|
|
142
|
+
db_shard_result["redis_xrevrange_p50_seconds"], db_shard_result["redis_xrevrange_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xrevrange'])
|
|
143
|
+
db_shard_result["redis_xtrim_p50_seconds"], db_shard_result["redis_xtrim_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xtrim'])
|
|
144
|
+
|
|
145
|
+
metrics = OpenC3::EphemeralStore.instance(db_shard: index).info("all")
|
|
146
|
+
db_shard_result["redis_ephemeral_connected_clients_total"] = metrics['connected_clients']
|
|
147
|
+
db_shard_result["redis_ephemeral_used_memory_rss_total"] = metrics['used_memory_rss']
|
|
148
|
+
db_shard_result["redis_ephemeral_commands_processed_total"] = metrics['total_commands_processed']
|
|
149
|
+
db_shard_result["redis_ephemeral_iops"] = metrics['instantaneous_ops_per_sec']
|
|
150
|
+
db_shard_result["redis_ephemeral_instantaneous_input_kbps"] = metrics['instantaneous_input_kbps']
|
|
151
|
+
db_shard_result["redis_ephemeral_instantaneous_output_kbps"] = metrics['instantaneous_output_kbps']
|
|
152
|
+
db_shard_result["redis_ephemeral_instantaneous_eventloop_cps"] = metrics['instantaneous_eventloop_cycles_per_sec']
|
|
153
|
+
db_shard_result["redis_ephemeral_instantaneous_eventloop_duration_usec"] = metrics['instantaneous_eventloop_duration_usec']
|
|
154
|
+
db_shard_result["redis_ephemeral_cpu_sys"] = metrics['used_cpu_sys']
|
|
155
|
+
db_shard_result["redis_ephemeral_cpu_user"] = metrics['used_cpu_user']
|
|
156
|
+
db_shard_result["redis_ephemeral_error_noauth_total"] = metrics['errorstat_NOAUTH'].to_s.split("count=")[-1].to_i
|
|
157
|
+
db_shard_result["redis_ephemeral_error_noperm_total"] = metrics['errorstat_NOPERM'].to_s.split("count=")[-1].to_i
|
|
158
|
+
db_shard_result["redis_ephemeral_hget_p50_seconds"], db_shard_result["redis_ephemeral_hget_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hget'])
|
|
159
|
+
db_shard_result["redis_ephemeral_hgetall_p50_seconds"], db_shard_result["redis_ephemeral_hgetall_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hgetall'])
|
|
160
|
+
db_shard_result["redis_ephemeral_hset_p50_seconds"], db_shard_result["redis_ephemeral_hset_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hset'])
|
|
161
|
+
db_shard_result["redis_ephemeral_xadd_p50_seconds"], db_shard_result["redis_ephemeral_xadd_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xadd'])
|
|
162
|
+
db_shard_result["redis_ephemeral_xread_p50_seconds"], db_shard_result["redis_ephemeral_xread_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xread'])
|
|
163
|
+
db_shard_result["redis_ephemeral_xrevrange_p50_seconds"], db_shard_result["redis_ephemeral_xrevrange_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xrevrange'])
|
|
164
|
+
db_shard_result["redis_ephemeral_xtrim_p50_seconds"], db_shard_result["redis_ephemeral_xtrim_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xtrim'])
|
|
165
|
+
result[index] = db_shard_result
|
|
166
|
+
end
|
|
108
167
|
|
|
109
168
|
return result
|
|
110
169
|
end
|
|
@@ -41,6 +41,7 @@ module OpenC3
|
|
|
41
41
|
attr_accessor :disable_erb
|
|
42
42
|
attr_accessor :ignore_changes
|
|
43
43
|
attr_accessor :shard
|
|
44
|
+
attr_accessor :db_shard
|
|
44
45
|
attr_accessor :enabled
|
|
45
46
|
|
|
46
47
|
# NOTE: The following three class methods are used by the ModelController
|
|
@@ -104,6 +105,7 @@ module OpenC3
|
|
|
104
105
|
disable_erb: nil,
|
|
105
106
|
ignore_changes: nil,
|
|
106
107
|
shard: 0,
|
|
108
|
+
db_shard: 0,
|
|
107
109
|
enabled: true,
|
|
108
110
|
scope:
|
|
109
111
|
)
|
|
@@ -132,6 +134,7 @@ module OpenC3
|
|
|
132
134
|
@disable_erb = disable_erb
|
|
133
135
|
@ignore_changes = ignore_changes
|
|
134
136
|
@shard = shard.to_i # to_i to handle nil
|
|
137
|
+
@db_shard = db_shard.to_i # to_i to handle nil
|
|
135
138
|
@enabled = enabled
|
|
136
139
|
@enabled = true if @enabled.nil?
|
|
137
140
|
@bucket = Bucket.getClient()
|
|
@@ -158,6 +161,7 @@ module OpenC3
|
|
|
158
161
|
'disable_erb' => @disable_erb,
|
|
159
162
|
'ignore_changes' => @ignore_changes,
|
|
160
163
|
'shard' => @shard,
|
|
164
|
+
'db_shard' => @db_shard,
|
|
161
165
|
'enabled' => @enabled,
|
|
162
166
|
}
|
|
163
167
|
end
|
|
@@ -236,6 +240,9 @@ module OpenC3
|
|
|
236
240
|
when 'SHARD'
|
|
237
241
|
parser.verify_num_parameters(1, 1, "#{keyword} <Shard Number Starting from 0>")
|
|
238
242
|
@shard = Integer(parameters[0])
|
|
243
|
+
when 'DB_SHARD'
|
|
244
|
+
parser.verify_num_parameters(1, 1, "#{keyword} <DB_Shard Number Starting from 0>")
|
|
245
|
+
@db_shard = Integer(parameters[0])
|
|
239
246
|
when 'STOPPED'
|
|
240
247
|
parser.verify_num_parameters(0, 0, "#{keyword}")
|
|
241
248
|
@enabled = false
|
|
@@ -16,9 +16,12 @@
|
|
|
16
16
|
# if purchased from OpenC3, Inc.
|
|
17
17
|
|
|
18
18
|
require 'openc3/models/model'
|
|
19
|
+
require 'openc3/models/db_sharded_model'
|
|
19
20
|
|
|
20
21
|
module OpenC3
|
|
21
22
|
class MicroserviceStatusModel < Model
|
|
23
|
+
include DbShardedModel
|
|
24
|
+
|
|
22
25
|
PRIMARY_KEY = 'openc3_microservice_status'
|
|
23
26
|
|
|
24
27
|
attr_accessor :state
|
|
@@ -26,18 +29,34 @@ module OpenC3
|
|
|
26
29
|
attr_accessor :error
|
|
27
30
|
attr_accessor :custom
|
|
28
31
|
|
|
32
|
+
# Look up db_shard from the corresponding MicroserviceModel.
|
|
33
|
+
def self._lookup_db_shard(name, scope:) # NOSONAR
|
|
34
|
+
json = Store.hget('openc3_microservices', name)
|
|
35
|
+
json ? (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i : 0
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Collect all unique db_shard values from MicroserviceModels.
|
|
39
|
+
def self._collect_db_shards(scope:)
|
|
40
|
+
db_shards = Set.new([0])
|
|
41
|
+
Store.hgetall('openc3_microservices').each do |name, json|
|
|
42
|
+
next if scope and name.split("__")[0] != scope
|
|
43
|
+
db_shards << (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i
|
|
44
|
+
end
|
|
45
|
+
db_shards
|
|
46
|
+
end
|
|
47
|
+
|
|
29
48
|
# NOTE: The following three class methods are used by the ModelController
|
|
30
49
|
# and are reimplemented to enable various Model class methods to work
|
|
31
50
|
def self.get(name:, scope:)
|
|
32
|
-
|
|
51
|
+
_db_sharded_get("#{scope}__#{PRIMARY_KEY}", name: name, scope: scope)
|
|
33
52
|
end
|
|
34
53
|
|
|
35
54
|
def self.names(scope:)
|
|
36
|
-
|
|
55
|
+
_db_sharded_names("#{scope}__#{PRIMARY_KEY}", scope: scope)
|
|
37
56
|
end
|
|
38
57
|
|
|
39
58
|
def self.all(scope:)
|
|
40
|
-
|
|
59
|
+
_db_sharded_all("#{scope}__#{PRIMARY_KEY}", scope: scope)
|
|
41
60
|
end
|
|
42
61
|
|
|
43
62
|
def initialize(
|
|
@@ -57,6 +76,14 @@ module OpenC3
|
|
|
57
76
|
@custom = custom
|
|
58
77
|
end
|
|
59
78
|
|
|
79
|
+
def create(update: false, force: false, queued: false, isoformat: false)
|
|
80
|
+
_db_sharded_create(self.class._db_shard_for_name(@name, scope: @scope, use_cache: true), update: update, force: force, queued: queued, isoformat: isoformat)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def destroy
|
|
84
|
+
_db_sharded_destroy(self.class._db_shard_for_name(@name, scope: @scope))
|
|
85
|
+
end
|
|
86
|
+
|
|
60
87
|
def as_json(*a)
|
|
61
88
|
{
|
|
62
89
|
'name' => @name,
|