cosmos 5.0.2 → 5.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/cosmos +183 -42
- data/data/config/microservice.yaml +47 -35
- data/data/config/plugins.yaml +10 -147
- data/data/config/target.yaml +70 -0
- data/data/config/tool.yaml +37 -31
- data/ext/cosmos/ext/cosmos_io/cosmos_io.c +14 -14
- data/ext/cosmos/ext/packet/packet.c +3 -3
- data/ext/cosmos/ext/structure/structure.c +31 -31
- data/lib/cosmos/api/api.rb +1 -25
- data/lib/cosmos/api/cmd_api.rb +17 -6
- data/lib/cosmos/api/config_api.rb +10 -4
- data/lib/cosmos/api/limits_api.rb +1 -1
- data/lib/cosmos/api/settings_api.rb +19 -7
- data/lib/cosmos/api/target_api.rb +2 -2
- data/lib/cosmos/api/tlm_api.rb +65 -41
- data/lib/cosmos/config/config_parser.rb +19 -22
- data/lib/cosmos/config/meta_config_parser.rb +1 -1
- data/lib/cosmos/conversions/generic_conversion.rb +2 -2
- data/lib/cosmos/conversions/polynomial_conversion.rb +5 -8
- data/lib/cosmos/conversions/segmented_polynomial_conversion.rb +26 -9
- data/lib/cosmos/io/json_drb.rb +5 -1
- data/lib/cosmos/logs/log_writer.rb +78 -29
- data/lib/cosmos/microservices/cleanup_microservice.rb +28 -29
- data/lib/cosmos/microservices/decom_microservice.rb +1 -1
- data/lib/cosmos/microservices/interface_microservice.rb +0 -16
- data/lib/cosmos/microservices/microservice.rb +3 -3
- data/lib/cosmos/microservices/reducer_microservice.rb +12 -10
- data/lib/cosmos/models/cvt_model.rb +6 -6
- data/lib/cosmos/models/gem_model.rb +9 -3
- data/lib/cosmos/models/info_model.rb +1 -1
- data/lib/cosmos/models/interface_model.rb +16 -7
- data/lib/cosmos/models/interface_status_model.rb +1 -1
- data/lib/cosmos/models/metadata_model.rb +69 -219
- data/lib/cosmos/models/metric_model.rb +2 -2
- data/lib/cosmos/models/microservice_model.rb +7 -4
- data/lib/cosmos/models/microservice_status_model.rb +1 -1
- data/lib/cosmos/models/model.rb +23 -16
- data/lib/cosmos/models/note_model.rb +122 -0
- data/lib/cosmos/models/ping_model.rb +2 -1
- data/lib/cosmos/models/plugin_model.rb +108 -48
- data/lib/cosmos/models/process_status_model.rb +1 -1
- data/lib/cosmos/models/scope_model.rb +10 -25
- data/lib/cosmos/models/settings_model.rb +55 -0
- data/lib/cosmos/models/sorted_model.rb +167 -0
- data/lib/cosmos/models/target_model.rb +143 -27
- data/lib/cosmos/models/tool_config_model.rb +38 -0
- data/lib/cosmos/models/tool_model.rb +9 -9
- data/lib/cosmos/models/widget_model.rb +11 -11
- data/lib/cosmos/operators/microservice_operator.rb +2 -1
- data/lib/cosmos/packets/packet.rb +24 -1
- data/lib/cosmos/packets/packet_config.rb +2 -2
- data/lib/cosmos/packets/packet_item.rb +57 -0
- data/lib/cosmos/packets/packet_item_limits.rb +14 -2
- data/lib/cosmos/packets/parsers/packet_item_parser.rb +1 -1
- data/lib/cosmos/packets/parsers/packet_parser.rb +1 -1
- data/lib/cosmos/packets/parsers/xtce_parser.rb +1 -1
- data/lib/cosmos/packets/structure.rb +30 -33
- data/lib/cosmos/packets/structure_item.rb +10 -1
- data/lib/cosmos/script/api_shared.rb +30 -25
- data/lib/cosmos/script/calendar.rb +37 -15
- data/lib/cosmos/script/commands.rb +5 -7
- data/lib/cosmos/script/script.rb +19 -39
- data/lib/cosmos/script/storage.rb +92 -105
- data/lib/cosmos/system/system.rb +2 -1
- data/lib/cosmos/tools/table_manager/table_config.rb +16 -1
- data/lib/cosmos/tools/table_manager/table_item.rb +1 -1
- data/lib/cosmos/tools/table_manager/table_manager_core.rb +213 -309
- data/lib/cosmos/top_level.rb +5 -1
- data/lib/cosmos/topics/autonomic_topic.rb +2 -2
- data/lib/cosmos/topics/calendar_topic.rb +1 -1
- data/lib/cosmos/topics/command_decom_topic.rb +35 -1
- data/lib/cosmos/topics/command_topic.rb +6 -4
- data/lib/cosmos/topics/config_topic.rb +68 -0
- data/lib/cosmos/topics/interface_topic.rb +8 -8
- data/lib/cosmos/topics/limits_event_topic.rb +5 -3
- data/lib/cosmos/topics/notifications_topic.rb +1 -1
- data/lib/cosmos/topics/router_topic.rb +9 -9
- data/lib/cosmos/topics/telemetry_decom_topic.rb +5 -1
- data/lib/cosmos/topics/telemetry_topic.rb +1 -1
- data/lib/cosmos/topics/timeline_topic.rb +1 -1
- data/lib/cosmos/topics/topic.rb +23 -8
- data/lib/cosmos/utilities/logger.rb +4 -3
- data/lib/cosmos/utilities/metric.rb +32 -26
- data/lib/cosmos/utilities/s3.rb +61 -0
- data/lib/cosmos/utilities/s3_file_cache.rb +12 -6
- data/lib/cosmos/utilities/store.rb +1 -0
- data/lib/cosmos/utilities/store_autoload.rb +25 -134
- data/lib/cosmos/version.rb +5 -4
- data/templates/plugin-template/plugin.gemspec +0 -2
- metadata +12 -10
- data/bin/xtce_converter +0 -92
- data/lib/cosmos/models/narrative_model.rb +0 -280
@@ -31,17 +31,17 @@ module Cosmos
|
|
31
31
|
received_count: packet.received_count,
|
32
32
|
stored: packet.stored,
|
33
33
|
buffer: packet.buffer(false) }
|
34
|
-
|
34
|
+
Topic.write_topic(topic, msg_hash)
|
35
35
|
end
|
36
36
|
|
37
37
|
# @param command [Hash] Command hash structure read to be written to a topic
|
38
38
|
def self.send_command(command, scope:)
|
39
39
|
ack_topic = "{#{scope}__ACKCMD}TARGET__#{command['target_name']}"
|
40
|
-
|
40
|
+
Topic.update_topic_offsets([ack_topic])
|
41
41
|
# Save the existing cmd_params Hash and JSON generate before writing to the topic
|
42
42
|
cmd_params = command['cmd_params']
|
43
43
|
command['cmd_params'] = JSON.generate(command['cmd_params'].as_json)
|
44
|
-
cmd_id =
|
44
|
+
cmd_id = Topic.write_topic("{#{scope}__CMD}TARGET__#{command['target_name']}", command, '*', 100)
|
45
45
|
# TODO: This timeout is fine for most but can we get the write_timeout from the interface here?
|
46
46
|
time = Time.now
|
47
47
|
while (Time.now - time) < COMMAND_ACK_TIMEOUT_S
|
@@ -66,7 +66,7 @@ module Cosmos
|
|
66
66
|
###########################################################################
|
67
67
|
|
68
68
|
def self.raise_hazardous_error(msg_hash, target_name, cmd_name, cmd_params)
|
69
|
-
_, description,
|
69
|
+
_, description, formatted = msg_hash["result"].split("\n")
|
70
70
|
# Create and populate a new HazardousError and raise it up
|
71
71
|
# The _cmd method in script/commands.rb rescues this and calls prompt_for_hazardous
|
72
72
|
error = HazardousError.new
|
@@ -74,6 +74,8 @@ module Cosmos
|
|
74
74
|
error.cmd_name = cmd_name
|
75
75
|
error.cmd_params = cmd_params
|
76
76
|
error.hazardous_description = description
|
77
|
+
error.formatted = formatted
|
78
|
+
|
77
79
|
# No Logger.info because the error is already logged by the Logger.info "Ack Received ...
|
78
80
|
raise error
|
79
81
|
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
# encoding: ascii-8bit
|
2
|
+
|
3
|
+
# Copyright 2022 Ball Aerospace & Technologies Corp.
|
4
|
+
# All Rights Reserved.
|
5
|
+
#
|
6
|
+
# This program is free software; you can modify and/or redistribute it
|
7
|
+
# under the terms of the GNU Affero General Public License
|
8
|
+
# as published by the Free Software Foundation; version 3 with
|
9
|
+
# attribution addendums as found in the LICENSE.txt
|
10
|
+
#
|
11
|
+
# This program is distributed in the hope that it will be useful,
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
14
|
+
# GNU Affero General Public License for more details.
|
15
|
+
#
|
16
|
+
# This program may also be used under the terms of a commercial or
|
17
|
+
# enterprise edition license of COSMOS if purchased from the
|
18
|
+
# copyright holder
|
19
|
+
|
20
|
+
require 'cosmos/topics/topic'
|
21
|
+
|
22
|
+
module Cosmos
|
23
|
+
class ConfigTopic < Topic
|
24
|
+
PRIMARY_KEY = "__CONFIG"
|
25
|
+
|
26
|
+
# Helper method to initialize the stream and ensure a consistent key
|
27
|
+
def self.initialize_stream(scope)
|
28
|
+
self.initialize_streams(["#{scope}#{PRIMARY_KEY}"])
|
29
|
+
end
|
30
|
+
|
31
|
+
# Write a configuration change to the topic
|
32
|
+
# @param config [Hash] Hash with required keys 'kind', 'name', 'type'
|
33
|
+
def self.write(config, scope:)
|
34
|
+
unless config.keys.include?(:kind)
|
35
|
+
raise "ConfigTopic error, required key kind: not given"
|
36
|
+
end
|
37
|
+
unless ['created', 'deleted'].include?(config[:kind])
|
38
|
+
raise "ConfigTopic error unknown kind: #{config[:kind]}"
|
39
|
+
end
|
40
|
+
unless config.keys.include?(:name)
|
41
|
+
raise "ConfigTopic error, required key name: not given"
|
42
|
+
end
|
43
|
+
unless config.keys.include?(:type)
|
44
|
+
raise "ConfigTopic error, required key type: not given"
|
45
|
+
end
|
46
|
+
# Limit the configuration topics to 1000 entries
|
47
|
+
Topic.write_topic("#{scope}#{PRIMARY_KEY}", config, '*', 1000)
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.read(offset = nil, count: 100, scope:)
|
51
|
+
topic = "#{scope}#{PRIMARY_KEY}"
|
52
|
+
if offset
|
53
|
+
result = Topic.read_topics([topic], [offset], nil, count)
|
54
|
+
if result.empty?
|
55
|
+
[] # We want to return an empty array rather than an empty hash
|
56
|
+
else
|
57
|
+
# result is a hash with the topic key followed by an array of results
|
58
|
+
# This returns just the array of arrays [[offset, hash], [offset, hash], ...]
|
59
|
+
result[topic]
|
60
|
+
end
|
61
|
+
else
|
62
|
+
result = Topic.get_newest_message(topic)
|
63
|
+
return [result] if result
|
64
|
+
return []
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -34,38 +34,38 @@ module Cosmos
|
|
34
34
|
|
35
35
|
def self.receive_commands(interface, scope:)
|
36
36
|
while true
|
37
|
-
|
37
|
+
Topic.read_topics(InterfaceTopic.topics(interface, scope: scope)) do |topic, msg_id, msg_hash, redis|
|
38
38
|
result = yield topic, msg_hash
|
39
39
|
ack_topic = topic.split("__")
|
40
40
|
ack_topic[1] = 'ACK' + ack_topic[1]
|
41
41
|
ack_topic = ack_topic.join("__")
|
42
|
-
|
42
|
+
Topic.write_topic(ack_topic, { 'result' => result, 'id' => msg_id }, '*', 100)
|
43
43
|
end
|
44
44
|
end
|
45
45
|
end
|
46
46
|
|
47
47
|
def self.write_raw(interface_name, data, scope:)
|
48
|
-
|
48
|
+
Topic.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'raw' => data }, '*', 100)
|
49
49
|
end
|
50
50
|
|
51
51
|
def self.connect_interface(interface_name, scope:)
|
52
|
-
|
52
|
+
Topic.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'connect' => 'true' }, '*', 100)
|
53
53
|
end
|
54
54
|
|
55
55
|
def self.disconnect_interface(interface_name, scope:)
|
56
|
-
|
56
|
+
Topic.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'disconnect' => 'true' }, '*', 100)
|
57
57
|
end
|
58
58
|
|
59
59
|
def self.start_raw_logging(interface_name, scope:)
|
60
|
-
|
60
|
+
Topic.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'log_raw' => 'true' }, '*', 100)
|
61
61
|
end
|
62
62
|
|
63
63
|
def self.stop_raw_logging(interface_name, scope:)
|
64
|
-
|
64
|
+
Topic.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'log_raw' => 'false' }, '*', 100)
|
65
65
|
end
|
66
66
|
|
67
67
|
def self.shutdown(interface, scope:)
|
68
|
-
|
68
|
+
Topic.write_topic("{#{scope}__CMD}INTERFACE__#{interface.name}", { 'shutdown' => 'true' }, '*', 100)
|
69
69
|
sleep 1 # Give some time for the interface to shutdown
|
70
70
|
InterfaceTopic.clear_topics(InterfaceTopic.topics(interface, scope: scope))
|
71
71
|
end
|
@@ -52,13 +52,13 @@ module Cosmos
|
|
52
52
|
raise "Invalid limits event type '#{event[:type]}'"
|
53
53
|
end
|
54
54
|
|
55
|
-
|
55
|
+
Topic.write_topic("#{scope}__cosmos_limits_events", event, '*', 1000)
|
56
56
|
end
|
57
57
|
|
58
58
|
def self.read(offset = nil, count: 100, scope:)
|
59
59
|
topic = "#{scope}__cosmos_limits_events"
|
60
60
|
if offset
|
61
|
-
result =
|
61
|
+
result = Topic.read_topics([topic], [offset], nil, count)
|
62
62
|
if result.empty?
|
63
63
|
[] # We want to return an empty array rather than an empty hash
|
64
64
|
else
|
@@ -67,7 +67,9 @@ module Cosmos
|
|
67
67
|
result[topic]
|
68
68
|
end
|
69
69
|
else
|
70
|
-
|
70
|
+
result = Topic.get_newest_message(topic)
|
71
|
+
return [result] if result
|
72
|
+
return []
|
71
73
|
end
|
72
74
|
end
|
73
75
|
|
@@ -22,7 +22,7 @@ require 'cosmos/topics/topic'
|
|
22
22
|
module Cosmos
|
23
23
|
class NotificationsTopic < Topic
|
24
24
|
def self.write_notification(notification, scope:)
|
25
|
-
|
25
|
+
Topic.write_topic("#{scope}__cosmos_notifications", notification)
|
26
26
|
end
|
27
27
|
end
|
28
28
|
end
|
@@ -36,13 +36,13 @@ module Cosmos
|
|
36
36
|
|
37
37
|
def self.receive_telemetry(router, scope:)
|
38
38
|
while true
|
39
|
-
|
39
|
+
Topic.read_topics(RouterTopic.topics(router, scope: scope)) do |topic, msg_id, msg_hash, redis|
|
40
40
|
result = yield topic, msg_hash
|
41
41
|
if /CMD}ROUTER/.match?(topic)
|
42
42
|
ack_topic = topic.split("__")
|
43
43
|
ack_topic[1] = 'ACK' + ack_topic[1]
|
44
44
|
ack_topic = ack_topic.join("__")
|
45
|
-
|
45
|
+
Topic.write_topic(ack_topic, { 'result' => result }, msg_id, 100)
|
46
46
|
end
|
47
47
|
end
|
48
48
|
end
|
@@ -51,33 +51,33 @@ module Cosmos
|
|
51
51
|
def self.route_command(packet, target_names, scope:)
|
52
52
|
if packet.identified?
|
53
53
|
topic = "{#{scope}__CMD}TARGET__#{packet.target_name}"
|
54
|
-
|
54
|
+
Topic.write_topic(topic, { 'target_name' => packet.target_name, 'cmd_name' => packet.packet_name, 'cmd_buffer' => packet.buffer(false) }, '*', 100)
|
55
55
|
elsif target_names.length == 1
|
56
56
|
topic = "{#{scope}__CMD}TARGET__#{target_names[0]}"
|
57
|
-
|
57
|
+
Topic.write_topic(topic, { 'target_name' => packet.target_name, 'cmd_name' => 'UNKNOWN', 'cmd_buffer' => packet.buffer(false) }, '*', 100)
|
58
58
|
else
|
59
59
|
raise "No route for command: #{packet.target_name} #{packet.packet_name}"
|
60
60
|
end
|
61
61
|
end
|
62
62
|
|
63
63
|
def self.connect_router(router_name, scope:)
|
64
|
-
|
64
|
+
Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'connect' => true }, '*', 100)
|
65
65
|
end
|
66
66
|
|
67
67
|
def self.disconnect_router(router_name, scope:)
|
68
|
-
|
68
|
+
Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'disconnect' => true }, '*', 100)
|
69
69
|
end
|
70
70
|
|
71
71
|
def self.start_raw_logging(router_name, scope:)
|
72
|
-
|
72
|
+
Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'log_raw' => 'true' }, '*', 100)
|
73
73
|
end
|
74
74
|
|
75
75
|
def self.stop_raw_logging(router_name, scope:)
|
76
|
-
|
76
|
+
Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'log_raw' => 'false' }, '*', 100)
|
77
77
|
end
|
78
78
|
|
79
79
|
def self.shutdown(router, scope:)
|
80
|
-
|
80
|
+
Topic.write_topic("{#{scope}__CMD}ROUTER__#{router.name}", { 'shutdown' => 'true' }, '*', 100)
|
81
81
|
sleep 1 # Give some time for the interface to shutdown
|
82
82
|
RouterTopic.clear_topics(RouterTopic.topics(router, scope: scope))
|
83
83
|
end
|
@@ -21,6 +21,10 @@ require 'cosmos/topics/topic'
|
|
21
21
|
|
22
22
|
module Cosmos
|
23
23
|
class TelemetryDecomTopic < Topic
|
24
|
+
def self.topics(scope:)
|
25
|
+
super(scope, 'DECOM')
|
26
|
+
end
|
27
|
+
|
24
28
|
def self.write_packet(packet, id: nil, scope:)
|
25
29
|
# Need to build a JSON hash of the decommutated data
|
26
30
|
# Support "downward typing"
|
@@ -39,7 +43,7 @@ module Cosmos
|
|
39
43
|
:received_count => packet.received_count,
|
40
44
|
:json_data => JSON.generate(json_hash.as_json),
|
41
45
|
}
|
42
|
-
|
46
|
+
Topic.write_topic("#{scope}__DECOM__{#{packet.target_name}}__#{packet.packet_name}", msg_hash, id)
|
43
47
|
# Also update the current value table with the latest decommutated data
|
44
48
|
CvtModel.set(json_hash, target_name: packet.target_name, packet_name: packet.packet_name, scope: scope)
|
45
49
|
end
|
@@ -30,7 +30,7 @@ module Cosmos
|
|
30
30
|
:received_count => packet.received_count,
|
31
31
|
:buffer => packet.buffer(false),
|
32
32
|
}
|
33
|
-
|
33
|
+
Topic.write_topic("#{scope}__TELEMETRY__{#{packet.target_name}}__#{packet.packet_name}", msg_hash)
|
34
34
|
end
|
35
35
|
end
|
36
36
|
end
|
data/lib/cosmos/topics/topic.rb
CHANGED
@@ -21,18 +21,33 @@ require 'cosmos/utilities/store'
|
|
21
21
|
|
22
22
|
module Cosmos
|
23
23
|
class Topic
|
24
|
-
|
25
|
-
|
24
|
+
if RUBY_VERSION < "3"
|
25
|
+
# Delegate all unknown class methods to delegate to the EphemeralStore
|
26
|
+
def self.method_missing(message, *args, &block)
|
27
|
+
EphemeralStore.public_send(message, *args, &block)
|
28
|
+
end
|
29
|
+
else
|
30
|
+
# Delegate all unknown class methods to delegate to the EphemeralStore
|
31
|
+
def self.method_missing(message, *args, **kwargs, &block)
|
32
|
+
EphemeralStore.public_send(message, *args, **kwargs, &block)
|
33
|
+
end
|
26
34
|
end
|
27
35
|
|
28
|
-
def self.
|
29
|
-
|
36
|
+
def self.clear_topics(topics, maxlen = 0)
|
37
|
+
topics.each { |topic| EphemeralStore.xtrim(topic, maxlen) }
|
30
38
|
end
|
31
39
|
|
32
|
-
def self.
|
33
|
-
|
34
|
-
|
35
|
-
|
40
|
+
def self.topics(scope, key)
|
41
|
+
EphemeralStore
|
42
|
+
.scan_each(match: "#{scope}__#{key}__*", type: 'stream', count: 100)
|
43
|
+
.to_a # Change the enumerator into an array
|
44
|
+
.uniq # Scan can return duplicates so ensure unique
|
45
|
+
.sort # Sort not entirely necessary but nice
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.get_cnt(topic)
|
49
|
+
_, packet = EphemeralStore.get_newest_message(topic)
|
50
|
+
packet ? packet["received_count"].to_i : 0
|
36
51
|
end
|
37
52
|
end
|
38
53
|
end
|
@@ -19,10 +19,11 @@
|
|
19
19
|
|
20
20
|
require 'cosmos/core_ext/class'
|
21
21
|
require 'cosmos/core_ext/time'
|
22
|
-
require 'cosmos/
|
22
|
+
require 'cosmos/topics/topic'
|
23
23
|
require 'socket'
|
24
24
|
require 'logger'
|
25
25
|
require 'time'
|
26
|
+
require 'json'
|
26
27
|
|
27
28
|
module Cosmos
|
28
29
|
# Supports different levels of logging and only writes if the level
|
@@ -173,11 +174,11 @@ module Cosmos
|
|
173
174
|
puts data.to_json if @stdout
|
174
175
|
unless @no_store
|
175
176
|
if scope
|
176
|
-
|
177
|
+
Topic.write_topic("#{scope}__cosmos_log_messages", data)
|
177
178
|
else
|
178
179
|
# The base cosmos_log_messages doesn't have an associated logger
|
179
180
|
# so it must be limited to prevent unbounded stream growth
|
180
|
-
|
181
|
+
Topic.write_topic("cosmos_log_messages", data, '*', 1000)
|
181
182
|
end
|
182
183
|
end
|
183
184
|
end
|
@@ -18,6 +18,7 @@
|
|
18
18
|
# copyright holder
|
19
19
|
|
20
20
|
require 'cosmos/models/metric_model'
|
21
|
+
require 'thread'
|
21
22
|
|
22
23
|
module Cosmos
|
23
24
|
class Metric
|
@@ -48,6 +49,7 @@ module Cosmos
|
|
48
49
|
@scope = scope
|
49
50
|
@microservice = microservice
|
50
51
|
@size = 5000
|
52
|
+
@mutex = Mutex.new
|
51
53
|
end
|
52
54
|
|
53
55
|
def add_sample(name:, value:, labels:)
|
@@ -69,15 +71,17 @@ module Cosmos
|
|
69
71
|
# the value is added to @items and the count of the value is increased
|
70
72
|
# if the count of the values exceed the size of the array it sets the
|
71
73
|
# count back to zero and the array will over write older data.
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
74
|
+
@mutex.synchronize do
|
75
|
+
key = "#{name}|" + labels.map { |k, v| "#{k}=#{v}" }.join(',')
|
76
|
+
if not @items.has_key?(key)
|
77
|
+
Logger.debug("new data for #{@scope}, #{key}")
|
78
|
+
@items[key] = { 'values' => Array.new(@size), 'count' => 0 }
|
79
|
+
end
|
80
|
+
count = @items[key]['count']
|
81
|
+
# Logger.info("adding data for #{@scope}, #{count} #{key}, #{value}")
|
82
|
+
@items[key]['values'][count] = value
|
83
|
+
@items[key]['count'] = count + 1 >= @size ? 0 : count + 1
|
76
84
|
end
|
77
|
-
count = @items[key]['count']
|
78
|
-
# Logger.info("adding data for #{@scope}, #{count} #{key}, #{value}")
|
79
|
-
@items[key]['values'][count] = value
|
80
|
-
@items[key]['count'] = count + 1 >= @size ? 0 : count + 1
|
81
85
|
end
|
82
86
|
|
83
87
|
def percentile(sorted_values, percentile)
|
@@ -106,24 +110,26 @@ module Cosmos
|
|
106
110
|
# array. to store the array as the value with the metric name again joined
|
107
111
|
# with the @microservice and @scope.
|
108
112
|
Logger.debug("#{@microservice} #{@scope} sending metrics to redis, #{@items.length}") if @items.length > 0
|
109
|
-
@
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
113
|
+
@mutex.synchronize do
|
114
|
+
@items.each do |key, values|
|
115
|
+
label_list = []
|
116
|
+
name, labels = key.split('|')
|
117
|
+
metric_labels = labels.nil? ? {} : labels.split(',').map { |x| x.split('=') }.map { |k, v| { k => v } }.reduce({}, :merge)
|
118
|
+
sorted_values = values['values'].compact.sort
|
119
|
+
for percentile_value in [10, 50, 90, 95, 99]
|
120
|
+
percentile_result = percentile(sorted_values, percentile_value)
|
121
|
+
labels = metric_labels.clone.merge({ 'scope' => @scope, 'microservice' => @microservice })
|
122
|
+
labels['percentile'] = percentile_value
|
123
|
+
labels['metric__value'] = percentile_result
|
124
|
+
label_list.append(labels)
|
125
|
+
end
|
126
|
+
begin
|
127
|
+
Logger.debug("sending metrics summary to redis key: #{@microservice}")
|
128
|
+
metric = MetricModel.new(name: @microservice, scope: @scope, metric_name: name, label_list: label_list)
|
129
|
+
metric.create(force: true)
|
130
|
+
rescue RuntimeError
|
131
|
+
Logger.error("failed attempt to update metric, #{key}, #{name} #{@scope}")
|
132
|
+
end
|
127
133
|
end
|
128
134
|
end
|
129
135
|
end
|
data/lib/cosmos/utilities/s3.rb
CHANGED
@@ -22,6 +22,67 @@ require 'cosmos/models/reducer_model'
|
|
22
22
|
|
23
23
|
module Cosmos
|
24
24
|
class S3Utilities
|
25
|
+
def self.list_files_before_time(bucket, prefix, time)
|
26
|
+
rubys3_client = Aws::S3::Client.new
|
27
|
+
oldest_list = []
|
28
|
+
total_size = 0
|
29
|
+
|
30
|
+
# Return nothing if bucket doesn't exist (it won't at the very beginning)
|
31
|
+
begin
|
32
|
+
rubys3_client.head_bucket(bucket: bucket)
|
33
|
+
rescue Aws::S3::Errors::NotFound
|
34
|
+
return total_size, oldest_list
|
35
|
+
end
|
36
|
+
|
37
|
+
# Get List of Packet Names - Assumes prefix gets us to a folder of packet names
|
38
|
+
token = nil
|
39
|
+
folder_list = []
|
40
|
+
while true
|
41
|
+
resp = rubys3_client.list_objects_v2({
|
42
|
+
bucket: bucket,
|
43
|
+
max_keys: 1000,
|
44
|
+
prefix: prefix,
|
45
|
+
delimiter: '/',
|
46
|
+
continuation_token: token
|
47
|
+
})
|
48
|
+
|
49
|
+
resp.common_prefixes.each do |item|
|
50
|
+
folder_list << item.prefix
|
51
|
+
end
|
52
|
+
break unless resp.is_truncated
|
53
|
+
token = resp.next_continuation_token
|
54
|
+
end
|
55
|
+
|
56
|
+
# Go through each folder and keep files that end before time
|
57
|
+
folder_list.each do |folder|
|
58
|
+
token = nil
|
59
|
+
next_folder = false
|
60
|
+
while true
|
61
|
+
resp = rubys3_client.list_objects_v2({
|
62
|
+
bucket: bucket,
|
63
|
+
max_keys: 1000,
|
64
|
+
prefix: folder,
|
65
|
+
continuation_token: token
|
66
|
+
})
|
67
|
+
resp.contents.each do |item|
|
68
|
+
t = item.key.split('__')[1]
|
69
|
+
file_end_time = Time.utc(t[0..3], t[4..5], t[6..7], t[8..9], t[10..11], t[12..13])
|
70
|
+
if file_end_time < time
|
71
|
+
oldest_list << item
|
72
|
+
total_size += item.size
|
73
|
+
else
|
74
|
+
next_folder = true
|
75
|
+
break
|
76
|
+
end
|
77
|
+
end
|
78
|
+
break if !resp.is_truncated or next_folder
|
79
|
+
|
80
|
+
token = resp.next_continuation_token
|
81
|
+
end
|
82
|
+
end
|
83
|
+
return total_size, oldest_list
|
84
|
+
end
|
85
|
+
|
25
86
|
def self.get_total_size_and_oldest_list(bucket, prefix, max_list_length = 10000)
|
26
87
|
rubys3_client = Aws::S3::Client.new
|
27
88
|
oldest_list = []
|
@@ -18,6 +18,7 @@
|
|
18
18
|
# copyright holder
|
19
19
|
|
20
20
|
require 'fileutils'
|
21
|
+
require 'tmpdir'
|
21
22
|
require 'cosmos'
|
22
23
|
require 'cosmos/utilities/s3'
|
23
24
|
|
@@ -48,7 +49,7 @@ class S3File
|
|
48
49
|
|
49
50
|
def retrieve
|
50
51
|
local_path = "#{S3FileCache.instance.cache_dir}/#{File.basename(@s3_path)}"
|
51
|
-
Cosmos::Logger.
|
52
|
+
Cosmos::Logger.debug "Retrieving #{@s3_path} from logs bucket"
|
52
53
|
@rubys3_client.get_object(bucket: "logs", key: @s3_path, response_target: local_path)
|
53
54
|
if File.exist?(local_path)
|
54
55
|
@size = File.size(local_path)
|
@@ -57,6 +58,7 @@ class S3File
|
|
57
58
|
rescue => err
|
58
59
|
@error = err
|
59
60
|
Cosmos::Logger.error "Failed to retrieve #{@s3_path}\n#{err.formatted}"
|
61
|
+
raise err
|
60
62
|
end
|
61
63
|
|
62
64
|
def reserve
|
@@ -165,11 +167,11 @@ class S3FileCache
|
|
165
167
|
end
|
166
168
|
|
167
169
|
# Create local file cache location
|
168
|
-
@cache_dir =
|
170
|
+
@cache_dir = Dir.mktmpdir
|
169
171
|
FileUtils.mkdir_p(@cache_dir)
|
170
|
-
|
171
|
-
|
172
|
-
|
172
|
+
at_exit do
|
173
|
+
FileUtils.remove_dir(@cache_dir, true)
|
174
|
+
end
|
173
175
|
|
174
176
|
@cached_files = S3FileCollection.new
|
175
177
|
|
@@ -178,7 +180,11 @@ class S3FileCache
|
|
178
180
|
file = @cached_files.get_next_to_retrieve
|
179
181
|
# Cosmos::Logger.debug "Next file: #{file}"
|
180
182
|
if file and (file.size + @cached_files.current_disk_usage()) <= @max_disk_usage
|
181
|
-
|
183
|
+
begin
|
184
|
+
file.retrieve
|
185
|
+
rescue
|
186
|
+
# Will be automatically retried
|
187
|
+
end
|
182
188
|
else
|
183
189
|
sleep(1)
|
184
190
|
end
|