glazed 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +3 -0
- data/.yardopts +5 -0
- data/CHANGELOG.md +18 -0
- data/LICENSE.txt +202 -0
- data/NOTICE +16 -0
- data/README.md +3 -0
- data/Rakefile +12 -0
- data/exe/glazed +8 -0
- data/lib/glazed/cli_session.rb +178 -0
- data/lib/glazed/collector.rb +113 -0
- data/lib/glazed/command/base.rb +146 -0
- data/lib/glazed/command/orchestrate.rb +78 -0
- data/lib/glazed/command/run.rb +51 -0
- data/lib/glazed/command/send_signal.rb +69 -0
- data/lib/glazed/command/status.rb +179 -0
- data/lib/glazed/command/version.rb +23 -0
- data/lib/glazed/command.rb +39 -0
- data/lib/glazed/command_result.rb +26 -0
- data/lib/glazed/dialect/actions.rb +95 -0
- data/lib/glazed/dialect/communication_config.rb +63 -0
- data/lib/glazed/dialect/interaction.rb +32 -0
- data/lib/glazed/dialect/run.rb +84 -0
- data/lib/glazed/dialect/runbook.rb +97 -0
- data/lib/glazed/dialect/sensitive_pattern.rb +31 -0
- data/lib/glazed/dialect.rb +12 -0
- data/lib/glazed/dialects/awplus.rb +84 -0
- data/lib/glazed/dialects/base.rb +246 -0
- data/lib/glazed/dialects/ios.rb +173 -0
- data/lib/glazed/dialects/junos.rb +78 -0
- data/lib/glazed/dialects/nec_ix.rb +71 -0
- data/lib/glazed/dialects/wlc.rb +56 -0
- data/lib/glazed/dialects.rb +65 -0
- data/lib/glazed/finder.rb +62 -0
- data/lib/glazed/must.rb +54 -0
- data/lib/glazed/orchestrator/policy.rb +184 -0
- data/lib/glazed/orchestrator.rb +288 -0
- data/lib/glazed/plugin_registry.rb +70 -0
- data/lib/glazed/proto.rb +20 -0
- data/lib/glazed/proto_json.rb +34 -0
- data/lib/glazed/redactor.rb +158 -0
- data/lib/glazed/runner.rb +176 -0
- data/lib/glazed/secret.rb +74 -0
- data/lib/glazed/secrets.rb +59 -0
- data/lib/glazed/shell_command.rb +90 -0
- data/lib/glazed/sources/base.rb +79 -0
- data/lib/glazed/sources/command.rb +67 -0
- data/lib/glazed/sources/inline.rb +56 -0
- data/lib/glazed/sources/json_files.rb +87 -0
- data/lib/glazed/sources.rb +105 -0
- data/lib/glazed/state_backends/base.rb +72 -0
- data/lib/glazed/state_backends/sqlite.rb +299 -0
- data/lib/glazed/state_backends.rb +75 -0
- data/lib/glazed/step_context.rb +65 -0
- data/lib/glazed/steps/base.rb +89 -0
- data/lib/glazed/steps/batch_shell.rb +57 -0
- data/lib/glazed/steps/fetch.rb +67 -0
- data/lib/glazed/steps/publish.rb +94 -0
- data/lib/glazed/steps/redact.rb +77 -0
- data/lib/glazed/steps/shell.rb +107 -0
- data/lib/glazed/steps.rb +83 -0
- data/lib/glazed/storages/base.rb +167 -0
- data/lib/glazed/storages/filesystem.rb +144 -0
- data/lib/glazed/storages/git.rb +181 -0
- data/lib/glazed/storages/github.rb +269 -0
- data/lib/glazed/storages/memory.rb +125 -0
- data/lib/glazed/storages.rb +88 -0
- data/lib/glazed/transports/base.rb +103 -0
- data/lib/glazed/transports/ssh.rb +177 -0
- data/lib/glazed/transports/telnet.rb +100 -0
- data/lib/glazed/transports.rb +28 -0
- data/lib/glazed/v1/config_pb.rb +25 -0
- data/lib/glazed/v1/inventory_pb.rb +27 -0
- data/lib/glazed/v1/orchestrator_pb.rb +27 -0
- data/lib/glazed/v1/redaction_pb.rb +23 -0
- data/lib/glazed/v1/secret_pb.rb +22 -0
- data/lib/glazed/v1/state_pb.rb +25 -0
- data/lib/glazed/v1/steps_pb.rb +25 -0
- data/lib/glazed/v1/storage_pb.rb +34 -0
- data/lib/glazed/v1/workflow_pb.rb +29 -0
- data/lib/glazed/validation.rb +31 -0
- data/lib/glazed/version.rb +9 -0
- data/lib/glazed/workflow.rb +70 -0
- data/lib/glazed.rb +18 -0
- metadata +245 -0
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require 'google/protobuf'
|
|
7
|
+
require 'google/protobuf/well_known_types'
|
|
8
|
+
require 'glazed'
|
|
9
|
+
require 'glazed/validation'
|
|
10
|
+
|
|
11
|
+
module Glazed
|
|
12
|
+
# Implementations of one kind of plugin, keyed by the full name of the
|
|
13
|
+
# settings message a config packs into a google.protobuf.Any.
|
|
14
|
+
#
|
|
15
|
+
# @example A plugin gem registering an inventory provider
|
|
16
|
+
# Glazed::Sources.register(GlazedNetbox::V1::NetboxInventory, GlazedNetbox::Source)
|
|
17
|
+
class PluginRegistry
|
|
18
|
+
# Raised when no implementation is registered for a settings type.
|
|
19
|
+
class UnknownPlugin < Glazed::Error; end
|
|
20
|
+
|
|
21
|
+
# @param kind [String] what the plugins implement, for error messages
|
|
22
|
+
#: (String kind) -> void
|
|
23
|
+
def initialize(kind)
|
|
24
|
+
@kind = kind #: String
|
|
25
|
+
@implementations = {} #: Hash[String, Module[untyped]]
|
|
26
|
+
@lock = Mutex.new #: Thread::Mutex
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Registers the implementation for a settings message type.
|
|
30
|
+
#
|
|
31
|
+
# @param settings_class [Class] generated message class of the settings
|
|
32
|
+
# @param implementation [Module] implementation built from those settings
|
|
33
|
+
# @return [void]
|
|
34
|
+
#: (singleton(Google::Protobuf::AbstractMessage) settings_class, Module[untyped] implementation) -> void
|
|
35
|
+
def register(settings_class, implementation)
|
|
36
|
+
@lock.synchronize { @implementations[settings_class.descriptor.name] = implementation }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Removes the implementation for a settings message type.
|
|
40
|
+
#
|
|
41
|
+
# @param settings_class [Class] generated message class of the settings
|
|
42
|
+
# @return [void]
|
|
43
|
+
#: (singleton(Google::Protobuf::AbstractMessage) settings_class) -> void
|
|
44
|
+
def unregister(settings_class)
|
|
45
|
+
@lock.synchronize { @implementations.delete(settings_class.descriptor.name) }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Finds the implementation for packed settings and unpacks them.
|
|
49
|
+
#
|
|
50
|
+
# @param any [Google::Protobuf::Any] packed settings
|
|
51
|
+
# @return [Array(Module, Google::Protobuf::MessageExts)] the implementation and the settings message
|
|
52
|
+
# @raise [UnknownPlugin] when no implementation is registered for the settings type
|
|
53
|
+
# @raise [InvalidMessage] when the settings break the rules of their schema
|
|
54
|
+
#: (Google::Protobuf::Any any) -> [Module[untyped], Google::Protobuf::MessageExts]
|
|
55
|
+
def resolve(any)
|
|
56
|
+
type = any.type_name.to_s
|
|
57
|
+
implementation = @lock.synchronize { @implementations[type] }
|
|
58
|
+
unless implementation
|
|
59
|
+
raise UnknownPlugin, "no #{@kind} plugin is registered for #{type}; require the gem providing it with `requires`"
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
settings_class = Google::Protobuf::DescriptorPool.generated_pool.lookup(type)&.msgclass
|
|
63
|
+
settings = settings_class && any.unpack(settings_class)
|
|
64
|
+
raise UnknownPlugin, "cannot unpack #{type} settings" unless settings.is_a?(Google::Protobuf::MessageExts)
|
|
65
|
+
|
|
66
|
+
Validation.validate!(settings, origin: type)
|
|
67
|
+
[implementation, settings]
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
data/lib/glazed/proto.rb
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require 'glazed/v1/config_pb'
|
|
7
|
+
require 'glazed/v1/inventory_pb'
|
|
8
|
+
require 'glazed/v1/orchestrator_pb'
|
|
9
|
+
require 'glazed/v1/redaction_pb'
|
|
10
|
+
require 'glazed/v1/secret_pb'
|
|
11
|
+
require 'glazed/v1/state_pb'
|
|
12
|
+
require 'glazed/v1/steps_pb'
|
|
13
|
+
require 'glazed/v1/storage_pb'
|
|
14
|
+
require 'glazed/v1/workflow_pb'
|
|
15
|
+
|
|
16
|
+
module Glazed
|
|
17
|
+
# Message classes generated from proto/glazed/v1 (Glazed::Proto::V1).
|
|
18
|
+
module Proto
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require 'google/protobuf'
|
|
7
|
+
|
|
8
|
+
module Glazed
|
|
9
|
+
# ProtoJSON codec that keeps proto field names (snake_case) on output and
|
|
10
|
+
# rejects unknown fields on input.
|
|
11
|
+
module ProtoJson
|
|
12
|
+
# Decodes ProtoJSON into a message.
|
|
13
|
+
#
|
|
14
|
+
# @param klass [Class<Google::Protobuf::AbstractMessage>] message class
|
|
15
|
+
# @param json [String] ProtoJSON document
|
|
16
|
+
# @return [Google::Protobuf::AbstractMessage] decoded message
|
|
17
|
+
# @raise [Google::Protobuf::ParseError] when the document is malformed or has unknown fields
|
|
18
|
+
# @example
|
|
19
|
+
# Glazed::ProtoJson.load(Glazed::Proto::V1::Node, '{"name": "rt1"}')
|
|
20
|
+
#: [M < Google::Protobuf::AbstractMessage] (singleton(Google::Protobuf::AbstractMessage) & Class[M] klass, String json) -> M
|
|
21
|
+
def self.load(klass, json)
|
|
22
|
+
klass.decode_json(json)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Encodes a message into ProtoJSON.
|
|
26
|
+
#
|
|
27
|
+
# @param message [Google::Protobuf::AbstractMessage] message to encode
|
|
28
|
+
# @return [String] ProtoJSON document
|
|
29
|
+
#: (Google::Protobuf::AbstractMessage message) -> String
|
|
30
|
+
def self.dump(message)
|
|
31
|
+
message.class.encode_json(message, preserve_proto_fieldnames: true)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require 'openssl'
|
|
7
|
+
require 'glazed'
|
|
8
|
+
require 'glazed/dialect/sensitive_pattern'
|
|
9
|
+
require 'glazed/proto'
|
|
10
|
+
require 'glazed/secret'
|
|
11
|
+
|
|
12
|
+
module Glazed
|
|
13
|
+
# Hides secrets located by a dialect's sensitive patterns.
|
|
14
|
+
#
|
|
15
|
+
# Only the secret within each match is replaced, by +[~REDACTED~]+ in :remove
|
|
16
|
+
# mode or +[~REDACTED:dgst=...~]+ in :hmac mode. The digest is a truncated
|
|
17
|
+
# HMAC-SHA256 of the secret in unpadded base64url, so a changed secret shows
|
|
18
|
+
# up in diffs without being revealed.
|
|
19
|
+
#
|
|
20
|
+
# @example
|
|
21
|
+
# Glazed::Redactor.new(patterns: Glazed::Dialects::Ios.sensitive_patterns, mode: :remove).call(config)
|
|
22
|
+
class Redactor
|
|
23
|
+
# Raised when the redaction settings are unusable.
|
|
24
|
+
class InvalidConfiguration < Glazed::Error; end
|
|
25
|
+
|
|
26
|
+
# Replacement in :remove mode.
|
|
27
|
+
REMOVED = '[~REDACTED~]' #: String
|
|
28
|
+
# Bytes of the HMAC kept in :hmac mode.
|
|
29
|
+
HMAC_BYTES = 8 #: Integer
|
|
30
|
+
|
|
31
|
+
# Supported modes.
|
|
32
|
+
MODES = %i[remove hmac].freeze #: Array[Symbol]
|
|
33
|
+
|
|
34
|
+
# @param patterns [Array<Dialect::SensitivePattern>] where secrets appear
|
|
35
|
+
# @param mode [Symbol] :remove or :hmac
|
|
36
|
+
# @param key [String, nil] HMAC key, required for :hmac
|
|
37
|
+
# @raise [InvalidConfiguration] when the mode is unknown or :hmac has no key
|
|
38
|
+
#: (patterns: Array[Dialect::SensitivePattern], mode: Symbol, ?key: String?) -> void
|
|
39
|
+
def initialize(patterns:, mode:, key: nil)
|
|
40
|
+
raise InvalidConfiguration, "unknown redaction mode #{mode.inspect}" unless MODES.include?(mode)
|
|
41
|
+
raise InvalidConfiguration, 'HMAC redaction requires a key' if mode == :hmac && (key.nil? || key.empty?)
|
|
42
|
+
|
|
43
|
+
@patterns = patterns #: Array[Dialect::SensitivePattern]
|
|
44
|
+
@mode = mode #: Symbol
|
|
45
|
+
@key = key #: String?
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Builds a redactor from settings.
|
|
49
|
+
#
|
|
50
|
+
# @param redaction [Proto::V1::Redaction] redaction settings
|
|
51
|
+
# @param patterns [Array<Dialect::SensitivePattern>] where secrets appear
|
|
52
|
+
# @return [Redactor] the redactor
|
|
53
|
+
# @raise [InvalidConfiguration] when the mode is unknown, or the HMAC key is missing or not base64
|
|
54
|
+
# @raise [Secret::Unavailable] when the HMAC key cannot be read
|
|
55
|
+
#: (Proto::V1::Redaction redaction, patterns: Array[Dialect::SensitivePattern]) -> Redactor
|
|
56
|
+
def self.from_proto(redaction, patterns:)
|
|
57
|
+
case redaction.mode
|
|
58
|
+
when :MODE_UNSPECIFIED, :MODE_REMOVE then new(patterns:, mode: :remove)
|
|
59
|
+
when :MODE_HMAC then new(patterns:, mode: :hmac, key: hmac_key(redaction))
|
|
60
|
+
else raise InvalidConfiguration, "unknown redaction mode #{redaction.mode.inspect}"
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# @param redaction [Proto::V1::Redaction] redaction settings
|
|
65
|
+
# @return [String, nil] the HMAC key, decoded from base64
|
|
66
|
+
# @raise [InvalidConfiguration] when the key is not base64
|
|
67
|
+
# @raise [Secret::Unavailable] when the key cannot be read
|
|
68
|
+
#: (Proto::V1::Redaction redaction) -> String?
|
|
69
|
+
private_class_method def self.hmac_key(redaction)
|
|
70
|
+
encoded = Secret.maybe(redaction.hmac_key)&.value or return nil
|
|
71
|
+
encoded.unpack1('m0')
|
|
72
|
+
rescue ArgumentError
|
|
73
|
+
raise InvalidConfiguration, 'hmac_key is not base64'
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# @return [Symbol] :remove or :hmac
|
|
77
|
+
#: Symbol
|
|
78
|
+
attr_reader :mode
|
|
79
|
+
|
|
80
|
+
# Redacts a configuration.
|
|
81
|
+
#
|
|
82
|
+
# @param text [String] configuration text
|
|
83
|
+
# @return [String] text with every located secret replaced
|
|
84
|
+
#: (String text) -> String
|
|
85
|
+
def call(text)
|
|
86
|
+
result = text.dup
|
|
87
|
+
secret_ranges(text).reverse_each do |start, finish|
|
|
88
|
+
result[start...finish] = replacement(text[start...finish] || '')
|
|
89
|
+
end
|
|
90
|
+
result
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# @param text [String] configuration text
|
|
94
|
+
# @return [Array<Array(Integer, Integer)>] non-overlapping secret start and end offsets, in order
|
|
95
|
+
#: (String text) -> Array[[Integer, Integer]]
|
|
96
|
+
private def secret_ranges(text)
|
|
97
|
+
ranges = @patterns.flat_map do |sensitive|
|
|
98
|
+
matches(sensitive.pattern, text).filter_map do |match|
|
|
99
|
+
case (secret = sensitive.determiner.call(match))
|
|
100
|
+
when Range
|
|
101
|
+
start = secret.begin
|
|
102
|
+
finish = secret.exclude_end? ? secret.end : secret.end + 1
|
|
103
|
+
[start, finish] if finish > start
|
|
104
|
+
when String then locate(match, secret) unless secret.empty?
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
kept = [] #: Array[[Integer, Integer]]
|
|
109
|
+
ranges.sort_by { |start, finish| [start, -finish] }.each do |start, finish|
|
|
110
|
+
last = kept.last
|
|
111
|
+
kept << [start, finish] if last.nil? || start >= last[1]
|
|
112
|
+
end
|
|
113
|
+
kept
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# @param pattern [Regexp] pattern to scan for
|
|
117
|
+
# @param text [String] text to scan
|
|
118
|
+
# @return [Array<MatchData>] every match
|
|
119
|
+
#: (Regexp pattern, String text) -> Array[MatchData]
|
|
120
|
+
private def matches(pattern, text)
|
|
121
|
+
found = [] #: Array[MatchData]
|
|
122
|
+
position = 0
|
|
123
|
+
while (match = pattern.match(text, position))
|
|
124
|
+
found << match
|
|
125
|
+
position = match.end(0) > match.begin(0) ? match.end(0) : match.end(0) + 1
|
|
126
|
+
break if position > text.size
|
|
127
|
+
end
|
|
128
|
+
found
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# Finds the secret in a match, preferring the last capture group equal to it,
|
|
132
|
+
# as secrets follow the names they belong to.
|
|
133
|
+
#
|
|
134
|
+
# @param match [MatchData] the match
|
|
135
|
+
# @param secret [String] secret returned by the determiner
|
|
136
|
+
# @return [Array(Integer, Integer), nil] start and end offsets of the secret, or nil when it is not in the match
|
|
137
|
+
#: (MatchData match, String secret) -> [Integer, Integer]?
|
|
138
|
+
private def locate(match, secret)
|
|
139
|
+
group = (match.size - 1).downto(1).find { match[it] == secret }
|
|
140
|
+
start = if group
|
|
141
|
+
match.begin(group)
|
|
142
|
+
elsif (offset = match[0]&.rindex(secret))
|
|
143
|
+
match.begin(0) + offset
|
|
144
|
+
end
|
|
145
|
+
[start, start + secret.size] if start
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# @param secret [String] secret to hide
|
|
149
|
+
# @return [String] its replacement
|
|
150
|
+
#: (String secret) -> String
|
|
151
|
+
private def replacement(secret)
|
|
152
|
+
return REMOVED unless @mode == :hmac
|
|
153
|
+
|
|
154
|
+
digest = OpenSSL::HMAC.digest('SHA256', @key.to_s, secret).byteslice(0, HMAC_BYTES)
|
|
155
|
+
"[~REDACTED:dgst=#{[digest].pack('m0').tr('+/', '-_').delete('=')}~]"
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require 'glazed'
|
|
7
|
+
require 'glazed/proto'
|
|
8
|
+
require 'glazed/steps'
|
|
9
|
+
require 'glazed/workflow'
|
|
10
|
+
|
|
11
|
+
module Glazed
|
|
12
|
+
# Runs a workflow in this process. Consecutive per-node steps run for each
|
|
13
|
+
# node on a thread pool; a node whose step fails is reported and left out of
|
|
14
|
+
# the rest of the workflow. Batch steps run with the remaining nodes, and
|
|
15
|
+
# their failures propagate.
|
|
16
|
+
#
|
|
17
|
+
# @example
|
|
18
|
+
# result = Glazed::Runner.new(workflow:, jobs: 4).call(nodes)
|
|
19
|
+
# result.failures.each { warn "#{it.node.name}: #{it.error.message}" }
|
|
20
|
+
class Runner
|
|
21
|
+
#: type event_value = String | Integer | bool | Array[String] | nil
|
|
22
|
+
#: type listener = ^(Hash[Symbol, Runner::event_value] event) -> void
|
|
23
|
+
|
|
24
|
+
# A node that failed a step.
|
|
25
|
+
class Failure
|
|
26
|
+
# @param node [Proto::V1::Node] the node
|
|
27
|
+
# @param step [String] name of the failed step
|
|
28
|
+
# @param error [StandardError] the error
|
|
29
|
+
#: (node: Proto::V1::Node, step: String, error: StandardError) -> void
|
|
30
|
+
def initialize(node:, step:, error:)
|
|
31
|
+
@node = node #: Proto::V1::Node
|
|
32
|
+
@step = step #: String
|
|
33
|
+
@error = error #: StandardError
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# @return [Proto::V1::Node] the node
|
|
37
|
+
#: Proto::V1::Node
|
|
38
|
+
attr_reader :node
|
|
39
|
+
# @return [String] name of the failed step
|
|
40
|
+
#: String
|
|
41
|
+
attr_reader :step
|
|
42
|
+
# @return [StandardError] the error
|
|
43
|
+
#: StandardError
|
|
44
|
+
attr_reader :error
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Outcome of a run.
|
|
48
|
+
class Result
|
|
49
|
+
# @param states [Array<Proto::V1::NodeState>] states of the nodes that completed every step
|
|
50
|
+
# @param failures [Array<Failure>] nodes that failed a step
|
|
51
|
+
#: (states: Array[Proto::V1::NodeState], failures: Array[Failure]) -> void
|
|
52
|
+
def initialize(states:, failures:)
|
|
53
|
+
@states = states #: Array[Proto::V1::NodeState]
|
|
54
|
+
@failures = failures #: Array[Failure]
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# @return [Array<Proto::V1::NodeState>] states of the nodes that completed every step
|
|
58
|
+
#: Array[Proto::V1::NodeState]
|
|
59
|
+
attr_reader :states
|
|
60
|
+
# @return [Array<Failure>] nodes that failed a step
|
|
61
|
+
#: Array[Failure]
|
|
62
|
+
attr_reader :failures
|
|
63
|
+
|
|
64
|
+
# @return [Array<Proto::V1::NodeState>] states with at least one artifact whose write changed stored content
|
|
65
|
+
#: () -> Array[Proto::V1::NodeState]
|
|
66
|
+
def changed = states.select { |state| state.artifacts.values.any?(&:changed) }
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# @param workflow [Workflow] the workflow
|
|
70
|
+
# @param jobs [Integer] nodes processed in parallel
|
|
71
|
+
# @param listener [Proc] receives an event hash per completed or failed step
|
|
72
|
+
#: (workflow: Workflow, jobs: Integer, ?listener: Runner::listener) -> void
|
|
73
|
+
def initialize(workflow:, jobs:, listener: ->(_event) {})
|
|
74
|
+
@workflow = workflow #: Workflow
|
|
75
|
+
@jobs = jobs #: Integer
|
|
76
|
+
@listener = listener #: Runner::listener
|
|
77
|
+
@lock = Mutex.new #: Thread::Mutex
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# @param nodes [Array<Proto::V1::Node>] nodes to run against
|
|
81
|
+
# @return [Result] the outcome
|
|
82
|
+
# @raise [Glazed::Error] when a batch step fails
|
|
83
|
+
#: (Array[Proto::V1::Node] nodes) -> Result
|
|
84
|
+
def call(nodes)
|
|
85
|
+
states = nodes.map { Proto::V1::NodeState.new(node: it) }
|
|
86
|
+
failures = [] #: Array[Failure]
|
|
87
|
+
@workflow.steps.chunk_while { |a, b| a.is_a?(Steps::NodeStep) && b.is_a?(Steps::NodeStep) }.each do |chunk|
|
|
88
|
+
first = chunk.first
|
|
89
|
+
states = case first
|
|
90
|
+
when Steps::BatchStep then run_batch(first, states)
|
|
91
|
+
else run_per_node(chunk.grep(Steps::NodeStep), states, failures)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
Result.new(states:, failures:)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# @param step [Steps::BatchStep] the step
|
|
98
|
+
# @param states [Array<Proto::V1::NodeState>] node states
|
|
99
|
+
# @return [Array<Proto::V1::NodeState>] node states after the step
|
|
100
|
+
#: (Steps::BatchStep step, Array[Proto::V1::NodeState] states) -> Array[Proto::V1::NodeState]
|
|
101
|
+
private def run_batch(step, states)
|
|
102
|
+
batch = step.call(Proto::V1::BatchState.new(nodes: states))
|
|
103
|
+
after = batch.nodes.to_a
|
|
104
|
+
if step.outputs.empty?
|
|
105
|
+
notify(level: 'info', step: step.name, nodes: after.size, message: 'done')
|
|
106
|
+
else
|
|
107
|
+
notify(level: 'info', step: step.name, nodes: after.size, changed: after.count { changed?(it, step) }, message: 'done')
|
|
108
|
+
end
|
|
109
|
+
after
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# @param steps [Array<Steps::NodeStep>] consecutive per-node steps
|
|
113
|
+
# @param states [Array<Proto::V1::NodeState>] node states
|
|
114
|
+
# @param failures [Array<Failure>] collects failed nodes
|
|
115
|
+
# @return [Array<Proto::V1::NodeState>] states of the nodes that completed the steps, in input order
|
|
116
|
+
#: (Array[Steps::NodeStep] steps, Array[Proto::V1::NodeState] states, Array[Failure] failures) -> Array[Proto::V1::NodeState]
|
|
117
|
+
private def run_per_node(steps, states, failures)
|
|
118
|
+
queue = Thread::Queue.new(states.each_with_index.to_a)
|
|
119
|
+
queue.close
|
|
120
|
+
results = {} #: Hash[Integer, Proto::V1::NodeState]
|
|
121
|
+
workers = Array.new([@jobs, states.size].min) do
|
|
122
|
+
Thread.new do
|
|
123
|
+
while (item = queue.pop)
|
|
124
|
+
state, index = item
|
|
125
|
+
Thread.current.name = state.node&.name
|
|
126
|
+
result = run_node(steps, state, failures)
|
|
127
|
+
@lock.synchronize { results[index] = result } if result
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
begin
|
|
132
|
+
workers.each(&:join)
|
|
133
|
+
rescue Interrupt
|
|
134
|
+
workers.select(&:alive?).each do |worker|
|
|
135
|
+
notify(level: 'error', node: worker.name, message: 'interrupted', backtrace: worker.backtrace || [])
|
|
136
|
+
end
|
|
137
|
+
raise
|
|
138
|
+
end
|
|
139
|
+
results.sort.map { |_, state| state }
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# @param steps [Array<Steps::NodeStep>] consecutive per-node steps
|
|
143
|
+
# @param state [Proto::V1::NodeState] the node's state
|
|
144
|
+
# @param failures [Array<Failure>] collects the node when it fails
|
|
145
|
+
# @return [Proto::V1::NodeState, nil] the final state, or nil when a step failed
|
|
146
|
+
#: (Array[Steps::NodeStep] steps, Proto::V1::NodeState state, Array[Failure] failures) -> Proto::V1::NodeState?
|
|
147
|
+
private def run_node(steps, state, failures)
|
|
148
|
+
node = state.node or raise ArgumentError, 'node state has no node'
|
|
149
|
+
steps.inject(state) do |current, step|
|
|
150
|
+
begin
|
|
151
|
+
current = step.call(current)
|
|
152
|
+
rescue StandardError => e
|
|
153
|
+
# One unreachable or misbehaving device must not stop the others.
|
|
154
|
+
@lock.synchronize { failures << Failure.new(node:, step: step.name, error: e) }
|
|
155
|
+
notify(level: 'error', step: step.name, node: node.name, error: e.class.name, message: e.message)
|
|
156
|
+
return nil
|
|
157
|
+
end
|
|
158
|
+
notify(level: 'info', step: step.name, node: node.name, changed: changed?(current, step), message: 'done')
|
|
159
|
+
current
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# @param state [Proto::V1::NodeState] state after the step
|
|
164
|
+
# @param step [Steps::Base] the step
|
|
165
|
+
# @return [Boolean] whether any artifact the step produced changed stored content
|
|
166
|
+
#: (Proto::V1::NodeState state, Steps::Base step) -> bool
|
|
167
|
+
private def changed?(state, step) = step.outputs.any? { state.artifacts[it]&.changed }
|
|
168
|
+
|
|
169
|
+
# @param event [Hash{Symbol => String, Integer, Boolean, nil}] event fields
|
|
170
|
+
# @return [void]
|
|
171
|
+
#: (**Runner::event_value event) -> void
|
|
172
|
+
private def notify(**event)
|
|
173
|
+
@lock.synchronize { @listener.call(event) }
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
end
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require 'glazed'
|
|
7
|
+
require 'glazed/proto'
|
|
8
|
+
require 'glazed/secrets'
|
|
9
|
+
|
|
10
|
+
module Glazed
|
|
11
|
+
# Secret resolved from a {Proto::V1::SecretRef} on each access. Its value
|
|
12
|
+
# never appears in #inspect or #to_s.
|
|
13
|
+
class Secret
|
|
14
|
+
# Raised when the referenced value cannot be read.
|
|
15
|
+
class Unavailable < Glazed::Error; end
|
|
16
|
+
|
|
17
|
+
# @param ref [Proto::V1::SecretRef] reference to resolve
|
|
18
|
+
#: (Proto::V1::SecretRef ref) -> void
|
|
19
|
+
def initialize(ref)
|
|
20
|
+
@ref = ref #: Proto::V1::SecretRef
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Wraps a reference that may be absent.
|
|
24
|
+
#
|
|
25
|
+
# @param ref [Proto::V1::SecretRef, nil] reference to wrap
|
|
26
|
+
# @return [Secret, nil] the secret, or nil when ref is nil
|
|
27
|
+
#: (Proto::V1::SecretRef? ref) -> Secret?
|
|
28
|
+
def self.maybe(ref)
|
|
29
|
+
ref && new(ref)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# @return [Proto::V1::SecretRef] the wrapped reference
|
|
33
|
+
#: Proto::V1::SecretRef
|
|
34
|
+
attr_reader :ref
|
|
35
|
+
|
|
36
|
+
# Reads the secret value.
|
|
37
|
+
#
|
|
38
|
+
# @return [String] the value
|
|
39
|
+
# @raise [Unavailable] when the environment variable or file is missing, or no source is set
|
|
40
|
+
#: () -> String
|
|
41
|
+
def value
|
|
42
|
+
case ref.source
|
|
43
|
+
when :value then ref.value
|
|
44
|
+
when :env then ENV.fetch(ref.env) { raise Unavailable, "environment variable #{ref.env} is not set" }
|
|
45
|
+
when :file then read_file(ref.file)
|
|
46
|
+
when :plugin then Secrets.build(ref.plugin || raise(Unavailable, 'secret plugin has no settings')).value
|
|
47
|
+
else raise Unavailable, 'secret has no source'
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# @return [String] description naming the source but not the value
|
|
52
|
+
#: () -> String
|
|
53
|
+
def inspect
|
|
54
|
+
case ref.source
|
|
55
|
+
when :env then "#<#{self.class} env:#{ref.env}>"
|
|
56
|
+
when :file then "#<#{self.class} file:#{ref.file}>"
|
|
57
|
+
when :plugin then "#<#{self.class} plugin:#{ref.plugin&.type_name}>"
|
|
58
|
+
else "#<#{self.class} value>"
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
alias to_s inspect
|
|
63
|
+
|
|
64
|
+
# @param path [String] file to read
|
|
65
|
+
# @return [String] file contents without one trailing newline
|
|
66
|
+
# @raise [Unavailable] when the file cannot be read
|
|
67
|
+
#: (String path) -> String
|
|
68
|
+
private def read_file(path)
|
|
69
|
+
File.read(path).delete_suffix("\n")
|
|
70
|
+
rescue SystemCallError => e
|
|
71
|
+
raise Unavailable, "cannot read secret file #{path}: #{e.message}"
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require 'google/protobuf'
|
|
7
|
+
require 'glazed'
|
|
8
|
+
require 'glazed/must'
|
|
9
|
+
require 'glazed/plugin_registry'
|
|
10
|
+
|
|
11
|
+
module Glazed
|
|
12
|
+
# Secret sources implemented by plugins, used by `plugin` secret references.
|
|
13
|
+
module Secrets
|
|
14
|
+
# A plugin source of one secret value, built from its settings message.
|
|
15
|
+
class Base
|
|
16
|
+
# Builds the source from its settings message.
|
|
17
|
+
#
|
|
18
|
+
# @param settings [Google::Protobuf::MessageExts] settings unpacked from the reference
|
|
19
|
+
# @return [Base] the source
|
|
20
|
+
#: (Google::Protobuf::MessageExts settings) -> Base
|
|
21
|
+
def self.from_proto(settings) = raise NotImplementedError
|
|
22
|
+
|
|
23
|
+
# Reads the secret, each time it is used.
|
|
24
|
+
#
|
|
25
|
+
# @return [String] the secret value
|
|
26
|
+
#: () -> String
|
|
27
|
+
def value = raise NotImplementedError
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Secret sources implemented by plugins, by settings message type.
|
|
31
|
+
PLUGINS = PluginRegistry.new('secret source') #: PluginRegistry
|
|
32
|
+
|
|
33
|
+
# Registers a plugin secret source, selected by `plugin` references whose
|
|
34
|
+
# settings are of the given type.
|
|
35
|
+
#
|
|
36
|
+
# @param settings_class [Class] generated message class of the source settings
|
|
37
|
+
# @param implementation [Class<Base>] source built with {Base.from_proto}
|
|
38
|
+
# @return [void]
|
|
39
|
+
# @example
|
|
40
|
+
# Glazed::Secrets.register(GlazedAws::V1::SsmParameter, GlazedAws::SsmParameterSecret)
|
|
41
|
+
#: (singleton(Google::Protobuf::AbstractMessage) settings_class, singleton(Base) implementation) -> void
|
|
42
|
+
def self.register(settings_class, implementation)
|
|
43
|
+
PLUGINS.register(settings_class, implementation)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Builds the source for packed settings.
|
|
47
|
+
#
|
|
48
|
+
# @param packed [Google::Protobuf::Any] settings from a secret reference
|
|
49
|
+
# @return [Base] the source
|
|
50
|
+
# @raise [PluginRegistry::UnknownPlugin] when no plugin is registered for the settings type
|
|
51
|
+
#: (Google::Protobuf::Any packed) -> Base
|
|
52
|
+
def self.build(packed)
|
|
53
|
+
implementation, settings = PLUGINS.resolve(packed)
|
|
54
|
+
raise Must::Violation, "#{implementation} is not a secret source" unless implementation.is_a?(Class) && implementation < Base
|
|
55
|
+
|
|
56
|
+
implementation.from_proto(settings)
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require 'open3'
|
|
7
|
+
require 'glazed'
|
|
8
|
+
|
|
9
|
+
module Glazed
|
|
10
|
+
# Runs a command line with /bin/sh -c, feeding stdin and capturing stdout.
|
|
11
|
+
#
|
|
12
|
+
# @example
|
|
13
|
+
# Glazed::ShellCommand.new(run: 'sort', env: {}, stdin: "b\na\n", timeout: 10).call #=> "a\nb\n"
|
|
14
|
+
class ShellCommand
|
|
15
|
+
# Raised when the command exits non-zero or is killed by a signal.
|
|
16
|
+
class Failed < Glazed::Error; end
|
|
17
|
+
|
|
18
|
+
# Bytes of stderr kept in error messages.
|
|
19
|
+
STDERR_TAIL = 1000 #: Integer
|
|
20
|
+
# Seconds between SIGTERM and SIGKILL when the command overruns.
|
|
21
|
+
KILL_GRACE = 5 #: Integer
|
|
22
|
+
|
|
23
|
+
# @param run [String] command line
|
|
24
|
+
# @param env [Hash{String => String}] environment added to the inherited one
|
|
25
|
+
# @param stdin [String] data written to stdin
|
|
26
|
+
# @param timeout [Numeric] seconds before the command is killed
|
|
27
|
+
#: (run: String, env: Hash[String, String], stdin: String, timeout: Numeric) -> void
|
|
28
|
+
def initialize(run:, env:, stdin:, timeout:)
|
|
29
|
+
@run = run #: String
|
|
30
|
+
@env = env #: Hash[String, String]
|
|
31
|
+
@stdin = stdin #: String
|
|
32
|
+
@timeout = timeout #: Numeric
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Runs the command to completion.
|
|
36
|
+
#
|
|
37
|
+
# @return [String] stdout
|
|
38
|
+
# @raise [Failed] when the command exits non-zero or is killed by a signal
|
|
39
|
+
# @raise [TimeoutError] when the command does not finish in time; its process group is killed
|
|
40
|
+
#: () -> String
|
|
41
|
+
def call
|
|
42
|
+
stdin, stdout, stderr, waiter = Open3.popen3(@env, '/bin/sh', '-c', @run, pgroup: true)
|
|
43
|
+
begin
|
|
44
|
+
writer = Thread.new do
|
|
45
|
+
stdin.write(@stdin)
|
|
46
|
+
rescue Errno::EPIPE
|
|
47
|
+
# The command exited or closed stdin without reading all of it.
|
|
48
|
+
nil
|
|
49
|
+
ensure
|
|
50
|
+
stdin.close
|
|
51
|
+
end
|
|
52
|
+
out = Thread.new { stdout.read }
|
|
53
|
+
err = Thread.new { stderr.read }
|
|
54
|
+
|
|
55
|
+
unless waiter.join(@timeout.to_f)
|
|
56
|
+
terminate(waiter)
|
|
57
|
+
raise TimeoutError, "#{@run.inspect} did not finish in #{@timeout}s"
|
|
58
|
+
end
|
|
59
|
+
writer.join
|
|
60
|
+
status = waiter.value #: Process::Status
|
|
61
|
+
output = out.value.to_s
|
|
62
|
+
return output if status.success?
|
|
63
|
+
|
|
64
|
+
raise Failed, "#{@run.inspect} #{describe(status)}: #{err.value.to_s[-STDERR_TAIL..] || err.value}"
|
|
65
|
+
ensure
|
|
66
|
+
[stdin, stdout, stderr].each { it.close unless it.closed? }
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Stops the command and everything it started, gracefully first.
|
|
71
|
+
#
|
|
72
|
+
# @param waiter [Process::Waiter] thread waiting on the process group leader
|
|
73
|
+
# @return [void]
|
|
74
|
+
#: (Process::Waiter waiter) -> void
|
|
75
|
+
private def terminate(waiter)
|
|
76
|
+
Process.kill('TERM', -waiter.pid)
|
|
77
|
+
Process.kill('KILL', -waiter.pid) unless waiter.join(KILL_GRACE)
|
|
78
|
+
rescue Errno::ESRCH
|
|
79
|
+
# Already gone.
|
|
80
|
+
nil
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# @param status [Process::Status] exit status
|
|
84
|
+
# @return [String] how the command ended
|
|
85
|
+
#: (Process::Status status) -> String
|
|
86
|
+
private def describe(status)
|
|
87
|
+
status.signaled? ? "was killed by signal #{status.termsig}" : "exited with #{status.exitstatus}"
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|