glazed 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. checksums.yaml +7 -0
  2. data/.rspec +3 -0
  3. data/.yardopts +5 -0
  4. data/CHANGELOG.md +18 -0
  5. data/LICENSE.txt +202 -0
  6. data/NOTICE +16 -0
  7. data/README.md +3 -0
  8. data/Rakefile +12 -0
  9. data/exe/glazed +8 -0
  10. data/lib/glazed/cli_session.rb +178 -0
  11. data/lib/glazed/collector.rb +113 -0
  12. data/lib/glazed/command/base.rb +146 -0
  13. data/lib/glazed/command/orchestrate.rb +78 -0
  14. data/lib/glazed/command/run.rb +51 -0
  15. data/lib/glazed/command/send_signal.rb +69 -0
  16. data/lib/glazed/command/status.rb +179 -0
  17. data/lib/glazed/command/version.rb +23 -0
  18. data/lib/glazed/command.rb +39 -0
  19. data/lib/glazed/command_result.rb +26 -0
  20. data/lib/glazed/dialect/actions.rb +95 -0
  21. data/lib/glazed/dialect/communication_config.rb +63 -0
  22. data/lib/glazed/dialect/interaction.rb +32 -0
  23. data/lib/glazed/dialect/run.rb +84 -0
  24. data/lib/glazed/dialect/runbook.rb +97 -0
  25. data/lib/glazed/dialect/sensitive_pattern.rb +31 -0
  26. data/lib/glazed/dialect.rb +12 -0
  27. data/lib/glazed/dialects/awplus.rb +84 -0
  28. data/lib/glazed/dialects/base.rb +246 -0
  29. data/lib/glazed/dialects/ios.rb +173 -0
  30. data/lib/glazed/dialects/junos.rb +78 -0
  31. data/lib/glazed/dialects/nec_ix.rb +71 -0
  32. data/lib/glazed/dialects/wlc.rb +56 -0
  33. data/lib/glazed/dialects.rb +65 -0
  34. data/lib/glazed/finder.rb +62 -0
  35. data/lib/glazed/must.rb +54 -0
  36. data/lib/glazed/orchestrator/policy.rb +184 -0
  37. data/lib/glazed/orchestrator.rb +288 -0
  38. data/lib/glazed/plugin_registry.rb +70 -0
  39. data/lib/glazed/proto.rb +20 -0
  40. data/lib/glazed/proto_json.rb +34 -0
  41. data/lib/glazed/redactor.rb +158 -0
  42. data/lib/glazed/runner.rb +176 -0
  43. data/lib/glazed/secret.rb +74 -0
  44. data/lib/glazed/secrets.rb +59 -0
  45. data/lib/glazed/shell_command.rb +90 -0
  46. data/lib/glazed/sources/base.rb +79 -0
  47. data/lib/glazed/sources/command.rb +67 -0
  48. data/lib/glazed/sources/inline.rb +56 -0
  49. data/lib/glazed/sources/json_files.rb +87 -0
  50. data/lib/glazed/sources.rb +105 -0
  51. data/lib/glazed/state_backends/base.rb +72 -0
  52. data/lib/glazed/state_backends/sqlite.rb +299 -0
  53. data/lib/glazed/state_backends.rb +75 -0
  54. data/lib/glazed/step_context.rb +65 -0
  55. data/lib/glazed/steps/base.rb +89 -0
  56. data/lib/glazed/steps/batch_shell.rb +57 -0
  57. data/lib/glazed/steps/fetch.rb +67 -0
  58. data/lib/glazed/steps/publish.rb +94 -0
  59. data/lib/glazed/steps/redact.rb +77 -0
  60. data/lib/glazed/steps/shell.rb +107 -0
  61. data/lib/glazed/steps.rb +83 -0
  62. data/lib/glazed/storages/base.rb +167 -0
  63. data/lib/glazed/storages/filesystem.rb +144 -0
  64. data/lib/glazed/storages/git.rb +181 -0
  65. data/lib/glazed/storages/github.rb +269 -0
  66. data/lib/glazed/storages/memory.rb +125 -0
  67. data/lib/glazed/storages.rb +88 -0
  68. data/lib/glazed/transports/base.rb +103 -0
  69. data/lib/glazed/transports/ssh.rb +177 -0
  70. data/lib/glazed/transports/telnet.rb +100 -0
  71. data/lib/glazed/transports.rb +28 -0
  72. data/lib/glazed/v1/config_pb.rb +25 -0
  73. data/lib/glazed/v1/inventory_pb.rb +27 -0
  74. data/lib/glazed/v1/orchestrator_pb.rb +27 -0
  75. data/lib/glazed/v1/redaction_pb.rb +23 -0
  76. data/lib/glazed/v1/secret_pb.rb +22 -0
  77. data/lib/glazed/v1/state_pb.rb +25 -0
  78. data/lib/glazed/v1/steps_pb.rb +25 -0
  79. data/lib/glazed/v1/storage_pb.rb +34 -0
  80. data/lib/glazed/v1/workflow_pb.rb +29 -0
  81. data/lib/glazed/validation.rb +31 -0
  82. data/lib/glazed/version.rb +9 -0
  83. data/lib/glazed/workflow.rb +70 -0
  84. data/lib/glazed.rb +18 -0
  85. metadata +245 -0
@@ -0,0 +1,70 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'google/protobuf'
7
+ require 'google/protobuf/well_known_types'
8
+ require 'glazed'
9
+ require 'glazed/validation'
10
+
11
+ module Glazed
12
+ # Implementations of one kind of plugin, keyed by the full name of the
13
+ # settings message a config packs into a google.protobuf.Any.
14
+ #
15
+ # @example A plugin gem registering an inventory provider
16
+ # Glazed::Sources.register(GlazedNetbox::V1::NetboxInventory, GlazedNetbox::Source)
17
+ class PluginRegistry
18
+ # Raised when no implementation is registered for a settings type.
19
+ class UnknownPlugin < Glazed::Error; end
20
+
21
+ # @param kind [String] what the plugins implement, for error messages
22
+ #: (String kind) -> void
23
+ def initialize(kind)
24
+ @kind = kind #: String
25
+ @implementations = {} #: Hash[String, Module[untyped]]
26
+ @lock = Mutex.new #: Thread::Mutex
27
+ end
28
+
29
+ # Registers the implementation for a settings message type.
30
+ #
31
+ # @param settings_class [Class] generated message class of the settings
32
+ # @param implementation [Module] implementation built from those settings
33
+ # @return [void]
34
+ #: (singleton(Google::Protobuf::AbstractMessage) settings_class, Module[untyped] implementation) -> void
35
+ def register(settings_class, implementation)
36
+ @lock.synchronize { @implementations[settings_class.descriptor.name] = implementation }
37
+ end
38
+
39
+ # Removes the implementation for a settings message type.
40
+ #
41
+ # @param settings_class [Class] generated message class of the settings
42
+ # @return [void]
43
+ #: (singleton(Google::Protobuf::AbstractMessage) settings_class) -> void
44
+ def unregister(settings_class)
45
+ @lock.synchronize { @implementations.delete(settings_class.descriptor.name) }
46
+ end
47
+
48
+ # Finds the implementation for packed settings and unpacks them.
49
+ #
50
+ # @param any [Google::Protobuf::Any] packed settings
51
+ # @return [Array(Module, Google::Protobuf::MessageExts)] the implementation and the settings message
52
+ # @raise [UnknownPlugin] when no implementation is registered for the settings type
53
+ # @raise [InvalidMessage] when the settings break the rules of their schema
54
+ #: (Google::Protobuf::Any any) -> [Module[untyped], Google::Protobuf::MessageExts]
55
+ def resolve(any)
56
+ type = any.type_name.to_s
57
+ implementation = @lock.synchronize { @implementations[type] }
58
+ unless implementation
59
+ raise UnknownPlugin, "no #{@kind} plugin is registered for #{type}; require the gem providing it with `requires`"
60
+ end
61
+
62
+ settings_class = Google::Protobuf::DescriptorPool.generated_pool.lookup(type)&.msgclass
63
+ settings = settings_class && any.unpack(settings_class)
64
+ raise UnknownPlugin, "cannot unpack #{type} settings" unless settings.is_a?(Google::Protobuf::MessageExts)
65
+
66
+ Validation.validate!(settings, origin: type)
67
+ [implementation, settings]
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,20 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'glazed/v1/config_pb'
7
+ require 'glazed/v1/inventory_pb'
8
+ require 'glazed/v1/orchestrator_pb'
9
+ require 'glazed/v1/redaction_pb'
10
+ require 'glazed/v1/secret_pb'
11
+ require 'glazed/v1/state_pb'
12
+ require 'glazed/v1/steps_pb'
13
+ require 'glazed/v1/storage_pb'
14
+ require 'glazed/v1/workflow_pb'
15
+
16
+ module Glazed
17
+ # Message classes generated from proto/glazed/v1 (Glazed::Proto::V1).
18
+ module Proto
19
+ end
20
+ end
@@ -0,0 +1,34 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'google/protobuf'
7
+
8
+ module Glazed
9
+ # ProtoJSON codec that keeps proto field names (snake_case) on output and
10
+ # rejects unknown fields on input.
11
+ module ProtoJson
12
+ # Decodes ProtoJSON into a message.
13
+ #
14
+ # @param klass [Class<Google::Protobuf::AbstractMessage>] message class
15
+ # @param json [String] ProtoJSON document
16
+ # @return [Google::Protobuf::AbstractMessage] decoded message
17
+ # @raise [Google::Protobuf::ParseError] when the document is malformed or has unknown fields
18
+ # @example
19
+ # Glazed::ProtoJson.load(Glazed::Proto::V1::Node, '{"name": "rt1"}')
20
+ #: [M < Google::Protobuf::AbstractMessage] (singleton(Google::Protobuf::AbstractMessage) & Class[M] klass, String json) -> M
21
+ def self.load(klass, json)
22
+ klass.decode_json(json)
23
+ end
24
+
25
+ # Encodes a message into ProtoJSON.
26
+ #
27
+ # @param message [Google::Protobuf::AbstractMessage] message to encode
28
+ # @return [String] ProtoJSON document
29
+ #: (Google::Protobuf::AbstractMessage message) -> String
30
+ def self.dump(message)
31
+ message.class.encode_json(message, preserve_proto_fieldnames: true)
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,158 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'openssl'
7
+ require 'glazed'
8
+ require 'glazed/dialect/sensitive_pattern'
9
+ require 'glazed/proto'
10
+ require 'glazed/secret'
11
+
12
+ module Glazed
13
+ # Hides secrets located by a dialect's sensitive patterns.
14
+ #
15
+ # Only the secret within each match is replaced, by +[~REDACTED~]+ in :remove
16
+ # mode or +[~REDACTED:dgst=...~]+ in :hmac mode. The digest is a truncated
17
+ # HMAC-SHA256 of the secret in unpadded base64url, so a changed secret shows
18
+ # up in diffs without being revealed.
19
+ #
20
+ # @example
21
+ # Glazed::Redactor.new(patterns: Glazed::Dialects::Ios.sensitive_patterns, mode: :remove).call(config)
22
+ class Redactor
23
+ # Raised when the redaction settings are unusable.
24
+ class InvalidConfiguration < Glazed::Error; end
25
+
26
+ # Replacement in :remove mode.
27
+ REMOVED = '[~REDACTED~]' #: String
28
+ # Bytes of the HMAC kept in :hmac mode.
29
+ HMAC_BYTES = 8 #: Integer
30
+
31
+ # Supported modes.
32
+ MODES = %i[remove hmac].freeze #: Array[Symbol]
33
+
34
+ # @param patterns [Array<Dialect::SensitivePattern>] where secrets appear
35
+ # @param mode [Symbol] :remove or :hmac
36
+ # @param key [String, nil] HMAC key, required for :hmac
37
+ # @raise [InvalidConfiguration] when the mode is unknown or :hmac has no key
38
+ #: (patterns: Array[Dialect::SensitivePattern], mode: Symbol, ?key: String?) -> void
39
+ def initialize(patterns:, mode:, key: nil)
40
+ raise InvalidConfiguration, "unknown redaction mode #{mode.inspect}" unless MODES.include?(mode)
41
+ raise InvalidConfiguration, 'HMAC redaction requires a key' if mode == :hmac && (key.nil? || key.empty?)
42
+
43
+ @patterns = patterns #: Array[Dialect::SensitivePattern]
44
+ @mode = mode #: Symbol
45
+ @key = key #: String?
46
+ end
47
+
48
+ # Builds a redactor from settings.
49
+ #
50
+ # @param redaction [Proto::V1::Redaction] redaction settings
51
+ # @param patterns [Array<Dialect::SensitivePattern>] where secrets appear
52
+ # @return [Redactor] the redactor
53
+ # @raise [InvalidConfiguration] when the mode is unknown, or the HMAC key is missing or not base64
54
+ # @raise [Secret::Unavailable] when the HMAC key cannot be read
55
+ #: (Proto::V1::Redaction redaction, patterns: Array[Dialect::SensitivePattern]) -> Redactor
56
+ def self.from_proto(redaction, patterns:)
57
+ case redaction.mode
58
+ when :MODE_UNSPECIFIED, :MODE_REMOVE then new(patterns:, mode: :remove)
59
+ when :MODE_HMAC then new(patterns:, mode: :hmac, key: hmac_key(redaction))
60
+ else raise InvalidConfiguration, "unknown redaction mode #{redaction.mode.inspect}"
61
+ end
62
+ end
63
+
64
+ # @param redaction [Proto::V1::Redaction] redaction settings
65
+ # @return [String, nil] the HMAC key, decoded from base64
66
+ # @raise [InvalidConfiguration] when the key is not base64
67
+ # @raise [Secret::Unavailable] when the key cannot be read
68
+ #: (Proto::V1::Redaction redaction) -> String?
69
+ private_class_method def self.hmac_key(redaction)
70
+ encoded = Secret.maybe(redaction.hmac_key)&.value or return nil
71
+ encoded.unpack1('m0')
72
+ rescue ArgumentError
73
+ raise InvalidConfiguration, 'hmac_key is not base64'
74
+ end
75
+
76
+ # @return [Symbol] :remove or :hmac
77
+ #: Symbol
78
+ attr_reader :mode
79
+
80
+ # Redacts a configuration.
81
+ #
82
+ # @param text [String] configuration text
83
+ # @return [String] text with every located secret replaced
84
+ #: (String text) -> String
85
+ def call(text)
86
+ result = text.dup
87
+ secret_ranges(text).reverse_each do |start, finish|
88
+ result[start...finish] = replacement(text[start...finish] || '')
89
+ end
90
+ result
91
+ end
92
+
93
+ # @param text [String] configuration text
94
+ # @return [Array<Array(Integer, Integer)>] non-overlapping secret start and end offsets, in order
95
+ #: (String text) -> Array[[Integer, Integer]]
96
+ private def secret_ranges(text)
97
+ ranges = @patterns.flat_map do |sensitive|
98
+ matches(sensitive.pattern, text).filter_map do |match|
99
+ case (secret = sensitive.determiner.call(match))
100
+ when Range
101
+ start = secret.begin
102
+ finish = secret.exclude_end? ? secret.end : secret.end + 1
103
+ [start, finish] if finish > start
104
+ when String then locate(match, secret) unless secret.empty?
105
+ end
106
+ end
107
+ end
108
+ kept = [] #: Array[[Integer, Integer]]
109
+ ranges.sort_by { |start, finish| [start, -finish] }.each do |start, finish|
110
+ last = kept.last
111
+ kept << [start, finish] if last.nil? || start >= last[1]
112
+ end
113
+ kept
114
+ end
115
+
116
+ # @param pattern [Regexp] pattern to scan for
117
+ # @param text [String] text to scan
118
+ # @return [Array<MatchData>] every match
119
+ #: (Regexp pattern, String text) -> Array[MatchData]
120
+ private def matches(pattern, text)
121
+ found = [] #: Array[MatchData]
122
+ position = 0
123
+ while (match = pattern.match(text, position))
124
+ found << match
125
+ position = match.end(0) > match.begin(0) ? match.end(0) : match.end(0) + 1
126
+ break if position > text.size
127
+ end
128
+ found
129
+ end
130
+
131
+ # Finds the secret in a match, preferring the last capture group equal to it,
132
+ # as secrets follow the names they belong to.
133
+ #
134
+ # @param match [MatchData] the match
135
+ # @param secret [String] secret returned by the determiner
136
+ # @return [Array(Integer, Integer), nil] start and end offsets of the secret, or nil when it is not in the match
137
+ #: (MatchData match, String secret) -> [Integer, Integer]?
138
+ private def locate(match, secret)
139
+ group = (match.size - 1).downto(1).find { match[it] == secret }
140
+ start = if group
141
+ match.begin(group)
142
+ elsif (offset = match[0]&.rindex(secret))
143
+ match.begin(0) + offset
144
+ end
145
+ [start, start + secret.size] if start
146
+ end
147
+
148
+ # @param secret [String] secret to hide
149
+ # @return [String] its replacement
150
+ #: (String secret) -> String
151
+ private def replacement(secret)
152
+ return REMOVED unless @mode == :hmac
153
+
154
+ digest = OpenSSL::HMAC.digest('SHA256', @key.to_s, secret).byteslice(0, HMAC_BYTES)
155
+ "[~REDACTED:dgst=#{[digest].pack('m0').tr('+/', '-_').delete('=')}~]"
156
+ end
157
+ end
158
+ end
@@ -0,0 +1,176 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'glazed'
7
+ require 'glazed/proto'
8
+ require 'glazed/steps'
9
+ require 'glazed/workflow'
10
+
11
+ module Glazed
12
+ # Runs a workflow in this process. Consecutive per-node steps run for each
13
+ # node on a thread pool; a node whose step fails is reported and left out of
14
+ # the rest of the workflow. Batch steps run with the remaining nodes, and
15
+ # their failures propagate.
16
+ #
17
+ # @example
18
+ # result = Glazed::Runner.new(workflow:, jobs: 4).call(nodes)
19
+ # result.failures.each { warn "#{it.node.name}: #{it.error.message}" }
20
+ class Runner
21
+ #: type event_value = String | Integer | bool | Array[String] | nil
22
+ #: type listener = ^(Hash[Symbol, Runner::event_value] event) -> void
23
+
24
+ # A node that failed a step.
25
+ class Failure
26
+ # @param node [Proto::V1::Node] the node
27
+ # @param step [String] name of the failed step
28
+ # @param error [StandardError] the error
29
+ #: (node: Proto::V1::Node, step: String, error: StandardError) -> void
30
+ def initialize(node:, step:, error:)
31
+ @node = node #: Proto::V1::Node
32
+ @step = step #: String
33
+ @error = error #: StandardError
34
+ end
35
+
36
+ # @return [Proto::V1::Node] the node
37
+ #: Proto::V1::Node
38
+ attr_reader :node
39
+ # @return [String] name of the failed step
40
+ #: String
41
+ attr_reader :step
42
+ # @return [StandardError] the error
43
+ #: StandardError
44
+ attr_reader :error
45
+ end
46
+
47
+ # Outcome of a run.
48
+ class Result
49
+ # @param states [Array<Proto::V1::NodeState>] states of the nodes that completed every step
50
+ # @param failures [Array<Failure>] nodes that failed a step
51
+ #: (states: Array[Proto::V1::NodeState], failures: Array[Failure]) -> void
52
+ def initialize(states:, failures:)
53
+ @states = states #: Array[Proto::V1::NodeState]
54
+ @failures = failures #: Array[Failure]
55
+ end
56
+
57
+ # @return [Array<Proto::V1::NodeState>] states of the nodes that completed every step
58
+ #: Array[Proto::V1::NodeState]
59
+ attr_reader :states
60
+ # @return [Array<Failure>] nodes that failed a step
61
+ #: Array[Failure]
62
+ attr_reader :failures
63
+
64
+ # @return [Array<Proto::V1::NodeState>] states with at least one artifact whose write changed stored content
65
+ #: () -> Array[Proto::V1::NodeState]
66
+ def changed = states.select { |state| state.artifacts.values.any?(&:changed) }
67
+ end
68
+
69
+ # @param workflow [Workflow] the workflow
70
+ # @param jobs [Integer] nodes processed in parallel
71
+ # @param listener [Proc] receives an event hash per completed or failed step
72
+ #: (workflow: Workflow, jobs: Integer, ?listener: Runner::listener) -> void
73
+ def initialize(workflow:, jobs:, listener: ->(_event) {})
74
+ @workflow = workflow #: Workflow
75
+ @jobs = jobs #: Integer
76
+ @listener = listener #: Runner::listener
77
+ @lock = Mutex.new #: Thread::Mutex
78
+ end
79
+
80
+ # @param nodes [Array<Proto::V1::Node>] nodes to run against
81
+ # @return [Result] the outcome
82
+ # @raise [Glazed::Error] when a batch step fails
83
+ #: (Array[Proto::V1::Node] nodes) -> Result
84
+ def call(nodes)
85
+ states = nodes.map { Proto::V1::NodeState.new(node: it) }
86
+ failures = [] #: Array[Failure]
87
+ @workflow.steps.chunk_while { |a, b| a.is_a?(Steps::NodeStep) && b.is_a?(Steps::NodeStep) }.each do |chunk|
88
+ first = chunk.first
89
+ states = case first
90
+ when Steps::BatchStep then run_batch(first, states)
91
+ else run_per_node(chunk.grep(Steps::NodeStep), states, failures)
92
+ end
93
+ end
94
+ Result.new(states:, failures:)
95
+ end
96
+
97
+ # @param step [Steps::BatchStep] the step
98
+ # @param states [Array<Proto::V1::NodeState>] node states
99
+ # @return [Array<Proto::V1::NodeState>] node states after the step
100
+ #: (Steps::BatchStep step, Array[Proto::V1::NodeState] states) -> Array[Proto::V1::NodeState]
101
+ private def run_batch(step, states)
102
+ batch = step.call(Proto::V1::BatchState.new(nodes: states))
103
+ after = batch.nodes.to_a
104
+ if step.outputs.empty?
105
+ notify(level: 'info', step: step.name, nodes: after.size, message: 'done')
106
+ else
107
+ notify(level: 'info', step: step.name, nodes: after.size, changed: after.count { changed?(it, step) }, message: 'done')
108
+ end
109
+ after
110
+ end
111
+
112
+ # @param steps [Array<Steps::NodeStep>] consecutive per-node steps
113
+ # @param states [Array<Proto::V1::NodeState>] node states
114
+ # @param failures [Array<Failure>] collects failed nodes
115
+ # @return [Array<Proto::V1::NodeState>] states of the nodes that completed the steps, in input order
116
+ #: (Array[Steps::NodeStep] steps, Array[Proto::V1::NodeState] states, Array[Failure] failures) -> Array[Proto::V1::NodeState]
117
+ private def run_per_node(steps, states, failures)
118
+ queue = Thread::Queue.new(states.each_with_index.to_a)
119
+ queue.close
120
+ results = {} #: Hash[Integer, Proto::V1::NodeState]
121
+ workers = Array.new([@jobs, states.size].min) do
122
+ Thread.new do
123
+ while (item = queue.pop)
124
+ state, index = item
125
+ Thread.current.name = state.node&.name
126
+ result = run_node(steps, state, failures)
127
+ @lock.synchronize { results[index] = result } if result
128
+ end
129
+ end
130
+ end
131
+ begin
132
+ workers.each(&:join)
133
+ rescue Interrupt
134
+ workers.select(&:alive?).each do |worker|
135
+ notify(level: 'error', node: worker.name, message: 'interrupted', backtrace: worker.backtrace || [])
136
+ end
137
+ raise
138
+ end
139
+ results.sort.map { |_, state| state }
140
+ end
141
+
142
+ # @param steps [Array<Steps::NodeStep>] consecutive per-node steps
143
+ # @param state [Proto::V1::NodeState] the node's state
144
+ # @param failures [Array<Failure>] collects the node when it fails
145
+ # @return [Proto::V1::NodeState, nil] the final state, or nil when a step failed
146
+ #: (Array[Steps::NodeStep] steps, Proto::V1::NodeState state, Array[Failure] failures) -> Proto::V1::NodeState?
147
+ private def run_node(steps, state, failures)
148
+ node = state.node or raise ArgumentError, 'node state has no node'
149
+ steps.inject(state) do |current, step|
150
+ begin
151
+ current = step.call(current)
152
+ rescue StandardError => e
153
+ # One unreachable or misbehaving device must not stop the others.
154
+ @lock.synchronize { failures << Failure.new(node:, step: step.name, error: e) }
155
+ notify(level: 'error', step: step.name, node: node.name, error: e.class.name, message: e.message)
156
+ return nil
157
+ end
158
+ notify(level: 'info', step: step.name, node: node.name, changed: changed?(current, step), message: 'done')
159
+ current
160
+ end
161
+ end
162
+
163
+ # @param state [Proto::V1::NodeState] state after the step
164
+ # @param step [Steps::Base] the step
165
+ # @return [Boolean] whether any artifact the step produced changed stored content
166
+ #: (Proto::V1::NodeState state, Steps::Base step) -> bool
167
+ private def changed?(state, step) = step.outputs.any? { state.artifacts[it]&.changed }
168
+
169
+ # @param event [Hash{Symbol => String, Integer, Boolean, nil}] event fields
170
+ # @return [void]
171
+ #: (**Runner::event_value event) -> void
172
+ private def notify(**event)
173
+ @lock.synchronize { @listener.call(event) }
174
+ end
175
+ end
176
+ end
@@ -0,0 +1,74 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'glazed'
7
+ require 'glazed/proto'
8
+ require 'glazed/secrets'
9
+
10
+ module Glazed
11
+ # Secret resolved from a {Proto::V1::SecretRef} on each access. Its value
12
+ # never appears in #inspect or #to_s.
13
+ class Secret
14
+ # Raised when the referenced value cannot be read.
15
+ class Unavailable < Glazed::Error; end
16
+
17
+ # @param ref [Proto::V1::SecretRef] reference to resolve
18
+ #: (Proto::V1::SecretRef ref) -> void
19
+ def initialize(ref)
20
+ @ref = ref #: Proto::V1::SecretRef
21
+ end
22
+
23
+ # Wraps a reference that may be absent.
24
+ #
25
+ # @param ref [Proto::V1::SecretRef, nil] reference to wrap
26
+ # @return [Secret, nil] the secret, or nil when ref is nil
27
+ #: (Proto::V1::SecretRef? ref) -> Secret?
28
+ def self.maybe(ref)
29
+ ref && new(ref)
30
+ end
31
+
32
+ # @return [Proto::V1::SecretRef] the wrapped reference
33
+ #: Proto::V1::SecretRef
34
+ attr_reader :ref
35
+
36
+ # Reads the secret value.
37
+ #
38
+ # @return [String] the value
39
+ # @raise [Unavailable] when the environment variable or file is missing, or no source is set
40
+ #: () -> String
41
+ def value
42
+ case ref.source
43
+ when :value then ref.value
44
+ when :env then ENV.fetch(ref.env) { raise Unavailable, "environment variable #{ref.env} is not set" }
45
+ when :file then read_file(ref.file)
46
+ when :plugin then Secrets.build(ref.plugin || raise(Unavailable, 'secret plugin has no settings')).value
47
+ else raise Unavailable, 'secret has no source'
48
+ end
49
+ end
50
+
51
+ # @return [String] description naming the source but not the value
52
+ #: () -> String
53
+ def inspect
54
+ case ref.source
55
+ when :env then "#<#{self.class} env:#{ref.env}>"
56
+ when :file then "#<#{self.class} file:#{ref.file}>"
57
+ when :plugin then "#<#{self.class} plugin:#{ref.plugin&.type_name}>"
58
+ else "#<#{self.class} value>"
59
+ end
60
+ end
61
+
62
+ alias to_s inspect
63
+
64
+ # @param path [String] file to read
65
+ # @return [String] file contents without one trailing newline
66
+ # @raise [Unavailable] when the file cannot be read
67
+ #: (String path) -> String
68
+ private def read_file(path)
69
+ File.read(path).delete_suffix("\n")
70
+ rescue SystemCallError => e
71
+ raise Unavailable, "cannot read secret file #{path}: #{e.message}"
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,59 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'google/protobuf'
7
+ require 'glazed'
8
+ require 'glazed/must'
9
+ require 'glazed/plugin_registry'
10
+
11
+ module Glazed
12
+ # Secret sources implemented by plugins, used by `plugin` secret references.
13
+ module Secrets
14
+ # A plugin source of one secret value, built from its settings message.
15
+ class Base
16
+ # Builds the source from its settings message.
17
+ #
18
+ # @param settings [Google::Protobuf::MessageExts] settings unpacked from the reference
19
+ # @return [Base] the source
20
+ #: (Google::Protobuf::MessageExts settings) -> Base
21
+ def self.from_proto(settings) = raise NotImplementedError
22
+
23
+ # Reads the secret, each time it is used.
24
+ #
25
+ # @return [String] the secret value
26
+ #: () -> String
27
+ def value = raise NotImplementedError
28
+ end
29
+
30
+ # Secret sources implemented by plugins, by settings message type.
31
+ PLUGINS = PluginRegistry.new('secret source') #: PluginRegistry
32
+
33
+ # Registers a plugin secret source, selected by `plugin` references whose
34
+ # settings are of the given type.
35
+ #
36
+ # @param settings_class [Class] generated message class of the source settings
37
+ # @param implementation [Class<Base>] source built with {Base.from_proto}
38
+ # @return [void]
39
+ # @example
40
+ # Glazed::Secrets.register(GlazedAws::V1::SsmParameter, GlazedAws::SsmParameterSecret)
41
+ #: (singleton(Google::Protobuf::AbstractMessage) settings_class, singleton(Base) implementation) -> void
42
+ def self.register(settings_class, implementation)
43
+ PLUGINS.register(settings_class, implementation)
44
+ end
45
+
46
+ # Builds the source for packed settings.
47
+ #
48
+ # @param packed [Google::Protobuf::Any] settings from a secret reference
49
+ # @return [Base] the source
50
+ # @raise [PluginRegistry::UnknownPlugin] when no plugin is registered for the settings type
51
+ #: (Google::Protobuf::Any packed) -> Base
52
+ def self.build(packed)
53
+ implementation, settings = PLUGINS.resolve(packed)
54
+ raise Must::Violation, "#{implementation} is not a secret source" unless implementation.is_a?(Class) && implementation < Base
55
+
56
+ implementation.from_proto(settings)
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,90 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'open3'
7
+ require 'glazed'
8
+
9
+ module Glazed
10
+ # Runs a command line with /bin/sh -c, feeding stdin and capturing stdout.
11
+ #
12
+ # @example
13
+ # Glazed::ShellCommand.new(run: 'sort', env: {}, stdin: "b\na\n", timeout: 10).call #=> "a\nb\n"
14
+ class ShellCommand
15
+ # Raised when the command exits non-zero or is killed by a signal.
16
+ class Failed < Glazed::Error; end
17
+
18
+ # Bytes of stderr kept in error messages.
19
+ STDERR_TAIL = 1000 #: Integer
20
+ # Seconds between SIGTERM and SIGKILL when the command overruns.
21
+ KILL_GRACE = 5 #: Integer
22
+
23
+ # @param run [String] command line
24
+ # @param env [Hash{String => String}] environment added to the inherited one
25
+ # @param stdin [String] data written to stdin
26
+ # @param timeout [Numeric] seconds before the command is killed
27
+ #: (run: String, env: Hash[String, String], stdin: String, timeout: Numeric) -> void
28
+ def initialize(run:, env:, stdin:, timeout:)
29
+ @run = run #: String
30
+ @env = env #: Hash[String, String]
31
+ @stdin = stdin #: String
32
+ @timeout = timeout #: Numeric
33
+ end
34
+
35
+ # Runs the command to completion.
36
+ #
37
+ # @return [String] stdout
38
+ # @raise [Failed] when the command exits non-zero or is killed by a signal
39
+ # @raise [TimeoutError] when the command does not finish in time; its process group is killed
40
+ #: () -> String
41
+ def call
42
+ stdin, stdout, stderr, waiter = Open3.popen3(@env, '/bin/sh', '-c', @run, pgroup: true)
43
+ begin
44
+ writer = Thread.new do
45
+ stdin.write(@stdin)
46
+ rescue Errno::EPIPE
47
+ # The command exited or closed stdin without reading all of it.
48
+ nil
49
+ ensure
50
+ stdin.close
51
+ end
52
+ out = Thread.new { stdout.read }
53
+ err = Thread.new { stderr.read }
54
+
55
+ unless waiter.join(@timeout.to_f)
56
+ terminate(waiter)
57
+ raise TimeoutError, "#{@run.inspect} did not finish in #{@timeout}s"
58
+ end
59
+ writer.join
60
+ status = waiter.value #: Process::Status
61
+ output = out.value.to_s
62
+ return output if status.success?
63
+
64
+ raise Failed, "#{@run.inspect} #{describe(status)}: #{err.value.to_s[-STDERR_TAIL..] || err.value}"
65
+ ensure
66
+ [stdin, stdout, stderr].each { it.close unless it.closed? }
67
+ end
68
+ end
69
+
70
+ # Stops the command and everything it started, gracefully first.
71
+ #
72
+ # @param waiter [Process::Waiter] thread waiting on the process group leader
73
+ # @return [void]
74
+ #: (Process::Waiter waiter) -> void
75
+ private def terminate(waiter)
76
+ Process.kill('TERM', -waiter.pid)
77
+ Process.kill('KILL', -waiter.pid) unless waiter.join(KILL_GRACE)
78
+ rescue Errno::ESRCH
79
+ # Already gone.
80
+ nil
81
+ end
82
+
83
+ # @param status [Process::Status] exit status
84
+ # @return [String] how the command ended
85
+ #: (Process::Status status) -> String
86
+ private def describe(status)
87
+ status.signaled? ? "was killed by signal #{status.termsig}" : "exited with #{status.exitstatus}"
88
+ end
89
+ end
90
+ end