glazed 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +3 -0
- data/.yardopts +5 -0
- data/CHANGELOG.md +18 -0
- data/LICENSE.txt +202 -0
- data/NOTICE +16 -0
- data/README.md +3 -0
- data/Rakefile +12 -0
- data/exe/glazed +8 -0
- data/lib/glazed/cli_session.rb +178 -0
- data/lib/glazed/collector.rb +113 -0
- data/lib/glazed/command/base.rb +146 -0
- data/lib/glazed/command/orchestrate.rb +78 -0
- data/lib/glazed/command/run.rb +51 -0
- data/lib/glazed/command/send_signal.rb +69 -0
- data/lib/glazed/command/status.rb +179 -0
- data/lib/glazed/command/version.rb +23 -0
- data/lib/glazed/command.rb +39 -0
- data/lib/glazed/command_result.rb +26 -0
- data/lib/glazed/dialect/actions.rb +95 -0
- data/lib/glazed/dialect/communication_config.rb +63 -0
- data/lib/glazed/dialect/interaction.rb +32 -0
- data/lib/glazed/dialect/run.rb +84 -0
- data/lib/glazed/dialect/runbook.rb +97 -0
- data/lib/glazed/dialect/sensitive_pattern.rb +31 -0
- data/lib/glazed/dialect.rb +12 -0
- data/lib/glazed/dialects/awplus.rb +84 -0
- data/lib/glazed/dialects/base.rb +246 -0
- data/lib/glazed/dialects/ios.rb +173 -0
- data/lib/glazed/dialects/junos.rb +78 -0
- data/lib/glazed/dialects/nec_ix.rb +71 -0
- data/lib/glazed/dialects/wlc.rb +56 -0
- data/lib/glazed/dialects.rb +65 -0
- data/lib/glazed/finder.rb +62 -0
- data/lib/glazed/must.rb +54 -0
- data/lib/glazed/orchestrator/policy.rb +184 -0
- data/lib/glazed/orchestrator.rb +288 -0
- data/lib/glazed/plugin_registry.rb +70 -0
- data/lib/glazed/proto.rb +20 -0
- data/lib/glazed/proto_json.rb +34 -0
- data/lib/glazed/redactor.rb +158 -0
- data/lib/glazed/runner.rb +176 -0
- data/lib/glazed/secret.rb +74 -0
- data/lib/glazed/secrets.rb +59 -0
- data/lib/glazed/shell_command.rb +90 -0
- data/lib/glazed/sources/base.rb +79 -0
- data/lib/glazed/sources/command.rb +67 -0
- data/lib/glazed/sources/inline.rb +56 -0
- data/lib/glazed/sources/json_files.rb +87 -0
- data/lib/glazed/sources.rb +105 -0
- data/lib/glazed/state_backends/base.rb +72 -0
- data/lib/glazed/state_backends/sqlite.rb +299 -0
- data/lib/glazed/state_backends.rb +75 -0
- data/lib/glazed/step_context.rb +65 -0
- data/lib/glazed/steps/base.rb +89 -0
- data/lib/glazed/steps/batch_shell.rb +57 -0
- data/lib/glazed/steps/fetch.rb +67 -0
- data/lib/glazed/steps/publish.rb +94 -0
- data/lib/glazed/steps/redact.rb +77 -0
- data/lib/glazed/steps/shell.rb +107 -0
- data/lib/glazed/steps.rb +83 -0
- data/lib/glazed/storages/base.rb +167 -0
- data/lib/glazed/storages/filesystem.rb +144 -0
- data/lib/glazed/storages/git.rb +181 -0
- data/lib/glazed/storages/github.rb +269 -0
- data/lib/glazed/storages/memory.rb +125 -0
- data/lib/glazed/storages.rb +88 -0
- data/lib/glazed/transports/base.rb +103 -0
- data/lib/glazed/transports/ssh.rb +177 -0
- data/lib/glazed/transports/telnet.rb +100 -0
- data/lib/glazed/transports.rb +28 -0
- data/lib/glazed/v1/config_pb.rb +25 -0
- data/lib/glazed/v1/inventory_pb.rb +27 -0
- data/lib/glazed/v1/orchestrator_pb.rb +27 -0
- data/lib/glazed/v1/redaction_pb.rb +23 -0
- data/lib/glazed/v1/secret_pb.rb +22 -0
- data/lib/glazed/v1/state_pb.rb +25 -0
- data/lib/glazed/v1/steps_pb.rb +25 -0
- data/lib/glazed/v1/storage_pb.rb +34 -0
- data/lib/glazed/v1/workflow_pb.rb +29 -0
- data/lib/glazed/validation.rb +31 -0
- data/lib/glazed/version.rb +9 -0
- data/lib/glazed/workflow.rb +70 -0
- data/lib/glazed.rb +18 -0
- metadata +245 -0
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require 'glazed/must'
|
|
7
|
+
require 'glazed/proto'
|
|
8
|
+
require 'glazed/steps'
|
|
9
|
+
require 'glazed/storages'
|
|
10
|
+
|
|
11
|
+
module Glazed
|
|
12
|
+
module Steps
|
|
13
|
+
# Batch: writes an artifact of every node into a storage as one change,
|
|
14
|
+
# such as one commit naming the nodes.
|
|
15
|
+
#
|
|
16
|
+
# @example Step definition
|
|
17
|
+
# {"name": "publish", "publish": {"input": "redacted", "storage": "team"}}
|
|
18
|
+
class Publish < BatchStep
|
|
19
|
+
# Node names listed in a commit message before it is shortened.
|
|
20
|
+
MESSAGE_NAMES = 5 #: Integer
|
|
21
|
+
|
|
22
|
+
# @param name [String] step name
|
|
23
|
+
# @param input [String] artifact to read from each node
|
|
24
|
+
# @param storage_name [String] storage to write to, by name
|
|
25
|
+
# @param output [String, nil] artifact recording the written locations
|
|
26
|
+
# @param context [StepContext] named storages
|
|
27
|
+
# @raise [StepContext::UnknownStorage] when the storage is not configured
|
|
28
|
+
#: (name: String, input: String, storage_name: String, output: String?, context: StepContext) -> void
|
|
29
|
+
def initialize(name:, input:, storage_name:, output:, context:)
|
|
30
|
+
super(name:)
|
|
31
|
+
@input = input #: String
|
|
32
|
+
@storage_name = storage_name #: String
|
|
33
|
+
@storage = context.storage(storage_name) #: Storages::Base
|
|
34
|
+
@output = output #: String?
|
|
35
|
+
@context = context #: StepContext
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Builds the step from {Proto::V1::PublishStep}.
|
|
39
|
+
#
|
|
40
|
+
# @param name [String] step name
|
|
41
|
+
# @param settings [Google::Protobuf::MessageExts] a {Proto::V1::PublishStep}
|
|
42
|
+
# @param context [StepContext] named storages
|
|
43
|
+
# @return [Publish] the step
|
|
44
|
+
#: (String name, Google::Protobuf::MessageExts settings, context: StepContext) -> Publish
|
|
45
|
+
def self.from_proto(name, settings, context:)
|
|
46
|
+
publish = Must.be_a(settings, Proto::V1::PublishStep)
|
|
47
|
+
new(name:, input: publish.input, storage_name: publish.storage, output: publish.output.empty? ? nil : publish.output, context:)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# @return [Array<String>] the read artifact
|
|
51
|
+
#: () -> Array[String]
|
|
52
|
+
def inputs = [@input]
|
|
53
|
+
|
|
54
|
+
# @return [Array<String>] the recorded artifact, if any
|
|
55
|
+
#: () -> Array[String]
|
|
56
|
+
def outputs = [@output].compact
|
|
57
|
+
|
|
58
|
+
# @param batch [Proto::V1::BatchState] node states holding the input artifact
|
|
59
|
+
# @return [Proto::V1::BatchState] the states, with the output artifact when configured
|
|
60
|
+
# @raise [MissingArtifact] when a node lacks the input artifact
|
|
61
|
+
# @raise [Glazed::Error] when reading or storing fails
|
|
62
|
+
#: (Proto::V1::BatchState batch) -> Proto::V1::BatchState
|
|
63
|
+
def call(batch)
|
|
64
|
+
states = batch.nodes.to_a
|
|
65
|
+
return batch if states.empty?
|
|
66
|
+
|
|
67
|
+
names = {} #: Hash[String, String]
|
|
68
|
+
files = states.map do |state|
|
|
69
|
+
node = state.node or raise ArgumentError, "#{name}: state has no node"
|
|
70
|
+
key = Storages.key_for(node)
|
|
71
|
+
names[key] = node.name
|
|
72
|
+
[key, @context.read(artifact(state, @input))] #: Storages::Base::file
|
|
73
|
+
end
|
|
74
|
+
results = @storage.write_all(files, message: ->(changed_keys) { message(changed_keys.map { names.fetch(it, it) }) })
|
|
75
|
+
output = @output
|
|
76
|
+
return batch unless output
|
|
77
|
+
|
|
78
|
+
nodes = states.zip(results).map do |state, result|
|
|
79
|
+
with_artifacts(state, output => stored(@storage_name, Must.expect(result)))
|
|
80
|
+
end
|
|
81
|
+
Proto::V1::BatchState.new(nodes:)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# @param names [Array<String>] names of the nodes whose content changed
|
|
85
|
+
# @return [String] commit message naming the nodes
|
|
86
|
+
#: (Array[String] names) -> String
|
|
87
|
+
private def message(names)
|
|
88
|
+
listed = names.first(MESSAGE_NAMES).join(', ')
|
|
89
|
+
rest = names.size - MESSAGE_NAMES
|
|
90
|
+
rest.positive? ? "#{name}: update #{listed} and #{rest} more" : "#{name}: update #{listed}"
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require 'glazed/dialects'
|
|
7
|
+
require 'glazed/must'
|
|
8
|
+
require 'glazed/proto'
|
|
9
|
+
require 'glazed/redactor'
|
|
10
|
+
require 'glazed/steps'
|
|
11
|
+
require 'glazed/storages'
|
|
12
|
+
|
|
13
|
+
module Glazed
|
|
14
|
+
module Steps
|
|
15
|
+
# Per node: hides secrets in an artifact using the node dialect's sensitive
|
|
16
|
+
# patterns, and stores the result as another artifact.
|
|
17
|
+
#
|
|
18
|
+
# @example Step definition
|
|
19
|
+
# {"name": "redact", "redact": {"input": "raw", "output": "redacted", "storage": "staging",
|
|
20
|
+
# "redaction": {"mode": "MODE_HMAC", "hmac_key": {"env": "GLAZED_HMAC_KEY"}}}}
|
|
21
|
+
class Redact < NodeStep
|
|
22
|
+
# @param name [String] step name
|
|
23
|
+
# @param input [String] artifact to read
|
|
24
|
+
# @param output [String] artifact to produce
|
|
25
|
+
# @param storage_name [String] storage to write to, by name
|
|
26
|
+
# @param redaction [Proto::V1::Redaction] how secrets are hidden
|
|
27
|
+
# @param context [StepContext] named storages
|
|
28
|
+
# @raise [StepContext::UnknownStorage] when the storage is not configured
|
|
29
|
+
#: (name: String, input: String, output: String, storage_name: String, redaction: Proto::V1::Redaction, context: StepContext) -> void
|
|
30
|
+
def initialize(name:, input:, output:, storage_name:, redaction:, context:)
|
|
31
|
+
super(name:)
|
|
32
|
+
@input = input #: String
|
|
33
|
+
@output = output #: String
|
|
34
|
+
@storage_name = storage_name #: String
|
|
35
|
+
@storage = context.storage(storage_name) #: Storages::Base
|
|
36
|
+
@redaction = redaction #: Proto::V1::Redaction
|
|
37
|
+
@context = context #: StepContext
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Builds the step from {Proto::V1::RedactStep}.
|
|
41
|
+
#
|
|
42
|
+
# @param name [String] step name
|
|
43
|
+
# @param settings [Google::Protobuf::MessageExts] a {Proto::V1::RedactStep}
|
|
44
|
+
# @param context [StepContext] named storages
|
|
45
|
+
# @return [Redact] the step
|
|
46
|
+
#: (String name, Google::Protobuf::MessageExts settings, context: StepContext) -> Redact
|
|
47
|
+
def self.from_proto(name, settings, context:)
|
|
48
|
+
redact = Must.be_a(settings, Proto::V1::RedactStep)
|
|
49
|
+
new(
|
|
50
|
+
name:, input: redact.input, output: redact.output, storage_name: redact.storage,
|
|
51
|
+
redaction: redact.redaction || Proto::V1::Redaction.new, context:,
|
|
52
|
+
)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# @return [Array<String>] the read artifact
|
|
56
|
+
#: () -> Array[String]
|
|
57
|
+
def inputs = [@input]
|
|
58
|
+
|
|
59
|
+
# @return [Array<String>] the produced artifact
|
|
60
|
+
#: () -> Array[String]
|
|
61
|
+
def outputs = [@output]
|
|
62
|
+
|
|
63
|
+
# @param state [Proto::V1::NodeState] the node and its input artifact
|
|
64
|
+
# @return [Proto::V1::NodeState] the state with the redacted artifact
|
|
65
|
+
# @raise [MissingArtifact] when the input artifact is absent
|
|
66
|
+
# @raise [Glazed::Error] when reading, redacting or storing fails
|
|
67
|
+
#: (Proto::V1::NodeState state) -> Proto::V1::NodeState
|
|
68
|
+
def call(state)
|
|
69
|
+
node = state.node or raise ArgumentError, "#{name}: state has no node"
|
|
70
|
+
redactor = Redactor.from_proto(@redaction, patterns: Dialects.find(node.dialect).sensitive_patterns)
|
|
71
|
+
content = redactor.call(@context.read(artifact(state, @input)))
|
|
72
|
+
result = @storage.write(key: Storages.key_for(node), content:, message: "#{node.name}: #{name}")
|
|
73
|
+
with_artifacts(state, @output => stored(@storage_name, result))
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require 'glazed/must'
|
|
7
|
+
require 'glazed/proto'
|
|
8
|
+
require 'glazed/proto_json'
|
|
9
|
+
require 'glazed/shell_command'
|
|
10
|
+
require 'glazed/steps'
|
|
11
|
+
require 'glazed/storages'
|
|
12
|
+
|
|
13
|
+
module Glazed
|
|
14
|
+
module Steps
|
|
15
|
+
# Per node: runs a command with /bin/sh -c. The input artifact's content is
|
|
16
|
+
# on stdin and stdout becomes the output artifact; both are optional.
|
|
17
|
+
#
|
|
18
|
+
# @example Step definition
|
|
19
|
+
# {"name": "normalize", "shell": {"run": "sed -e '/^ntp clock-period/d'",
|
|
20
|
+
# "input": "raw", "output": "normalized", "storage": "staging"}}
|
|
21
|
+
class Shell < NodeStep
|
|
22
|
+
# Seconds a command may run when the definition sets none.
|
|
23
|
+
DEFAULT_TIMEOUT = 300 #: Integer
|
|
24
|
+
|
|
25
|
+
# @param name [String] step name
|
|
26
|
+
# @param run [String] command line
|
|
27
|
+
# @param input [String, nil] artifact written to stdin
|
|
28
|
+
# @param output [String, nil] artifact produced from stdout
|
|
29
|
+
# @param storage_name [String, nil] storage for the output, by name
|
|
30
|
+
# @param context [StepContext] named storages
|
|
31
|
+
# @param timeout [Numeric] seconds before the command is killed
|
|
32
|
+
# @raise [StepContext::UnknownStorage] when the storage is not configured
|
|
33
|
+
#: (name: String, run: String, input: String?, output: String?, storage_name: String?, context: StepContext, ?timeout: Numeric) -> void
|
|
34
|
+
def initialize(name:, run:, input:, output:, storage_name:, context:, timeout: DEFAULT_TIMEOUT)
|
|
35
|
+
super(name:)
|
|
36
|
+
@run = run #: String
|
|
37
|
+
@input = input #: String?
|
|
38
|
+
@output = output #: String?
|
|
39
|
+
@storage_name = storage_name #: String?
|
|
40
|
+
@storage = (context.storage(storage_name) if storage_name) #: Storages::Base?
|
|
41
|
+
@context = context #: StepContext
|
|
42
|
+
@timeout = timeout #: Numeric
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Builds the step from {Proto::V1::ShellStep}.
|
|
46
|
+
#
|
|
47
|
+
# @param name [String] step name
|
|
48
|
+
# @param settings [Google::Protobuf::MessageExts] a {Proto::V1::ShellStep}
|
|
49
|
+
# @param context [StepContext] named storages
|
|
50
|
+
# @return [Shell] the step
|
|
51
|
+
#: (String name, Google::Protobuf::MessageExts settings, context: StepContext) -> Shell
|
|
52
|
+
def self.from_proto(name, settings, context:)
|
|
53
|
+
shell = Must.be_a(settings, Proto::V1::ShellStep)
|
|
54
|
+
new(
|
|
55
|
+
name:, run: shell.run, context:,
|
|
56
|
+
input: shell.input.empty? ? nil : shell.input,
|
|
57
|
+
output: shell.output.empty? ? nil : shell.output,
|
|
58
|
+
storage_name: shell.storage.empty? ? nil : shell.storage,
|
|
59
|
+
timeout: shell.timeout_seconds.zero? ? DEFAULT_TIMEOUT : shell.timeout_seconds,
|
|
60
|
+
)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# @return [Array<String>] the artifact written to stdin, if any
|
|
64
|
+
#: () -> Array[String]
|
|
65
|
+
def inputs = [@input].compact
|
|
66
|
+
|
|
67
|
+
# @return [Array<String>] the artifact produced from stdout, if any
|
|
68
|
+
#: () -> Array[String]
|
|
69
|
+
def outputs = [@output].compact
|
|
70
|
+
|
|
71
|
+
# @param state [Proto::V1::NodeState] the node and its artifacts
|
|
72
|
+
# @return [Proto::V1::NodeState] the state, with the output artifact when configured
|
|
73
|
+
# @raise [ShellCommand::Failed] when the command exits non-zero
|
|
74
|
+
# @raise [TimeoutError] when the command does not finish in time
|
|
75
|
+
# @raise [MissingArtifact] when the input artifact is absent
|
|
76
|
+
#: (Proto::V1::NodeState state) -> Proto::V1::NodeState
|
|
77
|
+
def call(state)
|
|
78
|
+
node = state.node or raise ArgumentError, "#{name}: state has no node"
|
|
79
|
+
input = @input
|
|
80
|
+
stdin = input ? @context.read(artifact(state, input)) : ''
|
|
81
|
+
stdout = ShellCommand.new(run: @run, env: env(node), stdin:, timeout: @timeout).call
|
|
82
|
+
|
|
83
|
+
output = @output
|
|
84
|
+
storage = @storage
|
|
85
|
+
storage_name = @storage_name
|
|
86
|
+
return state unless output && storage && storage_name
|
|
87
|
+
|
|
88
|
+
result = storage.write(key: Storages.key_for(node), content: stdout, message: "#{node.name}: #{name}")
|
|
89
|
+
with_artifacts(state, output => stored(storage_name, result))
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# @param node [Proto::V1::Node] the node
|
|
93
|
+
# @return [Hash{String => String}] environment describing the step and node
|
|
94
|
+
#: (Proto::V1::Node node) -> Hash[String, String]
|
|
95
|
+
private def env(node)
|
|
96
|
+
{
|
|
97
|
+
'GLAZED_STEP' => name,
|
|
98
|
+
'GLAZED_NODE_NAME' => node.name,
|
|
99
|
+
'GLAZED_NODE_GROUP' => node.group,
|
|
100
|
+
'GLAZED_NODE_DIALECT' => node.dialect,
|
|
101
|
+
'GLAZED_NODE_HOST' => node.host.empty? ? node.name : node.host,
|
|
102
|
+
'GLAZED_NODE' => ProtoJson.dump(node),
|
|
103
|
+
}
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
data/lib/glazed/steps.rb
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require 'glazed'
|
|
7
|
+
require 'glazed/finder'
|
|
8
|
+
require 'glazed/must'
|
|
9
|
+
require 'glazed/plugin_registry'
|
|
10
|
+
require 'glazed/proto'
|
|
11
|
+
require 'glazed/step_context'
|
|
12
|
+
require 'glazed/steps/base'
|
|
13
|
+
require 'glazed/validation'
|
|
14
|
+
|
|
15
|
+
module Glazed
|
|
16
|
+
# Units of work in a workflow. Each takes and returns a protobuf state
|
|
17
|
+
# ({Proto::V1::NodeState} or {Proto::V1::BatchState}), so a coordinator can
|
|
18
|
+
# pass them between processes as ProtoJSON.
|
|
19
|
+
module Steps
|
|
20
|
+
# Raised when a step definition is incomplete.
|
|
21
|
+
class InvalidStep < Glazed::Error; end
|
|
22
|
+
|
|
23
|
+
# Finds a step class by snake_case name.
|
|
24
|
+
#
|
|
25
|
+
# @param name [String, Symbol] step kind, such as "fetch"
|
|
26
|
+
# @return [Class<Base>] the step class
|
|
27
|
+
# @raise [Finder::NotFound] when there is none, for example when the gem providing it is not installed
|
|
28
|
+
# @raise [Must::Violation] when the name resolves to something other than a step
|
|
29
|
+
#: (String | Symbol name) -> singleton(Base)
|
|
30
|
+
def self.find(name)
|
|
31
|
+
found = Finder.find(self, 'glazed/steps', name)
|
|
32
|
+
return found if found.is_a?(Class) && found < Base && found != NodeStep && found != BatchStep
|
|
33
|
+
|
|
34
|
+
raise Must::Violation, "#{found} is not a step"
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Steps implemented by plugins, by settings message type.
|
|
38
|
+
PLUGINS = PluginRegistry.new('step') #: PluginRegistry
|
|
39
|
+
|
|
40
|
+
# Registers a plugin step, selected by `plugin` steps whose settings are of the given type.
|
|
41
|
+
#
|
|
42
|
+
# @param settings_class [Class] generated message class of the step settings
|
|
43
|
+
# @param implementation [Class<NodeStep>, Class<BatchStep>] step built with {Base.from_proto}
|
|
44
|
+
# @return [void]
|
|
45
|
+
#: (singleton(Google::Protobuf::AbstractMessage) settings_class, singleton(Base) implementation) -> void
|
|
46
|
+
def self.register(settings_class, implementation)
|
|
47
|
+
PLUGINS.register(settings_class, implementation)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Builds the step a definition selects.
|
|
51
|
+
#
|
|
52
|
+
# @param step [Proto::V1::Step] step definition
|
|
53
|
+
# @param context [StepContext] named storages the step refers to
|
|
54
|
+
# @return [Base] the step
|
|
55
|
+
# @raise [InvalidMessage] when the definition breaks the rules of its schema
|
|
56
|
+
# @raise [StepContext::UnknownStorage] when the step names an unknown storage
|
|
57
|
+
# @raise [PluginRegistry::UnknownPlugin] when no plugin is registered for the settings type
|
|
58
|
+
#: (Proto::V1::Step step, context: StepContext) -> Base
|
|
59
|
+
def self.build(step, context:)
|
|
60
|
+
Validation.validate!(step)
|
|
61
|
+
action = step.action or raise InvalidStep, "step #{step.name.inspect} sets no action"
|
|
62
|
+
if (packed = step.plugin)
|
|
63
|
+
implementation, settings = PLUGINS.resolve(packed)
|
|
64
|
+
return step_class(implementation).from_proto(step.name, settings, context:)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
settings = Must.be_a(step[action.to_s], Google::Protobuf::MessageExts)
|
|
68
|
+
find(action).from_proto(step.name, settings, context:)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# @param implementation [Module] registered implementation
|
|
72
|
+
# @return [Class<Base>] the implementation as a step class
|
|
73
|
+
# @raise [Must::Violation] when it is not a per-node or batch step
|
|
74
|
+
#: (Module[untyped] implementation) -> singleton(Base)
|
|
75
|
+
private_class_method def self.step_class(implementation)
|
|
76
|
+
if implementation.is_a?(Class) && (implementation < NodeStep || implementation < BatchStep)
|
|
77
|
+
return implementation
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
raise Must::Violation, "#{implementation} is not a step"
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require 'google/protobuf'
|
|
7
|
+
require 'glazed'
|
|
8
|
+
require 'glazed/must'
|
|
9
|
+
require 'glazed/proto'
|
|
10
|
+
|
|
11
|
+
module Glazed
|
|
12
|
+
module Storages
|
|
13
|
+
# Store of configuration files addressed by slash-separated keys, such as
|
|
14
|
+
# "core/rt1". Subclasses implement {.from_proto}, {#write_all} and {#read},
|
|
15
|
+
# and those reporting {#conditional?} also {#version_of}, {#read_versioned}
|
|
16
|
+
# and {#write_if}.
|
|
17
|
+
class Base
|
|
18
|
+
# Raised for a key that is empty or could escape the storage root.
|
|
19
|
+
class InvalidKey < Glazed::Error; end
|
|
20
|
+
# Raised by {#write_if} when the key no longer holds the expected version.
|
|
21
|
+
class Conflict < Glazed::Error; end
|
|
22
|
+
|
|
23
|
+
#: type file = [String, String]
|
|
24
|
+
#: type message_builder = ^(Array[String] changed_keys) -> String
|
|
25
|
+
|
|
26
|
+
# Outcome of writing one file.
|
|
27
|
+
class WriteResult
|
|
28
|
+
# @param pointer [Proto::V1::Pointer] location holding the content
|
|
29
|
+
# @param changed [Boolean] whether the content differs from what the key held before
|
|
30
|
+
#: (pointer: Proto::V1::Pointer, changed: bool) -> void
|
|
31
|
+
def initialize(pointer:, changed:)
|
|
32
|
+
@pointer = pointer #: Proto::V1::Pointer
|
|
33
|
+
@changed = changed #: bool
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# @return [Proto::V1::Pointer] location holding the content; the existing one when unchanged
|
|
37
|
+
#: Proto::V1::Pointer
|
|
38
|
+
attr_reader :pointer
|
|
39
|
+
|
|
40
|
+
# @return [Boolean] whether the content differs from what the key held before
|
|
41
|
+
#: () -> bool
|
|
42
|
+
def changed? = @changed
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Content of a key together with the version that identifies it.
|
|
46
|
+
class Versioned
|
|
47
|
+
# @param content [String] the content
|
|
48
|
+
# @param version [String] opaque version, such as an ETag
|
|
49
|
+
#: (content: String, version: String) -> void
|
|
50
|
+
def initialize(content:, version:)
|
|
51
|
+
@content = content #: String
|
|
52
|
+
@version = version #: String
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# @return [String] the content
|
|
56
|
+
#: String
|
|
57
|
+
attr_reader :content
|
|
58
|
+
|
|
59
|
+
# @return [String] opaque version, such as an ETag
|
|
60
|
+
#: String
|
|
61
|
+
attr_reader :version
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Builds a storage from its settings message.
|
|
65
|
+
#
|
|
66
|
+
# @param settings [Google::Protobuf::MessageExts] backend settings from {Proto::V1::StorageConfig}
|
|
67
|
+
# @return [Base] the storage
|
|
68
|
+
#: (Google::Protobuf::MessageExts settings) -> Base
|
|
69
|
+
def self.from_proto(settings) = raise NotImplementedError
|
|
70
|
+
|
|
71
|
+
# Writes one file.
|
|
72
|
+
#
|
|
73
|
+
# @param key [String] file key
|
|
74
|
+
# @param content [String] file content
|
|
75
|
+
# @param message [String] description of the change, such as a commit message
|
|
76
|
+
# @return [WriteResult] where the content is and whether it changed
|
|
77
|
+
# @raise [InvalidKey] for an unsafe key
|
|
78
|
+
#: (key: String, content: String, message: String) -> WriteResult
|
|
79
|
+
def write(key:, content:, message:)
|
|
80
|
+
write_all([[key, content]], message: ->(_changed_keys) { message }).fetch(0)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Writes several files as one change where the storage supports it. Files
|
|
84
|
+
# whose content equals what their key holds are not written again.
|
|
85
|
+
#
|
|
86
|
+
# @param files [Array<Array(String, String)>] keys and contents
|
|
87
|
+
# @param message [Proc] builds the description of the change, such as a
|
|
88
|
+
# commit message, from the keys whose content changed; not called when none did
|
|
89
|
+
# @return [Array<WriteResult>] results, in the order of files
|
|
90
|
+
# @raise [InvalidKey] for an unsafe key
|
|
91
|
+
#: (Array[Base::file] files, message: Base::message_builder) -> Array[WriteResult]
|
|
92
|
+
def write_all(files, message:) = raise NotImplementedError
|
|
93
|
+
|
|
94
|
+
# Reads content written earlier.
|
|
95
|
+
#
|
|
96
|
+
# @param pointer [Proto::V1::Pointer] location returned by a write
|
|
97
|
+
# @return [String] the content
|
|
98
|
+
#: (Proto::V1::Pointer pointer) -> String
|
|
99
|
+
def read(pointer) = raise NotImplementedError
|
|
100
|
+
|
|
101
|
+
# Whether the storage implements {#version_of}, {#read_versioned} and
|
|
102
|
+
# {#write_if}, so a single key can be updated by optimistic concurrency.
|
|
103
|
+
#
|
|
104
|
+
# @return [Boolean] true when supported
|
|
105
|
+
#: () -> bool
|
|
106
|
+
def conditional? = false
|
|
107
|
+
|
|
108
|
+
# Version the key currently holds, without reading the content.
|
|
109
|
+
#
|
|
110
|
+
# @param key [String] file key
|
|
111
|
+
# @return [String, nil] the version, or nil when the key is absent
|
|
112
|
+
# @raise [InvalidKey] for an unsafe key
|
|
113
|
+
#: (String key) -> String?
|
|
114
|
+
def version_of(key) = raise NotImplementedError
|
|
115
|
+
|
|
116
|
+
# Reads the current content of a key.
|
|
117
|
+
#
|
|
118
|
+
# @param key [String] file key
|
|
119
|
+
# @return [Versioned, nil] the content and its version, or nil when the key is absent
|
|
120
|
+
# @raise [InvalidKey] for an unsafe key
|
|
121
|
+
#: (String key) -> Versioned?
|
|
122
|
+
def read_versioned(key) = raise NotImplementedError
|
|
123
|
+
|
|
124
|
+
# Writes a key only if it still holds the expected version.
|
|
125
|
+
#
|
|
126
|
+
# @param key [String] file key
|
|
127
|
+
# @param content [String] new content
|
|
128
|
+
# @param expected_version [String, nil] version read earlier; nil to create a missing key only
|
|
129
|
+
# @return [String] version of the written content
|
|
130
|
+
# @raise [Conflict] when the key holds another version, or exists when nil was expected
|
|
131
|
+
# @raise [InvalidKey] for an unsafe key
|
|
132
|
+
#: (key: String, content: String, expected_version: String?) -> String
|
|
133
|
+
def write_if(key:, content:, expected_version:) = raise NotImplementedError
|
|
134
|
+
|
|
135
|
+
# Wraps a location in a pointer.
|
|
136
|
+
#
|
|
137
|
+
# @param location [Google::Protobuf::MessageExts] storage-specific location
|
|
138
|
+
# @return [Proto::V1::Pointer] the pointer
|
|
139
|
+
#: (Google::Protobuf::MessageExts location) -> Proto::V1::Pointer
|
|
140
|
+
private def pointer_to(location) = Proto::V1::Pointer.new(location: Google::Protobuf::Any.pack(location))
|
|
141
|
+
|
|
142
|
+
# Unwraps the location of a pointer.
|
|
143
|
+
#
|
|
144
|
+
# @param pointer [Proto::V1::Pointer] the pointer
|
|
145
|
+
# @param location_class [Class] location message this storage writes
|
|
146
|
+
# @return [Google::Protobuf::MessageExts] the location
|
|
147
|
+
# @raise [ArgumentError] when the pointer holds another kind of location
|
|
148
|
+
#: [L] (Proto::V1::Pointer pointer, Class[L] location_class) -> L
|
|
149
|
+
private def location_of(pointer, location_class)
|
|
150
|
+
any = pointer.location or raise ArgumentError, 'pointer has no location'
|
|
151
|
+
location = any.unpack(location_class) or raise ArgumentError, "#{self.class.name} cannot read a pointer to #{any.type_name}"
|
|
152
|
+
Must.be_a(location, location_class)
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# @param key [String] key to check
|
|
156
|
+
# @return [String] the key
|
|
157
|
+
# @raise [InvalidKey] when the key is empty, absolute, or has empty, "." or ".." segments
|
|
158
|
+
#: (String key) -> String
|
|
159
|
+
private def validate_key(key)
|
|
160
|
+
segments = key.split('/', -1)
|
|
161
|
+
raise InvalidKey, "invalid storage key #{key.inspect}" if segments.empty? || segments.any? { ['', '.', '..'].include?(it) }
|
|
162
|
+
|
|
163
|
+
key
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require 'digest'
|
|
7
|
+
require 'fileutils'
|
|
8
|
+
require 'securerandom'
|
|
9
|
+
require 'glazed'
|
|
10
|
+
require 'glazed/must'
|
|
11
|
+
require 'glazed/proto'
|
|
12
|
+
require 'glazed/storages/base'
|
|
13
|
+
|
|
14
|
+
module Glazed
|
|
15
|
+
module Storages
|
|
16
|
+
# Local directory holding the latest content of each key, for scripts and
|
|
17
|
+
# tools that read plain files. It keeps no history.
|
|
18
|
+
#
|
|
19
|
+
# @example
|
|
20
|
+
# storage = Glazed::Storages::Filesystem.new(path: '/var/lib/glazed/latest')
|
|
21
|
+
# storage.write(key: 'core/rt1', content: config, message: 'rt1')
|
|
22
|
+
class Filesystem < Base
|
|
23
|
+
# Raised when a file no longer holds the content its pointer names.
|
|
24
|
+
class StaleContent < Glazed::Error; end
|
|
25
|
+
|
|
26
|
+
# @param path [String] directory holding the files, created when missing
|
|
27
|
+
#: (path: String) -> void
|
|
28
|
+
def initialize(path:)
|
|
29
|
+
super()
|
|
30
|
+
@path = path #: String
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Builds a storage from {Proto::V1::FilesystemStorage}.
|
|
34
|
+
#
|
|
35
|
+
# @param settings [Google::Protobuf::MessageExts] a {Proto::V1::FilesystemStorage}
|
|
36
|
+
# @return [Filesystem] the storage
|
|
37
|
+
# @raise [Must::Violation] when settings is another message
|
|
38
|
+
#: (Google::Protobuf::MessageExts settings) -> Filesystem
|
|
39
|
+
def self.from_proto(settings)
|
|
40
|
+
filesystem = Must.be_a(settings, Proto::V1::FilesystemStorage)
|
|
41
|
+
new(path: filesystem.path)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# @return [String] directory holding the files
|
|
45
|
+
#: String
|
|
46
|
+
attr_reader :path
|
|
47
|
+
|
|
48
|
+
# Writes each changed file in place. A file is replaced atomically, so
|
|
49
|
+
# readers see either the old or the new content.
|
|
50
|
+
#
|
|
51
|
+
# @param files [Array<Array(String, String)>] keys and contents
|
|
52
|
+
# @param message [Proc] builds a description of the change, unused
|
|
53
|
+
# @return [Array<WriteResult>] the content's location and digest, and whether the file changed
|
|
54
|
+
# @raise [InvalidKey] for an unsafe key
|
|
55
|
+
#: (Array[Base::file] files, message: Base::message_builder) -> Array[WriteResult]
|
|
56
|
+
def write_all(files, message:)
|
|
57
|
+
files.each { |key, _| validate_key(key) }
|
|
58
|
+
files.map do |key, content|
|
|
59
|
+
file = ::File.join(path, key)
|
|
60
|
+
changed = !(::File.file?(file) && ::File.binread(file) == content.b)
|
|
61
|
+
replace(file, content) if changed
|
|
62
|
+
pointer = pointer_to(Proto::V1::FilesystemLocation.new(path: key, sha256: Digest::SHA256.hexdigest(content)))
|
|
63
|
+
WriteResult.new(pointer:, changed:)
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Reads the file a pointer names.
|
|
68
|
+
#
|
|
69
|
+
# @param pointer [Proto::V1::Pointer] pointer with a filesystem location
|
|
70
|
+
# @return [String] the content
|
|
71
|
+
# @raise [ArgumentError] when the pointer is not a filesystem location
|
|
72
|
+
# @raise [StaleContent] when the file has been rewritten with other content since
|
|
73
|
+
# @raise [Errno::ENOENT] when the file is gone
|
|
74
|
+
#: (Proto::V1::Pointer pointer) -> String
|
|
75
|
+
def read(pointer)
|
|
76
|
+
location = location_of(pointer, Proto::V1::FilesystemLocation)
|
|
77
|
+
content = ::File.read(::File.join(path, validate_key(location.path)))
|
|
78
|
+
unless Digest::SHA256.hexdigest(content) == location.sha256
|
|
79
|
+
raise StaleContent, "#{location.path} in #{path} no longer holds the content written"
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
content
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# @return [Boolean] true; versions are content digests
|
|
86
|
+
#: () -> bool
|
|
87
|
+
def conditional? = true
|
|
88
|
+
|
|
89
|
+
# @param key [String] file key
|
|
90
|
+
# @return [String, nil] SHA-256 of the file, or nil when it is missing
|
|
91
|
+
# @raise [InvalidKey] for an unsafe key
|
|
92
|
+
#: (String key) -> String?
|
|
93
|
+
def version_of(key) = read_versioned(key)&.version
|
|
94
|
+
|
|
95
|
+
# @param key [String] file key
|
|
96
|
+
# @return [Versioned, nil] the content and its SHA-256, or nil when the file is missing
|
|
97
|
+
# @raise [InvalidKey] for an unsafe key
|
|
98
|
+
#: (String key) -> Versioned?
|
|
99
|
+
def read_versioned(key)
|
|
100
|
+
content = ::File.binread(::File.join(path, validate_key(key)))
|
|
101
|
+
Versioned.new(content:, version: Digest::SHA256.hexdigest(content))
|
|
102
|
+
rescue Errno::ENOENT
|
|
103
|
+
nil
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Replaces the file while holding an exclusive lock on a sibling lock
|
|
107
|
+
# file, which every conditional writer of the key takes.
|
|
108
|
+
#
|
|
109
|
+
# @param key [String] file key
|
|
110
|
+
# @param content [String] new content
|
|
111
|
+
# @param expected_version [String, nil] SHA-256 read earlier; nil to create a missing file only
|
|
112
|
+
# @return [String] SHA-256 of the content
|
|
113
|
+
# @raise [Conflict] when the file holds other content
|
|
114
|
+
# @raise [InvalidKey] for an unsafe key
|
|
115
|
+
#: (key: String, content: String, expected_version: String?) -> String
|
|
116
|
+
def write_if(key:, content:, expected_version:)
|
|
117
|
+
file = ::File.join(path, validate_key(key))
|
|
118
|
+
FileUtils.mkdir_p(::File.dirname(file))
|
|
119
|
+
::File.open(::File.join(::File.dirname(file), ".#{::File.basename(file)}.lock"), ::File::RDWR | ::File::CREAT, 0o644) do |lock|
|
|
120
|
+
lock.flock(::File::LOCK_EX)
|
|
121
|
+
current = version_of(key)
|
|
122
|
+
raise Conflict, "#{key} in #{path} is at #{current || 'nothing'}, not #{expected_version || 'nothing'}" unless current == expected_version
|
|
123
|
+
|
|
124
|
+
replace(file, content)
|
|
125
|
+
end
|
|
126
|
+
Digest::SHA256.hexdigest(content)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# @param file [String] file to replace
|
|
130
|
+
# @param content [String] new content
|
|
131
|
+
# @return [void]
|
|
132
|
+
#: (String file, String content) -> void
|
|
133
|
+
private def replace(file, content)
|
|
134
|
+
dir = ::File.dirname(file)
|
|
135
|
+
FileUtils.mkdir_p(dir)
|
|
136
|
+
temporary = ::File.join(dir, ".#{::File.basename(file)}.#{SecureRandom.hex(8)}.tmp")
|
|
137
|
+
::File.binwrite(temporary, content)
|
|
138
|
+
::File.rename(temporary, file)
|
|
139
|
+
ensure
|
|
140
|
+
::File.unlink(temporary) if temporary && ::File.exist?(temporary)
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|