glazed 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. checksums.yaml +7 -0
  2. data/.rspec +3 -0
  3. data/.yardopts +5 -0
  4. data/CHANGELOG.md +18 -0
  5. data/LICENSE.txt +202 -0
  6. data/NOTICE +16 -0
  7. data/README.md +3 -0
  8. data/Rakefile +12 -0
  9. data/exe/glazed +8 -0
  10. data/lib/glazed/cli_session.rb +178 -0
  11. data/lib/glazed/collector.rb +113 -0
  12. data/lib/glazed/command/base.rb +146 -0
  13. data/lib/glazed/command/orchestrate.rb +78 -0
  14. data/lib/glazed/command/run.rb +51 -0
  15. data/lib/glazed/command/send_signal.rb +69 -0
  16. data/lib/glazed/command/status.rb +179 -0
  17. data/lib/glazed/command/version.rb +23 -0
  18. data/lib/glazed/command.rb +39 -0
  19. data/lib/glazed/command_result.rb +26 -0
  20. data/lib/glazed/dialect/actions.rb +95 -0
  21. data/lib/glazed/dialect/communication_config.rb +63 -0
  22. data/lib/glazed/dialect/interaction.rb +32 -0
  23. data/lib/glazed/dialect/run.rb +84 -0
  24. data/lib/glazed/dialect/runbook.rb +97 -0
  25. data/lib/glazed/dialect/sensitive_pattern.rb +31 -0
  26. data/lib/glazed/dialect.rb +12 -0
  27. data/lib/glazed/dialects/awplus.rb +84 -0
  28. data/lib/glazed/dialects/base.rb +246 -0
  29. data/lib/glazed/dialects/ios.rb +173 -0
  30. data/lib/glazed/dialects/junos.rb +78 -0
  31. data/lib/glazed/dialects/nec_ix.rb +71 -0
  32. data/lib/glazed/dialects/wlc.rb +56 -0
  33. data/lib/glazed/dialects.rb +65 -0
  34. data/lib/glazed/finder.rb +62 -0
  35. data/lib/glazed/must.rb +54 -0
  36. data/lib/glazed/orchestrator/policy.rb +184 -0
  37. data/lib/glazed/orchestrator.rb +288 -0
  38. data/lib/glazed/plugin_registry.rb +70 -0
  39. data/lib/glazed/proto.rb +20 -0
  40. data/lib/glazed/proto_json.rb +34 -0
  41. data/lib/glazed/redactor.rb +158 -0
  42. data/lib/glazed/runner.rb +176 -0
  43. data/lib/glazed/secret.rb +74 -0
  44. data/lib/glazed/secrets.rb +59 -0
  45. data/lib/glazed/shell_command.rb +90 -0
  46. data/lib/glazed/sources/base.rb +79 -0
  47. data/lib/glazed/sources/command.rb +67 -0
  48. data/lib/glazed/sources/inline.rb +56 -0
  49. data/lib/glazed/sources/json_files.rb +87 -0
  50. data/lib/glazed/sources.rb +105 -0
  51. data/lib/glazed/state_backends/base.rb +72 -0
  52. data/lib/glazed/state_backends/sqlite.rb +299 -0
  53. data/lib/glazed/state_backends.rb +75 -0
  54. data/lib/glazed/step_context.rb +65 -0
  55. data/lib/glazed/steps/base.rb +89 -0
  56. data/lib/glazed/steps/batch_shell.rb +57 -0
  57. data/lib/glazed/steps/fetch.rb +67 -0
  58. data/lib/glazed/steps/publish.rb +94 -0
  59. data/lib/glazed/steps/redact.rb +77 -0
  60. data/lib/glazed/steps/shell.rb +107 -0
  61. data/lib/glazed/steps.rb +83 -0
  62. data/lib/glazed/storages/base.rb +167 -0
  63. data/lib/glazed/storages/filesystem.rb +144 -0
  64. data/lib/glazed/storages/git.rb +181 -0
  65. data/lib/glazed/storages/github.rb +269 -0
  66. data/lib/glazed/storages/memory.rb +125 -0
  67. data/lib/glazed/storages.rb +88 -0
  68. data/lib/glazed/transports/base.rb +103 -0
  69. data/lib/glazed/transports/ssh.rb +177 -0
  70. data/lib/glazed/transports/telnet.rb +100 -0
  71. data/lib/glazed/transports.rb +28 -0
  72. data/lib/glazed/v1/config_pb.rb +25 -0
  73. data/lib/glazed/v1/inventory_pb.rb +27 -0
  74. data/lib/glazed/v1/orchestrator_pb.rb +27 -0
  75. data/lib/glazed/v1/redaction_pb.rb +23 -0
  76. data/lib/glazed/v1/secret_pb.rb +22 -0
  77. data/lib/glazed/v1/state_pb.rb +25 -0
  78. data/lib/glazed/v1/steps_pb.rb +25 -0
  79. data/lib/glazed/v1/storage_pb.rb +34 -0
  80. data/lib/glazed/v1/workflow_pb.rb +29 -0
  81. data/lib/glazed/validation.rb +31 -0
  82. data/lib/glazed/version.rb +9 -0
  83. data/lib/glazed/workflow.rb +70 -0
  84. data/lib/glazed.rb +18 -0
  85. metadata +245 -0
@@ -0,0 +1,94 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'glazed/must'
7
+ require 'glazed/proto'
8
+ require 'glazed/steps'
9
+ require 'glazed/storages'
10
+
11
+ module Glazed
12
+ module Steps
13
+ # Batch: writes an artifact of every node into a storage as one change,
14
+ # such as one commit naming the nodes.
15
+ #
16
+ # @example Step definition
17
+ # {"name": "publish", "publish": {"input": "redacted", "storage": "team"}}
18
+ class Publish < BatchStep
19
+ # Node names listed in a commit message before it is shortened.
20
+ MESSAGE_NAMES = 5 #: Integer
21
+
22
+ # @param name [String] step name
23
+ # @param input [String] artifact to read from each node
24
+ # @param storage_name [String] storage to write to, by name
25
+ # @param output [String, nil] artifact recording the written locations
26
+ # @param context [StepContext] named storages
27
+ # @raise [StepContext::UnknownStorage] when the storage is not configured
28
+ #: (name: String, input: String, storage_name: String, output: String?, context: StepContext) -> void
29
+ def initialize(name:, input:, storage_name:, output:, context:)
30
+ super(name:)
31
+ @input = input #: String
32
+ @storage_name = storage_name #: String
33
+ @storage = context.storage(storage_name) #: Storages::Base
34
+ @output = output #: String?
35
+ @context = context #: StepContext
36
+ end
37
+
38
+ # Builds the step from {Proto::V1::PublishStep}.
39
+ #
40
+ # @param name [String] step name
41
+ # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::PublishStep}
42
+ # @param context [StepContext] named storages
43
+ # @return [Publish] the step
44
+ #: (String name, Google::Protobuf::MessageExts settings, context: StepContext) -> Publish
45
+ def self.from_proto(name, settings, context:)
46
+ publish = Must.be_a(settings, Proto::V1::PublishStep)
47
+ new(name:, input: publish.input, storage_name: publish.storage, output: publish.output.empty? ? nil : publish.output, context:)
48
+ end
49
+
50
+ # @return [Array<String>] the read artifact
51
+ #: () -> Array[String]
52
+ def inputs = [@input]
53
+
54
+ # @return [Array<String>] the recorded artifact, if any
55
+ #: () -> Array[String]
56
+ def outputs = [@output].compact
57
+
58
+ # @param batch [Proto::V1::BatchState] node states holding the input artifact
59
+ # @return [Proto::V1::BatchState] the states, with the output artifact when configured
60
+ # @raise [MissingArtifact] when a node lacks the input artifact
61
+ # @raise [Glazed::Error] when reading or storing fails
62
+ #: (Proto::V1::BatchState batch) -> Proto::V1::BatchState
63
+ def call(batch)
64
+ states = batch.nodes.to_a
65
+ return batch if states.empty?
66
+
67
+ names = {} #: Hash[String, String]
68
+ files = states.map do |state|
69
+ node = state.node or raise ArgumentError, "#{name}: state has no node"
70
+ key = Storages.key_for(node)
71
+ names[key] = node.name
72
+ [key, @context.read(artifact(state, @input))] #: Storages::Base::file
73
+ end
74
+ results = @storage.write_all(files, message: ->(changed_keys) { message(changed_keys.map { names.fetch(it, it) }) })
75
+ output = @output
76
+ return batch unless output
77
+
78
+ nodes = states.zip(results).map do |state, result|
79
+ with_artifacts(state, output => stored(@storage_name, Must.expect(result)))
80
+ end
81
+ Proto::V1::BatchState.new(nodes:)
82
+ end
83
+
84
+ # @param names [Array<String>] names of the nodes whose content changed
85
+ # @return [String] commit message naming the nodes
86
+ #: (Array[String] names) -> String
87
+ private def message(names)
88
+ listed = names.first(MESSAGE_NAMES).join(', ')
89
+ rest = names.size - MESSAGE_NAMES
90
+ rest.positive? ? "#{name}: update #{listed} and #{rest} more" : "#{name}: update #{listed}"
91
+ end
92
+ end
93
+ end
94
+ end
@@ -0,0 +1,77 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'glazed/dialects'
7
+ require 'glazed/must'
8
+ require 'glazed/proto'
9
+ require 'glazed/redactor'
10
+ require 'glazed/steps'
11
+ require 'glazed/storages'
12
+
13
+ module Glazed
14
+ module Steps
15
+ # Per node: hides secrets in an artifact using the node dialect's sensitive
16
+ # patterns, and stores the result as another artifact.
17
+ #
18
+ # @example Step definition
19
+ # {"name": "redact", "redact": {"input": "raw", "output": "redacted", "storage": "staging",
20
+ # "redaction": {"mode": "MODE_HMAC", "hmac_key": {"env": "GLAZED_HMAC_KEY"}}}}
21
+ class Redact < NodeStep
22
+ # @param name [String] step name
23
+ # @param input [String] artifact to read
24
+ # @param output [String] artifact to produce
25
+ # @param storage_name [String] storage to write to, by name
26
+ # @param redaction [Proto::V1::Redaction] how secrets are hidden
27
+ # @param context [StepContext] named storages
28
+ # @raise [StepContext::UnknownStorage] when the storage is not configured
29
+ #: (name: String, input: String, output: String, storage_name: String, redaction: Proto::V1::Redaction, context: StepContext) -> void
30
+ def initialize(name:, input:, output:, storage_name:, redaction:, context:)
31
+ super(name:)
32
+ @input = input #: String
33
+ @output = output #: String
34
+ @storage_name = storage_name #: String
35
+ @storage = context.storage(storage_name) #: Storages::Base
36
+ @redaction = redaction #: Proto::V1::Redaction
37
+ @context = context #: StepContext
38
+ end
39
+
40
+ # Builds the step from {Proto::V1::RedactStep}.
41
+ #
42
+ # @param name [String] step name
43
+ # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::RedactStep}
44
+ # @param context [StepContext] named storages
45
+ # @return [Redact] the step
46
+ #: (String name, Google::Protobuf::MessageExts settings, context: StepContext) -> Redact
47
+ def self.from_proto(name, settings, context:)
48
+ redact = Must.be_a(settings, Proto::V1::RedactStep)
49
+ new(
50
+ name:, input: redact.input, output: redact.output, storage_name: redact.storage,
51
+ redaction: redact.redaction || Proto::V1::Redaction.new, context:,
52
+ )
53
+ end
54
+
55
+ # @return [Array<String>] the read artifact
56
+ #: () -> Array[String]
57
+ def inputs = [@input]
58
+
59
+ # @return [Array<String>] the produced artifact
60
+ #: () -> Array[String]
61
+ def outputs = [@output]
62
+
63
+ # @param state [Proto::V1::NodeState] the node and its input artifact
64
+ # @return [Proto::V1::NodeState] the state with the redacted artifact
65
+ # @raise [MissingArtifact] when the input artifact is absent
66
+ # @raise [Glazed::Error] when reading, redacting or storing fails
67
+ #: (Proto::V1::NodeState state) -> Proto::V1::NodeState
68
+ def call(state)
69
+ node = state.node or raise ArgumentError, "#{name}: state has no node"
70
+ redactor = Redactor.from_proto(@redaction, patterns: Dialects.find(node.dialect).sensitive_patterns)
71
+ content = redactor.call(@context.read(artifact(state, @input)))
72
+ result = @storage.write(key: Storages.key_for(node), content:, message: "#{node.name}: #{name}")
73
+ with_artifacts(state, @output => stored(@storage_name, result))
74
+ end
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,107 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'glazed/must'
7
+ require 'glazed/proto'
8
+ require 'glazed/proto_json'
9
+ require 'glazed/shell_command'
10
+ require 'glazed/steps'
11
+ require 'glazed/storages'
12
+
13
+ module Glazed
14
+ module Steps
15
+ # Per node: runs a command with /bin/sh -c. The input artifact's content is
16
+ # on stdin and stdout becomes the output artifact; both are optional.
17
+ #
18
+ # @example Step definition
19
+ # {"name": "normalize", "shell": {"run": "sed -e '/^ntp clock-period/d'",
20
+ # "input": "raw", "output": "normalized", "storage": "staging"}}
21
+ class Shell < NodeStep
22
+ # Seconds a command may run when the definition sets none.
23
+ DEFAULT_TIMEOUT = 300 #: Integer
24
+
25
+ # @param name [String] step name
26
+ # @param run [String] command line
27
+ # @param input [String, nil] artifact written to stdin
28
+ # @param output [String, nil] artifact produced from stdout
29
+ # @param storage_name [String, nil] storage for the output, by name
30
+ # @param context [StepContext] named storages
31
+ # @param timeout [Numeric] seconds before the command is killed
32
+ # @raise [StepContext::UnknownStorage] when the storage is not configured
33
+ #: (name: String, run: String, input: String?, output: String?, storage_name: String?, context: StepContext, ?timeout: Numeric) -> void
34
+ def initialize(name:, run:, input:, output:, storage_name:, context:, timeout: DEFAULT_TIMEOUT)
35
+ super(name:)
36
+ @run = run #: String
37
+ @input = input #: String?
38
+ @output = output #: String?
39
+ @storage_name = storage_name #: String?
40
+ @storage = (context.storage(storage_name) if storage_name) #: Storages::Base?
41
+ @context = context #: StepContext
42
+ @timeout = timeout #: Numeric
43
+ end
44
+
45
+ # Builds the step from {Proto::V1::ShellStep}.
46
+ #
47
+ # @param name [String] step name
48
+ # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::ShellStep}
49
+ # @param context [StepContext] named storages
50
+ # @return [Shell] the step
51
+ #: (String name, Google::Protobuf::MessageExts settings, context: StepContext) -> Shell
52
+ def self.from_proto(name, settings, context:)
53
+ shell = Must.be_a(settings, Proto::V1::ShellStep)
54
+ new(
55
+ name:, run: shell.run, context:,
56
+ input: shell.input.empty? ? nil : shell.input,
57
+ output: shell.output.empty? ? nil : shell.output,
58
+ storage_name: shell.storage.empty? ? nil : shell.storage,
59
+ timeout: shell.timeout_seconds.zero? ? DEFAULT_TIMEOUT : shell.timeout_seconds,
60
+ )
61
+ end
62
+
63
+ # @return [Array<String>] the artifact written to stdin, if any
64
+ #: () -> Array[String]
65
+ def inputs = [@input].compact
66
+
67
+ # @return [Array<String>] the artifact produced from stdout, if any
68
+ #: () -> Array[String]
69
+ def outputs = [@output].compact
70
+
71
+ # @param state [Proto::V1::NodeState] the node and its artifacts
72
+ # @return [Proto::V1::NodeState] the state, with the output artifact when configured
73
+ # @raise [ShellCommand::Failed] when the command exits non-zero
74
+ # @raise [TimeoutError] when the command does not finish in time
75
+ # @raise [MissingArtifact] when the input artifact is absent
76
+ #: (Proto::V1::NodeState state) -> Proto::V1::NodeState
77
+ def call(state)
78
+ node = state.node or raise ArgumentError, "#{name}: state has no node"
79
+ input = @input
80
+ stdin = input ? @context.read(artifact(state, input)) : ''
81
+ stdout = ShellCommand.new(run: @run, env: env(node), stdin:, timeout: @timeout).call
82
+
83
+ output = @output
84
+ storage = @storage
85
+ storage_name = @storage_name
86
+ return state unless output && storage && storage_name
87
+
88
+ result = storage.write(key: Storages.key_for(node), content: stdout, message: "#{node.name}: #{name}")
89
+ with_artifacts(state, output => stored(storage_name, result))
90
+ end
91
+
92
+ # @param node [Proto::V1::Node] the node
93
+ # @return [Hash{String => String}] environment describing the step and node
94
+ #: (Proto::V1::Node node) -> Hash[String, String]
95
+ private def env(node)
96
+ {
97
+ 'GLAZED_STEP' => name,
98
+ 'GLAZED_NODE_NAME' => node.name,
99
+ 'GLAZED_NODE_GROUP' => node.group,
100
+ 'GLAZED_NODE_DIALECT' => node.dialect,
101
+ 'GLAZED_NODE_HOST' => node.host.empty? ? node.name : node.host,
102
+ 'GLAZED_NODE' => ProtoJson.dump(node),
103
+ }
104
+ end
105
+ end
106
+ end
107
+ end
@@ -0,0 +1,83 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'glazed'
7
+ require 'glazed/finder'
8
+ require 'glazed/must'
9
+ require 'glazed/plugin_registry'
10
+ require 'glazed/proto'
11
+ require 'glazed/step_context'
12
+ require 'glazed/steps/base'
13
+ require 'glazed/validation'
14
+
15
+ module Glazed
16
+ # Units of work in a workflow. Each takes and returns a protobuf state
17
+ # ({Proto::V1::NodeState} or {Proto::V1::BatchState}), so a coordinator can
18
+ # pass them between processes as ProtoJSON.
19
+ module Steps
20
+ # Raised when a step definition is incomplete.
21
+ class InvalidStep < Glazed::Error; end
22
+
23
+ # Finds a step class by snake_case name.
24
+ #
25
+ # @param name [String, Symbol] step kind, such as "fetch"
26
+ # @return [Class<Base>] the step class
27
+ # @raise [Finder::NotFound] when there is none, for example when the gem providing it is not installed
28
+ # @raise [Must::Violation] when the name resolves to something other than a step
29
+ #: (String | Symbol name) -> singleton(Base)
30
+ def self.find(name)
31
+ found = Finder.find(self, 'glazed/steps', name)
32
+ return found if found.is_a?(Class) && found < Base && found != NodeStep && found != BatchStep
33
+
34
+ raise Must::Violation, "#{found} is not a step"
35
+ end
36
+
37
+ # Steps implemented by plugins, by settings message type.
38
+ PLUGINS = PluginRegistry.new('step') #: PluginRegistry
39
+
40
+ # Registers a plugin step, selected by `plugin` steps whose settings are of the given type.
41
+ #
42
+ # @param settings_class [Class] generated message class of the step settings
43
+ # @param implementation [Class<NodeStep>, Class<BatchStep>] step built with {Base.from_proto}
44
+ # @return [void]
45
+ #: (singleton(Google::Protobuf::AbstractMessage) settings_class, singleton(Base) implementation) -> void
46
+ def self.register(settings_class, implementation)
47
+ PLUGINS.register(settings_class, implementation)
48
+ end
49
+
50
+ # Builds the step a definition selects.
51
+ #
52
+ # @param step [Proto::V1::Step] step definition
53
+ # @param context [StepContext] named storages the step refers to
54
+ # @return [Base] the step
55
+ # @raise [InvalidMessage] when the definition breaks the rules of its schema
56
+ # @raise [StepContext::UnknownStorage] when the step names an unknown storage
57
+ # @raise [PluginRegistry::UnknownPlugin] when no plugin is registered for the settings type
58
+ #: (Proto::V1::Step step, context: StepContext) -> Base
59
+ def self.build(step, context:)
60
+ Validation.validate!(step)
61
+ action = step.action or raise InvalidStep, "step #{step.name.inspect} sets no action"
62
+ if (packed = step.plugin)
63
+ implementation, settings = PLUGINS.resolve(packed)
64
+ return step_class(implementation).from_proto(step.name, settings, context:)
65
+ end
66
+
67
+ settings = Must.be_a(step[action.to_s], Google::Protobuf::MessageExts)
68
+ find(action).from_proto(step.name, settings, context:)
69
+ end
70
+
71
+ # @param implementation [Module] registered implementation
72
+ # @return [Class<Base>] the implementation as a step class
73
+ # @raise [Must::Violation] when it is not a per-node or batch step
74
+ #: (Module[untyped] implementation) -> singleton(Base)
75
+ private_class_method def self.step_class(implementation)
76
+ if implementation.is_a?(Class) && (implementation < NodeStep || implementation < BatchStep)
77
+ return implementation
78
+ end
79
+
80
+ raise Must::Violation, "#{implementation} is not a step"
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,167 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'google/protobuf'
7
+ require 'glazed'
8
+ require 'glazed/must'
9
+ require 'glazed/proto'
10
+
11
+ module Glazed
12
+ module Storages
13
+ # Store of configuration files addressed by slash-separated keys, such as
14
+ # "core/rt1". Subclasses implement {.from_proto}, {#write_all} and {#read},
15
+ # and those reporting {#conditional?} also {#version_of}, {#read_versioned}
16
+ # and {#write_if}.
17
+ class Base
18
+ # Raised for a key that is empty or could escape the storage root.
19
+ class InvalidKey < Glazed::Error; end
20
+ # Raised by {#write_if} when the key no longer holds the expected version.
21
+ class Conflict < Glazed::Error; end
22
+
23
+ #: type file = [String, String]
24
+ #: type message_builder = ^(Array[String] changed_keys) -> String
25
+
26
+ # Outcome of writing one file.
27
+ class WriteResult
28
+ # @param pointer [Proto::V1::Pointer] location holding the content
29
+ # @param changed [Boolean] whether the content differs from what the key held before
30
+ #: (pointer: Proto::V1::Pointer, changed: bool) -> void
31
+ def initialize(pointer:, changed:)
32
+ @pointer = pointer #: Proto::V1::Pointer
33
+ @changed = changed #: bool
34
+ end
35
+
36
+ # @return [Proto::V1::Pointer] location holding the content; the existing one when unchanged
37
+ #: Proto::V1::Pointer
38
+ attr_reader :pointer
39
+
40
+ # @return [Boolean] whether the content differs from what the key held before
41
+ #: () -> bool
42
+ def changed? = @changed
43
+ end
44
+
45
+ # Content of a key together with the version that identifies it.
46
+ class Versioned
47
+ # @param content [String] the content
48
+ # @param version [String] opaque version, such as an ETag
49
+ #: (content: String, version: String) -> void
50
+ def initialize(content:, version:)
51
+ @content = content #: String
52
+ @version = version #: String
53
+ end
54
+
55
+ # @return [String] the content
56
+ #: String
57
+ attr_reader :content
58
+
59
+ # @return [String] opaque version, such as an ETag
60
+ #: String
61
+ attr_reader :version
62
+ end
63
+
64
+ # Builds a storage from its settings message.
65
+ #
66
+ # @param settings [Google::Protobuf::MessageExts] backend settings from {Proto::V1::StorageConfig}
67
+ # @return [Base] the storage
68
+ #: (Google::Protobuf::MessageExts settings) -> Base
69
+ def self.from_proto(settings) = raise NotImplementedError
70
+
71
+ # Writes one file.
72
+ #
73
+ # @param key [String] file key
74
+ # @param content [String] file content
75
+ # @param message [String] description of the change, such as a commit message
76
+ # @return [WriteResult] where the content is and whether it changed
77
+ # @raise [InvalidKey] for an unsafe key
78
+ #: (key: String, content: String, message: String) -> WriteResult
79
+ def write(key:, content:, message:)
80
+ write_all([[key, content]], message: ->(_changed_keys) { message }).fetch(0)
81
+ end
82
+
83
+ # Writes several files as one change where the storage supports it. Files
84
+ # whose content equals what their key holds are not written again.
85
+ #
86
+ # @param files [Array<Array(String, String)>] keys and contents
87
+ # @param message [Proc] builds the description of the change, such as a
88
+ # commit message, from the keys whose content changed; not called when none did
89
+ # @return [Array<WriteResult>] results, in the order of files
90
+ # @raise [InvalidKey] for an unsafe key
91
+ #: (Array[Base::file] files, message: Base::message_builder) -> Array[WriteResult]
92
+ def write_all(files, message:) = raise NotImplementedError
93
+
94
+ # Reads content written earlier.
95
+ #
96
+ # @param pointer [Proto::V1::Pointer] location returned by a write
97
+ # @return [String] the content
98
+ #: (Proto::V1::Pointer pointer) -> String
99
+ def read(pointer) = raise NotImplementedError
100
+
101
+ # Whether the storage implements {#version_of}, {#read_versioned} and
102
+ # {#write_if}, so a single key can be updated by optimistic concurrency.
103
+ #
104
+ # @return [Boolean] true when supported
105
+ #: () -> bool
106
+ def conditional? = false
107
+
108
+ # Version the key currently holds, without reading the content.
109
+ #
110
+ # @param key [String] file key
111
+ # @return [String, nil] the version, or nil when the key is absent
112
+ # @raise [InvalidKey] for an unsafe key
113
+ #: (String key) -> String?
114
+ def version_of(key) = raise NotImplementedError
115
+
116
+ # Reads the current content of a key.
117
+ #
118
+ # @param key [String] file key
119
+ # @return [Versioned, nil] the content and its version, or nil when the key is absent
120
+ # @raise [InvalidKey] for an unsafe key
121
+ #: (String key) -> Versioned?
122
+ def read_versioned(key) = raise NotImplementedError
123
+
124
+ # Writes a key only if it still holds the expected version.
125
+ #
126
+ # @param key [String] file key
127
+ # @param content [String] new content
128
+ # @param expected_version [String, nil] version read earlier; nil to create a missing key only
129
+ # @return [String] version of the written content
130
+ # @raise [Conflict] when the key holds another version, or exists when nil was expected
131
+ # @raise [InvalidKey] for an unsafe key
132
+ #: (key: String, content: String, expected_version: String?) -> String
133
+ def write_if(key:, content:, expected_version:) = raise NotImplementedError
134
+
135
+ # Wraps a location in a pointer.
136
+ #
137
+ # @param location [Google::Protobuf::MessageExts] storage-specific location
138
+ # @return [Proto::V1::Pointer] the pointer
139
+ #: (Google::Protobuf::MessageExts location) -> Proto::V1::Pointer
140
+ private def pointer_to(location) = Proto::V1::Pointer.new(location: Google::Protobuf::Any.pack(location))
141
+
142
+ # Unwraps the location of a pointer.
143
+ #
144
+ # @param pointer [Proto::V1::Pointer] the pointer
145
+ # @param location_class [Class] location message this storage writes
146
+ # @return [Google::Protobuf::MessageExts] the location
147
+ # @raise [ArgumentError] when the pointer holds another kind of location
148
+ #: [L] (Proto::V1::Pointer pointer, Class[L] location_class) -> L
149
+ private def location_of(pointer, location_class)
150
+ any = pointer.location or raise ArgumentError, 'pointer has no location'
151
+ location = any.unpack(location_class) or raise ArgumentError, "#{self.class.name} cannot read a pointer to #{any.type_name}"
152
+ Must.be_a(location, location_class)
153
+ end
154
+
155
+ # @param key [String] key to check
156
+ # @return [String] the key
157
+ # @raise [InvalidKey] when the key is empty, absolute, or has empty, "." or ".." segments
158
+ #: (String key) -> String
159
+ private def validate_key(key)
160
+ segments = key.split('/', -1)
161
+ raise InvalidKey, "invalid storage key #{key.inspect}" if segments.empty? || segments.any? { ['', '.', '..'].include?(it) }
162
+
163
+ key
164
+ end
165
+ end
166
+ end
167
+ end
@@ -0,0 +1,144 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require 'digest'
7
+ require 'fileutils'
8
+ require 'securerandom'
9
+ require 'glazed'
10
+ require 'glazed/must'
11
+ require 'glazed/proto'
12
+ require 'glazed/storages/base'
13
+
14
+ module Glazed
15
+ module Storages
16
+ # Local directory holding the latest content of each key, for scripts and
17
+ # tools that read plain files. It keeps no history.
18
+ #
19
+ # @example
20
+ # storage = Glazed::Storages::Filesystem.new(path: '/var/lib/glazed/latest')
21
+ # storage.write(key: 'core/rt1', content: config, message: 'rt1')
22
+ class Filesystem < Base
23
+ # Raised when a file no longer holds the content its pointer names.
24
+ class StaleContent < Glazed::Error; end
25
+
26
+ # @param path [String] directory holding the files, created when missing
27
+ #: (path: String) -> void
28
+ def initialize(path:)
29
+ super()
30
+ @path = path #: String
31
+ end
32
+
33
+ # Builds a storage from {Proto::V1::FilesystemStorage}.
34
+ #
35
+ # @param settings [Google::Protobuf::MessageExts] a {Proto::V1::FilesystemStorage}
36
+ # @return [Filesystem] the storage
37
+ # @raise [Must::Violation] when settings is another message
38
+ #: (Google::Protobuf::MessageExts settings) -> Filesystem
39
+ def self.from_proto(settings)
40
+ filesystem = Must.be_a(settings, Proto::V1::FilesystemStorage)
41
+ new(path: filesystem.path)
42
+ end
43
+
44
+ # @return [String] directory holding the files
45
+ #: String
46
+ attr_reader :path
47
+
48
+ # Writes each changed file in place. A file is replaced atomically, so
49
+ # readers see either the old or the new content.
50
+ #
51
+ # @param files [Array<Array(String, String)>] keys and contents
52
+ # @param message [Proc] builds a description of the change, unused
53
+ # @return [Array<WriteResult>] the content's location and digest, and whether the file changed
54
+ # @raise [InvalidKey] for an unsafe key
55
+ #: (Array[Base::file] files, message: Base::message_builder) -> Array[WriteResult]
56
+ def write_all(files, message:)
57
+ files.each { |key, _| validate_key(key) }
58
+ files.map do |key, content|
59
+ file = ::File.join(path, key)
60
+ changed = !(::File.file?(file) && ::File.binread(file) == content.b)
61
+ replace(file, content) if changed
62
+ pointer = pointer_to(Proto::V1::FilesystemLocation.new(path: key, sha256: Digest::SHA256.hexdigest(content)))
63
+ WriteResult.new(pointer:, changed:)
64
+ end
65
+ end
66
+
67
+ # Reads the file a pointer names.
68
+ #
69
+ # @param pointer [Proto::V1::Pointer] pointer with a filesystem location
70
+ # @return [String] the content
71
+ # @raise [ArgumentError] when the pointer is not a filesystem location
72
+ # @raise [StaleContent] when the file has been rewritten with other content since
73
+ # @raise [Errno::ENOENT] when the file is gone
74
+ #: (Proto::V1::Pointer pointer) -> String
75
+ def read(pointer)
76
+ location = location_of(pointer, Proto::V1::FilesystemLocation)
77
+ content = ::File.read(::File.join(path, validate_key(location.path)))
78
+ unless Digest::SHA256.hexdigest(content) == location.sha256
79
+ raise StaleContent, "#{location.path} in #{path} no longer holds the content written"
80
+ end
81
+
82
+ content
83
+ end
84
+
85
+ # @return [Boolean] true; versions are content digests
86
+ #: () -> bool
87
+ def conditional? = true
88
+
89
+ # @param key [String] file key
90
+ # @return [String, nil] SHA-256 of the file, or nil when it is missing
91
+ # @raise [InvalidKey] for an unsafe key
92
+ #: (String key) -> String?
93
+ def version_of(key) = read_versioned(key)&.version
94
+
95
+ # @param key [String] file key
96
+ # @return [Versioned, nil] the content and its SHA-256, or nil when the file is missing
97
+ # @raise [InvalidKey] for an unsafe key
98
+ #: (String key) -> Versioned?
99
+ def read_versioned(key)
100
+ content = ::File.binread(::File.join(path, validate_key(key)))
101
+ Versioned.new(content:, version: Digest::SHA256.hexdigest(content))
102
+ rescue Errno::ENOENT
103
+ nil
104
+ end
105
+
106
+ # Replaces the file while holding an exclusive lock on a sibling lock
107
+ # file, which every conditional writer of the key takes.
108
+ #
109
+ # @param key [String] file key
110
+ # @param content [String] new content
111
+ # @param expected_version [String, nil] SHA-256 read earlier; nil to create a missing file only
112
+ # @return [String] SHA-256 of the content
113
+ # @raise [Conflict] when the file holds other content
114
+ # @raise [InvalidKey] for an unsafe key
115
+ #: (key: String, content: String, expected_version: String?) -> String
116
+ def write_if(key:, content:, expected_version:)
117
+ file = ::File.join(path, validate_key(key))
118
+ FileUtils.mkdir_p(::File.dirname(file))
119
+ ::File.open(::File.join(::File.dirname(file), ".#{::File.basename(file)}.lock"), ::File::RDWR | ::File::CREAT, 0o644) do |lock|
120
+ lock.flock(::File::LOCK_EX)
121
+ current = version_of(key)
122
+ raise Conflict, "#{key} in #{path} is at #{current || 'nothing'}, not #{expected_version || 'nothing'}" unless current == expected_version
123
+
124
+ replace(file, content)
125
+ end
126
+ Digest::SHA256.hexdigest(content)
127
+ end
128
+
129
+ # @param file [String] file to replace
130
+ # @param content [String] new content
131
+ # @return [void]
132
+ #: (String file, String content) -> void
133
+ private def replace(file, content)
134
+ dir = ::File.dirname(file)
135
+ FileUtils.mkdir_p(dir)
136
+ temporary = ::File.join(dir, ".#{::File.basename(file)}.#{SecureRandom.hex(8)}.tmp")
137
+ ::File.binwrite(temporary, content)
138
+ ::File.rename(temporary, file)
139
+ ensure
140
+ ::File.unlink(temporary) if temporary && ::File.exist?(temporary)
141
+ end
142
+ end
143
+ end
144
+ end