kitsune-kit 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +27 -1
- data/README.md +8 -3
- data/docs/architecture/decisions/0007-compose-customization.md +30 -0
- data/docs/architecture.md +5 -1
- data/docs/commands.md +14 -0
- data/docs/configuration.md +16 -0
- data/docs/services/compose.md +164 -0
- data/lib/kitsune/kit/cli.rb +83 -6
- data/lib/kitsune/kit/configuration.rb +96 -12
- data/lib/kitsune/kit/operations/ensure_service.rb +33 -16
- data/lib/kitsune/kit/operations/service_files.rb +14 -10
- data/lib/kitsune/kit/scripts/docker.sh +11 -6
- data/lib/kitsune/kit/scripts/firewall.sh +8 -3
- data/lib/kitsune/kit/scripts/metrics.sh +6 -1
- data/lib/kitsune/kit/scripts/unattended.sh +8 -3
- data/lib/kitsune/kit/service_compose.rb +205 -5
- data/lib/kitsune/kit/version.rb +1 -1
- data/lib/kitsune/kit/workflows/eject_compose.rb +82 -0
- data/lib/kitsune/kit/workflows/initialize_project.rb +8 -0
- data/lib/kitsune/kit.rb +1 -0
- metadata +4 -1
|
@@ -1,32 +1,224 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
|
+
require "digest"
|
|
5
|
+
require "pathname"
|
|
4
6
|
require "yaml"
|
|
7
|
+
require_relative "errors"
|
|
5
8
|
|
|
6
9
|
module Kitsune
|
|
7
10
|
module Kit
|
|
8
11
|
class ServiceCompose
|
|
12
|
+
Document = Data.define(:filename, :content, :source)
|
|
13
|
+
MAX_FILE_SIZE = 262_144
|
|
14
|
+
MODES = %w[generated overlay custom].freeze
|
|
15
|
+
SENSITIVE_KEY = /(?:password|passwd|secret|token|api[_-]?key)/i
|
|
16
|
+
ENV_REFERENCE = /\A\$\{[A-Z][A-Z0-9_]*\}\z/
|
|
17
|
+
|
|
18
|
+
attr_reader :mode, :security_findings
|
|
19
|
+
|
|
9
20
|
def initialize(config:, type:, service:)
|
|
10
21
|
@config = config
|
|
11
|
-
@type = type
|
|
22
|
+
@type = type.to_s
|
|
12
23
|
@service = service
|
|
24
|
+
@mode = service.compose.mode
|
|
25
|
+
@security_findings = []
|
|
26
|
+
validate!
|
|
13
27
|
end
|
|
14
28
|
|
|
15
|
-
def content =
|
|
29
|
+
def content = documents.first.content
|
|
30
|
+
def generated_content = YAML.dump(generated_document)
|
|
16
31
|
def env_content(password) = "#{@service.password_env}=#{JSON.generate(password)}\n"
|
|
32
|
+
def filenames = documents.map(&:filename)
|
|
33
|
+
|
|
34
|
+
def fingerprint
|
|
35
|
+
Digest::SHA256.hexdigest([mode, *documents.flat_map { |item| [item.filename, item.content] }].join("\0"))
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def documents
|
|
39
|
+
@documents ||= case mode
|
|
40
|
+
when "generated"
|
|
41
|
+
[Document.new(filename: "compose.yml", content: generated_content, source: "generated")]
|
|
42
|
+
when "overlay"
|
|
43
|
+
[
|
|
44
|
+
Document.new(filename: "compose.yml", content: generated_content, source: "generated"),
|
|
45
|
+
Document.new(filename: "compose.override.yml", content: user_content,
|
|
46
|
+
source: @service.compose.file)
|
|
47
|
+
]
|
|
48
|
+
when "custom"
|
|
49
|
+
[Document.new(filename: "compose.yml", content: user_content, source: @service.compose.file)]
|
|
50
|
+
else
|
|
51
|
+
raise Errors::ConfigurationError, "unsupported Compose mode: #{mode}"
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def display
|
|
56
|
+
documents.map do |document|
|
|
57
|
+
"# --- #{document.filename} (#{document.source}) ---\n#{document.content}"
|
|
58
|
+
end.join("\n")
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def metadata
|
|
62
|
+
{
|
|
63
|
+
mode: mode,
|
|
64
|
+
files: documents.map { |document| { filename: document.filename, source: document.source } },
|
|
65
|
+
allow_unsafe: @service.compose.allow_unsafe,
|
|
66
|
+
security_findings: security_findings
|
|
67
|
+
}
|
|
68
|
+
end
|
|
17
69
|
|
|
18
70
|
private
|
|
19
71
|
|
|
20
|
-
def
|
|
72
|
+
def validate!
|
|
73
|
+
raise Errors::ConfigurationError, "unsupported Compose mode: #{mode}" unless MODES.include?(mode)
|
|
74
|
+
return if mode == "generated"
|
|
75
|
+
|
|
76
|
+
document = user_document
|
|
77
|
+
if mode == "custom"
|
|
78
|
+
services = document["services"]
|
|
79
|
+
unless services.is_a?(Hash) && services[@type].is_a?(Hash)
|
|
80
|
+
raise Errors::ConfigurationError.new(
|
|
81
|
+
"custom Compose file must define services.#{@type}",
|
|
82
|
+
hint: "Run `kit service #{@type} compose eject` to create a valid starting point."
|
|
83
|
+
)
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
detect_inline_secrets(document)
|
|
87
|
+
inspect_unsafe_options(document)
|
|
88
|
+
return if security_findings.empty? || @service.compose.allow_unsafe
|
|
89
|
+
|
|
90
|
+
raise Errors::UnsafeOperationError.new(
|
|
91
|
+
"Compose customization contains unsafe options",
|
|
92
|
+
hint: "Remove the options or set services.#{@type}.compose.allow_unsafe to true after review.",
|
|
93
|
+
context: { service: @type, findings: security_findings }
|
|
94
|
+
)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def user_content
|
|
98
|
+
@user_content ||= Pathname(@service.compose.file).binread
|
|
99
|
+
rescue SystemCallError => e
|
|
100
|
+
raise Errors::ConfigurationError, "unable to read Compose customization: #{e.class}"
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def user_document
|
|
104
|
+
@user_document ||= begin
|
|
105
|
+
if user_content.bytesize > MAX_FILE_SIZE
|
|
106
|
+
raise Errors::ConfigurationError, "Compose customization exceeds 256 KiB"
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
parsed = YAML.safe_load(user_content, permitted_classes: [], permitted_symbols: [], aliases: false)
|
|
110
|
+
unless parsed.is_a?(Hash)
|
|
111
|
+
raise Errors::ConfigurationError, "Compose customization must contain a YAML mapping"
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
stringify_keys(parsed)
|
|
115
|
+
rescue Psych::Exception => e
|
|
116
|
+
raise Errors::ConfigurationError, "invalid Compose customization YAML: #{e.message}"
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def detect_inline_secrets(value, path = [])
|
|
121
|
+
case value
|
|
122
|
+
when Hash
|
|
123
|
+
value.each do |key, child|
|
|
124
|
+
if key.match?(SENSITIVE_KEY) && child.is_a?(String) && !child.match?(ENV_REFERENCE)
|
|
125
|
+
raise Errors::ConfigurationError.new(
|
|
126
|
+
"Compose customization contains an inline secret at #{(path + [key]).join('.')}",
|
|
127
|
+
hint: "Reference an environment variable such as ${#{@service.password_env}} instead."
|
|
128
|
+
)
|
|
129
|
+
end
|
|
130
|
+
detect_inline_secrets(child, path + [key])
|
|
131
|
+
end
|
|
132
|
+
when Array
|
|
133
|
+
value.each_with_index { |child, index| detect_inline_secrets(child, path + [index.to_s]) }
|
|
134
|
+
when String
|
|
135
|
+
detect_inline_environment_secret(value, path)
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def detect_inline_environment_secret(value, path)
|
|
140
|
+
key, content = value.split("=", 2)
|
|
141
|
+
return unless content && key.match?(SENSITIVE_KEY) && !content.match?(ENV_REFERENCE)
|
|
142
|
+
|
|
143
|
+
raise Errors::ConfigurationError.new(
|
|
144
|
+
"Compose customization contains an inline secret at #{path.join('.')}",
|
|
145
|
+
hint: "Use mapping syntax and reference ${#{@service.password_env}} instead."
|
|
146
|
+
)
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def inspect_unsafe_options(document)
|
|
150
|
+
services = document["services"]
|
|
151
|
+
return unless services.is_a?(Hash)
|
|
152
|
+
|
|
153
|
+
services.each do |name, definition|
|
|
154
|
+
next unless definition.is_a?(Hash)
|
|
155
|
+
|
|
156
|
+
inspect_service(name, definition)
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def inspect_service(name, definition)
|
|
161
|
+
prefix = "services.#{name}"
|
|
162
|
+
finding("#{prefix}.privileged", "privileged containers") if definition["privileged"] == true
|
|
163
|
+
inspect_namespaces(prefix, definition)
|
|
164
|
+
inspect_capabilities(prefix, definition)
|
|
165
|
+
inspect_ports(prefix, name, definition)
|
|
166
|
+
finding("#{prefix}.devices", "host device access") if definition.key?("devices")
|
|
167
|
+
finding("#{prefix}.build", "remote image builds") if definition.key?("build")
|
|
168
|
+
finding("#{prefix}.env_file", "unmanaged environment files") if definition.key?("env_file")
|
|
169
|
+
Array(definition["volumes"]).each do |volume|
|
|
170
|
+
finding("#{prefix}.volumes", "host or Docker socket mount") if unsafe_volume?(volume)
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def inspect_namespaces(prefix, definition)
|
|
175
|
+
%w[network_mode pid ipc].each do |key|
|
|
176
|
+
finding("#{prefix}.#{key}", "host namespace access") if definition[key].to_s == "host"
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def inspect_capabilities(prefix, definition)
|
|
181
|
+
capabilities = Array(definition["cap_add"]).map { |item| item.to_s.upcase }
|
|
182
|
+
return unless capabilities.intersect?(%w[ALL SYS_ADMIN SYS_PTRACE NET_ADMIN])
|
|
183
|
+
|
|
184
|
+
finding("#{prefix}.cap_add", "elevated Linux capabilities")
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def inspect_ports(prefix, name, definition)
|
|
188
|
+
return unless definition.key?("ports")
|
|
189
|
+
return if name == @type && Array(definition["ports"]) == managed_ports
|
|
190
|
+
|
|
191
|
+
finding("#{prefix}.ports", "ports outside the managed firewall model")
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def managed_ports
|
|
195
|
+
return [] unless @service.publish
|
|
196
|
+
|
|
197
|
+
["#{@service.bind}:#{@service.port}:#{container_port}"]
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def unsafe_volume?(volume)
|
|
201
|
+
source = case volume
|
|
202
|
+
when String then volume.split(":", 2).first
|
|
203
|
+
when Hash then volume["source"] || volume["src"]
|
|
204
|
+
end
|
|
205
|
+
source.to_s.start_with?("/") || source.to_s.include?("docker.sock")
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def finding(path, message)
|
|
209
|
+
security_findings << { path: path, message: message }
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def generated_document
|
|
21
213
|
{
|
|
22
214
|
"name" => project_name,
|
|
23
|
-
"services" => { @type =>
|
|
215
|
+
"services" => { @type => generated_service_definition },
|
|
24
216
|
"volumes" => { "data" => nil },
|
|
25
217
|
"networks" => { "private" => { "external" => true, "name" => "kitsune-private" } }
|
|
26
218
|
}
|
|
27
219
|
end
|
|
28
220
|
|
|
29
|
-
def
|
|
221
|
+
def generated_service_definition
|
|
30
222
|
definition = @type == "postgres" ? postgres : redis
|
|
31
223
|
definition["ports"] = ["#{@service.bind}:#{@service.port}:#{container_port}"] if @service.publish
|
|
32
224
|
definition
|
|
@@ -63,6 +255,14 @@ module Kitsune
|
|
|
63
255
|
{ "test" => ["CMD-SHELL", command], "interval" => "10s", "timeout" => "5s", "retries" => 12 }
|
|
64
256
|
end
|
|
65
257
|
|
|
258
|
+
def stringify_keys(value)
|
|
259
|
+
case value
|
|
260
|
+
when Hash then value.to_h { |key, item| [key.to_s, stringify_keys(item)] }
|
|
261
|
+
when Array then value.map { |item| stringify_keys(item) }
|
|
262
|
+
else value
|
|
263
|
+
end
|
|
264
|
+
end
|
|
265
|
+
|
|
66
266
|
def project_name = "kitsune-#{@config.environment}-#{@type}"
|
|
67
267
|
def database = "app_#{@config.environment.tr('-', '_')}"
|
|
68
268
|
def container_port = @type == "postgres" ? 5432 : 6379
|
data/lib/kitsune/kit/version.rb
CHANGED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "pathname"
|
|
5
|
+
require "tempfile"
|
|
6
|
+
require "yaml"
|
|
7
|
+
require_relative "../errors"
|
|
8
|
+
require_relative "../service_compose"
|
|
9
|
+
|
|
10
|
+
module Kitsune
|
|
11
|
+
module Kit
|
|
12
|
+
module Workflows
|
|
13
|
+
class EjectCompose
|
|
14
|
+
TYPES = %w[postgres redis].freeze
|
|
15
|
+
|
|
16
|
+
def initialize(root:, config:, type:, config_path: nil)
|
|
17
|
+
@root = Pathname(root).expand_path
|
|
18
|
+
@config = config
|
|
19
|
+
@type = type.to_s
|
|
20
|
+
@config_path = config_path ? Pathname(config_path).expand_path(@root) : @root.join(".kitsune/config.yml")
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def call(force: false)
|
|
24
|
+
@backup_created = false
|
|
25
|
+
validate_target!(force)
|
|
26
|
+
content = ServiceCompose.new(config: config, type: type, service: service).generated_content
|
|
27
|
+
FileUtils.mkdir_p(output_path.dirname, mode: 0o700)
|
|
28
|
+
FileUtils.cp(config_path, backup_path)
|
|
29
|
+
@backup_created = true
|
|
30
|
+
write_atomic(output_path, content, 0o644)
|
|
31
|
+
update_configuration
|
|
32
|
+
{ compose_file: output_path.to_s, config_file: config_path.to_s, backup_file: backup_path.to_s }
|
|
33
|
+
rescue StandardError
|
|
34
|
+
FileUtils.cp(backup_path, config_path) if @backup_created && backup_path.file?
|
|
35
|
+
raise
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
attr_reader :root, :config, :type, :config_path
|
|
41
|
+
|
|
42
|
+
def service = config.services.public_send(type)
|
|
43
|
+
def output_path = root.join(".kitsune/compose/#{type}.yml")
|
|
44
|
+
def backup_path = Pathname("#{config_path}.backup")
|
|
45
|
+
|
|
46
|
+
def validate_target!(force)
|
|
47
|
+
raise Errors::ConfigurationError, "unknown service type: #{type}" unless TYPES.include?(type)
|
|
48
|
+
raise Errors::ConfigurationError, "configuration file does not exist: #{config_path}" unless config_path.file?
|
|
49
|
+
return if force || (!output_path.exist? && !backup_path.exist? && service.compose.mode == "generated")
|
|
50
|
+
|
|
51
|
+
raise Errors::UnsafeOperationError.new(
|
|
52
|
+
"refusing to replace an existing Compose customization or configuration backup",
|
|
53
|
+
hint: "Review #{output_path} and #{backup_path}, then repeat with --force if replacement is intentional."
|
|
54
|
+
)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def update_configuration
|
|
58
|
+
document = YAML.safe_load(config_path.read, permitted_classes: [], permitted_symbols: [], aliases: false)
|
|
59
|
+
service_config = document.fetch("services").fetch(type)
|
|
60
|
+
service_config["compose"] = {
|
|
61
|
+
"mode" => "custom",
|
|
62
|
+
"file" => ".kitsune/compose/#{type}.yml",
|
|
63
|
+
"allow_unsafe" => false
|
|
64
|
+
}
|
|
65
|
+
write_atomic(config_path, YAML.dump(document), 0o600)
|
|
66
|
+
rescue KeyError, Psych::Exception => e
|
|
67
|
+
raise Errors::ConfigurationError, "unable to update configuration for Compose ejection: #{e.message}"
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def write_atomic(path, content, mode)
|
|
71
|
+
Tempfile.create([path.basename.to_s, ".tmp"], path.dirname) do |file|
|
|
72
|
+
file.write(content)
|
|
73
|
+
file.flush
|
|
74
|
+
file.fsync
|
|
75
|
+
File.chmod(mode, file.path)
|
|
76
|
+
File.rename(file.path, path)
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
@@ -40,6 +40,10 @@ module Kitsune
|
|
|
40
40
|
allowed_cidrs: []
|
|
41
41
|
port: 5432
|
|
42
42
|
password_env: POSTGRES_PASSWORD
|
|
43
|
+
compose:
|
|
44
|
+
mode: generated
|
|
45
|
+
file:
|
|
46
|
+
allow_unsafe: false
|
|
43
47
|
redis:
|
|
44
48
|
enabled: false
|
|
45
49
|
mode: managed
|
|
@@ -50,6 +54,10 @@ module Kitsune
|
|
|
50
54
|
allowed_cidrs: []
|
|
51
55
|
port: 6379
|
|
52
56
|
password_env: REDIS_PASSWORD
|
|
57
|
+
compose:
|
|
58
|
+
mode: generated
|
|
59
|
+
file:
|
|
60
|
+
allow_unsafe: false
|
|
53
61
|
|
|
54
62
|
system:
|
|
55
63
|
swap_size_gb: 2
|
data/lib/kitsune/kit.rb
CHANGED
|
@@ -28,6 +28,7 @@ require_relative "kit/workflows/build_plan"
|
|
|
28
28
|
require_relative "kit/workflows/apply_plan"
|
|
29
29
|
require_relative "kit/workflows/doctor"
|
|
30
30
|
require_relative "kit/workflows/initialize_project"
|
|
31
|
+
require_relative "kit/workflows/eject_compose"
|
|
31
32
|
require_relative "kit/workflows/inspect_environment"
|
|
32
33
|
require_relative "kit/workflows/rollback"
|
|
33
34
|
require_relative "kit/workflows/destroy_server"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: kitsune-kit
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.6.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Omar Herrera
|
|
@@ -158,6 +158,7 @@ files:
|
|
|
158
158
|
- docs/architecture/decisions/0004-supported-platforms.md
|
|
159
159
|
- docs/architecture/decisions/0005-operation-semantics.md
|
|
160
160
|
- docs/architecture/decisions/0006-tui-backend.md
|
|
161
|
+
- docs/architecture/decisions/0007-compose-customization.md
|
|
161
162
|
- docs/architecture/provider-adapters.md
|
|
162
163
|
- docs/commands.md
|
|
163
164
|
- docs/configuration.md
|
|
@@ -167,6 +168,7 @@ files:
|
|
|
167
168
|
- docs/roadmap.md
|
|
168
169
|
- docs/security-audit.md
|
|
169
170
|
- docs/security.md
|
|
171
|
+
- docs/services/compose.md
|
|
170
172
|
- docs/services/postgres.md
|
|
171
173
|
- docs/services/redis.md
|
|
172
174
|
- docs/testing.md
|
|
@@ -233,6 +235,7 @@ files:
|
|
|
233
235
|
- lib/kitsune/kit/workflows/build_plan.rb
|
|
234
236
|
- lib/kitsune/kit/workflows/destroy_server.rb
|
|
235
237
|
- lib/kitsune/kit/workflows/doctor.rb
|
|
238
|
+
- lib/kitsune/kit/workflows/eject_compose.rb
|
|
236
239
|
- lib/kitsune/kit/workflows/environment_selection.rb
|
|
237
240
|
- lib/kitsune/kit/workflows/import_server.rb
|
|
238
241
|
- lib/kitsune/kit/workflows/initialize_project.rb
|