empeira 0.1.0.pre.alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +661 -0
- data/README.md +150 -0
- data/bin/empeira +18 -0
- data/config/defaults.yaml +229 -0
- data/docs/addons.md +331 -0
- data/docs/architecture.md +277 -0
- data/docs/configuration.md +1204 -0
- data/docs/development.md +472 -0
- data/docs/hiera-and-eyaml.md +361 -0
- data/docs/installation.md +280 -0
- data/docs/networking.md +362 -0
- data/docs/nodes.md +642 -0
- data/docs/proxy.md +136 -0
- data/docs/troubleshooting.md +361 -0
- data/lib/empeira/addons/environment.rb +82 -0
- data/lib/empeira/addons/error.rb +14 -0
- data/lib/empeira/addons/installation.rb +119 -0
- data/lib/empeira/addons/manifest.rb +112 -0
- data/lib/empeira/addons/service.rb +50 -0
- data/lib/empeira/addons/storage.rb +94 -0
- data/lib/empeira/agent/acquisition.rb +120 -0
- data/lib/empeira/agent/artifact.rb +41 -0
- data/lib/empeira/agent/authentication.rb +122 -0
- data/lib/empeira/agent/cache.rb +122 -0
- data/lib/empeira/agent/cache_storage.rb +82 -0
- data/lib/empeira/agent/download.rb +84 -0
- data/lib/empeira/agent/native_commands.rb +84 -0
- data/lib/empeira/agent/release_sources.rb +64 -0
- data/lib/empeira/agent/target.rb +66 -0
- data/lib/empeira/agent/versions.rb +29 -0
- data/lib/empeira/application.rb +86 -0
- data/lib/empeira/build.json +5 -0
- data/lib/empeira/build_info.rb +46 -0
- data/lib/empeira/cli/addons.rb +45 -0
- data/lib/empeira/cli/base.rb +73 -0
- data/lib/empeira/cli/completion.rb +42 -0
- data/lib/empeira/cli/config.rb +22 -0
- data/lib/empeira/cli/main.rb +114 -0
- data/lib/empeira/cli/node.rb +87 -0
- data/lib/empeira/cli/progress.rb +169 -0
- data/lib/empeira/cli/proxy.rb +36 -0
- data/lib/empeira/cli/update.rb +23 -0
- data/lib/empeira/completion/addons.rb +93 -0
- data/lib/empeira/completion/bash.rb +62 -0
- data/lib/empeira/completion/catalog.rb +170 -0
- data/lib/empeira/configuration/additional_services.rb +74 -0
- data/lib/empeira/configuration/addons.rb +85 -0
- data/lib/empeira/configuration/agent_schema.rb +163 -0
- data/lib/empeira/configuration/agent_values.rb +54 -0
- data/lib/empeira/configuration/bootstrap_guests.rb +42 -0
- data/lib/empeira/configuration/command_mocks.rb +65 -0
- data/lib/empeira/configuration/display.rb +20 -0
- data/lib/empeira/configuration/dns_rewrites.rb +58 -0
- data/lib/empeira/configuration/document.rb +67 -0
- data/lib/empeira/configuration/eyaml.rb +46 -0
- data/lib/empeira/configuration/hiera_modulepath.rb +40 -0
- data/lib/empeira/configuration/hiera_mount_schema.rb +62 -0
- data/lib/empeira/configuration/hiera_mounts.rb +83 -0
- data/lib/empeira/configuration/image_schema.rb +68 -0
- data/lib/empeira/configuration/loader.rb +70 -0
- data/lib/empeira/configuration/merge.rb +48 -0
- data/lib/empeira/configuration/network_redirects.rb +73 -0
- data/lib/empeira/configuration/node_images.rb +64 -0
- data/lib/empeira/configuration/schema.rb +215 -0
- data/lib/empeira/configuration/server_mounts.rb +106 -0
- data/lib/empeira/configuration/service_configuration.rb +40 -0
- data/lib/empeira/configuration/ssh_preferences.rb +83 -0
- data/lib/empeira/configuration/user.rb +46 -0
- data/lib/empeira/configuration/vm_interfaces.rb +147 -0
- data/lib/empeira/control_plane/additional_configurations.rb +106 -0
- data/lib/empeira/control_plane/browser.rb +73 -0
- data/lib/empeira/control_plane/controller.rb +763 -0
- data/lib/empeira/control_plane/discovery.rb +69 -0
- data/lib/empeira/control_plane/environment.rb +101 -0
- data/lib/empeira/control_plane/files.rb +263 -0
- data/lib/empeira/control_plane/health.rb +104 -0
- data/lib/empeira/control_plane/plan.rb +356 -0
- data/lib/empeira/control_plane/storage.rb +48 -0
- data/lib/empeira/errors.rb +9 -0
- data/lib/empeira/execution/console.rb +45 -0
- data/lib/empeira/execution/diagnostics.rb +210 -0
- data/lib/empeira/execution/launch_error.rb +14 -0
- data/lib/empeira/execution/result.rb +11 -0
- data/lib/empeira/execution/runner.rb +235 -0
- data/lib/empeira/execution/terminal.rb +35 -0
- data/lib/empeira/hostname_pattern.rb +14 -0
- data/lib/empeira/images/configuration.rb +44 -0
- data/lib/empeira/images/identity.rb +30 -0
- data/lib/empeira/images/recipe.rb +22 -0
- data/lib/empeira/images/reference.rb +43 -0
- data/lib/empeira/images/source.rb +19 -0
- data/lib/empeira/immutable.rb +19 -0
- data/lib/empeira/infrastructure/cleanup.rb +328 -0
- data/lib/empeira/infrastructure/definition.rb +59 -0
- data/lib/empeira/infrastructure/owned_files.rb +63 -0
- data/lib/empeira/infrastructure/recovery_inventory.rb +53 -0
- data/lib/empeira/infrastructure/service.rb +237 -0
- data/lib/empeira/infrastructure/status.rb +73 -0
- data/lib/empeira/infrastructure/store.rb +274 -0
- data/lib/empeira/modules/events.rb +67 -0
- data/lib/empeira/modules/git_safety.rb +60 -0
- data/lib/empeira/modules/host_git.rb +91 -0
- data/lib/empeira/modules/image.rb +27 -0
- data/lib/empeira/modules/installer.rb +67 -0
- data/lib/empeira/modules/request.rb +68 -0
- data/lib/empeira/modules/state.rb +87 -0
- data/lib/empeira/modules/storage.rb +35 -0
- data/lib/empeira/network/bootstrap_proxy.rb +157 -0
- data/lib/empeira/network/definition.rb +61 -0
- data/lib/empeira/network/direct_egress.rb +133 -0
- data/lib/empeira/network/egress.rb +27 -0
- data/lib/empeira/network/gateway.rb +79 -0
- data/lib/empeira/network/host_policy.rb +53 -0
- data/lib/empeira/network/naming.rb +28 -0
- data/lib/empeira/network/peer/adapter_image.rb +32 -0
- data/lib/empeira/network/peer/allocation.rb +45 -0
- data/lib/empeira/network/peer/attachment.rb +27 -0
- data/lib/empeira/network/peer/backend.rb +108 -0
- data/lib/empeira/network/peer/channel.rb +133 -0
- data/lib/empeira/network/peer/channel_server.rb +71 -0
- data/lib/empeira/network/peer/docker_adapter.rb +165 -0
- data/lib/empeira/network/peer/helper_security.rb +40 -0
- data/lib/empeira/network/peer/layout.rb +55 -0
- data/lib/empeira/network/peer/linux_podman.rb +114 -0
- data/lib/empeira/network/peer/podman_machine.rb +112 -0
- data/lib/empeira/network/policy.rb +39 -0
- data/lib/empeira/network/proxy_bindings.rb +52 -0
- data/lib/empeira/network/proxy_policy.rb +64 -0
- data/lib/empeira/network/redirects.rb +89 -0
- data/lib/empeira/network/resource.rb +50 -0
- data/lib/empeira/node/agent_installer.rb +31 -0
- data/lib/empeira/node/agent_package.rb +125 -0
- data/lib/empeira/node/agent_repository.rb +160 -0
- data/lib/empeira/node/apt_configuration.rb +85 -0
- data/lib/empeira/node/bootstrap.rb +17 -0
- data/lib/empeira/node/certificates.rb +98 -0
- data/lib/empeira/node/command_mock.rb +38 -0
- data/lib/empeira/node/command_mocks.rb +114 -0
- data/lib/empeira/node/container.rb +134 -0
- data/lib/empeira/node/container_agent_bootstrap.rb +43 -0
- data/lib/empeira/node/container_bootstrap.rb +52 -0
- data/lib/empeira/node/container_guest.rb +42 -0
- data/lib/empeira/node/container_lifecycle.rb +145 -0
- data/lib/empeira/node/container_preparation.rb +144 -0
- data/lib/empeira/node/container_state.rb +129 -0
- data/lib/empeira/node/definition.rb +20 -0
- data/lib/empeira/node/diagnostic_command.rb +19 -0
- data/lib/empeira/node/dnf_agent_repository.rb +202 -0
- data/lib/empeira/node/external_fact.rb +21 -0
- data/lib/empeira/node/image.rb +38 -0
- data/lib/empeira/node/interactive_tools.rb +33 -0
- data/lib/empeira/node/interface.rb +18 -0
- data/lib/empeira/node/inventory.rb +143 -0
- data/lib/empeira/node/package_bootstrap.rb +129 -0
- data/lib/empeira/node/package_proxy.rb +102 -0
- data/lib/empeira/node/package_sources.rb +19 -0
- data/lib/empeira/node/puppet_command.rb +12 -0
- data/lib/empeira/node/rpm_configuration.rb +62 -0
- data/lib/empeira/node/run_request.rb +52 -0
- data/lib/empeira/node/runtime_proxy.rb +86 -0
- data/lib/empeira/node/service.rb +72 -0
- data/lib/empeira/node/ssh_client.rb +25 -0
- data/lib/empeira/node/ssh_credentials.rb +78 -0
- data/lib/empeira/node/ssh_endpoint.rb +58 -0
- data/lib/empeira/node/ssh_identity.rb +37 -0
- data/lib/empeira/node/user_ssh.rb +60 -0
- data/lib/empeira/node/vm.rb +89 -0
- data/lib/empeira/node/vm_bootstrap.rb +53 -0
- data/lib/empeira/node/vm_lifecycle.rb +212 -0
- data/lib/empeira/node/vm_preflight.rb +37 -0
- data/lib/empeira/node/vm_preparation.rb +161 -0
- data/lib/empeira/node/vm_state.rb +161 -0
- data/lib/empeira/platform/facts.rb +66 -0
- data/lib/empeira/platform/kvm.rb +91 -0
- data/lib/empeira/platform/locations.rb +51 -0
- data/lib/empeira/platform/resolvers.rb +85 -0
- data/lib/empeira/platform/routes.rb +50 -0
- data/lib/empeira/progress.rb +70 -0
- data/lib/empeira/project_root.rb +47 -0
- data/lib/empeira/providers/lifecycle.rb +96 -0
- data/lib/empeira/providers/registry.rb +21 -0
- data/lib/empeira/providers/resource.rb +32 -0
- data/lib/empeira/providers.rb +55 -0
- data/lib/empeira/runtime/agent_helper.rb +34 -0
- data/lib/empeira/runtime/container.rb +184 -0
- data/lib/empeira/runtime/diagnostic_execution.rb +52 -0
- data/lib/empeira/runtime/docker.rb +87 -0
- data/lib/empeira/runtime/image_metadata.rb +83 -0
- data/lib/empeira/runtime/interface.rb +8 -0
- data/lib/empeira/runtime/mount_probe.rb +57 -0
- data/lib/empeira/runtime/podman.rb +122 -0
- data/lib/empeira/runtime/recipe_build.rb +103 -0
- data/lib/empeira/runtime/registry_diagnostics.rb +23 -0
- data/lib/empeira/runtime/service_arguments.rb +55 -0
- data/lib/empeira/runtime/service_images.rb +118 -0
- data/lib/empeira/runtime/service_inspection.rb +159 -0
- data/lib/empeira/runtime/services.rb +120 -0
- data/lib/empeira/runtime/update_helper.rb +97 -0
- data/lib/empeira/runtime/workspace_network.rb +69 -0
- data/lib/empeira/server/environment_configuration.rb +53 -0
- data/lib/empeira/server/http.rb +38 -0
- data/lib/empeira/server/puppetdb_runtime.rb +74 -0
- data/lib/empeira/server/relay_certificate.rb +76 -0
- data/lib/empeira/server/runtime.rb +90 -0
- data/lib/empeira/services/definition.rb +42 -0
- data/lib/empeira/updates/images.rb +104 -0
- data/lib/empeira/updates/modules.rb +45 -0
- data/lib/empeira/updates/service.rb +45 -0
- data/lib/empeira/version_requirement.rb +23 -0
- data/lib/empeira/vm/agent.rb +66 -0
- data/lib/empeira/vm/bootstrap_requirements.rb +69 -0
- data/lib/empeira/vm/certificate_runtime.rb +30 -0
- data/lib/empeira/vm/cloud_init.rb +171 -0
- data/lib/empeira/vm/disk.rb +136 -0
- data/lib/empeira/vm/guest.rb +123 -0
- data/lib/empeira/vm/guest_channel.rb +110 -0
- data/lib/empeira/vm/guest_network.rb +94 -0
- data/lib/empeira/vm/guest_output.rb +52 -0
- data/lib/empeira/vm/guest_shell.rb +79 -0
- data/lib/empeira/vm/guest_upload.rb +43 -0
- data/lib/empeira/vm/image_cache.rb +77 -0
- data/lib/empeira/vm/image_source.rb +91 -0
- data/lib/empeira/vm/interface.rb +8 -0
- data/lib/empeira/vm/interface_ownership.rb +124 -0
- data/lib/empeira/vm/interface_routes.rb +101 -0
- data/lib/empeira/vm/interfaces.rb +240 -0
- data/lib/empeira/vm/management.rb +73 -0
- data/lib/empeira/vm/prerequisites.rb +42 -0
- data/lib/empeira/vm/qemu.rb +83 -0
- data/lib/empeira/vm/qemu_runtime.rb +404 -0
- data/lib/empeira/vm/removal.rb +30 -0
- data/lib/empeira/vm/root_disk.rb +92 -0
- data/lib/empeira/vm/session_lock.rb +56 -0
- data/lib/empeira/vm/system_ssh.rb +29 -0
- data/lib/empeira/workspace.rb +42 -0
- data/lib/empeira.rb +209 -0
- data/resources/browser/relay.rb +22 -0
- data/resources/gateway/Containerfile +8 -0
- data/resources/gateway/bridge.rb +66 -0
- data/resources/gateway/configure.rb +323 -0
- data/resources/modules/Containerfile +15 -0
- data/resources/modules/Gemfile +5 -0
- data/resources/modules/Gemfile.lock +105 -0
- data/resources/modules/worker.rb +161 -0
- data/resources/network/adapter/Containerfile +13 -0
- data/resources/network/adapter/bridge_linux.go +51 -0
- data/resources/network/adapter/framing.go +45 -0
- data/resources/network/adapter/framing_test.go +53 -0
- data/resources/network/adapter/go.mod +3 -0
- data/resources/network/adapter/main_linux.go +273 -0
- data/resources/network/channel.rb +18 -0
- data/resources/network/ssh_connect.rb +32 -0
- data/resources/nodes/apt_sources.rb +66 -0
- data/resources/nodes/debian/Containerfile +10 -0
- data/resources/nodes/diagnostic_exec.rb +109 -0
- data/resources/nodes/interactive_tools.rb +139 -0
- data/resources/nodes/managed_file.rb +91 -0
- data/resources/nodes/management/guest_agent.py +415 -0
- data/resources/nodes/management/setup.sh +20 -0
- data/resources/nodes/management/shell_rc.sh +12 -0
- data/resources/nodes/management/system_account.sh +24 -0
- data/resources/nodes/management/system_home.sh +15 -0
- data/resources/nodes/rpm/Containerfile +10 -0
- data/resources/nodes/runtime_proxy.rb +244 -0
- data/resources/nodes/ssh/proxy.rb +16 -0
- data/resources/nodes/ssh/sshd_config +15 -0
- data/resources/nodes/ssh/start +6 -0
- data/resources/proxy/Containerfile +10 -0
- data/resources/relay/Containerfile +5 -0
- data/resources/relay/puppetdb.rb +81 -0
- data/resources/relay/server_environment.rb +10 -0
- data/resources/relay/server_start.sh +41 -0
- metadata +326 -0
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Empeira
|
|
4
|
+
module ControlPlane
|
|
5
|
+
# The plan holds desired, non-secret service definitions. IPs are runtime bindings.
|
|
6
|
+
# Declarative service specifications share one compatibility and naming boundary.
|
|
7
|
+
# rubocop:disable-next Metrics/ClassLength
|
|
8
|
+
class Plan
|
|
9
|
+
attr_accessor :browser_enabled, :subnet
|
|
10
|
+
attr_reader :context, :files, :naming, :server_runtime, :puppetdb_runtime, :additional_configurations
|
|
11
|
+
|
|
12
|
+
def initialize(context:)
|
|
13
|
+
@context = context
|
|
14
|
+
@files = Files.new(context: context)
|
|
15
|
+
@additional_configurations = AdditionalConfigurations.new(context: context)
|
|
16
|
+
@naming = Network::Naming.new
|
|
17
|
+
@server_runtime = Server::Runtime.resolve(config)
|
|
18
|
+
@puppetdb_runtime = Server::PuppetDBRuntime.resolve(config)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def config
|
|
22
|
+
context.configuration
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def network
|
|
26
|
+
Network::Definition.new(workspace: context.workspace, policy: Network::Policy.new).backend_name
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def proxy?
|
|
30
|
+
egress = config.dig('network', 'egress')
|
|
31
|
+
config.dig('proxy', 'enabled') || (egress.is_a?(Hash) && egress['mode'] == 'proxy')
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def database?
|
|
35
|
+
config.dig('puppetdb', 'enabled')
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def volume(key)
|
|
39
|
+
Services::Definition.new(key: "openvox-#{key}", workspace: context.workspace,
|
|
40
|
+
storage_revision: 1)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def volumes
|
|
44
|
+
keys = %w[ca ssl server-data]
|
|
45
|
+
keys.push('postgres-data', 'puppetdb-data') if database?
|
|
46
|
+
keys.to_h { |key| [key, volume(key)] }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# rubocop:disable-next Metrics/AbcSize -- Compose optional managed services into one provider-neutral plan.
|
|
50
|
+
def definitions(dns: nil)
|
|
51
|
+
common = { network: network, dns: dns, cpus: 1,
|
|
52
|
+
sysctls: { 'net.ipv6.conf.all.disable_ipv6' => '1', 'net.ipv6.conf.default.disable_ipv6' => '1' } }
|
|
53
|
+
result = {
|
|
54
|
+
'dns' => dns_service,
|
|
55
|
+
'gateway' => service('gateway', **common, **gateway),
|
|
56
|
+
'server' => service('server', **common, **server)
|
|
57
|
+
}
|
|
58
|
+
result['proxy'] = service('proxy', **common, **proxy) if proxy?
|
|
59
|
+
result.merge(database_services(common)).merge(additional_services(common))
|
|
60
|
+
.merge(browser_enabled ? Browser.new(self).definitions(dns: dns) : {})
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def additional_services(common = {})
|
|
64
|
+
config.dig('containers', 'additional').each_with_index.to_h do |entry, index|
|
|
65
|
+
options = { image: image_reference(entry.fetch('image')), memory: 256,
|
|
66
|
+
environment: proxy_environment_for('').merge(entry.fetch('environment', {})),
|
|
67
|
+
command: entry.fetch('command', []) }
|
|
68
|
+
location = "containers.additional[#{index}].configuration"
|
|
69
|
+
generated = additional_configurations.options(entry, mounts: common.fetch(:mounts, []), location: location)
|
|
70
|
+
[entry.fetch('name'), service(entry.fetch('name'), **common, **options, **generated)]
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def fingerprints
|
|
75
|
+
definitions.transform_values(&:fingerprint)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def validate_dns_rewrites!
|
|
79
|
+
entries = config.dig('dns', 'rewrites')
|
|
80
|
+
return if entries.empty?
|
|
81
|
+
|
|
82
|
+
targets = definitions.keys.map { |key| naming.hostname(key) }
|
|
83
|
+
entries.each_with_index do |entry, index|
|
|
84
|
+
target = Configuration::DNSRewrites.normalize(entry.fetch('to'))
|
|
85
|
+
next if targets.include?(target)
|
|
86
|
+
|
|
87
|
+
raise ConfigurationError, "dns.rewrites[#{index}].to target service #{target} is missing or disabled; " \
|
|
88
|
+
'configure the service before running empeira up'
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def validate_network_redirects!(subnet: self.subnet)
|
|
93
|
+
Network::Redirects.validate!(config.dig('network', 'redirects'), definitions: definitions, subnet: subnet)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def bind(source, target, readonly: true)
|
|
97
|
+
raise ConfigurationError, 'Mount paths must not contain commas or newlines' if source.to_s.match?(/[,\r\n]/)
|
|
98
|
+
|
|
99
|
+
"type=bind,src=#{source},dst=#{target}#{',readonly' if readonly}"
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def repository_mounts
|
|
103
|
+
[*environment.mounts, *hiera_mounts]
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def project_server_mounts
|
|
107
|
+
@project_server_mounts ||= validate_server_mounts!
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# rubocop:disable-next Metrics/AbcSize -- Compare resolved data mounts with the actual managed definition.
|
|
111
|
+
def validate_server_mounts!
|
|
112
|
+
resolved = Configuration::ServerMounts.new(entries: config.dig('server', 'mounts'),
|
|
113
|
+
project: context.project.path)
|
|
114
|
+
targets = managed_server_mounts.map { |mount| mount.split(',').find { |field| field.start_with?('dst=') }[4..] }
|
|
115
|
+
resolved.verify_targets!(targets)
|
|
116
|
+
@project_server_mounts = resolved.entries.map do |entry|
|
|
117
|
+
bind(entry.fetch('source'), entry.fetch('target'), readonly: entry.fetch('readonly'))
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def module_request
|
|
122
|
+
@module_request ||= Modules::Request.new(context: context)
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def environment
|
|
126
|
+
@environment ||= Environment.new(context: context, hiera: hiera)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def eyaml_mounts
|
|
130
|
+
destination = eyaml_staged? ? '/empeira-eyaml' : Configuration::Eyaml::DESTINATION
|
|
131
|
+
Configuration::Eyaml.new(config: config.fetch('eyaml'), project: context.project.path)
|
|
132
|
+
.mounts(destination: destination)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def eyaml_staged?
|
|
136
|
+
config.dig('eyaml', 'enabled') && server_runtime.startup.fetch('eyaml_keys', 'direct') == 'staged'
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def hiera
|
|
140
|
+
@hiera ||= Configuration::HieraMounts.new(config: config.fetch('hiera'), project: context.project.path,
|
|
141
|
+
environment: config.dig('server', 'environment'))
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def hiera_mounts
|
|
145
|
+
hiera.mounts
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def bootstrap_proxy(dns:)
|
|
149
|
+
options = proxy.merge(network: network, dns: dns, cpus: 1,
|
|
150
|
+
runtime_ip: subnet && Network::Gateway.address(subnet, 'bootstrap-proxy'),
|
|
151
|
+
configuration: 'authenticated bootstrap policy',
|
|
152
|
+
ports: [],
|
|
153
|
+
command: ['-N', '-f', '/empeira-bootstrap/squid.conf'],
|
|
154
|
+
mounts: [bind(files.path('bootstrap-squid.conf'), '/empeira-bootstrap/squid.conf'),
|
|
155
|
+
'type=tmpfs,dst=/var/log/squid', 'type=tmpfs,dst=/var/spool/squid'])
|
|
156
|
+
service('bootstrap-proxy', **options)
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# rubocop:disable-next Metrics/AbcSize -- Bind one managed recipe to its embedded helper source and image identity.
|
|
160
|
+
def gateway_artifact
|
|
161
|
+
registry = config.dig('images', 'registry')
|
|
162
|
+
artifact = Images::Configuration.artifact(config.dig('images', 'direct_egress'), registry: registry)
|
|
163
|
+
script = Pathname(__dir__).join('../../../resources/gateway/configure.rb').realpath
|
|
164
|
+
files = if artifact[:recipe]
|
|
165
|
+
{ 'configure.rb' => script.binread,
|
|
166
|
+
'bridge.rb' => script.dirname.join('bridge.rb').binread }
|
|
167
|
+
else
|
|
168
|
+
{}
|
|
169
|
+
end
|
|
170
|
+
recipe = artifact[:recipe]
|
|
171
|
+
recipe += "\n# configure-sha256: #{Infrastructure::Definition.fingerprint(files)}\n" if recipe
|
|
172
|
+
image = recipe ? Images::Configuration.local_image(recipe, purpose: 'gateway') : artifact[:image]
|
|
173
|
+
artifact.merge(image: image,
|
|
174
|
+
recipe: recipe, files: files)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def node_proxy_environment
|
|
178
|
+
return {} unless proxy?
|
|
179
|
+
|
|
180
|
+
proxy_environment_for("http://#{naming.hostname('proxy')}:3128")
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
private
|
|
184
|
+
|
|
185
|
+
def image_reference(data)
|
|
186
|
+
Images::Configuration.reference(data, registry: config.dig('images', 'registry'))
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def service(key, **options)
|
|
190
|
+
ipv6 = { 'net.ipv6.conf.all.disable_ipv6' => '1', 'net.ipv6.conf.default.disable_ipv6' => '1' }
|
|
191
|
+
options[:sysctls] = ipv6.merge(options.fetch(:sysctls, {}))
|
|
192
|
+
Services::Definition.new(key: key, workspace: context.workspace, hostname: naming.hostname(key), **options)
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def storage(key, destination)
|
|
196
|
+
"type=volume,src=#{volume(key).name},dst=#{destination}"
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def dns_service
|
|
200
|
+
service('dns', image: image_reference(config.dig('images', 'dns')), memory: 64,
|
|
201
|
+
network: network, runtime_ip: subnet && Network::Gateway.address(subnet, 'dns'),
|
|
202
|
+
sysctls: { 'net.ipv4.ip_forward' => '0', 'net.ipv6.conf.all.disable_ipv6' => '1',
|
|
203
|
+
'net.ipv6.conf.default.disable_ipv6' => '1' },
|
|
204
|
+
mounts: [bind(files.directory, '/empeira')],
|
|
205
|
+
configuration: files.configuration.fetch('Corefile'),
|
|
206
|
+
command: ['-conf', '/empeira/Corefile'])
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def database_services(common)
|
|
210
|
+
return {} unless database?
|
|
211
|
+
|
|
212
|
+
{ 'postgres' => service('postgres', **common, **postgres),
|
|
213
|
+
'puppetdb-backend' => service('puppetdb-backend', **common, **puppetdb),
|
|
214
|
+
'puppetdb' => service('puppetdb', **common, **puppetdb_relay(dns: common[:dns])) }
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def server
|
|
218
|
+
{ hiera: hiera.entries, **server_image, **server_startup, memory: config.dig('server', 'memory'),
|
|
219
|
+
cpus: config.dig('server', 'cpus'),
|
|
220
|
+
environment: server_environment, mounts: server_mounts }
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def server_mounts
|
|
224
|
+
[*managed_server_mounts, *project_server_mounts]
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
def managed_server_mounts
|
|
228
|
+
mounts = [storage('ca', '/etc/puppetlabs/puppetserver/ca'),
|
|
229
|
+
storage('ssl', '/etc/puppetlabs/puppet/ssl'),
|
|
230
|
+
storage('server-data', '/opt/puppetlabs/server/data/puppetserver'),
|
|
231
|
+
*repository_mounts, *eyaml_mounts]
|
|
232
|
+
mounts + managed_server_start_mounts
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def managed_server_start_mounts
|
|
236
|
+
mounts = []
|
|
237
|
+
mounts << "type=tmpfs,dst=#{Configuration::Eyaml::DESTINATION}" if eyaml_staged?
|
|
238
|
+
mounts << bind(files.path('server-puppetdb.conf'), '/empeira-server/puppetdb.conf') if database?
|
|
239
|
+
mounts << bind(server_start_script, '/empeira-server/start.sh')
|
|
240
|
+
mounts << bind(server_environment_script, Server::EnvironmentConfiguration::READER)
|
|
241
|
+
mounts
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
def server_startup
|
|
245
|
+
{ entrypoint: 'dumb-init',
|
|
246
|
+
command: ['/bin/sh', '/empeira-server/start.sh', server_runtime.startup.fetch('entrypoint'),
|
|
247
|
+
server_runtime.paths.fetch('puppetdb_config'), eyaml_staged? ? 'staged' : 'direct',
|
|
248
|
+
database? ? 'puppetdb' : 'no-puppetdb', *server_runtime.startup.fetch('arguments')],
|
|
249
|
+
configuration: [File.read(server_start_script), File.read(server_environment_script)].join("\n") }
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
def server_start_script
|
|
253
|
+
Pathname(__dir__).join('../../../resources/relay/server_start.sh').realpath
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
def server_environment_script
|
|
257
|
+
Pathname(__dir__).join('../../../resources/relay/server_environment.rb').realpath
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
def server_image
|
|
261
|
+
{ image: image_reference(config.dig('images', 'server')) }
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
def server_environment
|
|
265
|
+
server_runtime.environment(
|
|
266
|
+
certname: naming.hostname('server'), dns_alt_names: 'server',
|
|
267
|
+
ca_hostname: naming.hostname('server'), autosign: 'false',
|
|
268
|
+
server_hostname: naming.hostname('server'), environment_timeout: '0',
|
|
269
|
+
java_args: '-Xms256m -Xmx768m', max_active_instances: '1',
|
|
270
|
+
puppetdb_enabled: database?.to_s, storeconfigs: database?.to_s,
|
|
271
|
+
reports: database? ? 'puppetdb' : 'log',
|
|
272
|
+
# Upstream entrypoints otherwise rewrite puppetdb.conf from their default URL.
|
|
273
|
+
puppetdb_server_urls: ''
|
|
274
|
+
).merge(proxy_environment)
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
def proxy_environment
|
|
278
|
+
proxy_environment_for(proxy? ? "http://#{naming.hostname('proxy')}:3128" : '')
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
def proxy_environment_for(url)
|
|
282
|
+
bypass = Network::DirectEgress.destinations(config.dig('network', 'egress'))
|
|
283
|
+
no_proxy = naming.no_proxy(bypass)
|
|
284
|
+
{ 'http_proxy' => url, 'https_proxy' => url, 'HTTP_PROXY' => url, 'HTTPS_PROXY' => url,
|
|
285
|
+
'all_proxy' => '', 'ALL_PROXY' => '', 'no_proxy' => no_proxy, 'NO_PROXY' => no_proxy }
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
def postgres
|
|
289
|
+
{ image: image_reference(config.dig('images', 'postgres')), memory: 256,
|
|
290
|
+
env_file: files.secret_path,
|
|
291
|
+
configuration: files.configuration.fetch('database-setup.sql'),
|
|
292
|
+
environment: { 'POSTGRES_USER' => 'postgres', 'POSTGRES_DB' => 'puppetdb' },
|
|
293
|
+
mounts: [storage('postgres-data', '/var/lib/postgresql/data'),
|
|
294
|
+
bind(files.path('database-setup.sql'), '/empeira/setup.sql')],
|
|
295
|
+
command: ['postgres', '-c', 'shared_buffers=64MB', '-c', 'max_connections=50'] }
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
def puppetdb
|
|
299
|
+
{ configuration: files.configuration.slice('jetty.ini', 'database.conf', 'puppetdb.conf'),
|
|
300
|
+
image: image_reference(config.dig('images', 'puppetdb')), memory: config.dig('puppetdb', 'memory'),
|
|
301
|
+
user: puppetdb_runtime.user, entrypoint: puppetdb_runtime.entrypoint,
|
|
302
|
+
command: puppetdb_runtime.command, env_file: files.secret_path,
|
|
303
|
+
mounts: puppetdb_mounts }
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
def puppetdb_mounts
|
|
307
|
+
[storage('puppetdb-data', puppetdb_runtime.paths.fetch('data')),
|
|
308
|
+
bind(files.directory, '/empeira'),
|
|
309
|
+
*%w[jetty.ini database.conf puppetdb.conf].map do |name|
|
|
310
|
+
bind(files.path(name), "#{puppetdb_runtime.paths.fetch('config')}/#{name}")
|
|
311
|
+
end]
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
def puppetdb_relay(dns:)
|
|
315
|
+
script = Pathname(__dir__).join('../../../resources/relay/puppetdb.rb').realpath
|
|
316
|
+
certificate = Server::RelayCertificate.new(context: context)
|
|
317
|
+
Browser.new(self).relay_image.merge(
|
|
318
|
+
memory: 96, cpus: 1, user: '0', entrypoint: 'ruby', command: ['/empeira-relay/puppetdb.rb'],
|
|
319
|
+
configuration: File.read(script), runtime_environment: { 'EMPEIRA_DNS_IP' => dns },
|
|
320
|
+
sysctls: { 'net.ipv4.ip_forward' => '0' },
|
|
321
|
+
mounts: [bind(script, '/empeira-relay/puppetdb.rb'),
|
|
322
|
+
bind(certificate.path('cert.pem'), '/empeira-relay/cert.pem'),
|
|
323
|
+
bind(certificate.path('key.pem'), '/empeira-relay/key.pem')]
|
|
324
|
+
)
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
def proxy
|
|
328
|
+
configured = config.dig('images', 'proxy')
|
|
329
|
+
{
|
|
330
|
+
**Images::Configuration.artifact(configured, registry: config.dig('images', 'registry')),
|
|
331
|
+
memory: 128, runtime_ip: subnet && Network::Gateway.address(subnet, 'proxy'),
|
|
332
|
+
sysctls: { 'net.ipv4.ip_forward' => '0', 'net.ipv6.conf.all.disable_ipv6' => '1',
|
|
333
|
+
'net.ipv6.conf.default.disable_ipv6' => '1' }, entrypoint: '/usr/sbin/squid-gnutls',
|
|
334
|
+
command: ['-N', '-f', '/empeira-proxy/squid.conf'],
|
|
335
|
+
mounts: [bind(files.directory, '/empeira-proxy'),
|
|
336
|
+
'type=tmpfs,dst=/var/log/squid', 'type=tmpfs,dst=/var/spool/squid']
|
|
337
|
+
}
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
def gateway
|
|
341
|
+
artifact = gateway_artifact
|
|
342
|
+
{
|
|
343
|
+
**artifact,
|
|
344
|
+
runtime_ip: subnet && Network::Gateway.address(subnet),
|
|
345
|
+
build_files: artifact.fetch(:files), memory: 64, read_only: true, cap_drop: ['ALL'], cap_add: ['NET_ADMIN'],
|
|
346
|
+
security_options: ['no-new-privileges'], sysctls: { 'net.ipv4.ip_forward' => '1',
|
|
347
|
+
'net.ipv6.conf.all.disable_ipv6' => '1',
|
|
348
|
+
'net.ipv6.conf.default.disable_ipv6' => '1',
|
|
349
|
+
'net.ipv4.conf.all.rp_filter' => '0',
|
|
350
|
+
'net.ipv4.conf.default.rp_filter' => '0' },
|
|
351
|
+
mounts: [bind(files.directory, '/empeira-gateway'), 'type=tmpfs,dst=/run,tmpfs-size=1048576']
|
|
352
|
+
}.except(:files)
|
|
353
|
+
end
|
|
354
|
+
end
|
|
355
|
+
end
|
|
356
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Empeira
|
|
4
|
+
module ControlPlane
|
|
5
|
+
class Storage
|
|
6
|
+
def initialize(runtime:, plan:, inventory:, persist:)
|
|
7
|
+
@runtime = runtime
|
|
8
|
+
@plan = plan
|
|
9
|
+
@inventory = inventory
|
|
10
|
+
@persist = persist
|
|
11
|
+
@changed = false
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def prepare
|
|
15
|
+
definitions = @plan.volumes.values.to_h { |definition| [definition.key, definition] }
|
|
16
|
+
expected = @inventory.fetch('volumes').transform_values { |record| record['id'] }
|
|
17
|
+
@observed = @runtime.inspect_volumes(definitions, expected_ids: expected)
|
|
18
|
+
@plan.files.credentials(existing: database_recorded?) if @plan.database?
|
|
19
|
+
@plan.volumes.each_value { |definition| ensure_volume(definition) }
|
|
20
|
+
@changed
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def database_recorded?
|
|
24
|
+
definition = @plan.volume('postgres-data')
|
|
25
|
+
@inventory.fetch('volumes').key?(definition.key) || !@observed[definition.key].nil?
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def ensure_volume(definition)
|
|
29
|
+
expected = @inventory.fetch('volumes').dig(definition.key, 'id')
|
|
30
|
+
resource = @observed[definition.key]
|
|
31
|
+
if !resource && expected
|
|
32
|
+
raise Error, "Retained volume #{definition.key} is missing; restore it before starting services"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
resource ||= create_volume(definition)
|
|
36
|
+
@inventory.fetch('volumes')[definition.key] = { 'id' => resource.fetch('id') }
|
|
37
|
+
@persist.call
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def create_volume(definition)
|
|
41
|
+
@inventory.fetch('volumes')[definition.key] = { 'id' => nil }
|
|
42
|
+
@persist.call
|
|
43
|
+
@changed = true
|
|
44
|
+
@runtime.create_volume(definition)
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'io/console'
|
|
4
|
+
require 'socket'
|
|
5
|
+
|
|
6
|
+
module Empeira
|
|
7
|
+
module Execution
|
|
8
|
+
class Console
|
|
9
|
+
def attach(path, input:, output:, guidance: nil)
|
|
10
|
+
UNIXSocket.open(path) do |socket|
|
|
11
|
+
output.write(guidance) if guidance
|
|
12
|
+
output.flush
|
|
13
|
+
Terminal.with_interrupts do
|
|
14
|
+
if input.tty?
|
|
15
|
+
input.raw { transfer(socket, input, output) }
|
|
16
|
+
else
|
|
17
|
+
transfer(socket, input, output)
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
Result.new(stdout: '', stderr: '', exit_status: 0, timed_out: false)
|
|
22
|
+
rescue SystemCallError, IOError
|
|
23
|
+
raise ExecutionError, 'VM console connection failed or closed', cause: nil
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def transfer(socket, input, output)
|
|
27
|
+
loop do
|
|
28
|
+
IO.select([socket, input]).first.each do |source|
|
|
29
|
+
bytes = source.readpartial(4096)
|
|
30
|
+
if source == socket
|
|
31
|
+
output.write(bytes)
|
|
32
|
+
output.flush
|
|
33
|
+
else
|
|
34
|
+
before, escape, = bytes.partition("\x1d")
|
|
35
|
+
socket.write(before) unless before.empty?
|
|
36
|
+
return nil unless escape.empty?
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
rescue EOFError
|
|
41
|
+
nil
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'uri'
|
|
4
|
+
require 'base64'
|
|
5
|
+
|
|
6
|
+
module Empeira
|
|
7
|
+
module Execution
|
|
8
|
+
# Bounded, sanitized diagnostics for direct HTTP responses and native tool failures.
|
|
9
|
+
# rubocop:disable-next Metrics/ModuleLength -- HTTP, native tools and SSH share the existing redaction boundary.
|
|
10
|
+
module Diagnostics
|
|
11
|
+
RELEVANT = %r{permission\sdenied|could\snot|couldn't|cannot\s|unable\sto|failed|failure|fatal:|error:|not\sfound|
|
|
12
|
+
does\snot\sexist|not\sa\svalid|not\sappear|connection|timed?\sout|resolve|name\sor\sservice|
|
|
13
|
+
server\sresponded|HTTP[\s/]|certificate|host\skey|load\skey|invalid|conflicts|
|
|
14
|
+
unauthorized|forbidden|authentication\srequired|denied|insufficient_scope|manifest\sunknown|
|
|
15
|
+
name\sunknown|x509|unknown\sauthority|network\sunreachable|DNS\sfailure|timeout|
|
|
16
|
+
no\sbasic\sauth\scredentials|toomanyrequests|rate[\s-]limit|too\smany\srequests|
|
|
17
|
+
\A(?:Get|Head|Post)\s|
|
|
18
|
+
Puppetfile\smoduledir|\AModule\s[a-z][a-z0-9_]*[:\s]}ix
|
|
19
|
+
SECRET = /[a-z0-9_-]*(?:authorization|token|password|passwd|secret|api[-_]?key|access[-_]?key|credential|cookie)
|
|
20
|
+
[a-z0-9_-]*/ix
|
|
21
|
+
LIMIT = 8192
|
|
22
|
+
URL = %r{\b(?:https?|ssh|docker|oci)://[^\s<>"']+}
|
|
23
|
+
QUERY_SECRET = /#{SECRET}|auth|(?:\A|[-_])(?:sig|signature|key|session|jwt)\z/i
|
|
24
|
+
|
|
25
|
+
# rubocop:disable-next Metrics/ParameterLists -- Keep actual response fields explicit at the diagnostic boundary.
|
|
26
|
+
def self.http(operation:, url:, status:, reason:, body: nil, content_type: nil, secrets: [])
|
|
27
|
+
values = secrets + url_secrets(url)
|
|
28
|
+
response = body.nil? ? '[No response body available]' : response_text(body, content_type, values)
|
|
29
|
+
"Operation: #{safe_text(operation, secrets: values)}\nURL: #{safe_url(url)}\n" \
|
|
30
|
+
"HTTP status: #{safe_text("#{status} #{reason}", secrets: values).strip}\nResponse:\n" \
|
|
31
|
+
"#{response.lines.map { |line| " #{line}" }.join}"
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def self.http_response(response, operation:, url:, secrets: [])
|
|
35
|
+
http(operation: operation, url: url, status: response.code, reason: response.message,
|
|
36
|
+
body: read_body(response), content_type: response['Content-Type'], secrets: secrets)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def self.read_body(response)
|
|
40
|
+
body = +''.b
|
|
41
|
+
response.read_body do |chunk|
|
|
42
|
+
body << chunk
|
|
43
|
+
break if body.bytesize > LIMIT * 8
|
|
44
|
+
end
|
|
45
|
+
body
|
|
46
|
+
rescue IOError, SystemCallError, Timeout::Error, Net::ProtocolError => e
|
|
47
|
+
"#{body}\n[Response body incomplete: #{e.class.name}]"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def self.transport(operation:, url:, error:, response: nil, secrets: [])
|
|
51
|
+
response ||= error.response if error.respond_to?(:response) && error.response.respond_to?(:code)
|
|
52
|
+
status = response ? "#{response.code} #{response.message}" : 'No HTTP response received'
|
|
53
|
+
"Operation: #{safe_text(operation)}\nURL: #{safe_url(url)}\nConnection: #{connection(error)}\n" \
|
|
54
|
+
"HTTP status: #{safe_text(status)}\nDetails: " \
|
|
55
|
+
"#{bounded(safe_text(error.message, secrets: secrets + url_secrets(url)))}"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def self.connection(error)
|
|
59
|
+
case error
|
|
60
|
+
when OpenSSL::SSL::SSLError then 'TLS connection or certificate verification failed'
|
|
61
|
+
when SocketError then 'DNS resolution failed'
|
|
62
|
+
when Errno::ECONNREFUSED then 'Connection refused'
|
|
63
|
+
when Timeout::Error, Errno::ETIMEDOUT then 'Connection timed out'
|
|
64
|
+
when Net::HTTPFatalError then 'Proxy connection failed'
|
|
65
|
+
else 'Connection or transfer failed'
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def self.native(result, operation:, tool:, url: nil, secrets: [])
|
|
70
|
+
text = safe_text("#{result.stderr}\n#{result.stdout}", secrets: secrets)
|
|
71
|
+
location = url ? safe_url(url) : "Not reported separately by #{tool}; see native output"
|
|
72
|
+
"Operation: #{safe_text(operation)}\nURL: #{location}\n" \
|
|
73
|
+
"HTTP status / reason: Not reported separately by #{tool}; see native output\n" \
|
|
74
|
+
"Response: Not provided separately by #{tool}\n" \
|
|
75
|
+
"#{tool} output (exit=#{result.exit_status}, timeout=#{result.timed_out}):\n" \
|
|
76
|
+
"#{text.strip.empty? ? '[No native diagnostic available]' : bounded(text.strip)}"
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def self.command(result, operation:, tool:, secrets: [])
|
|
80
|
+
secrets += %w[EMPEIRA_AGENT_REPO_USERNAME EMPEIRA_AGENT_REPO_PASSWORD].filter_map { |key| ENV.fetch(key, nil) }
|
|
81
|
+
stdout = bounded(safe_text(result.stdout, secrets: secrets))
|
|
82
|
+
stderr = bounded(safe_text(result.stderr, secrets: secrets))
|
|
83
|
+
"Operation: #{safe_text(operation, secrets: secrets)}\nTool: #{safe_text(tool)}\n" \
|
|
84
|
+
"Exit code: #{result.exit_status || 'unavailable'}\nTimeout: #{result.timed_out}\n" \
|
|
85
|
+
"stdout:\n#{stdout.empty? ? '[Empty]' : stdout}\nstderr:\n#{stderr.empty? ? '[Empty]' : stderr}"
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def self.response_text(body, content_type, secrets)
|
|
89
|
+
return '[Empty response body]' if body.empty?
|
|
90
|
+
return '[Binary response body omitted]' if binary?(body, content_type)
|
|
91
|
+
|
|
92
|
+
text = safe_text(body, secrets: secrets)
|
|
93
|
+
prefix = content_type.to_s.match?(/html/i) ? "[HTML response]\n" : ''
|
|
94
|
+
"#{prefix}#{bounded(text)}"
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def self.binary?(body, content_type)
|
|
98
|
+
body.b.match?(/[\x00-\x08\x0b\x0c\x0e-\x1a\x1c-\x1f]/n) ||
|
|
99
|
+
content_type.to_s.match?(%r{\A(?:application/octet-stream|image/|audio/|video/)}) ||
|
|
100
|
+
(!body.dup.force_encoding('UTF-8').valid_encoding? && !content_type.to_s.match?(/text|json|xml/i))
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def self.bounded(text)
|
|
104
|
+
return text if text.bytesize <= LIMIT
|
|
105
|
+
|
|
106
|
+
"#{text.byteslice(0, LIMIT).scrub}\n[Truncated diagnostic after #{LIMIT} bytes]"
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def self.safe_url(value)
|
|
110
|
+
uri = URI(value.to_s)
|
|
111
|
+
uri.user = nil
|
|
112
|
+
uri.password = nil
|
|
113
|
+
uri.fragment = nil
|
|
114
|
+
if uri.query
|
|
115
|
+
uri.query = URI.encode_www_form(URI.decode_www_form(uri.query).map do |key, item|
|
|
116
|
+
[key, key.match?(QUERY_SECRET) ? '[REDACTED]' : item]
|
|
117
|
+
end)
|
|
118
|
+
end
|
|
119
|
+
uri.to_s
|
|
120
|
+
rescue URI::InvalidURIError, ArgumentError
|
|
121
|
+
'[Invalid URL omitted]'
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def self.url_secrets(value)
|
|
125
|
+
uri = URI(value.to_s)
|
|
126
|
+
fields = uri.userinfo.to_s.split(':', 2) + URI.decode_www_form(uri.query.to_s).filter_map do |key, item|
|
|
127
|
+
item if key.match?(QUERY_SECRET)
|
|
128
|
+
end
|
|
129
|
+
fields.flat_map { |field| [field, URI.decode_www_form_component(field)] }.reject(&:empty?)
|
|
130
|
+
rescue URI::InvalidURIError, ArgumentError
|
|
131
|
+
[]
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Redact before limiting output; retain actual URL hosts/paths and multiline server details.
|
|
135
|
+
def self.safe_text(text, secrets: [])
|
|
136
|
+
value = text.to_s.dup.force_encoding('UTF-8').scrub
|
|
137
|
+
values = known_values(value, secrets)
|
|
138
|
+
value = value.gsub(URL) { |url| safe_url(url) }
|
|
139
|
+
values.each { |item| value = value.gsub(item, '[REDACTED]') }
|
|
140
|
+
strip_controls(redact_fields(value))
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def self.known_values(text, secrets)
|
|
144
|
+
values = secrets + text.scan(URL).flat_map { |url| url_secrets(url) }
|
|
145
|
+
values << Base64.strict_encode64(secrets.join(':')) if secrets.size == 2
|
|
146
|
+
values.reject(&:empty?).uniq.sort_by { |item| -item.length }
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def self.redact_fields(value)
|
|
150
|
+
value = value.gsub(/-----BEGIN [^-]*PRIVATE KEY-----.*?(?:-----END [^-]*PRIVATE KEY-----|\z)/m, '[REDACTED]')
|
|
151
|
+
value = value.gsub(/^\s*(?:\S*-)?(?:Authorization|Cookie|Set-Cookie):[^\n]*/i, '[REDACTED HEADER]')
|
|
152
|
+
value = value.gsub(/\bno basic auth credentials\b/i, 'authentication required (no credentials)')
|
|
153
|
+
value = value.gsub(/\b(?:Bearer|Basic)\s+\S+/i, '[REDACTED]')
|
|
154
|
+
value = value.gsub(/((?:#{SECRET})["']?\s*[=:]\s*)(?:"[^"]*"|'[^']*'|\S+)/i, '\1[REDACTED]')
|
|
155
|
+
value.gsub(/\b(?:gh[pousr]_[A-Za-z0-9_]+|github_pat_[A-Za-z0-9_]+|glpat-[A-Za-z0-9_-]+)/, '[REDACTED]')
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def self.strip_controls(value)
|
|
159
|
+
value.gsub(/\e\][^\a]*(?:\a|\e\\)/, '').gsub(/\e\[[0-9;]*[A-Za-z]/, '')
|
|
160
|
+
.gsub(/[\p{Cc}\p{Cf}&&[^\n\t]]/, '')
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def self.registry_failure(diagnostic, timed_out: false)
|
|
164
|
+
return :network if timed_out || diagnostic.match?(/timed?\s*out|timeout/i)
|
|
165
|
+
return :tls if diagnostic.match?(/x509|certificate.*(?:failed|unknown|expired)|unknown authority/i)
|
|
166
|
+
if diagnostic.match?(/connection\srefused|network\s(?:is\s)?unreachable|no\ssuch\shost|DNS\sfailure|
|
|
167
|
+
(?:name|temporary)\sresolution|could\snot\sresolve\s(?:host|hostname)/ix)
|
|
168
|
+
return :network
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
auth = diagnostic.gsub(/permission denied/i, '')
|
|
172
|
+
if auth.match?(/\b(?:401|403|unauthorized|forbidden|denied|insufficient_scope)\b|
|
|
173
|
+
authentication\srequired|no\sbasic\sauth\scredentials/ix)
|
|
174
|
+
return :auth
|
|
175
|
+
end
|
|
176
|
+
return :missing if diagnostic.match?(/manifest unknown|name unknown|not found/i)
|
|
177
|
+
|
|
178
|
+
:unknown
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def self.clean(text)
|
|
182
|
+
value = text.to_s.encode('UTF-8', invalid: :replace, undef: :replace)
|
|
183
|
+
value = value.gsub(/-----BEGIN [^-]*PRIVATE KEY-----.*?(?:-----END [^-]*PRIVATE KEY-----|\z)/m, '[REDACTED]')
|
|
184
|
+
value.lines.filter_map { |line| diagnostic(line) }.first(20).join("\n")
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def self.diagnostic(line)
|
|
188
|
+
line = line.gsub(/\e\][^\a]*(?:\a|\e\\)/, '').gsub(/\e\[[0-9;]*[A-Za-z]/, '')
|
|
189
|
+
.gsub(/[\p{Cc}\p{Cf}]/, ' ').strip
|
|
190
|
+
return if line.empty? || !line.match?(RELEVANT) || line.match?(/authorization|cookie/i)
|
|
191
|
+
if line.match?(/(?:executing|command|running):|\b(?:git|ssh)\s+(?:-\S+\s+)*(?:clone|fetch|checkout|ls-remote)/i)
|
|
192
|
+
return
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
sanitize(line)[0, 500]
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def self.sanitize(line)
|
|
199
|
+
line = line.gsub(/\bno basic auth credentials\b/i, 'authentication required (no credentials)')
|
|
200
|
+
line = line.gsub(%r{\b(?:https?|ssh|docker|oci)://[^\s<>"']+}, '[REDACTED URL]')
|
|
201
|
+
line = line.gsub(/\b(?:Bearer|Basic)\s+\S+/i, '[REDACTED]')
|
|
202
|
+
line = line.gsub(/((?:#{SECRET})["']?\s*[=:]\s*)(?:"[^"]*"|'[^']*'|\S+)/i, '\1[REDACTED]')
|
|
203
|
+
line = line.gsub(/\b(?:gh[pousr]_[A-Za-z0-9_]+|github_pat_[A-Za-z0-9_]+|glpat-[A-Za-z0-9_-]+)/, '[REDACTED]')
|
|
204
|
+
line = line.gsub(/((?:load key|identity file|private key)\s*)["'][^"']*["']/i, '\1[REDACTED PATH]')
|
|
205
|
+
line = line.gsub(%r{"/[^"\n]*"|'/[^'\n]*'}, '[REDACTED PATH]')
|
|
206
|
+
line.gsub(%r{(?<![\w/])(?:~?/)[^\s"'<>:]+}, '[REDACTED PATH]')
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
end
|