rascal 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +4 -4
- data/.ruby-version +1 -1
- data/AGENTS.md +4 -0
- data/CHANGELOG.md +74 -0
- data/CONTRIBUTING.md +139 -0
- data/Gemfile.common +0 -3
- data/Gemfile.lock +1 -47
- data/Guardfile +0 -9
- data/README.md +679 -76
- data/Rakefile +1 -7
- data/lib/rascal/cli/base.rb +41 -20
- data/lib/rascal/cli/clean.rb +3 -8
- data/lib/rascal/cli/environments.rb +1 -3
- data/lib/rascal/cli/main.rb +106 -17
- data/lib/rascal/cli/run.rb +6 -7
- data/lib/rascal/cli/shell.rb +3 -3
- data/lib/rascal/cli/status.rb +81 -0
- data/lib/rascal/cli/update.rb +8 -8
- data/lib/rascal/cli.rb +1 -0
- data/lib/rascal/command_script.rb +83 -0
- data/lib/rascal/docker/command_containers.rb +101 -0
- data/lib/rascal/docker/container.rb +82 -101
- data/lib/rascal/docker/image.rb +71 -0
- data/lib/rascal/docker/interface.rb +4 -2
- data/lib/rascal/docker/network.rb +12 -0
- data/lib/rascal/docker/volume.rb +17 -10
- data/lib/rascal/docker.rb +11 -4
- data/lib/rascal/environment.rb +164 -40
- data/lib/rascal/gitlab/config_file.rb +199 -0
- data/lib/rascal/gitlab/environment_builder.rb +204 -0
- data/lib/rascal/gitlab/jobs.rb +299 -0
- data/lib/rascal/gitlab/variables.rb +155 -0
- data/lib/rascal/gitlab.rb +16 -0
- data/lib/rascal/io_helper.rb +11 -0
- data/lib/rascal/service.rb +52 -9
- data/lib/rascal/version.rb +1 -1
- data/lib/rascal.rb +2 -1
- data/rascal.gemspec +3 -2
- metadata +15 -8
- data/bin/cucumber +0 -29
- data/cucumber.yml +0 -1
- data/lib/rascal/environments_definition/gitlab.rb +0 -146
- data/lib/rascal/environments_definition.rb +0 -23
data/lib/rascal/environment.rb
CHANGED
|
@@ -1,32 +1,51 @@
|
|
|
1
1
|
require 'shellwords'
|
|
2
|
+
require 'tmpdir'
|
|
2
3
|
|
|
3
4
|
module Rascal
|
|
4
5
|
class Environment
|
|
5
|
-
|
|
6
|
+
include IOHelper
|
|
6
7
|
|
|
7
|
-
|
|
8
|
+
# Where the host's SSH agent socket appears in the container.
|
|
9
|
+
SSH_AGENT_SOCKET = '/run/rascal/ssh-agent.sock'
|
|
10
|
+
|
|
11
|
+
# Docker Desktop's path for the host's SSH agent. Sockets from macOS cannot be mounted directly.
|
|
12
|
+
DOCKER_DESKTOP_SSH_AGENT_SOCKET = '/run/host-services/ssh-auth.sock'
|
|
13
|
+
|
|
14
|
+
# How long to wait by default for a service to accept connections before running the command anyway.
|
|
15
|
+
DEFAULT_SERVICES_TIMEOUT = 30
|
|
16
|
+
|
|
17
|
+
attr_reader :name, :network, :image, :command_containers, :env_variables, :services, :volumes, :working_dir, :before_shell,
|
|
18
|
+
:problems, :warnings, :services_timeout
|
|
19
|
+
|
|
20
|
+
# `problems` prevent the environment from starting, but not from being cleaned.
|
|
21
|
+
# `warnings` are shown before starting it.
|
|
22
|
+
# With `ssh_agent`, the host's SSH agent and known hosts are available in the container,
|
|
23
|
+
# so it can fetch private dependencies over SSH. `services_timeout` is how many seconds to
|
|
24
|
+
# wait for services to accept connections.
|
|
25
|
+
def initialize(full_name, name:, image:, entrypoint: nil, env_variables: {}, services: [], volumes: [], before_shell: [], after_shell: [], working_dir: nil, problems: [], warnings: [], ssh_agent: false, services_timeout: DEFAULT_SERVICES_TIMEOUT)
|
|
8
26
|
@name = name
|
|
9
27
|
@network = Docker::Network.new(full_name)
|
|
10
|
-
@
|
|
28
|
+
@docker_name = full_name
|
|
29
|
+
@image = Docker::Image.new(image)
|
|
30
|
+
@command_containers = Docker::CommandContainers.new(full_name, @image, entrypoint: entrypoint)
|
|
11
31
|
@env_variables = env_variables
|
|
12
32
|
@services = services
|
|
13
33
|
@volumes = volumes
|
|
14
34
|
@working_dir = working_dir
|
|
15
35
|
@before_shell = before_shell
|
|
16
36
|
@after_shell = after_shell
|
|
37
|
+
@problems = problems
|
|
38
|
+
@warnings = warnings
|
|
39
|
+
@ssh_agent = ssh_agent
|
|
40
|
+
@services_timeout = services_timeout
|
|
17
41
|
end
|
|
18
42
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
network: @network,
|
|
26
|
-
volumes: @volumes,
|
|
27
|
-
working_dir: @working_dir,
|
|
28
|
-
allow_failure: true
|
|
29
|
-
)
|
|
43
|
+
# `env`, `unset_env` and `ssh_agent` change the container for this command only, like for
|
|
44
|
+
# updating dependencies. Services keep their variables. `ssh_agent` overrides the config's
|
|
45
|
+
# setting unless it is nil.
|
|
46
|
+
def run_shell(env: {}, unset_env: [], ssh_agent: nil)
|
|
47
|
+
starting_services = start
|
|
48
|
+
run_in_container(script('bash', starting_services), tty: true, env: env, unset_env: unset_env, ssh_agent: ssh_agent).tap { report_stopped_services }
|
|
30
49
|
end
|
|
31
50
|
|
|
32
51
|
# Runs a single command in the environment, without a TTY, and returns its
|
|
@@ -37,55 +56,160 @@ module Rascal
|
|
|
37
56
|
# The command is a list of arguments, which is escaped before being handed
|
|
38
57
|
# to bash, so arguments may contain spaces and shell metacharacters. Use
|
|
39
58
|
# e.g. `bash -c "foo && bar"` to run something the shell must interpret.
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
env:
|
|
45
|
-
network: @network,
|
|
46
|
-
volumes: @volumes,
|
|
47
|
-
working_dir: @working_dir,
|
|
48
|
-
allow_failure: true,
|
|
49
|
-
tty: false
|
|
50
|
-
)
|
|
59
|
+
# See #run_shell for the keyword arguments.
|
|
60
|
+
def run_command(*command, env: {}, unset_env: [], ssh_agent: nil)
|
|
61
|
+
starting_services = start
|
|
62
|
+
run_in_container(script(Shellwords.join(command), starting_services), tty: false, forward_stdin: stdin_redirected?,
|
|
63
|
+
env: env, unset_env: unset_env, ssh_agent: ssh_agent).tap { report_stopped_services }
|
|
51
64
|
end
|
|
52
65
|
|
|
53
66
|
def clean(clean_volumes: false)
|
|
67
|
+
@command_containers.clean
|
|
54
68
|
@services.each(&:clean)
|
|
55
69
|
@network.clean
|
|
56
70
|
@volumes.each(&:clean) if clean_volumes
|
|
57
71
|
end
|
|
58
72
|
|
|
73
|
+
# What rascal made of the CI config, and what of the environment exists in Docker,
|
|
74
|
+
# for `rascal status`. Starts and downloads nothing.
|
|
75
|
+
def status
|
|
76
|
+
{
|
|
77
|
+
'name' => @name,
|
|
78
|
+
'image' => @image.name,
|
|
79
|
+
'image_present' => @image.present?,
|
|
80
|
+
'entrypoint' => @command_containers.entrypoint,
|
|
81
|
+
# What runs before each command unless `entrypoint` replaces it.
|
|
82
|
+
'image_entrypoint' => (@image.entrypoint if @image.present?),
|
|
83
|
+
'problems' => @problems,
|
|
84
|
+
'warnings' => @warnings,
|
|
85
|
+
'network' => { 'name' => @network.prefixed_name, 'exists' => @network.exists? },
|
|
86
|
+
'services' => @services.collect(&:status),
|
|
87
|
+
'volumes' => @volumes.collect(&:status),
|
|
88
|
+
'command_containers' => @command_containers.names,
|
|
89
|
+
'working_dir' => @working_dir,
|
|
90
|
+
'variables' => @env_variables,
|
|
91
|
+
'before_shell' => @before_shell,
|
|
92
|
+
'after_shell' => @after_shell,
|
|
93
|
+
'ssh_agent' => @ssh_agent,
|
|
94
|
+
'services_timeout' => @services_timeout,
|
|
95
|
+
}
|
|
96
|
+
end
|
|
97
|
+
|
|
59
98
|
def update(skip: [])
|
|
99
|
+
raise_problems
|
|
60
100
|
[
|
|
61
101
|
*@services.collect { |s| s.update(skip: skip) },
|
|
62
|
-
*@
|
|
102
|
+
*@image.update(skip: skip),
|
|
63
103
|
]
|
|
64
104
|
end
|
|
65
105
|
|
|
66
106
|
private
|
|
67
107
|
|
|
68
|
-
#
|
|
69
|
-
# command
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
108
|
+
# Returns the services that may not be ready yet: those it started, and those another
|
|
109
|
+
# command started less than services_timeout ago.
|
|
110
|
+
def start
|
|
111
|
+
raise_problems
|
|
112
|
+
@warnings.each { |warning| warning(warning) }
|
|
113
|
+
download_missing
|
|
114
|
+
started = start_services
|
|
115
|
+
@services.select { |service| started.include?(service) || service.starting?(@services_timeout) }
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Only passes stdin on when it comes from a file or pipe, like `rascal run job -- psql < dump.sql`,
|
|
119
|
+
# or from a socket with input waiting, like from Node's child_process. A terminal, or a socket
|
|
120
|
+
# that stays open without input, would make commands that read stdin hang.
|
|
121
|
+
def stdin_redirected?
|
|
122
|
+
stat = stdin.stat
|
|
123
|
+
stat.pipe? || stat.file? || (stat.socket? && !IO.select([stdin], nil, nil, 0).nil?)
|
|
124
|
+
rescue NoMethodError, IOError, SystemCallError
|
|
125
|
+
false
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# A service that stopped, typically because of bad settings, makes commands fail in
|
|
129
|
+
# confusing ways, so point to its output.
|
|
130
|
+
def report_stopped_services
|
|
131
|
+
@services.reject(&:running?).each do |service|
|
|
132
|
+
warning("Service #{service.aliases.first} is not running anymore. See why with `docker logs #{service.container.prefixed_name}`, " \
|
|
133
|
+
"then fix its settings and run `rascal clean #{@name}`.")
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def raise_problems
|
|
138
|
+
raise Error, @problems.join("\n") if @problems.any?
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def run_in_container(script, tty:, env:, unset_env:, ssh_agent:, forward_stdin: tty)
|
|
142
|
+
ssh_volumes, ssh_env = ssh_agent_access(ssh_agent.nil? ? @ssh_agent : ssh_agent)
|
|
143
|
+
@command_containers.run('bash', '-c', script,
|
|
144
|
+
env: @env_variables.except(*unset_env).merge(env, ssh_env),
|
|
145
|
+
network: @network,
|
|
146
|
+
volumes: [*@volumes, *ssh_volumes],
|
|
147
|
+
working_dir: @working_dir,
|
|
148
|
+
tty: tty,
|
|
149
|
+
forward_stdin: forward_stdin,
|
|
150
|
+
)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# `starting_services` are those that may not accept connections yet.
|
|
154
|
+
def script(command, starting_services)
|
|
155
|
+
CommandScript.new(command,
|
|
156
|
+
working_dir: @working_dir,
|
|
157
|
+
before_shell: @before_shell,
|
|
158
|
+
after_shell: @after_shell,
|
|
159
|
+
waits: starting_services.collect { |service| [service.aliases.first, service.ports] },
|
|
160
|
+
wait_timeout: @services_timeout,
|
|
161
|
+
owner: project_owner,
|
|
162
|
+
).to_s
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# The user that should own files the container creates in the project: the host user, unless
|
|
166
|
+
# the project already appears in the container as theirs. With rootless Docker, the host user
|
|
167
|
+
# is root in the container, and Docker Desktop for Mac maps ownership itself.
|
|
168
|
+
def project_owner
|
|
169
|
+
[Process.uid, Process.gid] unless Process.uid == 0 || RUBY_PLATFORM.include?('darwin')
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# Volumes and variables that give the container access to the host's SSH agent.
|
|
173
|
+
def ssh_agent_access(enabled)
|
|
174
|
+
return [[], {}] unless enabled
|
|
175
|
+
|
|
176
|
+
socket = ENV['SSH_AUTH_SOCK'].to_s
|
|
177
|
+
if Docker.interface.run('info', '--format', '{{.OperatingSystem}}', output: :text, allow_failure: true).to_s.include?('Docker Desktop')
|
|
178
|
+
# Docker Desktop runs in a VM and provides the host's agent at a fixed path.
|
|
179
|
+
socket = DOCKER_DESKTOP_SSH_AGENT_SOCKET
|
|
180
|
+
elsif socket.empty? || !File.socket?(socket)
|
|
181
|
+
warning('ssh_agent is enabled, but there is no SSH agent at SSH_AUTH_SOCK. Start one with `eval $(ssh-agent)` and `ssh-add`.')
|
|
182
|
+
return [[], {}]
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
volumes = [Docker::Volume::Bind.new(socket, SSH_AGENT_SOCKET)]
|
|
186
|
+
known_hosts = File.join(Dir.home, '.ssh', 'known_hosts')
|
|
187
|
+
if File.file?(known_hosts)
|
|
188
|
+
volumes << Docker::Volume::Bind.new(known_hosts, '/etc/ssh/ssh_known_hosts', read_only: true)
|
|
189
|
+
end
|
|
190
|
+
[volumes, { 'SSH_AUTH_SOCK' => SSH_AGENT_SOCKET }]
|
|
79
191
|
end
|
|
80
192
|
|
|
81
193
|
def download_missing
|
|
82
|
-
@
|
|
194
|
+
@image.download_missing(user: @docker_name)
|
|
83
195
|
@services.each(&:download_missing)
|
|
84
196
|
end
|
|
85
197
|
|
|
198
|
+
# Several commands may start the same environment at once, like parallel test runs.
|
|
199
|
+
# A lock keeps them from creating the same network and services, or disconnecting
|
|
200
|
+
# a service another one just started.
|
|
86
201
|
def start_services
|
|
87
|
-
|
|
88
|
-
|
|
202
|
+
with_start_lock do
|
|
203
|
+
@network.create unless @network.exists?
|
|
204
|
+
@services.select { |service| service.start_if_stopped(network: @network) }
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def with_start_lock
|
|
209
|
+
File.open(File.join(Dir.tmpdir, "#{@network.prefixed_name}.lock"), File::RDWR | File::CREAT, 0o600) do |lock|
|
|
210
|
+
lock.flock(File::LOCK_EX)
|
|
211
|
+
yield
|
|
212
|
+
end
|
|
89
213
|
end
|
|
90
214
|
end
|
|
91
215
|
end
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
require 'open3'
|
|
2
|
+
require 'yaml'
|
|
3
|
+
|
|
4
|
+
module Rascal
|
|
5
|
+
module Gitlab
|
|
6
|
+
# Loads a .gitlab-ci.yml into a hash, like GitLab does before it looks at jobs:
|
|
7
|
+
#
|
|
8
|
+
# - Merges local files from `include:`. Other includes (from other projects, URLs,
|
|
9
|
+
# templates or components) are left out and listed in #skipped_includes.
|
|
10
|
+
# - Uses the config document after a header document with `spec:`.
|
|
11
|
+
#
|
|
12
|
+
# `!reference [key, ...]` tags become { REFERENCE_KEY => [key, ...] }. Like GitLab,
|
|
13
|
+
# Jobs resolves them after `extends:`.
|
|
14
|
+
class ConfigFile
|
|
15
|
+
# Marks where a `!reference` tag was, until it is resolved.
|
|
16
|
+
REFERENCE_KEY = '!reference'
|
|
17
|
+
|
|
18
|
+
# Include locations that are URLs.
|
|
19
|
+
REMOTE_LOCATION = %r{\A(https?|ssh|git)://}
|
|
20
|
+
|
|
21
|
+
attr_reader :skipped_includes
|
|
22
|
+
|
|
23
|
+
class << self
|
|
24
|
+
def reference?(value)
|
|
25
|
+
value.is_a?(Hash) && value.keys == [REFERENCE_KEY]
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Merges like ActiveSupport's deep_merge, which GitLab uses for includes and `extends:`.
|
|
29
|
+
# A reference is replaced as a whole, like GitLab's unresolved reference objects.
|
|
30
|
+
def deep_merge(hash1, hash2)
|
|
31
|
+
hash1.merge(hash2) do |_key, value1, value2|
|
|
32
|
+
if value1.is_a?(Hash) && value2.is_a?(Hash) && !reference?(value1) && !reference?(value2)
|
|
33
|
+
deep_merge(value1, value2)
|
|
34
|
+
else
|
|
35
|
+
value2
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def initialize(path)
|
|
42
|
+
@root = path.parent
|
|
43
|
+
@skipped_includes = []
|
|
44
|
+
@config = load_with_includes(path, [])
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def to_h
|
|
48
|
+
@config
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# The includes rascal skipped, like `project: group/templates, file: ruby.yml; https://example.com/ci.yml`.
|
|
52
|
+
def skipped_includes_description
|
|
53
|
+
@skipped_includes.collect do |entry|
|
|
54
|
+
entry.is_a?(Hash) ? entry.slice('project', 'file', 'remote', 'template', 'component', 'local').collect { |k, v| "#{k}: #{Array(v).join(', ')}" }.join(', ') : entry.to_s
|
|
55
|
+
end.join('; ')
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
|
|
60
|
+
def load_with_includes(path, including_paths)
|
|
61
|
+
config = parse(path.read, path)
|
|
62
|
+
includes = config.delete('include')
|
|
63
|
+
return config unless includes
|
|
64
|
+
|
|
65
|
+
included = include_paths(includes, path).inject({}) do |merged, included_path|
|
|
66
|
+
if including_paths.include?(included_path) || included_path == path
|
|
67
|
+
raise Error, "#{relative(included_path)} includes itself, directly or through other files."
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
self.class.deep_merge(merged, load_with_includes(included_path, [*including_paths, path]))
|
|
71
|
+
end
|
|
72
|
+
# Like GitLab, the including file wins over the files it includes.
|
|
73
|
+
self.class.deep_merge(included, config)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# The local files to include. Skips and remembers all other kinds of includes.
|
|
77
|
+
def include_paths(includes, including_path)
|
|
78
|
+
includes = [includes] unless includes.is_a?(Array)
|
|
79
|
+
includes.flat_map do |entry|
|
|
80
|
+
location = case entry
|
|
81
|
+
when String then entry.match?(REMOTE_LOCATION) ? nil : entry
|
|
82
|
+
when Hash then entry['local']
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
if location.nil?
|
|
86
|
+
@skipped_includes << entry
|
|
87
|
+
[]
|
|
88
|
+
elsif location.to_s.include?('*')
|
|
89
|
+
wildcard_paths(location.to_s, including_path)
|
|
90
|
+
else
|
|
91
|
+
local_path(location.to_s, entry)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# A local include is relative to the project root.
|
|
97
|
+
def local_path(location, entry)
|
|
98
|
+
path = @root.join(location.sub(%r{\A/+}, ''))
|
|
99
|
+
inside_root!(path, location)
|
|
100
|
+
if path.file?
|
|
101
|
+
[path]
|
|
102
|
+
elsif entry.is_a?(Hash) && entry.key?('rules')
|
|
103
|
+
# Rules like `exists:` may exclude a missing file in GitLab. rascal cannot evaluate rules.
|
|
104
|
+
@skipped_includes << entry
|
|
105
|
+
[]
|
|
106
|
+
else
|
|
107
|
+
raise Error, "The local include #{location} does not exist. Paths are relative to the project root."
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Like GitLab (see Repository#search_files_by_wildcard_path), `*` matches within a directory,
|
|
112
|
+
# `**` across directories, `**/` also matches no directory, and all of them match dotfiles.
|
|
113
|
+
# A pattern without matches includes nothing, and never the including file.
|
|
114
|
+
def wildcard_paths(location, including_path)
|
|
115
|
+
pattern = location.sub(%r{\A/+}, '')
|
|
116
|
+
regexp = Regexp.new('\A' + Regexp.escape(pattern).gsub('\*\*/', '(.*?/)?').gsub('\*\*', '.*').gsub('\*', '[^/]*') + '\z')
|
|
117
|
+
project_files
|
|
118
|
+
.select { |relative| relative.match?(regexp) }
|
|
119
|
+
.collect { |relative| @root.join(relative) }
|
|
120
|
+
.select { |path| path.file? && path != including_path }
|
|
121
|
+
.each { |path| inside_root!(path, location) }
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# The files GitLab would see: those in Git, plus new ones that are not ignored. Without Git,
|
|
125
|
+
# all files.
|
|
126
|
+
def project_files
|
|
127
|
+
@project_files ||= begin
|
|
128
|
+
output, status = Open3.capture2('git', '-C', @root.to_s, 'ls-files', '-z', '--cached', '--others', '--exclude-standard', err: File::NULL)
|
|
129
|
+
status.success? ? output.split("\0").sort : Dir.glob('**/*', File::FNM_DOTMATCH, base: @root.to_s).sort
|
|
130
|
+
rescue SystemCallError
|
|
131
|
+
Dir.glob('**/*', File::FNM_DOTMATCH, base: @root.to_s).sort
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def inside_root!(path, location)
|
|
136
|
+
root = @root.realpath.to_s
|
|
137
|
+
real = path.exist? ? path.realpath.to_s : path.cleanpath.to_s
|
|
138
|
+
unless real == root || real.start_with?("#{root}/")
|
|
139
|
+
raise Error, "The local include #{location} is outside the project."
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def relative(path)
|
|
144
|
+
path.relative_path_from(@root).to_s
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Parses YAML like YAML.safe_load with aliases, but keeps `!reference` tags
|
|
148
|
+
# as markers, and picks the config document like GitLab.
|
|
149
|
+
def parse(yaml, path)
|
|
150
|
+
document = config_document(Psych.parse_stream(yaml, filename: relative(path)).children)
|
|
151
|
+
return {} unless document
|
|
152
|
+
|
|
153
|
+
mark_references(document)
|
|
154
|
+
# Like GitLab, allows symbols, but no other Ruby objects.
|
|
155
|
+
class_loader = Psych::ClassLoader::Restricted.new(['Symbol'], [])
|
|
156
|
+
scanner = Psych::ScalarScanner.new(class_loader)
|
|
157
|
+
result = Psych::Visitors::ToRuby.new(scanner, class_loader).accept(document)
|
|
158
|
+
stringify_keys(result.is_a?(Hash) ? result : {})
|
|
159
|
+
rescue Psych::Exception => e
|
|
160
|
+
raise Error, "Could not read #{relative(path)}: #{e.message}"
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Like GitLab, ignores empty documents, and uses the second document only when
|
|
164
|
+
# the first one is a header with `spec:`.
|
|
165
|
+
def config_document(documents)
|
|
166
|
+
documents = documents.reject { |document| document.root.nil? || (document.root.is_a?(Psych::Nodes::Scalar) && document.root.value.empty?) }
|
|
167
|
+
first = documents.first
|
|
168
|
+
header = first&.root.is_a?(Psych::Nodes::Mapping) &&
|
|
169
|
+
first.root.children.each_slice(2).any? { |key, _| key.is_a?(Psych::Nodes::Scalar) && key.value == 'spec' }
|
|
170
|
+
header && documents.size > 1 ? documents[1] : first
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def mark_references(node)
|
|
174
|
+
node.children&.each_with_index do |child, index|
|
|
175
|
+
if child.respond_to?(:tag) && child.tag == '!reference'
|
|
176
|
+
child.tag = nil
|
|
177
|
+
# The anchor belongs to the reference, so aliases get the resolved value too.
|
|
178
|
+
marker = Psych::Nodes::Mapping.new(child.anchor)
|
|
179
|
+
child.anchor = nil
|
|
180
|
+
# A reference that is not a list gets resolved to a problem.
|
|
181
|
+
child = Psych::Nodes::Scalar.new('') unless child.is_a?(Psych::Nodes::Sequence)
|
|
182
|
+
marker.children << Psych::Nodes::Scalar.new(REFERENCE_KEY) << child
|
|
183
|
+
node.children[index] = marker
|
|
184
|
+
end
|
|
185
|
+
mark_references(child)
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
# YAML turns keys like `2.6:` into numbers. GitLab treats all keys as strings.
|
|
190
|
+
def stringify_keys(value)
|
|
191
|
+
case value
|
|
192
|
+
when Hash then value.to_h { |key, nested| [key.to_s, stringify_keys(nested)] }
|
|
193
|
+
when Array then value.collect { |nested| stringify_keys(nested) }
|
|
194
|
+
else value
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
end
|
|
198
|
+
end
|
|
199
|
+
end
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
module Rascal
|
|
2
|
+
module Gitlab
|
|
3
|
+
# Builds the Environment for a resolved Job: Docker names, volumes, the image and services,
|
|
4
|
+
# and the expanded variables. Collects what keeps the environment from starting as problems,
|
|
5
|
+
# and what may make commands fail as warnings.
|
|
6
|
+
class EnvironmentBuilder
|
|
7
|
+
INPUTS_ADVICE = "`spec: inputs:` (`$[[ inputs.name ]]`), which rascal does not support. " \
|
|
8
|
+
"Set the value in the `.rascal` block."
|
|
9
|
+
|
|
10
|
+
# `project_dir` is the directory of the .gitlab-ci.yml.
|
|
11
|
+
def initialize(job, project_dir:)
|
|
12
|
+
@job = job
|
|
13
|
+
@config = job.config
|
|
14
|
+
@project_dir = project_dir
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def build
|
|
18
|
+
@problems = @job.problems.dup
|
|
19
|
+
@warnings = @job.warnings.dup
|
|
20
|
+
repo_dir = @config.fetch('repo_dir', '/repo')
|
|
21
|
+
shared_volumes = [Docker::Volume::Bind.new(@project_dir, repo_dir), Docker::Volume::Named.new("#{docker_name}-builds", '/builds')]
|
|
22
|
+
variables = Variables.new(predefined_variables(repo_dir), *@job.variable_layers)
|
|
23
|
+
Environment.new(docker_name,
|
|
24
|
+
name: @job.name,
|
|
25
|
+
image: image_name(variables),
|
|
26
|
+
entrypoint: entrypoint,
|
|
27
|
+
volumes: [*shared_volumes, *named_volumes],
|
|
28
|
+
env_variables: resolve_variables(variables),
|
|
29
|
+
services: services(variables, shared_volumes),
|
|
30
|
+
before_shell: @config.fetch('before_shell', []),
|
|
31
|
+
after_shell: @config.fetch('after_shell', []),
|
|
32
|
+
working_dir: repo_dir,
|
|
33
|
+
ssh_agent: [true, 'true'].include?(@config.fetch('ssh_agent', false)),
|
|
34
|
+
services_timeout: services_timeout,
|
|
35
|
+
problems: @problems.uniq,
|
|
36
|
+
warnings: @warnings.uniq,
|
|
37
|
+
)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
# Like `rascal-myapp-rspec` without the prefix. Docker only allows some characters in names.
|
|
43
|
+
def docker_name
|
|
44
|
+
project = @project_dir.basename.to_s.gsub(/[^a-zA-Z0-9_.-]/, '_').sub(/\A[^a-zA-Z0-9]/, 'x\\0')
|
|
45
|
+
"#{project}-#{@job.name.gsub(/[^a-zA-Z0-9_.-]/, '_')}"
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# A subset of the variables GitLab predefines for every job.
|
|
49
|
+
# See https://docs.gitlab.com/ci/variables/predefined_variables/
|
|
50
|
+
def predefined_variables(repo_dir)
|
|
51
|
+
{
|
|
52
|
+
'CI' => 'true',
|
|
53
|
+
'GITLAB_CI' => 'true',
|
|
54
|
+
'CI_JOB_NAME' => @job.key,
|
|
55
|
+
'CI_PROJECT_NAME' => @project_dir.basename.to_s,
|
|
56
|
+
'CI_PROJECT_DIR' => repo_dir,
|
|
57
|
+
'CI_BUILDS_DIR' => '/builds',
|
|
58
|
+
}
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Returns the expanded variables. Records circular references as a problem, and
|
|
62
|
+
# references to undefined variables as warnings, only for the variables in `warn_for`
|
|
63
|
+
# if given.
|
|
64
|
+
def resolve_variables(variables, warn_for: nil)
|
|
65
|
+
resolved = variables.to_h
|
|
66
|
+
undefined = variables.undefined_references
|
|
67
|
+
undefined = undefined.slice(*warn_for) if warn_for
|
|
68
|
+
@warnings.concat(undefined_variable_warnings(undefined))
|
|
69
|
+
resolved.slice(*(warn_for || resolved.keys)).each do |name, value|
|
|
70
|
+
@warnings << "Variable #{name} uses #{INPUTS_ADVICE}" if value.include?('$[[')
|
|
71
|
+
end
|
|
72
|
+
resolved
|
|
73
|
+
rescue Error => e
|
|
74
|
+
@problems << e.message
|
|
75
|
+
{}
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# The expanded image name, or nil with the reason in the problems. Problems only matter
|
|
79
|
+
# when the environment is started, so `rascal clean` can still clean an environment whose
|
|
80
|
+
# image cannot be resolved.
|
|
81
|
+
def image_name(variables)
|
|
82
|
+
image = @config['image']
|
|
83
|
+
image = image['name'] if image.is_a?(Hash)
|
|
84
|
+
if image.nil? || image.to_s.empty?
|
|
85
|
+
# A job whose templates could not be resolved lacks its image as a consequence.
|
|
86
|
+
return if @job.problems.any?
|
|
87
|
+
|
|
88
|
+
@problems << "Job '#{@job.key}' has no image. #{@job.missing_config_hint}Set `image:` in the job, in `default:`, " \
|
|
89
|
+
"or in the `.rascal` block (`.rascal: jobs: #{@job.key}: image:`)."
|
|
90
|
+
return
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
if image.to_s.include?('$[[')
|
|
94
|
+
@problems << "The image of job '#{@job.key}' (#{image}) uses #{INPUTS_ADVICE}"
|
|
95
|
+
return
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
expand_name(variables, image) do |undefined|
|
|
99
|
+
"The image of job '#{@job.key}' (#{image}) uses the undefined variable #{undefined}. #{undefined_variable_advice}"
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def entrypoint
|
|
104
|
+
image = @config['image']
|
|
105
|
+
entrypoint = image['entrypoint'] if image.is_a?(Hash)
|
|
106
|
+
Array(entrypoint).collect(&:to_s) if entrypoint
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Expands a Docker image name, which must not reference undefined variables.
|
|
110
|
+
def expand_name(variables, name, &error_message)
|
|
111
|
+
undefined = []
|
|
112
|
+
expanded = variables.expand(name) { |variable| undefined << variable }
|
|
113
|
+
if undefined.any?
|
|
114
|
+
@problems << error_message.call(undefined.first)
|
|
115
|
+
nil
|
|
116
|
+
else
|
|
117
|
+
expanded
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def services_timeout
|
|
122
|
+
value = @config.fetch('services_timeout', Environment::DEFAULT_SERVICES_TIMEOUT)
|
|
123
|
+
Integer(value.to_s, 10).tap { |seconds| raise ArgumentError if seconds.negative? }
|
|
124
|
+
rescue ArgumentError
|
|
125
|
+
@problems << "services_timeout must be a number of seconds, not #{value.inspect}."
|
|
126
|
+
Environment::DEFAULT_SERVICES_TIMEOUT
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def named_volumes
|
|
130
|
+
hash_or_empty(@config['volumes']).collect do |volume_name, path|
|
|
131
|
+
Docker::Volume::Named.new("#{docker_name}-#{volume_name}", path)
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def services(variables, volumes)
|
|
136
|
+
Array(@config.fetch('services', [])).collect do |service_config|
|
|
137
|
+
# GitLab also allows services as a plain image name.
|
|
138
|
+
service_config = { 'name' => service_config.to_s } unless service_config.is_a?(Hash)
|
|
139
|
+
own_variables = Variables.null_as_empty(hash_or_empty(service_config['variables']))
|
|
140
|
+
service_variables = variables.merge(own_variables)
|
|
141
|
+
raw_image = service_config['name'].to_s
|
|
142
|
+
image = service_image(raw_image, service_variables)
|
|
143
|
+
aliases = service_aliases(service_config['alias'], image || raw_image)
|
|
144
|
+
env_variables = resolve_variables(service_variables, warn_for: own_variables.keys.collect(&:to_s))
|
|
145
|
+
Service.new("#{docker_name}_#{aliases.first}",
|
|
146
|
+
aliases: aliases,
|
|
147
|
+
image: image,
|
|
148
|
+
volumes: volumes,
|
|
149
|
+
env_variables: env_variables,
|
|
150
|
+
command: service_config['command'],
|
|
151
|
+
healthcheck_port: (env_variables['HEALTHCHECK_TCP_PORT'] if own_variables.key?('HEALTHCHECK_TCP_PORT')),
|
|
152
|
+
)
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def service_image(raw_image, variables)
|
|
157
|
+
if raw_image.empty?
|
|
158
|
+
@problems << "A service of job '#{@job.key}' has no image. Set `name:` for each service."
|
|
159
|
+
nil
|
|
160
|
+
elsif raw_image.include?('$[[')
|
|
161
|
+
@problems << "The service #{raw_image} of job '#{@job.key}' uses #{INPUTS_ADVICE}"
|
|
162
|
+
nil
|
|
163
|
+
else
|
|
164
|
+
expand_name(variables, raw_image) do |undefined|
|
|
165
|
+
"The service #{raw_image} of job '#{@job.key}' uses the undefined variable #{undefined}. #{undefined_variable_advice}"
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# GitLab allows several aliases, separated by commas or spaces. Without an alias, a
|
|
171
|
+
# service gets the host names GitLab derives from its image:
|
|
172
|
+
# `registry.example.com:5000/group/postgres:15` becomes `registry.example.com-group-postgres`
|
|
173
|
+
# and `registry.example.com__group__postgres`.
|
|
174
|
+
def service_aliases(alias_config, image)
|
|
175
|
+
aliases = alias_config.to_s.split(/[\s,]+/).reject(&:empty?)
|
|
176
|
+
return aliases if aliases.any?
|
|
177
|
+
|
|
178
|
+
name = image.sub(/@.*\z/, '') # digest
|
|
179
|
+
.sub(%r{:[^/:]*\z}, '') # tag
|
|
180
|
+
.sub(%r{\A([^/]+):\d+(?=/)}, '\\1') # registry port
|
|
181
|
+
.gsub(/[^a-zA-Z0-9_.\/-]/, '') # anything Docker does not allow in names, like an unresolved $
|
|
182
|
+
name = 'service' if name.empty?
|
|
183
|
+
[name.tr('/', '-'), name.gsub('/', '__')].uniq
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def undefined_variable_advice
|
|
187
|
+
"It is probably set by GitLab or in the project's CI/CD settings, which rascal cannot see. " \
|
|
188
|
+
"Define it for rascal under `variables:` in the `.rascal` block."
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def undefined_variable_warnings(undefined_references)
|
|
192
|
+
undefined_references.collect do |name, undefined|
|
|
193
|
+
"Variable #{name} references #{undefined.join(', ')}, which #{undefined.size == 1 ? 'is' : 'are'} undefined " \
|
|
194
|
+
"and expanded to an empty string. If the job needs #{undefined.size == 1 ? 'it' : 'them'}, " \
|
|
195
|
+
"define #{undefined.size == 1 ? 'it' : 'them'} under `variables:` in the `.rascal` block."
|
|
196
|
+
end
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def hash_or_empty(value)
|
|
200
|
+
value.is_a?(Hash) ? value : {}
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
end
|