rascal 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +4 -4
  3. data/.ruby-version +1 -1
  4. data/AGENTS.md +4 -0
  5. data/CHANGELOG.md +68 -0
  6. data/CONTRIBUTING.md +139 -0
  7. data/Gemfile.common +0 -3
  8. data/Gemfile.lock +1 -47
  9. data/Guardfile +0 -9
  10. data/README.md +679 -76
  11. data/Rakefile +1 -7
  12. data/lib/rascal/cli/base.rb +41 -20
  13. data/lib/rascal/cli/clean.rb +3 -8
  14. data/lib/rascal/cli/environments.rb +1 -3
  15. data/lib/rascal/cli/main.rb +106 -17
  16. data/lib/rascal/cli/run.rb +6 -7
  17. data/lib/rascal/cli/shell.rb +3 -3
  18. data/lib/rascal/cli/status.rb +81 -0
  19. data/lib/rascal/cli/update.rb +8 -8
  20. data/lib/rascal/cli.rb +1 -0
  21. data/lib/rascal/command_script.rb +83 -0
  22. data/lib/rascal/docker/command_containers.rb +101 -0
  23. data/lib/rascal/docker/container.rb +58 -107
  24. data/lib/rascal/docker/image.rb +71 -0
  25. data/lib/rascal/docker/interface.rb +4 -2
  26. data/lib/rascal/docker/network.rb +8 -0
  27. data/lib/rascal/docker/volume.rb +17 -10
  28. data/lib/rascal/docker.rb +11 -4
  29. data/lib/rascal/environment.rb +164 -40
  30. data/lib/rascal/gitlab/config_file.rb +199 -0
  31. data/lib/rascal/gitlab/environment_builder.rb +204 -0
  32. data/lib/rascal/gitlab/jobs.rb +299 -0
  33. data/lib/rascal/gitlab/variables.rb +155 -0
  34. data/lib/rascal/gitlab.rb +16 -0
  35. data/lib/rascal/io_helper.rb +11 -0
  36. data/lib/rascal/service.rb +51 -9
  37. data/lib/rascal/version.rb +1 -1
  38. data/lib/rascal.rb +2 -1
  39. data/rascal.gemspec +3 -2
  40. metadata +15 -8
  41. data/bin/cucumber +0 -29
  42. data/cucumber.yml +0 -1
  43. data/lib/rascal/environments_definition/gitlab.rb +0 -146
  44. data/lib/rascal/environments_definition.rb +0 -23
@@ -1,32 +1,51 @@
1
1
  require 'shellwords'
2
+ require 'tmpdir'
2
3
 
3
4
  module Rascal
4
5
  class Environment
5
- attr_reader :name, :network, :container, :env_variables, :services, :volumes, :working_dir, :before_shell
6
+ include IOHelper
6
7
 
7
- def initialize(full_name, name:, image:, env_variables: {}, services: [], volumes: [], before_shell: [], after_shell: [], working_dir: nil)
8
+ # Where the host's SSH agent socket appears in the container.
9
+ SSH_AGENT_SOCKET = '/run/rascal/ssh-agent.sock'
10
+
11
+ # Docker Desktop's path for the host's SSH agent. Sockets from macOS cannot be mounted directly.
12
+ DOCKER_DESKTOP_SSH_AGENT_SOCKET = '/run/host-services/ssh-auth.sock'
13
+
14
+ # How long to wait by default for a service to accept connections before running the command anyway.
15
+ DEFAULT_SERVICES_TIMEOUT = 30
16
+
17
+ attr_reader :name, :network, :image, :command_containers, :env_variables, :services, :volumes, :working_dir, :before_shell,
18
+ :problems, :warnings, :services_timeout
19
+
20
+ # `problems` prevent the environment from starting, but not from being cleaned.
21
+ # `warnings` are shown before starting it.
22
+ # With `ssh_agent`, the host's SSH agent and known hosts are available in the container,
23
+ # so it can fetch private dependencies over SSH. `services_timeout` is how many seconds to
24
+ # wait for services to accept connections.
25
+ def initialize(full_name, name:, image:, entrypoint: nil, env_variables: {}, services: [], volumes: [], before_shell: [], after_shell: [], working_dir: nil, problems: [], warnings: [], ssh_agent: false, services_timeout: DEFAULT_SERVICES_TIMEOUT)
8
26
  @name = name
9
27
  @network = Docker::Network.new(full_name)
10
- @container = Docker::Container.new(full_name, image)
28
+ @docker_name = full_name
29
+ @image = Docker::Image.new(image)
30
+ @command_containers = Docker::CommandContainers.new(full_name, @image, entrypoint: entrypoint)
11
31
  @env_variables = env_variables
12
32
  @services = services
13
33
  @volumes = volumes
14
34
  @working_dir = working_dir
15
35
  @before_shell = before_shell
16
36
  @after_shell = after_shell
37
+ @problems = problems
38
+ @warnings = warnings
39
+ @ssh_agent = ssh_agent
40
+ @services_timeout = services_timeout
17
41
  end
18
42
 
19
- def run_shell
20
- download_missing
21
- start_services
22
- command = [*@before_shell, 'bash', *@after_shell].join(';')
23
- @container.run_and_attach('bash', '-c', command,
24
- env: @env_variables,
25
- network: @network,
26
- volumes: @volumes,
27
- working_dir: @working_dir,
28
- allow_failure: true
29
- )
43
+ # `env`, `unset_env` and `ssh_agent` change the container for this command only, like for
44
+ # updating dependencies. Services keep their variables. `ssh_agent` overrides the config's
45
+ # setting unless it is nil.
46
+ def run_shell(env: {}, unset_env: [], ssh_agent: nil)
47
+ starting_services = start
48
+ run_in_container(script('bash', starting_services), tty: true, env: env, unset_env: unset_env, ssh_agent: ssh_agent).tap { report_stopped_services }
30
49
  end
31
50
 
32
51
  # Runs a single command in the environment, without a TTY, and returns its
@@ -37,55 +56,160 @@ module Rascal
37
56
  # The command is a list of arguments, which is escaped before being handed
38
57
  # to bash, so arguments may contain spaces and shell metacharacters. Use
39
58
  # e.g. `bash -c "foo && bar"` to run something the shell must interpret.
40
- def run_command(*command)
41
- download_missing
42
- start_services
43
- @container.run_and_attach('bash', '-c', command_script(Shellwords.join(command)),
44
- env: @env_variables,
45
- network: @network,
46
- volumes: @volumes,
47
- working_dir: @working_dir,
48
- allow_failure: true,
49
- tty: false
50
- )
59
+ # See #run_shell for the keyword arguments.
60
+ def run_command(*command, env: {}, unset_env: [], ssh_agent: nil)
61
+ starting_services = start
62
+ run_in_container(script(Shellwords.join(command), starting_services), tty: false, forward_stdin: stdin_redirected?,
63
+ env: env, unset_env: unset_env, ssh_agent: ssh_agent).tap { report_stopped_services }
51
64
  end
52
65
 
53
66
  def clean(clean_volumes: false)
67
+ @command_containers.clean
54
68
  @services.each(&:clean)
55
69
  @network.clean
56
70
  @volumes.each(&:clean) if clean_volumes
57
71
  end
58
72
 
73
+ # What rascal made of the CI config, and what of the environment exists in Docker,
74
+ # for `rascal status`. Starts and downloads nothing.
75
+ def status
76
+ {
77
+ 'name' => @name,
78
+ 'image' => @image.name,
79
+ 'image_present' => @image.present?,
80
+ 'entrypoint' => @command_containers.entrypoint,
81
+ # What runs before each command unless `entrypoint` replaces it.
82
+ 'image_entrypoint' => (@image.entrypoint if @image.present?),
83
+ 'problems' => @problems,
84
+ 'warnings' => @warnings,
85
+ 'network' => { 'name' => @network.prefixed_name, 'exists' => @network.exists? },
86
+ 'services' => @services.collect(&:status),
87
+ 'volumes' => @volumes.collect(&:status),
88
+ 'command_containers' => @command_containers.names,
89
+ 'working_dir' => @working_dir,
90
+ 'variables' => @env_variables,
91
+ 'before_shell' => @before_shell,
92
+ 'after_shell' => @after_shell,
93
+ 'ssh_agent' => @ssh_agent,
94
+ 'services_timeout' => @services_timeout,
95
+ }
96
+ end
97
+
59
98
  def update(skip: [])
99
+ raise_problems
60
100
  [
61
101
  *@services.collect { |s| s.update(skip: skip) },
62
- *@container.update(skip: skip),
102
+ *@image.update(skip: skip),
63
103
  ]
64
104
  end
65
105
 
66
106
  private
67
107
 
68
- # Runs before_shell/after_shell around the given command, but exits with the
69
- # command's status. Without saving it, an `after_shell` entry would determine
70
- # the exit status, and failures would go unnoticed.
71
- def command_script(command)
72
- [
73
- *@before_shell,
74
- command,
75
- '__rascal_status=$?',
76
- *@after_shell,
77
- 'exit $__rascal_status',
78
- ].join(';')
108
+ # Returns the services that may not be ready yet: those it started, and those another
109
+ # command started less than services_timeout ago.
110
+ def start
111
+ raise_problems
112
+ @warnings.each { |warning| warning(warning) }
113
+ download_missing
114
+ started = start_services
115
+ @services.select { |service| started.include?(service) || service.starting?(@services_timeout) }
116
+ end
117
+
118
+ # Only passes stdin on when it comes from a file or pipe, like `rascal run job -- psql < dump.sql`,
119
+ # or from a socket with input waiting, like from Node's child_process. A terminal, or a socket
120
+ # that stays open without input, would make commands that read stdin hang.
121
+ def stdin_redirected?
122
+ stat = stdin.stat
123
+ stat.pipe? || stat.file? || (stat.socket? && !IO.select([stdin], nil, nil, 0).nil?)
124
+ rescue NoMethodError, IOError, SystemCallError
125
+ false
126
+ end
127
+
128
+ # A service that stopped, typically because of bad settings, makes commands fail in
129
+ # confusing ways, so point to its output.
130
+ def report_stopped_services
131
+ @services.reject(&:running?).each do |service|
132
+ warning("Service #{service.aliases.first} is not running anymore. See why with `docker logs #{service.container.prefixed_name}`, " \
133
+ "then fix its settings and run `rascal clean #{@name}`.")
134
+ end
135
+ end
136
+
137
+ def raise_problems
138
+ raise Error, @problems.join("\n") if @problems.any?
139
+ end
140
+
141
+ def run_in_container(script, tty:, env:, unset_env:, ssh_agent:, forward_stdin: tty)
142
+ ssh_volumes, ssh_env = ssh_agent_access(ssh_agent.nil? ? @ssh_agent : ssh_agent)
143
+ @command_containers.run('bash', '-c', script,
144
+ env: @env_variables.except(*unset_env).merge(env, ssh_env),
145
+ network: @network,
146
+ volumes: [*@volumes, *ssh_volumes],
147
+ working_dir: @working_dir,
148
+ tty: tty,
149
+ forward_stdin: forward_stdin,
150
+ )
151
+ end
152
+
153
+ # `starting_services` are those that may not accept connections yet.
154
+ def script(command, starting_services)
155
+ CommandScript.new(command,
156
+ working_dir: @working_dir,
157
+ before_shell: @before_shell,
158
+ after_shell: @after_shell,
159
+ waits: starting_services.collect { |service| [service.aliases.first, service.ports] },
160
+ wait_timeout: @services_timeout,
161
+ owner: project_owner,
162
+ ).to_s
163
+ end
164
+
165
+ # The user that should own files the container creates in the project: the host user, unless
166
+ # the project already appears in the container as theirs. With rootless Docker, the host user
167
+ # is root in the container, and Docker Desktop for Mac maps ownership itself.
168
+ def project_owner
169
+ [Process.uid, Process.gid] unless Process.uid == 0 || RUBY_PLATFORM.include?('darwin')
170
+ end
171
+
172
+ # Volumes and variables that give the container access to the host's SSH agent.
173
+ def ssh_agent_access(enabled)
174
+ return [[], {}] unless enabled
175
+
176
+ socket = ENV['SSH_AUTH_SOCK'].to_s
177
+ if Docker.interface.run('info', '--format', '{{.OperatingSystem}}', output: :text, allow_failure: true).to_s.include?('Docker Desktop')
178
+ # Docker Desktop runs in a VM and provides the host's agent at a fixed path.
179
+ socket = DOCKER_DESKTOP_SSH_AGENT_SOCKET
180
+ elsif socket.empty? || !File.socket?(socket)
181
+ warning('ssh_agent is enabled, but there is no SSH agent at SSH_AUTH_SOCK. Start one with `eval $(ssh-agent)` and `ssh-add`.')
182
+ return [[], {}]
183
+ end
184
+
185
+ volumes = [Docker::Volume::Bind.new(socket, SSH_AGENT_SOCKET)]
186
+ known_hosts = File.join(Dir.home, '.ssh', 'known_hosts')
187
+ if File.file?(known_hosts)
188
+ volumes << Docker::Volume::Bind.new(known_hosts, '/etc/ssh/ssh_known_hosts', read_only: true)
189
+ end
190
+ [volumes, { 'SSH_AUTH_SOCK' => SSH_AGENT_SOCKET }]
79
191
  end
80
192
 
81
193
  def download_missing
82
- @container.download_missing
194
+ @image.download_missing(user: @docker_name)
83
195
  @services.each(&:download_missing)
84
196
  end
85
197
 
198
+ # Several commands may start the same environment at once, like parallel test runs.
199
+ # A lock keeps them from creating the same network and services, or disconnecting
200
+ # a service another one just started.
86
201
  def start_services
87
- @network.create unless @network.exists?
88
- @services.each { |s| s.start_if_stopped(network: @network) }
202
+ with_start_lock do
203
+ @network.create unless @network.exists?
204
+ @services.select { |service| service.start_if_stopped(network: @network) }
205
+ end
206
+ end
207
+
208
+ def with_start_lock
209
+ File.open(File.join(Dir.tmpdir, "#{@network.prefixed_name}.lock"), File::RDWR | File::CREAT, 0o600) do |lock|
210
+ lock.flock(File::LOCK_EX)
211
+ yield
212
+ end
89
213
  end
90
214
  end
91
215
  end
@@ -0,0 +1,199 @@
1
+ require 'open3'
2
+ require 'yaml'
3
+
4
+ module Rascal
5
+ module Gitlab
6
+ # Loads a .gitlab-ci.yml into a hash, like GitLab does before it looks at jobs:
7
+ #
8
+ # - Merges local files from `include:`. Other includes (from other projects, URLs,
9
+ # templates or components) are left out and listed in #skipped_includes.
10
+ # - Uses the config document after a header document with `spec:`.
11
+ #
12
+ # `!reference [key, ...]` tags become { REFERENCE_KEY => [key, ...] }. Like GitLab,
13
+ # Jobs resolves them after `extends:`.
14
+ class ConfigFile
15
+ # Marks where a `!reference` tag was, until it is resolved.
16
+ REFERENCE_KEY = '!reference'
17
+
18
+ # Include locations that are URLs.
19
+ REMOTE_LOCATION = %r{\A(https?|ssh|git)://}
20
+
21
+ attr_reader :skipped_includes
22
+
23
+ class << self
24
+ def reference?(value)
25
+ value.is_a?(Hash) && value.keys == [REFERENCE_KEY]
26
+ end
27
+
28
+ # Merges like ActiveSupport's deep_merge, which GitLab uses for includes and `extends:`.
29
+ # A reference is replaced as a whole, like GitLab's unresolved reference objects.
30
+ def deep_merge(hash1, hash2)
31
+ hash1.merge(hash2) do |_key, value1, value2|
32
+ if value1.is_a?(Hash) && value2.is_a?(Hash) && !reference?(value1) && !reference?(value2)
33
+ deep_merge(value1, value2)
34
+ else
35
+ value2
36
+ end
37
+ end
38
+ end
39
+ end
40
+
41
+ def initialize(path)
42
+ @root = path.parent
43
+ @skipped_includes = []
44
+ @config = load_with_includes(path, [])
45
+ end
46
+
47
+ def to_h
48
+ @config
49
+ end
50
+
51
+ # The includes rascal skipped, like `project: group/templates, file: ruby.yml; https://example.com/ci.yml`.
52
+ def skipped_includes_description
53
+ @skipped_includes.collect do |entry|
54
+ entry.is_a?(Hash) ? entry.slice('project', 'file', 'remote', 'template', 'component', 'local').collect { |k, v| "#{k}: #{Array(v).join(', ')}" }.join(', ') : entry.to_s
55
+ end.join('; ')
56
+ end
57
+
58
+ private
59
+
60
+ def load_with_includes(path, including_paths)
61
+ config = parse(path.read, path)
62
+ includes = config.delete('include')
63
+ return config unless includes
64
+
65
+ included = include_paths(includes, path).inject({}) do |merged, included_path|
66
+ if including_paths.include?(included_path) || included_path == path
67
+ raise Error, "#{relative(included_path)} includes itself, directly or through other files."
68
+ end
69
+
70
+ self.class.deep_merge(merged, load_with_includes(included_path, [*including_paths, path]))
71
+ end
72
+ # Like GitLab, the including file wins over the files it includes.
73
+ self.class.deep_merge(included, config)
74
+ end
75
+
76
+ # The local files to include. Skips and remembers all other kinds of includes.
77
+ def include_paths(includes, including_path)
78
+ includes = [includes] unless includes.is_a?(Array)
79
+ includes.flat_map do |entry|
80
+ location = case entry
81
+ when String then entry.match?(REMOTE_LOCATION) ? nil : entry
82
+ when Hash then entry['local']
83
+ end
84
+
85
+ if location.nil?
86
+ @skipped_includes << entry
87
+ []
88
+ elsif location.to_s.include?('*')
89
+ wildcard_paths(location.to_s, including_path)
90
+ else
91
+ local_path(location.to_s, entry)
92
+ end
93
+ end
94
+ end
95
+
96
+ # A local include is relative to the project root.
97
+ def local_path(location, entry)
98
+ path = @root.join(location.sub(%r{\A/+}, ''))
99
+ inside_root!(path, location)
100
+ if path.file?
101
+ [path]
102
+ elsif entry.is_a?(Hash) && entry.key?('rules')
103
+ # Rules like `exists:` may exclude a missing file in GitLab. rascal cannot evaluate rules.
104
+ @skipped_includes << entry
105
+ []
106
+ else
107
+ raise Error, "The local include #{location} does not exist. Paths are relative to the project root."
108
+ end
109
+ end
110
+
111
+ # Like GitLab (see Repository#search_files_by_wildcard_path), `*` matches within a directory,
112
+ # `**` across directories, `**/` also matches no directory, and all of them match dotfiles.
113
+ # A pattern without matches includes nothing, and never the including file.
114
+ def wildcard_paths(location, including_path)
115
+ pattern = location.sub(%r{\A/+}, '')
116
+ regexp = Regexp.new('\A' + Regexp.escape(pattern).gsub('\*\*/', '(.*?/)?').gsub('\*\*', '.*').gsub('\*', '[^/]*') + '\z')
117
+ project_files
118
+ .select { |relative| relative.match?(regexp) }
119
+ .collect { |relative| @root.join(relative) }
120
+ .select { |path| path.file? && path != including_path }
121
+ .each { |path| inside_root!(path, location) }
122
+ end
123
+
124
+ # The files GitLab would see: those in Git, plus new ones that are not ignored. Without Git,
125
+ # all files.
126
+ def project_files
127
+ @project_files ||= begin
128
+ output, status = Open3.capture2('git', '-C', @root.to_s, 'ls-files', '-z', '--cached', '--others', '--exclude-standard', err: File::NULL)
129
+ status.success? ? output.split("\0").sort : Dir.glob('**/*', File::FNM_DOTMATCH, base: @root.to_s).sort
130
+ rescue SystemCallError
131
+ Dir.glob('**/*', File::FNM_DOTMATCH, base: @root.to_s).sort
132
+ end
133
+ end
134
+
135
+ def inside_root!(path, location)
136
+ root = @root.realpath.to_s
137
+ real = path.exist? ? path.realpath.to_s : path.cleanpath.to_s
138
+ unless real == root || real.start_with?("#{root}/")
139
+ raise Error, "The local include #{location} is outside the project."
140
+ end
141
+ end
142
+
143
+ def relative(path)
144
+ path.relative_path_from(@root).to_s
145
+ end
146
+
147
+ # Parses YAML like YAML.safe_load with aliases, but keeps `!reference` tags
148
+ # as markers, and picks the config document like GitLab.
149
+ def parse(yaml, path)
150
+ document = config_document(Psych.parse_stream(yaml, filename: relative(path)).children)
151
+ return {} unless document
152
+
153
+ mark_references(document)
154
+ # Like GitLab, allows symbols, but no other Ruby objects.
155
+ class_loader = Psych::ClassLoader::Restricted.new(['Symbol'], [])
156
+ scanner = Psych::ScalarScanner.new(class_loader)
157
+ result = Psych::Visitors::ToRuby.new(scanner, class_loader).accept(document)
158
+ stringify_keys(result.is_a?(Hash) ? result : {})
159
+ rescue Psych::Exception => e
160
+ raise Error, "Could not read #{relative(path)}: #{e.message}"
161
+ end
162
+
163
+ # Like GitLab, ignores empty documents, and uses the second document only when
164
+ # the first one is a header with `spec:`.
165
+ def config_document(documents)
166
+ documents = documents.reject { |document| document.root.nil? || (document.root.is_a?(Psych::Nodes::Scalar) && document.root.value.empty?) }
167
+ first = documents.first
168
+ header = first&.root.is_a?(Psych::Nodes::Mapping) &&
169
+ first.root.children.each_slice(2).any? { |key, _| key.is_a?(Psych::Nodes::Scalar) && key.value == 'spec' }
170
+ header && documents.size > 1 ? documents[1] : first
171
+ end
172
+
173
+ def mark_references(node)
174
+ node.children&.each_with_index do |child, index|
175
+ if child.respond_to?(:tag) && child.tag == '!reference'
176
+ child.tag = nil
177
+ # The anchor belongs to the reference, so aliases get the resolved value too.
178
+ marker = Psych::Nodes::Mapping.new(child.anchor)
179
+ child.anchor = nil
180
+ # A reference that is not a list gets resolved to a problem.
181
+ child = Psych::Nodes::Scalar.new('') unless child.is_a?(Psych::Nodes::Sequence)
182
+ marker.children << Psych::Nodes::Scalar.new(REFERENCE_KEY) << child
183
+ node.children[index] = marker
184
+ end
185
+ mark_references(child)
186
+ end
187
+ end
188
+
189
+ # YAML turns keys like `2.6:` into numbers. GitLab treats all keys as strings.
190
+ def stringify_keys(value)
191
+ case value
192
+ when Hash then value.to_h { |key, nested| [key.to_s, stringify_keys(nested)] }
193
+ when Array then value.collect { |nested| stringify_keys(nested) }
194
+ else value
195
+ end
196
+ end
197
+ end
198
+ end
199
+ end
@@ -0,0 +1,204 @@
1
+ module Rascal
2
+ module Gitlab
3
+ # Builds the Environment for a resolved Job: Docker names, volumes, the image and services,
4
+ # and the expanded variables. Collects what keeps the environment from starting as problems,
5
+ # and what may make commands fail as warnings.
6
+ class EnvironmentBuilder
7
+ INPUTS_ADVICE = "`spec: inputs:` (`$[[ inputs.name ]]`), which rascal does not support. " \
8
+ "Set the value in the `.rascal` block."
9
+
10
+ # `project_dir` is the directory of the .gitlab-ci.yml.
11
+ def initialize(job, project_dir:)
12
+ @job = job
13
+ @config = job.config
14
+ @project_dir = project_dir
15
+ end
16
+
17
+ def build
18
+ @problems = @job.problems.dup
19
+ @warnings = @job.warnings.dup
20
+ repo_dir = @config.fetch('repo_dir', '/repo')
21
+ shared_volumes = [Docker::Volume::Bind.new(@project_dir, repo_dir), Docker::Volume::Named.new("#{docker_name}-builds", '/builds')]
22
+ variables = Variables.new(predefined_variables(repo_dir), *@job.variable_layers)
23
+ Environment.new(docker_name,
24
+ name: @job.name,
25
+ image: image_name(variables),
26
+ entrypoint: entrypoint,
27
+ volumes: [*shared_volumes, *named_volumes],
28
+ env_variables: resolve_variables(variables),
29
+ services: services(variables, shared_volumes),
30
+ before_shell: @config.fetch('before_shell', []),
31
+ after_shell: @config.fetch('after_shell', []),
32
+ working_dir: repo_dir,
33
+ ssh_agent: [true, 'true'].include?(@config.fetch('ssh_agent', false)),
34
+ services_timeout: services_timeout,
35
+ problems: @problems.uniq,
36
+ warnings: @warnings.uniq,
37
+ )
38
+ end
39
+
40
+ private
41
+
42
+ # Like `rascal-myapp-rspec` without the prefix. Docker only allows some characters in names.
43
+ def docker_name
44
+ project = @project_dir.basename.to_s.gsub(/[^a-zA-Z0-9_.-]/, '_').sub(/\A[^a-zA-Z0-9]/, 'x\\0')
45
+ "#{project}-#{@job.name.gsub(/[^a-zA-Z0-9_.-]/, '_')}"
46
+ end
47
+
48
+ # A subset of the variables GitLab predefines for every job.
49
+ # See https://docs.gitlab.com/ci/variables/predefined_variables/
50
+ def predefined_variables(repo_dir)
51
+ {
52
+ 'CI' => 'true',
53
+ 'GITLAB_CI' => 'true',
54
+ 'CI_JOB_NAME' => @job.key,
55
+ 'CI_PROJECT_NAME' => @project_dir.basename.to_s,
56
+ 'CI_PROJECT_DIR' => repo_dir,
57
+ 'CI_BUILDS_DIR' => '/builds',
58
+ }
59
+ end
60
+
61
+ # Returns the expanded variables. Records circular references as a problem, and
62
+ # references to undefined variables as warnings, only for the variables in `warn_for`
63
+ # if given.
64
+ def resolve_variables(variables, warn_for: nil)
65
+ resolved = variables.to_h
66
+ undefined = variables.undefined_references
67
+ undefined = undefined.slice(*warn_for) if warn_for
68
+ @warnings.concat(undefined_variable_warnings(undefined))
69
+ resolved.slice(*(warn_for || resolved.keys)).each do |name, value|
70
+ @warnings << "Variable #{name} uses #{INPUTS_ADVICE}" if value.include?('$[[')
71
+ end
72
+ resolved
73
+ rescue Error => e
74
+ @problems << e.message
75
+ {}
76
+ end
77
+
78
+ # The expanded image name, or nil with the reason in the problems. Problems only matter
79
+ # when the environment is started, so `rascal clean` can still clean an environment whose
80
+ # image cannot be resolved.
81
+ def image_name(variables)
82
+ image = @config['image']
83
+ image = image['name'] if image.is_a?(Hash)
84
+ if image.nil? || image.to_s.empty?
85
+ # A job whose templates could not be resolved lacks its image as a consequence.
86
+ return if @job.problems.any?
87
+
88
+ @problems << "Job '#{@job.key}' has no image. #{@job.missing_config_hint}Set `image:` in the job, in `default:`, " \
89
+ "or in the `.rascal` block (`.rascal: jobs: #{@job.key}: image:`)."
90
+ return
91
+ end
92
+
93
+ if image.to_s.include?('$[[')
94
+ @problems << "The image of job '#{@job.key}' (#{image}) uses #{INPUTS_ADVICE}"
95
+ return
96
+ end
97
+
98
+ expand_name(variables, image) do |undefined|
99
+ "The image of job '#{@job.key}' (#{image}) uses the undefined variable #{undefined}. #{undefined_variable_advice}"
100
+ end
101
+ end
102
+
103
+ def entrypoint
104
+ image = @config['image']
105
+ entrypoint = image['entrypoint'] if image.is_a?(Hash)
106
+ Array(entrypoint).collect(&:to_s) if entrypoint
107
+ end
108
+
109
+ # Expands a Docker image name, which must not reference undefined variables.
110
+ def expand_name(variables, name, &error_message)
111
+ undefined = []
112
+ expanded = variables.expand(name) { |variable| undefined << variable }
113
+ if undefined.any?
114
+ @problems << error_message.call(undefined.first)
115
+ nil
116
+ else
117
+ expanded
118
+ end
119
+ end
120
+
121
+ def services_timeout
122
+ value = @config.fetch('services_timeout', Environment::DEFAULT_SERVICES_TIMEOUT)
123
+ Integer(value.to_s, 10).tap { |seconds| raise ArgumentError if seconds.negative? }
124
+ rescue ArgumentError
125
+ @problems << "services_timeout must be a number of seconds, not #{value.inspect}."
126
+ Environment::DEFAULT_SERVICES_TIMEOUT
127
+ end
128
+
129
+ def named_volumes
130
+ hash_or_empty(@config['volumes']).collect do |volume_name, path|
131
+ Docker::Volume::Named.new("#{docker_name}-#{volume_name}", path)
132
+ end
133
+ end
134
+
135
+ def services(variables, volumes)
136
+ Array(@config.fetch('services', [])).collect do |service_config|
137
+ # GitLab also allows services as a plain image name.
138
+ service_config = { 'name' => service_config.to_s } unless service_config.is_a?(Hash)
139
+ own_variables = Variables.null_as_empty(hash_or_empty(service_config['variables']))
140
+ service_variables = variables.merge(own_variables)
141
+ raw_image = service_config['name'].to_s
142
+ image = service_image(raw_image, service_variables)
143
+ aliases = service_aliases(service_config['alias'], image || raw_image)
144
+ env_variables = resolve_variables(service_variables, warn_for: own_variables.keys.collect(&:to_s))
145
+ Service.new("#{docker_name}_#{aliases.first}",
146
+ aliases: aliases,
147
+ image: image,
148
+ volumes: volumes,
149
+ env_variables: env_variables,
150
+ command: service_config['command'],
151
+ healthcheck_port: (env_variables['HEALTHCHECK_TCP_PORT'] if own_variables.key?('HEALTHCHECK_TCP_PORT')),
152
+ )
153
+ end
154
+ end
155
+
156
+ def service_image(raw_image, variables)
157
+ if raw_image.empty?
158
+ @problems << "A service of job '#{@job.key}' has no image. Set `name:` for each service."
159
+ nil
160
+ elsif raw_image.include?('$[[')
161
+ @problems << "The service #{raw_image} of job '#{@job.key}' uses #{INPUTS_ADVICE}"
162
+ nil
163
+ else
164
+ expand_name(variables, raw_image) do |undefined|
165
+ "The service #{raw_image} of job '#{@job.key}' uses the undefined variable #{undefined}. #{undefined_variable_advice}"
166
+ end
167
+ end
168
+ end
169
+
170
+ # GitLab allows several aliases, separated by commas or spaces. Without an alias, a
171
+ # service gets the host names GitLab derives from its image:
172
+ # `registry.example.com:5000/group/postgres:15` becomes `registry.example.com-group-postgres`
173
+ # and `registry.example.com__group__postgres`.
174
+ def service_aliases(alias_config, image)
175
+ aliases = alias_config.to_s.split(/[\s,]+/).reject(&:empty?)
176
+ return aliases if aliases.any?
177
+
178
+ name = image.sub(/@.*\z/, '') # digest
179
+ .sub(%r{:[^/:]*\z}, '') # tag
180
+ .sub(%r{\A([^/]+):\d+(?=/)}, '\\1') # registry port
181
+ .gsub(/[^a-zA-Z0-9_.\/-]/, '') # anything Docker does not allow in names, like an unresolved $
182
+ name = 'service' if name.empty?
183
+ [name.tr('/', '-'), name.gsub('/', '__')].uniq
184
+ end
185
+
186
+ def undefined_variable_advice
187
+ "It is probably set by GitLab or in the project's CI/CD settings, which rascal cannot see. " \
188
+ "Define it for rascal under `variables:` in the `.rascal` block."
189
+ end
190
+
191
+ def undefined_variable_warnings(undefined_references)
192
+ undefined_references.collect do |name, undefined|
193
+ "Variable #{name} references #{undefined.join(', ')}, which #{undefined.size == 1 ? 'is' : 'are'} undefined " \
194
+ "and expanded to an empty string. If the job needs #{undefined.size == 1 ? 'it' : 'them'}, " \
195
+ "define #{undefined.size == 1 ? 'it' : 'them'} under `variables:` in the `.rascal` block."
196
+ end
197
+ end
198
+
199
+ def hash_or_empty(value)
200
+ value.is_a?(Hash) ? value : {}
201
+ end
202
+ end
203
+ end
204
+ end