rascal 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +4 -4
  3. data/.ruby-version +1 -1
  4. data/AGENTS.md +4 -0
  5. data/CHANGELOG.md +74 -0
  6. data/CONTRIBUTING.md +139 -0
  7. data/Gemfile.common +0 -3
  8. data/Gemfile.lock +1 -47
  9. data/Guardfile +0 -9
  10. data/README.md +679 -76
  11. data/Rakefile +1 -7
  12. data/lib/rascal/cli/base.rb +41 -20
  13. data/lib/rascal/cli/clean.rb +3 -8
  14. data/lib/rascal/cli/environments.rb +1 -3
  15. data/lib/rascal/cli/main.rb +106 -17
  16. data/lib/rascal/cli/run.rb +6 -7
  17. data/lib/rascal/cli/shell.rb +3 -3
  18. data/lib/rascal/cli/status.rb +81 -0
  19. data/lib/rascal/cli/update.rb +8 -8
  20. data/lib/rascal/cli.rb +1 -0
  21. data/lib/rascal/command_script.rb +83 -0
  22. data/lib/rascal/docker/command_containers.rb +101 -0
  23. data/lib/rascal/docker/container.rb +82 -101
  24. data/lib/rascal/docker/image.rb +71 -0
  25. data/lib/rascal/docker/interface.rb +4 -2
  26. data/lib/rascal/docker/network.rb +12 -0
  27. data/lib/rascal/docker/volume.rb +17 -10
  28. data/lib/rascal/docker.rb +11 -4
  29. data/lib/rascal/environment.rb +164 -40
  30. data/lib/rascal/gitlab/config_file.rb +199 -0
  31. data/lib/rascal/gitlab/environment_builder.rb +204 -0
  32. data/lib/rascal/gitlab/jobs.rb +299 -0
  33. data/lib/rascal/gitlab/variables.rb +155 -0
  34. data/lib/rascal/gitlab.rb +16 -0
  35. data/lib/rascal/io_helper.rb +11 -0
  36. data/lib/rascal/service.rb +52 -9
  37. data/lib/rascal/version.rb +1 -1
  38. data/lib/rascal.rb +2 -1
  39. data/rascal.gemspec +3 -2
  40. metadata +15 -8
  41. data/bin/cucumber +0 -29
  42. data/cucumber.yml +0 -1
  43. data/lib/rascal/environments_definition/gitlab.rb +0 -146
  44. data/lib/rascal/environments_definition.rb +0 -23
data/Rakefile CHANGED
@@ -1,12 +1,6 @@
1
1
  require 'bundler/gem_tasks'
2
2
  require 'rspec/core/rake_task'
3
- require 'cucumber'
4
- require 'cucumber/rake/task'
5
3
 
6
4
  RSpec::Core::RakeTask.new(:spec)
7
- Cucumber::Rake::Task.new(:features)
8
5
 
9
- desc 'Run specs and features'
10
- task :test => [:spec, :features]
11
-
12
- task default: :test
6
+ task default: :spec
@@ -3,13 +3,31 @@ require 'pathname'
3
3
  module Rascal
4
4
  module CLI
5
5
  class Base
6
- def initialize(thor, options)
7
- @thor = thor
6
+ include IOHelper
7
+
8
+ def initialize(options)
8
9
  @options = options
9
10
  end
10
11
 
11
12
  private
12
13
 
14
+ # Changes to the environment for a single command, from --env, --unset-env and --ssh-agent.
15
+ def command_overrides
16
+ env = Array(@options[:env]).to_h do |assignment|
17
+ name, value = assignment.split('=', 2)
18
+ fail_with_error('Use --env NAME=VALUE, like --env RAILS_ENV=test.') unless value && variable_name?(name)
19
+ [name, value]
20
+ end
21
+ unset_env = Array(@options[:unset_env]).each do |name|
22
+ fail_with_error('Use --unset-env NAME, like --unset-env CI.') unless variable_name?(name)
23
+ end
24
+ { env: env, unset_env: unset_env, ssh_agent: @options[:ssh_agent] }
25
+ end
26
+
27
+ def variable_name?(name)
28
+ name.match?(/\A[a-zA-Z_][a-zA-Z0-9_]*\z/)
29
+ end
30
+
13
31
  def config_location
14
32
  Pathname.new(@options[:config_file]).expand_path
15
33
  end
@@ -19,39 +37,42 @@ module Rascal
19
37
  end
20
38
 
21
39
  def find_environment(environment_name)
22
- definition = environment_definition
23
- if (environment = definition.environment(environment_name))
24
- return environment
25
- else
26
- available_environments = definition.available_environment_names.join(', ')
40
+ jobs = project_jobs
41
+ jobs.environment(environment_name) or begin
42
+ available_environments = jobs.names.join(', ')
27
43
  if environment_name
28
- fail_with_error("Unknown environment #{environment_name}. Available: #{available_environments}.")
44
+ fail_with_error("Unknown job #{environment_name}. Jobs: #{available_environments}.")
29
45
  else
30
- fail_with_error("Missing environment. Available: #{available_environments}.")
46
+ fail_with_error("Missing job. Jobs: #{available_environments}.")
31
47
  end
32
48
  end
33
49
  end
34
50
 
51
+ # The environment name, or :all with --all.
52
+ def name_or_all(environment_name)
53
+ return environment_name unless @options[:all]
54
+
55
+ fail_with_error('Give either a job or --all.') if environment_name
56
+ :all
57
+ end
58
+
35
59
  def each_environment(name, &block)
36
- return enum_for(:each_environment) unless block_given?
60
+ return enum_for(:each_environment, name) unless block_given?
37
61
 
38
62
  if name == :all
39
- definition = environment_definition
40
- definition.available_environment_names.each do |environment_name|
41
- yield definition.environment(environment_name)
63
+ jobs = project_jobs
64
+ jobs.names.each do |environment_name|
65
+ yield jobs.environment(environment_name)
42
66
  end
43
67
  else
44
68
  yield find_environment(name)
45
69
  end
46
70
  end
47
71
 
48
- def environment_definition
49
- if (definition = EnvironmentsDefinition.detect(config_location))
50
- definition
51
- else
52
- fail_with_error("Could not find an environment definition in current working directory.")
53
- nil
54
- end
72
+ def project_jobs
73
+ Gitlab::Jobs.find(config_location) or
74
+ fail_with_error("Could not find a .gitlab-ci.yml in #{config_location}. Run rascal in your project's root directory, " \
75
+ "or point to the file with --config-file.")
55
76
  end
56
77
  end
57
78
  end
@@ -3,14 +3,9 @@ require 'rascal'
3
3
  module Rascal
4
4
  module CLI
5
5
  class Clean < Base
6
- def initialize(thor, options, environment_name)
7
- @environment_name = if options[:all]
8
- fail_with_error('Cannot give --all and an environment name!') if environment_name
9
- :all
10
- else
11
- environment_name
12
- end
13
- super(thor, options)
6
+ def initialize(options, environment_name)
7
+ super(options)
8
+ @environment_name = name_or_all(environment_name)
14
9
  end
15
10
 
16
11
  def run
@@ -3,10 +3,8 @@ require 'rascal'
3
3
  module Rascal
4
4
  module CLI
5
5
  class Environments < Base
6
- include IOHelper
7
-
8
6
  def run
9
- environment_definition.available_environment_names.each do |name|
7
+ project_jobs.names.each do |name|
10
8
  say name
11
9
  end
12
10
  end
@@ -1,6 +1,17 @@
1
+ require 'rascal'
2
+
1
3
  module Rascal
2
4
  module CLI
3
5
  class Main < Thor
6
+ # Options that change a single `run` or `shell`, like for updating dependencies.
7
+ def self.command_environment_options
8
+ method_option :env, type: :string, repeatable: true, banner: 'NAME=VALUE',
9
+ desc: 'Set a variable in the container, for this command only. Repeat for more.'
10
+ method_option :unset_env, type: :string, repeatable: true, banner: 'NAME',
11
+ desc: 'Remove a variable from the container, like CI, for this command only. Repeat for more.'
12
+ method_option :ssh_agent, type: :boolean,
13
+ desc: 'Give the container access to your SSH agent, for this command only. --no-ssh-agent keeps it away, even if the config enables it.'
14
+ end
4
15
  def help(subcommand = false)
5
16
  if subcommand
6
17
  super
@@ -9,10 +20,12 @@ module Rascal
9
20
  say 'Usage:'
10
21
  say ' rascal <command> <args>'
11
22
  say 'For example:'
12
- say ' rascal shell 2.6'
23
+ say ' rascal run rspec -- bundle exec rspec spec/models/user_spec.rb'
13
24
  say
14
25
  super
15
- say 'For Further information about the commands, you can use "rascal help <command>".'
26
+ say 'For more about a command, run "rascal help <command>". rascal --version prints the version.'
27
+ say 'The README explains setup and troubleshooting:'
28
+ say ' https://github.com/makandra/rascal#readme'
16
29
  say
17
30
  end
18
31
  end
@@ -29,52 +42,128 @@ module Rascal
29
42
 
30
43
 
31
44
  map "shell" => "_shell"
32
- desc 'shell ENVIRONMENT', 'Start a docker shell for the given environment'
45
+ desc 'shell JOB', "Open an interactive bash in a job's environment"
46
+ long_desc <<~TEXT
47
+ Opens bash in the environment of a CI job, like for a Rails console or a debugger.
48
+ Needs a terminal. Without one, use rascal run.
49
+ TEXT
50
+ command_environment_options
33
51
  def _shell(environment_name = nil)
34
52
  handle_error do
35
- Shell.new(self, options, environment_name).run
53
+ Shell.new(options, environment_name).run
54
+ end
55
+ end
56
+
57
+ desc 'status [JOB]', "Show a job's environment as rascal sees it, and what of it is running"
58
+ long_desc <<~TEXT
59
+ Shows what rascal made of the CI config: the image, services, variables, before_shell and
60
+ after_shell, and any problems. Also shows what exists in Docker: whether images are
61
+ downloaded, the state of each service with the command to see its logs, the network,
62
+ volumes and running commands. Starts nothing. Without an environment, shows all of them.
63
+ TEXT
64
+ method_option :json, type: :boolean, desc: 'Print JSON, for scripts and agents'
65
+ def status(environment_name = nil)
66
+ handle_error do
67
+ Status.new(options, environment_name).run
36
68
  end
37
69
  end
38
70
 
39
- desc 'environments', 'List the names of all available environments'
71
+ desc 'environments', 'List the jobs you can use as environments'
40
72
  def environments
41
73
  handle_error do
42
- Environments.new(self, options).run
74
+ Environments.new(options).run
43
75
  end
44
76
  end
45
77
 
46
78
  # Thor::Actions already defines #run, hence the mapping.
47
79
  map 'run' => '_run'
48
80
  stop_on_unknown_option! :_run
49
- desc 'run ENVIRONMENT COMMAND', 'Run a command in the given environment and exit with its status'
81
+ desc 'run JOB -- COMMAND', "Run a command in a job's environment"
82
+ long_desc <<~TEXT
83
+ Runs a command in the environment of a CI job, without a terminal, and exits with its status.
84
+ Everything after -- is passed on as arguments. For shell syntax, use bash -c:
85
+
86
+ rascal run rspec -- bash -c 'bin/rails db:migrate && bundle exec rspec'
87
+
88
+ Options go before --. Variables and the SSH agent can change for a single command:
89
+
90
+ rascal run rspec --unset-env CI --ssh-agent -- bundle update rack --conservative
91
+ TEXT
92
+ command_environment_options
50
93
  def _run(environment_name = nil, *command)
51
94
  handle_error do
52
- command = command.drop(1) if command.first == '--'
53
- Run.new(self, options, environment_name, command).run
95
+ run_options, command = split_run_arguments(command)
96
+ Run.new(run_options, environment_name, command).run
54
97
  end
55
98
  end
56
99
 
57
- desc 'clean ENVIRONMENT', 'Stop and remove docker containers for the given environment'
58
- method_option :volumes, type: :boolean, default: false, desc: 'Remove (cache) volumes'
59
- method_option :all, type: :boolean, default: false, desc: 'Clean all environments'
100
+ desc 'clean JOB | --all', "Remove a job's environment"
101
+ long_desc <<~TEXT
102
+ Stops and removes the environment's services with their data, like databases, its network
103
+ and leftover command containers. Named volumes, like caches, stay unless you pass --volumes.
104
+ Files in the project stay. The next command starts the services fresh.
105
+ TEXT
106
+ method_option :volumes, type: :boolean, desc: 'Also remove volumes, like caches'
107
+ method_option :all, type: :boolean, desc: 'Clean the environments of all jobs'
60
108
  def clean(environment_name = nil)
61
109
  handle_error do
62
- Clean.new(self, options, environment_name).run
110
+ Clean.new(options, environment_name).run
63
111
  end
64
112
  end
65
113
 
66
- desc 'update ENVIRONMENT', 'Update all docker images'
67
- method_option :all, type: :boolean, default: false, desc: 'update all available environments'
114
+ desc 'update JOB | --all', "Pull the latest images of a job's environment"
115
+ long_desc <<~TEXT
116
+ Pulls the latest images of the job and its services. Services that already run keep their
117
+ old image until you run rascal clean.
118
+ TEXT
119
+ method_option :all, type: :boolean, desc: 'Update the environments of all jobs'
68
120
  def update(environment_name = nil)
69
121
  handle_error do
70
- Update.new(self, options, environment_name).run
122
+ Update.new(options, environment_name).run
123
+ end
124
+ end
125
+
126
+ # Thor's own command in newer versions, of no use here.
127
+ if Thor.method_defined?(:tree)
128
+ desc 'tree', 'Print a tree of all available commands', hide: true
129
+ def tree
130
+ super
71
131
  end
72
132
  end
73
133
 
74
- class_option :config_file, aliases: ['-c'], default: '.', required: true, desc: 'path to configuration file or directory containing it'
134
+ class_option :config_file, aliases: ['-c'], default: '.', required: true, desc: 'The .gitlab-ci.yml to use, or its directory'
75
135
 
76
136
  private
77
137
 
138
+ # Thor stops parsing options at the environment name of `run`, so the command can have
139
+ # options of its own. Options between the environment name and `--`, like
140
+ # `rascal run rspec -c path -- cmd`, are parsed here. A command without a leading `--`,
141
+ # like `rascal run rspec git checkout -- file`, is taken as it is.
142
+ def split_run_arguments(arguments)
143
+ return [options, arguments.drop(1)] if arguments.first == '--'
144
+ return [options, arguments] unless arguments.first&.start_with?('-')
145
+
146
+ separator = arguments.index('--') or
147
+ raise Thor::Error, "Put -- between rascal's options and the command, like: rascal run rspec #{arguments.first} ... -- COMMAND"
148
+ known_options = self.class.class_options.merge(self.class.commands['_run'].options)
149
+ parser = Thor::Options.new(known_options)
150
+ defaults = Thor::Options.new(known_options).parse([])
151
+ given = parser.parse(arguments.take(separator))
152
+ parser.check_unknown!
153
+ if parser.remaining.any?
154
+ raise Thor::Error, "Unexpected #{parser.remaining.join(' ')} before --. Usage: rascal run [OPTIONS] JOB -- COMMAND"
155
+ end
156
+
157
+ merged = options.to_h
158
+ given.each do |name, value|
159
+ next if defaults[name] == value
160
+
161
+ # Repeatable options, like --env, may come before and after the environment name.
162
+ merged[name] = value.is_a?(Array) ? Array(merged[name]) + value : value
163
+ end
164
+ [Thor::CoreExt::HashWithIndifferentAccess.new(merged), arguments.drop(separator + 1)]
165
+ end
166
+
78
167
  def handle_error
79
168
  yield
80
169
  rescue Rascal::Error => e
@@ -3,18 +3,17 @@ require 'rascal'
3
3
  module Rascal
4
4
  module CLI
5
5
  class Run < Base
6
- def initialize(thor, options, environment_name, command)
6
+ def initialize(options, environment_name, command)
7
7
  @environment_name = environment_name
8
8
  @command = command
9
- super(thor, options)
9
+ super(options)
10
10
  end
11
11
 
12
12
  def run
13
- if (environment = find_environment(@environment_name))
14
- fail_with_error('Missing command. Example: rascal run job -- bundle exec rake') if @command.empty?
15
- status = environment.run_command(*@command)
16
- exit(exit_code_for(status))
17
- end
13
+ environment = find_environment(@environment_name)
14
+ fail_with_error('Missing command. Example: rascal run job -- bundle exec rake') if @command.empty?
15
+ status = environment.run_command(*@command, **command_overrides)
16
+ exit(exit_code_for(status))
18
17
  end
19
18
 
20
19
  private
@@ -3,13 +3,13 @@ require 'rascal'
3
3
  module Rascal
4
4
  module CLI
5
5
  class Shell < Base
6
- def initialize(thor, options, environment_name)
6
+ def initialize(options, environment_name)
7
7
  @environment_name = environment_name
8
- super(thor, options)
8
+ super(options)
9
9
  end
10
10
 
11
11
  def run
12
- find_environment(@environment_name)&.run_shell
12
+ find_environment(@environment_name).run_shell(**command_overrides)
13
13
  end
14
14
  end
15
15
  end
@@ -0,0 +1,81 @@
1
+ require 'json'
2
+ require 'rascal'
3
+
4
+ module Rascal
5
+ module CLI
6
+ # Shows what rascal made of the CI config for an environment, and what of it exists in
7
+ # Docker, so you can see why something does not work without knowing rascal's names.
8
+ class Status < Base
9
+ def initialize(options, environment_name)
10
+ super(options)
11
+ @environment_name = environment_name || :all
12
+ end
13
+
14
+ def run
15
+ statuses = each_environment(@environment_name).collect(&:status)
16
+ if @options[:json]
17
+ say JSON.pretty_generate(@environment_name == :all ? statuses : statuses.first)
18
+ else
19
+ say statuses.collect { |status| text(status) }.join("\n")
20
+ end
21
+ end
22
+
23
+ private
24
+
25
+ def text(status)
26
+ lines = [status['name']]
27
+ hints = status['problems'].empty?
28
+ lines << " Image: #{status['image'] || '(none)'}#{image_note(status, hints)}"
29
+ image_entrypoint = status['image_entrypoint'] unless Array(status['image_entrypoint']).empty?
30
+ if status['entrypoint']
31
+ lines << " Entrypoint: #{status['entrypoint'].inspect}#{" (instead of the image's #{image_entrypoint.inspect})" if image_entrypoint}"
32
+ elsif image_entrypoint
33
+ lines << " Entrypoint: #{image_entrypoint.inspect} (from the image)"
34
+ end
35
+ status['problems'].each { |problem| lines << " Problem: #{problem}" }
36
+ status['warnings'].each { |warning| lines << " Warning: #{warning}" }
37
+ network = status['network']
38
+ lines << " Network: #{network['name']}#{' (does not exist)' unless network['exists']}"
39
+ lines << ' Services:' if status['services'].any?
40
+ status['services'].each { |service| lines.concat(service_text(service, hints)) }
41
+ lines << ' Volumes:'
42
+ status['volumes'].each do |volume|
43
+ source = volume['volume'] ? "volume #{volume['volume']}#{' (does not exist)' unless volume['exists']}" : volume['directory']
44
+ lines << " #{volume['path']}: #{source}"
45
+ end
46
+ lines << " Command containers: #{status['command_containers'].join(', ')}" if status['command_containers'].any?
47
+ lines << ' Variables:'
48
+ status['variables'].sort.each { |name, value| lines << " #{name}=#{value}" }
49
+ %w[before_shell after_shell].each do |key|
50
+ next if status[key].empty?
51
+
52
+ lines << " #{key}:"
53
+ status[key].each { |command| lines << " #{command}" }
54
+ end
55
+ lines << " ssh_agent: #{status['ssh_agent']}, services_timeout: #{status['services_timeout']}"
56
+ lines.join("\n")
57
+ end
58
+
59
+ def service_text(service, hints)
60
+ state = case service['state']
61
+ when 'running' then "running since #{service['started_at']}"
62
+ when 'missing' then hints ? 'not created, the next command starts it' : 'not created'
63
+ when 'created' then 'created, but never started'
64
+ else [service['state'], ("with exit code #{service['exit_code']}" if service['exit_code'])].compact.join(' ')
65
+ end
66
+ image = service['image'].to_s.empty? ? '(none)' : service['image']
67
+ lines = [" #{service['aliases'].join(', ')}: #{image}#{' (not downloaded)' unless service['image_present']}, #{state}"]
68
+ lines << " Error: #{service['error']}" if service['error']
69
+ lines << " Waits for port #{service['wait_ports'].join(' or ')}" if service['wait_ports']&.any?
70
+ lines << " Logs: #{service['logs']}" unless %w[missing created].include?(service['state'])
71
+ lines
72
+ end
73
+
74
+ def image_note(status, hints)
75
+ return unless status['image'] && !status['image_present']
76
+
77
+ hints ? ' (not downloaded, the next command downloads it)' : ' (not downloaded)'
78
+ end
79
+ end
80
+ end
81
+ end
@@ -3,19 +3,19 @@ require 'rascal'
3
3
  module Rascal
4
4
  module CLI
5
5
  class Update < Base
6
- def initialize(thor, options, environment_name)
7
- @environment_name = if options[:all]
8
- fail_with_error('Cannot give --all and an environment name!') if environment_name
9
- :all
10
- else
11
- environment_name
12
- end
13
- super(thor, options)
6
+ def initialize(options, environment_name)
7
+ super(options)
8
+ @environment_name = name_or_all(environment_name)
14
9
  end
15
10
 
16
11
  def run
17
12
  images = []
18
13
  each_environment(@environment_name) do |environment|
14
+ if @environment_name == :all && environment.problems.any?
15
+ # Typically a job that does not run tests, like a deploy job using $CI_REGISTRY_IMAGE.
16
+ info("Skipping #{environment.name}: #{environment.problems.join(' ')}")
17
+ next
18
+ end
19
19
  images += environment.update(skip: images)
20
20
  end
21
21
  end
data/lib/rascal/cli.rb CHANGED
@@ -8,6 +8,7 @@ module Rascal
8
8
  autoload :Main, 'rascal/cli/main'
9
9
  autoload :Run, 'rascal/cli/run'
10
10
  autoload :Shell, 'rascal/cli/shell'
11
+ autoload :Status, 'rascal/cli/status'
11
12
  autoload :Update, 'rascal/cli/update'
12
13
  end
13
14
  end
@@ -0,0 +1,83 @@
1
+ require 'shellwords'
2
+
3
+ module Rascal
4
+ # The bash script that runs a command in a job's container:
5
+ #
6
+ # 1. Waits until services that may not be ready yet accept connections, like a GitLab runner does.
7
+ # 2. Lets Git trust the project, which belongs to another user than the container's.
8
+ # 3. Runs before_shell, the command and after_shell, and exits with the command's status.
9
+ # Each on its own line, so entries may end in `&` or a comment. Without saving the status,
10
+ # an after_shell entry would determine it, and failures would go unnoticed.
11
+ # 4. Gives files the command created in the project back to their owner, since the container
12
+ # usually runs as root.
13
+ class CommandScript
14
+ # A bash function that waits until a host accepts TCP connections on one of the given ports:
15
+ # __rascal_wait_for HOST TIMEOUT_SECONDS PORT...
16
+ WAIT_FOR_FUNCTION = <<~BASH.gsub(/\n\s*/, ' ').strip
17
+ __rascal_wait_for() {
18
+ local host=$1 timeout=$2 deadline=$((SECONDS + $2)) port; shift 2;
19
+ while ((SECONDS < deadline)); do
20
+ for port in "$@"; do (exec 3<>"/dev/tcp/$host/$port") 2>/dev/null && return; done;
21
+ sleep 0.1;
22
+ done;
23
+ echo "rascal: Service $host does not accept connections on port $* after $timeout seconds, continuing anyway. If it listens on another port, set HEALTHCHECK_TCP_PORT in its variables. If it needs more time, set services_timeout in the .rascal block." >&2;
24
+ }
25
+ BASH
26
+
27
+ # `command` is a bash command line. `waits` are [host, ports] pairs, for which it waits up to
28
+ # `wait_timeout` seconds each. `owner` is the [uid, gid] that should own new files in
29
+ # `working_dir`, or nil to leave them as they are.
30
+ def initialize(command, working_dir:, before_shell: [], after_shell: [], waits: [], wait_timeout:, owner: nil)
31
+ @command = command
32
+ @working_dir = working_dir
33
+ @before_shell = before_shell
34
+ @after_shell = after_shell
35
+ @waits = waits.reject { |_host, ports| ports.empty? }
36
+ @wait_timeout = wait_timeout
37
+ @owner = owner
38
+ end
39
+
40
+ def to_s
41
+ [
42
+ *wait_for_services,
43
+ *trust_project_for_git,
44
+ *@before_shell,
45
+ @command,
46
+ '__rascal_status=$?',
47
+ *@after_shell,
48
+ *fix_ownership,
49
+ 'exit $__rascal_status',
50
+ ].join("\n")
51
+ end
52
+
53
+ private
54
+
55
+ def wait_for_services
56
+ return [] if @waits.empty?
57
+
58
+ [WAIT_FOR_FUNCTION, *@waits.collect { |host, ports| "__rascal_wait_for #{Shellwords.escape(host)} #{@wait_timeout.to_i} #{ports.join(' ')}" }]
59
+ end
60
+
61
+ # Git refuses a repository that belongs to another user ("detected dubious ownership").
62
+ # The project is mounted on purpose, so trust it in the container's global Git config,
63
+ # which goes away with the container. Older Git, as in Ubuntu 22.04, ignores
64
+ # safe.directory from anywhere else.
65
+ def trust_project_for_git
66
+ return [] if @working_dir.nil?
67
+
68
+ dir = Shellwords.escape(@working_dir)
69
+ ["if command -v git > /dev/null 2>&1 && ! git config --global --get-all safe.directory 2> /dev/null | grep -qxF #{dir}; " \
70
+ "then git config --global --add safe.directory #{dir} 2> /dev/null; fi"]
71
+ end
72
+
73
+ # Only when the project appears in the container as owned by that user, so it leaves a
74
+ # differently mapped project alone.
75
+ def fix_ownership
76
+ return [] if @owner.nil? || @working_dir.nil?
77
+
78
+ uid, gid = @owner
79
+ dir = Shellwords.escape(@working_dir)
80
+ ["[ \"$(stat -c %u #{dir} 2>/dev/null)\" = #{uid} ] && find #{dir} -xdev -user 0 -exec chown -h #{uid}:#{gid} {} + 2>/dev/null"]
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,101 @@
1
+ require 'securerandom'
2
+ require 'shellwords'
3
+
4
+ module Rascal
5
+ module Docker
6
+ # The one-off containers that run commands in an environment, each named like
7
+ # `rascal-myapp-rspec-run-0123abcd` and removed when its command ends.
8
+ class CommandContainers
9
+ include IOHelper
10
+
11
+ # Names of command containers end with this and a random suffix.
12
+ RUN_SUFFIX = '-run-'
13
+
14
+ attr_reader :entrypoint
15
+
16
+ # `image` is a Docker::Image. `entrypoint` overrides the image's entrypoint, like
17
+ # `image: entrypoint:` in GitLab. [] or [''] removes it.
18
+ def initialize(name, image, entrypoint: nil)
19
+ @name = name
20
+ @prefixed_name = "#{NAME_PREFIX}#{name}"
21
+ @image = image
22
+ @entrypoint = entrypoint && (entrypoint.empty? ? [''] : entrypoint)
23
+ end
24
+
25
+ # Runs a command in a new container, attached to rascal's output, and returns its
26
+ # Process::Status. If rascal gets a signal, like when a script or an agent gives up on a
27
+ # command, it removes the container, which would keep running otherwise. With `tty`, the
28
+ # container gets a terminal. With `forward_stdin`, it reads rascal's stdin.
29
+ def run(*command, env:, network:, volumes:, working_dir:, tty:, forward_stdin:)
30
+ name = "#{@prefixed_name}#{RUN_SUFFIX}#{SecureRandom.hex(4)}"
31
+ remove_on_signal(name) do
32
+ Docker.interface.run_and_attach(
33
+ 'container',
34
+ 'run',
35
+ '--rm',
36
+ '--name', name,
37
+ '-a', 'STDOUT',
38
+ '-a', 'STDERR',
39
+ *(['-a', 'STDIN', '--interactive'] if forward_stdin),
40
+ *('--tty' if tty),
41
+ *(['--entrypoint', @entrypoint.first.to_s] if @entrypoint),
42
+ *(['-w', working_dir] if working_dir),
43
+ *volumes.flat_map { |volume| ['-v', volume.to_param] },
44
+ *Docker.env_args(env),
45
+ '--network', network.id,
46
+ @image.name,
47
+ *@entrypoint&.drop(1),
48
+ # An entrypoint like `sh -c` takes the command as one string.
49
+ *(shell_entrypoint? ? [Shellwords.join(command)] : command),
50
+ redirect_io: { out: stdout, err: stderr, in: stdin },
51
+ allow_failure: true,
52
+ )
53
+ end
54
+ end
55
+
56
+ # Names of containers of commands that run now, or were left over.
57
+ def names
58
+ Docker.interface.run('container', 'ps', '--all', '--format', '{{.Names}}', '--filter', name_filter, output: :text).to_s.split
59
+ end
60
+
61
+ # Removes containers left over from commands that were killed.
62
+ def clean
63
+ ids = Docker.interface.run('container', 'ps', '--all', '--quiet', '--filter', name_filter, output: :text).to_s.split
64
+ return if ids.empty?
65
+
66
+ info "Removing #{ids.size} leftover command container#{'s' if ids.size > 1} for #{@name}"
67
+ Docker.interface.run('container', 'rm', '--force', '--volumes', *ids)
68
+ end
69
+
70
+ private
71
+
72
+ # Whether the entrypoint (from the CI config, or else the image's) is a shell that
73
+ # expects the command as one string, like `sh -c`, `bash -lc` or `sh -ec`.
74
+ def shell_entrypoint?
75
+ Array(@entrypoint || @image.entrypoint).last.to_s.match?(/\A-[a-zA-Z]*c[a-zA-Z]*\z/)
76
+ end
77
+
78
+ # Matches only containers of commands, not service containers of a job whose name starts the
79
+ # same, like `job-run-tests` for `job`.
80
+ def name_filter
81
+ "name=^/#{Regexp.escape(@prefixed_name)}#{RUN_SUFFIX}[0-9a-f]{8}$"
82
+ end
83
+
84
+ # Runs `docker` directly instead of through Docker.interface, which uses threads and locks
85
+ # that a signal handler must not.
86
+ def remove_on_signal(name)
87
+ previous_handlers = %w[INT TERM HUP].to_h do |signal|
88
+ handler = Signal.trap(signal) do
89
+ pid = Process.spawn('docker', 'container', 'rm', '--force', '--volumes', name, out: File::NULL, err: File::NULL)
90
+ Process.wait(pid)
91
+ exit(128 + Signal.list.fetch(signal))
92
+ end
93
+ [signal, handler]
94
+ end
95
+ yield
96
+ ensure
97
+ previous_handlers&.each { |signal, handler| Signal.trap(signal, handler) }
98
+ end
99
+ end
100
+ end
101
+ end