rascal 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +4 -4
- data/.ruby-version +1 -1
- data/AGENTS.md +4 -0
- data/CHANGELOG.md +74 -0
- data/CONTRIBUTING.md +139 -0
- data/Gemfile.common +0 -3
- data/Gemfile.lock +1 -47
- data/Guardfile +0 -9
- data/README.md +679 -76
- data/Rakefile +1 -7
- data/lib/rascal/cli/base.rb +41 -20
- data/lib/rascal/cli/clean.rb +3 -8
- data/lib/rascal/cli/environments.rb +1 -3
- data/lib/rascal/cli/main.rb +106 -17
- data/lib/rascal/cli/run.rb +6 -7
- data/lib/rascal/cli/shell.rb +3 -3
- data/lib/rascal/cli/status.rb +81 -0
- data/lib/rascal/cli/update.rb +8 -8
- data/lib/rascal/cli.rb +1 -0
- data/lib/rascal/command_script.rb +83 -0
- data/lib/rascal/docker/command_containers.rb +101 -0
- data/lib/rascal/docker/container.rb +82 -101
- data/lib/rascal/docker/image.rb +71 -0
- data/lib/rascal/docker/interface.rb +4 -2
- data/lib/rascal/docker/network.rb +12 -0
- data/lib/rascal/docker/volume.rb +17 -10
- data/lib/rascal/docker.rb +11 -4
- data/lib/rascal/environment.rb +164 -40
- data/lib/rascal/gitlab/config_file.rb +199 -0
- data/lib/rascal/gitlab/environment_builder.rb +204 -0
- data/lib/rascal/gitlab/jobs.rb +299 -0
- data/lib/rascal/gitlab/variables.rb +155 -0
- data/lib/rascal/gitlab.rb +16 -0
- data/lib/rascal/io_helper.rb +11 -0
- data/lib/rascal/service.rb +52 -9
- data/lib/rascal/version.rb +1 -1
- data/lib/rascal.rb +2 -1
- data/rascal.gemspec +3 -2
- metadata +15 -8
- data/bin/cucumber +0 -29
- data/cucumber.yml +0 -1
- data/lib/rascal/environments_definition/gitlab.rb +0 -146
- data/lib/rascal/environments_definition.rb +0 -23
data/Rakefile
CHANGED
|
@@ -1,12 +1,6 @@
|
|
|
1
1
|
require 'bundler/gem_tasks'
|
|
2
2
|
require 'rspec/core/rake_task'
|
|
3
|
-
require 'cucumber'
|
|
4
|
-
require 'cucumber/rake/task'
|
|
5
3
|
|
|
6
4
|
RSpec::Core::RakeTask.new(:spec)
|
|
7
|
-
Cucumber::Rake::Task.new(:features)
|
|
8
5
|
|
|
9
|
-
|
|
10
|
-
task :test => [:spec, :features]
|
|
11
|
-
|
|
12
|
-
task default: :test
|
|
6
|
+
task default: :spec
|
data/lib/rascal/cli/base.rb
CHANGED
|
@@ -3,13 +3,31 @@ require 'pathname'
|
|
|
3
3
|
module Rascal
|
|
4
4
|
module CLI
|
|
5
5
|
class Base
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
include IOHelper
|
|
7
|
+
|
|
8
|
+
def initialize(options)
|
|
8
9
|
@options = options
|
|
9
10
|
end
|
|
10
11
|
|
|
11
12
|
private
|
|
12
13
|
|
|
14
|
+
# Changes to the environment for a single command, from --env, --unset-env and --ssh-agent.
|
|
15
|
+
def command_overrides
|
|
16
|
+
env = Array(@options[:env]).to_h do |assignment|
|
|
17
|
+
name, value = assignment.split('=', 2)
|
|
18
|
+
fail_with_error('Use --env NAME=VALUE, like --env RAILS_ENV=test.') unless value && variable_name?(name)
|
|
19
|
+
[name, value]
|
|
20
|
+
end
|
|
21
|
+
unset_env = Array(@options[:unset_env]).each do |name|
|
|
22
|
+
fail_with_error('Use --unset-env NAME, like --unset-env CI.') unless variable_name?(name)
|
|
23
|
+
end
|
|
24
|
+
{ env: env, unset_env: unset_env, ssh_agent: @options[:ssh_agent] }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def variable_name?(name)
|
|
28
|
+
name.match?(/\A[a-zA-Z_][a-zA-Z0-9_]*\z/)
|
|
29
|
+
end
|
|
30
|
+
|
|
13
31
|
def config_location
|
|
14
32
|
Pathname.new(@options[:config_file]).expand_path
|
|
15
33
|
end
|
|
@@ -19,39 +37,42 @@ module Rascal
|
|
|
19
37
|
end
|
|
20
38
|
|
|
21
39
|
def find_environment(environment_name)
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
else
|
|
26
|
-
available_environments = definition.available_environment_names.join(', ')
|
|
40
|
+
jobs = project_jobs
|
|
41
|
+
jobs.environment(environment_name) or begin
|
|
42
|
+
available_environments = jobs.names.join(', ')
|
|
27
43
|
if environment_name
|
|
28
|
-
fail_with_error("Unknown
|
|
44
|
+
fail_with_error("Unknown job #{environment_name}. Jobs: #{available_environments}.")
|
|
29
45
|
else
|
|
30
|
-
fail_with_error("Missing
|
|
46
|
+
fail_with_error("Missing job. Jobs: #{available_environments}.")
|
|
31
47
|
end
|
|
32
48
|
end
|
|
33
49
|
end
|
|
34
50
|
|
|
51
|
+
# The environment name, or :all with --all.
|
|
52
|
+
def name_or_all(environment_name)
|
|
53
|
+
return environment_name unless @options[:all]
|
|
54
|
+
|
|
55
|
+
fail_with_error('Give either a job or --all.') if environment_name
|
|
56
|
+
:all
|
|
57
|
+
end
|
|
58
|
+
|
|
35
59
|
def each_environment(name, &block)
|
|
36
|
-
return enum_for(:each_environment) unless block_given?
|
|
60
|
+
return enum_for(:each_environment, name) unless block_given?
|
|
37
61
|
|
|
38
62
|
if name == :all
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
yield
|
|
63
|
+
jobs = project_jobs
|
|
64
|
+
jobs.names.each do |environment_name|
|
|
65
|
+
yield jobs.environment(environment_name)
|
|
42
66
|
end
|
|
43
67
|
else
|
|
44
68
|
yield find_environment(name)
|
|
45
69
|
end
|
|
46
70
|
end
|
|
47
71
|
|
|
48
|
-
def
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
fail_with_error("Could not find an environment definition in current working directory.")
|
|
53
|
-
nil
|
|
54
|
-
end
|
|
72
|
+
def project_jobs
|
|
73
|
+
Gitlab::Jobs.find(config_location) or
|
|
74
|
+
fail_with_error("Could not find a .gitlab-ci.yml in #{config_location}. Run rascal in your project's root directory, " \
|
|
75
|
+
"or point to the file with --config-file.")
|
|
55
76
|
end
|
|
56
77
|
end
|
|
57
78
|
end
|
data/lib/rascal/cli/clean.rb
CHANGED
|
@@ -3,14 +3,9 @@ require 'rascal'
|
|
|
3
3
|
module Rascal
|
|
4
4
|
module CLI
|
|
5
5
|
class Clean < Base
|
|
6
|
-
def initialize(
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
:all
|
|
10
|
-
else
|
|
11
|
-
environment_name
|
|
12
|
-
end
|
|
13
|
-
super(thor, options)
|
|
6
|
+
def initialize(options, environment_name)
|
|
7
|
+
super(options)
|
|
8
|
+
@environment_name = name_or_all(environment_name)
|
|
14
9
|
end
|
|
15
10
|
|
|
16
11
|
def run
|
data/lib/rascal/cli/main.rb
CHANGED
|
@@ -1,6 +1,17 @@
|
|
|
1
|
+
require 'rascal'
|
|
2
|
+
|
|
1
3
|
module Rascal
|
|
2
4
|
module CLI
|
|
3
5
|
class Main < Thor
|
|
6
|
+
# Options that change a single `run` or `shell`, like for updating dependencies.
|
|
7
|
+
def self.command_environment_options
|
|
8
|
+
method_option :env, type: :string, repeatable: true, banner: 'NAME=VALUE',
|
|
9
|
+
desc: 'Set a variable in the container, for this command only. Repeat for more.'
|
|
10
|
+
method_option :unset_env, type: :string, repeatable: true, banner: 'NAME',
|
|
11
|
+
desc: 'Remove a variable from the container, like CI, for this command only. Repeat for more.'
|
|
12
|
+
method_option :ssh_agent, type: :boolean,
|
|
13
|
+
desc: 'Give the container access to your SSH agent, for this command only. --no-ssh-agent keeps it away, even if the config enables it.'
|
|
14
|
+
end
|
|
4
15
|
def help(subcommand = false)
|
|
5
16
|
if subcommand
|
|
6
17
|
super
|
|
@@ -9,10 +20,12 @@ module Rascal
|
|
|
9
20
|
say 'Usage:'
|
|
10
21
|
say ' rascal <command> <args>'
|
|
11
22
|
say 'For example:'
|
|
12
|
-
say ' rascal
|
|
23
|
+
say ' rascal run rspec -- bundle exec rspec spec/models/user_spec.rb'
|
|
13
24
|
say
|
|
14
25
|
super
|
|
15
|
-
say 'For
|
|
26
|
+
say 'For more about a command, run "rascal help <command>". rascal --version prints the version.'
|
|
27
|
+
say 'The README explains setup and troubleshooting:'
|
|
28
|
+
say ' https://github.com/makandra/rascal#readme'
|
|
16
29
|
say
|
|
17
30
|
end
|
|
18
31
|
end
|
|
@@ -29,52 +42,128 @@ module Rascal
|
|
|
29
42
|
|
|
30
43
|
|
|
31
44
|
map "shell" => "_shell"
|
|
32
|
-
desc 'shell
|
|
45
|
+
desc 'shell JOB', "Open an interactive bash in a job's environment"
|
|
46
|
+
long_desc <<~TEXT
|
|
47
|
+
Opens bash in the environment of a CI job, like for a Rails console or a debugger.
|
|
48
|
+
Needs a terminal. Without one, use rascal run.
|
|
49
|
+
TEXT
|
|
50
|
+
command_environment_options
|
|
33
51
|
def _shell(environment_name = nil)
|
|
34
52
|
handle_error do
|
|
35
|
-
Shell.new(
|
|
53
|
+
Shell.new(options, environment_name).run
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
desc 'status [JOB]', "Show a job's environment as rascal sees it, and what of it is running"
|
|
58
|
+
long_desc <<~TEXT
|
|
59
|
+
Shows what rascal made of the CI config: the image, services, variables, before_shell and
|
|
60
|
+
after_shell, and any problems. Also shows what exists in Docker: whether images are
|
|
61
|
+
downloaded, the state of each service with the command to see its logs, the network,
|
|
62
|
+
volumes and running commands. Starts nothing. Without an environment, shows all of them.
|
|
63
|
+
TEXT
|
|
64
|
+
method_option :json, type: :boolean, desc: 'Print JSON, for scripts and agents'
|
|
65
|
+
def status(environment_name = nil)
|
|
66
|
+
handle_error do
|
|
67
|
+
Status.new(options, environment_name).run
|
|
36
68
|
end
|
|
37
69
|
end
|
|
38
70
|
|
|
39
|
-
desc 'environments', 'List the
|
|
71
|
+
desc 'environments', 'List the jobs you can use as environments'
|
|
40
72
|
def environments
|
|
41
73
|
handle_error do
|
|
42
|
-
Environments.new(
|
|
74
|
+
Environments.new(options).run
|
|
43
75
|
end
|
|
44
76
|
end
|
|
45
77
|
|
|
46
78
|
# Thor::Actions already defines #run, hence the mapping.
|
|
47
79
|
map 'run' => '_run'
|
|
48
80
|
stop_on_unknown_option! :_run
|
|
49
|
-
desc 'run
|
|
81
|
+
desc 'run JOB -- COMMAND', "Run a command in a job's environment"
|
|
82
|
+
long_desc <<~TEXT
|
|
83
|
+
Runs a command in the environment of a CI job, without a terminal, and exits with its status.
|
|
84
|
+
Everything after -- is passed on as arguments. For shell syntax, use bash -c:
|
|
85
|
+
|
|
86
|
+
rascal run rspec -- bash -c 'bin/rails db:migrate && bundle exec rspec'
|
|
87
|
+
|
|
88
|
+
Options go before --. Variables and the SSH agent can change for a single command:
|
|
89
|
+
|
|
90
|
+
rascal run rspec --unset-env CI --ssh-agent -- bundle update rack --conservative
|
|
91
|
+
TEXT
|
|
92
|
+
command_environment_options
|
|
50
93
|
def _run(environment_name = nil, *command)
|
|
51
94
|
handle_error do
|
|
52
|
-
command = command
|
|
53
|
-
Run.new(
|
|
95
|
+
run_options, command = split_run_arguments(command)
|
|
96
|
+
Run.new(run_options, environment_name, command).run
|
|
54
97
|
end
|
|
55
98
|
end
|
|
56
99
|
|
|
57
|
-
desc 'clean
|
|
58
|
-
|
|
59
|
-
|
|
100
|
+
desc 'clean JOB | --all', "Remove a job's environment"
|
|
101
|
+
long_desc <<~TEXT
|
|
102
|
+
Stops and removes the environment's services with their data, like databases, its network
|
|
103
|
+
and leftover command containers. Named volumes, like caches, stay unless you pass --volumes.
|
|
104
|
+
Files in the project stay. The next command starts the services fresh.
|
|
105
|
+
TEXT
|
|
106
|
+
method_option :volumes, type: :boolean, desc: 'Also remove volumes, like caches'
|
|
107
|
+
method_option :all, type: :boolean, desc: 'Clean the environments of all jobs'
|
|
60
108
|
def clean(environment_name = nil)
|
|
61
109
|
handle_error do
|
|
62
|
-
Clean.new(
|
|
110
|
+
Clean.new(options, environment_name).run
|
|
63
111
|
end
|
|
64
112
|
end
|
|
65
113
|
|
|
66
|
-
desc 'update
|
|
67
|
-
|
|
114
|
+
desc 'update JOB | --all', "Pull the latest images of a job's environment"
|
|
115
|
+
long_desc <<~TEXT
|
|
116
|
+
Pulls the latest images of the job and its services. Services that already run keep their
|
|
117
|
+
old image until you run rascal clean.
|
|
118
|
+
TEXT
|
|
119
|
+
method_option :all, type: :boolean, desc: 'Update the environments of all jobs'
|
|
68
120
|
def update(environment_name = nil)
|
|
69
121
|
handle_error do
|
|
70
|
-
Update.new(
|
|
122
|
+
Update.new(options, environment_name).run
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Thor's own command in newer versions, of no use here.
|
|
127
|
+
if Thor.method_defined?(:tree)
|
|
128
|
+
desc 'tree', 'Print a tree of all available commands', hide: true
|
|
129
|
+
def tree
|
|
130
|
+
super
|
|
71
131
|
end
|
|
72
132
|
end
|
|
73
133
|
|
|
74
|
-
class_option :config_file, aliases: ['-c'], default: '.', required: true, desc: '
|
|
134
|
+
class_option :config_file, aliases: ['-c'], default: '.', required: true, desc: 'The .gitlab-ci.yml to use, or its directory'
|
|
75
135
|
|
|
76
136
|
private
|
|
77
137
|
|
|
138
|
+
# Thor stops parsing options at the environment name of `run`, so the command can have
|
|
139
|
+
# options of its own. Options between the environment name and `--`, like
|
|
140
|
+
# `rascal run rspec -c path -- cmd`, are parsed here. A command without a leading `--`,
|
|
141
|
+
# like `rascal run rspec git checkout -- file`, is taken as it is.
|
|
142
|
+
def split_run_arguments(arguments)
|
|
143
|
+
return [options, arguments.drop(1)] if arguments.first == '--'
|
|
144
|
+
return [options, arguments] unless arguments.first&.start_with?('-')
|
|
145
|
+
|
|
146
|
+
separator = arguments.index('--') or
|
|
147
|
+
raise Thor::Error, "Put -- between rascal's options and the command, like: rascal run rspec #{arguments.first} ... -- COMMAND"
|
|
148
|
+
known_options = self.class.class_options.merge(self.class.commands['_run'].options)
|
|
149
|
+
parser = Thor::Options.new(known_options)
|
|
150
|
+
defaults = Thor::Options.new(known_options).parse([])
|
|
151
|
+
given = parser.parse(arguments.take(separator))
|
|
152
|
+
parser.check_unknown!
|
|
153
|
+
if parser.remaining.any?
|
|
154
|
+
raise Thor::Error, "Unexpected #{parser.remaining.join(' ')} before --. Usage: rascal run [OPTIONS] JOB -- COMMAND"
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
merged = options.to_h
|
|
158
|
+
given.each do |name, value|
|
|
159
|
+
next if defaults[name] == value
|
|
160
|
+
|
|
161
|
+
# Repeatable options, like --env, may come before and after the environment name.
|
|
162
|
+
merged[name] = value.is_a?(Array) ? Array(merged[name]) + value : value
|
|
163
|
+
end
|
|
164
|
+
[Thor::CoreExt::HashWithIndifferentAccess.new(merged), arguments.drop(separator + 1)]
|
|
165
|
+
end
|
|
166
|
+
|
|
78
167
|
def handle_error
|
|
79
168
|
yield
|
|
80
169
|
rescue Rascal::Error => e
|
data/lib/rascal/cli/run.rb
CHANGED
|
@@ -3,18 +3,17 @@ require 'rascal'
|
|
|
3
3
|
module Rascal
|
|
4
4
|
module CLI
|
|
5
5
|
class Run < Base
|
|
6
|
-
def initialize(
|
|
6
|
+
def initialize(options, environment_name, command)
|
|
7
7
|
@environment_name = environment_name
|
|
8
8
|
@command = command
|
|
9
|
-
super(
|
|
9
|
+
super(options)
|
|
10
10
|
end
|
|
11
11
|
|
|
12
12
|
def run
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
end
|
|
13
|
+
environment = find_environment(@environment_name)
|
|
14
|
+
fail_with_error('Missing command. Example: rascal run job -- bundle exec rake') if @command.empty?
|
|
15
|
+
status = environment.run_command(*@command, **command_overrides)
|
|
16
|
+
exit(exit_code_for(status))
|
|
18
17
|
end
|
|
19
18
|
|
|
20
19
|
private
|
data/lib/rascal/cli/shell.rb
CHANGED
|
@@ -3,13 +3,13 @@ require 'rascal'
|
|
|
3
3
|
module Rascal
|
|
4
4
|
module CLI
|
|
5
5
|
class Shell < Base
|
|
6
|
-
def initialize(
|
|
6
|
+
def initialize(options, environment_name)
|
|
7
7
|
@environment_name = environment_name
|
|
8
|
-
super(
|
|
8
|
+
super(options)
|
|
9
9
|
end
|
|
10
10
|
|
|
11
11
|
def run
|
|
12
|
-
find_environment(@environment_name)
|
|
12
|
+
find_environment(@environment_name).run_shell(**command_overrides)
|
|
13
13
|
end
|
|
14
14
|
end
|
|
15
15
|
end
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
require 'json'
|
|
2
|
+
require 'rascal'
|
|
3
|
+
|
|
4
|
+
module Rascal
|
|
5
|
+
module CLI
|
|
6
|
+
# Shows what rascal made of the CI config for an environment, and what of it exists in
|
|
7
|
+
# Docker, so you can see why something does not work without knowing rascal's names.
|
|
8
|
+
class Status < Base
|
|
9
|
+
def initialize(options, environment_name)
|
|
10
|
+
super(options)
|
|
11
|
+
@environment_name = environment_name || :all
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def run
|
|
15
|
+
statuses = each_environment(@environment_name).collect(&:status)
|
|
16
|
+
if @options[:json]
|
|
17
|
+
say JSON.pretty_generate(@environment_name == :all ? statuses : statuses.first)
|
|
18
|
+
else
|
|
19
|
+
say statuses.collect { |status| text(status) }.join("\n")
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
def text(status)
|
|
26
|
+
lines = [status['name']]
|
|
27
|
+
hints = status['problems'].empty?
|
|
28
|
+
lines << " Image: #{status['image'] || '(none)'}#{image_note(status, hints)}"
|
|
29
|
+
image_entrypoint = status['image_entrypoint'] unless Array(status['image_entrypoint']).empty?
|
|
30
|
+
if status['entrypoint']
|
|
31
|
+
lines << " Entrypoint: #{status['entrypoint'].inspect}#{" (instead of the image's #{image_entrypoint.inspect})" if image_entrypoint}"
|
|
32
|
+
elsif image_entrypoint
|
|
33
|
+
lines << " Entrypoint: #{image_entrypoint.inspect} (from the image)"
|
|
34
|
+
end
|
|
35
|
+
status['problems'].each { |problem| lines << " Problem: #{problem}" }
|
|
36
|
+
status['warnings'].each { |warning| lines << " Warning: #{warning}" }
|
|
37
|
+
network = status['network']
|
|
38
|
+
lines << " Network: #{network['name']}#{' (does not exist)' unless network['exists']}"
|
|
39
|
+
lines << ' Services:' if status['services'].any?
|
|
40
|
+
status['services'].each { |service| lines.concat(service_text(service, hints)) }
|
|
41
|
+
lines << ' Volumes:'
|
|
42
|
+
status['volumes'].each do |volume|
|
|
43
|
+
source = volume['volume'] ? "volume #{volume['volume']}#{' (does not exist)' unless volume['exists']}" : volume['directory']
|
|
44
|
+
lines << " #{volume['path']}: #{source}"
|
|
45
|
+
end
|
|
46
|
+
lines << " Command containers: #{status['command_containers'].join(', ')}" if status['command_containers'].any?
|
|
47
|
+
lines << ' Variables:'
|
|
48
|
+
status['variables'].sort.each { |name, value| lines << " #{name}=#{value}" }
|
|
49
|
+
%w[before_shell after_shell].each do |key|
|
|
50
|
+
next if status[key].empty?
|
|
51
|
+
|
|
52
|
+
lines << " #{key}:"
|
|
53
|
+
status[key].each { |command| lines << " #{command}" }
|
|
54
|
+
end
|
|
55
|
+
lines << " ssh_agent: #{status['ssh_agent']}, services_timeout: #{status['services_timeout']}"
|
|
56
|
+
lines.join("\n")
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def service_text(service, hints)
|
|
60
|
+
state = case service['state']
|
|
61
|
+
when 'running' then "running since #{service['started_at']}"
|
|
62
|
+
when 'missing' then hints ? 'not created, the next command starts it' : 'not created'
|
|
63
|
+
when 'created' then 'created, but never started'
|
|
64
|
+
else [service['state'], ("with exit code #{service['exit_code']}" if service['exit_code'])].compact.join(' ')
|
|
65
|
+
end
|
|
66
|
+
image = service['image'].to_s.empty? ? '(none)' : service['image']
|
|
67
|
+
lines = [" #{service['aliases'].join(', ')}: #{image}#{' (not downloaded)' unless service['image_present']}, #{state}"]
|
|
68
|
+
lines << " Error: #{service['error']}" if service['error']
|
|
69
|
+
lines << " Waits for port #{service['wait_ports'].join(' or ')}" if service['wait_ports']&.any?
|
|
70
|
+
lines << " Logs: #{service['logs']}" unless %w[missing created].include?(service['state'])
|
|
71
|
+
lines
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def image_note(status, hints)
|
|
75
|
+
return unless status['image'] && !status['image_present']
|
|
76
|
+
|
|
77
|
+
hints ? ' (not downloaded, the next command downloads it)' : ' (not downloaded)'
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
data/lib/rascal/cli/update.rb
CHANGED
|
@@ -3,19 +3,19 @@ require 'rascal'
|
|
|
3
3
|
module Rascal
|
|
4
4
|
module CLI
|
|
5
5
|
class Update < Base
|
|
6
|
-
def initialize(
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
:all
|
|
10
|
-
else
|
|
11
|
-
environment_name
|
|
12
|
-
end
|
|
13
|
-
super(thor, options)
|
|
6
|
+
def initialize(options, environment_name)
|
|
7
|
+
super(options)
|
|
8
|
+
@environment_name = name_or_all(environment_name)
|
|
14
9
|
end
|
|
15
10
|
|
|
16
11
|
def run
|
|
17
12
|
images = []
|
|
18
13
|
each_environment(@environment_name) do |environment|
|
|
14
|
+
if @environment_name == :all && environment.problems.any?
|
|
15
|
+
# Typically a job that does not run tests, like a deploy job using $CI_REGISTRY_IMAGE.
|
|
16
|
+
info("Skipping #{environment.name}: #{environment.problems.join(' ')}")
|
|
17
|
+
next
|
|
18
|
+
end
|
|
19
19
|
images += environment.update(skip: images)
|
|
20
20
|
end
|
|
21
21
|
end
|
data/lib/rascal/cli.rb
CHANGED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
require 'shellwords'
|
|
2
|
+
|
|
3
|
+
module Rascal
|
|
4
|
+
# The bash script that runs a command in a job's container:
|
|
5
|
+
#
|
|
6
|
+
# 1. Waits until services that may not be ready yet accept connections, like a GitLab runner does.
|
|
7
|
+
# 2. Lets Git trust the project, which belongs to another user than the container's.
|
|
8
|
+
# 3. Runs before_shell, the command and after_shell, and exits with the command's status.
|
|
9
|
+
# Each on its own line, so entries may end in `&` or a comment. Without saving the status,
|
|
10
|
+
# an after_shell entry would determine it, and failures would go unnoticed.
|
|
11
|
+
# 4. Gives files the command created in the project back to their owner, since the container
|
|
12
|
+
# usually runs as root.
|
|
13
|
+
class CommandScript
|
|
14
|
+
# A bash function that waits until a host accepts TCP connections on one of the given ports:
|
|
15
|
+
# __rascal_wait_for HOST TIMEOUT_SECONDS PORT...
|
|
16
|
+
WAIT_FOR_FUNCTION = <<~BASH.gsub(/\n\s*/, ' ').strip
|
|
17
|
+
__rascal_wait_for() {
|
|
18
|
+
local host=$1 timeout=$2 deadline=$((SECONDS + $2)) port; shift 2;
|
|
19
|
+
while ((SECONDS < deadline)); do
|
|
20
|
+
for port in "$@"; do (exec 3<>"/dev/tcp/$host/$port") 2>/dev/null && return; done;
|
|
21
|
+
sleep 0.1;
|
|
22
|
+
done;
|
|
23
|
+
echo "rascal: Service $host does not accept connections on port $* after $timeout seconds, continuing anyway. If it listens on another port, set HEALTHCHECK_TCP_PORT in its variables. If it needs more time, set services_timeout in the .rascal block." >&2;
|
|
24
|
+
}
|
|
25
|
+
BASH
|
|
26
|
+
|
|
27
|
+
# `command` is a bash command line. `waits` are [host, ports] pairs, for which it waits up to
|
|
28
|
+
# `wait_timeout` seconds each. `owner` is the [uid, gid] that should own new files in
|
|
29
|
+
# `working_dir`, or nil to leave them as they are.
|
|
30
|
+
def initialize(command, working_dir:, before_shell: [], after_shell: [], waits: [], wait_timeout:, owner: nil)
|
|
31
|
+
@command = command
|
|
32
|
+
@working_dir = working_dir
|
|
33
|
+
@before_shell = before_shell
|
|
34
|
+
@after_shell = after_shell
|
|
35
|
+
@waits = waits.reject { |_host, ports| ports.empty? }
|
|
36
|
+
@wait_timeout = wait_timeout
|
|
37
|
+
@owner = owner
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def to_s
|
|
41
|
+
[
|
|
42
|
+
*wait_for_services,
|
|
43
|
+
*trust_project_for_git,
|
|
44
|
+
*@before_shell,
|
|
45
|
+
@command,
|
|
46
|
+
'__rascal_status=$?',
|
|
47
|
+
*@after_shell,
|
|
48
|
+
*fix_ownership,
|
|
49
|
+
'exit $__rascal_status',
|
|
50
|
+
].join("\n")
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
def wait_for_services
|
|
56
|
+
return [] if @waits.empty?
|
|
57
|
+
|
|
58
|
+
[WAIT_FOR_FUNCTION, *@waits.collect { |host, ports| "__rascal_wait_for #{Shellwords.escape(host)} #{@wait_timeout.to_i} #{ports.join(' ')}" }]
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Git refuses a repository that belongs to another user ("detected dubious ownership").
|
|
62
|
+
# The project is mounted on purpose, so trust it in the container's global Git config,
|
|
63
|
+
# which goes away with the container. Older Git, as in Ubuntu 22.04, ignores
|
|
64
|
+
# safe.directory from anywhere else.
|
|
65
|
+
def trust_project_for_git
|
|
66
|
+
return [] if @working_dir.nil?
|
|
67
|
+
|
|
68
|
+
dir = Shellwords.escape(@working_dir)
|
|
69
|
+
["if command -v git > /dev/null 2>&1 && ! git config --global --get-all safe.directory 2> /dev/null | grep -qxF #{dir}; " \
|
|
70
|
+
"then git config --global --add safe.directory #{dir} 2> /dev/null; fi"]
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Only when the project appears in the container as owned by that user, so it leaves a
|
|
74
|
+
# differently mapped project alone.
|
|
75
|
+
def fix_ownership
|
|
76
|
+
return [] if @owner.nil? || @working_dir.nil?
|
|
77
|
+
|
|
78
|
+
uid, gid = @owner
|
|
79
|
+
dir = Shellwords.escape(@working_dir)
|
|
80
|
+
["[ \"$(stat -c %u #{dir} 2>/dev/null)\" = #{uid} ] && find #{dir} -xdev -user 0 -exec chown -h #{uid}:#{gid} {} + 2>/dev/null"]
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
require 'securerandom'
|
|
2
|
+
require 'shellwords'
|
|
3
|
+
|
|
4
|
+
module Rascal
|
|
5
|
+
module Docker
|
|
6
|
+
# The one-off containers that run commands in an environment, each named like
|
|
7
|
+
# `rascal-myapp-rspec-run-0123abcd` and removed when its command ends.
|
|
8
|
+
class CommandContainers
|
|
9
|
+
include IOHelper
|
|
10
|
+
|
|
11
|
+
# Names of command containers end with this and a random suffix.
|
|
12
|
+
RUN_SUFFIX = '-run-'
|
|
13
|
+
|
|
14
|
+
attr_reader :entrypoint
|
|
15
|
+
|
|
16
|
+
# `image` is a Docker::Image. `entrypoint` overrides the image's entrypoint, like
|
|
17
|
+
# `image: entrypoint:` in GitLab. [] or [''] removes it.
|
|
18
|
+
def initialize(name, image, entrypoint: nil)
|
|
19
|
+
@name = name
|
|
20
|
+
@prefixed_name = "#{NAME_PREFIX}#{name}"
|
|
21
|
+
@image = image
|
|
22
|
+
@entrypoint = entrypoint && (entrypoint.empty? ? [''] : entrypoint)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Runs a command in a new container, attached to rascal's output, and returns its
|
|
26
|
+
# Process::Status. If rascal gets a signal, like when a script or an agent gives up on a
|
|
27
|
+
# command, it removes the container, which would keep running otherwise. With `tty`, the
|
|
28
|
+
# container gets a terminal. With `forward_stdin`, it reads rascal's stdin.
|
|
29
|
+
def run(*command, env:, network:, volumes:, working_dir:, tty:, forward_stdin:)
|
|
30
|
+
name = "#{@prefixed_name}#{RUN_SUFFIX}#{SecureRandom.hex(4)}"
|
|
31
|
+
remove_on_signal(name) do
|
|
32
|
+
Docker.interface.run_and_attach(
|
|
33
|
+
'container',
|
|
34
|
+
'run',
|
|
35
|
+
'--rm',
|
|
36
|
+
'--name', name,
|
|
37
|
+
'-a', 'STDOUT',
|
|
38
|
+
'-a', 'STDERR',
|
|
39
|
+
*(['-a', 'STDIN', '--interactive'] if forward_stdin),
|
|
40
|
+
*('--tty' if tty),
|
|
41
|
+
*(['--entrypoint', @entrypoint.first.to_s] if @entrypoint),
|
|
42
|
+
*(['-w', working_dir] if working_dir),
|
|
43
|
+
*volumes.flat_map { |volume| ['-v', volume.to_param] },
|
|
44
|
+
*Docker.env_args(env),
|
|
45
|
+
'--network', network.id,
|
|
46
|
+
@image.name,
|
|
47
|
+
*@entrypoint&.drop(1),
|
|
48
|
+
# An entrypoint like `sh -c` takes the command as one string.
|
|
49
|
+
*(shell_entrypoint? ? [Shellwords.join(command)] : command),
|
|
50
|
+
redirect_io: { out: stdout, err: stderr, in: stdin },
|
|
51
|
+
allow_failure: true,
|
|
52
|
+
)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Names of containers of commands that run now, or were left over.
|
|
57
|
+
def names
|
|
58
|
+
Docker.interface.run('container', 'ps', '--all', '--format', '{{.Names}}', '--filter', name_filter, output: :text).to_s.split
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Removes containers left over from commands that were killed.
|
|
62
|
+
def clean
|
|
63
|
+
ids = Docker.interface.run('container', 'ps', '--all', '--quiet', '--filter', name_filter, output: :text).to_s.split
|
|
64
|
+
return if ids.empty?
|
|
65
|
+
|
|
66
|
+
info "Removing #{ids.size} leftover command container#{'s' if ids.size > 1} for #{@name}"
|
|
67
|
+
Docker.interface.run('container', 'rm', '--force', '--volumes', *ids)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
private
|
|
71
|
+
|
|
72
|
+
# Whether the entrypoint (from the CI config, or else the image's) is a shell that
|
|
73
|
+
# expects the command as one string, like `sh -c`, `bash -lc` or `sh -ec`.
|
|
74
|
+
def shell_entrypoint?
|
|
75
|
+
Array(@entrypoint || @image.entrypoint).last.to_s.match?(/\A-[a-zA-Z]*c[a-zA-Z]*\z/)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Matches only containers of commands, not service containers of a job whose name starts the
|
|
79
|
+
# same, like `job-run-tests` for `job`.
|
|
80
|
+
def name_filter
|
|
81
|
+
"name=^/#{Regexp.escape(@prefixed_name)}#{RUN_SUFFIX}[0-9a-f]{8}$"
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Runs `docker` directly instead of through Docker.interface, which uses threads and locks
|
|
85
|
+
# that a signal handler must not.
|
|
86
|
+
def remove_on_signal(name)
|
|
87
|
+
previous_handlers = %w[INT TERM HUP].to_h do |signal|
|
|
88
|
+
handler = Signal.trap(signal) do
|
|
89
|
+
pid = Process.spawn('docker', 'container', 'rm', '--force', '--volumes', name, out: File::NULL, err: File::NULL)
|
|
90
|
+
Process.wait(pid)
|
|
91
|
+
exit(128 + Signal.list.fetch(signal))
|
|
92
|
+
end
|
|
93
|
+
[signal, handler]
|
|
94
|
+
end
|
|
95
|
+
yield
|
|
96
|
+
ensure
|
|
97
|
+
previous_handlers&.each { |signal, handler| Signal.trap(signal, handler) }
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|