rascal 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +4 -4
- data/.ruby-version +1 -1
- data/AGENTS.md +4 -0
- data/CHANGELOG.md +74 -0
- data/CONTRIBUTING.md +139 -0
- data/Gemfile.common +0 -3
- data/Gemfile.lock +1 -47
- data/Guardfile +0 -9
- data/README.md +679 -76
- data/Rakefile +1 -7
- data/lib/rascal/cli/base.rb +41 -20
- data/lib/rascal/cli/clean.rb +3 -8
- data/lib/rascal/cli/environments.rb +1 -3
- data/lib/rascal/cli/main.rb +106 -17
- data/lib/rascal/cli/run.rb +6 -7
- data/lib/rascal/cli/shell.rb +3 -3
- data/lib/rascal/cli/status.rb +81 -0
- data/lib/rascal/cli/update.rb +8 -8
- data/lib/rascal/cli.rb +1 -0
- data/lib/rascal/command_script.rb +83 -0
- data/lib/rascal/docker/command_containers.rb +101 -0
- data/lib/rascal/docker/container.rb +82 -101
- data/lib/rascal/docker/image.rb +71 -0
- data/lib/rascal/docker/interface.rb +4 -2
- data/lib/rascal/docker/network.rb +12 -0
- data/lib/rascal/docker/volume.rb +17 -10
- data/lib/rascal/docker.rb +11 -4
- data/lib/rascal/environment.rb +164 -40
- data/lib/rascal/gitlab/config_file.rb +199 -0
- data/lib/rascal/gitlab/environment_builder.rb +204 -0
- data/lib/rascal/gitlab/jobs.rb +299 -0
- data/lib/rascal/gitlab/variables.rb +155 -0
- data/lib/rascal/gitlab.rb +16 -0
- data/lib/rascal/io_helper.rb +11 -0
- data/lib/rascal/service.rb +52 -9
- data/lib/rascal/version.rb +1 -1
- data/lib/rascal.rb +2 -1
- data/rascal.gemspec +3 -2
- metadata +15 -8
- data/bin/cucumber +0 -29
- data/cucumber.yml +0 -1
- data/lib/rascal/environments_definition/gitlab.rb +0 -146
- data/lib/rascal/environments_definition.rb +0 -23
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
module Rascal
|
|
2
|
+
module Gitlab
|
|
3
|
+
# A job, resolved like GitLab does, with rascal's settings applied.
|
|
4
|
+
# `variable_layers` are its variable definitions, from the lowest to the highest priority
|
|
5
|
+
# (see Jobs#variable_layers). `problems` keep its environment from starting, `warnings`
|
|
6
|
+
# may make commands fail. `missing_config_hint` starts a message about config the job
|
|
7
|
+
# lacks, which may come from includes rascal cannot read.
|
|
8
|
+
Job = Struct.new(:key, :name, :config, :variable_layers, :problems, :warnings, :missing_config_hint)
|
|
9
|
+
|
|
10
|
+
# The jobs of a .gitlab-ci.yml, resolved like GitLab does: `extends:`, `default:`,
|
|
11
|
+
# `inherit:` and `!reference`. rascal's settings from the `.rascal` block and a `.rascal.yml`
|
|
12
|
+
# override each job's config, and add variables on top of the job's.
|
|
13
|
+
class Jobs
|
|
14
|
+
# Top-level keys that configure the pipeline as a whole instead of defining a job.
|
|
15
|
+
# See https://docs.gitlab.com/ci/yaml/#global-keywords
|
|
16
|
+
GLOBAL_KEYS = %w[default include stages variables workflow].freeze
|
|
17
|
+
|
|
18
|
+
# Deprecated top-level keys that GitLab treats like the same keys in `default:`.
|
|
19
|
+
GLOBAL_DEFAULT_KEYS = %w[image services cache before_script after_script].freeze
|
|
20
|
+
|
|
21
|
+
# The keys of a job's config in which rascal resolves `!reference` tags. Others, like
|
|
22
|
+
# `script:`, may reference includes rascal cannot read, and rascal does not need them.
|
|
23
|
+
KEYS_WITH_REFERENCES = %w[image services before_shell after_shell volumes repo_dir].freeze
|
|
24
|
+
|
|
25
|
+
# Settings for rascal that extend the `.rascal` block, outside of the CI config.
|
|
26
|
+
LOCAL_CONFIG_FILE = '.rascal.yml'
|
|
27
|
+
|
|
28
|
+
# GitLab's limit for nested `extends:`.
|
|
29
|
+
MAX_EXTENDS_DEPTH = 10
|
|
30
|
+
|
|
31
|
+
# GitLab's documented limit for nested `!reference` tags.
|
|
32
|
+
MAX_REFERENCE_DEPTH = 10
|
|
33
|
+
|
|
34
|
+
# The jobs of the .gitlab-ci.yml at `path`, or in the directory `path`. Nil if there is none.
|
|
35
|
+
def self.find(path)
|
|
36
|
+
path = path.join('.gitlab-ci.yml') if path.directory?
|
|
37
|
+
new(path) if path.file?
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def initialize(config_path)
|
|
41
|
+
@project_dir = config_path.parent
|
|
42
|
+
@file = ConfigFile.new(config_path)
|
|
43
|
+
@ci_config = @file.to_h
|
|
44
|
+
# A .rascal.yml next to .gitlab-ci.yml extends the .rascal block, for settings
|
|
45
|
+
# that should not be part of the project's CI config.
|
|
46
|
+
rascal_block = hash_or_empty(@ci_config['.rascal'])
|
|
47
|
+
rascal_file = local_rascal_config
|
|
48
|
+
@rascal_config = deep_merge(rascal_block.except('jobs'), rascal_file.except('jobs'))
|
|
49
|
+
block_jobs = hash_or_empty(rascal_block['jobs'])
|
|
50
|
+
file_jobs = hash_or_empty(rascal_file['jobs'])
|
|
51
|
+
# An empty entry in .rascal.yml, like one with all its settings commented out, changes nothing.
|
|
52
|
+
@rascal_job_configs = (block_jobs.keys | file_jobs.keys).to_h do |key|
|
|
53
|
+
[key, deep_merge(hash_or_empty(block_jobs[key]), hash_or_empty(file_jobs[key]))]
|
|
54
|
+
end
|
|
55
|
+
# Images from rascal's settings, from the lowest to the highest priority, for #combined_image.
|
|
56
|
+
@rascal_images = lambda do |key|
|
|
57
|
+
[rascal_block, rascal_file, hash_or_empty(block_jobs[key]), hash_or_empty(file_jobs[key])]
|
|
58
|
+
.select { |settings| settings.key?('image') }.collect { |settings| settings['image'] }
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# The names of the jobs rascal can use as environments, including ones whose environment
|
|
63
|
+
# cannot start.
|
|
64
|
+
def names
|
|
65
|
+
jobs.collect(&:name).sort
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# The Job with the given name, or nil.
|
|
69
|
+
def job(name)
|
|
70
|
+
jobs.detect { |job| job.name == name }
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# The Environment of the job with the given name, or nil.
|
|
74
|
+
def environment(name)
|
|
75
|
+
if (job = job(name))
|
|
76
|
+
EnvironmentBuilder.new(job, project_dir: @project_dir).build
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
private
|
|
81
|
+
|
|
82
|
+
def skipped_includes_warning
|
|
83
|
+
return if @file.skipped_includes.empty?
|
|
84
|
+
|
|
85
|
+
"rascal cannot read these includes, so config from them is missing: #{@file.skipped_includes_description}. " \
|
|
86
|
+
"If the job needs services or variables from them, set them in the `.rascal` block."
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Explains that missing config may come from an include rascal cannot read.
|
|
90
|
+
def skipped_includes_hint
|
|
91
|
+
return '' if @file.skipped_includes.empty?
|
|
92
|
+
|
|
93
|
+
"It may come from an include rascal cannot read (#{@file.skipped_includes_description}), since rascal only reads local includes. "
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def local_rascal_config
|
|
97
|
+
path = @project_dir.join(LOCAL_CONFIG_FILE)
|
|
98
|
+
path.file? ? ConfigFile.new(path).to_h : {}
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def jobs
|
|
102
|
+
@jobs ||= @ci_config.filter_map do |key, job_config|
|
|
103
|
+
next if key.start_with?('.') || GLOBAL_KEYS.include?(key) || GLOBAL_DEFAULT_KEYS.include?(key)
|
|
104
|
+
next unless job_config.is_a?(Hash)
|
|
105
|
+
|
|
106
|
+
problems = []
|
|
107
|
+
begin
|
|
108
|
+
job_config = resolve_extends(key)
|
|
109
|
+
rescue Error => e
|
|
110
|
+
job_config = job_config.except('extends')
|
|
111
|
+
# Only this job is affected, so keep the others usable.
|
|
112
|
+
problems << e.message
|
|
113
|
+
end
|
|
114
|
+
rascal_job_config = hash_or_empty(@rascal_job_configs[key])
|
|
115
|
+
# The .rascal block overrides the job, which overrides `default:`.
|
|
116
|
+
# Variables are resolved separately, see #variable_layers.
|
|
117
|
+
config = deep_merge(
|
|
118
|
+
inherit_default(job_config),
|
|
119
|
+
@rascal_config.except('variables'),
|
|
120
|
+
rascal_job_config.except('variables'),
|
|
121
|
+
)
|
|
122
|
+
if (rascal_images = @rascal_images.call(key)).any?
|
|
123
|
+
base = inherit_default(job_config)
|
|
124
|
+
job_images = base.key?('image') ? [base['image']] : []
|
|
125
|
+
config['image'] = combined_image(job_images + rascal_images, key, problems)
|
|
126
|
+
end
|
|
127
|
+
next if config['hide']
|
|
128
|
+
|
|
129
|
+
# Like GitLab, resolve !reference tags after `extends:`. Only in the keys rascal uses,
|
|
130
|
+
# so a reference into an unreadable include in `script:` does not get in the way.
|
|
131
|
+
resolve = ->(value) { resolve_references(value, key, problems) }
|
|
132
|
+
KEYS_WITH_REFERENCES.each { |config_key| config[config_key] = resolve.call(config[config_key]) if config.key?(config_key) }
|
|
133
|
+
# References inside lists may resolve to lists, which GitLab flattens.
|
|
134
|
+
%w[services before_shell after_shell].each do |list_key|
|
|
135
|
+
next unless config.key?(list_key)
|
|
136
|
+
|
|
137
|
+
value = config[list_key]
|
|
138
|
+
config[list_key] = value.is_a?(Array) ? value.flatten : [value].compact
|
|
139
|
+
end
|
|
140
|
+
# A variable a reference would define might not be needed, or the .rascal block
|
|
141
|
+
# might define it, so that is only worth a warning.
|
|
142
|
+
warnings = [*skipped_includes_warning]
|
|
143
|
+
resolve_variables = ->(value) { resolve_references(value, key, warnings) }
|
|
144
|
+
variable_layers = variable_layers(job_config, rascal_job_config, resolve_variables)
|
|
145
|
+
|
|
146
|
+
Job.new(key, config.fetch('name', key).to_s, config, variable_layers, problems, warnings, skipped_includes_hint)
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# Combines a job's image with the images from rascal's settings, given from the lowest to the
|
|
151
|
+
# highest priority. Each may be a name, the long form `{ name: ..., entrypoint: ... }`, or a
|
|
152
|
+
# `!reference`. A long form without a name changes only parts of the image below it, like
|
|
153
|
+
# `{ entrypoint: [''] }` to skip the entrypoint. A name, or nil for no image, replaces what is
|
|
154
|
+
# below it. Only resolves references in the images it needs, so a job whose image rascal's
|
|
155
|
+
# settings replace does not get problems from its own.
|
|
156
|
+
def combined_image(images, key, problems)
|
|
157
|
+
combined = {}
|
|
158
|
+
images.reverse_each do |image|
|
|
159
|
+
image = resolve_references(image, key, problems)
|
|
160
|
+
image = { 'name' => image } unless image.nil? || image.is_a?(Hash)
|
|
161
|
+
break if image.nil?
|
|
162
|
+
|
|
163
|
+
combined = deep_merge(image, combined)
|
|
164
|
+
break if combined.key?('name')
|
|
165
|
+
end
|
|
166
|
+
combined
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Replaces !reference tags in `value` with what they point to, after `extends:`
|
|
170
|
+
# (see Gitlab::Ci::Config::Yaml::Tags::Resolver). Adds a problem for each one that
|
|
171
|
+
# cannot be resolved.
|
|
172
|
+
def resolve_references(value, job_key, problems, in_progress = [])
|
|
173
|
+
if ConfigFile.reference?(value)
|
|
174
|
+
resolve_reference(value[ConfigFile::REFERENCE_KEY], job_key, problems, in_progress)
|
|
175
|
+
elsif value.is_a?(Hash)
|
|
176
|
+
value.to_h { |key, nested| [key, resolve_references(nested, job_key, problems, in_progress)] }
|
|
177
|
+
elsif value.is_a?(Array)
|
|
178
|
+
value.collect { |nested| resolve_references(nested, job_key, problems, in_progress) }
|
|
179
|
+
else
|
|
180
|
+
value
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
def resolve_reference(location, job_key, problems, in_progress)
|
|
185
|
+
unless location.is_a?(Array) && location.any? && location.none? { |key| key.is_a?(Hash) || key.is_a?(Array) }
|
|
186
|
+
problems << "#{job_key}: !reference needs a list of keys, like `!reference [.template, services]`."
|
|
187
|
+
return
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
location = location.collect(&:to_s)
|
|
191
|
+
description = "!reference [#{location.join(', ')}]"
|
|
192
|
+
if in_progress.include?(location)
|
|
193
|
+
problems << "#{job_key}: #{description} references itself, directly or through other references."
|
|
194
|
+
elsif in_progress.size >= MAX_REFERENCE_DEPTH
|
|
195
|
+
problems << "#{job_key}: #{description} is nested too deeply."
|
|
196
|
+
elsif (target = referenced_value(location)).nil?
|
|
197
|
+
problems << "#{job_key}: #{description} could not be found. #{skipped_includes_hint}" \
|
|
198
|
+
"Set the config in the `.rascal` block (`.rascal: jobs: #{job_key}:`), or hide the job with `hide: true` there."
|
|
199
|
+
else
|
|
200
|
+
return resolve_references(target, job_key, problems, [*in_progress, location])
|
|
201
|
+
end
|
|
202
|
+
nil
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# The value at a location like [.template, services], after `extends:`.
|
|
206
|
+
def referenced_value(location)
|
|
207
|
+
first, *rest = location
|
|
208
|
+
rest.inject(extended_config(first)) { |current, key| current.is_a?(Hash) ? current[key] : nil }
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def extended_config(key)
|
|
212
|
+
@extended_configs ||= {}
|
|
213
|
+
@extended_configs.fetch(key) do
|
|
214
|
+
@extended_configs[key] = begin
|
|
215
|
+
resolve_extends(key)
|
|
216
|
+
rescue Error
|
|
217
|
+
@ci_config[key]
|
|
218
|
+
end
|
|
219
|
+
end
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
# Merges the job's `extends:` templates into it, like GitLab does
|
|
223
|
+
# (see Gitlab::Ci::Config::Extendable::Entry).
|
|
224
|
+
def resolve_extends(key, ancestors = [])
|
|
225
|
+
config = @ci_config[key]
|
|
226
|
+
return config unless config.is_a?(Hash) && config.key?('extends')
|
|
227
|
+
|
|
228
|
+
if ancestors.include?(key)
|
|
229
|
+
raise Error, "#{key}: circular dependency detected in `extends`"
|
|
230
|
+
elsif ancestors.size > MAX_EXTENDS_DEPTH
|
|
231
|
+
raise Error, "#{key}: nesting too deep in `extends`"
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
bases = Array(config['extends']).collect(&:to_s)
|
|
235
|
+
if (unknown = bases.reject { |base| @ci_config.key?(base) }).any?
|
|
236
|
+
raise Error, "#{key}: unknown keys in `extends` (#{unknown.join(', ')}). #{skipped_includes_hint}" \
|
|
237
|
+
"Set the missing config in the `.rascal` block (`.rascal: jobs: #{ancestors.first || key}:`), " \
|
|
238
|
+
"or hide the job with `hide: true` there."
|
|
239
|
+
elsif (invalid = bases.reject { |base| @ci_config[base].is_a?(Hash) }).any?
|
|
240
|
+
raise Error, "#{key}: invalid base hashes in `extends` (#{invalid.join(', ')})"
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
merged = bases.inject({}) do |result, base|
|
|
244
|
+
deep_merge(result, resolve_extends(base, [*ancestors, key]))
|
|
245
|
+
end
|
|
246
|
+
deep_merge(merged, config)
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
# Like GitLab, a key the job sets replaces the same key in `default:` as a whole
|
|
250
|
+
# (see Gitlab::Ci::Config::Entry::Processable#overwrite_entry), and `inherit: default:`
|
|
251
|
+
# limits what a job takes from `default:`.
|
|
252
|
+
def inherit_default(job_config)
|
|
253
|
+
inherited = default_config.slice(*inherited_keys(job_config, 'default', default_config.keys))
|
|
254
|
+
inherited.merge(job_config)
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
def default_config
|
|
258
|
+
@default_config ||= @ci_config.slice(*GLOBAL_DEFAULT_KEYS).merge(hash_or_empty(@ci_config['default']))
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
# The keys a job inherits, according to its `inherit:` setting for `kind`
|
|
262
|
+
# (`default` or `variables`), which may be true, false or a list of keys.
|
|
263
|
+
def inherited_keys(job_config, kind, all_keys)
|
|
264
|
+
case (setting = hash_or_empty(job_config['inherit'])[kind])
|
|
265
|
+
when false then []
|
|
266
|
+
when Array then all_keys & setting.collect(&:to_s)
|
|
267
|
+
else all_keys
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
# Variable definitions from the lowest to the highest priority. A variable that
|
|
272
|
+
# references its own name gets its value from a lower layer.
|
|
273
|
+
#
|
|
274
|
+
# GitLab merges global and job variables by name before expanding them (see
|
|
275
|
+
# Gitlab::Ci::Variables::Helpers.merge_variables), so they form one layer.
|
|
276
|
+
# The .rascal block adds its own layers on top, so it can extend a job's
|
|
277
|
+
# variables, like `RUBYOPT: $RUBYOPT -W:deprecated`, or remove one with `CI: null`.
|
|
278
|
+
# In GitLab's own variables, null means an empty string.
|
|
279
|
+
def variable_layers(job_config, rascal_job_config, resolve)
|
|
280
|
+
global_variables = hash_or_empty(resolve.call(@ci_config['variables']))
|
|
281
|
+
global_variables = global_variables.slice(*inherited_keys(job_config, 'variables', global_variables.keys))
|
|
282
|
+
default_variables = hash_or_empty(resolve.call(hash_or_empty(@ci_config['default'])['variables'])) # not supported by GitLab, kept for compatibility
|
|
283
|
+
[
|
|
284
|
+
Variables.null_as_empty(global_variables.merge(default_variables).merge(hash_or_empty(resolve.call(job_config['variables'])))),
|
|
285
|
+
hash_or_empty(resolve.call(@rascal_config['variables'])),
|
|
286
|
+
hash_or_empty(resolve.call(rascal_job_config['variables'])),
|
|
287
|
+
]
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
def deep_merge(*hashes)
|
|
291
|
+
hashes.inject { |result, hash| ConfigFile.deep_merge(hash_or_empty(result), hash_or_empty(hash)) }
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
def hash_or_empty(value)
|
|
295
|
+
value.is_a?(Hash) ? value : {}
|
|
296
|
+
end
|
|
297
|
+
end
|
|
298
|
+
end
|
|
299
|
+
end
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
module Rascal
|
|
2
|
+
module Gitlab
|
|
3
|
+
# Resolves CI variables the way a GitLab job sees them:
|
|
4
|
+
#
|
|
5
|
+
# - Definitions come in layers. Later layers override earlier ones.
|
|
6
|
+
# - A value may reference other variables as `$NAME`, `${NAME}` or `%NAME%`.
|
|
7
|
+
# - A value that references its own name gets the definition from the layer below,
|
|
8
|
+
# so the .rascal block can extend a job's variable with `RUBYOPT: $RUBYOPT -W:deprecated`.
|
|
9
|
+
# - `$NAME` and `${NAME}` expand to an empty string if NAME is undefined.
|
|
10
|
+
# `%NAME%` stays as it is, so strings like `%Y%m%d` survive.
|
|
11
|
+
# - `$$` stands for a literal `$`.
|
|
12
|
+
# - A variable defined as `{ value: ..., expand: false }` is used as is.
|
|
13
|
+
# - Variables that reference each other in a circle are an error.
|
|
14
|
+
# - A variable defined as nil is removed, as if no lower layer defined it. This is
|
|
15
|
+
# rascal's own extension, for removing predefined variables like `CI`.
|
|
16
|
+
#
|
|
17
|
+
# GitLab expands in two steps: the GitLab server expands references to defined variables
|
|
18
|
+
# (see Gitlab::Ci::Variables::Collection#sort_and_expand_all), then the runner expands
|
|
19
|
+
# the rest and turns `$$` into `$`. This class does both in one step.
|
|
20
|
+
class Variables
|
|
21
|
+
NAME = /[a-zA-Z_][a-zA-Z0-9_]*/
|
|
22
|
+
REFERENCE = /\$\$|%%|\$(?<name>#{NAME})|\$\{(?<name>#{NAME})\}|%(?<percent_name>#{NAME})%/
|
|
23
|
+
|
|
24
|
+
Definition = Struct.new(:value, :expand)
|
|
25
|
+
|
|
26
|
+
# Variable definitions as GitLab reads them, where null means an empty string. (In
|
|
27
|
+
# rascal's own settings, null removes a variable.)
|
|
28
|
+
def self.null_as_empty(definitions)
|
|
29
|
+
definitions.transform_values { |value| value.nil? ? '' : value }
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Takes layers of variable definitions as hashes, from the lowest to the highest priority.
|
|
33
|
+
def initialize(*layers)
|
|
34
|
+
# For each name, all its definitions from the lowest to the highest layer.
|
|
35
|
+
@definitions = Hash.new { |hash, name| hash[name] = [] }
|
|
36
|
+
layers.each { |layer| add(layer) }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Returns a new instance with another layer of definitions on top.
|
|
40
|
+
def merge(definitions)
|
|
41
|
+
merged = self.class.new
|
|
42
|
+
@definitions.each { |name, layers| merged.layers(name).concat(layers) }
|
|
43
|
+
merged.add(definitions)
|
|
44
|
+
merged
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# All variables, with their references expanded.
|
|
48
|
+
def to_h
|
|
49
|
+
defined_names.to_h { |name| [name, resolve(name)] }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# The variables whose values directly reference undefined variables, as a hash
|
|
53
|
+
# from each variable's name to the names of the undefined ones. A variable that
|
|
54
|
+
# references its own name without a lower definition does not count, since
|
|
55
|
+
# appending to an unset variable is common.
|
|
56
|
+
def undefined_references
|
|
57
|
+
defined_names.each_with_object({}) do |name, result|
|
|
58
|
+
layer = @definitions[name].size - 1
|
|
59
|
+
definition = @definitions[name][layer]
|
|
60
|
+
next unless definition.expand
|
|
61
|
+
|
|
62
|
+
undefined = []
|
|
63
|
+
expand_with(definition.value, [name, layer], [[name, layer]], direct_only: true) { |undefined_name| undefined << undefined_name }
|
|
64
|
+
result[name] = undefined.uniq if undefined.any?
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Expands variable references in the given string. Yields the name of each
|
|
69
|
+
# undefined variable it encounters.
|
|
70
|
+
def expand(string, &on_undefined)
|
|
71
|
+
expand_with(string.to_s, nil, [], &on_undefined)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
protected
|
|
75
|
+
|
|
76
|
+
def layers(name)
|
|
77
|
+
@definitions[name]
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def add(definitions)
|
|
81
|
+
(definitions || {}).each do |name, value|
|
|
82
|
+
@definitions[name.to_s] << normalize(value)
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
private
|
|
87
|
+
|
|
88
|
+
def defined_names
|
|
89
|
+
@definitions.keys.reject { |name| @definitions[name].last.value.nil? }
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def normalize(value)
|
|
93
|
+
if value.nil?
|
|
94
|
+
Definition.new(nil, true)
|
|
95
|
+
elsif value.is_a?(Hash)
|
|
96
|
+
Definition.new(value['value'].to_s, value.fetch('expand', true))
|
|
97
|
+
else
|
|
98
|
+
Definition.new(value.to_s, true)
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Resolves the given layer of a variable, the latest one by default.
|
|
103
|
+
def resolve(name, layer = @definitions[name].size - 1, visiting = [], &on_undefined)
|
|
104
|
+
definition = @definitions[name][layer]
|
|
105
|
+
if !definition.expand
|
|
106
|
+
definition.value
|
|
107
|
+
elsif visiting.include?([name, layer])
|
|
108
|
+
circle = [*visiting.drop_while { |entry| entry != [name, layer] }, [name, layer]].collect(&:first)
|
|
109
|
+
raise Error, "The variables #{circle.join(' -> ')} reference each other in a circle. GitLab rejects this as well."
|
|
110
|
+
else
|
|
111
|
+
expand_with(definition.value, [name, layer], [*visiting, [name, layer]], &on_undefined)
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# `current` is the variable whose value is being expanded, as [name, layer].
|
|
116
|
+
# With `direct_only`, only undefined variables in `string` itself are yielded, not
|
|
117
|
+
# those in the values of variables it references.
|
|
118
|
+
def expand_with(string, current, visiting, direct_only: false, &on_undefined)
|
|
119
|
+
string.gsub(REFERENCE) do
|
|
120
|
+
match = Regexp.last_match
|
|
121
|
+
if match[0] == '$$'
|
|
122
|
+
'$'
|
|
123
|
+
elsif match[0] == '%%'
|
|
124
|
+
'%%'
|
|
125
|
+
else
|
|
126
|
+
name = match[:name] || match[:percent_name]
|
|
127
|
+
layer = referenced_layer(name, current)
|
|
128
|
+
if layer
|
|
129
|
+
direct_only ? resolve(name, layer, visiting) : resolve(name, layer, visiting, &on_undefined)
|
|
130
|
+
elsif match[:name]
|
|
131
|
+
# Not for variables that are only defined lower or removed on purpose.
|
|
132
|
+
on_undefined&.call(name) unless @definitions.key?(name)
|
|
133
|
+
''
|
|
134
|
+
else
|
|
135
|
+
match[0]
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# The layer a reference to `name` points to, or nil if it is undefined or removed.
|
|
142
|
+
def referenced_layer(name, current)
|
|
143
|
+
return unless @definitions.key?(name)
|
|
144
|
+
|
|
145
|
+
layer = if current && current.first == name
|
|
146
|
+
# A variable referencing itself means its definition in a lower layer.
|
|
147
|
+
current.last.positive? ? current.last - 1 : nil
|
|
148
|
+
else
|
|
149
|
+
@definitions[name].size - 1
|
|
150
|
+
end
|
|
151
|
+
layer unless layer.nil? || @definitions[name][layer].value.nil?
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
module Rascal
|
|
2
|
+
# Turns a project's .gitlab-ci.yml into environments:
|
|
3
|
+
#
|
|
4
|
+
# - ConfigFile reads the YAML with its local includes.
|
|
5
|
+
# - Jobs resolves each job like GitLab does (`extends:`, `default:`, `!reference`), and adds
|
|
6
|
+
# rascal's settings from the `.rascal` block and `.rascal.yml`.
|
|
7
|
+
# - EnvironmentBuilder turns a resolved Job into an Environment with Docker names, volumes and
|
|
8
|
+
# services, expanding variables with Variables.
|
|
9
|
+
module Gitlab
|
|
10
|
+
autoload :ConfigFile, 'rascal/gitlab/config_file'
|
|
11
|
+
autoload :EnvironmentBuilder, 'rascal/gitlab/environment_builder'
|
|
12
|
+
autoload :Job, 'rascal/gitlab/jobs'
|
|
13
|
+
autoload :Jobs, 'rascal/gitlab/jobs'
|
|
14
|
+
autoload :Variables, 'rascal/gitlab/variables'
|
|
15
|
+
end
|
|
16
|
+
end
|
data/lib/rascal/io_helper.rb
CHANGED
|
@@ -15,6 +15,17 @@ module Rascal
|
|
|
15
15
|
stdout.puts(message)
|
|
16
16
|
end
|
|
17
17
|
|
|
18
|
+
# Reports what rascal does, like starting a service. Goes to stderr, so the
|
|
19
|
+
# output of a command stays clean for whoever reads it.
|
|
20
|
+
def info(message)
|
|
21
|
+
stderr.puts(message)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Reports something that may make commands fail, on stderr.
|
|
25
|
+
def warning(message)
|
|
26
|
+
stderr.puts("Warning: #{message}")
|
|
27
|
+
end
|
|
28
|
+
|
|
18
29
|
def stdout
|
|
19
30
|
IOHelper.stdout
|
|
20
31
|
end
|
data/lib/rascal/service.rb
CHANGED
|
@@ -1,35 +1,78 @@
|
|
|
1
1
|
module Rascal
|
|
2
2
|
class Service
|
|
3
|
-
attr_reader :name, :container, :
|
|
3
|
+
attr_reader :name, :image, :container, :aliases, :env_variables, :command
|
|
4
4
|
|
|
5
|
-
def initialize(name, env_variables: {}, image:,
|
|
5
|
+
def initialize(name, env_variables: {}, image:, aliases:, volumes: [], command: nil, healthcheck_port: nil)
|
|
6
6
|
@name = name
|
|
7
|
-
@
|
|
8
|
-
@
|
|
7
|
+
@image = Docker::Image.new(image)
|
|
8
|
+
@container = Docker::Container.new(name, @image)
|
|
9
|
+
@aliases = aliases
|
|
10
|
+
@healthcheck_port = healthcheck_port
|
|
9
11
|
@volumes = volumes
|
|
10
12
|
@env_variables = env_variables
|
|
11
13
|
@command = [*command]
|
|
12
14
|
end
|
|
13
15
|
|
|
16
|
+
# The TCP ports rascal waits for before running commands, of which one must accept
|
|
17
|
+
# connections. Like the GitLab runner, takes HEALTHCHECK_TCP_PORT from the service's
|
|
18
|
+
# own variables, or else the ports the image exposes.
|
|
19
|
+
def ports
|
|
20
|
+
if (port = @healthcheck_port.to_s[/\A\d+\z/])
|
|
21
|
+
[port.to_i]
|
|
22
|
+
else
|
|
23
|
+
@image.exposed_tcp_ports
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
14
27
|
def download_missing
|
|
15
|
-
@
|
|
28
|
+
@image.download_missing(user: @name)
|
|
16
29
|
end
|
|
17
30
|
|
|
18
|
-
|
|
19
|
-
|
|
31
|
+
# Returns whether it started the service.
|
|
32
|
+
def start_if_stopped(network:)
|
|
33
|
+
if @container.running?
|
|
34
|
+
false
|
|
35
|
+
else
|
|
36
|
+
@container.remove_if_network_missing(network)
|
|
20
37
|
if (container_id = @container.id)
|
|
21
38
|
network.disconnect(container_id)
|
|
22
39
|
end
|
|
23
|
-
@container.start(network: network,
|
|
40
|
+
@container.start(network: network, network_aliases: @aliases, volumes: @volumes, env: @env_variables, command: @command)
|
|
41
|
+
true
|
|
24
42
|
end
|
|
25
43
|
end
|
|
26
44
|
|
|
45
|
+
def running?
|
|
46
|
+
@container.running?
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Whether the service is running, but started so recently that it may not be ready yet.
|
|
50
|
+
def starting?(seconds)
|
|
51
|
+
started_at = @container.started_at
|
|
52
|
+
running? && started_at && started_at > Time.now - seconds
|
|
53
|
+
end
|
|
54
|
+
|
|
27
55
|
def clean
|
|
28
56
|
@container.clean
|
|
29
57
|
end
|
|
30
58
|
|
|
59
|
+
# For `rascal status`.
|
|
60
|
+
def status
|
|
61
|
+
{
|
|
62
|
+
'aliases' => @aliases,
|
|
63
|
+
'image' => @image.name,
|
|
64
|
+
'image_present' => @image.present?,
|
|
65
|
+
'command' => @command,
|
|
66
|
+
'variables' => @env_variables,
|
|
67
|
+
# Unknown until the image is downloaded.
|
|
68
|
+
'wait_ports' => (ports if @image.present? || @healthcheck_port),
|
|
69
|
+
**@container.status,
|
|
70
|
+
'logs' => "docker logs #{@container.prefixed_name}",
|
|
71
|
+
}
|
|
72
|
+
end
|
|
73
|
+
|
|
31
74
|
def update(**args)
|
|
32
|
-
@
|
|
75
|
+
@image.update(**args)
|
|
33
76
|
end
|
|
34
77
|
end
|
|
35
78
|
end
|
data/lib/rascal/version.rb
CHANGED
data/lib/rascal.rb
CHANGED
|
@@ -3,9 +3,10 @@ require "rascal/version"
|
|
|
3
3
|
module Rascal
|
|
4
4
|
class Error < StandardError; end
|
|
5
5
|
|
|
6
|
+
autoload :CommandScript, 'rascal/command_script'
|
|
6
7
|
autoload :Docker, 'rascal/docker'
|
|
7
8
|
autoload :Environment, 'rascal/environment'
|
|
8
|
-
autoload :
|
|
9
|
+
autoload :Gitlab, 'rascal/gitlab'
|
|
9
10
|
autoload :IOHelper, 'rascal/io_helper'
|
|
10
11
|
autoload :Service, 'rascal/service'
|
|
11
12
|
end
|
data/rascal.gemspec
CHANGED
|
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
|
|
|
18
18
|
if spec.respond_to?(:metadata)
|
|
19
19
|
spec.metadata["homepage_uri"] = spec.homepage
|
|
20
20
|
spec.metadata["source_code_uri"] = "https://github.com/makandra/rascal"
|
|
21
|
-
spec.metadata["changelog_uri"] = "https://github.com/makandra/rascal/blob/
|
|
21
|
+
spec.metadata["changelog_uri"] = "https://github.com/makandra/rascal/blob/main/CHANGELOG.md"
|
|
22
22
|
spec.metadata["rubygems_mfa_required"] = 'true'
|
|
23
23
|
else
|
|
24
24
|
raise "RubyGems 2.0 or newer is required to protect against " \
|
|
@@ -28,11 +28,12 @@ Gem::Specification.new do |spec|
|
|
|
28
28
|
# Specify which files should be added to the gem when it is released.
|
|
29
29
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
30
30
|
spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
|
|
31
|
-
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec
|
|
31
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec)/}) }
|
|
32
32
|
end
|
|
33
33
|
spec.bindir = "exe"
|
|
34
34
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
35
35
|
spec.require_paths = ["lib"]
|
|
36
|
+
spec.required_ruby_version = ">= 3.3"
|
|
36
37
|
|
|
37
38
|
spec.add_dependency "thor", ">= 1.0.0"
|
|
38
39
|
end
|