rascal 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +4 -4
  3. data/.ruby-version +1 -1
  4. data/AGENTS.md +4 -0
  5. data/CHANGELOG.md +74 -0
  6. data/CONTRIBUTING.md +139 -0
  7. data/Gemfile.common +0 -3
  8. data/Gemfile.lock +1 -47
  9. data/Guardfile +0 -9
  10. data/README.md +679 -76
  11. data/Rakefile +1 -7
  12. data/lib/rascal/cli/base.rb +41 -20
  13. data/lib/rascal/cli/clean.rb +3 -8
  14. data/lib/rascal/cli/environments.rb +1 -3
  15. data/lib/rascal/cli/main.rb +106 -17
  16. data/lib/rascal/cli/run.rb +6 -7
  17. data/lib/rascal/cli/shell.rb +3 -3
  18. data/lib/rascal/cli/status.rb +81 -0
  19. data/lib/rascal/cli/update.rb +8 -8
  20. data/lib/rascal/cli.rb +1 -0
  21. data/lib/rascal/command_script.rb +83 -0
  22. data/lib/rascal/docker/command_containers.rb +101 -0
  23. data/lib/rascal/docker/container.rb +82 -101
  24. data/lib/rascal/docker/image.rb +71 -0
  25. data/lib/rascal/docker/interface.rb +4 -2
  26. data/lib/rascal/docker/network.rb +12 -0
  27. data/lib/rascal/docker/volume.rb +17 -10
  28. data/lib/rascal/docker.rb +11 -4
  29. data/lib/rascal/environment.rb +164 -40
  30. data/lib/rascal/gitlab/config_file.rb +199 -0
  31. data/lib/rascal/gitlab/environment_builder.rb +204 -0
  32. data/lib/rascal/gitlab/jobs.rb +299 -0
  33. data/lib/rascal/gitlab/variables.rb +155 -0
  34. data/lib/rascal/gitlab.rb +16 -0
  35. data/lib/rascal/io_helper.rb +11 -0
  36. data/lib/rascal/service.rb +52 -9
  37. data/lib/rascal/version.rb +1 -1
  38. data/lib/rascal.rb +2 -1
  39. data/rascal.gemspec +3 -2
  40. metadata +15 -8
  41. data/bin/cucumber +0 -29
  42. data/cucumber.yml +0 -1
  43. data/lib/rascal/environments_definition/gitlab.rb +0 -146
  44. data/lib/rascal/environments_definition.rb +0 -23
@@ -1,115 +1,91 @@
1
+ require 'time'
2
+
1
3
  module Rascal
2
4
  module Docker
5
+ # A long-lived container that runs a service, like a database, under a fixed name.
3
6
  class Container
4
7
  include IOHelper
5
8
 
6
- attr_reader :image
9
+ attr_reader :prefixed_name
7
10
 
11
+ # `image` is a Docker::Image.
8
12
  def initialize(name, image)
9
13
  @name = name
10
14
  @prefixed_name = "#{NAME_PREFIX}#{name}"
11
15
  @image = image
12
16
  end
13
17
 
14
- def download_missing
15
- unless image_exists?
16
- say "Downloading image for #{@name}"
17
- Docker.interface.run(
18
- 'pull',
19
- @image,
20
- stdout: stdout,
21
- )
22
- end
18
+ # When the container was last started, or nil.
19
+ def started_at
20
+ started_at = inspection&.dig('State', 'StartedAt')
21
+ Time.parse(started_at) if started_at && !started_at.start_with?('0001-')
23
22
  end
24
23
 
25
24
  def running?
26
- if id
27
- container_info = Docker.interface.run(
28
- 'container',
29
- 'inspect',
30
- id,
31
- output: :json,
32
- ).first
33
- !!container_info.dig('State', 'Running')
34
- else
35
- false
36
- end
25
+ !!inspection&.dig('State', 'Running')
37
26
  end
38
27
 
39
- def exists?
40
- !!id
28
+ # For `rascal status`: the container's name, and its state like "running", "exited"
29
+ # or "missing", with when it started and how it exited.
30
+ def status
31
+ state = inspection&.dig('State')
32
+ return { 'container' => @prefixed_name, 'state' => 'missing' } unless state
33
+
34
+ status = state['Status'] || (state['Running'] ? 'running' : 'exited')
35
+ {
36
+ 'container' => @prefixed_name,
37
+ 'state' => status,
38
+ 'started_at' => started_at&.iso8601,
39
+ 'exit_code' => (state['ExitCode'] if %w[exited dead].include?(status)),
40
+ # Why Docker could not start it, like a missing command.
41
+ 'error' => (state['Error'] unless state['Error'].to_s.empty?),
42
+ }.compact
41
43
  end
42
44
 
43
- def start(network: nil, network_alias: nil, volumes: [], env: {}, command: [])
44
- say "Starting container for #{@name}"
45
- create(network: network, network_alias: network_alias, volumes: volumes, env: env, command: command) unless exists?
46
- Docker.interface.run(
47
- 'container',
48
- 'start',
49
- id,
50
- )
45
+ def exists?
46
+ !!id
51
47
  end
52
48
 
53
- def create(network: nil, network_alias: nil, volumes: [], env: {}, command: [])
54
- @id = Docker.interface.run(
55
- 'container',
56
- 'create',
57
- '--name', @prefixed_name,
58
- *(volumes.flat_map { |v| ['-v', v.to_param] }),
59
- *env_args(env),
60
- *(['--network', network.id] if network),
61
- *(['--network-alias', network_alias] if network_alias),
62
- @image,
63
- *command,
64
- output: :id,
65
- )
49
+ # Docker resolves a container's network by the id it recorded when the
50
+ # container was created, and refuses to start it once that network is
51
+ # gone:
52
+ #
53
+ # failed to set up container networking: network 36243f82141a... not found
54
+ #
55
+ # `docker network prune` (and `docker system prune`) leaves exactly that
56
+ # behind, because it removes networks whose containers are all stopped —
57
+ # the normal state of an environment between runs. Reconnecting does not
58
+ # help, since docker resolves the recorded id before it looks at the
59
+ # container's current endpoints, so the container has to go. Nothing in a
60
+ # service container is worth preserving; #start creates a new one.
61
+ def remove_if_network_missing(network)
62
+ return unless exists?
63
+ attached = attached_network_ids
64
+ return if attached.empty? || attached.include?(network.id)
65
+
66
+ info "Removing container for #{@name}, its network no longer exists"
67
+ remove_container
68
+ @id = nil
66
69
  end
67
70
 
68
- def run_and_attach(*command, env: {}, network: nil, volumes: [], working_dir: nil, allow_failure: false, tty: true)
69
- Docker.interface.run_and_attach(
70
- 'container',
71
- 'run',
72
- '--rm',
73
- '-a', 'STDOUT',
74
- '-a', 'STDERR',
75
- *(['-a', 'STDIN', '--interactive', '--tty'] if tty),
76
- *(['-w', working_dir] if working_dir),
77
- *(volumes.flat_map { |v| ['-v', v.to_param] }),
78
- *env_args(env),
79
- *(['--network', network.id] if network),
80
- @image,
81
- *command,
82
- redirect_io: {
83
- out: stdout,
84
- err: stderr,
85
- in: stdin,
86
- },
87
- allow_failure: allow_failure,
88
- )
71
+ # Creates the container if it does not exist, and starts it.
72
+ def start(network:, network_aliases: [], volumes: [], env: {}, command: [])
73
+ info "Starting container for #{@name}"
74
+ create(network: network, network_aliases: network_aliases, volumes: volumes, env: env, command: command) unless exists?
75
+ Docker.interface.run('container', 'start', id)
89
76
  end
90
77
 
91
78
  def clean
92
79
  if running?
93
- say "Stopping container for #{@name}"
94
- stop_container
80
+ info "Stopping container for #{@name}"
81
+ Docker.interface.run('container', 'stop', id)
95
82
  end
96
83
  if exists?
97
- say "Removing container for #{@name}"
84
+ info "Removing container for #{@name}"
98
85
  remove_container
99
86
  end
100
87
  end
101
88
 
102
- def update(skip: [])
103
- return if skip.include?(@image)
104
- say "Updating image #{@image}"
105
- Docker.interface.run(
106
- 'pull',
107
- @image,
108
- stdout: stdout,
109
- )
110
- @image
111
- end
112
-
113
89
  def id
114
90
  @id ||= Docker.interface.run(
115
91
  'container',
@@ -123,34 +99,39 @@ module Rascal
123
99
 
124
100
  private
125
101
 
126
- def image_exists?
127
- Docker.interface.run(
128
- 'image',
129
- 'inspect',
130
- @image,
131
- output: :json,
132
- allow_failure: true,
133
- ).first
134
- end
135
-
136
- def stop_container
137
- Docker.interface.run(
102
+ def create(network:, network_aliases:, volumes:, env:, command:)
103
+ @id = Docker.interface.run(
138
104
  'container',
139
- 'stop',
140
- id,
105
+ 'create',
106
+ '--name', @prefixed_name,
107
+ *volumes.flat_map { |volume| ['-v', volume.to_param] },
108
+ *Docker.env_args(env),
109
+ '--network', network.id,
110
+ *network_aliases.flat_map { |network_alias| ['--network-alias', network_alias] },
111
+ @image.name,
112
+ *command,
113
+ output: :id,
141
114
  )
142
115
  end
143
116
 
144
- def remove_container
145
- Docker.interface.run(
146
- 'container',
147
- 'rm',
148
- id,
149
- )
117
+ # A fresh `docker container inspect`, or nil if there is no container.
118
+ def inspection
119
+ return unless id
120
+
121
+ Docker.interface.run('container', 'inspect', id, output: :json).first
122
+ end
123
+
124
+ # Ids of the networks the container is attached to. Empty for a container
125
+ # that was created but never started.
126
+ def attached_network_ids
127
+ networks = inspection&.dig('NetworkSettings', 'Networks') || {}
128
+ networks.each_value.filter_map { |n| n['NetworkID'] }.reject(&:empty?).uniq
150
129
  end
151
130
 
152
- def env_args(env)
153
- env.flat_map { |key, value| ['-e', "#{key}=#{value}"] }
131
+ # Also removes the container's anonymous volumes, like the data volume that
132
+ # the postgres and redis images declare. Named volumes are kept.
133
+ def remove_container
134
+ Docker.interface.run('container', 'rm', '--volumes', id)
154
135
  end
155
136
  end
156
137
  end
@@ -0,0 +1,71 @@
1
+ module Rascal
2
+ module Docker
3
+ # An image, as named in the CI config. Downloads it, and reads what rascal needs from it.
4
+ class Image
5
+ include IOHelper
6
+
7
+ attr_reader :name
8
+
9
+ def initialize(name)
10
+ @name = name
11
+ end
12
+
13
+ def present?
14
+ !@name.to_s.empty? && !inspection.nil?
15
+ end
16
+
17
+ # `user` names what needs the image in the message, like `myapp-rspec_db`.
18
+ def download_missing(user:)
19
+ return if present?
20
+
21
+ info "Downloading image for #{user}"
22
+ pull
23
+ end
24
+
25
+ # Pulls the latest version, unless `skip` has it already. Returns the name of the image.
26
+ def update(skip: [])
27
+ return if skip.include?(@name)
28
+
29
+ info "Updating image #{@name}"
30
+ pull
31
+ @name
32
+ end
33
+
34
+ # The TCP ports the image declares with EXPOSE, like [5432] for postgres.
35
+ def exposed_tcp_ports
36
+ ports = inspection&.dig('Config', 'ExposedPorts') || {}
37
+ ports.keys.filter_map { |port| port[%r{\A(\d+)/tcp\z}, 1]&.to_i }
38
+ end
39
+
40
+ # The image's own entrypoint, like ["sh", "-c"], or nil.
41
+ def entrypoint
42
+ inspection&.dig('Config', 'Entrypoint')
43
+ end
44
+
45
+ private
46
+
47
+ # A fresh `docker image inspect`, or nil if the image is not there.
48
+ def inspection
49
+ Docker.interface.run('image', 'inspect', @name, output: :json, allow_failure: true).first
50
+ end
51
+
52
+ def pull
53
+ Docker.interface.run('pull', @name, stdout: stderr)
54
+ rescue Interface::Error => e
55
+ case e.message
56
+ when /docker daemon|docker\.sock/i
57
+ # Like no permission to use Docker. Docker's message says enough.
58
+ raise
59
+ when /unauthorized|authentication required|denied|no basic auth credentials/i
60
+ registry = @name.include?('/') && @name.split('/').first.match?(/[.:]|\Alocalhost\z/) ? @name.split('/').first : 'docker.io'
61
+ raise Error, "#{e.message.strip}\nDocker may need credentials for #{registry}. Log in with `docker login #{registry}`, then try again."
62
+ when /manifest unknown|name unknown|failed to resolve reference.*: not found/i
63
+ raise Error, "#{e.message.strip}\nDocker found no image #{@name} in its registry. Its tag may have been deleted, or the name is wrong. " \
64
+ 'Set another image in the .rascal block (`jobs: <job>: image:` or `services:`), or build the image yourself under this name.'
65
+ else
66
+ raise
67
+ end
68
+ end
69
+ end
70
+ end
71
+ end
@@ -25,8 +25,8 @@ module Rascal
25
25
  parse_output(output, save_stdout, save_stderr, command: command.join(' '))
26
26
  end
27
27
 
28
- def run_and_attach(*command, stdout: nil, stderr: nil, stdin: nil, env: {}, network: nil, volumes: [], working_dir: nil, redirect_io: {}, allow_failure: false)
29
- exit_status = spawn(env, 'docker', *stringify_command(command), redirect_io)
28
+ def run_and_attach(*command, redirect_io: {}, allow_failure: false)
29
+ exit_status = spawn('docker', *stringify_command(command), redirect_io)
30
30
  unless allow_failure || exit_status.success?
31
31
  raise Error, "docker container run failed"
32
32
  end
@@ -69,6 +69,8 @@ module Rascal
69
69
  end
70
70
  when :id
71
71
  stdout[/[0-9a-f]+/]
72
+ when :text
73
+ stdout.strip
72
74
  when :ignore
73
75
  nil
74
76
  else
@@ -1,6 +1,10 @@
1
1
  module Rascal
2
2
  module Docker
3
3
  class Network
4
+ include IOHelper
5
+
6
+ attr_reader :prefixed_name
7
+
4
8
  def initialize(name)
5
9
  @name = name
6
10
  @prefixed_name = "#{NAME_PREFIX}#{name}"
@@ -12,6 +16,9 @@ module Rascal
12
16
  'create',
13
17
  @prefixed_name,
14
18
  )
19
+ rescue Interface::Error => e
20
+ # Another rascal command for the same environment was faster.
21
+ raise unless e.message.include?('already exists')
15
22
  end
16
23
 
17
24
  def disconnect(container_id)
@@ -31,6 +38,7 @@ module Rascal
31
38
 
32
39
  def clean
33
40
  if exists?
41
+ info "Removing network #{@prefixed_name}"
34
42
  Docker.interface.run(
35
43
  'network',
36
44
  'rm',
@@ -39,11 +47,15 @@ module Rascal
39
47
  end
40
48
  end
41
49
 
50
+ # The full id, not the short one docker prints by default: a container
51
+ # records the network it is attached to by full id, and
52
+ # Container#remove_if_network_missing compares the two.
42
53
  def id
43
54
  @id ||= Docker.interface.run(
44
55
  'network',
45
56
  'ls',
46
57
  '--quiet',
58
+ '--no-trunc',
47
59
  '--filter', "name=^#{@prefixed_name}$",
48
60
  output: :id,
49
61
  )
@@ -1,11 +1,9 @@
1
1
  module Rascal
2
2
  module Docker
3
3
  module Volume
4
- class Base
4
+ class Named
5
5
  include IOHelper
6
- end
7
6
 
8
- class Named < Base
9
7
  def initialize(name, container_path)
10
8
  @prefixed_name = "#{NAME_PREFIX}#{name}"
11
9
  @container_path = container_path
@@ -15,9 +13,13 @@ module Rascal
15
13
  "#{@prefixed_name}:#{@container_path}"
16
14
  end
17
15
 
16
+ def status
17
+ { 'path' => @container_path, 'volume' => @prefixed_name, 'exists' => exists? }
18
+ end
19
+
18
20
  def clean
19
21
  if exists?
20
- say "Removing volume #{@prefixed_name}"
22
+ info "Removing volume #{@prefixed_name}"
21
23
  Docker.interface.run(
22
24
  'volume',
23
25
  'rm',
@@ -27,24 +29,29 @@ module Rascal
27
29
  end
28
30
 
29
31
  def exists?
30
- Docker.interface.run(
32
+ !Docker.interface.run(
31
33
  'volume',
32
34
  'ls',
33
35
  '--quiet',
34
36
  '--filter', "name=^#{@prefixed_name}$",
35
- output: :id,
36
- )
37
+ output: :text,
38
+ ).to_s.empty?
37
39
  end
38
40
  end
39
41
 
40
- class Bind < Base
41
- def initialize(local_path, container_path)
42
+ class Bind
43
+ def initialize(local_path, container_path, read_only: false)
42
44
  @local_path = local_path
43
45
  @container_path = container_path
46
+ @read_only = read_only
44
47
  end
45
48
 
46
49
  def to_param
47
- "#{@local_path}:#{@container_path}"
50
+ "#{@local_path}:#{@container_path}#{':ro' if @read_only}"
51
+ end
52
+
53
+ def status
54
+ { 'path' => @container_path, 'directory' => @local_path.to_s }
48
55
  end
49
56
 
50
57
  def clean
data/lib/rascal/docker.rb CHANGED
@@ -2,10 +2,12 @@ module Rascal
2
2
  module Docker
3
3
  NAME_PREFIX = 'rascal-'
4
4
 
5
- autoload :Container, 'rascal/docker/container'
6
- autoload :Interface, 'rascal/docker/interface'
7
- autoload :Network, 'rascal/docker/network'
8
- autoload :Volume, 'rascal/docker/volume'
5
+ autoload :CommandContainers, 'rascal/docker/command_containers'
6
+ autoload :Container, 'rascal/docker/container'
7
+ autoload :Image, 'rascal/docker/image'
8
+ autoload :Interface, 'rascal/docker/interface'
9
+ autoload :Network, 'rascal/docker/network'
10
+ autoload :Volume, 'rascal/docker/volume'
9
11
 
10
12
  class << self
11
13
  attr_writer :interface
@@ -13,6 +15,11 @@ module Rascal
13
15
  def interface
14
16
  @interface ||= Interface.new
15
17
  end
18
+
19
+ # Arguments for `docker container create` or `run` that set environment variables.
20
+ def env_args(env)
21
+ env.flat_map { |key, value| ['-e', "#{key}=#{value}"] }
22
+ end
16
23
  end
17
24
  end
18
25
  end