moby-derp 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 99faf7a4a2ccef6a30df32e08d424bf89c5693278100484a75e21276136e6a3a
-  data.tar.gz: 379c96e91a43e3b19db58d0e61d1324749969f58a2946a72f687429cda959f34
+  metadata.gz: 513a47c0b56a264731bf12050cbc651c7799c4ec60955c06e8cff46377328825
+  data.tar.gz: 01d2f26f2fa6300cdddf599b9f544616d1781c4a53879ec009bec1d4dd340b9d
 SHA512:
-  metadata.gz: 6d7b1071d6a38f8aa880ad7287c7355460a32dc4a66250e13bcec612d1f11ee91530d2cfedbba0bb4e8bdf54f5784fbaac00f807a46a967a35ddf45193feb6a6
-  data.tar.gz: ef1b7425e29551844fe383159a19010cc611ffc8dc07566f4c7e30afa4d3d02882692a3beaed35d6490263d51e093decd662e59b9e0169f177f2c221eb9079c4
+  metadata.gz: 307556e56b26e97437df00b9db1277f254554ba88af5a22d9b786de2daba8a30d91f5c932c3b28f4cebb972175958c7c1be81d3cd8ad718a23638d8f696aaa12
+  data.tar.gz: eb0c7f7f7514dbb93911cc65037a129aadfc1888454529071f2811e606dd3291dc57e5f2ced377df3896bbaccbac22c163bd5fad4e90cc0435304bb32b65ef6a

data/example.yml

@@ -213,6 +213,33 @@ containers:
       ulimit-rttime: 15:16
       ulimit-stack: 17:18
 
+    # If you're a bit suss as to whether or not one of your containers will
+    # successfully start up, you can use the following section to define a
+    # start-time health checking regime.
+    #
+    # How it works is that the defined command is run in the container (using
+    # `exec`), and if-and-when that command returns a `0` exit status, the
+    # container is considered to be healthy and we're done.  If the command
+    # returns a non-zero exit status, we wait for `interval` seconds and then
+    # retry.  If the command executes `attempts` times without receiving a `0`
+    # exit status, the container is considered "failed", and no further
+    # containers in the pod will be processed, and the `moby-derp` execution
+    # will itself exit with a non-zero status.
+    startup_health_check:
+      # The command to run inside the container via `exec`.  You can specify
+      # this as a string, or as an array of strings if you prefer to avoid
+      # shell quoting hell.
+      command: '/usr/local/bin/r-u-ok'
+
+      # How many seconds to wait between invocations of the command, when
+      # it fails.  Can be any non-negative number.  Defaults to 3.
+      interval: 3
+
+      # How many times to attempt to execute the health-check command before
+      # declaring the container hopelessly busticated, and aborting the
+      # `moby-derp` run.  Must be a positive integer.  Defaults to 10.
+      attempts: 10
+
 # SECTION 2: POD-LEVEL CONFIGURATION
 #
 # The remainder of the configuration items in this file correspond to settings

data/lib/moby_derp/container.rb

@@ -43,7 +43,35 @@ module MobyDerp
 			end
 
 			begin
-				Docker::Container.create(hash_labelled(container_creation_parameters)).start!.id
+				c = Docker::Container.create(hash_labelled(container_creation_parameters))
+				c.start!.object_id
+
+				if @config.startup_health_check
+					attempts = @config.startup_health_check[:attempts]
+
+					while attempts > 0
+						stdout, stderr, exitstatus = c.exec(@config.startup_health_check[:command])
+						if exitstatus > 0
+							stdout_lines = stdout.empty? ? [] : ["stdout:"] + stdout.join("\n").split("\n").map { |l| "  #{l}" }
+							stderr_lines = stderr.empty? ? [] : ["stderr:"] + stderr.join("\n").split("\n").map { |l| "  #{l}" }
+							output_lines = stdout_lines + stderr_lines
+							@logger.warn(logloc) { "Startup health check failed on #{container_name} with status #{exitstatus}." + (output_lines.empty? ? "" : (["  Output:"] + output_lines.join("\n  "))) }
+
+							attempts -= 1
+							sleep @config.startup_health_check[:interval]
+						else
+							@logger.info(logloc) { "Startup health check passed." }
+							break
+						end
+					end
+
+					if attempts == 0
+						raise MobyDerp::StartupHealthCheckError,
+							"Container #{container_name} has failed the startup health check command #{@config.startup_health_check[:attempts]} times.  Aborting."
+					end
+				end
+
+				c.id
 			rescue Docker::Error::ClientError => ex
 				raise MobyDerp::ContainerError,
 				      "moby daemon returned error: #{ex.message}"

data/lib/moby_derp/container_config.rb

@@ -8,7 +8,8 @@ require "shellwords"
 module MobyDerp
 	class ContainerConfig
 		attr_reader :name, :image, :update_image, :command, :environment, :mounts,
-		            :labels, :readonly, :stop_signal, :stop_timeout, :user, :restart, :limits
+		            :labels, :readonly, :stop_signal, :stop_timeout, :user, :restart, :limits,
+		            :startup_health_check
 
 		def initialize(system_config:,
 		               pod_config:,
@@ -24,13 +25,14 @@ module MobyDerp
 		               stop_timeout: 10,
 		               user: nil,
 		               restart: "no",
-		               limits: {}
+		               limits: {},
+		               startup_health_check: nil
 		              )
 			@system_config, @pod_config, @name, @image = system_config, pod_config, "#{pod_config.name}.#{container_name}", image
 
 			@update_image, @command, @environment, @mounts, @labels = update_image, command, environment, mounts, labels
 			@readonly, @stop_signal, @stop_timeout, @user, @restart = readonly, stop_signal, stop_timeout, user, restart
-			@limits = limits
+			@limits, @startup_health_check = limits, startup_health_check
 
 			validate_image
 			validate_update_image
@@ -44,6 +46,7 @@ module MobyDerp
 			validate_user
 			validate_restart
 			validate_limits
+			validate_startup_health_check
 		end
 
 		private
@@ -331,6 +334,51 @@ module MobyDerp
 			end
 		end
 
+		def validate_startup_health_check
+			if @startup_health_check.nil?
+				# This is fine
+				return
+			end
+
+			unless @startup_health_check.is_a?(Hash)
+				raise ConfigurationError,
+				      "startup_health_check must be a hash"
+			end
+
+			case @startup_health_check[:command]
+			when String
+				@startup_health_check[:command] = Shellwords.split(@startup_health_check[:command])
+			when Array
+				unless @startup_health_check[:command].all? { |c| String === c }
+					raise ConfigurationError, "all elements of the health check command array must be strings"
+				end
+			when NilClass
+				raise ConfigurationError, "health check command must be specified"
+			else
+				raise ConfigurationError,
+				      "health check command must be string or array of strings"
+			end
+
+			@startup_health_check[:interval] ||= 3
+			@startup_health_check[:attempts] ||= 10
+
+			unless Numeric === @startup_health_check[:interval]
+				raise ConfigurationError, "startup health check interval must be a number"
+			end
+
+			if @startup_health_check[:interval] < 0
+				raise ConfigurationError, "startup health check interval cannot be negative"
+			end
+
+			unless Integer === @startup_health_check[:attempts]
+				raise ConfigurationError, "startup health check attempt count must be an integer"
+			end
+
+			if @startup_health_check[:attempts] < 1
+				raise ConfigurationError, "startup health check attempt count must be a positive integer"
+			end
+		end
+
 		def validate_boolean(name)
 			v = instance_variable_get(:"@#{name}")
 			unless v == true || v == false

data/lib/moby_derp/error.rb

@@ -9,6 +9,10 @@ module MobyDerp
 	# Indicates there was a problem manipulating a live container
 	class ContainerError < Error; end
 
+	# Raised when the startup health check has failed spectacularly for
+	# a container
+	class StartupHealthCheckError < Error; end
+
 	# Only appears when an inviolable assertion is invalid, and indicates
 	# there is a bug in the code
 	class BugError < Error; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: moby-derp
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Matt Palmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-10 00:00:00.000000000 Z
+date: 2020-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: docker-api