moby-derp 0.4.0 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 31c2c89cac1b196c48dc2c2183367d61578b1e4e1bae1ff7d698dd7ef6c23cb5
-  data.tar.gz: c7b83663a971a01f3e8b5e8a5420b12b6192d207107ce1cd3047539644051f3e
+  metadata.gz: f4f3f67740bc9579d1b4e2af6e95801e2f019f65b2fddcc495e5ef1c24084565
+  data.tar.gz: a78bf5f24219288581f7bc24137b372e2c9738a9ba2245266b176c800e8db0f8
 SHA512:
-  metadata.gz: 34c40eb8d1d8cfa668a3048c6a8ceaf231c4fae6a607d389bf073d3a3874a74a10a31e728c0c322476a154c9c425fd92869d0fb5849707c1dee2b4c0970f15af
-  data.tar.gz: 01d15f3f197b48fe5748f27718a8e8c7035ff0713113742bf1ae7a43a480197ab7a97559e4ae19fbec8ca5711d22ebd38f02865194e9ce73a817b11a0899bf9a
+  metadata.gz: 724548d882433981999f8c6ba64ba48b3e227de6697a23fb2bd01819acf50a711fd880f685825099e25864aad0736fdc55cd7098ed4618eae83a38bf6a63d472
+  data.tar.gz: be5f791cc9b56efb0be63fc56bd6d788ad4d6617aed29360452a8c5c44d7d83228734edb3c0ad0c8bbbfda58c680e71bf65503ced36db2547876efa5e95bff93

data/README.md

@@ -134,7 +134,7 @@ wrapper script, like this:
 
     set -e
 
-    MOBY_DERP_SYSTEM_CONFIG_FILE=/opt/srv/etc/moby-derp/moby-derp.yaml
+    export MOBY_DERP_SYSTEM_CONFIG_FILE=/opt/srv/etc/moby-derp/moby-derp.yaml
 
     exec /usr/local/bin/moby-derp "$@"
 
@@ -185,7 +185,7 @@ to `moby-derp`.  This means that, yes, different users need to use different
 filenames.  The benefit of this is that the `sudo` configuration becomes a lot
 easier to audit -- the pod name is right there.
 
-This means that no matter a user does, they cannot have any effect on any
+This means that no matter what a user does, they cannot have any effect on any
 container which is not named for the pod they're manipulating.  There are also
 safety valves around `moby-derp`-managed containers being labelled as such, so
 that in the event that someone does inadvertently name a container in such a
@@ -220,7 +220,7 @@ still publish to ephemeral ports (using the `:containerPort` syntax, or
 `publish_all: true`) if they wish.
 
 If a pod *does* need to bind to a specific host port, then that pod/port pair
-should be whitelisted in the system configuration file.
+should be whitelisted in the [system configuration file](#system-configuration).
 
 
 # Contributing

data/example.yml

@@ -213,6 +213,33 @@ containers:
       ulimit-rttime: 15:16
       ulimit-stack: 17:18
 
+    # If you're a bit suss as to whether or not one of your containers will
+    # successfully start up, you can use the following section to define a
+    # start-time health checking regime.
+    #
+    # How it works is that the defined command is run in the container (using
+    # `exec`), and if-and-when that command returns a `0` exit status, the
+    # container is considered to be healthy and we're done.  If the command
+    # returns a non-zero exit status, we wait for `interval` seconds and then
+    # retry.  If the command executes `attempts` times without receiving a `0`
+    # exit status, the container is considered "failed", and no further
+    # containers in the pod will be processed, and the `moby-derp` execution
+    # will itself exit with a non-zero status.
+    startup_health_check:
+      # The command to run inside the container via `exec`.  You can specify
+      # this as a string, or as an array of strings if you prefer to avoid
+      # shell quoting hell.
+      command: '/usr/local/bin/r-u-ok'
+
+      # How many seconds to wait between invocations of the command, when
+      # it fails.  Can be any non-negative number.  Defaults to 3.
+      interval: 3
+
+      # How many times to attempt to execute the health-check command before
+      # declaring the container hopelessly busticated, and aborting the
+      # `moby-derp` run.  Must be a positive integer.  Defaults to 10.
+      attempts: 10
+
 # SECTION 2: POD-LEVEL CONFIGURATION
 #
 # The remainder of the configuration items in this file correspond to settings

data/lib/moby_derp/config_file.rb

@@ -1,7 +1,7 @@
 require_relative "./error"
 require_relative "./logging_helpers"
 
-require "safe_yaml"
+require "yaml"
 
 module MobyDerp
 	class ConfigFile
@@ -12,7 +12,7 @@ module MobyDerp
 		def initialize(filename)
 			begin
 				@logger.debug(logloc) { "Reading configuration file #{filename}" }
-				@config = SafeYAML.load(File.read(filename))
+				@config = YAML.safe_load(File.read(filename))
 			rescue Errno::ENOENT
 				raise ConfigurationError,
 				      "file does not exist"

data/lib/moby_derp/container.rb

@@ -5,6 +5,8 @@ require "docker-api"
 require "ipaddr"
 require "json/canonicalization"
 
+require_relative "./freedom_patches/docker/credential"
+
 module MobyDerp
 	class Container
 		include LoggingHelpers
@@ -43,7 +45,35 @@ module MobyDerp
 			end
 
 			begin
-				Docker::Container.create(hash_labelled(container_creation_parameters)).start!.id
+				c = Docker::Container.create(hash_labelled(container_creation_parameters))
+				c.start!.object_id
+
+				if @config.startup_health_check
+					attempts = @config.startup_health_check[:attempts]
+
+					while attempts > 0
+						stdout, stderr, exitstatus = c.exec(@config.startup_health_check[:command])
+						if exitstatus > 0
+							stdout_lines = stdout.empty? ? [] : ["stdout:"] + stdout.join("\n").split("\n").map { |l| "  #{l}" }
+							stderr_lines = stderr.empty? ? [] : ["stderr:"] + stderr.join("\n").split("\n").map { |l| "  #{l}" }
+							output_lines = stdout_lines + stderr_lines
+							@logger.warn(logloc) { "Startup health check failed on #{container_name} with status #{exitstatus}." + (output_lines.empty? ? "" : (["  Output:"] + output_lines.join("\n  "))) }
+
+							attempts -= 1
+							sleep @config.startup_health_check[:interval]
+						else
+							@logger.info(logloc) { "Startup health check passed." }
+							break
+						end
+					end
+
+					if attempts == 0
+						raise MobyDerp::StartupHealthCheckError,
+							"Container #{container_name} has failed the startup health check command #{@config.startup_health_check[:attempts]} times.  Aborting."
+					end
+				end
+
+				c.id
 			rescue Docker::Error::ClientError => ex
 				raise MobyDerp::ContainerError,
 				      "moby daemon returned error: #{ex.message}"
@@ -60,7 +90,7 @@ module MobyDerp
 						"Init"        => true,
 					}
 					params["MacAddress"] = container_mac_address
-					if network_uses_ipv6?
+					if network_uses_ipv6? && user_defined_network?
 						params["NetworkingConfig"] = {
 							"EndpointsConfig" => {
 								@pod.network_name => {
@@ -127,7 +157,9 @@ module MobyDerp
 				params["Labels"] = @pod.common_labels.merge(@config.labels)
 				params["Labels"]["org.hezmatt.moby-derp.pod-name"] = @pod.name
 
-				unless @root_container
+				if @root_container
+					params["Labels"] = @pod.root_labels.merge(params["Labels"])
+				else
 					params["Labels"]["org.hezmatt.moby-derp.root-container-id"] = @pod.root_container_id
 				end
 			end
@@ -222,6 +254,10 @@ module MobyDerp
 			docker_network.info["EnableIPv6"]
 		end
 
+		def user_defined_network?
+			!%w{bridge host none}.include?(@pod.network_name)
+		end
+
 		def container_ipv6_address
 			network, masklen = ipv6_network.split("/", 2)
 			network = IPAddr.new(network)

data/lib/moby_derp/container_config.rb

@@ -1,4 +1,4 @@
-require_relative "../freedom_patches/docker/image"
+require_relative "./freedom_patches/docker/image"
 require_relative "./error"
 require_relative "./mount"
 
@@ -8,7 +8,8 @@ require "shellwords"
 module MobyDerp
 	class ContainerConfig
 		attr_reader :name, :image, :update_image, :command, :environment, :mounts,
-		            :labels, :readonly, :stop_signal, :stop_timeout, :user, :restart, :limits
+		            :labels, :readonly, :stop_signal, :stop_timeout, :user, :restart, :limits,
+		            :startup_health_check
 
 		def initialize(system_config:,
 		               pod_config:,
@@ -24,13 +25,14 @@ module MobyDerp
 		               stop_timeout: 10,
 		               user: nil,
 		               restart: "no",
-		               limits: {}
+		               limits: {},
+		               startup_health_check: nil
 		              )
 			@system_config, @pod_config, @name, @image = system_config, pod_config, "#{pod_config.name}.#{container_name}", image
 
 			@update_image, @command, @environment, @mounts, @labels = update_image, command, environment, mounts, labels
 			@readonly, @stop_signal, @stop_timeout, @user, @restart = readonly, stop_signal, stop_timeout, user, restart
-			@limits = limits
+			@limits, @startup_health_check = limits, startup_health_check
 
 			validate_image
 			validate_update_image
@@ -44,6 +46,7 @@ module MobyDerp
 			validate_user
 			validate_restart
 			validate_limits
+			validate_startup_health_check
 		end
 
 		private
@@ -331,6 +334,51 @@ module MobyDerp
 			end
 		end
 
+		def validate_startup_health_check
+			if @startup_health_check.nil?
+				# This is fine
+				return
+			end
+
+			unless @startup_health_check.is_a?(Hash)
+				raise ConfigurationError,
+				      "startup_health_check must be a hash"
+			end
+
+			case @startup_health_check[:command]
+			when String
+				@startup_health_check[:command] = Shellwords.split(@startup_health_check[:command])
+			when Array
+				unless @startup_health_check[:command].all? { |c| String === c }
+					raise ConfigurationError, "all elements of the health check command array must be strings"
+				end
+			when NilClass
+				raise ConfigurationError, "health check command must be specified"
+			else
+				raise ConfigurationError,
+				      "health check command must be string or array of strings"
+			end
+
+			@startup_health_check[:interval] ||= 3
+			@startup_health_check[:attempts] ||= 10
+
+			unless Numeric === @startup_health_check[:interval]
+				raise ConfigurationError, "startup health check interval must be a number"
+			end
+
+			if @startup_health_check[:interval] < 0
+				raise ConfigurationError, "startup health check interval cannot be negative"
+			end
+
+			unless Integer === @startup_health_check[:attempts]
+				raise ConfigurationError, "startup health check attempt count must be an integer"
+			end
+
+			if @startup_health_check[:attempts] < 1
+				raise ConfigurationError, "startup health check attempt count must be a positive integer"
+			end
+		end
+
 		def validate_boolean(name)
 			v = instance_variable_get(:"@#{name}")
 			unless v == true || v == false

data/lib/moby_derp/error.rb

@@ -9,6 +9,10 @@ module MobyDerp
 	# Indicates there was a problem manipulating a live container
 	class ContainerError < Error; end
 
+	# Raised when the startup health check has failed spectacularly for
+	# a container
+	class StartupHealthCheckError < Error; end
+
 	# Only appears when an inviolable assertion is invalid, and indicates
 	# there is a bug in the code
 	class BugError < Error; end

data/lib/moby_derp/freedom_patches/docker/credential.rb

@@ -0,0 +1,91 @@
+require "json"
+require "open3"
+require "pathname"
+require "uri"
+
+module Docker
+	module Credential
+		#:nocov:
+		def self.for(ref)
+			image_cred(ref)
+		end
+
+		private
+
+		def self.image_cred(ref)
+			cred_helper = hunt_for_image_domain_cred(ref, docker_config.fetch("credHelpers", {}))
+
+			if cred_helper
+				out, rv = Open3.capture2e("docker-credential-#{cred_helper}", "get", stdin_data: image_domain(ref))
+
+				if rv.exitstatus == 0
+					cred_data = JSON.parse(out)
+
+					{ username: cred_data["Username"], password: cred_data["Secret"], serveraddress: image_domain(ref) }
+				else
+					raise RuntimeError, "Credential helper docker-credential-#{cred_helper} exited with #{rv.exitstatus}: #{out}"
+				end
+			else
+				cred = hunt_for_image_domain_cred(ref, docker_config.fetch("auths", {}))
+
+				if cred
+					user, pass = JSON.parse(cred.fetch("auth", "null"))&.unpack("m")&.first&.split(":", 2)
+
+					if user && pass
+						{ username: user, password: pass, serveraddress: image_domain(ref) }
+					else
+						{}
+					end
+				else
+					{}
+				end
+			end
+		end
+
+		def self.hunt_for_image_domain_cred(ref, section)
+			section.find do |k, v|
+				if k =~ /:\/\//
+					# Doin' it URL style
+					URI(k).host == image_domain(ref)
+				else
+					k == image_domain(ref)
+				end
+			end&.last
+		end
+
+		def self.image_domain(ref)
+			if match_data = ref.match(Docker::Image::IMAGE_REFERENCE)
+				if match_data[1] =~ /[.:]/
+					match_data[1].gsub(/\/\z/, '')
+				else
+					"index.docker.io"
+				end
+			else
+				raise ArgumentError, "Could not parse image ref #{ref.inspect}"
+			end
+		end
+
+		def self.docker_config
+			if (f = Pathname.new(ENV.fetch("DOCKER_CONFIG", "~/.docker")).expand_path.join("config.json")).exist?
+				JSON.parse(f.read)
+			else
+				{}
+			end
+		end
+
+		module ImageClassMixin
+			def create(opts = {}, creds = nil, conn = Docker.connection, &block)
+				if creds.nil?
+					image = opts["fromImage"] || opts[:fromImage]
+
+					creds = Docker::Credential.for(image)
+				end
+
+				super(opts, creds, conn, &block)
+			end
+		end
+		#:nocov:
+	end
+end
+
+Docker::Image.singleton_class.prepend(Docker::Credential::ImageClassMixin)

data/lib/moby_derp/pod.rb

@@ -18,6 +18,20 @@ module MobyDerp
 
 			@logger.debug(logloc) { "Root container ID is #{@root_container_id}" }
 
+			desired_container_names = @config.containers.map(&:name)
+
+			Docker::Container.all(all: true).each do |c|
+				c_name = c.info["Names"].first.sub(/^\//, '')
+
+				if c.info["Labels"]["org.hezmatt.moby-derp.pod-name"] == name &&
+						!c.info["Labels"]["org.hezmatt.moby-derp.root-container-id"].nil? &&
+						!desired_container_names.include?(c_name.split(".", 2).last)
+					@logger.info(logloc) { "Removing stale container #{c_name}" }
+					c.stop
+					c.delete
+				end
+			end
+
 			@config.containers.each do |cfg|
 				@logger.info(logloc) { "Checking container #{cfg.name}" }
 
@@ -48,6 +62,10 @@ module MobyDerp
 			@config.common_labels
 		end
 
+		def root_labels
+			@config.root_labels
+		end
+
 		def common_environment
 			@config.common_environment
 		end

data/lib/moby_derp/pod_config.rb

@@ -3,13 +3,24 @@ require_relative "./container_config"
 require_relative "./logging_helpers"
 require_relative "./mount"
 
-require "safe_yaml"
 require "socket"
 
 module MobyDerp
 	class PodConfig < ConfigFile
 		include LoggingHelpers
 
+		VALID_CONFIG_KEYS = %w{
+			containers
+			hostname
+			common_environment
+			common_labels
+			root_labels
+			common_mounts
+			expose
+			publish
+			publish_all
+		}
+
 		attr_reader :name,
 		            :containers,
 		            :hostname,
@@ -34,6 +45,10 @@ module MobyDerp
 			@name = File.basename(filename, ".*")
 			validate_name
 
+			unless (bad_keys = @config.keys - VALID_CONFIG_KEYS).empty?
+				raise ConfigurationError,
+				      "Invalid pod configuration key(s): #{bad_keys.inspect}"
+			end
 
 			unless @config.has_key?("containers")
 				raise ConfigurationError,
@@ -100,6 +115,11 @@ module MobyDerp
 					raise ConfigurationError,
 						"container name #{name.inspect} is invalid (must contain only alphanumerics, underscores, and hyphens)"
 				end
+
+				unless data.is_a?(Hash)
+					raise ConfigurationError,
+					      "container data must be a hash"
+				end
 			end
 
 			begin

data/lib/moby_derp/system_config.rb

@@ -1,16 +1,21 @@
 require_relative "./config_file"
 
-require "safe_yaml"
-
 module MobyDerp
 	class SystemConfig < ConfigFile
 		attr_reader :mount_root, :port_whitelist, :network_name, :use_host_resolv_conf,
 		            :cpu_count, :cpu_bits
 
-		def initialize(filename, moby_info, logger)
+		def initialize(config_data_or_filename, moby_info, logger)
 			@logger = logger
 
-			super(filename)
+			case config_data_or_filename
+			when String
+				super(config_data_or_filename)
+			when Hash
+				@config = stringify_keys(config_data_or_filename)
+			else
+				raise ArgumentError, "Unsupported type for config_data_or_filename parameter"
+			end
 
 			@mount_root           = @config["mount_root"]
 			@port_whitelist       = stringify_keys(@config["port_whitelist"] || {})

data/moby-derp.gemspec

@@ -25,7 +25,6 @@ Gem::Specification.new do |s|
 
 	s.add_runtime_dependency "docker-api"
 	s.add_runtime_dependency "json-canonicalization"
-	s.add_runtime_dependency "safe_yaml"
 
 	s.add_development_dependency 'bundler'
 	s.add_development_dependency 'deep_merge'

data/smoke_tests/root_labels.bats

@@ -0,0 +1,27 @@
+load test_helper
+
+@test "Core labels" {
+	config_file <<-'EOF'
+		root_labels:
+		  foo: booblee
+		containers:
+		  bob:
+		    image: busybox:latest
+		    command: sleep 1
+		common_labels:
+		  moby-derp-smoke-test: ayup
+EOF
+
+	run $MOBY_DERP_BIN $TEST_CONFIG_FILE
+
+	echo "status: $status"
+	echo "output: $output"
+
+	[ "$status" = "0" ]
+	container_running "mdst"
+	container_running "mdst.bob"
+
+	docker inspect mdst --format='{{.Config.Labels}}'
+	docker inspect mdst --format='{{.Config.Labels}}' | grep 'map.*foo:booblee'
+	! docker inspect mdst.bob --format='{{.Config.Labels}}' | grep 'map.*foo:booblee'
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: moby-derp
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.7.1
 platform: ruby
 authors:
 - Matt Palmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-20 00:00:00.000000000 Z
+date: 2020-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: docker-api
@@ -38,20 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: safe_yaml
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -208,13 +194,13 @@ files:
 - README.md
 - bin/moby-derp
 - example.yml
-- lib/freedom_patches/docker/image.rb
 - lib/moby_derp/config_file.rb
 - lib/moby_derp/container.rb
 - lib/moby_derp/container_config.rb
 - lib/moby_derp/error.rb
+- lib/moby_derp/freedom_patches/docker/credential.rb
+- lib/moby_derp/freedom_patches/docker/image.rb
 - lib/moby_derp/logging_helpers.rb
-- lib/moby_derp/moby_info.rb
 - lib/moby_derp/mount.rb
 - lib/moby_derp/pod.rb
 - lib/moby_derp/pod_config.rb
@@ -223,6 +209,7 @@ files:
 - smoke_tests/exposed.bats
 - smoke_tests/minimal.bats
 - smoke_tests/no_file.bats
+- smoke_tests/root_labels.bats
 - smoke_tests/test_helper.bash
 homepage: http://github.com/mpalmer/moby-derp
 licenses: []
@@ -242,7 +229,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: A simple management system for a pod of moby containers

data/lib/moby_derp/moby_info.rb

@@ -1,12 +0,0 @@
-module MobyDerp
-	class MobyInfo
-		attr_reader :cpu_count, :cpu_bits
-
-		def initialize(info)
-			@cpu_count = info["NCPU"]
-			# As far as I can tell, the only 32-bit platform Moby supports is
-			# armhf; if that turns out to be incorrect, amend the list below.
-			@cpu_bits  = %w{armhf}.include?(info["Architecture"]) ? 32 : 64
-		end
-	end
-end