moby-derp 0.1.0

data/lib/moby_derp/container.rb

@@ -0,0 +1,245 @@
+require_relative "./logging_helpers"
+
+require "digest/sha2"
+require "docker-api"
+require "ipaddr"
+require "json/canonicalization"
+
+module MobyDerp
+	class Container
+		include LoggingHelpers
+
+		def initialize(pod:, container_config:, root_container: false)
+			@logger = pod.logger
+
+			@pod, @config, @root_container = pod, container_config, root_container
+		end
+
+		def run
+			container_name = @root_container ? @pod.name : @config.name
+			@logger.debug(logloc) { "Calculated container name is #{container_name} (@root_container: #{@root_container.inspect}, @config.name: #{@config.name}, @pod.name: #{@pod.name}" }
+
+			begin
+				existing_container = Docker::Container.get(container_name)
+				@logger.debug(logloc) { "Config hash for existing container #{container_name} is #{existing_container.info["Config"]["Labels"]["org.hezmatt.moby-derp.config-hash"].inspect}" }
+				@logger.debug(logloc) { "New config hash is #{params_hash(container_creation_parameters).inspect}" }
+
+				if existing_container.info["Config"]["Labels"]["org.hezmatt.moby-derp.config-hash"] == params_hash(container_creation_parameters)
+					# Container is up-to-date
+					@logger.info(logloc) { "Container #{container_name} is up-to-date" }
+					return existing_container.id
+				end
+
+				if existing_container.info["Config"]["Labels"]["org.hezmatt.moby-derp.pod-name"] != @pod.name
+					raise ContainerError,
+					      "container #{container_name} is not tagged as being part of this pod"
+				end
+				@logger.info(logloc) { "Deleting container #{container_name} (#{existing_container.id[0..11]}) because it is out-of-date" }
+				existing_container.delete(force: true)
+				@logger.info(logloc) { "Creating new container #{container_name}" }
+			rescue Docker::Error::NotFoundError
+				@logger.info(logloc) { "Container #{container_name} does not exist; creating it" }
+				# Container doesn't exist, need to create it
+			end
+
+			begin
+				Docker::Container.create(hash_labelled(container_creation_parameters)).start!.id
+			rescue Docker::Error::ClientError => ex
+				raise MobyDerp::ContainerError,
+				      "moby daemon returned error: #{ex.message}"
+			end
+		end
+
+		private
+
+		def container_creation_parameters
+			{}.tap do |params|
+				if @root_container
+					params["HostConfig"] = {
+						"NetworkMode" => @pod.network_name,
+						"Init"        => true,
+					}
+					params["MacAddress"] = container_mac_address
+					if network_uses_ipv6?
+						params["NetworkingConfig"] = {
+							"EndpointsConfig" => {
+								@pod.network_name => {
+									"IPAMConfig" => {
+										"IPv6Address" => container_ipv6_address,
+									}
+								}
+							}
+						}
+					end
+				else
+					params["HostConfig"] = {
+						"NetworkMode"   => "container:#{@pod.root_container_id}",
+						"PidMode"       => "container:#{@pod.root_container_id}",
+						"IpcMode"       => "container:#{@pod.root_container_id}",
+					}
+				end
+
+				params["HostConfig"]["RestartPolicy"] = parsed_restart_policy
+				params["HostConfig"]["Mounts"]        = merged_mounts.map { |mount| mount_structure(mount) }
+
+				params["Env"]        = @pod.common_environment.merge(@config.environment).map { |k, v| "#{k}=#{v}" }
+				params["Volumes"]    = {}
+
+				params["name"]  = @root_container ? @pod.name : @config.name
+				params["Image"] = image_id
+
+				params["Cmd"]         = @config.command
+				params["StopSignal"]  = @config.stop_signal
+				params["StopTimeout"] = @config.stop_timeout
+
+				if @config.readonly
+					params["HostConfig"]["ReadonlyRootfs"] = true
+				end
+
+				if @config.limits["cpus"]
+					params["HostConfig"]["NanoCPUs"] = @config.limits["cpus"] * 10 ** 9
+				end
+
+				{
+					"cpu-shares"         => "CpuShares",
+					"oom-score-adj"      => "OomScoreAdj",
+					"pids"               => "PidsLimit",
+					"memory"             => "Memory",
+					"memory-swap"        => "MemorySwap",
+					"memory-reservation" => "MemoryReservation",
+					"shm-size"           => "ShmSize",
+				}.each do |limit_name, moby_limit_name|
+					if @config.limits[limit_name]
+						params["HostConfig"][moby_limit_name] = @config.limits[limit_name]
+					end
+				end
+
+				@config.limits.keys.grep(/^ulimit-/).each do |ulimit|
+					params["HostConfig"]["Ulimits"] ||= []
+					params["HostConfig"]["Ulimits"] << ulimit_structure(ulimit)
+				end
+
+				params["Labels"] = @pod.common_labels.merge(@config.labels)
+				params["Labels"]["org.hezmatt.moby-derp.pod-name"] = @pod.name
+
+				unless @root_container
+					params["Labels"]["org.hezmatt.moby-derp.root-container-id"] = @pod.root_container_id
+				end
+			end
+		end
+
+		def hash_labelled(params)
+			params.tap do |params|
+				config_hash = params_hash(params)
+
+				params["Labels"] ||= {}
+				params["Labels"]["org.hezmatt.moby-derp.config-hash"] = config_hash
+			end
+		end
+
+		def params_hash(params)
+			"sha256:#{Digest::SHA256.hexdigest(params.to_json_c14n)}"
+		end
+
+		def image_id
+			if @config.image =~ /\A#{Docker::Image::DIGEST}\z/
+				@config.image
+			else
+				if @config.update_image
+					begin
+						Docker::Image.create(fromImage: @config.image).id
+					rescue Docker::Error::NotFoundError
+						raise ContainerError,
+								"image #{@config.image} for container #{@config.name} cannot be downloaded"
+					end
+				else
+					begin
+						Docker::Image.get(@config.image).id
+					rescue Docker::Error::NotFoundError
+						# Image doesn't exist locally, so we'll have to pull it after all
+						begin
+							Docker::Image.create(fromImage: @config.image).id
+						rescue Docker::Error::NotFoundError
+							raise ContainerError,
+							      "image #{@config.image} for container #{@config.name} cannot be downloaded"
+						end
+					end
+				end
+			end
+		end
+
+		def parsed_restart_policy
+			@config.restart =~ /\A([a-z-]+)(:(\d+))?\z/
+			{ "Name" => $1 }.tap do |policy|
+				if $3
+					policy["MaximumRetryCount"] = $3.to_i
+				end
+			end
+		end
+
+		def merged_mounts
+			container_mount_targets = @config.mounts.map { |m| m.target }
+
+			@config.mounts + @pod.common_mounts.select { |m| !container_mount_targets.include?(m.target) }
+		end
+
+		def mount_structure(mount)
+			{
+				"Type"     => "bind",
+				"Source"   => "#{@pod.mount_root}/#{mount.source}",
+				"Target"   => mount.target,
+				"ReadOnly" => mount.readonly,
+			}
+		end
+
+		def ulimit_structure(limit_key)
+			{
+				"Name" => limit_key.sub(/^ulimit-/, ''),
+				"Soft" => @config.limits[limit_key].first,
+				"Hard" => @config.limits[limit_key].last,
+			}
+		end
+
+		def container_mac_address
+			"02:" + Digest::SHA256.hexdigest(@pod.name + Socket.gethostname)[0..9].scan(/../).join(":")
+		end
+
+		def docker_network
+			begin
+				network = Docker::Network.get(@pod.network_name)
+			rescue Docker::Error::NotFoundError
+				raise ContainerError,
+				      "network #{@pod.network_name} does not exist"
+			end
+		end
+
+		def network_uses_ipv6?
+			docker_network.info["EnableIPv6"]
+		end
+
+		def container_ipv6_address
+			network, masklen = ipv6_network.split("/", 2)
+			network = IPAddr.new(network)
+			masklen = masklen.to_i
+
+			(network | Digest::SHA256.hexdigest(container_mac_address).to_i(16) % 2**masklen).to_s
+		end
+
+		def ipv6_network
+			ipam = docker_network.info["IPAM"]
+			unless ipam["Driver"] == "default"
+				raise ContainerError,
+				      "Unsupported IPAM driver #{ipam["Driver"]} on network #{@pod.network_name}"
+			end
+
+			begin
+				ipam["Config"].find do |cfg|
+					IPAddr.new(cfg["Subnet"]).ipv6?
+				end["Subnet"]
+			rescue NoMethodError
+				raise ContainerError,
+				      "No IPv6 subnet found on network #{@pod.network_name}"
+			end
+		end
+	end
+end

data/lib/moby_derp/container_config.rb

@@ -0,0 +1,377 @@
+require_relative "../freedom_patches/docker/image"
+require_relative "./error"
+require_relative "./mount"
+
+require "docker-api"
+
+module MobyDerp
+	class ContainerConfig
+		attr_reader :name, :image, :update_image, :command, :environment, :mounts,
+		            :labels, :readonly, :stop_signal, :stop_timeout, :restart, :limits
+
+		def initialize(system_config:,
+		               pod_config:,
+		               container_name:,
+		               image:,
+		               update_image: true,
+		               command: [],
+		               environment: {},
+		               mounts: [],
+		               labels: {},
+		               readonly: false,
+		               stop_signal: "SIGTERM",
+		               stop_timeout: 10,
+		               restart: "no",
+		               limits: {}
+		              )
+			@system_config, @pod_config, @name, @image = system_config, pod_config, "#{pod_config.name}.#{container_name}", image
+
+			@update_image, @command, @environment, @mounts, @labels = update_image, command, environment, mounts, labels
+			@readonly, @stop_signal, @stop_timeout, @restart = readonly, stop_signal, stop_timeout, restart
+			@limits = limits
+
+			validate_image
+			validate_update_image
+			validate_command
+			validate_environment
+			validate_mounts
+			validate_labels
+			validate_readonly
+			validate_stop_signal
+			validate_stop_timeout
+			validate_restart
+			validate_limits
+		end
+
+		private
+
+		def validate_image
+			unless @image.is_a?(String)
+				raise ConfigurationError,
+				      "image must be a string"
+			end
+
+			unless @image =~ Docker::Image::IMAGE_REFERENCE
+				raise ConfigurationError,
+				      "image is not a valid image reference"
+			end
+		end
+
+		def validate_update_image
+			validate_boolean(:update_image)
+		end
+
+		def validate_command
+			case @command
+			when String
+				true
+			when Array
+				unless @command.all? { |c| String === c }
+					raise ConfigurationError, "all elements of the command array must be strings"
+				end
+			else
+				raise ConfigurationError,
+				      "command must be string or array of strings"
+			end
+		end
+
+		def validate_environment
+			validate_hash(:environment)
+
+			if (bad_vars = @environment.keys.select { |k| k =~ /=/ }) != []
+				raise ConfigurationError,
+				      "environment variable names cannot include equals signs: #{bad_vars.inspect}"
+			end
+		end
+
+		def validate_mounts
+			unless @mounts.is_a?(Array)
+				raise ConfigurationError,
+				      "mounts must be an array"
+			end
+
+			begin
+				@mounts.map! { |m| Mount.new(**symbolize_keys(m)) }
+			rescue ArgumentError => ex
+				case ex.message
+				when /unknown keywords?: (.*)$/
+					raise ConfigurationError,
+						   "unknown mount option(s): #{$1}"
+				when /missing keywords?: (.*)$/
+					raise ConfigurationError,
+					      "missing mount option(s): #{$1}"
+				else
+					#:nocov:
+					raise
+					#:nocov:
+				end
+			end
+		end
+
+		def validate_labels
+			validate_hash(:labels)
+		end
+
+		def validate_readonly
+			validate_boolean(:readonly)
+		end
+
+		def validate_stop_signal
+			if @stop_signal.is_a?(String)
+				signame = @stop_signal.sub(/\ASIG/, "")
+				# This is not 100% accurate, because in theory moby-derp could
+				# be running on a different platform to the Moby server it is
+				# controlling, but we'll worry about that if we ever come to it.
+				unless Signal.list.has_key?(signame)
+					raise ConfigurationError,
+					      "unknown signal name: #{@stop_signal.inspect}"
+				end
+			elsif @stop_signal.is_a?(Integer)
+				unless Signal.list.values.include?(@stop_signal)
+					raise ConfigurationError,
+					      "unknown signal ID #{@stop_signal}"
+				end
+			else
+				raise ConfigurationError,
+				      "stop_signal must be a string or integer"
+			end
+		end
+
+		def validate_stop_timeout
+			unless @stop_timeout.is_a?(Integer)
+				raise ConfigurationError,
+				      "stop_timeout must be an integer"
+			end
+
+			if @stop_timeout < 0
+				raise ConfigurationError,
+				      "stop_timeout cannot be negative"
+			end
+		end
+
+		def validate_restart
+			unless @restart.is_a?(String)
+				raise ConfigurationError,
+				      "restart must be a string"
+			end
+
+			unless @restart =~ /\Ano|on-failure(:\d+)?|always|unless-stopped\z/
+				raise ConfigurationError,
+				      "invalid value for restart parameter"
+			end
+		end
+
+		KNOWN_LIMITS = %w{
+			cpus cpu-shares memory memory-swap memory-reservation oom-score-adj
+			pids shm-size ulimit-core ulimit-cpu ulimit-data ulimit-fsize
+			ulimit-memlock ulimit-msgqueue ulimit-nofile ulimit-rttime ulimit-stack
+		}
+
+		def validate_limits
+			unless @limits.is_a?(Hash)
+				raise ConfigurationError,
+				      "limits must be a map"
+			end
+
+			if (bad_keys = @limits.keys - KNOWN_LIMITS) != []
+				raise ConfigurationError,
+				      "unknown limit(s): #{bad_keys.inspect}"
+			end
+
+			validate_cpus_limit
+			validate_cpushares_limit
+			validate_memory_limit
+			validate_memoryswap_limit
+			validate_memoryreservation_limit
+			validate_oomscoreadj_limit
+			validate_pids_limit
+			validate_shmsize_limit
+			validate_ulimits
+		end
+
+		def validate_cpus_limit
+			return unless @limits.has_key?("cpus")
+
+			unless @limits["cpus"].is_a?(Numeric)
+				raise ConfigurationError,
+				      "cpus limit must be a number"
+			end
+
+			if @limits["cpus"] <= 0
+				raise ConfigurationError,
+				      "cpus limit must be a positive number"
+			end
+
+			if @limits["cpus"] > @system_config.cpu_count
+				raise ConfigurationError,
+					"cannot use #{@limits["cpus"]}, as the system only has #{@system_config.cpu_count} CPUs"
+			end
+		end
+
+		def validate_cpushares_limit
+			return unless @limits.has_key?("cpu-shares")
+
+			unless @limits["cpu-shares"].is_a?(Integer)
+				raise ConfigurationError,
+				      "cpu-shares limit must be an integer"
+			end
+
+			unless (2..1024).include?(@limits["cpu-shares"])
+				raise ConfigurationError,
+				      "cpu-shares limit must be an integer between 2 and 1024 inclusive"
+			end
+		end
+
+		def validate_memory_limit
+			validate_memory_type_limit("memory")
+		end
+
+		def validate_memoryswap_limit
+			validate_memory_type_limit("memory-swap")
+		end
+
+		def validate_memoryreservation_limit
+			validate_memory_type_limit("memory-reservation")
+		end
+
+		def validate_oomscoreadj_limit
+			return unless @limits.has_key?("oom-score-adj")
+
+			unless @limits["oom-score-adj"].is_a?(Integer)
+				raise ConfigurationError,
+				      "oom-score-adj limit must be an integer"
+			end
+
+			unless (0..1000).include?(@limits["oom-score-adj"])
+				raise ConfigurationError,
+				      "oom-score-adj limit must be an integer between 0 and 1000 inclusive"
+			end
+		end
+
+		def validate_pids_limit
+			return unless @limits.has_key?("pids")
+
+			unless @limits["pids"].is_a?(Integer)
+				raise ConfigurationError,
+				      "pids limit must be an integer"
+			end
+
+			# As far as I can see, the only 32-bit platform that Moby supports is
+			# armhf.  Extend the list if required.
+			max_pids = @system_config.cpu_bits == 32 ? 2**15 : 2**22
+
+			unless (-1..max_pids).include?(@limits["pids"])
+				raise ConfigurationError,
+				      "pids limit must be an integer between -1 and #{max_pids} inclusive"
+			end
+		end
+
+		def validate_shmsize_limit
+			validate_memory_type_limit("shm-size")
+		end
+
+		def validate_ulimits
+			@limits.keys.grep(/\Aulimit-.*\z/).each do |ulimit|
+				unless @limits[ulimit] =~ /\A(unlimited|\d+)(:(unlimited|\d+))?\z/
+					raise ConfigurationError,
+					      "invalid limit syntax for #{ulimit}: must be <softlimit>[:<hardlimit>]"
+				end
+
+				@limits[ulimit] = [ulimit_value($1)]
+
+				if $2.nil?
+					@limits[ulimit][1] = @limits[ulimit][0]
+				else
+					@limits[ulimit][1] = ulimit_value($3)
+				end
+			end
+		end
+
+		def ulimit_value(s)
+			if s == "unlimited"
+				-1
+			else
+				s.to_i
+			end
+		end
+
+		def validate_memory_type_limit(name)
+			return unless @limits.has_key?(name)
+
+			case @limits[name]
+			when Integer
+				if @limits[name] < 0
+					raise ConfigurationError,
+					      "#{name} limit must not be a negative number"
+				end
+			when String
+				unless @limits[name] =~ /\A(\d+(\.\d+)?)([kKmMgGtTpP]?)[bB]?\z/
+					raise ConfigurationError,
+					      "invalid value for #{name} limit: #{@limits[name]}"
+				end
+				@limits[name] = ($1.to_f * multiplier($3)).to_i
+			else
+				raise ConfigurationError,
+				      "#{name} limit must be a string or an integer"
+			end
+		end
+
+		def validate_boolean(name)
+			v = instance_variable_get(:"@#{name}")
+			unless v == true || v == false
+				raise ConfigurationError,
+				      "#{name} setting must be a boolean"
+			end
+		end
+
+		def validate_hash(name)
+			h = instance_variable_get(:"@#{name}")
+
+			unless h.is_a?(Hash)
+				raise ConfigurationError,
+				      "#{h} is not a map"
+			end
+
+			unless (bad_keys = h.keys.select { |k| !k.is_a?(String) }) == []
+				raise ConfigurationError,
+				      "#{h} contains non-string key(s): #{bad_keys.inspect}"
+			end
+
+			unless (bad_values = h.values.select { |v| !v.is_a?(String) }) == []
+				raise ConfigurationError,
+				      "#{h} contains non-string value(s): #{bad_values.inspect}"
+			end
+		end
+
+		def symbolize_keys(h)
+			{}.tap do |res|
+				h.keys.each do |k|
+					res[k.to_sym] = h[k]
+				end
+			end
+		end
+
+		def multiplier(s)
+			case s.upcase
+			when ''
+				1
+			when 'K'
+				1024
+			when 'M'
+				1024 * 1024
+			when 'G'
+				1024 * 1024 * 1024
+			when 'T'
+				1024 * 1024 * 1024 * 1024
+			when 'P'
+				1024 * 1024 * 1024 * 1024 * 1024
+			else
+				#:nocov:
+				raise ConfigurationError,
+				      "Unknown suffix #{s.inspect}"
+				#:nocov:
+			end
+		end
+
+	end
+end

data/lib/moby_derp/error.rb

@@ -0,0 +1,15 @@
+module MobyDerp
+	# Base class for all MobyDerp-specific errors
+	class Error < StandardError; end
+
+	# Raised when something isn't quite right with the system or pod
+	# configuration
+	class ConfigurationError < Error; end
+
+	# Indicates there was a problem manipulating a live container
+	class ContainerError < Error; end
+
+	# Only appears when an inviolable assertion is invalid, and indicates
+	# there is a bug in the code
+	class BugError < Error; end
+end

data/lib/moby_derp/logging_helpers.rb

@@ -0,0 +1,26 @@
+module MobyDerp
+  module LoggingHelpers
+    private
+
+    def log_exception(ex, progname = nil)
+		#:nocov:
+      progname ||= "#{self.class.to_s}##{caller_locations(2, 1).first.label}"
+
+      logger.error(progname) do
+        explanation = if block_given?
+          yield
+        else
+          nil
+        end
+
+        (["#{explanation}#{explanation ? ": " : ""}#{ex.message} (#{ex.class})"] + ex.backtrace).join("\n  ")
+      end
+		#:nocov:
+    end
+
+    def logloc
+      loc = caller_locations.first
+      "#{self.class}##{loc.label}"
+    end
+  end
+end

data/lib/moby_derp/moby_info.rb

@@ -0,0 +1,12 @@
+module MobyDerp
+	class MobyInfo
+		attr_reader :cpu_count, :cpu_bits
+
+		def initialize(info)
+			@cpu_count = info["NCPU"]
+			# As far as I can tell, the only 32-bit platform Moby supports is
+			# armhf; if that turns out to be incorrect, amend the list below.
+			@cpu_bits  = %w{armhf}.include?(info["Architecture"]) ? 32 : 64
+		end
+	end
+end