twistlock-control 0.0.1

data/lib/twistlock_control/collections.rb

@@ -0,0 +1,22 @@
+module TwistlockControl
+	# Collections is an interface for querying the collections
+	module Collections
+		class << self
+			def provisioners
+				Entities::Provisioner.repository.table
+			end
+
+			def services
+				Entities::Service.repository.table
+			end
+
+			def service_instances
+				Entities::ServiceInstance.repository.table
+			end
+
+			def container_instances
+				Entities::ContainerInstance.repository.table
+			end
+		end
+	end
+end

data/lib/twistlock_control/entities.rb

@@ -0,0 +1,11 @@
+require_relative 'entity'
+
+%w(
+	provisioning_configuration
+	service
+	provisioner
+	composite_service
+	service_instance
+	container
+	container_instance
+).each { |entity| require_relative("entities/#{entity}") }

data/lib/twistlock_control/entities/composite_service.rb

@@ -0,0 +1,86 @@
+module TwistlockControl
+	module Entities
+		# A service link can for example be, the 'MySQL' container exposes a 'mysql' port.
+		# The 'RubyForum' container consumes this service by listening on the 'mysql' port.
+		# The accompanying ServiceLink would be:
+		# {
+		# provider_name: "MySQL",
+		# provider_port_name: "mysql",
+		# consumer_name: "RubyForum",
+		# consumer_port: "mysql"
+		# }
+		class ServiceLink < Entity
+			attribute :provider_name, String
+			attribute :consumer_name, String
+			attribute :provider_port_name, String
+			attribute :consumer_port_name, String
+		end
+
+		# A CompositeService is a service that consists of a number of services working together to
+		# provide a single service. For example a web forum service might consist of a MySQL service,
+		# for persistant storage, and a Ruby HTTP service that serves HTML sites and queries the storage.
+		# In the CompositeService you may choose to only expose the HTTP service, making it only possible
+		# to query the MySQL database through the Ruby application, which might be considered proper
+		# encapsulation.
+		#
+		# Relations between services are described by the links attribute. A link is characterized by
+		# a producer and a consumer, the consumer will connect to the producers provided service.
+		class CompositeService < Service
+			attribute :service_type, Symbol, default: :composite
+			attribute :id, String, default: :generate_id
+			attribute :name, String
+
+			# Link cases:
+			#
+			#   1. Multi-consumer: a webservice might have 10 Ruby frontend apps, all connecting
+			#      to the same database. Simply increasing the amount of service instances with
+			#      the same links will solve this case. Question: do we want to specify this
+			#      possibility in each service link, or can we simply assume all services are
+			#      scalable in this way? Not all are, but maybe that's a service property, not
+			#      a relation property.
+			#   2. Multi-producer: A MongoDB cluster might have multiple master nodes. A Ruby
+			#      frontend app may want to connect to any of these. The problem is that it will
+			#      have to know before starting on which ports this potentially infinite number
+			#      of servers has, and somehow choose between them. A solution might be to couple
+			#      each Ruby app with a random master.
+			attribute :service_relations, Hash[String => String]
+			attribute :links, [ServiceLink]
+
+			# A provided service basically means this composite service exposes a port of one
+			# of its services. In the forum example, it could be the port 80 http service of
+			# the RubyForum container.
+			#
+			# {
+			# 	http: { RubyForum: 'http' }
+			# }
+			#
+			attribute :provided_services, Hash[String => String]
+
+			def services
+				Service.find_with_ids(service_relations.values)
+			end
+
+			def containers
+				result = []
+				services = self.services.map(&:service)
+				composites = services.select { |s| s.service_type == :composite }
+				containers = services.select { |s| s.service_type == :container }
+				result += containers
+				composites.each do |c|
+					result += c.containers
+				end
+				result
+			end
+
+			def serialize
+				super.merge! links: links.map(&:attributes)
+			end
+
+			private
+
+			def generate_id
+				name.downcase.gsub(' ', '-')
+			end
+		end
+	end
+end

data/lib/twistlock_control/entities/container.rb

@@ -0,0 +1,66 @@
+require 'digest'
+require 'securerandom'
+require 'fileutils'
+require 'yaml'
+
+module TwistlockControl
+	module Entities
+		# A RelatedServiceDescription is used to describe a provided
+		# or consumed service.
+		class RelatedServiceDescription < Entity
+			attribute :port
+			attribute :description
+		end
+
+		# A ContainerDescription represents the container description
+		# file that's used to describe properties of containers.
+		class ContainerDescription < Entity
+			attribute :name, String
+			attribute :description, String
+			attribute :provided_services, Hash[String => RelatedServiceDescription]
+			attribute :consumed_services, Hash[String => RelatedServiceDescription]
+
+			def serialize
+				provided_services = (provided_services || {}).inject({}) { |r, (k, v)| r[k] = v.attributes }
+				consumed_services = (consumed_services || {}).inject({}) { |r, (k, v)| r[k] = v.attributes }
+				attributes.dup.merge!(
+					provided_services: provided_services,
+					consumed_services: consumed_services
+				)
+			end
+		end
+
+		# A container is a service that can be provisioned on a Twistlock provisioner node.
+		class Container < Service
+			attribute :service_type, Symbol, default: :container
+			attribute :id, String, default: :generate_id
+			attribute :url, String
+			attribute :name, String
+			attribute :description, ContainerDescription
+
+			# The network services provided by this service. Each service is
+			# identified with a name, for example: offers HTTP on port 80.
+			def provided_services
+				description.provided_services
+			end
+
+			# The service can depend on any other services. For example it might
+			# require a MySQL service to be linked in on port 3047.
+			def consumed_services
+				description.consumed_services
+			end
+
+			def serialize
+				super.merge!(
+					description: description ? description.serialize : nil
+				)
+			end
+
+			private
+
+			def generate_id
+				Digest::SHA256.hexdigest(url)
+			end
+		end
+	end
+end

data/lib/twistlock_control/entities/container_instance.rb

@@ -0,0 +1,22 @@
+module TwistlockControl
+	module Entities
+		# A container instance represents a container currently
+		# running on a provisioner.
+		class ContainerInstance < PersistedEntity
+			repository RethinkDBRepository['container_instances']
+
+			attribute :id, String, default: :generate_id
+
+			# Attributes as dictated by provisioner
+			attribute :container_id
+			attribute :ip_address
+			attribute :provisioner_id
+
+			private
+
+			def generate_id
+				Digest::SHA256.hexdigest("#{container_id}-#{provisioner_id}")
+			end
+		end
+	end
+end

data/lib/twistlock_control/entities/provisioner.rb

@@ -0,0 +1,24 @@
+require 'digest'
+
+module TwistlockControl
+	module Entities
+		# A provisioner is a machine capable of provisioning containers
+		class Provisioner < PersistedEntity
+			repository RethinkDBRepository['provisioners']
+
+			attribute :id, String, default: :generate_id
+			attribute :name, String
+			attribute :url, String
+
+			def self.api
+				@api ||= ProvisionerAPI.new(url)
+			end
+
+			private
+
+			def generate_id
+				Digest::SHA256.hexdigest(url)
+			end
+		end
+	end
+end

data/lib/twistlock_control/entities/provisioning_configuration.rb

@@ -0,0 +1,65 @@
+module TwistlockControl
+	module Entities
+		# ProvisioningConfiguration holds service instance configuration that
+		# pertains to the provisioning of containers.
+		class ProvisioningConfiguration < Entity
+			attribute :service_id
+
+			def self.new(attrs)
+				if attrs['configurations'] || attrs[:configurations]
+					obj = CompositeConfiguration.allocate
+				else
+					obj = ContainerConfiguration.allocate
+				end
+				obj.send :initialize, attrs
+			end
+		end
+
+		# Maybe we want ContainerConfiguration to be an entity with its
+		# own repository, so we can simply refer to it by id.
+		# That will make getting events from the provisioner easier
+		class ContainerConfiguration < ProvisioningConfiguration
+			attribute :provisioner_id
+			attribute :container_instance_id
+
+			attribute :mount_points
+			attribute :environment_variables
+
+			def provisioner
+				@provisioner ||= Provisioner.find_by_id(provisioner_id)
+			end
+
+			def provisioner=(provisioner)
+				@provisioner = provisioner
+				@provisioner_id = provisioner.id
+			end
+
+			def container
+				@container ||= Container.find_by_id(service_id)
+			end
+
+			def container_instance
+				@container_instance ||= ContainerInstance.find_by_id(container_instance_id)
+			end
+
+			def container_configurations
+				[self]
+			end
+		end
+
+		# Configuration for a composite service
+		class CompositeConfiguration < ProvisioningConfiguration
+			attribute :configurations, [ProvisioningConfiguration]
+
+			def serialize
+				serialized = super
+				serialized[:configurations] = configurations.map(&:serialize)
+				serialized
+			end
+
+			def container_configurations
+				configurations.flat_map(&:container_configurations)
+			end
+		end
+	end
+end

data/lib/twistlock_control/entities/service.rb

@@ -0,0 +1,19 @@
+module TwistlockControl
+	module Entities
+		# A Service class describes a provisionable network service.
+		class Service < PersistedEntity
+			repository RethinkDBRepository['services']
+
+			def self.deserialize(attrs)
+				return nil if attrs.nil?
+
+				case attrs['service_type']
+				when 'container' then Container.new(attrs)
+				when 'composite' then CompositeService.new(attrs)
+				else
+					fail "Unknown service_type: #{attrs[:service_type]}"
+				end
+			end
+		end
+	end
+end

data/lib/twistlock_control/entities/service_instance.rb

@@ -0,0 +1,122 @@
+module TwistlockControl
+	module Entities
+		# A service instance is an entity that represents an instance of a service
+		# that can be started and stopped. For example, an operator might define a Forum
+		# service and then spawn a Forum service instance for each of his customers.
+		# Each of the Forum services can be referenced by name and stopped and started
+		# independantly, and consist of separate container instances.
+		#
+		# A service instance has all runtime configuration such as mount points and
+		# environment variables.
+		#
+		# An operator should be able to assign containers to provisioners, and configure
+		# their runtime configuration.
+		#
+		# The configuration has a tree structure. For each composite service there will
+		# be a branch element, for every container a leaf.
+		class ServiceInstance < PersistedEntity
+			repository RethinkDBRepository['service_instances']
+
+			attribute :id, String, default: :generate_id
+			attribute :name, String
+			attribute :service_id, String
+			attribute :configuration, ProvisioningConfiguration
+
+			# We want to tell all containers how they are linked to eachother.
+			# Composite services have the information about which links exist.
+			# How many instances there are of a container should be configured
+			# at runtime. Can we just do it by adding ContainerConfigurations
+			# to a CompositeConfiguration? That would mean the build_configuration
+			# method would have to only build composite configurations, leaving
+			# the filling in of container configurations to the interactive
+			# resource allocation process. I.E. the user would create a composite
+			# configuration, then for each container needed of each composite
+			# service they would select on which machine(s) any containers will
+			# be ran. When a container configuration is created it can be
+			# determined to which other container configuration it is linked.
+			#
+			# So the next step is to change build_configuration to reflect that,
+			# then we add methods to CompositeConfiguration that allow to convenient
+			# addition of ContainerConfigurations. Including a way to enumerate
+			# which containers are needed.
+			#
+			# We also need to think about the linking, at the moment the provisioner
+			# can link a container to any ip address. When the containers are
+			# on separate machines, we can not usually link the containers directly
+			# on ip, a link would first have to be established. I envisioned this
+			# would ideally be through a simple TLS tunnel established by an ambassador
+			# container.
+			#
+			# If we would go for the ambassador approach the Twistlock system would
+			# have to be aware of this as it would have to provision ambassador nodes
+			# and use the ip addresses of the ambassador nodes to connect across machines.
+			#
+			# Alternatively, we could assume all machines in the cluster are in the
+			# same IP space and simply link them together. This would move the encryption
+			# and network management to a separate level and would ideally be a superior
+			# architecture, but in practice there is no simple way of achieving this
+			# in a way that is compatible with all container providers and all hosts.
+			# Since we want Twistlock to be an easy to deploy integrated solution,
+			# Twistlock would have to supply an automatic way of configuring such a
+			# datacenter without messing with existing architecture too much. A complex
+			# task that's not guaranteed to have a perfect solution.
+			#
+			# We could also for now simply assume a flat ip space, and work on the
+			# ambassador system later. A downside of that is that we might miss some
+			# architectural decision would enable the ambassador system to be more neatly
+			# integrated. So let's thing about the ambassador approach first.
+			#
+			# During the provisioning assignment step, it would become clear on which
+			# machine a process is going to be provisioned. If a linked container is
+			# on a different machine, the system detects it and links the container
+			# into an ambassador.
+			#
+			# Easiest will be to have a single ambassador per host. We could inform the
+			# ambassador of a cross-machine link via a HTTP POST, to which it would respond
+			# with the port numbers it will use for the link.
+			# Nice thing about this approach is that we can simply assume the ambassador
+			# is always there, so no complex logic for spawning it. Also it would be really
+			# easy to disable it and work with a flat ip space.
+			#
+			# So now the provisioning assignment step will be like this, for each container:
+			#    - pick a machine it will run on
+			#    - select any mounts
+			#    - configure any environment variables
+			#
+			# Then the provisioning preparation process will be:
+			#    - make sure all hosts have container descriptions/images
+			#    - determine all cross-machine links
+			#    - inform ambassadors of links
+			#
+			# Then the provisioning process itself:
+			#    - start each container on its host
+			#    - fill in ip-addresses of container instances
+			#    - whenever a container is live, determine if any of its links can be
+			#      established, and if so establish them
+			#
+			# Basically the only thing we need to make sure in the architecture is that it can
+			# be deduced from the link information whether the link is remote or local, so
+			# the system can decide if it has to go through an ambassador.
+
+			def container_configurations
+				configuration.container_configurations
+			end
+
+			def service
+				Service.find_by_id(service_id)
+			end
+
+			def serialize
+				serialized = attributes.dup
+				serialized[:configuration] = configuration.serialize
+				serialized
+			end
+
+			private
+
+			def generate_id
+				name
+			end
+		end
+	end
+end