picsolve_docker_builder 0.1.1 → 0.2.0

data/lib/picsolve_docker_builder/composer/image.rb

@@ -19,16 +19,18 @@ module PicsolveDockerBuilder
         config['image']
       end
 
-      def kubernetes
-        composer.kubernetes
+      def rc(kubernetes)
+        @rcs = {} if @rcs.nil?
+        @rcs[kubernetes] = kubernetes.rc(self) \
+          unless @rcs.key?(kubernetes)
+        @rcs[kubernetes]
       end
 
-      def rc
-        @rc ||= kubernetes.rc self
-      end
-
-      def service
-        @service ||= kubernetes.service self
+      def service(kubernetes)
+        @services = {} if @rcs.nil?
+        @services[kubernetes] = kubernetes.service(self) \
+          unless @services.key?(kubernetes)
+        @services[kubernetes]
       end
 
       def ports

data/lib/picsolve_docker_builder/frame.rb

@@ -249,6 +249,18 @@ module PicsolveDockerBuilder
       config['docker']['image_name']
     end
 
+    def runtime_image_name
+      name = config['docker']['runtime_image'] || image_name
+
+      if name.match(/:[a-z0-9\-_]+$/)
+        name
+      else
+        "#{name}:latest"
+      end
+    rescue NoMethodError
+      nil
+    end
+
     def image_name
       name = config['docker']['base_image']
 
@@ -267,7 +279,7 @@ module PicsolveDockerBuilder
 FROM #{image_name}
 MAINTAINER Picsolve Onlineops <onlineops@picsolve.com>
 #{dockerfile_hooks_asset_build_early}
-RUN useradd -d #{build_user_home} -u #{build_user_uid} #{build_user}
+RUN useradd -m -d #{build_user_home} -u #{build_user_uid} #{build_user}
 #{dockerfile_hooks_asset_build_late}
 EOS
       begin

data/lib/picsolve_docker_builder/helpers/config/base.rb

@@ -0,0 +1,14 @@
+module PicsolveDockerBuilder
+  module Helpers
+    module Config
+      # Base class for a dynamic config object
+      class Base
+        def self.name
+          nil
+        end
+        def initialize(_identifier)
+        end
+      end
+    end
+  end
+end

data/lib/picsolve_docker_builder/helpers/config/secret.rb

@@ -0,0 +1,14 @@
+module PicsolveDockerBuilder
+  module Helpers
+    module Config
+      # Config object that is retrieved from kubernetes secrets store
+      class Secret < Base
+        def self.name
+          'secret'
+        end
+        def initialize(_identifier)
+        end
+      end
+    end
+  end
+end

data/lib/picsolve_docker_builder/helpers/config/variable_object.rb

@@ -0,0 +1,68 @@
+module PicsolveDockerBuilder
+  module Helpers
+    module Config
+      # represents a config string that contains variables
+      class VariableObject
+        # Regex that finds variabales
+        def self.regex_full
+          /^\$\{([^\}]*)\}$/
+        end
+        def self.regex
+          /\$\{[^\}]*\}/
+        end
+        # Checks if a string needs to be replaced by an VariableObject
+        def self.replace_string(obj)
+          return obj unless VariableObject.regex.match(obj)
+          VariableObject.new(obj)
+        end
+
+        def list
+          @list ||= create_list
+        end
+
+        # Split the strings accordingly
+        def create_list
+          lst = split_list
+          replace_variables(lst)
+        end
+
+        # Replace variables
+        def replace_variables(list)
+          list.map! do |elem|
+            m = elem.match(VariableObject.regex_full)
+            if m
+              "var=#{m[1]}"
+            else
+              elem
+            end
+          end
+        end
+
+        def to_s
+          # TODO: implement var evaluation here
+          "<#VariableObject#{list}>"
+        end
+
+        def inspect
+          "<#VariableObject#{list}>"
+        end
+
+        # Split the strings accordingly
+        def split_list(existing_list = [])
+          if existing_list.length == 0
+            str = @str
+          else
+            str = existing_list.pop
+            return existing_list if str.length == 0
+          end
+          list = existing_list + str.partition(VariableObject.regex)
+          split_list(list)
+        end
+
+        def initialize(str)
+          @str = str
+        end
+      end
+    end
+  end
+end

data/lib/picsolve_docker_builder/helpers/config_manager.rb

@@ -0,0 +1,96 @@
+require 'picsolve_docker_builder/base'
+require 'picsolve_docker_builder/helpers/config/variable_object'
+require 'picsolve_docker_builder/composer/container'
+
+module PicsolveDockerBuilder
+  module Helpers
+    # Parses the config file
+    class ConfigManager
+      include PicsolveDockerBuilder::Base
+      def initialize(path, stage)
+        @path = path
+        # TODO: Do this properly, this is a very very dirty hack
+        stage = (
+          !ENV['STAGE'].nil? &&
+          ENV['STAGE'].downcase
+        ) || 'ci' if stage.nil?
+        @stage = stage
+      end
+
+      def default
+        {}
+      end
+
+      def config
+        return @config unless @config.nil?
+        read
+        parse_variables
+      end
+
+      def eval_container_config(config)
+        config['environment'] = eval_container_env(config['environment'])
+        config
+      end
+
+      def eval_container_env_split(env)
+        output = {}
+        env.each do |key, lst|
+          output[key] = {}
+          lst.each do |elem|
+            ikey, val = elem.split('=', 2)
+            output[key][ikey] = val
+          end
+        end
+        output
+      end
+
+      def eval_container_env(env)
+        env = eval_container_env_split(env)
+        output = {}
+        output = output.update(env['default']) if env.key?('default')
+        output = output.update(env[@stage]) if env.key?(@stage)
+        output
+      end
+
+      def containers(composer)
+        config['compose']['containers'].map do |name, raw_config|
+          PicsolveDockerBuilder::Composer::Container.new(
+            name,
+            eval_container_config(raw_config),
+            composer
+          )
+        end
+      end
+
+      def parse_variables
+        parse_variables_real(@config)
+      end
+
+      def parse_variables_real(obj)
+        if obj.is_a?(Hash)
+          obj.each do |k, v|
+            obj[k] = parse_variables_real(v)
+          end
+        elsif obj.is_a?(Array)
+          obj.map do |v|
+            parse_variables_real(v)
+          end
+        elsif obj.is_a?(String)
+          Config::VariableObject.replace_string(obj)
+        else
+          obj
+        end
+      end
+
+      def read
+        @config = default
+        begin
+          yaml = Psych.load_file @path
+          @config = @config.deep_merge(yaml)
+        rescue Errno::ENOENT
+          raise "cannot find config at '#{path}'"
+        end
+      end
+    end
+  end
+end

data/lib/picsolve_docker_builder/helpers/kubernetes/rc.rb

@@ -42,11 +42,17 @@ module PicsolveDockerBuilder
         end
 
         def containers
-          [{
+          c = {
             'name'  => @image.name,
             'image' => @image.repo_tag_unique,
             'ports' => @image.ports_rc
-          }]
+          }
+          # append environment variables if in place
+          if @image.class.instance_methods.include? :environment
+            env = @image.environment
+            c['env'] = env unless env.nil?
+          end
+          [c]
         end
 
         def deploy
@@ -69,17 +75,33 @@ module PicsolveDockerBuilder
           true
         end
 
-        def pods
-          # TODO: better handling of selectors
+        def pods_by_selector(selector)
           client.get_pods(
             namespace: @rc.metadata.namespace,
-            label_selector: "name=#{@rc.spec.selector.name}" \
-            ",deployment=#{@rc.spec.selector.deployment}"
+            label_selector: selector
           ).map do |pod|
             Pod.new(pod, @kubernetes)
           end
         end
 
+        def pods
+          pods_by_selector(
+            "name=#{@rc.spec.selector.name}," \
+            "deployment=#{@rc.spec.selector.deployment}"
+          )
+        end
+
+        def pods_orphan
+          pods_by_selector(
+            "name=#{@rc.spec.selector.name}," \
+            "deployment!=#{@rc.spec.selector.deployment}"
+          )
+        end
+
+        def remove_pods_orphan
+          pods_orphan.each(&:remove)
+        end
+
         def remove_pods
           pods.each(&:remove)
         end

data/lib/picsolve_docker_builder/helpers/kubernetes_manager.rb

@@ -11,32 +11,82 @@ module PicsolveDockerBuilder
     class KubernetesManager
       include PicsolveDockerBuilder::Base
 
+      attr_reader :port, :host, :type
+
+      def initialize(composer, host, type = nil, port = nil)
+        # default type is http
+        type ||= :http
+
+        if type == :ssh_forward
+          @type = type
+          @port = port || 22
+        end
+
+        if type == :http
+          @type = type
+          @port = port || 8080
+        end
+
+        @host = host
+        @composer = composer
+      end
+
       def create_client
-        Kubeclient::Client.new kubernetes_url, 'v1beta3'
+        Kubeclient::Client.new kubernetes_url, 'v1'
       end
 
       def kubernetes_url
+        if type == :ssh_forward
+          kubernetes_url_ssh_forward
+        elsif type == :http
+          kubernetes_url_http
+        end
+      end
+
+      def kubernetes_url_http
+        "http://#{host}:#{port}"
+      end
+
+      def kubernetes_url_ssh_forward_stop
+        @ssh_forward.stop unless @ssh_forward.nil?
+        @ssh_forward = nil
+      end
+
+      def stop
+        kubernetes_url_ssh_forward_stop if type == :ssh_forward
+      end
+
+      def kubernetes_url_ssh_forward
         @ssh_forward = SshForward.new(
-          ssh_host: kubernetes_host,
+          ssh_host: host,
+          ssh_port: port,
+          local_port: 10_000,
           remote_host: '127.0.0.1',
           remote_port: 8080
         )
         port = @ssh_forward.start
         at_exit do
-          @ssh_forward.stop unless @ssh_forward.nil?
+          kubernetes_url_ssh_forward_stop
         end
         "http://127.0.0.1:#{port}"
       end
 
-      def kubernetes_host
-        # TODO: Need to be touched for multi cluster support
-        config['compose']['clusters'].first
-      end
-
       def client
         @client ||= create_client
       end
 
+      def images
+        @composer.images
+      end
+
+      def app_name
+        @composer.app_name
+      end
+
+      def stage
+        @composer.stage
+      end
+
       def service(image)
         Kubernetes::Service.new(image, self)
       end
@@ -45,6 +95,74 @@ module PicsolveDockerBuilder
         Kubernetes::Rc.new(image, self)
       end
 
+      def wait_for_deployed_rcs
+        wait_timeout = 300
+        wait_sleep = 2
+        wait_beginning = 15
+        wait_count = wait_beginning
+        unready_images = images.dup
+        log.info \
+          'Wait for the new pods to be up and running for at least 20 seconds'
+        sleep wait_beginning
+        loop do
+          unready_images.each do |i|
+            unready_images.delete(i) if i.rc(self).ready?
+          end
+          log.debug "Still waiting for: #{unready_images.map(&:name)}"
+          $stdout.flush
+          $stderr.flush
+          wait_count += wait_sleep
+          break if wait_count > wait_timeout
+          break if unready_images.length == 0
+          sleep wait_sleep
+        end
+        successful_images = images.dup
+        unready_images.each do |u|
+          successful_images.delete(u)
+        end
+        [successful_images, unready_images]
+      end
+
+      def remove_old_rcs(successful_images)
+        # build list of successful deplyoed images
+        successful_images.each do |i|
+          log.debug "Remove old pods for #{i.name}"
+          i.rc(self).remove_old_rcs
+          log.debug "Remove orphaned pods for #{i.name}"
+          i.rc(self).remove_pods_orphan
+        end
+      end
+
+      def deploy
+        log.info "start deploying app_name=#{app_name} to stage #{stage}"
+
+        # reset hash
+        @hash = nil
+
+        deploy_services
+
+        deploy_rcs
+
+        successful_images, unready_images = wait_for_deployed_rcs
+
+        remove_old_rcs(successful_images)
+
+        fail "Failed to deploy this service: #{unready_images.map(&:name)}" \
+          if unready_images.length > 0
+      end
+
+      def deploy_services
+        images.each do |i|
+          i.service(self).deploy
+        end
+      end
+
+      def deploy_rcs
+        images.each do |i|
+          i.rc(self).deploy
+        end
+      end
+
       def template_labels_pods(i)
         c = template_labels(i)
         c['deployment'] = i.composer.hash
@@ -58,45 +176,6 @@ module PicsolveDockerBuilder
           'stage' => i.composer.stage
         }
       end
-
-      def config_service(s, i)
-        # workaround a bug on updating TODO: remove when fixed upstream
-        s.name = i.name
-
-        # configure the service
-        s.metadata = {} unless s.metadata
-        s.metadata.name = i.name
-        s.metadata.namespace = i.composer.namespace
-        s.metadata.labels = template_labels(i)
-        s.spec = {} unless s.spec
-        s.spec.ports = i.ports
-        s.spec.selector = {
-          'name' => i.name
-        }
-      end
-
-      def deploy_service(i)
-        existing = false
-        begin
-          s = client.get_service i.name, i.composer.namespace
-          existing = true
-        rescue KubeException
-          s = Kubeclient::Service.new
-        end
-
-        # config config
-        config_service(s, i)
-
-        if existing
-          log.debug \
-            "update service '#{i.composer.namespace}/#{i.name}' on kubernetes"
-          client.update_service s
-        else
-          log.debug \
-            "create service '#{i.composer.namespace}/#{i.name}' on kubernetes"
-          client.create_service s
-        end
-      end
     end
   end
 end

data/lib/picsolve_docker_builder/helpers/repository.rb

@@ -6,18 +6,104 @@ module PicsolveDockerBuilder
     # (as they are not very well defined)
     class Repository
       include PicsolveDockerBuilder::Base
+
+      attr_reader :tag
+
       def initialize(name)
         parse_input(name)
       end
 
       def parse_input(input)
-        input.split('/')
+        split = input.split('/')
+        if split.length == 1
+          split_length_1(split)
+        elsif split.length == 2
+          split_length_2(split)
+        elsif  split.length == 3
+          split_length_3(split)
+        end
+      end
+
+      def split_length_1(split)
+        parse_repo_tag(split[0])
+      end
+
+      def split_length_2(split)
+        if dot?(split[0])
+          if double_dot?(split[1])
+            # registry1.com/image1:tag
+            @registry = split[0]
+            parse_repo_tag(split[1])
+          else
+            # registry1.com/image1
+            @registry = split[0]
+            parse_repo_tag(split[1])
+          end
+        else
+          if double_dot?(split[1])
+            # user1/image1:tag1
+            @user = split[0]
+            parse_repo_tag(split[1])
+          else
+            # user1/image1
+            @user = split[0]
+            parse_repo_tag(split[1])
+          end
+        end
+      end
+
+      def split_length_3(split)
+        # registry1.com/user1/image1:tag1
+        # registry1.com:8080/user1/image1:tag1
+        @registry = split[0]
+        @user = split[1]
+        parse_repo_tag(split[2])
+      end
+
+      # Methods to check special characters
+      def slash?(input)
+        input.include? '/'
+      end
+
+      def double_dot?(input)
+        input.include? ':'
+      end
+
+      def dot?(input)
+        input.include? '.'
+      end
+
+      # Methods that returns the user name
+      def name
+        if @user.nil?
+          return @repo
+        else
+          return "#{@user}/#{@repo}"
+        end
+      end
+
+      # Methods that returns default values
+      def repo
+        @repo || 'image1'
+      end
+
+      def tag
+        @tag || 'latest'
+      end
+
+      def registry
+        @registry || :default
       end
 
-      def name=(_name)
+      def user
+        @user || 'library'
       end
 
-      def repo_tag
+      # Method to parse the repo tag
+      def parse_repo_tag(str)
+        split = str.split(':')
+        @repo = split[0]
+        @tag = split[1] if split.length > 1
       end
     end
   end

data/lib/picsolve_docker_builder/helpers/ssh_forward.rb

@@ -12,7 +12,11 @@ module PicsolveDockerBuilder
       end
 
       def connection
-        @connection ||= Net::SSH.start(ssh_host, ssh_user)
+        @connection ||= Net::SSH.start(
+          ssh_host,
+          ssh_user,
+          port: ssh_port
+        )
       end
 
       def bind_port
@@ -59,6 +63,10 @@ module PicsolveDockerBuilder
         options[:ssh_host]
       end
 
+      def ssh_port
+        options[:ssh_port] || 22
+      end
+
       def remote_host
         options[:remote_host]
       end