picsolve_docker_builder 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 15bcfea24b318c9f934f15dcfdd65d5ed4be8d99
-  data.tar.gz: cb6e0b70bc6ca5ff68ebb873a73266bfe763c441
+  metadata.gz: 47c0bcbcd965d2446b0c1c9254a67bbd23a9569d
+  data.tar.gz: 1a4f008a15c22534b990a77bca137fb4b04c0464
 SHA512:
-  metadata.gz: 55b9b50d6c5409344b7fa6e6c40e06ad9f4c7904ee2e30c9c47b0b6456f522e5ac274fc24b510edb5e89f4e6d4d24bac274d39e247d428d81fdf40b83df6268b
-  data.tar.gz: 5b306bfde7d4bd052037da1e16c8de9a9c27c7acfa917bace4838f72ab8fa5a74fe162d93feadb9177cd2f006ce0974aef586f96f859100c56c60004b6a75f80
+  metadata.gz: 34289ce6a0dfea6cb6ac0b4bf23f76bad8a7300b91e3c75dbb5d63ece96f17d61e5dd9dd08d0997cf668948e98d5bf4728065c9915f8714c6ac0fe1dae473836
+  data.tar.gz: d042eaacd90ded220dd645d5763dccc4cc0ff37e5bdd1c25f742ead7f1ed2483ab44f0d33cc6eff9ecd6d5573834979cd76554054e61452b656ca8819bf3be13

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Change Log
 
+## [0.2.0] - 2015-08-26
+### Added
+- Upgrade kubernetes api to v1
+- Create integration tests for most used cases
+- Implement cleanup of orphaned pods (fixes DE128)
+- Implement alpha support of new config file for composing (support of env vars per stage)
+
 ## [0.1.1] - 2015-07-03
 ### Fixed
 - Flush buffers for better jenkins experience

data/README.md

@@ -3,9 +3,52 @@
 ## Example config file `.docker-builder.yml`
 
 ```yaml
+---
+compose:
+  app_name: hello_world
+  clusters:
+    - dev-1b.kubernetes.picsolve.net
+    - dev-1c.kubernetes.picsolve.net
+  containers:
+    container1:
+      image: docker.picsolve.net/image1
+      environment:
+        default:
+          - COOL=${variable:cool}
+          - TERM=xterm
+          - "DB_USER=#{variable:db_username}"
+          - "DB_PASSWORD=#{secret:database:password}"
+          - "DB_NAME=#{stage}-#{app_name}"
+        qa:
+          - ENABLE_QA=1
+        ci:
+          - ENABLE_CI=1
+      ports:
+       - container_port: 9000
+         service_port: 9000
+      requirements:
+        bucket:
+          type: s3bucket
+        database:
+          type: postgres
+  variables:
+    default:
+      cool: yes
+      db_username: dbuser_default
+    ci:
+      db_username: dbuser_ci
+    qa:
+      db_username: dbuser_qa
 docker:
   base_image: base-image-name
   image_name: dest-image-name
+  build_environment:
+    - TERM=xterm
+  build_requirements:
+    bucket:
+      type: s3bucket
+    database:
+      type: postgres
   dockerfile_hooks:
     asset_build:
       early: |
@@ -20,8 +63,8 @@ docker:
 scala:
   sbt:
     build_task:
-     - clean
-     - universal:packageZipTarball
+      - clean
+      - "universal:packageZipTarball"
 ```
 
 ## Release process

data/integration/compose_hello_world_new_config/.docker-builder.yml

@@ -0,0 +1,18 @@
+compose:
+  app_name: hello_world_test_new
+  clusters:
+  - dev-1b.kubernetes.picsolve.net
+  containers:
+    helloworld:
+      image: docker.picsolve.net/hello-world
+      environment:
+        default:
+          - DEFAULT=true
+          - ENV=default
+        qa:
+          - ENV=qa
+        ci:
+          - ENV=ci
+      ports:
+       - container_port: 80
+         service_port: 80

data/integration/compose_hello_world_new_config/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/integration/compose_hello_world_new_config/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gem 'picsolve_docker_builder', path: '../../'

data/integration/compose_hello_world_new_config/Rakefile

@@ -0,0 +1,3 @@
+# Load upstream tasks
+spec = Gem::Specification.find_by_name 'picsolve_docker_builder'
+load "#{spec.gem_dir}/lib/tasks/compose.rake"

data/integration/compose_hello_world_old_config/.docker-builder.yml

@@ -0,0 +1,4 @@
+compose:
+  app_name: hello_world_test
+  clusters:
+  - dev-1b.kubernetes.picsolve.net

data/integration/compose_hello_world_old_config/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/integration/compose_hello_world_old_config/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gem 'picsolve_docker_builder', path: '../../'

data/integration/compose_hello_world_old_config/Rakefile

@@ -0,0 +1,3 @@
+# Load upstream tasks
+spec = Gem::Specification.find_by_name 'picsolve_docker_builder'
+load "#{spec.gem_dir}/lib/tasks/compose.rake"

data/integration/compose_hello_world_old_config/docker-compose.yml

@@ -0,0 +1,4 @@
+helloworld:
+  image: docker.picsolve.net/hello-world
+  ports:
+    - 80:80

data/integration/integration_compose_spec.rb

@@ -0,0 +1,25 @@
+require_relative 'spec_helper'
+
+describe 'Integration tests' do
+  before(:each) do
+    docker_needed!
+  end
+  describe 'compose:deploy' do
+    describe 'hello world image (old config)' do
+      let :path do
+        'integration/compose_hello_world_old_config'
+      end
+      it 'runs successfully' do
+        bundle_exec 'compose:deploy'
+      end
+    end
+    describe 'hello world image (new config)' do
+      let :path do
+        'integration/compose_hello_world_new_config'
+      end
+      it 'runs successfully' do
+        bundle_exec 'compose:deploy'
+      end
+    end
+  end
+end

data/integration/integration_helpers.rb

@@ -0,0 +1,45 @@
+require 'docker'
+require 'open3'
+
+# Helpers for integration tests
+module IntegrationHelpers
+  def sh(command)
+    env = {
+      'PATH' => ENV['PATH'],
+      'GEM_PATH' => ENV['GEM_PATH'],
+      'HOME' => ENV['HOME'],
+      'SSH_AUTH_SOCK' => ENV['SSH_AUTH_SOCK']
+    }
+    Open3.popen3(
+      env,
+      command,
+      unsetenv_others: true
+    ) do |stdin, stdout, stderr, wait_thr|
+      stdin.close
+      stdout.each_line do |line|
+        $stdout.write(line)
+        $stdout.flush
+      end
+      stderr.each_line do |line|
+        $stderr.write(line)
+        $stderr.flush
+      end
+      exit_status = wait_thr.value
+      fail 'Exit code is not zero' if exit_status != 0
+    end
+  end
+
+  def bundle_exec(task)
+    Dir.chdir(path) do
+      sh 'rm -f Gemfile.lock'
+      sh 'bundle install'
+      sh "bundle exec rake #{task}"
+    end
+  end
+
+  def docker_needed!
+    Docker.info.inspect
+  rescue Excon::Errors::Error
+    skip 'No docker daemon available'
+  end
+end

data/integration/integration_play_spec.rb

@@ -1,50 +1,16 @@
-require 'picsolve_docker_builder/helpers/repository'
-require 'docker'
-require 'open3'
-
-def sh(command)
-  env = {
-    'PATH' => ENV['PATH'],
-    'GEM_PATH' => ENV['GEM_PATH']
-  }
-  Open3.popen3(
-    env,
-    command,
-    unsetenv_others: true
-  ) do |stdin, stdout, stderr, wait_thr|
-    stdin.close
-    stdout.each_line do |line|
-      $stdout.write(line)
-      $stdout.flush
-    end
-    stderr.each_line do |line|
-      $stderr.write(line)
-      $stderr.flush
-    end
-    exit_status = wait_thr.value
-    fail 'Exit code is not zero' if exit_status != 0
-  end
-end
+require_relative 'spec_helper'
 
 describe 'Integration tests' do
   before(:each) do
-    begin
-      Docker.info.inspect
-    rescue Excon::Errors::Error
-      skip 'No docker daemon available'
-    end
+    docker_needed!
   end
   describe 'docker build' do
-    describe 'play hello world' do
+    describe 'play app hello world' do
       let :path do
         'integration/play_hello_world'
       end
       it 'builds successfully' do
-        Dir.chdir(path) do
-          sh 'rm -f Gemfile.lock'
-          sh 'bundle install'
-          sh 'bundle exec rake docker:build'
-        end
+        bundle_exec 'docker:build'
       end
     end
   end

data/integration/lib_hello_world/.gitignore

@@ -0,0 +1,12 @@
+logs
+/.ivy2
+/.m2
+/.sbt
+target
+/.idea
+/.idea_modules
+/.classpath
+/.project
+/.settings
+/RUNNING_PID
+Gemfile.lock

data/integration/lib_hello_world/README

@@ -0,0 +1,4 @@
+This is your new library module
+=================================
+
+This file will be packaged with your application, when using `activator dist`.

data/integration/lib_hello_world/app/picsolve/helloworld/Person.scala

@@ -0,0 +1,8 @@
+package picsolve.helloworld
+
+import java.util.Date
+
+/**
+ * Created by nicolas on 17/07/15.
+ */
+case class Person(name: String, birthDay: Date)

data/integration/lib_hello_world/build.sbt

@@ -0,0 +1,42 @@
+releaseSettings
+
+organization := "picsolve"
+
+val artifactId = "lib.hello_world"
+
+name := artifactId
+
+moduleName := artifactId
+
+lazy val root = (project in file(".")).enablePlugins(PlayScala)
+
+scalaVersion := "2.11.6"
+
+lazy val picsolveArtifactoryRepository = "http://artifactory.dev.services.picsolve.net:8081/artifactory"
+
+lazy val picsolveResolvers = Seq (
+  "Picsolve releases repository" at s"$picsolveArtifactoryRepository/libs-release/",
+  "Picsolve snapshots repository" at s"$picsolveArtifactoryRepository/libs-snapshot/"
+)
+
+resolvers += "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/"
+
+resolvers ++= picsolveResolvers
+
+resolvers += "scalaz-bintray" at "http://dl.bintray.com/scalaz/releases"
+
+
+libraryDependencies ++= Seq(
+  specs2 % Test,
+  "com.typesafe.scala-logging"  %% "scala-logging" % "3.1.0"
+)
+
+
+publishTo <<= version {
+  (v: String) =>
+    val repo = "http://artifactory.dev.services.picsolve.net:8081/artifactory/"
+    if (v.trim.endsWith("SNAPSHOT"))
+      Some("snapshots" at repo + "libs-snapshot-local")
+    else
+      Some("releases" at repo + "libs-release-local")
+}

data/integration/lib_hello_world/conf/logback.xml

@@ -0,0 +1,22 @@
+<configuration>
+    
+  <conversionRule conversionWord="coloredLevel" converterClass="play.api.Logger$ColoredLevel" />
+
+  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder>
+      <pattern>%coloredLevel - %logger - %message%n%xException</pattern>
+    </encoder>
+  </appender>
+
+  <!--
+    The logger name is typically the Java/Scala package name.
+    This configures the log level to log at for a package and its children packages.
+  -->
+  <logger name="play" level="INFO" />
+  <logger name="application" level="DEBUG" />
+
+  <root level="ERROR">
+    <appender-ref ref="STDOUT" />
+  </root>
+
+</configuration>

data/integration/lib_hello_world/project/build.properties

@@ -0,0 +1,4 @@
+#Activator-generated Properties
+#Mon Jun 29 12:18:44 BST 2015
+template.uuid=49a31253-85ed-4c4f-9041-f2f030591a8d
+sbt.version=0.13.8

data/integration/lib_hello_world/project/plugins.sbt

@@ -0,0 +1,4 @@
+// The Play plugin
+addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.4.0")
+
+addSbtPlugin("com.github.gseitz" %% "sbt-release" % "0.8.4")

data/integration/lib_hello_world/test/PersonSpec.scala

@@ -0,0 +1,24 @@
+import java.util.Date
+
+import org.junit.runner._
+import org.specs2.mutable._
+import org.specs2.runner._
+import picsolve.helloworld.Person
+import play.api.test._
+
+@RunWith(classOf[JUnitRunner])
+class PersonSpec extends Specification {
+
+  "Person" should {
+
+    "return their name correclty" in new WithApplication{
+      val name = "A Name"
+      Person(name,new Date()).name === name
+    }
+
+    "return their birthDay correclty" in new WithApplication{
+      val birthDay = new Date()
+      Person("A Name",birthDay).birthDay === birthDay
+    }
+  }
+}

data/integration/lib_hello_world/version.sbt

@@ -0,0 +1 @@
+version in ThisBuild := "0.0.1-SNAPSHOT"

data/integration/spec_helper.rb

@@ -1,3 +1,5 @@
+require_relative 'integration_helpers'
 RSpec.configure do |config|
+  config.include IntegrationHelpers
   config.mock_with :rspec
 end

data/lib/picsolve_docker_builder/builder/builder.rb

@@ -21,6 +21,7 @@ module PicsolveDockerBuilder
       end
 
       def ports
+        return if @opts[:ports].nil?
         ports = @opts[:ports]
         ports = [ports] unless ports.is_a?(Array)
         ports.map do |port|
@@ -37,7 +38,8 @@ module PicsolveDockerBuilder
       end
 
       def command
-        "[\"#{@opts[:command].join('", "')}\"]"
+        return '' if @opts[:command].nil?
+        "CMD [\"#{@opts[:command].join('", "')}\"]"
       end
 
       def hook(name)
@@ -103,7 +105,7 @@ MAINTAINER #{maintainer}
 #{ports}
 
 # call run script
-CMD #{command}
+#{command}
 
 #{hook(:late)}
 EOS

data/lib/picsolve_docker_builder/composer/composer.rb

@@ -1,5 +1,6 @@
 require 'picsolve_docker_builder/base'
 require 'picsolve_docker_builder/helpers/kubernetes_manager'
+require 'picsolve_docker_builder/helpers/config_manager'
 require 'picsolve_docker_builder/composer/image'
 require 'excon'
 require 'psych'
@@ -24,72 +25,6 @@ module PicsolveDockerBuilder
         @stage = my_stage.downcase
       end
 
-      def deploy_services
-        images.each do |i|
-          i.service.deploy
-        end
-      end
-
-      def deploy_rcs
-        images.each do |i|
-          i.rc.deploy
-        end
-      end
-
-      def wait_for_deployed_rcs
-        wait_timeout = 300
-        wait_sleep = 2
-        wait_beginning = 15
-        wait_count = wait_beginning
-        unready_images = images.dup
-        log.info \
-          'Wait for the new pods to be up and running for at least 20 seconds'
-        sleep wait_beginning
-        loop do
-          unready_images.each do |i|
-            unready_images.delete(i) if i.rc.ready?
-          end
-          log.debug "Still waiting for: #{unready_images.map(&:name)}"
-          $stdout.flush
-          $stderr.flush
-          wait_count += wait_sleep
-          break if wait_count > wait_timeout
-          break if unready_images.length == 0
-          sleep wait_sleep
-        end
-        successful_images = images.dup
-        unready_images.each do |u|
-          successful_images.delete(u)
-        end
-        [successful_images, unready_images]
-      end
-
-      def remove_old_rcs(successful_images)
-        # build list of successful deplyoed images
-        successful_images.each do |i|
-          log.debug "Remove pods for #{i.name}"
-          i.rc.remove_old_rcs
-        end
-      end
-
-      def deploy
-        log.info "start deploying app_name=#{app_name} to stage #{stage}"
-
-        # reset hash
-        @hash = nil
-
-        deploy_services
-
-        deploy_rcs
-
-        successful_images, unready_images = wait_for_deployed_rcs
-
-        remove_old_rcs(successful_images)
-
-        fail "Failed to deploy this service: #{unready_images.map(&:name)}" \
-          if unready_images.length > 0
-      end
-
       def app_name
         config['compose']['app_name'].downcase.gsub(/[^a-z0-9\-]/, '-')
       end
@@ -98,10 +33,6 @@ module PicsolveDockerBuilder
         "#{app_name}-#{stage}".downcase
       end
 
-      def kubernetes
-        @kubernetes ||= PicsolveDockerBuilder::Helpers::KubernetesManager.new
-      end
-
       def hash
         @hash ||= SecureRandom.hex[0..7]
       end
@@ -114,12 +45,51 @@ module PicsolveDockerBuilder
         @images ||= read_images
       end
 
+      def kubernetes_clusters
+        config['compose']['clusters'].map do |cluster|
+          PicsolveDockerBuilder::Helpers::KubernetesManager.new(
+            self,
+            cluster,
+            :ssh_forward, # TODO: support other types here
+          )
+        end
+      end
+
+      def deploy
+        errors = 0
+        kubernetes_clusters.each do |cluster|
+          begin
+            log.info "Deploy to kubernetes cluster #{cluster.host}"
+            cluster.deploy
+            cluster.stop
+          rescue StandardError => e
+            errors += 1
+            log.error "Failed deployment to kubernetes cluster: #{e}"
+            e.backtrace.each do |line|
+              log.error line
+            end
+          end
+        end
+        fail 'deployment to at least one cluster failed' if errors > 0
+      end
+
       def read_images
-        i = []
-        docker_compose.each do |name, config|
-          i << Image.new(name, config, self)
+        # classic config
+        containers = docker_compose
+        unless containers.nil?
+          i = []
+          containers.each do |name, config|
+            i << Image.new(name, config, self)
+          end
+          return i
         end
-        i
+
+        # new approach
+        c = PicsolveDockerBuilder::Helpers::ConfigManager.new(
+          config_path,
+          stage
+        )
+        c.containers(self)
       end
 
       def docker_compose
@@ -139,6 +109,7 @@ module PicsolveDockerBuilder
         Psych.load_file path
       rescue Errno::ENOENT
         log.debug "can not find docker-compose.yml at '#{path}'"
+        nil
       end
 
       def query_registry

data/lib/picsolve_docker_builder/composer/container.rb

@@ -0,0 +1,79 @@
+require 'picsolve_docker_builder/base'
+require 'picsolve_docker_builder/composer/registry'
+
+module PicsolveDockerBuilder
+  module Composer
+    # Represents an container in an namespace
+    class Container
+      include PicsolveDockerBuilder::Base
+      attr_reader :config, :name, :composer
+      def initialize(name, config, composer)
+        log.info "Image name=#{name} config=#{config}"
+        @config = config
+        @name = name
+        @composer = composer
+        log.info "repo_tag_uniqe is #{repo_tag_unique}"
+      end
+
+      def repo_tag
+        config['image']
+      end
+
+      def rc(kubernetes)
+        @rcs = {} if @rcs.nil?
+        @rcs[kubernetes] = kubernetes.rc(self) \
+          unless @rcs.key?(kubernetes)
+        @rcs[kubernetes]
+      end
+
+      def service(kubernetes)
+        @services = {} if @rcs.nil?
+        @services[kubernetes] = kubernetes.service(self) \
+          unless @services.key?(kubernetes)
+        @services[kubernetes]
+      end
+
+      def environment
+        return nil unless config.key? 'environment'
+        config['environment'].map do |key, value|
+          {
+            'name' => key,
+            'value' => value
+          }
+        end
+      end
+
+      def ports
+        config['ports'].map do |port|
+          {
+            'port' => port['container_port'].to_i
+          }
+        end
+      rescue StandardError
+        []
+      end
+
+      def ports_rc
+        p = []
+        ports.each do |port|
+          p << {
+            'containerPort' => port['port']
+          }
+        end
+        p
+      end
+
+      def repo_tag_hash_unique
+        @repo_tag_hash_unique ||= Registry.repo_tag_unique(config['image'])
+      end
+
+      def repo_tag_unique
+        repo_tag_hash_unique[:tag_unique]
+      end
+
+      def hash
+        repo_tag_hash_unique[:hash]
+      end
+    end
+  end
+end