marathon_deploy 0.0.1

data/examples/run.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+ruby -I.:lib deploy.rb  -v -f jondeploy.yaml

data/examples/testout.json

@@ -0,0 +1,68 @@
+{
+  "args": [
+    "properties.py && FOO=\"`cat /opt/etc/public-search-germany-webapp.properties | grep -v '^\\s*#' | grep -E '.+=.+'`\"; echo $FOO; echo; env $FOO; echo; env $FOO hostName=$HOSTNAME backend.instance.name=$HOSTNAME java -Dhttp.port=8080 -Dajp.port=8009 -Dinstance.confdir=file:///opt/etc -Dbackend.logdir=$MESOS_SANDBOX -Dlog.rootDir=$MESOS_SANDBOX -DlogDir=$MESOS_SANDBOX $CMD_OPTS -Xmx$JAVA_XMX -Xms$JAVA_XMS -jar $MESOS_SANDBOX/public-search-germany-webapp*.jar"
+  ],
+  "cmd": null,
+  "container": {
+    "docker": {
+      "image": "dockerregistry.mobile.rz/mobile-java8:latest",
+      "network": "BRIDGE",
+      "portMappings": [
+        {
+          "containerPort": 8080,
+          "hostPort": 0,
+          "protocol": "tcp"
+        },
+        {
+          "containerPort": 8009,
+          "hostPort": 0,
+          "protocol": "tcp"
+        }
+      ]
+    },
+    "type": "DOCKER"
+  },
+  "cpus": 0.1,
+  "env": {
+    "APPLICATION_NAME": "public-search-germany-webapp",
+    "CMD_OPTS": "",
+    "CONFIG_ASSEMBLER_BASE_URL": "http://mobile-config-assembler.service.consul/config-assembler",
+    "DATACENTER_NUMBER": "44",
+    "JAVA_XMS": "252m",
+    "JAVA_XMX": "504m",
+    "SERVERCLASS_NAME": "pubse",
+    "SERVICE_8009_NAME": "public-search-germany-webapp",
+    "SERVICE_8009_TAGS": "ajp",
+    "SERVICE_8080_CHECK_SCRIPT": "curl --fail --silent $HOST_IP:$SERVICE_PORT/fz/release-info",
+    "SERVICE_8080_NAME": "public-search-germany-webapp",
+    "SERVICE_8080_TAGS": "http,haproxy"
+  },
+  "healthChecks": [
+    {
+      "gracePeriodSeconds": 30,
+      "intervalSeconds": 10,
+      "maxConsecutiveFailures": 3,
+      "portIndex": 0,
+      "protocol": "TCP",
+      "timeoutSeconds": 30
+    },
+    {
+      "gracePeriodSeconds": 30,
+      "intervalSeconds": 10,
+      "maxConsecutiveFailures": 3,
+      "path": "/fz/release-info",
+      "portIndex": 0,
+      "protocol": "HTTP",
+      "timeoutSeconds": 30
+    }
+  ],
+  "id": "somename",
+  "instances": 4,
+  "mem": 512,
+  "storeUrls": [
+    "http://maven-download.mobile.rz/maven/hosted-mobile-deployment-productive-releases/de/mobile/public-search-germany-webapp/ecs-1558-gb70c745/public-search-germany-webapp-ecs-1558-gb70c745.jar"
+  ],
+  "uris": [
+
+  ]
+}

data/examples/testout.yaml

@@ -0,0 +1,54 @@
+---
+args:
+- properties.py && FOO="`cat /opt/etc/public-search-germany-webapp.properties | grep
+  -v '^\s*#' | grep -E '.+=.+'`"; echo $FOO; echo; env $FOO; echo; env $FOO hostName=$HOSTNAME
+  backend.instance.name=$HOSTNAME java -Dhttp.port=8080 -Dajp.port=8009 -Dinstance.confdir=file:///opt/etc
+  -Dbackend.logdir=$MESOS_SANDBOX -Dlog.rootDir=$MESOS_SANDBOX -DlogDir=$MESOS_SANDBOX
+  $CMD_OPTS -Xmx$JAVA_XMX -Xms$JAVA_XMS -jar $MESOS_SANDBOX/public-search-germany-webapp*.jar
+cmd: 
+container:
+  docker:
+    image: dockerregistry.mobile.rz/mobile-java8:latest
+    network: BRIDGE
+    portMappings:
+    - containerPort: 8080
+      hostPort: 0
+      protocol: tcp
+    - containerPort: 8009
+      hostPort: 0
+      protocol: tcp
+  type: DOCKER
+cpus: 0.1
+env:
+  APPLICATION_NAME: public-search-germany-webapp
+  CMD_OPTS: ''
+  CONFIG_ASSEMBLER_BASE_URL: http://mobile-config-assembler.service.consul/config-assembler
+  DATACENTER_NUMBER: '44'
+  JAVA_XMS: 252m
+  JAVA_XMX: 504m
+  SERVERCLASS_NAME: pubse
+  SERVICE_8009_NAME: public-search-germany-webapp
+  SERVICE_8009_TAGS: ajp
+  SERVICE_8080_CHECK_SCRIPT: curl --fail --silent $HOST_IP:$SERVICE_PORT/fz/release-info
+  SERVICE_8080_NAME: public-search-germany-webapp
+  SERVICE_8080_TAGS: http,haproxy
+healthChecks:
+- gracePeriodSeconds: 30
+  intervalSeconds: 10
+  maxConsecutiveFailures: 3
+  portIndex: 0
+  protocol: TCP
+  timeoutSeconds: 30
+- gracePeriodSeconds: 30
+  intervalSeconds: 10
+  maxConsecutiveFailures: 3
+  path: "/fz/release-info"
+  portIndex: 0
+  protocol: HTTP
+  timeoutSeconds: 30
+id: public-search-germany-webapp
+instances: 99
+mem: 512
+storeUrls:
+- http://maven-download.mobile.rz/maven/hosted-mobile-deployment-productive-releases/de/mobile/public-search-germany-webapp/ecs-1558-gb70c745/public-search-germany-webapp-ecs-1558-gb70c745.jar
+uris: []

data/input.txt

@@ -0,0 +1,66 @@
+{
+    "args": [
+        "properties.py && FOO=\"`cat /opt/etc/public-search-germany-webapp-%FIRST_VAR%.properties | grep -v '^\\s*#' | grep -E '.+=.+'`\"; echo $FOO; echo; env $FOO; echo; env %SECOND_VAR%$FOO hostName=$HOSTNAME backend.instance.name=$HOSTNAME java -Dhttp.port=8080 -Dajp.port=8009 -Dinstance.confdir=file:///opt/etc -Dbackend.logdir=$MESOS_SANDBOX -Dlog.rootDir=$MESOS_SANDBOX -DlogDir=$MESOS_SANDBOX $CMD_OPTS -Xmx$JAVA_XMX -Xms$JAVA_XMS -jar $MESOS_SANDBOX/public-search-germany-webapp*.jar"
+    ],
+    "cmd": null,
+    "container": {
+        "docker": {
+            "image": "dockerregistry.mobile.rz/mobile-java8:latest",
+            "network": "BRIDGE",
+            "portMappings": [
+                {
+                    "containerPort": 8080,
+                    "hostPort": 0,
+                    "protocol": "tcp"
+                },
+                {
+                    "containerPort": 8009,
+                    "hostPort": 0,
+                    "protocol": "tcp"
+                }
+            ]
+        },
+        "type": "DOCKER"
+    },
+    "cpus": 0.1,
+    "env": {
+        "APPLICATION_NAME": "public-search-germany-webapp",
+        "CMD_OPTS": "",
+        "CONFIG_ASSEMBLER_BASE_URL": "http://mobile-config-assembler.service.consul/config-assembler",
+        "DATACENTER_NUMBER": "44",
+        "JAVA_XMS": "252m",
+        "JAVA_XMX": "504m",
+        "SERVERCLASS_NAME": "pubse",
+        "SERVICE_8009_NAME": "public-search-germany-webapp",
+        "SERVICE_8009_TAGS": "ajp",
+        "SERVICE_8080_CHECK_SCRIPT": "curl --fail --silent $HOST_IP:%SERVICE_PORT%/fz/release-info",
+        "SERVICE_8080_NAME": "public-search-germany-webapp",
+        "SERVICE_8080_TAGS": "http,haproxy"
+    },
+    "healthChecks": [
+        {
+            "gracePeriodSeconds": 30,
+            "intervalSeconds": 10,
+            "maxConsecutiveFailures": 3,
+            "portIndex": 0,
+            "protocol": "TCP",
+            "timeoutSeconds": 30
+        },
+        {
+            "gracePeriodSeconds": 30,
+            "intervalSeconds": 10,
+            "maxConsecutiveFailures": 3,
+            "path": "/%CONTEXT%%TRICKY%/release-info",
+            "portIndex": 0,
+            "protocol": "HTTP",
+            "timeoutSeconds": 30
+        }
+    ],
+    "id": "public-search-germany-webapp",
+    "instances": 4,
+    "mem": 512,
+    "storeUrls": [
+        "http://maven-download.mobile.rz/maven/hosted-mobile-deployment-productive-releases/de/mobile/public-search-germany-webapp/ecs-1558-gb70c745/public-search-germany-webapp-ecs-1558-gb70c745.jar"
+    ],
+    "uris": []
+}

data/lib/marathon_deploy.rb

@@ -0,0 +1,5 @@
+require "marathon_deploy/version"
+
+module MarathonDeploy
+  # Your code goes here...
+end

data/lib/marathon_deploy/application.rb

@@ -0,0 +1,85 @@
+require 'marathon_deploy/marathon_defaults'
+require 'marathon_deploy/yaml_json'
+require 'marathon_deploy/error'
+require 'marathon_deploy/utils'
+
+class Application
+  
+  attr_reader :json, :id
+  attr_accessor :envs
+  
+  def initialize(deployfile)
+    
+    if (!File.exist?(File.join(Dir.pwd,deployfile)))
+      message = "#{deployfile} not found in current directory #{File.join(Dir.pwd)}"
+      raise Error::IOError, message, caller
+    end
+
+    extension = File.extname(deployfile)
+    
+    case extension
+      when '.json'
+        @json = YamlJson.read_json(deployfile)
+      when '.yaml'
+        @json = YamlJson.yaml2json(deployfile)
+      else
+        message = "File extension #{extension} is not supported for deployment file #{deployfile}"
+        raise Error::UnsupportedFileExtension, message, caller
+    end 
+    
+    missing_attributes = MarathonDefaults.missing_attributes(@json) 
+    
+    if(!missing_attributes.empty?)
+      message = "#{deployfile} is missing required marathon API attributes: #{missing_attributes.join(',')}"
+      raise Error::MissingMarathonAttributesError, message, caller
+    end
+    
+    missing_envs = MarathonDefaults.missing_envs(@json)
+    if(!missing_envs.empty?)
+      message = "#{deployfile} is missing required environment variables: #{missing_envs.join(',')}"
+      raise Error::MissingMarathonAttributesError, message, caller
+    end
+     
+    @deployfile = deployfile                
+    @json =  Utils.deep_symbolize(@json)    
+    add_identifier
+   
+  end
+  
+  def overlay_preproduction_settings
+    @json = MarathonDefaults.overlay_preproduction_settings(@json)
+  end
+  
+  def add_identifier
+    random = Utils.random
+    # Time.now.to_i
+    json[:env]['UNIQUE_ID'] = "#{id}_#{random}"
+  end
+  
+  def to_s
+    return id
+  end
+  
+  def id
+    if (@json[:id])
+      return @json[:id]
+    end
+  end
+  
+  def add_envs(envs)
+    if (envs.is_a?(Hash))
+      envs.each do |key,value|
+        @json[:env][key] = value
+      end 
+    else
+      raise Error::BadFormatError, "argument must be a hash", caller
+    end
+  end
+    
+  def instances
+    if (@json[:instances])
+      return @json[:instances]
+    end    
+  end  
+     
+end

data/lib/marathon_deploy/deployment.rb

@@ -0,0 +1,269 @@
+require 'marathon_deploy/http_util'
+require 'marathon_deploy/utils'
+require 'marathon_deploy/marathon_defaults'
+require 'timeout'
+
+class Deployment
+  
+  DEPLOYMENT_RECHECK_INTERVAL = MarathonDefaults::DEPLOYMENT_RECHECK_INTERVAL
+  DEPLOYMENT_TIMEOUT = MarathonDefaults::DEPLOYMENT_TIMEOUT
+  HEALTHY_WAIT_TIMEOUT = MarathonDefaults::HEALTHY_WAIT_TIMEOUT
+  HEALTHY_WAIT_RECHECK_INTERVAL = MarathonDefaults::HEALTHY_WAIT_RECHECK_INTERVAL
+  
+  attr_reader :url, :application, :deploymentId
+  
+  def initialize(url, application)
+    raise ArgumentError, "second argument to deployment object must be an Application", caller unless (!application.nil? && application.class == Application)
+    raise Error::BadURLError, "invalid url => #{url}", caller if (!HttpUtil.valid_url(url))    
+    @url = HttpUtil.clean_url(url)
+    @application = application
+  end
+  
+  def timeout
+    return DEPLOYMENT_TIMEOUT
+  end
+  
+  def healthcheck_timeout
+    return HEALTHY_WAIT_TIMEOUT
+  end
+  
+  def versions  
+    if (!applicationExists?)  
+      response = HttpUtil.get(@url + MarathonDefaults::MARATHON_APPS_REST_PATH + @application.id + '/versions')  
+      response_body = Utils.response_body(response)
+      return response_body[:versions]
+    else
+      return Array.new
+    end
+  end
+     
+  def wait_for_deployment_id(message = "Deployment with deploymentId #{@deploymentId} in progress")
+      startTime = Time.now
+      deployment_seen = false  
+      Timeout::timeout(DEPLOYMENT_TIMEOUT) do
+        while running_for_deployment_id?
+
+          deployment_seen = true
+          #response = list_all
+          #STDOUT.print "." if ( $LOG.level == 1 )
+          elapsedTime = '%.2f' % (Time.now - startTime)
+          $LOG.info(message + " (elapsed time #{elapsedTime}s)")
+          deployments = deployments_for_deployment_id
+          deployments.each do |item|
+            $LOG.debug(deployment_string(item))
+          end   
+          sleep(DEPLOYMENT_RECHECK_INTERVAL)
+        end        
+        #STDOUT.puts "" if ( $LOG.level == 1 )
+        if (deployment_seen)
+          elapsedTime = '%.2f' % (Time.now - startTime)
+          $LOG.info("Deployment with deploymentId #{@deploymentId} ended (Total time #{elapsedTime}s )")  
+        end
+      end    
+  end
+  
+  def wait_for_application(message = "Deployment of application #{@application.id} in progress")
+      deployment_seen = false  
+      Timeout::timeout(DEPLOYMENT_TIMEOUT) do
+        while running_for_application_id?
+          deployment_seen = true
+          #response = list_all
+          #STDOUT.print "." if ( $LOG.level == 1 )
+          $LOG.info(message)
+          deployments_for_application_id.each do |item|
+            $LOG.debug(deployment_string(item))
+          end
+          #$LOG.debug(JSON.pretty_generate(JSON.parse(response.body)))       
+          sleep(DEPLOYMENT_RECHECK_INTERVAL)
+        end                
+        #STDOUT.puts "" if ( $LOG.level == 1 )
+        if (deployment_seen)
+          $LOG.info("Deployment of application #{@application.id} ended")  
+        end
+      end    
+  end
+  
+  def wait_until_healthy    
+    Timeout::timeout(HEALTHY_WAIT_TIMEOUT) do
+      loop do
+        break if (!health_checks_defined?)
+        sick = get_alive("false")
+        if (!sick.empty?)
+          $LOG.info("#{sick.size}/#{@application.instances} instances are not healthy => " + sick.join(','))
+        else
+          healthy = get_alive("true")
+          if (healthy.size == @application.instances)
+            $LOG.info("#{healthy.size}/#{@application.instances} instances are healthy => " + healthy.join(','))
+            break
+          else
+            $LOG.info("#{healthy.size}/#{@application.instances} healthy instances seen, retrying")
+          end
+        end      
+        sleep(HEALTHY_WAIT_RECHECK_INTERVAL)
+      end                         
+    end  
+  end
+    
+  def cancel(deploymentId,force=false)
+    raise Error::BadURLError, "deploymentId must be specified to cancel deployment", caller if (deploymentId.empty?)
+    if (running_for_deployment_id?(deploymentId))
+      response = HttpUtil.delete(@url + MarathonDefaults::MARATHON_DEPLOYMENT_REST_PATH + deploymentId + "?force=#{force}")
+      $LOG.debug("Cancellation response [#{response.code}] => " + JSON.pretty_generate(JSON.parse(response.body)))
+    end
+    return response
+  end
+  
+  def applicationExists?
+    response = list_app
+    if (response.code.to_i == 200)
+      return true
+    end
+      return false
+  end
+       
+  def create_app
+    response = HttpUtil.post(@url + MarathonDefaults::MARATHON_APPS_REST_PATH,@application.json)
+    @deploymentId = get_deployment_id
+    return response
+  end
+  
+  def update_app(force=false)
+    url = @url + MarathonDefaults::MARATHON_APPS_REST_PATH + @application.id
+    url += force ? '?force=true' : ''
+    $LOG.debug("Updating app #{@application.id}  #{url}")
+    response = HttpUtil.put(url,@application.json)    
+    @deploymentId = Utils.response_body(response)[:deploymentId]
+    return response
+    
+  end
+  
+  def rolling_restart
+    url = @url + MarathonDefaults::MARATHON_APPS_REST_PATH + @application.id + '/restart'
+    $LOG.debug("Calling marathon api with url: #{url}") 
+    response = HttpUtil.post(url,{})
+    $LOG.info("Restart of #{@application.id} returned status code: #{response.code}")
+    $LOG.info(JSON.pretty_generate(JSON.parse(response.body)))
+  end  
+  
+  def health_checks_defined?
+    response = list_app
+    response_body = Utils.response_body(response)
+    return response_body[:app][:healthChecks].size == 0 ? false : true
+  end  
+  
+  ####### PRIVATE METHODS ##########
+  private
+
+  def get_alive(value)        
+    state = Array.new
+    
+    if (health_checks_defined?)     
+      response = list_app
+      response_body = Utils.response_body(response)
+        if (response_body[:app].empty?)
+          raise Error::DeploymentError, "Marathon returned an empty app json object", caller
+        else
+          get_healthcheck_results.flatten.each do |result|
+            next if result.nil?
+            alive = result[:alive].to_s
+            taskId = result[:taskId].to_s              
+            if (!alive.nil? && !taskId.nil?)
+              state << taskId if (alive == value)
+            end          
+          end         
+        end
+    else
+      $LOG.info("No health checks defined. Cannot determine application health of #{@application.id}.")    
+    end
+    return state
+  end
+  
+  def get_task_ids
+    response = list_app
+    response_body = Utils.response_body(response)
+    return response_body[:app][:tasks].collect { |task| task[:id]}
+  end
+  
+  def get_healthcheck_results
+    response = list_app
+    response_body = Utils.response_body(response)
+    return response_body[:app][:tasks].collect { |task| task[:healthCheckResults]}
+  end
+    
+  def get_deployment_id
+    response = list_app
+    payload = Utils.response_body(response)
+    return payload[:app][:deployments].first[:id] unless (payload[:app].nil?)
+    return nil
+  end
+    
+  def list_all
+    HttpUtil.get(@url + MarathonDefaults::MARATHON_DEPLOYMENT_REST_PATH)
+  end 
+  
+  def running_for_application_id?
+    if (deployment_running? && !deployments_for_application_id.empty?)
+      return true
+    end
+      return false
+  end
+  
+  def running_for_deployment_id?
+    if (deployment_running? && !deployments_for_deployment_id.empty?)
+      return true
+    end
+      return false
+  end
+  
+  def deployment_running?
+    response = list_all
+    body = JSON.parse(response.body)
+    return false if body.empty?
+    return true
+  end   
+   
+  def get_deployment_ids
+    response = list_all
+    payload = JSON.parse(response.body)
+    return payload.collect { |d| d['id'] }
+  end
+  
+  def list_app
+    HttpUtil.get(@url + MarathonDefaults::MARATHON_APPS_REST_PATH + @application.id)
+  end
+  
+  # DONT USE: the response seems to be broken in marathon for /v2/apps/application-id/tasks
+  #def get_tasks
+  #  HttpUtil.get(@url + MarathonDefaults::MARATHON_APPS_REST_PATH + @application.id + '/tasks')
+  #end
+  
+  def deployment_string(deploymentJsonObject)  
+    string = "\n" + "+-" * 25 + " DEPLOYMENT INFO  " + "+-" * 25 + "\n"
+    deploymentJsonObject.sort.each do |k,v|
+      case v
+      when String
+        string += k + " => " + v + "\n"
+      when Fixnum
+        string += k + " => " + v.to_s + "\n"
+      when Array
+        string += k + " => " + v.join(',') + "\n"
+      else
+        string += "#{k} + #{v}\n"
+      end
+    end  
+   return string 
+  end
+    
+  def deployments_for_deployment_id
+    response = list_all
+    payload = JSON.parse(response.body)
+    return payload.find_all { |d| d['id'] == @deploymentId }
+  end
+  
+  def deployments_for_application_id
+    response = list_all
+    payload = JSON.parse(response.body)
+    return payload.find_all { |d| d['affectedApps'].include?('/' + @application.id) }
+  end
+  
+end