marathon_deploy 0.0.1

data/lib/marathon_deploy/environment.rb

@@ -0,0 +1,28 @@
+require 'marathon_deploy/marathon_defaults'
+
+class Environment
+  
+  attr_reader :name
+  
+  def initialize(name)
+    if (!name.is_a? String)
+      raise Error::BadFormatError, "argument for environment must be a string", caller      
+    end
+    @name = name.upcase
+  end
+  
+  def marathon_endpoints
+    return 
+  end
+  
+  def is_production?
+    if (@name.casecmp(MarathonDefaults::PRODUCTION_ENVIRONMENT_NAME) == 0)
+      return true
+    end
+    return false
+  end
+  
+  def to_s
+    @name
+  end
+end

data/lib/marathon_deploy/error.rb

@@ -0,0 +1,21 @@
+module Error
+  
+  class MarathonError < StandardError; end
+   
+  class TimeoutError < MarathonError; end
+    
+  class IOError < MarathonError; end
+    
+  class BadURLError < MarathonError; end
+    
+  class UnsupportedFileExtension < MarathonError; end
+    
+  class MissingMarathonAttributesError < MarathonError; end
+    
+  class BadFormatError < MarathonError; end
+  
+  class DeploymentError < MarathonError ; end
+    
+  class UndefinedMacroError < MarathonError ; end     
+
+end

data/lib/marathon_deploy/http_util.rb

@@ -0,0 +1,91 @@
+require 'net/http'
+require 'uri'
+require 'marathon_deploy/error'
+
+module HttpUtil
+
+  def self.put(url,payload)
+    uri = construct_uri url 
+    begin
+      http = Net::HTTP.new(uri.host, uri.port)
+      req = Net::HTTP::Put.new(uri.request_uri)
+      req.body = payload.to_json
+      req["Content-Type"] = "application/json"
+      response = http.request(req)
+    rescue Errno::ECONNREFUSED => e
+      $LOG.error("Error calling marathon api: #{e.message}")
+      exit!
+    end
+    return response
+  end
+  
+  def self.post(url, payload)
+    uri = construct_uri url 
+    begin
+      http = Net::HTTP.new(uri.host, uri.port)
+      req = Net::HTTP::Post.new(uri.request_uri)
+      req.body = payload.to_json
+      req["Content-Type"] = "application/json"
+      response = http.request(req)
+    rescue Errno::ECONNREFUSED => e
+      message = "Error calling marathon api: #{e.message}"
+      $LOG.error(message)
+      raise Error::MarathonError, message, caller
+    end
+    return response
+  end
+  
+  def self.construct_uri(url)
+    raise Error::BadURLError unless (valid_url(url))
+    return URI.parse(url)
+  end
+  
+  def self.delete(url)
+    uri = construct_uri url 
+       begin
+         http = Net::HTTP.new(uri.host, uri.port)
+         req = Net::HTTP::Delete.new(uri.request_uri)         
+         response = http.request(req)
+       rescue Errno::ECONNREFUSED => e
+         message = "Error calling marathon api: #{e.message}"
+         $LOG.error(message)
+         raise Error::MarathonError, message, caller
+       end
+       return response    
+  end
+  
+  def self.clean_url(url)
+    url.sub(/(\/)+$/,'')
+  end
+  
+  def self.get(url)
+    uri = construct_uri url
+    begin
+    http = Net::HTTP.new(uri.host, uri.port)
+    req = Net::HTTP::Get.new(uri.request_uri)
+    req["Content-Type"] = "application/json"
+    response = http.request(req)
+    rescue  Errno::ECONNREFUSED => e
+      message = "Error calling marathon api: #{e.message}"
+      $LOG.error(message)
+      raise Error::MarathonError, message, caller
+    end
+    return  response
+  end
+  
+  def self.valid_url(url)
+    if (url =~ /\A#{URI::regexp}\z/)
+      return true
+    end
+    return false
+  end
+ 
+  def self.print(response)
+    begin
+      puts JSON.pretty_generate(JSON.parse(response.body))
+    rescue
+      puts response
+    end
+  end
+  
+end

data/lib/marathon_deploy/macro.rb

@@ -0,0 +1,71 @@
+require 'marathon_deploy/error'
+
+module Macro
+  
+  MACRO_BOUNDARY = '%%'
+  
+  def self.get_macros(data)
+    macros = Array.new
+    data.each do |line|
+      (macros  << line.scan(/(#{MACRO_BOUNDARY}\w+#{MACRO_BOUNDARY})/)).flatten!
+    end
+    return macros
+  end
+  
+  def self.get_env_keys
+    return ENV.keys
+  end
+  
+  def self.env_defined?(key)
+    if (ENV.has_key?(key) && (!ENV[key].nil? && !ENV[key].empty? ))
+      return true
+    else
+      return false
+    end
+  end
+  
+  def self.get_undefined_macros(macros)
+    raise ArgumentError, "argument must be an array", caller if (!macros.class == Array)
+    undefined = Array.new
+    return macros.select { |m| !has_env?(m) }
+  end
+  
+  def self.has_env?(macro)
+    raise ArgumentError, "argument must be a String", caller if (!macro.class == String)
+    env_name = strip(macro)
+    if (env_defined?(strip(env_name)))
+      return true
+    else
+      return false
+    end
+  end
+  
+  def self.strip(str)
+    return str.gsub(MACRO_BOUNDARY,'')
+  end
+  
+  def self.expand_macros(data)
+    processed = ""
+    macros = get_macros(data)
+    undefined = get_undefined_macros(macros)
+    if (!undefined.empty?)
+      raise Error::UndefinedMacroError, "macros found in deploy file without defined environment variables: #{undefined.uniq.join(',')}", caller
+    end
+  
+    data.each do |line|
+      macros.each { |m| line.gsub!(m,ENV[strip(m)]) }
+      processed += line
+    end
+    return processed  
+  end
+  
+  def self.process_macros(filename)
+    file = File.open(filename,'r')
+    data = file.readlines
+    file.close()
+    return expand_macros(data)
+  end
+  
+  private_class_method :get_macros, :get_env_keys, :strip, :has_env?, :get_undefined_macros, :env_defined?, :expand_macros
+  
+end

data/lib/marathon_deploy/marathon_client.rb

@@ -0,0 +1,79 @@
+require 'marathon_deploy/http_util'
+require 'marathon_deploy/deployment'
+require 'marathon_deploy/utils'
+require 'marathon_deploy/marathon_defaults'
+
+class MarathonClient
+
+  attr_reader :marathon_url, :options
+  attr_accessor :application
+    
+  # TODO:  Options will contain environment, datacenter
+  def initialize(url, options = {})
+    
+    raise Error::BadURLError, "invalid url => #{url}", caller if (!HttpUtil.valid_url(url))
+   
+    @marathon_url = HttpUtil.clean_url(url)
+    @options = options
+    if @options[:username] and @options[:password]
+      @options[:basic_auth] = {
+        :username => @options[:username],
+        :password => @options[:password]
+      }
+      @options.delete(:username)
+      @options.delete(:password)
+    end
+  end
+
+  def deploy
+    
+    deployment = Deployment.new(@marathon_url,application)
+    
+    $LOG.info("Checking if any deployments are already running for application #{application.id}")    
+    begin
+      deployment.wait_for_application("Deployment already running for application #{application.id}")
+    rescue Timeout::Error => e
+      raise Timeout::Error, "Timed out after #{deployment.timeout}s waiting for existing deployment of #{application.id} to finish. Check marathon #{@marathon_url + '/#deployments'} for stuck deployments!", caller
+    end
+    
+    $LOG.info("Starting deployment of #{application.id}")
+
+    # if application with this ID already exists
+    if (deployment.applicationExists?)  
+      $LOG.info("#{application.id} already exists. Performing update.")
+      response = deployment.update_app      
+         
+    # if no application with this ID is seen in marathon
+    else     
+      response = deployment.create_app
+    end
+    
+    if ((300..999).include?(response.code.to_i))
+      $LOG.error("Deployment response body => " + JSON.pretty_generate(JSON.parse(response.body)))
+      raise Error::DeploymentError, "Deployment returned response code #{response.code}", caller
+    end
+    
+    $LOG.info("Deployment started for #{application.id} with deployment id #{deployment.deploymentId}") unless (deployment.deploymentId.nil?)
+    
+    # wait for deployment to finish, according to marathon deployment API call
+    begin
+      deployment.wait_for_deployment_id 
+    rescue Timeout::Error => e
+      $LOG.error("Timed out waiting for deployment of #{application.id} to complete. Canceling deploymentId #{deployment.deploymentId} and rolling back!")
+      deployment.cancel(deployment.deploymentId)
+      raise Timeout::Error, "Deployment of #{application.id} timed out after #{deployment.timeout} seconds", caller
+    end 
+     
+    # wait for all instances with defined health checks to be healthy
+    if (!deployment.health_checks_defined?)
+      $LOG.warn("No health checks were defined for application #{application.id}. No health checking will be performed.")
+    end
+    
+    begin
+      deployment.wait_until_healthy
+    rescue Timeout::Error => e
+      raise Timeout::Error, "Timed out after #{deployment.healthcheck_timeout}s waiting for #{application.instances} instances of #{application.id} to become healthy", caller
+    end
+
+  end
+end

data/lib/marathon_deploy/marathon_defaults.rb

@@ -0,0 +1,83 @@
+require 'marathon_deploy/utils'
+require 'marathon_deploy/error'
+require 'logger'
+
+module MarathonDefaults
+ 
+  DEPLOYMENT_RECHECK_INTERVAL = 3
+  DEPLOYMENT_TIMEOUT = 300
+  HEALTHY_WAIT_TIMEOUT = 300
+  HEALTHY_WAIT_RECHECK_INTERVAL = 3
+  PRODUCTION_ENVIRONMENT_NAME = 'PRODUCTION'
+  DEFAULT_ENVIRONMENT_NAME = 'INTEGRATION'
+  DEFAULT_PREPRODUCTION_MARATHON_ENDPOINTS = ['http://192.168.59.103:8080']
+  DEFAULT_PRODUCTION_MARATHON_ENDPOINTS = ['http://paasmaster46-1.mobile.rz:8080']
+  DEFAULT_DEPLOYFILE = 'deploy.yaml'
+  DEFAULT_LOGFILE = false
+  DEFAULT_LOGLEVEL = Logger::INFO
+  MARATHON_APPS_REST_PATH = '/v2/apps/'
+  MARATHON_DEPLOYMENT_REST_PATH = '/v2/deployments/'
+
+  @@preproduction_override = {
+    :instances => 20,
+    :mem => 512,
+    :cpus => 0.1      
+  } 
+  
+  @@preproduction_env = {
+    :DATACENTER_NUMBER => "44",
+    :JAVA_XMS => "64m",
+    :JAVA_XMX => "128m"
+  }  
+  
+  @@required_marathon_env_variables = %w[
+    DATACENTER_NUMBER
+    APPLICATION_NAME
+  ]
+  
+  #@@required_marathon_attributes = %w[id env container healthChecks args storeUrls].map(&:to_sym)
+  @@required_marathon_attributes = %w[id].map(&:to_sym)
+   
+  def self.missing_attributes(json)
+    json = Utils.symbolize(json)
+    missing = []
+    @@required_marathon_attributes.each do |att|
+      if (!json[att])
+        missing << att 
+      end
+    end
+    return missing
+  end
+  
+  def self.missing_envs(json)
+    json = Utils.symbolize(json)
+    
+    if (!json.key?(:env))
+      raise Error::MissingMarathonAttributesError, "no env attribute found in deployment file", caller 
+    end
+    
+    missing = []
+    @@required_marathon_env_variables.each do |variable|
+      if (!json[:env][variable])
+        missing << variable 
+      end
+    end
+    return missing
+  end  
+  
+  def self.overlay_preproduction_settings(json)
+    json = Utils.deep_symbolize(json)
+      @@preproduction_override.each do |property,value|
+        given_value = json[property]
+        if (given_value > @@preproduction_override[property])
+          $LOG.debug("Overriding property [#{property}: #{json[property]}] with preproduction default [#{property}: #{@@preproduction_override[property]}]")
+          json[property] = @@preproduction_override[property]
+        end
+      end
+      @@preproduction_env.each do |name,value|
+        json[:env][name] = value
+      end
+      return json
+  end
+  
+end

data/lib/marathon_deploy/utils.rb

@@ -0,0 +1,31 @@
+module Utils
+  
+  def self.symbolize(data) 
+    data.inject({}){|h,(k,v)| h[k.to_sym] = v; h}
+  end
+  
+  def self.deep_symbolize(obj)
+    return obj.reduce({}) do |memo, (k, v)|
+      memo.tap { |m| m[k.to_sym] = deep_symbolize(v) }
+    end if obj.is_a? Hash
+    
+    return obj.reduce([]) do |memo, v| 
+      memo << deep_symbolize(v); memo
+    end if obj.is_a? Array
+  
+    obj
+  end
+  
+  def self.random
+    range = [*'0'..'9',*'A'..'Z',*'a'..'z']
+    return Array.new(30){ range.sample }.join
+  end
+  
+  def self.response_body(response)
+    if (response.is_a?(Net::HTTPResponse) && !response.body.nil?)
+      return deep_symbolize((JSON.parse(response.body)))
+    end
+    return nil
+  end
+  
+end

data/lib/marathon_deploy/version.rb

@@ -0,0 +1,3 @@
+module MarathonDeploy
+  VERSION = "0.0.1"
+end

data/lib/marathon_deploy/yaml_json.rb

@@ -0,0 +1,45 @@
+require 'yaml'
+require 'json'
+require 'marathon_deploy/macro'
+
+module YamlJson
+  
+  def self.yaml2json(filename, process_macros=true)  
+    if (process_macros)
+      begin
+        data = YAML::load(Macro::process_macros(filename))
+      rescue Error::UndefinedMacroError => e
+        abort(e.message)     
+      end
+    else
+      data = YAML::load_file(filename)
+    end
+    JSON.parse(JSON.dump(data))
+  end
+  
+  def self.json2yaml(filename, process_macros=true)
+    if (process_macros)
+      json = read_json_w_macros(filename)
+    else
+      json = read_json(filename)
+    end
+    yml = YAML::dump(json)
+  end
+  
+  def self.read_json(filename)
+    file = File.open(filename,'r')
+    data = file.read 
+    file.close
+    return JSON.parse(data)
+  end  
+  
+  def self.read_json_w_macros(filename)
+    begin
+      data = Macro::process_macros(filename)
+    rescue Error::UndefinedMacroError => e
+      abort(e.message)
+    end
+    return JSON.parse(data)
+  end  
+  
+end