marathon_deploy 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,28 @@
1
+ require 'marathon_deploy/marathon_defaults'
2
+
3
+ class Environment
4
+
5
+ attr_reader :name
6
+
7
+ def initialize(name)
8
+ if (!name.is_a? String)
9
+ raise Error::BadFormatError, "argument for environment must be a string", caller
10
+ end
11
+ @name = name.upcase
12
+ end
13
+
14
+ def marathon_endpoints
15
+ return
16
+ end
17
+
18
+ def is_production?
19
+ if (@name.casecmp(MarathonDefaults::PRODUCTION_ENVIRONMENT_NAME) == 0)
20
+ return true
21
+ end
22
+ return false
23
+ end
24
+
25
+ def to_s
26
+ @name
27
+ end
28
+ end
@@ -0,0 +1,21 @@
1
+ module Error
2
+
3
+ class MarathonError < StandardError; end
4
+
5
+ class TimeoutError < MarathonError; end
6
+
7
+ class IOError < MarathonError; end
8
+
9
+ class BadURLError < MarathonError; end
10
+
11
+ class UnsupportedFileExtension < MarathonError; end
12
+
13
+ class MissingMarathonAttributesError < MarathonError; end
14
+
15
+ class BadFormatError < MarathonError; end
16
+
17
+ class DeploymentError < MarathonError ; end
18
+
19
+ class UndefinedMacroError < MarathonError ; end
20
+
21
+ end
@@ -0,0 +1,91 @@
1
+ require 'net/http'
2
+ require 'uri'
3
+ require 'marathon_deploy/error'
4
+
5
+ module HttpUtil
6
+
7
+ def self.put(url,payload)
8
+ uri = construct_uri url
9
+ begin
10
+ http = Net::HTTP.new(uri.host, uri.port)
11
+ req = Net::HTTP::Put.new(uri.request_uri)
12
+ req.body = payload.to_json
13
+ req["Content-Type"] = "application/json"
14
+ response = http.request(req)
15
+ rescue Errno::ECONNREFUSED => e
16
+ $LOG.error("Error calling marathon api: #{e.message}")
17
+ exit!
18
+ end
19
+ return response
20
+ end
21
+
22
+ def self.post(url, payload)
23
+ uri = construct_uri url
24
+ begin
25
+ http = Net::HTTP.new(uri.host, uri.port)
26
+ req = Net::HTTP::Post.new(uri.request_uri)
27
+ req.body = payload.to_json
28
+ req["Content-Type"] = "application/json"
29
+ response = http.request(req)
30
+ rescue Errno::ECONNREFUSED => e
31
+ message = "Error calling marathon api: #{e.message}"
32
+ $LOG.error(message)
33
+ raise Error::MarathonError, message, caller
34
+ end
35
+ return response
36
+ end
37
+
38
+ def self.construct_uri(url)
39
+ raise Error::BadURLError unless (valid_url(url))
40
+ return URI.parse(url)
41
+ end
42
+
43
+ def self.delete(url)
44
+ uri = construct_uri url
45
+ begin
46
+ http = Net::HTTP.new(uri.host, uri.port)
47
+ req = Net::HTTP::Delete.new(uri.request_uri)
48
+ response = http.request(req)
49
+ rescue Errno::ECONNREFUSED => e
50
+ message = "Error calling marathon api: #{e.message}"
51
+ $LOG.error(message)
52
+ raise Error::MarathonError, message, caller
53
+ end
54
+ return response
55
+ end
56
+
57
+ def self.clean_url(url)
58
+ url.sub(/(\/)+$/,'')
59
+ end
60
+
61
+ def self.get(url)
62
+ uri = construct_uri url
63
+ begin
64
+ http = Net::HTTP.new(uri.host, uri.port)
65
+ req = Net::HTTP::Get.new(uri.request_uri)
66
+ req["Content-Type"] = "application/json"
67
+ response = http.request(req)
68
+ rescue Errno::ECONNREFUSED => e
69
+ message = "Error calling marathon api: #{e.message}"
70
+ $LOG.error(message)
71
+ raise Error::MarathonError, message, caller
72
+ end
73
+ return response
74
+ end
75
+
76
+ def self.valid_url(url)
77
+ if (url =~ /\A#{URI::regexp}\z/)
78
+ return true
79
+ end
80
+ return false
81
+ end
82
+
83
+ def self.print(response)
84
+ begin
85
+ puts JSON.pretty_generate(JSON.parse(response.body))
86
+ rescue
87
+ puts response
88
+ end
89
+ end
90
+
91
+ end
@@ -0,0 +1,71 @@
1
+ require 'marathon_deploy/error'
2
+
3
+ module Macro
4
+
5
+ MACRO_BOUNDARY = '%%'
6
+
7
+ def self.get_macros(data)
8
+ macros = Array.new
9
+ data.each do |line|
10
+ (macros << line.scan(/(#{MACRO_BOUNDARY}\w+#{MACRO_BOUNDARY})/)).flatten!
11
+ end
12
+ return macros
13
+ end
14
+
15
+ def self.get_env_keys
16
+ return ENV.keys
17
+ end
18
+
19
+ def self.env_defined?(key)
20
+ if (ENV.has_key?(key) && (!ENV[key].nil? && !ENV[key].empty? ))
21
+ return true
22
+ else
23
+ return false
24
+ end
25
+ end
26
+
27
+ def self.get_undefined_macros(macros)
28
+ raise ArgumentError, "argument must be an array", caller if (!macros.class == Array)
29
+ undefined = Array.new
30
+ return macros.select { |m| !has_env?(m) }
31
+ end
32
+
33
+ def self.has_env?(macro)
34
+ raise ArgumentError, "argument must be a String", caller if (!macro.class == String)
35
+ env_name = strip(macro)
36
+ if (env_defined?(strip(env_name)))
37
+ return true
38
+ else
39
+ return false
40
+ end
41
+ end
42
+
43
+ def self.strip(str)
44
+ return str.gsub(MACRO_BOUNDARY,'')
45
+ end
46
+
47
+ def self.expand_macros(data)
48
+ processed = ""
49
+ macros = get_macros(data)
50
+ undefined = get_undefined_macros(macros)
51
+ if (!undefined.empty?)
52
+ raise Error::UndefinedMacroError, "macros found in deploy file without defined environment variables: #{undefined.uniq.join(',')}", caller
53
+ end
54
+
55
+ data.each do |line|
56
+ macros.each { |m| line.gsub!(m,ENV[strip(m)]) }
57
+ processed += line
58
+ end
59
+ return processed
60
+ end
61
+
62
+ def self.process_macros(filename)
63
+ file = File.open(filename,'r')
64
+ data = file.readlines
65
+ file.close()
66
+ return expand_macros(data)
67
+ end
68
+
69
+ private_class_method :get_macros, :get_env_keys, :strip, :has_env?, :get_undefined_macros, :env_defined?, :expand_macros
70
+
71
+ end
@@ -0,0 +1,79 @@
1
+ require 'marathon_deploy/http_util'
2
+ require 'marathon_deploy/deployment'
3
+ require 'marathon_deploy/utils'
4
+ require 'marathon_deploy/marathon_defaults'
5
+
6
+ class MarathonClient
7
+
8
+ attr_reader :marathon_url, :options
9
+ attr_accessor :application
10
+
11
+ # TODO: Options will contain environment, datacenter
12
+ def initialize(url, options = {})
13
+
14
+ raise Error::BadURLError, "invalid url => #{url}", caller if (!HttpUtil.valid_url(url))
15
+
16
+ @marathon_url = HttpUtil.clean_url(url)
17
+ @options = options
18
+ if @options[:username] and @options[:password]
19
+ @options[:basic_auth] = {
20
+ :username => @options[:username],
21
+ :password => @options[:password]
22
+ }
23
+ @options.delete(:username)
24
+ @options.delete(:password)
25
+ end
26
+ end
27
+
28
+ def deploy
29
+
30
+ deployment = Deployment.new(@marathon_url,application)
31
+
32
+ $LOG.info("Checking if any deployments are already running for application #{application.id}")
33
+ begin
34
+ deployment.wait_for_application("Deployment already running for application #{application.id}")
35
+ rescue Timeout::Error => e
36
+ raise Timeout::Error, "Timed out after #{deployment.timeout}s waiting for existing deployment of #{application.id} to finish. Check marathon #{@marathon_url + '/#deployments'} for stuck deployments!", caller
37
+ end
38
+
39
+ $LOG.info("Starting deployment of #{application.id}")
40
+
41
+ # if application with this ID already exists
42
+ if (deployment.applicationExists?)
43
+ $LOG.info("#{application.id} already exists. Performing update.")
44
+ response = deployment.update_app
45
+
46
+ # if no application with this ID is seen in marathon
47
+ else
48
+ response = deployment.create_app
49
+ end
50
+
51
+ if ((300..999).include?(response.code.to_i))
52
+ $LOG.error("Deployment response body => " + JSON.pretty_generate(JSON.parse(response.body)))
53
+ raise Error::DeploymentError, "Deployment returned response code #{response.code}", caller
54
+ end
55
+
56
+ $LOG.info("Deployment started for #{application.id} with deployment id #{deployment.deploymentId}") unless (deployment.deploymentId.nil?)
57
+
58
+ # wait for deployment to finish, according to marathon deployment API call
59
+ begin
60
+ deployment.wait_for_deployment_id
61
+ rescue Timeout::Error => e
62
+ $LOG.error("Timed out waiting for deployment of #{application.id} to complete. Canceling deploymentId #{deployment.deploymentId} and rolling back!")
63
+ deployment.cancel(deployment.deploymentId)
64
+ raise Timeout::Error, "Deployment of #{application.id} timed out after #{deployment.timeout} seconds", caller
65
+ end
66
+
67
+ # wait for all instances with defined health checks to be healthy
68
+ if (!deployment.health_checks_defined?)
69
+ $LOG.warn("No health checks were defined for application #{application.id}. No health checking will be performed.")
70
+ end
71
+
72
+ begin
73
+ deployment.wait_until_healthy
74
+ rescue Timeout::Error => e
75
+ raise Timeout::Error, "Timed out after #{deployment.healthcheck_timeout}s waiting for #{application.instances} instances of #{application.id} to become healthy", caller
76
+ end
77
+
78
+ end
79
+ end
@@ -0,0 +1,83 @@
1
+ require 'marathon_deploy/utils'
2
+ require 'marathon_deploy/error'
3
+ require 'logger'
4
+
5
+ module MarathonDefaults
6
+
7
+ DEPLOYMENT_RECHECK_INTERVAL = 3
8
+ DEPLOYMENT_TIMEOUT = 300
9
+ HEALTHY_WAIT_TIMEOUT = 300
10
+ HEALTHY_WAIT_RECHECK_INTERVAL = 3
11
+ PRODUCTION_ENVIRONMENT_NAME = 'PRODUCTION'
12
+ DEFAULT_ENVIRONMENT_NAME = 'INTEGRATION'
13
+ DEFAULT_PREPRODUCTION_MARATHON_ENDPOINTS = ['http://192.168.59.103:8080']
14
+ DEFAULT_PRODUCTION_MARATHON_ENDPOINTS = ['http://paasmaster46-1.mobile.rz:8080']
15
+ DEFAULT_DEPLOYFILE = 'deploy.yaml'
16
+ DEFAULT_LOGFILE = false
17
+ DEFAULT_LOGLEVEL = Logger::INFO
18
+ MARATHON_APPS_REST_PATH = '/v2/apps/'
19
+ MARATHON_DEPLOYMENT_REST_PATH = '/v2/deployments/'
20
+
21
+ @@preproduction_override = {
22
+ :instances => 20,
23
+ :mem => 512,
24
+ :cpus => 0.1
25
+ }
26
+
27
+ @@preproduction_env = {
28
+ :DATACENTER_NUMBER => "44",
29
+ :JAVA_XMS => "64m",
30
+ :JAVA_XMX => "128m"
31
+ }
32
+
33
+ @@required_marathon_env_variables = %w[
34
+ DATACENTER_NUMBER
35
+ APPLICATION_NAME
36
+ ]
37
+
38
+ #@@required_marathon_attributes = %w[id env container healthChecks args storeUrls].map(&:to_sym)
39
+ @@required_marathon_attributes = %w[id].map(&:to_sym)
40
+
41
+ def self.missing_attributes(json)
42
+ json = Utils.symbolize(json)
43
+ missing = []
44
+ @@required_marathon_attributes.each do |att|
45
+ if (!json[att])
46
+ missing << att
47
+ end
48
+ end
49
+ return missing
50
+ end
51
+
52
+ def self.missing_envs(json)
53
+ json = Utils.symbolize(json)
54
+
55
+ if (!json.key?(:env))
56
+ raise Error::MissingMarathonAttributesError, "no env attribute found in deployment file", caller
57
+ end
58
+
59
+ missing = []
60
+ @@required_marathon_env_variables.each do |variable|
61
+ if (!json[:env][variable])
62
+ missing << variable
63
+ end
64
+ end
65
+ return missing
66
+ end
67
+
68
+ def self.overlay_preproduction_settings(json)
69
+ json = Utils.deep_symbolize(json)
70
+ @@preproduction_override.each do |property,value|
71
+ given_value = json[property]
72
+ if (given_value > @@preproduction_override[property])
73
+ $LOG.debug("Overriding property [#{property}: #{json[property]}] with preproduction default [#{property}: #{@@preproduction_override[property]}]")
74
+ json[property] = @@preproduction_override[property]
75
+ end
76
+ end
77
+ @@preproduction_env.each do |name,value|
78
+ json[:env][name] = value
79
+ end
80
+ return json
81
+ end
82
+
83
+ end
@@ -0,0 +1,31 @@
1
+ module Utils
2
+
3
+ def self.symbolize(data)
4
+ data.inject({}){|h,(k,v)| h[k.to_sym] = v; h}
5
+ end
6
+
7
+ def self.deep_symbolize(obj)
8
+ return obj.reduce({}) do |memo, (k, v)|
9
+ memo.tap { |m| m[k.to_sym] = deep_symbolize(v) }
10
+ end if obj.is_a? Hash
11
+
12
+ return obj.reduce([]) do |memo, v|
13
+ memo << deep_symbolize(v); memo
14
+ end if obj.is_a? Array
15
+
16
+ obj
17
+ end
18
+
19
+ def self.random
20
+ range = [*'0'..'9',*'A'..'Z',*'a'..'z']
21
+ return Array.new(30){ range.sample }.join
22
+ end
23
+
24
+ def self.response_body(response)
25
+ if (response.is_a?(Net::HTTPResponse) && !response.body.nil?)
26
+ return deep_symbolize((JSON.parse(response.body)))
27
+ end
28
+ return nil
29
+ end
30
+
31
+ end
@@ -0,0 +1,3 @@
1
+ module MarathonDeploy
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,45 @@
1
+ require 'yaml'
2
+ require 'json'
3
+ require 'marathon_deploy/macro'
4
+
5
+ module YamlJson
6
+
7
+ def self.yaml2json(filename, process_macros=true)
8
+ if (process_macros)
9
+ begin
10
+ data = YAML::load(Macro::process_macros(filename))
11
+ rescue Error::UndefinedMacroError => e
12
+ abort(e.message)
13
+ end
14
+ else
15
+ data = YAML::load_file(filename)
16
+ end
17
+ JSON.parse(JSON.dump(data))
18
+ end
19
+
20
+ def self.json2yaml(filename, process_macros=true)
21
+ if (process_macros)
22
+ json = read_json_w_macros(filename)
23
+ else
24
+ json = read_json(filename)
25
+ end
26
+ yml = YAML::dump(json)
27
+ end
28
+
29
+ def self.read_json(filename)
30
+ file = File.open(filename,'r')
31
+ data = file.read
32
+ file.close
33
+ return JSON.parse(data)
34
+ end
35
+
36
+ def self.read_json_w_macros(filename)
37
+ begin
38
+ data = Macro::process_macros(filename)
39
+ rescue Error::UndefinedMacroError => e
40
+ abort(e.message)
41
+ end
42
+ return JSON.parse(data)
43
+ end
44
+
45
+ end