marathon_deploy 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +16 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/NOTES +5 -0
- data/README.md +31 -0
- data/Rakefile +2 -0
- data/TODO +11 -0
- data/bin/deploy +3 -0
- data/bin/deploy.rb +106 -0
- data/bin/json2yaml.rb +5 -0
- data/examples/deploy.json +66 -0
- data/examples/deploy.yaml +54 -0
- data/examples/jondeploy.yaml +16 -0
- data/examples/jondeploy2.yaml +24 -0
- data/examples/nohealthchecks.yaml +18 -0
- data/examples/public-search-germany-webapp.pre.json +67 -0
- data/examples/run.sh +3 -0
- data/examples/testout.json +68 -0
- data/examples/testout.yaml +54 -0
- data/input.txt +66 -0
- data/lib/marathon_deploy.rb +5 -0
- data/lib/marathon_deploy/application.rb +85 -0
- data/lib/marathon_deploy/deployment.rb +269 -0
- data/lib/marathon_deploy/environment.rb +28 -0
- data/lib/marathon_deploy/error.rb +21 -0
- data/lib/marathon_deploy/http_util.rb +91 -0
- data/lib/marathon_deploy/macro.rb +71 -0
- data/lib/marathon_deploy/marathon_client.rb +79 -0
- data/lib/marathon_deploy/marathon_defaults.rb +83 -0
- data/lib/marathon_deploy/utils.rb +31 -0
- data/lib/marathon_deploy/version.rb +3 -0
- data/lib/marathon_deploy/yaml_json.rb +45 -0
- data/marathon_deploy.gemspec +28 -0
- metadata +136 -0
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'marathon_deploy/marathon_defaults'
|
2
|
+
|
3
|
+
class Environment
|
4
|
+
|
5
|
+
attr_reader :name
|
6
|
+
|
7
|
+
def initialize(name)
|
8
|
+
if (!name.is_a? String)
|
9
|
+
raise Error::BadFormatError, "argument for environment must be a string", caller
|
10
|
+
end
|
11
|
+
@name = name.upcase
|
12
|
+
end
|
13
|
+
|
14
|
+
def marathon_endpoints
|
15
|
+
return
|
16
|
+
end
|
17
|
+
|
18
|
+
def is_production?
|
19
|
+
if (@name.casecmp(MarathonDefaults::PRODUCTION_ENVIRONMENT_NAME) == 0)
|
20
|
+
return true
|
21
|
+
end
|
22
|
+
return false
|
23
|
+
end
|
24
|
+
|
25
|
+
def to_s
|
26
|
+
@name
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Error
|
2
|
+
|
3
|
+
class MarathonError < StandardError; end
|
4
|
+
|
5
|
+
class TimeoutError < MarathonError; end
|
6
|
+
|
7
|
+
class IOError < MarathonError; end
|
8
|
+
|
9
|
+
class BadURLError < MarathonError; end
|
10
|
+
|
11
|
+
class UnsupportedFileExtension < MarathonError; end
|
12
|
+
|
13
|
+
class MissingMarathonAttributesError < MarathonError; end
|
14
|
+
|
15
|
+
class BadFormatError < MarathonError; end
|
16
|
+
|
17
|
+
class DeploymentError < MarathonError ; end
|
18
|
+
|
19
|
+
class UndefinedMacroError < MarathonError ; end
|
20
|
+
|
21
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'uri'
|
3
|
+
require 'marathon_deploy/error'
|
4
|
+
|
5
|
+
module HttpUtil
|
6
|
+
|
7
|
+
def self.put(url,payload)
|
8
|
+
uri = construct_uri url
|
9
|
+
begin
|
10
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
11
|
+
req = Net::HTTP::Put.new(uri.request_uri)
|
12
|
+
req.body = payload.to_json
|
13
|
+
req["Content-Type"] = "application/json"
|
14
|
+
response = http.request(req)
|
15
|
+
rescue Errno::ECONNREFUSED => e
|
16
|
+
$LOG.error("Error calling marathon api: #{e.message}")
|
17
|
+
exit!
|
18
|
+
end
|
19
|
+
return response
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.post(url, payload)
|
23
|
+
uri = construct_uri url
|
24
|
+
begin
|
25
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
26
|
+
req = Net::HTTP::Post.new(uri.request_uri)
|
27
|
+
req.body = payload.to_json
|
28
|
+
req["Content-Type"] = "application/json"
|
29
|
+
response = http.request(req)
|
30
|
+
rescue Errno::ECONNREFUSED => e
|
31
|
+
message = "Error calling marathon api: #{e.message}"
|
32
|
+
$LOG.error(message)
|
33
|
+
raise Error::MarathonError, message, caller
|
34
|
+
end
|
35
|
+
return response
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.construct_uri(url)
|
39
|
+
raise Error::BadURLError unless (valid_url(url))
|
40
|
+
return URI.parse(url)
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.delete(url)
|
44
|
+
uri = construct_uri url
|
45
|
+
begin
|
46
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
47
|
+
req = Net::HTTP::Delete.new(uri.request_uri)
|
48
|
+
response = http.request(req)
|
49
|
+
rescue Errno::ECONNREFUSED => e
|
50
|
+
message = "Error calling marathon api: #{e.message}"
|
51
|
+
$LOG.error(message)
|
52
|
+
raise Error::MarathonError, message, caller
|
53
|
+
end
|
54
|
+
return response
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.clean_url(url)
|
58
|
+
url.sub(/(\/)+$/,'')
|
59
|
+
end
|
60
|
+
|
61
|
+
def self.get(url)
|
62
|
+
uri = construct_uri url
|
63
|
+
begin
|
64
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
65
|
+
req = Net::HTTP::Get.new(uri.request_uri)
|
66
|
+
req["Content-Type"] = "application/json"
|
67
|
+
response = http.request(req)
|
68
|
+
rescue Errno::ECONNREFUSED => e
|
69
|
+
message = "Error calling marathon api: #{e.message}"
|
70
|
+
$LOG.error(message)
|
71
|
+
raise Error::MarathonError, message, caller
|
72
|
+
end
|
73
|
+
return response
|
74
|
+
end
|
75
|
+
|
76
|
+
def self.valid_url(url)
|
77
|
+
if (url =~ /\A#{URI::regexp}\z/)
|
78
|
+
return true
|
79
|
+
end
|
80
|
+
return false
|
81
|
+
end
|
82
|
+
|
83
|
+
def self.print(response)
|
84
|
+
begin
|
85
|
+
puts JSON.pretty_generate(JSON.parse(response.body))
|
86
|
+
rescue
|
87
|
+
puts response
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'marathon_deploy/error'
|
2
|
+
|
3
|
+
module Macro
|
4
|
+
|
5
|
+
MACRO_BOUNDARY = '%%'
|
6
|
+
|
7
|
+
def self.get_macros(data)
|
8
|
+
macros = Array.new
|
9
|
+
data.each do |line|
|
10
|
+
(macros << line.scan(/(#{MACRO_BOUNDARY}\w+#{MACRO_BOUNDARY})/)).flatten!
|
11
|
+
end
|
12
|
+
return macros
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.get_env_keys
|
16
|
+
return ENV.keys
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.env_defined?(key)
|
20
|
+
if (ENV.has_key?(key) && (!ENV[key].nil? && !ENV[key].empty? ))
|
21
|
+
return true
|
22
|
+
else
|
23
|
+
return false
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.get_undefined_macros(macros)
|
28
|
+
raise ArgumentError, "argument must be an array", caller if (!macros.class == Array)
|
29
|
+
undefined = Array.new
|
30
|
+
return macros.select { |m| !has_env?(m) }
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.has_env?(macro)
|
34
|
+
raise ArgumentError, "argument must be a String", caller if (!macro.class == String)
|
35
|
+
env_name = strip(macro)
|
36
|
+
if (env_defined?(strip(env_name)))
|
37
|
+
return true
|
38
|
+
else
|
39
|
+
return false
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.strip(str)
|
44
|
+
return str.gsub(MACRO_BOUNDARY,'')
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.expand_macros(data)
|
48
|
+
processed = ""
|
49
|
+
macros = get_macros(data)
|
50
|
+
undefined = get_undefined_macros(macros)
|
51
|
+
if (!undefined.empty?)
|
52
|
+
raise Error::UndefinedMacroError, "macros found in deploy file without defined environment variables: #{undefined.uniq.join(',')}", caller
|
53
|
+
end
|
54
|
+
|
55
|
+
data.each do |line|
|
56
|
+
macros.each { |m| line.gsub!(m,ENV[strip(m)]) }
|
57
|
+
processed += line
|
58
|
+
end
|
59
|
+
return processed
|
60
|
+
end
|
61
|
+
|
62
|
+
def self.process_macros(filename)
|
63
|
+
file = File.open(filename,'r')
|
64
|
+
data = file.readlines
|
65
|
+
file.close()
|
66
|
+
return expand_macros(data)
|
67
|
+
end
|
68
|
+
|
69
|
+
private_class_method :get_macros, :get_env_keys, :strip, :has_env?, :get_undefined_macros, :env_defined?, :expand_macros
|
70
|
+
|
71
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
require 'marathon_deploy/http_util'
|
2
|
+
require 'marathon_deploy/deployment'
|
3
|
+
require 'marathon_deploy/utils'
|
4
|
+
require 'marathon_deploy/marathon_defaults'
|
5
|
+
|
6
|
+
class MarathonClient
|
7
|
+
|
8
|
+
attr_reader :marathon_url, :options
|
9
|
+
attr_accessor :application
|
10
|
+
|
11
|
+
# TODO: Options will contain environment, datacenter
|
12
|
+
def initialize(url, options = {})
|
13
|
+
|
14
|
+
raise Error::BadURLError, "invalid url => #{url}", caller if (!HttpUtil.valid_url(url))
|
15
|
+
|
16
|
+
@marathon_url = HttpUtil.clean_url(url)
|
17
|
+
@options = options
|
18
|
+
if @options[:username] and @options[:password]
|
19
|
+
@options[:basic_auth] = {
|
20
|
+
:username => @options[:username],
|
21
|
+
:password => @options[:password]
|
22
|
+
}
|
23
|
+
@options.delete(:username)
|
24
|
+
@options.delete(:password)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def deploy
|
29
|
+
|
30
|
+
deployment = Deployment.new(@marathon_url,application)
|
31
|
+
|
32
|
+
$LOG.info("Checking if any deployments are already running for application #{application.id}")
|
33
|
+
begin
|
34
|
+
deployment.wait_for_application("Deployment already running for application #{application.id}")
|
35
|
+
rescue Timeout::Error => e
|
36
|
+
raise Timeout::Error, "Timed out after #{deployment.timeout}s waiting for existing deployment of #{application.id} to finish. Check marathon #{@marathon_url + '/#deployments'} for stuck deployments!", caller
|
37
|
+
end
|
38
|
+
|
39
|
+
$LOG.info("Starting deployment of #{application.id}")
|
40
|
+
|
41
|
+
# if application with this ID already exists
|
42
|
+
if (deployment.applicationExists?)
|
43
|
+
$LOG.info("#{application.id} already exists. Performing update.")
|
44
|
+
response = deployment.update_app
|
45
|
+
|
46
|
+
# if no application with this ID is seen in marathon
|
47
|
+
else
|
48
|
+
response = deployment.create_app
|
49
|
+
end
|
50
|
+
|
51
|
+
if ((300..999).include?(response.code.to_i))
|
52
|
+
$LOG.error("Deployment response body => " + JSON.pretty_generate(JSON.parse(response.body)))
|
53
|
+
raise Error::DeploymentError, "Deployment returned response code #{response.code}", caller
|
54
|
+
end
|
55
|
+
|
56
|
+
$LOG.info("Deployment started for #{application.id} with deployment id #{deployment.deploymentId}") unless (deployment.deploymentId.nil?)
|
57
|
+
|
58
|
+
# wait for deployment to finish, according to marathon deployment API call
|
59
|
+
begin
|
60
|
+
deployment.wait_for_deployment_id
|
61
|
+
rescue Timeout::Error => e
|
62
|
+
$LOG.error("Timed out waiting for deployment of #{application.id} to complete. Canceling deploymentId #{deployment.deploymentId} and rolling back!")
|
63
|
+
deployment.cancel(deployment.deploymentId)
|
64
|
+
raise Timeout::Error, "Deployment of #{application.id} timed out after #{deployment.timeout} seconds", caller
|
65
|
+
end
|
66
|
+
|
67
|
+
# wait for all instances with defined health checks to be healthy
|
68
|
+
if (!deployment.health_checks_defined?)
|
69
|
+
$LOG.warn("No health checks were defined for application #{application.id}. No health checking will be performed.")
|
70
|
+
end
|
71
|
+
|
72
|
+
begin
|
73
|
+
deployment.wait_until_healthy
|
74
|
+
rescue Timeout::Error => e
|
75
|
+
raise Timeout::Error, "Timed out after #{deployment.healthcheck_timeout}s waiting for #{application.instances} instances of #{application.id} to become healthy", caller
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
require 'marathon_deploy/utils'
|
2
|
+
require 'marathon_deploy/error'
|
3
|
+
require 'logger'
|
4
|
+
|
5
|
+
module MarathonDefaults
|
6
|
+
|
7
|
+
DEPLOYMENT_RECHECK_INTERVAL = 3
|
8
|
+
DEPLOYMENT_TIMEOUT = 300
|
9
|
+
HEALTHY_WAIT_TIMEOUT = 300
|
10
|
+
HEALTHY_WAIT_RECHECK_INTERVAL = 3
|
11
|
+
PRODUCTION_ENVIRONMENT_NAME = 'PRODUCTION'
|
12
|
+
DEFAULT_ENVIRONMENT_NAME = 'INTEGRATION'
|
13
|
+
DEFAULT_PREPRODUCTION_MARATHON_ENDPOINTS = ['http://192.168.59.103:8080']
|
14
|
+
DEFAULT_PRODUCTION_MARATHON_ENDPOINTS = ['http://paasmaster46-1.mobile.rz:8080']
|
15
|
+
DEFAULT_DEPLOYFILE = 'deploy.yaml'
|
16
|
+
DEFAULT_LOGFILE = false
|
17
|
+
DEFAULT_LOGLEVEL = Logger::INFO
|
18
|
+
MARATHON_APPS_REST_PATH = '/v2/apps/'
|
19
|
+
MARATHON_DEPLOYMENT_REST_PATH = '/v2/deployments/'
|
20
|
+
|
21
|
+
@@preproduction_override = {
|
22
|
+
:instances => 20,
|
23
|
+
:mem => 512,
|
24
|
+
:cpus => 0.1
|
25
|
+
}
|
26
|
+
|
27
|
+
@@preproduction_env = {
|
28
|
+
:DATACENTER_NUMBER => "44",
|
29
|
+
:JAVA_XMS => "64m",
|
30
|
+
:JAVA_XMX => "128m"
|
31
|
+
}
|
32
|
+
|
33
|
+
@@required_marathon_env_variables = %w[
|
34
|
+
DATACENTER_NUMBER
|
35
|
+
APPLICATION_NAME
|
36
|
+
]
|
37
|
+
|
38
|
+
#@@required_marathon_attributes = %w[id env container healthChecks args storeUrls].map(&:to_sym)
|
39
|
+
@@required_marathon_attributes = %w[id].map(&:to_sym)
|
40
|
+
|
41
|
+
def self.missing_attributes(json)
|
42
|
+
json = Utils.symbolize(json)
|
43
|
+
missing = []
|
44
|
+
@@required_marathon_attributes.each do |att|
|
45
|
+
if (!json[att])
|
46
|
+
missing << att
|
47
|
+
end
|
48
|
+
end
|
49
|
+
return missing
|
50
|
+
end
|
51
|
+
|
52
|
+
def self.missing_envs(json)
|
53
|
+
json = Utils.symbolize(json)
|
54
|
+
|
55
|
+
if (!json.key?(:env))
|
56
|
+
raise Error::MissingMarathonAttributesError, "no env attribute found in deployment file", caller
|
57
|
+
end
|
58
|
+
|
59
|
+
missing = []
|
60
|
+
@@required_marathon_env_variables.each do |variable|
|
61
|
+
if (!json[:env][variable])
|
62
|
+
missing << variable
|
63
|
+
end
|
64
|
+
end
|
65
|
+
return missing
|
66
|
+
end
|
67
|
+
|
68
|
+
def self.overlay_preproduction_settings(json)
|
69
|
+
json = Utils.deep_symbolize(json)
|
70
|
+
@@preproduction_override.each do |property,value|
|
71
|
+
given_value = json[property]
|
72
|
+
if (given_value > @@preproduction_override[property])
|
73
|
+
$LOG.debug("Overriding property [#{property}: #{json[property]}] with preproduction default [#{property}: #{@@preproduction_override[property]}]")
|
74
|
+
json[property] = @@preproduction_override[property]
|
75
|
+
end
|
76
|
+
end
|
77
|
+
@@preproduction_env.each do |name,value|
|
78
|
+
json[:env][name] = value
|
79
|
+
end
|
80
|
+
return json
|
81
|
+
end
|
82
|
+
|
83
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Utils
|
2
|
+
|
3
|
+
def self.symbolize(data)
|
4
|
+
data.inject({}){|h,(k,v)| h[k.to_sym] = v; h}
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.deep_symbolize(obj)
|
8
|
+
return obj.reduce({}) do |memo, (k, v)|
|
9
|
+
memo.tap { |m| m[k.to_sym] = deep_symbolize(v) }
|
10
|
+
end if obj.is_a? Hash
|
11
|
+
|
12
|
+
return obj.reduce([]) do |memo, v|
|
13
|
+
memo << deep_symbolize(v); memo
|
14
|
+
end if obj.is_a? Array
|
15
|
+
|
16
|
+
obj
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.random
|
20
|
+
range = [*'0'..'9',*'A'..'Z',*'a'..'z']
|
21
|
+
return Array.new(30){ range.sample }.join
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.response_body(response)
|
25
|
+
if (response.is_a?(Net::HTTPResponse) && !response.body.nil?)
|
26
|
+
return deep_symbolize((JSON.parse(response.body)))
|
27
|
+
end
|
28
|
+
return nil
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'json'
|
3
|
+
require 'marathon_deploy/macro'
|
4
|
+
|
5
|
+
module YamlJson
|
6
|
+
|
7
|
+
def self.yaml2json(filename, process_macros=true)
|
8
|
+
if (process_macros)
|
9
|
+
begin
|
10
|
+
data = YAML::load(Macro::process_macros(filename))
|
11
|
+
rescue Error::UndefinedMacroError => e
|
12
|
+
abort(e.message)
|
13
|
+
end
|
14
|
+
else
|
15
|
+
data = YAML::load_file(filename)
|
16
|
+
end
|
17
|
+
JSON.parse(JSON.dump(data))
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.json2yaml(filename, process_macros=true)
|
21
|
+
if (process_macros)
|
22
|
+
json = read_json_w_macros(filename)
|
23
|
+
else
|
24
|
+
json = read_json(filename)
|
25
|
+
end
|
26
|
+
yml = YAML::dump(json)
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.read_json(filename)
|
30
|
+
file = File.open(filename,'r')
|
31
|
+
data = file.read
|
32
|
+
file.close
|
33
|
+
return JSON.parse(data)
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.read_json_w_macros(filename)
|
37
|
+
begin
|
38
|
+
data = Macro::process_macros(filename)
|
39
|
+
rescue Error::UndefinedMacroError => e
|
40
|
+
abort(e.message)
|
41
|
+
end
|
42
|
+
return JSON.parse(data)
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|