gv-valley 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,62 @@
1
+ require 'gv/bedrock/service'
2
+ require 'gv/bedrock/config'
3
+ require 'gv/common/docker_helper'
4
+ require 'gv/common/host_helper'
5
+
6
+ module GV
7
+ module Valley
8
+
9
+ ##
10
+ # Addon Service
11
+ #
12
+
13
+ class Addon < GV::Bedrock::Service
14
+
15
+ include GV::Common::HostHelper
16
+ include GV::Common::DockerHelper
17
+
18
+ PORT = nil
19
+
20
+ attr_reader :image, :params, :cmd
21
+
22
+ def initialize
23
+ super
24
+
25
+ pull_image_if_does_not_exists self.image
26
+
27
+ @home = GV::Bedrock::Config.service.get("home")
28
+ @name ||= File.basename(self.image)
29
+
30
+ end
31
+
32
+ def create app_name
33
+ @app_name = app_name
34
+ addon_name = "#{@name}.#{app_name}"
35
+ return nil if ps? addon_name
36
+ pipe "docker run --name #{addon_name} -d -p #{self.external_ip}::#{self.class::PORT} -e PORT=#{self.class::PORT} #{self.params} #{self.image} #{self.cmd}"
37
+ end
38
+
39
+ def destroy app_name
40
+ @app_name = app_name
41
+ addon_name = "#{@name}.#{app_name}"
42
+ batch addon_name, "stop", true
43
+ batch addon_name, "rm", true
44
+ end
45
+
46
+ def info app_name
47
+ @app_name = app_name
48
+ addon_name = "#{@name}.#{app_name}"
49
+ info(container_id(addon_name))
50
+ end
51
+
52
+ def port app_name
53
+ @app_name = app_name
54
+ addon_name = "#{@name}.#{app_name}"
55
+ container_port addon_name, self.external_ip, self.class::PORT
56
+ end
57
+
58
+
59
+ end
60
+
61
+ end
62
+ end
@@ -0,0 +1,108 @@
1
+ require 'gv/bedrock/config'
2
+ require 'gv/valley/etcd'
3
+ require 'gv/valley/file_server'
4
+ require 'gv/common/docker_helper'
5
+ require 'json'
6
+ require 'etcd'
7
+
8
+ module GV
9
+ module Valley
10
+
11
+ class App
12
+
13
+ PORT = 5000
14
+
15
+ class << self
16
+
17
+ def etcd_clt
18
+ @@etcd_clt ||= begin
19
+ server = GV::Valley::Etcd.service
20
+ ::Etcd.client(config = {
21
+ host: server.external_ip,
22
+ port: server.port
23
+ })
24
+ end
25
+ end
26
+
27
+ def all
28
+ etcd_clt.get("/apps").children.map{ |node| find(File.basename(node.key)) }
29
+ end
30
+
31
+ def find name
32
+ begin
33
+ find! name
34
+ rescue ::Etcd::KeyNotFound
35
+ nil
36
+ end
37
+ end
38
+
39
+ def find! name
40
+ value = etcd_clt.get("/apps/#{name}").value
41
+ new(ensure_defaults(JSON.load(value)))
42
+ end
43
+
44
+ def create name
45
+ save name, {"name" => name, "config" => default_config(name), "ps" => {}, "domains" => [default_domain(name)]}
46
+ find name
47
+ end
48
+
49
+ def save name, data
50
+ etcd_clt.set("/apps/#{name}", { value: JSON.dump(ensure_defaults(data)) })
51
+ end
52
+
53
+ def delete name
54
+ etcd_clt.delete("/apps/#{name}",{recursive: true}) rescue nil
55
+ end
56
+
57
+ def domain
58
+ GV::Bedrock::Config.service.get("domain")
59
+ end
60
+
61
+ def default_config name
62
+ config = {
63
+ "SLUG_URL" => "#{GV::Valley::FileServer.service.url}/#{name}/slug.tgz",
64
+ "PORT" => PORT
65
+ }
66
+ end
67
+
68
+ def default_domain name
69
+ "#{name}.#{self.domain}"
70
+ end
71
+
72
+ def ensure_defaults data
73
+ data["config"] ||= {}
74
+ data["config"].update(default_config(data["name"]))
75
+
76
+ data["domains"] ||= []
77
+ data["domains"] << default_domain(data["name"]) unless data["domains"].include? default_domain(data["name"])
78
+ data
79
+ end
80
+
81
+ end
82
+
83
+ def initialize attributes = {}
84
+ @attributes = attributes
85
+ @name = @attributes['name']
86
+ @attributes = attributes
87
+ end
88
+
89
+ def save
90
+ self.class.save(@name,@attributes)
91
+ end
92
+
93
+ def delete
94
+ self.class.delete @name
95
+ end
96
+
97
+ def [](key)
98
+ @attributes[key]
99
+ end
100
+
101
+ def []=(key,value)
102
+ @attributes[key]=value
103
+ end
104
+
105
+ end
106
+
107
+ end
108
+ end
@@ -0,0 +1,61 @@
1
+ require 'gv/bedrock/service'
2
+ require 'gv/valley/app'
3
+ require 'gv/common/pipe_helper'
4
+
5
+ module GV
6
+ module Valley
7
+
8
+ class Balancer < GV::Bedrock::Service
9
+
10
+ include GV::Common::PipeHelper
11
+ INDENT = " "
12
+
13
+ ##
14
+ # reloads haproxy config
15
+
16
+ def reload &block
17
+
18
+ @block = block
19
+
20
+ indicate "Loading Haproxy config"
21
+
22
+ config = File.read("#{GV::Valley.root}/scripts/haproxy.cfg")
23
+ target_file = "/etc/haproxy/haproxy.cfg"
24
+ acl = ""
25
+ backend = ""
26
+
27
+ App.all.each do |app|
28
+
29
+ app["domains"].each do |domain|
30
+ acl << "#{INDENT}use_backend b_#{app["name"]} if { hdr(host) -i #{domain} }\n"
31
+ end
32
+
33
+ backend << "backend b_#{app["name"]}\n"
34
+ app["ps"]["web"]["containers"].each do |container|
35
+ host = container['HostConfig']['PortBindings']["#{App::PORT}/tcp"].first
36
+ backend << "#{INDENT}server srv_#{container['ID'][0..6]} #{host['HostIp']}:#{host['HostPort']}\n"
37
+ end
38
+
39
+ end
40
+
41
+ config.gsub!(/^\#FRONT$/,acl)
42
+ config.gsub!(/^\#BACK$/,backend)
43
+
44
+ pipe "rm #{target_file}"
45
+ File.open(target_file,File::RDWR|File::CREAT){|f| f.puts config }
46
+
47
+ pipe "chmod 0770 #{target_file}"
48
+ pipe "chgrp haproxy #{target_file}"
49
+ pipe "service haproxy reload", &block
50
+
51
+ end
52
+
53
+ private
54
+
55
+
56
+
57
+ end
58
+
59
+ end
60
+
61
+ end
@@ -0,0 +1,104 @@
1
+ require 'yaml'
2
+ require 'gv/bedrock/service'
3
+ require 'gv/valley/app'
4
+ require 'gv/valley/runner'
5
+ require 'gv/common/pipe_helper'
6
+
7
+
8
+ module GV
9
+ module Valley
10
+
11
+ class Deployer < GV::Bedrock::Service
12
+
13
+ include GV::Common::PipeHelper
14
+
15
+ ##
16
+ # deploys app
17
+
18
+ def deploy name, &block
19
+
20
+ # find or create app
21
+ unless app = App.find(name)
22
+ app = App.create(name)
23
+ end
24
+
25
+ # set block for output helpers
26
+ @block = block
27
+
28
+ indicate "Deploying App"
29
+
30
+ # read procfile
31
+ host = GV::Valley::Runner.random_service
32
+ procfile = host.run(app["name"], "cat /app/Procfile")
33
+ procfile_types = YAML.load(procfile).keys
34
+
35
+ stop app
36
+
37
+ # add new Procfile process types or reset jobs array for existing types
38
+ procfile_types.each do |type|
39
+ unless app["ps"].keys.include? type
40
+ app["ps"][type] = {"scale" => 1, "containers" => []}
41
+ else
42
+ app["ps"][type]["containers"] = []
43
+ end
44
+ end
45
+
46
+ app.save
47
+
48
+ # remove the old types
49
+ app["ps"].keys.each do |type|
50
+ unless procfile_types.include? type
51
+ app["ps"].delete(type)
52
+ end
53
+ end
54
+
55
+ app.save
56
+
57
+ start app
58
+
59
+ app.save
60
+
61
+ end
62
+
63
+ ##
64
+ # stops all running procfile processes
65
+
66
+ def stop app, &block
67
+ # stop and remove all running procfile processes
68
+ tuple = [:ps, /#{app['name']}\./, nil, nil ]
69
+ while (self.class.space.read(tuple,0) rescue nil) do
70
+ if host = (self.class.space.take(tuple,0)[2] rescue nil)
71
+ host.remove app["name"]
72
+ end
73
+ end
74
+ end
75
+
76
+ ##
77
+ # starts all procfile processes
78
+
79
+ def start app, &block
80
+ # run available process types
81
+ app["ps"].each do |type,ps|
82
+ ps["scale"].times do |index|
83
+ host = GV::Valley::Runner.random_service
84
+ app["ps"][type]["containers"] << host.start(app["name"], type, index, &block)
85
+ end
86
+ end
87
+ end
88
+
89
+ private
90
+
91
+ def indicate string
92
+ say %(-----> #{string}), &@block
93
+ end
94
+
95
+ def say string
96
+ pipe %(echo '\e[1G#{string}'), &@block
97
+ end
98
+
99
+
100
+ end
101
+
102
+ end
103
+
104
+ end
@@ -0,0 +1,40 @@
1
+ require 'gv/bedrock/service'
2
+ require 'gv/bedrock/config'
3
+ require 'gv/common/docker_helper'
4
+ require 'gv/common/host_helper'
5
+
6
+ module GV
7
+ module Valley
8
+
9
+ ##
10
+ # Etcd Service
11
+ #
12
+
13
+ class Etcd < GV::Bedrock::Service
14
+
15
+ include GV::Common::HostHelper
16
+ include GV::Common::DockerHelper
17
+
18
+ PORT = 4001
19
+
20
+ def initialize
21
+ super
22
+
23
+ pull_image_if_does_not_exists "flynn/etcd"
24
+
25
+ home = GV::Bedrock::Config.service.get("home")
26
+
27
+ unless ps? 'etcd'
28
+ cleanup
29
+ pipe "docker run --name etcd -d -p #{self.external_ip}::#{PORT} -v #{home}/etcd:/data/db:rw flynn/etcd --name=greenvalley -data-dir=/data/db"
30
+ end
31
+ end
32
+
33
+ def port
34
+ container_port 'etcd', self.external_ip, PORT
35
+ end
36
+
37
+ end
38
+
39
+ end
40
+ end
@@ -0,0 +1,115 @@
1
+ require 'gv/bedrock/service'
2
+ require 'gv/bedrock/config'
3
+ require 'gv/common/host_helper'
4
+ require 'goliath/api'
5
+ require 'goliath/runner'
6
+ require 'uri'
7
+
8
+ module GV
9
+ module Valley
10
+
11
+ class FileServer < GV::Bedrock::Service
12
+
13
+ include GV::Common::HostHelper
14
+
15
+ class FileSystem
16
+ CHUNKSIZE = 65536
17
+
18
+ def initialize(path)
19
+ @path = path
20
+ end
21
+
22
+ def get
23
+ open(@path, 'rb') do |file|
24
+ yield file.read(CHUNKSIZE) until file.eof?
25
+ end
26
+ end
27
+
28
+ end
29
+
30
+ class Api < Goliath::API
31
+
32
+ use Goliath::Rack::DefaultMimeType
33
+ use Goliath::Rack::Render, 'json'
34
+ use Goliath::Rack::Heartbeat
35
+
36
+ use Goliath::Rack::Validation::RequestMethod, %w(GET PUT DELETE)
37
+
38
+ def on_headers(env, headers)
39
+ env['async-headers'] = headers
40
+ end
41
+
42
+ def on_body(env, data)
43
+ (env['async-body'] ||= '') << data
44
+ end
45
+
46
+ def response(env)
47
+
48
+ path = "#{ENV['GV_HOME']}/#{env['REQUEST_PATH']}"
49
+
50
+ case env['REQUEST_METHOD']
51
+
52
+ when 'GET'
53
+
54
+ headers = {'X-filename' => path}
55
+
56
+ raise Goliath::Validation::NotFoundError unless File.file?(path)
57
+
58
+ operation = proc do
59
+ FileSystem.new(path).get { |chunk| env.chunked_stream_send(chunk) }
60
+ end
61
+
62
+ callback = proc do |result|
63
+ env.chunked_stream_close
64
+ end
65
+
66
+ EM.defer operation, callback
67
+
68
+ headers.merge!( 'X-Stream' => 'Goliath')
69
+ chunked_streaming_response(200, headers)
70
+
71
+ when 'PUT'
72
+
73
+ File.delete(path) rescue nil
74
+ result = File.open(path, File::RDWR|File::CREAT){ |f| f.puts env['async-body'] }
75
+ [ 200, {}, {body: "OK"} ]
76
+
77
+ when 'DELETE'
78
+ result = File.delete(path)
79
+ [ 200, {}, {body: result } ]
80
+ end
81
+ end
82
+
83
+ end
84
+
85
+ def url
86
+ "http://#{self.external_ip}:#{self.port}"
87
+ end
88
+
89
+ def port
90
+ ENV['PORT'] ||= '9000'
91
+ end
92
+
93
+ def initialize
94
+ super
95
+ ENV['GV_HOME'] ||= GV::Bedrock::Config.service.get("home")
96
+ runner = Goliath::Runner.new(ARGV, nil)
97
+ runner.api = Api.new
98
+ runner.app = Goliath::Rack::Builder.build(Api, runner.api)
99
+ runner.port = self.port
100
+ runner.log_file = "/var/log/gv-file_server.log"
101
+ runner.pid_file = "/var/run/gv-file_server.pid"
102
+ runner.daemonize = true
103
+ runner.run
104
+ at_exit {
105
+ pid = File.read("/var/run/gv-file_server.pid").chomp.to_i
106
+ Process.kill("TERM",pid)
107
+ File.delete("/var/run/gv-file_server.pid")
108
+ File.delete("/var/log/gv-file_server.log")
109
+ File.delete("/var/log/gv-file_server.log_stdout.log")
110
+ }
111
+ end
112
+
113
+ end
114
+ end
115
+ end
@@ -0,0 +1,138 @@
1
+ require 'gv/bedrock/service'
2
+ require 'gv/common/host_helper'
3
+ require 'gv/common/docker_helper'
4
+ require 'gv/valley/app'
5
+ require 'yaml'
6
+
7
+ module GV
8
+ module Valley
9
+
10
+ class Runner < GV::Bedrock::Service
11
+
12
+ include GV::Common::HostHelper
13
+ include GV::Common::DockerHelper
14
+
15
+ def initialize
16
+ super
17
+ pull_image_if_does_not_exists "flynn/slugrunner"
18
+ register_all
19
+ end
20
+
21
+
22
+ ##
23
+ # runs one-off job
24
+
25
+ def run name, cmd, &block
26
+ app = App.find!(name)
27
+
28
+ cleanup
29
+
30
+ params = %(--rm -a stdout -a stderr ) #note that space at the end!
31
+ params << %(-i -a stdin) if cmd =~ Sticks::Pipe::INTERACTIVE_COMMANDS
32
+ cmd = %(docker run --name=#{app[:name]}.run.#{genuuid()} #{params} #{getenv(app)} flynn/slugrunner #{cmd})
33
+
34
+ debug "Runner#run name:#{name}, cmd:#{cmd}"
35
+
36
+ result = pipe cmd, &block
37
+
38
+ sleep 2
39
+
40
+ cleanup
41
+ result
42
+ end
43
+
44
+
45
+ ##
46
+ # force removes all matching jobs
47
+
48
+ def remove name, type = nil, &block
49
+ debug "Runner#remove name:#{name}, type:#{type}"
50
+
51
+ batch "#{name}.#{type}", "kill", true
52
+ batch "#{name}.#{type}", "rm", true
53
+ unregister_all(/#{name}\.#{type}\./)
54
+ end
55
+
56
+
57
+ ##
58
+ # starts process
59
+
60
+ def start name, type, index, &block
61
+ raise "AppNotFound" unless app = App.find(name)
62
+
63
+ info "Starting #{name}.#{type}.#{index}"
64
+
65
+ cleanup
66
+
67
+ params = %(-d -p #{self.external_ip}::#{App::PORT})
68
+ cmd = %(docker run --name=#{name}.#{type}.#{index} #{params} #{getenv(app)} flynn/slugrunner start #{type})
69
+
70
+ debug "Runner#start name:#{name}, type:#{type} index:#{index}"
71
+
72
+ container_id = pipe(cmd)
73
+
74
+ if container_id =~ /^[a-zA-Z0-9]{64}$/
75
+
76
+ register "#{name}.#{type}.#{index}"
77
+ info(container_id)
78
+
79
+ elsif container_id.include? "Error: Conflict"
80
+ error "Container name conflict: #{name}.#{type}.#{index}, docker service reloading..."
81
+
82
+ restart_docker!
83
+
84
+ remove name, type
85
+ start name, type, index, &block
86
+ else
87
+ raise container_id
88
+ end
89
+
90
+ end
91
+
92
+
93
+ ##
94
+ # retrives logs
95
+
96
+ def logs name, follow = false, &block
97
+ debug "Runner#logs name:#{name}, follow:#{follow}"
98
+
99
+ pipe "docker logs #{follow ? "-f" : nil} #{container_id(name)}", &block
100
+ end
101
+
102
+
103
+ private
104
+
105
+ def getenv app
106
+ app['config'].map{ |k,v| "-e #{k}=#{v}" }.join(" ")
107
+ end
108
+
109
+ def genuuid
110
+ rand(2**64).to_s(36) # from heroku/slugcompiler
111
+ end
112
+
113
+ def register_all
114
+ pipe("docker ps | grep runner/init").chomp.split("\n").each{|l|
115
+ cols = l.split(/\s{3,}/)
116
+ name = cols.last
117
+ unregister_all(/#{name}/)
118
+ register name
119
+ }
120
+ end
121
+
122
+ def unregister_all name
123
+ unregister name while (self.class.space.read([:ps, name, nil, self.external_ip],0) rescue nil)
124
+ end
125
+
126
+ def register name
127
+ self.class.space.write [:ps, name, self, self.external_ip ]
128
+ end
129
+
130
+ def unregister name
131
+ self.class.space.take([:ps, name, nil, self.external_ip],0) rescue nil
132
+ end
133
+
134
+
135
+ end
136
+
137
+ end
138
+ end
@@ -0,0 +1,5 @@
1
+ module GV
2
+ module Valley
3
+ VERSION = "0.0.1"
4
+ end
5
+ end
data/lib/gv/valley.rb ADDED
@@ -0,0 +1,10 @@
1
+ require "gv/common/logging"
2
+ require "gv/valley/version"
3
+
4
+ module GV
5
+ module Valley
6
+ def self.root
7
+ @@root ||= File.expand_path("../../../",__FILE__)
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,47 @@
1
+ global
2
+ log /dev/log local0
3
+ log /dev/log local1 notice
4
+ chroot /var/lib/haproxy
5
+ user haproxy
6
+ group haproxy
7
+ daemon
8
+
9
+ defaults
10
+ log global
11
+ mode http
12
+
13
+ contimeout 5000
14
+ clitimeout 50000
15
+ srvtimeout 50000
16
+
17
+ option httplog
18
+ option dontlognull
19
+ option redispatch
20
+ option abortonclose
21
+ option httpclose
22
+ option forwardfor
23
+
24
+ stats enable
25
+ stats auth admin:password
26
+ stats uri /monitor
27
+ stats refresh 5s
28
+ retries 5
29
+
30
+ balance roundrobin
31
+
32
+ errorfile 400 /etc/haproxy/errors/400.http
33
+ errorfile 403 /etc/haproxy/errors/403.http
34
+ errorfile 408 /etc/haproxy/errors/408.http
35
+ errorfile 500 /etc/haproxy/errors/500.http
36
+ errorfile 502 /etc/haproxy/errors/502.http
37
+ errorfile 503 /etc/haproxy/errors/503.http
38
+ errorfile 504 /etc/haproxy/errors/504.http
39
+
40
+
41
+ frontend http
42
+ bind *:80
43
+ monitor-uri /haproxy
44
+
45
+ #FRONT
46
+
47
+ #BACK