picsolve_docker_builder 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +11 -0
  3. data/.rspec +2 -0
  4. data/.rubocop.yml +8 -0
  5. data/.travis.yml +3 -0
  6. data/Gemfile +4 -0
  7. data/README.md +25 -0
  8. data/Rakefile +80 -0
  9. data/bin/docker_build +5 -0
  10. data/integration/integration_play_spec.rb +51 -0
  11. data/integration/play_hello_world/.docker-builder.yml +3 -0
  12. data/integration/play_hello_world/.gitignore +12 -0
  13. data/integration/play_hello_world/Gemfile +2 -0
  14. data/integration/play_hello_world/LICENSE +8 -0
  15. data/integration/play_hello_world/README +4 -0
  16. data/integration/play_hello_world/Rakefile +3 -0
  17. data/integration/play_hello_world/app/controllers/Application.scala +12 -0
  18. data/integration/play_hello_world/app/views/index.scala.html +7 -0
  19. data/integration/play_hello_world/app/views/main.scala.html +15 -0
  20. data/integration/play_hello_world/build.sbt +20 -0
  21. data/integration/play_hello_world/conf/application.conf +44 -0
  22. data/integration/play_hello_world/conf/logback.xml +22 -0
  23. data/integration/play_hello_world/conf/routes +9 -0
  24. data/integration/play_hello_world/project/build.properties +4 -0
  25. data/integration/play_hello_world/project/plugins.sbt +16 -0
  26. data/integration/play_hello_world/public/images/favicon.png +0 -0
  27. data/integration/play_hello_world/public/javascripts/hello.js +3 -0
  28. data/integration/play_hello_world/public/stylesheets/main.css +0 -0
  29. data/integration/play_hello_world/test/ApplicationSpec.scala +30 -0
  30. data/integration/play_hello_world/test/IntegrationSpec.scala +24 -0
  31. data/integration/spec_helper.rb +3 -0
  32. data/lib/picsolve_docker_builder.rb +5 -0
  33. data/lib/picsolve_docker_builder/base.rb +66 -0
  34. data/lib/picsolve_docker_builder/builder/builder.rb +113 -0
  35. data/lib/picsolve_docker_builder/builder/file.rb +46 -0
  36. data/lib/picsolve_docker_builder/composer/composer.rb +134 -0
  37. data/lib/picsolve_docker_builder/composer/image.rb +68 -0
  38. data/lib/picsolve_docker_builder/composer/registry.rb +80 -0
  39. data/lib/picsolve_docker_builder/frame.rb +298 -0
  40. data/lib/picsolve_docker_builder/helpers/kubeclient.rb +34 -0
  41. data/lib/picsolve_docker_builder/helpers/kubernetes/pod.rb +38 -0
  42. data/lib/picsolve_docker_builder/helpers/kubernetes/rc.rb +98 -0
  43. data/lib/picsolve_docker_builder/helpers/kubernetes/resource.rb +28 -0
  44. data/lib/picsolve_docker_builder/helpers/kubernetes/service.rb +50 -0
  45. data/lib/picsolve_docker_builder/helpers/kubernetes_manager.rb +102 -0
  46. data/lib/picsolve_docker_builder/helpers/repository.rb +24 -0
  47. data/lib/picsolve_docker_builder/helpers/ssh_forward.rb +75 -0
  48. data/lib/picsolve_docker_builder/scala.rb +196 -0
  49. data/lib/picsolve_docker_builder/version.rb +4 -0
  50. data/lib/tasks/compose.rake +25 -0
  51. data/lib/tasks/docker.rake +24 -0
  52. data/lib/tasks/scala.rake +11 -0
  53. data/picsolve_docker_builder.gemspec +35 -0
  54. metadata +250 -0
@@ -0,0 +1,98 @@
1
+ require 'picsolve_docker_builder/helpers/kubernetes/pod'
2
+ require 'picsolve_docker_builder/helpers/kubernetes/resource'
3
+
4
+ module PicsolveDockerBuilder
5
+ module Helpers
6
+ module Kubernetes
7
+ # Ruby representation of a kuberentes replication cluster
8
+ class Rc < Resource
9
+ def existing_rcs
10
+ client.get_replication_controllers(
11
+ namespace: @image.composer.namespace
12
+ ).select do |rc|
13
+ rc.metadata.name.match(/^#{@image.name}/)
14
+ end
15
+ end
16
+
17
+ def config_rc
18
+ # configure the service
19
+ @rc.metadata = {} unless @rc.metadata
20
+ @rc.metadata.name = "#{@image.name}-#{@image.composer.hash}"
21
+ @rc.metadata.namespace = @image.composer.namespace
22
+ @rc.metadata.labels = template_labels
23
+ @rc.spec = {} unless @rc.spec
24
+ @rc.spec.replicas = 1
25
+ @rc.spec.selector = template_labels_pods
26
+ @rc.spec.template = {
27
+ 'metadata' => {
28
+ 'labels' => template_labels_pods
29
+ },
30
+ 'spec' => {
31
+ 'containers' => containers
32
+ }
33
+ }
34
+ end
35
+
36
+ def containers
37
+ [{
38
+ 'name' => @image.name,
39
+ 'image' => @image.repo_tag_unique,
40
+ 'ports' => @image.ports_rc
41
+ }]
42
+ end
43
+
44
+ def deploy
45
+ @old_rcs = existing_rcs
46
+ @rc = Kubeclient::ReplicationController.new
47
+ config_rc
48
+ log.debug "create replication controller #{@rc.metadata.name}"
49
+ client.create_replication_controller @rc
50
+ end
51
+
52
+ def wait
53
+ ready?
54
+ end
55
+
56
+ def ready?
57
+ pods.each do |pod|
58
+ result = pod.ready?
59
+ return false unless result
60
+ end
61
+ true
62
+ end
63
+
64
+ def pods
65
+ # TODO: better handling of selectors
66
+ client.get_pods(
67
+ namespace: @rc.metadata.namespace,
68
+ label_selector: "name=#{@rc.spec.selector.name}" \
69
+ ",deployment=#{@rc.spec.selector.deployment}"
70
+ ).map do |pod|
71
+ Pod.new(pod, @kubernetes)
72
+ end
73
+ end
74
+
75
+ def remove
76
+ log.debug "remove replication controller #{@rc.metadata.name}"
77
+ client.delete_replication_controller(
78
+ @rc.metadata.name,
79
+ @rc.metadata.namespace
80
+ )
81
+ remove_pods_by_rc(@rc)
82
+ end
83
+
84
+ def remove_old_rcs
85
+ @old_rcs.each do |rc|
86
+ Pod.remove_by_rc(self, rc)
87
+ end
88
+ end
89
+
90
+ def template_labels_pods
91
+ c = template_labels
92
+ c['deployment'] = @image.composer.hash
93
+ c
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,28 @@
1
+ require 'picsolve_docker_builder/base'
2
+
3
+ module PicsolveDockerBuilder
4
+ module Helpers
5
+ module Kubernetes
6
+ # A generic kuberntes resource
7
+ class Resource
8
+ include PicsolveDockerBuilder::Base
9
+ def initialize(image, kubernetes)
10
+ @image = image
11
+ @kubernetes = kubernetes
12
+ end
13
+
14
+ def client
15
+ @kubernetes.client
16
+ end
17
+
18
+ def template_labels
19
+ {
20
+ 'name' => @image.name,
21
+ 'app_name' => @image.composer.app_name,
22
+ 'stage' => @image.composer.stage
23
+ }
24
+ end
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,50 @@
1
+ require 'picsolve_docker_builder/helpers/kubernetes/resource'
2
+
3
+ module PicsolveDockerBuilder
4
+ module Helpers
5
+ module Kubernetes
6
+ # A generic kuberntes resource
7
+ class Service < Resource
8
+ def config
9
+ # workaround a bug on updating TODO: remove when fixed upstream
10
+ @service.name = @image.name
11
+
12
+ # configure the service
13
+ @service.metadata = {} unless @service.metadata
14
+ @service.metadata.name = @image.name
15
+ @service.metadata.namespace = @image.composer.namespace
16
+ @service.metadata.labels = template_labels
17
+ @service.spec = {} unless @service.spec
18
+ @service.spec.ports = @image.ports
19
+ @service.spec.selector = {
20
+ 'name' => @image.name
21
+ }
22
+ end
23
+
24
+ def deploy
25
+ @existing = false
26
+ begin
27
+ @service = client.get_service @image.name, @image.composer.namespace
28
+ @existing = true
29
+ rescue KubeException
30
+ @service = Kubeclient::Service.new
31
+ end
32
+ # config service
33
+ config
34
+
35
+ if @existing
36
+ log.debug \
37
+ "update service '#{@image.composer.namespace}/#{@image.name}' " \
38
+ 'on kubernetes'
39
+ client.update_service @service
40
+ else
41
+ log.debug \
42
+ "create service '#{@image.composer.namespace}/#{@image.name}' " \
43
+ 'on kubernetes'
44
+ client.create_service @service
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,102 @@
1
+ require 'picsolve_docker_builder/helpers/kubeclient'
2
+ require 'picsolve_docker_builder/helpers/kubernetes/rc'
3
+ require 'picsolve_docker_builder/helpers/kubernetes/service'
4
+ require 'picsolve_docker_builder/helpers/kubernetes/pod'
5
+ require 'picsolve_docker_builder/helpers/ssh_forward'
6
+ require 'picsolve_docker_builder/base'
7
+
8
+ module PicsolveDockerBuilder
9
+ module Helpers
10
+ # Ruby representation of a kuberentes cluster
11
+ class KubernetesManager
12
+ include PicsolveDockerBuilder::Base
13
+
14
+ def create_client
15
+ Kubeclient::Client.new kubernetes_url, 'v1beta3'
16
+ end
17
+
18
+ def kubernetes_url
19
+ @ssh_forward = SshForward.new(
20
+ ssh_host: kubernetes_host,
21
+ remote_host: '127.0.0.1',
22
+ remote_port: 8080
23
+ )
24
+ port = @ssh_forward.start
25
+ at_exit do
26
+ @ssh_forward.stop unless @ssh_forward.nil?
27
+ end
28
+ "http://127.0.0.1:#{port}"
29
+ end
30
+
31
+ def kubernetes_host
32
+ # TODO: Need to be touched for multi cluster support
33
+ config['compose']['clusters'].first
34
+ end
35
+
36
+ def client
37
+ @client ||= create_client
38
+ end
39
+
40
+ def service(image)
41
+ Kubernetes::Service.new(image, self)
42
+ end
43
+
44
+ def rc(image)
45
+ Kubernetes::Rc.new(image, self)
46
+ end
47
+
48
+ def template_labels_pods(i)
49
+ c = template_labels(i)
50
+ c['deployment'] = i.composer.hash
51
+ c
52
+ end
53
+
54
+ def template_labels(i)
55
+ {
56
+ 'name' => i.name,
57
+ 'app_name' => i.composer.app_name,
58
+ 'stage' => i.composer.stage
59
+ }
60
+ end
61
+
62
+ def config_service(s, i)
63
+ # workaround a bug on updating TODO: remove when fixed upstream
64
+ s.name = i.name
65
+
66
+ # configure the service
67
+ s.metadata = {} unless s.metadata
68
+ s.metadata.name = i.name
69
+ s.metadata.namespace = i.composer.namespace
70
+ s.metadata.labels = template_labels(i)
71
+ s.spec = {} unless s.spec
72
+ s.spec.ports = i.ports
73
+ s.spec.selector = {
74
+ 'name' => i.name
75
+ }
76
+ end
77
+
78
+ def deploy_service(i)
79
+ existing = false
80
+ begin
81
+ s = client.get_service i.name, i.composer.namespace
82
+ existing = true
83
+ rescue KubeException
84
+ s = Kubeclient::Service.new
85
+ end
86
+
87
+ # config config
88
+ config_service(s, i)
89
+
90
+ if existing
91
+ log.debug \
92
+ "update service '#{i.composer.namespace}/#{i.name}' on kubernetes"
93
+ client.update_service s
94
+ else
95
+ log.debug \
96
+ "create service '#{i.composer.namespace}/#{i.name}' on kubernetes"
97
+ client.create_service s
98
+ end
99
+ end
100
+ end
101
+ end
102
+ end
@@ -0,0 +1,24 @@
1
+ require 'picsolve_docker_builder/base'
2
+
3
+ module PicsolveDockerBuilder
4
+ module Helpers
5
+ # This helper should help with paring of docker image names
6
+ # (as they are not very well defined)
7
+ class Repository
8
+ include PicsolveDockerBuilder::Base
9
+ def initialize(name)
10
+ parse_input(name)
11
+ end
12
+
13
+ def parse_input(input)
14
+ input.split('/')
15
+ end
16
+
17
+ def name=(_name)
18
+ end
19
+
20
+ def repo_tag
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,75 @@
1
+ require 'picsolve_docker_builder/base'
2
+ require 'net/ssh'
3
+
4
+ module PicsolveDockerBuilder
5
+ module Helpers
6
+ # Ruby class that can forward a remote port over SSH
7
+ class SshForward
8
+ include PicsolveDockerBuilder::Base
9
+ attr_reader :options
10
+ def initialize(opts)
11
+ @options = opts
12
+ end
13
+
14
+ def connection
15
+ @connection ||= Net::SSH.start(ssh_host, ssh_user)
16
+ end
17
+
18
+ def bind_port
19
+ tries = 0
20
+ port = local_port
21
+ begin
22
+ connection.forward.local(port, remote_host, remote_port)
23
+ return port
24
+ rescue Errno::EADDRINUSE => e
25
+ # raise after five failed tries
26
+ raise e if tries > 5
27
+ tries += 1
28
+ port += 1
29
+ retry
30
+ end
31
+ end
32
+
33
+ def start
34
+ log.debug "Connecting via ssh to host '#{ssh_user}@#{ssh_host}'"
35
+
36
+ # bind remote service to local port
37
+ port = bind_port
38
+
39
+ # start thread with ssh
40
+ @thread = Thread.new do
41
+ connection.loop { true }
42
+ end
43
+ port
44
+ end
45
+
46
+ def stop
47
+ return if @thread.nil?
48
+ @thread.kill
49
+ @thread.join
50
+ log.debug \
51
+ "Disconnected ssh connection to host '#{ssh_user}@#{ssh_host}'"
52
+ end
53
+
54
+ def ssh_user
55
+ options[:ssh_user] = 'core'
56
+ end
57
+
58
+ def ssh_host
59
+ options[:ssh_host]
60
+ end
61
+
62
+ def remote_host
63
+ options[:remote_host]
64
+ end
65
+
66
+ def remote_port
67
+ options[:remote_port]
68
+ end
69
+
70
+ def local_port
71
+ options[:local_port] || remote_port
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,196 @@
1
+ require 'rubygems/package'
2
+ require 'zlib'
3
+ require 'picsolve_docker_builder/frame'
4
+ require 'picsolve_docker_builder/builder/builder'
5
+
6
+ module PicsolveDockerBuilder
7
+ # Build a scala project
8
+ class Scala < Frame
9
+ def default_config
10
+ c = super
11
+ c['scala'] = {
12
+ 'sbt' => {
13
+ 'build_task' => [
14
+ 'clean test universal:packageZipTarball'
15
+ ]
16
+ }
17
+ }
18
+ c
19
+ end
20
+
21
+ def prepare_volumes
22
+ volumes = volumes_sbt.map do |v|
23
+ v[1]
24
+ end
25
+
26
+ cmd = [
27
+ 'chown',
28
+ '-c',
29
+ build_user,
30
+ build_user_home
31
+ ]
32
+
33
+ cmd += volumes
34
+
35
+ execute cmd
36
+ end
37
+
38
+ def docker_build_files
39
+ f = []
40
+ f << Builder::File.new(
41
+ 'app.tar.gz',
42
+ source: universal_tar_gz,
43
+ destination: '/opt'
44
+ )
45
+ unless run_script.nil?
46
+ f << Builder::File.new(
47
+ run_script[:destination],
48
+ run_script
49
+ )
50
+ end
51
+ f
52
+ end
53
+
54
+ def docker_builder
55
+ Builder::Builder.new(
56
+ base_image: image_name,
57
+ maintainer: 'Picsolve Onlineops <onlineops@picsolve.com>',
58
+ ports: [9000],
59
+ hooks: {
60
+ before_adds: [
61
+ '# add play users',
62
+ 'RUN groupadd -r play && \\',
63
+ ' useradd -r -g play play',
64
+ '# create pid dir',
65
+ 'RUN mkdir -p /var/run/play && \\',
66
+ ' chown play:play /var/run/play'
67
+ ],
68
+ after_adds: [
69
+ '# chown app directories',
70
+ 'RUN chown -R play:play /opt'
71
+ ]
72
+ },
73
+ command: ['gosu', 'play', bin_path],
74
+ files: docker_build_files
75
+ )
76
+ end
77
+
78
+ def docker_build
79
+ @docker_build = docker_builder.build
80
+ end
81
+
82
+ def dirs_sbt
83
+ [
84
+ '.sbt',
85
+ '.ivy2',
86
+ '.m2'
87
+ ]
88
+ end
89
+
90
+ def bin_path
91
+ @bin_path ||= detect_bin_path
92
+ end
93
+
94
+ def detect_bin_path
95
+ path = universal_bin_path
96
+ unless path.nil?
97
+ log.debug "run script detected in '#{path}'"
98
+ return path
99
+ end
100
+ path = run_script
101
+ unless path.nil?
102
+ path = run_script[:destination]
103
+ log.debug "run script detected in '#{path}'"
104
+ return path
105
+ end
106
+ fail 'No run script detected'
107
+ end
108
+
109
+ def run_script
110
+ @run_script ||= detect_run_script
111
+ end
112
+
113
+ def detect_run_script
114
+ path = run_script_path
115
+ return nil unless File.exist? path
116
+ basename = ::File.basename(path)
117
+ {
118
+ basename: basename,
119
+ source: path,
120
+ destination: "/#{basename}"
121
+ }
122
+ end
123
+
124
+ def run_script_path
125
+ File.join(base_dir, 'run.sh')
126
+ end
127
+
128
+ def volumes_sbt
129
+ dirs_sbt.map do |dir|
130
+ [
131
+ File.join(base_dir, dir),
132
+ File.join(build_user_home, dir)
133
+ ]
134
+ end
135
+ end
136
+
137
+ def volumes_array
138
+ super + volumes_sbt
139
+ end
140
+
141
+ def sbt_command
142
+ runs = config['scala']['sbt']['build_task'].map do |tasks|
143
+ "sbt #{tasks}"
144
+ end
145
+ runs.join ' && '
146
+ end
147
+
148
+ def asset_build
149
+ log.info "start asset building with image #{image_name}"
150
+
151
+ start
152
+
153
+ prepare_volumes
154
+
155
+ build_cmd = [
156
+ 'gosu', build_user, 'bash', '-c', sbt_command
157
+ ]
158
+
159
+ execute_attach build_cmd
160
+
161
+ log.info "finished asset building with image #{image_name}"
162
+
163
+ stop
164
+ end
165
+
166
+ def universal_bin_path
167
+ Zlib::GzipReader.open(universal_tar_gz) do |gunzip|
168
+ gunzip_io = StringIO.new(gunzip.read)
169
+ Gem::Package::TarReader.new gunzip_io do |tar|
170
+ tar.each do |entry|
171
+ next unless entry.file?
172
+ next unless entry.full_name.match(%r{/bin})
173
+ next if entry.full_name.match(/\.bat$/)
174
+ return File.join('/opt', entry.full_name)
175
+ end
176
+ end
177
+ end
178
+ nil
179
+ end
180
+
181
+ def universal_tar_gz
182
+ build_asset_glob = File.join(base_dir, 'target/**/*.tgz')
183
+ tgz_files = Dir.glob(build_asset_glob)
184
+ fail "no universal_tar_gz found: #{build_asset_glob}" \
185
+ if tgz_files.length < 1
186
+ fail "more than one universal_tar_gz found: #{build_asset_glob}" \
187
+ if tgz_files.length > 1
188
+ tgz_files[0]
189
+ end
190
+
191
+ def build
192
+ asset_build
193
+ docker_build
194
+ end
195
+ end
196
+ end