picsolve_docker_builder 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +11 -0
  3. data/.rspec +2 -0
  4. data/.rubocop.yml +8 -0
  5. data/.travis.yml +3 -0
  6. data/Gemfile +4 -0
  7. data/README.md +25 -0
  8. data/Rakefile +80 -0
  9. data/bin/docker_build +5 -0
  10. data/integration/integration_play_spec.rb +51 -0
  11. data/integration/play_hello_world/.docker-builder.yml +3 -0
  12. data/integration/play_hello_world/.gitignore +12 -0
  13. data/integration/play_hello_world/Gemfile +2 -0
  14. data/integration/play_hello_world/LICENSE +8 -0
  15. data/integration/play_hello_world/README +4 -0
  16. data/integration/play_hello_world/Rakefile +3 -0
  17. data/integration/play_hello_world/app/controllers/Application.scala +12 -0
  18. data/integration/play_hello_world/app/views/index.scala.html +7 -0
  19. data/integration/play_hello_world/app/views/main.scala.html +15 -0
  20. data/integration/play_hello_world/build.sbt +20 -0
  21. data/integration/play_hello_world/conf/application.conf +44 -0
  22. data/integration/play_hello_world/conf/logback.xml +22 -0
  23. data/integration/play_hello_world/conf/routes +9 -0
  24. data/integration/play_hello_world/project/build.properties +4 -0
  25. data/integration/play_hello_world/project/plugins.sbt +16 -0
  26. data/integration/play_hello_world/public/images/favicon.png +0 -0
  27. data/integration/play_hello_world/public/javascripts/hello.js +3 -0
  28. data/integration/play_hello_world/public/stylesheets/main.css +0 -0
  29. data/integration/play_hello_world/test/ApplicationSpec.scala +30 -0
  30. data/integration/play_hello_world/test/IntegrationSpec.scala +24 -0
  31. data/integration/spec_helper.rb +3 -0
  32. data/lib/picsolve_docker_builder.rb +5 -0
  33. data/lib/picsolve_docker_builder/base.rb +66 -0
  34. data/lib/picsolve_docker_builder/builder/builder.rb +113 -0
  35. data/lib/picsolve_docker_builder/builder/file.rb +46 -0
  36. data/lib/picsolve_docker_builder/composer/composer.rb +134 -0
  37. data/lib/picsolve_docker_builder/composer/image.rb +68 -0
  38. data/lib/picsolve_docker_builder/composer/registry.rb +80 -0
  39. data/lib/picsolve_docker_builder/frame.rb +298 -0
  40. data/lib/picsolve_docker_builder/helpers/kubeclient.rb +34 -0
  41. data/lib/picsolve_docker_builder/helpers/kubernetes/pod.rb +38 -0
  42. data/lib/picsolve_docker_builder/helpers/kubernetes/rc.rb +98 -0
  43. data/lib/picsolve_docker_builder/helpers/kubernetes/resource.rb +28 -0
  44. data/lib/picsolve_docker_builder/helpers/kubernetes/service.rb +50 -0
  45. data/lib/picsolve_docker_builder/helpers/kubernetes_manager.rb +102 -0
  46. data/lib/picsolve_docker_builder/helpers/repository.rb +24 -0
  47. data/lib/picsolve_docker_builder/helpers/ssh_forward.rb +75 -0
  48. data/lib/picsolve_docker_builder/scala.rb +196 -0
  49. data/lib/picsolve_docker_builder/version.rb +4 -0
  50. data/lib/tasks/compose.rake +25 -0
  51. data/lib/tasks/docker.rake +24 -0
  52. data/lib/tasks/scala.rake +11 -0
  53. data/picsolve_docker_builder.gemspec +35 -0
  54. metadata +250 -0
@@ -0,0 +1,80 @@
1
+ require 'picsolve_docker_builder/base'
2
+ require 'excon'
3
+ require 'json'
4
+ require 'base64'
5
+
6
+ module PicsolveDockerBuilder
7
+ module Composer
8
+ # This represents a remote registry, that is queried for
9
+ # for informations about images
10
+ class Registry
11
+ include PicsolveDockerBuilder::Base
12
+
13
+ attr_reader :config
14
+
15
+ def self.dockercfg_path
16
+ File.expand_path '~/.dockercfg'
17
+ end
18
+
19
+ def self.dockercfg
20
+ JSON.parse(
21
+ File.open(
22
+ dockercfg_path
23
+ ).read
24
+ )
25
+ end
26
+
27
+ def self.creds(*args)
28
+ auth = Base64.decode64(
29
+ get_login_basic(
30
+ *args
31
+ )
32
+ ).split(':')
33
+ {
34
+ 'username' => auth[0],
35
+ 'password' => auth[1]
36
+ }
37
+ end
38
+
39
+ def self.get_login_basic(registry = 'docker.picsolve.net')
40
+ dockercfg[registry]['auth']
41
+ rescue StandardError
42
+ nil
43
+ end
44
+
45
+ def self.repo_tag_unique(image_name)
46
+ repo_tag = image_name.split(%r{/})[1..-1].join('/')
47
+ repo_tag_split = repo_tag.split(/:/)
48
+ repo = repo_tag_split[0]
49
+ tag = repo_tag_split[1] || 'latest'
50
+
51
+ headers = {}
52
+ login_basic = get_login_basic
53
+ headers['Authorization'] = "Basic #{login_basic}" \
54
+ unless login_basic.nil?
55
+
56
+ connection = Excon.new(
57
+ 'https://docker.picsolve.net',
58
+ headers: headers
59
+ )
60
+ response = connection.get(path: "/v1/repositories/#{repo}/tags")
61
+
62
+ tags = JSON.parse(response.body)
63
+
64
+ hash = tags[tag]
65
+
66
+ tags.each do |t, h|
67
+ next if h != hash
68
+ next if t == tag
69
+ next unless t.match(/jenkins-[0-9]+/)
70
+ return {
71
+ tag_unique: "#{image_name.split(':').first}:#{t}",
72
+ hash: hash
73
+ }
74
+ end
75
+
76
+ fail "Can not find a uniqe tag for #{image_name}"
77
+ end
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,298 @@
1
+ require 'picsolve_docker_builder/composer/registry'
2
+ require 'picsolve_docker_builder/base'
3
+ require 'docker'
4
+ require 'logger'
5
+ require 'psych'
6
+
7
+ module PicsolveDockerBuilder
8
+ # Docker image building template
9
+ class Frame
10
+ include PicsolveDockerBuilder::Base
11
+
12
+ def initialize
13
+ # TODO: @christiansimon please report that upstream, second time
14
+ # i am facing that bug
15
+ Docker.options[:read_timeout] = 3600
16
+ Excon.defaults[:read_timeout] = 3600
17
+ config
18
+ end
19
+
20
+ def default_config
21
+ { 'docker' => {} }
22
+ end
23
+
24
+ def execute(cmd)
25
+ r = container.exec(cmd)
26
+ fail "Execution of cmd=#{cmd} failed" unless r[2] == 0
27
+ log.debug "executed container id=#{container.id} cmd=#{cmd} result=#{r}"
28
+ r
29
+ end
30
+
31
+ def execute_attach(cmd)
32
+ log.debug "execute and attach container id=#{container.id} cmd=#{cmd}"
33
+ r = container.exec(
34
+ cmd
35
+ ) do |_stream, chunk|
36
+ $stdout.write chunk
37
+ $stdout.flush
38
+ end
39
+ fail "Execution of cmd=#{cmd} failed" unless r[2] == 0
40
+ log.debug 'executed and attached container ' \
41
+ "id=#{container.id} cmd=#{cmd} exitcode=#{r[2]}"
42
+ end
43
+
44
+ def build_mode
45
+ return :template unless image_name.nil?
46
+
47
+ return :dockerfile if dockerfile_exists?
48
+
49
+ log.fatal 'No image_name configured and no Dockerfile present'
50
+ end
51
+
52
+ def dockerfile_hooks_docker_build_early
53
+ config['docker']['dockerfile_hooks']['docker_build']['early']
54
+ rescue NoMethodError
55
+ ''
56
+ end
57
+
58
+ def dockerfile_hooks_docker_build_late
59
+ config['docker']['dockerfile_hooks']['docker_build']['late']
60
+ rescue NoMethodError
61
+ ''
62
+ end
63
+
64
+ def dockerfile_hooks_asset_build_early
65
+ config['docker']['dockerfile_hooks']['asset_build']['early']
66
+ rescue NoMethodError
67
+ ''
68
+ end
69
+
70
+ def dockerfile_hooks_asset_build_late
71
+ config['docker']['dockerfile_hooks']['asset_build']['late']
72
+ rescue NoMethodError
73
+ ''
74
+ end
75
+
76
+ def jenkins_build_number
77
+ n = ENV['BUILD_NUMBER']
78
+ return nil if n.nil?
79
+ n.to_i
80
+ end
81
+
82
+ def tags
83
+ t = ['latest']
84
+ t << "jenkins-#{jenkins_build_number}" unless jenkins_build_number.nil?
85
+ t
86
+ end
87
+
88
+ def tag
89
+ fail 'No image found to tag' if @docker_build.nil?
90
+ tags.each do |tag|
91
+ log.info "tagging image with #{dest_image_name}:#{tag}"
92
+ @docker_build.tag(
93
+ repo: dest_image_name,
94
+ tag: tag,
95
+ force: true
96
+ )
97
+ end
98
+ end
99
+
100
+ def push
101
+ fail 'No image found to be pushed' if @docker_build.nil?
102
+ tags.each do |tag|
103
+ repotag = "#{dest_image_name}:#{tag}"
104
+ log.info "pushing image with #{repotag}"
105
+ @docker_build.push(
106
+ Composer::Registry.creds,
107
+ tag: tag
108
+ ) do |resp|
109
+ resp = JSON.parse resp
110
+
111
+ if resp.key? 'errorDetail'
112
+ message = "pushing image #{repotag} failed: #{resp['errorDetail']}"
113
+ log.fatal message
114
+ fail message
115
+ end
116
+
117
+ log.info "status pushing image #{repotag}: #{resp['status']}" \
118
+ if resp.key? 'status'
119
+ end
120
+ end
121
+ end
122
+
123
+ def docker_build_build
124
+ Docker::Image.build_from_dir(base_dir) do |stream|
125
+ s = JSON.parse(stream)['stream']
126
+ log.debug s.strip unless s.nil?
127
+ end
128
+ rescue StandardError => e
129
+ log.fatal "docker building failed: #{e}"
130
+ exit 1
131
+ end
132
+
133
+ def docker_build
134
+ dockerfile
135
+
136
+ log.info "start docker image building with path #{base_dir}"
137
+
138
+ @docker_build = docker_build_build
139
+ end
140
+
141
+ def build
142
+ docker_build
143
+ end
144
+
145
+ def dockerfile_path
146
+ File.join(base_dir, 'Dockerfile')
147
+ end
148
+
149
+ def dockerfile
150
+ return unless build_mode == :template
151
+
152
+ File.open('Dockerfile', 'w') do |file|
153
+ dockerfile_template.each_line do |line|
154
+ log.debug "Dockerfile: #{line.strip}"
155
+ end
156
+ file.write(dockerfile_template)
157
+ end
158
+ end
159
+
160
+ def dockerfile_exists?
161
+ File.exist? dockerfile_path
162
+ end
163
+
164
+ def dockerfile_template
165
+ fail NotImplementedError
166
+ end
167
+
168
+ def dockerignore_template
169
+ fail NotImplementedError
170
+ end
171
+
172
+ def validate_config(c)
173
+ validate_config_docker(c)
174
+ end
175
+
176
+ def validate_config_docker(c)
177
+ c
178
+ end
179
+
180
+ def start
181
+ container.start('Binds' => volumes)
182
+
183
+ log.debug "started container id=#{container.id} volumes=#{volumes}"
184
+ end
185
+
186
+ def stop
187
+ return if @container.nil?
188
+ container.stop
189
+ log.debug "stopped container id=#{container.id}"
190
+ container.remove
191
+ log.debug "removed container id=#{container.id}"
192
+ @container = nil
193
+ end
194
+
195
+ def volume_workspace
196
+ [
197
+ base_dir,
198
+ build_dir
199
+ ]
200
+ end
201
+
202
+ def volumes_array
203
+ [volume_workspace]
204
+ end
205
+
206
+ def volumes
207
+ volumes_array.map do |volume|
208
+ volume.join ':'
209
+ end
210
+ end
211
+
212
+ def build_user_uid
213
+ Process.uid.to_s
214
+ end
215
+
216
+ def build_user_home
217
+ '/home/build'
218
+ end
219
+
220
+ def build_user
221
+ 'build'
222
+ end
223
+
224
+ def build_dir
225
+ '/_build'
226
+ end
227
+
228
+ def container
229
+ @container ||= create_container
230
+ end
231
+
232
+ def create_container
233
+ command = ['/bin/sleep', '3600']
234
+ c = Docker::Container.create(
235
+ 'Image' => asset_image.id,
236
+ 'Cmd' => command,
237
+ 'OpenStdin' => false,
238
+ 'WorkingDir' => '/_build'
239
+ )
240
+ at_exit do
241
+ stop
242
+ end
243
+ log.debug "created a new container image=#{image_name} " \
244
+ "id=#{c.id} cmd=#{command}"
245
+ c
246
+ end
247
+
248
+ def dest_image_name
249
+ config['docker']['image_name']
250
+ end
251
+
252
+ def image_name
253
+ name = config['docker']['base_image']
254
+
255
+ if name.match(/:[a-z0-9\-_]+$/)
256
+ name
257
+ else
258
+ "#{name}:latest"
259
+ end
260
+ rescue NoMethodError
261
+ nil
262
+ end
263
+
264
+ def fetch_asset_image
265
+ log.debug "pulling image '#{image_name}' from registry"
266
+ dockerfile_asset = <<EOS
267
+ FROM #{image_name}
268
+ MAINTAINER Picsolve Onlineops <onlineops@picsolve.com>
269
+ #{dockerfile_hooks_asset_build_early}
270
+ RUN useradd -d #{build_user_home} -u #{build_user_uid} #{build_user}
271
+ #{dockerfile_hooks_asset_build_late}
272
+ EOS
273
+ begin
274
+ Docker::Image.build(dockerfile_asset) do |stream|
275
+ s = JSON.parse(stream)['stream']
276
+ log.debug s.strip unless s.nil?
277
+ end
278
+ rescue StandardError => e
279
+ log.fatal "asset building failed: #{e}"
280
+ exit 1
281
+ end
282
+ end
283
+
284
+ def asset_image
285
+ @asset_image ||= fetch_asset_image
286
+ end
287
+
288
+ def create_logger
289
+ log = Logger.new(STDOUT)
290
+ log.level = Logger::DEBUG
291
+ log
292
+ end
293
+
294
+ def log
295
+ @logger ||= create_logger
296
+ end
297
+ end
298
+ end
@@ -0,0 +1,34 @@
1
+ require 'kubeclient'
2
+
3
+ module Kubeclient
4
+ module Common
5
+ # Hot fix for kubeclient should be going upstream
6
+ # * https://github.com/abonas/kubeclient/pull/96
7
+ class Client
8
+ def get_entities(entity_type, klass, options)
9
+ params = {}
10
+ if options[:label_selector]
11
+ params['params'] = { labelSelector: options[:label_selector] }
12
+ end
13
+
14
+ ns_prefix = build_namespace_prefix(options[:namespace])
15
+ response = handle_exception do
16
+ rest_client[ns_prefix + resource_name(entity_type)]
17
+ .get(params.merge(@headers))
18
+ end
19
+
20
+ result = JSON.parse(response)
21
+
22
+ resource_version = result.fetch('resourceVersion', nil)
23
+ if resource_version.nil?
24
+ resource_version =
25
+ result.fetch('metadata', {}).fetch('resourceVersion', nil)
26
+ end
27
+
28
+ collection = result['items'].map { |item| new_entity(item, klass) }
29
+
30
+ EntityList.new(entity_type, resource_version, collection)
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,38 @@
1
+ require 'date'
2
+ require 'picsolve_docker_builder/helpers/kubernetes/resource'
3
+
4
+ module PicsolveDockerBuilder
5
+ module Helpers
6
+ module Kubernetes
7
+ # A generic kuberntes resource
8
+ class Pod < Resource
9
+ attr_reader :pod
10
+
11
+ def initialize(pod, kubernetes)
12
+ @pod = pod
13
+ @kubernetes = kubernetes
14
+ # seconds after a running pod consider as healthy
15
+ @grace_time = 20
16
+ end
17
+
18
+ def remove
19
+ log.debug "remove pod #{pod.metadata.name}"
20
+ client.delete_pod(@pod.metadata.name, @pod.metadata.namespace)
21
+ end
22
+
23
+ def ready?
24
+ # ensure pod is running
25
+ return false unless pod.status.phase == 'Running'
26
+
27
+ diff = DateTime.now - DateTime.strptime(pod.status.startTime)
28
+ diff_seconds = diff * 24 * 60 * 60
29
+
30
+ # ensure pod is running for more than 20 seconds
31
+ return false if diff_seconds < @grace_time
32
+
33
+ true
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end