picsolve_docker_builder 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.rspec +2 -0
- data/.rubocop.yml +8 -0
- data/.travis.yml +3 -0
- data/Gemfile +4 -0
- data/README.md +25 -0
- data/Rakefile +80 -0
- data/bin/docker_build +5 -0
- data/integration/integration_play_spec.rb +51 -0
- data/integration/play_hello_world/.docker-builder.yml +3 -0
- data/integration/play_hello_world/.gitignore +12 -0
- data/integration/play_hello_world/Gemfile +2 -0
- data/integration/play_hello_world/LICENSE +8 -0
- data/integration/play_hello_world/README +4 -0
- data/integration/play_hello_world/Rakefile +3 -0
- data/integration/play_hello_world/app/controllers/Application.scala +12 -0
- data/integration/play_hello_world/app/views/index.scala.html +7 -0
- data/integration/play_hello_world/app/views/main.scala.html +15 -0
- data/integration/play_hello_world/build.sbt +20 -0
- data/integration/play_hello_world/conf/application.conf +44 -0
- data/integration/play_hello_world/conf/logback.xml +22 -0
- data/integration/play_hello_world/conf/routes +9 -0
- data/integration/play_hello_world/project/build.properties +4 -0
- data/integration/play_hello_world/project/plugins.sbt +16 -0
- data/integration/play_hello_world/public/images/favicon.png +0 -0
- data/integration/play_hello_world/public/javascripts/hello.js +3 -0
- data/integration/play_hello_world/public/stylesheets/main.css +0 -0
- data/integration/play_hello_world/test/ApplicationSpec.scala +30 -0
- data/integration/play_hello_world/test/IntegrationSpec.scala +24 -0
- data/integration/spec_helper.rb +3 -0
- data/lib/picsolve_docker_builder.rb +5 -0
- data/lib/picsolve_docker_builder/base.rb +66 -0
- data/lib/picsolve_docker_builder/builder/builder.rb +113 -0
- data/lib/picsolve_docker_builder/builder/file.rb +46 -0
- data/lib/picsolve_docker_builder/composer/composer.rb +134 -0
- data/lib/picsolve_docker_builder/composer/image.rb +68 -0
- data/lib/picsolve_docker_builder/composer/registry.rb +80 -0
- data/lib/picsolve_docker_builder/frame.rb +298 -0
- data/lib/picsolve_docker_builder/helpers/kubeclient.rb +34 -0
- data/lib/picsolve_docker_builder/helpers/kubernetes/pod.rb +38 -0
- data/lib/picsolve_docker_builder/helpers/kubernetes/rc.rb +98 -0
- data/lib/picsolve_docker_builder/helpers/kubernetes/resource.rb +28 -0
- data/lib/picsolve_docker_builder/helpers/kubernetes/service.rb +50 -0
- data/lib/picsolve_docker_builder/helpers/kubernetes_manager.rb +102 -0
- data/lib/picsolve_docker_builder/helpers/repository.rb +24 -0
- data/lib/picsolve_docker_builder/helpers/ssh_forward.rb +75 -0
- data/lib/picsolve_docker_builder/scala.rb +196 -0
- data/lib/picsolve_docker_builder/version.rb +4 -0
- data/lib/tasks/compose.rake +25 -0
- data/lib/tasks/docker.rake +24 -0
- data/lib/tasks/scala.rake +11 -0
- data/picsolve_docker_builder.gemspec +35 -0
- metadata +250 -0
@@ -0,0 +1,80 @@
|
|
1
|
+
require 'picsolve_docker_builder/base'
|
2
|
+
require 'excon'
|
3
|
+
require 'json'
|
4
|
+
require 'base64'
|
5
|
+
|
6
|
+
module PicsolveDockerBuilder
|
7
|
+
module Composer
|
8
|
+
# This represents a remote registry, that is queried for
|
9
|
+
# for informations about images
|
10
|
+
class Registry
|
11
|
+
include PicsolveDockerBuilder::Base
|
12
|
+
|
13
|
+
attr_reader :config
|
14
|
+
|
15
|
+
def self.dockercfg_path
|
16
|
+
File.expand_path '~/.dockercfg'
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.dockercfg
|
20
|
+
JSON.parse(
|
21
|
+
File.open(
|
22
|
+
dockercfg_path
|
23
|
+
).read
|
24
|
+
)
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.creds(*args)
|
28
|
+
auth = Base64.decode64(
|
29
|
+
get_login_basic(
|
30
|
+
*args
|
31
|
+
)
|
32
|
+
).split(':')
|
33
|
+
{
|
34
|
+
'username' => auth[0],
|
35
|
+
'password' => auth[1]
|
36
|
+
}
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.get_login_basic(registry = 'docker.picsolve.net')
|
40
|
+
dockercfg[registry]['auth']
|
41
|
+
rescue StandardError
|
42
|
+
nil
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.repo_tag_unique(image_name)
|
46
|
+
repo_tag = image_name.split(%r{/})[1..-1].join('/')
|
47
|
+
repo_tag_split = repo_tag.split(/:/)
|
48
|
+
repo = repo_tag_split[0]
|
49
|
+
tag = repo_tag_split[1] || 'latest'
|
50
|
+
|
51
|
+
headers = {}
|
52
|
+
login_basic = get_login_basic
|
53
|
+
headers['Authorization'] = "Basic #{login_basic}" \
|
54
|
+
unless login_basic.nil?
|
55
|
+
|
56
|
+
connection = Excon.new(
|
57
|
+
'https://docker.picsolve.net',
|
58
|
+
headers: headers
|
59
|
+
)
|
60
|
+
response = connection.get(path: "/v1/repositories/#{repo}/tags")
|
61
|
+
|
62
|
+
tags = JSON.parse(response.body)
|
63
|
+
|
64
|
+
hash = tags[tag]
|
65
|
+
|
66
|
+
tags.each do |t, h|
|
67
|
+
next if h != hash
|
68
|
+
next if t == tag
|
69
|
+
next unless t.match(/jenkins-[0-9]+/)
|
70
|
+
return {
|
71
|
+
tag_unique: "#{image_name.split(':').first}:#{t}",
|
72
|
+
hash: hash
|
73
|
+
}
|
74
|
+
end
|
75
|
+
|
76
|
+
fail "Can not find a uniqe tag for #{image_name}"
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,298 @@
|
|
1
|
+
require 'picsolve_docker_builder/composer/registry'
|
2
|
+
require 'picsolve_docker_builder/base'
|
3
|
+
require 'docker'
|
4
|
+
require 'logger'
|
5
|
+
require 'psych'
|
6
|
+
|
7
|
+
module PicsolveDockerBuilder
|
8
|
+
# Docker image building template
|
9
|
+
class Frame
|
10
|
+
include PicsolveDockerBuilder::Base
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
# TODO: @christiansimon please report that upstream, second time
|
14
|
+
# i am facing that bug
|
15
|
+
Docker.options[:read_timeout] = 3600
|
16
|
+
Excon.defaults[:read_timeout] = 3600
|
17
|
+
config
|
18
|
+
end
|
19
|
+
|
20
|
+
def default_config
|
21
|
+
{ 'docker' => {} }
|
22
|
+
end
|
23
|
+
|
24
|
+
def execute(cmd)
|
25
|
+
r = container.exec(cmd)
|
26
|
+
fail "Execution of cmd=#{cmd} failed" unless r[2] == 0
|
27
|
+
log.debug "executed container id=#{container.id} cmd=#{cmd} result=#{r}"
|
28
|
+
r
|
29
|
+
end
|
30
|
+
|
31
|
+
def execute_attach(cmd)
|
32
|
+
log.debug "execute and attach container id=#{container.id} cmd=#{cmd}"
|
33
|
+
r = container.exec(
|
34
|
+
cmd
|
35
|
+
) do |_stream, chunk|
|
36
|
+
$stdout.write chunk
|
37
|
+
$stdout.flush
|
38
|
+
end
|
39
|
+
fail "Execution of cmd=#{cmd} failed" unless r[2] == 0
|
40
|
+
log.debug 'executed and attached container ' \
|
41
|
+
"id=#{container.id} cmd=#{cmd} exitcode=#{r[2]}"
|
42
|
+
end
|
43
|
+
|
44
|
+
def build_mode
|
45
|
+
return :template unless image_name.nil?
|
46
|
+
|
47
|
+
return :dockerfile if dockerfile_exists?
|
48
|
+
|
49
|
+
log.fatal 'No image_name configured and no Dockerfile present'
|
50
|
+
end
|
51
|
+
|
52
|
+
def dockerfile_hooks_docker_build_early
|
53
|
+
config['docker']['dockerfile_hooks']['docker_build']['early']
|
54
|
+
rescue NoMethodError
|
55
|
+
''
|
56
|
+
end
|
57
|
+
|
58
|
+
def dockerfile_hooks_docker_build_late
|
59
|
+
config['docker']['dockerfile_hooks']['docker_build']['late']
|
60
|
+
rescue NoMethodError
|
61
|
+
''
|
62
|
+
end
|
63
|
+
|
64
|
+
def dockerfile_hooks_asset_build_early
|
65
|
+
config['docker']['dockerfile_hooks']['asset_build']['early']
|
66
|
+
rescue NoMethodError
|
67
|
+
''
|
68
|
+
end
|
69
|
+
|
70
|
+
def dockerfile_hooks_asset_build_late
|
71
|
+
config['docker']['dockerfile_hooks']['asset_build']['late']
|
72
|
+
rescue NoMethodError
|
73
|
+
''
|
74
|
+
end
|
75
|
+
|
76
|
+
def jenkins_build_number
|
77
|
+
n = ENV['BUILD_NUMBER']
|
78
|
+
return nil if n.nil?
|
79
|
+
n.to_i
|
80
|
+
end
|
81
|
+
|
82
|
+
def tags
|
83
|
+
t = ['latest']
|
84
|
+
t << "jenkins-#{jenkins_build_number}" unless jenkins_build_number.nil?
|
85
|
+
t
|
86
|
+
end
|
87
|
+
|
88
|
+
def tag
|
89
|
+
fail 'No image found to tag' if @docker_build.nil?
|
90
|
+
tags.each do |tag|
|
91
|
+
log.info "tagging image with #{dest_image_name}:#{tag}"
|
92
|
+
@docker_build.tag(
|
93
|
+
repo: dest_image_name,
|
94
|
+
tag: tag,
|
95
|
+
force: true
|
96
|
+
)
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def push
|
101
|
+
fail 'No image found to be pushed' if @docker_build.nil?
|
102
|
+
tags.each do |tag|
|
103
|
+
repotag = "#{dest_image_name}:#{tag}"
|
104
|
+
log.info "pushing image with #{repotag}"
|
105
|
+
@docker_build.push(
|
106
|
+
Composer::Registry.creds,
|
107
|
+
tag: tag
|
108
|
+
) do |resp|
|
109
|
+
resp = JSON.parse resp
|
110
|
+
|
111
|
+
if resp.key? 'errorDetail'
|
112
|
+
message = "pushing image #{repotag} failed: #{resp['errorDetail']}"
|
113
|
+
log.fatal message
|
114
|
+
fail message
|
115
|
+
end
|
116
|
+
|
117
|
+
log.info "status pushing image #{repotag}: #{resp['status']}" \
|
118
|
+
if resp.key? 'status'
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def docker_build_build
|
124
|
+
Docker::Image.build_from_dir(base_dir) do |stream|
|
125
|
+
s = JSON.parse(stream)['stream']
|
126
|
+
log.debug s.strip unless s.nil?
|
127
|
+
end
|
128
|
+
rescue StandardError => e
|
129
|
+
log.fatal "docker building failed: #{e}"
|
130
|
+
exit 1
|
131
|
+
end
|
132
|
+
|
133
|
+
def docker_build
|
134
|
+
dockerfile
|
135
|
+
|
136
|
+
log.info "start docker image building with path #{base_dir}"
|
137
|
+
|
138
|
+
@docker_build = docker_build_build
|
139
|
+
end
|
140
|
+
|
141
|
+
def build
|
142
|
+
docker_build
|
143
|
+
end
|
144
|
+
|
145
|
+
def dockerfile_path
|
146
|
+
File.join(base_dir, 'Dockerfile')
|
147
|
+
end
|
148
|
+
|
149
|
+
def dockerfile
|
150
|
+
return unless build_mode == :template
|
151
|
+
|
152
|
+
File.open('Dockerfile', 'w') do |file|
|
153
|
+
dockerfile_template.each_line do |line|
|
154
|
+
log.debug "Dockerfile: #{line.strip}"
|
155
|
+
end
|
156
|
+
file.write(dockerfile_template)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
def dockerfile_exists?
|
161
|
+
File.exist? dockerfile_path
|
162
|
+
end
|
163
|
+
|
164
|
+
def dockerfile_template
|
165
|
+
fail NotImplementedError
|
166
|
+
end
|
167
|
+
|
168
|
+
def dockerignore_template
|
169
|
+
fail NotImplementedError
|
170
|
+
end
|
171
|
+
|
172
|
+
def validate_config(c)
|
173
|
+
validate_config_docker(c)
|
174
|
+
end
|
175
|
+
|
176
|
+
def validate_config_docker(c)
|
177
|
+
c
|
178
|
+
end
|
179
|
+
|
180
|
+
def start
|
181
|
+
container.start('Binds' => volumes)
|
182
|
+
|
183
|
+
log.debug "started container id=#{container.id} volumes=#{volumes}"
|
184
|
+
end
|
185
|
+
|
186
|
+
def stop
|
187
|
+
return if @container.nil?
|
188
|
+
container.stop
|
189
|
+
log.debug "stopped container id=#{container.id}"
|
190
|
+
container.remove
|
191
|
+
log.debug "removed container id=#{container.id}"
|
192
|
+
@container = nil
|
193
|
+
end
|
194
|
+
|
195
|
+
def volume_workspace
|
196
|
+
[
|
197
|
+
base_dir,
|
198
|
+
build_dir
|
199
|
+
]
|
200
|
+
end
|
201
|
+
|
202
|
+
def volumes_array
|
203
|
+
[volume_workspace]
|
204
|
+
end
|
205
|
+
|
206
|
+
def volumes
|
207
|
+
volumes_array.map do |volume|
|
208
|
+
volume.join ':'
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
def build_user_uid
|
213
|
+
Process.uid.to_s
|
214
|
+
end
|
215
|
+
|
216
|
+
def build_user_home
|
217
|
+
'/home/build'
|
218
|
+
end
|
219
|
+
|
220
|
+
def build_user
|
221
|
+
'build'
|
222
|
+
end
|
223
|
+
|
224
|
+
def build_dir
|
225
|
+
'/_build'
|
226
|
+
end
|
227
|
+
|
228
|
+
def container
|
229
|
+
@container ||= create_container
|
230
|
+
end
|
231
|
+
|
232
|
+
def create_container
|
233
|
+
command = ['/bin/sleep', '3600']
|
234
|
+
c = Docker::Container.create(
|
235
|
+
'Image' => asset_image.id,
|
236
|
+
'Cmd' => command,
|
237
|
+
'OpenStdin' => false,
|
238
|
+
'WorkingDir' => '/_build'
|
239
|
+
)
|
240
|
+
at_exit do
|
241
|
+
stop
|
242
|
+
end
|
243
|
+
log.debug "created a new container image=#{image_name} " \
|
244
|
+
"id=#{c.id} cmd=#{command}"
|
245
|
+
c
|
246
|
+
end
|
247
|
+
|
248
|
+
def dest_image_name
|
249
|
+
config['docker']['image_name']
|
250
|
+
end
|
251
|
+
|
252
|
+
def image_name
|
253
|
+
name = config['docker']['base_image']
|
254
|
+
|
255
|
+
if name.match(/:[a-z0-9\-_]+$/)
|
256
|
+
name
|
257
|
+
else
|
258
|
+
"#{name}:latest"
|
259
|
+
end
|
260
|
+
rescue NoMethodError
|
261
|
+
nil
|
262
|
+
end
|
263
|
+
|
264
|
+
def fetch_asset_image
|
265
|
+
log.debug "pulling image '#{image_name}' from registry"
|
266
|
+
dockerfile_asset = <<EOS
|
267
|
+
FROM #{image_name}
|
268
|
+
MAINTAINER Picsolve Onlineops <onlineops@picsolve.com>
|
269
|
+
#{dockerfile_hooks_asset_build_early}
|
270
|
+
RUN useradd -d #{build_user_home} -u #{build_user_uid} #{build_user}
|
271
|
+
#{dockerfile_hooks_asset_build_late}
|
272
|
+
EOS
|
273
|
+
begin
|
274
|
+
Docker::Image.build(dockerfile_asset) do |stream|
|
275
|
+
s = JSON.parse(stream)['stream']
|
276
|
+
log.debug s.strip unless s.nil?
|
277
|
+
end
|
278
|
+
rescue StandardError => e
|
279
|
+
log.fatal "asset building failed: #{e}"
|
280
|
+
exit 1
|
281
|
+
end
|
282
|
+
end
|
283
|
+
|
284
|
+
def asset_image
|
285
|
+
@asset_image ||= fetch_asset_image
|
286
|
+
end
|
287
|
+
|
288
|
+
def create_logger
|
289
|
+
log = Logger.new(STDOUT)
|
290
|
+
log.level = Logger::DEBUG
|
291
|
+
log
|
292
|
+
end
|
293
|
+
|
294
|
+
def log
|
295
|
+
@logger ||= create_logger
|
296
|
+
end
|
297
|
+
end
|
298
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'kubeclient'
|
2
|
+
|
3
|
+
module Kubeclient
|
4
|
+
module Common
|
5
|
+
# Hot fix for kubeclient should be going upstream
|
6
|
+
# * https://github.com/abonas/kubeclient/pull/96
|
7
|
+
class Client
|
8
|
+
def get_entities(entity_type, klass, options)
|
9
|
+
params = {}
|
10
|
+
if options[:label_selector]
|
11
|
+
params['params'] = { labelSelector: options[:label_selector] }
|
12
|
+
end
|
13
|
+
|
14
|
+
ns_prefix = build_namespace_prefix(options[:namespace])
|
15
|
+
response = handle_exception do
|
16
|
+
rest_client[ns_prefix + resource_name(entity_type)]
|
17
|
+
.get(params.merge(@headers))
|
18
|
+
end
|
19
|
+
|
20
|
+
result = JSON.parse(response)
|
21
|
+
|
22
|
+
resource_version = result.fetch('resourceVersion', nil)
|
23
|
+
if resource_version.nil?
|
24
|
+
resource_version =
|
25
|
+
result.fetch('metadata', {}).fetch('resourceVersion', nil)
|
26
|
+
end
|
27
|
+
|
28
|
+
collection = result['items'].map { |item| new_entity(item, klass) }
|
29
|
+
|
30
|
+
EntityList.new(entity_type, resource_version, collection)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'date'
|
2
|
+
require 'picsolve_docker_builder/helpers/kubernetes/resource'
|
3
|
+
|
4
|
+
module PicsolveDockerBuilder
|
5
|
+
module Helpers
|
6
|
+
module Kubernetes
|
7
|
+
# A generic kuberntes resource
|
8
|
+
class Pod < Resource
|
9
|
+
attr_reader :pod
|
10
|
+
|
11
|
+
def initialize(pod, kubernetes)
|
12
|
+
@pod = pod
|
13
|
+
@kubernetes = kubernetes
|
14
|
+
# seconds after a running pod consider as healthy
|
15
|
+
@grace_time = 20
|
16
|
+
end
|
17
|
+
|
18
|
+
def remove
|
19
|
+
log.debug "remove pod #{pod.metadata.name}"
|
20
|
+
client.delete_pod(@pod.metadata.name, @pod.metadata.namespace)
|
21
|
+
end
|
22
|
+
|
23
|
+
def ready?
|
24
|
+
# ensure pod is running
|
25
|
+
return false unless pod.status.phase == 'Running'
|
26
|
+
|
27
|
+
diff = DateTime.now - DateTime.strptime(pod.status.startTime)
|
28
|
+
diff_seconds = diff * 24 * 60 * 60
|
29
|
+
|
30
|
+
# ensure pod is running for more than 20 seconds
|
31
|
+
return false if diff_seconds < @grace_time
|
32
|
+
|
33
|
+
true
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|