picsolve_docker_builder 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +7 -0
  3. data/README.md +45 -2
  4. data/integration/compose_hello_world_new_config/.docker-builder.yml +18 -0
  5. data/integration/compose_hello_world_new_config/.gitignore +1 -0
  6. data/integration/compose_hello_world_new_config/Gemfile +2 -0
  7. data/integration/compose_hello_world_new_config/Rakefile +3 -0
  8. data/integration/compose_hello_world_old_config/.docker-builder.yml +4 -0
  9. data/integration/compose_hello_world_old_config/.gitignore +1 -0
  10. data/integration/compose_hello_world_old_config/Gemfile +2 -0
  11. data/integration/compose_hello_world_old_config/Rakefile +3 -0
  12. data/integration/compose_hello_world_old_config/docker-compose.yml +4 -0
  13. data/integration/integration_compose_spec.rb +25 -0
  14. data/integration/integration_helpers.rb +45 -0
  15. data/integration/integration_play_spec.rb +4 -38
  16. data/integration/lib_hello_world/.gitignore +12 -0
  17. data/integration/lib_hello_world/README +4 -0
  18. data/integration/lib_hello_world/app/picsolve/helloworld/Person.scala +8 -0
  19. data/integration/lib_hello_world/build.sbt +42 -0
  20. data/integration/lib_hello_world/conf/application.conf +0 -0
  21. data/integration/lib_hello_world/conf/logback.xml +22 -0
  22. data/integration/lib_hello_world/project/build.properties +4 -0
  23. data/integration/lib_hello_world/project/plugins.sbt +4 -0
  24. data/integration/lib_hello_world/test/PersonSpec.scala +24 -0
  25. data/integration/lib_hello_world/version.sbt +1 -0
  26. data/integration/spec_helper.rb +2 -0
  27. data/lib/picsolve_docker_builder/builder/builder.rb +4 -2
  28. data/lib/picsolve_docker_builder/composer/composer.rb +45 -74
  29. data/lib/picsolve_docker_builder/composer/container.rb +79 -0
  30. data/lib/picsolve_docker_builder/composer/image.rb +10 -8
  31. data/lib/picsolve_docker_builder/frame.rb +13 -1
  32. data/lib/picsolve_docker_builder/helpers/config/base.rb +14 -0
  33. data/lib/picsolve_docker_builder/helpers/config/secret.rb +14 -0
  34. data/lib/picsolve_docker_builder/helpers/config/variable_object.rb +68 -0
  35. data/lib/picsolve_docker_builder/helpers/config_manager.rb +96 -0
  36. data/lib/picsolve_docker_builder/helpers/kubernetes/rc.rb +28 -6
  37. data/lib/picsolve_docker_builder/helpers/kubernetes_manager.rb +126 -47
  38. data/lib/picsolve_docker_builder/helpers/repository.rb +89 -3
  39. data/lib/picsolve_docker_builder/helpers/ssh_forward.rb +9 -1
  40. data/lib/picsolve_docker_builder/nodejs.rb +135 -0
  41. data/lib/picsolve_docker_builder/version.rb +1 -1
  42. data/lib/tasks/compose.rake +2 -0
  43. data/lib/tasks/nodejs.rake +11 -0
  44. data/picsolve_docker_builder.gemspec +1 -1
  45. metadata +36 -8
@@ -19,16 +19,18 @@ module PicsolveDockerBuilder
19
19
  config['image']
20
20
  end
21
21
 
22
- def kubernetes
23
- composer.kubernetes
22
+ def rc(kubernetes)
23
+ @rcs = {} if @rcs.nil?
24
+ @rcs[kubernetes] = kubernetes.rc(self) \
25
+ unless @rcs.key?(kubernetes)
26
+ @rcs[kubernetes]
24
27
  end
25
28
 
26
- def rc
27
- @rc ||= kubernetes.rc self
28
- end
29
-
30
- def service
31
- @service ||= kubernetes.service self
29
+ def service(kubernetes)
30
+ @services = {} if @rcs.nil?
31
+ @services[kubernetes] = kubernetes.service(self) \
32
+ unless @services.key?(kubernetes)
33
+ @services[kubernetes]
32
34
  end
33
35
 
34
36
  def ports
@@ -249,6 +249,18 @@ module PicsolveDockerBuilder
249
249
  config['docker']['image_name']
250
250
  end
251
251
 
252
+ def runtime_image_name
253
+ name = config['docker']['runtime_image'] || image_name
254
+
255
+ if name.match(/:[a-z0-9\-_]+$/)
256
+ name
257
+ else
258
+ "#{name}:latest"
259
+ end
260
+ rescue NoMethodError
261
+ nil
262
+ end
263
+
252
264
  def image_name
253
265
  name = config['docker']['base_image']
254
266
 
@@ -267,7 +279,7 @@ module PicsolveDockerBuilder
267
279
  FROM #{image_name}
268
280
  MAINTAINER Picsolve Onlineops <onlineops@picsolve.com>
269
281
  #{dockerfile_hooks_asset_build_early}
270
- RUN useradd -d #{build_user_home} -u #{build_user_uid} #{build_user}
282
+ RUN useradd -m -d #{build_user_home} -u #{build_user_uid} #{build_user}
271
283
  #{dockerfile_hooks_asset_build_late}
272
284
  EOS
273
285
  begin
@@ -0,0 +1,14 @@
1
+ module PicsolveDockerBuilder
2
+ module Helpers
3
+ module Config
4
+ # Base class for a dynamic config object
5
+ class Base
6
+ def self.name
7
+ nil
8
+ end
9
+ def initialize(_identifier)
10
+ end
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,14 @@
1
+ module PicsolveDockerBuilder
2
+ module Helpers
3
+ module Config
4
+ # Config object that is retrieved from kubernetes secrets store
5
+ class Secret < Base
6
+ def self.name
7
+ 'secret'
8
+ end
9
+ def initialize(_identifier)
10
+ end
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,68 @@
1
+ module PicsolveDockerBuilder
2
+ module Helpers
3
+ module Config
4
+ # represents a config string that contains variables
5
+ class VariableObject
6
+ # Regex that finds variabales
7
+ def self.regex_full
8
+ /^\$\{([^\}]*)\}$/
9
+ end
10
+ def self.regex
11
+ /\$\{[^\}]*\}/
12
+ end
13
+ # Checks if a string needs to be replaced by an VariableObject
14
+ def self.replace_string(obj)
15
+ return obj unless VariableObject.regex.match(obj)
16
+ VariableObject.new(obj)
17
+ end
18
+
19
+ def list
20
+ @list ||= create_list
21
+ end
22
+
23
+ # Split the strings accordingly
24
+ def create_list
25
+ lst = split_list
26
+ replace_variables(lst)
27
+ end
28
+
29
+ # Replace variables
30
+ def replace_variables(list)
31
+ list.map! do |elem|
32
+ m = elem.match(VariableObject.regex_full)
33
+ if m
34
+ "var=#{m[1]}"
35
+ else
36
+ elem
37
+ end
38
+ end
39
+ end
40
+
41
+ def to_s
42
+ # TODO: implement var evaluation here
43
+ "<#VariableObject#{list}>"
44
+ end
45
+
46
+ def inspect
47
+ "<#VariableObject#{list}>"
48
+ end
49
+
50
+ # Split the strings accordingly
51
+ def split_list(existing_list = [])
52
+ if existing_list.length == 0
53
+ str = @str
54
+ else
55
+ str = existing_list.pop
56
+ return existing_list if str.length == 0
57
+ end
58
+ list = existing_list + str.partition(VariableObject.regex)
59
+ split_list(list)
60
+ end
61
+
62
+ def initialize(str)
63
+ @str = str
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,96 @@
1
+ require 'picsolve_docker_builder/base'
2
+ require 'picsolve_docker_builder/helpers/config/variable_object'
3
+ require 'picsolve_docker_builder/composer/container'
4
+
5
+ module PicsolveDockerBuilder
6
+ module Helpers
7
+ # Parses the config file
8
+ class ConfigManager
9
+ include PicsolveDockerBuilder::Base
10
+ def initialize(path, stage)
11
+ @path = path
12
+ # TODO: Do this properly, this is a very very dirty hack
13
+ stage = (
14
+ !ENV['STAGE'].nil? &&
15
+ ENV['STAGE'].downcase
16
+ ) || 'ci' if stage.nil?
17
+ @stage = stage
18
+ end
19
+
20
+ def default
21
+ {}
22
+ end
23
+
24
+ def config
25
+ return @config unless @config.nil?
26
+ read
27
+ parse_variables
28
+ end
29
+
30
+ def eval_container_config(config)
31
+ config['environment'] = eval_container_env(config['environment'])
32
+ config
33
+ end
34
+
35
+ def eval_container_env_split(env)
36
+ output = {}
37
+ env.each do |key, lst|
38
+ output[key] = {}
39
+ lst.each do |elem|
40
+ ikey, val = elem.split('=', 2)
41
+ output[key][ikey] = val
42
+ end
43
+ end
44
+ output
45
+ end
46
+
47
+ def eval_container_env(env)
48
+ env = eval_container_env_split(env)
49
+ output = {}
50
+ output = output.update(env['default']) if env.key?('default')
51
+ output = output.update(env[@stage]) if env.key?(@stage)
52
+ output
53
+ end
54
+
55
+ def containers(composer)
56
+ config['compose']['containers'].map do |name, raw_config|
57
+ PicsolveDockerBuilder::Composer::Container.new(
58
+ name,
59
+ eval_container_config(raw_config),
60
+ composer
61
+ )
62
+ end
63
+ end
64
+
65
+ def parse_variables
66
+ parse_variables_real(@config)
67
+ end
68
+
69
+ def parse_variables_real(obj)
70
+ if obj.is_a?(Hash)
71
+ obj.each do |k, v|
72
+ obj[k] = parse_variables_real(v)
73
+ end
74
+ elsif obj.is_a?(Array)
75
+ obj.map do |v|
76
+ parse_variables_real(v)
77
+ end
78
+ elsif obj.is_a?(String)
79
+ Config::VariableObject.replace_string(obj)
80
+ else
81
+ obj
82
+ end
83
+ end
84
+
85
+ def read
86
+ @config = default
87
+ begin
88
+ yaml = Psych.load_file @path
89
+ @config = @config.deep_merge(yaml)
90
+ rescue Errno::ENOENT
91
+ raise "cannot find config at '#{path}'"
92
+ end
93
+ end
94
+ end
95
+ end
96
+ end
@@ -42,11 +42,17 @@ module PicsolveDockerBuilder
42
42
  end
43
43
 
44
44
  def containers
45
- [{
45
+ c = {
46
46
  'name' => @image.name,
47
47
  'image' => @image.repo_tag_unique,
48
48
  'ports' => @image.ports_rc
49
- }]
49
+ }
50
+ # append environment variables if in place
51
+ if @image.class.instance_methods.include? :environment
52
+ env = @image.environment
53
+ c['env'] = env unless env.nil?
54
+ end
55
+ [c]
50
56
  end
51
57
 
52
58
  def deploy
@@ -69,17 +75,33 @@ module PicsolveDockerBuilder
69
75
  true
70
76
  end
71
77
 
72
- def pods
73
- # TODO: better handling of selectors
78
+ def pods_by_selector(selector)
74
79
  client.get_pods(
75
80
  namespace: @rc.metadata.namespace,
76
- label_selector: "name=#{@rc.spec.selector.name}" \
77
- ",deployment=#{@rc.spec.selector.deployment}"
81
+ label_selector: selector
78
82
  ).map do |pod|
79
83
  Pod.new(pod, @kubernetes)
80
84
  end
81
85
  end
82
86
 
87
+ def pods
88
+ pods_by_selector(
89
+ "name=#{@rc.spec.selector.name}," \
90
+ "deployment=#{@rc.spec.selector.deployment}"
91
+ )
92
+ end
93
+
94
+ def pods_orphan
95
+ pods_by_selector(
96
+ "name=#{@rc.spec.selector.name}," \
97
+ "deployment!=#{@rc.spec.selector.deployment}"
98
+ )
99
+ end
100
+
101
+ def remove_pods_orphan
102
+ pods_orphan.each(&:remove)
103
+ end
104
+
83
105
  def remove_pods
84
106
  pods.each(&:remove)
85
107
  end
@@ -11,32 +11,82 @@ module PicsolveDockerBuilder
11
11
  class KubernetesManager
12
12
  include PicsolveDockerBuilder::Base
13
13
 
14
+ attr_reader :port, :host, :type
15
+
16
+ def initialize(composer, host, type = nil, port = nil)
17
+ # default type is http
18
+ type ||= :http
19
+
20
+ if type == :ssh_forward
21
+ @type = type
22
+ @port = port || 22
23
+ end
24
+
25
+ if type == :http
26
+ @type = type
27
+ @port = port || 8080
28
+ end
29
+
30
+ @host = host
31
+ @composer = composer
32
+ end
33
+
14
34
  def create_client
15
- Kubeclient::Client.new kubernetes_url, 'v1beta3'
35
+ Kubeclient::Client.new kubernetes_url, 'v1'
16
36
  end
17
37
 
18
38
  def kubernetes_url
39
+ if type == :ssh_forward
40
+ kubernetes_url_ssh_forward
41
+ elsif type == :http
42
+ kubernetes_url_http
43
+ end
44
+ end
45
+
46
+ def kubernetes_url_http
47
+ "http://#{host}:#{port}"
48
+ end
49
+
50
+ def kubernetes_url_ssh_forward_stop
51
+ @ssh_forward.stop unless @ssh_forward.nil?
52
+ @ssh_forward = nil
53
+ end
54
+
55
+ def stop
56
+ kubernetes_url_ssh_forward_stop if type == :ssh_forward
57
+ end
58
+
59
+ def kubernetes_url_ssh_forward
19
60
  @ssh_forward = SshForward.new(
20
- ssh_host: kubernetes_host,
61
+ ssh_host: host,
62
+ ssh_port: port,
63
+ local_port: 10_000,
21
64
  remote_host: '127.0.0.1',
22
65
  remote_port: 8080
23
66
  )
24
67
  port = @ssh_forward.start
25
68
  at_exit do
26
- @ssh_forward.stop unless @ssh_forward.nil?
69
+ kubernetes_url_ssh_forward_stop
27
70
  end
28
71
  "http://127.0.0.1:#{port}"
29
72
  end
30
73
 
31
- def kubernetes_host
32
- # TODO: Need to be touched for multi cluster support
33
- config['compose']['clusters'].first
34
- end
35
-
36
74
  def client
37
75
  @client ||= create_client
38
76
  end
39
77
 
78
+ def images
79
+ @composer.images
80
+ end
81
+
82
+ def app_name
83
+ @composer.app_name
84
+ end
85
+
86
+ def stage
87
+ @composer.stage
88
+ end
89
+
40
90
  def service(image)
41
91
  Kubernetes::Service.new(image, self)
42
92
  end
@@ -45,6 +95,74 @@ module PicsolveDockerBuilder
45
95
  Kubernetes::Rc.new(image, self)
46
96
  end
47
97
 
98
+ def wait_for_deployed_rcs
99
+ wait_timeout = 300
100
+ wait_sleep = 2
101
+ wait_beginning = 15
102
+ wait_count = wait_beginning
103
+ unready_images = images.dup
104
+ log.info \
105
+ 'Wait for the new pods to be up and running for at least 20 seconds'
106
+ sleep wait_beginning
107
+ loop do
108
+ unready_images.each do |i|
109
+ unready_images.delete(i) if i.rc(self).ready?
110
+ end
111
+ log.debug "Still waiting for: #{unready_images.map(&:name)}"
112
+ $stdout.flush
113
+ $stderr.flush
114
+ wait_count += wait_sleep
115
+ break if wait_count > wait_timeout
116
+ break if unready_images.length == 0
117
+ sleep wait_sleep
118
+ end
119
+ successful_images = images.dup
120
+ unready_images.each do |u|
121
+ successful_images.delete(u)
122
+ end
123
+ [successful_images, unready_images]
124
+ end
125
+
126
+ def remove_old_rcs(successful_images)
127
+ # build list of successful deplyoed images
128
+ successful_images.each do |i|
129
+ log.debug "Remove old pods for #{i.name}"
130
+ i.rc(self).remove_old_rcs
131
+ log.debug "Remove orphaned pods for #{i.name}"
132
+ i.rc(self).remove_pods_orphan
133
+ end
134
+ end
135
+
136
+ def deploy
137
+ log.info "start deploying app_name=#{app_name} to stage #{stage}"
138
+
139
+ # reset hash
140
+ @hash = nil
141
+
142
+ deploy_services
143
+
144
+ deploy_rcs
145
+
146
+ successful_images, unready_images = wait_for_deployed_rcs
147
+
148
+ remove_old_rcs(successful_images)
149
+
150
+ fail "Failed to deploy this service: #{unready_images.map(&:name)}" \
151
+ if unready_images.length > 0
152
+ end
153
+
154
+ def deploy_services
155
+ images.each do |i|
156
+ i.service(self).deploy
157
+ end
158
+ end
159
+
160
+ def deploy_rcs
161
+ images.each do |i|
162
+ i.rc(self).deploy
163
+ end
164
+ end
165
+
48
166
  def template_labels_pods(i)
49
167
  c = template_labels(i)
50
168
  c['deployment'] = i.composer.hash
@@ -58,45 +176,6 @@ module PicsolveDockerBuilder
58
176
  'stage' => i.composer.stage
59
177
  }
60
178
  end
61
-
62
- def config_service(s, i)
63
- # workaround a bug on updating TODO: remove when fixed upstream
64
- s.name = i.name
65
-
66
- # configure the service
67
- s.metadata = {} unless s.metadata
68
- s.metadata.name = i.name
69
- s.metadata.namespace = i.composer.namespace
70
- s.metadata.labels = template_labels(i)
71
- s.spec = {} unless s.spec
72
- s.spec.ports = i.ports
73
- s.spec.selector = {
74
- 'name' => i.name
75
- }
76
- end
77
-
78
- def deploy_service(i)
79
- existing = false
80
- begin
81
- s = client.get_service i.name, i.composer.namespace
82
- existing = true
83
- rescue KubeException
84
- s = Kubeclient::Service.new
85
- end
86
-
87
- # config config
88
- config_service(s, i)
89
-
90
- if existing
91
- log.debug \
92
- "update service '#{i.composer.namespace}/#{i.name}' on kubernetes"
93
- client.update_service s
94
- else
95
- log.debug \
96
- "create service '#{i.composer.namespace}/#{i.name}' on kubernetes"
97
- client.create_service s
98
- end
99
- end
100
179
  end
101
180
  end
102
181
  end
@@ -6,18 +6,104 @@ module PicsolveDockerBuilder
6
6
  # (as they are not very well defined)
7
7
  class Repository
8
8
  include PicsolveDockerBuilder::Base
9
+
10
+ attr_reader :tag
11
+
9
12
  def initialize(name)
10
13
  parse_input(name)
11
14
  end
12
15
 
13
16
  def parse_input(input)
14
- input.split('/')
17
+ split = input.split('/')
18
+ if split.length == 1
19
+ split_length_1(split)
20
+ elsif split.length == 2
21
+ split_length_2(split)
22
+ elsif split.length == 3
23
+ split_length_3(split)
24
+ end
25
+ end
26
+
27
+ def split_length_1(split)
28
+ parse_repo_tag(split[0])
29
+ end
30
+
31
+ def split_length_2(split)
32
+ if dot?(split[0])
33
+ if double_dot?(split[1])
34
+ # registry1.com/image1:tag
35
+ @registry = split[0]
36
+ parse_repo_tag(split[1])
37
+ else
38
+ # registry1.com/image1
39
+ @registry = split[0]
40
+ parse_repo_tag(split[1])
41
+ end
42
+ else
43
+ if double_dot?(split[1])
44
+ # user1/image1:tag1
45
+ @user = split[0]
46
+ parse_repo_tag(split[1])
47
+ else
48
+ # user1/image1
49
+ @user = split[0]
50
+ parse_repo_tag(split[1])
51
+ end
52
+ end
53
+ end
54
+
55
+ def split_length_3(split)
56
+ # registry1.com/user1/image1:tag1
57
+ # registry1.com:8080/user1/image1:tag1
58
+ @registry = split[0]
59
+ @user = split[1]
60
+ parse_repo_tag(split[2])
61
+ end
62
+
63
+ # Methods to check special characters
64
+ def slash?(input)
65
+ input.include? '/'
66
+ end
67
+
68
+ def double_dot?(input)
69
+ input.include? ':'
70
+ end
71
+
72
+ def dot?(input)
73
+ input.include? '.'
74
+ end
75
+
76
+ # Methods that returns the user name
77
+ def name
78
+ if @user.nil?
79
+ return @repo
80
+ else
81
+ return "#{@user}/#{@repo}"
82
+ end
83
+ end
84
+
85
+ # Methods that returns default values
86
+ def repo
87
+ @repo || 'image1'
88
+ end
89
+
90
+ def tag
91
+ @tag || 'latest'
92
+ end
93
+
94
+ def registry
95
+ @registry || :default
15
96
  end
16
97
 
17
- def name=(_name)
98
+ def user
99
+ @user || 'library'
18
100
  end
19
101
 
20
- def repo_tag
102
+ # Method to parse the repo tag
103
+ def parse_repo_tag(str)
104
+ split = str.split(':')
105
+ @repo = split[0]
106
+ @tag = split[1] if split.length > 1
21
107
  end
22
108
  end
23
109
  end
@@ -12,7 +12,11 @@ module PicsolveDockerBuilder
12
12
  end
13
13
 
14
14
  def connection
15
- @connection ||= Net::SSH.start(ssh_host, ssh_user)
15
+ @connection ||= Net::SSH.start(
16
+ ssh_host,
17
+ ssh_user,
18
+ port: ssh_port
19
+ )
16
20
  end
17
21
 
18
22
  def bind_port
@@ -59,6 +63,10 @@ module PicsolveDockerBuilder
59
63
  options[:ssh_host]
60
64
  end
61
65
 
66
+ def ssh_port
67
+ options[:ssh_port] || 22
68
+ end
69
+
62
70
  def remote_host
63
71
  options[:remote_host]
64
72
  end