dapp 0.26.14 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/bin/dapp +0 -4
  3. data/config/en/common.yml +1 -0
  4. data/config/en/net_status.yml +2 -0
  5. data/lib/dapp.rb +1 -4
  6. data/lib/dapp/cli.rb +0 -4
  7. data/lib/dapp/cli/command/base.rb +1 -7
  8. data/lib/dapp/dapp.rb +4 -22
  9. data/lib/dapp/dapp/dappfile.rb +2 -2
  10. data/lib/dapp/dapp/deps/base.rb +6 -2
  11. data/lib/dapp/dapp/option_tags.rb +1 -7
  12. data/lib/dapp/dimg/artifact.rb +4 -0
  13. data/lib/dapp/dimg/build/stage/artifact_base.rb +7 -7
  14. data/lib/dapp/dimg/build/stage/artifact_default.rb +4 -5
  15. data/lib/dapp/dimg/build/stage/base.rb +1 -3
  16. data/lib/dapp/dimg/build/stage/before_install.rb +2 -3
  17. data/lib/dapp/dimg/build/stage/from.rb +31 -5
  18. data/lib/dapp/dimg/build/stage/ga_base.rb +1 -1
  19. data/lib/dapp/dimg/build/stage/ga_latest_patch.rb +1 -3
  20. data/lib/dapp/dimg/build/stage/import_artifact.rb +3 -4
  21. data/lib/dapp/dimg/builder/ansible.rb +167 -2
  22. data/lib/dapp/dimg/builder/ansible/assets.rb +332 -0
  23. data/lib/dapp/dimg/builder/chef.rb +2 -2
  24. data/lib/dapp/dimg/cli/command/base.rb +14 -0
  25. data/lib/dapp/dimg/cli/command/dimg/build.rb +15 -24
  26. data/lib/dapp/dimg/cli/command/dimg/run.rb +14 -1
  27. data/lib/dapp/dimg/config/directive/dimg/instance_methods.rb +2 -1
  28. data/lib/dapp/dimg/config/directive/dimg_group_base.rb +2 -2
  29. data/lib/dapp/dimg/config/directive/git_artifact_remote.rb +4 -3
  30. data/lib/dapp/dimg/dapp/command/cleanup.rb +2 -2
  31. data/lib/dapp/dimg/dapp/command/common.rb +16 -5
  32. data/lib/dapp/dimg/dapp/command/mrproper.rb +6 -8
  33. data/lib/dapp/dimg/dapp/command/push.rb +1 -1
  34. data/lib/dapp/dimg/dapp/command/run.rb +3 -2
  35. data/lib/dapp/dimg/dapp/command/stages/cleanup_local.rb +3 -0
  36. data/lib/dapp/dimg/dapp/command/tag.rb +1 -1
  37. data/lib/dapp/dimg/dapp/dimg.rb +8 -0
  38. data/lib/dapp/dimg/dimg.rb +14 -3
  39. data/lib/dapp/dimg/dimg/stages.rb +1 -1
  40. data/lib/dapp/dimg/docker_registry/base.rb +10 -0
  41. data/lib/dapp/dimg/docker_registry/base/authorization.rb +1 -16
  42. data/lib/dapp/dimg/git_artifact.rb +9 -2
  43. data/lib/dapp/dimg/git_repo/base.rb +2 -2
  44. data/lib/dapp/dimg/git_repo/local.rb +2 -2
  45. data/lib/dapp/kube/dapp/command/deploy.rb +0 -1
  46. data/lib/dapp/kube/dapp/command/lint.rb +11 -4
  47. data/lib/dapp/kube/helm/values.rb +7 -7
  48. data/lib/dapp/version.rb +2 -2
  49. metadata +5 -34
  50. data/lib/dapp/dapp/sentry.rb +0 -106
  51. data/lib/dapp/helper/url.rb +0 -23
@@ -127,7 +127,7 @@ module Dapp
127
127
  end
128
128
 
129
129
  def stage_cookbooks_checksum_path(stage)
130
- dimg.build_path.join("#{builder_cookbook.checksum}.#{dimg.config._name}.#{stage}.checksum")
130
+ dimg.build_path.join("#{builder_cookbook.checksum}.#{dimg.name}.#{stage}.checksum")
131
131
  end
132
132
 
133
133
  def stage_cookbooks_checksum(stage)
@@ -216,7 +216,7 @@ module Dapp
216
216
  end
217
217
 
218
218
  def stage_build_path(stage)
219
- dimg.tmp_path(dimg.config._name).join(stage.to_s)
219
+ dimg.tmp_path(dimg.name).join(stage.to_s)
220
220
  end
221
221
 
222
222
  def container_stage_build_path(_stage)
@@ -1,6 +1,16 @@
1
1
  module Dapp::Dimg::CLI
2
2
  module Command
3
3
  class Base < ::Dapp::CLI::Command::Base
4
+ DIMG_STAGES = [
5
+ :from, :before_install, :before_install_artifact, :g_a_archive, :g_a_pre_install_patch, :install,
6
+ :g_a_post_install_patch, :after_install_artifact, :before_setup, :before_setup_artifact,
7
+ :g_a_pre_setup_patch, :setup, :g_a_post_setup_patch, :after_setup_artifact, :g_a_latest_patch, :docker_instructions
8
+ ].freeze
9
+
10
+ STAGE_PROC = proc do |stages|
11
+ proc { |val| val.to_sym.tap { |v| in_validate!(v, stages) } }
12
+ end
13
+
4
14
  option :build_dir,
5
15
  long: "--build-dir PATH",
6
16
  description: "Directory where build cache stored ($HOME/.dapp/builds/<dapp name> by default)."
@@ -9,6 +19,10 @@ module Dapp::Dimg::CLI
9
19
  self.class.parse_options(self, argv)
10
20
  run_dapp_command(run_method, options: cli_options(dimgs_patterns: cli_arguments))
11
21
  end
22
+
23
+ def run_method
24
+ class_to_lowercase
25
+ end
12
26
  end
13
27
  end
14
28
  end
@@ -11,25 +11,16 @@ Usage:
11
11
 
12
12
  Options:
13
13
  BANNER
14
- introspected_stages = [
15
- :from, :before_install, :before_install_artifact, :g_a_archive, :g_a_pre_install_patch, :install,
16
- :g_a_post_install_patch, :after_install_artifact, :before_setup, :before_setup_artifact,
17
- :g_a_pre_setup_patch, :setup, :g_a_post_setup_patch, :after_setup_artifact, :g_a_latest_patch, :docker_instructions
18
- ]
19
- artifact_introspected_stages = [
14
+ artifact_stages = [
20
15
  :from, :before_install, :before_install_artifact, :g_a_archive, :g_a_pre_install_patch, :install,
21
16
  :g_a_post_install_patch, :after_install_artifact, :before_setup, :before_setup_artifact,
22
17
  :g_a_pre_setup_patch, :setup, :after_setup_artifact, :g_a_artifact_patch, :build_artifact
23
18
  ]
24
19
 
25
- introspect_stage_proc = proc do |stages|
26
- proc { |val| val.to_sym.tap { |v| in_validate!(v, stages) } }
27
- end
28
-
29
- introspect_before_proc = proc do |stages|
20
+ before_stage_proc = proc do |stages|
30
21
  proc do |val|
31
22
  val_sym = val.to_sym
32
- introspect_stage_proc.call(stages[1..-1]).call(val_sym)
23
+ STAGE_PROC.call(stages[1..-1]).call(val_sym)
33
24
  stages[stages.index(val_sym) - 1]
34
25
  end
35
26
  end
@@ -58,24 +49,24 @@ BANNER
58
49
  default: false
59
50
 
60
51
  option :introspect_stage,
61
- long: '--introspect-stage STAGE',
62
- description: "Introspect one of the following stages (#{list_msg_format(introspected_stages)})",
63
- proc: introspect_stage_proc.call(introspected_stages)
52
+ long: '--introspect-stage STAGE',
53
+ description: "Introspect one of the following stages (#{list_msg_format(DIMG_STAGES)})",
54
+ proc: STAGE_PROC.call(DIMG_STAGES)
64
55
 
65
56
  option :introspect_before,
66
- long: '--introspect-before STAGE',
67
- description: "Introspect stage before one of the following stages (#{list_msg_format(introspected_stages[1..-1])})",
68
- proc: introspect_before_proc.call(introspected_stages)
57
+ long: '--introspect-before STAGE',
58
+ description: "Introspect stage before one of the following stages (#{list_msg_format(DIMG_STAGES[1..-1])})",
59
+ proc: before_stage_proc.call(DIMG_STAGES)
69
60
 
70
61
  option :introspect_artifact_stage,
71
- long: '--introspect-artifact-stage STAGE',
72
- description: "Introspect one of the following stages (#{list_msg_format(artifact_introspected_stages)})",
73
- proc: introspect_stage_proc.call(artifact_introspected_stages)
62
+ long: '--introspect-artifact-stage STAGE',
63
+ description: "Introspect one of the following stages (#{list_msg_format(artifact_stages)})",
64
+ proc: STAGE_PROC.call(artifact_stages)
74
65
 
75
66
  option :introspect_artifact_before,
76
- long: '--introspect-artifact-before STAGE',
77
- description: "Introspect stage before one of the following stages (#{list_msg_format(artifact_introspected_stages[1..-1])})",
78
- proc: introspect_before_proc.call(artifact_introspected_stages)
67
+ long: '--introspect-artifact-before STAGE',
68
+ description: "Introspect stage before one of the following stages (#{list_msg_format(artifact_stages[1..-1])})",
69
+ proc: before_stage_proc.call(artifact_stages)
79
70
 
80
71
  option :ssh_key,
81
72
  long: '--ssh-key SSH_KEY',
@@ -12,6 +12,11 @@ Usage:
12
12
 
13
13
  Options:
14
14
  BANNER
15
+ option :stage,
16
+ long: '--stage STAGE',
17
+ description: "Run one of the following stages (#{list_msg_format(DIMG_STAGES)})",
18
+ proc: STAGE_PROC.call(DIMG_STAGES)
19
+
15
20
  option :ssh_key,
16
21
  long: '--ssh-key SSH_KEY',
17
22
  description: ['Enable only specified ssh keys ',
@@ -51,8 +56,16 @@ BANNER
51
56
  index = filtered_args.index('--') || filtered_args.count
52
57
  docker_options = index.nonzero? ? filtered_args.slice(0..index - 1) : []
53
58
  command = filtered_args.slice(index + 1..-1) || []
59
+
60
+ if docker_options.empty? && command.empty?
61
+ docker_options = %w(-ti --rm)
62
+ command = %w(/bin/bash)
63
+ end
64
+
65
+ stage_name = config.delete(:stage)
66
+
54
67
  run_dapp_command(nil, options: cli_options(dimgs_patterns: patterns), log_running_time: false) do |dapp|
55
- dapp.run(docker_options, command)
68
+ dapp.run(stage_name, docker_options, command)
56
69
  end
57
70
  end
58
71
  end
@@ -5,8 +5,9 @@ module Dapp
5
5
  class Dimg < Base
6
6
  module InstanceMethods
7
7
  attr_reader :_builder
8
- attr_reader :_chef, :_shell, :_docker, :_git_artifact, :_mount, :_artifact
8
+ attr_reader :_chef, :_shell, :_docker, :_git_artifact, :_mount, :_artifact, :_ansible
9
9
  attr_reader :_artifact_groups
10
+ attr_reader :_from_dimg, :_from_dimg_artifact
10
11
 
11
12
  def chef(&blk)
12
13
  builder(:chef)
@@ -20,11 +20,11 @@ module Dapp
20
20
  end
21
21
 
22
22
  def _dimg
23
- (@_dimg + @_dimg_group.map(&:_dimg)).flatten
23
+ (@_dimg + _dimg_group.map(&:_dimg)).flatten
24
24
  end
25
25
 
26
26
  def _dimg_group
27
- @_dimg_group
27
+ @_dimg_group||[]
28
28
  end
29
29
 
30
30
  protected
@@ -3,13 +3,14 @@ module Dapp
3
3
  module Config
4
4
  module Directive
5
5
  class GitArtifactRemote < GitArtifactLocal
6
- include ::Dapp::Helper::Url
7
-
8
6
  attr_reader :_url, :_name, :_branch, :_commit
9
7
 
10
8
  def initialize(url, **kwargs, &blk)
11
9
  @_url = url
12
- @_name = git_url_to_name(url)
10
+
11
+ url_without_scheme = url.split("://", 2).last
12
+ url_without_creds = url_without_scheme.split(":", 2).last
13
+ @_name = url_without_creds.gsub(%r{.*?([^\/ ]+\/[^\/ ]+)\.git}, '\\1')
13
14
 
14
15
  super(**kwargs, &blk)
15
16
  end
@@ -5,8 +5,8 @@ module Dapp
5
5
  module Cleanup
6
6
  def cleanup
7
7
  log_step_with_indent(:cleanup) do
8
- dapp_containers_flush
9
- dapp_dangling_images_flush
8
+ dapp_containers_flush_by_label('dapp')
9
+ dapp_dangling_images_flush_by_label('dapp')
10
10
  end
11
11
  end
12
12
  end
@@ -66,12 +66,16 @@ module Dapp
66
66
  project_images.map { |image| image[:dangling] ? image[:id] : image[:name] }
67
67
  end
68
68
 
69
- def dapp_containers_flush
70
- remove_containers_by_query(%(#{host_docker} ps -a -f "label=dapp" -q --no-trunc))
69
+ def dapp_containers_flush_by_label(label)
70
+ log_proper_containers do
71
+ remove_containers_by_query(%(#{host_docker} ps -a -f "label=#{label}" -q --no-trunc))
72
+ end
71
73
  end
72
74
 
73
- def dapp_dangling_images_flush
74
- remove_images_by_query(%(#{host_docker} images -f "dangling=true" -f "label=dapp" -q --no-trunc))
75
+ def dapp_dangling_images_flush_by_label(label)
76
+ log_proper_flush_dangling_images do
77
+ remove_images_by_query(%(#{host_docker} images -f "dangling=true" -f "label=#{label}" -q --no-trunc))
78
+ end
75
79
  end
76
80
 
77
81
  def dapp_tagless_images_flush
@@ -125,7 +129,6 @@ module Dapp
125
129
  end
126
130
 
127
131
  def remove_base(query_format, ids, force: false)
128
- return if ids.empty?
129
132
  force_option = force ? ' -f' : ''
130
133
  log(ids.join("\n")) if log_verbose? || dry_run?
131
134
  run_command(format(query_format, force_option: force_option, ids: ids.join(' ')))
@@ -178,6 +181,14 @@ module Dapp
178
181
  log_step_with_indent(:'proper repo cache', &blk)
179
182
  end
180
183
 
184
+ def log_proper_containers(&blk)
185
+ log_step_with_indent(:'proper containers', &blk)
186
+ end
187
+
188
+ def log_proper_flush_dangling_images(&blk)
189
+ log_step_with_indent(:'proper dangling', &blk)
190
+ end
191
+
181
192
  def push_format(dimg_name)
182
193
  if dimg_name.nil?
183
194
  spush_format
@@ -16,7 +16,7 @@ module Dapp
16
16
  proper_cache_version
17
17
  end
18
18
 
19
- dapp_dangling_images_flush
19
+ dapp_dangling_images_flush_by_label('dapp')
20
20
  dapp_tagless_images_flush
21
21
  end
22
22
  end
@@ -52,16 +52,14 @@ module Dapp
52
52
  end
53
53
 
54
54
  def flush_by_label(label)
55
- log_step_with_indent(:containers) { dapp_containers_flush_by_label(label) }
56
- log_step_with_indent(:images) { dapp_images_flush_by_label(label) }
57
- end
58
-
59
- def dapp_containers_flush_by_label(label)
60
- remove_containers_by_query(%(#{host_docker} ps -a -f "label=dapp" -f "label=#{label}" -q --no-trunc))
55
+ dapp_containers_flush_by_label(label)
56
+ dapp_images_flush_by_label(label)
61
57
  end
62
58
 
63
59
  def dapp_images_flush_by_label(label)
64
- remove_images(dapp_images_names_by_label(label))
60
+ log_step_with_indent('proper images') do
61
+ remove_images(dapp_images_names_by_label(label))
62
+ end
65
63
  end
66
64
 
67
65
  def proper_cache_version
@@ -8,7 +8,7 @@ module Dapp
8
8
  #RubyProf.start
9
9
  log_step_with_indent(:stages) { stages_push } if with_stages?
10
10
  dimg_import_export_base do |dimg|
11
- dimg.export!(option_repo, format: push_format(dimg.config._name))
11
+ dimg.export!(option_repo, format: push_format(dimg.name))
12
12
  end
13
13
  # FIXME: rework images cache, then profile
14
14
  #result = RubyProf.stop
@@ -3,10 +3,11 @@ module Dapp
3
3
  module Dapp
4
4
  module Command
5
5
  module Run
6
- def run(docker_options, command)
6
+ def run(stage_name, docker_options, command)
7
7
  one_dimg!
8
8
  setup_ssh_agent
9
- dimg(config: build_configs.first, ignore_git_fetch: true, should_be_built: true).run(docker_options, command)
9
+ dimg(config: build_configs.first, ignore_git_fetch: true, should_be_built: stage_name.nil?)
10
+ .run_stage(stage_name, docker_options, command)
10
11
  end
11
12
  end
12
13
  end
@@ -11,6 +11,9 @@ module Dapp
11
11
  proper_cache if proper_cache_version?
12
12
  stages_cleanup_by_repo if proper_repo_cache?
13
13
  proper_git_commit if proper_git_commit?
14
+
15
+ dapp_containers_flush_by_label("dapp=#{name}")
16
+ dapp_dangling_images_flush_by_label("dapp=#{name}")
14
17
  end
15
18
  end
16
19
 
@@ -5,7 +5,7 @@ module Dapp
5
5
  module Tag
6
6
  def tag
7
7
  dimg_import_export_base do |dimg|
8
- dimg.tag!(option_repo, format: push_format(dimg.config._name))
8
+ dimg.tag!(option_repo, format: push_format(dimg.name))
9
9
  end
10
10
  end
11
11
  end
@@ -10,6 +10,14 @@ module Dapp
10
10
  (@artifacts_dimgs ||= {})[config._name] ||= ::Dapp::Dimg::Artifact.new(config: config, dapp: self, **kwargs)
11
11
  end
12
12
 
13
+ def dimg_layer(config:, **kwargs)
14
+ (@dimg_layers ||= {})[config._name] ||= ::Dapp::Dimg::Dimg.new(config: config, dapp: self, **kwargs)
15
+ end
16
+
17
+ def artifact_dimg_layer(config:, **kwargs)
18
+ (@artifact_dimg_layers ||= {})[config._name] ||= ::Dapp::Dimg::Artifact.new(config: config, dapp: self, **kwargs)
19
+ end
20
+
13
21
  def _terminate_dimg_on_terminate(dimg)
14
22
  @_call_before_terminate << proc{dimg.terminate}
15
23
  end
@@ -25,6 +25,10 @@ module Dapp
25
25
  raise Error::Dimg, code: :dimg_not_built if should_be_built?
26
26
  end
27
27
 
28
+ def name
29
+ config._name
30
+ end
31
+
28
32
  def terminate
29
33
  cleanup_tmp
30
34
  end
@@ -74,7 +78,7 @@ module Dapp
74
78
  dapp.tags_by_scheme.each do |tag_scheme_name, tags|
75
79
  dapp.log_step_with_indent(tag_scheme_name) do
76
80
  tags.each do |tag|
77
- image_name = format(export_format, repo: repo, dimg_name: config._name, tag: tag)
81
+ image_name = format(export_format, repo: repo, dimg_name: name, tag: tag)
78
82
  export_base!(image_name, push: push) do
79
83
  export_image = build_export_image!(image_name, scheme_name: tag_scheme_name)
80
84
  if push
@@ -139,7 +143,13 @@ module Dapp
139
143
  end
140
144
 
141
145
  def run(docker_options, command)
142
- cmd = "#{dapp.host_docker} run #{[docker_options, last_stage.image.built_id, command].flatten.compact.join(' ')}"
146
+ run_stage(nil, docker_options, command)
147
+ end
148
+
149
+ def run_stage(stage_name, docker_options, command)
150
+ stage_image = (stage_name.nil? ? last_stage : stage_by_name(stage_name)).image
151
+ raise Error::Dimg, code: :dimg_stage_not_built, data: { stage_name: stage_name } unless stage_image.built?
152
+ cmd = "#{dapp.host_docker} run #{[docker_options, stage_image.built_id, command].flatten.compact.join(' ')}"
143
153
  if dapp.dry_run?
144
154
  dapp.log(cmd)
145
155
  else
@@ -164,7 +174,7 @@ module Dapp
164
174
  end
165
175
 
166
176
  def scratch?
167
- config._docker._from.nil?
177
+ config._docker._from.nil? && config._from_dimg.nil? && config._from_dimg_artifact.nil?
168
178
  end
169
179
 
170
180
  def dev_mode?
@@ -198,6 +208,7 @@ module Dapp
198
208
  cmd = "".tap do |cmd|
199
209
  cmd << "#{dapp.host_docker} run --rm"
200
210
  cmd << " --volume #{dapp.tmp_base_dir}:#{dapp.tmp_base_dir}"
211
+ cmd << " --label dapp=#{dapp.name}"
201
212
  cmd << " alpine:3.6"
202
213
  cmd << " rm -rf #{tmp_path}"
203
214
  end
@@ -34,7 +34,7 @@ module Dapp
34
34
  def last_stage
35
35
  @last_stage || begin
36
36
  (self.last_stage = last_stage_class.new(self)).tap do |stage|
37
- dapp.log_secondary_process("#{config._name || 'nameless'}: calculating stages signatures") do
37
+ dapp.log_secondary_process("#{name || 'nameless'}: calculating stages signatures") do
38
38
  stage.signature
39
39
  end
40
40
  end
@@ -35,6 +35,7 @@ module Dapp
35
35
  end
36
36
 
37
37
  def image_delete(tag)
38
+ image_blobs(tag).each { |hash| blob_delete(hash.values.first) }
38
39
  api_request(repo_suffix, "/manifests/#{image_digest(tag)}",
39
40
  method: :delete,
40
41
  expects: [202, 404],
@@ -53,6 +54,10 @@ module Dapp
53
54
  headers: { Accept: 'application/vnd.docker.distribution.manifest.v2+json' }).headers['Docker-Content-Digest']
54
55
  end
55
56
 
57
+ def image_blobs(tag)
58
+ manifest_v1(tag)['fsLayers']
59
+ end
60
+
56
61
  def manifest_v1(tag)
57
62
  api_request(repo_suffix, "/manifests/#{tag}")
58
63
  end
@@ -62,6 +67,11 @@ module Dapp
62
67
  headers: { Accept: 'application/vnd.docker.distribution.manifest.v2+json' })
63
68
  end
64
69
 
70
+ def blob_delete(id)
71
+ api_request(repo_suffix, "/blobs/#{id}",
72
+ method: :delete, expects: [202, 404])
73
+ end
74
+
65
75
  def api_request(*uri, **options)
66
76
  JSON.load(raw_api_request(*uri, **options).body)
67
77
  end
@@ -29,25 +29,10 @@ module Dapp
29
29
  [:realm, :service, :scope].map do |option|
30
30
  /#{option}="([[^"].]*)/ =~ header
31
31
  next unless Regexp.last_match(1)
32
-
33
- option_value = begin
34
- if option == :scope
35
- handle_scope_option(Regexp.last_match(1))
36
- else
37
- Regexp.last_match(1)
38
- end
39
- end
40
-
41
- [option, option_value]
32
+ [option, Regexp.last_match(1)]
42
33
  end.compact.to_h
43
34
  end
44
35
 
45
- def handle_scope_option(resourcescope)
46
- resource_type, resource_name, actions = resourcescope.split(":")
47
- actions = actions.split(",").map { |action| action == "delete" ? "*" : action }.join(",")
48
- [resource_type, resource_name, actions].join(":")
49
- end
50
-
51
36
  def authorization_auth
52
37
  @authorization_auth ||= begin
53
38
  if ::Dapp::Dapp.options_with_docker_credentials?