dapp 0.7.16 → 0.7.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: f5884787809c47ff8c6ced799f109912144814b0
4
- data.tar.gz: 5b94502ee922f44d9adbabc87e74b3e29db2e80a
3
+ metadata.gz: ac3cccdbfe9bb2fb9aab05958f1fa1d8f2db5ab5
4
+ data.tar.gz: c8518534ebe63840f47ff76327186f6b78d542bc
5
5
  SHA512:
6
- metadata.gz: 35020f1bd1c480ba266b0e500a8e41944de22ec889b4ae7c4e8617b0f9b81348be06ceb9962e526064e02464aac999199bb63b7e1b04745627380494d6931272
7
- data.tar.gz: 9c1bb01f8c90ec1d00a3131ae65af7a82fce6bbf7863d6a2d5e2326eee5a55df6322b579be0ceb3771023f3e9e9c0962b849d28ececec46c6004874087671370
6
+ metadata.gz: 043df0615ad8084c2deca2fec4bb98f932a60a3ce7969f2ad84e4a14fef2643ed6e5889a8387c67701b287ef945d0f7198b1143be1fb3e24b78aaa9e5ae7bc6b
7
+ data.tar.gz: 12099bf042bb1282be953c32f12bce147fe2a6103a90a82eb88dcf3e296bb1af4d782fde0aa12988d6caad8dc2b881dad8a222f703f82ce4ba7caf7f4b8a28ef
@@ -26,6 +26,7 @@ en:
26
26
  cannot_run_ssh_agent: "Cannot run ssh-agent"
27
27
  ssh_key_not_found: "Ssh key `%{path}` not exist!"
28
28
  mrproper_required_option: "Expected command option `--improper-dev-mode-cache`, `--improper-cache-version-stages` or `--all`!"
29
+ stages_cleanup_required_option: "Expected command option `--improper-cache-version`, `--improper-git-commit` or `--improper-repo-cache`!"
29
30
  config:
30
31
  dimg_name_required: 'Dimg name required!'
31
32
  dimg_name_incorrect: "Dimg has incorrect name `%{name}`: doesn't match regex `%{reg}`!"
@@ -42,6 +43,7 @@ en:
42
43
  stage_artifact_not_associated: "Artifact not associated with any stage: expected `before` or `after` attribute!"
43
44
  stage_artifact_double_associate: "Cannot use `%{stage}` stage for artifact, already used in `%{conflict_stage}` stage!"
44
45
  stage_artifact_not_supported_associated_stage: "Bad artifact stage `%{stage}`!"
46
+ git_artifact_remote_unsupported_protocol: "Remote git repo `%{url}`: unsupported protocol!"
45
47
  artifact_conflict: "Conflict between artifacts paths!"
46
48
  scratch_unsupported_directive: "Scratch dimg has unsupported directive `%{directive}`!"
47
49
  scratch_artifact_required: "Scratch dimg without artifacts!"
@@ -54,13 +56,13 @@ en:
54
56
  cookbook_berksfile_not_found: "Dapp cookbook Berksfile not found at %{path}"
55
57
  cookbook_metadata_not_found: "Dapp cookbook metadata.rb file not found at %{path}"
56
58
  cookbook_not_specified_in_berksfile: "Dapp cookbook `%{name}` not specified in Berksfile at %{path}"
57
- registry:
58
- incorrect_repo: 'Incorrect repository!'
59
- no_such_dimg: 'No such dimg in registry!'
60
- authenticate_type_not_supported: 'Registry authenticate type not supported!'
61
- page_not_found: "Page `%{url}` not found!"
62
- user_not_authorized: 'User not authorized!'
63
- response_with_error_status: 'Response with error status!'
64
59
  berksfile_absolute_path_forbidden: "Absolute paths in Berksfile are not allowed (cookbook `%{cookbook}`, path: `%{path}`)"
60
+ registry:
61
+ no_such_dimg: 'Registry `%{registry}`: no such dimg in registry!'
62
+ authenticate_type_not_supported: 'Registry `%{registry}`: authenticate type not supported!'
63
+ page_not_found: "Registry `%{registry}`: page `%{url}` not found!"
64
+ user_not_authorized: 'Registry `%{registry}`: user not authorized!'
65
+ rugged:
66
+ rugged_network_error: "Remote git repo `%{url}`: `%{message}`!"
65
67
  lock:
66
68
  timeout: "Could not obtain lock for `%{name}` within %{timeout} seconds"
data/lib/dapp.rb CHANGED
@@ -9,11 +9,13 @@ require 'mixlib/shellout'
9
9
  require 'securerandom'
10
10
  require 'excon'
11
11
  require 'json'
12
+ require 'uri'
12
13
  require 'ostruct'
13
14
  require 'time'
14
15
  require 'i18n'
15
16
  require 'paint'
16
17
  require 'inifile'
18
+ require 'rugged'
17
19
 
18
20
  require 'net_status'
19
21
 
@@ -32,6 +34,7 @@ require 'dapp/error/config'
32
34
  require 'dapp/error/project'
33
35
  require 'dapp/error/shellout'
34
36
  require 'dapp/error/registry'
37
+ require 'dapp/error/rugged'
35
38
  require 'dapp/exception/base'
36
39
  require 'dapp/exception/introspect_image'
37
40
  require 'dapp/exception/registry'
@@ -18,8 +18,19 @@ module Dapp
18
18
  image.add_service_change_label artifacts_labels
19
19
  end
20
20
 
21
- def images
22
- [image].concat(artifacts.map { |artifact| artifact[:dimg].images }.flatten)
21
+ def artifacts
22
+ @artifacts ||= begin
23
+ dimg.config.public_send("_#{name}").map do |artifact|
24
+ artifact_dimg = Dapp::Artifact.new(config: artifact._config,
25
+ project: dimg.project,
26
+ ignore_git_fetch: dimg.ignore_git_fetch)
27
+ { name: artifact._config._name, options: artifact._artifact_options, dimg: artifact_dimg }
28
+ end
29
+ end
30
+ end
31
+
32
+ def artifact?
33
+ true
23
34
  end
24
35
 
25
36
  protected
@@ -32,17 +43,6 @@ module Dapp
32
43
  true
33
44
  end
34
45
 
35
- def artifacts
36
- @artifacts ||= begin
37
- dimg.config.public_send("_#{name}").map do |artifact|
38
- artifact_dimg = Dapp::Artifact.new(config: artifact._config,
39
- project: dimg.project,
40
- ignore_git_fetch: dimg.ignore_git_fetch)
41
- { name: artifact._config._name, options: artifact._artifact_options, dimg: artifact_dimg }
42
- end
43
- end
44
- end
45
-
46
46
  def artifacts_signatures
47
47
  artifacts.map { |artifact| hashsum [artifact[:dimg].signature, artifact[:options]] }
48
48
  end
@@ -142,6 +142,10 @@ module Dapp
142
142
  []
143
143
  end
144
144
 
145
+ def artifact?
146
+ false
147
+ end
148
+
145
149
  protected
146
150
 
147
151
  def image_build
@@ -9,7 +9,7 @@ Version: #{Dapp::VERSION}
9
9
 
10
10
  Available subcommands: (for details, dapp SUB-COMMAND --help)
11
11
 
12
- dapp stages cleanup local [options] [DIMG ...] REPO
12
+ dapp stages cleanup local [options] [DIMG ...] [REPO]
13
13
  dapp stages cleanup repo [options] [DIMG ...] REPO
14
14
  dapp stages flush local [options] [DIMG ...]
15
15
  dapp stages flush repo [options] [DIMG ...] REPO
@@ -7,7 +7,7 @@ module Dapp
7
7
  Version: #{Dapp::VERSION}
8
8
 
9
9
  Usage:
10
- dapp stages cleanup local [options] [DIMG ...] REPO
10
+ dapp stages cleanup local [options] [DIMG ...] [REPO]
11
11
 
12
12
  DIMG Dapp image to process [default: *].
13
13
 
@@ -17,10 +17,26 @@ BANNER
17
17
  long: '--improper-cache-version',
18
18
  boolean: true
19
19
 
20
+ option :proper_git_commit,
21
+ long: '--improper-git-commit',
22
+ boolean: true
23
+
24
+ option :proper_repo_cache,
25
+ long: '--improper-repo-cache',
26
+ boolean: true
27
+
20
28
  def run(argv = ARGV)
21
29
  self.class.parse_options(self, argv)
22
- repo = self.class.required_argument(self)
23
- Project.new(cli_options: config, dimgs_patterns: cli_arguments).stages_cleanup_local(repo)
30
+ repository = repo
31
+ Project.new(cli_options: config, dimgs_patterns: cli_arguments).send(run_method, repository)
32
+ end
33
+
34
+ def repo
35
+ config[:proper_repo_cache] ? self.class.required_argument(self) : nil
36
+ end
37
+
38
+ def run_method
39
+ :stages_cleanup_local
24
40
  end
25
41
  end
26
42
  end
@@ -13,10 +13,12 @@ Usage:
13
13
 
14
14
  Options:
15
15
  BANNER
16
- def run(argv = ARGV)
17
- self.class.parse_options(self, argv)
18
- repo = self.class.required_argument(self)
19
- Project.new(cli_options: config, dimgs_patterns: cli_arguments).stages_cleanup_repo(repo)
16
+ def run_method
17
+ :stages_cleanup_repo
18
+ end
19
+
20
+ def repo
21
+ self.class.required_argument(self)
20
22
  end
21
23
  end
22
24
  end
@@ -43,7 +43,7 @@ module Dapp
43
43
 
44
44
  def git(url = nil, &blk)
45
45
  type = url.nil? ? :local : :remote
46
- _git_artifact.send(type, url, &blk)
46
+ _git_artifact.send(type, url.to_s, &blk)
47
47
  end
48
48
 
49
49
  def mount(to, &blk)
@@ -31,12 +31,13 @@ module Dapp
31
31
  end
32
32
 
33
33
  def validate_scratch_directives!
34
- directives = [:_shell, :_chef, :_git_artifact, :_install_dependencies, :_setup_dependencies,
35
- :_tmp_dir_mount, :_build_dir_mount]
36
- directives.each do |directive|
34
+ directives = [[:_shell, :shell], [:_chef, :chef], [:_git_artifact, :git],
35
+ [:_install_dependencies, :install_depends_on], [:_setup_dependencies, :setup_depends_on],
36
+ [:_tmp_dir_mount, :mount], [:_build_dir_mount, :mount]]
37
+ directives.each do |name, user_name|
37
38
  raise Error::Config,
38
39
  code: :scratch_unsupported_directive,
39
- data: { directive: directive[1..-1] } unless public_send(directive).send(:empty?)
40
+ data: { directive: user_name } unless public_send(name).send(:empty?)
40
41
  end
41
42
 
42
43
  docker_directives = [:_expose, :_env, :_cmd, :_onbuild, :_workdir, :_user, :_entrypoint]
@@ -6,6 +6,8 @@ module Dapp
6
6
  attr_reader :_url, :_name, :_branch, :_commit
7
7
 
8
8
  def initialize(url, **kwargs, &blk)
9
+ raise Error::Config, code: :git_artifact_remote_unsupported_protocol, data: { url: url } unless %w(http https).include? URI(url.to_s).scheme
10
+
9
11
  @_url = url
10
12
  @_name = url.gsub(%r{.*?([^\/ ]+)\.git}, '\\1')
11
13
 
data/lib/dapp/dimg.rb CHANGED
@@ -37,7 +37,7 @@ module Dapp
37
37
  end
38
38
  end
39
39
  ensure
40
- FileUtils.rm_rf(tmp_path)
40
+ cleanup_tmp
41
41
  end
42
42
 
43
43
  def tag!(tag)
@@ -132,6 +132,10 @@ module Dapp
132
132
  @builder ||= Builder.const_get(config._builder.capitalize).new(self)
133
133
  end
134
134
 
135
+ def artifacts
136
+ @artifacts ||= artifacts_stages.map { |stage| stage.artifacts.map { |artifact| artifact[:dimg] } }.flatten
137
+ end
138
+
135
139
  def artifact?
136
140
  false
137
141
  end
@@ -153,6 +157,11 @@ module Dapp
153
157
  system(cmd)
154
158
  end
155
159
 
160
+ def cleanup_tmp
161
+ FileUtils.rm_rf(tmp_path)
162
+ artifacts.each(&:cleanup_tmp)
163
+ end
164
+
156
165
  protected
157
166
 
158
167
  def should_be_built?
@@ -15,16 +15,8 @@ module Dapp
15
15
  project.stage_dapp_label
16
16
  end
17
17
 
18
- def images
19
- (@images ||= []).tap do |images|
20
- stages.each do |stage|
21
- if stage.respond_to?(:images)
22
- images.concat(stage.images)
23
- else
24
- images << stage.image
25
- end
26
- end
27
- end.uniq!(&:name)
18
+ def all_images
19
+ @all_images ||= all_stages.map(&:image).uniq!(&:name)
28
20
  end
29
21
 
30
22
  protected
@@ -38,11 +30,11 @@ module Dapp
38
30
  end
39
31
 
40
32
  def export_images
41
- images.select(&:tagged?)
33
+ all_images.select(&:tagged?)
42
34
  end
43
35
 
44
36
  def import_images
45
- images.select { |image| !image.tagged? }
37
+ all_images.select { |image| !image.tagged? }
46
38
  end
47
39
 
48
40
  def stages
@@ -54,6 +46,14 @@ module Dapp
54
46
  end
55
47
  end
56
48
  end
49
+
50
+ def artifacts_stages
51
+ @artifacts_stages ||= stages.select { |stage| stage.artifact? }
52
+ end
53
+
54
+ def all_stages
55
+ stages + artifacts.map(&:all_stages).flatten
56
+ end
57
57
  end # Stages
58
58
  end # Dimg
59
59
  end # Dapp
@@ -25,7 +25,7 @@ module Dapp
25
25
  def tags
26
26
  @tags ||= api_request(repo_suffix, 'tags/list')['tags'] || []
27
27
  rescue Error::Registry => e
28
- raise Exception::Registry, code: :dimg_not_found_in_registry if e.net_status[:code] == :page_not_found
28
+ raise Exception::Registry, code: :no_such_dimg, data: { registry: api_url } if e.net_status[:code] == :page_not_found
29
29
  raise
30
30
  end
31
31
 
@@ -50,6 +50,11 @@ module Dapp
50
50
  headers: { Accept: 'Accept: application/vnd.docker.distribution.manifest.v2+json' })
51
51
  end
52
52
 
53
+ def image_history(tag)
54
+ response = manifest_v1(tag)
55
+ JSON.load(response['history'].first['v1Compatibility'])
56
+ end
57
+
53
58
  protected
54
59
 
55
60
  def image_digest(tag)
@@ -70,11 +75,6 @@ module Dapp
70
75
  headers: { Accept: 'application/vnd.docker.distribution.manifest.v2+json' })
71
76
  end
72
77
 
73
- def image_history(tag)
74
- response = manifest_v1(tag)
75
- JSON.load(response['history'].first['v1Compatibility'])
76
- end
77
-
78
78
  def blob_delete(id)
79
79
  api_request(repo_suffix, "/blobs/#{id}",
80
80
  method: :delete, expects: [202, 404])
@@ -88,9 +88,9 @@ module Dapp
88
88
  url = api_url(*uri)
89
89
  request(url, **default_api_options.merge(options))
90
90
  rescue Excon::Error::NotFound
91
- raise Error::Registry, code: :page_not_found, data: { url: url }
91
+ raise Error::Registry, code: :page_not_found, data: { url: url, registry: api_url }
92
92
  rescue Excon::Error::Unauthorized
93
- raise Error::Registry, code: :user_not_authorized
93
+ user_not_authorized!
94
94
  end
95
95
 
96
96
  def api_url(*uri)
@@ -100,6 +100,10 @@ module Dapp
100
100
  def default_api_options
101
101
  { expects: [200] }
102
102
  end
103
+
104
+ def user_not_authorized!
105
+ raise Error::Registry, code: :user_not_authorized, data: { registry: api_url }
106
+ end
103
107
  end
104
108
  end # DockerRegistry
105
109
  end # Dapp
@@ -10,7 +10,7 @@ module Dapp
10
10
  when /Bearer/ then { headers: { Authorization: "Bearer #{authorization_token(authenticate_header)}" } }
11
11
  when /Basic/ then { headers: { Authorization: "Basic #{authorization_auth}" } }
12
12
  when nil then {}
13
- else raise Error::Registry, code: :authenticate_type_not_supported
13
+ else raise Error::Registry, code: :authenticate_type_not_supported, data: { registry: api_url }
14
14
  end
15
15
  end
16
16
  end
@@ -42,14 +42,14 @@ module Dapp
42
42
  r = chomp_name(r)
43
43
  end
44
44
  credential = (auths[r] || auths.find { |repo, _| repo == r })
45
- raise Error::Registry, code: :user_not_authorized if credential.nil?
45
+ user_not_authorized! if credential.nil?
46
46
  credential['auth']
47
47
  end
48
48
 
49
49
  def auths_section_from_docker_config
50
50
  file = Pathname(File.join(Dir.home, '.docker', 'config.json'))
51
- raise Error::Registry, code: :user_not_authorized unless file.exist?
52
- JSON.load(file.read)['auths'].tap { |auths| raise Error::Registry, code: :user_not_authorized if auths.nil? }
51
+ user_not_authorized! unless file.exist?
52
+ JSON.load(file.read)['auths'].tap { |auths| user_not_authorized! if auths.nil? }
53
53
  end
54
54
 
55
55
  private
@@ -0,0 +1,6 @@
1
+ module Dapp
2
+ module Error
3
+ # Rugged
4
+ class Rugged < Base; end
5
+ end
6
+ end
@@ -55,11 +55,11 @@ module Dapp
55
55
  end
56
56
 
57
57
  def any_changes?(from, to = latest_commit)
58
- !repo.git_bare(diff_command(from, to, quiet: true), returns: [0, 1]).status.success?
58
+ !repo.old_git_bare(diff_command(from, to, quiet: true), returns: [0, 1]).status.success?
59
59
  end
60
60
 
61
61
  def patch_size(from, to)
62
- repo.git_bare("#{diff_command(from, to)} | wc -c").stdout.strip.to_i
62
+ repo.old_git_bare("#{diff_command(from, to)} | wc -c").stdout.strip.to_i
63
63
  end
64
64
 
65
65
  def latest_commit
@@ -15,35 +15,40 @@ module Dapp
15
15
  end
16
16
 
17
17
  def path
18
- dimg.tmp_path "#{name}.git"
18
+ dimg.tmp_path("#{name}.git").to_s
19
19
  end
20
20
 
21
- def git_bare(command, **kwargs)
22
- git "--git-dir=#{path} #{command}", **kwargs
21
+ def git_bare
22
+ @git_bare ||= Rugged::Repository.new(path, bare: true)
23
23
  end
24
24
 
25
- def commit_at(commit)
26
- Time.at Integer git_bare("show -s --format=%ct #{commit}").stdout.strip
25
+ def old_git_bare(command, **kwargs)
26
+ old_git "--git-dir=#{path} #{command}", **kwargs
27
27
  end
28
28
 
29
- def latest_commit(branch)
30
- git_bare("rev-parse #{branch}").stdout.strip
29
+ def commit_at(commit)
30
+ git_bare.lookup(commit).time.to_i
31
31
  end
32
32
 
33
- def exist_in_commit?(path, commit)
34
- git_bare("cat-file -e #{commit}:#{path}", returns: [0, 128]).status.success?
33
+ def latest_commit(branch)
34
+ return git_bare.head.target_id if branch == 'HEAD'
35
+ git_bare.branches[branch].target_id
35
36
  end
36
37
 
37
38
  def cleanup!
38
39
  end
39
40
 
40
41
  def branch
41
- git_bare('rev-parse --abbrev-ref HEAD').stdout.strip
42
+ git_bare.head.name.sub(/^refs\/heads\//, '')
42
43
  end
43
44
 
44
45
  protected
45
46
 
46
- def git(command, **kwargs)
47
+ def git
48
+ @git ||= Rugged::Repository.new(path)
49
+ end
50
+
51
+ def old_git(command, **kwargs)
47
52
  dimg.system_shellout! "#{dimg.project.git_bin} #{command}", **kwargs
48
53
  end
49
54
  end
@@ -11,14 +11,7 @@ module Dapp
11
11
  end
12
12
 
13
13
  def path
14
- @path ||= begin
15
- git_repo_path = Pathname(git("-C #{dimg.home_path} rev-parse --git-dir").stdout.strip)
16
- if git_repo_path.relative?
17
- File.join(dimg.home_path, git_repo_path)
18
- else
19
- git_repo_path
20
- end
21
- end
14
+ @path ||= Rugged::Repository.discover(dimg.home_path.to_s).path
22
15
  end
23
16
 
24
17
  def latest_commit(branch = nil)
@@ -8,13 +8,18 @@ module Dapp
8
8
  @url = url
9
9
 
10
10
  dimg.project.log_secondary_process(dimg.project.t(code: 'process.git_artifact_clone', data: { name: name }), short: true) do
11
- git "clone --bare --depth 1 #{url} #{path}"
11
+ begin
12
+ Rugged::Repository.clone_at(url, path, bare: true)
13
+ rescue Rugged::NetworkError => e
14
+ raise Error::Rugged, code: :rugged_network_error, data: { message: e.message, url: url }
15
+ end
12
16
  end unless File.directory?(path)
13
17
  end
14
18
 
15
- def fetch!(branch = 'master')
19
+ def fetch!(branch = nil)
20
+ branch ||= self.branch
16
21
  dimg.project.log_secondary_process(dimg.project.t(code: 'process.git_artifact_fetch', data: { name: name }), short: true) do
17
- git_bare "fetch origin #{branch}:#{branch}"
22
+ git_bare.fetch('origin', [branch])
18
23
  end unless dimg.ignore_git_fetch || dimg.project.dry_run?
19
24
  end
20
25
 
@@ -8,12 +8,12 @@ module Dapp
8
8
  def bp(repo)
9
9
  bp_step(:build)
10
10
  bp_step(:push, repo)
11
- bp_step(:stages_cleanup_local, repo)
11
+ bp_step(:stages_cleanup_by_repo, repo)
12
12
  bp_step(:cleanup)
13
13
  end
14
14
 
15
15
  def bp_step(step, *args)
16
- log_step_with_indent(step) { public_send(step, *args) }
16
+ log_step_with_indent(step) { send(step, *args) }
17
17
  end
18
18
  end
19
19
  end
@@ -12,7 +12,7 @@ module Dapp
12
12
  project_dangling_images_flush
13
13
  remove_images_by_query([
14
14
  'docker images',
15
- %(--format '{{if ne "#{stage_cache}" .Repository }}{{.ID}}{{ end }}'),
15
+ %(--format '{{if ne "#{stage_cache}" .Repository }}{{.Repository}}:{{.Tag}}{{ end }}'),
16
16
  %(-f "label=dapp=#{stage_dapp_label}")
17
17
  ].join(' ')) # FIXME: negative filter is not currently supported by the Docker CLI
18
18
  end
@@ -7,8 +7,12 @@ module Dapp
7
7
  module Common
8
8
  protected
9
9
 
10
- def project_images
11
- shellout!(%(docker images --format="{{.Repository}}:{{.Tag}}" #{stage_cache})).stdout.strip
10
+ def project_images_names
11
+ shellout!(%(docker images --format="{{.Repository}}:{{.Tag}}" #{stage_cache})).stdout.lines.map(&:strip)
12
+ end
13
+
14
+ def project_images_ids
15
+ shellout!(%(docker images #{stage_cache} -q --no-trunc)).stdout.lines.map(&:strip)
12
16
  end
13
17
 
14
18
  def project_containers_flush
@@ -24,7 +28,7 @@ module Dapp
24
28
  end
25
29
 
26
30
  def remove_images(ids, force: false)
27
- remove_base('docker rmi%{force_option} %{ids}', ids, force: force)
31
+ remove_base('docker rmi%{force_option} %{ids}', ids.uniq, force: force)
28
32
  end
29
33
 
30
34
  def remove_containers_by_query(containers_query, force: false)
@@ -32,7 +36,7 @@ module Dapp
32
36
  end
33
37
 
34
38
  def remove_containers(ids, force: false)
35
- remove_base('docker rm%{force_option} %{ids}', ids, force: force)
39
+ remove_base('docker rm%{force_option} %{ids}', ids.uniq, force: force)
36
40
  end
37
41
 
38
42
  def remove_base(query_format, ids, force: false)
@@ -46,6 +50,10 @@ module Dapp
46
50
  yield(res)
47
51
  end
48
52
 
53
+ def image_labels(image_id)
54
+ Image::Stage.image_config_option(image_id: image_id, option: 'labels')
55
+ end
56
+
49
57
  def run_command(cmd)
50
58
  log(cmd) if log_verbose? || dry_run?
51
59
  shellout!(cmd) unless dry_run?
@@ -14,7 +14,7 @@ module Dapp
14
14
  elsif proper_cache_version?
15
15
  log_proper_cache do
16
16
  proper_cache_images = proper_cache_all_images
17
- remove_images(dapp_images_by_label('dapp').lines.select { |id| !proper_cache_images.lines.include?(id) }.map(&:strip))
17
+ remove_images(dapp_images_by_label('dapp').select { |id| !proper_cache_images.include?(id) }.map(&:strip))
18
18
  end
19
19
  else
20
20
  raise Error::Project, code: :mrproper_required_option
@@ -26,7 +26,7 @@ module Dapp
26
26
 
27
27
  def flush_by_label(label)
28
28
  log_step_with_indent(:containers) { dapp_containers_flush_by_label(label) }
29
- log_step_with_indent(:images) { dapp_images_flush_be_label(label) }
29
+ log_step_with_indent(:images) { dapp_images_flush_by_label(label) }
30
30
  end
31
31
 
32
32
  def proper_all?
@@ -45,13 +45,13 @@ module Dapp
45
45
  remove_images_by_query(%(docker images -f "dangling=true" -f "label=#{label}" -q), force: true)
46
46
  end
47
47
 
48
- def dapp_images_flush_be_label(label)
48
+ def dapp_images_flush_by_label(label)
49
49
  dapp_dangling_images_flush_by_label(label)
50
- remove_images(dapp_images_by_label(label).lines.map(&:strip), force: true)
50
+ remove_images(dapp_images_by_label(label), force: true)
51
51
  end
52
52
 
53
53
  def dapp_images_by_label(label)
54
- @dapp_images ||= shellout!(%(docker images -f "dangling=false" --format="{{.Repository}}:{{.Tag}}" -f "label=#{label}")).stdout.strip
54
+ @dapp_images ||= shellout!(%(docker images -f "dangling=false" --format="{{.Repository}}:{{.Tag}}" -f "label=#{label}")).stdout.lines.map(&:strip)
55
55
  end
56
56
 
57
57
  def proper_cache_all_images
@@ -59,7 +59,7 @@ module Dapp
59
59
  'docker images',
60
60
  '--format="{{.Repository}}:{{.Tag}}"',
61
61
  %(-f "label=dapp-cache-version=#{Dapp::BUILD_CACHE_VERSION}" -f "dangling=false")
62
- ].join(' ')).stdout.strip
62
+ ].join(' ')).stdout.lines.map(&:strip)
63
63
  end
64
64
  end
65
65
  end
@@ -8,43 +8,46 @@ module Dapp
8
8
  module CleanupLocal
9
9
  def stages_cleanup_local(repo)
10
10
  lock_repo(repo, readonly: true) do
11
- registry = registry(repo)
12
- repo_dimgs = repo_dimgs_images(registry)
13
- proper_cache if proper_cache_version?
14
- cleanup_project(repo_dimgs)
11
+ raise Error::Project, code: :stages_cleanup_required_option unless stages_cleanup_option?
12
+
13
+ project_containers_flush
14
+
15
+ proper_cache if proper_cache_version?
16
+ stages_cleanup_by_repo(repo) if proper_repo_cache?
17
+ proper_git_commit if proper_git_commit?
15
18
  end
16
19
  end
17
20
 
18
21
  protected
19
22
 
20
- def cleanup_project(repo_dimgs)
23
+ def proper_cache
24
+ log_proper_cache do
25
+ lock("#{name}.images") do
26
+ log_step_with_indent(name) do
27
+ remove_images(project_images_names.select { |image_name| !actual_cache_images.include?(image_name) })
28
+ end
29
+ end
30
+ end
31
+ end
32
+
33
+ def stages_cleanup_by_repo(repo)
34
+ registry = registry(repo)
35
+ repo_dimgs = repo_dimgs_images(registry)
36
+
21
37
  lock("#{name}.images") do
22
38
  log_step_with_indent(name) do
23
- project_containers_flush
24
39
  project_dangling_images_flush
25
40
  dimgs, stages = project_images_hash.partition { |_, image_id| repo_dimgs.values.include?(image_id) }
26
41
  dimgs = dimgs.to_h
27
42
  stages = stages.to_h
28
- dimgs.each { |_, aiid| clear_stages(aiid, stages) }
43
+ dimgs.each { |_, aiid| except_image_with_parents(aiid, stages) }
29
44
  remove_images(stages.keys)
30
45
  end
31
46
  end
32
47
  end
33
48
 
34
49
  def repo_dimgs_images(registry)
35
- repo_images(registry).first
36
- end
37
-
38
- def proper_cache
39
- log_proper_cache do
40
- lock("#{name}.images") do
41
- log_step_with_indent(name) do
42
- project_containers_flush
43
- actual_cache_images = actual_cache_images
44
- remove_images(project_images.lines.select { |id| !actual_cache_images.lines.include?(id) }.map(&:strip))
45
- end
46
- end
47
- end
50
+ repo_dimgs_and_cache(registry).first
48
51
  end
49
52
 
50
53
  def actual_cache_images
@@ -53,7 +56,7 @@ module Dapp
53
56
  '--format="{{.Repository}}:{{.Tag}}"',
54
57
  %(-f "label=dapp-cache-version=#{Dapp::BUILD_CACHE_VERSION}"),
55
58
  stage_cache
56
- ].join(' ')).stdout.strip
59
+ ].join(' ')).stdout.lines.map(&:strip)
57
60
  end
58
61
 
59
62
  def project_images_hash
@@ -62,9 +65,9 @@ module Dapp
62
65
  end.to_h
63
66
  end
64
67
 
65
- def clear_stages(image_id, stages)
68
+ def except_image_with_parents(image_id, stages)
66
69
  if image_exist?(image_id)
67
- image_dapp_artifacts_label(image_id).each { |aiid| clear_stages(aiid, stages) }
70
+ image_dapp_artifacts_label(image_id).each { |aiid| except_image_with_parents(aiid, stages) }
68
71
  iid = image_id
69
72
  loop do
70
73
  stages.delete_if { |_, siid| siid == iid }
@@ -89,6 +92,46 @@ module Dapp
89
92
  def image_parent(image_id)
90
93
  shellout!(%(docker inspect -f {{.Parent}} #{image_id})).stdout.strip
91
94
  end
95
+
96
+ def proper_git_commit
97
+ log_proper_git_commit do
98
+ unproper_images = []
99
+ project_images_detailed.each do |_, attrs|
100
+ attrs['Config']['Labels'].each do |repo_name, commit|
101
+ next if (repo = project_git_repositories[repo_name]).nil?
102
+ git = repo.name == 'own' ? :git : :git_bare
103
+ unproper_images.concat(image_hierarchy(attrs['Id'])) unless repo.send(git).exists?(commit)
104
+ end
105
+ end
106
+ remove_images(unproper_images.uniq)
107
+ end
108
+ end
109
+
110
+ def project_images_detailed
111
+ @project_images_detailed ||= {}.tap do |images|
112
+ project_images_names.each do |image_name|
113
+ shellout!(%(docker inspect --format='{{json .}}' #{image_name})).stdout.strip.tap do |output|
114
+ images[image_name] = output == 'null' ? {} : JSON.parse(output)
115
+ end
116
+ end
117
+ end
118
+ end
119
+
120
+ def image_hierarchy(image_id)
121
+ hierarchy = []
122
+ iids = [image_id]
123
+
124
+ loop do
125
+ hierarchy.concat(iids)
126
+ break if begin
127
+ iids.map! do |iid|
128
+ project_images_detailed.map { |_, attrs| attrs['Id'] if attrs['Parent'] == iid }.compact
129
+ end.flatten!.empty?
130
+ end
131
+ end
132
+
133
+ hierarchy
134
+ end
92
135
  end
93
136
  end
94
137
  end
@@ -8,19 +8,37 @@ module Dapp
8
8
  module CleanupRepo
9
9
  def stages_cleanup_repo(repo)
10
10
  lock_repo(repo) do
11
+ raise Error::Project, code: :stages_cleanup_required_option unless stages_cleanup_option?
12
+
11
13
  registry = registry(repo)
12
- repo_dimgs, repo_stages = repo_images(registry)
14
+ repo_dimgs, repo_stages = repo_dimgs_and_cache(registry)
13
15
  repo_stages.delete_if { |_, siid| repo_dimgs.values.include?(siid) } # ignoring stages with dimgs ids (v2)
14
- proper_repo_cache(registry, repo_stages) if proper_cache_version?
15
- log_step_with_indent(repo) do
16
- repo_dimgs.each { |image_tag, image_id| clear_repo_stages(registry, repo_stages, image_tag, image_id) }
17
- repo_stages.keys.each { |image_tag| delete_repo_image(registry, image_tag) }
18
- end
16
+
17
+ proper_repo_cache(registry, repo_stages) if proper_cache_version?
18
+ repo_stages_cleanup(registry, repo_dimgs, repo_stages) if proper_repo_cache?
19
+ proper_repo_git_commit(registry) if proper_git_commit?
19
20
  end
20
21
  end
21
22
 
22
23
  protected
23
24
 
25
+ def proper_repo_cache(registry, repo_stages)
26
+ log_proper_cache do
27
+ wrong_cache_images = repo_stages.select do |image_tag, _|
28
+ repo_image_dapp_cache_version_label(registry, image_tag) != Dapp::BUILD_CACHE_VERSION.to_s
29
+ end
30
+ wrong_cache_images.each { |image_tag, _| delete_repo_image(registry, image_tag) }
31
+ repo_stages.delete_if { |image_tag, _| wrong_cache_images.keys.include?(image_tag) }
32
+ end
33
+ end
34
+
35
+ def repo_stages_cleanup(registry, repo_dimgs, repo_stages)
36
+ log_step_with_indent(repo) do
37
+ repo_dimgs.each { |image_tag, image_id| clear_repo_stages(registry, repo_stages, image_tag, image_id) }
38
+ repo_stages.keys.each { |image_tag| delete_repo_image(registry, image_tag) }
39
+ end
40
+ end
41
+
24
42
  def clear_repo_stages(registry, repo_stages, image_tag, image_id)
25
43
  repo_image_dapp_artifacts_labels(registry, image_tag).each do |iid|
26
44
  itag = find_image_tag_by_id(repo_stages, iid)
@@ -49,13 +67,54 @@ module Dapp
49
67
  nil
50
68
  end
51
69
 
52
- def proper_repo_cache(registry, repo_stages)
53
- log_proper_cache do
54
- wrong_cache_images = repo_stages.select do |image_tag, _|
55
- repo_image_dapp_cache_version_label(registry, image_tag) != Dapp::BUILD_CACHE_VERSION.to_s
70
+ def proper_repo_git_commit(registry)
71
+ log_proper_git_commit do
72
+ unproper_images = []
73
+ repo_project_images_detailed(registry).each do |_, attrs|
74
+ attrs[:labels].each do |repo_name, commit|
75
+ next if (repo = project_git_repositories[repo_name]).nil?
76
+ git = repo.name == 'own' ? :git : :git_bare
77
+ unproper_images.concat(repo_image_tags_hierarchy(registry, attrs[:id])) unless repo.send(git).exists?(commit)
78
+ end
56
79
  end
57
- wrong_cache_images.each { |image_tag, _| delete_repo_image(registry, image_tag) }
58
- repo_stages.delete_if { |image_tag, _| wrong_cache_images.keys.include?(image_tag) }
80
+ remove_repo_images(registry, unproper_images.uniq)
81
+ end
82
+ end
83
+
84
+ def repo_project_images_detailed(registry)
85
+ @repo_project_images_detailed ||= begin
86
+ registry.tags.map do |tag|
87
+ image_history = registry.image_history(tag)
88
+ attrs = {
89
+ id: registry.image_id(tag),
90
+ parent: image_history['container_config']['Image'],
91
+ labels: image_history['config']['Labels']
92
+ }
93
+ [tag, attrs]
94
+ end
95
+ end
96
+ end
97
+
98
+ def repo_image_tags_hierarchy(registry, registry_image_id)
99
+ hierarchy = []
100
+ iids = [registry_image_id]
101
+
102
+ loop do
103
+ hierarchy.concat(iids)
104
+ break if begin
105
+ iids.map! do |iid|
106
+ repo_project_images_detailed(registry).map { |_, attrs| attrs[:id] if attrs[:parent] == iid }.compact
107
+ end.flatten!.empty?
108
+ end
109
+ end
110
+
111
+ repo_project_images_detailed(registry).map { |tag, attrs| tag if hierarchy.include? attrs[:id] }.compact
112
+ end
113
+
114
+ def remove_repo_images(registry, tags)
115
+ tags.each do |tag|
116
+ log(tag) if dry_run? || log_verbose?
117
+ registry.image_delete(tag) unless dry_run?
59
118
  end
60
119
  end
61
120
  end
@@ -13,7 +13,7 @@ module Dapp
13
13
  DockerRegistry.new(repo)
14
14
  end
15
15
 
16
- def repo_images(registry)
16
+ def repo_dimgs_and_cache(registry)
17
17
  format = proc do |arr|
18
18
  arr.map do |tag|
19
19
  if (id = registry.image_id(tag)).nil?
@@ -30,8 +30,8 @@ module Dapp
30
30
  def registry_tags(registry)
31
31
  registry.tags
32
32
  rescue Exception::Registry => e
33
- raise unless e.net_status[:code] == :dimg_not_found_in_registry
34
- log_warning(desc: { code: 'dimg_not_found_in_registry' })
33
+ raise unless e.net_status[:code] == :no_such_dimg
34
+ log_warning(desc: { code: :dimg_not_found_in_registry })
35
35
  []
36
36
  end
37
37
 
@@ -47,6 +47,35 @@ module Dapp
47
47
  hash.select { |k, _v| k.start_with?('dapp-artifact') }.values
48
48
  end
49
49
 
50
+ def project_git_repositories
51
+ @project_git_repositories ||= begin
52
+ {}.tap do |repositories|
53
+ dimgs = build_configs.map { |config| Dimg.new(config: config, project: self, ignore_git_fetch: true) }
54
+ dimgs.each do |dimg|
55
+ [dimg, dimg.artifacts].flatten
56
+ .map(&:git_artifacts).flatten
57
+ .map { |ga_artifact| repositories[ga_artifact.full_name] = ga_artifact.repo }
58
+ end
59
+ end
60
+ end
61
+ end
62
+
63
+ def proper_repo_cache?
64
+ !!cli_options[:proper_repo_cache]
65
+ end
66
+
67
+ def proper_git_commit?
68
+ !!cli_options[:proper_git_commit]
69
+ end
70
+
71
+ def stages_cleanup_option?
72
+ proper_git_commit? || proper_cache_version? || proper_repo_cache?
73
+ end
74
+
75
+ def log_proper_git_commit(&blk)
76
+ log_step_with_indent(:'proper git commit', &blk)
77
+ end
78
+
50
79
  def lock_repo(repo, *args, &blk)
51
80
  lock("repo.#{hashsum repo}", *args, &blk)
52
81
  end
@@ -11,7 +11,7 @@ module Dapp
11
11
  log_step_with_indent(name) do
12
12
  project_containers_flush
13
13
  project_dangling_images_flush
14
- remove_images(project_images.lines.map(&:strip))
14
+ remove_images(project_images_names)
15
15
  end
16
16
  end
17
17
  end
@@ -10,7 +10,7 @@ module Dapp
10
10
  lock_repo(repo) do
11
11
  log_step_with_indent(repo) do
12
12
  registry = registry(repo)
13
- repo_dimgs, repo_stages = repo_images(registry)
13
+ repo_dimgs, repo_stages = repo_dimgs_and_cache(registry)
14
14
  repo_dimgs.merge(repo_stages).keys.each { |image_tag| delete_repo_image(registry, image_tag) }
15
15
  end
16
16
  end
@@ -62,9 +62,11 @@ module Dapp
62
62
 
63
63
  def log_warning(*args, **kwargs)
64
64
  kwargs[:style] = :warning
65
- kwargs[:desc] ||= {}
66
- kwargs[:desc][:context] ||= :warning
67
65
  kwargs[:stream] ||= $stderr
66
+ if args.empty?
67
+ kwargs[:desc] ||= {}
68
+ kwargs[:desc][:context] ||= :warning
69
+ end
68
70
  log(*args, **kwargs)
69
71
  end
70
72
 
data/lib/dapp/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # Version
2
2
  module Dapp
3
- VERSION = '0.7.16'.freeze
3
+ VERSION = '0.7.17'.freeze
4
4
  BUILD_CACHE_VERSION = 6
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dapp
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.16
4
+ version: 0.7.17
5
5
  platform: ruby
6
6
  authors:
7
7
  - Dmitry Stolyarov
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-12-01 00:00:00.000000000 Z
11
+ date: 2016-12-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: mixlib-shellout
@@ -138,6 +138,20 @@ dependencies:
138
138
  - - "~>"
139
139
  - !ruby/object:Gem::Version
140
140
  version: 3.0.0
141
+ - !ruby/object:Gem::Dependency
142
+ name: rugged
143
+ requirement: !ruby/object:Gem::Requirement
144
+ requirements:
145
+ - - "~>"
146
+ - !ruby/object:Gem::Version
147
+ version: 0.24.0
148
+ type: :runtime
149
+ prerelease: false
150
+ version_requirements: !ruby/object:Gem::Requirement
151
+ requirements:
152
+ - - "~>"
153
+ - !ruby/object:Gem::Version
154
+ version: 0.24.0
141
155
  - !ruby/object:Gem::Dependency
142
156
  name: bundler
143
157
  requirement: !ruby/object:Gem::Requirement
@@ -460,6 +474,7 @@ files:
460
474
  - lib/dapp/error/image.rb
461
475
  - lib/dapp/error/project.rb
462
476
  - lib/dapp/error/registry.rb
477
+ - lib/dapp/error/rugged.rb
463
478
  - lib/dapp/error/shellout.rb
464
479
  - lib/dapp/exception/base.rb
465
480
  - lib/dapp/exception/introspect_image.rb