dapp 0.7.16 → 0.7.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/config/en/net_status.yml +9 -7
- data/lib/dapp.rb +3 -0
- data/lib/dapp/build/stage/artifact_base.rb +13 -13
- data/lib/dapp/build/stage/base.rb +4 -0
- data/lib/dapp/cli/stages.rb +1 -1
- data/lib/dapp/cli/stages/cleanup_local.rb +19 -3
- data/lib/dapp/cli/stages/cleanup_repo.rb +6 -4
- data/lib/dapp/config/dimg/instance_methods.rb +1 -1
- data/lib/dapp/config/dimg/validation.rb +5 -4
- data/lib/dapp/config/directive/git_artifact_remote.rb +2 -0
- data/lib/dapp/dimg.rb +10 -1
- data/lib/dapp/dimg/stages.rb +12 -12
- data/lib/dapp/docker_registry/base.rb +12 -8
- data/lib/dapp/docker_registry/mod/authorization.rb +4 -4
- data/lib/dapp/error/rugged.rb +6 -0
- data/lib/dapp/git_artifact.rb +2 -2
- data/lib/dapp/git_repo/base.rb +16 -11
- data/lib/dapp/git_repo/own.rb +1 -8
- data/lib/dapp/git_repo/remote.rb +8 -3
- data/lib/dapp/project/command/bp.rb +2 -2
- data/lib/dapp/project/command/cleanup.rb +1 -1
- data/lib/dapp/project/command/common.rb +12 -4
- data/lib/dapp/project/command/mrproper.rb +6 -6
- data/lib/dapp/project/command/stages/cleanup_local.rb +66 -23
- data/lib/dapp/project/command/stages/cleanup_repo.rb +71 -12
- data/lib/dapp/project/command/stages/common.rb +32 -3
- data/lib/dapp/project/command/stages/flush_local.rb +1 -1
- data/lib/dapp/project/command/stages/flush_repo.rb +1 -1
- data/lib/dapp/project/logging/base.rb +4 -2
- data/lib/dapp/version.rb +1 -1
- metadata +17 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ac3cccdbfe9bb2fb9aab05958f1fa1d8f2db5ab5
|
4
|
+
data.tar.gz: c8518534ebe63840f47ff76327186f6b78d542bc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 043df0615ad8084c2deca2fec4bb98f932a60a3ce7969f2ad84e4a14fef2643ed6e5889a8387c67701b287ef945d0f7198b1143be1fb3e24b78aaa9e5ae7bc6b
|
7
|
+
data.tar.gz: 12099bf042bb1282be953c32f12bce147fe2a6103a90a82eb88dcf3e296bb1af4d782fde0aa12988d6caad8dc2b881dad8a222f703f82ce4ba7caf7f4b8a28ef
|
data/config/en/net_status.yml
CHANGED
@@ -26,6 +26,7 @@ en:
|
|
26
26
|
cannot_run_ssh_agent: "Cannot run ssh-agent"
|
27
27
|
ssh_key_not_found: "Ssh key `%{path}` not exist!"
|
28
28
|
mrproper_required_option: "Expected command option `--improper-dev-mode-cache`, `--improper-cache-version-stages` or `--all`!"
|
29
|
+
stages_cleanup_required_option: "Expected command option `--improper-cache-version`, `--improper-git-commit` or `--improper-repo-cache`!"
|
29
30
|
config:
|
30
31
|
dimg_name_required: 'Dimg name required!'
|
31
32
|
dimg_name_incorrect: "Dimg has incorrect name `%{name}`: doesn't match regex `%{reg}`!"
|
@@ -42,6 +43,7 @@ en:
|
|
42
43
|
stage_artifact_not_associated: "Artifact not associated with any stage: expected `before` or `after` attribute!"
|
43
44
|
stage_artifact_double_associate: "Cannot use `%{stage}` stage for artifact, already used in `%{conflict_stage}` stage!"
|
44
45
|
stage_artifact_not_supported_associated_stage: "Bad artifact stage `%{stage}`!"
|
46
|
+
git_artifact_remote_unsupported_protocol: "Remote git repo `%{url}`: unsupported protocol!"
|
45
47
|
artifact_conflict: "Conflict between artifacts paths!"
|
46
48
|
scratch_unsupported_directive: "Scratch dimg has unsupported directive `%{directive}`!"
|
47
49
|
scratch_artifact_required: "Scratch dimg without artifacts!"
|
@@ -54,13 +56,13 @@ en:
|
|
54
56
|
cookbook_berksfile_not_found: "Dapp cookbook Berksfile not found at %{path}"
|
55
57
|
cookbook_metadata_not_found: "Dapp cookbook metadata.rb file not found at %{path}"
|
56
58
|
cookbook_not_specified_in_berksfile: "Dapp cookbook `%{name}` not specified in Berksfile at %{path}"
|
57
|
-
registry:
|
58
|
-
incorrect_repo: 'Incorrect repository!'
|
59
|
-
no_such_dimg: 'No such dimg in registry!'
|
60
|
-
authenticate_type_not_supported: 'Registry authenticate type not supported!'
|
61
|
-
page_not_found: "Page `%{url}` not found!"
|
62
|
-
user_not_authorized: 'User not authorized!'
|
63
|
-
response_with_error_status: 'Response with error status!'
|
64
59
|
berksfile_absolute_path_forbidden: "Absolute paths in Berksfile are not allowed (cookbook `%{cookbook}`, path: `%{path}`)"
|
60
|
+
registry:
|
61
|
+
no_such_dimg: 'Registry `%{registry}`: no such dimg in registry!'
|
62
|
+
authenticate_type_not_supported: 'Registry `%{registry}`: authenticate type not supported!'
|
63
|
+
page_not_found: "Registry `%{registry}`: page `%{url}` not found!"
|
64
|
+
user_not_authorized: 'Registry `%{registry}`: user not authorized!'
|
65
|
+
rugged:
|
66
|
+
rugged_network_error: "Remote git repo `%{url}`: `%{message}`!"
|
65
67
|
lock:
|
66
68
|
timeout: "Could not obtain lock for `%{name}` within %{timeout} seconds"
|
data/lib/dapp.rb
CHANGED
@@ -9,11 +9,13 @@ require 'mixlib/shellout'
|
|
9
9
|
require 'securerandom'
|
10
10
|
require 'excon'
|
11
11
|
require 'json'
|
12
|
+
require 'uri'
|
12
13
|
require 'ostruct'
|
13
14
|
require 'time'
|
14
15
|
require 'i18n'
|
15
16
|
require 'paint'
|
16
17
|
require 'inifile'
|
18
|
+
require 'rugged'
|
17
19
|
|
18
20
|
require 'net_status'
|
19
21
|
|
@@ -32,6 +34,7 @@ require 'dapp/error/config'
|
|
32
34
|
require 'dapp/error/project'
|
33
35
|
require 'dapp/error/shellout'
|
34
36
|
require 'dapp/error/registry'
|
37
|
+
require 'dapp/error/rugged'
|
35
38
|
require 'dapp/exception/base'
|
36
39
|
require 'dapp/exception/introspect_image'
|
37
40
|
require 'dapp/exception/registry'
|
@@ -18,8 +18,19 @@ module Dapp
|
|
18
18
|
image.add_service_change_label artifacts_labels
|
19
19
|
end
|
20
20
|
|
21
|
-
def
|
22
|
-
|
21
|
+
def artifacts
|
22
|
+
@artifacts ||= begin
|
23
|
+
dimg.config.public_send("_#{name}").map do |artifact|
|
24
|
+
artifact_dimg = Dapp::Artifact.new(config: artifact._config,
|
25
|
+
project: dimg.project,
|
26
|
+
ignore_git_fetch: dimg.ignore_git_fetch)
|
27
|
+
{ name: artifact._config._name, options: artifact._artifact_options, dimg: artifact_dimg }
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def artifact?
|
33
|
+
true
|
23
34
|
end
|
24
35
|
|
25
36
|
protected
|
@@ -32,17 +43,6 @@ module Dapp
|
|
32
43
|
true
|
33
44
|
end
|
34
45
|
|
35
|
-
def artifacts
|
36
|
-
@artifacts ||= begin
|
37
|
-
dimg.config.public_send("_#{name}").map do |artifact|
|
38
|
-
artifact_dimg = Dapp::Artifact.new(config: artifact._config,
|
39
|
-
project: dimg.project,
|
40
|
-
ignore_git_fetch: dimg.ignore_git_fetch)
|
41
|
-
{ name: artifact._config._name, options: artifact._artifact_options, dimg: artifact_dimg }
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
46
|
def artifacts_signatures
|
47
47
|
artifacts.map { |artifact| hashsum [artifact[:dimg].signature, artifact[:options]] }
|
48
48
|
end
|
data/lib/dapp/cli/stages.rb
CHANGED
@@ -9,7 +9,7 @@ Version: #{Dapp::VERSION}
|
|
9
9
|
|
10
10
|
Available subcommands: (for details, dapp SUB-COMMAND --help)
|
11
11
|
|
12
|
-
dapp stages cleanup local [options] [DIMG ...] REPO
|
12
|
+
dapp stages cleanup local [options] [DIMG ...] [REPO]
|
13
13
|
dapp stages cleanup repo [options] [DIMG ...] REPO
|
14
14
|
dapp stages flush local [options] [DIMG ...]
|
15
15
|
dapp stages flush repo [options] [DIMG ...] REPO
|
@@ -7,7 +7,7 @@ module Dapp
|
|
7
7
|
Version: #{Dapp::VERSION}
|
8
8
|
|
9
9
|
Usage:
|
10
|
-
dapp stages cleanup local [options] [DIMG ...] REPO
|
10
|
+
dapp stages cleanup local [options] [DIMG ...] [REPO]
|
11
11
|
|
12
12
|
DIMG Dapp image to process [default: *].
|
13
13
|
|
@@ -17,10 +17,26 @@ BANNER
|
|
17
17
|
long: '--improper-cache-version',
|
18
18
|
boolean: true
|
19
19
|
|
20
|
+
option :proper_git_commit,
|
21
|
+
long: '--improper-git-commit',
|
22
|
+
boolean: true
|
23
|
+
|
24
|
+
option :proper_repo_cache,
|
25
|
+
long: '--improper-repo-cache',
|
26
|
+
boolean: true
|
27
|
+
|
20
28
|
def run(argv = ARGV)
|
21
29
|
self.class.parse_options(self, argv)
|
22
|
-
|
23
|
-
Project.new(cli_options: config, dimgs_patterns: cli_arguments).
|
30
|
+
repository = repo
|
31
|
+
Project.new(cli_options: config, dimgs_patterns: cli_arguments).send(run_method, repository)
|
32
|
+
end
|
33
|
+
|
34
|
+
def repo
|
35
|
+
config[:proper_repo_cache] ? self.class.required_argument(self) : nil
|
36
|
+
end
|
37
|
+
|
38
|
+
def run_method
|
39
|
+
:stages_cleanup_local
|
24
40
|
end
|
25
41
|
end
|
26
42
|
end
|
@@ -13,10 +13,12 @@ Usage:
|
|
13
13
|
|
14
14
|
Options:
|
15
15
|
BANNER
|
16
|
-
def
|
17
|
-
|
18
|
-
|
19
|
-
|
16
|
+
def run_method
|
17
|
+
:stages_cleanup_repo
|
18
|
+
end
|
19
|
+
|
20
|
+
def repo
|
21
|
+
self.class.required_argument(self)
|
20
22
|
end
|
21
23
|
end
|
22
24
|
end
|
@@ -31,12 +31,13 @@ module Dapp
|
|
31
31
|
end
|
32
32
|
|
33
33
|
def validate_scratch_directives!
|
34
|
-
directives = [:_shell, :_chef, :
|
35
|
-
:
|
36
|
-
|
34
|
+
directives = [[:_shell, :shell], [:_chef, :chef], [:_git_artifact, :git],
|
35
|
+
[:_install_dependencies, :install_depends_on], [:_setup_dependencies, :setup_depends_on],
|
36
|
+
[:_tmp_dir_mount, :mount], [:_build_dir_mount, :mount]]
|
37
|
+
directives.each do |name, user_name|
|
37
38
|
raise Error::Config,
|
38
39
|
code: :scratch_unsupported_directive,
|
39
|
-
data: { directive:
|
40
|
+
data: { directive: user_name } unless public_send(name).send(:empty?)
|
40
41
|
end
|
41
42
|
|
42
43
|
docker_directives = [:_expose, :_env, :_cmd, :_onbuild, :_workdir, :_user, :_entrypoint]
|
@@ -6,6 +6,8 @@ module Dapp
|
|
6
6
|
attr_reader :_url, :_name, :_branch, :_commit
|
7
7
|
|
8
8
|
def initialize(url, **kwargs, &blk)
|
9
|
+
raise Error::Config, code: :git_artifact_remote_unsupported_protocol, data: { url: url } unless %w(http https).include? URI(url.to_s).scheme
|
10
|
+
|
9
11
|
@_url = url
|
10
12
|
@_name = url.gsub(%r{.*?([^\/ ]+)\.git}, '\\1')
|
11
13
|
|
data/lib/dapp/dimg.rb
CHANGED
@@ -37,7 +37,7 @@ module Dapp
|
|
37
37
|
end
|
38
38
|
end
|
39
39
|
ensure
|
40
|
-
|
40
|
+
cleanup_tmp
|
41
41
|
end
|
42
42
|
|
43
43
|
def tag!(tag)
|
@@ -132,6 +132,10 @@ module Dapp
|
|
132
132
|
@builder ||= Builder.const_get(config._builder.capitalize).new(self)
|
133
133
|
end
|
134
134
|
|
135
|
+
def artifacts
|
136
|
+
@artifacts ||= artifacts_stages.map { |stage| stage.artifacts.map { |artifact| artifact[:dimg] } }.flatten
|
137
|
+
end
|
138
|
+
|
135
139
|
def artifact?
|
136
140
|
false
|
137
141
|
end
|
@@ -153,6 +157,11 @@ module Dapp
|
|
153
157
|
system(cmd)
|
154
158
|
end
|
155
159
|
|
160
|
+
def cleanup_tmp
|
161
|
+
FileUtils.rm_rf(tmp_path)
|
162
|
+
artifacts.each(&:cleanup_tmp)
|
163
|
+
end
|
164
|
+
|
156
165
|
protected
|
157
166
|
|
158
167
|
def should_be_built?
|
data/lib/dapp/dimg/stages.rb
CHANGED
@@ -15,16 +15,8 @@ module Dapp
|
|
15
15
|
project.stage_dapp_label
|
16
16
|
end
|
17
17
|
|
18
|
-
def
|
19
|
-
|
20
|
-
stages.each do |stage|
|
21
|
-
if stage.respond_to?(:images)
|
22
|
-
images.concat(stage.images)
|
23
|
-
else
|
24
|
-
images << stage.image
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end.uniq!(&:name)
|
18
|
+
def all_images
|
19
|
+
@all_images ||= all_stages.map(&:image).uniq!(&:name)
|
28
20
|
end
|
29
21
|
|
30
22
|
protected
|
@@ -38,11 +30,11 @@ module Dapp
|
|
38
30
|
end
|
39
31
|
|
40
32
|
def export_images
|
41
|
-
|
33
|
+
all_images.select(&:tagged?)
|
42
34
|
end
|
43
35
|
|
44
36
|
def import_images
|
45
|
-
|
37
|
+
all_images.select { |image| !image.tagged? }
|
46
38
|
end
|
47
39
|
|
48
40
|
def stages
|
@@ -54,6 +46,14 @@ module Dapp
|
|
54
46
|
end
|
55
47
|
end
|
56
48
|
end
|
49
|
+
|
50
|
+
def artifacts_stages
|
51
|
+
@artifacts_stages ||= stages.select { |stage| stage.artifact? }
|
52
|
+
end
|
53
|
+
|
54
|
+
def all_stages
|
55
|
+
stages + artifacts.map(&:all_stages).flatten
|
56
|
+
end
|
57
57
|
end # Stages
|
58
58
|
end # Dimg
|
59
59
|
end # Dapp
|
@@ -25,7 +25,7 @@ module Dapp
|
|
25
25
|
def tags
|
26
26
|
@tags ||= api_request(repo_suffix, 'tags/list')['tags'] || []
|
27
27
|
rescue Error::Registry => e
|
28
|
-
raise Exception::Registry, code: :
|
28
|
+
raise Exception::Registry, code: :no_such_dimg, data: { registry: api_url } if e.net_status[:code] == :page_not_found
|
29
29
|
raise
|
30
30
|
end
|
31
31
|
|
@@ -50,6 +50,11 @@ module Dapp
|
|
50
50
|
headers: { Accept: 'Accept: application/vnd.docker.distribution.manifest.v2+json' })
|
51
51
|
end
|
52
52
|
|
53
|
+
def image_history(tag)
|
54
|
+
response = manifest_v1(tag)
|
55
|
+
JSON.load(response['history'].first['v1Compatibility'])
|
56
|
+
end
|
57
|
+
|
53
58
|
protected
|
54
59
|
|
55
60
|
def image_digest(tag)
|
@@ -70,11 +75,6 @@ module Dapp
|
|
70
75
|
headers: { Accept: 'application/vnd.docker.distribution.manifest.v2+json' })
|
71
76
|
end
|
72
77
|
|
73
|
-
def image_history(tag)
|
74
|
-
response = manifest_v1(tag)
|
75
|
-
JSON.load(response['history'].first['v1Compatibility'])
|
76
|
-
end
|
77
|
-
|
78
78
|
def blob_delete(id)
|
79
79
|
api_request(repo_suffix, "/blobs/#{id}",
|
80
80
|
method: :delete, expects: [202, 404])
|
@@ -88,9 +88,9 @@ module Dapp
|
|
88
88
|
url = api_url(*uri)
|
89
89
|
request(url, **default_api_options.merge(options))
|
90
90
|
rescue Excon::Error::NotFound
|
91
|
-
raise Error::Registry, code: :page_not_found, data: { url: url }
|
91
|
+
raise Error::Registry, code: :page_not_found, data: { url: url, registry: api_url }
|
92
92
|
rescue Excon::Error::Unauthorized
|
93
|
-
|
93
|
+
user_not_authorized!
|
94
94
|
end
|
95
95
|
|
96
96
|
def api_url(*uri)
|
@@ -100,6 +100,10 @@ module Dapp
|
|
100
100
|
def default_api_options
|
101
101
|
{ expects: [200] }
|
102
102
|
end
|
103
|
+
|
104
|
+
def user_not_authorized!
|
105
|
+
raise Error::Registry, code: :user_not_authorized, data: { registry: api_url }
|
106
|
+
end
|
103
107
|
end
|
104
108
|
end # DockerRegistry
|
105
109
|
end # Dapp
|
@@ -10,7 +10,7 @@ module Dapp
|
|
10
10
|
when /Bearer/ then { headers: { Authorization: "Bearer #{authorization_token(authenticate_header)}" } }
|
11
11
|
when /Basic/ then { headers: { Authorization: "Basic #{authorization_auth}" } }
|
12
12
|
when nil then {}
|
13
|
-
else raise Error::Registry, code: :authenticate_type_not_supported
|
13
|
+
else raise Error::Registry, code: :authenticate_type_not_supported, data: { registry: api_url }
|
14
14
|
end
|
15
15
|
end
|
16
16
|
end
|
@@ -42,14 +42,14 @@ module Dapp
|
|
42
42
|
r = chomp_name(r)
|
43
43
|
end
|
44
44
|
credential = (auths[r] || auths.find { |repo, _| repo == r })
|
45
|
-
|
45
|
+
user_not_authorized! if credential.nil?
|
46
46
|
credential['auth']
|
47
47
|
end
|
48
48
|
|
49
49
|
def auths_section_from_docker_config
|
50
50
|
file = Pathname(File.join(Dir.home, '.docker', 'config.json'))
|
51
|
-
|
52
|
-
JSON.load(file.read)['auths'].tap { |auths|
|
51
|
+
user_not_authorized! unless file.exist?
|
52
|
+
JSON.load(file.read)['auths'].tap { |auths| user_not_authorized! if auths.nil? }
|
53
53
|
end
|
54
54
|
|
55
55
|
private
|
data/lib/dapp/git_artifact.rb
CHANGED
@@ -55,11 +55,11 @@ module Dapp
|
|
55
55
|
end
|
56
56
|
|
57
57
|
def any_changes?(from, to = latest_commit)
|
58
|
-
!repo.
|
58
|
+
!repo.old_git_bare(diff_command(from, to, quiet: true), returns: [0, 1]).status.success?
|
59
59
|
end
|
60
60
|
|
61
61
|
def patch_size(from, to)
|
62
|
-
repo.
|
62
|
+
repo.old_git_bare("#{diff_command(from, to)} | wc -c").stdout.strip.to_i
|
63
63
|
end
|
64
64
|
|
65
65
|
def latest_commit
|
data/lib/dapp/git_repo/base.rb
CHANGED
@@ -15,35 +15,40 @@ module Dapp
|
|
15
15
|
end
|
16
16
|
|
17
17
|
def path
|
18
|
-
dimg.tmp_path
|
18
|
+
dimg.tmp_path("#{name}.git").to_s
|
19
19
|
end
|
20
20
|
|
21
|
-
def git_bare
|
22
|
-
|
21
|
+
def git_bare
|
22
|
+
@git_bare ||= Rugged::Repository.new(path, bare: true)
|
23
23
|
end
|
24
24
|
|
25
|
-
def
|
26
|
-
|
25
|
+
def old_git_bare(command, **kwargs)
|
26
|
+
old_git "--git-dir=#{path} #{command}", **kwargs
|
27
27
|
end
|
28
28
|
|
29
|
-
def
|
30
|
-
git_bare(
|
29
|
+
def commit_at(commit)
|
30
|
+
git_bare.lookup(commit).time.to_i
|
31
31
|
end
|
32
32
|
|
33
|
-
def
|
34
|
-
git_bare
|
33
|
+
def latest_commit(branch)
|
34
|
+
return git_bare.head.target_id if branch == 'HEAD'
|
35
|
+
git_bare.branches[branch].target_id
|
35
36
|
end
|
36
37
|
|
37
38
|
def cleanup!
|
38
39
|
end
|
39
40
|
|
40
41
|
def branch
|
41
|
-
git_bare(
|
42
|
+
git_bare.head.name.sub(/^refs\/heads\//, '')
|
42
43
|
end
|
43
44
|
|
44
45
|
protected
|
45
46
|
|
46
|
-
def git
|
47
|
+
def git
|
48
|
+
@git ||= Rugged::Repository.new(path)
|
49
|
+
end
|
50
|
+
|
51
|
+
def old_git(command, **kwargs)
|
47
52
|
dimg.system_shellout! "#{dimg.project.git_bin} #{command}", **kwargs
|
48
53
|
end
|
49
54
|
end
|
data/lib/dapp/git_repo/own.rb
CHANGED
@@ -11,14 +11,7 @@ module Dapp
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def path
|
14
|
-
@path ||=
|
15
|
-
git_repo_path = Pathname(git("-C #{dimg.home_path} rev-parse --git-dir").stdout.strip)
|
16
|
-
if git_repo_path.relative?
|
17
|
-
File.join(dimg.home_path, git_repo_path)
|
18
|
-
else
|
19
|
-
git_repo_path
|
20
|
-
end
|
21
|
-
end
|
14
|
+
@path ||= Rugged::Repository.discover(dimg.home_path.to_s).path
|
22
15
|
end
|
23
16
|
|
24
17
|
def latest_commit(branch = nil)
|
data/lib/dapp/git_repo/remote.rb
CHANGED
@@ -8,13 +8,18 @@ module Dapp
|
|
8
8
|
@url = url
|
9
9
|
|
10
10
|
dimg.project.log_secondary_process(dimg.project.t(code: 'process.git_artifact_clone', data: { name: name }), short: true) do
|
11
|
-
|
11
|
+
begin
|
12
|
+
Rugged::Repository.clone_at(url, path, bare: true)
|
13
|
+
rescue Rugged::NetworkError => e
|
14
|
+
raise Error::Rugged, code: :rugged_network_error, data: { message: e.message, url: url }
|
15
|
+
end
|
12
16
|
end unless File.directory?(path)
|
13
17
|
end
|
14
18
|
|
15
|
-
def fetch!(branch =
|
19
|
+
def fetch!(branch = nil)
|
20
|
+
branch ||= self.branch
|
16
21
|
dimg.project.log_secondary_process(dimg.project.t(code: 'process.git_artifact_fetch', data: { name: name }), short: true) do
|
17
|
-
git_bare
|
22
|
+
git_bare.fetch('origin', [branch])
|
18
23
|
end unless dimg.ignore_git_fetch || dimg.project.dry_run?
|
19
24
|
end
|
20
25
|
|
@@ -8,12 +8,12 @@ module Dapp
|
|
8
8
|
def bp(repo)
|
9
9
|
bp_step(:build)
|
10
10
|
bp_step(:push, repo)
|
11
|
-
bp_step(:
|
11
|
+
bp_step(:stages_cleanup_by_repo, repo)
|
12
12
|
bp_step(:cleanup)
|
13
13
|
end
|
14
14
|
|
15
15
|
def bp_step(step, *args)
|
16
|
-
log_step_with_indent(step) {
|
16
|
+
log_step_with_indent(step) { send(step, *args) }
|
17
17
|
end
|
18
18
|
end
|
19
19
|
end
|
@@ -12,7 +12,7 @@ module Dapp
|
|
12
12
|
project_dangling_images_flush
|
13
13
|
remove_images_by_query([
|
14
14
|
'docker images',
|
15
|
-
%(--format '{{if ne "#{stage_cache}" .Repository }}{{.
|
15
|
+
%(--format '{{if ne "#{stage_cache}" .Repository }}{{.Repository}}:{{.Tag}}{{ end }}'),
|
16
16
|
%(-f "label=dapp=#{stage_dapp_label}")
|
17
17
|
].join(' ')) # FIXME: negative filter is not currently supported by the Docker CLI
|
18
18
|
end
|
@@ -7,8 +7,12 @@ module Dapp
|
|
7
7
|
module Common
|
8
8
|
protected
|
9
9
|
|
10
|
-
def
|
11
|
-
shellout!(%(docker images --format="{{.Repository}}:{{.Tag}}" #{stage_cache})).stdout.strip
|
10
|
+
def project_images_names
|
11
|
+
shellout!(%(docker images --format="{{.Repository}}:{{.Tag}}" #{stage_cache})).stdout.lines.map(&:strip)
|
12
|
+
end
|
13
|
+
|
14
|
+
def project_images_ids
|
15
|
+
shellout!(%(docker images #{stage_cache} -q --no-trunc)).stdout.lines.map(&:strip)
|
12
16
|
end
|
13
17
|
|
14
18
|
def project_containers_flush
|
@@ -24,7 +28,7 @@ module Dapp
|
|
24
28
|
end
|
25
29
|
|
26
30
|
def remove_images(ids, force: false)
|
27
|
-
remove_base('docker rmi%{force_option} %{ids}', ids, force: force)
|
31
|
+
remove_base('docker rmi%{force_option} %{ids}', ids.uniq, force: force)
|
28
32
|
end
|
29
33
|
|
30
34
|
def remove_containers_by_query(containers_query, force: false)
|
@@ -32,7 +36,7 @@ module Dapp
|
|
32
36
|
end
|
33
37
|
|
34
38
|
def remove_containers(ids, force: false)
|
35
|
-
remove_base('docker rm%{force_option} %{ids}', ids, force: force)
|
39
|
+
remove_base('docker rm%{force_option} %{ids}', ids.uniq, force: force)
|
36
40
|
end
|
37
41
|
|
38
42
|
def remove_base(query_format, ids, force: false)
|
@@ -46,6 +50,10 @@ module Dapp
|
|
46
50
|
yield(res)
|
47
51
|
end
|
48
52
|
|
53
|
+
def image_labels(image_id)
|
54
|
+
Image::Stage.image_config_option(image_id: image_id, option: 'labels')
|
55
|
+
end
|
56
|
+
|
49
57
|
def run_command(cmd)
|
50
58
|
log(cmd) if log_verbose? || dry_run?
|
51
59
|
shellout!(cmd) unless dry_run?
|
@@ -14,7 +14,7 @@ module Dapp
|
|
14
14
|
elsif proper_cache_version?
|
15
15
|
log_proper_cache do
|
16
16
|
proper_cache_images = proper_cache_all_images
|
17
|
-
remove_images(dapp_images_by_label('dapp').
|
17
|
+
remove_images(dapp_images_by_label('dapp').select { |id| !proper_cache_images.include?(id) }.map(&:strip))
|
18
18
|
end
|
19
19
|
else
|
20
20
|
raise Error::Project, code: :mrproper_required_option
|
@@ -26,7 +26,7 @@ module Dapp
|
|
26
26
|
|
27
27
|
def flush_by_label(label)
|
28
28
|
log_step_with_indent(:containers) { dapp_containers_flush_by_label(label) }
|
29
|
-
log_step_with_indent(:images) {
|
29
|
+
log_step_with_indent(:images) { dapp_images_flush_by_label(label) }
|
30
30
|
end
|
31
31
|
|
32
32
|
def proper_all?
|
@@ -45,13 +45,13 @@ module Dapp
|
|
45
45
|
remove_images_by_query(%(docker images -f "dangling=true" -f "label=#{label}" -q), force: true)
|
46
46
|
end
|
47
47
|
|
48
|
-
def
|
48
|
+
def dapp_images_flush_by_label(label)
|
49
49
|
dapp_dangling_images_flush_by_label(label)
|
50
|
-
remove_images(dapp_images_by_label(label)
|
50
|
+
remove_images(dapp_images_by_label(label), force: true)
|
51
51
|
end
|
52
52
|
|
53
53
|
def dapp_images_by_label(label)
|
54
|
-
@dapp_images ||= shellout!(%(docker images -f "dangling=false" --format="{{.Repository}}:{{.Tag}}" -f "label=#{label}")).stdout.strip
|
54
|
+
@dapp_images ||= shellout!(%(docker images -f "dangling=false" --format="{{.Repository}}:{{.Tag}}" -f "label=#{label}")).stdout.lines.map(&:strip)
|
55
55
|
end
|
56
56
|
|
57
57
|
def proper_cache_all_images
|
@@ -59,7 +59,7 @@ module Dapp
|
|
59
59
|
'docker images',
|
60
60
|
'--format="{{.Repository}}:{{.Tag}}"',
|
61
61
|
%(-f "label=dapp-cache-version=#{Dapp::BUILD_CACHE_VERSION}" -f "dangling=false")
|
62
|
-
].join(' ')).stdout.strip
|
62
|
+
].join(' ')).stdout.lines.map(&:strip)
|
63
63
|
end
|
64
64
|
end
|
65
65
|
end
|
@@ -8,43 +8,46 @@ module Dapp
|
|
8
8
|
module CleanupLocal
|
9
9
|
def stages_cleanup_local(repo)
|
10
10
|
lock_repo(repo, readonly: true) do
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
11
|
+
raise Error::Project, code: :stages_cleanup_required_option unless stages_cleanup_option?
|
12
|
+
|
13
|
+
project_containers_flush
|
14
|
+
|
15
|
+
proper_cache if proper_cache_version?
|
16
|
+
stages_cleanup_by_repo(repo) if proper_repo_cache?
|
17
|
+
proper_git_commit if proper_git_commit?
|
15
18
|
end
|
16
19
|
end
|
17
20
|
|
18
21
|
protected
|
19
22
|
|
20
|
-
def
|
23
|
+
def proper_cache
|
24
|
+
log_proper_cache do
|
25
|
+
lock("#{name}.images") do
|
26
|
+
log_step_with_indent(name) do
|
27
|
+
remove_images(project_images_names.select { |image_name| !actual_cache_images.include?(image_name) })
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def stages_cleanup_by_repo(repo)
|
34
|
+
registry = registry(repo)
|
35
|
+
repo_dimgs = repo_dimgs_images(registry)
|
36
|
+
|
21
37
|
lock("#{name}.images") do
|
22
38
|
log_step_with_indent(name) do
|
23
|
-
project_containers_flush
|
24
39
|
project_dangling_images_flush
|
25
40
|
dimgs, stages = project_images_hash.partition { |_, image_id| repo_dimgs.values.include?(image_id) }
|
26
41
|
dimgs = dimgs.to_h
|
27
42
|
stages = stages.to_h
|
28
|
-
dimgs.each { |_, aiid|
|
43
|
+
dimgs.each { |_, aiid| except_image_with_parents(aiid, stages) }
|
29
44
|
remove_images(stages.keys)
|
30
45
|
end
|
31
46
|
end
|
32
47
|
end
|
33
48
|
|
34
49
|
def repo_dimgs_images(registry)
|
35
|
-
|
36
|
-
end
|
37
|
-
|
38
|
-
def proper_cache
|
39
|
-
log_proper_cache do
|
40
|
-
lock("#{name}.images") do
|
41
|
-
log_step_with_indent(name) do
|
42
|
-
project_containers_flush
|
43
|
-
actual_cache_images = actual_cache_images
|
44
|
-
remove_images(project_images.lines.select { |id| !actual_cache_images.lines.include?(id) }.map(&:strip))
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
50
|
+
repo_dimgs_and_cache(registry).first
|
48
51
|
end
|
49
52
|
|
50
53
|
def actual_cache_images
|
@@ -53,7 +56,7 @@ module Dapp
|
|
53
56
|
'--format="{{.Repository}}:{{.Tag}}"',
|
54
57
|
%(-f "label=dapp-cache-version=#{Dapp::BUILD_CACHE_VERSION}"),
|
55
58
|
stage_cache
|
56
|
-
].join(' ')).stdout.strip
|
59
|
+
].join(' ')).stdout.lines.map(&:strip)
|
57
60
|
end
|
58
61
|
|
59
62
|
def project_images_hash
|
@@ -62,9 +65,9 @@ module Dapp
|
|
62
65
|
end.to_h
|
63
66
|
end
|
64
67
|
|
65
|
-
def
|
68
|
+
def except_image_with_parents(image_id, stages)
|
66
69
|
if image_exist?(image_id)
|
67
|
-
image_dapp_artifacts_label(image_id).each { |aiid|
|
70
|
+
image_dapp_artifacts_label(image_id).each { |aiid| except_image_with_parents(aiid, stages) }
|
68
71
|
iid = image_id
|
69
72
|
loop do
|
70
73
|
stages.delete_if { |_, siid| siid == iid }
|
@@ -89,6 +92,46 @@ module Dapp
|
|
89
92
|
def image_parent(image_id)
|
90
93
|
shellout!(%(docker inspect -f {{.Parent}} #{image_id})).stdout.strip
|
91
94
|
end
|
95
|
+
|
96
|
+
def proper_git_commit
|
97
|
+
log_proper_git_commit do
|
98
|
+
unproper_images = []
|
99
|
+
project_images_detailed.each do |_, attrs|
|
100
|
+
attrs['Config']['Labels'].each do |repo_name, commit|
|
101
|
+
next if (repo = project_git_repositories[repo_name]).nil?
|
102
|
+
git = repo.name == 'own' ? :git : :git_bare
|
103
|
+
unproper_images.concat(image_hierarchy(attrs['Id'])) unless repo.send(git).exists?(commit)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
remove_images(unproper_images.uniq)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
def project_images_detailed
|
111
|
+
@project_images_detailed ||= {}.tap do |images|
|
112
|
+
project_images_names.each do |image_name|
|
113
|
+
shellout!(%(docker inspect --format='{{json .}}' #{image_name})).stdout.strip.tap do |output|
|
114
|
+
images[image_name] = output == 'null' ? {} : JSON.parse(output)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def image_hierarchy(image_id)
|
121
|
+
hierarchy = []
|
122
|
+
iids = [image_id]
|
123
|
+
|
124
|
+
loop do
|
125
|
+
hierarchy.concat(iids)
|
126
|
+
break if begin
|
127
|
+
iids.map! do |iid|
|
128
|
+
project_images_detailed.map { |_, attrs| attrs['Id'] if attrs['Parent'] == iid }.compact
|
129
|
+
end.flatten!.empty?
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
hierarchy
|
134
|
+
end
|
92
135
|
end
|
93
136
|
end
|
94
137
|
end
|
@@ -8,19 +8,37 @@ module Dapp
|
|
8
8
|
module CleanupRepo
|
9
9
|
def stages_cleanup_repo(repo)
|
10
10
|
lock_repo(repo) do
|
11
|
+
raise Error::Project, code: :stages_cleanup_required_option unless stages_cleanup_option?
|
12
|
+
|
11
13
|
registry = registry(repo)
|
12
|
-
repo_dimgs, repo_stages =
|
14
|
+
repo_dimgs, repo_stages = repo_dimgs_and_cache(registry)
|
13
15
|
repo_stages.delete_if { |_, siid| repo_dimgs.values.include?(siid) } # ignoring stages with dimgs ids (v2)
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
end
|
16
|
+
|
17
|
+
proper_repo_cache(registry, repo_stages) if proper_cache_version?
|
18
|
+
repo_stages_cleanup(registry, repo_dimgs, repo_stages) if proper_repo_cache?
|
19
|
+
proper_repo_git_commit(registry) if proper_git_commit?
|
19
20
|
end
|
20
21
|
end
|
21
22
|
|
22
23
|
protected
|
23
24
|
|
25
|
+
def proper_repo_cache(registry, repo_stages)
|
26
|
+
log_proper_cache do
|
27
|
+
wrong_cache_images = repo_stages.select do |image_tag, _|
|
28
|
+
repo_image_dapp_cache_version_label(registry, image_tag) != Dapp::BUILD_CACHE_VERSION.to_s
|
29
|
+
end
|
30
|
+
wrong_cache_images.each { |image_tag, _| delete_repo_image(registry, image_tag) }
|
31
|
+
repo_stages.delete_if { |image_tag, _| wrong_cache_images.keys.include?(image_tag) }
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def repo_stages_cleanup(registry, repo_dimgs, repo_stages)
|
36
|
+
log_step_with_indent(repo) do
|
37
|
+
repo_dimgs.each { |image_tag, image_id| clear_repo_stages(registry, repo_stages, image_tag, image_id) }
|
38
|
+
repo_stages.keys.each { |image_tag| delete_repo_image(registry, image_tag) }
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
24
42
|
def clear_repo_stages(registry, repo_stages, image_tag, image_id)
|
25
43
|
repo_image_dapp_artifacts_labels(registry, image_tag).each do |iid|
|
26
44
|
itag = find_image_tag_by_id(repo_stages, iid)
|
@@ -49,13 +67,54 @@ module Dapp
|
|
49
67
|
nil
|
50
68
|
end
|
51
69
|
|
52
|
-
def
|
53
|
-
|
54
|
-
|
55
|
-
|
70
|
+
def proper_repo_git_commit(registry)
|
71
|
+
log_proper_git_commit do
|
72
|
+
unproper_images = []
|
73
|
+
repo_project_images_detailed(registry).each do |_, attrs|
|
74
|
+
attrs[:labels].each do |repo_name, commit|
|
75
|
+
next if (repo = project_git_repositories[repo_name]).nil?
|
76
|
+
git = repo.name == 'own' ? :git : :git_bare
|
77
|
+
unproper_images.concat(repo_image_tags_hierarchy(registry, attrs[:id])) unless repo.send(git).exists?(commit)
|
78
|
+
end
|
56
79
|
end
|
57
|
-
|
58
|
-
|
80
|
+
remove_repo_images(registry, unproper_images.uniq)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def repo_project_images_detailed(registry)
|
85
|
+
@repo_project_images_detailed ||= begin
|
86
|
+
registry.tags.map do |tag|
|
87
|
+
image_history = registry.image_history(tag)
|
88
|
+
attrs = {
|
89
|
+
id: registry.image_id(tag),
|
90
|
+
parent: image_history['container_config']['Image'],
|
91
|
+
labels: image_history['config']['Labels']
|
92
|
+
}
|
93
|
+
[tag, attrs]
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def repo_image_tags_hierarchy(registry, registry_image_id)
|
99
|
+
hierarchy = []
|
100
|
+
iids = [registry_image_id]
|
101
|
+
|
102
|
+
loop do
|
103
|
+
hierarchy.concat(iids)
|
104
|
+
break if begin
|
105
|
+
iids.map! do |iid|
|
106
|
+
repo_project_images_detailed(registry).map { |_, attrs| attrs[:id] if attrs[:parent] == iid }.compact
|
107
|
+
end.flatten!.empty?
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
repo_project_images_detailed(registry).map { |tag, attrs| tag if hierarchy.include? attrs[:id] }.compact
|
112
|
+
end
|
113
|
+
|
114
|
+
def remove_repo_images(registry, tags)
|
115
|
+
tags.each do |tag|
|
116
|
+
log(tag) if dry_run? || log_verbose?
|
117
|
+
registry.image_delete(tag) unless dry_run?
|
59
118
|
end
|
60
119
|
end
|
61
120
|
end
|
@@ -13,7 +13,7 @@ module Dapp
|
|
13
13
|
DockerRegistry.new(repo)
|
14
14
|
end
|
15
15
|
|
16
|
-
def
|
16
|
+
def repo_dimgs_and_cache(registry)
|
17
17
|
format = proc do |arr|
|
18
18
|
arr.map do |tag|
|
19
19
|
if (id = registry.image_id(tag)).nil?
|
@@ -30,8 +30,8 @@ module Dapp
|
|
30
30
|
def registry_tags(registry)
|
31
31
|
registry.tags
|
32
32
|
rescue Exception::Registry => e
|
33
|
-
raise unless e.net_status[:code] == :
|
34
|
-
log_warning(desc: { code:
|
33
|
+
raise unless e.net_status[:code] == :no_such_dimg
|
34
|
+
log_warning(desc: { code: :dimg_not_found_in_registry })
|
35
35
|
[]
|
36
36
|
end
|
37
37
|
|
@@ -47,6 +47,35 @@ module Dapp
|
|
47
47
|
hash.select { |k, _v| k.start_with?('dapp-artifact') }.values
|
48
48
|
end
|
49
49
|
|
50
|
+
def project_git_repositories
|
51
|
+
@project_git_repositories ||= begin
|
52
|
+
{}.tap do |repositories|
|
53
|
+
dimgs = build_configs.map { |config| Dimg.new(config: config, project: self, ignore_git_fetch: true) }
|
54
|
+
dimgs.each do |dimg|
|
55
|
+
[dimg, dimg.artifacts].flatten
|
56
|
+
.map(&:git_artifacts).flatten
|
57
|
+
.map { |ga_artifact| repositories[ga_artifact.full_name] = ga_artifact.repo }
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def proper_repo_cache?
|
64
|
+
!!cli_options[:proper_repo_cache]
|
65
|
+
end
|
66
|
+
|
67
|
+
def proper_git_commit?
|
68
|
+
!!cli_options[:proper_git_commit]
|
69
|
+
end
|
70
|
+
|
71
|
+
def stages_cleanup_option?
|
72
|
+
proper_git_commit? || proper_cache_version? || proper_repo_cache?
|
73
|
+
end
|
74
|
+
|
75
|
+
def log_proper_git_commit(&blk)
|
76
|
+
log_step_with_indent(:'proper git commit', &blk)
|
77
|
+
end
|
78
|
+
|
50
79
|
def lock_repo(repo, *args, &blk)
|
51
80
|
lock("repo.#{hashsum repo}", *args, &blk)
|
52
81
|
end
|
@@ -10,7 +10,7 @@ module Dapp
|
|
10
10
|
lock_repo(repo) do
|
11
11
|
log_step_with_indent(repo) do
|
12
12
|
registry = registry(repo)
|
13
|
-
repo_dimgs, repo_stages =
|
13
|
+
repo_dimgs, repo_stages = repo_dimgs_and_cache(registry)
|
14
14
|
repo_dimgs.merge(repo_stages).keys.each { |image_tag| delete_repo_image(registry, image_tag) }
|
15
15
|
end
|
16
16
|
end
|
@@ -62,9 +62,11 @@ module Dapp
|
|
62
62
|
|
63
63
|
def log_warning(*args, **kwargs)
|
64
64
|
kwargs[:style] = :warning
|
65
|
-
kwargs[:desc] ||= {}
|
66
|
-
kwargs[:desc][:context] ||= :warning
|
67
65
|
kwargs[:stream] ||= $stderr
|
66
|
+
if args.empty?
|
67
|
+
kwargs[:desc] ||= {}
|
68
|
+
kwargs[:desc][:context] ||= :warning
|
69
|
+
end
|
68
70
|
log(*args, **kwargs)
|
69
71
|
end
|
70
72
|
|
data/lib/dapp/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dapp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.17
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dmitry Stolyarov
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-12-
|
11
|
+
date: 2016-12-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mixlib-shellout
|
@@ -138,6 +138,20 @@ dependencies:
|
|
138
138
|
- - "~>"
|
139
139
|
- !ruby/object:Gem::Version
|
140
140
|
version: 3.0.0
|
141
|
+
- !ruby/object:Gem::Dependency
|
142
|
+
name: rugged
|
143
|
+
requirement: !ruby/object:Gem::Requirement
|
144
|
+
requirements:
|
145
|
+
- - "~>"
|
146
|
+
- !ruby/object:Gem::Version
|
147
|
+
version: 0.24.0
|
148
|
+
type: :runtime
|
149
|
+
prerelease: false
|
150
|
+
version_requirements: !ruby/object:Gem::Requirement
|
151
|
+
requirements:
|
152
|
+
- - "~>"
|
153
|
+
- !ruby/object:Gem::Version
|
154
|
+
version: 0.24.0
|
141
155
|
- !ruby/object:Gem::Dependency
|
142
156
|
name: bundler
|
143
157
|
requirement: !ruby/object:Gem::Requirement
|
@@ -460,6 +474,7 @@ files:
|
|
460
474
|
- lib/dapp/error/image.rb
|
461
475
|
- lib/dapp/error/project.rb
|
462
476
|
- lib/dapp/error/registry.rb
|
477
|
+
- lib/dapp/error/rugged.rb
|
463
478
|
- lib/dapp/error/shellout.rb
|
464
479
|
- lib/dapp/exception/base.rb
|
465
480
|
- lib/dapp/exception/introspect_image.rb
|