dapp 0.35.17 → 0.35.18
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/dapp/dapp/ruby2go.rb +4 -0
- data/lib/dapp/dapp/slug.rb +4 -71
- data/lib/dapp/dimg/dapp/command/cleanup_repo.rb +12 -159
- data/lib/dapp/dimg/dapp/command/flush_local.rb +3 -3
- data/lib/dapp/dimg/dapp/command/ruby2go_cleanup.rb +3 -4
- data/lib/dapp/dimg/dapp/command/stages/flush_local.rb +1 -1
- data/lib/dapp/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7e9352f864e0861e11219e81099f33c917de38e6f678360d7a6a85c503c8c99c
|
4
|
+
data.tar.gz: 463867abfb9f02c469ba3ad1b386bf6fdb8d4d14c552d4138b2f1065e2952722
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 48bb9636eccebbd3548b7ec1f1aead9da7c1f0a537935c8c6763b1a3f8d698cd46dfa488d4e5ffc8a193680ea3ac282612764c6862466a75e802fbd5b9428758
|
7
|
+
data.tar.gz: e6a1c8c23fa3ecb145eea35198b8df9ad3872964867c429db3ac7d765b9b9c54a13fc150227f616a29b005caa1e975473d3355fc55b4bb58d99f7a4ae14d3076
|
data/lib/dapp/dapp/ruby2go.rb
CHANGED
data/lib/dapp/dapp/slug.rb
CHANGED
@@ -1,77 +1,10 @@
|
|
1
1
|
module Dapp
|
2
2
|
class Dapp
|
3
3
|
module Slug
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
if ENV['DAPP_SLUG_V3']
|
9
|
-
base.include(V3)
|
10
|
-
else
|
11
|
-
base.include(V1V2)
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
module V1V2
|
16
|
-
def consistent_uniq_slugify(s)
|
17
|
-
return s unless should_be_slugged?(s)
|
18
|
-
consistent_uniq_slug_reg =~ s.tr('/', '-').slugify.squeeze('--')
|
19
|
-
consistent_uniq_slug = Regexp.last_match(1)
|
20
|
-
murmur_hash = MurmurHash3::V32.str_hexdigest(s)
|
21
|
-
[].tap do |slug|
|
22
|
-
slug << begin
|
23
|
-
unless consistent_uniq_slug.nil?
|
24
|
-
index = ENV['DAPP_SLUG_V2'] ? SLUG_V2_LIMIT_LENGTH - murmur_hash.length - SLUG_SEPARATOR.length - 1 : -1
|
25
|
-
consistent_uniq_slug[0..index]
|
26
|
-
end
|
27
|
-
end
|
28
|
-
slug << murmur_hash
|
29
|
-
end.compact.join(SLUG_SEPARATOR)
|
30
|
-
end
|
31
|
-
|
32
|
-
def should_be_slugged?(s)
|
33
|
-
!(/^#{consistent_uniq_slug_reg}$/ =~ s)
|
34
|
-
end
|
35
|
-
|
36
|
-
def consistent_uniq_slug_reg
|
37
|
-
/(?!-)((-?[a-z0-9]+)+)(?<!-)/
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
module V3
|
42
|
-
def consistent_uniq_slugify(s)
|
43
|
-
return s unless should_be_slugged?(s)
|
44
|
-
slug = consistent_uniq_slug(s)
|
45
|
-
murmur_hash = MurmurHash3::V32.str_hexdigest(s)
|
46
|
-
[].tap do |res|
|
47
|
-
res << begin
|
48
|
-
unless slug.empty?
|
49
|
-
index = SLUG_V2_LIMIT_LENGTH - murmur_hash.length - SLUG_SEPARATOR.length - 1
|
50
|
-
slug[0..index]
|
51
|
-
end
|
52
|
-
end
|
53
|
-
res << murmur_hash
|
54
|
-
end.compact.join(SLUG_SEPARATOR)
|
55
|
-
end
|
56
|
-
|
57
|
-
def should_be_slugged?(s)
|
58
|
-
consistent_uniq_slug(s) != s || s.length > SLUG_V2_LIMIT_LENGTH
|
59
|
-
end
|
60
|
-
|
61
|
-
def consistent_uniq_slug(s)
|
62
|
-
''.tap do |res|
|
63
|
-
status = :empty
|
64
|
-
s.to_s.chars.each do |ch|
|
65
|
-
next if (s_ch = ch.slugify).empty?
|
66
|
-
|
67
|
-
if s_ch !~ /[[:punct:]|[:blank:]]/
|
68
|
-
res << s_ch
|
69
|
-
status = :non_empty if status == :empty
|
70
|
-
elsif status == :non_empty && res[-1] != '-'
|
71
|
-
res << '-'
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end.chomp('-')
|
4
|
+
def consistent_uniq_slugify(s)
|
5
|
+
ruby2go_slug({ options: { data: s } }).tap do |res|
|
6
|
+
raise Error::Build, code: :ruby2go_slug_failed_unexpected_error, data: { message: res["error"] } unless res["error"].nil?
|
7
|
+
break res['data']
|
75
8
|
end
|
76
9
|
end
|
77
10
|
end # Slug
|
@@ -3,167 +3,20 @@ module Dapp
|
|
3
3
|
module Dapp
|
4
4
|
module Command
|
5
5
|
module CleanupRepo
|
6
|
-
GIT_TAGS_LIMIT_POLICY = 10
|
7
|
-
EXPIRY_DATE_PERIOD_POLICY = 60 * 60 * 24 * 30
|
8
|
-
GIT_COMMITS_LIMIT_POLICY = 50
|
9
|
-
GIT_COMMITS_EXPIRY_DATE_PERIOD_POLICY = 60 * 60 * 24 * 30
|
10
|
-
|
11
6
|
def cleanup_repo
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
cleanup_repo_by_policies(registry, actual_detailed_dimgs_images_by_scheme(registry))
|
25
|
-
end
|
26
|
-
|
27
|
-
repo_dimgstages_cleanup(registry, repo_dimgs, repo_dimgstages) if with_stages?
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
def actual_detailed_dimgs_images_by_scheme(registry)
|
33
|
-
{}.tap do |detailed_dimgs_images_by_scheme|
|
34
|
-
tagging_schemes.each { |scheme| detailed_dimgs_images_by_scheme[scheme] = [] }
|
35
|
-
repo_detailed_dimgs_images(registry).each do |image|
|
36
|
-
image_repository = [option_repo, image[:dimg]].compact.join('/')
|
37
|
-
image_name = [image_repository, image[:tag]].join(':')
|
38
|
-
|
39
|
-
should_be_ignored = deployed_docker_images.include?(image_name)
|
40
|
-
|
41
|
-
if should_be_ignored
|
42
|
-
log "Keep in repo image that is being used in kubernetes: #{image_name}"
|
43
|
-
next
|
44
|
-
end
|
45
|
-
|
46
|
-
(detailed_dimgs_images_by_scheme[image[:labels]['dapp-tag-scheme']] ||= []) << image
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
def cleanup_repo_by_nonexistent_git_primitive(registry, detailed_dimgs_images_by_scheme)
|
52
|
-
%w(git_tag git_branch git_commit).each do |scheme|
|
53
|
-
cleanup_repo_by_nonexistent_git_base(registry, detailed_dimgs_images_by_scheme, scheme) do |detailed_dimg_image|
|
54
|
-
case scheme
|
55
|
-
when 'git_tag' then consistent_git_tags.include?(detailed_dimg_image[:tag])
|
56
|
-
when 'git_branch' then consistent_git_remote_branches.include?(detailed_dimg_image[:tag])
|
57
|
-
when 'git_commit' then git_own_repo.commit_exists?(detailed_dimg_image[:tag])
|
58
|
-
else
|
59
|
-
raise
|
60
|
-
end
|
61
|
-
end unless detailed_dimgs_images_by_scheme[scheme].empty?
|
62
|
-
end
|
63
|
-
end
|
64
|
-
|
65
|
-
def consistent_git_tags
|
66
|
-
git_tag_by_consistent_tag_name.keys
|
67
|
-
end
|
68
|
-
|
69
|
-
def consistent_git_remote_branches
|
70
|
-
@consistent_git_remote_branches ||= git_own_repo.remote_branches.map(&method(:consistent_uniq_slugify))
|
71
|
-
end
|
72
|
-
|
73
|
-
def cleanup_repo_by_nonexistent_git_base(registry, repo_dimgs_images_by_scheme, dapp_tag_scheme)
|
74
|
-
nonexist_repo_images = repo_dimgs_images_by_scheme[dapp_tag_scheme]
|
75
|
-
.select { |dimg_image| dimg_image[:labels]['dapp-tag-scheme'] == dapp_tag_scheme }
|
76
|
-
.select { |dimg_image| !(yield dimg_image) }
|
77
|
-
|
78
|
-
log_step_with_indent(:"#{dapp_tag_scheme.split('_').join(' ')} nonexistent") do
|
79
|
-
nonexist_repo_images.each { |repo_image| delete_repo_image(registry, repo_image) }
|
80
|
-
end unless nonexist_repo_images.empty?
|
81
|
-
end
|
82
|
-
|
83
|
-
def cleanup_repo_by_policies(registry, detailed_dimgs_images_by_scheme)
|
84
|
-
cleanup_repo_tags_by_policies(registry, detailed_dimgs_images_by_scheme)
|
85
|
-
cleanup_repo_commits_by_policies(registry, detailed_dimgs_images_by_scheme)
|
86
|
-
end
|
87
|
-
|
88
|
-
def cleanup_repo_tags_by_policies(registry, detailed_dimgs_images_by_scheme)
|
89
|
-
cleanup_repo_by_policies_base(
|
90
|
-
registry,
|
91
|
-
detailed_dimgs_images_by_scheme['git_tag'].select { |dimg| consistent_git_tags.include?(dimg[:tag]) },
|
92
|
-
expiry_date_policy: git_tags_expiry_date_policy,
|
93
|
-
limit_policy: git_tags_limit_policy,
|
94
|
-
log_primitive: 'tag'
|
95
|
-
)
|
96
|
-
end
|
97
|
-
|
98
|
-
def cleanup_repo_commits_by_policies(registry, detailed_dimgs_images_by_scheme)
|
99
|
-
cleanup_repo_by_policies_base(
|
100
|
-
registry,
|
101
|
-
detailed_dimgs_images_by_scheme['git_commit'].select { |dimg| git_own_repo.commit_exists?(dimg[:tag]) },
|
102
|
-
expiry_date_policy: git_commits_expiry_date_policy,
|
103
|
-
limit_policy: git_commits_limit_policy,
|
104
|
-
log_primitive: 'commit'
|
105
|
-
)
|
106
|
-
end
|
107
|
-
|
108
|
-
def cleanup_repo_by_policies_base(registry, detailed_dimgs_images, expiry_date_policy:, limit_policy:, log_primitive:)
|
109
|
-
sorted_detailed_dimgs_images = detailed_dimgs_images.sort_by { |dimg| dimg[:created_at] }.reverse
|
110
|
-
|
111
|
-
expired_dimgs_images, not_expired_dimgs_images = sorted_detailed_dimgs_images.partition do |dimg_image|
|
112
|
-
dimg_image[:created_at] < expiry_date_policy
|
7
|
+
git_repo_option = git_own_repo_exist? ? JSON.dump(git_own_repo.get_ruby2go_state_hash) : nil
|
8
|
+
ruby2go_cleanup_command(:gc, ruby2go_cleanup_gc_options_dump, local_repo: git_repo_option)
|
9
|
+
end
|
10
|
+
|
11
|
+
def ruby2go_cleanup_gc_options_dump
|
12
|
+
ruby2go_cleanup_common_repo_options.merge(
|
13
|
+
mode: {
|
14
|
+
with_stages: with_stages?
|
15
|
+
},
|
16
|
+
deployed_docker_images: deployed_docker_images
|
17
|
+
).tap do |data|
|
18
|
+
break JSON.dump(data)
|
113
19
|
end
|
114
|
-
|
115
|
-
log_step_with_indent(:"git #{log_primitive} date policy (before #{DateTime.strptime(expiry_date_policy.to_s, '%s')})") do
|
116
|
-
expired_dimgs_images.each { |dimg| delete_repo_image(registry, dimg) }
|
117
|
-
end unless expired_dimgs_images.empty?
|
118
|
-
|
119
|
-
not_expired_dimgs_images
|
120
|
-
.each_with_object({}) { |dimg, images_by_dimg| (images_by_dimg[dimg[:dimg]] ||= []) << dimg }
|
121
|
-
.each do |dimg_name, images|
|
122
|
-
next if images[limit_policy..-1].nil?
|
123
|
-
log_step_with_indent(:"git #{log_primitive} limit policy (> #{limit_policy}) (`#{dimg_name || 'nameless'}` dimg)") do
|
124
|
-
images[limit_policy..-1].each { |dimg| delete_repo_image(registry, dimg) }
|
125
|
-
end
|
126
|
-
end
|
127
|
-
end
|
128
|
-
|
129
|
-
def git_tags_expiry_date_policy
|
130
|
-
@git_tags_expiry_date_policy ||= expiry_date_policy_value('EXPIRY_DATE_PERIOD_POLICY', default: EXPIRY_DATE_PERIOD_POLICY)
|
131
|
-
end
|
132
|
-
|
133
|
-
def git_tags_limit_policy
|
134
|
-
@git_tags_limit_policy ||= policy_value('GIT_TAGS_LIMIT_POLICY', default: GIT_TAGS_LIMIT_POLICY)
|
135
|
-
end
|
136
|
-
|
137
|
-
def git_commits_expiry_date_policy
|
138
|
-
@git_commits_expiry_date_policy ||= expiry_date_policy_value('GIT_COMMITS_EXPIRY_DATE_PERIOD_POLICY', default: GIT_COMMITS_EXPIRY_DATE_PERIOD_POLICY)
|
139
|
-
end
|
140
|
-
|
141
|
-
def git_commits_limit_policy
|
142
|
-
@git_commits_limit_policy ||= policy_value('GIT_COMMITS_LIMIT_POLICY', default: GIT_COMMITS_LIMIT_POLICY)
|
143
|
-
end
|
144
|
-
|
145
|
-
def expiry_date_policy_value(env_key, default:)
|
146
|
-
expiry_date_period_policy = policy_value(env_key, default: default)
|
147
|
-
Time.now.to_i - expiry_date_period_policy
|
148
|
-
end
|
149
|
-
|
150
|
-
def policy_value(env_key, default:)
|
151
|
-
return default if (val = ENV[env_key]).nil?
|
152
|
-
|
153
|
-
if val.to_i.to_s == val
|
154
|
-
val.to_i
|
155
|
-
else
|
156
|
-
log_warning("WARNING: `#{env_key}` value `#{val}` is ignored (using default value `#{default}`)!")
|
157
|
-
default
|
158
|
-
end
|
159
|
-
end
|
160
|
-
|
161
|
-
def git_tag_by_consistent_git_tag(consistent_git_tag)
|
162
|
-
git_tag_by_consistent_tag_name[consistent_git_tag]
|
163
|
-
end
|
164
|
-
|
165
|
-
def git_tag_by_consistent_tag_name
|
166
|
-
@git_consistent_tags ||= git_own_repo.tags.map { |t| [consistent_uniq_slugify(t), t] }.to_h
|
167
20
|
end
|
168
21
|
|
169
22
|
def deployed_docker_images
|
@@ -8,14 +8,14 @@ module Dapp
|
|
8
8
|
end
|
9
9
|
|
10
10
|
def ruby2go_cleanup_flush_local_options_dump
|
11
|
-
ruby2go_cleanup_common_project_options
|
11
|
+
ruby2go_cleanup_common_project_options.merge(
|
12
12
|
mode: {
|
13
13
|
with_dimgs: true,
|
14
14
|
with_stages: with_stages?,
|
15
15
|
only_repo: false,
|
16
16
|
},
|
17
|
-
).tap do |
|
18
|
-
break JSON.dump(
|
17
|
+
).tap do |data|
|
18
|
+
break JSON.dump(data)
|
19
19
|
end
|
20
20
|
end
|
21
21
|
end
|
@@ -3,11 +3,11 @@ module Dapp
|
|
3
3
|
module Dapp
|
4
4
|
module Command
|
5
5
|
module Ruby2GoCleanup
|
6
|
-
def ruby2go_cleanup_command(command, command_options)
|
6
|
+
def ruby2go_cleanup_command(command, command_options, **options)
|
7
7
|
options = {
|
8
8
|
command: command,
|
9
9
|
command_options: command_options,
|
10
|
-
options: { host_docker_config_dir: self.class.host_docker_config_dir }
|
10
|
+
options: { host_docker_config_dir: self.class.host_docker_config_dir }.merge(options)
|
11
11
|
}
|
12
12
|
|
13
13
|
ruby2go_cleanup(options).tap do |res|
|
@@ -15,13 +15,12 @@ module Dapp
|
|
15
15
|
end
|
16
16
|
end
|
17
17
|
|
18
|
-
def ruby2go_cleanup_common_project_options
|
18
|
+
def ruby2go_cleanup_common_project_options
|
19
19
|
{
|
20
20
|
common_project_options: {
|
21
21
|
project_name: name,
|
22
22
|
common_options: {
|
23
23
|
dry_run: dry_run?,
|
24
|
-
force: force
|
25
24
|
}
|
26
25
|
},
|
27
26
|
}
|
data/lib/dapp/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dapp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.35.
|
4
|
+
version: 0.35.18
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dmitry Stolyarov
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-10-
|
11
|
+
date: 2018-10-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mixlib-shellout
|