environments-list-builder 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +35 -0
- data/.rakeTasks +7 -0
- data/.rspec +1 -0
- data/ChangeLog.md +4 -0
- data/Gemfile +11 -0
- data/Gemfile.lock +187 -0
- data/LICENSE +201 -0
- data/LICENSE.txt +203 -0
- data/README.md +2 -0
- data/Rakefile +36 -0
- data/bin/environmentslistbuilder +34 -0
- data/environments-list-builder.gemspec +37 -0
- data/features/.gitkeep +0 -0
- data/features/manifestrepo-builder.feature +1 -0
- data/features/step_definitions/.gitkeep +0 -0
- data/features/step_definitions/manifestrepo-builder_steps.rb +1 -0
- data/lib/cicd/builder/environments-list.rb +50 -0
- data/lib/cicd/builder/environments-list/mixlib/build.rb +163 -0
- data/lib/cicd/builder/environments-list/mixlib/lib/mixins/no_commands.rb +309 -0
- data/lib/cicd/builder/environments-list/mixlib/lib/update_bucket_policy.rb +452 -0
- data/lib/cicd/builder/environments-list/mixlib/repo.rb +49 -0
- data/lib/cicd/builder/environments-list/mixlib/repo/artifactory.rb +49 -0
- data/lib/cicd/builder/environments-list/version.rb +10 -0
- data/openssl +0 -0
- data/spec/builder_spec.rb +8 -0
- data/spec/spec_helper.rb +4 -0
- data/tests/infinite-etag.rb +166 -0
- metadata +49 -9
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module CiCd
|
4
|
+
module Builder
|
5
|
+
# noinspection RubySuperCallWithoutSuperclassInspection
|
6
|
+
module EnvironmentsList
|
7
|
+
module Repo
|
8
|
+
require 'cicd/builder/mixlib/repo/base'
|
9
|
+
require 'cicd/builder/mixlib/repo/S3'
|
10
|
+
# noinspection RubyResolve
|
11
|
+
if ENV.has_key?('REPO_TYPE') and (not ENV['REPO_TYPE'].capitalize.eql?('S3'))
|
12
|
+
require "cicd/builder/environments-list/mixlib/repo/#{ENV['REPO_TYPE'].downcase}"
|
13
|
+
end
|
14
|
+
|
15
|
+
# ---------------------------------------------------------------------------------------------------------------
|
16
|
+
def getRepoClass(type = nil)
|
17
|
+
@logger.step CLASS+'::'+__method__.to_s
|
18
|
+
if type.nil?
|
19
|
+
type ||= 'S3'
|
20
|
+
if ENV.has_key?('REPO_TYPE')
|
21
|
+
type = ENV['REPO_TYPE']
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
@logger.info "#{type} repo interface"
|
26
|
+
clazz = begin
|
27
|
+
Object.const_get("#{self.class.name.gsub(%r'::\w+$', '')}::Repo::#{type}")
|
28
|
+
rescue NameError => e
|
29
|
+
begin
|
30
|
+
# Object.const_get("#{self.class.name.gsub(%r'::\w+$', '')}::Repo::#{type}")
|
31
|
+
Object.const_get("CiCd::Builder::Manifest::Repo::#{type}")
|
32
|
+
rescue NameError #=> e
|
33
|
+
Object.const_get("CiCd::Builder::Repo::#{type}")
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
if block_given?
|
38
|
+
if clazz.is_a?(Class) and not clazz.nil?
|
39
|
+
yield
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
clazz
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'artifactory'
|
2
|
+
require 'tmpdir'
|
3
|
+
require "cicd/builder/manifest/mixlib/repo/artifactory"
|
4
|
+
|
5
|
+
module CiCd
|
6
|
+
module Builder
|
7
|
+
# noinspection RubySuperCallWithoutSuperclassInspection
|
8
|
+
module EnvironmentsList
|
9
|
+
module Repo
|
10
|
+
class Artifactory < CiCd::Builder::Manifest::Repo::Artifactory
|
11
|
+
|
12
|
+
alias_method :super_uploadToRepo, :uploadToRepo
|
13
|
+
# ---------------------------------------------------------------------------------------------------------------
|
14
|
+
def uploadToRepo(artifacts)
|
15
|
+
@logger.step CLASS+'::'+__method__.to_s
|
16
|
+
# super_uploadToRepo(artifacts) get's the immediate parent class
|
17
|
+
cicd_uploadToRepo(artifacts)
|
18
|
+
# CiCd::Builder::Repo::Artifactory.instance_method(:uploadToRepo).bind(self).call(artifacts)
|
19
|
+
if @vars[:environments][:changed]
|
20
|
+
data = {
|
21
|
+
name: ENV['ARTIFACTORY_ENVIRONMENTS_MODULE'],
|
22
|
+
module: ENV['ARTIFACTORY_ENVIRONMENTS_MODULE'],
|
23
|
+
file: @vars[:environments][:file],
|
24
|
+
version: @vars[:environments][:version],
|
25
|
+
build: @vars[:build_num],
|
26
|
+
properties: @properties_matrix,
|
27
|
+
temp: false,
|
28
|
+
sha1: Digest::SHA1.file(@vars[:environments][:file]).hexdigest,
|
29
|
+
md5: Digest::MD5.file(@vars[:environments][:file]).hexdigest,
|
30
|
+
}
|
31
|
+
|
32
|
+
cicd_maybeUploadArtifactoryObject(
|
33
|
+
data: data,
|
34
|
+
artifact_module: data[:module],
|
35
|
+
artifact_version: data[:version],
|
36
|
+
file_name: '',
|
37
|
+
file_ext: File.extname(data[:file]).gsub(/^\./,''),
|
38
|
+
repo: ENV['ARTIFACTORY_RELEASE_REPO'],
|
39
|
+
copy: false
|
40
|
+
)
|
41
|
+
end
|
42
|
+
@vars[:return_code]
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/openssl
ADDED
File without changes
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,166 @@
|
|
1
|
+
require 'aws-sdk-core'
|
2
|
+
require 'aws-sdk-resources'
|
3
|
+
require 'uri'
|
4
|
+
|
5
|
+
# ---------------------------------------------------------------------------------------------------------------
|
6
|
+
def getS3()
|
7
|
+
region = ENV['AWS_REGION'] || ::Aws.config[:region] || 'us-east-1'
|
8
|
+
unless @s3
|
9
|
+
# noinspection RubyArgCount
|
10
|
+
@s3 = ::Aws::S3::Client.new(region: region)
|
11
|
+
end
|
12
|
+
unless @s3 and ((@s3.config.access_key_id and @s3.config.secret_access_key) or @s3.config.credentials)
|
13
|
+
@logger.warn "Unable to find AWS credentials in standard locations:
|
14
|
+
ENV['AWS_ACCESS_KEY'] and ENV['AWS_SECRET_ACCESS_KEY']
|
15
|
+
Aws.config[:credentials]
|
16
|
+
Shared credentials file, ~/.aws/credentials
|
17
|
+
EC2 Instance profile
|
18
|
+
"
|
19
|
+
if ENV['AWS_PROFILE']
|
20
|
+
@logger.info "Trying profile '#{ENV['AWS_PROFILE']}' explicitly"
|
21
|
+
creds = Aws::SharedCredentials.new( path: File.expand_path('~/.aws/credentials'), profile: ENV['AWS_PROFILE'] )
|
22
|
+
if creds.loadable?
|
23
|
+
# noinspection RubyArgCount
|
24
|
+
@s3 = ::Aws::S3::Client.new(region: region, credentials: creds)
|
25
|
+
end
|
26
|
+
else
|
27
|
+
@logger.warn 'No AWS_PROFILE defined'
|
28
|
+
end
|
29
|
+
end
|
30
|
+
unless @s3 and ((@s3.config.access_key_id and @s3.config.secret_access_key) or @s3.config.credentials)
|
31
|
+
raise 'Unable to find AWS credentials!'
|
32
|
+
end
|
33
|
+
@s3
|
34
|
+
end
|
35
|
+
|
36
|
+
# ---------------------------------------------------------------------------------------------------------------
|
37
|
+
def getBucket(name = nil)
|
38
|
+
@s3 = getS3()
|
39
|
+
begin
|
40
|
+
::Aws::S3::Bucket.new(name: name || ENV['AWS_S3_BUCKET'], client: @s3)
|
41
|
+
rescue Aws::S3::Errors::NotFound
|
42
|
+
@vars[:return_code] = Errors::BUCKET
|
43
|
+
nil
|
44
|
+
rescue Exception => e
|
45
|
+
@logger.error "S3 Bucket resource API error: #{e.class.name} #{e.message}"
|
46
|
+
raise e
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
# ---------------------------------------------------------------------------------------------------------------
|
51
|
+
def getObjects(artifact, path)
|
52
|
+
parts = URI(path).path.gsub(%r'^#{File::SEPARATOR}', '').split(File::SEPARATOR)
|
53
|
+
name = parts.shift
|
54
|
+
bucket = getBucket(name)
|
55
|
+
key = File.join(parts, '')
|
56
|
+
@logger.info "S3://#{name}:#{key} URL: #{path} #{artifact}"
|
57
|
+
objects = []
|
58
|
+
bucket.objects(prefix: key).each do |object|
|
59
|
+
if artifact.empty? or (not artifact.empty? and object.key =~ %r'#{key}#{artifact}')
|
60
|
+
objects << object
|
61
|
+
end
|
62
|
+
end
|
63
|
+
@logger.debug "S3://#{name}:#{key} has #{objects.size} objects"
|
64
|
+
return key, name, objects
|
65
|
+
end
|
66
|
+
|
67
|
+
# ---------------------------------------------------------------------------------------------------------------
|
68
|
+
def calcLocalETag(etag, local, size = nil)
|
69
|
+
if size == nil
|
70
|
+
stat = File.stat(local)
|
71
|
+
size = stat.size
|
72
|
+
end
|
73
|
+
@logger.debug "Calculate etag to match #{etag}"
|
74
|
+
match = etag.match(%r'-(\d+)$')
|
75
|
+
check = if match
|
76
|
+
require 's3etag'
|
77
|
+
parts = match[1].to_i
|
78
|
+
chunk = size.to_f / parts.to_f
|
79
|
+
mbs = (chunk.to_f / 1024 /1024 + 0.5).to_i
|
80
|
+
part_size = mbs * 1024 * 1024
|
81
|
+
chkit = S3Etag.calc(file: local, threshold: part_size, min_part_size: part_size, max_parts: parts)
|
82
|
+
@logger.debug "S3Etag Calculated #{chkit} : (#{size} / #{part_size}) <= #{parts}"
|
83
|
+
chunks = size / part_size
|
84
|
+
while chkit != etag and chunks <= parts and chunks > 0 and (size > part_size)
|
85
|
+
# Go one larger if a modulus remains and we have the right number of parts
|
86
|
+
mbs += 1
|
87
|
+
part_size = mbs * 1024 * 1024
|
88
|
+
chunks = size.to_f / part_size
|
89
|
+
chkit = S3Etag.calc(file: local, threshold: part_size, min_part_size: part_size, max_parts: parts)
|
90
|
+
@logger.debug "S3Etag Calculated #{chkit} : (#{size} / #{part_size}) <= #{parts}"
|
91
|
+
end
|
92
|
+
#raise "Unable to match etag #{etag}!" if chkit != etag
|
93
|
+
chkit
|
94
|
+
else
|
95
|
+
Digest::MD5.file(local).hexdigest
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
# ---------------------------------------------------------------------------------------------------------------
|
100
|
+
def shouldDownload?(etag, local, object)
|
101
|
+
if File.exists?(local)
|
102
|
+
@logger.debug "\t\tchecking etag on #{local}"
|
103
|
+
stat = File.stat(local)
|
104
|
+
check = calcLocalETag(etag, local, stat.size)
|
105
|
+
if etag != check or object.size != stat.size or object.last_modified > stat.mtime
|
106
|
+
@logger.debug "\t\t#{etag} != \"#{check}\" #{object.size} != #{stat.size} #{object.last_modified} > #{stat.mtime}"
|
107
|
+
true
|
108
|
+
else
|
109
|
+
@logger.debug "\t\tmatched #{etag}"
|
110
|
+
false
|
111
|
+
end
|
112
|
+
else
|
113
|
+
true
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
# ---------------------------------------------------------------------------------------------------------------
|
118
|
+
def doDownload(etag, local, object)
|
119
|
+
@logger.info "\t\tdownload #{object.size} bytes"
|
120
|
+
response = object.get(:response_target => local)
|
121
|
+
File.utime(response.last_modified, response.last_modified, local)
|
122
|
+
@logger.info "\t\tdone"
|
123
|
+
check = calcLocalETag(etag, local)
|
124
|
+
if check.eql?(etag)
|
125
|
+
false
|
126
|
+
else
|
127
|
+
@logger.info "\tETag different: #{etag} != #{check}"
|
128
|
+
true
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
|
133
|
+
class FakeLogger
|
134
|
+
def initialize
|
135
|
+
|
136
|
+
end
|
137
|
+
|
138
|
+
def method_missing(*args)
|
139
|
+
puts "#{args[0]}: #{args[1..-1].join(' ')}"
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
@logger = FakeLogger.new()
|
144
|
+
artifact, path = ['', 'https://s3.amazonaws.com/wgen-sto-artifacts/release/com/amplify/learning/enrollment/1.3.0-265']
|
145
|
+
local_dir = File.join('/tmp', 'enrollment', '')
|
146
|
+
Dir.mkdir(local_dir, 0700) unless File.directory?(local_dir)
|
147
|
+
artifacts = []
|
148
|
+
|
149
|
+
key, name, objects = getObjects(artifact, path)
|
150
|
+
# 1 or more objects on the key/ path
|
151
|
+
if objects.size > 0
|
152
|
+
objects.each do |object|
|
153
|
+
@logger.info "\tchecking #{object.key}"
|
154
|
+
local = File.join(local_dir, File.basename(object.key))
|
155
|
+
etag = object.etag.gsub(%r/['"]/, '')
|
156
|
+
download = shouldDownload?(etag, local, object)
|
157
|
+
if download
|
158
|
+
changed = doDownload(etag, local, object)
|
159
|
+
else
|
160
|
+
@logger.info "\t\tunchanged"
|
161
|
+
end
|
162
|
+
artifacts << local
|
163
|
+
end
|
164
|
+
else
|
165
|
+
@logger.fatal "Artifact not found: s3://#{name}/#{key}#{artifact}"
|
166
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: environments-list-builder
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Christo De Lange
|
@@ -16,7 +16,7 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0.6.
|
19
|
+
version: 0.6.9
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: '1.1'
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version: 0.6.
|
29
|
+
version: 0.6.9
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '1.1'
|
@@ -161,19 +161,25 @@ dependencies:
|
|
161
161
|
- !ruby/object:Gem::Version
|
162
162
|
version: '0'
|
163
163
|
- !ruby/object:Gem::Dependency
|
164
|
-
name: aws-sdk
|
164
|
+
name: aws-sdk
|
165
165
|
requirement: !ruby/object:Gem::Requirement
|
166
166
|
requirements:
|
167
|
-
- - "
|
167
|
+
- - ">="
|
168
168
|
- !ruby/object:Gem::Version
|
169
169
|
version: '2.0'
|
170
|
+
- - "<"
|
171
|
+
- !ruby/object:Gem::Version
|
172
|
+
version: '2.1'
|
170
173
|
type: :runtime
|
171
174
|
prerelease: false
|
172
175
|
version_requirements: !ruby/object:Gem::Requirement
|
173
176
|
requirements:
|
174
|
-
- - "
|
177
|
+
- - ">="
|
175
178
|
- !ruby/object:Gem::Version
|
176
179
|
version: '2.0'
|
180
|
+
- - "<"
|
181
|
+
- !ruby/object:Gem::Version
|
182
|
+
version: '2.1'
|
177
183
|
- !ruby/object:Gem::Dependency
|
178
184
|
name: dldinternet-mixlib-logging
|
179
185
|
requirement: !ruby/object:Gem::Requirement
|
@@ -291,10 +297,38 @@ dependencies:
|
|
291
297
|
description: Jenkins builder of the environments manifest for Continuous Integration/Continuous
|
292
298
|
Delivery artifact promotion style deployments
|
293
299
|
email: rubygems@dldinternet.com
|
294
|
-
executables:
|
300
|
+
executables:
|
301
|
+
- environmentslistbuilder
|
295
302
|
extensions: []
|
296
303
|
extra_rdoc_files: []
|
297
|
-
files:
|
304
|
+
files:
|
305
|
+
- ".gitignore"
|
306
|
+
- ".rakeTasks"
|
307
|
+
- ".rspec"
|
308
|
+
- ChangeLog.md
|
309
|
+
- Gemfile
|
310
|
+
- Gemfile.lock
|
311
|
+
- LICENSE
|
312
|
+
- LICENSE.txt
|
313
|
+
- README.md
|
314
|
+
- Rakefile
|
315
|
+
- bin/environmentslistbuilder
|
316
|
+
- environments-list-builder.gemspec
|
317
|
+
- features/.gitkeep
|
318
|
+
- features/manifestrepo-builder.feature
|
319
|
+
- features/step_definitions/.gitkeep
|
320
|
+
- features/step_definitions/manifestrepo-builder_steps.rb
|
321
|
+
- lib/cicd/builder/environments-list.rb
|
322
|
+
- lib/cicd/builder/environments-list/mixlib/build.rb
|
323
|
+
- lib/cicd/builder/environments-list/mixlib/lib/mixins/no_commands.rb
|
324
|
+
- lib/cicd/builder/environments-list/mixlib/lib/update_bucket_policy.rb
|
325
|
+
- lib/cicd/builder/environments-list/mixlib/repo.rb
|
326
|
+
- lib/cicd/builder/environments-list/mixlib/repo/artifactory.rb
|
327
|
+
- lib/cicd/builder/environments-list/version.rb
|
328
|
+
- openssl
|
329
|
+
- spec/builder_spec.rb
|
330
|
+
- spec/spec_helper.rb
|
331
|
+
- tests/infinite-etag.rb
|
298
332
|
homepage: https://rubygems.org/gems/environments-list-builder
|
299
333
|
licenses:
|
300
334
|
- Apachev2
|
@@ -319,4 +353,10 @@ rubygems_version: 2.4.3
|
|
319
353
|
signing_key:
|
320
354
|
specification_version: 4
|
321
355
|
summary: Jenkins builder task for CI/CD
|
322
|
-
test_files:
|
356
|
+
test_files:
|
357
|
+
- features/.gitkeep
|
358
|
+
- features/manifestrepo-builder.feature
|
359
|
+
- features/step_definitions/.gitkeep
|
360
|
+
- features/step_definitions/manifestrepo-builder_steps.rb
|
361
|
+
- spec/builder_spec.rb
|
362
|
+
- spec/spec_helper.rb
|