manifest-builder 0.4.0 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (8) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +2 -2
  3. data/lib/cicd/builder/manifest/mixlib/build.rb +304 -318
  4. data/lib/cicd/builder/manifest/mixlib/repo/artifactory.rb +163 -0
  5. data/lib/cicd/builder/manifest/mixlib/repo.rb +44 -0
  6. data/lib/cicd/builder/manifest/version.rb +1 -1
  7. data/lib/cicd/builder/manifest.rb +6 -2
  8. metadata +4 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 25d294dbd696497265d4cbd29c06158d121ae09e
4
- data.tar.gz: 6a3b1ec3431326915f54bc3d50adab16872512b4
3
+ metadata.gz: 17e7567ae1c78580e2ecc9f74f593764ff2dab7f
4
+ data.tar.gz: db1ca34f46fc756802efbc861ae6666345328109
5
5
  SHA512:
6
- metadata.gz: 5a7425bea022457dd3c2788c4dc83b3b663fc4d37e565945a9d50e41d08adafa681987a0e97f3a0bf7e2a5d32c80b9f2775d2febb2d65ea8622030be4740537d
7
- data.tar.gz: e71d7292a1da2612bbad013543de6b79d06381802d4744b306ab7d9b552820d5e2c84583bf798a2686effe5eb6068e19e1dff5f0f57002324e5ee42498147aa8
6
+ metadata.gz: 70b1e865b1d772c0077e2af4291bc4131f3199dd3792c0dafb540b5b96743d496abb54e40745e93cdc0e9709a1474fd1eb40760d30b8d0174ea25625f537ff82
7
+ data.tar.gz: 7fb9a46b85de867913b9f22766b124e18f99b7b497a5e1a75d69c82b6fb9425fbc3a184178943b99f1655ec9c3c22e37adf0e1a43354d9971b27cf3c5c764326
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- manifest-builder (0.4.0)
4
+ manifest-builder (0.5.0)
5
5
  archive-tar-minitar (= 0.5.2)
6
6
  chefrepo-builder (>= 0.9.27, < 1.1)
7
7
  cicd-builder (>= 0.9.33, < 1.1)
@@ -18,7 +18,7 @@ PATH
18
18
  PATH
19
19
  remote: ../cicd-builder
20
20
  specs:
21
- cicd-builder (0.9.33)
21
+ cicd-builder (0.9.35)
22
22
  artifactory (>= 2.2.1, < 2.3)
23
23
  awesome_print (>= 1.6, < 2.0)
24
24
  aws-sdk (>= 2.0, < 2.1)
data/lib/cicd/builder/manifest/mixlib/build.rb CHANGED
@@ -5,376 +5,362 @@ module CiCd
5
5
  # noinspection RubySuperCallWithoutSuperclassInspection
6
6
  module Manifest
7
7
  module Build
8
- # module ClassMethods
9
- # ---------------------------------------------------------------------------------------------------------------
10
- def self.included(includer)
11
- end
12
8
 
13
- # ---------------------------------------------------------------------------------------------------------------
14
- # noinspection RubyHashKeysTypesInspection
15
- def prepareBuild()
16
- ret = super
17
- if ret == 0
18
- @vars[:artifacts] = []
19
- yaml = YAML.load(IO.read(ENV['MANIFEST_FILE']))
20
- keys = Hash[yaml.keys.map.with_index.to_a].keys.sort
21
- # @logger.info keys.ai
22
- ordr = []
23
- bads = []
24
- apps = {}
25
- vars = {}
9
+ # ---------------------------------------------------------------------------------------------------------------
10
+ # noinspection RubyHashKeysTypesInspection
11
+ def prepareBuild()
12
+ ret = super
13
+ if ret == 0
14
+ @vars[:artifacts] = []
15
+ yaml = YAML.load(IO.read(ENV['MANIFEST_FILE']))
16
+ keys = Hash[yaml.keys.map.with_index.to_a].keys.sort
17
+ # @logger.info keys.ai
18
+ ordr = []
19
+ bads = []
20
+ apps = {}
21
+ vars = {}
26
22
 
27
- rmap = {
28
- sha256: %w[_sha256],
29
- base_url: %w[_repo_base_url],
30
- url: %w[_url],
31
- version: %w[_app_version],
32
- build: %w[_app_build],
33
- }
23
+ rmap = {
24
+ sha256: %w[_sha256],
25
+ base_url: %w[_repo_base_url],
26
+ url: %w[_url],
27
+ version: %w[_app_version],
28
+ build: %w[_app_build],
29
+ }
34
30
 
35
- keys.each do |prod|
36
- rmap.keys.each do |var|
37
- vars[var] = ''
38
- end
39
- name = ''
40
- match = nil
41
- rmap.each do |var,lst|
42
- lst.each do |regexstr|
43
- match = prod.match(%r'^(.*?)#{regexstr}$')
44
- if match
45
- name = match[1]
46
- vars[var] = yaml[prod]
47
- break
48
- end
31
+ keys.each do |prod|
32
+ rmap.keys.each do |var|
33
+ vars[var] = ''
34
+ end
35
+ name = ''
36
+ match = nil
37
+ rmap.each do |var,lst|
38
+ lst.each do |regexstr|
39
+ match = prod.match(%r'^(.*?)#{regexstr}$')
40
+ if match
41
+ name = match[1]
42
+ vars[var] = yaml[prod]
43
+ break
49
44
  end
50
- break if match
51
- end
52
- if match
53
- ordr << name
54
- unless apps[name]
55
- apps[name] = { name: name, }
56
- end
57
- rmap.keys.each do |var|
58
- apps[name][var] = vars[var] unless vars[var].empty?
59
- end
60
- else
61
- bads << prod
62
45
  end
46
+ break if match
63
47
  end
64
- @logger.debug "App entries: #{apps.ai}"
65
- if bads.size > 0
66
- @logger.fatal "Bad entries: #{bads.map{|p| "#{p}: #{yaml[p]}"}.ai}"
67
- ret = Errors::BAD_ARTIFACTS
48
+ if match
49
+ ordr << name
50
+ unless apps[name]
51
+ apps[name] = { name: name, }
52
+ end
53
+ rmap.keys.each do |var|
54
+ apps[name][var] = vars[var] unless vars[var].empty?
55
+ end
68
56
  else
69
- @vars[:components] = apps
57
+ bads << prod
70
58
  end
71
59
  end
72
- @vars[:return_code] = ret
60
+ @logger.debug "App entries: #{apps.ai}"
61
+ if bads.size > 0
62
+ @logger.fatal "Bad entries: #{bads.map{|p| "#{p}: #{yaml[p]}"}.ai}"
63
+ ret = Errors::BAD_ARTIFACTS
64
+ else
65
+ @vars[:components] = apps
66
+ end
73
67
  end
68
+ @vars[:return_code] = ret
69
+ end
74
70
 
75
- # # ---------------------------------------------------------------------------------------------------------------
76
- # def makeBuild()
77
- # super
78
- # end
79
-
80
- VER_RGX = %r'^\d+\.\d+(\.?\d)*$'
81
- MMP_RGX = %r'^(\d+\.?){2,3}$'
82
- # ---------------------------------------------------------------------------------------------------------------
83
- def getVersionBuildFromName(artifact)
84
- version = artifact.dup
85
- version.gsub!(%r'\.*(tar\.gz|tgz|bzip2|bz2|jar|war|[a-z]+)$', '')
86
- # if artifact =~ %r'^#{comp[0]}'
87
- # version.gsub!(%r'^#{comp[0]}\.*-*','')
88
- # else
89
- # version.gsub!(%r'^[a-zA-Z\-._]+','')
90
- # end
91
- version.gsub!(%r'^[a-zA-Z\-._]+', '')
92
-
93
- # build = if version.match(VER_RGX)
94
- # if version.match(%r'^(\d+\.?){2,3}$')
95
- # 0
96
- # elsif version.match(%r'\-')
97
- # version,build = version.split(/-/)
98
- # build
99
- # else
100
- # 0
101
- # end
102
- # else
103
- # 0
104
- # end
105
- build = ''
106
- if version.match(VER_RGX)
107
- if version.match(%r'\-')
108
- version,build = version.split(/-/)
109
- end
110
- # else
111
- # match = version.match(%r'^(\d+)-(\d{4}-\d{2}-\d{2}[_]\d{2}-\d{2}-\d{2})\.(release|snapshot)$')
112
- # if match
113
- # build = match[1]
114
- # # version.gsub!(/^#{build}-/, '')
115
- # version = match[2]
116
- # end
71
+ VER_RGX = %r'^\d+\.\d+(\.?\d)*$'
72
+ MMP_RGX = %r'^(\d+\.?){2,3}$'
73
+ # ---------------------------------------------------------------------------------------------------------------
74
+ def getVersionBuildFromName(artifact)
75
+ version = artifact.dup
76
+ version.gsub!(%r'\.*(tar\.gz|tgz|bzip2|bz2|jar|war|[a-z]+)$', '')
77
+ version.gsub!(%r'^[a-zA-Z\-._]+', '')
78
+ build = ''
79
+ if version.match(VER_RGX)
80
+ if version.match(%r'\-')
81
+ version,build = version.split(/-/)
117
82
  end
118
- [version,build]
119
83
  end
84
+ [version,build]
85
+ end
120
86
 
121
- # ---------------------------------------------------------------------------------------------------------------
122
- def getVersionBuild(path,artifact,comp)
123
- version,build = File.split(path)
124
- if build.match(%r'^\d+$') and version.match(%r'/?\d+\.\d+\.?\d*$') # Hole in one!
125
- version = File.basename(version)
87
+ # ---------------------------------------------------------------------------------------------------------------
88
+ def getVersionBuild(path,artifact,comp)
89
+ version,build = File.split(path)
90
+ if build.match(%r'^\d+$') and version.match(%r'/?\d+\.\d+\.?\d*$') # Hole in one!
91
+ version = File.basename(version)
92
+ else
93
+ if build.match(VER_RGX)
94
+ version = build
95
+ build = ''
126
96
  else
127
- if build.match(VER_RGX)
128
- version = build
129
- build = ''
130
- else
131
- version = comp[1][:build].nil? ? '' : ( comp[1][:build] > 0 ? build.to_s : '' )
132
- end
133
- unless version.match(VER_RGX)
134
- version = comp[1][:version] || ''
135
- end
136
- ver,bld = getVersionBuildFromName(artifact)
97
+ version = comp[1][:build].nil? ? '' : ( comp[1][:build] > 0 ? build.to_s : '' )
98
+ end
99
+ unless version.match(VER_RGX)
100
+ version = comp[1][:version] || ''
101
+ end
102
+ ver,bld = getVersionBuildFromName(artifact)
103
+ if version.empty?
104
+ version,build = [ver,bld]
137
105
  if version.empty?
138
- version,build = [ver,bld]
139
- if version.empty?
140
- version = @vars[:build_ver]
141
- else
142
- uri,ver = File.split(path)
143
- if version =~ %r'^#{ver}'
144
- if version =~ VER_RGX
145
- if version =~ %r'^#{build}' # prob the major part of version
146
- build = ''
147
- end
148
- else
149
- unless version.eql?(ver)
150
- build = version.dup
151
- version = ver
152
- build = build.gsub(%r'^#{version}(\.|-)*','')
153
- end
154
- end
155
- else
156
- build = version.dup
157
- version = ver
158
- build = build.gsub(%r'^#{version}(\.|-)*','')
159
- end
160
- end
106
+ version = @vars[:build_ver]
161
107
  else
162
- if ver.match(VER_RGX)
163
- if ver.match(MMP_RGX)
164
- if version.length < ver.length
165
- version = ver # Guessing it is the better version
108
+ _,ver = File.split(path)
109
+ if version =~ %r'^#{ver}'
110
+ if version =~ VER_RGX
111
+ if version =~ %r'^#{build}' # prob the major part of version
112
+ build = ''
166
113
  end
167
114
  else
168
- build = ver.dup
169
- # version.gsub!(/\.d+$/, '')
170
- build.gsub!(/^#{version}\.?/, '')
115
+ unless version.eql?(ver)
116
+ build = version.dup
117
+ version = ver
118
+ build = build.gsub(%r'^#{version}(\.|-)*','')
119
+ end
171
120
  end
121
+ else
122
+ build = version.dup
123
+ version = ver
124
+ build = build.gsub(%r'^#{version}(\.|-)*','')
172
125
  end
173
126
  end
174
- unless build.match(%r'^[1-9]\d*$')
175
- build = comp[1][:build]
176
- build = @vars[:build_num] if (build.nil? or build.empty? or build.to_i == 0)
127
+ else
128
+ if ver.match(VER_RGX)
129
+ if ver.match(MMP_RGX)
130
+ if version.length < ver.length
131
+ version = ver # Guessing it is the better version
132
+ end
133
+ else
134
+ build = ver.dup
135
+ # version.gsub!(/\.d+$/, '')
136
+ build.gsub!(/^#{version}\.?/, '')
137
+ end
177
138
  end
178
139
  end
179
- [version,build]
140
+ unless build.match(%r'^[1-9]\d*$')
141
+ build = comp[1][:build]
142
+ build = @vars[:build_num] if (build.nil? or build.empty? or build.to_i == 0)
143
+ end
180
144
  end
145
+ [version,build]
146
+ end
181
147
 
182
- # ---------------------------------------------------------------------------------------------------------------
183
- def packageBuild()
184
- @logger.step __method__.to_s
185
- if isSameDirectory(Dir.pwd, ENV['WORKSPACE'])
186
- if @vars.has_key?(:components) and not @vars[:components].empty?
187
- @vars[:return_code] = 0
148
+ # ---------------------------------------------------------------------------------------------------------------
149
+ def packageBuild()
150
+ @logger.step __method__.to_s
151
+ if isSameDirectory(Dir.pwd, ENV['WORKSPACE'])
152
+ if @vars.has_key?(:components) and not @vars[:components].empty?
153
+ @vars[:return_code] = 0
188
154
 
189
- clazz = getRepoClass('S3')
190
- if clazz.is_a?(Class) and not clazz.nil?
191
- @repo = clazz.new(self)
155
+ clazz = getRepoClass('S3')
156
+ if clazz.is_a?(Class) and not clazz.nil?
157
+ @repo = clazz.new(self)
192
158
 
159
+ if @vars[:return_code] == 0
160
+ lines = []
161
+ @vars[:artifacts] = []
162
+ # Deal with all artifacts of each component
163
+ @vars[:components].each { |comp|
164
+ processComponent(comp, lines)
165
+ }
193
166
  if @vars[:return_code] == 0
194
- lines = []
195
- @vars[:artifacts] = []
196
- # Deal with all artifacts of each component
197
- @vars[:components].each { |comp|
198
- artifact, path, version, build = parseComponent(comp)
199
-
200
- require 'uri'
201
- begin
202
- parts = URI(path).path.gsub(%r'^#{File::SEPARATOR}','').split(File::SEPARATOR)
203
- name = parts.shift
204
- bucket = getBucket(name)
205
- key = File.join(parts, '')
206
- @logger.info "S3://#{name}:#{key} URL: #{path} #{artifact}"
207
- objects = []
208
- bucket.objects(prefix: key).each do |object|
209
- if artifact.empty? or (not artifact.empty? and object.key =~ %r'#{key}#{artifact}')
210
- objects << object
211
- end
212
- end
213
- @logger.debug "S3://#{name}:#{key} has #{objects.size} objects"
214
- local_dir = File.join(@vars[:local_dirs]['artifacts'],comp[0], '')
215
- Dir.mkdir(local_dir, 0700) unless File.directory?(local_dir)
216
- artifacts = []
217
- changed = false
218
- # 1 or more objects on the key/ path
219
- if objects.size > 0
220
- lines << "#{comp[0]}:#{artifact} v#{version} b#{build} - #{path}"
221
- # When we start pulling the artifacts then everything that is build 0 get this build number, in fact all artifacts get this build number!
222
- objects.each do |object|
223
- @logger.info "\tchecking #{object.key}"
224
- local = File.join(local_dir,File.basename(object.key))
225
- etag = object.etag.gsub(%r/['"]/, '')
226
- download = if File.exists?(local)
227
- @logger.debug "\t\tchecking etag on #{local}"
228
- stat = File.stat(local)
229
- check = calcLocalETag(etag, local, stat.size)
230
- if etag != check or object.size != stat.size or object.last_modified > stat.mtime
231
- @logger.debug "\t\t#{etag} != \"#{check}\" #{object.size} != #{stat.size} #{object.last_modified} > #{stat.mtime}"
232
- true
233
- else
234
- @logger.debug "\t\tmatched #{etag}"
235
- false
236
- end
237
- else
238
- true
239
- end
240
- if download
241
- @logger.info "\t\tdownload #{object.size} bytes"
242
- response = object.get(:response_target => local)
243
- File.utime(response.last_modified, response.last_modified, local)
244
- @logger.info "\t\tdone"
245
- check = calcLocalETag(etag, local)
246
- unless check.eql?(etag)
247
- @logger.info "\tETag different: #{etag} != #{check}"
248
- changed = true
249
- end
250
- else
251
- @logger.info "\t\tunchanged"
252
- end
253
- artifacts << local
254
- end
255
- # The local file will be 1 artifact or an archive of the local artifacts when artifacts.size > 1
256
- local = if artifacts.size > 0
257
- if artifacts.size > 1
258
- begin
259
- # require 'zlib'
260
- # require 'archive/tar/minitar'
261
- file = File.join(local_dir, "#{comp[0]}-#{version}.zip")
262
- if changed or not File.exists?(file)
263
- # output = File.open(file, 'wb')
264
- # output = Zlib::GzipWriter.new(output, Zlib::BEST_COMPRESSION, Zlib::RLE)
265
- # Dir.chdir(local_dir) do
266
- # Archive::Tar::Minitar.pack(artifacts.map{|f| f.gsub(%r'^#{local_dir}','')}, output, false )
267
- # end
268
- zipped_files = artifacts.map{|f| f.gsub(%r'^#{local_dir}','')}.join(' ')
269
- Dir.chdir(local_dir) do
270
- res = %x(zip -o9X #{file} #{zipped_files})
271
- end
272
- raise "Failed to zip #{file} containting #{zipped_files}" unless $?.exitstatus == 0
273
- end
274
- file
275
- rescue Exception => e
276
- @logger.error "Artifact error: #{file} #{e.class.name} #{e.message}"
277
- File.unlink(file)
278
- raise e
279
- # ensure
280
- # output.close if output and not output.closed?
281
- end
282
- else
283
- artifacts[0]
284
- end
285
- else
286
- end
287
- addArtifact(@vars[:artifacts], local, local_dir, { module: comp[0], name: comp[0], build: build, version: version, file: local})
288
- else
289
- @logger.fatal "Artifact not found: s3://#{name}/#{key}#{artifact}"
290
- @vars[:return_code] = Errors::ARTIFACT_NOT_FOUND
291
- end
292
- # rescue Aws::S3::Errors::NotFound => e
293
- # @logger.fatal "Artifact S3 error: #{artifact} #{e.class.name} #{e.message}"
294
- # raise e
295
- # rescue Aws::S3::Errors::NoSuchKey => e
296
- # @logger.error "Artifact S3 error: #{artifact} #{e.class.name} #{e.message}"
297
- rescue Exception => e
298
- @logger.error "Artifact error: #{artifact} #{e.class.name} #{e.message}"
299
- raise e
300
- end
301
- }
302
- if @vars[:return_code] == 0
303
- cleanupAfterPackaging(lines)
304
- end
305
-
306
- else
307
- @logger.fatal "S3 repo error: Bucket #{ENV['AWS_S3_BUCKET']}"
167
+ cleanupAfterPackaging(lines)
308
168
  end
169
+
309
170
  else
310
- @logger.error "CiCd::Builder::Repo::#{type} is not a valid repo class"
311
- @vars[:return_code] = Errors::BUILDER_REPO_TYPE
171
+ @logger.fatal "S3 repo error: Bucket #{ENV['AWS_S3_BUCKET']}"
312
172
  end
313
173
  else
314
- @logger.error 'No components found during preparation?'
315
- @vars[:return_code] = Errors::NO_COMPONENTS
174
+ @logger.error "CiCd::Builder::Repo::#{type} is not a valid repo class"
175
+ @vars[:return_code] = Errors::BUILDER_REPO_TYPE
316
176
  end
317
177
  else
318
- @logger.error "Not in WORKSPACE? '#{pwd}' does not match WORKSPACE='#{workspace}'"
319
- @vars[:return_code] = Errors::WORKSPACE_DIR
178
+ @logger.error 'No components found during preparation?'
179
+ @vars[:return_code] = Errors::NO_COMPONENTS
320
180
  end
181
+ else
182
+ @logger.error "Not in WORKSPACE? '#{pwd}' does not match WORKSPACE='#{workspace}'"
183
+ @vars[:return_code] = Errors::WORKSPACE_DIR
184
+ end
185
+
186
+ @vars[:return_code]
187
+ end
321
188
 
322
- @vars[:return_code]
189
+ # ---------------------------------------------------------------------------------------------------------------
190
+ def cleanupAfterPackaging(lines)
191
+ begin
192
+ unless IO.write(@vars[:build_mff], lines.join("\n")) > 0
193
+ @logger.error "Nothing was written to build manifest '#{@vars[:build_mff]}'"
194
+ @vars[:return_code] = Errors::MANIFEST_EMPTY
195
+ end
196
+ rescue => e
197
+ @logger.error "Failed to write manifest '#{@vars[:build_mff]}' (#{e.message})"
198
+ @vars[:return_code] = Errors::MANIFEST_WRITE
199
+ end
200
+ FileUtils.rmtree(@vars[:build_dir])
201
+ @vars[:return_code] = File.directory?(@vars[:build_dir]) ? Errors::BUILD_DIR : 0
202
+ unless @vars[:return_code] == 0
203
+ @logger.warn "Remove manifest '#{@vars[:build_mff]}' due to error"
204
+ FileUtils.rm_f(@vars[:build_mff])
205
+ # @vars[:return_code] = File.exists?(@vars[:build_mff]) ? Errors::MANIFEST_DELETE : 0
323
206
  end
207
+ end
324
208
 
325
- def cleanupAfterPackaging(lines)
326
- begin
327
- unless IO.write(@vars[:build_mff], lines.join("\n")) > 0
328
- @logger.error "Nothing was written to build manifest '#{@vars[:build_mff]}'"
329
- @vars[:return_code] = Errors::MANIFEST_EMPTY
209
+ private
210
+
211
+ # ---------------------------------------------------------------------------------------------------------------
212
+ def processComponent(comp, lines)
213
+ artifact, path, version, build = parseComponent(comp)
214
+
215
+ require 'uri'
216
+ begin
217
+ key, name, objects = getObjects(artifact, path)
218
+ local_dir = File.join(@vars[:local_dirs]['artifacts'], comp[0], '')
219
+ Dir.mkdir(local_dir, 0700) unless File.directory?(local_dir)
220
+ artifacts = []
221
+ changed = false
222
+ # 1 or more objects on the key/ path
223
+ if objects.size > 0
224
+ lines << "#{comp[0]}:#{artifact} v#{version} b#{build} - #{path}"
225
+ # When we start pulling the artifacts then everything that is build 0 get this build number, in fact all artifacts get this build number!
226
+ objects.each do |object|
227
+ @logger.info "\tchecking #{object.key}"
228
+ local = File.join(local_dir, File.basename(object.key))
229
+ etag = object.etag.gsub(%r/['"]/, '')
230
+ download = shouldDownload?(etag, local, object)
231
+ if download
232
+ changed = doDownload(etag, local, object)
233
+ else
234
+ @logger.info "\t\tunchanged"
235
+ end
236
+ artifacts << local
237
+ end
238
+ # The local file will be 1 artifact or an archive of the local artifacts when artifacts.size > 1
239
+ if artifacts.size > 0
240
+ local = getLocalArtifact(artifacts, changed, comp, local_dir, version)
241
+ addArtifact(@vars[:artifacts], local, local_dir, {module: comp[0], name: comp[0], build: build, version: version, file: local})
330
242
  end
331
- rescue => e
332
- @logger.error "Failed to write manifest '#{@vars[:build_mff]}' (#{e.message})"
333
- @vars[:return_code] = Errors::MANIFEST_WRITE
243
+ else
244
+ @logger.fatal "Artifact not found: s3://#{name}/#{key}#{artifact}"
245
+ @vars[:return_code] = Errors::ARTIFACT_NOT_FOUND
334
246
  end
335
- FileUtils.rmtree(@vars[:build_dir])
336
- @vars[:return_code] = File.directory?(@vars[:build_dir]) ? Errors::BUILD_DIR : 0
337
- unless @vars[:return_code] == 0
338
- @logger.warn "Remove manifest '#{@vars[:build_mff]}' due to error"
339
- FileUtils.rm_f(@vars[:build_mff])
340
- # @vars[:return_code] = File.exists?(@vars[:build_mff]) ? Errors::MANIFEST_DELETE : 0
247
+ rescue Exception => e
248
+ @logger.error "Artifact error: #{artifact} #{e.class.name} #{e.message}"
249
+ raise e
250
+ end
251
+ end
252
+
253
+ # ---------------------------------------------------------------------------------------------------------------
254
+ def getObjects(artifact, path)
255
+ parts = URI(path).path.gsub(%r'^#{File::SEPARATOR}', '').split(File::SEPARATOR)
256
+ name = parts.shift
257
+ bucket = getBucket(name)
258
+ key = File.join(parts, '')
259
+ @logger.info "S3://#{name}:#{key} URL: #{path} #{artifact}"
260
+ objects = []
261
+ bucket.objects(prefix: key).each do |object|
262
+ if artifact.empty? or (not artifact.empty? and object.key =~ %r'#{key}#{artifact}')
263
+ objects << object
341
264
  end
342
265
  end
266
+ @logger.debug "S3://#{name}:#{key} has #{objects.size} objects"
267
+ return key, name, objects
268
+ end
343
269
 
344
- def parseComponent(comp)
345
- if comp[1][:url]
346
- path, artifact = File.split(comp[1][:url])
347
- version, build = getVersionBuild(path, artifact, comp)
348
- elsif comp[1][:base_url]
349
- artifact = ''
350
- if comp[1][:build].nil?
351
- version, build = comp[1][:version].split(%r'-')
352
- path = File.join(comp[1][:base_url], comp[1][:version])
353
- else
354
- version, build = [comp[1][:version], comp[1][:build]]
355
- path = File.join(comp[1][:base_url], comp[1][:version], comp[1][:build])
356
- end
270
+ # ---------------------------------------------------------------------------------------------------------------
271
+ def doDownload(etag, local, object)
272
+ @logger.info "\t\tdownload #{object.size} bytes"
273
+ response = object.get(:response_target => local)
274
+ File.utime(response.last_modified, response.last_modified, local)
275
+ @logger.info "\t\tdone"
276
+ check = calcLocalETag(etag, local)
277
+ if check.eql?(etag)
278
+ false
279
+ else
280
+ @logger.info "\tETag different: #{etag} != #{check}"
281
+ true
282
+ end
283
+ end
284
+
285
+ # ---------------------------------------------------------------------------------------------------------------
286
+ def shouldDownload?(etag, local, object)
287
+ if File.exists?(local)
288
+ @logger.debug "\t\tchecking etag on #{local}"
289
+ stat = File.stat(local)
290
+ check = calcLocalETag(etag, local, stat.size)
291
+ if etag != check or object.size != stat.size or object.last_modified > stat.mtime
292
+ @logger.debug "\t\t#{etag} != \"#{check}\" #{object.size} != #{stat.size} #{object.last_modified} > #{stat.mtime}"
293
+ true
357
294
  else
358
- path = ''
359
- artifact = ''
360
- version, build = getVersionBuild(path, artifact, comp)
295
+ @logger.debug "\t\tmatched #{etag}"
296
+ false
361
297
  end
362
- return artifact, path, version, build
298
+ else
299
+ true
363
300
  end
301
+ end
364
302
 
365
- def getBucket(name = nil)
366
- @s3 = @repo.getS3()
367
- bucket = begin
368
- ::Aws::S3::Bucket.new(name: name || ENV['AWS_S3_BUCKET'], client: @s3)
369
- rescue Aws::S3::Errors::NotFound
370
- @vars[:return_code] = Errors::BUCKET
371
- nil
303
+ # ---------------------------------------------------------------------------------------------------------------
304
+ def getLocalArtifact(artifacts, changed, comp, local_dir, version)
305
+ if artifacts.size > 1
306
+ begin
307
+ file = File.join(local_dir, "#{comp[0]}-#{version}.zip")
308
+ if changed or not File.exists?(file)
309
+ zipped_files = artifacts.map { |f| f.gsub(%r'^#{local_dir}', '') }.join(' ')
310
+ Dir.chdir(local_dir) do
311
+ res = %x(zip -o9X #{file} #{zipped_files} 2>&1)
312
+ @logger.info res
313
+ end
314
+ raise "Failed to zip #{file} containting #{zipped_files}" unless $?.exitstatus == 0
315
+ end
316
+ file
372
317
  rescue Exception => e
373
- @logger.error "S3 Bucket resource API error: #{e.class.name} #{e.message}"
318
+ @logger.error "Artifact error: #{file} #{e.class.name} #{e.message}"
319
+ File.unlink(file)
374
320
  raise e
375
321
  end
376
- bucket
322
+ else
323
+ artifacts[0]
324
+ end
325
+ end
326
+
327
+ # ---------------------------------------------------------------------------------------------------------------
328
+ def parseComponent(comp)
329
+ if comp[1][:url]
330
+ path, artifact = File.split(comp[1][:url])
331
+ version, build = getVersionBuild(path, artifact, comp)
332
+ elsif comp[1][:base_url]
333
+ artifact = ''
334
+ if comp[1][:build].nil?
335
+ # noinspection RubyUnusedLocalVariable
336
+ version, build = comp[1][:version].split(%r'-')
337
+ # noinspection RubyUnusedLocalVariable
338
+ path = File.join(comp[1][:base_url], comp[1][:version])
339
+ else
340
+ version, build = [comp[1][:version], comp[1][:build]]
341
+ path = File.join(comp[1][:base_url], comp[1][:version], comp[1][:build])
342
+ end
343
+ else
344
+ path = ''
345
+ artifact = ''
346
+ version, build = getVersionBuild(path, artifact, comp)
347
+ end
348
+ return artifact, path, version, build
349
+ end
350
+
351
+ # ---------------------------------------------------------------------------------------------------------------
352
+ def getBucket(name = nil)
353
+ @s3 = @repo.getS3()
354
+ begin
355
+ ::Aws::S3::Bucket.new(name: name || ENV['AWS_S3_BUCKET'], client: @s3)
356
+ rescue Aws::S3::Errors::NotFound
357
+ @vars[:return_code] = Errors::BUCKET
358
+ nil
359
+ rescue Exception => e
360
+ @logger.error "S3 Bucket resource API error: #{e.class.name} #{e.message}"
361
+ raise e
377
362
  end
363
+ end
378
364
  end
379
365
  end
380
366
  end
data/lib/cicd/builder/manifest/mixlib/repo/artifactory.rb ADDED
@@ -0,0 +1,163 @@
1
+ require 'artifactory'
2
+
3
+ module CiCd
4
+ module Builder
5
+ # noinspection RubySuperCallWithoutSuperclassInspection
6
+ module Manifest
7
+ module Repo
8
+ class Artifactory < CiCd::Builder::Repo::Artifactory
9
+ # include ::Artifactory::Resource
10
+
11
+ # ---------------------------------------------------------------------------------------------------------------
12
+ def initialize(builder)
13
+ super
14
+ end
15
+
16
+ # ---------------------------------------------------------------------------------------------------------------
17
+ def uploadToRepo(artifacts)
18
+ @manifest = {}
19
+ super
20
+ if @vars[:return_code] == 0
21
+ # Preserve the manifest (maybeUploadArtifactoryObject will add everything we upload to the instance var)
22
+ manifest = @manifest.dup
23
+ # Create a manifest for each product and store it.
24
+ createProductManifests(manifest)
25
+ # Get a super manifest of all products and store as learning-manifest
26
+ createSuperManifest(manifest) # -#{@vars[:variant]}
27
+ @manifest = manifest
28
+ end
29
+ # If we are asked to produce a PROJECTS_FILE then we do that from the manifest and components.
30
+ unless ENV['PROJECTS_FILE'].nil?
31
+ if @vars[:return_code] == 0
32
+ if File.directory?(File.realdirpath(File.dirname(ENV['PROJECTS_FILE'])))
33
+ createProjectsFile
34
+ else
35
+ @logger.error "The path to the PROJECTS_FILE (#{File.dirname(ENV['PROJECTS_FILE'])}) does not exist!"
36
+ @vars[:return_code] = Errors::NO_PROJECTS_PATH
37
+ end
38
+ end
39
+ end
40
+ @vars[:return_code]
41
+ end
42
+
43
+ # ---------------------------------------------------------------------------------------------------------------
44
+ def createProjectsFile
45
+ projects = {}
46
+ project_names = {}
47
+ unless ENV['PROJECT_NAMES'].nil?
48
+ if File.exists?(ENV['PROJECT_NAMES'])
49
+ project_names = JSON.load(IO.read(ENV['PROJECT_NAMES']))
50
+ else
51
+ @logger.error "The PROJECT_NAMES file (#{ENV['PROJECT_NAMES']}) does not exist!"
52
+ @vars[:return_code] = Errors::NO_PROJECT_NAMES
53
+ end
54
+ end
55
+ exts = Hash[@vars[:artifacts].map { |a| [a[:data][:name], File.extname(File.basename(a[:data][:file])).gsub(%r'^\.', '')] }]
56
+ @manifest.each do |mod, man|
57
+ man.each do |prod, build|
58
+ projects[prod] = {
59
+ name: project_names[prod] || prod,
60
+ module: mod,
61
+ ext: exts[prod],
62
+ class_filter: '',
63
+ }
64
+ end
65
+ end
66
+ IO.write(ENV['PROJECTS_FILE'], JSON.pretty_generate(projects, {indent: "\t", space: ' '}))
67
+ end
68
+
69
+ def createSuperManifest(manifest)
70
+ manifest_data = ''
71
+ manifest.each do |mod, man|
72
+ man.each do |k, v|
73
+ manifest_data += "#{k}=#{v}\n"
74
+ end
75
+ end
76
+ amn = artifactory_manifest_name # Just using a local iso invoking method_missing repeatedly ... ;)
77
+ data = {module: amn, data: manifest_data, version: @vars[:build_ver], build: @vars[:build_num], properties: @properties_matrix}
78
+ tempArtifactFile(amn, data)
79
+ data[:sha1] = Digest::SHA1.file(data[:file]).hexdigest
80
+ data[:md5] = Digest::MD5.file(data[:file]).hexdigest
81
+ data[:name] = amn
82
+ maybeUploadArtifactoryObject(data: data, artifact_module: amn, artifact_version: data[:version] || @vars[:version], file_name: '', file_ext: 'properties')
83
+ end
84
+
85
+ # ---------------------------------------------------------------------------------------------------------------
86
+ def createProductManifests(manifest)
87
+ manifest.each do |mod, man|
88
+ manifest_data = ''
89
+ man.each do |k, v|
90
+ manifest_data += "#{k}=#{v}\n"
91
+ end
92
+ data = {module: mod, data: manifest_data, version: @vars[:build_ver], build: @vars[:build_num], properties: @properties_matrix}
93
+ tempArtifactFile("#{mod}-manifest", data)
94
+ data[:sha1] = Digest::SHA1.file(data[:file]).hexdigest
95
+ data[:md5] = Digest::MD5.file(data[:file]).hexdigest
96
+ data[:name] = "#{mod}-manifest"
97
+ maybeUploadArtifactoryObject(data: data, artifact_module: data[:name], artifact_version: data[:version] || @vars[:version], file_name: '', file_ext: 'properties') # -#{@vars[:variant]}
98
+ end
99
+ end
100
+
101
+ # ---------------------------------------------------------------------------------------------------------------
102
+ def maybeUploadArtifactoryObject(args)
103
+ super
104
+ if @vars[:return_code] == 0
105
+ data = args[:data]
106
+ artifact_module = args[:artifact_module]
107
+ artifact_version = args[:artifact_version]
108
+ # file_ext = args[:file_ext]
109
+ # file_name = args[:file_name]
110
+ if @manifest[artifact_module].nil?
111
+ @manifest[artifact_module] = {}
112
+ file_name = artifact_module
113
+ else
114
+ file_name, _ = get_artifact_file_name_ext(data)
115
+ if file_name.empty?
116
+ file_name = artifact_module
117
+ else
118
+ file_name = "#{artifact_module}#{file_name}"
119
+ end
120
+ end
121
+ @manifest[artifact_module][file_name] = artifact_version
122
+ end
123
+
124
+ @vars[:return_code]
125
+ end
126
+
127
+ private :createProductManifests, :createProjectsFile, :createSuperManifest
128
+ end
129
+ end
130
+ end
131
+ end
132
+ end
133
+
134
+ =begin
135
+
136
+ {
137
+ "test-project": {
138
+ "name": "test-project",
139
+ "module": "test-server",
140
+ "ext": "war",
141
+ "class_filter": "role.role-1"
142
+ },
143
+ "test-project-2": {
144
+ "name": "test-project-2",
145
+ "module": "test-server2",
146
+ "ext": "zip",
147
+ "class_filter": "role.role-2"
148
+ },
149
+ "test-manifest": {
150
+ "name": "test-manifest",
151
+ "module": "test-manifest",
152
+ "ext": "properties",
153
+ "class_filter": ""
154
+ },
155
+ "test-manifest2": {
156
+ "name": "test-manifest2",
157
+ "module": "test-manifest2",
158
+ "ext": "properties",
159
+ "class_filter": ""
160
+ }
161
+ }
162
+
163
+ =end
data/lib/cicd/builder/manifest/mixlib/repo.rb ADDED
@@ -0,0 +1,44 @@
1
+ require 'json'
2
+
3
+ module CiCd
4
+ module Builder
5
+ # noinspection RubySuperCallWithoutSuperclassInspection
6
+ module Manifest
7
+ module Repo
8
+ require 'cicd/builder/mixlib/repo/base'
9
+ require 'cicd/builder/mixlib/repo/S3'
10
+ # noinspection RubyResolve
11
+ if ENV.has_key?('REPO_TYPE') and (not ENV['REPO_TYPE'].capitalize.eql?('S3'))
12
+ require "cicd/builder/manifest/mixlib/repo/#{ENV['REPO_TYPE'].downcase}"
13
+ end
14
+
15
+ # ---------------------------------------------------------------------------------------------------------------
16
+ def getRepoClass(type = nil)
17
+ @logger.info __method__.to_s
18
+ if type.nil?
19
+ type ||= 'S3'
20
+ if ENV.has_key?('REPO_TYPE')
21
+ type = ENV['REPO_TYPE']
22
+ end
23
+ end
24
+
25
+ @logger.info "#{type} repo interface"
26
+ clazz = begin
27
+ Object.const_get("#{self.class.name.gsub(%r'::\w+$', '')}::Repo::#{type}")
28
+ rescue NameError #=> e
29
+ Object.const_get("CiCd::Builder::Repo::#{type}")
30
+ end
31
+
32
+ if block_given?
33
+ if clazz.is_a?(Class) and not clazz.nil?
34
+ yield
35
+ end
36
+ end
37
+
38
+ clazz
39
+ end
40
+
41
+ end
42
+ end
43
+ end
44
+ end
data/lib/cicd/builder/manifest/version.rb CHANGED
@@ -2,7 +2,7 @@ module CiCd
2
2
  module Builder
3
3
  module Manifest
4
4
  # manifest-builder version
5
- VERSION = '0.4.0'
5
+ VERSION = '0.5.0'
6
6
  MAJOR, MINOR, TINY = VERSION.split('.')
7
7
  PATCH = TINY
8
8
  end
data/lib/cicd/builder/manifest.rb CHANGED
@@ -10,8 +10,11 @@ module CiCd
10
10
  module Manifest
11
11
  class Runner < ChefRepo::Runner
12
12
  require 'cicd/builder/manifest/mixlib/build'
13
+ require 'cicd/builder/manifest/mixlib/repo'
13
14
  include CiCd::Builder::Manifest::Build
15
+ include CiCd::Builder::Manifest::Repo
14
16
 
17
+ # ---------------------------------------------------------------------------------------------------------------
15
18
  def initialize()
16
19
  super
17
20
  @default_options[:builder] = VERSION
@@ -26,12 +29,13 @@ module CiCd
26
29
  patch: PATCH,
27
30
  }
28
31
  end
32
+
29
33
  # ---------------------------------------------------------------------------------------------------------------
30
34
  def setup()
31
35
  $stdout.write("ManifestBuilder v#{CiCd::Builder::Manifest::VERSION}\n")
32
36
  @default_options[:env_keys] << %w(
33
- MANIFEST_FILE
34
- )
37
+ MANIFEST_FILE
38
+ )
35
39
  super
36
40
  end
37
41
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: manifest-builder
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.0
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Christo De Lange
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-03-15 00:00:00.000000000 Z
11
+ date: 2015-03-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: cicd-builder
@@ -229,6 +229,8 @@ files:
229
229
  - features/step_definitions/manifestrepo-builder_steps.rb
230
230
  - lib/cicd/builder/manifest.rb
231
231
  - lib/cicd/builder/manifest/mixlib/build.rb
232
+ - lib/cicd/builder/manifest/mixlib/repo.rb
233
+ - lib/cicd/builder/manifest/mixlib/repo/artifactory.rb
232
234
  - lib/cicd/builder/manifest/version.rb
233
235
  - manifest-builder.gemspec
234
236
  - spec/builder_spec.rb