environments-list-builder 0.1.8 → 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,49 @@
1
+ require 'json'
2
+
3
+ module CiCd
4
+ module Builder
5
+ # noinspection RubySuperCallWithoutSuperclassInspection
6
+ module EnvironmentsList
7
+ module Repo
8
+ require 'cicd/builder/mixlib/repo/base'
9
+ require 'cicd/builder/mixlib/repo/S3'
10
+ # noinspection RubyResolve
11
+ if ENV.has_key?('REPO_TYPE') and (not ENV['REPO_TYPE'].capitalize.eql?('S3'))
12
+ require "cicd/builder/environments-list/mixlib/repo/#{ENV['REPO_TYPE'].downcase}"
13
+ end
14
+
15
+ # ---------------------------------------------------------------------------------------------------------------
16
+ def getRepoClass(type = nil)
17
+ @logger.step CLASS+'::'+__method__.to_s
18
+ if type.nil?
19
+ type ||= 'S3'
20
+ if ENV.has_key?('REPO_TYPE')
21
+ type = ENV['REPO_TYPE']
22
+ end
23
+ end
24
+
25
+ @logger.info "#{type} repo interface"
26
+ clazz = begin
27
+ Object.const_get("#{self.class.name.gsub(%r'::\w+$', '')}::Repo::#{type}")
28
+ rescue NameError => e
29
+ begin
30
+ # Object.const_get("#{self.class.name.gsub(%r'::\w+$', '')}::Repo::#{type}")
31
+ Object.const_get("CiCd::Builder::Manifest::Repo::#{type}")
32
+ rescue NameError #=> e
33
+ Object.const_get("CiCd::Builder::Repo::#{type}")
34
+ end
35
+ end
36
+
37
+ if block_given?
38
+ if clazz.is_a?(Class) and not clazz.nil?
39
+ yield
40
+ end
41
+ end
42
+
43
+ clazz
44
+ end
45
+
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,49 @@
1
+ require 'artifactory'
2
+ require 'tmpdir'
3
+ require "cicd/builder/manifest/mixlib/repo/artifactory"
4
+
5
+ module CiCd
6
+ module Builder
7
+ # noinspection RubySuperCallWithoutSuperclassInspection
8
+ module EnvironmentsList
9
+ module Repo
10
+ class Artifactory < CiCd::Builder::Manifest::Repo::Artifactory
11
+
12
+ alias_method :super_uploadToRepo, :uploadToRepo
13
+ # ---------------------------------------------------------------------------------------------------------------
14
+ def uploadToRepo(artifacts)
15
+ @logger.step CLASS+'::'+__method__.to_s
16
+ # super_uploadToRepo(artifacts) get's the immediate parent class
17
+ cicd_uploadToRepo(artifacts)
18
+ # CiCd::Builder::Repo::Artifactory.instance_method(:uploadToRepo).bind(self).call(artifacts)
19
+ if @vars[:environments][:changed]
20
+ data = {
21
+ name: ENV['ARTIFACTORY_ENVIRONMENTS_MODULE'],
22
+ module: ENV['ARTIFACTORY_ENVIRONMENTS_MODULE'],
23
+ file: @vars[:environments][:file],
24
+ version: @vars[:environments][:version],
25
+ build: @vars[:build_num],
26
+ properties: @properties_matrix,
27
+ temp: false,
28
+ sha1: Digest::SHA1.file(@vars[:environments][:file]).hexdigest,
29
+ md5: Digest::MD5.file(@vars[:environments][:file]).hexdigest,
30
+ }
31
+
32
+ cicd_maybeUploadArtifactoryObject(
33
+ data: data,
34
+ artifact_module: data[:module],
35
+ artifact_version: data[:version],
36
+ file_name: '',
37
+ file_ext: File.extname(data[:file]).gsub(/^\./,''),
38
+ repo: ENV['ARTIFACTORY_RELEASE_REPO'],
39
+ copy: false
40
+ )
41
+ end
42
+ @vars[:return_code]
43
+ end
44
+
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,10 @@
1
+ module CiCd
2
+ module Builder
3
+ module EnvironmentsList
4
+ # environments-list-builder version
5
+ VERSION = '0.1.9'
6
+ MAJOR, MINOR, TINY = VERSION.split('.')
7
+ PATCH = TINY
8
+ end
9
+ end
10
+ end
data/openssl ADDED
File without changes
@@ -0,0 +1,8 @@
1
+ require 'spec_helper'
2
+ require 'cicd/builder'
3
+
4
+ describe ManifestBuilder::Builder do
5
+ it "should have a VERSION constant" do
6
+ subject.const_get('VERSION').should_not be_empty
7
+ end
8
+ end
@@ -0,0 +1,4 @@
1
+ require 'rspec'
2
+ require 'cicd/builder/version'
3
+
4
+ include ManifestBuilder::Builder
@@ -0,0 +1,166 @@
1
+ require 'aws-sdk-core'
2
+ require 'aws-sdk-resources'
3
+ require 'uri'
4
+
5
+ # ---------------------------------------------------------------------------------------------------------------
6
+ def getS3()
7
+ region = ENV['AWS_REGION'] || ::Aws.config[:region] || 'us-east-1'
8
+ unless @s3
9
+ # noinspection RubyArgCount
10
+ @s3 = ::Aws::S3::Client.new(region: region)
11
+ end
12
+ unless @s3 and ((@s3.config.access_key_id and @s3.config.secret_access_key) or @s3.config.credentials)
13
+ @logger.warn "Unable to find AWS credentials in standard locations:
14
+ ENV['AWS_ACCESS_KEY'] and ENV['AWS_SECRET_ACCESS_KEY']
15
+ Aws.config[:credentials]
16
+ Shared credentials file, ~/.aws/credentials
17
+ EC2 Instance profile
18
+ "
19
+ if ENV['AWS_PROFILE']
20
+ @logger.info "Trying profile '#{ENV['AWS_PROFILE']}' explicitly"
21
+ creds = Aws::SharedCredentials.new( path: File.expand_path('~/.aws/credentials'), profile: ENV['AWS_PROFILE'] )
22
+ if creds.loadable?
23
+ # noinspection RubyArgCount
24
+ @s3 = ::Aws::S3::Client.new(region: region, credentials: creds)
25
+ end
26
+ else
27
+ @logger.warn 'No AWS_PROFILE defined'
28
+ end
29
+ end
30
+ unless @s3 and ((@s3.config.access_key_id and @s3.config.secret_access_key) or @s3.config.credentials)
31
+ raise 'Unable to find AWS credentials!'
32
+ end
33
+ @s3
34
+ end
35
+
36
+ # ---------------------------------------------------------------------------------------------------------------
37
+ def getBucket(name = nil)
38
+ @s3 = getS3()
39
+ begin
40
+ ::Aws::S3::Bucket.new(name: name || ENV['AWS_S3_BUCKET'], client: @s3)
41
+ rescue Aws::S3::Errors::NotFound
42
+ @vars[:return_code] = Errors::BUCKET
43
+ nil
44
+ rescue Exception => e
45
+ @logger.error "S3 Bucket resource API error: #{e.class.name} #{e.message}"
46
+ raise e
47
+ end
48
+ end
49
+
50
+ # ---------------------------------------------------------------------------------------------------------------
51
+ def getObjects(artifact, path)
52
+ parts = URI(path).path.gsub(%r'^#{File::SEPARATOR}', '').split(File::SEPARATOR)
53
+ name = parts.shift
54
+ bucket = getBucket(name)
55
+ key = File.join(parts, '')
56
+ @logger.info "S3://#{name}:#{key} URL: #{path} #{artifact}"
57
+ objects = []
58
+ bucket.objects(prefix: key).each do |object|
59
+ if artifact.empty? or (not artifact.empty? and object.key =~ %r'#{key}#{artifact}')
60
+ objects << object
61
+ end
62
+ end
63
+ @logger.debug "S3://#{name}:#{key} has #{objects.size} objects"
64
+ return key, name, objects
65
+ end
66
+
67
+ # ---------------------------------------------------------------------------------------------------------------
68
+ def calcLocalETag(etag, local, size = nil)
69
+ if size == nil
70
+ stat = File.stat(local)
71
+ size = stat.size
72
+ end
73
+ @logger.debug "Calculate etag to match #{etag}"
74
+ match = etag.match(%r'-(\d+)$')
75
+ check = if match
76
+ require 's3etag'
77
+ parts = match[1].to_i
78
+ chunk = size.to_f / parts.to_f
79
+ mbs = (chunk.to_f / 1024 /1024 + 0.5).to_i
80
+ part_size = mbs * 1024 * 1024
81
+ chkit = S3Etag.calc(file: local, threshold: part_size, min_part_size: part_size, max_parts: parts)
82
+ @logger.debug "S3Etag Calculated #{chkit} : (#{size} / #{part_size}) <= #{parts}"
83
+ chunks = size / part_size
84
+ while chkit != etag and chunks <= parts and chunks > 0 and (size > part_size)
85
+ # Go one larger if a modulus remains and we have the right number of parts
86
+ mbs += 1
87
+ part_size = mbs * 1024 * 1024
88
+ chunks = size.to_f / part_size
89
+ chkit = S3Etag.calc(file: local, threshold: part_size, min_part_size: part_size, max_parts: parts)
90
+ @logger.debug "S3Etag Calculated #{chkit} : (#{size} / #{part_size}) <= #{parts}"
91
+ end
92
+ #raise "Unable to match etag #{etag}!" if chkit != etag
93
+ chkit
94
+ else
95
+ Digest::MD5.file(local).hexdigest
96
+ end
97
+ end
98
+
99
+ # ---------------------------------------------------------------------------------------------------------------
100
+ def shouldDownload?(etag, local, object)
101
+ if File.exists?(local)
102
+ @logger.debug "\t\tchecking etag on #{local}"
103
+ stat = File.stat(local)
104
+ check = calcLocalETag(etag, local, stat.size)
105
+ if etag != check or object.size != stat.size or object.last_modified > stat.mtime
106
+ @logger.debug "\t\t#{etag} != \"#{check}\" #{object.size} != #{stat.size} #{object.last_modified} > #{stat.mtime}"
107
+ true
108
+ else
109
+ @logger.debug "\t\tmatched #{etag}"
110
+ false
111
+ end
112
+ else
113
+ true
114
+ end
115
+ end
116
+
117
+ # ---------------------------------------------------------------------------------------------------------------
118
+ def doDownload(etag, local, object)
119
+ @logger.info "\t\tdownload #{object.size} bytes"
120
+ response = object.get(:response_target => local)
121
+ File.utime(response.last_modified, response.last_modified, local)
122
+ @logger.info "\t\tdone"
123
+ check = calcLocalETag(etag, local)
124
+ if check.eql?(etag)
125
+ false
126
+ else
127
+ @logger.info "\tETag different: #{etag} != #{check}"
128
+ true
129
+ end
130
+ end
131
+
132
+
133
+ class FakeLogger
134
+ def initialize
135
+
136
+ end
137
+
138
+ def method_missing(*args)
139
+ puts "#{args[0]}: #{args[1..-1].join(' ')}"
140
+ end
141
+ end
142
+
143
+ @logger = FakeLogger.new()
144
+ artifact, path = ['', 'https://s3.amazonaws.com/wgen-sto-artifacts/release/com/amplify/learning/enrollment/1.3.0-265']
145
+ local_dir = File.join('/tmp', 'enrollment', '')
146
+ Dir.mkdir(local_dir, 0700) unless File.directory?(local_dir)
147
+ artifacts = []
148
+
149
+ key, name, objects = getObjects(artifact, path)
150
+ # 1 or more objects on the key/ path
151
+ if objects.size > 0
152
+ objects.each do |object|
153
+ @logger.info "\tchecking #{object.key}"
154
+ local = File.join(local_dir, File.basename(object.key))
155
+ etag = object.etag.gsub(%r/['"]/, '')
156
+ download = shouldDownload?(etag, local, object)
157
+ if download
158
+ changed = doDownload(etag, local, object)
159
+ else
160
+ @logger.info "\t\tunchanged"
161
+ end
162
+ artifacts << local
163
+ end
164
+ else
165
+ @logger.fatal "Artifact not found: s3://#{name}/#{key}#{artifact}"
166
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: environments-list-builder
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.8
4
+ version: 0.1.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - Christo De Lange
@@ -16,7 +16,7 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 0.6.8
19
+ version: 0.6.9
20
20
  - - "<"
21
21
  - !ruby/object:Gem::Version
22
22
  version: '1.1'
@@ -26,7 +26,7 @@ dependencies:
26
26
  requirements:
27
27
  - - ">="
28
28
  - !ruby/object:Gem::Version
29
- version: 0.6.8
29
+ version: 0.6.9
30
30
  - - "<"
31
31
  - !ruby/object:Gem::Version
32
32
  version: '1.1'
@@ -161,19 +161,25 @@ dependencies:
161
161
  - !ruby/object:Gem::Version
162
162
  version: '0'
163
163
  - !ruby/object:Gem::Dependency
164
- name: aws-sdk-core
164
+ name: aws-sdk
165
165
  requirement: !ruby/object:Gem::Requirement
166
166
  requirements:
167
- - - "~>"
167
+ - - ">="
168
168
  - !ruby/object:Gem::Version
169
169
  version: '2.0'
170
+ - - "<"
171
+ - !ruby/object:Gem::Version
172
+ version: '2.1'
170
173
  type: :runtime
171
174
  prerelease: false
172
175
  version_requirements: !ruby/object:Gem::Requirement
173
176
  requirements:
174
- - - "~>"
177
+ - - ">="
175
178
  - !ruby/object:Gem::Version
176
179
  version: '2.0'
180
+ - - "<"
181
+ - !ruby/object:Gem::Version
182
+ version: '2.1'
177
183
  - !ruby/object:Gem::Dependency
178
184
  name: dldinternet-mixlib-logging
179
185
  requirement: !ruby/object:Gem::Requirement
@@ -291,10 +297,38 @@ dependencies:
291
297
  description: Jenkins builder of the environments manifest for Continuous Integration/Continuous
292
298
  Delivery artifact promotion style deployments
293
299
  email: rubygems@dldinternet.com
294
- executables: []
300
+ executables:
301
+ - environmentslistbuilder
295
302
  extensions: []
296
303
  extra_rdoc_files: []
297
- files: []
304
+ files:
305
+ - ".gitignore"
306
+ - ".rakeTasks"
307
+ - ".rspec"
308
+ - ChangeLog.md
309
+ - Gemfile
310
+ - Gemfile.lock
311
+ - LICENSE
312
+ - LICENSE.txt
313
+ - README.md
314
+ - Rakefile
315
+ - bin/environmentslistbuilder
316
+ - environments-list-builder.gemspec
317
+ - features/.gitkeep
318
+ - features/manifestrepo-builder.feature
319
+ - features/step_definitions/.gitkeep
320
+ - features/step_definitions/manifestrepo-builder_steps.rb
321
+ - lib/cicd/builder/environments-list.rb
322
+ - lib/cicd/builder/environments-list/mixlib/build.rb
323
+ - lib/cicd/builder/environments-list/mixlib/lib/mixins/no_commands.rb
324
+ - lib/cicd/builder/environments-list/mixlib/lib/update_bucket_policy.rb
325
+ - lib/cicd/builder/environments-list/mixlib/repo.rb
326
+ - lib/cicd/builder/environments-list/mixlib/repo/artifactory.rb
327
+ - lib/cicd/builder/environments-list/version.rb
328
+ - openssl
329
+ - spec/builder_spec.rb
330
+ - spec/spec_helper.rb
331
+ - tests/infinite-etag.rb
298
332
  homepage: https://rubygems.org/gems/environments-list-builder
299
333
  licenses:
300
334
  - Apachev2
@@ -319,4 +353,10 @@ rubygems_version: 2.4.3
319
353
  signing_key:
320
354
  specification_version: 4
321
355
  summary: Jenkins builder task for CI/CD
322
- test_files: []
356
+ test_files:
357
+ - features/.gitkeep
358
+ - features/manifestrepo-builder.feature
359
+ - features/step_definitions/.gitkeep
360
+ - features/step_definitions/manifestrepo-builder_steps.rb
361
+ - spec/builder_spec.rb
362
+ - spec/spec_helper.rb