environments-list-builder 0.1.8 → 0.1.9

data/lib/cicd/builder/environments-list/mixlib/repo.rb

@@ -0,0 +1,49 @@
+require 'json'
+
+module CiCd
+	module Builder
+    # noinspection RubySuperCallWithoutSuperclassInspection
+    module EnvironmentsList
+      module Repo
+        require 'cicd/builder/mixlib/repo/base'
+        require 'cicd/builder/mixlib/repo/S3'
+        # noinspection RubyResolve
+        if ENV.has_key?('REPO_TYPE') and (not ENV['REPO_TYPE'].capitalize.eql?('S3'))
+          require "cicd/builder/environments-list/mixlib/repo/#{ENV['REPO_TYPE'].downcase}"
+        end
+
+        # ---------------------------------------------------------------------------------------------------------------
+        def getRepoClass(type = nil)
+          @logger.step CLASS+'::'+__method__.to_s
+          if type.nil?
+            type ||= 'S3'
+            if ENV.has_key?('REPO_TYPE')
+              type = ENV['REPO_TYPE']
+            end
+          end
+
+          @logger.info "#{type} repo interface"
+          clazz = begin
+                    Object.const_get("#{self.class.name.gsub(%r'::\w+$', '')}::Repo::#{type}")
+                  rescue NameError => e
+                    begin
+                      # Object.const_get("#{self.class.name.gsub(%r'::\w+$', '')}::Repo::#{type}")
+                      Object.const_get("CiCd::Builder::Manifest::Repo::#{type}")
+                    rescue NameError #=> e
+                      Object.const_get("CiCd::Builder::Repo::#{type}")
+                    end
+                  end
+
+          if block_given?
+            if clazz.is_a?(Class) and not clazz.nil?
+              yield
+            end
+          end
+
+          clazz
+        end
+
+      end
+    end
+  end
+end

data/lib/cicd/builder/environments-list/mixlib/repo/artifactory.rb

@@ -0,0 +1,49 @@
+require 'artifactory'
+require 'tmpdir'
+require "cicd/builder/manifest/mixlib/repo/artifactory"
+
+module CiCd
+  module Builder
+    # noinspection RubySuperCallWithoutSuperclassInspection
+    module EnvironmentsList
+      module Repo
+        class Artifactory < CiCd::Builder::Manifest::Repo::Artifactory
+
+          alias_method :super_uploadToRepo, :uploadToRepo
+          # ---------------------------------------------------------------------------------------------------------------
+          def uploadToRepo(artifacts)
+            @logger.step CLASS+'::'+__method__.to_s
+            # super_uploadToRepo(artifacts) get's the immediate parent class
+            cicd_uploadToRepo(artifacts)
+            # CiCd::Builder::Repo::Artifactory.instance_method(:uploadToRepo).bind(self).call(artifacts)
+            if @vars[:environments][:changed]
+              data = {
+                         name: ENV['ARTIFACTORY_ENVIRONMENTS_MODULE'],
+                       module: ENV['ARTIFACTORY_ENVIRONMENTS_MODULE'],
+                         file: @vars[:environments][:file],
+                      version: @vars[:environments][:version],
+                        build: @vars[:build_num],
+                   properties: @properties_matrix,
+                         temp: false,
+                         sha1: Digest::SHA1.file(@vars[:environments][:file]).hexdigest,
+                          md5: Digest::MD5.file(@vars[:environments][:file]).hexdigest,
+              }
+
+              cicd_maybeUploadArtifactoryObject(
+                              data: data,
+                   artifact_module: data[:module],
+                  artifact_version: data[:version],
+                         file_name: '',
+                          file_ext: File.extname(data[:file]).gsub(/^\./,''),
+                              repo: ENV['ARTIFACTORY_RELEASE_REPO'],
+                              copy: false
+              )
+            end
+            @vars[:return_code]
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/cicd/builder/environments-list/version.rb

@@ -0,0 +1,10 @@
+module CiCd
+  module Builder
+    module EnvironmentsList
+      # environments-list-builder version
+      VERSION = '0.1.9'
+      MAJOR, MINOR, TINY = VERSION.split('.')
+      PATCH = TINY
+    end
+  end
+end

data/spec/builder_spec.rb

@@ -0,0 +1,8 @@
+require 'spec_helper'
+require 'cicd/builder'
+
+describe ManifestBuilder::Builder do
+  it "should have a VERSION constant" do
+    subject.const_get('VERSION').should_not be_empty
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+require 'rspec'
+require 'cicd/builder/version'
+
+include ManifestBuilder::Builder

data/tests/infinite-etag.rb

@@ -0,0 +1,166 @@
+require 'aws-sdk-core'
+require 'aws-sdk-resources'
+require 'uri'
+
+# ---------------------------------------------------------------------------------------------------------------
+def getS3()
+  region = ENV['AWS_REGION'] || ::Aws.config[:region] || 'us-east-1'
+  unless @s3
+    # noinspection RubyArgCount
+    @s3 = ::Aws::S3::Client.new(region: region)
+  end
+  unless @s3 and ((@s3.config.access_key_id and @s3.config.secret_access_key) or @s3.config.credentials)
+    @logger.warn "Unable to find AWS credentials in standard locations:
+ENV['AWS_ACCESS_KEY'] and ENV['AWS_SECRET_ACCESS_KEY']
+Aws.config[:credentials]
+Shared credentials file, ~/.aws/credentials
+EC2 Instance profile
+"
+    if ENV['AWS_PROFILE']
+      @logger.info "Trying profile '#{ENV['AWS_PROFILE']}' explicitly"
+      creds = Aws::SharedCredentials.new( path: File.expand_path('~/.aws/credentials'), profile: ENV['AWS_PROFILE'] )
+      if creds.loadable?
+        # noinspection RubyArgCount
+        @s3 = ::Aws::S3::Client.new(region: region, credentials: creds)
+      end
+    else
+      @logger.warn 'No AWS_PROFILE defined'
+    end
+  end
+  unless @s3 and ((@s3.config.access_key_id and @s3.config.secret_access_key) or @s3.config.credentials)
+    raise 'Unable to find AWS credentials!'
+  end
+  @s3
+end
+
+# ---------------------------------------------------------------------------------------------------------------
+def getBucket(name = nil)
+  @s3 = getS3()
+  begin
+    ::Aws::S3::Bucket.new(name: name || ENV['AWS_S3_BUCKET'], client: @s3)
+  rescue Aws::S3::Errors::NotFound
+    @vars[:return_code] = Errors::BUCKET
+    nil
+  rescue Exception => e
+    @logger.error "S3 Bucket resource API error: #{e.class.name} #{e.message}"
+    raise e
+  end
+end
+
+# ---------------------------------------------------------------------------------------------------------------
+def getObjects(artifact, path)
+  parts = URI(path).path.gsub(%r'^#{File::SEPARATOR}', '').split(File::SEPARATOR)
+  name = parts.shift
+  bucket = getBucket(name)
+  key = File.join(parts, '')
+  @logger.info "S3://#{name}:#{key} URL: #{path} #{artifact}"
+  objects = []
+  bucket.objects(prefix: key).each do |object|
+    if artifact.empty? or (not artifact.empty? and object.key =~ %r'#{key}#{artifact}')
+      objects << object
+    end
+  end
+  @logger.debug "S3://#{name}:#{key} has #{objects.size} objects"
+  return key, name, objects
+end
+
+# ---------------------------------------------------------------------------------------------------------------
+def calcLocalETag(etag, local, size = nil)
+  if size == nil
+    stat = File.stat(local)
+    size = stat.size
+  end
+  @logger.debug "Calculate etag to match #{etag}"
+  match = etag.match(%r'-(\d+)$')
+  check = if match
+            require 's3etag'
+            parts = match[1].to_i
+            chunk = size.to_f / parts.to_f
+            mbs = (chunk.to_f / 1024 /1024 + 0.5).to_i
+            part_size = mbs * 1024 * 1024
+            chkit = S3Etag.calc(file: local, threshold: part_size, min_part_size: part_size, max_parts: parts)
+            @logger.debug "S3Etag Calculated #{chkit} : (#{size} / #{part_size}) <= #{parts}"
+            chunks = size / part_size
+            while chkit != etag and chunks <= parts and chunks > 0 and (size > part_size)
+              # Go one larger if a modulus remains and we have the right number of parts
+              mbs += 1
+              part_size = mbs * 1024 * 1024
+              chunks = size.to_f / part_size
+              chkit = S3Etag.calc(file: local, threshold: part_size, min_part_size: part_size, max_parts: parts)
+              @logger.debug "S3Etag Calculated #{chkit} : (#{size} / #{part_size}) <= #{parts}"
+            end
+            #raise "Unable to match etag #{etag}!" if chkit != etag
+            chkit
+          else
+            Digest::MD5.file(local).hexdigest
+          end
+end
+
+# ---------------------------------------------------------------------------------------------------------------
+def shouldDownload?(etag, local, object)
+  if File.exists?(local)
+    @logger.debug "\t\tchecking etag on #{local}"
+    stat = File.stat(local)
+    check = calcLocalETag(etag, local, stat.size)
+    if etag != check or object.size != stat.size or object.last_modified > stat.mtime
+      @logger.debug "\t\t#{etag} != \"#{check}\" #{object.size} != #{stat.size} #{object.last_modified} > #{stat.mtime}"
+      true
+    else
+      @logger.debug "\t\tmatched #{etag}"
+      false
+    end
+  else
+    true
+  end
+end
+
+# ---------------------------------------------------------------------------------------------------------------
+def doDownload(etag, local, object)
+  @logger.info "\t\tdownload #{object.size} bytes"
+  response = object.get(:response_target => local)
+  File.utime(response.last_modified, response.last_modified, local)
+  @logger.info "\t\tdone"
+  check = calcLocalETag(etag, local)
+  if check.eql?(etag)
+    false
+  else
+    @logger.info "\tETag different: #{etag} != #{check}"
+    true
+  end
+end
+
+
+class FakeLogger
+  def initialize
+
+  end
+
+  def method_missing(*args)
+    puts "#{args[0]}: #{args[1..-1].join(' ')}"
+  end
+end
+
+@logger = FakeLogger.new()
+artifact, path = ['', 'https://s3.amazonaws.com/wgen-sto-artifacts/release/com/amplify/learning/enrollment/1.3.0-265']
+local_dir = File.join('/tmp', 'enrollment', '')
+Dir.mkdir(local_dir, 0700) unless File.directory?(local_dir)
+artifacts = []
+
+key, name, objects = getObjects(artifact, path)
+# 1 or more objects on the key/ path
+if objects.size > 0
+  objects.each do |object|
+    @logger.info "\tchecking #{object.key}"
+    local = File.join(local_dir, File.basename(object.key))
+    etag = object.etag.gsub(%r/['"]/, '')
+    download = shouldDownload?(etag, local, object)
+    if download
+      changed = doDownload(etag, local, object)
+    else
+      @logger.info "\t\tunchanged"
+    end
+    artifacts << local
+  end
+else
+  @logger.fatal "Artifact not found: s3://#{name}/#{key}#{artifact}"
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: environments-list-builder
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors:
 - Christo De Lange
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.8
+        version: 0.6.9
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.1'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.8
+        version: 0.6.9
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.1'
@@ -161,19 +161,25 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-core
+  name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '2.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '2.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: dldinternet-mixlib-logging
   requirement: !ruby/object:Gem::Requirement
@@ -291,10 +297,38 @@ dependencies:
 description: Jenkins builder of the environments manifest for Continuous Integration/Continuous
   Delivery artifact promotion style deployments
 email: rubygems@dldinternet.com
-executables: []
+executables:
+- environmentslistbuilder
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- ".gitignore"
+- ".rakeTasks"
+- ".rspec"
+- ChangeLog.md
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/environmentslistbuilder
+- environments-list-builder.gemspec
+- features/.gitkeep
+- features/manifestrepo-builder.feature
+- features/step_definitions/.gitkeep
+- features/step_definitions/manifestrepo-builder_steps.rb
+- lib/cicd/builder/environments-list.rb
+- lib/cicd/builder/environments-list/mixlib/build.rb
+- lib/cicd/builder/environments-list/mixlib/lib/mixins/no_commands.rb
+- lib/cicd/builder/environments-list/mixlib/lib/update_bucket_policy.rb
+- lib/cicd/builder/environments-list/mixlib/repo.rb
+- lib/cicd/builder/environments-list/mixlib/repo/artifactory.rb
+- lib/cicd/builder/environments-list/version.rb
+- openssl
+- spec/builder_spec.rb
+- spec/spec_helper.rb
+- tests/infinite-etag.rb
 homepage: https://rubygems.org/gems/environments-list-builder
 licenses:
 - Apachev2
@@ -319,4 +353,10 @@ rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: Jenkins builder task for CI/CD
-test_files: []
+test_files:
+- features/.gitkeep
+- features/manifestrepo-builder.feature
+- features/step_definitions/.gitkeep
+- features/step_definitions/manifestrepo-builder_steps.rb
+- spec/builder_spec.rb
+- spec/spec_helper.rb