manifest-builder 0.5.8 → 0.5.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d9a09d37c265bbc389e8d5d8a0b4c7ee3a495a42
-  data.tar.gz: 838a631b3faff67079fba98b268a1f7951547d3c
+  metadata.gz: e60728ab2daaf198e0540b734c90ef98133e6e76
+  data.tar.gz: b1a51a36abf4c4dee729cf74f50f6f4b93d43bd3
 SHA512:
-  metadata.gz: 0ae28e498ac10100dbde4fe533ef17049bfe36233242fd4ee6d024ce1a5ef7615e1a5bd5775e22de13b00eb97a795e8aabf18c059e18e86d82b2a6ba2494f4ac
-  data.tar.gz: 20a3d1960607a42ecc2053ddf32ec8da18eb78d6b359a0cad2271c52da033d75a354fdd4001b2ca1cd241191fea16d4beedbff559c28c1253bec3a0743da6216
+  metadata.gz: 5189f9610b02429a865ed37e417f5d332e228d7eec4e0856347871a9eeb4bfc2dc3566d9302d0c79eb31fa21b8a3c3fea003a641d48b5076fbd068b98bff383a
+  data.tar.gz: 49749634786c6eeb406a3a3c98cc6b85c7fc1288a02cec37b51b6674a60d2b2bae2bb56ffb78a3cf837e6430917aafb4cb2e9dc75da7d39219e76b0fff445ff9

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    manifest-builder (0.5.8)
+    manifest-builder (0.5.10)
       archive-tar-minitar (= 0.5.2)
       chefrepo-builder (>= 0.9.27, < 1.1)
-      cicd-builder (>= 0.9.38, < 1.1)
+      cicd-builder (>= 0.9.40, < 1.1)
       json (>= 1.8.1, < 1.9)
       s3etag (>= 0.0.1, < 0.1.0)
 
@@ -18,7 +18,7 @@ PATH
 PATH
   remote: ../cicd-builder
   specs:
-    cicd-builder (0.9.38)
+    cicd-builder (0.9.40)
       artifactory (>= 2.2.1, < 2.3)
       awesome_print (>= 1.6, < 2.0)
       aws-sdk (>= 2.0, < 2.1)

data/lib/cicd/builder/manifest/version.rb

@@ -2,7 +2,7 @@ module CiCd
   module Builder
     module Manifest
       # manifest-builder version
-      VERSION = '0.5.8'
+      VERSION = '0.5.10'
       MAJOR, MINOR, TINY = VERSION.split('.')
       PATCH = TINY
     end

data/manifest-builder.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ['lib']
 
-  gem.add_dependency 'cicd-builder', '>= 0.9.38', '< 1.1'
+  gem.add_dependency 'cicd-builder', '>= 0.9.40', '< 1.1'
   gem.add_dependency 'chefrepo-builder', '>= 0.9.27', '< 1.1'
   # gem.add_dependency 'ansiblerepo-builder', '>= 0.3.7', '< 0.5'
   gem.add_dependency 'json', '>= 1.8.1', '< 1.9'

data/tests/infinite-etag.rb

@@ -0,0 +1,166 @@
+require 'aws-sdk-core'
+require 'aws-sdk-resources'
+require 'uri'
+
+# ---------------------------------------------------------------------------------------------------------------
+def getS3()
+  region = ENV['AWS_REGION'] || ::Aws.config[:region] || 'us-east-1'
+  unless @s3
+    # noinspection RubyArgCount
+    @s3 = ::Aws::S3::Client.new(region: region)
+  end
+  unless @s3 and ((@s3.config.access_key_id and @s3.config.secret_access_key) or @s3.config.credentials)
+    @logger.warn "Unable to find AWS credentials in standard locations:
+ENV['AWS_ACCESS_KEY'] and ENV['AWS_SECRET_ACCESS_KEY']
+Aws.config[:credentials]
+Shared credentials file, ~/.aws/credentials
+EC2 Instance profile
+"
+    if ENV['AWS_PROFILE']
+      @logger.info "Trying profile '#{ENV['AWS_PROFILE']}' explicitly"
+      creds = Aws::SharedCredentials.new( path: File.expand_path('~/.aws/credentials'), profile: ENV['AWS_PROFILE'] )
+      if creds.loadable?
+        # noinspection RubyArgCount
+        @s3 = ::Aws::S3::Client.new(region: region, credentials: creds)
+      end
+    else
+      @logger.warn 'No AWS_PROFILE defined'
+    end
+  end
+  unless @s3 and ((@s3.config.access_key_id and @s3.config.secret_access_key) or @s3.config.credentials)
+    raise 'Unable to find AWS credentials!'
+  end
+  @s3
+end
+
+# ---------------------------------------------------------------------------------------------------------------
+def getBucket(name = nil)
+  @s3 = getS3()
+  begin
+    ::Aws::S3::Bucket.new(name: name || ENV['AWS_S3_BUCKET'], client: @s3)
+  rescue Aws::S3::Errors::NotFound
+    @vars[:return_code] = Errors::BUCKET
+    nil
+  rescue Exception => e
+    @logger.error "S3 Bucket resource API error: #{e.class.name} #{e.message}"
+    raise e
+  end
+end
+
+# ---------------------------------------------------------------------------------------------------------------
+def getObjects(artifact, path)
+  parts = URI(path).path.gsub(%r'^#{File::SEPARATOR}', '').split(File::SEPARATOR)
+  name = parts.shift
+  bucket = getBucket(name)
+  key = File.join(parts, '')
+  @logger.info "S3://#{name}:#{key} URL: #{path} #{artifact}"
+  objects = []
+  bucket.objects(prefix: key).each do |object|
+    if artifact.empty? or (not artifact.empty? and object.key =~ %r'#{key}#{artifact}')
+      objects << object
+    end
+  end
+  @logger.debug "S3://#{name}:#{key} has #{objects.size} objects"
+  return key, name, objects
+end
+
+# ---------------------------------------------------------------------------------------------------------------
+def calcLocalETag(etag, local, size = nil)
+  if size == nil
+    stat = File.stat(local)
+    size = stat.size
+  end
+  @logger.debug "Calculate etag to match #{etag}"
+  match = etag.match(%r'-(\d+)$')
+  check = if match
+            require 's3etag'
+            parts = match[1].to_i
+            chunk = size.to_f / parts.to_f
+            mbs = (chunk.to_f / 1024 /1024 + 0.5).to_i
+            part_size = mbs * 1024 * 1024
+            chkit = S3Etag.calc(file: local, threshold: part_size, min_part_size: part_size, max_parts: parts)
+            @logger.debug "S3Etag Calculated #{chkit} : (#{size} / #{part_size}) <= #{parts}"
+            chunks = size / part_size
+            while chkit != etag and chunks <= parts and chunks > 0 and (size > part_size)
+              # Go one larger if a modulus remains and we have the right number of parts
+              mbs += 1
+              part_size = mbs * 1024 * 1024
+              chunks = size.to_f / part_size
+              chkit = S3Etag.calc(file: local, threshold: part_size, min_part_size: part_size, max_parts: parts)
+              @logger.debug "S3Etag Calculated #{chkit} : (#{size} / #{part_size}) <= #{parts}"
+            end
+            raise "Unable to match etag #{etag}!" if chkit != etag
+            chkit
+          else
+            Digest::MD5.file(local).hexdigest
+          end
+end
+
+# ---------------------------------------------------------------------------------------------------------------
+def shouldDownload?(etag, local, object)
+  if File.exists?(local)
+    @logger.debug "\t\tchecking etag on #{local}"
+    stat = File.stat(local)
+    check = calcLocalETag(etag, local, stat.size)
+    if etag != check or object.size != stat.size or object.last_modified > stat.mtime
+      @logger.debug "\t\t#{etag} != \"#{check}\" #{object.size} != #{stat.size} #{object.last_modified} > #{stat.mtime}"
+      true
+    else
+      @logger.debug "\t\tmatched #{etag}"
+      false
+    end
+  else
+    true
+  end
+end
+
+# ---------------------------------------------------------------------------------------------------------------
+def doDownload(etag, local, object)
+  @logger.info "\t\tdownload #{object.size} bytes"
+  response = object.get(:response_target => local)
+  File.utime(response.last_modified, response.last_modified, local)
+  @logger.info "\t\tdone"
+  check = calcLocalETag(etag, local)
+  if check.eql?(etag)
+    false
+  else
+    @logger.info "\tETag different: #{etag} != #{check}"
+    true
+  end
+end
+
+
+class FakeLogger
+  def initialize
+
+  end
+
+  def method_missing(*args)
+    puts "#{args[0]}: #{args[1..-1].join(' ')}"
+  end
+end
+
+@logger = FakeLogger.new()
+artifact, path = ['', 'https://s3.amazonaws.com/wgen-sto-artifacts/release/com/amplify/learning/enrollment/1.3.0-265']
+local_dir = File.join('/tmp', 'enrollment', '')
+Dir.mkdir(local_dir, 0700) unless File.directory?(local_dir)
+artifacts = []
+
+key, name, objects = getObjects(artifact, path)
+# 1 or more objects on the key/ path
+if objects.size > 0
+  objects.each do |object|
+    @logger.info "\tchecking #{object.key}"
+    local = File.join(local_dir, File.basename(object.key))
+    etag = object.etag.gsub(%r/['"]/, '')
+    download = shouldDownload?(etag, local, object)
+    if download
+      changed = doDownload(etag, local, object)
+    else
+      @logger.info "\t\tunchanged"
+    end
+    artifacts << local
+  end
+else
+  @logger.fatal "Artifact not found: s3://#{name}/#{key}#{artifact}"
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: manifest-builder
 version: !ruby/object:Gem::Version
-  version: 0.5.8
+  version: 0.5.10
 platform: ruby
 authors:
 - Christo De Lange
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.38
+        version: 0.9.40
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.1'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.38
+        version: 0.9.40
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.1'
@@ -235,6 +235,7 @@ files:
 - manifest-builder.gemspec
 - spec/builder_spec.rb
 - spec/spec_helper.rb
+- tests/infinite-etag.rb
 homepage: https://rubygems.org/gems/manifest-builder
 licenses:
 - Apachev2