manifest-builder 0.6.2 → 0.6.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0b10b3d8ea6cfc58960b69110ceb02434f017b35
-  data.tar.gz: 1d2dc07ea272e2d8ffde9016586873cd5e834350
+  metadata.gz: 4f8692350e883642c61400675d5b04780b553a5b
+  data.tar.gz: 61d48a5874ff2b1c8aed3bd1c4accda5a1d6032c
 SHA512:
-  metadata.gz: f545e10d18b032a1c4f2de06f63f684cff93e2d1fd3d9ddea06e4c976a7f7192e73d6bce5be5b3ff76300f76ddda5a093308c55b472bbf1e287a43f5283781cd
-  data.tar.gz: 6e88d6c42454c00614ced3186896e6744682e451c8557e8f482b26c1b69f36664aa5c951cf75704873247dbb5ed67b69512c5594ab483f46cd285644a2ab8d1f
+  metadata.gz: 06cba7f7f72e817a3343664c3c8cf9dd9218a0006d71a0b35e6722c09685771bc839f48ab5c9377df8f898b2ada4acb344531fd9ffc6648d69d08f09deb015e2
+  data.tar.gz: 32ecf4772c0abb9180b9f965a6cbec2d481786cd6d55fdc8dc60e3f08c06d0444d4815f4809347d5913061c769d93573982f1d66543b3801c2467ecccc89e20c

data/Gemfile.lock

@@ -1,23 +1,23 @@
 PATH
   remote: .
   specs:
-    manifest-builder (0.6.2)
+    manifest-builder (0.6.4)
       archive-tar-minitar (= 0.5.2)
-      chefrepo-builder (>= 0.9.32, < 1.1)
+      chefrepo-builder (>= 0.9.34, < 1.1)
       json (>= 1.8.1, < 1.9)
       s3etag (>= 0.0.1, < 0.1.0)
 
 PATH
   remote: ../chefrepo-builder
   specs:
-    chefrepo-builder (0.9.32)
-      cicd-builder (>= 0.9.47, < 1.1)
+    chefrepo-builder (0.9.34)
+      cicd-builder (>= 0.9.50, < 1.1)
       json (= 1.8.1)
 
 PATH
   remote: ../cicd-builder
   specs:
-    cicd-builder (0.9.48)
+    cicd-builder (0.9.50)
       artifactory (>= 2.2.1, < 2.3)
       awesome_print (>= 1.6, < 2.0)
       aws-sdk (>= 2.0, < 2.1)
@@ -26,6 +26,8 @@ PATH
       inifile (>= 3.0.0, < 3.1)
       json (>= 1.8.1, < 1.9)
       logging (>= 1.8.2, < 1.9)
+      parallel (>= 1.4.1, < 1.5)
+      ruby-progressbar (>= 1.7.5, < 2)
       semverse (>= 1.2.1, < 1.3)
       yajl-ruby (>= 1.2.1, < 1.3)
 
@@ -35,14 +37,14 @@ GEM
     archive-tar-minitar (0.5.2)
     artifactory (2.2.1)
     awesome_print (1.6.1)
-    aws-sdk (2.0.39)
-      aws-sdk-resources (= 2.0.39)
-    aws-sdk-core (2.0.39)
+    aws-sdk (2.0.41)
+      aws-sdk-resources (= 2.0.41)
+    aws-sdk-core (2.0.41)
       builder (~> 3.0)
       jmespath (~> 1.0)
       multi_json (~> 1.0)
-    aws-sdk-resources (2.0.39)
-      aws-sdk-core (= 2.0.39)
+    aws-sdk-resources (2.0.41)
+      aws-sdk-core (= 2.0.41)
     builder (3.2.2)
     chef (12.0.3)
       chef-zero (~> 3.2)
@@ -95,7 +97,7 @@ GEM
       little-plugger (>= 1.1.3)
       multi_json (>= 1.8.4)
     method_source (0.8.2)
-    mime-types (2.4.3)
+    mime-types (2.5)
     mixlib-authentication (1.3.0)
       mixlib-log
     mixlib-cli (1.5.0)
@@ -109,7 +111,7 @@ GEM
     net-ssh-multi (1.2.1)
       net-ssh (>= 2.6.5)
       net-ssh-gateway (>= 1.2.0)
-    ohai (8.2.0)
+    ohai (8.3.0)
       ffi (~> 1.9)
       ffi-yajl (>= 1.1, < 3.0)
       ipaddress
@@ -121,6 +123,7 @@ GEM
       rake (~> 10.1)
       systemu (~> 2.6.4)
       wmi-lite (~> 1.0)
+    parallel (1.4.1)
     plist (3.1.0)
     pry (0.10.1)
       coderay (~> 1.1.0)
@@ -136,6 +139,7 @@ GEM
     rspec-expectations (2.99.2)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.99.3)
+    ruby-progressbar (1.7.5)
     rubygems-tasks (0.2.4)
     s3etag (0.0.1)
     semverse (1.2.1)

data/lib/cicd/builder/manifest.rb

@@ -8,6 +8,7 @@ module CiCd
     require 'cicd/builder/manifest/version'
 
     module Manifest
+      CLASS = 'CiCd::Builder::Manifest'
       class Runner < ChefRepo::Runner
         require 'cicd/builder/manifest/mixlib/build'
         require 'cicd/builder/manifest/mixlib/repo'

data/lib/cicd/builder/manifest/mixlib/build.rb

@@ -9,6 +9,7 @@ module CiCd
         # ---------------------------------------------------------------------------------------------------------------
         # noinspection RubyHashKeysTypesInspection
         def prepareBuild()
+          @logger.step CLASS+'::'+__method__.to_s
           ret = super
           if ret == 0
             @vars[:artifacts] = []
@@ -157,33 +158,22 @@ module CiCd
 
         # ---------------------------------------------------------------------------------------------------------------
         def packageBuild()
-          @logger.step __method__.to_s
+          @logger.info CLASS+'::'+__method__.to_s
           if isSameDirectory(Dir.pwd, ENV['WORKSPACE'])
             if @vars.has_key?(:components) and not @vars[:components].empty?
               @vars[:return_code] = 0
 
-              clazz = getRepoClass('S3')
-              @logger.debug "Repo class == '#{clazz}'"
-              if clazz.is_a?(Class) and not clazz.nil?
-                @repo = clazz.new(self)
-
+              getRepoInstance('S3')
+              if 0 == @vars[:return_code]
+                lines             = []
+                @vars[:artifacts] = []
+                # Deal with all artifacts of each component
+                @vars[:components].each { |comp|
+                  processComponent(comp, lines)
+                }
                 if @vars[:return_code] == 0
-                  lines             = []
-                  @vars[:artifacts] = []
-                  # Deal with all artifacts of each component
-                  @vars[:components].each { |comp|
-                    processComponent(comp, lines)
-                  }
-                  if @vars[:return_code] == 0
-                    cleanupAfterPackaging(lines)
-                  end
-
-                else
-                  @logger.fatal "S3 repo error: Bucket #{ENV['AWS_S3_BUCKET']}"
+                  cleanupAfterPackaging(lines)
                 end
-              else
-                @logger.error "CiCd::Builder::Repo::#{type} is not a valid repo class"
-                @vars[:return_code] = Errors::BUILDER_REPO_TYPE
               end
             else
               @logger.error 'No components found during preparation?'
@@ -217,7 +207,7 @@ module CiCd
           end
         end
 
-        private
+        protected
 
         # ---------------------------------------------------------------------------------------------------------------
         def processComponent(comp, lines)
@@ -227,7 +217,7 @@ module CiCd
           require 'uri'
           require 'digest'
           begin
-            key, name, objects = getObjects(artifact, path)
+            key, bucket, objects = getObjects(artifact, path,cdata)
             local_dir = File.join(@vars[:local_dirs]['artifacts'], cname, '')
             Dir.mkdir(local_dir, 0700) unless File.directory?(local_dir)
             artifacts = []
@@ -244,7 +234,7 @@ module CiCd
                 etag = object.etag.gsub(%r/['"]/, '')
                 download = shouldDownload?(etag, local, object)
                 if download
-                  changed = doDownload(etag, local, object)
+                  changed = doDownload(etag, local, object, bucket)
                 else
                   @logger.info "\t\tunchanged"
                 end
@@ -261,7 +251,7 @@ module CiCd
                 artifacts.each do |local|
                   base = File.basename(local)
                   if sha256[base].is_a?(String)
-                    msg = "Artifact checksum is invalid or manifest is incorrect. Artifact: s3://#{name}/#{key}#{artifact}, SHA256: Manifest=#{cdata[:sha256]}, Actual=#{sha256[base]}"
+                    msg = "Artifact checksum is invalid or manifest is incorrect. Artifact: s3://#{bucket}/#{key}#{artifact}, SHA256: Manifest=#{cdata[:sha256]}, Actual=#{sha256[base]}"
                     if ENV['ENFORCE_CHECKSUMS'] and ENV['ENFORCE_CHECKSUMS'].downcase =~ %r/^(yes|on|set|1|enable|active|enforced?)$/
                       @logger.fatal msg
                       @vars[:return_code] = Errors::ARTIFACT_CHECKSUM_BAD
@@ -280,12 +270,14 @@ module CiCd
                                                                            build: build,
                                                                          version: version,
                                                                             file: local,
-                                                                          sha256: (sha256.keys.include?(base)) ? (sha256[base].is_a?(TrueClass) ? cdata[:sha256] : sha256[base]) : Digest::SHA256.file(local).hexdigest
+                                                                          sha256: (sha256.keys.include?(base)) ? (sha256[base].is_a?(TrueClass) ? cdata[:sha256] : sha256[base]) : Digest::SHA256.file(local).hexdigest,
+                                                                       file_name: cdata[:file_name],
+                                                                        file_ext: cdata[:file_ext],
                                                                    })
                 end
               end
             else
-              @logger.fatal "Artifact not found: s3://#{name}/#{key}#{artifact}"
+              @logger.fatal "Artifact not found: s3://#{bucket}/#{key}#{artifact}"
               @vars[:return_code] = Errors::ARTIFACT_NOT_FOUND
             end
           rescue Exception => e
@@ -295,26 +287,38 @@ module CiCd
         end
 
         # ---------------------------------------------------------------------------------------------------------------
-        def getObjects(artifact, path)
+        def getObjects(artifact, path, cdata)
           parts = URI(path).path.gsub(%r'^#{File::SEPARATOR}', '').split(File::SEPARATOR)
-          name = parts.shift
-          bucket = getBucket(name)
-          key = File.join(parts, '')
-          @logger.info "S3://#{name}:#{key} URL: #{path} #{artifact}"
+          bucket,prefix = if cdata[:s3_bucket] and cdata[:s3_key]
+                            [cdata[:s3_bucket],File.join(File.dirname(cdata[:s3_key]),'')]
+                          else
+                            name = parts.shift
+                            [name,File.join(parts, '')]
+                          end
+          @logger.info "S3://#{bucket}:#{prefix} URL: #{path} #{artifact}"
           objects = []
-          bucket.objects(prefix: key).each do |object|
-            if artifact.empty? or (not artifact.empty? and object.key =~ %r'#{key}#{artifact}')
-              objects << object
+          @s3 = @repo.getS3() #bucket_client = getBucket(bucket)
+          bucket_objects = @s3.list_objects(bucket: bucket, prefix: prefix)
+          if bucket_objects
+            bucket_objects = bucket_objects[:contents]
+            bucket_objects.each do |object|
+              if artifact.empty? or (not artifact.empty? and object.key =~ %r'#{prefix}#{artifact}')
+                objects << object
+              end
             end
           end
-          @logger.debug "S3://#{name}:#{key} has #{objects.size} objects"
-          return key, name, objects
+          @logger.debug "S3://#{bucket}:#{prefix} has #{objects.size} objects"
+          return prefix, bucket, objects
         end
 
         # ---------------------------------------------------------------------------------------------------------------
-        def doDownload(etag, local, object)
+        def doDownload(etag, local, object, bucket)
           @logger.info "\t\tdownload #{object.size} bytes"
-          response = object.get(:response_target => local)
+          begin
+            response = object.get(:response_target => local)
+          rescue NoMethodError
+            response = @s3.get_object(bucket: bucket, key: object.key, response_target: local)
+          end
           File.utime(response.last_modified, response.last_modified, local)
           @logger.info "\t\tdone"
           check = calcLocalETag(etag, local)
@@ -374,7 +378,7 @@ module CiCd
           cname,cdata = comp
           if cdata[:url]
             path, artifact = File.split(cdata[:url])
-            version, build = getVersionBuild(path, artifact, comp)
+            version, build = (cdata[:version] and cdata[:build]) ? [cdata[:version], cdata[:build]] : getVersionBuild(path, artifact, comp)
           elsif cdata[:base_url]
             artifact = ''
             if cdata[:build].nil?
@@ -389,7 +393,7 @@ module CiCd
           else
             path           = ''
             artifact       = ''
-            version, build = getVersionBuild(path, artifact, comp)
+            version, build = (cdata[:version] and cdata[:build]) ? [cdata[:version], cdata[:build]] : getVersionBuild(path, artifact, comp)
           end
           return artifact, path, version, build
         end

data/lib/cicd/builder/manifest/mixlib/repo.rb

@@ -14,7 +14,7 @@ module CiCd
 
         # ---------------------------------------------------------------------------------------------------------------
         def getRepoClass(type = nil)
-          @logger.info __method__.to_s
+          @logger.step CLASS+'::'+__method__.to_s
           if type.nil?
             type ||= 'S3'
             if ENV.has_key?('REPO_TYPE')

data/lib/cicd/builder/manifest/mixlib/repo/artifactory.rb

@@ -18,16 +18,15 @@ module CiCd
           def uploadToRepo(artifacts)
             @manifest = {}
             super
-            if @vars[:return_code] == 0
-              # Preserve the manifest (maybeUploadArtifactoryObject will add everything we upload to the instance var)
-              manifest = @manifest.dup
-              # Create a manifest for each product and store it.
-              createProductManifests(manifest)
-              # Get a super manifest of all products and store as learning-manifest
-              createSuperManifest(manifest) # -#{@vars[:variant]}
-              @manifest = manifest
-            end
+            handleManifests()
             # If we are asked to produce a PROJECTS_FILE then we do that from the manifest and components.
+            handleProjectsFile()
+            cleanupTempFiles()
+            @vars[:return_code]
+          end
+
+          # ---------------------------------------------------------------------------------------------------------------
+          def handleProjectsFile
             unless ENV['PROJECTS_FILE'].nil?
               if @vars[:return_code] == 0
                 if File.directory?(File.realdirpath(File.dirname(ENV['PROJECTS_FILE'])))
@@ -38,19 +37,19 @@ module CiCd
                 end
               end
             end
-            @vars[:artifacts].each do |art|
-              if art[:data][:temp].is_a?(FalseClass)
-                if File.exists?(art[:data][:file])
-                  File.unlink(art[:data][:file]) if File.exists?(art[:data][:file])
-                  art[:data].delete(:file)
-                  art[:data].delete(:temp)
-                else
-                  @logger.warn "Temporary file disappeared: #{data.ai}"
-                  @vars[:return_code] = Errors::TEMP_FILE_MISSING
-                end
-              end
+          end
+
+          # ---------------------------------------------------------------------------------------------------------------
+          def handleManifests
+            if @vars[:return_code] == 0
+              # Preserve the manifest (maybeUploadArtifactoryObject will add everything we upload to the instance var)
+              manifest = @manifest.dup
+              # Create a manifest for each product and store it.
+              createProductManifests(manifest)
+              # Get a super manifest of all products and store as learning-manifest
+              createSuperManifest(manifest) # -#{@vars[:variant]}
+              @manifest = manifest
             end
-            @vars[:return_code]
           end
 
           # ---------------------------------------------------------------------------------------------------------------
@@ -191,12 +190,16 @@ module CiCd
           end
 
           def createSuperManifest(manifest)
-            manifest_data = ''
-            manifest.each do |mod, man|
-              man.each do |k, v|
-                manifest_data += "#{k}=#{v}\n"
+            assemble = lambda{|mani|
+              manifest_data = ''
+              mani.each do |_, man|
+                man.each do |k, v|
+                  manifest_data += "#{k}=#{v}\n"
+                end
               end
-            end
+              manifest_data
+            }
+            manifest_data = assemble.call(manifest)
             amn = artifactory_manifest_name # Just using a local iso invoking method_missing repeatedly ... ;)
             data = {module: amn, data: manifest_data, version: @vars[:build_ver], build: @vars[:build_num], properties: @properties_matrix}
             # tempArtifactFile(amn, data)
@@ -215,18 +218,22 @@ module CiCd
                                     key: amn,
                                    data: data.dup,
                                  }
-            # manifest[amn]={ amn => "#{@vars[:build_ver]}-#{@vars[:build_num]}" }
-
-            maybeUploadArtifactoryObject(data: data, artifact_module: amn, artifact_version: data[:version] || @vars[:version], file_name: '', file_ext: 'properties')
+            manifest = maybeUpdateManifest(data, manifest, assemble, amn)
+            if 0 == @vars[:return_code]
+              maybeUploadArtifactoryObject(data: data, artifact_module: amn, artifact_version: data[:version] || @vars[:version], file_name: '', file_ext: 'properties')
+            end
           end
-
           # ---------------------------------------------------------------------------------------------------------------
           def createProductManifests(manifest)
             manifest.dup.each do |mod, man|
-              manifest_data = ''
-              man.each do |k, v|
-                manifest_data += "#{k}=#{v}\n"
-              end
+              assemble = lambda{ |mani|
+                manifest_data = ''
+                mani.each do |k, v|
+                  manifest_data += "#{k}=#{v}\n"
+                end
+                manifest_data
+              }
+              manifest_data = assemble.call(man)
               data = {
                             name: "#{mod}-manifest",
                           module: "#{mod}-manifest",
@@ -236,7 +243,6 @@ module CiCd
                            build: @vars[:build_num],
                       properties: @properties_matrix
               }
-              # tempArtifactFile("#{mod}-manifest", data)
 
               data[:file] = Dir::Tmpname.create("#{mod}-manifest") do |tmpnam, n, opts|
                 mode = File::RDWR|File::CREAT|File::EXCL
@@ -248,14 +254,72 @@ module CiCd
               data[:sha1] = Digest::SHA1.file(data[:file]).hexdigest
               data[:md5]  = Digest::MD5.file(data[:file]).hexdigest
               @vars[:artifacts] << {
-                                       key: "#{mod}-manifest",
-                                      data: data.dup,
-                                   }
-              # noinspection RubyStringKeysInHashInspection
-              # manifest["#{mod}-manifest"]={ "#{mod}-manifest" => "#{@vars[:build_ver]}-#{@vars[:build_num]}" }
+                  key: "#{mod}-manifest",
+                  data: data.dup,
+              }
+
+              maybeUpdateManifest(data, man, assemble)
+              if 0 == @vars[:return_code]
+                maybeUploadArtifactoryObject(data: data, artifact_module: data[:name], artifact_version: data[:version] || @vars[:version], file_name: '', file_ext: 'properties') # -#{@vars[:variant]}
+              end
+            end
+          end
 
-              maybeUploadArtifactoryObject(data: data, artifact_module: data[:name], artifact_version: data[:version] || @vars[:version], file_name: '', file_ext: 'properties') # -#{@vars[:variant]}
+          # ---------------------------------------------------------------------------------------------------------------
+          def maybeUpdateManifest(data, man, assemble, supman=nil)
+            artifact_name = getArtifactName(data[:name], '', data[:version], 'properties')
+            artifact_path = getArtifactPath(data[:module], data[:version], data[:name])
+            objects = maybeArtifactoryObject(data[:module], @vars[:build_ver], false)
+            unless objects.nil? or objects.size == 0
+              @logger.info "#{artifact_name}(#{artifactory_endpoint()}/#{artifactory_repo()}/#{artifact_path}) exists - #{objects.size} results"
+              @logger.info "\t#{objects.map { |o| o.attributes[:uri] }.join("\n\t")}"
+              if objects.size > 1
+                @logger.error "Matched more than one artifact: #{artifact_path}/#{artifact_name}"
+                @vars[:return_code] = Errors::ARTIFACT_MULTI_MATCH
+              else
+                # Download over our temp file
+                local_dir, local_fil = File.split(data[:file])
+                @logger.info "Download #{data[:name]} to #{local_fil}"
+                @arti_download_artifact = false
+                3.times{
+                  monitor(30){
+                    IO.write(data[:file], '')
+                    objects[0].download(local_dir, filename: local_fil)
+                    @arti_download_artifact = true
+                  }
+                  break if @arti_download_artifact
+                }
+                # Add our manifest
+                data[:data] = IO.read(data[:file])
+                raise "Failed to download #{data[:name]}" if data[:data] == '' or not @arti_download_artifact
+                manifest = {}
+                data[:data].split(/\n+/).each do |line|
+                  k, v = line.chomp.split(/=/)
+                  manifest[k] = v #unless man.has_key?(k)
+                end
+                man = if supman
+                        manifest.delete(supman)
+                        man.each do |_,m|
+                          m.each do |k,v|
+                            manifest[k] = v
+                          end
+                        end
+                        { supman => manifest }
+                      else
+                        man.each do |k,v|
+                          manifest[k] = v #unless man.has_key?(k)
+                        end
+                        manifest
+                      end
+                data[:data] = assemble.call(man)
+                # Save the new file
+                IO.write(data[:file], data[:data])
+                data[:temp] = false
+                data[:sha1] = Digest::SHA1.file(data[:file]).hexdigest
+                data[:md5] = Digest::MD5.file(data[:file]).hexdigest
+              end
             end
+            man
           end
 
           # ---------------------------------------------------------------------------------------------------------------
@@ -284,7 +348,7 @@ module CiCd
             @vars[:return_code]
           end
 
-          private :createProductManifests, :createProjectsFile, :createSuperManifest
+          protected :createProductManifests, :createProjectsFile, :createSuperManifest
         end
       end
     end

data/lib/cicd/builder/manifest/version.rb

@@ -2,7 +2,7 @@ module CiCd
   module Builder
     module Manifest
       # manifest-builder version
-      VERSION = '0.6.2'
+      VERSION = '0.6.4'
       MAJOR, MINOR, TINY = VERSION.split('.')
       PATCH = TINY
     end

data/manifest-builder.gemspec

@@ -17,8 +17,8 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ['lib']
 
-  # gem.add_dependency 'cicd-builder', '>= 0.9.48', '< 1.1'
-  gem.add_dependency 'chefrepo-builder', '>= 0.9.32', '< 1.1'
+  # gem.add_dependency 'cicd-builder', '>= 0.9.50', '< 1.1'
+  gem.add_dependency 'chefrepo-builder', '>= 0.9.34', '< 1.1'
   gem.add_dependency 'json', '>= 1.8.1', '< 1.9'
   gem.add_dependency 's3etag', '>= 0.0.1', '< 0.1.0'
   gem.add_dependency 'archive-tar-minitar', '= 0.5.2'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: manifest-builder
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.4
 platform: ruby
 authors:
 - Christo De Lange
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-28 00:00:00.000000000 Z
+date: 2015-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chefrepo-builder
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.32
+        version: 0.9.34
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.1'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.32
+        version: 0.9.34
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.1'