manifest-builder 0.6.0 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ffd8bef49ca4e0886fd5532da54da27c4b6bfbd7
-  data.tar.gz: 50f0f52111d62b9850957f4e498f181edb3e80c6
+  metadata.gz: 0b10b3d8ea6cfc58960b69110ceb02434f017b35
+  data.tar.gz: 1d2dc07ea272e2d8ffde9016586873cd5e834350
 SHA512:
-  metadata.gz: e29f04b796c242f71f6feced462cf3d3a2f09228578b0ecfe0293967d984d09cc835687f7c96511fb5ec28843fc86f6861b3208bff13be9698009f3c008fe660
-  data.tar.gz: 5a9d3cfc3e4eeffd30f9204850ff60f0aefdab23b9381fb426a60e7e9e0d5a3c69cd53a0239bd50ff1bf0871358ae96abf2f5b2b8d06584321841fbf721051b7
+  metadata.gz: f545e10d18b032a1c4f2de06f63f684cff93e2d1fd3d9ddea06e4c976a7f7192e73d6bce5be5b3ff76300f76ddda5a093308c55b472bbf1e287a43f5283781cd
+  data.tar.gz: 6e88d6c42454c00614ced3186896e6744682e451c8557e8f482b26c1b69f36664aa5c951cf75704873247dbb5ed67b69512c5594ab483f46cd285644a2ab8d1f

data/Gemfile.lock

@@ -1,24 +1,23 @@
 PATH
   remote: .
   specs:
-    manifest-builder (0.6.0)
+    manifest-builder (0.6.2)
       archive-tar-minitar (= 0.5.2)
-      chefrepo-builder (>= 0.9.31, < 1.1)
-      cicd-builder (>= 0.9.47, < 1.1)
+      chefrepo-builder (>= 0.9.32, < 1.1)
       json (>= 1.8.1, < 1.9)
       s3etag (>= 0.0.1, < 0.1.0)
 
 PATH
   remote: ../chefrepo-builder
   specs:
-    chefrepo-builder (0.9.31)
+    chefrepo-builder (0.9.32)
       cicd-builder (>= 0.9.47, < 1.1)
       json (= 1.8.1)
 
 PATH
   remote: ../cicd-builder
   specs:
-    cicd-builder (0.9.47)
+    cicd-builder (0.9.48)
       artifactory (>= 2.2.1, < 2.3)
       awesome_print (>= 1.6, < 2.0)
       aws-sdk (>= 2.0, < 2.1)
@@ -84,7 +83,7 @@ GEM
       json (>= 1.4.6)
     git (1.2.9.1)
     hashie (2.1.2)
-    highline (1.7.1)
+    highline (1.7.2)
     inifile (3.0.0)
     ipaddress (0.8.0)
     jmespath (1.0.2)

data/lib/cicd/builder/manifest.rb

@@ -35,6 +35,8 @@ module CiCd
           $stdout.write("ManifestBuilder v#{CiCd::Builder::Manifest::VERSION}\n")
           @default_options[:env_keys] << %w(
                                             MANIFEST_FILE
+                                            CLASSES_FILE
+                                            CLASSES_MANIFEST_FILE
                                            )
           super
         end

data/lib/cicd/builder/manifest/mixlib/build.rb

@@ -88,6 +88,7 @@ module CiCd
 
         # ---------------------------------------------------------------------------------------------------------------
         def getVersionBuild(path,artifact,comp)
+          cname,cdata = comp
           version,build  = File.split(path)
           if build.match(%r'^\d+$') and version.match(%r'/?\d+\.\d+\.?\d*$') # Hole in one!
             version = File.basename(version)
@@ -96,10 +97,10 @@ module CiCd
               version = build
               build   = ''
             else
-              version = comp[1][:build].nil? ? '' : ( comp[1][:build] > 0 ? build.to_s : '' )
+              version = cdata[:build].nil? ? '' : ( cdata[:build] > 0 ? build.to_s : '' )
             end
             unless version.match(VER_RGX)
-              version = comp[1][:version] || ''
+              version = cdata[:version] || ''
             end
             ver,bld = getVersionBuildFromName(artifact)
             if version.empty?
@@ -147,7 +148,7 @@ module CiCd
               end
             end
             unless build.match(%r'^[1-9]\d*$')
-              build = comp[1][:build]
+              build = cdata[:build]
               build = @vars[:build_num] if (build.nil? or build.empty? or build.to_i == 0)
             end
           end
@@ -221,21 +222,25 @@ module CiCd
         # ---------------------------------------------------------------------------------------------------------------
         def processComponent(comp, lines)
           artifact, path, version, build = parseComponent(comp)
+          cname,cdata = comp
 
           require 'uri'
+          require 'digest'
           begin
             key, name, objects = getObjects(artifact, path)
-            local_dir = File.join(@vars[:local_dirs]['artifacts'], comp[0], '')
+            local_dir = File.join(@vars[:local_dirs]['artifacts'], cname, '')
             Dir.mkdir(local_dir, 0700) unless File.directory?(local_dir)
             artifacts = []
+            sha256 = {}
             changed = false
             # 1 or more objects on the key/ path
             if objects.size > 0
-              lines << "#{comp[0]}:#{artifact} v#{version} b#{build} - #{path}"
+              lines << "#{cname}:#{artifact} v#{version} b#{build} - #{path}"
               # When we start pulling the artifacts then everything that is build 0 get this build number, in fact all artifacts get this build number!
               objects.each do |object|
                 @logger.info "\tchecking #{object.key}"
-                local = File.join(local_dir, File.basename(object.key))
+                base = File.basename(object.key)
+                local = File.join(local_dir, base)
                 etag = object.etag.gsub(%r/['"]/, '')
                 download = shouldDownload?(etag, local, object)
                 if download
@@ -243,12 +248,41 @@ module CiCd
                 else
                   @logger.info "\t\tunchanged"
                 end
+                sha256[base] = nil
+                if cdata[:sha256]
+                  hd = Digest::SHA256.file(local).hexdigest
+                  sha256[base] = hd == cdata[:sha256] ? true : hd
+                end
+
                 artifacts << local
               end
               # The local file will be 1 artifact or an archive of the local artifacts when artifacts.size > 1
               if artifacts.size > 0
-                local = getLocalArtifact(artifacts, changed, comp, local_dir, version)
-                addArtifact(@vars[:artifacts], local, local_dir, {module: comp[0], name: comp[0], build: build, version: version, file: local})
+                artifacts.each do |local|
+                  base = File.basename(local)
+                  if sha256[base].is_a?(String)
+                    msg = "Artifact checksum is invalid or manifest is incorrect. Artifact: s3://#{name}/#{key}#{artifact}, SHA256: Manifest=#{cdata[:sha256]}, Actual=#{sha256[base]}"
+                    if ENV['ENFORCE_CHECKSUMS'] and ENV['ENFORCE_CHECKSUMS'].downcase =~ %r/^(yes|on|set|1|enable|active|enforced?)$/
+                      @logger.fatal msg
+                      @vars[:return_code] = Errors::ARTIFACT_CHECKSUM_BAD
+                    else
+                      @logger.warn msg
+                    end
+                  end
+                end
+                if 0 == @vars[:return_code]
+                  local = getLocalArtifact(artifacts, changed, comp, local_dir, version)
+                  base = File.basename(local)
+                  addArtifact(@vars[:artifacts], local, local_dir, {
+                                                                       component: cname,
+                                                                          module: cname,
+                                                                            name: cname,
+                                                                           build: build,
+                                                                         version: version,
+                                                                            file: local,
+                                                                          sha256: (sha256.keys.include?(base)) ? (sha256[base].is_a?(TrueClass) ? cdata[:sha256] : sha256[base]) : Digest::SHA256.file(local).hexdigest
+                                                                   })
+                end
               end
             else
               @logger.fatal "Artifact not found: s3://#{name}/#{key}#{artifact}"
@@ -312,9 +346,10 @@ module CiCd
 
         # ---------------------------------------------------------------------------------------------------------------
         def getLocalArtifact(artifacts, changed, comp, local_dir, version)
+          cname,cdata = comp
           if artifacts.size > 1
             begin
-              file = File.join(local_dir, "#{comp[0]}-#{version}.zip")
+              file = File.join(local_dir, "#{cname}-#{version}.zip")
               if changed or not File.exists?(file)
                 zipped_files = artifacts.map { |f| f.gsub(%r'^#{local_dir}', '') }.join(' ')
                 Dir.chdir(local_dir) do
@@ -336,23 +371,24 @@ module CiCd
 
         # ---------------------------------------------------------------------------------------------------------------
         def parseComponent(comp)
-          if comp[1][:url]
-            path, artifact = File.split(comp[1][:url])
+          cname,cdata = comp
+          if cdata[:url]
+            path, artifact = File.split(cdata[:url])
             version, build = getVersionBuild(path, artifact, comp)
-          elsif comp[1][:base_url]
+          elsif cdata[:base_url]
             artifact = ''
-            if comp[1][:build].nil?
+            if cdata[:build].nil?
               # noinspection RubyUnusedLocalVariable
-              version, build = comp[1][:version].split(%r'-')
+              version, build = cdata[:version].split(%r'-')
               # noinspection RubyUnusedLocalVariable
-              path = File.join(comp[1][:base_url], comp[1][:version])
+              path = File.join(cdata[:base_url], cdata[:version])
             else
-              version, build = [comp[1][:version], comp[1][:build]]
-              path = File.join(comp[1][:base_url], comp[1][:version], comp[1][:build])
+              version, build = [cdata[:version], cdata[:build]]
+              path = File.join(cdata[:base_url], cdata[:version], cdata[:build])
             end
           else
-            path = ''
-            artifact = ''
+            path           = ''
+            artifact       = ''
             version, build = getVersionBuild(path, artifact, comp)
           end
           return artifact, path, version, build

data/lib/cicd/builder/manifest/mixlib/repo/artifactory.rb

@@ -57,36 +57,31 @@ module CiCd
           def createProjectsFile
             @logger.info __method__.to_s
             projects = {}
-            project_names = {}
-            unless ENV['PROJECT_NAMES'].nil?
-              if File.exists?(ENV['PROJECT_NAMES'])
-                @logger.info "Load PROJECT_NAMES: #{ENV['PROJECT_NAMES']}"
-                project_names = JSON.load(IO.read(ENV['PROJECT_NAMES'])) || {}
-              else
-                @logger.error "The PROJECT_NAMES file (#{ENV['PROJECT_NAMES']}) does not exist!"
-                @vars[:return_code] = Errors::NO_PROJECT_NAMES
-              end
-            end
+            project_names = loadProjectNames()
             exts = {}
             exts = Hash[@vars[:artifacts].map { |a| [a[:data][:name], File.basename(a[:data][:file]).match(CiCd::Builder::Manifest::Build::EXT_RGX)[1]] }]
+
+            createClassesFile()
+
             @vars[:artifacts].each do |art|
               prod = art[:data][:name]
               mod  = art[:data][:module]
               projects[prod] = {
-                  name: project_names[prod] || prod,
-                  module: mod,
-                  ext: exts[prod],
-                  class_filter: '',
-              }
+                                         name: project_names[prod] || prod,
+                                       module: mod,
+                                          ext: exts[prod],
+                                 class_filter: @vars[:filters][prod] || @vars[:filters][prod.gsub(/-manifest$/, '')],
+                               }
             end
 
             require 'chef/mash'
             require 'chef/mixin/deep_merge'
-            old_projects = Chef::Mash.new(File.exists?(ENV['PROJECTS_FILE']) ? JSON.load(IO.read(ENV['PROJECTS_FILE'])) : {})
-            projects = Chef::Mash.new(projects)
-            projects = Chef::Mash.new(Chef::Mixin::DeepMerge.deep_merge(projects, old_projects))
-            @logger.info "Save PROJECTS_FILE: #{ENV['PROJECTS_FILE']}"
-            IO.write(ENV['PROJECTS_FILE'], JSON.pretty_generate(projects.to_hash, {indent: "\t", space: ' '}))
+
+            projects_hash = File.exists?(ENV['PROJECTS_FILE']) ? loadConfigFile(ENV['PROJECTS_FILE']) : {}
+            old_projects = ::Chef::Mash.new(projects_hash)
+            projects = ::Chef::Mash.new(projects)
+            projects = ::Chef::Mash.new(::Chef::Mixin::DeepMerge.deep_merge(projects, old_projects))
+            saveConfigFile(ENV['PROJECTS_FILE'], projects)
             data = {
                           name: 'projects-file',
                         module: 'projects-file',
@@ -99,7 +94,100 @@ module CiCd
                            md5: Digest::MD5.file(ENV['PROJECTS_FILE']).hexdigest,
                    }
 
-            maybeUploadArtifactoryObject(data: data, artifact_module: data[:name], artifact_version: data[:version] || @vars[:version], file_name: '', file_ext: 'json')
+            maybeUploadArtifactoryObject(
+                                                     data: data,
+                                          artifact_module: data[:name],
+                                         artifact_version: data[:version] || @vars[:version],
+                                                file_name: '',
+                                                 file_ext: (ENV['PROJECTS_FILE'] and ENV['PROJECTS_FILE'].downcase.match(/\.ya?ml$/)) ? 'yaml' : 'json'
+                                        )
+          end
+
+          # ---------------------------------------------------------------------------------------------------------------
+          # noinspection RubyHashKeysTypesInspection
+          def createClassesFile()
+            @logger.info __method__.to_s
+            project_names = loadProjectNames()
+
+            @vars[:classes] = YAML.load(IO.read(ENV['CLASSES_MANIFEST_FILE']))
+            # keys = Hash[classes.keys.map.with_index.to_a].keys.sort
+
+            @vars[:filters] = {}
+            filters = {}
+            @vars[:classes].each do |role,apps|
+              apps.map{ |app|
+                filters[app] ||= []
+                filters[app] << role
+              }
+            end
+            filters.each do |app,roles|
+              @vars[:filters][app] = Hash[roles.map.with_index.to_a].keys.join('|')
+            end
+
+            saveConfigFile(ENV['CLASSES_FILE'],@vars[:classes])
+            data = {
+                              name: 'classes-file',
+                            module: 'classes-file',
+                              file: ENV['CLASSES_FILE'],
+                           version: @vars[:build_ver],
+                             build: @vars[:build_num],
+                        properties: @properties_matrix,
+                              temp: false,
+                              sha1: Digest::SHA1.file(ENV['CLASSES_FILE']).hexdigest,
+                               md5: Digest::MD5.file(ENV['CLASSES_FILE']).hexdigest,
+                    }
+
+            maybeUploadArtifactoryObject(
+                                                     data: data,
+                                          artifact_module: data[:name],
+                                         artifact_version: data[:version] || @vars[:version],
+                                                file_name: '',
+                                                 file_ext: 'yaml'
+                                        )
+          end
+
+          def saveConfigFile(file, projects)
+            @logger.info "Save config file: #{file}"
+            ext = file.gsub(/\.(\w+)$/, '\1')
+            IO.write(file, case ext.downcase
+                           when /ya?ml/
+                             projects.to_hash.to_yaml line_width: 1024, indentation: 4, canonical: false
+                           when /json|js/
+                             JSON.pretty_generate(projects.to_hash, {indent: "\t", space: ' '})
+                           else
+                             raise "Unsupported extension: #{ext}"
+                           end)
+          end
+
+          def loadConfigFile(file)
+            ext = file.gsub(/\.(\w+)$/, '\1')
+            hash = case ext.downcase
+                   when /ya?ml/
+                     YAML.load_file(ENV['PROJECTS_FILE'])
+                   when /json|js/
+                     JSON.load(IO.read(ENV['PROJECTS_FILE']))
+                   else
+                     raise "Unsupported extension: #{ext}"
+                   end
+          end
+
+          def loadProjectNames(fresh=false)
+            if fresh
+              @project_names = nil
+            end
+            unless @project_names
+              @project_names = {}
+              unless ENV['PROJECT_NAMES'].nil?
+                if File.exists?(ENV['PROJECT_NAMES'])
+                  @logger.info "Load PROJECT_NAMES: #{ENV['PROJECT_NAMES']}"
+                  @project_names = JSON.load(IO.read(ENV['PROJECT_NAMES'])) || {}
+                else
+                  @logger.error "The PROJECT_NAMES file (#{ENV['PROJECT_NAMES']}) does not exist!"
+                  @vars[:return_code] = Errors::NO_PROJECT_NAMES
+                end
+              end
+            end
+            @project_names
           end
 
           def createSuperManifest(manifest)
@@ -142,6 +230,7 @@ module CiCd
               data = {
                             name: "#{mod}-manifest",
                           module: "#{mod}-manifest",
+                       component: mod,
                             data: manifest_data,
                          version: @vars[:build_ver],
                            build: @vars[:build_num],

data/lib/cicd/builder/manifest/version.rb

@@ -2,7 +2,7 @@ module CiCd
   module Builder
     module Manifest
       # manifest-builder version
-      VERSION = '0.6.0'
+      VERSION = '0.6.2'
       MAJOR, MINOR, TINY = VERSION.split('.')
       PATCH = TINY
     end

data/manifest-builder.gemspec

@@ -17,8 +17,8 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ['lib']
 
-  gem.add_dependency 'cicd-builder', '>= 0.9.47', '< 1.1'
-  gem.add_dependency 'chefrepo-builder', '>= 0.9.31', '< 1.1'
+  # gem.add_dependency 'cicd-builder', '>= 0.9.48', '< 1.1'
+  gem.add_dependency 'chefrepo-builder', '>= 0.9.32', '< 1.1'
   gem.add_dependency 'json', '>= 1.8.1', '< 1.9'
   gem.add_dependency 's3etag', '>= 0.0.1', '< 0.1.0'
   gem.add_dependency 'archive-tar-minitar', '= 0.5.2'

metadata

@@ -1,42 +1,22 @@
 --- !ruby/object:Gem::Specification
 name: manifest-builder
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.2
 platform: ruby
 authors:
 - Christo De Lange
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-27 00:00:00.000000000 Z
+date: 2015-04-28 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: cicd-builder
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.9.47
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.9.47
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: chefrepo-builder
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.31
+        version: 0.9.32
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.1'
@@ -46,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.31
+        version: 0.9.32
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.1'