cicd-builder 0.9.23 → 0.9.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 61ad72216b3d5b2ff2a2e52780638a8caee66e4b
-  data.tar.gz: b3f2f57f65776b850cc026b1a8d9dc6af726878e
+  metadata.gz: 0eab5918292fa3a2bf62313f8abd03f761f3f45d
+  data.tar.gz: 01182f3afc46fddde83438647fad3719779099fd
 SHA512:
-  metadata.gz: 87b31bbd6491c1a63674014eec0e0871a357cd5f6b080d666d7555fab1e2da7c87d93e38c484db053e7d844b18f1d18dc6111180c39d32c45c3e905d694b0255
-  data.tar.gz: 905a2508e6fd8587f4a287779c3b596a1cf5c1b26cc127c53ffb843ea0587413fad27db85376709fde6dc02902f2f389c295cf2ca8d986808f505276fb94347c
+  metadata.gz: cb9cbefa0fc88d44b1ec6853fb414f12c87b063cf782a94ada69032688d810e45820eb3c37ef9f06ba4366460242ac3f32833b3c04620c061b1a9387af7acb8b
+  data.tar.gz: 00e6ad0652ddc63c2443d2f6ebaf82fe12e86df11f115d7ebeb450abc56d08097f39e2284a1265ad1b24e2583d364d8dd67493b43cd17c32f2855359b0d591a0

data/Gemfile.lock

@@ -0,0 +1,136 @@
+PATH
+  remote: .
+  specs:
+    cicd-builder (0.9.25)
+      artifactory (>= 2.2.1, < 2.3)
+      awesome_print (= 1.2.0, < 1.3)
+      aws-sdk (>= 2.0, < 2.1)
+      chef (>= 11.18.2, < 12)
+      git (>= 1.2.7, < 1.3)
+      inifile (>= 3.0.0, < 3.1)
+      json (= 1.8.1, < 1.9)
+      logging (>= 1.8.2, < 1.9)
+      semverse (>= 1.2.1, < 1.3)
+      yajl-ruby (>= 1.2.1, < 1.3)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    artifactory (2.2.1)
+    awesome_print (1.2.0)
+    aws-sdk (2.0.28)
+      aws-sdk-resources (= 2.0.28)
+    aws-sdk-core (2.0.28)
+      builder (~> 3.0)
+      jmespath (~> 1.0)
+      multi_json (~> 1.0)
+      multi_xml (~> 0.5)
+    aws-sdk-resources (2.0.28)
+      aws-sdk-core (= 2.0.28)
+    builder (3.2.2)
+    chef (11.18.6)
+      chef-zero (~> 2.2, >= 2.2.1)
+      diff-lcs (~> 1.2, >= 1.2.4)
+      erubis (~> 2.7)
+      ffi-yajl (~> 1.2)
+      highline (~> 1.6, >= 1.6.9)
+      mime-types (~> 1.16)
+      mixlib-authentication (~> 1.3)
+      mixlib-cli (~> 1.4)
+      mixlib-config (~> 2.0)
+      mixlib-log (~> 1.3)
+      mixlib-shellout (~> 1.4)
+      net-ssh (~> 2.6)
+      net-ssh-multi (~> 1.1)
+      ohai (~> 7.4)
+      plist (~> 3.1.0)
+      pry (~> 0.9)
+      rest-client (>= 1.0.4, <= 1.6.7)
+    chef-zero (2.2.1)
+      ffi-yajl (~> 1.1)
+      hashie (~> 2.0)
+      mixlib-log (~> 1.3)
+      rack
+    coderay (1.1.0)
+    cucumber (0.10.7)
+      builder (>= 2.1.2)
+      diff-lcs (>= 1.1.2)
+      gherkin (~> 2.4.0)
+      json (>= 1.4.6)
+      term-ansicolor (>= 1.0.5)
+    diff-lcs (1.2.5)
+    erubis (2.7.0)
+    ffi (1.9.6)
+    ffi-yajl (1.4.0)
+      ffi (~> 1.5)
+      libyajl2 (~> 1.2)
+    gherkin (2.4.21)
+      json (>= 1.4.6)
+    git (1.2.9.1)
+    hashie (2.1.2)
+    highline (1.7.1)
+    inifile (3.0.0)
+    ipaddress (0.8.0)
+    jmespath (1.0.2)
+      multi_json (~> 1.0)
+    json (1.8.1)
+    libyajl2 (1.2.0)
+    little-plugger (1.1.3)
+    logging (1.8.2)
+      little-plugger (>= 1.1.3)
+      multi_json (>= 1.8.4)
+    method_source (0.8.2)
+    mime-types (1.25.1)
+    mixlib-authentication (1.3.0)
+      mixlib-log
+    mixlib-cli (1.5.0)
+    mixlib-config (2.1.0)
+    mixlib-log (1.6.0)
+    mixlib-shellout (1.6.1)
+    multi_json (1.10.1)
+    multi_xml (0.5.5)
+    net-ssh (2.9.2)
+    net-ssh-gateway (1.2.0)
+      net-ssh (>= 2.6.5)
+    net-ssh-multi (1.2.0)
+      net-ssh (>= 2.6.5)
+      net-ssh-gateway (>= 1.2.0)
+    ohai (7.4.1)
+      ffi (~> 1.9)
+      ffi-yajl (~> 1.1)
+      ipaddress
+      mime-types (~> 1.16)
+      mixlib-cli
+      mixlib-config (~> 2.0)
+      mixlib-log
+      mixlib-shellout (~> 1.2)
+      systemu (~> 2.6.4)
+      wmi-lite (~> 1.0)
+    plist (3.1.0)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rack (1.6.0)
+    rake (10.3.2)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    rubygems-tasks (0.2.4)
+    semverse (1.2.1)
+    slop (3.6.0)
+    systemu (2.6.4)
+    term-ansicolor (1.3.0)
+      tins (~> 1.0)
+    tins (1.3.4)
+    wmi-lite (1.0.0)
+    yajl-ruby (1.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (>= 1.6, < 1.7)
+  cicd-builder!
+  cucumber (>= 0.10.7, < 0.11)
+  rake (>= 10.3, < 11)
+  rubygems-tasks (>= 0.2, < 1.1)

data/cicd-builder.gemspec

@@ -17,17 +17,19 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ['lib']
 
-  gem.add_dependency 'awesome_print', '= 1.2.0'
-  gem.add_dependency 'inifile', '>= 2.0.2'
-  gem.add_dependency 'logging', '>= 0.0.0'
-  gem.add_dependency 'json', '= 1.8.1'
-  gem.add_dependency 'chef', '>= 11.8.2'
-  gem.add_dependency 'aws-sdk', '>= 1.49.0'
-  gem.add_dependency 'yajl-ruby', '>= 1.2.1'
-  gem.add_dependency 'git', '>= 1.2.7'
+  gem.add_dependency 'awesome_print', '= 1.2.0', '< 1.3'
+  gem.add_dependency 'inifile', '>= 3.0.0', '< 3.1'
+  gem.add_dependency 'logging', '>= 1.8.2', '< 1.9'
+  gem.add_dependency 'json', '= 1.8.1', '< 1.9'
+  gem.add_dependency 'chef', '>= 11.18.2', '< 12'
+  gem.add_dependency 'aws-sdk', '>= 2.0', '< 2.1'
+  gem.add_dependency 'yajl-ruby', '>= 1.2.1', '< 1.3'
+  gem.add_dependency 'git', '>= 1.2.7', '< 1.3'
+  gem.add_dependency 'semverse', '>= 1.2.1', '< 1.3'
+  gem.add_dependency 'artifactory', '>= 2.2.1', '< 2.3'
 
-  gem.add_development_dependency 'bundler', '~> 1.6'
-  gem.add_development_dependency 'rake', '~> 10.3'
-  gem.add_development_dependency 'rubygems-tasks', '~> 0.2'
-  gem.add_development_dependency 'cucumber', '>= 0.10.7'
+  gem.add_development_dependency 'bundler', '>= 1.6', '< 1.7'
+  gem.add_development_dependency 'rake', '>= 10.3', '< 11'
+  gem.add_development_dependency 'rubygems-tasks', '>= 0.2', '< 1.1'
+  gem.add_development_dependency 'cucumber', '>= 0.10.7', '< 0.11'
 end

data/lib/cicd/builder.rb

@@ -13,7 +13,8 @@ module CiCd
 		require 'fileutils'
 		require 'digest'
 		require 'yajl/json_gem'
-		require 'aws-sdk'
+		require 'aws-sdk-core'
+		require 'aws-sdk-resources'
 
 		_lib=File.dirname(__FILE__)
 		$:.unshift(_lib) unless $:.include?(_lib)
@@ -56,28 +57,21 @@ module CiCd
       end
 
 			# ---------------------------------------------------------------------------------------------------------------
-			def run()
+			def setup()
 				$stdout.write("CiCd::Builder v#{VERSION}\n")
+        @default_options[:env_keys] = Hash[@default_options[:env_keys].flatten.map.with_index.to_a].keys
 				parseOptions()
+			end
+
+			# ---------------------------------------------------------------------------------------------------------------
+			def run()
+				setup()
 
-				ret = checkEnvironment()
-				if 0 == ret
-					ret = getVars()
-          if 0 == ret
-            ret = prepareBuild()
-            if 0 == ret
-              ret = makeBuild()
-              if 0 == ret
-                ret = saveBuild()
-                if 0 == ret
-                  ret = uploadBuildArtifacts()
-                  if 0 == ret
-                    # noop
-                  end
-                end
-              end
-            end
-          end
+        ret = 0
+				%w(checkEnvironment getVars prepareBuild makeBuild saveBuild uploadBuildArtifacts).each do |step|
+          @logger.step "#{step}"
+          ret = send(step)
+          break unless ret == 0
         end
 
 				@vars[:return_code]
@@ -85,5 +79,14 @@ module CiCd
 
 		end
 
+    def isSameDirectory(pwd, workspace)
+      pwd = File.realdirpath(File.expand_path(pwd))
+      workspace = File.realdirpath(File.expand_path(workspace))
+      unless pwd == workspace
+
+      end
+      return pwd, workspace
+    end
+
 	end
 end

data/lib/cicd/builder/mixlib/build.rb

@@ -5,6 +5,7 @@ module CiCd
 
 		# ---------------------------------------------------------------------------------------------------------------
 		def cleanupBuild()
+      @logger.step __method__.to_s
       [ :build_pkg, :build_chk, :build_mdf, :build_mff ].each do |fil|
         if File.exists?(@vars[fil])
           begin
@@ -12,7 +13,7 @@ module CiCd
           rescue => e
             @logger.error e.to_s
             #raise e
-            return -96
+            return Errors::CLEANUPBUILD_EXCEPTION
           end
         end
       end
@@ -22,7 +23,7 @@ module CiCd
 				rescue => e
 					@logger.error e.to_s
 					#raise e
-					return -95
+					return Errors::CLEANUPBUILD_EXCEPTION
 				end
 			end
 			0
@@ -31,48 +32,71 @@ module CiCd
     # ---------------------------------------------------------------------------------------------------------------
     def prepareBuild()
       meta = {}
+      @vars[:return_code] = 0
       %w[ WORKSPACE PROJECT_NAME ].each do |e|
         unless ENV.has_key?(e)
-          raise "#{e} environment variable is required"
+          puts "#{e} environment variable is required"
+          @vars[:return_code] = Errors::MISSING_ENV_VAR
         end
       end
       meta[:Version] = @vars[:version]
       meta[:Release] = @vars[:release]
 
-      place = ''
-      begin
-        place = 'require "git"'
-        eval place
-
-        # Assuming we are in the workspace ...
-        place = "Git.open('#{ENV['WORKSPACE']}')"
-        git = Git.open(ENV['WORKSPACE'], :log => @logger)
-        place = 'git.log'
-        meta[:Commit] = git.log[0].sha
-        place = 'git.current_branch'
-        meta[:Branch] = git.current_branch
-
-        @vars[:build_ext] = 'tar.gz'
-        @vars[:build_bra] = meta[:Branch].gsub(%r([/|]),'.')
-        @vars[:build_ver] = "#{meta[:Version]}"
-        @vars[:build_vrb] = "#{@vars[:build_ver]}-release-#{meta[:Release]}-#{@vars[:build_bra]}-#{@vars[:variant]}" #
-        @vars[:build_nam] = "#{@vars[:project_name]}-#{@vars[:build_vrb]}"
-        @vars[:build_rel] = "#{@vars[:build_nam]}-build-#{@vars[:build_num]}"
-        @vars[:build_dir] = "#{ENV['WORKSPACE']}/#{@vars[:build_rel]}"
-        @vars[:latest_pkg]= "#{@vars[:build_store]}/#{@vars[:build_rel]}.#{@vars[:build_ext]}"
-        @vars[:build_pkg] = "#{@vars[:build_rel]}.#{@vars[:build_ext]}"
-        @vars[:build_chk] = "#{@vars[:build_rel]}.checksum"
-        @vars[:build_mff] = "#{@vars[:build_rel]}.manifest"
-        @vars[:build_mdf] = "#{@vars[:build_rel]}.meta"
-        @vars[:build_mdd] = meta.dup
-        #noinspection RubyArgCount
-        @vars[:build_mds] = Digest::SHA256.hexdigest(meta.to_s)
-
-        @vars[:return_code] = 0
-
-      rescue Exception => e
-        @logger.error "#{e.class}:: '#{place}' - #{e.message}"
-        @vars[:return_code] = -98
+      if @vars[:return_code] == 0
+
+        if File.exists?(ENV['WORKSPACE']) and (File.directory?(ENV['WORKSPACE']) or File.symlink?(ENV['WORKSPACE']))
+
+          place = ''
+          begin
+            req = 'require "git"'
+            eval req
+
+            # Assuming we are in the workspace ...
+            place = "Git.open('#{ENV['WORKSPACE']}')"
+            git = Git.open(ENV['WORKSPACE'], :log => @logger)
+            place = 'git.log'
+            meta[:Commit] = git.log[0].sha
+            place = 'git.current_branch'
+            meta[:Branch] = git.current_branch
+
+            @vars[:build_ext] = 'tar.gz'
+            @vars[:build_bra] = meta[:Branch].gsub(%r([/|]),'.')
+            @vars[:build_ver] = "#{meta[:Version]}"
+            @vars[:build_rel] = "#{meta[:Release]}"
+            @vars[:build_vrb] = "#{@vars[:build_ver]}-release-#{meta[:Release]}-#{@vars[:build_bra]}-#{@vars[:variant]}" #
+            @vars[:build_nam] = "#{@vars[:project_name]}-#{@vars[:build_vrb]}"
+            @vars[:build_nmn] = "#{@vars[:build_nam]}-build-#{@vars[:build_num]}"
+            @vars[:build_dir] = "#{ENV['WORKSPACE']}/#{@vars[:build_nmn]}"
+            @vars[:latest_pkg]= "#{@vars[:build_store]}/#{@vars[:build_nmn]}.#{@vars[:build_ext]}"
+            @vars[:build_pkg] = "#{@vars[:build_nmn]}.#{@vars[:build_ext]}"
+            @vars[:build_chk] = "#{@vars[:build_nmn]}.checksum"
+            @vars[:build_mff] = "#{@vars[:build_nmn]}.manifest"
+            @vars[:build_mdf] = "#{@vars[:build_nmn]}.meta"
+            @vars[:build_mdd] = meta.dup
+            #noinspection RubyArgCount
+            @vars[:build_mds] = Digest::SHA256.hexdigest(meta.to_s)
+
+            @vars[:return_code] = 0
+
+          rescue Exception => e
+            @logger.error "#{e.class}:: '#{place}' - #{e.message}"
+            @vars[:return_code] = Errors::PREPAREBUILD_EXCEPTION
+          end
+
+        else
+          puts "Invalid workspace: '#{ENV['WORKSPACE']}'"
+          @vars[:return_code] = Errors::INVALID_WORKSPACE
+        end
+      end
+
+      if @vars[:return_code] == 0
+        @vars[:local_dirs] ||= {}
+        %w(artifacts latest).each do |dir|
+          @vars[:local_dirs][dir] = "#{ENV['WORKSPACE']}/#{dir}"
+          unless File.directory?(dir)
+            Dir.mkdir(dir)
+          end
+        end
       end
 
       @vars[:return_code]
@@ -83,54 +107,100 @@ module CiCd
       if @vars.has_key?(:build_dir) and @vars.has_key?(:build_pkg)
         begin
           do_build = false
-          if File.exists?(@vars[:build_chk])
-            @vars[:build_sha] = IO.readlines(@vars[:build_chk])
-            unless @vars[:build_sha].is_a?(Array)
-              @logger.error "Unable to parse build checksum from #{@vars[:build_chk]}"
-              return -97
+          loadCheckSumFile()
+          if 0 == @vars[:return_code]
+            do_build = true if @vars[:build_sha].empty?
+            do_build = true unless File.exists?(@vars[:build_pkg]) and (Digest::SHA256.file(@vars[:build_pkg]).hexdigest() == @vars[:build_sha])
+            if do_build
+              @vars[:return_code] = cleanupBuild()
+              if 0 == @vars[:return_code]
+                @vars[:build_dte] = DateTime.now.strftime('%F %T%:z')
+                createMetaData()
+                if 0 == @vars[:return_code]
+                  @vars[:return_code] = packageBuild()
+                  if 0 == @vars[:return_code]
+                    @vars[:check_sha] = @vars[:build_sha]
+                    @vars[:build_sha] = if File.exists?(@vars[:build_pkg])
+                                          Digest::SHA256.file(@vars[:build_pkg]).hexdigest()
+                                        else
+                                          '0'
+                                        end
+                    unless IO.write(@vars[:build_chk], @vars[:build_sha]) > 0
+                      @logger.error "Unable to store checksum in '#{@vars[:build_chk]}'"
+                      @vars[:return_code] = Errors::STORING_BUILD_CHECKSUM
+                    end
+                  end
+                end
+              end
             end
-            @vars[:build_sha] = @vars[:build_sha][0].chomp()
-          else
-            @vars[:build_sha] = ''
-            do_build = true
-          end
-          unless File.exists?(@vars[:build_pkg])
-            do_build = true
           end
+
+          # Report status regardless of return code.
+          reportStatus()
+
           if do_build
-            @vars[:return_code] = cleanupBuild()
-            return @vars[:return_code] unless @vars[:return_code] == 0
-            @vars[:build_dte] = DateTime.now.strftime("%F %T%:z")
-            createMetaData()
-            @vars[:return_code] = packageBuild()
-            if 0 == @vars[:return_code]
-              @vars[:check_sha] = @vars[:build_sha]
-              @vars[:build_sha] = Digest::SHA256.file(@vars[:build_pkg]).hexdigest()
-              IO.write(@vars[:build_chk], @vars[:build_sha])
-            end
-            reportStatus()
             reportResult()
           else
-            reportStatus()
-
-            # No need to build again :)
+            # Was no need to build :) or a failure :(
             @logger.info "NO_CHANGE: #{ENV['JOB_NAME']} #{ENV['BUILD_NUMBER']} #{@vars[:build_nam]} #{@vars[:build_pkg]} #{@vars[:build_chk]} [#{@vars[:build_sha]}]"
-            @vars[:return_code] = 0
+            # @vars[:return_code] = 0
             return 1
           end
         rescue => e
-          @logger.error "#{e.class.name} #{e.message}"
-          @vars[:return_code] = -99
+          @logger.error "makeBuild failure: #{e.class.name} #{e.message}"
+          @vars[:return_code] = Errors::MAKEBUILD_EXCEPTION
         end
       else
-        @logger.error ":build_dir or :build_pkg is unknown"
-        @vars[:return_code] = 2
+        @logger.error ':build_dir or :build_pkg is unknown'
+        @vars[:return_code] = Errors::MAKEBUILD_PREPARATION
       end
       @vars[:return_code]
     end
 
+    # ---------------------------------------------------------------------------------------------------------------
+    def loadCheckSumFile
+      if File.exists?(@vars[:build_chk])
+        @vars[:build_sha] = IO.readlines(@vars[:build_chk])
+        unless @vars[:build_sha].is_a?(Array)
+          @logger.error "Unable to parse build checksum from #{@vars[:build_chk]}"
+          @vars[:return_code] = Errors::PARSING_BUILD_CHECKSUM
+        end
+        @vars[:build_sha] = @vars[:build_sha][0].chomp()
+      else
+        @vars[:build_sha] = ''
+      end
+      @vars[:return_code]
+    end
+
+    # ---------------------------------------------------------------------------------------------------------------
+    def calcLocalETag(etag, local, size = nil)
+      if size == nil
+        stat = File.stat(local)
+        size = stat.size
+      end
+      match = etag.match(%r'-(\d+)$')
+      check = if match
+                require 's3etag'
+                parts = match[1].to_i
+                chunk = size.to_f / parts.to_f
+                mbs = (chunk.to_f / 1024 /1024 + 0.5).to_i
+                part_size = mbs * 1024 * 1024
+                chkit = S3Etag.calc(file: local, threshold: part_size, min_part_size: part_size, max_parts: parts)
+                while chkit != etag and (size / part_size) <= parts
+                  # Go one larger if a modulus remains and we have the right number of parts
+                  mbs += 1
+                  part_size = mbs * 1024 * 1024
+                  chkit = S3Etag.calc(file: local, threshold: part_size, min_part_size: part_size, max_parts: parts)
+                end
+                chkit
+              else
+                Digest::MD5.file(local).hexdigest
+              end
+    end
+
     # ---------------------------------------------------------------------------------------------------------------
 		def packageBuild()
+      @logger.step __method__.to_s
 			excludes=%w(*.iml *.txt *.sh *.md .gitignore .editorconfig .jshintrc *.deprecated adminer doc)
 			excludes = excludes.map{ |e| "--exclude=#{@vars[:build_nam]}/#{e}" }.join(' ')
 			cmd = %(cd #{ENV['WORKSPACE']}; tar zcvf #{@vars[:build_pkg]} #{excludes} #{@vars[:build_nam]} 1>#{@vars[:build_pkg]}.manifest)
@@ -144,6 +214,7 @@ module CiCd
 
     # ---------------------------------------------------------------------------------------------------------------
     def createMetaData()
+      @logger.step __method__.to_s
       @vars[:build_mdd].merge!({
                                 :Generation => @options[:gen],
                                 :Project => @vars[:project_name],
@@ -154,7 +225,11 @@ module CiCd
 
                                       })
       json = JSON.pretty_generate( @vars[:build_mdd], { indent: "\t", space: ' '})
-      IO.write(@vars[:build_mdf], json)
+      unless IO.write(@vars[:build_mdf], json) > 0
+        @logger.error "Unable to store metadata in '#{@vars[:build_mdf]}'"
+        @vars[:return_code] = Errors::STORING_BUILD_METADATA
+      end
+      @vars[:return_code]
     end
 
   end