chefrepo-manifest-builder 0.1.0 → 0.1.1

data/Rakefile

@@ -0,0 +1,36 @@
+# encoding: utf-8
+
+require 'rubygems'
+
+begin
+  require 'bundler'
+rescue LoadError => e
+  warn e.message
+  warn "Run `gem install bundler` to install Bundler."
+  exit -1
+end
+
+begin
+  Bundler.setup(:development)
+rescue Bundler::BundlerError => e
+  warn e.message
+  warn "Run `bundle install` to install missing gems."
+  exit e.status_code
+end
+
+require 'rake'
+
+require 'rubygems/tasks'
+Gem::Tasks.new
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new
+
+task :test    => :spec
+task :default => :spec
+
+require 'cucumber/rake/task'
+
+Cucumber::Rake::Task.new do |t|
+  t.cucumber_opts = %w[--format pretty]
+end

data/bin/chefrepomanifestbuilder

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+
+path = File.dirname(__FILE__)
+if File.exists?("#{path}/../.gitignore")
+  %w(chefrepo-builder cicd-builder).each do |mod|
+    add_path = File.expand_path(File.join(path, "../../#{mod}", "lib"))
+    $:.unshift(add_path)
+  end
+else
+  # Borrowing from "whiches" gem ...
+  cmd  = File.basename(__FILE__, '.rb')
+  exes = []
+  exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
+  ENV['PATH'].split(File::PATH_SEPARATOR).each do |pth|
+    exts.each { |ext|
+      exe = File.join(pth, "#{cmd}#{ext}")
+      exes << exe if File.executable? exe
+    }
+  end
+  if exes.size > 0
+    path = File.dirname(exes[0])
+  end
+
+end
+add_path = File.expand_path(File.join(path, "..", "lib"))
+$:.unshift(add_path)
+
+require 'cicd/builder/chefrepo-manifest'
+
+# =====================================================================================================================
+exit CiCd::Builder::ChefRepoManifest::Runner.new().run()

data/chefrepo-manifest-builder.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+
+require File.expand_path('../lib/cicd/builder/chefrepo-manifest/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = 'chefrepo-manifest-builder'
+  gem.version       = CiCd::Builder::ChefRepoManifest::VERSION
+  gem.summary       = %q{ChefRepo builder for a software manifest}
+  gem.description   = %q{ChefRepo builder of the software manifest for Continuous Integration/Continuous Delivery artifact promotion style deployments}
+  gem.license       = "Apachev2"
+  gem.authors       = ["Christo De Lange"]
+  gem.email         = "rubygems@dldinternet.com"
+  gem.homepage      = "https://rubygems.org/gems/manifest-builder"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.add_dependency 'manifest-builder', '>= 0.6.2', '< 1.1'
+  # gem.add_dependency 'cicd-builder', '>= 0.9.48', '< 1.1'
+  # gem.add_dependency 'chefrepo-builder', '>= 0.9.31', '< 1.1'
+  gem.add_dependency 'json', '>= 1.8.1', '< 1.9'
+  gem.add_dependency 's3etag', '>= 0.0.1', '< 0.1.0'
+  gem.add_dependency 'archive-tar-minitar', '= 0.5.2'
+
+  gem.add_development_dependency 'bundler', '>= 1.7', '< 2.0'
+  gem.add_development_dependency 'rake', '>= 10.3', '< 11'
+  gem.add_development_dependency 'rubygems-tasks', '>= 0.2', '< 1.1'
+  gem.add_development_dependency 'cucumber', '>= 0.10.7', '< 0.11'
+  gem.add_development_dependency 'rspec', '>= 2.99', '< 3.0'
+end

data/features/manifestrepo-builder.feature

@@ -0,0 +1 @@
+Feature: Blah blah blah

data/features/step_definitions/manifestrepo-builder_steps.rb

@@ -0,0 +1 @@
+@wip

data/lib/cicd/builder/chefrepo-manifest/mixlib/build.rb

@@ -0,0 +1,62 @@
+
+module CiCd
+  module Builder
+    # noinspection RubySuperCallWithoutSuperclassInspection
+    module ChefRepoManifest
+      module Build
+
+        # ---------------------------------------------------------------------------------------------------------------
+        # noinspection RubyHashKeysTypesInspection
+        def prepareBuild()
+          super
+          if @vars[:return_code] == 0
+          end
+          @vars[:return_code]
+        end
+
+        # ---------------------------------------------------------------------------------------------------------------
+        def packageBuild()
+          @logger.step __method__.to_s
+          if isSameDirectory(Dir.pwd, ENV['WORKSPACE'])
+            if @vars.has_key?(:components) and not @vars[:components].empty?
+              @vars[:return_code] = 0
+
+              clazz = getRepoClass('S3')
+              @logger.debug "Repo class == '#{clazz}'"
+              if clazz.is_a?(Class) and not clazz.nil?
+                @repo = clazz.new(self)
+
+                if @vars[:return_code] == 0
+                  lines             = []
+                  @vars[:artifacts] = []
+                  # Deal with all artifacts of each component
+                  @vars[:components].each { |comp|
+                    processComponent(comp, lines)
+                  }
+                  if @vars[:return_code] == 0
+                    cleanupAfterPackaging(lines)
+                  end
+
+                else
+                  @logger.fatal "S3 repo error: Bucket #{ENV['AWS_S3_BUCKET']}"
+                end
+              else
+                @logger.error "CiCd::Builder::Repo::#{type} is not a valid repo class"
+                @vars[:return_code] = Errors::BUILDER_REPO_TYPE
+              end
+            else
+              @logger.error 'No components found during preparation?'
+              @vars[:return_code] = Errors::NO_COMPONENTS
+            end
+          else
+            @logger.error "Not in WORKSPACE? '#{pwd}' does not match WORKSPACE='#{workspace}'"
+            @vars[:return_code] = Errors::WORKSPACE_DIR
+          end
+
+          @vars[:return_code]
+        end
+
+      end
+    end
+  end
+end

data/lib/cicd/builder/chefrepo-manifest/mixlib/repo/artifactory.rb

@@ -0,0 +1,323 @@
+require 'artifactory'
+require 'tmpdir'
+
+module CiCd
+  module Builder
+    # noinspection RubySuperCallWithoutSuperclassInspection
+    module Manifest
+      module Repo
+        class Artifactory < CiCd::Builder::Repo::Artifactory
+          # include ::Artifactory::Resource
+
+          # ---------------------------------------------------------------------------------------------------------------
+          def initialize(builder)
+            super
+          end
+
+          # ---------------------------------------------------------------------------------------------------------------
+          def uploadToRepo(artifacts)
+            @manifest = {}
+            super
+            if @vars[:return_code] == 0
+              # Preserve the manifest (maybeUploadArtifactoryObject will add everything we upload to the instance var)
+              manifest = @manifest.dup
+              # Create a manifest for each product and store it.
+              createProductManifests(manifest)
+              # Get a super manifest of all products and store as learning-manifest
+              createSuperManifest(manifest) # -#{@vars[:variant]}
+              @manifest = manifest
+            end
+            # If we are asked to produce a PROJECTS_FILE then we do that from the manifest and components.
+            unless ENV['PROJECTS_FILE'].nil?
+              if @vars[:return_code] == 0
+                if File.directory?(File.realdirpath(File.dirname(ENV['PROJECTS_FILE'])))
+                  createProjectsFile
+                else
+                  @logger.error "The path to the PROJECTS_FILE (#{File.dirname(ENV['PROJECTS_FILE'])}) does not exist!"
+                  @vars[:return_code] = Errors::NO_PROJECTS_PATH
+                end
+              end
+            end
+            @vars[:artifacts].each do |art|
+              if art[:data][:temp].is_a?(FalseClass)
+                if File.exists?(art[:data][:file])
+                  File.unlink(art[:data][:file]) if File.exists?(art[:data][:file])
+                  art[:data].delete(:file)
+                  art[:data].delete(:temp)
+                else
+                  @logger.warn "Temporary file disappeared: #{data.ai}"
+                  @vars[:return_code] = Errors::TEMP_FILE_MISSING
+                end
+              end
+            end
+            @vars[:return_code]
+          end
+
+          # ---------------------------------------------------------------------------------------------------------------
+          def createProjectsFile
+            @logger.info __method__.to_s
+            projects = {}
+            project_names = loadProjectNames()
+            exts = {}
+            exts = Hash[@vars[:artifacts].map { |a| [a[:data][:name], File.basename(a[:data][:file]).match(CiCd::Builder::Manifest::Build::EXT_RGX)[1]] }]
+
+            createClassesFile()
+
+            @vars[:artifacts].each do |art|
+              prod = art[:data][:name]
+              mod  = art[:data][:module]
+              projects[prod] = {
+                                         name: project_names[prod] || prod,
+                                       module: mod,
+                                          ext: exts[prod],
+                                 class_filter: @vars[:filters][prod] || @vars[:filters][prod.gsub(/-manifest$/, '')],
+                               }
+            end
+
+            require 'chef/mash'
+            require 'chef/mixin/deep_merge'
+
+            projects_hash = File.exists?(ENV['PROJECTS_FILE']) ? loadConfigFile(ENV['PROJECTS_FILE']) : {}
+            old_projects = ::Chef::Mash.new(projects_hash)
+            projects = ::Chef::Mash.new(projects)
+            projects = ::Chef::Mash.new(::Chef::Mixin::DeepMerge.deep_merge(projects, old_projects))
+            saveConfigFile(ENV['PROJECTS_FILE'], projects)
+            data = {
+                          name: 'projects-file',
+                        module: 'projects-file',
+                          file: ENV['PROJECTS_FILE'],
+                       version: @vars[:build_ver],
+                         build: @vars[:build_num],
+                    properties: @properties_matrix,
+                          temp: false,
+                          sha1: Digest::SHA1.file(ENV['PROJECTS_FILE']).hexdigest,
+                           md5: Digest::MD5.file(ENV['PROJECTS_FILE']).hexdigest,
+                   }
+
+            maybeUploadArtifactoryObject(
+                                                     data: data,
+                                          artifact_module: data[:name],
+                                         artifact_version: data[:version] || @vars[:version],
+                                                file_name: '',
+                                                 file_ext: (ENV['PROJECTS_FILE'] and ENV['PROJECTS_FILE'].downcase.match(/\.ya?ml$/)) ? 'yaml' : 'json'
+                                        )
+          end
+
+          # ---------------------------------------------------------------------------------------------------------------
+          # noinspection RubyHashKeysTypesInspection
+          def createClassesFile()
+            @logger.info __method__.to_s
+            project_names = loadProjectNames()
+
+            @vars[:classes] = YAML.load(IO.read(ENV['CLASSES_MANIFEST_FILE']))
+            # keys = Hash[classes.keys.map.with_index.to_a].keys.sort
+
+            @vars[:filters] = {}
+            filters = {}
+            @vars[:classes].each do |role,apps|
+              apps.map{ |app|
+                filters[app] ||= []
+                filters[app] << role
+              }
+            end
+            filters.each do |app,roles|
+              @vars[:filters][app] = Hash[roles.map.with_index.to_a].keys.join('|')
+            end
+
+            saveConfigFile(ENV['CLASSES_FILE'],@vars[:classes])
+            data = {
+                              name: 'classes-file',
+                            module: 'classes-file',
+                              file: ENV['CLASSES_FILE'],
+                           version: @vars[:build_ver],
+                             build: @vars[:build_num],
+                        properties: @properties_matrix,
+                              temp: false,
+                              sha1: Digest::SHA1.file(ENV['CLASSES_FILE']).hexdigest,
+                               md5: Digest::MD5.file(ENV['CLASSES_FILE']).hexdigest,
+                    }
+
+            maybeUploadArtifactoryObject(
+                                                     data: data,
+                                          artifact_module: data[:name],
+                                         artifact_version: data[:version] || @vars[:version],
+                                                file_name: '',
+                                                 file_ext: 'yaml'
+                                        )
+          end
+
+          def saveConfigFile(file, projects)
+            @logger.info "Save config file: #{file}"
+            ext = file.gsub(/\.(\w+)$/, '\1')
+            IO.write(file, case ext.downcase
+                           when /ya?ml/
+                             projects.to_hash.to_yaml line_width: 1024, indentation: 4, canonical: false
+                           when /json|js/
+                             JSON.pretty_generate(projects.to_hash, {indent: "\t", space: ' '})
+                           else
+                             raise "Unsupported extension: #{ext}"
+                           end)
+          end
+
+          def loadConfigFile(file)
+            ext = file.gsub(/\.(\w+)$/, '\1')
+            hash = case ext.downcase
+                   when /ya?ml/
+                     YAML.load_file(ENV['PROJECTS_FILE'])
+                   when /json|js/
+                     JSON.load(IO.read(ENV['PROJECTS_FILE']))
+                   else
+                     raise "Unsupported extension: #{ext}"
+                   end
+          end
+
+          def loadProjectNames(fresh=false)
+            if fresh
+              @project_names = nil
+            end
+            unless @project_names
+              @project_names = {}
+              unless ENV['PROJECT_NAMES'].nil?
+                if File.exists?(ENV['PROJECT_NAMES'])
+                  @logger.info "Load PROJECT_NAMES: #{ENV['PROJECT_NAMES']}"
+                  @project_names = JSON.load(IO.read(ENV['PROJECT_NAMES'])) || {}
+                else
+                  @logger.error "The PROJECT_NAMES file (#{ENV['PROJECT_NAMES']}) does not exist!"
+                  @vars[:return_code] = Errors::NO_PROJECT_NAMES
+                end
+              end
+            end
+            @project_names
+          end
+
+          def createSuperManifest(manifest)
+            manifest_data = ''
+            manifest.each do |mod, man|
+              man.each do |k, v|
+                manifest_data += "#{k}=#{v}\n"
+              end
+            end
+            amn = artifactory_manifest_name # Just using a local iso invoking method_missing repeatedly ... ;)
+            data = {module: amn, data: manifest_data, version: @vars[:build_ver], build: @vars[:build_num], properties: @properties_matrix}
+            # tempArtifactFile(amn, data)
+
+            data[:file] = Dir::Tmpname.create(amn) do |tmpnam, n, opts|
+              mode = File::RDWR|File::CREAT|File::EXCL
+              perm = 0600
+              opts = perm
+            end + '.properties'
+            IO.write(data[:file], data[:data])
+            data[:temp] = false
+            data[:sha1] = Digest::SHA1.file(data[:file]).hexdigest
+            data[:md5]  = Digest::MD5.file(data[:file]).hexdigest
+            data[:name] = amn
+            @vars[:artifacts] << {
+                                    key: amn,
+                                   data: data.dup,
+                                 }
+            # manifest[amn]={ amn => "#{@vars[:build_ver]}-#{@vars[:build_num]}" }
+
+            maybeUploadArtifactoryObject(data: data, artifact_module: amn, artifact_version: data[:version] || @vars[:version], file_name: '', file_ext: 'properties')
+          end
+
+          # ---------------------------------------------------------------------------------------------------------------
+          def createProductManifests(manifest)
+            manifest.dup.each do |mod, man|
+              manifest_data = ''
+              man.each do |k, v|
+                manifest_data += "#{k}=#{v}\n"
+              end
+              data = {
+                            name: "#{mod}-manifest",
+                          module: "#{mod}-manifest",
+                       component: mod,
+                            data: manifest_data,
+                         version: @vars[:build_ver],
+                           build: @vars[:build_num],
+                      properties: @properties_matrix
+              }
+              # tempArtifactFile("#{mod}-manifest", data)
+
+              data[:file] = Dir::Tmpname.create("#{mod}-manifest") do |tmpnam, n, opts|
+                mode = File::RDWR|File::CREAT|File::EXCL
+                perm = 0600
+                opts = perm
+              end + '.properties'
+              IO.write(data[:file], data[:data])
+              data[:temp] = false
+              data[:sha1] = Digest::SHA1.file(data[:file]).hexdigest
+              data[:md5]  = Digest::MD5.file(data[:file]).hexdigest
+              @vars[:artifacts] << {
+                                       key: "#{mod}-manifest",
+                                      data: data.dup,
+                                   }
+              # noinspection RubyStringKeysInHashInspection
+              # manifest["#{mod}-manifest"]={ "#{mod}-manifest" => "#{@vars[:build_ver]}-#{@vars[:build_num]}" }
+
+              maybeUploadArtifactoryObject(data: data, artifact_module: data[:name], artifact_version: data[:version] || @vars[:version], file_name: '', file_ext: 'properties') # -#{@vars[:variant]}
+            end
+          end
+
+          # ---------------------------------------------------------------------------------------------------------------
+          def maybeUploadArtifactoryObject(args)
+            super
+            if @vars[:return_code] == 0
+              data             = args[:data]
+              artifact_module  = args[:artifact_module]
+              artifact_version = args[:artifact_version]
+              # file_ext         = args[:file_ext]
+              # file_name        = args[:file_name]
+              if @manifest[artifact_module].nil?
+                @manifest[artifact_module] = {}
+                file_name = artifact_module
+              else
+                file_name, _ = get_artifact_file_name_ext(data)
+                if file_name.empty?
+                  file_name = artifact_module
+                else
+                  file_name = "#{artifact_module}#{file_name}"
+                end
+              end
+              @manifest[artifact_module][file_name] = artifact_version
+            end
+
+            @vars[:return_code]
+          end
+
+          private :createProductManifests, :createProjectsFile, :createSuperManifest
+        end
+      end
+    end
+  end
+end
+
+=begin
+
+                {
+                    "test-project": {
+                        "name": "test-project",
+                        "module": "test-server",
+                        "ext": "war",
+                        "class_filter": "role.role-1"
+                    },
+                    "test-project-2": {
+                        "name": "test-project-2",
+                        "module": "test-server2",
+                        "ext": "zip",
+                        "class_filter": "role.role-2"
+                    },
+                    "test-manifest": {
+                        "name": "test-manifest",
+                        "module": "test-manifest",
+                        "ext": "properties",
+                        "class_filter": ""
+                    },
+                    "test-manifest2": {
+                        "name": "test-manifest2",
+                        "module": "test-manifest2",
+                        "ext": "properties",
+                        "class_filter": ""
+                    }
+                }
+
+=end

data/lib/cicd/builder/chefrepo-manifest/version.rb

@@ -0,0 +1,10 @@
+module CiCd
+  module Builder
+    module ChefRepoManifest
+      # chefrepo-manifest-builder version
+      VERSION = '0.1.1'
+      MAJOR, MINOR, TINY = VERSION.split('.')
+      PATCH = TINY
+    end
+  end
+end

data/lib/cicd/builder/chefrepo-manifest.rb

@@ -0,0 +1,39 @@
+require 'cicd/builder/manifest'
+
+module CiCd
+  module Builder
+    _lib=File.dirname(__FILE__)
+    $:.unshift(_lib) unless $:.include?(_lib)
+
+    require 'cicd/builder/chefrepo-manifest/version'
+
+    module ChefRepoManifest
+      class Runner < Manifest::Runner
+
+        # ---------------------------------------------------------------------------------------------------------------
+        def initialize()
+          super
+          @default_options[:builder] = VERSION
+        end
+
+        # ---------------------------------------------------------------------------------------------------------------
+        def getBuilderVersion
+          {
+              version:  VERSION,
+              major:    MAJOR,
+              minor:    MINOR,
+              patch:    PATCH,
+          }
+        end
+
+        # ---------------------------------------------------------------------------------------------------------------
+        def setup()
+          $stdout.write("ChefRepoManifestBuilder v#{CiCd::Builder::ChefRepoManifest::VERSION}\n")
+          super
+        end
+
+      end
+    end
+
+  end
+end

data/spec/builder_spec.rb

@@ -0,0 +1,8 @@
+require 'spec_helper'
+require 'cicd/builder'
+
+describe ManifestBuilder::Builder do
+  it "should have a VERSION constant" do
+    subject.const_get('VERSION').should_not be_empty
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+require 'rspec'
+require 'cicd/builder/version'
+
+include ManifestBuilder::Builder