gitchefsync 0.6.1

data/lib/gitchefsync/config.rb

@@ -0,0 +1,88 @@
+require 'gitchefsync/opts'
+require 'logger'
+#Not available till ruby 2.0
+#require 'syslog/logger'
+require 'syslog'
+require 'gitchefsync/log'
+
+#Central point of configuration
+module Gitchefsync
+  module Configuration
+
+    REL_BRANCH = 'master'
+
+        
+    def initialize(opts)
+      @git_bin = 'git'
+
+    end
+
+    def configure(options)
+      
+      @options = options
+      config = options[:config]
+      @git_bin = config['git']
+      @berks = config['berks']
+      @knife = config['knife']
+      @git_local = options[:git_local]
+      @token =  options[:private_token]
+      @stage_dir = config['stage_dir']
+      @rel_branch = config['release_branch']
+      @rel_branch ||= 'master'
+      @stage_cb_dir = options[:stage_cookbook_dir]
+      @stage_cb_dir ||= '/tmp/cookbooks'
+      @berks_upload = false
+      @audit_keep_trim = config['audit_keep_trim'] 
+      @audit_keep_trim ||= 20
+        
+      #backward compatibility for "sync_local" attribute
+      if config['sync_local'].is_a? String 
+        if config['sync_local'] == "true"
+          config['sync_local'] = true
+          config[:sync_local] = true
+        else
+          config['sync_local'] = false
+          config[:sync_local] = false
+        end
+      end
+      options[:syslog] ?
+        @log = Gitchefsync::Log::SysLogger.new('gitchefsync') :
+        @log = Logger.new(STDOUT)
+      #json based configuration
+      @config = config
+     
+    end
+
+    def parseAndConfigure(args)
+      include Parser
+
+      $args = args.clone
+      $opts = Parser.parseOpts args
+      configure $opts
+      
+      #instantiate audit
+      #@audit = Audit.new(@config['stage_dir'] )
+         
+      return $opts
+    end
+    
+    def logger
+      @log
+    end
+    
+    def self.log 
+      Gitchefsync.logger()
+      
+    end
+    
+    def configuration
+      @config
+    end
+    
+    def options
+      @options
+    end
+    
+    
+  end
+end

data/lib/gitchefsync/env_sync.rb

@@ -0,0 +1,339 @@
+require 'gitchefsync/git_util'
+require 'gitchefsync/errors'
+require 'gitchefsync/io_util'
+require 'gitchefsync/audit'
+require 'gitchefsync/config'
+require 'gitchefsync/common'
+
+module Gitchefsync
+
+  def self.included base
+    base.extend ClassMethods
+  end
+
+  class EnvRepo
+    
+    attr_reader :git_delta
+    
+    def initialize(https_url_to_repo)
+      options = Gitchefsync.options
+      config = Gitchefsync.configuration
+
+      Gitlab.private_token = options[:private_token]
+      @https_url_to_repo = https_url_to_repo
+      @git_group, @git_project = https_url_to_repo.split(File::SEPARATOR).last(2)
+      @stage_filepath = config['stage_dir']
+      @stage_target_path = File.join(@stage_filepath, [@git_group, @git_project.chomp(".git")].join('_'))
+      @git_default_branch = config['release_branch']
+      @git_delta = true
+      @git_bin = config['git']
+    end
+
+    def sync_repo
+      @git_delta = true
+      if Git.gitInit(@stage_target_path)
+        @git_delta = Gitchefsync.gitDelta(@stage_target_path, @git_default_branch)
+        msg = "cd #{@stage_target_path} && #{@git_bin} pull origin #{@git_default_branch}"
+        git_pull = lambda { |msg| Git.cmd msg }
+        if @git_delta
+          git_pull.call(msg)
+          Gitchefsync.logger.info "event_id=git_pull_repo_due_to_new_delta:repo=#{@stage_target_path}:git_delta=#{@git_delta}"
+        else
+          Gitchefsync.logger.info "event_id=skip_git_pull_repo_since_zero_delta:repo=#{@stage_target_path}:git_delta=#{@git_delta}"
+        end
+      else
+        stage_basename = @stage_target_path.split(File::SEPARATOR).last()
+        git_clone = Git.cmd "cd #{@stage_filepath} && #{@git_bin} clone #{@https_url_to_repo} #{stage_basename}"
+        check_default_branch = Git.cmd "cd #{@stage_target_path} && #{@git_bin} ls-remote origin #{@git_default_branch}"
+        Gitchefsync.logger.info "event_id=git_clone_repo_first_time:repo=#{@stage_target_path}:git_default_branch=#{@git_default_branch}"
+
+        #remove EnvRepo project in staging directory if default_branch does not exit
+        if check_default_branch.empty?
+          Gitchefsync.logger.fatal "event_id=rel_branch_does_not_exist=#{@git_default_branch}"
+          Gitchefsync.logger.fatal "event_id=removing_env_repo=#{@https_url_to_repo}, path: #{@stage_target_path}"
+          FS.cmd "rm -rf #{@stage_target_path}"
+          raise("#{@git_default_branch} does not exist in env_repo: #{@https_url_to_repo}")
+        end
+      end
+    end
+
+    def chef_path
+      return File.join(@stage_target_path, "chef-repo")
+    end
+
+    def validate_structure
+      if !File.directory?(self.chef_path)
+        Gitchefsync.logger.fatal "event_id=chef_repo_structure_problem"
+        raise("#{self.chef_path} is not a chef-repo path")
+      end
+    end
+  end
+
+  class EnvSync
+    def initialize(repo_list)
+      options = Gitchefsync.options
+      config = Gitchefsync.configuration
+
+      FS.knifeReady(config['stage_dir'],options[:knife_config])
+
+      repo_list.each do |repo|
+        repo.validate_structure
+      end
+
+      @knife = config['knife']
+      @stage_filepath = config['stage_dir']
+      @force_upload = config['force_upload']
+      @repo_list = repo_list
+      @audit = Audit.new(config['stage_dir'], 'env' )
+      @audit_keep_trim = config['audit_keep_trim']
+      @audit_keep_trim ||= 20
+      @env_file_list = Array.new()
+      @db_file_list = Array.new()
+      @role_file_list = Array.new()
+    end
+
+    def reject_json file
+      file_json = nil
+      begin
+        json = File.read file
+        file_json = JSON.parse json
+      rescue Exception => e
+        Gitchefsync.logger.error "event_id=env_parse_error:file=#{file}"
+        @audit.addEnv(file,'UPDATE', e )
+      end
+      file_json
+    end
+
+    def json_type file
+      return "env" unless FS.getBasePath(file, "environments").nil?
+      return "db" unless FS.getBasePath(file, "data_bags").nil?
+      return "role" unless FS.getBasePath(file, "roles").nil?
+    end
+
+    def validate_json(f, iden)
+      if f['basename'] != f['json'][iden]
+        raise ValidationError, "The file json's #{iden} attribute does not match basename: #{f['basename']}"
+      end
+      Gitchefsync.logger.debug "event_id=json_is_valid:iden=#{iden}:basename=#{f['basename']}"
+    end
+
+    def upload_env(f, git_delta)
+      Gitchefsync.logger.debug "event_id=upload_env:filepath=#{f['fullpath']}"
+      begin
+        validate_json(f, 'name')
+        @env_file_list << f['json']['name']
+        show_out = FS.cmdNoError "#{@knife} environment show #{f['json']['name']}"
+        if show_out.match("ERROR:") || git_delta || @force_upload
+          FS.cmd "#{@knife} environment from file #{f['fullpath']} --yes"
+          Gitchefsync.logger.info "event_id=environment_uploaded:file_json_name=#{f['json']['name']}:file=#{f['fullpath']}"
+          @audit.addEnv(f['fullpath'],'UPDATE' )
+        else
+          Gitchefsync.logger.debug "event_id=environment_not_uploaded:file_json_name=#{f['json']['name']}:file=#{f['fullpath']}"
+          @audit.addEnv(f['fullpath'],'EXISTING' )
+        end
+      rescue ValidationError => e
+        Gitchefsync.logger.error("event_id=validation_error:msg=#{e.message}")
+        @audit.addEnv(f['fullpath'],'UPDATE', e )
+      end
+    end
+
+    def data_bag_iden fullpath
+      chef_repo, data_bag = false, false
+      fullpath.split(File::SEPARATOR).each do |dir|
+        return dir if chef_repo && data_bag
+        chef_repo = dir.eql? "chef-repo" unless chef_repo
+        data_bag = dir.eql? "data_bags" if chef_repo
+      end
+      raise ValidationError, "event_id=invalid_path_to_data_bag_json:path=#{fullpath}"
+    end
+
+    def upload_db(f, git_delta)
+      Gitchefsync.logger.debug "event_id=upload_data_bag:filepath=#{f['fullpath']}"
+      db_iden = data_bag_iden(f['fullpath'])
+      begin
+        validate_json(f, 'id')
+        @db_file_list << [db_iden, f['json']['id']]
+        show_out = FS.cmdNoError "#{@knife} data bag show #{db_iden} #{f['json']['id']}"
+        if show_out.match("ERROR:") || git_delta || @force_upload
+          FS.cmd "#{@knife} data bag create #{db_iden}"
+          FS.cmd "#{@knife} data bag from file #{db_iden} #{f['fullpath']}"
+          Gitchefsync.logger.info "event_id=databag_uploaded:file_json_name=#{f['json']['id']}:file=#{f['fullpath']}"
+          @audit.addEnv(f['fullpath'],'UPDATE' )
+        else
+          Gitchefsync.logger.debug "event_id=data_bag_not_uploaded:file_json_name=#{f['json']['id']}:file=#{f['fullpath']}"
+          @audit.addEnv(f['fullpath'],'EXISTING')
+        end
+      rescue ValidationError => e
+        Gitchefsync.logger.error("event_id=validation_error:msg=#{e.message}")
+        @audit.addEnv(f['fullpath'],'UPDATE', e )
+      end
+    end
+
+    def upload_role(f, git_delta)
+      Gitchefsync.logger.debug "event_id=upload_role:fullpath=#{f['fullpath']}"
+      begin
+        validate_json(f, 'name')
+        @role_file_list << f['json']['name']
+        show_out = FS.cmdNoError "#{@knife} role show #{f['json']['name']}"
+        if show_out.match("ERROR:") || git_delta || @force_upload
+          FS.cmd "#{@knife} role from file #{f['fullpath']} --yes"
+          Gitchefsync.logger.info "event_id=role_uploaded:file_json_name=#{f['json']['name']}:file=#{f['fullpath']}"
+          @audit.addEnv(f['fullpath'],'UPDATE' )
+        else
+          Gitchefsync.logger.debug "event_id=role_not_uploaded:file_json_name=#{f['json']['name']}:file=#{f['fullpath']}"
+          @audit.addEnv(f['fullpath'],'EXISTING' )
+        end
+      rescue ValidationError => e
+        Gitchefsync.logger.error("event_id=validation_error:msg=#{e.message}")
+        @audit.addEnv(f['fullpath'],'UPDATE', e )
+      end
+    end
+
+    def cleanup_json_files
+      Gitchefsync.logger.info "cleanup_json_files"
+
+      #list env and compare on the server, deleting ones that aren't in git
+      knifeUtil = KnifeUtil.new(@knife, @stage_filepath)
+
+      delta_env_list = knifeUtil.listEnv() - @env_file_list
+      Gitchefsync.logger.info "event_id=env_diff:delta=#{delta_env_list}"
+      # MAND-672
+      delta_env_list.each do |env_name|
+        # TODO: Audit file may not be correct if someone manually
+        #       uploaded an environment file with json 'name' variable different
+        #       then actual environment filename.
+        a = AuditItem.new(env_name,'',nil)
+        a.setAction "DEL"
+        FS.cmd "#{@knife} environment delete #{env_name} --yes"
+        Gitchefsync.logger.info "event_id=environment_deleted:env_name=#{env_name}"
+        @audit.add(a)
+      end
+     
+      delta_db_list = knifeUtil.listDB() - @db_file_list
+      Gitchefsync.logger.info "event_id=data_bag_item_diff:delta=#{delta_db_list}"
+      delta_db_list.each do |bag, item|
+        # TODO: Audit file may not be correct if someone manually
+        #       uploaded an data bag with item json 'id' variable different
+        #       then actual json filename.
+        a = AuditItem.new("BAG: #{bag} ITEM: #{item}",'',nil)
+        a.setAction "DEL"
+        FS.cmd "#{@knife} data bag delete #{bag} #{item} --yes"
+        Gitchefsync.logger.info "event_id=data_bag_item_deleted:bag=#{bag}:item=#{item}"
+        @audit.add(a)
+        
+        items_remaining = knifeUtil.showDBItem(bag)
+        if items_remaining.empty?
+          a = AuditItem.new("BAG: #{bag}",'',nil)
+          a.setAction "DEL"
+          FS.cmd "#{@knife} data bag delete #{bag} --yes"
+          Gitchefsync.logger.info "event_id=data_bag_deleted:bag=#{bag}"
+          @audit.add(a)
+        end
+      end
+ 
+      delta_role_list = knifeUtil.listRole() - @role_file_list
+      Gitchefsync.logger.info "event_id=role_diff:delta=#{delta_role_list}"
+      delta_role_list.each do |role_name|
+        # TODO: Audit file may not be correct if someone manually
+        #       uploaded an role file with json 'name' variable different
+        #       then actual role filename.
+        a = AuditItem.new(role_name,'',nil)
+        a.setAction "DEL"
+        FS.cmd "#{@knife} role delete #{role_name} --yes"
+        Gitchefsync.logger.info "event_id=role_deleted:role_name=#{role_name}"
+        @audit.add(a)
+      end
+
+      #TODO: must create audit for removal
+
+      @audit.write
+      #trim the audit file
+      @audit.trim(@audit_keep_trim)
+    end
+
+    def update_json_files
+      Gitchefsync.logger.info "event_id=update_json_files"
+      @env_file_list.clear
+      @env_file_list << "_default"
+      @db_file_list.clear
+      @role_file_list.clear
+
+      @repo_list.each do |repo|
+        env_dir = repo.chef_path + "/**/*json"
+
+        Dir.glob(env_dir).each  do |file|
+
+          file_attr = Hash.new()
+          file_attr['json'] = reject_json(file)
+          next if file_attr['json'].nil?
+          file_attr['type'] = json_type(file)
+          file_attr['filename'] = File.basename(file)
+          file_attr['basename'] = File.basename(file).chomp(".json")
+          file_attr['fullpath'] = file
+
+          if file_attr['type'].eql? "env"
+            upload_env(file_attr, repo.git_delta)
+          elsif file_attr['type'].eql? "db"
+            upload_db(file_attr, repo.git_delta)
+          elsif file_attr['type'].eql? "role"
+            upload_role(file_attr, repo.git_delta)
+          end
+        end
+      end
+
+      self.cleanup_json_files
+    end
+  end
+
+  # sync all environment, data_bags and roles json in repo(s)
+  def self.syncEnv
+    logger.info "event_id=env_sync_start"
+
+    #TODO: Auto discouver `chef-repo` type repositories known by chefbot
+    url = @config['git_env_repo']
+    url.gsub!("http://", "https://") if url.start_with? "http://"
+    envRepo = EnvRepo.new(https_url_to_repo=url)
+    if !@config['sync_local']
+      envRepo.sync_repo
+    else
+      logger.info "event_id=Skip_syncing_env_repos_from_git"
+    end
+
+    envSync = EnvSync.new(repo_list=[envRepo])
+    logger.info "event_id=start_to_update_json_files"
+    envSync.update_json_files
+  end
+
+  #Adding functionality to merge environment repos together
+  #by introspecting the "working directory" for chef-repo
+  #It's been assumed that all repositories have have pulled
+  def self.mergeEnvRepos
+    include FS,Git
+
+    global_env_path =  @git_local + "/global_chef_env"
+
+    working_dir = Dir.entries(@git_local).reject! {|item| item.start_with?(".") || item.eql?("global_chef_env")}
+    working_dir.each  do |dir|
+      path = File.join(@git_local, dir)
+      chef_repo_dir = path + "/chef-repo"
+      if Dir.exists?(chef_repo_dir)
+        logger.info("event_id=processing_child_env_repo:dir=#{dir}")
+        begin
+          #add this repository as a
+          Git.cmd "cd #{global_env_path} && #{@git_bin} remote add #{dir} file://#{File.join(path,dir)}"
+        rescue Exception => e
+          logger.info "event_id=git_remote_already_exists:#{e.message}"
+        end
+        begin
+          #Merge the content via pull
+          logger.info"event_id=env_merge:src=#{dir}"
+          output  = Git.cmd "cd #{global_env_path} && #{@git_bin} pull #{dir} master"
+          logger.info "event_id=env_merge_sucess:msg=#{output}"
+        rescue Exception => e
+          logger.error "event_id=env:output=#{output}"
+          Git.cmd "cd #{global_env_path} && #{@git_bin} reset --hard origin/master"
+        end
+      end
+    end
+  end
+end

data/lib/gitchefsync/errors.rb

@@ -0,0 +1,35 @@
+
+#Define custom exception classes here
+module Gitchefsync
+
+  # Custom error class for rescuing from all Gitlab errors.
+  class Error < StandardError; end
+
+  # command, system error
+  class CmdError < Error; end
+
+  #A Git error has occurred
+  #or "fatal/error condition"
+  class GitError < Error; end
+
+  #A knife cookbook is frozen error
+  class FrozenError < CmdError; end
+
+  class BerksError < CmdError; end
+
+  class NoBerksError < BerksError; end
+
+  class KnifeError < CmdError; end
+
+  class NoMetaDataError < KnifeError; end
+
+  class InvalidTar < Error; end
+
+  class AuditError < Error;  end
+
+  class NoGitGroups < Error; end
+
+  class ValidationError < Error; end
+
+  class ConfigError < Error; end
+end

data/lib/gitchefsync/git_util.rb

@@ -0,0 +1,102 @@
+require 'gitchefsync/config'
+require 'gitlab'
+#Git helper module
+module Gitchefsync
+  module Git
+
+    def self.hasGit()
+      begin
+        git_ver = `git --version`
+        return git_ver.match('git version .*')
+      rescue
+        #TODO
+        raise NoGit, "Git must be installed"
+      end
+    end
+
+    #check that path exists and git is intializated
+    def self.gitInit (path)
+      return File.directory?(path + "/.git")
+    end
+
+    #a check to determine if a repository exists
+    #this is inherintly dangerous operation, as network issues could
+    #prevent the real issue
+    #will provide a 2 stage check, one a pull (if successful) return true
+    #second, if that failts
+    def self.remoteExists(path, remote)
+      Gitchefsync.logger.debug "event_id=checkRemoteExists:path=#{path}"
+      begin
+        self.cmd("cd #{path} && git ls-remote")
+        return true
+      rescue GitError => e
+        Gitchefsync.logger.warn "event_id=git_pull_err:msg=#{e.message}"
+      end
+      # we've passed the first
+      begin
+        #arbitrary gitlab command
+        Gitlab.users()
+        #successfully called Gitlab and not been able to pull remotely: give up
+        return false
+      rescue Exception => e
+        #in the face of not being able to contact Gitlab (for whatever reason) assume repository alive
+        return true
+      end
+      return true
+    end
+
+    #Return all git tags (which map to a SHA-1 hash) that only exist on monitoring branch (@rel_branch)
+    #Solves MAND-602 "ChefSync - Tagged Cookbooks on non-targeted git branches get synced"
+    def self.branchTags(path, branch)
+      self.cmd "cd #{path} && git clean -xdf"
+
+      self.cmd "cd #{path} && git checkout #{branch}"
+      git_graph = "git log --graph --oneline --branches=master --pretty='%H'"
+      branch_tags = "#{git_graph} | grep '^*' |" +
+      " tr -d '|' | awk '{ print $2 }' |" +
+      " xargs -n1 git describe --tags --exact-match 2>/dev/null"
+
+      status = `cd #{path} && git status`.split(/\n/)
+      graph = `cd #{path} &&  #{git_graph}`.split(/\n/)
+      Gitchefsync.logger.debug "event_id=branchTags: path=#{path}, status=#{status}"
+      Gitchefsync.logger.debug "event_id=branchTags: path=#{path}, graph=#{graph}"
+
+      tags = self.cmd("cd #{path} && #{branch_tags}").split(/\n/)
+      Gitchefsync.logger.info "event_id=branchTags: path=#{path}, tags=#{tags}"
+      tags
+    end
+
+    #executes a command line process
+    #raises and exception on stderr
+    #returns the sys output
+    def self.cmd(x)
+      ret = nil
+      err = nil
+      Open3.popen3(x) do |stdin, stdout, stderr, wait_thr|
+        ret = stdout.read
+        err = stderr.read
+      end
+      ret << err
+
+      if ret.start_with?"error:"
+        raise GitError, "stderr=#{ret}:cmd=#{x}"
+      end
+      if ret.start_with?"fatal:"
+        raise GitError, "stderr=#{ret}:cmd=#{x}"
+      end
+      ret
+    end
+
+    def self.cmdNoError(x)
+      ret = nil
+      err = nil
+      Open3.popen3(x) do |stdin, stdout, stderr, wait_thr|
+        ret = stdout.read
+        err = stderr.read
+      end
+
+      ret << err
+      ret
+    end
+  end
+end

data/lib/gitchefsync/io_util.rb

@@ -0,0 +1,83 @@
+require 'open3'
+require 'gitchefsync/errors'
+# raise an excetpion here
+
+module Gitchefsync
+  module FS
+
+    #copy the knife file over
+    #TODO: do this the ruby way
+    def self.knifeReady (working_dir,knife_file)
+      chef_dir = working_dir + "/" + ".chef"
+      if !File.exists?(chef_dir)
+        self.cmd "mkdir -p #{chef_dir}"
+      end
+      if !File.exists?(knife_file)
+        raise(KnifeError, "knife file must be defined")
+      end
+
+      self.cmd "cp -f #{knife_file} #{chef_dir}/knife.rb"
+      #check for knife readiness
+      self.cmd "cd #{working_dir} && knife client list"
+    end
+
+    #executes a command line process
+    #raises and exception on stderr
+    #returns the sys output
+    def self.cmd(x, env={})
+      ret = nil
+      err = nil
+      Open3.popen3(env, x) do |stdin, stdout, stderr, wait_thr|
+        ret = stdout.read
+        err = stderr.read
+      end
+      if err.to_s != ''
+        raise CmdError, "stdout=#{err}:cmd=#{x}"
+      end
+      ret
+    end
+
+    #there is a host of errors associated with berks
+    #"DEPRECATED: Your Berksfile contains a site location ..."
+    #this method is to allow filtering on the message to determine
+    #what is a real error versus what is just a warning - at this time
+    #will just have no checking
+    def self.cmdBerks(x)
+      self.cmdNoError(x)
+    end
+
+    def self.cmdNoError(x)
+      ret = nil
+      err = nil
+      Open3.popen3(x) do |stdin, stdout, stderr, wait_thr|
+        ret = stdout.read
+        err = stderr.read
+      end
+
+      ret << err
+      ret
+    end
+
+    def self.flatten(path, find)
+      arr_path = Array.new
+      arr_path.unshift(File.basename(path, ".*"))
+      fp = path
+      while true do
+        fp = File.expand_path("..", fp)
+        return nil if fp == "/"
+        break if File.basename(fp) == find
+        arr_path.unshift(File.basename(fp))
+      end
+      arr_path.join("_")
+    end
+
+    def self.getBasePath(path, find)
+      fp = path
+      while true do
+        fp = File.expand_path("..", fp)
+        return nil if fp == "/"
+        return fp if File.basename(fp) == find
+      end
+    end
+  end
+end