gitchefsync 0.6.1

data/lib/gitchefsync/knife_util.rb

@@ -0,0 +1,209 @@
+#helper file for parsing knife commands
+#this may get refactored
+
+require 'gitchefsync/git_util'
+require 'gitchefsync/errors'
+require 'gitchefsync/io_util'
+require 'gitchefsync/config'
+
+module Gitchefsync
+
+  class KnifeUtil
+
+    def initialize(knife, wd)
+      @knife = knife
+      @wd = wd
+    end
+
+    #needs knife on the command path
+    #parses the knife command "cookbook list -a"
+    #returns a list of cookbooks
+    def listCookbooks
+      list = Array.new
+      str = FS.cmd "cd #{@wd} && #{@knife} cookbook list -a"
+      arr_str = str.split(/\n/)
+      arr_str.each do |line|
+        cb_name, *cb_versions = line.split(' ')        
+        cb_versions.each do |cb_version|
+          list << Cookbook.new(cb_name, cb_version)
+        end
+      end
+      list
+    end
+
+    #get a list of existing environment names on chef server
+    def listEnv
+      list = Array.new
+      str = FS.cmd "cd #{@wd} && #{@knife} environment list"
+      environments = str.split(/\n/)
+      environments.each do |env|
+        list << env.strip
+      end
+      list
+    end
+
+    #get a list of existing data bags items (in [bag, item] pairs) on chef server 
+    def listDB
+      list = Array.new
+      str = FS.cmd "cd #{@wd} && #{@knife} data bag list"
+      data_bags = str.split(/\n/)
+      data_bags.each do |bag|
+        data_bag_items = showDBItem bag.strip
+        data_bag_items.each do |item|
+          list << [bag.strip, item.strip]
+        end
+      end
+      list
+    end
+    
+    #get a list of existing data bag items (from specified bag) on chef server
+    def showDBItem bag
+      list = Array.new
+      str = FS.cmd "cd #{@wd} && #{@knife} data bag show #{bag}"
+      data_bag_items = str.split(/\n/)
+      data_bag_items.each do |item|
+        list << item.strip
+      end
+      list
+    end
+    
+    #get a list of existing role names on chef server
+    def listRole
+      list = Array.new
+      str = FS.cmd "cd #{@wd} && #{@knife} role list"
+      roles = str.split(/\n/)
+      roles.each do |role|
+        list << role.strip
+      end
+      list
+    end
+
+    #checks if the cookbook name and version exist in the
+    #array of cookbooks
+    #@param name - name of cookbook
+    #@param version - version of cookbook
+    #@param list - the list of cookbooks - from listCookbooks
+    def inList( name, version, list)
+      found = false
+      list.each do |item|
+        found = true if ( (name == item.name) and (version == item.version))
+      end
+      found
+    end
+
+    #Checks to see if cookbook given is in list
+    #uses inList method to determine it
+    def isCBinList(cookbook, list)
+      return inList( cookbook.name, cookbook.version, list)
+    end
+
+    #returns a list of are in list1 that are not in list2
+    def subtract(list1,list2)
+      list = Array.new
+      list1.each do |cookbook|
+        if !isCBinList(cookbook,list2)
+          list << cookbook
+        end
+      end
+      list
+    end
+
+    #delete a cookbook from the server
+    def delCookbook(cb)
+      begin
+        FS.cmd("cd #{@wd} && knife cookbook delete #{cb.name} #{cb.version} -y" )
+      rescue CmdError => e
+        Gitchefsync.logger.error "event_id=cb_del:#{e.message}:e=#{e.backtrace}"
+      end
+    end
+
+    #Parse metadata.rb from a given directory path
+    def parseMetaData(path)
+      #Gitchefsync.logger.debug "Parsing metadata: #{path}"
+      if !File.exists?(File.join(path, "metadata.rb"))
+        raise NoMetaDataError
+      end
+      contents = ""
+      begin
+        file = File.new(File.join(path, "metadata.rb"), "r")
+
+        contents = file.read
+        version = attr_val(contents,"version")
+        name = attr_val(contents,"name")
+
+        if name.nil?
+          Gitchefsync.logger.warn "event_id=parse_meta_err_name:msg=cannot be resolved, deferring to directory name #{path}"
+          name = File.basename path
+        end
+        #parse maintainer information
+        maintainer = attr_val(contents,"maintainer")
+        email = attr_val(contents,"maintainer_email")
+
+        #puts "matched:#{name}:#{version}"
+        return Cookbook.new(name, version,maintainer,email)
+      rescue Exception => e
+        puts e.backtrace
+        Gitchefsync.logger.error "#{e.backtrace}"
+        raise KnifeError, "Unable to parse data: file=#{path}/metadata.rb #{contents}"
+      ensure
+        file.close unless file.nil?
+      end
+    end
+
+    def attr_val(contents, name)
+      m1 = contents.match(/#{name}\s+['|"](.*)['|"]/)
+      val = nil
+      if m1 != nil && m1.length == 2
+        val = m1[1]
+      else
+        Gitchefsync.logger.warn "event_id=parse_warn:name=#{name}"
+      end
+      val
+    end
+  end #end class
+
+  #A cookbook description
+  #may include a hash or other description
+  class Cookbook
+
+    def	initialize(name,version,maintainer = "", maintainer_email = "")
+      @name = name
+      @version = version
+      @maintainer = maintainer
+      @maintainer_email = maintainer_email
+    end
+
+    def name
+      @name
+    end
+
+    def version
+      @version
+    end
+
+    def maintainer
+      @maintainer
+    end
+
+    def setMaintainer(maintainer)
+      @maintainer = maintainer
+    end
+
+    def maintainer_email
+      @maintainer_email
+    end
+
+    def setMaintainer_email(maintainer_email)
+      @maintainer_email = maintainer_email
+    end
+
+    def to_s
+      return @name + "_" + @version
+    end
+
+    #name convention of how berks packages
+    def berksTar
+      return @name + "-" + @version + ".tar.gz"
+    end
+  end
+end

data/lib/gitchefsync/log.rb

@@ -0,0 +1,46 @@
+module Gitchefsync
+  module Log
+    #Wrapped sys log Logger
+    #Overload all the logger methods - although substitution is not covered
+    class SysLogger
+
+      #TODO: may define syslog open at this point
+      def initialize(name)
+        begin
+          Syslog.open(name, Syslog::LOG_PID, Syslog::LOG_LOCAL1)
+        rescue Exception => e
+          puts "Syslog error: #{e.message}"
+        end
+
+      end
+
+      def debug(*args)
+        log(Syslog::LOG_DEBUG,args[0])
+      end
+
+      def info(*args)
+        log(Syslog::LOG_INFO,args[0])
+      end
+
+      def warn(*args)
+        log(Syslog::LOG_WARNING,args[0])
+      end
+
+      def error(*args)
+        log(Syslog::LOG_ERR,args[0])
+      end
+
+      def fatal(*args)
+        log(Syslog::LOG_EMERG,args[0])
+      end
+
+      def log ( level, msg)
+        begin
+          Syslog.log(level, msg)
+        ensure
+          #Syslog.close
+        end
+      end
+    end
+  end
+end

data/lib/gitchefsync/notify.rb

@@ -0,0 +1,74 @@
+require 'gitchefsync/audit'
+require 'gitchefsync/io_util'
+require 'net/smtp'
+
+module Gitchefsync
+  
+  class Notification
+    
+    def initialize(smtp="mail.rim.net", from="mandolin@blackberry.com",to='mandolin@blackberry.com', msg="")
+      @to = to
+      @from = from
+      @smtp = smtp = Net::SMTP.start(smtp, 25)
+      @hostname = FS.cmd "hostname"
+    end
+
+    def notifyFromAudit(audit_dir, audit_type)
+      audit = Audit.new(audit_dir,audit_type)
+      audit_list = audit.latestAuditItems
+
+      audit_list.each do |audit_item|
+        if audit_item.ex != nil
+          h = audit_item.to_hash
+          msg = "From: gichefsync <mandolin@blackberry.com>\nTo: #{h[:maintainer]} #{h[:maintainer_email]}\nSubject: gitchefsync failure\n"
+          msg << "Alert from Hostname: #{@hostname}\n\n"
+          msg << "Attention!\n\n"
+          msg << "gitchefsync has identified you as the maintainer of this artifact\n"
+          msg << "====================================\n"
+          msg << "#{h[:name]}:#{h[:version]}\n"
+          msg << "====================================\n"
+          msg << "#{h[:exception]}"
+
+          sendTo(h[:maintainer_email],msg)
+          Gitchefsync.logger.info("event_id=email_sent=#{h[:maintainer_email]} ")
+        end
+      end
+      Gitchefsync.logger.info("event_id=notification_complete:audit_type=#{audit_type}")
+    end
+
+    #Aggregates a single email to the "to" email parameter
+    def singleNotifyFromAudit(audit_dir,audit_type,to)
+      audit = Audit.new(audit_dir,audit_type)
+      audit_list = audit.latestAuditItems
+      msg = "From: gichefsync <mandolin@blackberry.com>\nTo: #{to}\nSubject: gitchefsync failure: summary\n\n"
+      msg << "Alert from Hostname: #{@hostname}\n\n"
+      audit_list.each do |audit_item|
+        h = audit_item.to_hash
+
+        if h[:exception] != nil
+          ver = ""
+          if !h[:version].empty? then ver = ":" + h[:version] end
+
+          msg << "item: #{h[:name]}#{ver} was NOT processed with status #{h[:action]} "
+          msg << "\nERROR #{h[:exception]}"
+        else
+          msg << "item: #{h[:name]} was NOT processed with status #{h[:action]} "
+        end
+        msg << "\n\n"
+      end
+      sendTo(to,msg)
+    end
+
+    def send(body)
+      @smtp.send_message body, @from, @to
+    end
+
+    def sendTo(send_to, body)
+      @smtp.send_message body, @from, send_to
+    end
+
+    def close
+      @smtp.finish
+    end
+  end
+end

data/lib/gitchefsync/opts.rb

@@ -0,0 +1,64 @@
+require 'gitlab'
+require 'gitchefsync/errors'
+module Gitchefsync
+
+  module Parser
+
+    def self.parseOpts (args)
+      options = {}
+      begin
+
+        opt_parser = OptionParser.new do |opts|
+          opts.banner = "Usage: sync_all.rb --private_token=xyz OR LDAP credentials"
+          options[:private_token] = ''
+          opts.on('-t', '--private_token token','gitlab private token') do |token|
+            options[:private_token] = token
+          end
+          options[:config_file] = './sync-config.json'
+          opts.on('-c', '--config file','path to config file') do |token|
+            options[:config_file] = token
+          end
+          options[:login] = ''
+          opts.on('-l','--login login','Required when token not set') do |login|
+            options[:login] = login
+          end
+          options[:password] = ''
+          opts.on('-p','--password password','Required when token not set') do |pass|
+            options[:password] = pass
+          end
+          opts.on('-s','--syslog',"Enable syslog") do |syslog|
+            options[:syslog] = true
+          end
+          opts.on('-u', '--giturl', "Gitlab url") do |giturl|
+            Gitlab.endpoint = giturl
+          end
+        end
+
+        opt_parser.parse! (args)
+
+        json = File.read(options[:config_file])
+
+        j_config = JSON.parse(json)
+        options[:config] = j_config
+        options[:git_local] = j_config['working_directory']
+        options[:knife_config] = j_config['knife_file']
+        options[:groups] = j_config['git_groups']
+        options[:stage_cookbook_dir] = j_config['tmp_dir']
+
+        #set gitlab token and ensure git url for backward compatibility
+        if Gitlab.endpoint == nil
+          Gitlab.endpoint = 'https://gitlab.rim.net/api/v3'
+        end
+        if options[:private_token].empty? && !options[:login].empty? && !options[:password].empty?
+          puts "using credentials"
+          options[:private_token] = Gitlab.session(options[:login],options[:password]).to_hash['private_token']
+        end
+
+      rescue Exception => e
+        puts e.backtrace
+        raise(ConfigError,e.message,e.backtrace)
+      end
+      options
+    end
+  end
+end

data/lib/gitchefsync/schedule.rb

@@ -0,0 +1,161 @@
+require 'timeout'
+
+module Gitchefsync
+  class ScheduleSync
+    
+    def initialize()
+      options = Gitchefsync.options
+      config = Gitchefsync.configuration
+      
+      @lock_filename = config['lock_filename'] || 'sync.lock'
+      @lock_timeout = config['lock_timeout'] || 10
+      @lock_timeout = @lock_timeout.to_i
+      
+      @master_sync_timeout = config['master_sync_timeout'] || 600
+      @master_sync_timeout = @master_sync_timeout.to_i
+            
+      @sous_rsync_user = config['sous_rsync_user'] || 'chefsync'
+      @sous_rsync_host = config['sous_rsync_host'] || ''
+      @sous_rsync_src = config['sous_rsync_src'] || config['stage_dir'] || '/opt/gitchefsync/staging/'
+      @sous_rsync_dest = config['sous_rsync_dest'] || 
+                         File.join(File::SEPARATOR, config['stage_dir'].split(File::SEPARATOR)[1..-2]) || 
+                         '/opt/gitchefsync/'
+      @sous_rsync_options = config['sous_rsync_options'] || '-ar --delete'
+      @sous_rsync_excludes = config['sous_rsync_excludes'] || '.chef .snapshot'
+      
+      @sous_sync_timeout = config['sous_sync_timeout'] || 600
+      @sous_sync_timeout = @sous_sync_timeout.to_i 
+    end
+        
+    def obtainExclusiveLock 
+      Gitchefsync.logger.info "event_id=attempt_to_lock_file:lock_filename=#{@lock_filename}"
+      lock_file = File.open(@lock_filename, File::RDWR|File::CREAT, 0644)
+  
+      begin
+        Timeout::timeout(@lock_timeout) { lock_file.flock(File::LOCK_EX) }
+      rescue
+        Gitchefsync.logger.fatal "event_id=unable_to_lock_file:lock_filename=#{@lock_filename}"
+        exit 1
+      end
+      
+      lock_file
+    end
+  
+    def master
+      lock_file = obtainExclusiveLock
+      
+      begin
+        Timeout::timeout(@master_sync_timeout) do
+          Gitchefsync.logger.info "event_id=master_sync_starting"
+  
+          #Setup and check Gitlab API endpoint
+          Gitlab.endpoint = 'http://gitlab.rim.net/api/v3'
+          Gitchefsync.checkGit
+  
+          Gitchefsync.syncEnv
+          Gitchefsync.syncCookbooks
+          Gitchefsync.reconcile
+          cleanTmp()
+          Gitchefsync.logger.info "event_id=master_sync_completed"
+          
+          # MAND-615 - This file will signal its ok for this directory to be sous sync target
+          File.write(File.join(@sous_rsync_src, "master_sync_completed"), "")
+        end
+      rescue Timeout::Error
+        Gitchefsync.logger.fatal "event_id=master_sync_timed_out:master_sync_timeout=#{@master_sync_timeout}"
+        exit 1
+      rescue => e
+        Gitchefsync.logger.error "event_id=caught_exception:msg=#{e.message}"
+      end
+      lock_file.close
+    end
+  
+    def sous
+      lock_file = obtainExclusiveLock
+      
+      begin
+        Timeout::timeout(@sous_sync_timeout) do
+          Gitchefsync.logger.info "event_id=sous_sync_starting"
+          
+          exclude = ""
+          @sous_rsync_excludes.split(" ").each do |pattern|
+            exclude = "#{exclude} --exclude #{pattern}"
+          end
+          
+          if @sous_rsync_host.empty?
+            Gitchefsync.logger.fatal "event_id=sous_rsync_host_not_configured"
+            exit 1
+          end
+          
+          master_sync_completed = File.join(@sous_rsync_src, "master_sync_completed")
+          master_sync_completed_cmd = "ssh #{@sous_rsync_user}@#{@sous_rsync_host} ls #{master_sync_completed} 2>/dev/null"
+          Gitchefsync.logger.info "event_id=check_master_sync_completed:cmd=#{master_sync_completed_cmd}"
+          master_sync_completed_stdout = FS.cmd "#{master_sync_completed_cmd}"
+          
+          # MAND-615 - Do not perform an rsync on #{master_sync_completed} target if empty.
+          # Avoid doing rsync command and staged cookbook/env upload in situation where master 
+          # chef server was re-instantiated and master sync has yet to run once.
+          if master_sync_completed_stdout.empty?
+            Gitchefsync.logger.fatal "event_id=missing_master_sync_completed_file:master_sync_completed=#{master_sync_completed}"
+            exit 1
+          end
+            
+          rsync_cmd = "rsync #{@sous_rsync_options} #{exclude} #{@sous_rsync_user}@#{@sous_rsync_host}:#{@sous_rsync_src} #{@sous_rsync_dest} 2>/dev/null"
+          Gitchefsync.logger.info "event_id=execute_rsync:cmd=#{rsync_cmd}"
+          FS.cmd "#{rsync_cmd}"
+  
+          Gitchefsync.stagedUpload
+          Gitchefsync.syncEnv
+          Gitchefsync.reconcile
+          cleanTmp()
+          Gitchefsync.logger.info "event_id=sous_sync_completed"
+        end
+      rescue Timeout::Error
+        Gitchefsync.logger.fatal "event_id=sous_sync_timed_out:sous_sync_timeout=#{@sous_sync_timeout}"
+        exit 1
+      rescue => e
+        Gitchefsync.logger.error "event_id=caught_exception:msg=#{e.message}"
+      end
+      lock_file.close
+    end
+  
+    #Due to ridley bug (4.0.0+ possibly earlier) -clean the tmp directories
+    #takes current "Date" and cleans up
+    def cleanTmp
+      ts_str = "/tmp/d" + Date.today.strftime("%Y%m%d") + "-*"
+      Gitchefsync.logger.info "clean up of #{ts_str}"
+      FS.cmdNoError "sudo rm -fr #{ts_str}"
+    end
+  end
+  
+  def self.runMasterSync
+    scheduleSync = ScheduleSync.new()
+    scheduleSync.master
+  end
+    
+  def self.runSousSync
+    #force loocal sync
+    options[:config]['sync_local'] = "true"
+      
+    #Make sure sous sync only runs on the primary node
+    drbd_connection_state = FS.cmd("sudo drbdadm cstate chef_disk", 
+        { "TERM" => "xterm", "PATH" => "/usr/sbin:/usr/bin:/bin:/sbin" })
+    drbd_role = FS.cmd("sudo drbdadm role chef_disk", 
+        { "TERM" => "xterm", "PATH" => "/usr/sbin:/usr/bin:/bin:/sbin" })
+    
+    drbd_connection_state.delete!("\n")
+    drbd_role.delete!("\n")
+    
+    connected = drbd_connection_state.match(/Connected/)
+    role = drbd_role.match(/^Primary/)
+    
+    if connected and role
+      Gitchefsync.logger.info "event_id=proceed_to_sous_sync:drbd_connection_state=#{connected}:drbd_role=#{role}"
+      scheduleSync = ScheduleSync.new()
+      scheduleSync.sous
+    elsif   
+      Gitchefsync.logger.fatal "event_id=abort_sous_sync:drbd_connection_state=#{connected}:drbd_role=#{role}"
+      exit 1
+    end
+  end
+end

data/lib/gitchefsync/version.rb

@@ -0,0 +1,3 @@
+module Gitchefsync
+  VERSION = "0.6.1"
+end