vcseif 1.2.0

data/lib/vcseif/vcs_connector_driver.rb

@@ -0,0 +1,227 @@
+# Copyright 2001-2013 Rally Software Development Corp. All Rights Reserved.
+require "yaml"
+require "fileutils"
+
+require_relative "vcs_connector_runner"
+require_relative "utils/rally_logger"
+require_relative "utils/exceptions"
+
+class VCSConnectorDriver
+
+    NUMBER_OF_REQUIRED_VCS_CONNECTOR_SECTIONS = 3
+    VCS_APP_ROOT = 'VCS_APP_ROOT'
+
+    attr_reader :script_name, :uber_log_name
+    attr_accessor :log
+
+    def initialize(script_name)
+        rationalizeEnvironment()
+        @script_name = script_name
+        logs_directory = "%s/logs" % ENV[VCS_APP_ROOT]
+        if not Dir.exists?(logs_directory)
+            begin
+                FileUtils.mkdir(logs_directory)
+            rescue
+            end
+        end
+        @uber_log_name = "%s/%s.log"  % [logs_directory, script_name]
+    end
+
+    def rationalizeEnvironment()
+        # set vcs_app_root to the directory where either <vcs>2rally.exe or <vcs>2rally.rb is located
+        if ENV.keys.include?('OCRA_EXECUTABLE')
+          vcs_app_root = File.dirname(ENV['OCRA_EXECUTABLE'].gsub(/\\/, '/'))
+        else
+          vcs_app_root = File.expand_path(File.dirname($PROGRAM_NAME))
+        end
+
+        unless ENV.keys.include?(VCS_APP_ROOT)
+          ENV[VCS_APP_ROOT] = vcs_app_root  # make this available everywhere...
+          $LOAD_PATH.unshift(vcs_app_root)
+          $LOAD_PATH.unshift("#{vcs_app_root}/lib")
+        end
+    end
+
+    def execute(args)
+        @log = RallyLogger.new(@uber_log_name)
+        @log.info("#{@script_name} started with parameters of: #{args.join(', ')}")
+
+        ret_code = 0
+        for config_name in args do
+            config = getConnectorConfiguration(config_name)
+            next if config.nil?
+            ret_code = delegateToConnectorRunner(config)
+            @log.info("finished processing config #{config_name} with status code of #{ret_code}")
+        end
+        @log.info("#{@script_name} COMPLETED")
+        return 0
+    end
+
+    def getConnectorConfiguration(config_name)
+        """
+            Allow a config_name to map to a file with the same value 
+            exsiting within the ENV[VCS_APP_ROOT]/configs directory.
+            If such a file exists, it must be in YAML format and have at least 3 sections.
+            Allow specifying a config_name by 'name', 'name.yml', 'name.yaml' or 'name.txt'.
+        """
+        # suffix-full options: in ENV[VCS_APP_ROOT]/configs subdir
+        # suffix-less options: try '%s.yml'  % config_name in ENV[VCS_APP_ROOT]/configs subdir
+        #                      try '%s.yaml' % config_name in ENV[VCS_APP_ROOT]/configs subdir
+        #                      try '%s.txt'  % config_name in ENV[VCS_APP_ROOT]/configs subdir
+        #  TODO: consider cleaning this processing up in another method devoted solely to the sussing the options above
+
+        config_dir = "%s/configs" % ENV[VCS_APP_ROOT]
+        ext_name = File.extname(config_name)
+        if ext_name != ""  # suffix-full
+            config_file_path = '%s/%s' % [config_dir, config_name]
+            if Dir.glob("#{config_dir}/*").include?(config_name)
+                config_name = config_name.sub(/#{ext_name}$/, '')
+            end
+        else  # suffix-less
+            conf_root = "%s/%s" % [config_dir, config_name]
+            config_file_path = conf_root
+            if File.exists?('%s.yml' % conf_root)
+                config_file_path = "%s.yml" % conf_root
+            elsif File.exists?('%s.yaml' % conf_root)
+                config_file_path = "%s.yaml" % conf_root
+            elsif File.exists?('%s.txt' % conf_root)
+                config_file_path = "%s.txt" % conf_root
+            end
+        end
+
+        if not File.exists?(config_file_path)
+          problem = "Config file for name: %s not found" % config_name
+          @log.error(problem)
+          return nil
+        end
+
+        if not File.file?(config_file_path)
+          problem = 'Config file: %s is not a file' % config_file_path
+          @log.error(problem)
+          return nil
+        end
+
+        if not File.readable?(config_file_path)
+          problem = 'Config file: %s not a readable file' % config_file_path
+          @log.error(problem)
+          return nil
+        end
+
+        config = loadConfiguration(config_file_path)
+        config_file_name = config_file_path.split("/")[-1]
+        if not config.nil?
+            config['ConfigName'] = File.basename(config_file_name, '.*')
+            config['ConfigPath'] = config_file_path
+        end
+
+        return config
+    end
+
+    def loadConfiguration(config_file_path)
+        begin
+          content = "" and File.open(config_file_path, 'r') { |f| content = f.read() }
+          # As YAML forbids tab chars, we have to rid the content of tab chars,
+          # but emit a log warning that tabs were detected and transformed if we do
+          if content =~ /\t/
+            content.gsub!(/\t/, '    ')  
+            @log.warning('One or more tab characters (\\t) was detected in the config file and transformed to spaces.  YAML does not allow tab characters.')
+          end
+          config = YAML.load(content)
+          raise "config file: #{} is not a valid YAML file" unless config.class.name == 'Hash' 
+          sections = config.keys
+        rescue Exception => ex
+           @log.error("Unable to load YAML from config_file '%s' : %s" % [config_file_path, ex.message])
+           return nil
+        end
+
+        if config.keys.length < NUMBER_OF_REQUIRED_VCS_CONNECTOR_SECTIONS
+           @log.error("config '%s' not in required YAML format or has too few sections" % config_file_path)
+           return nil
+        end
+
+        return config
+    end
+
+    def delegateToConnectorRunner(config)
+        """
+            Given a Hash with the configuration content, 
+            obtain the correct _X_ConnectorRunner instance and feed it
+            the config and call its run method.
+        """
+        hits = config.keys.select {|key| key =~ /^\w+Connector$/}
+        if hits.nil? or hits.empty?
+            @log.error("Config %s lacks an entry identifying the EIF Connector type" % config['ConfigName'])
+            return 1
+        end
+        connector_type = hits.first
+
+        runner_name = '%sRunner' % connector_type
+        begin
+            connector_runner_class = Kernel.const_get(runner_name)
+        rescue => ex
+            problem = 'Unable to locate/load the %sRunner class.  Is your vcseif gem properly installed?'
+            @log.fatal(problem % connector_type)
+            return 2
+        end
+
+        begin
+            connector_runner = connector_runner_class.new(config)
+        rescue => ex
+            @log.error("#{ex.message}")
+            problem = 'Unable to obtain an instance of the %sRunner class.  Is your vcseif gem properly installed?'
+            @log.fatal(problem % connector_type)
+            return 3
+        end
+
+        ret_code = 0
+        blurtage = ""
+        begin
+            connector_runner.run()
+        rescue VCSEIF_Exceptions::ConfigurationError => ex
+            blurtage  = ex.verbiage
+            err_text  = "ERROR: #{@script_name} detected a FATAL configuration error, #{blurtage}."
+            log_entry = "FATAL: Configuration error: #{blurtage}"
+            recordAnomaly(ex, err_text, blurtage, log_entry)
+            ret_code = 4
+        rescue VCSEIF_Exceptions::UnrecoverableException => ex
+            blurtage  = ex.verbiage
+            err_text  = "ERROR: #{@script_name}, #{blurtage}."
+            log_entry = "UnrecoverableException: #{blurtage}"
+            recordAnomaly(ex, err_text, blurtage, log_entry)
+            ret_code = 5
+        rescue VCSEIF_Exceptions::OperationalError => ex
+            blurtage  = ex.verbiage
+            err_text  = "ERROR: #{@script_name} detected an OperationalError condition"
+            log_entry = "OperationalError: #{blurtage}"
+            recordAnomaly(ex, err_text, blurtage, log_entry)
+            ret_code = 6
+        rescue Exception => ex
+            blurtage  = ex.message
+            err_text  = "ERROR: #{@script_name} detected an Exception condition"
+            recordAnomaly(ex, err_text, blurtage, "")
+            ret_code = 9
+        end
+
+        return ret_code
+    end
+
+    def recordAnomaly(ex, issue, blurtage, log_entry)
+        $stderr.write("%s\n" % issue)
+        #$stderr.write("ERROR: #{blurtage}\n") if !blurtage.empty?
+        #$stderr.write("#{ex.message}\n")
+        @log.error(log_entry) if not log_entry.empty?
+
+        @log.entry("ERROR: #{blurtage} #{ex.message}")
+        @log.entry(ex.backtrace.join("\n"))
+        @log.write("call chain follows:")
+        # blurt out a formatted stack trace in outermost to innermost call sequence
+        for bt_line in ex.backtrace.reverse do
+            short_bt =  bt_line.sub(/#{Dir.pwd}\//, "... /")
+            short_bt = short_bt.sub(/^.*\/gems\//,  "... gems/")
+               @log.write("    #{short_bt}") 
+            $stderr.write("    #{short_bt}\n")
+        end
+        @log.write("!!!") 
+    end
+
+end

data/lib/vcseif/vcs_connector_runner.rb

@@ -0,0 +1,283 @@
+# Copyright 2001-2013 Rally Software Development Corp. All Rights Reserved.
+
+############################################################################################################
+#
+#  vcs_connector_runner - detect commits in a VCS Repository (non-Rally) and add 
+#                         any unrecorded changeset (and change) information to 
+#                         an SCMRepository in a Rally Workspace
+#
+############################################################################################################
+
+require 'date'
+require 'time'
+
+############################################################################################################
+
+class VCSConnectorRunner
+
+    attr_reader :log, :config
+    # the following declarations are necessary for testing, otherwise nothing accesses them
+    # except the instance
+    attr_reader :logfile_name, :lockfile_name, :preview, :connector, :extension, :time_file
+
+    EXISTENCE_PROCLAMATION = %{
+    ************************************************************************************************************
+
+         Rally VCSConnector starting at: %s  with pid: %s
+               curdir: %s
+              command: %s
+
+    ************************************************************************************************************
+    }
+
+    STD_TS_FMT = '%Y-%m-%d %H:%M:%S Z'
+
+    DEFAULT_ORIGIN_TIME = (DateTime.now - 3).to_time
+    ############################################################################################################
+
+
+    def initialize(config)
+        """
+            An instance of this class is instantiated with a Hash containing the 
+            specifics of a configuration to direct the execution.  Some rudimentary 
+            validation of the configuration is done, via use of an instance of a
+            Konfabulator, with any detected problem noted and a ConfigurationError is 
+            raised in that eventuality.
+            Once those items are taken care of, the instance obtains an instance of a
+            an AuxiliaryClassLoader instance and an instance of a VCSConnector.
+            The AuxiliaryClassLoader is delegated the task of pulling in any Extension classes.
+            These instances are then provided to the VCSConnector instance.
+            This instance then runs the operate_service method that directs the VCSConnector 
+            instance to obtain unrecorded changesets from the target VCS and reflect them in 
+            the Rally server.
+        """
+        @logfile_name  = "%s/logs/%s.log" % [ENV['VCS_APP_ROOT'], config['ConfigName']]
+        @lockfile_name = '%s/%s.LOCK'     % [ENV['VCS_APP_ROOT'], config['ConfigName']]
+        @log = RallyLogger.new(@logfile_name)
+        VCSEIF_Exceptions::logAllExceptions(true, @log)
+        @config    = config
+        @preview   = false
+        @connector = nil
+        @extension = {}
+        @time_file = nil
+    end
+
+    def proclaim_existence
+        proc = ProcTable.target_process($$) # $$ is the Ruby shorthand for Process.pid
+        start_time = Time.now.utc.strftime(STD_TS_FMT)
+        cmd_elements = proc.cmdline.split()
+        cmd_elements[0] = File.basename(cmd_elements.first)
+        proc.cmdline = cmd_elements.join(" ")
+        @log.write(EXISTENCE_PROCLAMATION % [start_time, proc.pid, Dir.pwd, proc.cmdline])
+    end
+
+    def acquireLock
+        """
+            Check for conditions to proceed based on lock file absence/presence status.
+            Acquisition of the lock is a green-light to proceed.
+            Failure to acquire the lock prevents any further operation.
+        """
+        if LockFile.exists?(@lockfile_name)
+            @log.warning("A %s file exists" % @lockfile_name)
+            locker = LockFile.current_lock_holder(@lockfile_name)
+            if not LockFile.locker_is_running?(@lockfile_name, locker)
+                message =  "A prior connector process (%s) did not clean up "
+                message << "the lock file on termination, proceeding with this run"
+                @log.warn(message % locker)
+            else
+                @log.error("Another connector process [%s] is still running, unable to proceed" % locker)
+                problem = "Simultaneous processes for this connector are prohibited"
+                boomex = VCSEIF_Exceptions::UnrecoverableException.new(problem)
+                raise boomex, problem
+            end
+        end
+
+        LockFile.create_lock(@lockfile_name)
+        return true
+    end
+
+    def releaseLock
+        LockFile.destroy_lock(@lockfile_name)
+    end
+
+
+    public
+    def run
+        proclaim_existence()
+        own_lock = false
+        own_lock = acquireLock()
+
+        begin
+            operateService()
+        rescue Exception => ex
+            raise
+        ensure
+            begin
+                releaseLock() if own_lock
+            rescue Exception => ex
+                problem = "ERROR: unable to remove lock file '%s', %s" % [@lockfile_name, ex.message]
+                shux = VCSEIF_Exceptions::OperationalError.new(problem)
+                raise shux, problem
+            end
+        end
+
+    end
+
+
+    private
+    def operateService()
+        started = finished = elapsed = nil
+        connector = nil
+        @log.info("processing to commence using content from the %s.yml config file" % @config['ConfigPath'])
+
+        conf_name = @config['ConfigName']
+        conf_path = @config['ConfigPath']
+        last_conf_mod  = File.mtime(conf_path).utc.strftime(STD_TS_FMT)
+        blurb_vars = [ conf_path, last_conf_mod, File.size(conf_path) ]
+        @log.info("%s last modified %s,  size: %d chars" % blurb_vars)
+        @config = inflateConfiguration(@config)  # transform @config from Hash to Konfabulator instance
+
+        this_run = Time.now.utc # be optimistic that reflectChangesetsInRally will succeed
+        now_zulu = this_run.strftime(STD_TS_FMT) # zulu <-- universal coordinated time <-- UTC
+
+        @time_file = TimeFile.new(buildTimeFileName(conf_name), @log)
+        if File.exists?(@time_file.filename)
+            #last_commit_zulu_seconds = (@time_file.read() - 3600) # the 3600 seconds is a DST compensator
+            last_commit_zulu_seconds = @time_file.read()
+            last_commit = Time.at(last_commit_zulu_seconds).utc # convert epoch seconds value to Zulu time 
+        else
+            last_commit = DEFAULT_ORIGIN_TIME
+        end
+        last_commit_zulu = last_commit.strftime(STD_TS_FMT)
+        log_msg = "Time File value %s --- Now %s" % [last_commit_zulu, now_zulu]
+        @log.info(log_msg)
+
+        connector = VCSConnector.new(@config, @log)
+        @log.debug("got an VCSConnector instance, calling its run method...")
+        status, repo_changesets = connector.run(last_commit, @extension)
+        finished = Time.now.utc
+        elapsed  = finished - this_run
+##
+##        puts "after VCSConnector.run, status: |#{status}| repo_changesets created to follow:"
+##        target_repo = repo_changesets.keys.first
+##        for repo_chgset in repo_changesets[target_repo] do
+##            name   = repo_chgset['Name']
+##            tstamp = repo_chgset['CommitTimestamp'].sub('T', ' ')
+##            puts "%36.36s  %-19.19s Z" % [name, tstamp]
+##        end
+##
+        logServiceStatistics(repo_changesets, elapsed.round)
+
+        if @preview
+            @log.info("Preview mode in effect, Time File not written/updated")
+            return
+        end
+        if status != true
+            # Not writing the time.file may cause repetitive detection of changesets, 
+            # but that is better than missing out on changesets altogether
+            @log.info("There was an error in processing so Time File was not written")
+            problem = "OperationalErrors detected during this processing run"
+            operr = VCSEIF_Exceptions::OperationalError.new(problem)
+            raise operr, problem
+        end
+
+        repo_name  = repo_changesets.keys().first
+        changesets = repo_changesets[repo_name]
+
+        # we've added changesets successfully, so update the Time File (x_time.file)
+        # the time_file instance converts epoch seconds to a readable representation
+        # but we also emit a readable representation of this_run in the log with the desired context
+        updated_last_commit = calc_last_commit_time(changesets, last_commit)
+        @time_file.write(updated_last_commit)
+        @log.info("time file written with value of %s" % updated_last_commit)
+    end
+
+
+    public
+
+    def calc_last_commit_time(changesets, default)
+      return default if changesets.nil? || changesets.empty?
+      last = changesets.first # default
+      last = changesets[-2] if changesets.length >= 2
+      #the last commit we get is a timestamp value from Rally which is already in Zulu with milliseconds
+      calctime = DateTime.parse(last["CommitTimestamp"]).to_time
+      @log.debug("Calculated timefile value of #{calctime}")
+      calctime
+    end
+
+    def buildTimeFileName(config_name)
+        tf_name = "%s/time.file" % ENV['VCS_APP_ROOT']
+        if not config_name.empty?
+            tf_name = "%s/%s_time.file" % [ENV['VCS_APP_ROOT'], config_name]
+        end
+        return tf_name
+    end
+
+
+    private
+    def logServiceStatistics(repo_changeset, elapsed)
+        """
+            what we intend to append to the log...  
+               vcs_conn_config.yml: 32 additional VCS changesets reflected in Rally
+            and a line with the elapsed time taken in human readable form  
+            (elapsed is in seconds (a float value))
+        """
+        repo_changeset.each_pair do |repo, changesets|
+            preview_reminder = "" # default
+            preview_reminder = "(Preview Mode)" if @preview == true
+            cs_length = changesets.nil? ? 0 : changesets.length
+            @log.info("%d changesets created for %s %s" % [cs_length, repo, preview_reminder])
+        end
+        hours, rem = elapsed.divmod(3600)
+        mins, secs =     rem.divmod(  60)
+        duration = "%d:%02d:%02d (hms)" % [hours, mins, secs]
+        @log.info("service run took %s" % duration)
+    end
+
+
+    public
+    def inflateConfiguration(config)
+        """
+            Given a Hash with config info, obtain and return a Konfabulator instance
+            that incorporates that config information into a form that provides an easy to
+            use interface to access the configuration specification elements.
+        """
+        begin
+            konfig = Konfabulator.new(config, @log)
+        rescue VCSEIF_Exceptions::ConfigurationError => ex
+            # info for this will have already been logged or blurted
+            raise
+        rescue VCSEIF_Exceptions::NonFatalConfigurationError => ex
+            # info for this will have already been logged or blurted
+            raise
+        rescue Exception => ex
+            raise StandardError, ex.message
+        end
+
+        svc_conf = konfig.topLevel('Services')
+        @preview = false
+        @preview = true if svc_conf and svc_conf['Preview'] == true
+        @log.info('Preview mode in effect') if @preview
+        @log_level = 'Info'
+        ll = svc_conf['LogLevel'] || nil
+        ll = ll.downcase.sub(/^(.)/) {$1.capitalize} unless ll.nil? # title case the result
+        if ['Fatal', 'Error', 'Warn', 'Info', 'Debug'].include?(ll)
+            @log_level = ll
+            @log.set_level(@log_level)
+            @log.info("LogLevel set to '%s'" % @log_level)
+        #else
+        #   bad LogLevel specified, do nothing, we've already defaulted to Info level
+        end
+
+        # TODO: enable this when a well-defined need for it has been established
+        #if svc_conf.contains('PostBatchExtension')
+        #    pba_class_name = svc_conf.get('PostBatchExtension')
+        #    pba_class = ExtensionLoader.getExtension(pba_class_name)
+        #    @extension['PostBatch'] = pba_class.new()
+        #end
+
+        return konfig
+    end
+
+end
+