vcseif 1.2.0

data/lib/vcseif/utils/time_file.rb

@@ -0,0 +1,80 @@
+# Copyright 2001-2013 Rally Software Development Corp. All Rights Reserved.
+require "date"
+require "time"
+
+class TimeFile
+
+  # Store the last time the connector was run - just in case the connector stops and restarts
+  # Time (as UTC) is stored as a string in YYYY-MM-DD HH:MM:SS UTC" format
+  ISO8601_FORMAT = "%Y-%m-%dT%H:%M:%S.%L%z"
+  STD_TS_FORMAT  = "%Y-%m-%d %H:%M:%S Z"
+  STD_STRPTIME_FMT = "%Y-%m-%d %H:%M:%S %z"
+
+  %{
+     An instance of this class is used to record a timestamp in a file.
+     The timestamp is an ASCII representation that is derived from the ISO-8601 format.
+   }
+
+  attr_reader :filename
+  attr_reader :log
+
+  def initialize(filename, logger)
+    @filename = filename
+    @log      = logger
+  end
+
+  def exists?
+    return File.exists?(@filename)
+  end
+
+  def read()
+    %{
+       Returns the time stored in the file as an epoch seconds value
+       or a default value of the time 5 minutes ago (again in epoch seconds form).
+     }
+    # default to 5 minutes ago if file non-existent or empty
+    default = Time.now - (5 * 60)
+    last_run_timestamp = nil
+
+    if exists?
+      begin
+        File.open(@filename, "r") do |file|
+          line = file.gets
+          if !line.nil? and !line.empty?
+            line = line.strip
+            if line =~ /\d+.*T\d+:\d+:\d+.\d+/  # in old-style ISO8601 format
+              last_run_timestamp = Time.strptime(line, ISO8601_FORMAT)
+            else
+              begin
+                last_run_timestamp = Time.strptime(line, STD_STRPTIME_FMT)
+              rescue => ex
+                problem = "Invalid format for timefile entry: #{line}"
+                action  = "reverting to default of #{last_run_timestamp}"
+                @log.error("%s, %s" % [problem, action])
+              end
+            end
+          end
+        end
+      rescue => ex
+        problem = "Could not read time entry from #{@filename}"
+        action  = "rewriting time file with default value"
+        @log.error("%s, %s" % [problem, action])
+        write(last_run_timestamp)
+      end
+    end
+
+    last_run_timestamp = default if last_run_timestamp.nil?
+    return last_run_timestamp
+  end
+
+  def write(timehack=nil)
+    %{
+       Writes the time (expected as a Ruby Time object)
+       to the file in ISO 8601 format
+     }
+    timehack = Time.now if timehack.nil?
+    stamp    = timehack.utc.strftime(STD_TS_FORMAT)
+    File.open(@filename, 'w') { |file| file.puts "%s\n" % stamp }
+  end
+
+end

data/lib/vcseif/vcs_connector.rb

@@ -0,0 +1,487 @@
+# Copyright 2001-2013 Rally Software Development Corp. All Rights Reserved.
+
+# This is the large stone level operation of the VCSEIF infrastructure,
+# wherein the two connections held by the connector get instantiated, 
+# connected to their respective systems and exercised.
+
+require 'time'
+require 'open3'   # so that ocra will pick this up
+
+# require_relative "../vcseif"   #dsmith - this should not be needed - available when you require in vcseif
+# which will make the following available:
+#    connection.VCSConnection
+#    utils/exceptions.UnrecoverableException
+#    utils/exceptions.ConfigurationError
+#    utils/exceptions.OperationalError
+#    utils/rally_logger.RallyLogger
+#    utils/auxloader.PluginLoader 
+#    utils/auxloader.ClassLoader
+
+##############################################################################################
+
+class VCSConnector
+
+    VERSION = VCSEIF::Version
+
+    PLUGIN_SPEC_PATTERN       = Regexp.compile('^(?<plugin_class>\w+)\s*\((?<plugin_config>[^\)]*)\)\s*$')
+    PLAIN_PLUGIN_SPEC_PATTERN = Regexp.compile('(?<plugin_class>\w+)\s*$')
+
+    RALLY_TIMESTAMP   = "%Y-%m-%dT%H:%M:%S.%L %Z"
+    ISO8601_TIMESTAMP = "%Y-%m-%dT%H:%M:%S"
+
+    @@transformable_attributes = ['Author']
+
+    attr_reader :config, :log
+    attr_reader :rally_conn, :vcs_conn, :rally_conn_class, :vcs_conn_class
+    attr_reader :vcs_name
+    attr_reader :rally_conf, :vcs_conf, :svc_conf, :txfm_conf
+    attr_reader :transforms
+
+    def initialize(config, logger)
+        @config = config
+        @log    = logger
+
+        conn_sections = []
+        begin
+            non_conn_sections = ['Services', 'Transforms']
+            conn_sections = config.topLevels.select {|section| !non_conn_sections.include?(section)}
+            if conn_sections.length != 2
+                problem = 'Config does not contain two connection sections'
+                confex = VCSEIF_Exceptions::ConfigurationError.new(problem)
+                raise confex, problem
+            end    
+        rescue Exception => ex
+            problem = 'Config file lacks sufficient information for VCSConnector operation'
+            confex = VCSEIF_Exceptions::ConfigurationError.new(problem)
+            raise confex, problem
+        end
+
+        @rally_conn = nil
+        @vcs_conn   = nil
+
+        @vcs_name = conn_sections.select {|section_name| section_name !~ /^Rally/}.first
+        @log.info("Rally VCS Connector for %s, version %s" % [@vcs_name, VERSION])
+        @log.info("Ruby platform: #{RUBY_PLATFORM}")
+        @log.info("Ruby  version: #{RUBY_VERSION}")
+
+        internalizeConfig(config)
+
+        rally_conn_class_name = config.connectionClassName('Rally')
+        vcs_conn_class_name   = config.connectionClassName(@vcs_name)
+
+        @log.debug("Loading #{rally_conn_class_name} class")
+        begin
+            @rally_conn_class = ClassLoader.loadConnectionClass('lib', rally_conn_class_name, 'rally_vcs_connection')
+        rescue Exception => ex
+            problem = 'Unable to load RallyVCSConnection class, %s' % ex.message
+            boomex = VCSEIF_Exceptions::UnrecoverableException.new(problem)
+            raise boomex, problem
+        end
+
+        @log.debug("Loading #{vcs_conn_class_name} class")
+        begin
+            @vcs_conn_class = ClassLoader.loadConnectionClass('lib', vcs_conn_class_name)
+        rescue Exception => ex
+            problem = 'Unable to load %sConnection class, %s' % [@vcs_name, ex.message]
+            boomex = VCSEIF_Exceptions::UnrecoverableException.new(problem)
+            raise boomex, problem
+        end
+
+        @log.debug('Obtaining Rally and VCS connections...')
+        establishConnections()
+        @log.debug('Rally and VCS connections established')
+        validate()  # basically just calls validate on both connection instances
+
+        @log.debug("loading Transform class ...")
+        @transforms = loadTransforms(@txfm_conf)
+        @log.debug("Transform loaded")
+
+        @log.info("Initialization complete: Delegate connections operational, " +\
+                  "aux facilities loaded, ready for scan/reflect ops")
+    end
+
+
+    def internalizeConfig(config)
+        @rally_conf = config.topLevel('Rally')
+        @vcs_conf   = config.topLevel(@vcs_name)
+        @svc_conf   = config.topLevel('Services')
+        @txfm_conf  = config.topLevel('Transforms') || {}
+        # default the Author transform to Passthru if not specified in @txfm_conf
+        if not @txfm_conf.keys.include?('Author')
+            @txfm_conf['Author'] = 'Passthru'
+        end
+##
+##        @log.debug("config.Services has |#{@svc_conf.inspect}|")
+##
+    end
+
+
+    def establishConnections()
+        @rally_conn = @rally_conn_class.new(@rally_conf, @log)
+        @vcs_conn   =   @vcs_conn_class.new(  @vcs_conf, @log)
+        @vcs_conn.connect()  # we do this before rally_conn to be able to get the vcs backend version
+        vcs_backend_version = @vcs_conn.getBackendVersion()
+
+        if @rally_conn != nil and @vcs_conn != nil and @rally_conn.respond_to?('set_integration_header')
+            # so we can use it in our X-Rally-Integrations header items here
+            rally_headers = {'name'    => 'Rally VCSConnector for %s' % @vcs_name,
+                             'version' => VERSION,
+                             'vendor'  => 'Rally Software',
+                             'other_version' => vcs_backend_version
+                            }
+            @rally_conn.set_integration_header(rally_headers)
+        end
+        @rally_conn.setSourceIdentification(@vcs_conn.name, @vcs_conn.uri)
+        @rally_conn.connect()  
+    end
+
+
+    private
+    def loadTransforms(txfms)
+        """
+            Given a txfms parm (nil or Hash with keys for each transformable field)
+            that specifies the plugin class and plugin config (Rally User entity attributes
+            names that could be used in the transformation process), validate the 
+            sytactical validity of the transformation specification, and load the 
+            module containing the transformer class and obtain an instance of that class.
+            Populate a Hash keyed by attribute name with the value for each key 
+            an instance of the transform class.
+
+            The field names are from the Rally Changeset entity.  They must have their
+            first letter capitalized and any other words in the entity name must also
+            have their first letter capitalized and there can be no embedded spaces.
+        """
+        attribute_transformer = {}
+        return attribute_transformer if txfms.nil?
+
+        txfms.each_pair do |attributeName, txfm_spec|
+            @log.debug("target attribute: %s txfm_spec: %s" % [attributeName, txfm_spec])
+            md = PLUGIN_SPEC_PATTERN.match(txfm_spec)
+            if not md.nil?
+                plugin_class_name, plugin_config = md[:plugin_class], md[:plugin_config]
+            else
+                md = PLAIN_PLUGIN_SPEC_PATTERN.match(txfm_spec)
+                if md.nil?
+                    problem = "Bad Plugin Spec for %s : %s" % [attributeName, txfm_spec]
+                    raise VCSEIF_Exceptions::ConfigurationError.new(problem)
+                end
+                plugin_class_name, plugin_config = md[:plugin_class], ""
+            end
+
+            begin
+                plugin_class = PluginLoader.getPlugin(plugin_class_name)
+            rescue => ex
+                raise 
+            end
+            if plugin_class.nil?
+                problem = 'Unable to load plugin for %s' % plugin_class_name
+                raise VCSEIF_Exceptions::UnrecoverableException.new(problem)
+            end
+
+            @log.debug("Transform Class: %s" % plugin_class.name)
+            kwargs = { 'config'    => plugin_config, 
+                       'vcs_ident' => @vcs_conn.name,
+                       'rally'     => @rally_conn,
+                       'logger'    => @rally_conn.log
+                     }
+            begin
+                transformer = plugin_class.new(kwargs)
+            rescue Exception => ex
+                problem =  "Caught exception on attempt to get instance of #{plugin_class_name} "
+                problem << "plugin, #{ex.message}"
+                raise StandardError, problem
+            end
+            attribute_transformer[attributeName] = (transformer)
+        end
+
+        return attribute_transformer
+    end
+
+
+    public
+    def validate()
+        """
+            This calls the validate method on both the Rally and the VCS connections
+        """
+        valid = true
+
+        @log.info("Connector validation starting")
+
+        if not @rally_conn.validate()
+            @log.debug("RallyConnection validation failed")
+            return false
+        end
+        @log.debug("RallyConnection validation succeeded")
+
+        if not @vcs_conn.validate()
+            @log.debug("%sConnection validation failed" % @vcs_name)
+            return false
+        end
+        @log.debug("%sConnection validation succeeded" % @vcs_name)
+        @log.info("Connector validation completed")
+
+        return valid
+    end
+
+
+    def run(last_commit_timestamp, extension)
+        """
+            Where the rubber meets the road, or rather, controlling 
+            the machinery that puts the rubber on the road...
+        """
+        preBatch(extension)
+        status, repo_changesets = reflectChangesetsInRally(last_commit_timestamp)
+        postBatch(extension, status, repo_changesets)
+        @rally_conn.disconnect()
+        @vcs_conn.disconnect()
+        return [status, repo_changesets]
+    end
+
+
+    def preBatch(extension)
+        """
+        """
+        if extension != nil and extension =~ /PreBatchAction/
+            preba = extension['PreBatchAction']
+            preba.service()
+        end
+    end
+
+
+    def postBatch(extension, status, repo_changesets)
+        """
+        """
+        if extension != nil and extension =~ /PostBatchAction/
+            postba = extension['PostBatchAction']
+            postba.service(status, repo_changesets)
+        end
+    end
+
+    #last_commit_timestamp is an instance of Time
+    def reflectChangesetsInRally(last_commit_timestamp)
+        """
+            The last commit timestamm is passed to Connection objects in UTC;
+            they are responsible for converting if necessary. 
+            Time in log messages is always reported in UTC (aka Z or Zulu time).
+        """
+        status = false
+        rally  = @rally_conn
+        vcs    = @vcs_conn
+
+        rally_repo   = @rally_conf['RepositoryName']
+        preview_mode = false
+        preview_item = @svc_conf['Preview'].to_s.downcase
+        preview_mode = true if ['true', 'yes', '1', 'ok', 'on'].include?(preview_item)
+        pm_tag = ''
+        action = 'adding'
+        if preview_mode == true
+            pm_tag = "Preview: "
+            action = "would add"
+            @log.info('***** Preview Mode *****   (no Changesets will be created in Rally)')
+        end
+
+        begin
+            rally_ref_time, vcs_ref_time = getRefTimes(last_commit_timestamp)
+            time_info = "    ref times -->  rally_ref_time: |%s|  vcs_ref_time: |%s|"
+            @log.debug(time_info % [rally_ref_time, vcs_ref_time])
+            recent_rally_changesets = rally.getRecentChangesets(rally_ref_time)
+            @log.debug("Obtained Rally recent changesets info")
+            recent_vcs_changesets   =   vcs.getRecentChangesets(  vcs_ref_time)
+            @log.debug("Obtained VCS recent changesets info")
+            unrecorded_changesets = identifyUnrecordedChangesets(recent_rally_changesets, 
+                                                                 recent_vcs_changesets, 
+                                                                 vcs_ref_time)
+            @log.info("%d unrecorded Changesets" % unrecorded_changesets.length)
+            
+            recorded_changesets = []
+            unrecorded_changesets.each do |changeset|
+                cts = changeset.commit_timestamp.sub('Z', ' Z').sub('T', ' ')
+                desc = '%sChangeset %16.16s | %s | %s  not yet reflected in Rally'
+                @log.debug(desc % [pm_tag, changeset.ident, cts, changeset.author])
+                committer = changeset.author
+                transformEligibleAttributes(changeset)
+
+                adds = collect_files_and_links(changeset.ident, changeset.adds, 'add')
+                mods = collect_files_and_links(changeset.ident, changeset.mods, 'mod')
+                dels = collect_files_and_links(changeset.ident, changeset.dels, 'del')
+
+                info = { 
+                         'Revision'        => changeset.ident,
+                         'CommitTimestamp' => changeset.commit_timestamp,
+                         'Committer'       => committer,
+                         'Author'          => changeset.author,
+                         'Message'         => changeset.message,
+                         'Additions'       => adds,
+                         'Modifications'   => mods,
+                         'Deletions'       => dels,
+                         'Uri'             => @vcs_conn.get_rev_uri(changeset.ident)
+                       }
+
+                desc = '%sChangeset %16.16s | %s | %s  %s to Rally...'
+                @log.info(desc % [pm_tag, changeset.ident, cts, changeset.author, action])
+                @log.debug("  Rev URI: #{info['Uri']}")
+
+                if not preview_mode
+                    begin
+                        if not rally.changesetExists?(changeset.ident)
+                            rally_changeset = rally.createChangeset(info)
+                            recorded_changesets << rally_changeset
+                        end
+                    rescue Exception => ex
+                        raise VCSEIF_Exceptions::OperationalError.new(ex.message)
+                    end
+                end
+            end
+            status = true
+        rescue VCSEIF_Exceptions::OperationalError => ex
+            # already resulted in a log message
+        rescue VCSEIF_Exceptions::UnrecoverableException => ex
+            raise
+        rescue Exception => ex
+            raise
+        end    
+        status = false if rally.operational_errors > 0
+
+        repo_changesets = {rally_repo => recorded_changesets} 
+        return [status, repo_changesets]
+    end
+
+    def collect_files_and_links(revnum, file_list, operation)
+      return_list = []
+      file_list.each do |rev_file|
+        file_uri =  @vcs_conn.get_file_rev_uri(revnum, rev_file, operation)
+        log.debug(" Change file URI: #{file_uri}")
+        return_list << {:file => rev_file, :link => file_uri}
+      end
+      return_list
+    end
+
+    def getRefTimes(last_commit_timestamp)
+        """
+            last_commit_timestamp is provided as an epoch seconds value. 
+            Return a two-tuple of the reference time to be used for obtaining the 
+            recent changesets in Rally and the reference time to be used for 
+            obtaining the recent changesets in the target VCS system.
+        """
+        rally_lookback = get_config_lookback(@rally_conf)
+        vcs_lookback   = get_config_lookback(@vcs_conf)
+        rally_ref_time = Time.at(last_commit_timestamp - rally_lookback).gmtime
+        vcs_ref_time   = Time.at(last_commit_timestamp -   vcs_lookback).gmtime 
+        return [rally_ref_time, vcs_ref_time]
+    end
+
+    def get_config_lookback(config)
+      default_value = 3600  # 1 hour in seconds
+      lookback_val = config['Lookback']
+      return default_value if lookback_val.nil? || lookback_val.to_i <= 0
+
+      #default assumption is in minutes for lookback setting
+      lookback_val  = lookback_val.to_s.downcase.strip
+      converted_val = lookback_val.to_i  #rubyism:  "5days".to_i = 5   "number".to_i = 0
+      return_val    = (converted_val <= 0) ? default_value : converted_val * 60  #seconds per minute
+
+      if lookback_val.include?("d")     #5 days, 5days, 5d, 5 d
+        return_val = (converted_val <= 0) ? default_value : converted_val * 86400  #seconds per day
+      elsif lookback_val.include?("h")  #24 hours, 24 hours, 24h, 24 h
+        return_val = (converted_val <= 0) ? default_value : converted_val * 3600  #seconds per hour
+      end
+      return_val
+    end
+
+    def transformEligibleAttributes(changeset)
+        """
+            Check for a transformer for each transformable attribute, 
+            if no such transformer, leave the changeset.<attribute> value unchanged.
+            Otherwise, get the transformer for the transformable attribute,
+            call it's service method with:
+               getattr(changeset, transformable_attribute.lower())
+            and use the result as the value for transformable_attribute,
+            modifying the changeset item.
+        """
+        @@transformable_attributes.each do |transformable_attribute|
+        @log.debug("tranformable attribute: |#{transformable_attribute}|")
+            if @transforms.include?(transformable_attribute)
+                @log.debug("tranforms has a transformer class for #{transformable_attribute}")
+                transformer = @transforms[transformable_attribute] 
+                target_attr = transformable_attribute.downcase()
+                @log.debug("transformable target attribute: |#{target_attr}|")
+                target_value = changeset.send(target_attr)
+                @log.debug("transform target_value: |#{target_value}|")
+                transformed = transformer.service(target_value)
+                @log.debug("transformed to value: |#{transformed}|")
+                if not transformed.nil?
+                    begin
+##
+##                        puts "#{changeset.class.name} attributes: #{changeset.instance_variables}"
+##
+                        changeset.send("#{target_attr}=", transformed)
+                    rescue Exception => ex
+                        puts "#{ex.message}"
+                        for bt_line in ex.backtrace do
+                            puts bt_line.sub(Dir.pwd, '${CWD}')
+                        end
+                        puts "#{ex.inspect}"
+                    end
+##
+##                puts "Changeset.#{target_attr} new value: |#{changeset.send(target_attr)}| should be set to |#{transformed}| ?"
+##
+                else
+                    # TODO: should we log this?
+                end
+            end
+        end
+    end
+
+
+    def identifyUnrecordedChangesets(rally_changesets, vcs_changesets, ref_time)
+        """
+            Assume that for purposes of matching that the changeset identity in either system
+            can be determined by the rev_ident (as opposed to the rev_num).
+            If there are items in the rally_changesets for which there is  a counterpart in 
+            the vcs_changesets, the information has already been reflected in Rally.  --> NOOP
+            If there are items in the rally_changesets for which there is no counterpart in 
+            the vcs_changesets, information has been lost,  dat be berry berry bad... --> ERROR
+            If there are items in the vcs_changesets   for which there is no counterpart in
+            the rally_changesets, those items are candidates to be reflected in Rally --> REFLECT
+        """
+        rally_tank = {}
+        # requirement:  rc.ident <-- rc.Revision
+        rally_changesets.each {|rally_chgset| rally_tank[rally_chgset.ident] = rally_chgset}
+
+        vcs_tank   = {}
+        # requirement:  vc.ident <-- {node} or {hash} or {number}, etc .
+##
+##        puts "vcs_changesets ..."
+##        vcs_changesets.each { |cset| { puts cset.details }
+##
+        vcs_changesets.each {|vcs_chgset| vcs_tank[vcs_chgset.ident] = vcs_chgset}
+
+        reflected_changesets  = rally_changesets.select {|rcs|   vcs_tank.has_key?(rcs.ident) == true }
+        unpaired_changesets   = rally_changesets.select {|rcs|   vcs_tank.has_key?(rcs.ident) == false}
+        unrecorded_changesets =   vcs_changesets.select {|vcs| rally_tank.has_key?(vcs.ident) == false}
+
+        if unpaired_changesets.length > 0
+            #lost_changesets = unpaired_changesets.select {|rcs| rcs.CommitTimestamp > ref_time}
+            lost_changesets = []
+            unpaired_changesets.each do |rcs|
+                rally_commit_time = Time.strptime(rcs.CommitTimestamp[0..18], ISO8601_TIMESTAMP)
+                lost_changesets << rcs if rally_commit_time > ref_time
+            end
+            # Hold off on blurting this information in the log. It may just lead to confusion.
+            # There _could_ be other systems throwing info into the Rally Change/Changeset pool
+            # (although we wouldn't expect that to happen very often).
+            #if not lost_changesets.empty?
+            #  problem =  "Changesets exist in Rally for which there is no "
+            #  problem << "longer a counterpart changeset in the target VCS\n    "
+            #  cs_idents = lost_changesets.collect {|cs| cs.ident}
+            #  @log.error(problem + cs_idents.join("\n    "))
+            #end
+        end
+    
+        return unrecorded_changesets
+    end
+
+end   # VCSConnector class
+
+####################################################################################
+