perforce2svn 0.7.0

data/lib/perforce2svn/migrator.rb

@@ -0,0 +1,71 @@
+require 'perforce2svn/logging'
+require 'perforce2svn/errors'
+require 'perforce2svn/environment'
+require 'perforce2svn/mapping/mapping_file'
+require 'perforce2svn/perforce/commit_builder'
+require 'perforce2svn/subversion/svn_repo'
+require 'perforce2svn/version_range'
+require 'choosy/terminal'
+
+module Perforce2Svn
+  class Migrator
+    include Logging
+    include Choosy::Terminal
+    
+    def initialize(migrator_file, options)
+      Logging.configure(options[:debug])
+      Environment.new.check!
+
+      @migration_file = Mapping::MappingFile.new(migrator_file, options)
+      @svnRepo = Perforce2Svn::Subversion::SvnRepo.new(options[:repository])
+      @commit_builder = Perforce::CommitBuilder.new(@migration_file.mappings)
+      @version_range = options[:changes] || VersionRange.new(1)
+      @options = options
+    end
+
+    def run!
+      begin
+        @commit_builder.commits_in(@version_range) do |commit|
+          migrate_commit(commit)
+        end unless @options[:skip_perforce]
+
+        execute_commands unless @options[:skip_commands]
+      rescue SystemExit
+        raise
+      rescue Interrupt
+        @svnRepo.clean_transactions!
+        die "Interrupted. Not continuing."
+      rescue Exception => e
+        puts e.backtrace
+        log.error e
+        die "Unable to complete migration."
+      end
+    end
+
+    private
+    def migrate_commit(commit)
+      commit.log!
+      @svnRepo.transaction(commit.author, commit.time, commit.message) do |txn|
+        commit.files.each do |file|
+          if file.deleted?
+            txn.delete(file.dest)
+          elsif file.symlink?
+            txn.symlink(file.dest, file.symlink_target)
+          else
+            file.streamed_contents do |fstream|
+              txn.update(file.dest, fstream, file.binary?)
+            end
+          end
+        end
+      end
+    end
+
+    def execute_commands
+      @svnRepo.transaction(@migration_file.author, Time.now, @migration_file.message) do |txn|
+        @migration_file.commands.each do |command|
+          command.execute!(txn)
+        end
+      end
+    end
+  end
+end

data/lib/perforce2svn/perforce/commit_builder.rb

@@ -0,0 +1,159 @@
+require 'perforce2svn/logging'
+require 'perforce2svn/perforce/perforce_file'
+require 'perforce2svn/perforce/p4_depot'
+require 'perforce2svn/mapping/branch_mapping'
+require 'iconv'
+require 'svn/core'
+
+module Perforce2Svn::Perforce
+  # The standard commit message
+  class PerforceCommit
+    include Perforce2Svn::Logging
+
+    attr_reader :author, :message, :time, :revision, :files
+    
+    def initialize(revision, author, message, time, files)
+      @author = author
+      @revision = revision
+      @message = message
+      @time = time
+      @files = files
+    end
+
+    def log!
+      to_s.each_line do |line|
+        log.info line.chomp
+      end
+    end
+
+    def to_s
+      header = <<EOF
+================================================================
+Perforce Revision: #{revision}
+Time: #{time}
+User: #{author}
+Message:
+EOF
+      @message.each_line do |line|
+        header << "    #{line}"
+      end
+
+      header << "\nTotal Files: #{@files.length}\n"
+      @files.each do |file|
+        header << "    #{file}\n"
+      end
+
+      header
+    end
+  end
+
+  # Used to build commit information from the pretty much
+  # crazy data the P4 library returns
+  class CommitBuilder
+    include Perforce2Svn::Logging
+
+    def initialize(mappings)
+      @mappings = mappings
+      @log_converter = Iconv.new('UTF-8//IGNORE/TRANSLIT', 'UTF-8')
+    end
+
+    def commits_in(version_range, &block)
+      raise ArgumentError, "Requires a block" unless block_given?
+      if version_range.synced_to_head?
+        version_range.reset_to_head(P4Depot.instance.latest_revision)
+      end
+
+      skipped_previous = false
+      version_range.min.upto(version_range.max) do |revision|
+        commit = commit_at(revision)
+        if commit
+          if skipped_previous
+            print "\n"
+          end
+          skipped_previous = false
+          yield commit
+        else
+          if log.debug? 
+            log.info "Skipping irrelevant revision: #{revision}"
+          elsif skipped_previous
+            print "\r[INFO]  Skipping irrelevant revision: #{revision}"
+          else
+            print "[INFO]  Skipping irrelevant revision: #{revision}"
+          end
+          skipped_previous = true
+        end
+      end
+    end
+
+    def commit_at(revision)
+      P4Depot.instance.query do |p4|
+        log.debug "PERFORCE: Inspecting revision: #{revision}"
+        raw_commit = p4.run('describe', '-s', "#{revision}")[0]
+        return build_from(raw_commit)
+      end
+    end
+
+    # Builds from a raw P4 library return
+    def build_from(raw_commit)
+      changes = unpack_file_changes(raw_commit)
+      return nil unless changes
+
+      revision = raw_commit['change'].to_i
+      author = raw_commit['user']
+      commit_log = @log_converter.iconv(raw_commit['desc'].gsub(/\r/, ''))
+      time = Time.at(raw_commit['time'].to_i)
+      
+      PerforceCommit.new(revision, author, commit_log, time, changes)
+    end
+
+    private
+    # The data structures returned from the P4 library
+    # are pretty much unusable, so we have to munge them
+    # into better objects
+    def unpack_file_changes(raw_commit)
+      depot_files = raw_commit['depotFile']
+      if depot_files.nil? || depot_files.length == 0
+        log.debug "No files present"
+        return nil
+      end
+
+      file_changes = []
+      actions = raw_commit['action']
+      types = raw_commit['type']
+      revisions = raw_commit['rev']
+
+      filter_changes(depot_files) do |i, src, dest|
+        action = actions[i]
+        type = types[i]
+        rev = revisions[i].to_i
+        file_changes << PerforceFile.new(rev, src, dest, type, action)
+      end
+
+      if file_changes.empty?
+        log.debug "No relevant files"
+        nil
+      else
+        file_changes
+      end
+    end
+
+    def filter_changes(depot_files, &block)
+      depot_files.each_index do |i|
+        src = depot_files[i]
+        dest = find_svn_path(src)
+        yield i, src, dest if dest
+      end
+    end
+
+    # TODO: Add regular expression joining
+    def find_svn_path(perforce_path)
+      @mappings.each do |mapping|
+        log.debug "Checking path: #{perforce_path}"
+        if mapping.matches_perforce_path? perforce_path
+          return mapping.to_svn_path perforce_path
+        end
+      end
+      nil
+    end
+  end#CommitBuilder
+end

data/lib/perforce2svn/perforce/p4_depot.rb

@@ -0,0 +1,69 @@
+require 'perforce2svn/logging'
+require 'perforce2svn/errors'
+require 'P4'
+require 'singleton'
+
+module Perforce2Svn::Perforce
+  class P4Depot
+    include Singleton
+    include Perforce2Svn::Logging
+
+    def initialize
+      @p4 = P4.new
+    end
+
+    def connect!
+      handle_errors do
+        @p4.connect unless @p4.connected?
+      end
+    end
+
+    def disconnect!
+      begin
+        @p4.disconnect if @p4.connected?
+      rescue Exception => e
+        log.fatal(e)
+        exit 1
+      end
+    end
+
+    def query(&block)
+      raise Perforce2Svn::P4Error, "Requires a block" unless block_given?
+      connect!
+      handle_errors do
+        yield @p4
+      end
+    end
+
+    # Retrieves the latest revision on the Perforce server
+    def latest_revision
+      if @latest_revision.nil?
+        query do |p4|
+          log.debug "Retrieving latest perforce revision"
+          output = p4.run("changes", "-m1")[0]
+          @latest_revision = output['change'].to_i
+        end
+      end
+      @latest_revision
+    end
+
+    private
+    def handle_errors(&block)
+      begin
+        yield
+      rescue P4Exception => e
+        @p4.warnings.each do |warning|
+          log.debug "PERFORCE: Skipping warning: #{warning}"
+        end
+        if @p4.errors.length > 0
+          log.error e
+          @p4.errors.each do |error|
+            log.error "PERFORCE: #{error}"
+          end
+          log.fatal "PERFORCE: Are you currently logged into the Perforce server? "
+          raise Perforce2Svn::P4Error, "Error while interacting with the Perforce server"
+        end
+      end
+    end
+  end # P4Depot
+end

data/lib/perforce2svn/perforce/perforce_file.rb

@@ -0,0 +1,81 @@
+require 'perforce2svn/errors'
+require 'perforce2svn/logging'
+require 'perforce2svn/perforce/p4_depot'
+require 'tmpdir'
+
+module Perforce2Svn::Perforce
+  # The collection of properties about a Perforce file
+  class PerforceFile
+    include Perforce2Svn::Logging
+
+    attr_reader :revision, :src, :dest, :type, :action
+
+    def initialize(revision, src, dest, type, action)
+      @revision = revision
+      @src = src
+      @dest = dest
+      @type = type
+      @action = action
+    end
+
+    # Is this a binary file?
+    def binary?
+      type =~ /binary/
+    end
+
+    # Is this a symlink
+    def symlink?
+      type =~ /symlink/
+    end
+
+    # Was it deleted in the last commit?
+    def deleted?
+      action == 'delete'
+    end
+
+    def to_s
+      "(#{@action}:#{@type}\##{@revision})\t#{@src}"
+    end
+
+    # Retrieves the target of a symlink
+    def symlink_target
+      p4query do |p4|
+        return p4.run('print', '-q', "#{@src}\##{@revision}")[0].strip
+      end
+    end
+
+    # Pulls the file contents for a given path for the
+    # specific file revision
+    def contents
+      streamed_contents do |stream|
+        return stream.read
+      end
+    end
+
+    # Pull a stream from a file at a specified file revision
+    def streamed_contents(&block)
+      raise Perforce2Svn::P4Error, "Requires a block to pull the file stream" unless block_given?
+      log.debug {  "PERFORCE: Reading file: #{@src}\##{@revision}" }
+
+      tmpfile = File.join(Dir.tmpdir, ".p4file-#{rand}")
+      begin
+        P4Depot.instance.query do |p4|
+          p4.run('print', '-o', tmpfile, '-q', "#{@src}\##{@revision}")
+        end
+
+        if !File.file? tmpfile
+          raise Perforce2Svn::P4Error, "Unable to retrieve the file contents: #{src}\##{revision}"
+        end
+
+        mode = binary? ? 'rb' : 'r'
+        File.open(tmpfile, mode) do |file|
+          yield file
+        end
+      ensure
+        if File.file? tmpfile
+          File.delete(tmpfile)
+        end
+      end
+    end
+  end#PerforceFile
+end

data/lib/perforce2svn/subversion/svn_repo.rb

@@ -0,0 +1,156 @@
+require 'perforce2svn/logging'
+require 'perforce2svn/errors'
+require 'perforce2svn/subversion/svn_transaction'
+require 'svn/repos'
+require 'svn/core'
+require 'fileutils'
+require 'iconv'
+
+module Perforce2Svn::Subversion
+  class SvnRepo
+    include Perforce2Svn::Logging
+
+    attr_reader :repository_path
+
+    # Initializes a repository at a given path.
+    # IF that repository does not exist, it creates one.
+    def initialize(repository_path)
+      raise ArgumentError, "No path given" if repository_path.nil?
+
+      @repository_path = repository_path
+
+      unless File.directory? repository_path
+        fs_config = {
+          Svn::Fs::CONFIG_FS_TYPE => Svn::Fs::TYPE_FSFS
+        }
+        Svn::Repos.create(@repository_path, {}, fs_config)
+      end
+    end
+
+    def transaction(author, time, log_message, &block)
+      if not block_given?
+        raise Perforce2Svn::SvnTransactionError, "Transactions must be block-scoped"
+      end
+
+      if author.nil? or author == '' or time.nil?
+        raise "The author or time was empty"
+      end
+
+      props = {
+        Svn::Core::PROP_REVISION_AUTHOR => author,
+        Svn::Core::PROP_REVISION_DATE => time.to_svn_format,
+        Svn::Core::PROP_REVISION_LOG => sanitize(log_message)
+      }
+    
+      begin
+        # Yield the transaction
+        txn = repository.transaction_for_commit(props)
+        svn_txn = SvnTransaction.new(txn, repository.fs.root)
+        yield svn_txn
+        # Finalize the transaction
+        svn_txn.send(:finalize!)
+
+        if not repository.fs.transactions.include?(txn.name)
+          log.fatal "Unable to commit the transaction to the repository (#{txn.name}): #{author} #{time}"
+          raise Perforce2Svn::SvnTransactionError, "Transaction doesn't exist in repository."
+        end
+    
+        svn_revision = repository.commit(txn)
+        # It doesn't look like the 'svn:date' 
+        # property gets set correctly during
+        # a commit, so we need to update it now
+        repository.set_prop(author, # Person authorizing change
+                            Svn::Core::PROP_REVISION_DATE, # Property to modify
+                            time.to_svn_format, # value
+                            svn_revision,   # revision
+                            nil,            # callbacks
+                            false,          # run pre-commit hooks
+                            false)          # run post-commit hooks
+        log.info("Committed Subversion revision: #{svn_revision}")
+        return svn_revision
+      rescue Exception => e
+        clean_transactions!
+        raise
+      end
+    end
+
+    # Deletes a repository
+    def delete!
+      if File.exists? @repository_path
+        FileUtils.rm_rf @repository_path
+      end
+    end
+
+    # Occasionally, we may interrupt a transaction in
+    # process.  In that case, we should make sure
+    # to clean up after we are done.
+    def clean_transactions!
+      `svnadmin lstxns #{@repository_path}`.each do |txn|
+        `svnadmin rmtxns #{@repository_path} #{txn}`
+        if $? != 0
+          log.error "Unable to clean transaction: #{txn}"
+        end
+      end
+    end
+
+    # Retrieves the current contents of the file 
+    # in the SVN repository at the given path.
+    # You can optionally supply the revision number
+    #
+    # Raises a Svn::Error::FsNotFound if the specific file path cannot
+    # be found.
+    #
+    # Raises a Svn::Error::FsNoSuchRevision if the revision cannot 
+    # be found.
+    def pull_contents(file_path, revision = nil)
+      repository.fs.root(revision).file_contents(file_path){|f| f.read}
+    end
+
+    # Checks that a path exists at a revision
+    def exists?(path, revision = nil)
+      repository.fs.root(revision).check_path(path) != 0
+    end
+
+    # Retrieve the commit log for a given revision number
+    def commit_log(revision)
+      author = repository.prop(Svn::Core::PROP_REVISION_AUTHOR, revision)
+      date = repository.prop(Svn::Core::PROP_REVISION_DATE, revision)
+      commit_log = repository.prop(Svn::Core::PROP_REVISION_LOG, revision)
+      timestamp = Time.parse_svn_format(date)
+        
+      SvnCommitInfo.new(revision, author, timestamp, commit_log)
+    end
+
+    def prop_get(path, prop_name, revision = nil)
+      repository.fs.root(revision).node_prop(path, prop_name)
+    end
+
+    def children(path, revision = nil)
+      repository.fs.root(revision).dir_entries(path).keys
+    end
+
+    private
+    def repository
+      @repository ||= Svn::Repos.open(@repository_path)
+    end
+
+    # There can be weird stuff in the log messages, so
+    # we make sure that it doesn't bork when committing
+    # FIXME: This is replicated in the PerforceCommit, refactor!
+    def sanitize(text)
+      @sanitizer ||= Iconv.new('UTF-8//IGNORE/TRANSLIT', 'UTF-8')
+      @sanitizer.iconv(text.gsub(/\r/, ''))
+    end
+  end # SvnRepo
+
+  class SvnCommitInfo
+    attr_reader :revision, :author, :timestamp, :log
+  
+    def initialize(revision, author, timestamp, log)
+      @revision = revision
+      @author = author
+      @timestamp = timestamp
+      @log = log
+    end
+  end
+end