tractive 1.0.0

data/lib/tractive/migrator/engine/migrate_to_file.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Migrator
+  class Engine
+    module MigrateToFile
+      def migrate_to_file
+        Tractive::GracefulQuit.enable
+        migrate_tickets_to_file(@start_ticket, @filter_closed)
+      rescue RuntimeError => e
+        $logger.error e.message
+      end
+
+      private
+
+      # Creates github issues for trac tickets.
+      def migrate_tickets_to_file(start_ticket, filterout_closed)
+        $logger.info("migrating issues")
+        # We match the issue title to determine whether an issue exists already.
+        tractickets = @trac.tickets
+                           .for_migration(start_ticket, filterout_closed, @filter_options)
+                           .all
+        begin
+          lasttracid = tractickets.last[:id]
+        rescue StandardError
+          raise("trac has no ticket #{start_ticket}")
+        end
+
+        (start_ticket.to_i..lasttracid).each do |ticket_id|
+          ticket = tractickets.select { |i| i[:id] == ticket_id }.first
+
+          @current_ticket_id = ticket_id # used to build filename for attachments
+
+          if ticket.nil?
+            next unless @mockdeleted
+
+            ticket = mock_ticket_details(ticket_id)
+          end
+
+          raise("tickets out of sync #{ticket_id} - #{ticket[:id]}") if ticket[:id] != ticket_id
+
+          next if filterout_closed && (ticket[:status] == "closed")
+
+          Tractive::GracefulQuit.check("quitting after processing ticket ##{@last_created_issue}") do
+            @output_file.puts "}"
+          end
+
+          $logger.info(%{creating issue for trac #{ticket[:id]} "#{ticket[:summary]}" (#{ticket[:reporter]})})
+          # API details: https://gist.github.com/jonmagic/5282384165e0f86ef105
+          request = Migrator::Converter::TracToGithub.new(@config).compose(ticket)
+
+          @output_file.puts @delimiter
+          @output_file.puts({ @current_ticket_id => request }.to_json[1...-1])
+          @delimiter = "," if @delimiter != ","
+          response = { "status" => "added to file", "issue_url" => "/#{ticket[:id]}" }
+
+          $logger.info("Status: #{response["status"]}")
+
+          issue_id = response["issue_url"].match(/\d+$/).to_s.to_i
+          $logger.info("created issue ##{issue_id} for trac ticket #{ticket[:id]}")
+
+          @last_created_issue = ticket[:id]
+        end
+
+        @output_file.puts "}"
+      end
+    end
+  end
+end

data/lib/tractive/migrator/engine.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require_relative "engine/migrate_from_db"
+require_relative "engine/migrate_from_file"
+require_relative "engine/migrate_to_file"
+
+require_relative "converter/trac_to_github"
+require_relative "converter/twf_to_markdown"
+
+# Service to perform migrations
+module Migrator
+  class Engine
+    include Migrator::Engine::MigrateFromDb
+    include Migrator::Engine::MigrateToFile
+    include Migrator::Engine::MigrateFromFile
+
+    def initialize(args)
+      # def initialize(trac, github, users, labels, revmap, attachurl, singlepost, safetychecks, mockdeleted = false)
+      @config = args
+
+      db                = args[:db]
+      github            = args[:cfg]["github"]
+      safetychecks      = !(args[:opts][:fast])
+      mockdeleted       = args[:opts][:mockdeleted]
+      start_ticket      = args[:opts][:start]
+      filter_closed     = args[:opts][:openedonly]
+      input_file_name   = args[:opts][:importfromfile]
+
+      @filter_applied   = args[:opts][:filter]
+      @filter_options   = { column_name: args[:opts][:columnname], operator: args[:opts][:operator], column_value: args[:opts][:columnvalue] }
+
+      @trac  = Tractive::Trac.new(db)
+      @repo  = github["repo"]
+      @client = GithubApi::Client.new(access_token: github["token"])
+
+      if input_file_name
+        @from_file = input_file_name
+        file = File.open(@from_file, "r")
+        @input_file = JSON.parse(file.read)
+        file.close
+      end
+
+      @ticket_to_issue   = {}
+      @mockdeleted       = mockdeleted || @filter_applied
+
+      $logger.debug("Get highest in #{@repo}")
+      issues = @client.issues(@repo, { filter: "all",
+                                       state: "all",
+                                       sort: "created",
+                                       direction: "desc" })
+
+      @last_created_issue = issues.empty? ? 0 : issues[0]["number"].to_i
+
+      $logger.info("created issue on GitHub is '#{@last_created_issue}' #{issues.count}")
+
+      dry_run_output_file = args[:cfg][:dry_run_output_file] || "#{Dir.pwd}/dryrun_out.json"
+
+      @dry_run          = args[:opts][:dryrun]
+      @output_file      = File.new(dry_run_output_file, "w+")
+      @delimiter        = "{"
+      @revmap           = load_revmap_file(args[:opts][:revmapfile] || args[:cfg]["revmapfile"])
+      @safetychecks     = safetychecks
+      @start_ticket     = (start_ticket || (@last_created_issue + 1)).to_i
+      @filter_closed    = filter_closed
+    end
+
+    def migrate
+      if @dry_run
+        migrate_to_file
+      elsif @from_file
+        migrate_from_file
+      else
+        migrate_from_db
+      end
+    end
+
+    private
+
+    def load_revmap_file(revmapfile)
+      # load revision mapping file and convert it to a hash.
+      # This revmap file allows to map between SVN revisions (rXXXX)
+      # and git commit sha1 hashes.
+      revmap = nil
+      if revmapfile
+        File.open(revmapfile, "r:UTF-8") do |f|
+          $logger.info("loading revision map #{revmapfile}")
+
+          revmap = f.each_line
+                    .map { |line| line.split(/\s+\|\s+/) }
+                    .map { |rev, sha, _| [rev.gsub(/^r/, ""), sha] }.to_h # remove leading "r" if present
+        end
+      end
+
+      revmap
+    end
+
+    def mock_ticket_details(ticket_id)
+      summary = if @filter_applied
+                  "Not available in trac #{ticket_id}"
+                else
+                  "DELETED in trac #{ticket_id}"
+                end
+      {
+        id: ticket_id,
+        summary: summary,
+        time: Time.now.to_i,
+        status: "closed",
+        reporter: "tractive"
+      }
+    end
+
+    def update_comment_ref(issue_id)
+      comments = @client.issue_comments(@repo, issue_id)
+      comments.each do |comment|
+        next unless comment["body"].include?("Replying to [comment:")
+
+        updated_comment_body = create_update_comment_params(comment, comments, issue_id)
+        @client.update_issue_comment(@repo, comment["id"], updated_comment_body)
+      end
+    end
+
+    def create_update_comment_params(comment, comments, issue_id)
+      body = comment["body"]
+      matcher = body.match(/Replying to \[comment:(?<comment_number>\d+).*\]/)
+      matched_comment = comments[matcher[:comment_number].to_i - 1]
+      body.gsub!(/Replying to \[comment:(\d+).*\]/, "Replying to [#{@repo}##{issue_id} (comment:\\1)](#{matched_comment["html_url"]})")
+
+      body
+    end
+  end
+end

data/lib/tractive/migrator.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "migrator/engine"

data/lib/tractive/models/attachment.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Tractive
+  class Attachment < Sequel::Model(:attachment)
+    dataset_module do
+      where(:tickets_attachments, type: "ticket")
+      select(:for_export, :id, :filename)
+    end
+  end
+end

data/lib/tractive/models/milestone.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Tractive
+  class Milestone < Sequel::Model(:milestone)
+  end
+end

data/lib/tractive/models/report.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Tractive
+  class Report < Sequel::Model(:report)
+  end
+end

data/lib/tractive/models/revision.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Tractive
+  class Revision < Sequel::Model(:revision)
+  end
+end

data/lib/tractive/models/session.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Tractive
+  class Session < Sequel::Model(:session_attribute)
+  end
+end

data/lib/tractive/models/ticket.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Tractive
+  class Ticket < Sequel::Model(:ticket)
+    one_to_many :changes, class: Tractive::TicketChange, key: :ticket
+    one_to_many :attachments, class: Attachment, key: :id, conditions: { type: "ticket" }
+
+    dataset_module do
+      def for_migration(start_ticket, filterout_closed, filter_options)
+        tickets = order(:id)
+                  .where { id >= start_ticket }
+                  .filter_column(filter_options)
+
+        tickets = tickets.exclude(status: "closed") if filterout_closed
+
+        tickets
+      end
+
+      def filter_column(options)
+        return self if options.nil? || options.values.compact.empty?
+
+        if options[:operator].downcase == "like"
+          where { Sequel.like(options[:column_name].to_sym, options[:column_value]) }
+        else
+          where { Sequel.lit("#{options[:column_name]} #{options[:operator]} '#{options[:column_value]}'") }
+        end
+      end
+    end
+
+    def all_changes
+      # combine the changes and attachment table results and sort them by date
+      change_arr = changes + attachments
+      change_arr.sort_by { |change| change[:time] }
+    end
+  end
+end

data/lib/tractive/models/ticket_change.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Tractive
+  class TicketChange < Sequel::Model(:ticket_change)
+    many_to_one :ticket, key: :ticket
+  end
+end

data/lib/tractive/revmap_generator.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Tractive
+  class RevmapGenerator
+    def initialize(input_file, svn_url, svn_local_path, git_local_repo_path, output_file = "revmap.txt")
+      @input_file = input_file
+      @git_local_repo_path = git_local_repo_path
+      @svn_url = svn_url
+      @svn_local_path = svn_local_path
+      @duplicate_commits = {}
+      @duplicate_message_commits = {}
+      @last_revision = nil
+      @pinwheel = %w[| / - \\]
+      @output_file = output_file
+    end
+
+    def generate
+      line_count = File.read(@input_file).scan(/\n/).count
+      i = 0
+
+      File.open(@output_file, "w+") do |file|
+        File.foreach(@input_file) do |line|
+          info = extract_info_from_line(line)
+          next if @last_revision == info[:revision]
+
+          @last_revision = info[:revision]
+          print_revmap_info(info, file)
+
+          percent = ((i.to_f / line_count) * 100).round(2)
+          progress = "=" * (percent.to_i / 2) unless i < 2
+          printf("\rProgress: [%<progress>-50s] %<percent>.2f%% %<spinner>s", progress: progress, percent: percent, spinner: @pinwheel.rotate!.first)
+          i += 1
+        end
+      end
+    end
+
+    private
+
+    def extract_info_from_line(line)
+      info = {}
+
+      info[:revision], timestamp_author = line.split
+      info[:revision], info[:revision_count] = info[:revision].split(".")
+      info[:revision].gsub!("SVN:", "r")
+      info[:timestamp], author_count = timestamp_author.split("!")
+      info[:author], info[:count] = author_count.split(":")
+
+      info
+    end
+
+    def print_revmap_info(info, file)
+      # get sha from git api
+      commits = git_commits(info)
+
+      if commits.count == 1
+        file.puts "#{info[:revision]} | #{commits.values[0].join(",")}"
+      else
+        message = commit_message_from_svn(info[:revision])
+        file.puts "#{info[:revision]} | #{@duplicate_commits[info[:timestamp]][message].join(",")}"
+      end
+    end
+
+    def git_commits(info)
+      return @duplicate_commits[info[:timestamp]] if @duplicate_commits[info[:timestamp]]
+
+      # get commits from git dir
+      commits = commits_from_git_repo(info)
+
+      commits_hash = {}
+      commits.each do |commit|
+        message = commit[:message]
+        sha = commit[:sha]
+
+        if commits_hash[message]
+          commits_hash[message] << sha
+        else
+          $logger.warn("'#{sha}' has same timestamp, commiter and commit messgae as '#{commits_hash[message]}'") unless commits_hash[message].nil?
+          commits_hash[message] = [sha]
+        end
+      end
+
+      @duplicate_commits[info[:timestamp]] = commits_hash if commits.count > 1
+
+      commits_hash
+    end
+
+    def commit_message_from_svn(revision)
+      svn_logs = Tractive::Utilities.svn_log(@svn_url, @svn_local_path, "-r": revision, "--xml": "")
+      h = Ox.load(svn_logs, mode: :hash)
+      h[:log][:logentry][3][:msg]
+    end
+
+    def commits_from_git_repo(info)
+      command = "git rev-list --after=#{info[:timestamp]} --until=#{info[:timestamp]} --committer=#{info[:author]} --all --format='%cd|%h~|~%s' --date=format:'%Y-%m-%dT%H:%M:%SZ'"
+      commits = Dir.chdir(@git_local_repo_path) do
+        `#{command}`
+      end
+
+      commits_arr = []
+      commits.split("\n").each_slice(2) do |sha_hash, commit_info|
+        commit_hash = {}
+        commit_hash[:sha] = sha_hash.split.last
+        commit_hash[:short_sha], commit_hash[:message] = commit_info.split("~|~")
+
+        commits_arr << commit_hash
+      end
+
+      commits_arr
+    end
+  end
+end

data/lib/tractive/trac.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Tractive
+  class Trac
+    attr_reader :tickets, :changes, :sessions, :attachments
+
+    def initialize(db)
+      $logger.info("loading tickets")
+      @db          = db
+      @tickets     = Ticket
+      @changes     = TicketChange
+      @sessions    = Session
+      @attachments = Attachment
+    end
+  end
+end

data/lib/tractive/utilities.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Tractive
+  class Utilities
+    class << self
+      def make_hash(prefix, array)
+        array.map { |i| [i, "#{prefix}#{i}"] }.to_h
+      end
+
+      def setup_db!(db_url)
+        files_to_load = [
+          "lib/tractive/models/attachment.rb",
+          "lib/tractive/models/milestone.rb",
+          "lib/tractive/models/report.rb",
+          "lib/tractive/models/revision.rb",
+          "lib/tractive/models/session.rb",
+          "lib/tractive/models/ticket_change.rb",
+          "lib/tractive/models/ticket.rb"
+        ]
+        db = Sequel.connect(db_url) if db_url
+
+        raise("could not connect to tractive database") unless db
+
+        files_to_load.each do |file|
+          require_relative "../../#{file}"
+        end
+
+        db
+      end
+
+      def setup_logger(options = {})
+        $logger           = Logger.new(options[:output_stream])
+        $logger.level     = options[:verbose] ? Logger::DEBUG : Logger::INFO
+        $logger.formatter = proc do |severity, datetime, _progname, msg|
+          time = datetime.strftime("%Y-%m-%d %H:%M:%S")
+          "[#{time}] #{severity}#{" " * (5 - severity.size + 1)}| #{msg}\n"
+        end
+      end
+
+      # returns the git commit hash for a specified revision (using revmap hash)
+      def map_changeset(str)
+        if @revmap&.key?(str)
+          "[r#{str}](../commit/#{@revmap[str]}) #{@revmap[str]}"
+        else
+          str
+        end
+      end
+
+      def svn_log(url, local_path, flags = {})
+        command = "svn log"
+        command += " #{url}" if url
+
+        flags.each do |key, value|
+          command += " #{key}"
+          command += " #{value}" if value
+        end
+
+        if local_path
+          Dir.chdir(local_path) do
+            `#{command}`
+          end
+        else
+          `#{command}`
+        end
+      end
+    end
+  end
+end

data/lib/tractive/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Tractive
+  VERSION = "1.0.0"
+end

data/lib/tractive.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative "tractive/graceful_quit"
+require_relative "tractive/attachment_exporter"
+require_relative "tractive/migrator"
+require_relative "tractive/trac"
+require_relative "tractive/info"
+require_relative "tractive/version"
+require_relative "tractive/main"
+require_relative "tractive/utilities"
+require_relative "tractive/github_api"
+require_relative "tractive/revmap_generator"
+require "json"
+require "logger"
+require "yaml"
+require "rest-client"
+require "optparse"
+require "sequel"
+require "set"
+require "singleton"
+require "uri"
+require "pry"
+require "thor"
+require "ox"
+
+module Tractive
+  class Error < StandardError; end
+  # Your code goes here...
+end

data/tractive.gemspec

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "lib/tractive/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "tractive"
+  spec.version       = Tractive::VERSION
+  spec.authors       = ["Ribose"]
+  spec.email         = ["open.source@ribose.com"]
+
+  spec.summary       = "Exporting tool for Trac"
+  # spec.description   = "TODO: Write a longer description or delete this line."
+  spec.homepage      = "https://github.com/ietf-ribose/tractive"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/ietf-ribose/tractive"
+  spec.metadata["changelog_uri"] = "https://github.com/ietf-ribose/tractive"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "mysql2"
+  spec.add_dependency "ox"
+  spec.add_dependency "rest-client"
+  spec.add_dependency "sequel"
+  spec.add_dependency "sqlite3"
+  spec.add_dependency "thor"
+
+  spec.add_development_dependency "pry"
+end