tractive 1.0.0

data/lib/tractive/migrator/converter/trac_to_github.rb

@@ -0,0 +1,307 @@
+# frozen_string_literal: true
+
+module Migrator
+  module Converter
+    class TracToGithub
+      def initialize(args)
+        @tracticketbaseurl    = args[:cfg]["trac"]["ticketbaseurl"]
+        @attachurl            = args[:opts][:attachurl] || args[:cfg].dig("attachments", "url")
+        @changeset_base_url   = args[:cfg]["trac"]["changeset_base_url"]
+        @singlepost           = args[:opts][:singlepost]
+        @labels_cfg           = args[:cfg]["labels"].transform_values(&:to_h)
+        @milestonesfromtrac   = args[:cfg]["milestones"]
+        @users                = args[:cfg]["users"].to_h
+        @trac_mails_cache     = {}
+        @repo                 = args[:cfg]["github"]["repo"]
+        @client               = GithubApi::Client.new(access_token: args[:cfg]["github"]["token"])
+        @wiki_attachments_url = args[:cfg]["trac"]["wiki_attachments_url"]
+
+        load_milestone_map
+
+        @uri_parser = URI::Parser.new
+        @twf_to_markdown = Migrator::Converter::TwfToMarkdown.new(@tracticketbaseurl, @attachurl, @changeset_base_url, @wiki_attachments_url)
+      end
+
+      def compose(ticket)
+        body   = ""
+        closed = nil
+
+        # summary line:
+        # body += %i[id component priority resolution].map do |cat|
+        #   ticket[cat] and !ticket[cat].to_s.lstrip.empty? and
+        #     "**#{cat}:** #{ticket[cat]}"
+        # end.select { |x| x }.join(" | ")
+
+        # Initial report
+        # TODO: respect ticket[:changetime]
+        body += "\n\n" unless @singlepost
+        body += ticket_change(@singlepost, {
+                                ticket: ticket[:id],
+                                time: ticket[:time],
+                                author: ticket[:reporter],
+                                assigne: ticket[:owner],
+                                field: :initial,
+                                oldvalue: nil,
+                                newvalue: ticket[:description]
+                              })["body"]
+
+        changes = if ticket.is_a? Hash
+                    []
+                  else
+                    ticket.all_changes
+                  end
+
+        # replay all changes in chronological order:
+        comments = changes.map { |x| ticket_change(@singlepost, x) }.select { |x| x }.to_a
+        if @singlepost
+          body += comments.map { |x| x["body"] }.join("\n")
+          comments = []
+        end
+
+        labels = Set[]
+        changes.each do |x|
+          del = @labels_cfg.fetch(x[:field], {})[x[:oldvalue]]
+          # add = @labels_cfg.fetch(x[:field], {})[x[:newvalue]]
+          @labels_cfg.fetch(x[:field], {})[x[:newvalue]]
+          labels.delete(del) if del
+          # labels.add(add) if add
+          closed = x[:time] if (x[:field] == "status") && (x[:newvalue] == "closed")
+        end
+
+        # we separate labels from badges
+        # labels: are changed frequently in the lifecycle of a ticket, therefore are transferred to github lables
+        # badges: are basically fixed  and are transferred to a metadata table in the ticket
+
+        badges = Set[]
+
+        badges.add(@labels_cfg.fetch("component", {})[ticket[:component]])
+        badges.add(@labels_cfg.fetch("type", {})[ticket[:type]])
+        badges.add(@labels_cfg.fetch("resolution", {})[ticket[:resolution]])
+        badges.add(@labels_cfg.fetch("version", {})[ticket[:version]])
+
+        labels.add(@labels_cfg.fetch("severity", {})[ticket[:severity]])
+        labels.add(@labels_cfg.fetch("priority", {})[ticket[:priority]])
+        labels.add(@labels_cfg.fetch("tracstate", {})[ticket[:status]])
+        labels.delete(nil)
+
+        keywords = ticket[:keywords]
+        if keywords
+          if ticket[:keywords].downcase == "discuss"
+            labels.add(@labels_cfg.fetch("keywords", {})[ticket[:keywords].downcase])
+          else
+            badges.add(@labels_cfg.fetch("keywords", {})[ticket[:keywords]])
+          end
+        end
+        # If the field is not set, it will be nil and generate an unprocessable json
+
+        milestone = @milestonemap[ticket[:milestone]]
+
+        # compute footer
+        footer = "_Issue migrated from trac:#{ticket[:id]} at #{Time.now}_"
+
+        # compute badgetabe
+        #
+
+        github_assignee = map_assignee(ticket[:owner])
+
+        badges     = badges.to_a.compact.sort
+        badgetable = badges.map { |i| %(`#{i}`) }.join(" ")
+        badgetable += begin
+          "   |    by #{trac_mail(ticket[:reporter])}"
+        rescue StandardError
+          "deleted Ticket"
+        end
+        # badgetable += "   |   **->#{ticket[:owner]}**"  # note that from github to gitlab we loose the assigne
+
+        # compose body
+        body = [badgetable, body, footer].join("\n\n___\n")
+
+        labels.add("owner:#{github_assignee}")
+
+        issue = {
+          "title" => ticket[:summary],
+          "body" => body,
+          "labels" => labels.to_a,
+          "closed" => ticket[:status] == "closed",
+          "created_at" => format_time(ticket[:time]),
+          "milestone" => milestone
+        }
+
+        if @users.key?(ticket[:owner])
+          owner = trac_mail(ticket[:owner])
+          github_owner = @users[owner]
+          $logger.debug("..owner in trac: #{owner}")
+          $logger.debug("..assignee in GitHub: #{github_owner}")
+          issue["assignee"] = github_owner
+        end
+
+        ### as the assignee stuff is pretty fragile, we do not assign at all
+        # issue['assignee'] = github_assignee if github_assignee
+
+        if ticket[:changetime]
+          # issue["updated_at"] = format_time(ticket[:changetime])
+        end
+
+        if issue["closed"] && closed
+          #  issue["closed_at"] = format_time(closed)
+        end
+
+        {
+          "issue" => issue,
+          "comments" => comments
+        }
+      end
+
+      private
+
+      def map_user(user)
+        @users[user] || user
+      end
+
+      def map_assignee(user)
+        @users[user]
+      end
+
+      def load_milestone_map
+        read_milestones_from_github
+
+        newmilestonekeys = @milestonesfromtrac.keys - @milestonemap.keys
+
+        newmilestonekeys.each do |milestonelabel|
+          milestone           = {
+            "title" => milestonelabel.to_s,
+            "state" => @milestonesfromtrac[milestonelabel][:completed].nil? ? "open" : "closed",
+            "description" => @milestonesfromtrac[milestonelabel][:description] || "no description in trac",
+            "due_on" => "2012-10-09T23:39:01Z"
+          }
+          due                 = @milestonesfromtrac[milestonelabel][:due]
+          milestone["due_on"] = Time.at(due / 1_000_000).strftime("%Y-%m-%dT%H:%M:%SZ") if due
+
+          $logger.info "creating #{milestone}"
+
+          @client.create_milestone(@repo, milestone)
+        end
+
+        read_milestones_from_github
+        nil
+      end
+
+      def read_milestones_from_github
+        milestonesongithub = @client.milestones(@repo, { state: "all",
+                                                         sort: "due_on",
+                                                         direction: "desc" })
+        @milestonemap = milestonesongithub.map { |i| [i["title"], i["number"]] }.to_h
+        nil
+      end
+
+      def ticket_change(append, meta)
+        # kind
+        kind = if meta[:ticket]
+                 meta[:field].to_s
+               else
+                 "attachment"
+               end
+        kind = "title" if kind == "summary"
+
+        # don't care
+        return unless interested_in_change?(kind, meta[:newvalue])
+
+        # author
+        author = meta[:author]
+        author = trac_mail(author)
+        author = "@#{map_user(author)}" if @users.key?(author)
+
+        text = ""
+
+        if kind != "initial"
+          text += "\n___\n" if append
+          text += "_#{author}_ " if author
+        end
+
+        case kind
+        when "owner", "status", "title", "resolution", "priority", "component", "type", "severity", "platform", "milestone"
+          old = meta[:oldvalue]
+          new = meta[:newvalue]
+          if old && new
+            text += "_changed #{kind} from `#{old}` to `#{new}`_"
+          elsif old
+            text += "_removed #{kind} (was `#{old}`)_"
+          elsif new
+            text += "_set #{kind} to `#{new}`_"
+          end
+
+        when :initial, "initial"
+          body = meta[:newvalue]
+          # text += "created the issue\n\n"
+          if body && !body.lstrip.empty?
+            # text += "\n___\n" if not append
+            text += @twf_to_markdown.convert(body)
+          end
+
+        when "comment"
+          body      = meta[:newvalue]
+          changeset = body.match(/In \[changeset:"(\d+)/).to_a[1]
+          text += if changeset
+                    # changesethash = @revmap[changeset]
+                    "_committed #{Tractive::Utilities.map_changeset(changeset)}_"
+                  else
+                    "_commented_\n\n"
+                  end
+
+          text += "\n___\n" unless append
+          text += @twf_to_markdown.convert(body) if body
+
+        when "attachment"
+          text += "_uploaded file "
+          name = meta[:filename]
+          body = meta[:description]
+          if @attachurl
+            url = @uri_parser.escape("#{@attachurl}/#{meta[:id]}/#{name}")
+            text += "[`#{name}`](#{url})"
+            body += "\n![#{name}](#{url})" if [".png", ".jpg", ".gif"].include? File.extname(name).downcase
+          else
+            text += "`#{name}`"
+          end
+          text += " (#{(meta[:size] / 1024.0).round(1)} KiB)_"
+          text += "\n\n#{body}"
+
+        when "description"
+          # (ticket[:description] already contains the new value,
+          # so there is no need to update)
+          text += "_edited the issue description_"
+
+        else
+          # this should not happen
+          text += "changed #{kind} which not transferred by tractive"
+        end
+
+        {
+          "body" => text,
+          "created_at" => format_time(meta[:time])
+        }
+      end
+
+      # returns the author mail if found, otherwise author itself
+      def trac_mail(author)
+        return @trac_mails_cache[author] if @trac_mails_cache.key?(author)
+
+        # tries to retrieve the email from trac db
+        data = Tractive::Session.select(:value).where(Sequel.lit('name = "email" AND sid = ?', author))
+        return (@trac_mails_cache[author] = data.first[:value]) if data.count == 1
+
+        (@trac_mails_cache[author] = author) # not found
+      end
+
+      # Format time for github API
+      def format_time(time)
+        time = Time.at(time / 1e6, time % 1e6)
+        time.strftime("%FT%TZ")
+      end
+
+      def interested_in_change?(kind, newvalue)
+        !(%w[keywords cc reporter version].include?(kind) ||
+          (kind == "comment" && (newvalue.nil? || newvalue.lstrip.empty?)))
+      end
+    end
+  end
+end

data/lib/tractive/migrator/converter/twf_to_markdown.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Migrator
+  module Converter
+    # twf => Trac wiki format
+    class TwfToMarkdown
+      def initialize(base_url, attach_url, changeset_base_url, wiki_attachments_url)
+        @base_url = base_url
+        @attach_url = attach_url
+        @changeset_base_url = changeset_base_url
+        @wiki_attachments_url = wiki_attachments_url
+      end
+
+      def convert(str)
+        convert_newlines(str)
+        convert_code_snippets(str)
+        convert_headings(str)
+        convert_links(str)
+        convert_font_styles(str)
+        convert_changeset(str, @changeset_base_url)
+        convert_image(str, @base_url, @attach_url, @wiki_attachments_url)
+        convert_ticket(str, @base_url)
+
+        str
+      end
+
+      private
+
+      # CommitTicketReference
+      def convert_ticket_reference(str)
+        str.gsub!(/\{\{\{\n(#!CommitTicketReference .+?)\}\}\}/m, '\1')
+        str.gsub!(/#!CommitTicketReference .+\n/, "")
+      end
+
+      # Ticket
+      def convert_ticket(str, base_url)
+        # replace a full ticket id with the github short refrence
+        if base_url
+          baseurlpattern = base_url.gsub("/", "\\/")
+          str.gsub!(%r{#{baseurlpattern}/(\d+)}) { "ticket:#{Regexp.last_match[1]}" }
+        end
+
+        # Ticket
+        str.gsub!(/ticket:(\d+)/, '#\1')
+      end
+
+      # Headings
+      def convert_headings(str)
+        str.gsub!(/======\s(.+?)\s======/, '###### \1')
+        str.gsub!(/=====\s(.+?)\s=====/, '##### \1')
+        str.gsub!(/====\s(.+?)\s====/, '#### \1')
+        str.gsub!(/===\s(.+?)\s===/, '### \1')
+        str.gsub!(/==\s(.+?)\s==/, '## \1')
+        str.gsub!(/=\s(.+?)\s=/, '# \1')
+      end
+
+      # Line endings
+      def convert_newlines(str)
+        str.gsub!(/\[\[br\]\]/i, "\n")
+        str.gsub!("\r\n", "\n")
+      end
+
+      # Code
+      def convert_code_snippets(str)
+        str.gsub!(/\{\{\{([^\n]+?)\}\}\}/, '`\1`')
+        str.gsub!(/\{\{\{(.+?)\}\}\}/m, '```\1```')
+        str.gsub!(/(?<=```)#!/m, "")
+      end
+
+      # Changeset
+      def convert_changeset(str, changeset_base_url)
+        str.gsub!(%r{#{Regexp.quote(changeset_base_url)}/(\d+)/?}, '[changeset:\1]') if changeset_base_url
+        str.gsub!(/\[changeset:"r(\d+)".*\]/, '[changeset:\1]')
+        str.gsub!(/\[changeset:r(\d+)\]/, '[changeset:\1]')
+        str.gsub!(/\br(\d+)\b/) { Tractive::Utilities.map_changeset(Regexp.last_match[1]) }
+        str.gsub!(/\[changeset:"(\d+)".*\]/) { Tractive::Utilities.map_changeset(Regexp.last_match[1]) }
+        str.gsub!(/\[changeset:"(\d+).*\]/) { Tractive::Utilities.map_changeset(Regexp.last_match[1]) }
+      end
+
+      # Font styles
+      def convert_font_styles(str)
+        str.gsub!(/'''(.+?)'''/, '**\1**')
+        str.gsub!(/''(.+?)''/, '*\1*')
+        str.gsub!(%r{[^:]//(.+?[^:])//}, '_\1_')
+      end
+
+      # Links
+      def convert_links(str)
+        str.gsub!(/\[(http[^\s\[\]]+)\s([^\[\]]+)\]/, '[\2](\1)')
+        str.gsub!(/!(([A-Z][a-z0-9]+){2,})/, '\1')
+      end
+
+      def convert_image(str, base_url, attach_url, wiki_attachments_url)
+        # https://trac.edgewall.org/wiki/WikiFormatting#Images
+        # [[Image(picture.gif)]] Current page (Ticket, Wiki, Comment)
+        # [[Image(wiki:WikiFormatting:picture.gif)]] (referring to attachment on another page)
+        # [[Image(ticket:1:picture.gif)]] (file attached to a ticket)
+
+        image_regex = /\[\[Image\((?:(?<module>(?:source|wiki)):)?(?<path>[^)]+)\)\]\]/
+        d = image_regex.match(str)
+        return if d.nil?
+
+        path = d[:path]
+        mod = d[:module]
+
+        image_path = if mod == "source"
+                       "![#{path.split("/").last}](#{base_url}#{path})"
+                     elsif mod == "wiki"
+                       _, file = path.split(":")
+                       upload_path = "#{wiki_attachments_url}/#{file}"
+                       "![#{file}](#{upload_path})"
+                     elsif path.start_with?("http")
+                       # [[Image(http://example.org/s.jpg)]]
+                       "![#{d[:path]}](#{d[:path]})"
+                     else
+                       _, id, file = path.split(":")
+                       file_path = "#{attach_url}/#{id}/#{file}"
+                       "![#{d[:path]}](#{file_path})"
+                     end
+
+        str.gsub!(image_regex, image_path)
+      end
+    end
+  end
+end

data/lib/tractive/migrator/converter.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "converter/trac_to_github"

data/lib/tractive/migrator/engine/migrate_from_db.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Migrator
+  class Engine
+    module MigrateFromDb
+      def migrate_from_db
+        Tractive::GracefulQuit.enable
+        migrate_tickets_from_db(@start_ticket, @filter_closed)
+      rescue RuntimeError => e
+        $logger.error e.message
+      end
+
+      private
+
+      # Creates github issues for trac tickets.
+      def migrate_tickets_from_db(start_ticket, filterout_closed)
+        $logger.info("migrating issues")
+        # We match the issue title to determine whether an issue exists already.
+        tractickets = @trac.tickets
+                           .for_migration(start_ticket, filterout_closed, @filter_options)
+                           .all
+
+        begin
+          lasttracid = tractickets.last[:id]
+        rescue StandardError
+          raise("trac has no ticket #{start_ticket}")
+        end
+
+        (start_ticket.to_i..lasttracid).each do |ticket_id|
+          ticket = tractickets.select { |i| i[:id] == ticket_id }.first
+          @current_ticket_id = ticket_id # used to build filename for attachments
+
+          if ticket.nil?
+            next unless @mockdeleted
+
+            ticket = mock_ticket_details(ticket_id)
+          end
+
+          raise("tickets out of sync #{ticket_id} - #{ticket[:id]}") if ticket[:id] != ticket_id
+
+          Tractive::GracefulQuit.check("quitting after processing ticket ##{@last_created_issue}")
+
+          if @safetychecks
+            begin
+              # issue exists already:
+              @client.issue(@repo, ticket[:id])
+              $logger.info("found ticket #{ticket[:id]}")
+              next
+            rescue StandardError
+            end
+          end
+
+          $logger.info(%{creating issue for trac #{ticket[:id]} "#{ticket[:summary]}" (#{ticket[:reporter]})})
+          # API details: https://gist.github.com/jonmagic/5282384165e0f86ef105
+          request = Migrator::Converter::TracToGithub.new(@config).compose(ticket)
+
+          response = @client.create_issue(@repo, request)
+
+          if @safetychecks # - it is not really faster if we do not wait for the processing
+            while response["status"] == "pending"
+              sleep 1
+              $logger.info("Checking import status: #{response["id"]}")
+              $logger.debug("you can manually check: #{response["url"]}")
+              response = @client.issue_import_status(@repo, response["id"])
+            end
+
+            $logger.info("Status: #{response["status"]}")
+
+            if response["status"] == "failed"
+              $logger.error(response["errors"])
+              exit 1
+            end
+
+            issue_id = response["issue_url"].match(/\d+$/).to_s.to_i
+
+            $logger.info("created issue ##{issue_id} for trac ticket #{ticket[:id]}")
+
+            update_comment_ref(issue_id) if request.to_s.include?("Replying to [comment:")
+
+            # assert correct issue number
+            if issue_id != ticket[:id]
+              $logger.warn("mismatch issue ##{issue_id} for ticket #{ticket[:id]}")
+              exit 1
+            end
+          else
+            # to allow manual verification:
+            $logger.info(response["url"])
+          end
+
+          @last_created_issue = ticket[:id]
+        end
+      end
+    end
+  end
+end

data/lib/tractive/migrator/engine/migrate_from_file.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Migrator
+  class Engine
+    module MigrateFromFile
+      def migrate_from_file
+        Tractive::GracefulQuit.enable
+        migrate_tickets_from_file(@start_ticket, @filter_closed)
+      rescue RuntimeError => e
+        $logger.error e.message
+      end
+
+      private
+
+      # Creates github issues for trac tickets.
+      def migrate_tickets_from_file(start_ticket, filterout_closed)
+        $logger.info("migrating issues")
+        # We match the issue title to determine whether an issue exists already.
+        tractickets = @trac.tickets.order(:id).where { id >= start_ticket }.all
+        begin
+          lasttracid = @input_file.keys.map(&:to_i).max
+        rescue StandardError
+          raise("trac has no ticket #{start_ticket}")
+        end
+
+        (start_ticket.to_i..lasttracid).each do |ticket_id|
+          ticket = tractickets.select { |i| i[:id] == ticket_id }.first
+
+          @current_ticket_id = ticket_id # used to build filename for attachments
+
+          if ticket.nil?
+            next unless @mockdeleted
+
+            ticket = {
+              id: ticket_id,
+              summary: "DELETED in trac #{ticket_id}",
+              time: Time.now.to_i,
+              status: "closed",
+              reporter: "tractive"
+            }
+          end
+
+          raise("tickets out of sync #{ticket_id} - #{ticket[:id]}") if ticket[:id] != ticket_id
+
+          next if filterout_closed && (ticket[:status] == "closed")
+
+          Tractive::GracefulQuit.check("quitting after processing ticket ##{@last_created_issue}")
+
+          if @safetychecks
+            begin
+              # issue exists already:
+              @client.issue(@repo, ticket[:id])
+              $logger.info("found ticket #{ticket[:id]}")
+              next
+            rescue StandardError
+            end
+          end
+
+          $logger.info(%{creating issue from file for trac #{ticket[:id]} "#{ticket[:summary]}" (#{ticket[:reporter]})})
+          # API details: https://gist.github.com/jonmagic/5282384165e0f86ef105
+          request = @input_file[@current_ticket_id.to_s]
+          response = @client.create_issue(@repo, request)
+
+          if @safetychecks # - it is not really faster if we do not wait for the processing
+            while response["status"] == "pending"
+              sleep 1
+              $logger.info("Checking import status: #{response["id"]}")
+              $logger.info("you can manually check: #{response["url"]}")
+              response = @client.issue_import_status(@repo, response["id"])
+            end
+
+            $logger.info("Status: #{response["status"]}")
+
+            if response["status"] == "failed"
+              $logger.error(response["errors"])
+              exit 1
+            end
+
+            issue_id = response["issue_url"].match(/\d+$/).to_s.to_i
+
+            $logger.info("created issue ##{issue_id} for trac ticket #{ticket[:id]}")
+
+            update_comment_ref(issue_id) if request.to_s.include?("Replying to [comment:")
+
+            # assert correct issue number
+            if issue_id != ticket[:id]
+              $logger.warn("mismatch issue ##{issue_id} for ticket #{ticket[:id]}")
+              exit 1
+            end
+          else
+            # to allow manual verification:
+            $logger.info(response["url"])
+          end
+
+          @last_created_issue = ticket[:id]
+        end
+      end
+    end
+  end
+end