tractive 1.0.11 → 1.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8ebb8048967ee1732b68d54ed6c10798e0959d533e7ed9744a843f1a129f764d
-  data.tar.gz: 4799587764dcaa848e76971b50df220e72ffa914a5f1bc7c3cad20ef2e5de83d
+  metadata.gz: 07fb75035716d30de8f58206ad93cd0b932d74807646d5577d519d848eb2bfa0
+  data.tar.gz: c4e50252da3aaf67b919041cdc682e8559abb28163dd847671d30c91808d15e0
 SHA512:
-  metadata.gz: 0c052a1f936f6e4d4749e4fd9f8ec9fcab3042f4220e2c4f10e6a7ed127f2c1ee8a856f49a58ae8b960a38f12ffdc35d9097a55361142b39bfa605a269ea2d1b
-  data.tar.gz: 6d0025c7682c74526aa3a0fb8a4d93d4f46fad4cab5e560898e3b2c504e8981cffe10203e59afb7c8d818d3598efe3729b854c2939d7b8ca64105846de55f131
+  metadata.gz: efc63d0e9e8b08d3780cccd3877554c7a1df4408cd8db3c5bcf4fd11634eab73ac7f29fa300fb9191e77a892ba5d795c80e0d1a5a97283e6c541370372ef5725
+  data.tar.gz: b1dc726435943c2b0cf70f369c21ffd049d07e13147a5d88f162dc725fb1ad7d62a8a034b2adb05903cc9e393dd421497af457fbaf2ad8cf45684b783b63b5ba

data/.rubocop.yml

@@ -18,6 +18,7 @@ Metrics/ClassLength:
 
 Metrics/BlockLength:
   Max: 500
+  IgnoredMethods: ['describe']
 
 Style/Documentation:
   Enabled: false
@@ -37,6 +38,9 @@ Metrics/MethodLength:
 Metrics/ModuleLength:
   Max: 150
 
+Metrics/ParameterLists:
+  Max: 6
+
 Security/Open:
   Enabled: false
 

data/README.adoc

@@ -802,6 +802,14 @@ If attachment files are reachable via a URL we reference this here.
 | Path to config file.
 | String
 
+| `-e`, `--wiki-extensions`
+| Space separated list of extensions or filenames (if the file don't have an extension). This is required to convert file SVN source links to Github links. This is used to determine if a path belongs to a file or a directory.
+| Array
+
+| `-f`, `--source-folders`
+| Space separated list of non standard folders in SVN that are used to find if a path is complete or partial.
+| Array
+
 |===
 
 

data/exe/tractive

@@ -58,6 +58,12 @@ class TractiveCommand < CommandBase
                                       desc: "Full path of the Trac sqlite3 database export file"
   method_option "repo-path", type: :string, aliases: ["-r"], banner: "/GIT/ROOT/DIR",
                              desc: "Full path to the root of the git-repository that is our destination"
+  method_option "home-page-name", type: :string, aliases: ["-h"], default: "WikiStart",
+                                  desc: "Name of the SVN wiki to map to the home page in Github wiki"
+  method_option "wiki-extensions", type: :array, aliases: ["-e"], default: [".py", "changelog", "expire-ids"],
+                                   desc: "Array of strings to determinte whether a given path is a file path or a directory in wiki"
+  method_option "source-folders", type: :array, aliases: ["-f"], default: ["personal", "attic", "sprint", "branch/hawk"],
+                                  desc: "Array of strings to figure out if a path is complete or partial"
   def migrate_wikis
     Tractive::Main.new(options).migrate_wikis
   end

data/lib/tractive/attachment_exporter.rb

@@ -53,9 +53,10 @@ module Tractive
         $logger.info "Saving attachments of ticket #{attachment.id}... "
         FileUtils.mkdir_p "#{output_dir}/#{attachment.id}"
 
-        File.open("#{output_dir}/#{attachment.id}/#{attachment.filename}", "wb") do |file|
-          file.write URI.open(uri_parser.escape("#{trac_url}/#{attachment.id}/#{attachment.filename}")).read
-        end
+        File.binwrite(
+          "#{output_dir}/#{attachment.id}/#{attachment.filename}",
+          URI.open(uri_parser.escape("#{trac_url}/#{attachment.id}/#{attachment.filename}")).read
+        )
       end
     end
   end

data/lib/tractive/migrator/converter/trac_to_github.rb

@@ -33,7 +33,8 @@ module Migrator
           @attachment_options,
           @changeset_base_url,
           @wiki_attachments_url,
-          @revmap_file_path
+          @revmap_file_path,
+          git_repo: @repo, home_page_name: args[:opts]["home-page-name"]
         )
       end
 

data/lib/tractive/migrator/converter/twf_to_markdown.rb

@@ -1,27 +1,41 @@
 # frozen_string_literal: true
 
+require "cgi"
+
 module Migrator
   module Converter
     # twf => Trac wiki format
     class TwfToMarkdown
-      def initialize(base_url, attachment_options, changeset_base_url, wiki_attachments_url, revmap_file_path)
+      def initialize(base_url, attachment_options, changeset_base_url, wiki_attachments_url, revmap_file_path, options = {})
         @base_url = base_url
         @attach_url = attachment_options[:url]
         @attach_hashed = attachment_options[:hashed]
         @changeset_base_url = changeset_base_url
         @wiki_attachments_url = wiki_attachments_url
         @revmap = load_revmap_file(revmap_file_path)
+
+        @git_repo = options[:git_repo]
+        @home_page_name = options[:home_page_name]
+        @wiki_extensions = options[:wiki_extensions] # || [".py", "changelog", "expire-ids"]
+        @source_folders = options[:source_folders] # || %w[personal attic sprint branch/hawk]
       end
 
       def convert(str)
+        # Fix 'Windows EOL' to 'Linux EOL'
+        str.gsub!("\r\n", "\n")
+
+        convert_tables(str)
         convert_newlines(str)
+        convert_comments(str)
+        convert_html_snippets(str)
         convert_code_snippets(str)
         convert_headings(str)
-        convert_links(str)
+        convert_links(str, @git_repo)
         convert_font_styles(str)
         convert_changeset(str, @changeset_base_url)
         convert_image(str, @base_url, @attach_url, @wiki_attachments_url)
         convert_ticket(str, @base_url)
+        revert_intermediate_references(str)
 
         str
       end
@@ -46,18 +60,12 @@ module Migrator
         revmap
       end
 
-      # CommitTicketReference
-      def convert_ticket_reference(str)
-        str.gsub!(/\{\{\{\n(#!CommitTicketReference .+?)\}\}\}/m, '\1')
-        str.gsub!(/#!CommitTicketReference .+\n/, "")
-      end
-
       # Ticket
       def convert_ticket(str, base_url)
         # replace a full ticket id with the github short refrence
         if base_url
           baseurlpattern = base_url.gsub("/", "\\/")
-          str.gsub!(%r{#{baseurlpattern}/(\d+)}) { "ticket:#{Regexp.last_match[1]}" }
+          str.gsub!(%r{#{baseurlpattern}/(\d+)}, '#\1')
         end
 
         # Ticket
@@ -80,16 +88,35 @@ module Migrator
         str.gsub!("\r\n", "\n")
       end
 
+      # Comments
+      def convert_comments(str)
+        str.gsub!(/\{\{\{(?>((?!(?:}}}|{{{)).+?|\g<0>))*\}\}\}/m) do |str_match|
+          str_match.gsub(/\{\{\{\s*#!comment(\s*)(.*)\}\}\}/m, '<!--\1\2\1-->')
+        end
+      end
+
+      # HTML Snippets
+      def convert_html_snippets(str)
+        str.gsub!(/\{\{\{#!html(.*?)\}\}\}/m, '\1')
+      end
+
+      # CommitTicketReference
+      def convert_ticket_reference(str)
+        str.gsub!(/\{\{\{\n(#!CommitTicketReference .+?)\}\}\}/m, '\1')
+        str.gsub!(/#!CommitTicketReference .+\n/, "")
+      end
+
       # Code
       def convert_code_snippets(str)
         str.gsub!(/\{\{\{([^\n]+?)\}\}\}/, '`\1`')
+        str.gsub!(/\{\{\{#!(.*?)\n(.+?)\}\}\}/m, "```\\1\n\\2\n```")
         str.gsub!(/\{\{\{(.+?)\}\}\}/m, '```\1```')
         str.gsub!(/(?<=```)#!/m, "")
       end
 
       # Changeset
       def convert_changeset(str, changeset_base_url)
-        str.gsub!(%r{#{Regexp.quote(changeset_base_url)}/(\d+)/?}, '[changeset:\1]') if changeset_base_url
+        str.gsub!(%r{#{Regexp.quote(changeset_base_url)}/(\d+)/?}, '[changeset:\1]') if changeset_base_url && !changeset_base_url.empty?
         str.gsub!(/\[changeset:"r(\d+)".*\]/, '[changeset:\1]')
         str.gsub!(/\[changeset:r(\d+)\]/, '[changeset:\1]')
         str.gsub!(/\br(\d+)\b/) { Tractive::Utilities.map_changeset(Regexp.last_match[1], @revmap, changeset_base_url) }
@@ -103,15 +130,170 @@ module Migrator
       def convert_font_styles(str)
         str.gsub!(/'''(.+?)'''/, '**\1**')
         str.gsub!(/''(.+?)''/, '*\1*')
-        str.gsub!(%r{[^:]//(.+?[^:])//}, '_\1_')
+        str.gsub!(%r{([^:])//(.+?[^:])//}, '\1_\2_')
+      end
+
+      # Tables
+      def convert_tables(str)
+        str.gsub!(/^( *\|\|[^\n]+\|\| *[^\n|]+$)+$/, '\1 ||')
+
+        str.gsub!(/(?:^( *\|\|[^\n]+\|\| *)\n?)+/) do |match_result|
+          rows = match_result.gsub("||", "|").split("\n")
+          rows.insert(1, "| #{"--- | " * (rows[0].split("|").size - 1)}".strip)
+
+          "#{rows.join("\n")}\n"
+        end
       end
 
       # Links
-      def convert_links(str)
-        str.gsub!(/\[(http[^\s\[\]]+)\s([^\[\]]+)\]/, '[\2](\1)')
+      def convert_links(str, git_repo)
+        convert_camel_case_links(str, git_repo)
+        convert_double_bracket_wiki_links(str, git_repo)
+        convert_single_bracket_wiki_links(str, git_repo)
+
+        str.gsub!(/(^!)\[(http[^\s\[\]]+)\s([^\[\]]+)\]/, '[\2](\1)')
         str.gsub!(/!(([A-Z][a-z0-9]+){2,})/, '\1')
       end
 
+      def convert_single_bracket_wiki_links(str, git_repo)
+        str.gsub!(/(!?)\[((?:wiki|source):)?([^\s\]]*) ?(.*?)\]/) do |match_result|
+          source = Regexp.last_match[2]
+          path = Regexp.last_match[3]
+          name = Regexp.last_match[4]
+
+          formatted_link(
+            match_result,
+            git_repo,
+            { source: source, path: path, name: name }
+          )
+        end
+      end
+
+      def convert_double_bracket_wiki_links(str, git_repo)
+        str.gsub!(/(!?)\[\[((?:wiki|source):)?([^|\n]*)\|?(.*?)\]\]/) do |match_result|
+          source = Regexp.last_match[2]
+          path = Regexp.last_match[3]
+          name = Regexp.last_match[4]
+
+          formatted_link(
+            match_result,
+            git_repo,
+            { source: source, path: path, name: name }
+          )
+        end
+      end
+
+      def formatted_link(unformatted_text, git_repo, url_options = {})
+        return unformatted_text.gsub("!", "{~") if unformatted_text.start_with?("!")
+
+        if url_options[:source] == "wiki:"
+          link, internal_link = url_options[:path].split("#")
+          link = "Home" if link == @home_page_name
+          internal_link = Tractive::Utilities.dasharize(internal_link) if internal_link
+          url_options[:name] = link if url_options[:name].empty?
+          "{{#{url_options[:name]}}}(https://github.com/#{git_repo}/wiki/#{link}##{internal_link})"
+        elsif url_options[:source] == "source:"
+          url_options[:name] = url_options[:path] if url_options[:name].empty?
+          "{{#{url_options[:name]}}}(https://github.com/#{git_repo}/#{source_git_path(url_options[:path])})"
+        elsif url_options[:path].start_with?("http")
+          url_options[:name] = url_options[:path] if url_options[:name].empty?
+          "{{#{url_options[:name]}}}(#{url_options[:path]})"
+        else
+          unformatted_text
+        end
+      end
+
+      def source_git_path(trac_path)
+        trac_path = trac_path.gsub("trunk/", "main/")
+        trac_path = trac_path.delete_prefix("/").delete_suffix("/")
+
+        return "" if trac_path.empty?
+
+        uri = URI.parse(trac_path)
+
+        trac_path = uri.path
+        line_number = uri.fragment
+        trac_path, revision = trac_path.split("@")
+
+        if trac_path.split("/").count <= 1
+          wiki_path(trac_path)
+        else
+          unless trac_path.start_with?("tags")
+            params = CGI.parse(uri.query || "")
+            revision ||= params["rev"].first
+
+            # TODO: Currently @ does not work with file paths except for main branch
+            sha = @revmap[revision]&.strip
+
+            trac_path = if sha && file?(trac_path)
+                          trac_path.gsub("main/", "#{sha}/")
+                        else
+                          sha || trac_path
+                        end
+          end
+
+          wiki_path(trac_path.delete_prefix("tags/"), line_number)
+        end
+      end
+
+      def index_paths
+        @index_paths ||= {
+          "tags" => "tags",
+          "tags/" => "tags",
+          "branch" => "branches/all"
+        }
+      end
+
+      def file?(trac_path)
+        return false unless trac_path
+
+        @wiki_extensions.any? { |extension| trac_path.end_with?(extension) }
+      end
+
+      def wiki_path(path, line_number = "")
+        # TODO: This will not work for folders given in the source_folder parameter and
+        # will not work for subfolders paths like `personal/rjs` unless given in the parameters.
+        return "branches/all?query=#{path}" if @source_folders.any? { |folder| folder == path }
+        return index_paths[path] if index_paths[path]
+
+        prefix = if file?(path)
+                   "blob"
+                 else
+                   "tree"
+                 end
+
+        "#{prefix}/#{path}#{"#" unless line_number.to_s.empty?}#{line_number}"
+      end
+
+      # CamelCase page names follow these rules:
+      #   1. The name must consist of alphabetic characters only;
+      #      no digits, spaces, punctuation or underscores are allowed.
+      #   2. A name must have at least two capital letters.
+      #   3. The first character must be capitalized.
+      #   4. Every capital letter must be followed by one or more lower-case letters.
+      #   5. The use of slash ( / ) is permitted in page names, where it typically represents a hierarchy.
+      def convert_camel_case_links(str, git_repo)
+        name_regex = %r{(^| )(!?)(/?[A-Z][a-z]+(/?[A-Z][a-z]+)+/?)}
+        wiki_pages_names = Tractive::Wiki.select(:name).distinct.map(:name)
+        str.gsub!(name_regex) do
+          start = Regexp.last_match[2]
+          name = Regexp.last_match[3]
+
+          wiki_link = if start != "!" && wiki_pages_names.include?(name)
+                        make_wiki_link(name, git_repo)
+                      else
+                        name
+                      end
+
+          "#{Regexp.last_match[1]}#{wiki_link}"
+        end
+      end
+
+      def make_wiki_link(wiki_name, git_repo)
+        wiki_name = "Home" if wiki_name == @home_page_name
+        "[#{wiki_name}](https://github.com/#{git_repo}/wiki/#{wiki_name})"
+      end
+
       def convert_image(str, base_url, attach_url, wiki_attachments_url)
         # https://trac.edgewall.org/wiki/WikiFormatting#Images
         # [[Image(picture.gif)]] Current page (Ticket, Wiki, Comment)
@@ -119,28 +301,37 @@ module Migrator
         # [[Image(ticket:1:picture.gif)]] (file attached to a ticket)
 
         image_regex = /\[\[Image\((?:(?<module>(?:source|wiki)):)?(?<path>[^)]+)\)\]\]/
-        d = image_regex.match(str)
-        return if d.nil?
-
-        path = d[:path]
-        mod = d[:module]
-
-        image_path = if mod == "source"
-                       "![#{path.split("/").last}](#{base_url}#{path})"
-                     elsif mod == "wiki"
-                       id, file = path.split(":")
-                       upload_path = "#{wiki_attachments_url}/#{Tractive::Utilities.attachment_path(id, file, hashed: @attach_hashed)}"
-                       "![#{file}](#{upload_path})"
-                     elsif path.start_with?("http")
-                       # [[Image(http://example.org/s.jpg)]]
-                       "![#{d[:path]}](#{d[:path]})"
-                     else
-                       _, id, file = path.split(":")
-                       file_path = "#{attach_url}/#{Tractive::Utilities.attachment_path(id, file, hashed: @attach_hashed)}"
-                       "![#{d[:path]}](#{file_path})"
-                     end
-
-        str.gsub!(image_regex, image_path)
+
+        str.gsub!(image_regex) do
+          path = Regexp.last_match[:path]
+          mod = Regexp.last_match[:module]
+
+          converted_image = if mod == "source"
+                              "!{{#{path.split("/").last}}}(#{base_url}#{path})"
+                            elsif mod == "wiki"
+                              id, file = path.split(":")
+                              upload_path = "#{wiki_attachments_url}/#{Tractive::Utilities.attachment_path(id, file, hashed: @attach_hashed)}"
+                              "!{{#{file}}}(#{upload_path})"
+                            elsif path.start_with?("http")
+                              # [[Image(http://example.org/s.jpg)]]
+                              "!{{#{path}}}(#{path})"
+                            else
+                              _, id, file = path.split(":")
+                              file_path = "#{attach_url}/#{Tractive::Utilities.attachment_path(id, file, hashed: @attach_hashed)}"
+                              "!{{#{path}}}(#{file_path})"
+                            end
+
+          # There are also ticket references in the format of ticket:1 so
+          # changing this now and will revert it at the end again
+          converted_image.gsub(/ticket:(\d+)/, 'ImageTicket~\1')
+        end
+      end
+
+      def revert_intermediate_references(str)
+        str.gsub!(/ImageTicket~(\d)/, 'ticket:\1')
+        str.gsub!("{{", "[")
+        str.gsub!("}}", "]")
+        str.gsub!(/(\{~)*/, "")
       end
     end
   end

data/lib/tractive/migrator/wikis/migrate_from_db.rb

@@ -13,12 +13,17 @@ module Migrator
         @authors_map = @config["users"].to_h
 
         @tracticketbaseurl    = @config["trac"]["ticketbaseurl"]
+        @git_repo             = @config["github"]["repo"]
         @changeset_base_url   = @config["trac"]["changeset_base_url"] || ""
-        @wiki_attachments_url = @options["attachment-base-url"] || @config.dig("wiki", "attachments", "url") || ""
-        @repo_path            = @options["repo-path"] || ""
         @revmap_path          = @config["revmap_path"]
         @attachments_hashed   = @config.dig("wiki", "attachments", "hashed")
 
+        @wiki_attachments_url = @options["attachment-base-url"] || @config.dig("wiki", "attachments", "url") || ""
+        @repo_path            = @options["repo-path"] || ""
+        @home_page_name       = @options["home-page-name"]
+        @wiki_extensions      = @options["wiki-extensions"]
+        @source_folders       = @options["source-folders"]
+
         @attachment_options   = {
           hashed: @attachments_hashed
         }
@@ -26,7 +31,17 @@ module Migrator
         verify_options
         verify_locations
 
-        @twf_to_markdown = Migrator::Converter::TwfToMarkdown.new(@tracticketbaseurl, @attachment_options, @changeset_base_url, @wiki_attachments_url, @revmap_path)
+        @twf_to_markdown = Migrator::Converter::TwfToMarkdown.new(
+          @tracticketbaseurl,
+          @attachment_options,
+          @changeset_base_url,
+          @wiki_attachments_url,
+          @revmap_path,
+          git_repo: @git_repo,
+          home_page_name: @home_page_name,
+          wiki_extensions: @wiki_extensions,
+          source_folders: @source_folders
+        )
       end
 
       def migrate_wikis
@@ -43,7 +58,8 @@ module Migrator
                         wiki[:comment].gsub('"', '\"')
                       end
 
-            file_name = "#{cleanse_filename(wiki[:name])}.md"
+            file_name = filename_for_wiki(wiki)
+
             $logger.info("Working with file [#{file_name}]")
             $logger.debug("Object: #{wiki}")
 
@@ -75,6 +91,12 @@ module Migrator
 
       private
 
+      def filename_for_wiki(wiki)
+        return "Home.md" if @home_page_name == wiki[:name]
+
+        "#{cleanse_filename(wiki[:name])}.md"
+      end
+
       def verify_options
         $logger.info("Verifying options...")
 

data/lib/tractive/utilities.rb

@@ -36,6 +36,10 @@ module Tractive
         db
       end
 
+      def dasharize(str)
+        str.gsub(/([a-z\d])([A-Z])/, '\1-\2').downcase
+      end
+
       def attachment_path(id, filename, options = {})
         return "#{id}/#{filename}" unless options[:hashed]
 

data/lib/tractive/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tractive
-  VERSION = "1.0.11"
+  VERSION = "1.0.12"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tractive
 version: !ruby/object:Gem::Version
-  version: 1.0.11
+  version: 1.0.12
 platform: ruby
 authors:
 - Ribose
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-12-17 00:00:00.000000000 Z
+date: 2022-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mysql2