tractive 1.0.8 → 1.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f8e921094cb307df3ab46beb5ca61451f70dff070d002524d01311830fdfa2c
-  data.tar.gz: 3a41b37b4e9abdc7f54182023a5d8051bfa5cc0179e9a54a69f55acf64b140dc
+  metadata.gz: 07fb75035716d30de8f58206ad93cd0b932d74807646d5577d519d848eb2bfa0
+  data.tar.gz: c4e50252da3aaf67b919041cdc682e8559abb28163dd847671d30c91808d15e0
 SHA512:
-  metadata.gz: 9be506ff5ac3b8dd8613805202b10da56276fbe955415f395b11aef693367b95ed46dd19edc9e7ac3f22ff1eadb26a1be3deb68cb59a4ddbbb5c2fb06fdc754d
-  data.tar.gz: 427ac9b2b341dae9bad18073b310a716c6def1604ecd63b96b718eb73fe09b3c6688579b635403d008f36b75683c99709f7b38c57b5f60df6d1b4fd08eefeca0
+  metadata.gz: efc63d0e9e8b08d3780cccd3877554c7a1df4408cd8db3c5bcf4fd11634eab73ac7f29fa300fb9191e77a892ba5d795c80e0d1a5a97283e6c541370372ef5725
+  data.tar.gz: b1dc726435943c2b0cf70f369c21ffd049d07e13147a5d88f162dc725fb1ad7d62a8a034b2adb05903cc9e393dd421497af457fbaf2ad8cf45684b783b63b5ba

data/.rubocop.yml

@@ -18,6 +18,7 @@ Metrics/ClassLength:
 
 Metrics/BlockLength:
   Max: 500
+  IgnoredMethods: ['describe']
 
 Style/Documentation:
   Enabled: false
@@ -34,6 +35,12 @@ Metrics/AbcSize:
 Metrics/MethodLength:
   Enabled: false
 
+Metrics/ModuleLength:
+  Max: 150
+
+Metrics/ParameterLists:
+  Max: 6
+
 Security/Open:
   Enabled: false
 

data/README.adoc

@@ -174,7 +174,7 @@ With this information you can now build the RevMap with:
 
 [source,sh]
 ----
-tractive generate-revmap \
+tractive generate revmap \
   --svn-url <url of the SVN repository> \
   --rev-timestamp-file <reposurgeon timestamp map, e.g. {name-of-repo}.fo> \
   --git-local-repo-path <path to converted Git repository, e.g. {name-of-repo}-git> \
@@ -186,7 +186,7 @@ issues and commits will replace references to SVN revisions with the
 corresponding Git commit SHA, enabling GitHub to expose those linkages in the
 user interface.
 
-.Options for `tractive generate-revmap` command
+.Options for `tractive generate revmap` command
 [cols="3a,5a,2a",options="header"]
 |===
 | Option | Description | Type
@@ -315,7 +315,7 @@ e.g. 'ghp_fpsc4de1f0c46e01576810740c9242097cba4619486'.
 e.g. '/Users/user/repo-git'.
 
 `revmap_path:`::: Local path to the RevMap file generated via the
-`tractive generate-revmap` command.
+`tractive generate revmap` command.
 
 
 EXAMPLE:
@@ -326,7 +326,8 @@ github:
   repo: 'example-org/target-repository'
   token: 'ghp_fpsc4de1f0c46e01576810740c9242097cba4619486'
   local_repo_path: '/Users/user/repo-git'
-  revmap_path: ./example-revmap.txt
+
+revmap_path: ./example-revmap.txt
 ----
 
 
@@ -359,8 +360,12 @@ The pattern of a mapping is like:
 +
 [source,yaml]
 ----
-  configuration:      conf
-  documentation:      doc
+  configuration:      
+    name: conf
+    color: ff00ff
+  documentation:      
+    name: doc
+    color: 00ff00
 ----
 
 `resolution:`::: Resolution of the Trac ticket. e.g.
@@ -433,7 +438,7 @@ The pattern of a mapping is like:
 ----
 
 
-NOTE: As `severity`, `priority` and `tracstate` are converted into `labels` on github so there is an option to specify the `color` for those labels.
+NOTE: As `component`, `severity`, `priority` and `tracstate` are converted into `labels` on github so there is an option to specify the `color` for those labels.
 
 
 ==== User mapping
@@ -499,22 +504,32 @@ milestones:
 
 
 ==== Attachments migration configuration
+`ticket | wiki:`:: specifies the options for the tickets or wikis
+
+  `attachments:`::: specifies method of obtaining attachments from Trac.
 
-`attachments:`:: specifies method of obtaining attachments from Trac.
+    `url:`:::: URL to obtain Trac attachments from
 
-`url:`::: URL to obtain Trac attachments from
+    `hashed:`:::: Whether the url has hased or plain image names and ids
 
-`export_folder:`::: folder where the attachments will be downloaded to from Trac.
+    `export_folder:`:::: folder where the attachments will be downloaded to from Trac.
 
-`export_script:`::: output of a script that utilizes `trac-admin` to download
-all attachments from Trac.
+    `export_script:`:::: output of a script that utilizes `trac-admin` to download 
+    all attachments from Trac.
 
 [source,yaml]
 ----
-attachments:
-  url: https://abc.com/raw-attachment/ticket
-  export_folder: ./attachments
-  export_script: attachments.sh
+ticket:
+  attachments:
+    url: https://abc.com/raw-attachment/ticket
+    hashed: true
+    export_folder: ./attachments
+    export_script: attachments.sh
+    
+wiki:
+  attachments:
+    url: https://abc.com/raw-attachment/wiki
+    hashed: true
 ----
 
 By using the <<gather-info,`-i` option>>, you can easily produce a YAML file
@@ -682,6 +697,7 @@ The following options are allowed (at least one necessary):
 * `column-name`
 * `operator`
 * `column-value`
+* `include-null`
 
 | Boolean
 
@@ -697,6 +713,10 @@ The following options are allowed (at least one necessary):
 | Value of the column to filter.
 | String
 
+| `--include-null`
+| Include rows having null value in filtered column.
+| Boolean
+
 | `-h`, `help`
 | Display the Tractive help message, or you can provide a command to know more
   about a single command via `tractive help {command}`.
@@ -741,6 +761,58 @@ The following options are allowed (at least one necessary):
 |===
 
 
+=== Migrating Wikis
+
+You need the following for wiki migration
+
+* Clone of the github wiki repo.
+
+Then you can run the migration with:
+
+[source,sh]
+----
+tractive migrate-wikis \
+  --attachment-base-url <url for the attachments> \
+  --trac-database-path <full path of trac database> \
+  --repo-path <path to cloned Git wiki repository, e.g. {name-of-repo}.wiki>
+----
+
+After that open a terminal in the git wiki folder and run `git push`
+
+.Options for `tractive migrate-wikis` command
+[cols="3a,5a,2a",options="header"]
+|===
+| Option | Description | Type
+
+| `a`, `--attachment-base-url`
+| 
+(required unless specified in config file) 
+If attachment files are reachable via a URL we reference this here.
+| String
+
+| `-d`, `--trac-database-path`
+| (required unless specified in config file) Full path of the Trac sqlite3 database export file.
+| String
+
+| `-r`, `--repo-path`
+| (required) Full path to the root of the git-repository that is our destination.
+| String
+
+| `-c`, `--config`
+| Path to config file.
+| String
+
+| `-e`, `--wiki-extensions`
+| Space separated list of extensions or filenames (if the file don't have an extension). This is required to convert file SVN source links to Github links. This is used to determine if a path belongs to a file or a directory.
+| Array
+
+| `-f`, `--source-folders`
+| Space separated list of non standard folders in SVN that are used to find if a path is complete or partial.
+| Array
+
+|===
+
+
 == Implementation details
 
 === Usage of GitHub Issue Import API

data/exe/command_base.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative "../lib/tractive"
+
+class CommandBase < Thor
+  class_option "logfile", type: :string, aliases: ["-L", "--log-file"],
+                          desc: "Name of the logfile to output logs to."
+  class_option "config", type: :string, default: "tractive.config.yaml", banner: "<PATH>", aliases: "-c",
+                         desc: "Set the configuration file"
+  class_option "verbose", type: :boolean, aliases: ["-v", "--verbose"], desc: "Verbose mode"
+end

data/exe/generate.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative "./command_base"
+
+class Generate < CommandBase
+  desc "revmap <OPTIONS>", "Generate a mapping from svn revision number to git sha hash."
+  method_option "svnurl", type: :string, aliases: ["--svn-url"],
+                          desc: "Svn url that should be used in revmap generation"
+  method_option "svnlocalpath", type: :string, aliases: ["--svn-local-path"],
+                                desc: "Local SVN repo path"
+  method_option "gitlocalrepopath", type: :string, aliases: ["--git-local-repo-path"],
+                                    desc: "Local git repo path that should be used in revmap generation"
+  method_option "revtimestampfile", type: :string, aliases: ["--rev-timestamp-file"],
+                                    desc: "File containing svn revision and timestamps that should be used in revmap generation"
+  method_option "revoutputfile", type: :string, aliases: ["--revmap-output-file"],
+                                 desc: "File to output the generated revmap"
+  def revmap
+    verify_revmap_generator_options!(options)
+
+    Tractive::Utilities.setup_logger(output_stream: options[:log_file] || $stderr, verbose: options[:verbose])
+    Tractive::RevmapGenerator.new(
+      options["revtimestampfile"],
+      options["svnurl"],
+      options["svnlocalpath"],
+      options["gitlocalrepopath"],
+      options["revoutputfile"]
+    ).generate
+  end
+
+  no_commands do
+    def verify_revmap_generator_options!(options)
+      required_options = {}
+      required_options["--svn-url OR --svn-local-path"] = options["svnurl"] || options["svnlocalpath"]
+      required_options["--git-local-repo-path"] = options["gitlocalrepopath"]
+      required_options["--rev-timestamp-file"] = options["revtimestampfile"]
+      required_options["--revmap-output-file"] = options["revoutputfile"]
+
+      missing_options = {}
+      required_options.each do |key, value|
+        missing_options[key] = value if value.nil? || value.strip.empty?
+      end
+
+      return if missing_options.empty?
+
+      warn("missing revmap generator options (#{missing_options.keys}).\nRun with `--help` or `-h` to see available options")
+      exit 1
+    end
+  end
+end

data/exe/tractive

@@ -1,20 +1,20 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require_relative "../lib/tractive"
+require_relative "./command_base"
+require_relative "./generate"
 
-class TractiveCommand < Thor
-  default_command :migrate
-  class_option "logfile", type: :string, aliases: ["-L", "--log-file"],
-                          desc: "Name of the logfile to output logs to."
+class TractiveCommand < CommandBase
+  default_command :migrate_tickets
+
+  desc "generate", "Generate different files/scripts"
+  subcommand "generate", Generate
 
   desc "<OPTIONS>", "Migrate Trac instances to modern Git management platforms like GitHub and GitLab"
   method_option "attachmentexporter", type: :string, aliases: ["-A", "--attachment-exporter"],
                                       desc: "Generate an attachment exporter script according to config.yaml"
   method_option "attachurl", type: :string, aliases: ["-a", "--attachment-url"], banner: "<URL>",
                              desc: "If attachment files are reachable via a URL we reference this here"
-  method_option "config", type: :string, default: "tractive.config.yaml", banner: "<PATH>", aliases: "-c",
-                          desc: "Set the configuration file"
   method_option "dryrun", type: :boolean, aliases: ["-d", "--dry-run"],
                           desc: "Write data to a file instead of pushing it to github"
   method_option "exportattachments", type: :string, aliases: ["-e", "--export-attachments"],
@@ -30,6 +30,8 @@ class TractiveCommand < Thor
                             desc: "Operator for filter."
   method_option "columnvalue", type: :string, aliases: ["--column-value"],
                                desc: "Value of the column to filter."
+  method_option "includenull", type: :boolean, aliases: ["--include-null"],
+                               desc: "Flag for including null values in the filter result."
 
   method_option "importfromfile", type: :string, aliases: ["-I", "--import-from-file"],
                                   desc: "Import issues from a json file"
@@ -45,33 +47,25 @@ class TractiveCommand < Thor
                               desc: "Put all issue comments in the first message."
   method_option "start", type: :numeric, aliases: ["-s", "--start-at"], banner: "<ID>",
                          desc: "Start migration from ticket with number <ID>"
-  method_option "verbose", type: :boolean, aliases: ["-v", "--verbose"], desc: "Verbose mode"
-  def migrate
+  def migrate_tickets
     Tractive::Main.new(options).run
   end
 
-  desc "generate-revmap <OPTIONS>", "Generate a mapping from svn revision number to git sha hash."
-  method_option "svnurl", type: :string, aliases: ["--svn-url"],
-                          desc: "Svn url that should be used in revmap generation"
-  method_option "svnlocalpath", type: :string, aliases: ["--svn-local-path"],
-                                desc: "Local SVN repo path"
-  method_option "gitlocalrepopath", type: :string, aliases: ["--git-local-repo-path"],
-                                    desc: "Local git repo path that should be used in revmap generation"
-  method_option "revtimestampfile", type: :string, aliases: ["--rev-timestamp-file"],
-                                    desc: "File containing svn revision and timestamps that should be used in revmap generation"
-  method_option "revoutputfile", type: :string, aliases: ["--revmap-output-file"],
-                                 desc: "File to output the generated revmap"
-  def generate_revmap
-    verify_revmap_generator_options!(options)
-
-    Tractive::Utilities.setup_logger(output_stream: options[:log_file] || $stderr, verbose: options[:verbose])
-    Tractive::RevmapGenerator.new(
-      options["revtimestampfile"],
-      options["svnurl"],
-      options["svnlocalpath"],
-      options["gitlocalrepopath"],
-      options["revoutputfile"]
-    ).generate
+  desc "migrate-wikis", "Migrate Trac wikis to a git repository, preserving history (including authors, dates, and comments)"
+  method_option "attachment-base-url", type: :string, aliases: ["-a"], banner: "http://<base-url>",
+                                       desc: "If attachment files are reachable via a URL we reference this here"
+  method_option "trac-database-path", type: :string, aliases: ["-d"], banner: "/PATH/TO/EXPORTFILE",
+                                      desc: "Full path of the Trac sqlite3 database export file"
+  method_option "repo-path", type: :string, aliases: ["-r"], banner: "/GIT/ROOT/DIR",
+                             desc: "Full path to the root of the git-repository that is our destination"
+  method_option "home-page-name", type: :string, aliases: ["-h"], default: "WikiStart",
+                                  desc: "Name of the SVN wiki to map to the home page in Github wiki"
+  method_option "wiki-extensions", type: :array, aliases: ["-e"], default: [".py", "changelog", "expire-ids"],
+                                   desc: "Array of strings to determinte whether a given path is a file path or a directory in wiki"
+  method_option "source-folders", type: :array, aliases: ["-f"], default: ["personal", "attic", "sprint", "branch/hawk"],
+                                  desc: "Array of strings to figure out if a path is complete or partial"
+  def migrate_wikis
+    Tractive::Main.new(options).migrate_wikis
   end
 
   def self.exit_on_failure?
@@ -87,26 +81,6 @@ class TractiveCommand < Thor
   def respond_to_missing?
     true
   end
-
-  no_commands do
-    def verify_revmap_generator_options!(options)
-      required_options = {}
-      required_options["--svn-url OR --svn-local-path"] = options["svnurl"] || options["svnlocalpath"]
-      required_options["--git-local-repo-path"] = options["gitlocalrepopath"]
-      required_options["--rev-timestamp-file"] = options["revtimestampfile"]
-      required_options["--revmap-output-file"] = options["revoutputfile"]
-
-      missing_options = {}
-      required_options.each do |key, value|
-        missing_options[key] = value if value.nil? || value.strip.empty?
-      end
-
-      return if missing_options.empty?
-
-      warn("missing revmap generator options (#{missing_options.keys}).\nRun with `--help` or `-h` to see available options")
-      exit 1
-    end
-  end
 end
 
 TractiveCommand.start(ARGV)

data/lib/tractive/attachment_exporter.rb

@@ -53,9 +53,10 @@ module Tractive
         $logger.info "Saving attachments of ticket #{attachment.id}... "
         FileUtils.mkdir_p "#{output_dir}/#{attachment.id}"
 
-        File.open("#{output_dir}/#{attachment.id}/#{attachment.filename}", "wb") do |file|
-          file.write URI.open(uri_parser.escape("#{trac_url}/#{attachment.id}/#{attachment.filename}")).read
-        end
+        File.binwrite(
+          "#{output_dir}/#{attachment.id}/#{attachment.filename}",
+          URI.open(uri_parser.escape("#{trac_url}/#{attachment.id}/#{attachment.filename}")).read
+        )
       end
     end
   end

data/lib/tractive/github_api/client/issues.rb

@@ -6,7 +6,7 @@ module GithubApi
     module Issues
       def create_issue(repo, params)
         JSON.parse(
-          RestClient.post(
+          Http::Client::Request.post(
             "https://api.github.com/repos/#{repo}/import/issues",
             params.to_json,
             {
@@ -20,7 +20,7 @@ module GithubApi
 
       def list_issues(repo, params)
         JSON.parse(
-          RestClient.get(
+          Http::Client::Request.get(
             "https://api.github.com/repos/#{repo}/issues",
             {
               "Authorization" => "token #{@token}",
@@ -33,7 +33,7 @@ module GithubApi
 
       def issue(repo, number)
         JSON.parse(
-          RestClient.get(
+          Http::Client::Request.get(
             "https://api.github.com/repos/#{repo}/issues/#{number}",
             { "Authorization" => "token #{@token}" }
           )
@@ -42,7 +42,7 @@ module GithubApi
 
       def issue_import_status(repo, id)
         JSON.parse(
-          RestClient.get(
+          Http::Client::Request.get(
             "https://api.github.com/repos/#{repo}/import/issues/#{id}",
             {
               "Authorization" => "token #{@token}",
@@ -54,7 +54,7 @@ module GithubApi
 
       def issue_comments(repo, issue_id)
         JSON.parse(
-          RestClient.get(
+          Http::Client::Request.get(
             "https://api.github.com/repos/#{repo}/issues/#{issue_id}/comments",
             {
               "Authorization" => "token #{@token}",
@@ -66,7 +66,7 @@ module GithubApi
 
       def update_issue_comment(repo, comment_id, comment_body)
         JSON.parse(
-          RestClient.patch(
+          Http::Client::Request.patch(
             "https://api.github.com/repos/#{repo}/issues/comments/#{comment_id}",
             { body: comment_body }.to_json,
             { "Authorization" => "token #{@token}" }

data/lib/tractive/github_api/client/labels.rb

@@ -5,7 +5,7 @@ module GithubApi
     module Labels
       def list_labels(repo, params = {})
         JSON.parse(
-          RestClient.get(
+          Http::Client::Request.get(
             "https://api.github.com/repos/#{repo}/labels",
             {
               "Authorization" => "token #{@token}",
@@ -18,7 +18,7 @@ module GithubApi
 
       def create_label(repo, params)
         JSON.parse(
-          RestClient.post(
+          Http::Client::Request.post(
             "https://api.github.com/repos/#{repo}/labels",
             params.to_json,
             {

data/lib/tractive/github_api/client/milestones.rb

@@ -6,7 +6,7 @@ module GithubApi
     module Milestones
       def list_milestones(repo, params)
         JSON.parse(
-          RestClient.get(
+          Http::Client::Request.get(
             "https://api.github.com/repos/#{repo}/milestones?per_page=100",
             {
               "Authorization" => "token #{@token}",
@@ -19,7 +19,7 @@ module GithubApi
 
       def create_milestone(repo, params)
         JSON.parse(
-          RestClient.post(
+          Http::Client::Request.post(
             "https://api.github.com/repos/#{repo}/milestones",
             params.to_json,
             {

data/lib/tractive/github_api/client.rb

@@ -4,6 +4,8 @@ require_relative "client/issues"
 require_relative "client/labels"
 require_relative "client/milestones"
 
+require_relative "../http/client"
+
 # Service to perform github actions
 module GithubApi
   class Client

data/lib/tractive/http/client/request.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Http
+  module Client
+    class Request
+      def initialize(args)
+        @args = args
+        @max_retries = args[:max_retries] || 3
+      end
+
+      def execute(&block)
+        retries = 0
+
+        begin
+          retries += 1
+          RestClient::Request.execute(@args, &block)
+        rescue RestClient::Forbidden => e
+          retry_after = e.http_headers[:x_ratelimit_reset].to_i - Time.now.to_i
+          raise e if retry_after.negative? || retries > @max_retries
+
+          while retry_after.positive?
+            minutes = retry_after / 60
+            seconds = retry_after % 60
+
+            print "\rRate Limit Exceeded, Will retry in #{minutes} min #{seconds} sec"
+            sleep(1)
+
+            retry_after = e.http_headers[:x_ratelimit_reset].to_i - Time.now.to_i
+          end
+          retry if retries <= @max_retries
+        end
+      end
+
+      attr_reader :response
+
+      class << self
+        def get(url, headers = {}, &block)
+          execute(method: :get, url: url, headers: headers, &block)
+        end
+
+        def post(url, payload, headers = {}, &block)
+          execute(method: :post, url: url, payload: payload, headers: headers, &block)
+        end
+
+        def put(url, payload, headers = {}, &block)
+          execute(method: :put, url: url, payload: payload, headers: headers, &block)
+        end
+
+        def patch(url, payload, headers = {}, &block)
+          execute(method: :patch, url: url, payload: payload, headers: headers, &block)
+        end
+
+        def execute(args, &block)
+          new(args).execute(&block)
+        end
+      end
+    end
+  end
+end

data/lib/tractive/http/client.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "client/request"

data/lib/tractive/main.rb

@@ -9,7 +9,7 @@ module Tractive
       @cfg  = YAML.load_file(@opts[:config])
 
       Tractive::Utilities.setup_logger(output_stream: @opts[:logfile] || $stderr, verbose: @opts[:verbose])
-      @db = Tractive::Utilities.setup_db!(@cfg["trac"]["database"])
+      @db = Tractive::Utilities.setup_db!(@opts["trac-database-path"] || @cfg["trac"]["database"])
     rescue Sequel::DatabaseConnectionError, Sequel::AdapterNotFound, URI::InvalidURIError, Sequel::DatabaseError => e
       $logger.error e.message
       exit 1
@@ -33,6 +33,10 @@ module Tractive
       Migrator::Engine.new(opts: @opts, cfg: @cfg, db: @db).migrate
     end
 
+    def migrate_wikis
+      Migrator::Wikis::MigrateFromDb.new(opts: @opts, cfg: @cfg).migrate_wikis
+    end
+
     def info
       Tractive::Info.new.print
     end