tractive 1.0.6 → 1.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fdbed889bbd1a1727140d2f1a537141b67199f956448952df2b6ac6ebf8976ca
-  data.tar.gz: 3e211800b03fc1b8eeeb268c9198de662e8ef3cba0a3301231b278bee15e400f
+  metadata.gz: 4d048cfbc81f42620ae076e33691d303559fb14b097048e2d4dcfb9574ed5bae
+  data.tar.gz: 3015ae96353d4e65c10569115d3c7b43d08474cfdf9223e82f8c2710e1663efc
 SHA512:
-  metadata.gz: fd5f9acbc163d4509d2b75a61d64071cfee00c6167e6c709710f66999ece41e3d343fa0664c192a427a85fde02666d41a14f9618557fac1781acf26d8dd11349
-  data.tar.gz: '081c6dc2373891c5190a6e43f93822174082990e37e253e110ccba0ed9801e13b425ec2e188ef512a75a0de0d955e96134946c7a6ae55f0331aa3f53cd88fdfd'
+  metadata.gz: f8ff314239f2b28ba8b2d87dadb07136c050d80a32a34e214e02d9826b375db886f09fe911cb8f42a63cbdf2a4ba244b8745d65bbf7be9177bc79c3bed20a835
+  data.tar.gz: 474c1ecad5be50f06599fca50aeee7c33bdfeba778310c77f0830653235de30c34fed85f144a24044555cdce551ab28e645af268afbf1da55674e2e38c48f372

data/.rubocop.yml

@@ -34,6 +34,9 @@ Metrics/AbcSize:
 Metrics/MethodLength:
   Enabled: false
 
+Metrics/ModuleLength:
+  Max: 150
+
 Security/Open:
   Enabled: false
 

data/README.adoc

@@ -174,7 +174,7 @@ With this information you can now build the RevMap with:
 
 [source,sh]
 ----
-tractive generate-revmap \
+tractive generate revmap \
   --svn-url <url of the SVN repository> \
   --rev-timestamp-file <reposurgeon timestamp map, e.g. {name-of-repo}.fo> \
   --git-local-repo-path <path to converted Git repository, e.g. {name-of-repo}-git> \
@@ -186,7 +186,7 @@ issues and commits will replace references to SVN revisions with the
 corresponding Git commit SHA, enabling GitHub to expose those linkages in the
 user interface.
 
-.Options for `tractive generate-revmap` command
+.Options for `tractive generate revmap` command
 [cols="3a,5a,2a",options="header"]
 |===
 | Option | Description | Type
@@ -315,7 +315,7 @@ e.g. 'ghp_fpsc4de1f0c46e01576810740c9242097cba4619486'.
 e.g. '/Users/user/repo-git'.
 
 `revmap_path:`::: Local path to the RevMap file generated via the
-`tractive generate-revmap` command.
+`tractive generate revmap` command.
 
 
 EXAMPLE:
@@ -326,7 +326,8 @@ github:
   repo: 'example-org/target-repository'
   token: 'ghp_fpsc4de1f0c46e01576810740c9242097cba4619486'
   local_repo_path: '/Users/user/repo-git'
-  revmap_path: ./example-revmap.txt
+
+revmap_path: ./example-revmap.txt
 ----
 
 
@@ -359,8 +360,12 @@ The pattern of a mapping is like:
 +
 [source,yaml]
 ----
-  configuration:      conf
-  documentation:      doc
+  configuration:      
+    name: conf
+    color: ff00ff
+  documentation:      
+    name: doc
+    color: 00ff00
 ----
 
 `resolution:`::: Resolution of the Trac ticket. e.g.
@@ -433,7 +438,7 @@ The pattern of a mapping is like:
 ----
 
 
-NOTE: As `severity`, `priority` and `tracstate` are converted into `labels` on github so there is an option to specify the `color` for those labels.
+NOTE: As `component`, `severity`, `priority` and `tracstate` are converted into `labels` on github so there is an option to specify the `color` for those labels.
 
 
 ==== User mapping
@@ -499,22 +504,32 @@ milestones:
 
 
 ==== Attachments migration configuration
+`ticket | wiki:`:: specifies the options for the tickets or wikis
+
+  `attachments:`::: specifies method of obtaining attachments from Trac.
 
-`attachments:`:: specifies method of obtaining attachments from Trac.
+    `url:`:::: URL to obtain Trac attachments from
 
-`url:`::: URL to obtain Trac attachments from
+    `hashed:`:::: Whether the url has hased or plain image names and ids
 
-`export_folder:`::: folder where the attachments will be downloaded to from Trac.
+    `export_folder:`:::: folder where the attachments will be downloaded to from Trac.
 
-`export_script:`::: output of a script that utilizes `trac-admin` to download
-all attachments from Trac.
+    `export_script:`:::: output of a script that utilizes `trac-admin` to download 
+    all attachments from Trac.
 
 [source,yaml]
 ----
-attachments:
-  url: https://abc.com/raw-attachment/ticket
-  export_folder: ./attachments
-  export_script: attachments.sh
+ticket:
+  attachments:
+    url: https://abc.com/raw-attachment/ticket
+    hashed: true
+    export_folder: ./attachments
+    export_script: attachments.sh
+    
+wiki:
+  attachments:
+    url: https://abc.com/raw-attachment/wiki
+    hashed: true
 ----
 
 By using the <<gather-info,`-i` option>>, you can easily produce a YAML file
@@ -682,6 +697,7 @@ The following options are allowed (at least one necessary):
 * `column-name`
 * `operator`
 * `column-value`
+* `include-null`
 
 | Boolean
 
@@ -697,6 +713,10 @@ The following options are allowed (at least one necessary):
 | Value of the column to filter.
 | String
 
+| `--include-null`
+| Include rows having null value in filtered column.
+| Boolean
+
 | `-h`, `help`
 | Display the Tractive help message, or you can provide a command to know more
   about a single command via `tractive help {command}`.
@@ -741,6 +761,50 @@ The following options are allowed (at least one necessary):
 |===
 
 
+=== Migrating Wikis
+
+You need the following for wiki migration
+
+* Clone of the github wiki repo.
+
+Then you can run the migration with:
+
+[source,sh]
+----
+tractive migrate-wikis \
+  --attachment-base-url <url for the attachments> \
+  --trac-database-path <full path of trac database> \
+  --repo-path <path to cloned Git wiki repository, e.g. {name-of-repo}.wiki>
+----
+
+After that open a terminal in the git wiki folder and run `git push`
+
+.Options for `tractive migrate-wikis` command
+[cols="3a,5a,2a",options="header"]
+|===
+| Option | Description | Type
+
+| `a`, `--attachment-base-url`
+| 
+(required unless specified in config file) 
+If attachment files are reachable via a URL we reference this here.
+| String
+
+| `-d`, `--trac-database-path`
+| (required unless specified in config file) Full path of the Trac sqlite3 database export file.
+| String
+
+| `-r`, `--repo-path`
+| (required) Full path to the root of the git-repository that is our destination.
+| String
+
+| `-c`, `--config`
+| Path to config file.
+| String
+
+|===
+
+
 == Implementation details
 
 === Usage of GitHub Issue Import API

data/exe/command_base.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative "../lib/tractive"
+
+class CommandBase < Thor
+  class_option "logfile", type: :string, aliases: ["-L", "--log-file"],
+                          desc: "Name of the logfile to output logs to."
+  class_option "config", type: :string, default: "tractive.config.yaml", banner: "<PATH>", aliases: "-c",
+                         desc: "Set the configuration file"
+  class_option "verbose", type: :boolean, aliases: ["-v", "--verbose"], desc: "Verbose mode"
+end

data/exe/generate.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative "./command_base"
+
+class Generate < CommandBase
+  desc "revmap <OPTIONS>", "Generate a mapping from svn revision number to git sha hash."
+  method_option "svnurl", type: :string, aliases: ["--svn-url"],
+                          desc: "Svn url that should be used in revmap generation"
+  method_option "svnlocalpath", type: :string, aliases: ["--svn-local-path"],
+                                desc: "Local SVN repo path"
+  method_option "gitlocalrepopath", type: :string, aliases: ["--git-local-repo-path"],
+                                    desc: "Local git repo path that should be used in revmap generation"
+  method_option "revtimestampfile", type: :string, aliases: ["--rev-timestamp-file"],
+                                    desc: "File containing svn revision and timestamps that should be used in revmap generation"
+  method_option "revoutputfile", type: :string, aliases: ["--revmap-output-file"],
+                                 desc: "File to output the generated revmap"
+  def revmap
+    verify_revmap_generator_options!(options)
+
+    Tractive::Utilities.setup_logger(output_stream: options[:log_file] || $stderr, verbose: options[:verbose])
+    Tractive::RevmapGenerator.new(
+      options["revtimestampfile"],
+      options["svnurl"],
+      options["svnlocalpath"],
+      options["gitlocalrepopath"],
+      options["revoutputfile"]
+    ).generate
+  end
+
+  no_commands do
+    def verify_revmap_generator_options!(options)
+      required_options = {}
+      required_options["--svn-url OR --svn-local-path"] = options["svnurl"] || options["svnlocalpath"]
+      required_options["--git-local-repo-path"] = options["gitlocalrepopath"]
+      required_options["--rev-timestamp-file"] = options["revtimestampfile"]
+      required_options["--revmap-output-file"] = options["revoutputfile"]
+
+      missing_options = {}
+      required_options.each do |key, value|
+        missing_options[key] = value if value.nil? || value.strip.empty?
+      end
+
+      return if missing_options.empty?
+
+      warn("missing revmap generator options (#{missing_options.keys}).\nRun with `--help` or `-h` to see available options")
+      exit 1
+    end
+  end
+end

data/exe/tractive

@@ -1,20 +1,20 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require_relative "../lib/tractive"
+require_relative "./command_base"
+require_relative "./generate"
 
-class TractiveCommand < Thor
-  default_command :migrate
-  class_option "logfile", type: :string, aliases: ["-L", "--log-file"],
-                          desc: "Name of the logfile to output logs to."
+class TractiveCommand < CommandBase
+  default_command :migrate_tickets
+
+  desc "generate", "Generate different files/scripts"
+  subcommand "generate", Generate
 
   desc "<OPTIONS>", "Migrate Trac instances to modern Git management platforms like GitHub and GitLab"
   method_option "attachmentexporter", type: :string, aliases: ["-A", "--attachment-exporter"],
                                       desc: "Generate an attachment exporter script according to config.yaml"
   method_option "attachurl", type: :string, aliases: ["-a", "--attachment-url"], banner: "<URL>",
                              desc: "If attachment files are reachable via a URL we reference this here"
-  method_option "config", type: :string, default: "tractive.config.yaml", banner: "<PATH>", aliases: "-c",
-                          desc: "Set the configuration file"
   method_option "dryrun", type: :boolean, aliases: ["-d", "--dry-run"],
                           desc: "Write data to a file instead of pushing it to github"
   method_option "exportattachments", type: :string, aliases: ["-e", "--export-attachments"],
@@ -30,6 +30,8 @@ class TractiveCommand < Thor
                             desc: "Operator for filter."
   method_option "columnvalue", type: :string, aliases: ["--column-value"],
                                desc: "Value of the column to filter."
+  method_option "includenull", type: :boolean, aliases: ["--include-null"],
+                               desc: "Flag for including null values in the filter result."
 
   method_option "importfromfile", type: :string, aliases: ["-I", "--import-from-file"],
                                   desc: "Import issues from a json file"
@@ -45,33 +47,19 @@ class TractiveCommand < Thor
                               desc: "Put all issue comments in the first message."
   method_option "start", type: :numeric, aliases: ["-s", "--start-at"], banner: "<ID>",
                          desc: "Start migration from ticket with number <ID>"
-  method_option "verbose", type: :boolean, aliases: ["-v", "--verbose"], desc: "Verbose mode"
-  def migrate
+  def migrate_tickets
     Tractive::Main.new(options).run
   end
 
-  desc "generate-revmap <OPTIONS>", "Generate a mapping from svn revision number to git sha hash."
-  method_option "svnurl", type: :string, aliases: ["--svn-url"],
-                          desc: "Svn url that should be used in revmap generation"
-  method_option "svnlocalpath", type: :string, aliases: ["--svn-local-path"],
-                                desc: "Local SVN repo path"
-  method_option "gitlocalrepopath", type: :string, aliases: ["--git-local-repo-path"],
-                                    desc: "Local git repo path that should be used in revmap generation"
-  method_option "revtimestampfile", type: :string, aliases: ["--rev-timestamp-file"],
-                                    desc: "File containing svn revision and timestamps that should be used in revmap generation"
-  method_option "revoutputfile", type: :string, aliases: ["--revmap-output-file"],
-                                 desc: "File to output the generated revmap"
-  def generate_revmap
-    verify_revmap_generator_options!(options)
-
-    Tractive::Utilities.setup_logger(output_stream: options[:log_file] || $stderr, verbose: options[:verbose])
-    Tractive::RevmapGenerator.new(
-      options["revtimestampfile"],
-      options["svnurl"],
-      options["svnlocalpath"],
-      options["gitlocalrepopath"],
-      options["revoutputfile"]
-    ).generate
+  desc "migrate-wikis", "Migrate Trac wikis to a git repository, preserving history (including authors, dates, and comments)"
+  method_option "attachment-base-url", type: :string, aliases: ["-a"], banner: "http://<base-url>",
+                                       desc: "If attachment files are reachable via a URL we reference this here"
+  method_option "trac-database-path", type: :string, aliases: ["-d"], banner: "/PATH/TO/EXPORTFILE",
+                                      desc: "Full path of the Trac sqlite3 database export file"
+  method_option "repo-path", type: :string, aliases: ["-r"], banner: "/GIT/ROOT/DIR",
+                             desc: "Full path to the root of the git-repository that is our destination"
+  def migrate_wikis
+    Tractive::Main.new(options).migrate_wikis
   end
 
   def self.exit_on_failure?
@@ -87,26 +75,6 @@ class TractiveCommand < Thor
   def respond_to_missing?
     true
   end
-
-  no_commands do
-    def verify_revmap_generator_options!(options)
-      required_options = {}
-      required_options["--svn-url OR --svn-local-path"] = options["svnurl"] || options["svnlocalpath"]
-      required_options["--git-local-repo-path"] = options["gitlocalrepopath"]
-      required_options["--rev-timestamp-file"] = options["revtimestampfile"]
-      required_options["--revmap-output-file"] = options["revoutputfile"]
-
-      missing_options = {}
-      required_options.each do |key, value|
-        missing_options[key] = value if value.nil? || value.strip.empty?
-      end
-
-      return if missing_options.empty?
-
-      warn("missing revmap generator options (#{missing_options.keys}).\nRun with `--help` or `-h` to see available options")
-      exit 1
-    end
-  end
 end
 
 TractiveCommand.start(ARGV)

data/lib/tractive/github_api/client/issues.rb

@@ -6,7 +6,7 @@ module GithubApi
     module Issues
       def create_issue(repo, params)
         JSON.parse(
-          RestClient.post(
+          Http::Client::Request.post(
             "https://api.github.com/repos/#{repo}/import/issues",
             params.to_json,
             {
@@ -20,7 +20,7 @@ module GithubApi
 
       def list_issues(repo, params)
         JSON.parse(
-          RestClient.get(
+          Http::Client::Request.get(
             "https://api.github.com/repos/#{repo}/issues",
             {
               "Authorization" => "token #{@token}",
@@ -33,7 +33,7 @@ module GithubApi
 
       def issue(repo, number)
         JSON.parse(
-          RestClient.get(
+          Http::Client::Request.get(
             "https://api.github.com/repos/#{repo}/issues/#{number}",
             { "Authorization" => "token #{@token}" }
           )
@@ -42,7 +42,7 @@ module GithubApi
 
       def issue_import_status(repo, id)
         JSON.parse(
-          RestClient.get(
+          Http::Client::Request.get(
             "https://api.github.com/repos/#{repo}/import/issues/#{id}",
             {
               "Authorization" => "token #{@token}",
@@ -54,7 +54,7 @@ module GithubApi
 
       def issue_comments(repo, issue_id)
         JSON.parse(
-          RestClient.get(
+          Http::Client::Request.get(
             "https://api.github.com/repos/#{repo}/issues/#{issue_id}/comments",
             {
               "Authorization" => "token #{@token}",
@@ -66,7 +66,7 @@ module GithubApi
 
       def update_issue_comment(repo, comment_id, comment_body)
         JSON.parse(
-          RestClient.patch(
+          Http::Client::Request.patch(
             "https://api.github.com/repos/#{repo}/issues/comments/#{comment_id}",
             { body: comment_body }.to_json,
             { "Authorization" => "token #{@token}" }

data/lib/tractive/github_api/client/labels.rb

@@ -5,7 +5,7 @@ module GithubApi
     module Labels
       def list_labels(repo, params = {})
         JSON.parse(
-          RestClient.get(
+          Http::Client::Request.get(
             "https://api.github.com/repos/#{repo}/labels",
             {
               "Authorization" => "token #{@token}",
@@ -18,7 +18,7 @@ module GithubApi
 
       def create_label(repo, params)
         JSON.parse(
-          RestClient.post(
+          Http::Client::Request.post(
             "https://api.github.com/repos/#{repo}/labels",
             params.to_json,
             {

data/lib/tractive/github_api/client/milestones.rb

@@ -6,7 +6,7 @@ module GithubApi
     module Milestones
       def list_milestones(repo, params)
         JSON.parse(
-          RestClient.get(
+          Http::Client::Request.get(
             "https://api.github.com/repos/#{repo}/milestones?per_page=100",
             {
               "Authorization" => "token #{@token}",
@@ -19,7 +19,7 @@ module GithubApi
 
       def create_milestone(repo, params)
         JSON.parse(
-          RestClient.post(
+          Http::Client::Request.post(
             "https://api.github.com/repos/#{repo}/milestones",
             params.to_json,
             {

data/lib/tractive/github_api/client.rb

@@ -4,6 +4,8 @@ require_relative "client/issues"
 require_relative "client/labels"
 require_relative "client/milestones"
 
+require_relative "../http/client"
+
 # Service to perform github actions
 module GithubApi
   class Client

data/lib/tractive/http/client/request.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Http
+  module Client
+    class Request
+      def initialize(args)
+        @args = args
+        @max_retries = args[:max_retries] || 3
+      end
+
+      def execute(&block)
+        retries = 0
+
+        begin
+          retries += 1
+          RestClient::Request.execute(@args, &block)
+        rescue RestClient::Forbidden => e
+          retry_after = e.http_headers[:x_ratelimit_reset].to_i - Time.now.to_i
+          raise e if retry_after.negative? || retries > @max_retries
+
+          while retry_after.positive?
+            minutes = retry_after / 60
+            seconds = retry_after % 60
+
+            print "\rRate Limit Exceeded, Will retry in #{minutes} min #{seconds} sec"
+            sleep(1)
+
+            retry_after = e.http_headers[:x_ratelimit_reset].to_i - Time.now.to_i
+          end
+          retry if retries <= @max_retries
+        end
+      end
+
+      attr_reader :response
+
+      class << self
+        def get(url, headers = {}, &block)
+          execute(method: :get, url: url, headers: headers, &block)
+        end
+
+        def post(url, payload, headers = {}, &block)
+          execute(method: :post, url: url, payload: payload, headers: headers, &block)
+        end
+
+        def put(url, payload, headers = {}, &block)
+          execute(method: :put, url: url, payload: payload, headers: headers, &block)
+        end
+
+        def patch(url, payload, headers = {}, &block)
+          execute(method: :patch, url: url, payload: payload, headers: headers, &block)
+        end
+
+        def execute(args, &block)
+          new(args).execute(&block)
+        end
+      end
+    end
+  end
+end

data/lib/tractive/http/client.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "client/request"

data/lib/tractive/main.rb

@@ -9,7 +9,7 @@ module Tractive
       @cfg  = YAML.load_file(@opts[:config])
 
       Tractive::Utilities.setup_logger(output_stream: @opts[:logfile] || $stderr, verbose: @opts[:verbose])
-      @db = Tractive::Utilities.setup_db!(@cfg["trac"]["database"])
+      @db = Tractive::Utilities.setup_db!(@opts["trac-database-path"] || @cfg["trac"]["database"])
     rescue Sequel::DatabaseConnectionError, Sequel::AdapterNotFound, URI::InvalidURIError, Sequel::DatabaseError => e
       $logger.error e.message
       exit 1
@@ -33,6 +33,10 @@ module Tractive
       Migrator::Engine.new(opts: @opts, cfg: @cfg, db: @db).migrate
     end
 
+    def migrate_wikis
+      Migrator::Wikis::MigrateFromDb.new(opts: @opts, cfg: @cfg).migrate_wikis
+    end
+
     def info
       Tractive::Info.new.print
     end

data/lib/tractive/migrator/converter/trac_to_github.rb

@@ -4,8 +4,8 @@ module Migrator
   module Converter
     class TracToGithub
       def initialize(args)
-        @tracticketbaseurl    = args[:cfg]["trac"]["ticketbaseurl"]
-        @attachurl            = args[:opts][:attachurl] || args[:cfg].dig("attachments", "url")
+        @trac_ticket_base_url = args[:cfg]["trac"]["ticketbaseurl"]
+        @attachurl            = args[:opts][:attachurl] || args[:cfg].dig("ticket", "attachments", "url")
         @changeset_base_url   = args[:cfg]["trac"]["changeset_base_url"] || ""
         @singlepost           = args[:opts][:singlepost]
         @labels_cfg           = args[:cfg]["labels"].transform_values(&:to_h)
@@ -14,8 +14,12 @@ module Migrator
         @trac_mails_cache     = {}
         @repo                 = args[:cfg]["github"]["repo"]
         @client               = GithubApi::Client.new(access_token: args[:cfg]["github"]["token"])
-        @wiki_attachments_url = args[:cfg]["trac"]["wiki_attachments_url"]
+        @wiki_attachments_url = args[:cfg].dig("wiki", "attachments", "url")
         @revmap_file_path     = args[:opts][:revmapfile] || args[:cfg]["revmap_path"]
+        @attachment_options   = {
+          url: @attachurl,
+          hashed: args[:cfg].dig("ticket", "attachments", "hashed")
+        }
 
         load_milestone_map
         create_labels_on_github(@labels_cfg["severity"].values)
@@ -24,12 +28,18 @@ module Migrator
         create_labels_on_github(@labels_cfg["component"].values)
 
         @uri_parser = URI::Parser.new
-        @twf_to_markdown = Migrator::Converter::TwfToMarkdown.new(@tracticketbaseurl, @attachurl, @changeset_base_url, @wiki_attachments_url, @revmap_file_path)
+        @twf_to_markdown = Migrator::Converter::TwfToMarkdown.new(
+          @trac_ticket_base_url,
+          @attachment_options,
+          @changeset_base_url,
+          @wiki_attachments_url,
+          @revmap_file_path
+        )
       end
 
       def compose(ticket)
-        body   = ""
-        closed = nil
+        body = ""
+        closed_time = nil
 
         # summary line:
         # body += %i[id component priority resolution].map do |cat|
@@ -70,7 +80,7 @@ module Migrator
           @labels_cfg.fetch(x[:field], {})[x[:newvalue]]
           labels.delete(del) if del
           # labels.add(add) if add
-          closed = x[:time] if (x[:field] == "status") && (x[:newvalue] == "closed")
+          closed_time = x[:time] if x[:field] == "status" && x[:newvalue] == "closed"
         end
 
         # we separate labels from badges
@@ -149,8 +159,12 @@ module Migrator
           # issue["updated_at"] = format_time(ticket[:changetime])
         end
 
-        if issue["closed"] && closed
-          #  issue["closed_at"] = format_time(closed)
+        if issue["closed"]
+          issue["closed_at"] = if closed_time
+                                 format_time(closed_time)
+                               else
+                                 format_time(ticket[:closed_at].to_i)
+                               end
         end
 
         {
@@ -204,7 +218,16 @@ module Migrator
       def create_labels_on_github(labels)
         return if labels.nil? || labels.empty?
 
-        existing_labels = @client.labels(@repo, per_page: 100).map { |label| label["name"] }
+        page = 1
+        existing_labels = []
+        result = @client.labels(@repo, per_page: 100, page: page).map { |label| label["name"] }
+
+        until result.empty?
+          existing_labels += result
+          page += 1
+          result = @client.labels(@repo, per_page: 100, page: page).map { |label| label["name"] }
+        end
+
         new_labels = labels.reject { |label| existing_labels.include?(label["name"]&.strip) }
 
         new_labels.each do |label|
@@ -278,7 +301,7 @@ module Migrator
           name = meta[:filename]
           body = meta[:description]
           if @attachurl
-            url = @uri_parser.escape("#{@attachurl}/#{meta[:id]}/#{name}")
+            url = @uri_parser.escape("#{@attachurl}/#{Tractive::Utilities.attachment_path(meta[:id], name, @attachment_options)}")
             text += "[`#{name}`](#{url})"
             body += "\n![#{name}](#{url})" if [".png", ".jpg", ".gif"].include? File.extname(name).downcase
           else
@@ -326,9 +349,9 @@ module Migrator
       end
 
       def trac_ticket_link(ticket)
-        return "trac:#{ticket[:id]}" unless @tracticketbaseurl
+        return "trac:#{ticket[:id]}" unless @trac_ticket_base_url
 
-        "[trac:#{ticket[:id]}](#{@tracticketbaseurl}/#{ticket[:id]})"
+        "[trac:#{ticket[:id]}](#{@trac_ticket_base_url}/#{ticket[:id]})"
       end
     end
   end

data/lib/tractive/migrator/converter/twf_to_markdown.rb

@@ -4,9 +4,10 @@ module Migrator
   module Converter
     # twf => Trac wiki format
     class TwfToMarkdown
-      def initialize(base_url, attach_url, changeset_base_url, wiki_attachments_url, revmap_file_path)
+      def initialize(base_url, attachment_options, changeset_base_url, wiki_attachments_url, revmap_file_path)
         @base_url = base_url
-        @attach_url = attach_url
+        @attach_url = attachment_options[:url]
+        @attach_hashed = attachment_options[:hashed]
         @changeset_base_url = changeset_base_url
         @wiki_attachments_url = wiki_attachments_url
         @revmap = load_revmap_file(revmap_file_path)
@@ -95,6 +96,7 @@ module Migrator
         str.gsub!(/\[changeset:"(\d+)".*\]/) { Tractive::Utilities.map_changeset(Regexp.last_match[1], @revmap, changeset_base_url) }
         str.gsub!(/\[changeset:(\d+).*\]/) { Tractive::Utilities.map_changeset(Regexp.last_match[1], @revmap, changeset_base_url) }
         str.gsub!(/\[(\d+)\]/) { Tractive::Utilities.map_changeset(Regexp.last_match[1], @revmap, changeset_base_url) }
+        str.gsub!(%r{\[(\d+)/.*\]}) { Tractive::Utilities.map_changeset(Regexp.last_match[1], @revmap, changeset_base_url) }
       end
 
       # Font styles
@@ -126,15 +128,15 @@ module Migrator
         image_path = if mod == "source"
                        "![#{path.split("/").last}](#{base_url}#{path})"
                      elsif mod == "wiki"
-                       _, file = path.split(":")
-                       upload_path = "#{wiki_attachments_url}/#{file}"
+                       id, file = path.split(":")
+                       upload_path = "#{wiki_attachments_url}/#{Tractive::Utilities.attachment_path(id, file, hashed: @attach_hashed)}"
                        "![#{file}](#{upload_path})"
                      elsif path.start_with?("http")
                        # [[Image(http://example.org/s.jpg)]]
                        "![#{d[:path]}](#{d[:path]})"
                      else
                        _, id, file = path.split(":")
-                       file_path = "#{attach_url}/#{id}/#{file}"
+                       file_path = "#{attach_url}/#{Tractive::Utilities.attachment_path(id, file, hashed: @attach_hashed)}"
                        "![#{d[:path]}](#{file_path})"
                      end
 

data/lib/tractive/migrator/engine.rb

@@ -27,7 +27,7 @@ module Migrator
       input_file_name   = args[:opts][:importfromfile]
 
       @filter_applied   = args[:opts][:filter]
-      @filter_options   = { column_name: args[:opts][:columnname], operator: args[:opts][:operator], column_value: args[:opts][:columnvalue] }
+      @filter_options   = { column_name: args[:opts][:columnname], operator: args[:opts][:operator], column_value: args[:opts][:columnvalue], include_null: args[:opts][:includenull] }
 
       @trac  = Tractive::Trac.new(db)
       @repo  = github["repo"]
@@ -96,7 +96,7 @@ module Migrator
 
     def mock_ticket_details(ticket_id)
       summary = if @filter_applied
-                  "Not available in trac #{ticket_id}"
+                  "Placeholder issue #{ticket_id} created to align Github issue and trac ticket numbers during migration."
                 else
                   "DELETED in trac #{ticket_id}"
                 end
@@ -105,7 +105,8 @@ module Migrator
         summary: summary,
         time: Time.now.to_i,
         status: "closed",
-        reporter: "tractive"
+        reporter: "tractive",
+        closed_at: Time.at(0).utc
       }
     end
 

data/lib/tractive/migrator/wikis/migrate_from_db.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Migrator
+  module Wikis
+    class MigrateFromDb
+      def initialize(args)
+        $logger.debug("OPTIONS = #{args}")
+
+        @config = args[:cfg]
+        @options = args[:opts]
+        @authors_map = @config["users"].to_h
+
+        @tracticketbaseurl    = @config["trac"]["ticketbaseurl"]
+        @changeset_base_url   = @config["trac"]["changeset_base_url"] || ""
+        @wiki_attachments_url = @options["attachment-base-url"] || @config.dig("wiki", "attachments", "url") || ""
+        @repo_path            = @options["repo-path"] || ""
+        @revmap_path          = @config["revmap_path"]
+
+        @attachment_options   = {
+          hashed: @config.dig("ticket", "attachments", "hashed")
+        }
+
+        verify_options
+        verify_locations
+
+        @twf_to_markdown = Migrator::Converter::TwfToMarkdown.new(@tracticketbaseurl, @attachment_options, @changeset_base_url, @wiki_attachments_url, @revmap_path)
+      end
+
+      def migrate_wikis
+        $logger.info("Processing the wiki...")
+
+        Dir.chdir(@options["repo-path"]) do
+          # For every version of every file in the wiki...
+          Tractive::Wiki.for_migration.each do |wiki|
+            next if skip_file(wiki[:name])
+
+            comment = if wiki[:comment].nil? || wiki[:comment].empty?
+                        "Initial load of version #{wiki[:version]} of trac-file #{wiki[:name]}"
+                      else
+                        wiki[:comment].gsub('"', '\"')
+                      end
+
+            file_name = "#{cleanse_filename(wiki[:name])}.md"
+            $logger.info("Working with file [#{file_name}]")
+            $logger.debug("Object: #{wiki}")
+
+            wiki_markdown_text = @twf_to_markdown.convert(wiki[:text])
+
+            # Create file with content
+            File.open(file_name, "w") do |f|
+              f.puts(wiki_markdown_text)
+            end
+
+            # git-add it
+            unless execute_command("git add #{file_name}").success?
+              $logger.error("ERROR at git-add #{file_name}!!!")
+              exit(1)
+            end
+
+            author = generate_author(wiki[:author])
+
+            # git-commit it
+            commit_command = "git commit --allow-empty -m \"#{comment}\" --author \"#{author}\" --date \"#{wiki[:fixeddate]}\""
+            unless execute_command(commit_command).success?
+              $logger.error("ERROR at git-commit #{file_name}!!!")
+              exit(1)
+            end
+          end
+        end
+      end
+
+      private
+
+      def verify_options
+        $logger.info("Verifying options...")
+
+        missing_options = []
+        missing_options << "attachment-base-url" if @wiki_attachments_url.empty?
+        missing_options << "repo-path" if @repo_path.empty?
+
+        return if missing_options.empty?
+
+        $logger.error("Following options are missing: #{missing_options} - exiting...")
+        exit(1)
+      end
+
+      def verify_locations
+        $logger.info("Verifying locations...")
+        missing_directories = []
+
+        # git-root exists?
+        missing_directories << "repo-path" unless Dir.exist?(@repo_path)
+
+        return if missing_directories.empty?
+
+        $logger.error("Following directories are missing: #{missing_directories} - exiting ...")
+        exit(1)
+      end
+
+      def cleanse_filename(name)
+        # Get rid of 'magic' characters from potential filename - replace with '_'
+        # Magic: [ /<>- ]
+        name.gsub(%r{[/<>-]}, "_")
+      end
+
+      def skip_file(file_name)
+        file_name.start_with?("Trac") || (file_name.start_with?("Wiki") && !file_name.start_with?("WikiStart"))
+      end
+
+      def generate_author(author)
+        return "" if author.nil? || author.empty?
+
+        author_name = @authors_map[author]&.[]("name") || author.split("@").first
+        author_email = @authors_map[author]&.[]("email") || author
+
+        "#{author_name} <#{author_email}>"
+      end
+
+      def execute_command(command)
+        `#{command}`
+        $CHILD_STATUS
+      end
+    end
+  end
+end

data/lib/tractive/migrator/wikis.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "wikis/migrate_from_db"

data/lib/tractive/migrator.rb

@@ -1,3 +1,4 @@
 # frozen_string_literal: true
 
 require_relative "migrator/engine"
+require_relative "migrator/wikis"

data/lib/tractive/models/attachment.rb

@@ -4,6 +4,7 @@ module Tractive
   class Attachment < Sequel::Model(:attachment)
     dataset_module do
       where(:tickets_attachments, type: "ticket")
+      where(:wiki_attachments, type: "wiki")
       select(:for_export, :id, :filename)
     end
   end

data/lib/tractive/models/ticket.rb

@@ -19,14 +19,18 @@ module Tractive
       def filter_column(options)
         return self if options.nil? || options.values.compact.empty?
 
-        case options[:operator].downcase
-        when "like"
-          where { Sequel.like(options[:column_name].to_sym, options[:column_value]) }
-        when "not like"
-          where { ~Sequel.like(options[:column_name].to_sym, options[:column_value]) }
-        else
-          where { Sequel.lit("#{options[:column_name]} #{options[:operator]} '#{options[:column_value]}'") }
-        end
+        query = case options[:operator].downcase
+                when "like"
+                  Sequel.like(options[:column_name].to_sym, options[:column_value])
+                when "not like"
+                  ~Sequel.like(options[:column_name].to_sym, options[:column_value])
+                else
+                  Sequel.lit("#{options[:column_name]} #{options[:operator]} '#{options[:column_value]}'")
+                end
+
+        query = Sequel.|(query, { options[:column_name].to_sym => nil }) if options[:include_null]
+
+        where { query }
       end
     end
 
@@ -35,5 +39,9 @@ module Tractive
       change_arr = changes + attachments
       change_arr.sort_by { |change| change[:time] }
     end
+
+    def closed_comments
+      changes_dataset.where(field: "status", newvalue: "closed")
+    end
   end
 end

data/lib/tractive/models/wiki.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Tractive
+  class Wiki < Sequel::Model(:wiki)
+    set_primary_key :name
+    one_to_many :attachments, class: Attachment, key: :id, conditions: { type: "wiki" }
+
+    dataset_module do
+      def for_migration
+        select(:name, :version, :author, :comment, Sequel.lit("datetime(time/1000000, 'unixepoch')").as(:fixeddate), :text).order(:name, :version)
+      end
+
+      def latest_versions
+        select(:name, :version, :text).group(:name).having { version =~ MAX(version) }
+      end
+    end
+  end
+end

data/lib/tractive/trac.rb

@@ -2,7 +2,7 @@
 
 module Tractive
   class Trac
-    attr_reader :tickets, :changes, :sessions, :attachments
+    attr_reader :tickets, :changes, :sessions, :attachments, :wikis
 
     def initialize(db)
       $logger.info("loading tickets")
@@ -11,6 +11,7 @@ module Tractive
       @changes     = TicketChange
       @sessions    = Session
       @attachments = Attachment
+      @wikis       = Wiki
     end
   end
 end

data/lib/tractive/utilities.rb

@@ -22,7 +22,8 @@ module Tractive
           "lib/tractive/models/revision.rb",
           "lib/tractive/models/session.rb",
           "lib/tractive/models/ticket_change.rb",
-          "lib/tractive/models/ticket.rb"
+          "lib/tractive/models/ticket.rb",
+          "lib/tractive/models/wiki.rb"
         ]
         db = Sequel.connect(db_url) if db_url
 
@@ -35,6 +36,17 @@ module Tractive
         db
       end
 
+      def attachment_path(id, filename, options = {})
+        return "#{id}/#{filename}" unless options[:hashed]
+
+        folder_name = Digest::SHA1.hexdigest(id)
+        parent_folder_name = folder_name[0..2]
+        hashed_filename = Digest::SHA1.hexdigest(filename)
+        file_extension = File.extname(filename)
+
+        "#{parent_folder_name}/#{folder_name}/#{hashed_filename}#{file_extension}"
+      end
+
       def setup_logger(options = {})
         $logger           = Logger.new(options[:output_stream])
         $logger.level     = options[:verbose] ? Logger::DEBUG : Logger::INFO
@@ -46,7 +58,7 @@ module Tractive
 
       # returns the git commit hash for a specified revision (using revmap hash)
       def map_changeset(str, revmap, changeset_base_url = "")
-        if revmap&.key?(str)
+        if revmap&.key?(str) && !revmap[str].nil?
           base_url = changeset_base_url
           base_url += "/" if base_url[-1] && base_url[-1] != "/"
           "#{base_url}#{revmap[str].strip}"

data/lib/tractive/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tractive
-  VERSION = "1.0.6"
+  VERSION = "1.0.10"
 end

data/lib/tractive.rb

@@ -10,6 +10,7 @@ require_relative "tractive/main"
 require_relative "tractive/utilities"
 require_relative "tractive/github_api"
 require_relative "tractive/revmap_generator"
+require "English"
 require "json"
 require "logger"
 require "yaml"

data/tractive.gemspec

@@ -16,6 +16,7 @@ Gem::Specification.new do |spec|
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = "https://github.com/ietf-ribose/tractive"
   spec.metadata["changelog_uri"] = "https://github.com/ietf-ribose/tractive"
+  spec.metadata["rubygems_mfa_required"] = "true"
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tractive
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.0.10
 platform: ruby
 authors:
 - Ribose
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-11-01 00:00:00.000000000 Z
+date: 2021-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mysql2
@@ -112,6 +112,8 @@ description:
 email:
 - open.source@ribose.com
 executables:
+- command_base.rb
+- generate.rb
 - tractive
 extensions: []
 extra_rdoc_files: []
@@ -133,6 +135,8 @@ files:
 - db/trac-test.db
 - docker/Dockerfile
 - docker/docker-compose.yml
+- exe/command_base.rb
+- exe/generate.rb
 - exe/tractive
 - lib/tractive.rb
 - lib/tractive/attachment_exporter.rb
@@ -142,6 +146,8 @@ files:
 - lib/tractive/github_api/client/labels.rb
 - lib/tractive/github_api/client/milestones.rb
 - lib/tractive/graceful_quit.rb
+- lib/tractive/http/client.rb
+- lib/tractive/http/client/request.rb
 - lib/tractive/info.rb
 - lib/tractive/main.rb
 - lib/tractive/migrator.rb
@@ -152,6 +158,8 @@ files:
 - lib/tractive/migrator/engine/migrate_from_db.rb
 - lib/tractive/migrator/engine/migrate_from_file.rb
 - lib/tractive/migrator/engine/migrate_to_file.rb
+- lib/tractive/migrator/wikis.rb
+- lib/tractive/migrator/wikis/migrate_from_db.rb
 - lib/tractive/models/attachment.rb
 - lib/tractive/models/milestone.rb
 - lib/tractive/models/report.rb
@@ -159,6 +167,7 @@ files:
 - lib/tractive/models/session.rb
 - lib/tractive/models/ticket.rb
 - lib/tractive/models/ticket_change.rb
+- lib/tractive/models/wiki.rb
 - lib/tractive/revmap_generator.rb
 - lib/tractive/trac.rb
 - lib/tractive/utilities.rb
@@ -170,6 +179,7 @@ metadata:
   homepage_uri: https://github.com/ietf-ribose/tractive
   source_code_uri: https://github.com/ietf-ribose/tractive
   changelog_uri: https://github.com/ietf-ribose/tractive
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths: