tractive 1.0.0 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b305885e7567a145dc78a8185cd6d4f8d8389f40abf8aac87ff4ae02dec108b
-  data.tar.gz: 1611f6914e94894f5458b71bb1d59d3fcf7e004959242d803e067f863a36f4f7
+  metadata.gz: 73d210f072099b2362c1b39ebb5dfd7d6096fcb76b5096e9fc3db14dec34f14e
+  data.tar.gz: 39b0716034e7ddd0604bc7390f8a4332c2bb5c0664ba88834d7381761dd759aa
 SHA512:
-  metadata.gz: 275ca097252255c2853c147f6e9d0890ff606fd2bc3a0cb39afaaadba6e11c70d2425d0b24d849193cea563f20f82c83a7a885857550566bb7b1fd61f92c89ba
-  data.tar.gz: 06f34858165d8202fabbc16ae7f3cce721f668fef78fe39acee4ffefb864e873d22dea68a9b11644b08cf776062f3974272e11726248fca0099fb4852eb60ed6
+  metadata.gz: c765fe77b184c12c4e161691df30181eea49b0ebd757893b28718b13cb8ccd89f25eec0e7037398bdabe1fa84e9a0df513706ddcc8cef21b27bc0c37812b9d62
+  data.tar.gz: 88eb6e7049f269feab229e39ab0de8ff9835648cf92f300b64ff07b94ad25e8274991bf9106f0c2050b75fceef430965b98d7ba493690965a7306b4e0932bc25

data/.ruby-gemset

@@ -0,0 +1 @@
+tractive

data/.ruby-version

@@ -0,0 +1 @@
+2.7.0

data/Gemfile.lock

@@ -0,0 +1,100 @@
+PATH
+  remote: .
+  specs:
+    tractive (1.0.4)
+      mysql2
+      ox
+      rest-client
+      sequel
+      sqlite3
+      thor
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    ast (2.4.2)
+    coderay (1.1.3)
+    crack (0.4.5)
+      rexml
+    diff-lcs (1.4.4)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    hashdiff (1.0.1)
+    http-accept (1.7.0)
+    http-cookie (1.0.4)
+      domain_name (~> 0.5)
+    method_source (1.0.0)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2021.0901)
+    mysql2 (0.5.3)
+    netrc (0.11.0)
+    ox (2.14.5)
+    parallel (1.21.0)
+    parser (3.0.2.0)
+      ast (~> 2.4.1)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    public_suffix (4.0.6)
+    rainbow (3.0.0)
+    rake (13.0.6)
+    regexp_parser (2.1.1)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    rexml (3.2.5)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    rubocop (1.22.1)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.12.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.12.0)
+      parser (>= 3.0.1.1)
+    ruby-progressbar (1.11.0)
+    sequel (5.49.0)
+    sqlite3 (1.4.2)
+    thor (1.1.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.8)
+    unicode-display_width (2.1.0)
+    webmock (3.14.0)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  pry
+  rake (~> 13.0)
+  rspec (~> 3.0)
+  rubocop (~> 1.7)
+  tractive!
+  webmock (~> 3.14)
+
+BUNDLED WITH
+   2.1.2

data/README.adoc

@@ -1,5 +1,4 @@
 = Tractive: migrating from Trac to GitHub
-
 == Purpose
 
 Tractive is a tool that helps you migrate Trac instances to GitHub.
@@ -194,15 +193,17 @@ user interface.
 
 | `--svn-url`
 |
-(required) SVN repository URL that should be used in RevMap generation.
-The URL must start with the `http://` or `https://` prefix (not the `svn://`
-prefix).
+(required unless `--svn-local-path` set)
+SVN repository URL that should be used in RevMap generation. The URL must start
+with the `http://` or `https://` prefix (not the `svn://` prefix).
 | String
 
 | `--svn-local-path`
-| SVN local repository path that should be used in RevMap generation. You can clone
-the svn repo locally and provide its local path if the svn repository is offline. (`--svn-url` is not
-required if this option is given).
+|
+(required unless `-svn-url` set)
+SVN local repository path that should be used in RevMap generation. You can
+clone the svn repo locally and provide its local path if the svn repository is
+offline.
 | String
 
 | `--rev-timestamp-file`
@@ -384,25 +385,55 @@ The pattern of a mapping is like:
 +
 [source,yaml]
 ----
-  blocker:  '#blocker'
-  critical: '#critical'
-  major:    '#major'
-  medium:   '#medium'
-  minor:    '#minor'
-  trivial:  '#easy'
-  waiting:  '#pending'
-  n/a:
+  trivial:
+    name: trivial
+    color: ff0000
+  major:
+    name: major
+    color: b44647
+  minor:
+    name: minor
+    color: f7347a
+  medium:
+    name: medium
+    color: f3c77c
 ----
 
 `priority:`::: Priority of the Trac ticket.
 +
 [source,yaml]
 ----
-  Low:  '@low'
-  High: '@high'
+  Low:
+    name: low
+    color: 22dd00
+  High:
+    name: high
+    color: ff0000
+----
+
+
+`tracstate:`::: Status of the Trac ticket.
++
+[source,yaml]
+----
+  accepted:
+    name: accepted
+    color: 22dd00
+  assigned:
+    name: assigned
+    color: aadd88
+  closed:
+    name: closed
+    color: ee00aa
+  new:
+    name: new
+    color:
 ----
 
 
+NOTE: As `severity`, `priority` and `tracstate` are converted into `labels` on github so there is an option to specify the `color` for those labels.
+
+
 ==== User mapping
 
 `users:`:: a one-to-one mapping between Trac usernames or email addresses to
@@ -411,7 +442,10 @@ GitHub usernames for users, in the following pattern:
 [source,yaml]
 ----
 users:
-  {Trac email or username}: {username on GitHub}
+  {Trac email or username}:
+    email: {Github email}
+    name: {name of the person}
+    username: {username on GitHub}
   ...
 ----
 
@@ -420,8 +454,23 @@ EXAMPLE:
 [source,yaml]
 ----
 users:
-  matthew@example.org: example-matt
-  valencia: example-vale
+  matthew@gmail.org:
+    email: matthew@example.org
+    name: Matthew
+    username: example-matt
+  valencia:
+    email: valencia
+    name: Valencia
+    username: example-vale
+----
+
+If you don't want to map a user, you can just leave the `username` empty like below:
+----
+users:
+  matthew@gmail.org:
+    email: matthew@example.org
+    name: Matthew
+    username:
 ----
 
 ==== Milestone mapping

data/config.example.yaml

@@ -24,9 +24,16 @@ revmap_path: ./example-revmap.txt
 #   the issue migration process will fail if the GitHub user specified as owner
 #   (assignee) does not exist.
 users:
-  # <Trac email or username>: <username on GitHub>
-  matthew@example.org: example-matt
-  valencia: example-vale
+  # email: <Trac email or username>
+  # name: <Name of the person (optional)>
+  # username: <username on GitHub>
+  - email: matthew@example.org
+    name: Matthew
+    username: example-matt
+
+  - email: valencia
+    name: Valencia
+    username: example-vale
 
 # Label mapping from Trac ticket to GitHub label
 labels:
@@ -47,14 +54,38 @@ labels:
 
   # less useful, but also possible:
   priority:
-    Low:                '@low'
-    High:               '@high'
+    Low:
+      name:             low
+      color:            22dd00
+    High:
+      name:             high
+      color:            ff0000
   severity:
-    blocker:            '#high'
-    critical:           '#critical'
-    major:              '#major'
-    minor:              '#minor'
-    trivial:            '#trivial'
+    trivial:
+      name:             trivial
+      color:            ff0000
+    major:
+      name:             major
+      color:            b44647
+    minor:
+      name:             minor
+      color:            f7347a
+    medium:
+      name:             medium
+      color:            f3c77c
+  tracstate:
+    accepted:
+      name:             accepted
+      color:            22dd00
+    assigned:
+      name:             assigned
+      color:            aadd88
+    closed:
+      name:             closed
+      color:            ee00aa
+    new:
+      name:             new
+      color:
   version:
     '1.3':              v1.3
     '1.4':              v1.4

data/exe/tractive

@@ -5,6 +5,8 @@ require_relative "../lib/tractive"
 
 class TractiveCommand < Thor
   default_command :migrate
+  class_option "logfile", type: :string, aliases: ["-L", "--log-file"],
+                          desc: "Name of the logfile to output logs to."
 
   desc "<OPTIONS>", "Migrate Trac instances to modern Git management platforms like GitHub and GitLab"
   method_option "attachmentexporter", type: :string, aliases: ["-A", "--attachment-exporter"],
@@ -33,8 +35,6 @@ class TractiveCommand < Thor
                                   desc: "Import issues from a json file"
   method_option "info", type: :boolean, aliases: ["-i", "--info"],
                         desc: "Reports existing labels and users in the database"
-  method_option "logfile", type: :string, aliases: ["-L", "--log-file"],
-                           desc: "Name of the logfile to output logs to."
   method_option "mockdeleted", type: :boolean, aliases: ["-M", "--mockup"],
                                desc: "Import from 0 and mocking tickets deleted on trac"
   method_option "openedonly", type: :boolean, aliases: ["-o", "--opened-only"],
@@ -64,6 +64,7 @@ class TractiveCommand < Thor
   def generate_revmap
     verify_revmap_generator_options!(options)
 
+    Tractive::Utilities.setup_logger(output_stream: options[:log_file] || $stderr, verbose: options[:verbose])
     Tractive::RevmapGenerator.new(
       options["revtimestampfile"],
       options["svnurl"],

data/lib/tractive/github_api/client/labels.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module GithubApi
+  class Client
+    module Labels
+      def list_labels(repo, params = {})
+        JSON.parse(
+          RestClient.get(
+            "https://api.github.com/repos/#{repo}/labels",
+            {
+              "Authorization" => "token #{@token}",
+              params: params
+            }
+          )
+        )
+      end
+      alias labels list_labels
+
+      def create_label(repo, params)
+        JSON.parse(
+          RestClient.post(
+            "https://api.github.com/repos/#{repo}/labels",
+            params.to_json,
+            {
+              "Authorization" => "token #{@token}",
+              "Accept" => "application/vnd.github.v3+json"
+            }
+          )
+        )
+      end
+    end
+  end
+end

data/lib/tractive/github_api/client.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
 
 require_relative "client/issues"
+require_relative "client/labels"
 require_relative "client/milestones"
 
 # Service to perform github actions
 module GithubApi
   class Client
     include GithubApi::Client::Issues
+    include GithubApi::Client::Labels
     include GithubApi::Client::Milestones
 
     def initialize(options = {})

data/lib/tractive/info.rb

@@ -30,15 +30,15 @@ module Tractive
       tracstates  = Ticket.distinct.select(:status).select_map(:status).compact
 
       {
-        "users" => Utilities.make_hash("", users),
+        "users" => Utilities.make_each_hash(users, %w[email name username]),
         "milestones" => milestones,
         "labels" => {
           "type" => Utilities.make_hash("type_", types),
           "component" => Utilities.make_hash("component_", components),
           "resolution" => Utilities.make_hash("resolution_", resolutions),
-          "severity" => Utilities.make_hash("severity", severity),
-          "priority" => Utilities.make_hash("priority_", priorities),
-          "tracstate" => Utilities.make_hash("tracstate_", tracstates)
+          "severity" => Utilities.make_each_hash(severity, %w[name color]),
+          "priority" => Utilities.make_each_hash(priorities, %w[name color]),
+          "tracstate" => Utilities.make_each_hash(tracstates, %w[name color])
         }
       }
     end

data/lib/tractive/migrator/converter/trac_to_github.rb

@@ -17,6 +17,9 @@ module Migrator
         @wiki_attachments_url = args[:cfg]["trac"]["wiki_attachments_url"]
 
         load_milestone_map
+        create_labels_on_github(@labels_cfg["severity"].values)
+        create_labels_on_github(@labels_cfg["priority"].values)
+        create_labels_on_github(@labels_cfg["tracstate"].values)
 
         @uri_parser = URI::Parser.new
         @twf_to_markdown = Migrator::Converter::TwfToMarkdown.new(@tracticketbaseurl, @attachurl, @changeset_base_url, @wiki_attachments_url)
@@ -116,7 +119,8 @@ module Migrator
         # compose body
         body = [badgetable, body, footer].join("\n\n___\n")
 
-        labels.add("owner:#{github_assignee}")
+        labels.add("name" => "owner:#{github_assignee}") unless github_assignee.nil? || github_assignee.empty?
+        labels = labels.map { |label| label["name"] }
 
         issue = {
           "title" => ticket[:summary],
@@ -129,7 +133,7 @@ module Migrator
 
         if @users.key?(ticket[:owner])
           owner = trac_mail(ticket[:owner])
-          github_owner = @users[owner]
+          github_owner = @users[owner]["username"]
           $logger.debug("..owner in trac: #{owner}")
           $logger.debug("..assignee in GitHub: #{github_owner}")
           issue["assignee"] = github_owner
@@ -155,11 +159,11 @@ module Migrator
       private
 
       def map_user(user)
-        @users[user] || user
+        @users.fetch(user, {})["email"] || user
       end
 
       def map_assignee(user)
-        @users[user]
+        @users.fetch(user, {})["email"]
       end
 
       def load_milestone_map
@@ -168,9 +172,9 @@ module Migrator
         newmilestonekeys = @milestonesfromtrac.keys - @milestonemap.keys
 
         newmilestonekeys.each do |milestonelabel|
-          milestone           = {
+          milestone = {
             "title" => milestonelabel.to_s,
-            "state" => @milestonesfromtrac[milestonelabel][:completed].nil? ? "open" : "closed",
+            "state" => @milestonesfromtrac[milestonelabel][:completed].to_i.zero? ? "open" : "closed",
             "description" => @milestonesfromtrac[milestonelabel][:description] || "no description in trac",
             "due_on" => "2012-10-09T23:39:01Z"
           }
@@ -194,6 +198,21 @@ module Migrator
         nil
       end
 
+      def create_labels_on_github(labels)
+        return if labels.nil? || labels.empty?
+
+        existing_labels = @client.labels(@repo).map { |label| label["name"] }
+        new_labels = labels.reject { |label| existing_labels.include?(label["name"]) }
+
+        new_labels.each do |label|
+          params = { name: label["name"] }
+          params["color"] = label["color"] unless label["color"].nil?
+
+          @client.create_label(@repo, params)
+          $logger.info("Created label: #{label["name"]}")
+        end
+      end
+
       def ticket_change(append, meta)
         # kind
         kind = if meta[:ticket]

data/lib/tractive/models/ticket.rb

@@ -19,8 +19,11 @@ module Tractive
       def filter_column(options)
         return self if options.nil? || options.values.compact.empty?
 
-        if options[:operator].downcase == "like"
+        case options[:operator].downcase
+        when "like"
           where { Sequel.like(options[:column_name].to_sym, options[:column_value]) }
+        when "not like"
+          where { ~Sequel.like(options[:column_name].to_sym, options[:column_value]) }
         else
           where { Sequel.lit("#{options[:column_name]} #{options[:operator]} '#{options[:column_value]}'") }
         end

data/lib/tractive/revmap_generator.rb

@@ -11,6 +11,7 @@ module Tractive
       @duplicate_message_commits = {}
       @last_revision = nil
       @pinwheel = %w[| / - \\]
+      @skipped = []
       @output_file = output_file
     end
 
@@ -20,6 +21,7 @@ module Tractive
 
       File.open(@output_file, "w+") do |file|
         File.foreach(@input_file) do |line|
+          i += 1
           info = extract_info_from_line(line)
           next if @last_revision == info[:revision]
 
@@ -29,9 +31,10 @@ module Tractive
           percent = ((i.to_f / line_count) * 100).round(2)
           progress = "=" * (percent.to_i / 2) unless i < 2
           printf("\rProgress: [%<progress>-50s] %<percent>.2f%% %<spinner>s", progress: progress, percent: percent, spinner: @pinwheel.rotate!.first)
-          i += 1
         end
       end
+
+      $logger.info "\n\nFollowing revisions are skipped because they don't have a corresponding git commit. #{@skipped}"
     end
 
     private
@@ -52,11 +55,13 @@ module Tractive
       # get sha from git api
       commits = git_commits(info)
 
-      if commits.count == 1
-        file.puts "#{info[:revision]} | #{commits.values[0].join(",")}"
+      if commits.empty?
+        @skipped << info[:revision]
+      elsif commits.count == 1
+        file.puts "#{info[:revision]} | #{commits.values[0]&.join(",")}"
       else
         message = commit_message_from_svn(info[:revision])
-        file.puts "#{info[:revision]} | #{@duplicate_commits[info[:timestamp]][message].join(",")}"
+        file.puts "#{info[:revision]} | #{@duplicate_commits[info[:timestamp]][message]&.join(",")}"
       end
     end
 
@@ -91,16 +96,24 @@ module Tractive
     end
 
     def commits_from_git_repo(info)
-      command = "git rev-list --after=#{info[:timestamp]} --until=#{info[:timestamp]} --committer=#{info[:author]} --all --format='%cd|%h~|~%s' --date=format:'%Y-%m-%dT%H:%M:%SZ'"
+      shas_command = "git rev-list --after=#{info[:timestamp]} --until=#{info[:timestamp]} --committer=#{info[:author]} --all"
+      shas = Dir.chdir(@git_local_repo_path) do
+        `#{shas_command}`
+      end
+
+      commits_command = "git rev-list --after=#{info[:timestamp]} --until=#{info[:timestamp]} --committer=#{info[:author]} --all --format='medium'"
       commits = Dir.chdir(@git_local_repo_path) do
-        `#{command}`
+        `#{commits_command}`
       end
 
+      regex = /#{shas.split("\n").map { |sha| "(?=commit #{sha})" }.join "|"}/
+
       commits_arr = []
-      commits.split("\n").each_slice(2) do |sha_hash, commit_info|
+      commits.split(regex).each do |commit_info|
         commit_hash = {}
-        commit_hash[:sha] = sha_hash.split.last
-        commit_hash[:short_sha], commit_hash[:message] = commit_info.split("~|~")
+        info = commit_info.split("\n", 4)
+        commit_hash[:sha] = info[0].split.last
+        commit_hash[:message] = info.last.strip.gsub("\n", "").gsub(/\s+/, " ")
 
         commits_arr << commit_hash
       end

data/lib/tractive/utilities.rb

@@ -7,6 +7,13 @@ module Tractive
         array.map { |i| [i, "#{prefix}#{i}"] }.to_h
       end
 
+      def make_each_hash(values, keys)
+        values.map do |value|
+          value = [value] unless value.is_a?(Array)
+          [value[0], keys.zip(value).to_h]
+        end.to_h
+      end
+
       def setup_db!(db_url)
         files_to_load = [
           "lib/tractive/models/attachment.rb",

data/lib/tractive/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tractive
-  VERSION = "1.0.0"
+  VERSION = "1.0.4"
 end

data/lib/tractive.rb

@@ -19,7 +19,6 @@ require "sequel"
 require "set"
 require "singleton"
 require "uri"
-require "pry"
 require "thor"
 require "ox"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tractive
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.4
 platform: ruby
 authors:
 - Ribose
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-11 00:00:00.000000000 Z
+date: 2021-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mysql2
@@ -120,8 +120,11 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
+- ".ruby-gemset"
+- ".ruby-version"
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.md
 - README.adoc
 - Rakefile
@@ -137,6 +140,7 @@ files:
 - lib/tractive/github_api.rb
 - lib/tractive/github_api/client.rb
 - lib/tractive/github_api/client/issues.rb
+- lib/tractive/github_api/client/labels.rb
 - lib/tractive/github_api/client/milestones.rb
 - lib/tractive/graceful_quit.rb
 - lib/tractive/info.rb