tractive 1.0.0

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "tractive"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/config.example.yaml

@@ -0,0 +1,73 @@
+# Trac-specific information
+trac:
+  # Trac database location
+  database: sqlite://db/trac.db
+  # database: mysql2://user:password@host:port/database
+  # database: mysql2://root:password@mysql:3306/foobar
+
+  # URL of the Trac "tickets" interface
+  ticketbaseurl: https://example.org/trac/foobar/ticket
+
+# GitHub-specific information
+github:
+  # Target GitHub organization and repo name
+  repo: 'example-org/target-repository'
+
+  # GitHub user Personal Access Token
+  token: 'ghp_fpsc4de1f0c46e01576810740c9242097cba4619486'
+
+# RevMap file to use for migration
+revmap_path: ./example-revmap.txt
+
+# User map: Trac username to GitHub username
+# - Please ensure the GitHub usernames are correct and valid, otherwise
+#   the issue migration process will fail if the GitHub user specified as owner
+#   (assignee) does not exist.
+users:
+  # <Trac email or username>: <username on GitHub>
+  matthew@example.org: example-matt
+  valencia: example-vale
+
+# Label mapping from Trac ticket to GitHub label
+labels:
+  type:
+    Feature Request:    feature request
+    Defect:             defect
+    Task:               task
+  component:
+    configuration:      conf
+    documentation:      doc
+  resolution:
+    Invalid:            invalid
+    Duplicate:          duplicate
+    Wontfix:            wontfix
+  platform:
+    Linux:              Linux
+    Windows:            Windows
+
+  # less useful, but also possible:
+  priority:
+    Low:                '@low'
+    High:               '@high'
+  severity:
+    blocker:            '#high'
+    critical:           '#critical'
+    major:              '#major'
+    minor:              '#minor'
+    trivial:            '#trivial'
+  version:
+    '1.3':              v1.3
+    '1.4':              v1.4
+
+milestones:
+  '2021_02':
+    name: '2021_02'
+    due: 1392595200000000
+    completed: 1415959156000000
+    description: ''
+
+attachments:
+  url: "https://github.com/example-org/raw/main/from_tractive"
+  # export-folder is relative to export-script
+  export_folder: "./attachments"
+  export_script: ./attachments.sh

data/docker/Dockerfile

@@ -0,0 +1,19 @@
+FROM ruby:2.5
+
+WORKDIR /usr/src/app
+
+COPY Gemfile Gemfile.lock ./
+RUN bundle install
+
+
+# throw errors if Gemfile has been modified since Gemfile.lock
+RUN bundle config --global frozen 1
+
+COPY . .
+
+WORKDIR /usr/src/myapp/trac-hub
+
+ENTRYPOINT ./trac-hub
+CMD -h
+
+

data/docker/docker-compose.yml

@@ -0,0 +1,68 @@
+version: "2"
+
+services:
+  tracd:
+    image: solsson/trac:engine
+
+    restart: unless-stopped
+    volumes:
+      - ../trac-env/:/trac
+    ports:
+      - 81:80
+    links:
+      - mysql
+    depends_on:
+      - mysql
+    command: ["-s", "/trac", "--auth", "*,/trac/conf/digest_passwd,pjtadmin"]
+
+  mysql:
+    image: mysql:5.7
+    restart: unless-stopped
+    environment:
+      - MYSQL_ROOT_PASSWORD=password
+    ports:
+      - 3306:3306
+    volumes:
+      - mysql:/var/lib/mysql
+
+  gitlab:
+    image: gitlab/gitlab-ce:latest
+    ports:
+      - 443:443
+      - 80:80
+      - 22:22
+    container_name: gitlab
+
+    restart: unless-stopped
+    volumes:
+      - /Users/beweiche/gitlab/config:/etc/gitlab
+      - /Users/beweiche/gitlab/logs:/var/log/gitlab
+      - /Users/beweiche/gitlab/data:/var/opt/gitlab
+
+  trac-hub:
+    image: bwl21/trac-hub
+    build: .
+
+    restart: unless-stopped
+
+    volumes:
+      - ..:/usr/src/myapp
+      - /Users/beweiche/gitlab/data:/var/opt/gitlab
+    links:
+      - mysql
+    depends_on:
+      - mysql
+
+    working_dir: /usr/src/myapp/trac-hub
+    entrypoint: ./trac-hub
+    command: -h
+
+
+
+
+volumes:
+  mysql: {
+
+  }
+  www: {}
+  log: {}

data/exe/tractive

@@ -0,0 +1,111 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "../lib/tractive"
+
+class TractiveCommand < Thor
+  default_command :migrate
+
+  desc "<OPTIONS>", "Migrate Trac instances to modern Git management platforms like GitHub and GitLab"
+  method_option "attachmentexporter", type: :string, aliases: ["-A", "--attachment-exporter"],
+                                      desc: "Generate an attachment exporter script according to config.yaml"
+  method_option "attachurl", type: :string, aliases: ["-a", "--attachment-url"], banner: "<URL>",
+                             desc: "If attachment files are reachable via a URL we reference this here"
+  method_option "config", type: :string, default: "tractive.config.yaml", banner: "<PATH>", aliases: "-c",
+                          desc: "Set the configuration file"
+  method_option "dryrun", type: :boolean, aliases: ["-d", "--dry-run"],
+                          desc: "Write data to a file instead of pushing it to github"
+  method_option "exportattachments", type: :string, aliases: ["-e", "--export-attachments"],
+                                     desc: "Export attachments from the database according to config.yaml"
+  method_option "fast", type: :boolean, aliases: ["-F", "--fast-import"],
+                        desc: "Import without safety-checking issue numbers."
+
+  method_option "filter", type: :boolean, aliases: ["-f", "--filter"],
+                          desc: "Filter records that you want to import."
+  method_option "columnname", type: :string, aliases: ["--column-name"],
+                              desc: "Name of the column to filter."
+  method_option "operator", type: :string, aliases: ["--operator"],
+                            desc: "Operator for filter."
+  method_option "columnvalue", type: :string, aliases: ["--column-value"],
+                               desc: "Value of the column to filter."
+
+  method_option "importfromfile", type: :string, aliases: ["-I", "--import-from-file"],
+                                  desc: "Import issues from a json file"
+  method_option "info", type: :boolean, aliases: ["-i", "--info"],
+                        desc: "Reports existing labels and users in the database"
+  method_option "logfile", type: :string, aliases: ["-L", "--log-file"],
+                           desc: "Name of the logfile to output logs to."
+  method_option "mockdeleted", type: :boolean, aliases: ["-M", "--mockup"],
+                               desc: "Import from 0 and mocking tickets deleted on trac"
+  method_option "openedonly", type: :boolean, aliases: ["-o", "--opened-only"],
+                              desc: "Skips the import of closed tickets"
+  method_option "revmapfile", type: :string, aliases: ["-r", "--rev-map-file"], banner: "<PATH>",
+                              desc: "Allows to specify a commit revision mapping FILE"
+  method_option "singlepost", type: :boolean, aliases: ["-S", "--single-post"],
+                              desc: "Put all issue comments in the first message."
+  method_option "start", type: :numeric, aliases: ["-s", "--start-at"], banner: "<ID>",
+                         desc: "Start migration from ticket with number <ID>"
+  method_option "verbose", type: :boolean, aliases: ["-v", "--verbose"], desc: "Verbose mode"
+  def migrate
+    Tractive::Main.new(options).run
+  end
+
+  desc "generate-revmap <OPTIONS>", "Generate a mapping from svn revision number to git sha hash."
+  method_option "svnurl", type: :string, aliases: ["--svn-url"],
+                          desc: "Svn url that should be used in revmap generation"
+  method_option "svnlocalpath", type: :string, aliases: ["--svn-local-path"],
+                                desc: "Local SVN repo path"
+  method_option "gitlocalrepopath", type: :string, aliases: ["--git-local-repo-path"],
+                                    desc: "Local git repo path that should be used in revmap generation"
+  method_option "revtimestampfile", type: :string, aliases: ["--rev-timestamp-file"],
+                                    desc: "File containing svn revision and timestamps that should be used in revmap generation"
+  method_option "revoutputfile", type: :string, aliases: ["--revmap-output-file"],
+                                 desc: "File to output the generated revmap"
+  def generate_revmap
+    verify_revmap_generator_options!(options)
+
+    Tractive::RevmapGenerator.new(
+      options["revtimestampfile"],
+      options["svnurl"],
+      options["svnlocalpath"],
+      options["gitlocalrepopath"],
+      options["revoutputfile"]
+    ).generate
+  end
+
+  def self.exit_on_failure?
+    true
+  end
+
+  def method_missing(*args)
+    warn "No method found named: #{args[0]}"
+    warn "Run with `--help` or `-h` to see available options"
+    exit 1
+  end
+
+  def respond_to_missing?
+    true
+  end
+
+  no_commands do
+    def verify_revmap_generator_options!(options)
+      required_options = {}
+      required_options["--svn-url OR --svn-local-path"] = options["svnurl"] || options["svnlocalpath"]
+      required_options["--git-local-repo-path"] = options["gitlocalrepopath"]
+      required_options["--rev-timestamp-file"] = options["revtimestampfile"]
+      required_options["--revmap-output-file"] = options["revoutputfile"]
+
+      missing_options = {}
+      required_options.each do |key, value|
+        missing_options[key] = value if value.nil? || value.strip.empty?
+      end
+
+      return if missing_options.empty?
+
+      warn("missing revmap generator options (#{missing_options.keys}).\nRun with `--help` or `-h` to see available options")
+      exit 1
+    end
+  end
+end
+
+TractiveCommand.start(ARGV)

data/lib/tractive/attachment_exporter.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "open-uri"
+
+module Tractive
+  class AttachmentExporter
+    def initialize(cfg, db)
+      @cfg = cfg
+      @db = db
+    end
+
+    # Produce a shell script to export attachments, to be executed on the Trac
+    # instance
+    def generate_script
+      outfile   = @cfg.dig("attachments", "export_script")
+      outfolder = @cfg.dig("attachments", "export_folder")
+
+      raise("missing attachments.export_script entry in configuration") unless outfile
+      raise("missing attachments.export_folder entry in configuration") unless outfolder
+
+      attachments     = Attachment.tickets_attachments.for_export
+      export_commands = attachments.map do |attachment|
+        %(mkdir -p #{outfolder}/#{attachment[:id]}
+        trac-admin /trac attachment export ticket:#{attachment[:id]} '#{attachment[:filename]}' > '#{outfolder}/#{attachment[:id]}/#{attachment[:filename]}')
+      end
+
+      File.open(outfile, "w") do |f|
+        f.puts("mkdir -p #{outfolder}")
+        f.puts(export_commands.join("\n"))
+      end
+
+      $logger.info "created attachment exporter in #{outfile}"
+    rescue StandardError => e
+      $logger.error(e.message)
+      exit(1)
+    end
+
+    # export the images from the database into a folder
+    def export
+      output_dir = @cfg.dig("attachments", "export_folder") || "#{Dir.pwd}/tmp/trac"
+      trac_url = @cfg.dig("attachments", "url")
+
+      raise("attachments url is required in config.yaml to export attachments.") unless trac_url
+
+      FileUtils.mkdir_p output_dir
+      attachments = Attachment.tickets_attachments.for_export
+
+      # using URI::Parser.new because URI.encode raise warning: URI.escape is obsolete
+      uri_parser = URI::Parser.new
+
+      attachments.each do |attachment|
+        $logger.info "Saving attachments of ticket #{attachment.id}... "
+        FileUtils.mkdir_p "#{output_dir}/#{attachment.id}"
+
+        File.open("#{output_dir}/#{attachment.id}/#{attachment.filename}", "wb") do |file|
+          file.write URI.open(uri_parser.escape("#{trac_url}/#{attachment.id}/#{attachment.filename}")).read
+        end
+      end
+    end
+  end
+end

data/lib/tractive/github_api/client/issues.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module GithubApi
+  class Client
+    # Methods for the Issues API
+    module Issues
+      def create_issue(repo, params)
+        JSON.parse(
+          RestClient.post(
+            "https://api.github.com/repos/#{repo}/import/issues",
+            params.to_json,
+            {
+              "Authorization" => "token #{@token}",
+              "Content-Type" => "application/json",
+              "Accept" => "application/vnd.github.golden-comet-preview+json"
+            }
+          )
+        )
+      end
+
+      def list_issues(repo, params)
+        JSON.parse(
+          RestClient.get(
+            "https://api.github.com/repos/#{repo}/issues",
+            {
+              "Authorization" => "token #{@token}",
+              params: params
+            }
+          )
+        )
+      end
+      alias issues list_issues
+
+      def issue(repo, number)
+        JSON.parse(
+          RestClient.get(
+            "https://api.github.com/repos/#{repo}/issues/#{number}",
+            { "Authorization" => "token #{@token}" }
+          )
+        )
+      end
+
+      def issue_import_status(repo, id)
+        JSON.parse(
+          RestClient.get(
+            "https://api.github.com/repos/#{repo}/import/issues/#{id}",
+            {
+              "Authorization" => "token #{@token}",
+              "Accept" => "application/vnd.github.golden-comet-preview+json"
+            }
+          )
+        )
+      end
+
+      def issue_comments(repo, issue_id)
+        JSON.parse(
+          RestClient.get(
+            "https://api.github.com/repos/#{repo}/issues/#{issue_id}/comments",
+            {
+              "Authorization" => "token #{@token}",
+              "Accept" => "application/vnd.github.golden-comet-preview+json"
+            }
+          )
+        )
+      end
+
+      def update_issue_comment(repo, comment_id, comment_body)
+        JSON.parse(
+          RestClient.patch(
+            "https://api.github.com/repos/#{repo}/issues/comments/#{comment_id}",
+            { body: comment_body }.to_json,
+            { "Authorization" => "token #{@token}" }
+          )
+        )
+      end
+    end
+  end
+end

data/lib/tractive/github_api/client/milestones.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module GithubApi
+  class Client
+    # Methods for the Issues API
+    module Milestones
+      def list_milestones(repo, params)
+        JSON.parse(
+          RestClient.get(
+            "https://api.github.com/repos/#{repo}/milestones?per_page=100",
+            {
+              "Authorization" => "token #{@token}",
+              params: params
+            }
+          )
+        )
+      end
+      alias milestones list_milestones
+
+      def create_milestone(repo, params)
+        JSON.parse(
+          RestClient.post(
+            "https://api.github.com/repos/#{repo}/milestones",
+            params.to_json,
+            {
+              "Authorization" => "token #{@token}",
+              "Content-Type" => "application/json",
+              "Accept" => "application/vnd.github.golden-comet-preview+json"
+            }
+          )
+        )
+      end
+    end
+  end
+end

data/lib/tractive/github_api/client.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "client/issues"
+require_relative "client/milestones"
+
+# Service to perform github actions
+module GithubApi
+  class Client
+    include GithubApi::Client::Issues
+    include GithubApi::Client::Milestones
+
+    def initialize(options = {})
+      @token = options[:access_token]
+    end
+  end
+end

data/lib/tractive/github_api.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "github_api/client"

data/lib/tractive/graceful_quit.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "singleton"
+
+module Tractive
+  class GracefulQuit
+    include Singleton
+
+    attr_accessor :breaker
+
+    def initialize
+      self.breaker = false
+    end
+
+    def self.enable
+      trap("INT") do
+        yield if block_given?
+        instance.breaker = true
+      end
+    end
+
+    def self.check(message = "Quitting")
+      return unless instance.breaker
+
+      yield if block_given?
+      $logger.info message
+      exit
+    end
+  end
+end

data/lib/tractive/info.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Tractive
+  class Info
+    def print
+      $logger.info result_hash.to_yaml
+    end
+
+    private
+
+    def result_hash
+      users = [
+        Ticket.distinct.select(:reporter).select_map(:reporter),
+        Ticket.distinct.select(:owner).select_map(:owner),
+        TicketChange.distinct.select(:author).select_map(:author),
+        TicketChange.distinct.select(:newvalue).where(field: "reporter").select_map(:newvalue),
+        Revision.distinct.select(:author).select_map(:author),
+        Report.distinct.select(:author).select_map(:author),
+        Attachment.distinct.select(:author).select_map(:author)
+      ].flatten.uniq.compact
+
+      milestones = {}
+      Milestone.each { |r| milestones[r.name] = r.to_hash }
+
+      types       = Ticket.distinct.select(:type).select_map(:type).compact
+      components  = Ticket.distinct.select(:component).select_map(:component).compact
+      resolutions = Ticket.distinct.select(:resolution).select_map(:resolution).compact
+      severity    = Ticket.distinct.select(:severity).select_map(:severity).compact
+      priorities  = Ticket.distinct.select(:priority).select_map(:priority).compact
+      tracstates  = Ticket.distinct.select(:status).select_map(:status).compact
+
+      {
+        "users" => Utilities.make_hash("", users),
+        "milestones" => milestones,
+        "labels" => {
+          "type" => Utilities.make_hash("type_", types),
+          "component" => Utilities.make_hash("component_", components),
+          "resolution" => Utilities.make_hash("resolution_", resolutions),
+          "severity" => Utilities.make_hash("severity", severity),
+          "priority" => Utilities.make_hash("priority_", priorities),
+          "tracstate" => Utilities.make_hash("tracstate_", tracstates)
+        }
+      }
+    end
+  end
+end

data/lib/tractive/main.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Tractive
+  class Main
+    def initialize(opts)
+      verify_options!(opts)
+
+      @opts = opts
+      @cfg  = YAML.load_file(@opts[:config])
+
+      Tractive::Utilities.setup_logger(output_stream: @opts[:log_file] || $stderr, verbose: @opts[:verbose])
+      @db = Tractive::Utilities.setup_db!(@cfg["trac"]["database"])
+    rescue Sequel::DatabaseConnectionError, Sequel::AdapterNotFound, URI::InvalidURIError, Sequel::DatabaseError => e
+      $logger.error e.message
+      exit 1
+    end
+
+    def run
+      if @opts[:info]
+        info
+      elsif @opts[:attachmentexporter]
+        create_attachment_exporter_script
+      elsif @opts[:exportattachments]
+        export_attachments
+      elsif @opts[:generaterevmap]
+        generate_revmap_file
+      else
+        migrate
+      end
+    end
+
+    def migrate
+      Migrator::Engine.new(opts: @opts, cfg: @cfg, db: @db).migrate
+    end
+
+    def info
+      Tractive::Info.new.print
+    end
+
+    def export_attachments
+      Tractive::AttachmentExporter.new(@cfg, @db).export
+    end
+
+    def create_attachment_exporter_script
+      Tractive::AttachmentExporter.new(@cfg, @db).generate_script
+    end
+
+    private
+
+    def verify_options!(options)
+      verify_config_options!(options)
+      verify_filter_options!(options)
+    end
+
+    def verify_config_options!(options)
+      return if File.exist?(options[:config])
+
+      warn_and_exit("missing configuration file (#{options[:config]})", 1)
+    end
+
+    def verify_filter_options!(options)
+      required_options = { columnname: "--column-name",
+                           operator: "--operator",
+                           columnvalue: "--column-value" }
+      missing_options = {}
+      required_options.each do |key, value|
+        missing_options[key] = value if !options[key] || options[key].strip.empty?
+      end
+
+      return if !options[:filter] || missing_options.empty?
+
+      warn_and_exit("missing filter options #{missing_options.values}", 1)
+    end
+
+    def warn_and_exit(message, exit_code)
+      warn message
+      warn "Run with `--help` or `-h` to see available options"
+      exit exit_code
+    end
+  end
+end