rails_den 2.0.0

data/lib/rails_den/import/discourse/transformer.rb

@@ -0,0 +1,940 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "set"
+
+module RailsDen
+  module Import
+    module Discourse
+      class Transformer
+        Result = Data.define(
+          :users,
+          :categories,
+          :boards,
+          :topics,
+          :posts,
+          :renamed_users,
+          :discourse_admins,
+          :discourse_moderators
+        )
+
+        USER_COLUMNS = %w[
+          source_id
+          username
+          email
+          created_at
+          updated_at
+          admin
+          moderator
+        ].freeze
+
+        CATEGORY_COLUMNS = %w[
+          source_id
+          name
+          description
+          position
+          parent_category_id
+          created_at
+          updated_at
+        ].freeze
+
+        TOPIC_COLUMNS = %w[
+          source_id
+          title
+          source_user_id
+          source_category_id
+          closed
+          pinned_at
+          created_at
+          updated_at
+        ].freeze
+
+        POST_COLUMNS = %w[
+          source_id
+          source_user_id
+          source_topic_id
+          post_number
+          reply_to_post_number
+          body
+          created_at
+          updated_at
+        ].freeze
+
+        def initialize(source_database:, out: $stdout)
+          @source_database = source_database
+          @out = out
+
+          @user_map = {}
+          @category_map = {}
+          @board_map = {}
+          @topic_map = {}
+          @post_map = {}
+          @post_number_map = {}
+
+          @renamed_users = []
+          @discourse_admins = []
+          @discourse_moderators = []
+
+          @used_usernames = Set.new
+          @used_category_slugs = Set.new
+          @used_board_slugs = Hash.new { |hash, key| hash[key] = Set.new }
+          @used_topic_slugs = Hash.new { |hash, key| hash[key] = Set.new }
+        end
+
+        def call
+          ensure_supported_target!
+          ensure_empty_target!
+
+          @out.puts
+          @out.puts "Importing RailsDen records..."
+
+          ActiveRecord::Base.transaction do
+            import_users
+            import_taxonomy
+            import_topics
+            import_posts
+            connect_replies
+            assign_opening_posts
+          end
+
+          Result.new(
+            users: @user_map.size,
+            categories: @category_map.values.uniq.size,
+            boards: @board_map.values.uniq.size,
+            topics: @topic_map.size,
+            posts: @post_map.size,
+            renamed_users: @renamed_users,
+            discourse_admins: @discourse_admins.size,
+            discourse_moderators: @discourse_moderators.size
+          )
+        end
+
+        private
+
+        def user_class
+          @user_class ||=
+            RailsDen
+              .config
+              .user_class
+              .constantize
+        end
+
+        def polymorphic_user_type
+          user_class.polymorphic_name
+        end
+
+        def ensure_supported_target!
+          required_columns = %w[
+            username
+            email_address
+            password_digest
+          ]
+
+          missing =
+            required_columns -
+            user_class.column_names
+
+          return if missing.empty?
+
+          raise ImportError,
+                "The default Discourse importer requires a standalone " \
+                "RailsDen user model with these columns: " \
+                "#{required_columns.join(', ')}. Missing: #{missing.join(', ')}."
+        end
+
+        def ensure_empty_target!
+          populated = []
+
+          populated << user_class.name if user_class.exists?
+          populated << RailsDen::Category.name if RailsDen::Category.exists?
+          populated << RailsDen::Board.name if RailsDen::Board.exists?
+          populated << RailsDen::Topic.name if RailsDen::Topic.exists?
+          populated << RailsDen::Post.name if RailsDen::Post.exists?
+
+          return if populated.empty?
+
+          raise ImportError,
+                "The default Discourse import must run against a new, empty " \
+                "standalone RailsDen application. Existing records were found in: " \
+                "#{populated.join(', ')}."
+        end
+
+        def import_users
+          @out.puts "  Users..."
+
+          rows =
+            @source_database.rows(
+              <<~SQL,
+                SELECT
+                  u.id,
+                  u.username,
+                  e.email,
+                  u.created_at,
+                  u.updated_at,
+                  u.admin,
+                  u.moderator
+                FROM users u
+                LEFT JOIN user_emails e
+                  ON e.user_id = u.id
+                 AND e."primary" IN ('t', 'true', '1')
+                ORDER BY u.id::bigint
+              SQL
+              columns: USER_COLUMNS
+            )
+
+          used_emails = Set.new
+
+          rows.each do |row|
+            source_id = integer!(row.fetch("source_id"), "user id")
+            source_username = row.fetch("username").to_s
+
+            username =
+              normalized_username(
+                source_username,
+                source_id
+              )
+
+            email =
+              normalized_email(
+                row["email"],
+                source_id,
+                used_emails
+              )
+
+            password =
+              SecureRandom.urlsafe_base64(36)
+
+            created_at =
+              parsed_time(row["created_at"])
+
+            updated_at =
+              parsed_time(row["updated_at"]) ||
+              created_at
+
+            user =
+              user_class.create!(
+                username: username,
+                email_address: email,
+                password: password,
+                password_confirmation: password,
+                created_at: created_at,
+                updated_at: updated_at
+              )
+
+            @user_map[source_id] = user.id
+
+            if username != source_username.downcase
+              @renamed_users << [
+                source_username,
+                username
+              ]
+            end
+
+            if truthy?(row["admin"])
+              @discourse_admins << user.id
+            end
+
+            if truthy?(row["moderator"])
+              @discourse_moderators << user.id
+            end
+          end
+        end
+
+        def import_taxonomy
+          @out.puts "  Categories and boards..."
+
+          rows =
+            @source_database.rows(
+              <<~SQL,
+                SELECT
+                  id,
+                  name,
+                  description,
+                  position,
+                  parent_category_id,
+                  created_at,
+                  updated_at
+                FROM categories
+                ORDER BY
+                  COALESCE(NULLIF(position, '')::integer, 0),
+                  id::bigint
+              SQL
+              columns: CATEGORY_COLUMNS
+            )
+
+          source_categories =
+            rows.index_by do |row|
+              integer!(row.fetch("source_id"), "category id")
+            end
+
+          roots =
+            rows.select do |row|
+              blank?(row["parent_category_id"])
+            end
+
+          roots.each do |row|
+            source_id =
+              integer!(row.fetch("source_id"), "category id")
+
+            category_id =
+              insert_category(row)
+
+            @category_map[source_id] = category_id
+          end
+
+          rows.each do |row|
+            source_id =
+              integer!(row.fetch("source_id"), "category id")
+
+            root_source_id =
+              root_category_id(
+                source_id,
+                source_categories
+              )
+
+            rails_den_category_id =
+              @category_map.fetch(root_source_id)
+
+            board_title =
+              board_title_for(
+                source_id,
+                root_source_id,
+                source_categories
+              )
+
+            board_id =
+              insert_board(
+                row,
+                category_id: rails_den_category_id,
+                title: board_title
+              )
+
+            @board_map[source_id] = board_id
+          end
+        end
+
+        def import_topics
+          @out.puts "  Topics..."
+
+          rows =
+            @source_database.rows(
+              <<~SQL,
+                SELECT
+                  id,
+                  title,
+                  user_id,
+                  category_id,
+                  closed,
+                  pinned_at,
+                  created_at,
+                  updated_at
+                FROM topics
+                WHERE archetype = 'regular'
+                ORDER BY id::bigint
+              SQL
+              columns: TOPIC_COLUMNS
+            )
+
+          rows.each do |row|
+            source_id =
+              integer!(row.fetch("source_id"), "topic id")
+
+            board_id =
+              board_for_topic(row)
+
+            author_id =
+              target_user_id!(
+                row.fetch("source_user_id"),
+                "topic #{source_id}"
+              )
+
+            title =
+              row.fetch("title").to_s.presence ||
+              "Imported topic #{source_id}"
+
+            slug =
+              unique_slug(
+                title,
+                @used_topic_slugs[board_id],
+                "topic-#{source_id}"
+              )
+
+            created_at =
+              parsed_time(row["created_at"])
+
+            updated_at =
+              parsed_time(row["updated_at"]) ||
+              created_at
+
+            pinned_at =
+              parsed_time(row["pinned_at"])
+
+            result =
+              RailsDen::Topic.insert!(
+                {
+                  board_id: board_id,
+                  author_type: polymorphic_user_type,
+                  author_id: author_id,
+                  title: title,
+                  slug: slug,
+                  locked: truthy?(row["closed"]),
+                  pinned: pinned_at.present?,
+                  pinned_at: pinned_at,
+                  pinned_by_type: nil,
+                  pinned_by_id: nil,
+                  deleted_at: nil,
+                  deleted_by_type: nil,
+                  deleted_by_id: nil,
+                  merged_into_id: nil,
+                  opening_post_id: nil,
+                  created_at: created_at,
+                  updated_at: updated_at
+                },
+                returning: %w[id],
+                record_timestamps: false
+              )
+
+            @topic_map[source_id] =
+              result.rows.first.first
+          end
+        end
+
+        def import_posts
+          @out.puts "  Posts..."
+
+          rows =
+            source_post_rows
+
+          rows.each do |row|
+            source_id =
+              integer!(row.fetch("source_id"), "post id")
+
+            source_topic_id =
+              integer!(
+                row.fetch("source_topic_id"),
+                "post topic id"
+              )
+
+            target_topic_id =
+              @topic_map[source_topic_id]
+
+            next unless target_topic_id
+
+            author_id =
+              target_user_id!(
+                row.fetch("source_user_id"),
+                "post #{source_id}"
+              )
+
+            created_at =
+              parsed_time(row["created_at"])
+
+            updated_at =
+              parsed_time(row["updated_at"]) ||
+              created_at
+
+            result =
+              RailsDen::Post.insert!(
+                {
+                  topic_id: target_topic_id,
+                  author_type: polymorphic_user_type,
+                  author_id: author_id,
+                  body: row["body"].to_s,
+                  created_at: created_at,
+                  updated_at: updated_at,
+                  deleted_at: nil,
+                  deleted_by_id: nil,
+                  deleted_by_type: nil,
+                  reply_to_id: nil,
+                  quoted_post_id: nil
+                },
+                returning: %w[id],
+                record_timestamps: false
+              )
+
+            target_post_id =
+              result.rows.first.first
+
+            @post_map[source_id] =
+              target_post_id
+
+            post_number =
+              integer!(
+                row.fetch("post_number"),
+                "post number"
+              )
+
+            @post_number_map[
+              [source_topic_id, post_number]
+            ] = target_post_id
+          end
+        end
+
+        def connect_replies
+          @out.puts "  Reply relationships..."
+
+          source_post_rows.each do |row|
+            next if blank?(row["reply_to_post_number"])
+
+            source_post_id =
+              integer!(row.fetch("source_id"), "post id")
+
+            source_topic_id =
+              integer!(
+                row.fetch("source_topic_id"),
+                "post topic id"
+              )
+
+            reply_number =
+              integer!(
+                row.fetch("reply_to_post_number"),
+                "reply post number"
+              )
+
+            target_post_id =
+              @post_map[source_post_id]
+
+            reply_target_id =
+              @post_number_map[
+                [source_topic_id, reply_number]
+              ]
+
+            next unless target_post_id && reply_target_id
+
+            RailsDen::Post
+              .where(id: target_post_id)
+              .update_all(
+                reply_to_id: reply_target_id
+              )
+          end
+        end
+
+        def assign_opening_posts
+          @out.puts "  Opening posts..."
+
+          @topic_map.each do |source_topic_id, target_topic_id|
+            opening_post_id =
+              @post_number_map[
+                [source_topic_id, 1]
+              ]
+
+            unless opening_post_id
+              candidates =
+                @post_number_map.select do |key, _value|
+                  key.first == source_topic_id
+                end
+
+              opening_post_id =
+                candidates
+                  .min_by { |key, _value| key.last }
+                  &.last
+            end
+
+            next unless opening_post_id
+
+            RailsDen::Topic
+              .where(id: target_topic_id)
+              .update_all(
+                opening_post_id: opening_post_id
+              )
+          end
+        end
+
+        def source_post_rows
+          @source_post_rows ||=
+            @source_database.rows(
+              <<~SQL,
+                SELECT
+                  p.id,
+                  p.user_id,
+                  p.topic_id,
+                  p.post_number,
+                  p.reply_to_post_number,
+                  p.raw,
+                  p.created_at,
+                  p.updated_at
+                FROM posts p
+                INNER JOIN topics t
+                  ON t.id = p.topic_id
+                WHERE t.archetype = 'regular'
+                ORDER BY
+                  p.topic_id::bigint,
+                  p.post_number::integer,
+                  p.id::bigint
+              SQL
+              columns: POST_COLUMNS
+            )
+        end
+
+        def insert_category(row)
+          source_id =
+            integer!(row.fetch("source_id"), "category id")
+
+          title =
+            row.fetch("name").to_s.presence ||
+            "Imported category #{source_id}"
+
+          slug =
+            unique_slug(
+              title,
+              @used_category_slugs,
+              "category-#{source_id}"
+            )
+
+          created_at =
+            parsed_time(row["created_at"])
+
+          updated_at =
+            parsed_time(row["updated_at"]) ||
+            created_at
+
+          result =
+            RailsDen::Category.insert!(
+              {
+                title: title,
+                slug: slug,
+                description: row["description"].presence,
+                position: integer_or_zero(row["position"]),
+                enabled: true,
+                visibility: "public",
+                created_at: created_at,
+                updated_at: updated_at
+              },
+              returning: %w[id],
+              record_timestamps: false
+            )
+
+          result.rows.first.first
+        end
+
+        def insert_board(row, category_id:, title:)
+          source_id =
+            integer!(row.fetch("source_id"), "category id")
+
+          slug =
+            unique_slug(
+              title,
+              @used_board_slugs[category_id],
+              "board-#{source_id}"
+            )
+
+          created_at =
+            parsed_time(row["created_at"])
+
+          updated_at =
+            parsed_time(row["updated_at"]) ||
+            created_at
+
+          result =
+            RailsDen::Board.insert!(
+              {
+                category_id: category_id,
+                title: title,
+                slug: slug,
+                description: row["description"].presence,
+                position: integer_or_zero(row["position"]),
+                enabled: true,
+                visibility: "public",
+                icon: nil,
+                created_at: created_at,
+                updated_at: updated_at
+              },
+              returning: %w[id],
+              record_timestamps: false
+            )
+
+          result.rows.first.first
+        end
+
+        def root_category_id(source_id, categories)
+          seen = Set.new
+          current_id = source_id
+
+          loop do
+            if seen.include?(current_id)
+              raise ImportError,
+                    "Discourse category hierarchy contains a cycle at " \
+                    "category #{current_id}."
+            end
+
+            seen << current_id
+
+            row =
+              categories[current_id]
+
+            unless row
+              raise ImportError,
+                    "Discourse category #{source_id} references missing " \
+                    "category #{current_id}."
+            end
+
+            parent_id =
+              row["parent_category_id"]
+
+            return current_id if blank?(parent_id)
+
+            current_id =
+              integer!(
+                parent_id,
+                "parent category id"
+              )
+          end
+        end
+
+        def board_title_for(source_id, root_source_id, categories)
+          return "General" if source_id == root_source_id
+
+          names = []
+          current_id = source_id
+
+          while current_id != root_source_id
+            row =
+              categories.fetch(current_id)
+
+            names.unshift(
+              row.fetch("name").to_s.presence ||
+              "Category #{current_id}"
+            )
+
+            current_id =
+              integer!(
+                row.fetch("parent_category_id"),
+                "parent category id"
+              )
+          end
+
+          names.join(" / ")
+        end
+
+        def board_for_topic(row)
+          source_category_id =
+            row["source_category_id"]
+
+          unless blank?(source_category_id)
+            board =
+              @board_map[
+                integer!(
+                  source_category_id,
+                  "topic category id"
+                )
+              ]
+
+            return board if board
+          end
+
+          fallback_board
+        end
+
+        def fallback_board
+          return @fallback_board if @fallback_board
+
+          now = Time.current
+
+          category_slug =
+            unique_slug(
+              "General",
+              @used_category_slugs,
+              "general"
+            )
+
+          category_result =
+            RailsDen::Category.insert!(
+              {
+                title: "General",
+                slug: category_slug,
+                description: nil,
+                position: 0,
+                enabled: true,
+                visibility: "public",
+                created_at: now,
+                updated_at: now
+              },
+              returning: %w[id],
+              record_timestamps: false
+            )
+
+          category_id =
+            category_result.rows.first.first
+
+          board_slug =
+            unique_slug(
+              "General Discussion",
+              @used_board_slugs[category_id],
+              "general-discussion"
+            )
+
+          board_result =
+            RailsDen::Board.insert!(
+              {
+                category_id: category_id,
+                title: "General Discussion",
+                slug: board_slug,
+                description: nil,
+                position: 0,
+                enabled: true,
+                visibility: "public",
+                icon: nil,
+                created_at: now,
+                updated_at: now
+              },
+              returning: %w[id],
+              record_timestamps: false
+            )
+
+          @fallback_board =
+            board_result.rows.first.first
+        end
+
+        def normalized_username(source_username, source_id)
+          base =
+            source_username
+              .to_s
+              .downcase
+              .gsub(/[^a-z0-9_]+/, "_")
+              .gsub(/\A_+|_+\z/, "")
+
+          base =
+            "user_#{source_id.abs}" if base.blank?
+
+          if base.length < 3
+            base =
+              "#{base}_#{source_id.abs}"
+          end
+
+          base =
+            base[0, 30]
+
+          candidate = base
+          counter = 2
+
+          while @used_usernames.include?(candidate)
+            suffix = "_#{counter}"
+
+            candidate =
+              "#{base[0, 30 - suffix.length]}#{suffix}"
+
+            counter += 1
+          end
+
+          @used_usernames << candidate
+
+          candidate
+        end
+
+        def normalized_email(source_email, source_id, used_emails)
+          candidate =
+            source_email
+              .to_s
+              .strip
+              .downcase
+
+          if candidate.blank? ||
+             used_emails.include?(candidate)
+            candidate =
+              "discourse-user-#{source_id.abs}@invalid.railsden"
+          end
+
+          counter = 2
+          base_candidate = candidate
+
+          while used_emails.include?(candidate)
+            local, domain =
+              base_candidate.split("@", 2)
+
+            candidate =
+              "#{local}-#{counter}@#{domain || 'invalid.railsden'}"
+
+            counter += 1
+          end
+
+          used_emails << candidate
+
+          candidate
+        end
+
+        def target_user_id!(source_user_id, context)
+          source_id =
+            integer!(
+              source_user_id,
+              "#{context} author id"
+            )
+
+          target_id =
+            @user_map[source_id]
+
+          return target_id if target_id
+
+          raise ImportError,
+                "#{context} references Discourse user #{source_id}, " \
+                "but that user was not present in the backup."
+        end
+
+        def unique_slug(title, used, fallback)
+          base =
+            title
+              .to_s
+              .parameterize
+
+          base =
+            fallback if base.blank?
+
+          candidate = base
+          counter = 2
+
+          while used.include?(candidate)
+            candidate =
+              "#{base}-#{counter}"
+
+            counter += 1
+          end
+
+          used << candidate
+
+          candidate
+        end
+
+        def parsed_time(value)
+          return Time.current if blank?(value)
+
+          Time.zone.parse(value.to_s)
+        rescue ArgumentError
+          raise ImportError,
+                "Could not parse Discourse timestamp: #{value.inspect}"
+        end
+
+        def integer!(value, label)
+          Integer(value.to_s, 10)
+        rescue ArgumentError
+          raise ImportError,
+                "Invalid #{label}: #{value.inspect}"
+        end
+
+        def integer_or_zero(value)
+          return 0 if blank?(value)
+
+          Integer(value.to_s, 10)
+        rescue ArgumentError
+          0
+        end
+
+        def truthy?(value)
+          %w[
+            1
+            t
+            true
+            yes
+          ].include?(
+            value
+              .to_s
+              .downcase
+          )
+        end
+
+        def blank?(value)
+          value.nil? ||
+            value.to_s.empty?
+        end
+      end
+    end
+  end
+end