rails_den 2.0.0

data/lib/rails_den/import/discourse/finalizer.rb

@@ -0,0 +1,617 @@
+# frozen_string_literal: true
+
+require "marcel"
+require "set"
+
+module RailsDen
+  module Import
+    module Discourse
+      class Finalizer
+        Result = Data.define(
+          :administrators,
+          :moderators,
+          :uploads,
+          :post_attachments,
+          :avatars,
+          :unmatched_uploads,
+          :missing_files
+        )
+
+        USER_COLUMNS = %w[
+          source_id
+          email
+          admin
+          moderator
+          uploaded_avatar_id
+        ].freeze
+
+        POST_COLUMNS = %w[
+          source_id
+          body
+        ].freeze
+
+        UPLOAD_COLUMNS = %w[
+          source_id
+          original_filename
+          sha1
+          extension
+          url
+          access_control_post_id
+        ].freeze
+
+        BASE62_KEYS =
+          "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+        def initialize(
+          source_database:,
+          backup_root:,
+          user_class:,
+          out: $stdout
+        )
+          @source_database = source_database
+          @backup_root = backup_root
+          @user_class = user_class
+          @out = out
+        end
+
+        def call
+          users = source_users
+          user_map = target_user_map(users)
+          post_map = target_post_map
+
+          administrators,
+            moderators =
+            import_staff(
+              users,
+              user_map
+            )
+
+          upload_result =
+            import_uploads(
+              users: users,
+              user_map: user_map,
+              post_map: post_map
+            )
+
+          Result.new(
+            administrators: administrators,
+            moderators: moderators,
+            uploads: upload_result.fetch(:uploads),
+            post_attachments: upload_result.fetch(:post_attachments),
+            avatars: upload_result.fetch(:avatars),
+            unmatched_uploads: upload_result.fetch(:unmatched_uploads),
+            missing_files: upload_result.fetch(:missing_files)
+          )
+        end
+
+        private
+
+        def source_users
+          @source_users ||=
+            @source_database.rows(
+              <<~SQL,
+                SELECT
+                  u.id,
+                  e.email,
+                  u.admin,
+                  u.moderator,
+                  u.uploaded_avatar_id
+                FROM users u
+                LEFT JOIN user_emails e
+                  ON e.user_id = u.id
+                 AND e."primary" IN ('t', 'true', '1')
+                ORDER BY u.id::bigint
+              SQL
+              columns: USER_COLUMNS
+            )
+        end
+
+        def source_posts
+          @source_posts ||=
+            @source_database.rows(
+              <<~SQL,
+                SELECT
+                  p.id,
+                  p.raw
+                FROM posts p
+                INNER JOIN topics t
+                  ON t.id = p.topic_id
+                WHERE t.archetype = 'regular'
+                ORDER BY
+                  p.topic_id::bigint,
+                  p.post_number::integer,
+                  p.id::bigint
+              SQL
+              columns: POST_COLUMNS
+            )
+        end
+
+        def source_uploads
+          @source_uploads ||=
+            @source_database.rows(
+              <<~SQL,
+                SELECT
+                  id,
+                  original_filename,
+                  sha1,
+                  extension,
+                  url,
+                  access_control_post_id
+                FROM uploads
+                ORDER BY id::bigint
+              SQL
+              columns: UPLOAD_COLUMNS
+            )
+        end
+
+        def target_user_map(users)
+          used_emails = Set.new
+
+          users.each_with_object({}) do |row, map|
+            source_id =
+              integer!(
+                row.fetch("source_id"),
+                "user id"
+              )
+
+            email =
+              normalized_email(
+                row["email"],
+                source_id,
+                used_emails
+              )
+
+            user =
+              @user_class.find_by(
+                email_address: email
+              )
+
+            unless user
+              raise ImportError,
+                    "Could not find imported RailsDen user for " \
+                    "Discourse user #{source_id}."
+            end
+
+            map[source_id] = user
+          end
+        end
+
+        def target_post_map
+          source = source_posts
+          target = RailsDen::Post.order(:id).to_a
+
+          unless source.length == target.length
+            raise ImportError,
+                  "Imported post count does not match Discourse source " \
+                  "(#{target.length} RailsDen posts vs #{source.length} source posts)."
+          end
+
+          source
+            .zip(target)
+            .to_h do |row, post|
+              [
+                integer!(
+                  row.fetch("source_id"),
+                  "post id"
+                ),
+                post
+              ]
+            end
+        end
+
+        def import_staff(users, user_map)
+          @out.puts "  Staff roles..."
+
+
+          RailsDen::BuiltInRoleProvisioner.call
+          administrator_role =
+            RailsDen::Role.find_by!(
+              key: "administrator",
+              built_in: true
+            )
+
+          moderator_role =
+            RailsDen::Role.find_by!(
+              key: "moderator",
+              built_in: true
+            )
+
+          administrators = 0
+          moderators = 0
+
+          users.each do |row|
+            source_id =
+              integer!(
+                row.fetch("source_id"),
+                "user id"
+              )
+
+            user =
+              user_map.fetch(source_id)
+
+            if truthy?(row["admin"])
+              RailsDen::Administrator.find_or_create_by!(
+                user: user
+              )
+
+              RailsDen::RoleAssignment.find_or_create_by!(
+                user: user,
+                role: administrator_role
+              )
+
+              administrators += 1
+            end
+
+            if truthy?(row["moderator"])
+              RailsDen::RoleAssignment.find_or_create_by!(
+                user: user,
+                role: moderator_role
+              )
+
+              moderators += 1
+            end
+          end
+
+          [
+            administrators,
+            moderators
+          ]
+        end
+
+        def import_uploads(users:, user_map:, post_map:)
+          @out.puts "  Uploads and avatars..."
+
+          avatar_users =
+            avatar_users_by_upload(
+              users,
+              user_map
+            )
+
+          posts =
+            source_posts
+
+          imported_uploads = 0
+          post_attachments = 0
+          avatars = 0
+          unmatched_uploads = 0
+          missing_files = []
+
+          source_uploads.each do |upload|
+            upload_id =
+              integer!(
+                upload.fetch("source_id"),
+                "upload id"
+              )
+
+            target_posts =
+              referenced_posts(
+                upload,
+                posts,
+                post_map
+              )
+
+            target_users =
+              avatar_users.fetch(
+                upload_id,
+                []
+              )
+
+            if target_posts.empty? &&
+               target_users.empty?
+              unmatched_uploads += 1
+              next
+            end
+
+            path =
+              locate_upload_file(
+                upload
+              )
+
+            unless path
+              missing_files << upload_id
+              next
+            end
+
+            blob =
+              create_blob(
+                upload,
+                path
+              )
+
+            target_posts.uniq.each do |post|
+              post.attachments.attach(blob)
+              post_attachments += 1
+            end
+
+            target_users.uniq.each do |user|
+              profile =
+                RailsDen::MemberProfile.find_or_create_by!(
+                  profile_owner: user
+                )
+
+              profile.avatar.attach(blob)
+              avatars += 1
+            end
+
+            imported_uploads += 1
+          end
+
+          {
+            uploads: imported_uploads,
+            post_attachments: post_attachments,
+            avatars: avatars,
+            unmatched_uploads: unmatched_uploads,
+            missing_files: missing_files
+          }
+        end
+
+        def avatar_users_by_upload(users, user_map)
+          users.each_with_object(
+            Hash.new { |hash, key| hash[key] = [] }
+          ) do |row, map|
+            next if blank?(
+              row["uploaded_avatar_id"]
+            )
+
+            upload_id =
+              integer!(
+                row.fetch("uploaded_avatar_id"),
+                "uploaded avatar id"
+              )
+
+            source_user_id =
+              integer!(
+                row.fetch("source_id"),
+                "user id"
+              )
+
+            map[upload_id] <<
+              user_map.fetch(
+                source_user_id
+              )
+          end
+        end
+
+        def referenced_posts(upload, posts, post_map)
+          source_ids =
+            posts.filter_map do |post_row|
+              next unless upload_referenced_in?(
+                post_row["body"],
+                upload
+              )
+
+              integer!(
+                post_row.fetch("source_id"),
+                "post id"
+              )
+            end
+
+          unless blank?(
+            upload["access_control_post_id"]
+          )
+            source_ids <<
+              integer!(
+                upload.fetch("access_control_post_id"),
+                "access-control post id"
+              )
+          end
+
+          source_ids
+            .uniq
+            .filter_map do |source_post_id|
+              post_map[source_post_id]
+            end
+        end
+
+        def upload_referenced_in?(body, upload)
+          value =
+            body.to_s
+
+          return false if value.empty?
+
+          sha1 =
+            upload["sha1"]
+              .to_s
+              .downcase
+
+          url =
+            upload["url"]
+              .to_s
+
+          short =
+            if sha1.match?(/\A[0-9a-f]{40}\z/)
+              base62_encode(
+                sha1.to_i(16)
+              )
+            end
+
+          return true if !sha1.empty? &&
+                         value.include?(sha1)
+
+          return true if !url.empty? &&
+                         value.include?(url)
+
+          return false if short.nil?
+
+          value.include?(
+            "upload://#{short}"
+          ) ||
+            value.include?(
+              "/uploads/short-url/#{short}"
+            )
+        end
+
+        def locate_upload_file(upload)
+          url =
+            upload["url"]
+              .to_s
+
+          unless url.empty?
+            relative_url =
+              url
+                .sub(
+                  %r{\Ahttps?://[^/]+},
+                  ""
+                )
+                .sub(
+                  %r{\A//[^/]+},
+                  ""
+                )
+                .delete_prefix("/")
+
+            direct =
+              File.join(
+                @backup_root,
+                relative_url
+              )
+
+            return direct if File.file?(direct)
+          end
+
+          sha1 =
+            upload["sha1"]
+              .to_s
+              .downcase
+
+          return if sha1.empty?
+
+          candidates =
+            Dir
+              .glob(
+                File.join(
+                  @backup_root,
+                  "**",
+                  "#{sha1}*"
+                )
+              )
+              .select do |path|
+                File.file?(path)
+              end
+
+          candidates.find do |path|
+            path.include?(
+              "#{File::SEPARATOR}original#{File::SEPARATOR}"
+            )
+          end ||
+            candidates.first
+        end
+
+        def create_blob(upload, path)
+          filename =
+            File.basename(
+              upload["original_filename"]
+                .to_s
+                .presence ||
+              path
+            )
+
+          File.open(
+            path,
+            "rb"
+          ) do |io|
+            content_type =
+              Marcel::MimeType.for(
+                io,
+                name: filename
+              )
+
+            io.rewind
+
+            ActiveStorage::Blob.create_and_upload!(
+              io: io,
+              filename: filename,
+              content_type: content_type
+            )
+          end
+        end
+
+        def base62_encode(number)
+          return "0" if number.zero?
+
+          value = +""
+
+          while number.positive?
+            value.prepend(
+              BASE62_KEYS[
+                number % 62
+              ]
+            )
+
+            number /= 62
+          end
+
+          value
+        end
+
+        def normalized_email(
+          source_email,
+          source_id,
+          used_emails
+        )
+          candidate =
+            source_email
+              .to_s
+              .strip
+              .downcase
+
+          if candidate.empty? ||
+             used_emails.include?(candidate)
+            candidate =
+              "discourse-user-#{source_id.abs}@invalid.railsden"
+          end
+
+          counter = 2
+          base_candidate = candidate
+
+          while used_emails.include?(
+            candidate
+          )
+            local,
+              domain =
+              base_candidate.split(
+                "@",
+                2
+              )
+
+            candidate =
+              "#{local}-#{counter}@#{domain || 'invalid.railsden'}"
+
+            counter += 1
+          end
+
+          used_emails << candidate
+
+          candidate
+        end
+
+        def integer!(value, label)
+          Integer(
+            value.to_s,
+            10
+          )
+        rescue ArgumentError
+          raise ImportError,
+                "Invalid #{label}: #{value.inspect}"
+        end
+
+        def truthy?(value)
+          %w[
+            1
+            t
+            true
+            yes
+          ].include?(
+            value
+              .to_s
+              .downcase
+          )
+        end
+
+        def blank?(value)
+          value.nil? ||
+            value.to_s.empty?
+        end
+      end
+    end
+  end
+end