rails_den 2.0.0

data/lib/rails_den/import/discourse/importer.rb

@@ -0,0 +1,266 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "rails_den/import/discourse"
+require "rails_den/import/discourse/backup"
+require "rails_den/import/discourse/finalizer"
+require "rails_den/import/discourse/source_database"
+require "rails_den/import/discourse/transformer"
+
+module RailsDen
+  module Import
+    module Discourse
+      class Importer
+        REQUIRED_SOURCE_TABLES = %w[
+          users
+          user_emails
+          categories
+          topics
+          posts
+          uploads
+        ].freeze
+
+        SOURCE_COUNT_TABLES = %w[
+          users
+          categories
+          topics
+          posts
+          uploads
+        ].freeze
+
+        def initialize(
+          backup_path:,
+          out: $stdout
+        )
+          @backup_path =
+            File.expand_path(
+              backup_path.to_s
+            )
+
+          @out = out
+        end
+
+        def call
+          configuration =
+            target_postgresql_configuration
+
+          @out.puts "RailsDen Discourse Import"
+          @out.puts "Backup: #{@backup_path}"
+          @out.puts
+
+          transformation_result = nil
+          finalizer_result = nil
+
+          Backup
+            .new(path: @backup_path)
+            .with_extracted do |backup|
+              @out.puts "Backup validated."
+              @out.puts(
+                "Database dump: #{File.basename(backup.dump_path)}"
+              )
+              @out.puts
+
+              transformation_result,
+                finalizer_result =
+                import_from_source(
+                  backup: backup,
+                  configuration: configuration
+                )
+            end
+
+          print_completion_summary(
+            transformation_result,
+            finalizer_result
+          )
+
+          true
+        end
+
+        private
+
+        def import_from_source(
+          backup:,
+          configuration:
+        )
+          source_database =
+            SourceDatabase.new(
+              target_configuration: configuration,
+              out: @out
+            )
+
+          begin
+            source_database.create!
+            source_database.restore!(
+              backup.dump_path
+            )
+
+            source_database.verify_tables!(
+              REQUIRED_SOURCE_TABLES
+            )
+
+            print_source_summary(
+              source_database.counts(
+                SOURCE_COUNT_TABLES
+              )
+            )
+
+            transformation_result = nil
+            finalizer_result = nil
+
+            ActiveRecord::Base.transaction do
+              transformation_result =
+                Transformer.new(
+                  source_database: source_database,
+                  out: @out
+                ).call
+
+              finalizer_result =
+                Finalizer.new(
+                  source_database: source_database,
+                  backup_root: backup.root_path,
+                  user_class: RailsDen
+                    .config
+                    .user_class
+                    .constantize,
+                  out: @out
+                ).call
+            end
+
+            [
+              transformation_result,
+              finalizer_result
+            ]
+          ensure
+            source_database.drop!
+          end
+        end
+
+        def target_postgresql_configuration
+          configuration =
+            ActiveRecord::Base
+              .connection_db_config
+              .configuration_hash
+              .symbolize_keys
+
+          adapter =
+            configuration[
+              :adapter
+            ].to_s
+
+          unless adapter.start_with?(
+            "postgres"
+          )
+            raise ImportError,
+                  "Discourse import currently requires a PostgreSQL " \
+                  "RailsDen application database."
+          end
+
+          if configuration[
+               :database
+             ].blank?
+            raise ImportError,
+                  "RailsDen's PostgreSQL database name could not be determined."
+          end
+
+          configuration
+        end
+
+        def print_source_summary(counts)
+          @out.puts
+          @out.puts "Discourse source verified:"
+
+          SOURCE_COUNT_TABLES.each do |table_name|
+            value =
+              counts.fetch(
+                table_name
+              )
+
+            label =
+              table_name
+                .tr("_", " ")
+                .capitalize
+
+            if value.nil?
+              @out.puts(
+                "  #{label}: not present"
+              )
+            else
+              @out.puts(
+                "  #{label}: #{value}"
+              )
+            end
+          end
+        end
+
+        def print_completion_summary(
+          result,
+          finalizer
+        )
+          @out.puts
+          @out.puts "RailsDen import complete:"
+          @out.puts "  Users: #{result.users}"
+          @out.puts "  Categories: #{result.categories}"
+          @out.puts "  Boards: #{result.boards}"
+          @out.puts "  Topics: #{result.topics}"
+          @out.puts "  Posts: #{result.posts}"
+
+          @out.puts(
+            "  Administrators: #{finalizer.administrators}"
+          )
+
+          @out.puts(
+            "  Moderators: #{finalizer.moderators}"
+          )
+
+          @out.puts(
+            "  Uploads imported: #{finalizer.uploads}"
+          )
+
+          @out.puts(
+            "  Post attachments: #{finalizer.post_attachments}"
+          )
+
+          @out.puts(
+            "  Profile avatars: #{finalizer.avatars}"
+          )
+
+          if result.renamed_users.any?
+            @out.puts
+            @out.puts(
+              "Usernames adjusted for RailsDen compatibility:"
+            )
+
+            result
+              .renamed_users
+              .each do |source, target|
+                @out.puts(
+                  "  #{source} -> #{target}"
+                )
+              end
+          end
+
+          if finalizer.unmatched_uploads.positive?
+            @out.puts
+            @out.puts(
+              "Uploads not associated with an imported post or user avatar: " \
+              "#{finalizer.unmatched_uploads}"
+            )
+          end
+
+          if finalizer.missing_files.any?
+            @out.puts
+            @out.puts(
+              "WARNING: Backup files were missing for Discourse upload IDs: " \
+              "#{finalizer.missing_files.join(', ')}"
+            )
+          end
+
+          @out.puts
+          @out.puts(
+            "Imported users must use RailsDen password reset before first sign-in."
+          )
+        end
+      end
+    end
+  end
+end

data/lib/rails_den/import/discourse/source_database.rb

@@ -0,0 +1,320 @@
+# frozen_string_literal: true
+
+require "csv"
+require "fileutils"
+require "open3"
+require "rails_den/import/discourse"
+require "securerandom"
+
+module RailsDen
+  module Import
+    module Discourse
+      class SourceDatabase
+        STAGING_TABLES = %w[
+          users
+          user_emails
+          categories
+          topics
+          posts
+          uploads
+        ].freeze
+
+        COPY_HEADER =
+          /\ACOPY public\.([a-zA-Z0-9_]+) \((.+)\) FROM stdin;\z/
+
+        attr_reader :database_name
+
+        def initialize(target_configuration:, out: $stdout)
+          @target_configuration = target_configuration
+          @out = out
+          @database_name =
+            "rails_den_discourse_#{Process.pid}_#{SecureRandom.hex(6)}"
+          @created = false
+        end
+
+        def create!
+          ensure_postgresql_tools!
+
+          @out.puts "Creating temporary Discourse source database..."
+
+          run_command!(
+            "createdb",
+            "--maintenance-db=#{target_database}",
+            "--encoding=UTF8",
+            "--template=template0",
+            database_name
+          )
+
+          @created = true
+        end
+
+        def restore!(dump_path)
+          @out.puts "Extracting required Discourse data..."
+
+          staging_dump_path =
+            File.join(
+              File.dirname(dump_path),
+              "rails-den-staging-dump.sql"
+            )
+
+          extract_staging_dump!(
+            dump_path,
+            staging_dump_path
+          )
+
+          @out.puts "Loading Discourse staging data..."
+
+          run_command!(
+            "psql",
+            "--quiet",
+            "--dbname=#{database_name}",
+            "--single-transaction",
+            "--variable=ON_ERROR_STOP=1",
+            "--file=#{staging_dump_path}"
+          )
+        ensure
+          FileUtils.rm_f(staging_dump_path) if staging_dump_path
+        end
+
+        def verify_tables!(required_tables)
+          missing_tables =
+            required_tables.reject do |table_name|
+              table_exists?(table_name)
+            end
+
+          return if missing_tables.empty?
+
+          raise ImportError,
+                "Backup is missing required Discourse tables: " \
+                "#{missing_tables.join(', ')}"
+        end
+
+        def counts(table_names)
+          table_names.to_h do |table_name|
+            count =
+              if table_exists?(table_name)
+                query_scalar(
+                  "SELECT COUNT(*) FROM #{quote_identifier(table_name)}"
+                ).to_i
+              end
+
+            [table_name, count]
+          end
+        end
+
+        def rows(sql, columns:)
+          stdout =
+            run_command!(
+              "psql",
+              "--dbname=#{database_name}",
+              "--csv",
+              "--tuples-only",
+              "--command=#{sql}"
+            )
+
+          CSV.parse(stdout).map do |values|
+            columns.zip(values).to_h
+          end
+        end
+
+        def query_scalar(sql)
+          stdout =
+            run_command!(
+              "psql",
+              "--dbname=#{database_name}",
+              "--tuples-only",
+              "--no-align",
+              "--command=#{sql}"
+            )
+
+          stdout.to_s.strip
+        end
+
+        def drop!
+          return true unless @created
+
+          @out.puts "Removing temporary Discourse source database..."
+
+          _stdout, stderr, status =
+            Open3.capture3(
+              postgres_environment,
+              "dropdb",
+              "--maintenance-db=#{target_database}",
+              "--if-exists",
+              database_name
+            )
+
+          @created = false if status.success?
+
+          unless status.success?
+            @out.puts(
+              "WARNING: Could not remove temporary database " \
+              "#{database_name}: #{stderr.to_s.strip}"
+            )
+          end
+
+          status.success?
+        end
+
+        private
+
+        def target_database
+          @target_configuration.fetch(:database).to_s
+        end
+
+        def ensure_postgresql_tools!
+          %w[createdb dropdb psql].each do |command|
+            next if executable_available?(command)
+
+            raise ImportError,
+                  "Required PostgreSQL command is not available: #{command}"
+          end
+        end
+
+        def executable_available?(command)
+          _stdout, _stderr, status =
+            Open3.capture3(
+              "sh",
+              "-c",
+              "command -v \"$1\" >/dev/null 2>&1",
+              "rails-den-command-check",
+              command
+            )
+
+          status.success?
+        end
+
+        def postgres_environment
+          environment = {}
+
+          {
+            "PGHOST" => @target_configuration[:host],
+            "PGPORT" => @target_configuration[:port],
+            "PGUSER" => @target_configuration[:username],
+            "PGPASSWORD" => @target_configuration[:password],
+            "PGSSLMODE" => @target_configuration[:sslmode]
+          }.each do |name, value|
+            environment[name] = value.to_s if value.present?
+          end
+
+          environment
+        end
+
+        def run_command!(*command)
+          stdout, stderr, status =
+            Open3.capture3(
+              postgres_environment,
+              *command
+            )
+
+          return stdout if status.success?
+
+          detail =
+            stderr.to_s.strip.presence ||
+            stdout.to_s.strip.presence
+
+          detail ||=
+            "command exited with status #{status.exitstatus}"
+
+          raise ImportError,
+                "#{command.first} failed: #{detail}"
+        end
+
+        def table_exists?(table_name)
+          query_scalar(
+            "SELECT to_regclass('public.#{table_name}') IS NOT NULL"
+          ) == "t"
+        end
+
+        def extract_staging_dump!(source_path, destination_path)
+          found_tables = []
+
+          File.open(destination_path, "wb") do |destination|
+            copying_table = nil
+
+            File.foreach(source_path) do |line|
+              if copying_table
+                destination.write(line)
+
+                copying_table = nil if line.chomp == "\\."
+
+                next
+              end
+
+              match =
+                COPY_HEADER.match(
+                  line.chomp
+                )
+
+              next unless match
+
+              table_name = match[1]
+
+              next unless STAGING_TABLES.include?(table_name)
+
+              columns =
+                parse_copy_columns(
+                  match[2]
+                )
+
+              destination.puts(
+                staging_table_sql(
+                  table_name,
+                  columns
+                )
+              )
+
+              destination.write(line)
+
+              copying_table = table_name
+              found_tables << table_name
+            end
+          end
+
+          if found_tables.empty?
+            raise ImportError,
+                  "No supported Discourse data tables were found in the backup."
+          end
+        end
+
+        def parse_copy_columns(column_list)
+          column_list
+            .split(",")
+            .map(&:strip)
+            .map do |column_name|
+              unquote_identifier(
+                column_name
+              )
+            end
+        end
+
+        def staging_table_sql(table_name, columns)
+          column_definitions =
+            columns.map do |column_name|
+              "#{quote_identifier(column_name)} text"
+            end
+
+          "CREATE TABLE IF NOT EXISTS " \
+            "#{quote_identifier(table_name)} " \
+            "(#{column_definitions.join(', ')});"
+        end
+
+        def unquote_identifier(identifier)
+          value = identifier.to_s
+
+          return value unless value.start_with?('"') &&
+                              value.end_with?('"')
+
+          value
+            .delete_prefix('"')
+            .delete_suffix('"')
+            .gsub('""', '"')
+        end
+
+        def quote_identifier(identifier)
+          %Q{"#{identifier.to_s.gsub('"', '""')}"}
+        end
+      end
+    end
+  end
+end