data_migration_for_rails 0.1.1

data/app/controllers/data_migration/migration_steps_controller.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module DataMigration
+  class MigrationStepsController < ApplicationController
+    include DataMigration::PunditAuthorization
+
+    before_action :set_migration_plan
+    before_action :set_migration_step, only: %i[show edit update destroy]
+
+    def index
+      @migration_steps = policy_scope(MigrationStep).where(migration_plan_id: params[:migration_plan_id]).order(:sequence)
+    end
+
+    def show
+      authorize @migration_step
+    end
+
+    def new
+      @migration_step = @migration_plan.migration_steps.new
+      authorize @migration_step
+    end
+
+    def create
+      @migration_step = @migration_plan.migration_steps.new(migration_step_params)
+      authorize @migration_step
+
+      # Validate JSON fields before saving
+      validation_errors = validate_json_fields(@migration_step)
+      if validation_errors.any?
+        validation_errors.each { |error| @migration_step.errors.add(:base, error) }
+        render :new, status: :unprocessable_entity
+        return
+      end
+
+      if @migration_step.save
+        redirect_to "/data_migration/migration_plans/#{@migration_plan.id}",
+                    notice: 'Migration step was successfully created.'
+      else
+        render :new, status: :unprocessable_entity
+      end
+    end
+
+    def edit
+      authorize @migration_step
+    end
+
+    def update
+      authorize @migration_step
+
+      # Assign attributes first
+      @migration_step.assign_attributes(migration_step_params)
+
+      # Validate JSON fields before saving
+      validation_errors = validate_json_fields(@migration_step)
+      if validation_errors.any?
+        validation_errors.each { |error| @migration_step.errors.add(:base, error) }
+        render :edit, status: :unprocessable_entity
+        return
+      end
+
+      if @migration_step.save
+        redirect_to "/data_migration/migration_plans/#{@migration_plan.id}",
+                    notice: 'Migration step was successfully updated.'
+      else
+        render :edit, status: :unprocessable_entity
+      end
+    end
+
+    def destroy
+      authorize @migration_step
+
+      @migration_step.destroy
+      redirect_to "/data_migration/migration_plans/#{@migration_plan.id}",
+                  notice: 'Migration step was successfully destroyed.'
+    end
+
+    private
+
+    def set_migration_plan
+      @migration_plan = MigrationPlan.find(params[:migration_plan_id])
+    end
+
+    def set_migration_step
+      @migration_step = @migration_plan.migration_steps.find(params[:id])
+    end
+
+    def migration_step_params
+      params.require(:migration_step).permit(
+        :source_model_name, :sequence, :filter_query, :dependee_id, :migration_plan_id,
+        :dependee_attribute_mapping, :column_overrides, :association_overrides,
+        :attachment_export_mode, :attachment_fields
+      )
+    end
+
+    def validate_json_fields(step)
+      errors = []
+
+      # Validate column_overrides
+      if step.column_overrides.present?
+        begin
+          parsed = step.column_overrides.is_a?(String) ? JSON.parse(step.column_overrides) : step.column_overrides
+          errors << 'Column Overrides must be a JSON object/hash' unless parsed.is_a?(Hash)
+        rescue JSON::ParserError
+          errors << 'Column Overrides contains invalid JSON'
+        end
+      end
+
+      # Validate association_overrides
+      if step.association_overrides.present?
+        begin
+          parsed = step.association_overrides.is_a?(String) ? JSON.parse(step.association_overrides) : step.association_overrides
+          if parsed.is_a?(Hash)
+            # Validate structure: each value should be a hash with proper keys
+            parsed.each do |key, value|
+              unless value.is_a?(Hash)
+                errors << "Association Overrides: '#{key}' must be a hash"
+                next
+              end
+
+              # Check for polymorphic associations
+              if value['polymorphic'] == true
+                # Polymorphic requires type_column and lookup_attributes hash
+                if value['type_column'].blank?
+                  errors << "Association Overrides: '#{key}' (polymorphic) must have a 'type_column' key"
+                end
+
+                if value['lookup_attributes'].blank?
+                  errors << "Association Overrides: '#{key}' (polymorphic) must have 'lookup_attributes' key"
+                elsif !value['lookup_attributes'].is_a?(Hash)
+                  errors << "Association Overrides: '#{key}' (polymorphic) 'lookup_attributes' must be a hash with model types as keys"
+                end
+              else
+                # Non-polymorphic requires model and lookup_attributes
+                errors << "Association Overrides: '#{key}' must have a 'model' key" if value['model'].blank?
+
+                if value['lookup_attributes'].blank?
+                  errors << "Association Overrides: '#{key}' must have 'lookup_attributes' key"
+                elsif !value['lookup_attributes'].is_a?(Array)
+                  errors << "Association Overrides: '#{key}' 'lookup_attributes' must be an array"
+                end
+              end
+            end
+          else
+            errors << 'Association Overrides must be a JSON object/hash'
+          end
+        rescue JSON::ParserError
+          errors << 'Association Overrides contains invalid JSON'
+        end
+      end
+
+      # Validate dependee_attribute_mapping
+      if step.dependee_attribute_mapping.present?
+        begin
+          parsed = step.dependee_attribute_mapping.is_a?(String) ? JSON.parse(step.dependee_attribute_mapping) : step.dependee_attribute_mapping
+          errors << 'Dependee Attribute Mapping must be a JSON object/hash' unless parsed.is_a?(Hash)
+        rescue JSON::ParserError
+          errors << 'Dependee Attribute Mapping contains invalid JSON'
+        end
+      end
+
+      errors
+    end
+  end
+end

data/app/controllers/data_migration/users_controller.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module DataMigration
+  class UsersController < ApplicationController
+    include DataMigration::PunditAuthorization
+
+    before_action :set_user, only: %i[edit update destroy]
+
+    def index
+      authorize DataMigrationUser
+      @users = policy_scope(DataMigrationUser).ordered_by_email
+    end
+
+    def new
+      authorize DataMigrationUser
+      @user = DataMigrationUser.new
+    end
+
+    def create
+      authorize DataMigrationUser
+      @user = DataMigrationUser.new(user_params)
+
+      if @user.save
+        redirect_to users_path, notice: 'User created successfully.'
+      else
+        render :new, status: :unprocessable_entity
+      end
+    end
+
+    def edit
+      authorize @user
+    end
+
+    def update
+      authorize @user
+
+      # Remove password if blank
+      params_to_update = if user_params[:password].blank?
+                           user_params.except(:password, :password_confirmation)
+                         else
+                           user_params
+                         end
+
+      if @user.update(params_to_update)
+        redirect_to users_path, notice: 'User updated successfully.'
+      else
+        render :edit, status: :unprocessable_entity
+      end
+    end
+
+    def destroy
+      authorize @user
+
+      if @user.destroy
+        redirect_to users_path, notice: 'User deleted successfully.'
+      else
+        redirect_to users_path, alert: @user.errors.full_messages.join(', ')
+      end
+    end
+
+    private
+
+    def set_user
+      @user = DataMigrationUser.find(params[:id])
+    end
+
+    def user_params
+      params.require(:user).permit(:name, :email, :password, :password_confirmation, :role)
+    end
+  end
+end

data/app/controllers/users/sessions_controller.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Users
+  class SessionsController < Devise::SessionsController
+    # Skip CSRF verification for the new action to handle Devise FailureApp redirects
+    skip_before_action :verify_authenticity_token, only: [:new]
+
+    # GET /resource/sign_in
+    # def new
+    #   super
+    # end
+
+    # POST /resource/sign_in
+    # def create
+    #   super
+    # end
+
+    # DELETE /resource/sign_out
+    # def destroy
+    #   super
+    # end
+
+    # protected
+
+    # If you have extra params to permit, append them to the sanitizer.
+    # def configure_sign_in_params
+    #   devise_parameter_sanitizer.permit(:sign_in, keys: [:attribute])
+    # end
+  end
+end

data/app/helpers/data_migration/application_helper.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module DataMigration
+  module ApplicationHelper
+    def execution_status_color(status)
+      case status.to_sym
+      when :pending
+        'secondary'
+      when :running
+        'primary'
+      when :completed
+        'success'
+      when :failed
+        'danger'
+      else
+        'secondary'
+      end
+    end
+
+    def execution_type_icon(type)
+      type.to_sym == :export ? '📤' : '📥'
+    end
+  end
+end

data/app/jobs/application_job.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class ApplicationJob < ActiveJob::Base
+  # Automatically retry jobs that encountered a deadlock
+  # retry_on ActiveRecord::Deadlocked
+
+  # Most jobs are safe to ignore if the underlying records are no longer available
+  # discard_on ActiveJob::DeserializationError
+end

data/app/jobs/export_job.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class ExportJob < ApplicationJob
+  queue_as :default
+
+  def perform(execution_id)
+    execution = MigrationExecution.find(execution_id)
+    migration_plan = execution.migration_plan
+
+    Exports::GeneratorService.new(migration_plan, execution).call
+  rescue StandardError => e
+    Rails.logger.error("ExportJob failed for execution #{execution_id}: #{e.message}")
+    Rails.logger.error(e.backtrace.join("\n"))
+
+    # Only update if the service didn't already mark it as failed
+    if execution.reload.status != 'failed'
+      execution.update!(
+        status: :failed,
+        completed_at: Time.current,
+        error_log: e.full_message
+      )
+    end
+
+    # Don't re-raise - allow the job to complete so Sidekiq doesn't retry
+    # The error is already logged and saved to the execution record
+  end
+end

data/app/jobs/import_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class ImportJob < ApplicationJob
+  queue_as :default
+
+  def perform(execution_id)
+    execution = MigrationExecution.find(execution_id)
+    migration_plan = execution.migration_plan
+    uploaded_file_path = execution.file_path
+
+    Imports::ProcessorService.new(migration_plan, execution, uploaded_file_path).call
+  rescue StandardError => e
+    Rails.logger.error("ImportJob failed for execution #{execution_id}: #{e.message}")
+    Rails.logger.error(e.backtrace.join("\n"))
+
+    # Only update if the service didn't already mark it as failed
+    if execution.reload.status != 'failed'
+      execution.update!(
+        status: :failed,
+        completed_at: Time.current,
+        error_log: e.full_message
+      )
+    end
+
+    # Don't re-raise - allow the job to complete so Sidekiq doesn't retry
+    # The error is already logged and saved to the execution record
+  end
+end

data/app/mailers/application_mailer.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+class ApplicationMailer < ActionMailer::Base
+  default from: 'from@example.com'
+  layout 'mailer'
+end

data/app/models/application_record.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class ApplicationRecord < ActiveRecord::Base
+  primary_abstract_class
+end

data/app/models/data_migration_user.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+class DataMigrationUser < ApplicationRecord
+  self.table_name = 'data_migration_users'
+
+  # Devise modules
+  devise :database_authenticatable, :registerable,
+         :recoverable, :rememberable, :validatable
+
+  # Associations
+  has_many :migration_plans, dependent: :restrict_with_error
+  has_many :migration_executions, dependent: :restrict_with_error
+
+  # Validations
+  validates :name, presence: true
+
+  # Role enum
+  enum role: { viewer: 0, operator: 1, admin: 2 }
+
+  # Callbacks
+  after_initialize :set_default_role, if: :new_record?
+
+  # Scopes
+  scope :ordered_by_email, -> { order(:email) }
+  scope :admins, -> { where(role: :admin) }
+  scope :operators, -> { where(role: :operator) }
+  scope :viewers, -> { where(role: :viewer) }
+
+  # Permission methods
+  def can_execute_migrations?
+    operator? || admin?
+  end
+
+  def can_manage_users?
+    admin?
+  end
+
+  private
+
+  def set_default_role
+    self.role ||= :viewer
+  end
+end

data/app/models/migration_execution.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+class MigrationExecution < ApplicationRecord
+  # Associations
+  belongs_to :migration_plan
+  belongs_to :user, class_name: 'DataMigrationUser'
+  has_many :migration_records, dependent: :destroy
+
+  # Enums
+  enum execution_type: { export: 0, import: 1 }
+  enum status: { pending: 0, running: 1, completed: 2, failed: 3 }
+
+  # Validations
+  validates :execution_type, presence: true
+  validates :status, presence: true
+
+  # Callbacks to set defaults
+  after_initialize :set_defaults, if: :new_record?
+
+  # Scopes
+  scope :recent, -> { order(created_at: :desc) }
+  scope :by_user, ->(user) { where(user: user) }
+  scope :by_plan, ->(plan) { where(migration_plan: plan) }
+
+  # Instance methods
+  def duration
+    return nil unless started_at && completed_at
+
+    completed_at - started_at
+  end
+
+  def progress_percentage
+    return 0 if stats.blank? || stats['total'].to_i.zero?
+
+    ((stats['processed'].to_f / stats['total']) * 100).round(2)
+  end
+
+  def display_name
+    "#{execution_type.titleize} - #{migration_plan.name}"
+  end
+
+  # Custom getter for filter_params
+  def filter_params
+    value = read_attribute(:filter_params)
+    return {} if value.blank?
+    return value if value.is_a?(Hash)
+
+    JSON.parse(value)
+  rescue JSON::ParserError => e
+    Rails.logger.error "Failed to parse filter_params for MigrationExecution #{id}: #{e.message}"
+    {}
+  end
+
+  # Custom setter for filter_params
+  def filter_params=(value)
+    if value.is_a?(String)
+      write_attribute(:filter_params, value)
+    elsif value.is_a?(Hash)
+      write_attribute(:filter_params, value.to_json)
+    else
+      write_attribute(:filter_params, {}.to_json)
+    end
+  end
+
+  # Custom getter for stats
+  def stats
+    value = read_attribute(:stats)
+    return {} if value.blank?
+    return value if value.is_a?(Hash)
+
+    JSON.parse(value)
+  rescue JSON::ParserError => e
+    Rails.logger.error "Failed to parse stats for MigrationExecution #{id}: #{e.message}"
+    {}
+  end
+
+  # Custom setter for stats
+  def stats=(value)
+    if value.is_a?(String)
+      write_attribute(:stats, value)
+    elsif value.is_a?(Hash)
+      write_attribute(:stats, value.to_json)
+    else
+      write_attribute(:stats, {}.to_json)
+    end
+  end
+
+  private
+
+  def set_defaults
+    # Defaults are handled by getters
+  end
+end

data/app/models/migration_plan.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class MigrationPlan < ApplicationRecord
+  # Associations
+  belongs_to :user, class_name: 'DataMigrationUser'
+  has_many :migration_steps, dependent: :destroy
+  has_many :migration_executions, dependent: :restrict_with_error
+
+  # Validations
+  validates :name, presence: true, uniqueness: true
+
+  # Scopes
+  scope :ordered_by_name, -> { order(:name) }
+
+  # Instance methods
+  def can_be_deleted?
+    migration_executions.empty?
+  end
+
+  def last_execution
+    migration_executions.recent.first
+  end
+end

data/app/models/migration_record.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+class MigrationRecord < ApplicationRecord
+  # Associations
+  belongs_to :migration_execution
+
+  # Enums
+  enum action: { created: 0, updated: 1, skipped: 2, failed: 3 }
+
+  # Validations
+  validates :migrated_model_name, presence: true
+  validates :record_identifier, presence: true
+  validates :action, presence: true
+
+  # Callbacks to set defaults
+  after_initialize :set_defaults, if: :new_record?
+
+  # Scopes
+  scope :by_action, ->(action) { where(action: action) }
+  scope :by_model, ->(model_name) { where(migrated_model_name: model_name) }
+  scope :errors_only, -> { where(action: :failed) }
+
+  # Custom getter for record_changes
+  def record_changes
+    value = read_attribute(:record_changes)
+    return {} if value.blank?
+    return value if value.is_a?(Hash)
+
+    JSON.parse(value)
+  rescue JSON::ParserError => e
+    Rails.logger.error "Failed to parse record_changes for MigrationRecord #{id}: #{e.message}"
+    {}
+  end
+
+  # Custom setter for record_changes
+  def record_changes=(value)
+    if value.is_a?(String)
+      write_attribute(:record_changes, value)
+    elsif value.is_a?(Hash)
+      write_attribute(:record_changes, value.to_json)
+    else
+      write_attribute(:record_changes, {}.to_json)
+    end
+  end
+
+  # Instance methods
+  def display_name
+    "#{migrated_model_name} #{record_identifier}"
+  end
+
+  def success?
+    !failed?
+  end
+
+  private
+
+  def set_defaults
+    # Defaults are handled by getters
+  end
+end