data_migration_for_rails 0.1.1

data/app/services/imports/processor_service.rb

@@ -0,0 +1,457 @@
+# frozen_string_literal: true
+
+require 'csv'
+require 'fileutils'
+require 'zlib'
+require 'rubygems/package'
+
+module Imports
+  class ProcessorService
+    attr_reader :migration_plan, :execution, :uploaded_file
+
+    def initialize(migration_plan, execution, uploaded_file_path)
+      @migration_plan = migration_plan
+      @execution = execution
+      @uploaded_file_path = uploaded_file_path
+      @stats = {
+        total_steps: migration_plan.migration_steps.count,
+        completed_steps: 0,
+        total_records: 0,
+        processed_records: 0,
+        created: 0,
+        updated: 0,
+        skipped: 0,
+        failed: 0,
+        total_attachments: 0,
+        processed_attachments: 0,
+        errors: []
+      }
+      @id_mapping = {} # Maps old IDs to new IDs for foreign key updates
+      @temp_dir = nil
+    end
+
+    def call
+      execution.update!(status: :running, started_at: Time.current)
+
+      Dir.mktmpdir do |temp_dir|
+        extract_archive(temp_dir)
+        import_all_steps(temp_dir)
+        finalize_success
+      rescue StandardError => e
+        finalize_failure(e)
+      end
+    end
+
+    private
+
+    def extract_archive(temp_dir)
+      Gem::Package::TarReader.new(Zlib::GzipReader.open(@uploaded_file_path)) do |tar|
+        tar.each do |entry|
+          next unless entry.file?
+
+          file_path = File.join(temp_dir, entry.full_name)
+          FileUtils.mkdir_p(File.dirname(file_path))
+
+          File.open(file_path, 'wb') do |f|
+            f.write(entry.read)
+          end
+        end
+      end
+    end
+
+    def import_all_steps(temp_dir)
+      @temp_dir = temp_dir
+      migration_plan.migration_steps.order(:sequence).each do |step|
+        import_step(step, temp_dir)
+        @stats[:completed_steps] += 1
+        update_progress
+      end
+    end
+
+    def import_step(step, temp_dir)
+      csv_path = File.join(temp_dir, "#{step.source_model_name}_export.csv")
+      return unless File.exist?(csv_path)
+
+      model_class = step.source_model_name.constantize
+      rows = CSV.read(csv_path, headers: true)
+
+      @stats[:total_records] += rows.count
+      update_progress
+
+      rows.each do |row|
+        process_row(row, step, model_class)
+        @stats[:processed_records] += 1
+        update_progress if (@stats[:processed_records] % 50).zero?
+      end
+    rescue StandardError => e
+      @stats[:errors] << { step: step.source_model_name, error: e.message }
+      raise
+    end
+
+    def process_row(row, step, model_class)
+      # Build attributes hash from CSV row
+      attributes = build_attributes(row, step, model_class)
+
+      # Update foreign key associations (pass CSV row for association attribute values)
+      update_foreign_keys(attributes, step, row)
+
+      # Find or create record
+      existing_record = find_existing_record(attributes, step, model_class)
+
+      if existing_record
+        handle_existing_record(existing_record, attributes, step, row)
+      else
+        handle_new_record(attributes, step, model_class, row)
+      end
+    rescue StandardError => e
+      @stats[:failed] += 1
+      record_migration_action(step, row, :failed, {}, e.message)
+    end
+
+    def build_attributes(row, _step, model_class)
+      attributes = {}
+
+      model_class.column_names.each do |column|
+        attributes[column] = row[column] if row.key?(column)
+      end
+
+      attributes
+    end
+
+    def update_foreign_keys(attributes, step, csv_row)
+      return unless step.association_overrides.present?
+
+      step.association_overrides.each do |fk_column, mapping_info|
+        next unless attributes[fk_column].present?
+
+        # Check if this is a polymorphic association
+        if mapping_info['polymorphic'] == true
+          handle_polymorphic_foreign_key(attributes, fk_column, mapping_info, csv_row)
+        else
+          handle_regular_foreign_key(attributes, fk_column, mapping_info, csv_row)
+        end
+      end
+    end
+
+    def handle_regular_foreign_key(attributes, fk_column, mapping_info, csv_row)
+      old_id = attributes[fk_column].to_i
+      target_model = mapping_info['model']
+      lookup_attributes = mapping_info['lookup_attributes']
+
+      # Find the new ID based on lookup attributes
+      new_id = find_mapped_id(old_id, target_model, lookup_attributes, csv_row)
+      attributes[fk_column] = new_id if new_id
+    end
+
+    def handle_polymorphic_foreign_key(attributes, fk_column, mapping_info, csv_row)
+      type_column = mapping_info['type_column']
+
+      # Get the polymorphic type from the attributes
+      polymorphic_type = attributes[type_column]
+      return unless polymorphic_type.present?
+
+      # Get the lookup attributes for this specific type
+      lookup_attributes_by_type = mapping_info['lookup_attributes']
+      return unless lookup_attributes_by_type.is_a?(Hash)
+
+      lookup_attributes = lookup_attributes_by_type[polymorphic_type]
+      return unless lookup_attributes.present?
+
+      # Find the new ID based on the polymorphic type and lookup attributes
+      old_id = attributes[fk_column].to_i
+      new_id = find_mapped_id(old_id, polymorphic_type, lookup_attributes, csv_row)
+      attributes[fk_column] = new_id if new_id
+    end
+
+    def find_mapped_id(old_id, target_model, lookup_attributes, csv_row)
+      # Check if we've already mapped this ID (cache for performance)
+      mapping_key = "#{target_model}_#{old_id}"
+      return @id_mapping[mapping_key] if @id_mapping.key?(mapping_key)
+
+      # Build conditions hash from CSV row using lookup attributes
+      # For association_overrides, the association name would be something like 'company'
+      # and CSV columns would be like 'company.name', 'company.code'
+      conditions = {}
+
+      # Determine association name from the target model
+      # e.g., "Company" -> "company", "Project" -> "project"
+      association_name = target_model.underscore
+
+      Array(lookup_attributes).each do |attr|
+        # Look for CSV column like "company.name"
+        csv_column = "#{association_name}.#{attr}"
+        conditions[attr] = csv_row[csv_column] if csv_row.key?(csv_column) && csv_row[csv_column].present?
+      end
+
+      # If we have conditions, query the database
+      if conditions.present?
+        begin
+          model_class = target_model.constantize
+          found_record = model_class.find_by(conditions)
+
+          if found_record
+            # Cache this mapping for future lookups
+            @id_mapping[mapping_key] = found_record.id
+            return found_record.id
+          else
+            Rails.logger.warn "Could not find #{target_model} with #{conditions.inspect} for old_id #{old_id}"
+          end
+        rescue NameError => e
+          Rails.logger.error "Model #{target_model} not found: #{e.message}"
+        end
+      end
+
+      # Fallback: return old_id (might work if IDs are the same in both DBs)
+      old_id
+    end
+
+    def find_existing_record(attributes, step, model_class)
+      return nil if step.column_overrides.blank? || step.column_overrides['uniq_record_id_cols'].blank?
+
+      unique_columns = Array(step.column_overrides['uniq_record_id_cols'])
+      conditions = {}
+
+      unique_columns.each do |column|
+        conditions[column] = attributes[column]
+      end
+
+      model_class.find_by(conditions)
+    end
+
+    def handle_existing_record(record, attributes, step, csv_row)
+      # Check if source record is newer
+      if should_update?(record, attributes)
+        # Remove columns that should be ignored on update
+        ignored_columns = step.column_overrides&.dig('ignore_on_update') || []
+        update_attrs = attributes.except(*ignored_columns, 'id', 'created_at')
+
+        changes = record.attributes.slice(*update_attrs.keys).to_h.reject { |k, v| v == update_attrs[k] }
+
+        if record.update(update_attrs)
+          @stats[:updated] += 1
+          record_migration_action(step, csv_row, :updated, changes)
+
+          # Import attachments if any
+          import_attachments(record, step, csv_row)
+        else
+          @stats[:failed] += 1
+          record_migration_action(step, csv_row, :failed, {}, record.errors.full_messages.join(', '))
+        end
+      else
+        @stats[:skipped] += 1
+        record_migration_action(step, csv_row, :skipped, {})
+      end
+
+      # Store ID mapping
+      return unless csv_row['id'].present?
+
+      mapping_key = "#{step.source_model_name}_#{csv_row['id']}"
+      @id_mapping[mapping_key] = record.id
+    end
+
+    def handle_new_record(attributes, step, model_class, csv_row)
+      # Remove id from attributes to let database assign new one
+      old_id = attributes.delete('id')
+      attributes.delete('created_at')
+      attributes.delete('updated_at')
+
+      record = model_class.new(attributes)
+
+      if record.save
+        @stats[:created] += 1
+        record_migration_action(step, csv_row, :created, attributes)
+
+        # Store ID mapping
+        if old_id.present?
+          mapping_key = "#{step.source_model_name}_#{old_id}"
+          @id_mapping[mapping_key] = record.id
+        end
+
+        # Import attachments if any
+        import_attachments(record, step, csv_row)
+      else
+        @stats[:failed] += 1
+        record_migration_action(step, csv_row, :failed, {}, record.errors.full_messages.join(', '))
+      end
+    end
+
+    def should_update?(record, attributes)
+      # If no updated_at in import data, always update
+      return true unless attributes['updated_at'].present?
+
+      source_updated_at = begin
+        Time.zone.parse(attributes['updated_at'].to_s)
+      rescue StandardError
+        nil
+      end
+      return true unless source_updated_at
+
+      # Update if source is newer
+      source_updated_at > record.updated_at
+    end
+
+    def record_migration_action(step, csv_row, action, changes, error_message = nil)
+      record_identifier = csv_row['id'].present? ? "#{step.source_model_name}##{csv_row['id']}" : 'unknown'
+
+      MigrationRecord.create!(
+        migration_execution: execution,
+        migrated_model_name: step.source_model_name,
+        record_identifier: record_identifier,
+        action: action,
+        record_changes: changes,
+        error_message: error_message
+      )
+    end
+
+    def update_progress
+      execution.update!(stats: @stats)
+      broadcast_progress
+    end
+
+    def broadcast_progress
+      ActionCable.server.broadcast(
+        "execution_#{execution.id}",
+        {
+          type: 'progress',
+          stats: @stats,
+          percentage: calculate_percentage,
+          message: progress_message
+        }
+      )
+    end
+
+    def calculate_percentage
+      return 0 if @stats[:total_records].zero?
+
+      ((@stats[:processed_records].to_f / @stats[:total_records]) * 100).round(2)
+    end
+
+    def progress_message
+      "Processing step #{@stats[:completed_steps]}/#{@stats[:total_steps]} - " \
+      "#{@stats[:processed_records]}/#{@stats[:total_records]} records imported " \
+      "(#{@stats[:created]} created, #{@stats[:updated]} updated, #{@stats[:skipped]} skipped, #{@stats[:failed]} failed)"
+    end
+
+    def finalize_success
+      execution.update!(
+        status: :completed,
+        completed_at: Time.current,
+        stats: @stats
+      )
+
+      broadcast_completion('Import completed successfully')
+    end
+
+    def finalize_failure(error)
+      @stats[:errors] << { general: error.message }
+
+      execution.update!(
+        status: :failed,
+        completed_at: Time.current,
+        error_log: error.full_message,
+        stats: @stats
+      )
+
+      broadcast_completion("Import failed: #{error.message}")
+    end
+
+    def broadcast_completion(message)
+      ActionCable.server.broadcast(
+        "execution_#{execution.id}",
+        {
+          type: 'completion',
+          status: execution.status,
+          message: message,
+          stats: @stats
+        }
+      )
+    end
+
+    # Import attachments for a record
+    def import_attachments(record, step, csv_row)
+      return if step.ignore?
+
+      model_class = record.class
+      return unless model_class.respond_to?(:reflect_on_all_attachments)
+
+      model_class.reflect_on_all_attachments.each do |attachment_reflection|
+        attachment_name = attachment_reflection.name
+
+        if step.url?
+          # Import from URL
+          url_column = "#{attachment_name}_url"
+          url = csv_row[url_column]
+
+          import_attachment_from_url(record, attachment_name, url) if url.present?
+        elsif step.raw_data?
+          # Import from raw file
+          path_column = "#{attachment_name}_path"
+          filename_column = "#{attachment_name}_filename"
+          content_type_column = "#{attachment_name}_content_type"
+
+          file_path = csv_row[path_column]
+          filename = csv_row[filename_column]
+          content_type = csv_row[content_type_column]
+
+          if file_path.present? && filename.present?
+            import_attachment_from_file(record, attachment_name, file_path, filename, content_type)
+          end
+        end
+      end
+    rescue StandardError => e
+      Rails.logger.error "Failed to import attachments for record #{record.id}: #{e.message}"
+      @stats[:errors] << {
+        step: step.source_model_name,
+        record_id: record.id,
+        error: "Attachment import failed: #{e.message}"
+      }
+    end
+
+    # Import attachment from URL
+    def import_attachment_from_url(record, attachment_name, url)
+      require 'open-uri'
+
+      URI.open(url) do |file|
+        record.send(attachment_name).attach(
+          io: file,
+          filename: File.basename(URI.parse(url).path),
+          content_type: file.content_type
+        )
+        @stats[:processed_attachments] += 1
+      end
+    rescue StandardError => e
+      Rails.logger.error "Failed to download attachment from URL #{url}: #{e.message}"
+      @stats[:errors] << {
+        attachment_name: attachment_name,
+        url: url,
+        error: e.message
+      }
+    end
+
+    # Import attachment from extracted file
+    def import_attachment_from_file(record, attachment_name, file_path, filename, content_type)
+      full_path = File.join(@temp_dir, file_path)
+
+      unless File.exist?(full_path)
+        Rails.logger.warn "Attachment file not found: #{full_path}"
+        return
+      end
+
+      File.open(full_path, 'rb') do |file|
+        record.send(attachment_name).attach(
+          io: file,
+          filename: filename,
+          content_type: content_type
+        )
+        @stats[:processed_attachments] += 1
+      end
+    rescue StandardError => e
+      Rails.logger.error "Failed to attach file #{full_path}: #{e.message}"
+      @stats[:errors] << {
+        attachment_name: attachment_name,
+        file_path: file_path,
+        error: e.message
+      }
+    end
+  end
+end

data/app/services/migration_plans/export_config_service.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module MigrationPlans
+  class ExportConfigService
+    attr_reader :migration_plan
+
+    def initialize(migration_plan)
+      @migration_plan = migration_plan
+    end
+
+    def call
+      {
+        version: '1.0',
+        exported_at: Time.current.iso8601,
+        plan: {
+          name: migration_plan.name,
+          description: migration_plan.description,
+          steps: export_steps
+        }
+      }.to_json
+    end
+
+    private
+
+    def export_steps
+      migration_plan.migration_steps.order(:sequence).map do |step|
+        {
+          source_model_name: step.source_model_name,
+          sequence: step.sequence,
+          filter_query: step.filter_query,
+          column_overrides: step.column_overrides,
+          association_overrides: step.association_overrides,
+          attachment_export_mode: step.attachment_export_mode,
+          attachment_fields: step.attachment_fields,
+          dependee_sequence: step.dependee&.sequence, # Reference by sequence, not ID
+          dependee_attribute_mapping: step.dependee_attribute_mapping
+        }
+      end
+    end
+  end
+end

data/app/services/migration_plans/import_config_service.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module MigrationPlans
+  class ImportConfigService
+    attr_reader :config_json, :current_user, :errors
+
+    def initialize(config_json, current_user)
+      @config_json = config_json
+      @current_user = current_user
+      @errors = []
+      @step_mapping = {} # Maps sequence to step objects for dependency resolution
+    end
+
+    def call
+      config = parse_config
+      return nil unless config
+
+      validate_config(config)
+      return nil if @errors.any?
+
+      create_plan_with_steps(config)
+    rescue StandardError => e
+      @errors << "Import failed: #{e.message}"
+      nil
+    end
+
+    def success?
+      @errors.empty?
+    end
+
+    private
+
+    def parse_config
+      JSON.parse(@config_json)
+    rescue JSON::ParserError => e
+      @errors << "Invalid JSON format: #{e.message}"
+      nil
+    end
+
+    def validate_config(config)
+      unless config['plan']
+        @errors << "Missing 'plan' section in configuration"
+        return
+      end
+
+      plan_data = config['plan']
+
+      @errors << 'Plan name is required' unless plan_data['name'].present?
+
+      unless plan_data['steps'].is_a?(Array)
+        @errors << 'Steps must be an array'
+        return
+      end
+
+      # Validate each step
+      plan_data['steps'].each_with_index do |step_data, index|
+        validate_step(step_data, index)
+      end
+    end
+
+    def validate_step(step_data, index)
+      unless step_data['source_model_name'].present?
+        @errors << "Step #{index + 1}: source_model_name is required"
+        return
+      end
+
+      # Check if model exists in this environment
+      begin
+        step_data['source_model_name'].constantize
+      rescue NameError
+        @errors << "Step #{index + 1}: Model '#{step_data['source_model_name']}' not found in this environment"
+      end
+
+      @errors << "Step #{index + 1}: sequence is required" unless step_data['sequence'].present?
+
+      # Validate attachment_export_mode if present
+      return unless step_data['attachment_export_mode'].present?
+
+      valid_modes = %w[ignore url raw_data]
+      return if valid_modes.include?(step_data['attachment_export_mode'])
+
+      @errors << "Step #{index + 1}: invalid attachment_export_mode '#{step_data['attachment_export_mode']}'"
+    end
+
+    def create_plan_with_steps(config)
+      plan_data = config['plan']
+
+      ActiveRecord::Base.transaction do
+        # Find or create the migration plan by name
+        plan = MigrationPlan.find_or_create_by!(name: plan_data['name']) do |p|
+          p.user = current_user
+        end
+
+        # Update plan attributes
+        plan.update!(
+          description: plan_data['description'],
+          user: current_user
+        )
+
+        # Create/update steps in sequence order (important for dependencies)
+        sorted_steps = plan_data['steps'].sort_by { |s| s['sequence'] }
+
+        sorted_steps.each do |step_data|
+          create_or_update_step(plan, step_data)
+        end
+
+        # Resolve dependencies after all steps are created/updated
+        resolve_dependencies(plan, sorted_steps)
+
+        plan
+      end
+    end
+
+    def create_or_update_step(plan, step_data)
+      # Find or create step by plan and sequence
+      step = MigrationStep.find_or_create_by!(
+        migration_plan: plan,
+        sequence: step_data['sequence']
+      )
+
+      # Update step attributes
+      step.update!(
+        source_model_name: step_data['source_model_name'],
+        filter_query: step_data['filter_query'] || '',
+        column_overrides: step_data['column_overrides'] || {},
+        association_overrides: step_data['association_overrides'] || {},
+        attachment_export_mode: step_data['attachment_export_mode'] || 'ignore',
+        attachment_fields: step_data['attachment_fields'],
+        dependee_attribute_mapping: step_data['dependee_attribute_mapping'] || {}
+      )
+
+      # Store in mapping for later dependency resolution
+      @step_mapping[step_data['sequence']] = {
+        step: step,
+        dependee_sequence: step_data['dependee_sequence']
+      }
+
+      step
+    end
+
+    def resolve_dependencies(_plan, _sorted_steps)
+      @step_mapping.each do |sequence, data|
+        step = data[:step]
+        dependee_sequence = data[:dependee_sequence]
+
+        next unless dependee_sequence.present?
+
+        # Find the dependee step by sequence
+        dependee_data = @step_mapping[dependee_sequence]
+        if dependee_data
+          step.update!(dependee: dependee_data[:step])
+        else
+          Rails.logger.warn "Could not resolve dependency for step #{sequence}: dependee sequence #{dependee_sequence} not found"
+        end
+      end
+    end
+  end
+end

data/app/views/data_migration/devise/registrations/edit.html.erb

@@ -0,0 +1,41 @@
+<div class="row justify-content-center">
+  <div class="col-md-6">
+    <div class="card shadow">
+      <div class="card-body p-4">
+        <h2 class="card-title mb-4">Change Password</h2>
+
+        <%= form_for(resource, as: resource_name, url: registration_path(resource_name), html: { method: :put }) do |f| %>
+          <%= render "data_migration/devise/shared/error_messages", resource: resource %>
+
+          <div class="mb-3">
+            <%= f.label :password, "New password", class: "form-label" %>
+            <%= f.password_field :password, autocomplete: "new-password", class: "form-control", placeholder: "Enter new password" %>
+            <small class="form-text text-muted">
+              (Minimum 6 characters)
+            </small>
+          </div>
+
+          <div class="mb-3">
+            <%= f.label :password_confirmation, "Confirm new password", class: "form-label" %>
+            <%= f.password_field :password_confirmation, autocomplete: "new-password", class: "form-control", placeholder: "Confirm new password" %>
+          </div>
+
+          <hr class="my-4">
+
+          <div class="mb-4">
+            <%= f.label :current_password, class: "form-label" %>
+            <%= f.password_field :current_password, autocomplete: "current-password", class: "form-control", placeholder: "Enter current password to confirm changes" %>
+            <small class="form-text text-muted">
+              We need your current password to confirm changes
+            </small>
+          </div>
+
+          <div class="d-flex gap-2">
+            <%= f.submit "Update Password", class: "btn btn-primary" %>
+            <%= link_to "Back", :back, class: "btn btn-outline-secondary" %>
+          </div>
+        <% end %>
+      </div>
+    </div>
+  </div>
+</div>