data_migration_for_rails 0.1.1

data/app/views/data_migration/migration_steps/_form.html.erb

@@ -0,0 +1,473 @@
+<%= form_with(model: [migration_plan, migration_step], local: true, class: "needs-validation", id: "migration-step-form") do |form| %>
+  <% if migration_step.errors.any? %>
+    <div class="alert alert-danger">
+      <h4><%= pluralize(migration_step.errors.count, "error") %> prohibited this step from being saved:</h4>
+      <ul class="mb-0">
+        <% migration_step.errors.full_messages.each do |message| %>
+          <li><%= message %></li>
+        <% end %>
+      </ul>
+    </div>
+  <% end %>
+
+  <div class="row">
+    <div class="col-md-8">
+      <div class="mb-3">
+        <%= form.label :source_model_name, "Model Name", class: "form-label" %>
+        <%= form.select :source_model_name,
+            options_for_select(model_registry.keys.sort, migration_step.source_model_name),
+            { include_blank: "Select a model..." },
+            { class: "form-select", required: true, id: "model_selector" } %>
+        <div class="form-text">The ActiveRecord model class name to migrate.</div>
+      </div>
+
+      <!-- Model Fields Display (dynamically shown when model is selected) -->
+      <div class="mb-3" id="model-fields-container" style="display: none;">
+        <div class="card">
+          <div class="card-header bg-light">
+            <h6 class="mb-0">📋 <span id="selected-model-name"></span> Fields</h6>
+          </div>
+          <div class="card-body">
+            <div class="row">
+              <div class="col-md-6">
+                <strong>Database Columns:</strong>
+                <ul id="model-columns-list" class="small mb-0" style="column-count: 2;"></ul>
+              </div>
+              <div class="col-md-6" id="model-attachments-container" style="display: none;">
+                <strong>📎 Attachments:</strong>
+                <ul id="model-attachments-list" class="small mb-0"></ul>
+              </div>
+            </div>
+          </div>
+        </div>
+      </div>
+
+      <div class="mb-3">
+        <%= form.label :filter_query, "Filter Query (optional)", class: "form-label" %>
+        <%= form.text_area :filter_query, class: "form-control font-monospace", rows: 3, placeholder: "e.g., where(active: true).where('created_at > ?', 1.year.ago)" %>
+        <div class="form-text">
+          Optional ActiveRecord query to filter which records to include. Leave blank to migrate all records.
+          <br><strong>Example:</strong> <code>where(status: "active")</code>
+          <br><strong>With placeholder:</strong> <code>where(status: "{{status_value}}")</code> or <code>where("created_at > ?", "{{cutoff_date}}")</code>
+          <br><small class="text-muted">Note: Placeholders must be inside string quotes.</small>
+        </div>
+      </div>
+
+      <!-- Column Overrides JSON Field -->
+      <div class="mb-3">
+        <div class="d-flex justify-content-between align-items-center mb-2">
+          <%= form.label :column_overrides, "Column Overrides (JSON)", class: "form-label mb-0" %>
+          <div class="btn-group btn-group-sm" role="group">
+            <button type="button" class="btn btn-outline-secondary" onclick="formatJSON('column_overrides')">
+              📋 Format
+            </button>
+            <button type="button" class="btn btn-outline-primary" onclick="insertTemplate('column_overrides', 'basic')">
+              ➕ Template
+            </button>
+            <button type="button" class="btn btn-outline-info" onclick="clearField('column_overrides')">
+              🗑️ Clear
+            </button>
+          </div>
+        </div>
+        <%= form.text_area :column_overrides,
+            value: migration_step.column_overrides.is_a?(Hash) ? JSON.pretty_generate(migration_step.column_overrides) : migration_step.column_overrides,
+            class: "form-control font-monospace json-field",
+            id: "column_overrides",
+            rows: 5,
+            placeholder: '{}',
+            data: { field_type: "column_overrides" } %>
+        <div class="invalid-feedback" id="column_overrides_error"></div>
+        <div class="form-text">
+          JSON object mapping associations to their attributes for export. Required for Association ID Mappings to work.
+          <br><strong>Example:</strong> <code>{"company": ["name", "code"], "role": ["title"]}</code>
+          <br><small class="text-muted">Note: For polymorphic associations like 'commentable', include the lookup attributes here so they're exported to CSV.</small>
+        </div>
+      </div>
+
+      <!-- Association Overrides JSON Field -->
+      <div class="mb-3">
+        <div class="d-flex justify-content-between align-items-center mb-2">
+          <%= form.label :association_overrides, "Association ID Mappings (JSON)", class: "form-label mb-0" %>
+          <div class="btn-group btn-group-sm" role="group">
+            <button type="button" class="btn btn-outline-secondary" onclick="formatJSON('association_overrides')">
+              📋 Format
+            </button>
+            <button type="button" class="btn btn-outline-primary" onclick="insertTemplate('association_overrides', 'regular')">
+              ➕ Regular
+            </button>
+            <button type="button" class="btn btn-outline-primary" onclick="insertTemplate('association_overrides', 'polymorphic')">
+              ➕ Polymorphic
+            </button>
+            <button type="button" class="btn btn-outline-info" onclick="clearField('association_overrides')">
+              🗑️ Clear
+            </button>
+          </div>
+        </div>
+        <%= form.text_area :association_overrides,
+            value: migration_step.association_overrides.is_a?(Hash) ? JSON.pretty_generate(migration_step.association_overrides) : migration_step.association_overrides,
+            class: "form-control font-monospace json-field",
+            id: "association_overrides",
+            rows: 8,
+            placeholder: '{}',
+            data: { field_type: "association_overrides" } %>
+        <div class="invalid-feedback" id="association_overrides_error"></div>
+        <div class="form-text">
+          JSON object for remapping foreign key IDs during import based on lookup attributes.
+          <br><strong>Regular Association:</strong> <code>{"company_id": {"model": "Company", "lookup_attributes": ["code"]}}</code>
+          <br><strong>Polymorphic:</strong> <code>{"commentable_id": {"polymorphic": true, "type_column": "commentable_type", "lookup_attributes": {"Post": ["title"], "Product": ["sku"]}}}</code>
+        </div>
+      </div>
+    </div>
+
+    <div class="col-md-4">
+      <div class="mb-3">
+        <%= form.label :sequence, "Execution Order", class: "form-label" %>
+        <%= form.number_field :sequence, class: "form-control", min: 1, required: true, value: migration_step.sequence || (migration_plan.migration_steps.maximum(:sequence) || 0) + 1 %>
+        <div class="form-text">
+          Steps are executed in ascending order. Dependencies should be migrated first.
+        </div>
+      </div>
+
+      <div class="mb-3">
+        <%= form.label :dependee_id, "Depends On Step (optional)", class: "form-label" %>
+        <%= form.collection_select :dependee_id,
+            migration_plan.migration_steps.where.not(id: migration_step.id).order(:sequence),
+            :id,
+            ->(step) { "#{step.sequence}. #{step.source_model_name}" },
+            { include_blank: "None - Independent Step" },
+            { class: "form-select" } %>
+        <div class="form-text">
+          Select a parent step if this step's records reference the parent step's records.
+        </div>
+      </div>
+
+      <!-- Dependee Attribute Mapping JSON Field -->
+      <div class="mb-3">
+        <div class="d-flex justify-content-between align-items-center mb-2">
+          <%= form.label :dependee_attribute_mapping, "Dependee Attribute Mapping (JSON)", class: "form-label mb-0" %>
+          <div class="btn-group btn-group-sm" role="group">
+            <button type="button" class="btn btn-outline-secondary" onclick="formatJSON('dependee_attribute_mapping')">
+              📋 Format
+            </button>
+            <button type="button" class="btn btn-outline-primary" onclick="insertTemplate('dependee_attribute_mapping', 'basic')">
+              ➕ Template
+            </button>
+            <button type="button" class="btn btn-outline-info" onclick="clearField('dependee_attribute_mapping')">
+              🗑️ Clear
+            </button>
+          </div>
+        </div>
+        <%= form.text_area :dependee_attribute_mapping,
+            value: migration_step.dependee_attribute_mapping.is_a?(Hash) ? JSON.pretty_generate(migration_step.dependee_attribute_mapping) : migration_step.dependee_attribute_mapping,
+            class: "form-control font-monospace json-field",
+            id: "dependee_attribute_mapping",
+            rows: 3,
+            placeholder: '{}',
+            data: { field_type: "dependee_attribute_mapping" } %>
+        <div class="invalid-feedback" id="dependee_attribute_mapping_error"></div>
+        <div class="form-text">
+          Maps this step's columns to the dependee step's columns for filtering during export.
+          <br><strong>Example:</strong> <code>{"company_id": "id"}</code>
+          <br><small class="text-muted">Only export records where company_id matches the IDs exported in the dependee step.</small>
+        </div>
+      </div>
+
+      <!-- Attachment Export Mode -->
+      <div class="mb-3" id="attachment-mode-container" style="display: none;">
+        <%= form.label :attachment_export_mode, "Attachment Export Mode", class: "form-label" %>
+        <%= form.select :attachment_export_mode,
+            options_for_select([
+              ['Ignore Attachments', 'ignore'],
+              ['Export as URLs', 'url'],
+              ['Export as Raw Data (compressed)', 'raw_data']
+            ], migration_step.attachment_export_mode),
+            {},
+            { class: "form-select" } %>
+        <div class="form-text">
+          Choose how to handle Active Storage attachments during export.
+        </div>
+      </div>
+
+      <div class="card bg-light">
+        <div class="card-body">
+          <h6 class="card-title">💡 Tips</h6>
+          <ul class="small mb-0">
+            <li>Migrate parent models before child models</li>
+            <li>Use filter queries to limit data scope</li>
+            <li>Column overrides include association data in exports</li>
+            <li>Use dependee mapping for referential integrity</li>
+            <li>Test with small datasets first</li>
+            <li>Click "Format" to prettify JSON</li>
+            <li>Use templates to get started quickly</li>
+          </ul>
+        </div>
+      </div>
+    </div>
+  </div>
+
+  <div class="mb-3">
+    <%= form.submit migration_step.new_record? ? "Create Step" : "Update Step", class: "btn btn-primary", id: "submit-button" %>
+  </div>
+<% end %>
+
+<style>
+  .json-field.is-valid {
+    border-color: #198754 !important;
+    background-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%23198754' d='M2.3 6.73.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e");
+    background-repeat: no-repeat;
+    background-position: right calc(.375em + .1875rem) center;
+    background-size: calc(.75em + .375rem) calc(.75em + .375rem);
+    padding-right: calc(1.5em + .75rem);
+  }
+
+  .json-field.is-invalid {
+    border-color: #dc3545 !important;
+    background-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 12 12' width='12' height='12' fill='none' stroke='%23dc3545'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e");
+    background-repeat: no-repeat;
+    background-position: right calc(.375em + .1875rem) center;
+    background-size: calc(.75em + .375rem) calc(.75em + .375rem);
+    padding-right: calc(1.5em + .75rem);
+  }
+
+  .json-field:focus {
+    box-shadow: 0 0 0 0.25rem rgba(13, 110, 253, 0.25);
+  }
+</style>
+
+<script>
+// Model Registry Data (cached from server)
+const modelRegistry = <%= raw model_registry.to_json %>;
+
+// JSON Templates
+const templates = {
+  column_overrides: {
+    basic: {
+      "association_name": ["attribute1", "attribute2"],
+      "another_association": ["attribute3"]
+    }
+  },
+  association_overrides: {
+    regular: {
+      "company_id": {
+        "model": "Company",
+        "lookup_attributes": ["code"]
+      },
+      "manager_id": {
+        "model": "Employee",
+        "lookup_attributes": ["email"]
+      }
+    },
+    polymorphic: {
+      "commentable_id": {
+        "polymorphic": true,
+        "type_column": "commentable_type",
+        "lookup_attributes": {
+          "Post": ["title"],
+          "Article": ["title"],
+          "Product": ["name", "sku"]
+        }
+      }
+    }
+  },
+  dependee_attribute_mapping: {
+    basic: {
+      "company_id": "id",
+      "parent_id": "id"
+    }
+  }
+};
+
+// Validate JSON field
+function validateJSON(fieldId) {
+  const field = document.getElementById(fieldId);
+  const errorDiv = document.getElementById(fieldId + '_error');
+  const value = field.value.trim();
+
+  // Empty is valid (optional field)
+  if (value === '' || value === '{}' || value === '[]') {
+    field.classList.remove('is-invalid');
+    field.classList.add('is-valid');
+    errorDiv.textContent = '';
+    return true;
+  }
+
+  try {
+    const parsed = JSON.parse(value);
+
+    // Additional validation: must be an object, not array
+    if (Array.isArray(parsed)) {
+      throw new Error('Must be a JSON object {}, not an array []');
+    }
+
+    field.classList.remove('is-invalid');
+    field.classList.add('is-valid');
+    errorDiv.textContent = '';
+    return true;
+  } catch (e) {
+    field.classList.remove('is-valid');
+    field.classList.add('is-invalid');
+    errorDiv.textContent = '⚠️ Invalid JSON: ' + e.message;
+    errorDiv.style.display = 'block';
+    return false;
+  }
+}
+
+// Format JSON with proper indentation
+function formatJSON(fieldId) {
+  const field = document.getElementById(fieldId);
+  const value = field.value.trim();
+
+  if (value === '' || value === '{}' || value === '[]') {
+    field.value = '{}';
+    validateJSON(fieldId);
+    return;
+  }
+
+  try {
+    const parsed = JSON.parse(value);
+    field.value = JSON.stringify(parsed, null, 2);
+    validateJSON(fieldId);
+  } catch (e) {
+    alert('Cannot format: Invalid JSON\n\n' + e.message);
+  }
+}
+
+// Insert template
+function insertTemplate(fieldId, templateType) {
+  const field = document.getElementById(fieldId);
+  const fieldType = field.dataset.fieldType;
+
+  if (templates[fieldType] && templates[fieldType][templateType]) {
+    const template = templates[fieldType][templateType];
+
+    // If field has content, ask for confirmation
+    if (field.value.trim() !== '' && field.value.trim() !== '{}') {
+      if (!confirm('This will replace the existing content. Continue?')) {
+        return;
+      }
+    }
+
+    field.value = JSON.stringify(template, null, 2);
+    validateJSON(fieldId);
+    field.focus();
+  }
+}
+
+// Clear field
+function clearField(fieldId) {
+  const field = document.getElementById(fieldId);
+
+  if (field.value.trim() !== '' && field.value.trim() !== '{}') {
+    if (!confirm('Clear all content in this field?')) {
+      return;
+    }
+  }
+
+  field.value = '{}';
+  field.classList.remove('is-valid', 'is-invalid');
+  document.getElementById(fieldId + '_error').textContent = '';
+}
+
+// Validate all JSON fields on page load and input
+document.addEventListener('DOMContentLoaded', function() {
+  const jsonFields = document.querySelectorAll('.json-field');
+
+  jsonFields.forEach(field => {
+    // Validate on load
+    validateJSON(field.id);
+
+    // Validate on blur
+    field.addEventListener('blur', function() {
+      validateJSON(this.id);
+    });
+
+    // Remove validation styling while typing
+    field.addEventListener('input', function() {
+      this.classList.remove('is-valid', 'is-invalid');
+    });
+  });
+
+  // Validate all fields before form submission
+  const form = document.getElementById('migration-step-form');
+  form.addEventListener('submit', function(e) {
+    let allValid = true;
+
+    jsonFields.forEach(field => {
+      if (!validateJSON(field.id)) {
+        allValid = false;
+      }
+    });
+
+    if (!allValid) {
+      e.preventDefault();
+      alert('Please fix the JSON errors before submitting.');
+      // Scroll to first error
+      const firstError = document.querySelector('.json-field.is-invalid');
+      if (firstError) {
+        firstError.scrollIntoView({ behavior: 'smooth', block: 'center' });
+        firstError.focus();
+      }
+    }
+  });
+
+  // Handle model selector change
+  const modelSelector = document.getElementById('model_selector');
+  if (modelSelector) {
+    // Show fields if model is already selected (edit mode)
+    if (modelSelector.value) {
+      displayModelFields(modelSelector.value);
+    }
+
+    modelSelector.addEventListener('change', function() {
+      const modelName = this.value;
+      if (modelName) {
+        displayModelFields(modelName);
+      } else {
+        // Hide fields if no model selected
+        document.getElementById('model-fields-container').style.display = 'none';
+        document.getElementById('attachment-mode-container').style.display = 'none';
+      }
+    });
+  }
+});
+
+// Display model fields and attachments
+function displayModelFields(modelName) {
+  const metadata = modelRegistry[modelName];
+  if (!metadata) return;
+
+  // Update model name display
+  document.getElementById('selected-model-name').textContent = modelName;
+
+  // Display columns
+  const columnsList = document.getElementById('model-columns-list');
+  columnsList.innerHTML = '';
+  metadata.columns.forEach(column => {
+    const li = document.createElement('li');
+    li.innerHTML = `<code>${column.name}</code> <small class="text-muted">(${column.type})</small>`;
+    columnsList.appendChild(li);
+  });
+
+  // Display attachments if any
+  const attachmentsContainer = document.getElementById('model-attachments-container');
+  const attachmentsList = document.getElementById('model-attachments-list');
+  if (metadata.attachments && metadata.attachments.length > 0) {
+    attachmentsList.innerHTML = '';
+    metadata.attachments.forEach(attachment => {
+      const li = document.createElement('li');
+      li.innerHTML = `<code>${attachment.name}</code> <small class="text-muted">(${attachment.type})</small>`;
+      attachmentsList.appendChild(li);
+    });
+    attachmentsContainer.style.display = 'block';
+
+    // Show attachment export mode selector
+    document.getElementById('attachment-mode-container').style.display = 'block';
+  } else {
+    attachmentsContainer.style.display = 'none';
+    document.getElementById('attachment-mode-container').style.display = 'none';
+  }
+
+  // Show the fields container
+  document.getElementById('model-fields-container').style.display = 'block';
+}
+</script>

data/app/views/data_migration/migration_steps/edit.html.erb

@@ -0,0 +1,12 @@
+<div class="row justify-content-center">
+  <div class="col-md-10">
+    <h1 class="mb-4">Edit Migration Step</h1>
+
+    <%= render 'form', migration_step: @migration_step, migration_plan: @migration_plan %>
+
+    <div class="mt-3">
+      <%= link_to "View Plan", @migration_plan, class: "btn btn-primary" %>
+      <%= link_to "Cancel", @migration_plan, class: "btn btn-secondary" %>
+    </div>
+  </div>
+</div>

data/app/views/data_migration/migration_steps/new.html.erb

@@ -0,0 +1,9 @@
+<div class="row justify-content-center">
+  <div class="col-md-10">
+    <h1 class="mb-4">New Migration Step for <%= @migration_plan.name %></h1>
+
+    <%= render 'form', migration_step: @migration_step, migration_plan: @migration_plan %>
+
+    <%= link_to "Cancel", @migration_plan, class: "btn btn-secondary mt-3" %>
+  </div>
+</div>

data/app/views/data_migration/users/_form.html.erb

@@ -0,0 +1,49 @@
+<%= form_with(model: user, url: user.new_record? ? users_path : user_path(user), local: true) do |form| %>
+  <% if user.errors.any? %>
+    <div class="alert alert-danger">
+      <h4><%= pluralize(user.errors.count, "error") %> prohibited this user from being saved:</h4>
+      <ul class="mb-0">
+        <% user.errors.full_messages.each do |message| %>
+          <li><%= message %></li>
+        <% end %>
+      </ul>
+    </div>
+  <% end %>
+
+  <div class="mb-3">
+    <%= form.label :name, class: "form-label" %>
+    <%= form.text_field :name, class: "form-control", required: true %>
+  </div>
+
+  <div class="mb-3">
+    <%= form.label :email, class: "form-label" %>
+    <%= form.email_field :email, class: "form-control", required: true %>
+  </div>
+
+  <div class="mb-3">
+    <%= form.label :password, class: "form-label" %>
+    <%= form.password_field :password, class: "form-control", autocomplete: "new-password" %>
+    <% unless user.new_record? %>
+      <div class="form-text">Leave blank to keep current password</div>
+    <% end %>
+  </div>
+
+  <div class="mb-3">
+    <%= form.label :password_confirmation, class: "form-label" %>
+    <%= form.password_field :password_confirmation, class: "form-control", autocomplete: "new-password" %>
+  </div>
+
+  <div class="mb-3">
+    <%= form.label :role, class: "form-label" %>
+    <%= form.select :role, DataMigrationUser.roles.keys.map { |role| [role.titleize, role] },
+        {}, { class: "form-select", required: true } %>
+    <div class="form-text">
+      <strong>Admin:</strong> Full access<br>
+      <strong>Operator:</strong> Can execute exports/imports<br>
+      <strong>Viewer:</strong> Read-only access
+    </div>
+  </div>
+
+  <%= form.submit user.new_record? ? "Create User" : "Update User", class: "btn btn-primary" %>
+  <%= link_to "Cancel", users_path, class: "btn btn-secondary" %>
+<% end %>

data/app/views/data_migration/users/edit.html.erb

@@ -0,0 +1,2 @@
+<h1>Edit User</h1>
+<%= render 'form', user: @user %>

data/app/views/data_migration/users/index.html.erb

@@ -0,0 +1,41 @@
+<div class="d-flex justify-content-between align-items-center mb-4">
+  <h1>Users</h1>
+  <%= link_to "New User", new_user_path, class: "btn btn-primary" %>
+</div>
+
+<div class="table-responsive">
+  <table class="table table-hover">
+    <thead class="table-light">
+      <tr>
+        <th>Name</th>
+        <th>Email</th>
+        <th>Role</th>
+        <th>Created</th>
+        <th>Actions</th>
+      </tr>
+    </thead>
+    <tbody>
+      <% @users.each do |user| %>
+        <tr>
+          <td><%= user.name %></td>
+          <td><%= user.email %></td>
+          <td>
+            <span class="badge bg-<%= user.admin? ? 'danger' : (user.operator? ? 'primary' : 'secondary') %>">
+              <%= user.role.titleize %>
+            </span>
+          </td>
+          <td><%= time_ago_in_words(user.created_at) %> ago</td>
+          <td>
+            <%= link_to "Edit", edit_user_path(user), class: "btn btn-sm btn-outline-warning" %>
+            <% if user != current_user %>
+              <%= button_to "Delete", user_path(user),
+                  method: :delete,
+                  class: "btn btn-sm btn-outline-danger",
+                  data: { confirm: "Are you sure?" } %>
+            <% end %>
+          </td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+</div>

data/app/views/data_migration/users/new.html.erb

@@ -0,0 +1,2 @@
+<h1>New User</h1>
+<%= render 'form', user: @user %>