schema-tools 1.0.0

data/lib/schema_tools/migrate/migrate_breaking_change.rb

@@ -0,0 +1,373 @@
+require_relative '../schema_files'
+require_relative 'migration_step'
+require_relative 'migrate_verify'
+require_relative '../diff'
+require_relative 'rollback'
+require 'json'
+
+module SchemaTools
+  # Custom logger that uses the migration's log() method
+  class MigrationLogger
+    attr_writer :migration_log_index
+
+    def initialize(migration_log_index, client)
+      @migration_log_index = migration_log_index
+      @client = client
+    end
+
+    def info(message)
+      log(message)
+    end
+
+    def warn(message)
+      log(message)
+    end
+
+    def error(message)
+      log(message)
+    end
+
+    def log(message)
+      puts message
+      log_to_log_index(message)
+    end
+
+    def log_to_log_index(message)
+      return unless @migration_log_index
+      doc = {
+        timestamp: Time.now.iso8601,
+        message: message.is_a?(String) ? message : message.to_json
+      }
+      @client.post("/#{@migration_log_index}/_doc", doc, suppress_logging: true)
+    end
+  end
+
+  class MigrateBreakingChange
+    def self.migrate(alias_name:, client:)
+      new(alias_name: alias_name, client: client).migrate
+    end
+
+    def initialize(alias_name:, client:)
+      @alias_name = alias_name
+      @client = client
+      @migration_log_index = nil
+      @current_step = nil
+      @rollback_attempted = false
+
+      @logger = MigrationLogger.new(nil, client)
+      @client.instance_variable_set(:@logger, @logger)
+    end
+
+    def migrate
+      log "=" * 60
+      log "Breaking Change Migration for #{@alias_name}"
+      log "=" * 60
+      
+      begin
+        setup
+        migration_steps.each do |step|
+          @current_step = step
+          step.execute(self)
+        end
+        SchemaTools.verify_migration(@alias_name, @client)
+      rescue => e
+        log("Migration failed: #{e.message}")
+        raise e
+      end
+    end
+
+    def setup
+      unless @client.alias_exists?(@alias_name)
+        raise "Alias '#{@alias_name}' does not exist"
+      end
+      
+      indices = @client.get_alias_indices(@alias_name)
+      if indices.length != 1
+        log "ERROR: Alias '#{@alias_name}' must point to exactly one index"
+        log "  Currently points to: #{indices.join(', ')}"
+        raise "Alias '#{@alias_name}' must point to exactly one index"
+      end
+
+      @new_timestamp = Time.now.strftime('%Y%m%d%H%M%S')
+      @migration_log_index = "#{@alias_name}-#{@new_timestamp}-migration-log"
+      log "Creating log index: #{@migration_log_index}"
+      @client.create_index(@migration_log_index, {}, {})
+      @logger.migration_log_index = @migration_log_index
+      log "Logging to '#{@migration_log_index}'"
+      
+      @current_index = indices.first
+      log "Alias '#{@alias_name}' points to index '#{@current_index}'"
+      
+      @new_index = "#{@alias_name}-#{@new_timestamp}"
+      log "new_index: #{@new_index}"
+
+      @catchup1_index = "#{@new_index}-catchup-1"
+      log "catchup1_index: #{@catchup1_index}"
+
+      @catchup2_index = "#{@new_index}-catchup-2"
+      log "catchup2_index: #{@catchup2_index}"
+
+      # Use current index settings and mappings when creating catchup indexes
+      # so that any reindex painless script logic will apply correctly to them.
+      @current_settings = @client.get_index_settings(@current_index)
+      @current_mappings = @client.get_index_mappings(@current_index)
+      raise "Schema files not found for #{@current_index}" unless @current_settings && @current_mappings
+      # Filter read-only settings
+      @current_settings = SettingsFilter.filter_internal_settings(@current_settings)
+      log "Current settings: #{JSON.generate(@current_settings)}"
+      log "Current mappings: #{JSON.generate(@current_mappings)}"
+
+      @new_settings = SchemaFiles.get_settings(@alias_name)
+      @new_mappings = SchemaFiles.get_mappings(@alias_name)
+      raise "Schema files not found for #{@alias_name}" unless @new_settings && @new_mappings
+      log "New settings: #{JSON.generate(@new_settings)}"
+      log "New mappings: #{JSON.generate(@new_mappings)}"
+
+      @reindex_script = SchemaFiles.get_reindex_script(@alias_name)
+      if @reindex_script
+        log "Using reindex painless script defined for #{@alias_name}"
+        log "reindex.painless script: #{@reindex_script}"
+      end
+    end
+
+    def log(message)
+      @logger.info(message)
+    end
+
+    def migration_steps
+      [
+        MigrationStep.new(
+          name: "STEP 1: Create catchup-1 index",
+          run: ->(logger) { step1_create_catchup1 }
+        ),
+        MigrationStep.new(
+          name: "STEP 2: Configure alias for write to catchup-1",
+          run: ->(logger) { step2_configure_alias_write_catchup1_read_both }
+        ),
+        MigrationStep.new(
+          name: "STEP 3: Reindex to new index",
+          run: ->(logger) { step3_reindex_to_new_index }
+        ),
+        MigrationStep.new(
+          name: "STEP 4: Create catchup-2 index",
+          run: ->(logger) { step4_create_catchup2 }
+        ),
+        MigrationStep.new(
+          name: "STEP 5: Configure alias for write to catchup-2",
+          run: ->(logger) { step5_configure_alias_write_catchup2_read_all }
+        ),
+        MigrationStep.new(
+          name: "STEP 6: Merge catchup-1 to new index",
+          run: ->(logger) { step6_merge_catchup1_to_new }
+        ),
+        MigrationStep.new(
+          name: "STEP 7: Configure alias with no write indexes",
+          run: ->(logger) { step7_configure_alias_no_write }
+        ),
+        MigrationStep.new(
+          name: "STEP 8: Merge catchup-2 to new index",
+          run: ->(logger) { step8_merge_catchup2_to_new }
+        ),
+        MigrationStep.new(
+          name: "STEP 9: Configure alias to new index only",
+          run: ->(logger) { step9_configure_alias_final }
+        ),
+        MigrationStep.new(
+          name: "STEP 10: Close unused indexes",
+          run: ->(logger) { step10_close_unused_indexes }
+        )
+      ]
+    end
+
+    def step1_create_catchup1
+      @client.create_index(@catchup1_index, @current_settings, @current_mappings)
+      log "Created catchup-1 index: #{@catchup1_index}"
+    end
+
+    def step2_configure_alias_write_catchup1_read_both
+      actions = [
+        {
+          add: {
+            index: @catchup1_index,
+            alias: @alias_name,
+            is_write_index: true
+          }
+        },
+        {
+          add: {
+            index: @current_index,
+            alias: @alias_name,
+            is_write_index: false
+          }
+        }
+      ]
+      update_aliases(actions)
+      log "Configured alias #{@alias_name} to write to #{@catchup1_index} and read from both indexes"
+    end
+
+    def update_aliases(actions)
+      response = @client.update_aliases(actions)
+      if response['errors']
+        log "ERROR: Failed to update aliases"
+        log actions
+        log response
+        raise "Failed to update aliases"
+      end
+    end
+    
+    def step3_reindex_to_new_index
+      @client.create_index(@new_index, @new_settings, @new_mappings)
+      begin
+        reindex(@current_index, @new_index, @reindex_script)
+      rescue => e
+        attempt_rollback(e)
+        raise e  # Re-raise the error after rollback
+      end
+    end
+
+    def reindex(current_index, new_index, reindex_script)
+      response = @client.reindex(current_index, new_index, reindex_script)
+      log response
+
+      if response['took']
+        log "Reindex task complete. Took: #{response['took']}"
+        return true
+      end
+      
+      task_id = response['task']
+      if !task_id
+        raise "No task ID from reindex. Reindex incomplete."
+      end
+
+      log "Reindex task started at #{Time.now}. task_id is #{task_id}. Fetch task status with GET #{@client.url}/_tasks/#{task_id}"
+      
+      timeout = 604800 # 1 week
+      @client.wait_for_task(response['task'], timeout)
+      log "Reindex complete"
+    end
+
+    def step4_create_catchup2
+      @client.create_index(@catchup2_index, @current_settings, @current_mappings)
+      log "Created catchup-2 index: #{@catchup2_index}"
+    end
+
+    def step5_configure_alias_write_catchup2_read_all
+      actions = [
+        # keep reading from current_index and catchup1_index
+        # add a new catchup2_index for writes
+        {
+          add: {
+            index: @catchup2_index,
+            alias: @alias_name,
+            is_write_index: true
+          }
+        },
+        {
+          add: {
+            index: @catchup1_index,
+            alias: @alias_name,
+            is_write_index: false
+          }
+        },
+        {
+          add: {
+            index: @current_index,
+            alias: @alias_name,
+            is_write_index: false
+          }
+        }
+      ]
+      update_aliases(actions)
+      log "Configured alias #{@alias_name} to write to #{@catchup2_index} and continue reading from current and catchup1 indexes"
+    end
+
+    def step6_merge_catchup1_to_new
+      reindex(@catchup1_index, @new_index, @reindex_script)
+      log "Catchup-1 merged to new index"
+    end
+
+    def step7_configure_alias_no_write
+      actions = [
+        {
+          add: {
+            index: @catchup2_index,
+            alias: @alias_name,
+            is_write_index: false
+          }
+        },
+        {
+          add: {
+            index: @catchup1_index,
+            alias: @alias_name,
+            is_write_index: false
+          }
+        },
+        {
+          add: {
+            index: @current_index,
+            alias: @alias_name,
+            is_write_index: false
+          }
+        }
+      ]
+      update_aliases(actions)
+      log "Configured alias #{@alias_name} with NO write indexes - writes will fail temporarily"
+    end
+
+    def step8_merge_catchup2_to_new
+      reindex_script = SchemaFiles.get_reindex_script(@alias_name)
+      reindex(@catchup2_index, @new_index, reindex_script)
+    end
+
+    def step9_configure_alias_final
+      actions = [
+        {
+          remove: {
+            index: @catchup2_index,
+            alias: @alias_name
+          }
+        },
+        {
+          remove: {
+            index: @catchup1_index,
+            alias: @alias_name
+          }
+        },
+        {
+          remove: {
+            index: @current_index,
+            alias: @alias_name
+          }
+        },
+        {
+          add: {
+            index: @new_index,
+            alias: @alias_name,
+            is_write_index: true
+          }
+        }
+      ]
+      update_aliases(actions)
+      log "Configured alias #{@alias_name} to write and read from #{@new_index} only"
+    end
+
+    def step10_close_unused_indexes
+      [@current_index, @catchup1_index, @catchup2_index, @migration_log_index].each do |index|
+        if @client.index_exists?(index)
+          log "Closing index: #{index}"
+          @client.close_index(index)
+        end
+      end
+      @migration_log_index = nil
+      @logger.migration_log_index = nil
+    end
+
+    private
+
+    def attempt_rollback(original_error)
+      rollback = Migrate::Rollback.new(@alias_name, @current_index, @catchup1_index, @new_index, @client, self)
+      rollback.attempt_rollback(original_error)
+    end
+
+  end
+end

data/lib/schema_tools/migrate/migrate_new.rb

@@ -0,0 +1,33 @@
+require_relative '../schema_files'
+require_relative 'migrate_breaking_change'
+require_relative '../diff'
+require_relative '../settings_diff'
+require_relative '../api_aware_mappings_diff'
+require 'json'
+
+module SchemaTools
+  def self.migrate_to_new_alias(alias_name, client)
+    timestamp = Time.now.strftime("%Y%m%d%H%M%S")
+    new_index_name = "#{alias_name}-#{timestamp}"
+    
+    settings = SchemaFiles.get_settings(alias_name)
+    mappings = SchemaFiles.get_mappings(alias_name)
+    
+    if settings.nil? || mappings.nil?
+      schema_path = File.join(Config.schemas_path, alias_name)
+      puts "ERROR: Could not load schema files for #{alias_name}"
+      puts "  Make sure settings.json and mappings.json exist in #{schema_path}"
+      raise "Could not load schema files for #{alias_name}"
+    end
+    
+    puts "Creating new index '#{new_index_name}' with provided schema..."
+    client.create_index(new_index_name, settings, mappings)
+    puts "✓ Index '#{new_index_name}' created"
+    
+    puts "Creating alias '#{alias_name}' pointing to '#{new_index_name}'..."
+    client.create_alias(alias_name, new_index_name)
+    puts "✓ Alias '#{alias_name}' created and configured"
+    
+    puts "Migration completed successfully!"
+  end
+end

data/lib/schema_tools/migrate/migrate_non_breaking_change.rb

@@ -0,0 +1,74 @@
+require_relative '../schema_files'
+require_relative 'migrate_verify'
+require_relative '../diff'
+require_relative '../settings_diff'
+require_relative '../api_aware_mappings_diff'
+require 'json'
+
+module SchemaTools
+  def self.attempt_non_breaking_migration(alias_name:, index_name:, client:)
+    settings = SchemaFiles.get_settings(alias_name)
+    mappings = SchemaFiles.get_mappings(alias_name)
+    
+    if settings.nil? || mappings.nil?
+      schema_path = File.join(Config.schemas_path, alias_name)
+      puts "ERROR: Could not load schema files for #{alias_name}"
+      puts "  Make sure settings.json and mappings.json exist in #{schema_path}"
+      raise "Could not load schema files for #{alias_name}"
+    end
+    
+    puts "Checking for differences between local schema and live alias..."
+    diff_result = Diff.generate_schema_diff(alias_name, client)
+    
+    if diff_result[:status] == :no_changes
+      puts "✓ No differences detected between local schema and live alias"
+      puts "✓ Migration skipped - index is already up to date"
+      return
+    end
+    
+    puts "Showing diff between local schema and live alias before migration:"
+    puts "-" * 60
+    Diff.print_schema_diff(diff_result)
+    puts "-" * 60
+    
+    puts "Attempting to update index '#{index_name}' in place with new schema as a non-breaking change..."
+    begin
+      remote_settings = client.get_index_settings(index_name)
+      filtered_remote_settings = SettingsFilter.filter_internal_settings(remote_settings)
+      
+      settings_diff = SettingsDiff.new(settings, filtered_remote_settings)
+      minimal_settings_changes = settings_diff.generate_minimal_changes
+      
+      if minimal_settings_changes.empty?
+        puts "✓ No settings changes needed - settings are already up to date"
+      else
+        puts "Applying minimal settings changes"
+        client.update_index_settings(index_name, minimal_settings_changes)
+        puts "✓ Settings updated successfully"
+      end
+      
+      remote_mappings = client.get_index_mappings(index_name)
+      mappings_diff = ApiAwareMappingsDiff.new(mappings, remote_mappings)
+      minimal_mappings_changes = mappings_diff.generate_minimal_changes
+      
+      if minimal_mappings_changes.empty?
+        puts "✓ No mappings changes needed - mappings are already up to date"
+      else
+        puts "Applying minimal mappings changes"
+        client.update_index_mappings(index_name, minimal_mappings_changes)
+        puts "✓ Mappings updated successfully"
+      end
+      
+      puts "✓ Index '#{index_name}' updated successfully"
+
+      SchemaTools.verify_migration(alias_name, client)
+    rescue => e
+      if e.message.include?("no settings to update")
+        puts "✓ No settings changes needed - index is already up to date"
+        puts "Migration completed successfully!"
+      else
+        raise e
+      end
+    end
+  end
+end

data/lib/schema_tools/migrate/migrate_verify.rb

@@ -0,0 +1,19 @@
+require_relative '../diff'
+
+module SchemaTools
+  def self.verify_migration(alias_name, client)
+    puts "Verifying migration by comparing local schema with remote index..."
+    diff_result = Diff.generate_schema_diff(alias_name, client)
+    
+    if diff_result[:status] == :no_changes
+      puts "✓ Migration verification successful - no differences detected"
+      puts "Migration completed successfully!"
+    else
+      puts "⚠️  Migration verification failed - differences detected:"
+      puts "-" * 60
+      Diff.print_schema_diff(diff_result)
+      puts "-" * 60
+      raise "Migration verification failed - local schema does not match remote index after migration"
+    end
+  end
+end

data/lib/schema_tools/migrate/migration_step.rb

@@ -0,0 +1,36 @@
+module SchemaTools
+  class MigrationStep
+    attr_reader :name, :before_actions, :run_actions, :after_actions
+
+    def initialize(name:, run:)
+      @name = name
+      @before_actions = []
+      @run_actions = [run]
+      @after_actions = []
+      add_default_logging
+    end
+
+    def add_before(action)
+      @before_actions << action
+      self
+    end
+
+    def add_after(action)
+      @after_actions << action
+      self
+    end
+
+    def execute(logger)
+      @before_actions.each { |action| action.call(logger) }
+      @run_actions.each { |action| action.call(logger) }
+      @after_actions.each { |action| action.call(logger) }
+    end
+
+    private
+
+    def add_default_logging
+      @before_actions << ->(logger) { logger.log("\nSTARTING: #{@name}") }
+      @after_actions << ->(logger) { logger.log("COMPLETED: #{@name}") }
+    end
+  end
+end