schema-tools 1.0.0

data/lib/schema_tools/migrate/rollback.rb

@@ -0,0 +1,211 @@
+module SchemaTools
+  module Migrate
+    class Rollback
+      def initialize(alias_name, current_index, catchup1_index, new_index, client, logger)
+        @alias_name = alias_name
+        @current_index = current_index
+        @catchup1_index = catchup1_index
+        @new_index = new_index
+        @client = client
+        @logger = logger
+      end
+
+      def attempt_rollback(original_error)
+        @logger.log "=" * 60
+        @logger.log "🔄 ATTEMPTING ROLLBACK DUE TO STEP 3 FAILURE"
+        @logger.log "=" * 60
+        @logger.log "Original error: #{original_error.message}"
+        @logger.log ""
+        @logger.log "Rolling back to original state..."
+        @logger.log "This will preserve any data written during migration and restore the alias to the original index."
+        @logger.log ""
+        
+        begin
+          # Step 1: Stop writes by making alias read-only
+          @logger.log "🔄 ROLLBACK STEP 1: Stopping writes to prevent data loss..."
+          stop_writes
+          
+          # Step 2: Reindex catchup changes back to original index
+          @logger.log "🔄 ROLLBACK STEP 2: Reindexing catchup changes back to original index..."
+          reindex_catchup_to_original
+          
+          # Step 3: Restore alias to original state
+          @logger.log "🔄 ROLLBACK STEP 3: Restoring alias to original index..."
+          restore_alias_to_original
+          
+          # Step 4: Clean up created indexes
+          @logger.log "🔄 ROLLBACK STEP 4: Cleaning up created indexes..."
+          cleanup_indexes
+          
+          @logger.log "=" * 60
+          @logger.log "✅ ROLLBACK COMPLETED SUCCESSFULLY"
+          @logger.log "=" * 60
+          @logger.log "The alias '#{@alias_name}' has been restored to point to the original index '#{@current_index}'"
+          @logger.log "All data written during migration has been preserved in the original index."
+          @logger.log "All created indexes have been cleaned up."
+          @logger.log "You can now:"
+          @logger.log "  1. Fix the issue that caused the reindex to fail"
+          @logger.log "  2. Re-run the migration: rake 'schema:migrate[#{@alias_name}]'"
+          @logger.log "  3. Check the migration log: #{@logger.instance_variable_get(:@migration_log_index)}"
+          @logger.log ""
+          
+        rescue => rollback_error
+          @logger.log "=" * 60
+          @logger.log "❌ ROLLBACK FAILED"
+          @logger.log "=" * 60
+          @logger.log "Rollback error: #{rollback_error.message}"
+          @logger.log ""
+          log_rollback_instructions(original_error, rollback_error)
+        end
+      end
+
+      private
+
+      def stop_writes
+        # Configure alias to read-only: read from both original and catchup, write to neither
+        actions = []
+        
+        # Remove write access from catchup-1 index
+        actions << {
+          remove: {
+            index: @catchup1_index,
+            alias: @alias_name
+          }
+        }
+        
+        # Add read-only access to original index
+        actions << {
+          add: {
+            index: @current_index,
+            alias: @alias_name,
+            is_write_index: false
+          }
+        }
+        
+        # Add read-only access to catchup-1 index (if it exists)
+        if @client.index_exists?(@catchup1_index)
+          actions << {
+            add: {
+              index: @catchup1_index,
+              alias: @alias_name,
+              is_write_index: false
+            }
+          }
+        end
+        
+        @client.update_aliases(actions)
+        @logger.log "✓ Writes stopped - alias is now read-only"
+      end
+
+      def reindex_catchup_to_original
+        # Check if catchup-1 index has any documents
+        doc_count = @client.get_index_doc_count(@catchup1_index)
+        
+        if doc_count > 0
+          @logger.log "📊 Found #{doc_count} documents in catchup-1 index - reindexing to original..."
+          
+          # Reindex from catchup-1 to original index
+          response = @client.reindex(@catchup1_index, @current_index, nil)
+          @logger.log "Reindex task started - task_id: #{response['task']}"
+          
+          # Wait for reindex to complete
+          @client.wait_for_task(response['task'])
+          @logger.log "✓ Catchup data successfully reindexed to original index"
+        else
+          @logger.log "✓ No documents in catchup-1 index - skipping reindex"
+        end
+      end
+
+      def restore_alias_to_original
+        # Remove all aliases and restore to original state only
+        actions = []
+        
+        # Remove alias from catchup-1 index (if it exists)
+        if @client.index_exists?(@catchup1_index)
+          actions << {
+            remove: {
+              index: @catchup1_index,
+              alias: @alias_name
+            }
+          }
+        end
+        
+        # Add alias back to original index only
+        actions << {
+          add: {
+            index: @current_index,
+            alias: @alias_name,
+            is_write_index: true
+          }
+        }
+        
+        @client.update_aliases(actions)
+        @logger.log "✓ Alias restored to original index: #{@current_index}"
+      end
+
+      def cleanup_indexes
+        # Clean up catchup-1 index
+        if @client.index_exists?(@catchup1_index)
+          @client.delete_index(@catchup1_index)
+          @logger.log "✓ Deleted catchup-1 index: #{@catchup1_index}"
+        else
+          @logger.log "⚠️  Catchup-1 index does not exist: #{@catchup1_index}"
+        end
+        
+        # Clean up new index if it was created
+        if @client.index_exists?(@new_index)
+          @client.delete_index(@new_index)
+          @logger.log "✓ Deleted new index: #{@new_index}"
+        else
+          @logger.log "⚠️  New index does not exist: #{@new_index}"
+        end
+      end
+
+      def log_rollback_instructions(original_error, rollback_error = nil)
+        @logger.log "=" * 60
+        @logger.log "❌ MANUAL ROLLBACK REQUIRED"
+        @logger.log "=" * 60
+        @logger.log "The automatic rollback failed. You need to manually restore the system."
+        @logger.log ""
+        @logger.log "Current state:"
+        @logger.log "  - Alias: #{@alias_name}"
+        @logger.log "  - Original index: #{@current_index}"
+        @logger.log "  - Catchup-1 index: #{@catchup1_index}"
+        @logger.log "  - New index: #{@new_index}"
+        @logger.log ""
+        @logger.log "Manual rollback steps:"
+        @logger.log "1. Stop writes by making alias read-only:"
+        @logger.log "   curl -X POST 'http://localhost:9200/_aliases' -H 'Content-Type: application/json' -d '{"
+        @logger.log "     \"actions\": ["
+        @logger.log "       { \"remove\": { \"index\": \"#{@catchup1_index}\", \"alias\": \"#{@alias_name}\" } },"
+        @logger.log "       { \"add\": { \"index\": \"#{@current_index}\", \"alias\": \"#{@alias_name}\", \"is_write_index\": false } }"
+        @logger.log "     ]"
+        @logger.log "   }'"
+        @logger.log ""
+        @logger.log "2. Reindex catchup data to original (if needed):"
+        @logger.log "   curl -X POST 'http://localhost:9200/_reindex' -H 'Content-Type: application/json' -d '{"
+        @logger.log "     \"source\": { \"index\": \"#{@catchup1_index}\" },"
+        @logger.log "     \"dest\": { \"index\": \"#{@current_index}\" }"
+        @logger.log "   }'"
+        @logger.log ""
+        @logger.log "3. Restore alias to original index:"
+        @logger.log "   curl -X POST 'http://localhost:9200/_aliases' -H 'Content-Type: application/json' -d '{"
+        @logger.log "     \"actions\": ["
+        @logger.log "       { \"remove\": { \"index\": \"#{@catchup1_index}\", \"alias\": \"#{@alias_name}\" } },"
+        @logger.log "       { \"add\": { \"index\": \"#{@current_index}\", \"alias\": \"#{@alias_name}\", \"is_write_index\": true } }"
+        @logger.log "     ]"
+        @logger.log "   }'"
+        @logger.log ""
+        @logger.log "4. Clean up created indexes:"
+        @logger.log "   curl -X DELETE 'http://localhost:9200/#{@catchup1_index}'"
+        @logger.log "   curl -X DELETE 'http://localhost:9200/#{@new_index}'"
+        @logger.log ""
+        @logger.log "Original error: #{original_error.message}"
+        if rollback_error
+          @logger.log "Rollback error: #{rollback_error.message}"
+        end
+        @logger.log ""
+      end
+    end
+  end
+end

data/lib/schema_tools/new_alias.rb

@@ -0,0 +1,165 @@
+require 'json'
+require 'fileutils'
+require 'time'
+require_relative 'config'
+require_relative 'settings_filter'
+
+module SchemaTools
+  def self.new_alias(client:)
+    puts "Warning: This tool only supports migrating aliases."
+    puts "Create an alias for this index by running:"
+    puts "rake schema:alias"
+    puts "\nDownload this index schema anyway? Y/n"
+    
+    choice = STDIN.gets&.chomp&.downcase
+    if choice.nil? || choice.empty? || choice == 'y' || choice == 'yes'
+      # User wants to proceed with creating a new alias
+      puts "\nEnter a new alias name:"
+      alias_name = STDIN.gets&.chomp
+      if alias_name.nil? || alias_name.empty?
+        puts "No alias name provided. Exiting."
+        exit 1
+      end
+      
+      if client.alias_exists?(alias_name)
+        puts "Alias '#{alias_name}' already exists."
+        exit 1
+      end
+      
+      timestamp = Time.now.strftime("%Y%m%d%H%M%S")
+      index_name = "#{alias_name}-#{timestamp}"
+      
+      puts "Creating index '#{index_name}' with alias '#{alias_name}'..."
+      
+      sample_settings = {
+        "number_of_shards" => 1,
+        "number_of_replicas" => 0,
+        "analysis" => {
+          "analyzer" => {
+            "default" => {
+              "type" => "standard"
+            }
+          }
+        }
+      }
+      
+      sample_mappings = {
+        "properties" => {
+          "id" => {
+            "type" => "keyword"
+          },
+          "created_at" => {
+            "type" => "date"
+          },
+          "updated_at" => {
+            "type" => "date"
+          }
+        }
+      }
+      
+      client.create_index(index_name, sample_settings, sample_mappings)
+      client.create_alias(alias_name, index_name)
+      
+      puts "✓ Created index '#{index_name}' with alias '#{alias_name}'"
+      
+      schema_path = File.join(Config.schemas_path, alias_name)
+      FileUtils.mkdir_p(schema_path)
+      
+      settings_file = File.join(schema_path, 'settings.json')
+      mappings_file = File.join(schema_path, 'mappings.json')
+      
+      File.write(settings_file, JSON.pretty_generate(sample_settings))
+      File.write(mappings_file, JSON.pretty_generate(sample_mappings))
+      
+      puts "✓ Sample schema created at #{schema_path}"
+      puts "  - settings.json"
+      puts "  - mappings.json"
+    else
+      puts "Exiting. Run 'rake schema:alias' to create an alias for an existing index."
+    end
+  end
+
+  def self.create_alias_for_index(client:)
+    aliases = client.list_aliases
+    indices = client.list_indices
+    
+    unaliased_indices = indices.reject { |index| aliases.values.flatten.include?(index) || index.start_with?('.') || client.index_closed?(index) }
+    
+    puts "\nIndexes not part of any aliases:"
+    if unaliased_indices.empty?
+      puts "  (none)"
+      puts "\nNo unaliased indices available to create aliases for."
+      return
+    end
+    
+    unaliased_indices.each_with_index do |index_name, index|
+      puts "  #{index + 1}. #{index_name}"
+    end
+    
+    puts "\nPlease choose an index to create an alias for:"
+    puts "Enter the number (1-#{unaliased_indices.length}):"
+    
+    choice = STDIN.gets&.chomp
+    if choice.nil?
+      puts "No input provided. Exiting."
+      exit 1
+    end
+    
+    choice_num = choice.to_i
+    if choice_num < 1 || choice_num > unaliased_indices.length
+      puts "Invalid choice. Please enter a number between 1 and #{unaliased_indices.length}."
+      exit 1
+    end
+    
+    selected_index = unaliased_indices[choice_num - 1]
+    
+    puts "\nType the name of a new alias to create for this index:"
+    new_alias_name = STDIN.gets&.chomp
+    if new_alias_name.nil? || new_alias_name.empty?
+      puts "No alias name provided. Exiting."
+      exit 1
+    end
+    
+    if client.alias_exists?(new_alias_name)
+      puts "Alias '#{new_alias_name}' already exists."
+      exit 1
+    end
+    
+    puts "Creating alias '#{new_alias_name}' for index '#{selected_index}'..."
+    client.create_alias(new_alias_name, selected_index)
+    
+    puts "✓ Created alias '#{new_alias_name}' -> '#{selected_index}'"
+    
+    # Download the schema for the newly aliased index
+    download_schema(new_alias_name, selected_index, client)
+  end
+
+  private
+
+  def self.download_schema(folder_name, index_name, client)
+    settings = client.get_index_settings(index_name)
+    mappings = client.get_index_mappings(index_name)
+    
+    if settings.nil? || mappings.nil?
+      puts "Failed to retrieve settings or mappings for #{index_name}"
+      exit 1
+    end
+    
+    # Filter out internal settings
+    filtered_settings = SettingsFilter.filter_internal_settings(settings)
+    
+    schema_path = File.join(Config.schemas_path, folder_name)
+    FileUtils.mkdir_p(schema_path)
+    
+    settings_file = File.join(schema_path, 'settings.json')
+    mappings_file = File.join(schema_path, 'mappings.json')
+    
+    File.write(settings_file, JSON.pretty_generate(filtered_settings))
+    File.write(mappings_file, JSON.pretty_generate(mappings))
+    
+    puts "✓ Schema downloaded to #{schema_path}"
+    puts "  - settings.json"
+    puts "  - mappings.json"
+  end
+
+end

data/lib/schema_tools/painless_scripts_delete.rb

@@ -0,0 +1,21 @@
+module SchemaTools
+  def self.painless_scripts_delete(script_name:, client:)
+    raise "script_name parameter is required" unless script_name && !script_name.strip.empty?
+    
+    # Remove .painless extension if provided
+    script_name = script_name.gsub(/\.painless$/, '')
+    
+    puts "Deleting painless script '#{script_name}' from cluster..."
+    
+    begin
+      client.delete_script(script_name)
+      puts "Successfully deleted painless script '#{script_name}' from cluster"
+    rescue => e
+      if e.message.include?('404') || e.message.include?('not found')
+        puts "Script '#{script_name}' not found in cluster"
+      else
+        raise e
+      end
+    end
+  end
+end

data/lib/schema_tools/painless_scripts_download.rb

@@ -0,0 +1,26 @@
+require 'fileutils'
+
+module SchemaTools
+  def self.painless_scripts_download(client:)
+    painless_scripts_path = Config.painless_scripts_path
+    
+    puts "Downloading all painless scripts from cluster..."
+    
+    scripts = client.get_stored_scripts
+    
+    if scripts.empty?
+      puts "No painless scripts found in cluster."
+      return
+    end
+    
+    FileUtils.mkdir_p(painless_scripts_path)
+    
+    scripts.each do |script_name, script_content|
+      script_file_path = File.join(painless_scripts_path, "#{script_name}.painless")
+      File.write(script_file_path, script_content)
+      puts "Downloaded script: #{script_name}"
+    end
+    
+    puts "Successfully downloaded #{scripts.length} painless script(s) to #{painless_scripts_path}"
+  end
+end

data/lib/schema_tools/painless_scripts_upload.rb

@@ -0,0 +1,31 @@
+require 'fileutils'
+
+module SchemaTools
+  def self.painless_scripts_upload(client:)
+    painless_scripts_path = Config.painless_scripts_path
+    
+    unless Dir.exist?(painless_scripts_path)
+      puts "Painless scripts directory #{painless_scripts_path} does not exist."
+      return
+    end
+    
+    puts "Uploading all painless scripts from #{painless_scripts_path} to cluster..."
+    
+    script_files = Dir.glob(File.join(painless_scripts_path, '*.painless'))
+    
+    if script_files.empty?
+      puts "No painless script files found in #{painless_scripts_path}"
+      return
+    end
+    
+    script_files.each do |script_file_path|
+      script_name = File.basename(script_file_path, '.painless')
+      script_content = File.read(script_file_path)
+      
+      client.put_script(script_name, script_content)
+      puts "Uploaded script: #{script_name}"
+    end
+    
+    puts "Successfully uploaded #{script_files.length} painless script(s) to cluster"
+  end
+end

data/lib/schema_tools/rake_tasks.rb

@@ -0,0 +1,15 @@
+require 'rake'
+require 'rake/tasklib'
+
+module SchemaTools
+  module RakeTasks
+    def self.load_tasks
+      Dir.glob(File.join(File.dirname(__FILE__), '..', 'tasks', '*.rake')).each do |rake_file|
+        load rake_file
+      end
+    end
+  end
+end
+
+# Auto-load tasks when this file is required
+SchemaTools::RakeTasks.load_tasks

data/lib/schema_tools/schema_files.rb

@@ -0,0 +1,53 @@
+require 'json'
+require 'fileutils'
+
+module SchemaTools
+  class SchemaFiles
+    def self.get_settings(alias_name)
+      settings_path = File.join(Config.schemas_path, alias_name, 'settings.json')
+      return nil unless File.exist?(settings_path)
+      
+      JSON.parse(File.read(settings_path))
+    end
+
+    def self.get_mappings(alias_name)
+      mappings_path = File.join(Config.schemas_path, alias_name, 'mappings.json')
+      return nil unless File.exist?(mappings_path)
+      
+      JSON.parse(File.read(mappings_path))
+    end
+
+    def self.get_reindex_script(alias_name)
+      script_path = File.join(Config.schemas_path, alias_name, 'reindex.painless')
+      
+      File.exist?(script_path) ? File.read(script_path) : nil
+    end
+
+    def self.discover_all_schemas
+      return [] unless Dir.exist?(Config.schemas_path)
+      
+      schemas = []
+      
+      Dir.glob(File.join(Config.schemas_path, '*'))
+         .select { |d| File.directory?(d) }
+         .each do |schema_dir|
+        alias_name = File.basename(schema_dir)
+        
+        if has_schema_files?(alias_name)
+          schemas << alias_name
+        end
+      end
+      
+      schemas
+    end
+
+    private
+
+    def self.has_schema_files?(alias_name)
+      settings_path = File.join(Config.schemas_path, alias_name, 'settings.json')
+      mappings_path = File.join(Config.schemas_path, alias_name, 'mappings.json')
+      
+      File.exist?(settings_path) && File.exist?(mappings_path)
+    end
+  end
+end

data/lib/schema_tools/seed.rb

@@ -0,0 +1,64 @@
+module SchemaTools
+  def self.seed(client:)
+    # List available indices (connection already validated during client initialization)
+    puts "Connecting to #{Config.connection_url}..."
+    indices = client.list_indices
+    
+    if indices.empty?
+      puts "No indices found in the cluster."
+      puts "Please create an index first."
+      exit 0
+    end
+    
+    puts "Available indices:"
+    indices.each_with_index do |index_name, index|
+      puts "#{index + 1}. #{index_name}"
+    end
+    
+    puts "\nPlease select an index by number (1-#{indices.length}):"
+    selection_input = STDIN.gets&.chomp
+    if selection_input.nil?
+      puts "No input provided. Exiting."
+      exit 1
+    end
+    selection = selection_input.to_i
+    
+    if selection < 1 || selection > indices.length
+      puts "Invalid selection. Please run the task again and select a valid number."
+      exit 1
+    end
+    
+    selected_index = indices[selection - 1]
+    puts "Selected index: #{selected_index}"
+    
+    # Fetch the mappings for the selected index
+    puts "Fetching mappings for #{selected_index}..."
+    mappings = client.get_index_mappings(selected_index)
+    
+    if mappings.nil?
+      puts "Failed to fetch mappings for #{selected_index}"
+      exit 1
+    end
+    
+    puts "Mappings fetched successfully."
+    
+    # Prompt user for number of documents to seed
+    puts "\nHow many documents would you like to seed?"
+    num_docs_input = STDIN.gets&.chomp
+    if num_docs_input.nil?
+      puts "No input provided. Exiting."
+      exit 1
+    end
+    
+    num_docs = num_docs_input.to_i
+    if num_docs <= 0
+      puts "Invalid number of documents. Please enter a positive integer."
+      exit 1
+    end
+    
+    puts "Seeding #{num_docs} documents from #{selected_index}..."
+    
+    # Call the seeding function
+    Seed.seed_data(num_docs, mappings, client, selected_index)
+  end
+end

data/lib/schema_tools/settings_diff.rb

@@ -0,0 +1,64 @@
+require 'json'
+require_relative 'diff'
+
+module SchemaTools
+  class SettingsDiff
+    def initialize(local_schema, remote_schema)
+      @local_schema = local_schema
+      @remote_schema = remote_schema
+    end
+
+    def generate_minimal_changes
+      return {} unless @local_schema.is_a?(Hash)
+      
+      # Normalize local schema to always have "index" wrapper
+      local_index = normalize_local_schema(@local_schema)
+      return {} if local_index.nil?
+      
+      remote_index = @remote_schema.is_a?(Hash) && @remote_schema.key?("index") ? @remote_schema["index"] : {}
+      
+      # Normalize both sides to ensure consistent comparison
+      normalized_remote = Diff.normalize_values(remote_index)
+      normalized_local = Diff.normalize_values(local_index)
+      
+      changes = find_changes(normalized_remote, normalized_local)
+      changes.empty? ? {} : { "index" => changes }
+    end
+
+    private
+
+    def normalize_local_schema(local_schema)
+      # If local schema already has "index" wrapper, use it only if it's a valid hash
+      if local_schema.key?("index")
+        return local_schema["index"] if local_schema["index"].is_a?(Hash)
+        # If index exists but is not a hash, return nil to indicate invalid schema
+        return nil
+      end
+      
+      # If local schema doesn't have "index" wrapper, treat the entire schema as index settings
+      # This handles cases like { "number_of_shards": 1 } which is equivalent to { "index": { "number_of_shards": 1 } }
+      return local_schema
+    end
+
+    def find_changes(remote, local)
+      changes = {}
+      
+      return changes unless local.is_a?(Hash) && remote.is_a?(Hash)
+      
+      local.each do |key, value|
+        if !remote.key?(key)
+          changes[key] = value
+        elsif value != remote[key]
+          if value.is_a?(Hash) && remote[key].is_a?(Hash)
+            nested_changes = find_changes(remote[key], value)
+            changes[key] = nested_changes unless nested_changes.empty?
+          else
+            changes[key] = value
+          end
+        end
+      end
+      
+      changes
+    end
+  end
+end

data/lib/schema_tools/settings_filter.rb

@@ -0,0 +1,27 @@
+require 'json'
+
+module SchemaTools
+  class SettingsFilter
+    # Remove read-only OpenSearch/Elasticsearch internal fields
+    def self.filter_internal_settings(settings)
+      return settings unless settings.is_a?(Hash)
+      
+      filtered_settings = JSON.parse(JSON.generate(settings))
+      
+      internal_fields = [
+        'creation_date',
+        'provided_name', 
+        'uuid',
+        'version'
+      ]
+      
+      if filtered_settings['index']
+        internal_fields.each do |field|
+          filtered_settings['index'].delete(field)
+        end
+      end
+      
+      filtered_settings
+    end
+  end
+end