schema-tools 1.0.0

data/lib/schema_tools/diff.rb

@@ -0,0 +1,263 @@
+require 'json'
+require_relative 'json_diff'
+require_relative 'schema_files'
+require_relative 'client'
+require_relative 'settings_filter'
+
+module SchemaTools
+  def self.diff_all_schemas(client:)
+    Diff.diff_all_schemas(client)
+  end
+
+  def self.diff_schema(alias_name, client:)
+    print_schema_diff(Diff.generate_schema_diff(alias_name, client))
+    nil  # Console output method, returns nil
+  end
+
+  class Diff
+    # Compare all schemas to their corresponding downloaded alias settings and mappings
+    def self.diff_all_schemas(client)
+      schemas = SchemaFiles.discover_all_schemas
+      
+      if schemas.empty?
+        puts "No schemas found in #{Config.schemas_path}"
+        return
+      end
+
+      puts "Found #{schemas.length} schema(s) to compare:"
+      schemas.each do |schema|
+        puts "  - #{schema}"
+      end
+      puts
+
+      schemas.each do |alias_name|
+        begin
+          print_schema_diff(generate_schema_diff(alias_name, client))
+        rescue => e
+          puts "✗ Diff failed for #{alias_name}: #{e.message}"
+        end
+        puts
+      end
+    end
+
+    # Generate a nicely formatted diff representation for a single schema
+    # Compare a single schema to its corresponding downloaded alias settings and mappings
+    def self.generate_schema_diff(alias_name, client)
+      result = {
+        alias_name: alias_name,
+        status: nil,
+        settings_diff: nil,
+        mappings_diff: nil,
+        error: nil
+      }
+      json_diff = JsonDiff.new
+
+      begin
+        unless client.alias_exists?(alias_name)
+          result[:status] = :alias_not_found
+          result[:error] = "Alias '#{alias_name}' not found in cluster"
+          return result
+        end
+
+        alias_indices = client.get_alias_indices(alias_name)
+        
+        if alias_indices.length > 1
+          result[:status] = :multiple_indices
+          result[:error] = "Alias '#{alias_name}' points to multiple indices: #{alias_indices.join(', ')}. This configuration is not supported for diffing."
+          return result
+        end
+
+        index_name = alias_indices.first
+
+        local_settings = SchemaFiles.get_settings(alias_name)
+        local_mappings = SchemaFiles.get_mappings(alias_name)
+
+        if local_settings.nil? || local_mappings.nil?
+          result[:status] = :local_files_not_found
+          result[:error] = "Local schema files not found for #{alias_name}"
+          return result
+        end
+
+        remote_settings = client.get_index_settings(index_name)
+        remote_mappings = client.get_index_mappings(index_name)
+
+        if remote_settings.nil? || remote_mappings.nil?
+          result[:status] = :remote_fetch_failed
+          result[:error] = "Failed to retrieve remote settings or mappings for #{index_name}"
+          return result
+        end
+
+        # Filter remote settings to match local format
+        filtered_remote_settings = SettingsFilter.filter_internal_settings(remote_settings)
+
+        # Normalize both local and remote settings to ensure consistent comparison
+        normalized_local_settings = self.normalize_local_settings(local_settings)
+        normalized_remote_settings = self.normalize_remote_settings(filtered_remote_settings)
+
+        result[:settings_diff] = json_diff.generate_diff(normalized_remote_settings, normalized_local_settings)
+        result[:mappings_diff] = json_diff.generate_diff(remote_mappings, local_mappings)
+        
+        result[:comparison_context] = {
+          new_files: {
+            settings: "#{alias_name}/settings.json",
+            mappings: "#{alias_name}/mappings.json"
+          },
+          old_api: {
+            settings: "GET /#{index_name}/_settings",
+            mappings: "GET /#{index_name}/_mappings"
+          }
+        }
+        
+        if result[:settings_diff] == "No changes detected" && result[:mappings_diff] == "No changes detected"
+          result[:status] = :no_changes
+        else
+          result[:status] = :changes_detected
+        end
+
+      rescue => e
+        result[:status] = :error
+        result[:error] = e.message
+      end
+
+      result
+    end
+
+    # print_schema_diff(generate_schema_diff(alias_name))
+    def self.print_schema_diff(schema_diff)
+      puts "=" * 60
+      puts "Comparing schema: #{schema_diff[:alias_name]}"
+      puts "=" * 60
+
+      # Handle errors by printing and returning
+      if schema_diff[:status] == :alias_not_found
+        puts "❌ #{schema_diff[:error]}"
+        return
+      elsif schema_diff[:status] == :multiple_indices
+        puts "⚠️  #{schema_diff[:error]}"
+        return
+      elsif schema_diff[:status] == :local_files_not_found
+        puts "❌ #{schema_diff[:error]}"
+        return
+      elsif schema_diff[:status] == :remote_fetch_failed
+        puts "❌ #{schema_diff[:error]}"
+        return
+      elsif schema_diff[:status] == :error
+        puts "❌ Error: #{schema_diff[:error]}"
+        return
+      end
+
+      # Show what's being compared
+      puts "New (Local Files):"
+      puts "   #{schema_diff[:comparison_context][:new_files][:settings]}"
+      puts "   #{schema_diff[:comparison_context][:new_files][:mappings]}"
+      puts
+      puts "Old (Remote API):"
+      puts "   #{schema_diff[:comparison_context][:old_api][:settings]}"
+      puts "   #{schema_diff[:comparison_context][:old_api][:mappings]}"
+      puts
+
+      # Display the diffs
+      puts "Settings Comparison:"
+      puts schema_diff[:settings_diff]
+      puts
+
+      puts "Mappings Comparison:"
+      puts schema_diff[:mappings_diff]
+    end
+
+    private
+
+    def self.normalize_local_settings(local_settings)
+      return local_settings unless local_settings.is_a?(Hash)
+      
+      # If local settings already have "index" wrapper, use it
+      if local_settings.key?("index")
+        if local_settings["index"].is_a?(Hash)
+          # Normalize the index settings and return
+          normalized_index = normalize_values(local_settings["index"])
+          return { "index" => normalized_index }
+        else
+          # If index exists but is not a hash, return as-is (invalid format)
+          return local_settings
+        end
+      end
+      
+      # If local settings are empty, don't add index wrapper
+      # This prevents empty {} from becoming { "index" => {} }
+      return local_settings if local_settings.empty?
+      
+      # If local settings don't have "index" wrapper, wrap them in "index"
+      # This handles cases like { "number_of_shards": 1 } which should be compared as { "index": { "number_of_shards": 1 } }
+      normalized_settings = normalize_values(local_settings)
+      { "index" => normalized_settings }
+    end
+
+    # Normalize remote settings to ensure consistent comparison
+    # Remote settings may come back as strings from ES API, so normalize them too
+    def self.normalize_remote_settings(remote_settings)
+      return remote_settings unless remote_settings.is_a?(Hash)
+      
+      # If remote settings already have "index" wrapper, normalize it
+      if remote_settings.key?("index")
+        if remote_settings["index"].is_a?(Hash)
+          normalized_index = normalize_values(remote_settings["index"])
+          return { "index" => normalized_index }
+        else
+          return remote_settings
+        end
+      end
+      
+      # If remote settings don't have "index" wrapper, normalize and wrap them
+      normalized_settings = normalize_values(remote_settings)
+      { "index" => normalized_settings }
+    end
+
+    # Normalize string values to their proper types for Elasticsearch comparison
+    # Handles the gotcha cases where ES accepts string inputs but returns canonical types
+    def self.normalize_values(obj)
+      case obj
+      when Hash
+        normalized = {}
+        obj.each do |key, value|
+          normalized[key] = normalize_values(value)
+        end
+        normalized
+      when Array
+        obj.map { |item| normalize_values(item) }
+      when String
+        normalize_string_value(obj)
+      else
+        obj
+      end
+    end
+
+    # Convert string values to their proper types based on Elasticsearch behavior
+    def self.normalize_string_value(str)
+      # Handle boolean values
+      case str.downcase
+      when "true"
+        true
+      when "false"
+        false
+      when "1"
+        # Could be boolean true or numeric 1, default to numeric for settings
+        1
+      when "0"
+        # Could be boolean false or numeric 0, default to numeric for settings
+        0
+      else
+        # Handle numeric strings
+        if str.match?(/\A-?\d+\z/)
+          # Integer string
+          str.to_i
+        elsif str.match?(/\A-?\d*\.\d+\z/)
+          # Float string
+          str.to_f
+        else
+          # Keep as string if it doesn't match any pattern
+          str
+        end
+      end
+    end
+  end
+end

data/lib/schema_tools/download.rb

@@ -0,0 +1,114 @@
+require 'json'
+require 'fileutils'
+require_relative 'config'
+require_relative 'settings_filter'
+
+module SchemaTools
+  def self.download(client:)
+    aliases = client.list_aliases
+    indices = client.list_indices
+    
+    single_aliases = aliases.select { |alias_name, indices| indices.length == 1 && !alias_name.start_with?('.') }
+    multi_aliases = aliases.select { |alias_name, indices| indices.length > 1 && !alias_name.start_with?('.') }
+    unaliased_indices = indices.reject { |index| aliases.values.flatten.include?(index) || index.start_with?('.') || client.index_closed?(index) }
+    
+    # Create a combined list with sequential numbering
+    options = []
+    
+    puts "\nAliases pointing to 1 index:"
+    if single_aliases.empty?
+      puts "  (none)"
+    else
+      single_aliases.each_with_index do |(alias_name, indices), index|
+        option_number = options.length + 1
+        options << { type: :alias, name: alias_name, index: indices.first }
+        puts "  #{option_number}. #{alias_name} -> #{indices.first}"
+      end
+    end
+    
+    puts "\nIndexes not part of any aliases:"
+    if unaliased_indices.empty?
+      puts "  (none)"
+    else
+      unaliased_indices.each_with_index do |index_name, index|
+        option_number = options.length + 1
+        options << { type: :index, name: index_name, index: index_name }
+        puts "  #{option_number}. #{index_name}"
+      end
+    end
+    
+    if multi_aliases.any?
+      puts "\nAliases pointing to more than 1 index (cannot choose):"
+      multi_aliases.each do |alias_name, indices|
+        puts "  - #{alias_name} -> #{indices.join(', ')}"
+      end
+    end
+    
+    if options.empty?
+      puts "\nNo aliases or indices available to download."
+      return
+    end
+    
+    puts "\nPlease choose an alias or index to download:"
+    puts "Enter the number (1-#{options.length}):"
+    
+    choice = STDIN.gets&.chomp
+    if choice.nil?
+      puts "No input provided. Exiting."
+      exit 1
+    end
+    
+    choice_num = choice.to_i
+    if choice_num < 1 || choice_num > options.length
+      puts "Invalid choice. Please enter a number between 1 and #{options.length}."
+      exit 1
+    end
+    
+    selected_option = options[choice_num - 1]
+    
+    if selected_option[:type] == :alias
+      download_alias(selected_option[:name], selected_option[:index], client)
+    else
+      download_index(selected_option[:name], client)
+    end
+  end
+
+  private
+
+  def self.download_alias(alias_name, index_name, client)
+    puts "Downloading alias '#{alias_name}' (index: #{index_name})..."
+    download_schema(alias_name, index_name, client)
+  end
+
+  def self.download_index(index_name, client)
+    puts "Downloading index '#{index_name}'..."
+    download_schema(index_name, index_name, client)
+  end
+
+  def self.download_schema(folder_name, index_name, client)
+    settings = client.get_index_settings(index_name)
+    mappings = client.get_index_mappings(index_name)
+    
+    if settings.nil? || mappings.nil?
+      puts "Failed to retrieve settings or mappings for #{index_name}"
+      exit 1
+    end
+    
+    # Filter out internal settings
+    filtered_settings = SettingsFilter.filter_internal_settings(settings)
+    
+    schema_path = File.join(Config.schemas_path, folder_name)
+    FileUtils.mkdir_p(schema_path)
+    
+    settings_file = File.join(schema_path, 'settings.json')
+    mappings_file = File.join(schema_path, 'mappings.json')
+    
+    File.write(settings_file, JSON.pretty_generate(filtered_settings))
+    File.write(mappings_file, JSON.pretty_generate(mappings))
+    
+    puts "✓ Schema downloaded to #{schema_path}"
+    puts "  - settings.json"
+    puts "  - mappings.json"
+  end
+
+end

data/lib/schema_tools/json_diff.rb

@@ -0,0 +1,234 @@
+require 'json'
+
+module SchemaTools
+  class JsonDiff
+    def initialize()      
+    end
+
+    # Keys to ignore in diff comparisons (noisy metadata)
+    IGNORED_KEYS = [].freeze
+
+    # Generate a detailed diff between two JSON objects
+    # Returns a formatted string showing additions, removals, and modifications
+    def generate_diff(old_json, new_json, context: {})
+      old_normalized = normalize_json(old_json)
+      new_normalized = normalize_json(new_json)
+      
+      # Filter out ignored keys
+      old_filtered = filter_ignored_keys(old_normalized)
+      new_filtered = filter_ignored_keys(new_normalized)
+      
+      if old_filtered == new_filtered
+        return "No changes detected"
+      end
+
+      diff_lines = []
+      diff_lines << "=== Changes Detected ==="
+      diff_lines << ""
+      
+      # Generate detailed diff
+      changes = compare_objects(old_filtered, new_filtered, "")
+      
+      if changes.empty?
+        diff_lines << "No changes detected"
+      else
+        changes.each { |change| diff_lines << change }
+      end
+      
+      diff_lines.join("\n")
+    end
+
+    private
+
+    def normalize_json(json_obj)
+      return {} unless json_obj
+      normalized = JSON.parse(JSON.generate(json_obj))
+      
+      # Normalize OpenSearch/Elasticsearch-specific behavior
+      if normalized.is_a?(Hash) && normalized.key?('properties')
+        normalized = normalize_mappings(normalized)
+      end
+      
+      normalized
+    end
+
+    def normalize_mappings(mappings)
+      return mappings unless mappings.is_a?(Hash) && mappings.key?('properties')
+      
+      normalized = mappings.dup
+      normalized['properties'] = normalize_properties(mappings['properties'])
+      normalized
+    end
+
+    def normalize_properties(properties)
+      return properties unless properties.is_a?(Hash)
+      
+      normalized = {}
+      properties.each do |key, value|
+        if value.is_a?(Hash)
+          # Remove implicit "type": "object" if the field has "properties"
+          if value.key?('properties') && value['type'] == 'object'
+            normalized_value = value.dup
+            normalized_value.delete('type')
+            normalized[key] = normalize_properties(normalized_value)
+          else
+            normalized[key] = normalize_properties(value)
+          end
+        else
+          normalized[key] = value
+        end
+      end
+      
+      normalized
+    end
+
+    def filter_ignored_keys(obj, path_prefix = "")
+      return obj unless obj.is_a?(Hash)
+      
+      filtered = {}
+      obj.each do |key, value|
+        current_path = path_prefix.empty? ? key : "#{path_prefix}.#{key}"
+        
+        # Skip ignored keys
+        next if IGNORED_KEYS.any? { |ignored_key| current_path == ignored_key }
+        
+        # Recursively filter nested objects
+        if value.is_a?(Hash)
+          filtered[key] = filter_ignored_keys(value, current_path)
+        else
+          filtered[key] = value
+        end
+      end
+      
+      filtered
+    end
+
+    def compare_objects(old_obj, new_obj, path_prefix)
+      changes = []
+      
+      # Handle different object types
+      if old_obj.is_a?(Hash) && new_obj.is_a?(Hash)
+        changes.concat(compare_hashes(old_obj, new_obj, path_prefix))
+      elsif old_obj.is_a?(Array) && new_obj.is_a?(Array)
+        changes.concat(compare_arrays(old_obj, new_obj, path_prefix))
+      elsif old_obj != new_obj
+        changes << format_change(path_prefix, old_obj, new_obj)
+      end
+      
+      changes
+    end
+
+    def compare_hashes(old_hash, new_hash, path_prefix)
+      changes = []
+      all_keys = (old_hash.keys + new_hash.keys).uniq.sort
+      
+      all_keys.each do |key|
+        current_path = path_prefix.empty? ? key : "#{path_prefix}.#{key}"
+        
+        # Skip ignored keys
+        next if IGNORED_KEYS.any? { |ignored_key| current_path == ignored_key }
+        
+        old_value = old_hash[key]
+        new_value = new_hash[key]
+        
+        if old_value.nil? && !new_value.nil?
+          changes << "➕ ADDED: #{current_path}"
+          changes.concat(format_value_details("New value", new_value, "  "))
+        elsif !old_value.nil? && new_value.nil?
+          changes << "➖ REMOVED: #{current_path}"
+          changes.concat(format_value_details("Old value", old_value, "  "))
+        elsif old_value != new_value
+          if old_value.is_a?(Hash) && new_value.is_a?(Hash) ||
+             old_value.is_a?(Array) && new_value.is_a?(Array)
+            changes.concat(compare_objects(old_value, new_value, current_path))
+          else
+            changes << "🔄 MODIFIED: #{current_path}"
+            changes.concat(format_value_details("Old value", old_value, "  "))
+            changes.concat(format_value_details("New value", new_value, "  "))
+          end
+        end
+      end
+      
+      changes
+    end
+
+    def compare_arrays(old_array, new_array, path_prefix)
+      changes = []
+      
+      if old_array.length != new_array.length
+        changes << "ARRAY LENGTH CHANGED: #{path_prefix} (#{old_array.length} → #{new_array.length})"
+      end
+      
+      # Compare elements up to the minimum length
+      min_length = [old_array.length, new_array.length].min
+      (0...min_length).each do |index|
+        current_path = "#{path_prefix}[#{index}]"
+        old_value = old_array[index]
+        new_value = new_array[index]
+        
+        if old_value != new_value
+          if old_value.is_a?(Hash) && new_value.is_a?(Hash) ||
+             old_value.is_a?(Array) && new_value.is_a?(Array)
+            changes.concat(compare_objects(old_value, new_value, current_path))
+          else
+            changes << "🔄 MODIFIED: #{current_path}"
+            changes.concat(format_value_details("Old value", old_value, "  "))
+            changes.concat(format_value_details("New value", new_value, "  "))
+          end
+        end
+      end
+      
+      # Handle added elements
+      if new_array.length > old_array.length
+        (old_array.length...new_array.length).each do |index|
+          current_path = "#{path_prefix}[#{index}]"
+          changes << "➕ ADDED: #{current_path}"
+          changes.concat(format_value_details("New value", new_array[index], "  "))
+        end
+      end
+      
+      # Handle removed elements
+      if old_array.length > new_array.length
+        (new_array.length...old_array.length).each do |index|
+          current_path = "#{path_prefix}[#{index}]"
+          changes << "➖ REMOVED: #{current_path}"
+          changes.concat(format_value_details("Old value", old_array[index], "  "))
+        end
+      end
+      
+      changes
+    end
+
+    def format_change(path, old_value, new_value)
+      "🔄 MODIFIED: #{path}"
+    end
+
+    def format_value_details(label, value, indent)
+      details = []
+      details << "#{indent}#{label}:"
+      
+      if value.is_a?(String)
+        # Handle multiline strings
+        if value.include?("\n")
+          details << "#{indent}  \"\"\""
+          value.split("\n").each do |line|
+            details << "#{indent}  #{line}"
+          end
+          details << "#{indent}  \"\"\""
+        else
+          details << "#{indent}  \"#{value}\""
+        end
+      elsif value.is_a?(Hash) || value.is_a?(Array)
+        # Format complex objects with proper indentation
+        formatted = JSON.pretty_generate(value)
+        formatted.split("\n").each do |line|
+          details << "#{indent}  #{line}"
+        end
+      else
+        details << "#{indent}  #{value.inspect}"
+      end
+      
+      details
+    end
+  end
+end

data/lib/schema_tools/migrate/migrate.rb

@@ -0,0 +1,90 @@
+require_relative '../schema_files'
+require_relative 'migrate_breaking_change'
+require_relative 'migrate_non_breaking_change'
+require_relative 'migrate_new'
+require_relative '../diff'
+require_relative '../settings_diff'
+require_relative '../api_aware_mappings_diff'
+require 'json'
+
+module SchemaTools
+  def self.migrate_all(client:)
+    puts "Discovering all schemas and migrating each to their latest revisions..."
+    
+    schemas = SchemaFiles.discover_all_schemas
+    
+    if schemas.empty?
+      puts "No schemas found in #{Config.schemas_path}"
+      return
+    end
+    
+    puts "Found #{schemas.length} schema(s) to migrate:"
+    schemas.each do |schema|
+      puts "  - #{schema}"
+    end
+    puts
+    
+    schemas.each do |alias_name|
+      begin
+        migrate_one_schema(alias_name: alias_name, client: client)
+      rescue => e
+        puts "✗ Migration failed for #{alias_name}: #{e.message}"
+        raise e
+      end
+      puts
+    end
+  end
+
+  def self.migrate_one_schema(alias_name:, client:)
+    puts "=" * 60
+    puts "Migrating alias #{alias_name}"
+    puts "=" * 60
+    
+    schema_path = File.join(Config.schemas_path, alias_name)
+    unless Dir.exist?(schema_path)
+      raise "Schema folder not found: #{schema_path}"
+    end
+    
+    # Check if it's an index name (not an alias)
+    if !client.alias_exists?(alias_name) && client.index_exists?(alias_name)
+      puts "ERROR: Migration not run for index \"#{alias_name}\""
+      puts "  To prevent downtime, this tool only migrates aliased indexes."
+      puts ""
+      puts "  Create a new alias for your index by running:"
+      puts "  rake schema:alias"
+      puts ""
+      puts "  Then rename the schema folder to the alias name and re-run:"
+      puts "  rake schema:migrate"
+      puts ""
+      puts "  Then change your application to read and write to the alias name instead of the index name `#{alias_name}`."
+      raise "Migration not run for alias #{alias_name} because #{alias_name} is an index, not an alias"
+    end
+    
+    unless client.alias_exists?(alias_name)
+      puts "Alias '#{alias_name}' not found. Creating new index and alias..."
+      migrate_to_new_alias(alias_name, client)
+      return
+    end
+    
+    indices = client.get_alias_indices(alias_name)
+    if indices.length > 1
+      puts "This tool can only migrate aliases that point at one index."
+      raise "Alias '#{alias_name}' points to multiple indices: #{indices.join(', ')}"
+    end
+    
+    if indices.length == 0
+      raise "Alias '#{alias_name}' points to no indices."
+    end
+    
+    index_name = indices.first
+    puts "Alias '#{alias_name}' points to index '#{index_name}'"
+    begin
+      attempt_non_breaking_migration(alias_name:, index_name:, client:)
+    rescue => e
+      puts "✗ Failed to update index '#{index_name}': #{e.message}"
+      puts "This appears to be a breaking change. Starting breaking change migration..."
+      
+      MigrateBreakingChange.migrate(alias_name:, client:)
+    end
+  end
+end