schema-tools 1.0.6 → 1.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a977e1d5ae35640094087dff8eb1b1e5481db2b6bc0d35315f219fea02210d6
-  data.tar.gz: d8a40fbc727ede8614d607a22ca9aa830b32254598f8900d8f0e0b57fc315589
+  metadata.gz: 844c9578ff5adeafdd8d34da1df7ec2df9114a9db572680b32fb856a92c58b9d
+  data.tar.gz: cf21d35fc65c62922911bc18e50be33515e19dbd599ca9714d6c47d10b5d7199
 SHA512:
-  metadata.gz: 07d90a197672dff508c6db4e6e40d90da18ce06dad381b0f46c6687d3e1d38eb2f36a305d06717863da6f5f60fcb4cdae25d9df4a5862a54338887a819255924
-  data.tar.gz: cd758d32d70d00af78c8f9580ff66a02f7c4bc86de7dde7306f90eaa7f4ae88dd431987adbfd984bd7e8f12b5d9f648c66c89729443b47a87204f61b4fc65dd2
+  metadata.gz: 1743b2f92814bea0e6f368cc0b807b6dc4734a615ce2c752aca37dc50e9523db206b51882e596a4489df33bfb5178bc5a39729f7b688fa22a62dabe3548058cc
+  data.tar.gz: 0e5a6babe517b01f61773a4be3a679aabcd74f3acebf2200cfa08dbc4c9b4d9ed23f30f3e38ce29d5c5cd02847eea06dfd8d231989e96b220154c5b3722f9763

data/README.md

@@ -7,6 +7,8 @@
 - Create new aliases with sample schemas.
 - Manage painless scripts independently from schema migrations.
 
+A sample app that uses schema-tools is available at: https://github.com/richkuz/schema-tools-sample-app
+
 ## Quick start
 
 Install this Ruby gem.
@@ -41,23 +43,6 @@ export OPENSEARCH_USERNAME=your_username
 export OPENSEARCH_PASSWORD=your_password
 ```
 
-### View available rake tasks
-
-```sh
-rake -T | grep " schema:"
-```
-
-Available schema tasks:
-- `schema:migrate[alias_name]` - Migrate to a specific alias schema or migrate all schemas
-- `schema:new` - Create a new alias with sample schema
-- `schema:close[name]` - Close an index or alias
-- `schema:delete[name]` - Hard delete an index (only works on closed indexes) or delete an alias
-- `schema:drop[alias_name]` - Delete an alias (does not delete the index)
-- `schema:download` - Download schema from an existing alias or index
-- `schema:alias` - Create an alias for an existing index
-- `schema:seed` - Seed data to a live index
-- `schema:diff` - Compare all schemas to their corresponding downloaded alias settings and mappings
-
 ### Download an existing schema
 
 Run `rake schema:download` to download a schema from an existing alias or index:
@@ -115,6 +100,24 @@ $ rake schema:new
 #   - mappings.json
 ```
 
+### View available rake tasks
+
+```sh
+rake -T | grep " schema:"
+```
+
+Available schema tasks:
+- `schema:download` - Download schema from an existing alias or index
+- `schema:migrate` - Migrate all schemas to match local schema files
+- `schema:migrate[alias_name]` - Migrate a specific alias to match its local schema files
+- `schema:new` - Create a new alias with sample schema
+- `schema:alias` - Create an alias for an existing index
+- `schema:diff` - Compare all schemas to their corresponding downloaded alias settings and mappings
+- `schema:seed` - Seed data to a live index
+- `schema:close[name]` - Close an index or alias
+- `schema:delete[name]` - Hard delete an index (only works on closed indexes) or delete an alias
+- `schema:drop[alias_name]` - Delete an alias (does not delete the index)
+
 ## Directory structure reference
 
 Example directory structure with multiple aliases:
@@ -298,6 +301,8 @@ Use case:
 
 Rake `schema:migrate` solves this use case through the following procedure.
 
+See: [Migration Procedure Diagram](https://github.com/richkuz/schema-tools/blob/main/docs/schema-tools-migration.svg)
+
 First, some terms:
 - `alias_name`: Alias containing the index to migrate
 	- `products`
@@ -317,20 +322,20 @@ SETUP
 Create `log_index` to log the migration state.
 - The migration logs when it starts and completes a step along with a description.
 
-STEP 1
+STEP 0
 
 Attempt to reindex 1 document to a throwaway index to catch obvious configuration errors and abort early if possible.
 
-STEP 2
+STEP 1
 
 Create `catchup1_index` using the new schema.
 - This index will preserve writes during the reindex.
 
-STEP 3
+STEP 2
 
 Configure `alias_name` to only write to `catchup1_index` and read from `current_index` and `catchup1_index`.
 
-STEP 4
+STEP 3
 
 Create `new_index` using the new schema.
 
@@ -346,38 +351,38 @@ POST _reindex
 }
 ```
 
-STEP 5
+STEP 4
 
 Create `catchup2_index` using the new schema.
 - This index ensures a place for ongoing writes while flushing `catchup1_index`.
 
-STEP 6
+STEP 5
 
 Configure `alias_name` to only write to `catchup2_index` and continue reading from `current_index` and `catchup1_index`.
 
-STEP 7
+STEP 6
 
 Reindex `catchup1_index` into `new_index`.
 - Merge the first catchup index into the new canonical index.
 
-STEP 8
+STEP 7
 
 Configure `alias_name` so there are NO write indexes
 - This guarantees that no writes can sneak into an obsolete catchup index during the second (quick) merge.
 - Any write operations will fail during this time with: `"reason": "Alias [FOO] has more than one index associated with it [...], can't execute a single index op"`
 - Clients must retry any failed writes.
 
-STEP 9
+STEP 8
 
 Reindex `catchup2_index` into `new_index`
 - Final sync to merge the second catchup index into the new canonical index.
 
-STEP 10
+STEP 9
 
 Configure `alias_name` to write to and read from `new_index` only.
 - Writes resume to the single new index. All data and deletes are consistent.
 
-STEP 11
+STEP 10
 
 Close unused indexes to avoid accidental writes.
 - Close `catchup1_index`

data/lib/schema_tools/client.rb

@@ -196,6 +196,11 @@ module SchemaTools
       
       response = post(url, body)
 
+      # In dry run mode, post returns a task response, not a completed response
+      if @dryrun
+        return response
+      end
+
       if response['failures'] && !response['failures'].empty?
         failure_reason = response['failures'].map { |f| f['cause']['reason'] }.join("; ")
         raise "Reindex failed with internal errors. Failures: #{failure_reason}"
@@ -205,14 +210,25 @@ module SchemaTools
       created = response['created'].to_i
       updated = response['updated'].to_i
       
-      if total != 1
+      if total == 0
+        # Verify that the source index actually has 0 documents
+        source_doc_count = get_index_doc_count(source_index)
+        if source_doc_count == 0
+          # Handle the case where the source index has no documents
+          # This is a valid scenario - just log and continue
+          @logger.info("Source index has no documents to reindex")
+        else
+          # This shouldn't happen - if source has docs but reindex found 0, something went wrong
+          raise "Reindex query found 0 documents but source index has #{source_doc_count} documents. This indicates a reindex configuration issue."
+        end
+      elsif total != 1
         raise "Reindex query found #{total} documents. Expected to find 1."
-      elsif created + updated != 1
+      elsif total == 1 && created + updated != 1
         raise "Reindex failed to index the document (created: #{created}, updated: #{updated}). Noops: #{response.fetch('noops', 0)}."
       elsif response['timed_out'] == true
         raise "Reindex operation timed out."
       end
-      
+
       response
     end
 

data/lib/schema_tools/diff.rb

@@ -94,7 +94,11 @@ module SchemaTools
         normalized_local_settings = self.normalize_local_settings(local_settings)
         normalized_remote_settings = self.normalize_remote_settings(filtered_remote_settings)
 
-        result[:settings_diff] = json_diff.generate_diff(normalized_remote_settings, normalized_local_settings)
+        # Check for replica count differences first
+        replica_warning = check_replica_count_difference(normalized_remote_settings, normalized_local_settings)
+        
+        # Generate diff while ignoring number_of_replicas
+        result[:settings_diff] = json_diff.generate_diff(normalized_remote_settings, normalized_local_settings, ignored_keys: ["index.number_of_replicas"])
         result[:mappings_diff] = json_diff.generate_diff(remote_mappings, local_mappings)
         
         result[:comparison_context] = {
@@ -108,8 +112,10 @@ module SchemaTools
           }
         }
         
+        # Set status based on diff results
         if result[:settings_diff] == "No changes detected" && result[:mappings_diff] == "No changes detected"
           result[:status] = :no_changes
+          result[:replica_warning] = replica_warning if replica_warning
         else
           result[:status] = :changes_detected
         end
@@ -163,10 +169,38 @@ module SchemaTools
 
       puts "Mappings Comparison:"
       puts schema_diff[:mappings_diff]
+      
+      # Display replica warning if present
+      if schema_diff[:replica_warning]
+        puts
+        puts "⚠️  #{schema_diff[:replica_warning]}"
+      end
     end
 
     private
 
+    def self.check_replica_count_difference(remote_settings, local_settings)
+      remote_replicas = get_replica_count(remote_settings)
+      local_replicas = get_replica_count(local_settings)
+      
+      if remote_replicas && local_replicas && remote_replicas != local_replicas
+        "WARNING: The specified number of replicas #{local_replicas} in the schema could not be applied to the cluster, likely because the cluster isn't running enough nodes."
+      else
+        nil
+      end
+    end
+
+    def self.get_replica_count(settings)
+      return nil unless settings.is_a?(Hash)
+      
+      if settings.key?("index") && settings["index"].is_a?(Hash)
+        settings["index"]["number_of_replicas"]
+      else
+        settings["number_of_replicas"]
+      end
+    end
+
+
     def self.normalize_local_settings(local_settings)
       return local_settings unless local_settings.is_a?(Hash)
       

data/lib/schema_tools/json_diff.rb

@@ -10,13 +10,13 @@ module SchemaTools
 
     # Generate a detailed diff between two JSON objects
     # Returns a formatted string showing additions, removals, and modifications
-    def generate_diff(old_json, new_json, context: {})
+    def generate_diff(old_json, new_json, context: {}, ignored_keys: [])
       old_normalized = normalize_json(old_json)
       new_normalized = normalize_json(new_json)
       
       # Filter out ignored keys
-      old_filtered = filter_ignored_keys(old_normalized)
-      new_filtered = filter_ignored_keys(new_normalized)
+      old_filtered = filter_ignored_keys(old_normalized, "", ignored_keys)
+      new_filtered = filter_ignored_keys(new_normalized, "", ignored_keys)
       
       if old_filtered == new_filtered
         return "No changes detected"
@@ -27,7 +27,7 @@ module SchemaTools
       diff_lines << ""
       
       # Generate detailed diff
-      changes = compare_objects(old_filtered, new_filtered, "")
+      changes = compare_objects(old_filtered, new_filtered, "", ignored_keys)
       
       if changes.empty?
         diff_lines << "No changes detected"
@@ -82,19 +82,22 @@ module SchemaTools
       normalized
     end
 
-    def filter_ignored_keys(obj, path_prefix = "")
+    def filter_ignored_keys(obj, path_prefix = "", ignored_keys = [])
       return obj unless obj.is_a?(Hash)
       
+      # Combine class-level ignored keys with parameter ignored keys
+      all_ignored_keys = IGNORED_KEYS + ignored_keys
+      
       filtered = {}
       obj.each do |key, value|
         current_path = path_prefix.empty? ? key : "#{path_prefix}.#{key}"
         
         # Skip ignored keys
-        next if IGNORED_KEYS.any? { |ignored_key| current_path == ignored_key }
+        next if all_ignored_keys.any? { |ignored_key| current_path == ignored_key }
         
         # Recursively filter nested objects
         if value.is_a?(Hash)
-          filtered[key] = filter_ignored_keys(value, current_path)
+          filtered[key] = filter_ignored_keys(value, current_path, ignored_keys)
         else
           filtered[key] = value
         end
@@ -103,14 +106,14 @@ module SchemaTools
       filtered
     end
 
-    def compare_objects(old_obj, new_obj, path_prefix)
+    def compare_objects(old_obj, new_obj, path_prefix, ignored_keys = [])
       changes = []
       
       # Handle different object types
       if old_obj.is_a?(Hash) && new_obj.is_a?(Hash)
-        changes.concat(compare_hashes(old_obj, new_obj, path_prefix))
+        changes.concat(compare_hashes(old_obj, new_obj, path_prefix, ignored_keys))
       elsif old_obj.is_a?(Array) && new_obj.is_a?(Array)
-        changes.concat(compare_arrays(old_obj, new_obj, path_prefix))
+        changes.concat(compare_arrays(old_obj, new_obj, path_prefix, ignored_keys))
       elsif old_obj != new_obj
         changes << format_change(path_prefix, old_obj, new_obj)
       end
@@ -118,15 +121,18 @@ module SchemaTools
       changes
     end
 
-    def compare_hashes(old_hash, new_hash, path_prefix)
+    def compare_hashes(old_hash, new_hash, path_prefix, ignored_keys = [])
       changes = []
       all_keys = (old_hash.keys + new_hash.keys).uniq.sort
       
+      # Combine class-level ignored keys with parameter ignored keys
+      all_ignored_keys = IGNORED_KEYS + ignored_keys
+      
       all_keys.each do |key|
         current_path = path_prefix.empty? ? key : "#{path_prefix}.#{key}"
         
         # Skip ignored keys
-        next if IGNORED_KEYS.any? { |ignored_key| current_path == ignored_key }
+        next if all_ignored_keys.any? { |ignored_key| current_path == ignored_key }
         
         old_value = old_hash[key]
         new_value = new_hash[key]
@@ -140,7 +146,7 @@ module SchemaTools
         elsif old_value != new_value
           if old_value.is_a?(Hash) && new_value.is_a?(Hash) ||
              old_value.is_a?(Array) && new_value.is_a?(Array)
-            changes.concat(compare_objects(old_value, new_value, current_path))
+            changes.concat(compare_objects(old_value, new_value, current_path, ignored_keys))
           else
             changes << "🔄 MODIFIED: #{current_path}"
             changes.concat(format_value_details("Old value", old_value, "  "))
@@ -152,7 +158,7 @@ module SchemaTools
       changes
     end
 
-    def compare_arrays(old_array, new_array, path_prefix)
+    def compare_arrays(old_array, new_array, path_prefix, ignored_keys = [])
       changes = []
       
       if old_array.length != new_array.length
@@ -169,7 +175,7 @@ module SchemaTools
         if old_value != new_value
           if old_value.is_a?(Hash) && new_value.is_a?(Hash) ||
              old_value.is_a?(Array) && new_value.is_a?(Array)
-            changes.concat(compare_objects(old_value, new_value, current_path))
+            changes.concat(compare_objects(old_value, new_value, current_path, ignored_keys))
           else
             changes << "🔄 MODIFIED: #{current_path}"
             changes.concat(format_value_details("Old value", old_value, "  "))

data/lib/schema_tools/migrate/migrate_verify.rb

@@ -7,6 +7,9 @@ module SchemaTools
     
     if diff_result[:status] == :no_changes
       puts "✓ Migration verification successful - no differences detected"
+      if diff_result[:replica_warning]
+        puts "⚠️  #{diff_result[:replica_warning]}"
+      end
       puts "Migration completed successfully!"
     else
       puts "⚠️  Migration verification failed - differences detected:"

data/lib/schema_tools/new_alias.rb

@@ -5,6 +5,21 @@ require_relative 'config'
 require_relative 'settings_filter'
 
 module SchemaTools
+  EXAMPLE_PAINLESS_SCRIPT = <<~PAINLESS
+      // Example reindex script for transforming data during migration
+      // Modify this script to transform your data as needed
+      //
+      // Example: Rename a field
+      // if (ctx._source.containsKey('old_field_name')) {
+      //   ctx._source.new_field_name = ctx._source.old_field_name;
+      //   ctx._source.remove('old_field_name');
+      // }
+      //
+      // Example: Add a new field
+      // ctx._source.new_field = 'default_value';
+      long timestamp = System.currentTimeMillis();
+    PAINLESS
+  
   def self.new_alias(client:)
     puts "\nEnter a new alias name:"
     alias_name = STDIN.gets&.chomp
@@ -68,20 +83,7 @@ module SchemaTools
     File.write(mappings_file, JSON.pretty_generate(sample_mappings))
     
     # Create example reindex.painless file
-    reindex_content = <<~PAINLESS
-      // Example reindex script for transforming data during migration
-      // Modify this script to transform your data as needed
-      //
-      // Example: Rename a field
-      // if (ctx._source.containsKey('old_field_name')) {
-      //   ctx._source.new_field_name = ctx._source.old_field_name;
-      //   ctx._source.remove('old_field_name');
-      // }
-      //
-      // Example: Add a new field
-      // ctx._source.new_field = 'default_value';
-      long timestamp = System.currentTimeMillis();
-    PAINLESS
+    reindex_content = EXAMPLE_PAINLESS_SCRIPT
     
     File.write(reindex_file, reindex_content)
     
@@ -170,20 +172,7 @@ module SchemaTools
     File.write(settings_file, JSON.pretty_generate(filtered_settings))
     File.write(mappings_file, JSON.pretty_generate(mappings))
     
-    # Create example reindex.painless file
-    reindex_content = <<~PAINLESS
-      # Example reindex script for transforming data during migration
-      # Modify this script to transform your data as needed
-      #
-      # Example: Rename a field
-      # if (ctx._source.containsKey('old_field_name')) {
-      #   ctx._source.new_field_name = ctx._source.old_field_name;
-      #   ctx._source.remove('old_field_name');
-      # }
-      #
-      # Example: Add a new field
-      # ctx._source.new_field = 'default_value';
-    PAINLESS
+    reindex_content = EXAMPLE_PAINLESS_SCRIPT
     
     File.write(reindex_file, reindex_content)
     

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: schema-tools
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.0.7
 platform: ruby
 authors:
 - Rich Kuzsma
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-10-13 00:00:00.000000000 Z
+date: 2025-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake