schema-tools 1.0.0

data/lib/schema_tools/api_aware_mappings_diff.rb

@@ -0,0 +1,79 @@
+require 'json'
+
+module SchemaTools
+  class ApiAwareMappingsDiff
+    def initialize(local_mappings, remote_mappings)
+      @local_mappings = local_mappings
+      @remote_mappings = remote_mappings
+    end
+
+    def generate_minimal_changes
+      return {} unless @local_mappings.is_a?(Hash) && @local_mappings.key?("properties")
+      
+      remote_properties = @remote_mappings.is_a?(Hash) && @remote_mappings.key?("properties") ? @remote_mappings["properties"] : {}
+      changes = find_api_aware_changes(remote_properties, @local_mappings["properties"])
+      
+      # Check if dynamic setting is different
+      dynamic_changed = false
+      if @local_mappings.key?("dynamic")
+        remote_dynamic = @remote_mappings.is_a?(Hash) && @remote_mappings.key?("dynamic") ? @remote_mappings["dynamic"] : nil
+        dynamic_changed = @local_mappings["dynamic"] != remote_dynamic
+      end
+      
+      if changes.empty? && !dynamic_changed
+        {}
+      else
+        result = {}
+        result["properties"] = changes unless changes.empty?
+        result["dynamic"] = @local_mappings["dynamic"] if dynamic_changed
+        result
+      end
+    end
+
+    private
+
+    def find_api_aware_changes(remote, local)
+      changes = {}
+      
+      return changes unless local.is_a?(Hash) && remote.is_a?(Hash)
+      
+      local.each do |key, value|
+        if !remote.key?(key)
+          # New field - include complete definition
+          changes[key] = value
+        elsif value != remote[key]
+          if value.is_a?(Hash) && remote[key].is_a?(Hash)
+            # Field exists but has changes
+            if is_field_definition?(value)
+              # For field definitions, always include complete definition for API compatibility
+              changes[key] = value
+            else
+              # For nested objects, try to be more selective
+              nested_changes = find_api_aware_changes(remote[key], value)
+              if nested_changes.empty?
+                # No nested changes, but the values are different
+                # Only include if this is a significant change
+                changes[key] = value
+              else
+                changes[key] = nested_changes
+              end
+            end
+          else
+            # Simple value change
+            changes[key] = value
+          end
+        end
+      end
+      
+      changes
+    end
+
+    def is_field_definition?(field_value)
+      return false unless field_value.is_a?(Hash)
+      
+      # Field definitions are objects that have a "type" property
+      # These require complete definitions for OpenSearch/Elasticsearch API compatibility
+      field_value.key?("type")
+    end
+  end
+end

data/lib/schema_tools/catchup.rb

@@ -0,0 +1,23 @@
+module SchemaTools
+  def self.catchup(index_name:, client:)
+    raise "index_name parameter is required" unless index_name
+    
+    index_config = SchemaFiles.get_index_config(index_name)
+    raise "Index configuration not found for #{index_name}" unless index_config
+    
+    from_index = index_config['from_index_name']
+    raise "from_index_name not specified in index configuration" unless from_index
+    
+    unless client.index_exists?(from_index)
+      raise "Source index #{from_index} does not exist. Cannot perform catchup reindex to #{index_name}."
+    end
+    
+    reindex_script = SchemaFiles.get_reindex_script(index_name)
+    
+    puts "Starting catchup reindex from #{from_index} to #{index_name}"
+    # TODO NOT IMPLEMENTED YET
+    # Do a reindex by query
+    puts "TODO IMPLEMENT ME"
+    response = client.reindex(from_index, index_name, reindex_script)
+  end
+end

data/lib/schema_tools/client.rb

@@ -0,0 +1,472 @@
+require 'net/http'
+require 'json'
+require 'uri'
+require_relative 'settings_filter'
+
+class SimpleLogger
+  def initialize(output = STDOUT)
+    @output = output
+  end
+
+  def info(message)
+    @output.puts message
+  end
+
+  def warn(message)
+    @output.puts message
+  end
+
+  def error(message)
+    @output.puts message
+  end
+end
+
+module SchemaTools
+  class Client
+    attr_reader :url
+    
+    def initialize(url, dryrun: false, logger: SimpleLogger.new, username: nil, password: nil)
+      @url = url
+      @dryrun = dryrun
+      @logger = logger
+      @username = username
+      @password = password
+      @logger.info "Client is running in DRYRUN mode. No mutating operations will be performed." if dryrun
+    end
+
+    def get(path)
+      uri = URI("#{@url}#{path}")
+      request = Net::HTTP::Get.new(uri)
+      add_auth_header(request)
+      response = make_http_request(uri) { |http| http.request(request) }
+      
+      case response.code.to_i
+      when 200
+        JSON.parse(response.body)
+      when 404
+        nil
+      else
+        raise "HTTP #{response.code}: #{response.body}"
+      end
+    end
+
+    def put(path, body, suppress_logging: false)
+      unless suppress_logging
+        @logger.info("DRYRUN=true, simulation only") if @dryrun
+        log_operation('PUT', path, body)
+        await_user_input if interactive_mode?
+      end
+      if @dryrun
+        return { 'acknowledged' => true } # Return mock response for dry run
+      end
+
+      uri = URI("#{@url}#{path}")
+      request = Net::HTTP::Put.new(uri)
+      request['Content-Type'] = 'application/json'
+      request.body = body.to_json
+      add_auth_header(request)
+      
+      response = make_http_request(uri) { |http| http.request(request) }
+      
+      case response.code.to_i
+      when 200, 201
+        JSON.parse(response.body) if response.body && !response.body.empty?
+      else
+        raise "HTTP #{response.code}: #{response.body}"
+      end
+    end
+
+    def post(path, body, suppress_logging: false)
+      unless suppress_logging
+        @logger.info("DRYRUN=true, simulation only") if @dryrun
+        log_operation('POST', path, body)
+        await_user_input if interactive_mode?
+      end
+      if @dryrun
+        return {"task"=>"FEl-TdjcTpmIvnE5_1fv4Q:164963"} if path.start_with?('/_reindex')
+        return { 'acknowledged' => true } # Return mock response for dry run
+      end
+
+      uri = URI("#{@url}#{path}")
+      request = Net::HTTP::Post.new(uri)
+      request['Content-Type'] = 'application/json'
+      request.body = body.to_json
+      add_auth_header(request)
+      
+      response = make_http_request(uri) { |http| http.request(request) }
+      
+      case response.code.to_i
+      when 200, 201
+        JSON.parse(response.body) if response.body && !response.body.empty?
+      else
+        raise "HTTP #{response.code}: #{response.body}"
+      end
+    end
+
+    def delete(path, suppress_logging: false)
+      unless suppress_logging
+        @logger.info("DRYRUN=true, simulation only") if @dryrun
+        log_operation('DELETE', path)
+        await_user_input if interactive_mode?
+      end
+      if @dryrun
+        return { 'acknowledged' => true } # Return mock response for dry run
+      end
+
+      uri = URI("#{@url}#{path}")
+      request = Net::HTTP::Delete.new(uri)
+      add_auth_header(request)
+      
+      response = make_http_request(uri) { |http| http.request(request) }
+      
+      case response.code.to_i
+      when 200, 404
+        JSON.parse(response.body) if response.body && !response.body.empty?
+      else
+        raise "HTTP #{response.code}: #{response.body}"
+      end
+    end
+
+    def index_exists?(index_name)
+      get("/#{index_name}") != nil
+    end
+
+    def get_index_settings(index_name)
+      response = get("/#{index_name}")
+      return nil unless response
+      response[index_name]['settings']
+    end
+
+    def get_index_mappings(index_name)
+      response = get("/#{index_name}")
+      return nil unless response
+      response[index_name]['mappings']
+    end
+
+    def get_index_doc_count(index_name)
+      response = get("/#{index_name}/_count")
+      return 0 unless response
+      response['count'] || 0
+    end
+
+    def create_index(index_name, settings, mappings)
+      body = {
+        settings: settings,
+        mappings: mappings
+      }
+      put("/#{index_name}", body)
+    end
+
+    def update_index_settings(index_name, settings)
+      put("/#{index_name}/_settings", settings)
+    end
+
+
+    def reindex(source_index, dest_index, script = nil)
+      body = {
+        source: { index: source_index },
+        dest: { index: dest_index },
+        conflicts: "proceed"
+      }
+      body[:script] = { source: script } if script
+      
+      url = "/_reindex?wait_for_completion=false&refresh=false"
+      
+      post(url, body)
+    end
+
+    def get_task_status(task_id)
+      get("/_tasks/#{task_id}")
+    end
+
+    def put_script(script_name, script_content)
+      body = { script: { lang: "painless", source: script_content } }
+      put("/_scripts/#{script_name}", body)
+    end
+
+    def delete_script(script_name, suppress_logging: false)
+      unless suppress_logging
+        @logger.info("DRYRUN=true, simulation only") if @dryrun
+        log_operation('DELETE', "/_scripts/#{script_name}")
+        await_user_input if interactive_mode?
+      end
+      if @dryrun
+        return { 'acknowledged' => true } # Return mock response for dry run
+      end
+
+      uri = URI("#{@url}/_scripts/#{script_name}")
+      request = Net::HTTP::Delete.new(uri)
+      add_auth_header(request)
+      
+      response = make_http_request(uri) { |http| http.request(request) }
+      
+      case response.code.to_i
+      when 200
+        JSON.parse(response.body) if response.body && !response.body.empty?
+      when 404
+        raise "HTTP 404: Script '#{script_name}' not found"
+      else
+        raise "HTTP #{response.code}: #{response.body}"
+      end
+    end
+
+    def get_stored_scripts
+      # Try the legacy Elasticsearch API first (works for Elasticsearch and older OpenSearch)
+      begin
+        response = get("/_scripts")
+        return {} unless response
+        
+        scripts = {}
+        response.each do |script_id, script_data|
+          scripts[script_id] = script_data.dig('script', 'source')
+        end
+        
+        return scripts
+      rescue => e
+        # If the legacy API fails (e.g., OpenSearch 2.x), try the new API
+        begin
+          response = get("/_cluster/state/metadata?filter_path=metadata.stored_scripts")
+          return {} unless response
+          
+          stored_scripts_data = response.dig('metadata', 'stored_scripts')
+          return {} unless stored_scripts_data
+          
+          scripts = {}
+          stored_scripts_data.each do |script_id, script_data|
+            scripts[script_id] = script_data['source']
+          end
+          
+          return scripts
+        rescue => fallback_error
+          # If both APIs fail, log the original error and return empty hash
+          @logger.warn("Could not retrieve stored scripts: #{e.message}. Fallback error: #{fallback_error}") if @logger
+          {}
+        end
+      end
+    end
+
+    def delete_index(index_name)
+      delete("/#{index_name}")
+    end
+
+    def list_indices
+      response = get("/_cat/indices?format=json")
+      return [] unless response && response.is_a?(Array)
+      
+      response.map { |index| index['index'] }
+              .reject { |name| name.start_with?('.') || name.start_with?('top_queries-') } # Exclude system indices
+              .sort
+    end
+
+    def list_aliases
+      response = get("/_aliases")
+      return {} unless response
+      
+      aliases = {}
+      response.each do |index_name, index_data|
+        index_aliases = index_data['aliases']
+        next unless index_aliases && !index_aliases.empty?
+        
+        index_aliases.each do |alias_name, alias_data|
+          aliases[alias_name] ||= []
+          aliases[alias_name] << index_name
+        end
+      end
+      
+      aliases
+    end
+
+    def get_alias_indices(alias_name)
+      response = get("/_alias/#{alias_name}")
+      return [] unless response
+      
+      response.keys
+    end
+
+    def create_alias(alias_name, index_name)
+      body = {
+        actions: [
+          {
+            add: {
+              index: index_name,
+              alias: alias_name
+            }
+          }
+        ]
+      }
+      post("/_aliases", body)
+    end
+
+    def alias_exists?(alias_name)
+      response = get("/_alias/#{alias_name}")
+      response && !response.empty?
+    end
+
+    def delete_alias(alias_name, indices = nil)
+      # If no indices specified, get all indices for this alias
+      if indices.nil?
+        indices = get_alias_indices(alias_name)
+      end
+      
+      actions = indices.map do |index_name|
+        {
+          remove: {
+            index: index_name,
+            alias: alias_name
+          }
+        }
+      end
+      
+      body = { actions: actions }
+      post("/_aliases", body)
+    end
+
+    def update_index_settings(index_name, settings)
+      # Filter out internal settings that can't be updated
+      filtered_settings = SettingsFilter.filter_internal_settings(settings)
+      
+      body = { index: filtered_settings['index'] || {} }
+      put("/#{index_name}/_settings", body)
+    end
+
+    def update_index_mappings(index_name, mappings)
+      body = { properties: mappings['properties'] || {} }
+      put("/#{index_name}/_mapping", body)
+    end
+
+    def test_connection
+      path = "/_cluster/health"
+      puts "Testing connection to #{@url}#{path}"
+      get(path)
+      true
+    rescue => e
+      puts e
+      false
+    end
+
+    def close_index(index_name)
+      post("/#{index_name}/_close", {})
+    end
+
+    def update_by_query(source_index, dest_index, script = nil)
+      body = {
+        source: { index: source_index },
+        dest: { index: dest_index }
+      }
+      body[:script] = { source: script } if script
+      
+      url = "/_update_by_query?wait_for_completion=false"
+      
+      post(url, body)
+    end
+
+    def update_aliases(actions)
+      body = { actions: actions }
+      post("/_aliases", body)
+    end
+
+    def wait_for_task(task_id, timeout = 3600)
+      start_time = Time.now
+      
+      loop do
+        task_status = get_task_status(task_id)
+        
+        if task_status['completed']
+          return task_status
+        end
+        
+        if Time.now - start_time > timeout
+          raise "Task #{task_id} timed out after #{timeout} seconds"
+        end
+        
+        sleep 5
+      end
+    end
+
+    def bulk_index(documents, index_name, suppress_logging: false)
+      unless suppress_logging
+        @logger.info("DRYRUN=true, simulation only") if @dryrun
+        log_operation('POST', '/_bulk') # including documents would be too noisy
+        await_user_input if interactive_mode?
+      end
+      if @dryrun
+        return { 'items' => documents.map { |doc| { 'index' => { 'status' => 201 } } } }
+      end
+
+      bulk_body = documents.map do |doc|
+        [
+          { index: { _index: index_name } },
+          doc
+        ]
+      end.flatten
+
+      ndjson = bulk_body.map(&:to_json).join("\n") + "\n"
+
+      uri = URI("#{@url}/_bulk")
+      request = Net::HTTP::Post.new(uri)
+      request['Content-Type'] = 'application/x-ndjson'
+      request.body = ndjson
+      add_auth_header(request)
+      
+      response = make_http_request(uri) { |http| http.request(request) }
+      
+      case response.code.to_i
+      when 200
+        JSON.parse(response.body)
+      else
+        raise "HTTP #{response.code}: #{response.body}"
+      end
+    end
+
+    def index_closed?(index_name)
+      response = get("/#{index_name}")
+      return false unless response
+      
+      # Check if the index is closed by looking at the index status
+      # Closed indices have a specific status in the response
+      index_info = response[index_name]
+      return false unless index_info
+      
+      # Check if the index is closed by looking at the settings
+      settings = index_info['settings']
+      return false unless settings
+      
+      # An index is closed if it has the 'verified_before_close' setting set to true
+      settings.dig('index', 'verified_before_close') == 'true'
+    end
+
+    private
+
+    def make_http_request(uri)
+      use_ssl = uri.scheme == 'https'
+      port = uri.port || (use_ssl ? 443 : 80)
+      
+      Net::HTTP.start(uri.hostname, port, use_ssl: use_ssl) do |http|
+        yield(http)
+      end
+    end
+
+    def add_auth_header(request)
+      if @username && @password
+        request.basic_auth(@username, @password)
+      end
+    end
+
+    def log_operation(method, path, body = nil)
+      message = "\e[1m\e[37m#{method} #{path}\e[0m" # Bold White color
+      if body
+        message += "\n#{body.is_a?(String) ? body : JSON.pretty_generate(body)}"
+      end
+      @logger.info message
+    end
+
+    def interactive_mode?
+      ENV['INTERACTIVE'] == 'true'
+    end
+
+    def await_user_input
+      print "\nPress Enter to continue... "
+      STDIN.gets
+    end
+  end
+end

data/lib/schema_tools/close.rb

@@ -0,0 +1,28 @@
+module SchemaTools
+  def self.close(name:, client:)
+    raise "name parameter is required" unless name
+    
+    # Check if it's an alias
+    if client.alias_exists?(name)
+      indices = client.get_alias_indices(name)
+      puts "Closing alias '#{name}' (points to: #{indices.join(', ')})"
+      puts "This will close all underlying index(es)."
+      
+      indices.each do |index_name|
+        if client.index_exists?(index_name)
+          client.close_index(index_name)
+          puts "✓ Index #{index_name} closed"
+        else
+          puts "⚠ Index #{index_name} does not exist"
+        end
+      end
+      puts "✓ All index(es) in alias '#{name}' closed"
+    elsif client.index_exists?(name)
+      puts "Closing index #{name}"
+      client.close_index(name)
+      puts "✓ Index #{name} closed"
+    else
+      raise "Neither alias nor index exists: #{name}"
+    end
+  end
+end

data/lib/schema_tools/config.rb

@@ -0,0 +1,46 @@
+module SchemaTools
+  module Config
+    # e.g. http://localhost:9200
+    CONNECTION_URL = ENV['OPENSEARCH_URL'] || ENV['ELASTICSEARCH_URL']
+
+    # Optional username for HTTP basic authentication
+    CONNECTION_USERNAME = ENV['OPENSEARCH_USERNAME'] || ENV['ELASTICSEARCH_USERNAME']
+
+    # Optional password for HTTP basic authentication
+    CONNECTION_PASSWORD = ENV['OPENSEARCH_PASSWORD'] || ENV['ELASTICSEARCH_PASSWORD']
+
+    # Folder on disk where all schema definitions are stored
+    SCHEMAS_PATH = ENV['SCHEMAS_PATH'] || 'schemas'
+
+    # Folder on disk where painless scripts are stored
+    PAINLESS_SCRIPTS_PATH = ENV['PAINLESS_SCRIPTS_PATH'] || 'painless_scripts'
+
+    # Descriptive name for operations (kept for backward compatibility)
+    SCHEMA_TOOLS_USER = ENV['SCHEMA_TOOLS_USER'] || 'rake task'
+
+    def self.schema_tools_user
+      SCHEMA_TOOLS_USER
+    end
+
+    def self.connection_url
+      CONNECTION_URL
+    end
+
+    def self.connection_username
+      CONNECTION_USERNAME
+    end
+
+    def self.connection_password
+      CONNECTION_PASSWORD
+    end
+    
+    def self.schemas_path
+      SCHEMAS_PATH
+    end
+
+    def self.painless_scripts_path
+      PAINLESS_SCRIPTS_PATH
+    end
+
+  end
+end

data/lib/schema_tools/delete.rb

@@ -0,0 +1,28 @@
+module SchemaTools
+  def self.delete(name:, client:)
+    raise "name parameter is required" unless name
+    
+    # Check if it's an alias
+    if client.alias_exists?(name)
+      indices = client.get_alias_indices(name)
+      puts "Deleting alias '#{name}' (points to: #{indices.join(', ')})"
+      puts "The underlying index(es) will remain intact."
+      
+      client.delete_alias(name)
+      puts "✓ Alias '#{name}' deleted"
+      puts "Index(es) #{indices.join(', ')} remain(s) intact"
+    elsif client.index_exists?(name)
+      puts "Checking that index #{name} is closed before proceeding"
+      unless client.index_closed?(name)
+        raise "Hard delete only allowed on closed indexes. Please run rake 'schema:close[#{name}]' first."
+      end
+      puts "Index #{name} is closed"
+      
+      puts "Hard deleting index #{name}"
+      client.delete_index(name)
+      puts "✓ Index #{name} hard deleted"
+    else
+      raise "Neither alias nor index exists: #{name}"
+    end
+  end
+end