elasticsearch_record 1.0.2 → 1.1.0

data/lib/active_record/connection_adapters/elasticsearch/table_statements.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Elasticsearch
+      # extend adapter with table-related statements
+      module TableStatements
+        extend ActiveSupport::Concern
+
+        included do
+          # ORIGINAL methods untouched:
+          #
+          # SUPPORTED but not used:
+          #
+          # UNSUPPORTED methods that will be ignored:
+          # - native_database_types
+          # - table_options
+          # - table_comment
+          # - table_alias_for
+          #
+          # UNSUPPORTED methods that will fail:
+          # - create_join_table
+          # - drop_join_table
+          # - create_alter_table
+          # - change_column_default
+          # - change_column_null
+          # - rename_column
+          #
+          # UPCOMING future methods:
+          # - clone (option -> close, or read-only (#lock / unlock) )
+          # - refresh
+          # - rename_table
+
+          define_unsupported_method :rename_table
+
+          # Opens a closed index.
+          # @param [String] table_name
+          # @return [Boolean] acknowledged status
+          def open_table(table_name)
+            schema_cache.clear_data_source_cache!(table_name.to_s)
+            api(:indices, :open, { index: table_name }, 'OPEN TABLE').dig('acknowledged')
+          end
+
+          # Opens closed indices.
+          # @param [Array] table_names
+          # @return [Array] acknowledged status for each provided table
+          def open_tables(*table_names)
+            table_names -= [schema_migration.table_name, InternalMetadata.table_name]
+            return if table_names.empty?
+
+            table_names.map { |table_name| open_table(table_name) }
+          end
+
+          # Closes an index.
+          # @param [String] table_name
+          # @return [Boolean] acknowledged status
+          def close_table(table_name)
+            schema_cache.clear_data_source_cache!(table_name.to_s)
+            api(:indices, :close, { index: table_name }, 'CLOSE TABLE').dig('acknowledged')
+          end
+
+          # Closes indices by provided names.
+          # @param [Array] table_names
+          # @return [Array] acknowledged status for each provided table
+          def close_tables(*table_names)
+            table_names -= [schema_migration.table_name, InternalMetadata.table_name]
+            return if table_names.empty?
+
+            table_names.map { |table_name| close_table(table_name) }
+          end
+
+          # truncates index by provided name.
+          # HINT: Elasticsearch does not have a +truncate+ concept:
+          # - so we have to store the current index' schema
+          # - drop the index
+          # - and create it again
+          # @param [String] table_name
+          # @return [Boolean] acknowledged status
+          def truncate_table(table_name)
+            # force: automatically drops an existing index
+            create_table(table_name, force: true, **table_schema(table_name))
+          end
+
+          alias :truncate :truncate_table
+
+          # truncate indices by provided names.
+          # @param [Array] table_names
+          # @return [Array] acknowledged status for each provided table
+          def truncate_tables(*table_names)
+            table_names -= [schema_migration.table_name, InternalMetadata.table_name]
+            return if table_names.empty?
+
+            table_names.map { |table_name| truncate_table(table_name) }
+          end
+
+          # drops an index
+          # [<tt>:if_exists</tt>]
+          #   Set to +true+ to only drop the table if it exists.
+          #   Defaults to false.
+          # @param [String] table_name
+          # @param [Hash] options
+          # @return [Array] acknowledged status for provided table
+          def drop_table(table_name, **options)
+            schema_cache.clear_data_source_cache!(table_name.to_s)
+            api(:indices, :delete, { index: table_name, ignore: (options[:if_exists] ? 404 : nil) }, 'DROP TABLE').dig('acknowledged')
+          end
+
+          # creates a new table (index).
+          # [<tt>:force</tt>]
+          #   Set to +true+ to drop an existing table
+          #   Defaults to false.
+          # [<tt>:copy_from</tt>]
+          #   Set to an existing index, to copy it's schema.
+          # [<tt>:if_not_exists</tt>]
+          #   Set to +true+ to skip creation if table already exists.
+          #   Defaults to false.
+          # @param [String] table_name
+          # @param [Boolean] force - force a drop on the existing table (default: false)
+          # @param [nil, String] copy_from - copy schema from existing table
+          # @param [Hash] options
+          # @return [Boolean] acknowledged status
+          def create_table(table_name, force: false, copy_from: nil, if_not_exists: false, **options)
+            return if if_not_exists && table_exists?(table_name)
+
+            # copy schema from existing table
+            options.merge!(table_schema(copy_from)) if copy_from
+
+            # create new definition
+            definition = create_table_definition(table_name, **extract_table_options!(options))
+
+            # yield optional block
+            if block_given?
+              definition.assign do |d|
+                yield d
+              end
+            end
+
+            # force drop existing table
+            if force
+              drop_table(table_name, if_exists: true)
+            else
+              schema_cache.clear_data_source_cache!(table_name.to_s)
+            end
+
+            # execute definition query(ies)
+            definition.exec!
+          end
+
+          # A block for changing mappings, settings & aliases in +table+.
+          #
+          #   # change_table() yields a ChangeTableDefinition instance
+          #   change_table(:suppliers) do |t|
+          #     t.mapping :name, :string
+          #     # Other column alterations here
+          #   end
+          def change_table(table_name, **options)
+            definition = update_table_definition(table_name, self, **options)
+
+            # yield optional block
+            if block_given?
+              definition.assign do |d|
+                yield d
+              end
+            end
+
+            # execute definition query(ies)
+            definition.exec!
+          end
+
+          # -- mapping -------------------------------------------------------------------------------------------------
+
+          def add_mapping(table_name, name, type, **options, &block)
+            _exec_change_table_with(:add_mapping, table_name, name, type, **options, &block)
+          end
+
+          alias :add_column :add_mapping
+
+          def change_mapping(table_name, name, type, **options, &block)
+            _exec_change_table_with(:change_mapping, table_name, name, type, **options, &block)
+          end
+
+          alias :change_column :change_mapping
+
+          def change_mapping_meta(table_name, name, **options)
+            _exec_change_table_with(:change_mapping_meta, table_name, name, **options)
+          end
+
+          def change_mapping_attributes(table_name, name, **options,&block)
+            _exec_change_table_with(:change_mapping_attributes, table_name, name, **options, &block)
+          end
+          alias :change_mapping_attribute :change_mapping_attributes
+
+          # -- setting -------------------------------------------------------------------------------------------------
+
+          def add_setting(table_name, name, value, **options, &block)
+            _exec_change_table_with(:add_setting, table_name, name, value, **options, &block)
+          end
+
+          def change_setting(table_name, name, value, **options, &block)
+            _exec_change_table_with(:change_setting, table_name, name, value, **options, &block)
+          end
+
+          def delete_setting(table_name, name, **options, &block)
+            _exec_change_table_with(:delete_setting, table_name, name, **options, &block)
+          end
+
+          # -- alias ---------------------------------------------------------------------------------------------------
+
+          def add_alias(table_name, name, **options, &block)
+            _exec_change_table_with(:add_alias, table_name, name, **options, &block)
+          end
+
+          def change_alias(table_name, name, **options, &block)
+            _exec_change_table_with(:change_alias, table_name, name, **options, &block)
+          end
+
+          def delete_alias(table_name, name, **options, &block)
+            _exec_change_table_with(:delete_alias, table_name, name, **options, &block)
+          end
+
+          private
+
+          def _exec_change_table_with(method, table_name, *args, **kwargs, &block)
+            change_table(table_name) do |t|
+              t.send(method, *args, **kwargs, &block)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/elasticsearch/type/multicast_value.rb

@@ -4,6 +4,8 @@ module ActiveRecord
       module Type # :nodoc:
         class MulticastValue < ActiveRecord::Type::Value
 
+          delegate :user_input_in_time_zone, to: :nested_type
+
           attr_reader :nested_type
 
           def initialize(nested_type: nil, **)

data/lib/active_record/connection_adapters/elasticsearch/unsupported_implementation.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Elasticsearch
+
+      class UnsupportedImplementationError < StandardError
+        def initialize(method_name)
+          super "Unsupported implementation of method: #{method_name}."
+        end
+      end
+
+      module UnsupportedImplementation
+        extend ActiveSupport::Concern
+
+        class_methods do
+          def define_unsupported_method(*method_names)
+            method_names.each do |method_name|
+              module_eval <<-RUBY, __FILE__, __LINE__ + 1
+                def #{method_name}(*args)
+                  raise NotImplementedError, "'##{method_name}' is originally defined by 'ActiveRecord::ConnectionAdapters' but is not supported by Elasticsearch. Choose a different solution to prevent the execution of this method!"
+                end
+              RUBY
+            end
+          end
+
+          private :define_unsupported_method
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/elasticsearch_adapter.rb

@@ -2,12 +2,17 @@
 
 require 'active_record/connection_adapters'
 
-# new
+require 'active_record/connection_adapters/elasticsearch/unsupported_implementation'
+
 require 'active_record/connection_adapters/elasticsearch/column'
 require 'active_record/connection_adapters/elasticsearch/database_statements'
 require 'active_record/connection_adapters/elasticsearch/quoting'
+require 'active_record/connection_adapters/elasticsearch/schema_creation'
+require 'active_record/connection_adapters/elasticsearch/schema_definitions'
+require 'active_record/connection_adapters/elasticsearch/schema_dumper'
 require 'active_record/connection_adapters/elasticsearch/schema_statements'
 require 'active_record/connection_adapters/elasticsearch/type'
+require 'active_record/connection_adapters/elasticsearch/table_statements'
 
 require 'arel/visitors/elasticsearch'
 require 'arel/collectors/elasticsearch_query'
@@ -40,15 +45,17 @@ module ActiveRecord # :nodoc:
 
       # defines the Elasticsearch 'base' structure, which is always included but cannot be resolved through mappings ...
       BASE_STRUCTURE = [
-        { 'name' => '_id', 'type' => 'string', 'null' => false, 'primary' => true },
-        { 'name' => '_index', 'type' => 'string', 'null' => false, 'virtual' => true },
-        { 'name' => '_score', 'type' => 'float', 'null' => false, 'virtual' => true },
-        { 'name' => '_type', 'type' => 'string', 'null' => false, 'virtual' => true }
+        { 'name' => '_id', 'type' => 'keyword', 'virtual' => true, 'meta' => { 'primary_key' => 'true' } },
+        { 'name' => '_index', 'type' => 'keyword', 'virtual' => true },
+        { 'name' => '_score', 'type' => 'float', 'virtual' => true },
+        { 'name' => '_type', 'type' => 'keyword', 'virtual' => true }
       ].freeze
 
+      include Elasticsearch::UnsupportedImplementation
       include Elasticsearch::Quoting
-      include Elasticsearch::SchemaStatements
       include Elasticsearch::DatabaseStatements
+      include Elasticsearch::SchemaStatements
+      include Elasticsearch::TableStatements
 
       class << self
         def base_structure_keys
@@ -58,8 +65,10 @@ module ActiveRecord # :nodoc:
         def new_client(config)
           # IMPORTANT: remove +adapter+ from config - otherwise we mess up with Faraday::AdapterRegistry
           client = ::Elasticsearch::Client.new(config.except(:adapter))
-          client.ping
+          client.ping unless config[:ping] == false
           client
+        rescue ::Elastic::Transport::Transport::Errors::Unauthorized
+          raise ActiveRecord::DatabaseConnectionError.username_error(config[:username])
         rescue ::Elastic::Transport::Transport::ServerError => error
           raise ::ActiveRecord::ConnectionNotEstablished, error.message
         end
@@ -123,6 +132,18 @@ module ActiveRecord # :nodoc:
       # reinitialize the constant with new types
       TYPE_MAP = ActiveRecord::Type::HashLookupTypeMap.new.tap { |m| initialize_type_map(m) }
 
+      # define native types - which will be used for schema-dumping
+      NATIVE_DATABASE_TYPES = {
+        primary_key: { name: 'long' },
+        string:      { name: 'keyword' },
+        blob:        { name: 'binary' },
+        datetime:    { name: 'date' },
+        bigint:      { name: 'long' },
+        json:        { name: 'object' }
+      }.merge(
+        TYPE_MAP.keys.map { |key| [key.to_sym, { name: key }] }.to_h
+      )
+
       def initialize(*args)
         super(*args)
 
@@ -131,20 +152,78 @@ module ActiveRecord # :nodoc:
         @prepared_statements = false
       end
 
-      def migrations_paths # :nodoc:
+      # overwrite method to provide a Elasticsearch path
+      def migrations_paths
         @config[:migrations_paths] || ['db/migrate_elasticsearch']
       end
 
-      # temporary workaround
-      # toDo: fixme
+      # Does this adapter support explain?
+      def supports_explain?
+        false
+      end
+
+      # Does this adapter support creating indexes in the same statement as
+      # creating the table?
+      def supports_indexes_in_create?
+        false
+      end
+
+      # Does this adapter support metadata comments on database objects (tables)?
+      # PLEASE NOTE: Elasticsearch does only support comments on mappings as 'meta' information.
+      # This method only relies to create comments on tables (indices) and is therefore not supported.
+      # see @ ActiveRecord::ConnectionAdapters::SchemaStatements#create_table
+      def supports_comments?
+        false
+      end
+
+      # Can comments for tables, columns, and indexes be specified in create/alter table statements?
+      # see @ ActiveRecord::ConnectionAdapters::ElasticsearchAdapter#supports_comments?
+      def supports_comments_in_create?
+        false
+      end
+
+      # disable metadata tables
       def use_metadata_table? # :nodoc:
         false
       end
 
-      # temporary workaround
-      # toDo: fixme
-      def schema_migration # :nodoc:
-        @schema_migration ||= ElasticsearchRecord::SchemaMigration
+      # returns a hash of 'ActiveRecord types' -> 'Elasticsearch types' (defined @ +NATIVE_DATABASE_TYPES+)
+      # @return [Hash]
+      def native_database_types # :nodoc:
+        NATIVE_DATABASE_TYPES
+      end
+
+      # calls the +elasticsearch-api+ endpoints by provided namespace and action.
+      # if a block was provided it'll yield the response.body and returns the blocks result.
+      # otherwise it will return the response itself...
+      # @param [Symbol] namespace - the API namespace (e.g. indices, nodes, sql, ...)
+      # @param [Symbol] action - the API action to call in tha namespace
+      # @param [Hash] arguments - action arguments
+      # @param [String (frozen)] name - the logging name
+      # @param [Boolean] async - send async (default: false) - currently not supported
+      # @return [Elasticsearch::API::Response, Object]
+      def api(namespace, action, arguments = {}, name = 'API', async: false)
+        raise ::StandardError, 'ASYNC api calls are not supported' if async
+
+        # resolve the API target
+        target = namespace == :core ? @connection : @connection.__send__(namespace)
+
+        log("#{namespace}.#{action}", arguments, name, async: async) do
+          response = ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+            target.__send__(action, arguments)
+          end
+
+          if response.is_a?(::Elasticsearch::API::Response)
+            # reverse information for the LogSubscriber - shows the 'query-time' in the logs
+            # this works, since we use a referenced hash ...
+            arguments[:_qt] = response['took']
+
+            # raise timeouts
+            raise(ActiveRecord::StatementTimeout, "Elasticsearch api request failed due a timeout") if response['timed_out']
+          end
+
+          response
+        end
       end
 
       private
@@ -156,7 +235,22 @@ module ActiveRecord # :nodoc:
       # catch Elasticsearch Transport-errors to be treated as +StatementInvalid+ (the original message is still readable ...)
       def translate_exception(exception, message:, sql:, binds:)
         case exception
-        when Elastic::Transport::Transport::ServerError
+        when ::Elastic::Transport::Transport::Errors::ClientClosedRequest
+          ::ActiveRecord::QueryCanceled.new(message, sql: sql, binds: binds)
+        when ::Elastic::Transport::Transport::Errors::RequestTimeout
+          ::ActiveRecord::StatementTimeout.new(message, sql: sql, binds: binds)
+        when ::Elastic::Transport::Transport::Errors::Conflict
+          ::ActiveRecord::RecordNotUnique.new(message, sql: sql, binds: binds)
+        when ::Elastic::Transport::Transport::Errors::BadRequest
+          if exception.message.match?(/resource_already_exists_exception/)
+            ::ActiveRecord::DatabaseAlreadyExists.new(message, sql: sql, binds: binds)
+          else
+            ::ActiveRecord::StatementInvalid.new(message, sql: sql, binds: binds)
+          end
+        when ::Elastic::Transport::Transport::Errors::Unauthorized
+          ::ActiveRecord::DatabaseConnectionError.username_error(@config[:username])
+          # must be last 'Elastic' error
+        when ::Elastic::Transport::Transport::ServerError
           ::ActiveRecord::StatementInvalid.new(message, sql: sql, binds: binds)
         else
           # just forward the exception ...
@@ -181,10 +275,10 @@ module ActiveRecord # :nodoc:
       # returns a new collector for the Arel visitor.
       # @return [Arel::Collectors::ElasticsearchQuery]
       def collector
-        # IMPORTANT: since prepared statements doesn't make sense for elasticsearch,
-        # we don't have to check for +prepared_statements+ here.
+        # IMPORTANT: prepared statements doesn't make sense for elasticsearch,
+        # so we don't have to check for +prepared_statements+ here.
         # Also, bindings are (currently) not supported.
-        # So, we just need a query collector...
+        # So, we just need a single, simple query collector...
         Arel::Collectors::ElasticsearchQuery.new
       end
 
@@ -195,7 +289,6 @@ module ActiveRecord # :nodoc:
       end
 
       # Builds the result object.
-      #
       # This is an internal hook to make possible connection adapters to build
       # custom result objects with response-specific data.
       # @return [ElasticsearchRecord::Result]

data/lib/arel/collectors/elasticsearch_query.rb

@@ -60,7 +60,6 @@ module Arel # :nodoc: all
       end
 
       # used by the +Arel::Visitors::Elasticsearch#compile+ method (and default Arel visitors)
-      # todo: maybe return arguments with :_meta information instead of self ...
       def value
         self
       end