elasticsearch_record 1.0.0

data/lib/active_record/connection_adapters/elasticsearch/quoting.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Elasticsearch
+      module Quoting # :nodoc:
+        # Quotes the column value to help prevent
+        def quote(value)
+          case value
+            # those values do not need to be quoted
+          when BigDecimal, Numeric, String, Symbol, nil, true, false then value
+          when ActiveSupport::Duration then value.to_i
+          when Array then value.map { |val| quote(val) }
+          when Hash then value.transform_values { |val| quote(val) }
+          else
+            super
+          end
+        end
+
+        def quoted_true
+          true
+        end
+
+        def unquoted_true
+          true
+        end
+
+        def quoted_false
+          false
+        end
+
+        def unquoted_false
+          false
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/elasticsearch/schema_statements.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Elasticsearch
+      module SchemaStatements # :nodoc:
+        # Returns the relation names usable to back Active Record models.
+        # For Elasticsearch this means all indices - which also includes system +dot+ '.' indices.
+        # @see ActiveRecord::ConnectionAdapters::SchemaStatements#data_sources
+        # @return [Array<String>]
+        def data_sources
+          api(:indices, :get_settings, { index: :_all }, 'SCHEMA').keys
+        end
+
+        # returns a hash of all mappings by provided index_name
+        # @param [String] index_name
+        # @return [Hash]
+        def mappings(index_name)
+          api(:indices, :get_mapping, { index: index_name }, 'SCHEMA').dig(index_name, 'mappings')
+        end
+
+        # returns a hash of all settings by provided index_name
+        # @param [String] index_name
+        # @return [Hash]
+        def settings(index_name)
+          api(:indices, :get_settings, { index: index_name }, 'SCHEMA').dig(index_name, 'settings','index')
+        end
+
+        # Returns the list of a table's column names, data types, and default values.
+        # @see ActiveRecord::ConnectionAdapters::SchemaStatements#columns
+        # @see ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter#column_definitions
+        # @param [String] table_name
+        # @return [Array<Hash>]
+        def column_definitions(table_name)
+          structure = mappings(table_name)
+          raise(ActiveRecord::StatementInvalid, "Could not find elasticsearch index '#{table_name}'") if structure.blank? || structure['properties'].blank?
+
+          # since the received mappings do not have the "primary" +_id+-column we manually need to add this here
+          # The BASE_STRUCTURE will also include some meta keys like '_score', '_type', ...
+          ActiveRecord::ConnectionAdapters::ElasticsearchAdapter::BASE_STRUCTURE + structure['properties'].map { |key, prop|
+            # mappings can have +fields+ - we also want them for 'query-conditions'
+            # that can be resolved through +.column_names+
+            fields = prop.delete('fields') || []
+
+            # we need to merge the name & possible nested fields (which are also searchable)
+            prop.merge('name' => key, 'fields' => fields.map { |fkey, _field| "#{key}.#{fkey}" })
+          }
+        end
+
+        # creates a new column object from provided field Hash
+        # @see ActiveRecord::ConnectionAdapters::SchemaStatements#columns
+        # @see ActiveRecord::ConnectionAdapters::MySQL::SchemaStatements#new_column_from_field
+        # @param [String] _table_name
+        # @param [Hash] field
+        # @return [ActiveRecord::ConnectionAdapters::Column]
+        def new_column_from_field(_table_name, field)
+          # fallback for possible empty type
+          field_type = field['type'].presence || (field['properties'].present? ? 'nested' : 'object')
+
+          ActiveRecord::ConnectionAdapters::Elasticsearch::Column.new(
+            field["name"],
+            field["null_value"],
+            fetch_type_metadata(field_type),
+            field['null'].nil? ? true : field['null'],
+            nil,
+            comment: field['meta'] ? field['meta'].map { |k, v| "#{k}: #{v}" }.join(' | ') : nil,
+            virtual: field['virtual'],
+            fields:  field['fields']
+          )
+        end
+
+        # lookups from building the @columns_hash.
+        # since Elasticsearch has the "feature" to provide multicast values on any type, we need to fetch them ...
+        # you know, ES can return an integer or an array of integers for any column ...
+        # @param [ActiveRecord::ConnectionAdapters::Elasticsearch::Column] column
+        # @return [ActiveRecord::ConnectionAdapters::Elasticsearch::Type::MulticastValue]
+        def lookup_cast_type_from_column(column)
+          type_map.lookup(:multicast_value, super)
+        end
+
+        # Returns a array of tables primary keys.
+        # PLEASE NOTE: Elasticsearch does not have a concept of primary key.
+        # The only thing that uniquely identifies a document is the index together with the +_id+.
+        # To not break the "ConnectionAdapters" concept we simulate this through the BASE_STRUCTURE.
+        # We know, we can just return '_id' here ...
+        # @see ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter#primary_keys
+        # @param [String] _table_name
+        def primary_keys(_table_name)
+          ActiveRecord::ConnectionAdapters::ElasticsearchAdapter::BASE_STRUCTURE
+            .select { |f| f["primary"] }
+            .map { |f| f["name"] }
+        end
+
+        # Checks to see if the data source +name+ exists on the database.
+        #
+        #   data_source_exists?(:ebooks)
+        # @see ActiveRecord::ConnectionAdapters::SchemaStatements#data_source_exists?
+        # @param [String, Symbol] name
+        # @return [Boolean]
+        def data_source_exists?(name)
+          # response returns boolean
+          api(:indices, :exists?, { index: name }, 'SCHEMA')
+        end
+
+        # Returns an array of table names defined in the database.
+        # For Elasticsearch this means all normal indices (no system +dot+ '.' indices)
+        # @see ActiveRecord::ConnectionAdapters::SchemaStatements#tables
+        # @return [Array<String>]
+        def tables
+          data_sources.reject { |key| key[0] == '.' }
+        end
+
+        # Checks to see if the table +table_name+ exists on the database.
+        #
+        #   table_exists?(:developers)
+        #
+        # @see ActiveRecord::ConnectionAdapters::SchemaStatements#table_exists?
+        # @param [String, Symbol] table_name
+        # @return [Boolean]
+        def table_exists?(table_name)
+          # just reference to the data sources
+          data_source_exists?(table_name)
+        end
+
+        # returns the maximum allowed size for queries.
+        # The query will raise an ActiveRecord::StatementInvalid if the requested limit is above this value.
+        # @return [Integer]
+        def max_result_window
+          10000
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/elasticsearch/type/format_string.rb

@@ -0,0 +1,28 @@
+module ActiveRecord
+  module ConnectionAdapters
+    module Elasticsearch
+      module Type # :nodoc:
+        class FormatString < ActiveRecord::Type::String
+          attr_reader :format
+
+          def initialize(**args)
+            @format = args.delete(:format).presence || /.*/
+            super
+          end
+
+          def type
+            :format_string
+          end
+
+          private
+
+          def cast_value(value)
+            return value unless ::String === value
+            return '' unless value.match(format)
+            value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/elasticsearch/type/multicast_value.rb

@@ -0,0 +1,52 @@
+module ActiveRecord
+  module ConnectionAdapters
+    module Elasticsearch
+      module Type # :nodoc:
+        class MulticastValue < ActiveRecord::Type::Value
+
+          attr_reader :nested_type
+
+          def initialize(nested_type: nil, **)
+            @nested_type = nested_type || ActiveRecord::Type::Value.new
+          end
+
+          def type
+            nested_type.type
+          end
+
+          # overwrites the default deserialize behaviour
+          # @param [Object] value
+          # @return [Object,nil] deserialized object
+          def deserialize(value)
+            cast(_deserialize(value))
+          end
+
+          private
+
+          def _deserialize(value)
+            # check for the special +object+ type which is forced to be casted
+            return _deserialize_by_nested_type(value) if nested_type.type == :object && nested_type.forced?
+
+            if value.is_a?(Array)
+              value.map { |val| _deserialize_by_nested_type(val) }
+            elsif value.is_a?(Hash)
+              value.reduce({}) { |m, (key, val)|
+                m[key] = _deserialize_by_nested_type(val)
+                m
+              }
+            else
+              _deserialize_by_nested_type(value)
+            end
+          end
+
+          # in some cases we cannot deserialize, since the ES-type don't match well with the provided value
+          # but the result should be ok, as it is (e.g. 'Hash')...
+          # so we rescue here with just the provided value
+          def _deserialize_by_nested_type(value)
+            nested_type.deserialize(value) rescue value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/elasticsearch/type/object.rb

@@ -0,0 +1,44 @@
+module ActiveRecord
+  module ConnectionAdapters
+    module Elasticsearch
+      module Type # :nodoc:
+        class Object < ActiveRecord::Type::Value
+          attr_reader :cast_type, :default
+
+          # creates a new object type which can be natively every value.
+          # Providing a +cast+ will call the value with this method (or callback)
+          # @param [nil, Symbol, String, Proc] cast - the cast type
+          # @param [nil, Object] default
+          # @param [Boolean] force - force value to be casted (used by the MulticastValue type) - (default: false)
+          def initialize(cast: nil, default: nil, force: false, **)
+            @cast_type = cast
+            @default   = default
+            @force     = force
+          end
+
+          def type
+            :object
+          end
+
+          def forced?
+            @force
+          end
+
+          private
+
+          # cast value by provided cast_method
+          def cast_value(value)
+            case self.cast_type
+            when Symbol, String
+              value.public_send(self.cast_type) rescue default
+            when Proc
+              self.cast_type.(value) rescue default
+            else
+              value
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/elasticsearch/type/range.rb

@@ -0,0 +1,42 @@
+module ActiveRecord
+  module ConnectionAdapters
+    module Elasticsearch
+      module Type # :nodoc:
+        class Range < MulticastValue
+
+          def type
+            "range_#{nested_type.type}".to_sym
+          end
+
+          private
+
+          def cast_value(value)
+            return (0..0) unless value.is_a?(Hash)
+            # check for existing gte & lte
+
+            min_value = if value['gte']
+                          value['gte']
+                        elsif value['gt']
+                          value['gt'] + 1
+                        else
+                          nil
+                        end
+
+            max_value = if value['lte']
+                          value['lte']
+                        elsif value['lt']
+                          value['lt'] - 1
+                        else
+                          nil
+                        end
+
+            return (0..0) if min_value.nil? || max_value.nil?
+
+            # build & return range
+            (min_value..max_value)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/elasticsearch/type.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'active_record/connection_adapters/elasticsearch/type/format_string'
+require 'active_record/connection_adapters/elasticsearch/type/multicast_value'
+require 'active_record/connection_adapters/elasticsearch/type/object'
+require 'active_record/connection_adapters/elasticsearch/type/range'
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Elasticsearch
+      module Type # :nodoc:
+
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/elasticsearch_adapter.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+require 'active_record/connection_adapters'
+
+# new
+require 'active_record/connection_adapters/elasticsearch/column'
+require 'active_record/connection_adapters/elasticsearch/database_statements'
+require 'active_record/connection_adapters/elasticsearch/quoting'
+require 'active_record/connection_adapters/elasticsearch/schema_statements'
+require 'active_record/connection_adapters/elasticsearch/type'
+
+require 'arel/visitors/elasticsearch'
+require 'arel/collectors/elasticsearch_query'
+
+gem 'elasticsearch'
+require 'elasticsearch'
+
+module ActiveRecord # :nodoc:
+  module ConnectionHandling # :nodoc:
+    def elasticsearch_connection(config)
+      config          = config.symbolize_keys
+
+      # move 'host' to 'hosts'
+      config[:hosts]  = config.delete(:host) if config[:host]
+
+      # enable logging (Rails.logger)
+      config[:logger] = logger if config.delete(:log)
+
+      ConnectionAdapters::ElasticsearchAdapter.new(
+        ConnectionAdapters::ElasticsearchAdapter.new_client(config),
+        logger,
+        config
+      )
+    end
+  end
+
+  module ConnectionAdapters # :nodoc:
+    class ElasticsearchAdapter < AbstractAdapter
+      ADAPTER_NAME = "Elasticsearch"
+
+      # defines the Elasticsearch 'base' structure, which is always included but cannot be resolved through mappings ...
+      BASE_STRUCTURE = [
+        { 'name' => '_id', 'type' => 'string', 'null' => false, 'primary' => true },
+        { 'name' => '_index', 'type' => 'string', 'null' => false, 'virtual' => true },
+        { 'name' => '_score', 'type' => 'float', 'null' => false, 'virtual' => true },
+        { 'name' => '_type', 'type' => 'string', 'null' => false, 'virtual' => true }
+      ].freeze
+
+      include Elasticsearch::Quoting
+      include Elasticsearch::SchemaStatements
+      include Elasticsearch::DatabaseStatements
+
+      class << self
+        def base_structure_keys
+          @base_structure_keys ||= BASE_STRUCTURE.map { |struct| struct['name'] }.freeze
+        end
+
+        def new_client(config)
+          # IMPORTANT: remove +adapter+ from config - otherwise we mess up with Faraday::AdapterRegistry
+          client = ::Elasticsearch::Client.new(config.except(:adapter))
+          client.ping
+          client
+        rescue ::Elastic::Transport::Transport::ServerError => error
+          raise ::ActiveRecord::ConnectionNotEstablished, error.message
+        end
+
+        private
+
+        def initialize_type_map(m)
+          m.register_type 'binary', Type::Binary.new
+          m.register_type 'boolean', Type::Boolean.new
+          m.register_type 'keyword', Type::String.new
+
+          m.alias_type 'constant_keyword', 'keyword'
+          m.alias_type 'wildcard', 'keyword'
+
+          # maybe use integer 8 here ...
+          m.register_type 'long', Type::BigInteger.new
+          m.register_type 'integer', Type::Integer.new
+          m.register_type 'short', Type::Integer.new(limit: 2)
+          m.register_type 'byte', Type::Integer.new(limit: 1)
+          m.register_type 'double', Type::Float.new(limit: 8)
+          m.register_type 'float', Type::Float.new(limit: 4)
+          m.register_type 'half_float', Type::Float.new(limit: 2)
+          m.register_type 'scaled_float', Type::Float.new(limit: 8, scale: 8)
+          m.register_type 'unsigned_long', Type::UnsignedInteger.new
+
+          m.register_type 'date', Type::DateTime.new
+
+          # force a hash
+          m.register_type 'object', ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Object.new(cast: :to_h, force: true)
+          m.alias_type 'flattened', "object"
+
+          # array of objects
+          m.register_type 'nested', ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Object.new(cast: :to_h)
+
+          ip_type = ActiveRecord::ConnectionAdapters::Elasticsearch::Type::FormatString.new(format: /^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}$/)
+
+          m.register_type 'integer_range', ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Range.new(nested_type: Type::Integer.new)
+          m.register_type 'float_range', ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Range.new(nested_type: Type::Float.new(limit: 4))
+          m.register_type 'long_range', ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Range.new(nested_type: Type::Integer.new(limit: 8))
+          m.register_type 'double_range', ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Range.new(nested_type: Type::Float.new(limit: 8))
+          m.register_type 'date_range', ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Range.new(nested_type: Type::DateTime.new)
+          m.register_type 'ip_range', ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Range.new(nested_type: ip_type)
+
+          m.register_type 'ip', ip_type
+          m.register_type 'version', ActiveRecord::ConnectionAdapters::Elasticsearch::Type::FormatString.new(format: /^\d+\.\d+\.\d+[\-\+A-Za-z\.]*$/)
+          # m.register_type 'murmur3', Murmur3.new
+
+          m.register_type 'text', Type::Text.new
+
+          # this special Type is required to parse a ES-value into the +nested_type+, array or hash.
+          # For arrays & hashes it tries to cast the values with the provided +nested_type+
+          # but falls back to provided value if cast fails.
+          # This type cannot be accessed through the mapping and is only called @ #lookup_cast_type_from_column
+          # @see ActiveRecord::ConnectionAdapters::Elasticsearch::SchemaStatements#lookup_cast_type_from_column
+          m.register_type :multicast_value do |_type, nested_type|
+            ActiveRecord::ConnectionAdapters::Elasticsearch::Type::MulticastValue.new(nested_type: nested_type)
+          end
+        end
+      end
+
+      # reinitialize the constant with new types
+      TYPE_MAP = ActiveRecord::Type::HashLookupTypeMap.new.tap { |m| initialize_type_map(m) }
+
+      def initialize(*args)
+        super(*args)
+
+        # prepared statements are not supported by Elasticsearch.
+        # documentation for mysql prepares statements @ https://dev.mysql.com/doc/refman/8.0/en/sql-prepared-statements.html
+        @prepared_statements = false
+      end
+
+      private
+
+      def type_map
+        TYPE_MAP
+      end
+
+      # catch Elasticsearch Transport-errors to be treated as +StatementInvalid+ (the original message is still readable ...)
+      def translate_exception(exception, message:, sql:, binds:)
+        case exception
+        when Elastic::Transport::Transport::ServerError
+          ::ActiveRecord::StatementInvalid.new(message, sql: sql, binds: binds)
+        else
+          # just forward the exception ...
+          exception
+        end
+      end
+
+      # provide a custom log instrumenter for elasticsearch subscribers
+      def log(gate, arguments, name, async: false, &block)
+        @instrumenter.instrument(
+          "query.elasticsearch_record",
+          gate:      gate,
+          name:      name,
+          arguments: gate == 'core.msearch' ? arguments.deep_dup : arguments,
+          async:     async) do
+          @lock.synchronize(&block)
+        rescue => e
+          raise translate_exception_class(e, arguments, [])
+        end
+      end
+
+      # returns a new collector for the Arel visitor.
+      # @return [Arel::Collectors::ElasticsearchQuery]
+      def collector
+        # IMPORTANT: since prepared statements doesn't make sense for elasticsearch,
+        # we don't have to check for +prepared_statements+ here.
+        # Also, bindings are (currently) not supported.
+        # So, we just need a query collector...
+        Arel::Collectors::ElasticsearchQuery.new
+      end
+
+      # returns a new visitor to compile Arel into Elasticsearch Hashes (in this case we use a query object)
+      # @return [Arel::Visitors::Elasticsearch]
+      def arel_visitor
+        Arel::Visitors::Elasticsearch.new(self)
+      end
+
+      # Builds the result object.
+      #
+      # This is an internal hook to make possible connection adapters to build
+      # custom result objects with response-specific data.
+      # @return [ElasticsearchRecord::Result]
+      def build_result(response, columns: [], column_types: {})
+        ElasticsearchRecord::Result.new(response, columns, column_types)
+      end
+
+      # register native types
+      ActiveRecord::Type.register(:format_string, ActiveRecord::ConnectionAdapters::Elasticsearch::Type::FormatString, adapter: :elasticsearch)
+      ActiveRecord::Type.register(:multicast_value, ActiveRecord::ConnectionAdapters::Elasticsearch::Type::MulticastValue, adapter: :elasticsearch)
+      ActiveRecord::Type.register(:object, ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Object, adapter: :elasticsearch, override: false)
+      ActiveRecord::Type.register(:range, ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Range, adapter: :elasticsearch)
+    end
+  end
+end

data/lib/arel/collectors/elasticsearch_query.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require 'elasticsearch_record/query'
+
+module Arel # :nodoc: all
+  module Collectors
+    class ElasticsearchQuery < ::ElasticsearchRecord::Query
+
+      # required for ActiveRecord
+      attr_accessor :preparable
+
+      def initialize
+        # force initialize a body as hash
+        super(body: {})
+
+        # @binds = []
+        @bind_index = 1
+      end
+
+      # send a proposal to this query
+      # @param [Symbol] action - the claim action
+      # @param [Array] args - args to claim
+      def claim(action, *args)
+        case action
+        when :index
+          # change the index name
+          @index = args[0]
+        when :type
+          # change the query type
+          @type = args[0]
+        when :status
+          # change the query status
+          @status = args[0]
+        when :columns
+          # change the query columns
+          @columns = args[0]
+        when :arguments
+          # change the query arguments
+          @arguments = args[0]
+        when :argument
+          # adds / sets any argument
+          if args.length == 2
+            @arguments[args[0]] = args[1]
+          else # should be a hash
+            @arguments.merge!(args[0])
+          end
+        when :body
+          # set the body var
+          @body = args[0]
+        when :assign
+          # calls a assign on the body
+          assign(*args)
+        else
+          raise "Unsupported claim action: #{action}"
+        end
+      end
+
+      def <<(claim)
+        self.claim(claim[0], *claim[1])
+      end
+
+      # used by the +Arel::Visitors::Elasticsearch#compile+ method (and default Arel visitors)
+      # todo: maybe return arguments with :_meta information instead of self ...
+      def value
+        self
+      end
+
+      # IMPORTANT: For SQL defaults (see @ Arel::Collectors::SubstituteBinds) a value
+      # will +not+ be directly assigned (see @ Arel::Visitors::ToSql#visit_Arel_Nodes_HomogeneousIn).
+      # instead it will be send as bind and then re-delegated to the SQL collector.
+      #
+      # This only works for linear SQL-queries and not nested Hashes
+      # (otherwise we have to collect those binds, and replace them afterwards).
+      #
+      # This will be ignored by the ElasticsearchQuery collector, but supports statement caches on the other side
+      # (see @ ActiveRecord::StatementCache::PartialQueryCollector)
+      def add_bind(bind, &block)
+        @bind_index += 1
+
+        self
+      end
+
+      # @see Arel::Collectors::ElasticsearchQuery#add_bind
+      def add_binds(binds, proc_for_binds = nil, &block)
+        @bind_index += binds.size
+        self
+      end
+
+      private
+
+      # calls a assign on the body
+      def assign(key, value)
+        # check for special provided key, to claim through an assign
+        if key == :__claim__
+          if value.is_a?(Array)
+            value.each do |arg|
+              vkey = arg.keys.first
+              claim(vkey, arg[vkey])
+            end
+          else
+            vkey = value.keys.first
+            claim(vkey, value[vkey])
+          end
+        elsif value.nil?
+          @body.delete(key)
+        else
+          @body[key] = value
+        end
+      end
+    end
+  end
+end

data/lib/arel/nodes/select_agg.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Arel # :nodoc: all
+  module Nodes
+    class SelectAgg < Unary
+
+      def left
+        expr[0]
+      end
+
+      def right
+        return expr[1].reduce({}) { |m, data| m.merge(data) } if expr[1].is_a?(Array)
+
+        expr[1]
+      end
+
+      def opts
+        expr[2]
+      end
+    end
+  end
+end