elasticsearch_record 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9bd7858c5e59826439234e17437d5b8ff62eebbced936a5effee9da19159de5e
-  data.tar.gz: ae5d249a7d888363381bee306184c40979445aea26c5e1eec98ac94e11009e88
+  metadata.gz: 0d9553144d67b0b52192c1da8165a42ee27feeadfde5b8e6cc186466c52606ce
+  data.tar.gz: bf7bcb6fd5bbb97d8595066260349f08c0d8a1fa53da3fc3686b6e6e7a9bced5
 SHA512:
-  metadata.gz: 23d1062c88940694455c0ef93b7fa7f9c648b20011ed69125234304b9e5e1f61a3ccffa87967af8b757fac7e9846c66051b3517b210d3a91f3e41d71ccd7c057
-  data.tar.gz: c975c08bfe47faa8b098f7a457ceb828a8314e3a51c0e0554caa6f73d08b85ce168b4ca3466a54f392336debdae5c44ddebe267c83bda1eb2ee94c9f2b059362
+  metadata.gz: 709fb46f9e622a0d0eefff1eae49879a72301cd12cb6c92f465d689a7016269aa6f4b1a80773116c6d6c1e22f77ce40d3688ab12ec3757cca514984421fc7d26
+  data.tar.gz: 8ddc6805b580aabc3ea085cf61446ec498b712280d5bf7ac2899aa1ef8c9fa02478282c3c807749703406d8bc898f9b9154c32d04394eff3aa813ea395a53cf7

data/README.md

@@ -1,8 +1,10 @@
 # ElasticsearchRecord
 
+[![Gem Version](https://badge.fury.io/rb/elasticsearch_record.svg)](https://badge.fury.io/rb/elasticsearch_record)
+
 ActiveRecord functionality for Elasticsearch indexes & documents.
 
-_ElasticsearchRecord is a ActiveRecord-fork and tries to provide the same functionality for Elasticsearch._
+_ElasticsearchRecord is a ActiveRecord-fork and provides similar functionality for Elasticsearch._
 
 -----
 
@@ -118,7 +120,7 @@ Different to the default where-method you can now use it in different ways.
 
 Using it by default with a Hash, the method decides itself to either add a filter, or must_not query.
 
-_Hint: If not overwritten through ```kind(...)``` a default kind **:bool** will be used._
+_Hint: If not provided through ```#kind```-method a default kind **:bool** will be used._
 ```ruby
 # use it by default
 Search.where(name: 'A nice object')

data/docs/CHANGELOG.md

@@ -1,5 +1,16 @@
 # ElasticsearchRecord - CHANGELOG
 
+## [1.0.1] - 2022-10-19
+* [add] ```ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Nested``` class to cast nested values
+* [add] **properties** to column definition (so they are also searchable by _Relation_ conditions)
+* [add] exception for _Relation_ #pit_results if batch size is too large
+* [add] a default _#find_by_id_-method to proved a 'fallback' functionality for the primary '_id' column
+* [fix] nested properties are not cast for column-type "object"
+* [fix] ```ActiveRecord::ConnectionAdapters::Elasticsearch::SchemaStatements``` fields and property detection
+* [fix] ```ElasticsearchRecord::StatementCache::PartialQuery``` reference manipulation of cached hash _(missing .deep_dup )_
+* [ref] ```ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Object``` class to only cast object values
+* [ref] ```Arel::Visitors::Elasticsearch#visit_Sort``` to detect a random sort with correct keyword: "**\_\_rand\_\_**"
+
 ## [1.0.0] - 2022-10-18
 * [add] patch for ```ActiveRecord::Relation::Merger``` - to support AR-relations
 * [add] ```ElasticsearchRecord::Relation#pit_results``` _offset_ & _yield_ support

data/lib/active_record/connection_adapters/elasticsearch/column.rb

@@ -5,11 +5,12 @@ module ActiveRecord
     module Elasticsearch
       class Column < ConnectionAdapters::Column # :nodoc:
 
-        attr_reader :virtual, :fields
+        attr_reader :virtual, :fields, :properties
 
         def initialize(name, default, sql_type_metadata = nil, null = true, default_function = nil, **kwargs)
-          @virtual = kwargs.delete(:virtual)
-          @fields  = kwargs.delete(:fields)
+          @virtual    = kwargs.delete(:virtual)
+          @fields     = kwargs.delete(:fields)
+          @properties = kwargs.delete(:properties)
           super(name, default, sql_type_metadata, null, default_function, **kwargs)
         end
 
@@ -26,6 +27,29 @@ module ActiveRecord
         def fields?
           fields.present?
         end
+
+        # returns true if this column has nested properties
+        # To receive the nested names just call +#properties+ on this object.
+        # @return [Boolean]
+        def properties?
+          properties.present?
+        end
+
+        # returns a array of field names
+        # @return [Array]
+        def field_names
+          return [] unless fields?
+
+          fields.map { |field| field['name'] }
+        end
+
+        # returns a array of property names
+        # @return [Array]
+        def property_names
+          return [] unless properties?
+
+          properties.map { |property| property['name'] }
+        end
       end
     end
   end

data/lib/active_record/connection_adapters/elasticsearch/schema_statements.rb

@@ -23,7 +23,7 @@ module ActiveRecord
         # @param [String] index_name
         # @return [Hash]
         def settings(index_name)
-          api(:indices, :get_settings, { index: index_name }, 'SCHEMA').dig(index_name, 'settings','index')
+          api(:indices, :get_settings, { index: index_name }, 'SCHEMA').dig(index_name, 'settings', 'index')
         end
 
         # Returns the list of a table's column names, data types, and default values.
@@ -38,12 +38,14 @@ module ActiveRecord
           # since the received mappings do not have the "primary" +_id+-column we manually need to add this here
           # The BASE_STRUCTURE will also include some meta keys like '_score', '_type', ...
           ActiveRecord::ConnectionAdapters::ElasticsearchAdapter::BASE_STRUCTURE + structure['properties'].map { |key, prop|
-            # mappings can have +fields+ - we also want them for 'query-conditions'
-            # that can be resolved through +.column_names+
-            fields = prop.delete('fields') || []
+            # resolve (nested) fields and properties
+            fields, properties = resolve_fields_and_properties(key, prop, true)
 
-            # we need to merge the name & possible nested fields (which are also searchable)
-            prop.merge('name' => key, 'fields' => fields.map { |fkey, _field| "#{key}.#{fkey}" })
+            # fallback for possible empty type
+            type = prop['type'].presence || (properties.present? ? 'object' : 'nested')
+
+            # return a new hash
+            prop.merge('name' => key, 'type' => type, 'fields' => fields, 'properties' => properties)
           }
         end
 
@@ -54,18 +56,16 @@ module ActiveRecord
         # @param [Hash] field
         # @return [ActiveRecord::ConnectionAdapters::Column]
         def new_column_from_field(_table_name, field)
-          # fallback for possible empty type
-          field_type = field['type'].presence || (field['properties'].present? ? 'nested' : 'object')
-
           ActiveRecord::ConnectionAdapters::Elasticsearch::Column.new(
             field["name"],
             field["null_value"],
-            fetch_type_metadata(field_type),
+            fetch_type_metadata(field["type"]),
             field['null'].nil? ? true : field['null'],
             nil,
-            comment: field['meta'] ? field['meta'].map { |k, v| "#{k}: #{v}" }.join(' | ') : nil,
-            virtual: field['virtual'],
-            fields:  field['fields']
+            comment:    field['meta'] ? field['meta'].map { |k, v| "#{k}: #{v}" }.join(' | ') : nil,
+            virtual:    field['virtual'],
+            fields:     field['fields'],
+            properties: field['properties']
           )
         end
 
@@ -128,6 +128,59 @@ module ActiveRecord
         def max_result_window
           10000
         end
+
+        private
+
+        # returns a multidimensional array with fields & properties from the provided +prop+.
+        # Nested fields & properties will be also detected.
+        # .
+        #   resolve_fields_and_properties('user', {...})
+        #   # > [
+        #     # fields
+        #     [0] [
+        #         [0] {
+        #             "name" => "user.name.analyzed",
+        #             "type" => "text"
+        #         }
+        #     ],
+        #     # properties
+        #     [1] [
+        #         [0] {
+        #             "name" => "user.id",
+        #             "type" => "integer"
+        #         },
+        #         [1] {
+        #             "name" => "user.name",
+        #             "type" => "keyword"
+        #         }
+        #     ]
+        # ]
+        #
+        # @param [String] key
+        # @param [Hash] prop
+        # @param [Boolean] root - provide true, if this is a top property entry (default: false)
+        # @return [[Array, Array]]
+        def resolve_fields_and_properties(key, prop, root = false)
+          # mappings can have +fields+ - we also want them for 'query-conditions'
+          fields = (prop['fields'] || {}).map { |field_key, field_def|
+            { 'name' => "#{key}.#{field_key}", 'type' => field_def['type'] }
+          }
+
+          # initial empty array
+          properties = []
+
+          if prop['properties'].present?
+            prop['properties'].each do |nested_key, nested_prop|
+              nested_fields, nested_properties = resolve_fields_and_properties("#{key}.#{nested_key}", nested_prop)
+              fields                           |= nested_fields
+              properties                       |= nested_properties
+            end
+          elsif !root # don't add the root property as sub-property
+            properties << { 'name' => key, 'type' => prop['type'] }
+          end
+
+          [fields, properties]
+        end
       end
     end
   end

data/lib/active_record/connection_adapters/elasticsearch/type/multicast_value.rb

@@ -25,7 +25,7 @@ module ActiveRecord
 
           def _deserialize(value)
             # check for the special +object+ type which is forced to be casted
-            return _deserialize_by_nested_type(value) if nested_type.type == :object && nested_type.forced?
+            return _deserialize_by_nested_type(value) if nested_type.type == :object
 
             if value.is_a?(Array)
               value.map { |val| _deserialize_by_nested_type(val) }

data/lib/active_record/connection_adapters/elasticsearch/type/nested.rb

@@ -0,0 +1,21 @@
+module ActiveRecord
+  module ConnectionAdapters
+    module Elasticsearch
+      module Type # :nodoc:
+        class Nested < ActiveRecord::Type::Value
+
+          def type
+            :nested
+          end
+
+          private
+
+          # cast value
+          def cast_value(value)
+            value.to_h
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/elasticsearch/type/object.rb

@@ -3,39 +3,15 @@ module ActiveRecord
     module Elasticsearch
       module Type # :nodoc:
         class Object < ActiveRecord::Type::Value
-          attr_reader :cast_type, :default
-
-          # creates a new object type which can be natively every value.
-          # Providing a +cast+ will call the value with this method (or callback)
-          # @param [nil, Symbol, String, Proc] cast - the cast type
-          # @param [nil, Object] default
-          # @param [Boolean] force - force value to be casted (used by the MulticastValue type) - (default: false)
-          def initialize(cast: nil, default: nil, force: false, **)
-            @cast_type = cast
-            @default   = default
-            @force     = force
-          end
-
           def type
             :object
           end
 
-          def forced?
-            @force
-          end
-
           private
 
           # cast value by provided cast_method
           def cast_value(value)
-            case self.cast_type
-            when Symbol, String
-              value.public_send(self.cast_type) rescue default
-            when Proc
-              self.cast_type.(value) rescue default
-            else
-              value
-            end
+            value.to_h
           end
         end
       end

data/lib/active_record/connection_adapters/elasticsearch/type.rb

@@ -2,6 +2,7 @@
 
 require 'active_record/connection_adapters/elasticsearch/type/format_string'
 require 'active_record/connection_adapters/elasticsearch/type/multicast_value'
+require 'active_record/connection_adapters/elasticsearch/type/nested'
 require 'active_record/connection_adapters/elasticsearch/type/object'
 require 'active_record/connection_adapters/elasticsearch/type/range'
 

data/lib/active_record/connection_adapters/elasticsearch_adapter.rb

@@ -88,11 +88,11 @@ module ActiveRecord # :nodoc:
           m.register_type 'date', Type::DateTime.new
 
           # force a hash
-          m.register_type 'object', ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Object.new(cast: :to_h, force: true)
+          m.register_type 'object', ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Object.new
           m.alias_type 'flattened', "object"
 
           # array of objects
-          m.register_type 'nested', ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Object.new(cast: :to_h)
+          m.register_type 'nested', ActiveRecord::ConnectionAdapters::Elasticsearch::Type::Nested.new
 
           ip_type = ActiveRecord::ConnectionAdapters::Elasticsearch::Type::FormatString.new(format: /^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}$/)
 

data/lib/arel/visitors/elasticsearch.rb

@@ -410,8 +410,8 @@ module Arel # :nodoc: all
           key = visit(o.expr)
           dir = visit(o.direction)
 
-          # we support a special key: _rand to create a simple random method ...
-          if key == '_rand'
+          # we support a special key: __rand__ to create a simple random method ...
+          if key == '__rand__'
             assign({
                      "_script" => {
                        "script" => "Math.random()",

data/lib/elasticsearch_record/gem_version.rb

@@ -9,7 +9,7 @@ module ElasticsearchRecord
   module VERSION
     MAJOR = 1
     MINOR = 0
-    TINY  = 0
+    TINY  = 1
     PRE   = nil
 
     STRING = [MAJOR, MINOR, TINY, PRE].compact.join(".")

data/lib/elasticsearch_record/model_schema.rb

@@ -20,12 +20,13 @@ module ElasticsearchRecord
         @source_column_names ||= columns.reject(&:virtual).map(&:name) - ActiveRecord::ConnectionAdapters::ElasticsearchAdapter.base_structure_keys
       end
 
-      # returns an array with columns names, that are searchable (also includes nested )
+      # returns an array with columns names, that are searchable (also includes nested fields & properties )
       def searchable_column_names
         @searchable_column_names ||= columns.reject(&:virtual).reduce([]) { |m, column|
           m << column.name
-          m += column.fields if column.fields?
-          m
+          m += column.field_names
+          m += column.property_names
+          m.uniq
         }
       end
 

data/lib/elasticsearch_record/querying.rb

@@ -16,6 +16,13 @@ module ElasticsearchRecord
       ].freeze # :nodoc:
       delegate(*ES_QUERYING_METHODS, to: :all)
 
+      # finds a single record by provided id.
+      # This method is overwritten to support the primary key column (+_id+).
+      # @param [Object] id
+      def find_by_id(id)
+        has_attribute?('id') ? super(id) : public_send(:find_by__id, id)
+      end
+
       # finds records by sql, query-arguments or query-object.
       #
       # PLEASE NOTE: This method is used by different other methods:

data/lib/elasticsearch_record/relation/calculation_methods.rb

@@ -3,10 +3,10 @@ module ElasticsearchRecord
     module CalculationMethods
       # Count the records.
       #
-      #   Person.count
+      #   Person.all.count
       #   => the total count of all people
       #
-      #   Person.count(:age)
+      #   Person.all.count(:age)
       #   => returns the total count of all people whose age is present in database
       def count(column_name = nil)
         # fallback to default
@@ -46,7 +46,7 @@ module ElasticsearchRecord
       # percentiles over numeric values extracted from the aggregated documents.
       # Returns a hash with empty values (but keys still exists) if there is no row.
       #
-      #   Person.percentiles(:year)
+      #   Person.all.percentiles(:year)
       #   > {
       #      "1.0" => 2016.0,
       #      "5.0" => 2016.0,
@@ -68,7 +68,7 @@ module ElasticsearchRecord
       # For example, if a value is greater than or equal to 95% of the observed values it is
       # said to be at the 95th percentile rank.
       #
-      #   Person.percentile_ranks(:year, [500,600])
+      #   Person.all.percentile_ranks(:year, [500,600])
       #   > {
       #      "1.0" => 2016.0,
       #      "5.0" => 2016.0,
@@ -86,7 +86,7 @@ module ElasticsearchRecord
 
       # Calculates the cardinality on a given column. Returns +0+ if there's no row.
       #
-      #   Person.cardinality(:age)
+      #   Person.all.cardinality(:age)
       #   > 12
       #
       # @param [Symbol, String] column_name
@@ -96,7 +96,7 @@ module ElasticsearchRecord
 
       # Calculates the average value on a given column. Returns +nil+ if there's no row. See #calculate for examples with options.
       #
-      #   Person.average(:age) # => 35.8
+      #   Person.all.average(:age) # => 35.8
       #
       # @param [Symbol, String] column_name
       def average(column_name)
@@ -106,7 +106,7 @@ module ElasticsearchRecord
       # Calculates the minimum value on a given column. The value is returned
       # with the same data type of the column, or +nil+ if there's no row.
       #
-      #   Person.minimum(:age)
+      #   Person.all.minimum(:age)
       #   > 7
       #
       # @param [Symbol, String] column_name
@@ -118,7 +118,7 @@ module ElasticsearchRecord
       # with the same data type of the column, or +nil+ if there's no row. See
       # #calculate for examples with options.
       #
-      #   Person.maximum(:age) # => 93
+      #   Person.all.maximum(:age) # => 93
       #
       # @param [Symbol, String] column_name
       def maximum(column_name)
@@ -129,7 +129,7 @@ module ElasticsearchRecord
       # with the same data type of the column, +0+ if there's no row. See
       # #calculate for examples with options.
       #
-      #   Person.sum(:age) # => 4562
+      #   Person.all.sum(:age) # => 4562
       #
       # @param [Symbol, String] column_name (optional)
       def sum(column_name)

data/lib/elasticsearch_record/relation/result_methods.rb

@@ -89,6 +89,8 @@ module ElasticsearchRecord
       # @param [String] keep_alive - how long to keep alive (for each single request) - default: '1m'
       # @param [Integer] batch_size - how many results per query (default: 1000 - this means at least 10 queries before reaching the +max_result_window+)
       def pit_results(keep_alive: '1m', batch_size: 1000)
+        raise ArgumentError, "Batch size cannot be above the 'max_result_window' (#{klass.connection.max_result_window}) !" if batch_size > klass.connection.max_result_window
+
         # check if a limit or offset values was provided
         results_limit  = limit_value ? limit_value : Float::INFINITY
         results_offset = offset_value ? offset_value : 0
@@ -116,7 +118,7 @@ module ElasticsearchRecord
             current_response = relation.spawn.configure!(current_pit_hash).limit!(batch_size).resolve('Pit').response
 
             # resolve only data from hits->hits[{_source}]
-            current_results        = current_response['hits']['hits'].map { |result| result['_source'] }
+            current_results        = current_response['hits']['hits'].map { |result| result['_source'].merge('_id' => result['_id']) }
             current_results_length = current_results.length
 
             # check if we reached the required offset

data/lib/elasticsearch_record/statement_cache.rb

@@ -11,7 +11,7 @@ module ElasticsearchRecord
 
       def sql_for(binds, connection)
         # dup original array
-        claims = @values.dup
+        claims = @values.deep_dup
 
         # substitute binds
         claims.each do |claim|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticsearch_record
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Tobias Gonsior
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-17 00:00:00.000000000 Z
+date: 2022-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -80,8 +80,8 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '13.0'
-description: 'ElasticsearchRecord is a ActiveRecord-fork and tries to provide the
-  same functionality for Elasticsearch.
+description: 'ElasticsearchRecord is a ActiveRecord-fork and provides similar functionality
+  for Elasticsearch.
 
   '
 email:
@@ -106,6 +106,7 @@ files:
 - lib/active_record/connection_adapters/elasticsearch/type.rb
 - lib/active_record/connection_adapters/elasticsearch/type/format_string.rb
 - lib/active_record/connection_adapters/elasticsearch/type/multicast_value.rb
+- lib/active_record/connection_adapters/elasticsearch/type/nested.rb
 - lib/active_record/connection_adapters/elasticsearch/type/object.rb
 - lib/active_record/connection_adapters/elasticsearch/type/range.rb
 - lib/active_record/connection_adapters/elasticsearch_adapter.rb