elasticsearch_record 1.3.1 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8e00649a9c1e9451d8ea006078889081a70cb178a3d0f23336d157929eefafc
-  data.tar.gz: '0596785a16b974b95a48ccb2bb3cb6a334ce553e8d80b1b82691ee98df18fe3b'
+  metadata.gz: 3f50568400141462093f5d91bf5b26efb44a52b9529577d5e72a4d3699a57717
+  data.tar.gz: 89536ffacc4ddd5fb4d0334eda843d206c25fc867664f6f9454ee07d921ec712
 SHA512:
-  metadata.gz: 61301e89b67d34fd4ed604a782c85b84bd2316f98c7b67ddf293ad2a27e17e30531b905e7ecdc5a4d9dad34099d9feab8603269b2ee0e680b2a0cd2ff39835d3
-  data.tar.gz: d7d208605dbbe0c09acccacda2ac6deba8a69f85540046ac098b555b9bc5212e7c19c137df1957f0b692112dc694f2a3913538671183e18052061f417f1dbd4b
+  metadata.gz: 20d82567d738f3f5fa78cdd1bb2595ea7cd068dd16a14fbda56b4c88450a33cac5b32106652401854e0085e1c4378cce6b3ef23983b04ec71fb14f931628c19e
+  data.tar.gz: 3bcd72650c2d0817048edf3eff0be6ba8de4cc568b04203c3ec9631ccd4ef269b1c91f6f70d90dc05419357e5f4d3c3f164b4b4345f72c54c9d0cfabe15d3103

data/README.md

@@ -14,10 +14,8 @@ _ElasticsearchRecord is a ActiveRecord adapter and provides similar functionalit
 
 **PLEASE NOTE:**
 
-- This is still in **development**!
-- Specs & documentation will follow. 
-- You might experience BUGs and Exceptions...
-- Currently supports only ActiveRecord 7.0 + Elasticsearch 8.4 _(downgrade for rails 6.x is planned in future versions)_
+- Specs & documentation are still missing, but will follow.
+- Currently supports ActiveRecord ~> 7.0 + Elasticsearch >= 7.17
 
 -----
 
@@ -55,10 +53,6 @@ Or install it yourself as:
   * logs Elasticsearch API-calls
   * shows Runtime in logs
 
-## Contra - what it _(currently)_ can not
-* Joins to other indexes or databases
-* complex, combined or nested queries ```and, or, Model.arel ...```
-
 ## Setup
 
 ### a) Update your **database.yml** and add a elasticsearch connection:
@@ -67,7 +61,7 @@ Or install it yourself as:
  
  development:
    primary:
-    # <...>
+     # <...>
 
    # elasticsearch
    elasticsearch:
@@ -75,10 +69,24 @@ Or install it yourself as:
      host: localhost:9200
      user: elastic
      password: '****'
-     log: true
+     
+     # enable ES verbose logging
+     # log: true
+     
+     # add table (index) prefix & suffix to all 'tables'
+     # table_name_prefix: 'app-'
+     # table_name_suffix: '-development'
  
  production:
-   ...
+   # <...>
+
+   # elasticsearch
+   elasticsearch:
+     # <...>
+   
+     # add table (index) prefix & suffix to all 'tables'
+     # table_name_prefix: 'app-'
+     # table_name_suffix: '-production'
  
  test:
    ...
@@ -86,15 +94,19 @@ Or install it yourself as:
  
 ```
 
-### b) Require ```elasticsearch_record/instrumentation``` in your application.rb (if you want to...):
+### b) Require `elasticsearch_record/instrumentation` in your application.rb (if you want to...):
+
 ```ruby
 # config/application.rb
+
 require_relative "boot"
 
 require "rails"
 # Pick the frameworks you want:
 
-# ...
+# <...>
+
+# add instrumentation
 require 'elasticsearch_record/instrumentation'
 
 module Application
@@ -102,14 +114,24 @@ module Application
 end
 ```
 
-### c) Create a model that inherits from ```ElasticsearchRecord::Base``` model.
+### c) Create a model that inherits from `ElasticsearchRecord::Base` model.
 ```ruby
 # app/models/application_elasticsearch_record.rb
-   
-class Search < ElasticsearchRecord::Base
-   
+
+class ApplicationElasticsearchRecord < ElasticsearchRecord::Base
+  # needs to be abstract
+  self.abstract_class = true
 end
+```
+
+Example class, that inherits from **ApplicationElasticsearchRecord**
 
+```ruby
+# app/models/search.rb
+   
+class Search < ApplicationElasticsearchRecord
+  
+end
 ```
 
 ### d) have FUN with your model:
@@ -243,11 +265,80 @@ _see simple documentation about these methods @ [rubydoc](https://rubydoc.info/g
 
 -----
 
-### Useful model class attributes
-- index_base_name
-- relay_id_attribute
+## Useful model class attributes
+
+### index_base_name
+Rails resolves a pluralized underscore table_name from the class name by default - which will not work for some models.
+
+To support a generic +table_name_prefix+ & +table_name_suffix+ from the _database.yml_, 
+the 'index_base_name' provides a possibility to chain prefix, **base** and suffix.
+
+```ruby
+class UnusalStat < ApplicationElasticsearchRecord
+  self.index_base_name = 'unusal-stats'
+end
+
+UnusalStat.where(year: 2023).to_query
+# => {:index=>"app-unusal-stats-development", :body ...
+```
+
+### delegate_id_attribute
+Rails resolves the primary_key's value by accessing the **#id** method.
+
+Since Elasticsearch also supports an additional, independent **id** attribute,
+it would only be able to access this through `_read_attribute(:id)`.
+
+To also have the ability of accessing this attribute through the default, this flag can be enabled.
+
+```ruby
+class SearchUser < ApplicationElasticsearchRecord
+  # attributes: id, name
+end
+
+# create new user within the index
+user = SearchUser.create(id: 8, name: 'Parker')
+
+# accessing the id, does NOT return the stored id by default - this will be delegated to the primary_key '_id'.
+user.id
+# => 'b2e34xa2'
+
+# -- ENABLE delegation -------------------------------------------------------------------
+SearchUser.delegate_id_attribute = true
+
+# create new user within the index
+user = SearchUser.create(id: 9, name: 'Pam')
+
+# accessing the id accesses the stored attribute now
+user.id
+# => 9
+
+# accessing the ES index id
+user._id
+# => 'xtf31bh8x'
+```
+
+## delegate_query_nil_limit
+Elasticsearch's default value for queries without a **size** is forced to **10**.
+To provide a similar behaviour as the (my)SQL interface,
+this can be automatically set to the `max_result_window` value by calling `.limit(nil)` on the models' relation.
+
+
+```ruby
+SearchUser.where(name: 'Peter').limit(nil)
+# returns a maximum of 10 items ...
+# => [...]
 
-### Useful model class methods
+# -- ENABLE delegation -------------------------------------------------------------------
+SearchUser.delegate_query_nil_limit = true
+
+SearchUser.where(name: 'Peter').limit(nil)
+# returns up to 10_000 items ...
+# => [...]
+
+# hint: if you want more than 10_000 use the +#pit_results+ method!
+```
+
+## Useful model class methods
 - auto_increment?
 - max_result_window
 - source_column_names
@@ -255,12 +346,55 @@ _see simple documentation about these methods @ [rubydoc](https://rubydoc.info/g
 - find_by_query
 - msearch
 
+## Useful model API methods
+Quick access to model-related methods for easier access without creating a overcomplicated method call on the models connection...
+
+Access these methods through the model class method `.api`.
+```ruby
+# returns mapping of model class
+klass.api.mappings
+
+# e.g. for ElasticUser model
+SearchUser.api.mappings
+
+# insert new raw data
+SearchUser.api.insert([{name: 'Hans', age: 34}, {name: 'Peter', age: 22}])
+```
+
+* open!
+* close!
+* refresh!
+* block!
+* unblock!
+* drop!(confirm: true)
+* truncate!(confirm: true)
+* mappings
+* metas
+* settings
+* aliases
+* state
+* schema
+* exists?
+* alias_exists?
+* setting_exists?
+* mapping_exists?
+* meta_exists?
+
+Fast insert, update, delete raw data
+* index
+* insert
+* update
+* delete
+* bulk
+
+-----
+
 ## ActiveRecord ConnectionAdapters table-methods
-Access these methods through the model's connection.
+Access these methods through the model class method `.connection`.
 
 ```ruby
-  # returns mapping of provided table (index)
-  model.connection.table_mappings('table-name')
+# returns mapping of provided table (index)
+klass.connection.table_mappings('table-name')
 ```
 
 - table_mappings
@@ -281,7 +415,7 @@ Access these methods through the model's connection.
 ## Active Record Schema migration methods
 Access these methods through the model's connection or within any `Migration`.
 
-**cluster actions:**
+### cluster actions:
 - open_table
 - open_tables
 - close_table
@@ -298,7 +432,7 @@ Access these methods through the model's connection or within any `Migration`.
 - change_table
 - rename_table
 
-**table actions:**
+### table actions:
 - change_meta
 - remove_meta
 - add_mapping
@@ -313,8 +447,10 @@ Access these methods through the model's connection or within any `Migration`.
 - change_alias
 - remove_alias
 
+
+**Example migration:**
+
 ```ruby
-# Example migration
 class AddTests < ActiveRecord::Migration[7.0]
   def up
     create_table "assignments", if_not_exists: true do |t|
@@ -387,6 +523,47 @@ class AddTests < ActiveRecord::Migration[7.0]
 end
 ```
 
+
+## environment-related-table-name:
+Using the `_env_table_name`-method will resolve the table (index) name within the current environment,
+even if the environments shares the same cluster ...
+
+This can be provided through the `database.yml` by using the `table_name_prefix/suffix` configuration keys.
+Within the migration the `_env_table_name`-method must be used in combination with the table (index) base name.
+
+**Example:**
+Production uses a index suffix with '-pro', development uses '-dev' - they share the same cluster, but different indexes.
+For the **settings** table:
+- settings-pro
+- settings-dev
+
+A single migration can be created to be used within each environment:
+```ruby
+# Example migration
+class AddSettings < ActiveRecord::Migration[7.0]
+  def up
+    create_table _env_table_name("settings"), force: true do |t|
+      t.mapping :created_at, :date
+      t.mapping :key, :integer do |m|
+        m.primary_key = true
+        m.auto_increment = 10
+      end
+      t.mapping :status, :keyword
+      t.mapping :updated_at, :date
+      t.mapping :value, :text
+
+      t.setting "index.number_of_replicas", "0"
+      t.setting "index.number_of_shards", "1"
+      t.setting "index.routing.allocation.include._tier_preference", "data_content"
+    end
+  end 
+  
+  def down
+    drop_table _env_table_name("settings")
+  end
+end 
+```
+
 ## Docs
 
 [CHANGELOG](docs/CHANGELOG.md)

data/docs/CHANGELOG.md

@@ -1,5 +1,23 @@
 # ElasticsearchRecord - CHANGELOG
 
+## [1.5.0] - 2023-07-10
+* [add] additional `ElasticsearchRecord::ModelApi` methods **drop!** & **truncate!**, which have to be called with a `confirm:true` parameter
+* [add] `.ElasticsearchRecord::Base.delegate_query_nil_limit` to automatically delegate a relations `limit(nil)`-call to the **max_result_window** _(set to 10.000 as default)_
+* [add] `ActiveRecord::ConnectionAdapters::Elasticsearch::SchemaStatements#access_shard_doc?` which checks, if the **PIT**-shard_doc order is available
+* [add] support for **_shard_doc** as a default order for `ElasticsearchRecord::Relation#pit_results`
+* [ref] `.ElasticsearchRecord::Base.relay_id_attribute` to a more coherent name: `delegate_id_attribute` 
+* [ref] `ElasticsearchRecord::Relation#ordered_relation` to optimize already ordered relations
+* [ref] gemspecs to support different versions of Elasticsearch
+* [ref] improved README
+* [fix] `ElasticsearchRecord::Relation#pit_results` infinite loop _(caused by missing order)_
+* [fix] `ElasticsearchRecord::Relation#pit_results` results generation without 'uniq' check of the array
+
+## [1.4.0] - 2023-01-27
+* [add] `ElasticsearchRecord::ModelApi` for fast & easy access the elasticsearch index - callable through `.api` (e.g. ElasticUser.api.mappings)
+* [ref] `ElasticsearchRecord::Instrumentation::LogSubscriber` to truncate the query-string (default: 1000)
+* [ref] `ActiveRecord::ConnectionAdapters::ElasticsearchAdapter#log` with extra attribute (log: true) to prevent logging (e.g. on custom api calls)
+* [fix] `ElasticsearchRecord::Result#bucket` to prevent resolving additional meta key (key_as_string)
+
 ## [1.3.1] - 2023-01-18
 * [fix] `#none!` method to correctly invalidate the query (String(s) in where-queries like '1=0' will raise now)
 * [fix] missing 'ChangeSettingDefinition' & 'RemoveSettingDefinition' @ `ActiveRecord::ConnectionAdapters::Elasticsearch::UpdateTableDefinition::COMPOSITE_DEFINITIONS` to composite in a single query

data/elasticsearch_record.gemspec

@@ -32,8 +32,8 @@ DESC
 
   spec.require_paths = ["lib"]
 
-  spec.add_dependency 'activerecord', '~> 7.0.0'
-  spec.add_dependency 'elasticsearch', '~> 8.4'
+  spec.add_dependency 'activerecord', '~> 7.0'
+  spec.add_dependency 'elasticsearch', '>= 7.17'
 
   #spec.add_development_dependency 'coveralls_reborn', '~> 0.25'
   spec.add_development_dependency 'rspec', '~> 3.0'

data/lib/active_record/connection_adapters/elasticsearch/schema_statements.rb

@@ -376,6 +376,14 @@ module ActiveRecord
             @access_id_fielddata
           end
 
+          # returns true if +_shard_doc+ field can be accessed through PIT-search.
+          # @return [Boolean]
+          def access_shard_doc?
+            @access_shard_doc = cluster_info[:version] >= "7.12" if @access_shard_doc.nil?
+
+            @access_shard_doc
+          end
+
           # Returns basic information about the cluster.
           # @return [Hash{Symbol->Unknown}]
           def cluster_info

data/lib/active_record/connection_adapters/elasticsearch_adapter.rb

@@ -216,14 +216,15 @@ module ActiveRecord # :nodoc:
       # @param [Hash] arguments - action arguments
       # @param [String (frozen)] name - the logging name
       # @param [Boolean] async - send async (default: false) - currently not supported
+      # @param [Boolean] log - send log to instrumenter (default: true)
       # @return [Elasticsearch::API::Response, Object]
-      def api(namespace, action, arguments = {}, name = 'API', async: false)
+      def api(namespace, action, arguments = {}, name = 'API', async: false, log: true)
         raise ::StandardError, 'ASYNC api calls are not supported' if async
 
         # resolve the API target
         target = namespace == :core ? @connection : @connection.__send__(namespace)
 
-        log("#{namespace}.#{action}", arguments, name, async: async) do
+        log("#{namespace}.#{action}", arguments, name, async: async, log: log) do
           response = ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
             target.__send__(action, arguments)
           end
@@ -274,16 +275,24 @@ module ActiveRecord # :nodoc:
       end
 
       # provide a custom log instrumenter for elasticsearch subscribers
-      def log(gate, arguments, name, async: false, &block)
-        @instrumenter.instrument(
-          "query.elasticsearch_record",
-          gate:      gate,
-          name:      name,
-          arguments: gate == 'core.msearch' ? arguments.deep_dup : arguments,
-          async:     async) do
-          @lock.synchronize(&block)
-        rescue => e
-          raise translate_exception_class(e, arguments, [])
+      def log(gate, arguments, name, async: false, log: true, &block)
+        if log
+          @instrumenter.instrument(
+            "query.elasticsearch_record",
+            gate:      gate,
+            name:      name,
+            arguments: gate == 'core.msearch' ? arguments.deep_dup : arguments,
+            async:     async) do
+            @lock.synchronize(&block)
+          rescue => e
+            raise translate_exception_class(e, arguments, [])
+          end
+        else
+          begin
+            @lock.synchronize(&block)
+          rescue => e
+            raise translate_exception_class(e, arguments, [])
+          end
         end
       end
 

data/lib/elasticsearch_record/core.rb

@@ -3,23 +3,27 @@ module ElasticsearchRecord
     extend ActiveSupport::Concern
 
     included do
-
       # Rails resolves the primary_key's value by accessing the +#id+ method.
       # Since Elasticsearch also supports an additional, independent +id+ attribute, it would only be able to access
-      # this through +read_attribute(:id)+.
+      # this through +_read_attribute(:id)+.
       # To also have the ability of accessing this attribute through the default, this flag can be enabled.
       # @attribute! Boolean
-      class_attribute :relay_id_attribute, instance_writer: false, default: false
+      class_attribute :delegate_id_attribute, instance_writer: false, default: false
+
+      # Elasticsearch's default value for queries without a +size+ is forced to +10+.
+      # To provide a similar behaviour as SQL, this can be automatically set to the +max_result_window+ value.
+      # @attribute! Boolean
+      class_attribute :delegate_query_nil_limit, instance_writer: false, default: false
     end
 
     # overwrite to provide a Elasticsearch version of returning a 'primary_key' attribute.
     # Elasticsearch uses the static +_id+ column as primary_key, but also supports an additional +id+ column.
     # To provide functionality of returning the +id+ attribute, this method must also support it
-    # with enabled +relay_id_attribute+.
+    # with enabled +delegate_id_attribute+.
     # @return [Object]
     def id
       # check, if the model has a +id+ attribute
-      return _read_attribute('id') if relay_id_attribute? && has_attribute?('id')
+      return _read_attribute('id') if delegate_id_attribute? && has_attribute?('id')
 
       super
     end
@@ -27,11 +31,11 @@ module ElasticsearchRecord
     # overwrite to provide a Elasticsearch version of setting a 'primary_key' attribute.
     # Elasticsearch uses the static +_id+ column as primary_key, but also supports an additional +id+ column.
     # To provide functionality of setting the +id+ attribute, this method must also support it
-    # with enabled +relay_id_attribute+.
+    # with enabled +delegate_id_attribute+.
     # @param [Object] value
     def id=(value)
       # check, if the model has a +id+ attribute
-      return _write_attribute('id', value) if relay_id_attribute? && has_attribute?('id')
+      return _write_attribute('id', value) if delegate_id_attribute? && has_attribute?('id')
 
       # auxiliary update the +_id+ virtual column if we have a different primary_key
       _write_attribute('_id', value) if @primary_key != '_id'
@@ -42,13 +46,13 @@ module ElasticsearchRecord
     # overwrite to provide a Elasticsearch version of returning a 'primary_key' was attribute.
     # Elasticsearch uses the static +_id+ column as primary_key, but also supports an additional +id+ column.
     # To provide functionality of returning the +id_Was+ attribute, this method must also support it
-    # with enabled +relay_id_attribute+.
+    # with enabled +delegate_id_attribute+.
     def id_was
-      relay_id_attribute? && has_attribute?('id') ? attribute_was('id') : super
+      delegate_id_attribute? && has_attribute?('id') ? attribute_was('id') : super
     end
 
     # overwrite the write_attribute method to always write to the 'id'-attribute, if present.
-    # This methods does not check for +relay_id_attribute+ flag!
+    # This methods does not check for +delegate_id_attribute+ flag!
     # see @ ActiveRecord::AttributeMethods::Write#write_attribute
     def write_attribute(attr_name, value)
       return _write_attribute('id', value) if attr_name.to_s == 'id' && has_attribute?('id')
@@ -57,7 +61,7 @@ module ElasticsearchRecord
     end
 
     # overwrite read_attribute method to read from the 'id'-attribute, if present.
-    # This methods does not check for +relay_id_attribute+ flag!
+    # This methods does not check for +delegate_id_attribute+ flag!
     # see @ ActiveRecord::AttributeMethods::Read#read_attribute
     def read_attribute(attr_name, &block)
       return _read_attribute('id', &block) if attr_name.to_s == 'id' && has_attribute?('id')
@@ -84,10 +88,11 @@ module ElasticsearchRecord
         cache.compute_if_absent(key) { ElasticsearchRecord::StatementCache.create(connection, &block) }
       end
 
-      # not supported atm - maybe enable in future versions to resolve migrations easier
-      # def primary_class? # :nodoc:
-      #   self == ::ElasticsearchRecord::Base
-      # end
+      # used to provide fast access to the connection API without explicit providing table-related parameters.
+      # @return [anonymous Struct]
+      def api
+        ElasticsearchRecord::ModelApi.new(self)
+      end
 
       private
 

data/lib/elasticsearch_record/gem_version.rb

@@ -8,8 +8,8 @@ module ElasticsearchRecord
 
   module VERSION
     MAJOR = 1
-    MINOR = 3
-    TINY  = 1
+    MINOR = 5
+    TINY  = 0
     PRE   = nil
 
     STRING = [MAJOR, MINOR, TINY, PRE].compact.join(".")

data/lib/elasticsearch_record/instrumentation/log_subscriber.rb

@@ -5,7 +5,7 @@ module ElasticsearchRecord
     # attach to ElasticsearchRecord related events
     class LogSubscriber < ActiveSupport::LogSubscriber
 
-      IGNORE_PAYLOAD_NAMES = %w[SCHEMA EXPLAIN]
+      IGNORE_PAYLOAD_NAMES = %w[SCHEMA EXPLAIN EXCLUDE]
 
       def self.runtime=(value)
         Thread.current["elasticsearch_record_runtime"] = value
@@ -37,17 +37,18 @@ module ElasticsearchRecord
                end
         name = "CACHE #{name}" if payload[:cached]
 
-        # nice feature: displays the REAL query-time (_qt)
+        # nice feature: displays the REAL query-time from elasticsearch response (_qt)
+        # this is handled through the +::ActiveRecord::ConnectionAdapters::ElasticsearchAdapter#api+ method
         name = "#{name} (took: #{payload[:arguments][:_qt].round(1)}ms)" if payload[:arguments][:_qt]
 
         # build query
-        query = payload[:arguments].except(:_qt).inspect.gsub(/:(\w+)=>/, '\1: ').presence || '-'
+        query = payload[:arguments].except(:_qt).inspect.gsub(/:(\w+)=>/, '\1: ').truncate((payload[:truncate] || 1000), omission: color(' (pruned)', RED))
 
         # final coloring
         name  = color(name, name_color(payload[:name]), true)
         query = color(query, gate_color(payload[:gate]), true) if colorize_logging
 
-        debug "  #{name} #{query}"
+        debug "  #{name} #{query.presence || '-/-'}"
       end
 
       private

data/lib/elasticsearch_record/model_api.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module ElasticsearchRecord
+  class ModelApi
+    attr_reader :klass
+
+    def initialize(klass)
+      @klass = klass
+    end
+
+    # undelegated schema methods: clone rename create
+    # those should not be quick-accessible, since they might end in heavily broken index
+
+    # delegated dangerous methods (created with exclamation mark)
+    # not able to provide individual arguments - always the defaults will be used!
+    #
+    # @example
+    #   open!
+    #   close!
+    #   refresh!
+    #   block!
+    #   unblock!
+    %w(open close refresh block unblock).each do |method|
+      define_method("#{method}!") do
+        _connection.send("#{method}_table", _index_name)
+      end
+    end
+
+    # delegated dangerous methods with confirm parameter (created with exclamation mark)
+    # a exception will be raised, if +confirm:true+ is missing.
+    #
+    # @example
+    #   drop!(confirm: true)
+    #   truncate!(confirm: true)
+    %w(drop truncate).each do |method|
+      define_method("#{method}!") do |confirmed: false|
+        raise "#{method} of table '#{_index_name}' aborted!\nexecution not confirmed!\ncall with: #{klass}.api.#{method}!(confirmed: true)" unless confirmed
+        _connection.send("#{method}_table", _index_name)
+      end
+    end
+
+    # delegated table methods
+    #
+    # @example
+    #   mappings
+    #   metas
+    #   settings
+    #   aliases
+    #   state
+    #   schema
+    #   exists?
+    %w(mappings metas settings aliases state schema exists?).each do |method|
+      define_method(method) do |*args|
+        _connection.send("table_#{method}", _index_name, *args)
+      end
+    end
+
+    # delegated plain methods
+    #
+    # @example
+    #   alias_exists?
+    #   setting_exists?
+    #   mapping_exists?
+    #   meta_exists?
+    %w(alias_exists? setting_exists? mapping_exists? meta_exists?).each do |method|
+      define_method(method) do |*args|
+        _connection.send(method, _index_name, *args)
+      end
+    end
+
+    # fast insert/update data.
+    #
+    # @example
+    #   index([{name: 'Hans', age: 34}, {name: 'Peter', age: 22}])
+    #
+    #   index({id: 5, name: 'Georg', age: 87})
+    #
+    # @param [Array<Hash>,Hash] data
+    # @param [Hash] options
+    def index(data, **options)
+      bulk(data, :index, **options)
+    end
+
+    # fast insert new data.
+    #
+    # @example
+    #   insert([{name: 'Hans', age: 34}, {name: 'Peter', age: 22}])
+    #
+    #   insert({name: 'Georg', age: 87})
+    #
+    # @param [Array<Hash>,Hash] data
+    # @param [Hash] options
+    def insert(data, **options)
+      bulk(data, :create, **options)
+    end
+
+    # fast update existing data.
+    #
+    # @example
+    #   update([{id: 1, name: 'Hansi'}, {id: 2, name: 'Peter Parker', age: 42}])
+    #
+    #   update({id: 3, name: 'Georg McCain'})
+    #
+    # @param [Array<Hash>,Hash] data
+    # @param [Hash] options
+    def update(data, **options)
+      bulk(data, :update, **options)
+    end
+
+    # fast delete data.
+    #
+    # @example
+    #   delete([1,2,3,5])
+    #
+    #   delete(3)
+    #
+    #   delete({id: 2})
+    #
+    # @param [Array<Hash>,Hash] data
+    # @param [Hash] options
+    def delete(data, **options)
+      data = [data] unless data.is_a?(Array)
+
+      if data[0].is_a?(Hash)
+        bulk(data, :delete, **options)
+      else
+        bulk(data.map { |id| { id: id } }, :delete, **options)
+      end
+    end
+
+    # bulk handle provided data (single Hash or multiple Array<Hash>).
+    # @param [Hash,Array<Hash>] data - the data to insert/update/delete ...
+    # @param [Symbol] operation
+    # @param [Boolean, Symbol] refresh
+    def bulk(data, operation = :index, refresh: true, **options)
+      data = [data] unless data.is_a?(Array)
+
+      _connection.api(:core, :bulk, {
+        index:   _index_name,
+        body:    data.map { |item| { operation => { _id: item[:id], data: item.except(:id) } } },
+        refresh: refresh
+      }, "BULK #{operation.to_s.upcase}", **options)
+    end
+
+    private
+
+    def _index_name
+      klass.index_name
+    end
+
+    def _connection
+      klass.connection
+    end
+  end
+end

data/lib/elasticsearch_record/relation/core_methods.rb

@@ -99,21 +99,29 @@ module ElasticsearchRecord
       # overwrite original methods to provide a elasticsearch version:
       # checks against the +#access_id_fielddata?+ to ensure the Elasticsearch Cluster allows access on the +_id+ field.
       def ordered_relation
-        # resolve valid primary_key (either not the '_id' or +access_id_fielddata?+ is enabled)
+        # order values already exist
+        return self unless order_values.empty?
+
+        # resolve valid primary_key
+        # - either it is NOT the '_id' column
+        # OR
+        # - it is the '_id'-column, but +access_id_fielddata?+ is also enabled!
         valid_primary_key = if primary_key != '_id' || klass.connection.access_id_fielddata?
-               primary_key
-             else
-               nil
-             end
+                              primary_key
+                            else
+                              nil
+                            end
 
         # slightly changed original methods content
-        if order_values.empty? && (implicit_order_column || valid_primary_key)
+        if implicit_order_column || valid_primary_key
+          # order by +implicit_order_column+ AND +primary_key+
           if implicit_order_column && valid_primary_key && implicit_order_column != valid_primary_key
             order(table[implicit_order_column].asc, table[valid_primary_key].asc)
           else
             order(table[implicit_order_column || valid_primary_key].asc)
           end
         else
+          # order is not possible due restricted settings
           self
         end
       end

data/lib/elasticsearch_record/relation/result_methods.rb

@@ -91,15 +91,20 @@ module ElasticsearchRecord
       # @param [String] keep_alive - how long to keep alive (for each single request) - default: '1m'
       # @param [Integer] batch_size - how many results per query (default: 1000 - this means at least 10 queries before reaching the +max_result_window+)
       def pit_results(keep_alive: '1m', batch_size: 1000)
-        raise ArgumentError, "Batch size cannot be above the 'max_result_window' (#{klass.max_result_window}) !" if batch_size > klass.max_result_window
+        raise(ArgumentError, "Batch size cannot be above the 'max_result_window' (#{klass.max_result_window}) !") if batch_size > klass.max_result_window
 
-        # check if a limit or offset values was provided
+        # check if limit or offset values where provided
         results_limit  = limit_value ? limit_value : Float::INFINITY
         results_offset = offset_value ? offset_value : 0
 
         # search_after requires a order - we resolve a order either from provided value or by default ...
         relation = ordered_relation
 
+        # FALLBACK (without any order) for restricted access to the '_id' field.
+        # with PIT a order by '_shard_doc' can also be used
+        # see @ https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html
+        relation.order!(_shard_doc: :asc) if relation.order_values.empty? && klass.connection.access_shard_doc?
+
         # clear limit & offset
         relation.offset!(nil).limit!(nil)
 
@@ -135,7 +140,7 @@ module ElasticsearchRecord
               if block_given?
                 yield ranged_results
               else
-                results |= ranged_results
+                results += ranged_results
               end
 
               # add to total
@@ -150,8 +155,8 @@ module ElasticsearchRecord
             # we ran out of data
             break if current_results_length < batch_size
 
-            # additional security - required?
-            # break if current_pit_hash[:search_after] == current_response['hits']['hits'][-1]['sort']
+            # additional security - prevents infinite loops
+            raise(::ActiveRecord::StatementInvalid, "'pit_results' aborted due an infinite loop error (invalid or missing order)") if current_pit_hash[:search_after] == current_response['hits']['hits'][-1]['sort'] && current_pit_hash[:pit][:id] == current_response['pit_id']
 
             # -------- NEXT LOOP changes --------
 

data/lib/elasticsearch_record/relation/value_methods.rb

@@ -43,6 +43,15 @@ module ElasticsearchRecord
         @values[:aggs] = value
       end
 
+      # overwrite the limit_value setter, to provide a special behaviour of auto-setting the +max_result_window+.
+      def limit=(limit)
+        if limit == '__max__' || (limit.nil? && delegate_query_nil_limit?)
+          super(max_result_window)
+        else
+          super
+        end
+      end
+
       private
 
       # alternative method to avoid redefining the const +VALID_UNSCOPING_VALUES+

data/lib/elasticsearch_record/result.rb

@@ -256,7 +256,7 @@ module ElasticsearchRecord
       else
         # resolve sub-aggregations / nodes without 'meta' keys.
         # if this results in an empty hash, the return will be nil
-        node.except(:key, :doc_count, :doc_count_error_upper_bound, :sum_other_doc_count).transform_values { |val| _resolve_bucket(val) }.presence
+        node.except(:key, :doc_count, :doc_count_error_upper_bound, :sum_other_doc_count, :key_as_string).transform_values { |val| _resolve_bucket(val) }.presence
       end
     end
 

data/lib/elasticsearch_record.rb

@@ -23,6 +23,7 @@ module ElasticsearchRecord
     autoload :Base
     autoload :Core
     autoload :ModelSchema
+    autoload :ModelApi
     autoload :Persistence
     autoload :Querying
     autoload :Query

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticsearch_record
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.5.0
 platform: ruby
 authors:
 - Tobias Gonsior
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-01-18 00:00:00.000000000 Z
+date: 2023-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '8.4'
+        version: '7.17'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '8.4'
+        version: '7.17'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -97,7 +97,7 @@ dependencies:
 description: 'ElasticsearchRecord is a ActiveRecord adapter and provides similar functionality
   for Elasticsearch.
 
-  '
+'
 email:
 - info@ruby-smart.org
 executables: []
@@ -107,7 +107,6 @@ files:
 - ".rspec"
 - ".yardopts"
 - Gemfile
-- Gemfile.lock
 - README.md
 - Rakefile
 - docs/CHANGELOG.md
@@ -159,6 +158,7 @@ files:
 - lib/elasticsearch_record/instrumentation/controller_runtime.rb
 - lib/elasticsearch_record/instrumentation/log_subscriber.rb
 - lib/elasticsearch_record/instrumentation/railtie.rb
+- lib/elasticsearch_record/model_api.rb
 - lib/elasticsearch_record/model_schema.rb
 - lib/elasticsearch_record/patches/active_record/relation_merger_patch.rb
 - lib/elasticsearch_record/patches/arel/select_core_patch.rb
@@ -191,7 +191,7 @@ metadata:
   source_code_uri: https://github.com/ruby-smart/elasticsearch_record
   documentation_uri: https://rubydoc.info/gems/elasticsearch_record
   changelog_uri: https://github.com/ruby-smart/elasticsearch_record/blob/main/docs/CHANGELOG.md
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -206,8 +206,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
-signing_key:
+rubygems_version: 3.1.6
+signing_key: 
 specification_version: 4
 summary: ActiveRecord adapter for Elasticsearch
 test_files: []

data/Gemfile.lock

@@ -1,73 +0,0 @@
-PATH
-  remote: .
-  specs:
-    elasticsearch_record (1.2.4)
-      activerecord (~> 7.0.0)
-      elasticsearch (~> 8.4)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    activemodel (7.0.4)
-      activesupport (= 7.0.4)
-    activerecord (7.0.4)
-      activemodel (= 7.0.4)
-      activesupport (= 7.0.4)
-    activesupport (7.0.4)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 1.6, < 2)
-      minitest (>= 5.1)
-      tzinfo (~> 2.0)
-    concurrent-ruby (1.1.10)
-    diff-lcs (1.5.0)
-    elastic-transport (8.1.0)
-      faraday (< 3)
-      multi_json
-    elasticsearch (8.5.2)
-      elastic-transport (~> 8)
-      elasticsearch-api (= 8.5.2)
-    elasticsearch-api (8.5.2)
-      multi_json
-    faraday (2.7.2)
-      faraday-net_http (>= 2.0, < 3.1)
-      ruby2_keywords (>= 0.0.4)
-    faraday-net_http (3.0.2)
-    i18n (1.12.0)
-      concurrent-ruby (~> 1.0)
-    minitest (5.16.3)
-    multi_json (1.15.0)
-    rake (13.0.6)
-    rspec (3.11.0)
-      rspec-core (~> 3.11.0)
-      rspec-expectations (~> 3.11.0)
-      rspec-mocks (~> 3.11.0)
-    rspec-core (3.11.0)
-      rspec-support (~> 3.11.0)
-    rspec-expectations (3.11.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-support (3.11.1)
-    ruby2_keywords (0.0.5)
-    tzinfo (2.0.5)
-      concurrent-ruby (~> 1.0)
-    webrick (1.7.0)
-    yard (0.9.28)
-      webrick (~> 1.7.0)
-    yard-activesupport-concern (0.0.1)
-      yard (>= 0.8)
-
-PLATFORMS
-  x86_64-linux
-
-DEPENDENCIES
-  elasticsearch_record!
-  rake (~> 13.0)
-  rspec (~> 3.0)
-  yard (~> 0.9)
-  yard-activesupport-concern (~> 0.0.1)
-
-BUNDLED WITH
-   2.3.18